From 5ba3f43ea354af8ad55bea84372a2bc834d8757c Mon Sep 17 00:00:00 2001 From: Apple <opensource@apple.com> Date: Tue, 26 Sep 2017 16:45:51 +0000 Subject: [PATCH] xnu-4570.1.46.tar.gz --- .gitignore | 5 +- .upstream_base_commits | 2 + EXTERNAL_HEADERS/AvailabilityMacros.h | 2 + EXTERNAL_HEADERS/architecture/arm/Makefile | 21 + EXTERNAL_HEADERS/architecture/arm/arm_neon.h | 74267 ++++++++++++++++ EXTERNAL_HEADERS/corecrypto/cc.h | 26 +- EXTERNAL_HEADERS/corecrypto/cc_config.h | 137 +- EXTERNAL_HEADERS/corecrypto/cc_debug.h | 21 +- EXTERNAL_HEADERS/corecrypto/cc_priv.h | 35 +- .../corecrypto/cc_runtime_config.h | 48 + EXTERNAL_HEADERS/corecrypto/ccaes.h | 7 + EXTERNAL_HEADERS/corecrypto/ccasn1.h | 2 +- .../corecrypto/ccchacha20poly1305.h | 295 + EXTERNAL_HEADERS/corecrypto/cccmac.h | 92 - EXTERNAL_HEADERS/corecrypto/ccder.h | 122 +- EXTERNAL_HEADERS/corecrypto/ccdrbg.h | 6 +- EXTERNAL_HEADERS/corecrypto/cchmac.h | 23 + EXTERNAL_HEADERS/corecrypto/ccmode.h | 45 +- EXTERNAL_HEADERS/corecrypto/ccmode_factory.h | 13 +- EXTERNAL_HEADERS/corecrypto/ccmode_impl.h | 18 +- EXTERNAL_HEADERS/corecrypto/ccmode_siv.h | 6 +- EXTERNAL_HEADERS/corecrypto/ccn.h | 32 +- EXTERNAL_HEADERS/corecrypto/ccrng.h | 53 +- EXTERNAL_HEADERS/corecrypto/ccrsa.h | 2 +- EXTERNAL_HEADERS/corecrypto/cczp.h | 27 +- EXTERNAL_HEADERS/corecrypto/fipspost_trace.h | 45 + EXTERNAL_HEADERS/mach-o/arm/reloc.h | 60 + EXTERNAL_HEADERS/mach-o/arm64/reloc.h | 41 + EXTERNAL_HEADERS/mach-o/loader.h | 50 +- Makefile | 9 +- README.md | 7 + SETUP/kextsymboltool/kextsymboltool.c | 5 - bsd/arm/Makefile | 34 + bsd/arm/_limits.h | 9 + bsd/arm/_mcontext.h | 91 + bsd/arm/_param.h | 22 + bsd/arm/_types.h | 98 + bsd/arm/disklabel.h | 21 + bsd/arm/endian.h | 78 + bsd/arm/exec.h | 96 + bsd/arm/fasttrap_isa.h | 230 + bsd/arm/limits.h | 110 + bsd/arm/param.h | 147 + bsd/arm/profile.h | 32 + bsd/arm/psl.h | 16 + bsd/arm/ptrace.h | 43 + bsd/arm/reboot.h | 32 + bsd/arm/reg.h | 16 + bsd/arm/signal.h | 19 + bsd/arm/types.h | 151 + bsd/arm/vmparam.h | 35 + bsd/bsm/audit_kevents.h | 5 +- bsd/bsm/audit_record.h | 1 + bsd/conf/Makefile.arm | 13 + bsd/conf/Makefile.arm64 | 10 + bsd/conf/Makefile.template | 10 +- bsd/conf/files | 41 +- bsd/conf/files.arm | 20 + bsd/conf/files.arm64 | 21 + bsd/conf/files.x86_64 | 2 - bsd/conf/param.c | 9 + bsd/dev/arm/conf.c | 306 + bsd/dev/arm/cons.c | 111 + bsd/dev/arm/cpu_in_cksum.s | 444 + bsd/dev/arm/disassembler.c | 1097 + bsd/dev/arm/dtrace_isa.c | 631 + bsd/dev/arm/dtrace_subr_arm.c | 186 + bsd/dev/arm/fasttrap_isa.c | 1297 + bsd/dev/arm/fbt_arm.c | 681 + bsd/dev/arm/kern_machdep.c | 188 + bsd/dev/arm/km.c | 403 + bsd/dev/arm/munge.c | 767 + bsd/dev/arm/pci_device.h | 106 + bsd/dev/arm/pio.h | 224 + bsd/dev/arm/sdt_arm.c | 166 + bsd/dev/arm/stubs.c | 83 + bsd/dev/arm/sysctl.c | 60 + bsd/dev/arm/systemcalls.c | 651 + bsd/dev/arm/table_inline.h | 36 + bsd/dev/arm/unix_signal.c | 737 + bsd/dev/arm64/conf.c | 306 + bsd/dev/arm64/cpu_in_cksum.s | 404 + bsd/dev/arm64/disassembler.c | 1146 + bsd/dev/arm64/dtrace_isa.c | 696 + bsd/dev/arm64/dtrace_subr_arm.c | 219 + bsd/dev/arm64/fasttrap_isa.c | 2127 + bsd/dev/arm64/fbt_arm.c | 608 + bsd/dev/arm64/sdt_arm.c | 162 + bsd/dev/arm64/sysctl.c | 55 + bsd/dev/dtrace/dtrace.c | 325 +- bsd/dev/dtrace/dtrace_glue.c | 73 +- bsd/dev/dtrace/dtrace_ptss.c | 54 +- bsd/dev/dtrace/fasttrap.c | 459 +- bsd/dev/dtrace/fbt.c | 545 +- bsd/dev/dtrace/lockstat.c | 185 +- bsd/dev/dtrace/profile_prvd.c | 131 +- bsd/dev/dtrace/scripts/Makefile | 3 +- bsd/dev/dtrace/scripts/mptcp.d | 22 +- .../dtrace/scripts/vm_map_delete_permanent.d | 14 + bsd/dev/dtrace/sdt.c | 25 +- bsd/dev/dtrace/sdt_subr.c | 14 +- bsd/dev/dtrace/systrace.c | 180 +- bsd/dev/dtrace/systrace.h | 5 - bsd/dev/i386/conf.c | 171 +- bsd/dev/i386/dis_tables.c | 1081 +- bsd/dev/i386/dtrace_isa.c | 15 +- bsd/dev/i386/fasttrap_isa.c | 139 +- bsd/dev/i386/fbt_x86.c | 479 +- bsd/dev/i386/km.c | 9 +- bsd/dev/i386/sdt_x86.c | 2 +- bsd/dev/i386/sysctl.c | 7 + bsd/dev/i386/systemcalls.c | 18 + bsd/dev/i386/unix_signal.c | 113 +- bsd/dev/monotonic.c | 459 + bsd/dev/munge.c | 16 + bsd/dev/unix_startup.c | 14 +- bsd/i386/_mcontext.h | 51 +- bsd/i386/dis_tables.h | 10 +- bsd/i386/fasttrap_isa.h | 5 + bsd/kern/ast.h | 6 + bsd/kern/bsd_init.c | 62 +- bsd/kern/bsd_stubs.c | 17 +- bsd/kern/decmpfs.c | 19 +- bsd/kern/imageboot.c | 152 +- bsd/kern/kdebug.c | 496 +- bsd/kern/kern_acct.c | 1 - bsd/kern/kern_aio.c | 36 +- bsd/kern/kern_clock.c | 21 +- bsd/kern/kern_control.c | 20 +- bsd/kern/kern_core.c | 32 +- bsd/kern/kern_credential.c | 65 +- bsd/kern/kern_cs.c | 114 +- bsd/kern/kern_csr.c | 10 +- bsd/kern/kern_descrip.c | 114 +- bsd/kern/kern_event.c | 6462 +- bsd/kern/kern_exec.c | 293 +- bsd/kern/kern_exit.c | 259 +- bsd/kern/kern_fork.c | 47 +- bsd/kern/kern_guarded.c | 750 +- bsd/kern/kern_kpc.c | 192 +- bsd/kern/kern_ktrace.c | 117 +- bsd/kern/kern_lockf.c | 6 + bsd/kern/kern_malloc.c | 72 +- bsd/kern/kern_memorystatus.c | 1493 +- bsd/kern/kern_mib.c | 91 + bsd/kern/kern_mman.c | 216 +- bsd/kern/kern_newsysctl.c | 4 + bsd/kern/kern_ntptime.c | 782 + bsd/kern/kern_overrides.c | 1 - bsd/kern/kern_priv.c | 23 +- bsd/kern/kern_proc.c | 98 +- bsd/kern/kern_resource.c | 205 +- bsd/kern/kern_shutdown.c | 99 +- bsd/kern/kern_sig.c | 26 +- bsd/kern/kern_symfile.c | 4 +- bsd/kern/kern_synch.c | 2 - bsd/kern/kern_sysctl.c | 514 +- bsd/kern/kern_time.c | 72 +- bsd/kern/kern_xxx.c | 27 +- bsd/kern/kpi_mbuf.c | 23 +- bsd/kern/kpi_socket.c | 136 +- bsd/kern/kpi_socketfilter.c | 39 +- bsd/kern/mach_loader.c | 512 +- bsd/kern/mach_loader.h | 1 + bsd/kern/mach_process.c | 14 + bsd/kern/makesyscalls.sh | 83 +- bsd/kern/mcache.c | 133 +- bsd/kern/policy_check.c | 21 +- bsd/kern/posix_sem.c | 31 +- bsd/kern/posix_shm.c | 37 +- bsd/kern/proc_info.c | 279 +- bsd/kern/process_policy.c | 158 +- bsd/kern/pthread_shims.c | 79 +- bsd/kern/stackshot.c | 3 +- bsd/kern/subr_eventhandler.c | 359 + bsd/kern/subr_log.c | 19 +- bsd/kern/subr_prf.c | 9 +- bsd/kern/subr_prof.c | 1 - bsd/kern/sys_coalition.c | 34 +- bsd/kern/sys_generic.c | 22 +- bsd/kern/sys_pipe.c | 46 +- bsd/kern/sys_socket.c | 8 +- bsd/kern/sys_ulock.c | 24 +- bsd/kern/sys_work_interval.c | 135 +- bsd/kern/syscalls.master | 37 +- bsd/kern/sysv_msg.c | 6 +- bsd/kern/sysv_shm.c | 9 +- bsd/kern/trace_codes | 244 +- bsd/kern/tty.c | 455 +- bsd/kern/tty_ptmx.c | 354 +- bsd/kern/tty_pty.c | 36 +- bsd/kern/ubc_subr.c | 103 +- bsd/kern/uipc_domain.c | 50 +- bsd/kern/uipc_mbuf.c | 205 +- bsd/kern/uipc_mbuf2.c | 86 +- bsd/kern/uipc_socket.c | 333 +- bsd/kern/uipc_socket2.c | 150 +- bsd/kern/uipc_syscalls.c | 80 +- bsd/kern/uipc_usrreq.c | 36 +- bsd/libkern/libkern.h | 25 +- bsd/libkern/url_encode.c | 3 - bsd/machine/Makefile | 2 +- bsd/machine/_limits.h | 2 + bsd/machine/_mcontext.h | 2 + bsd/machine/_param.h | 2 + bsd/machine/_types.h | 2 + bsd/machine/disklabel.h | 2 + bsd/machine/endian.h | 2 + bsd/machine/exec.h | 2 + bsd/machine/fasttrap_isa.h | 2 + bsd/machine/limits.h | 2 + bsd/machine/param.h | 2 + bsd/machine/profile.h | 2 + bsd/machine/psl.h | 2 + bsd/machine/ptrace.h | 2 + bsd/machine/reboot.h | 2 + bsd/machine/reg.h | 2 + bsd/machine/signal.h | 2 + bsd/machine/smp.h | 39 + bsd/machine/types.h | 2 + bsd/machine/vmparam.h | 2 + bsd/man/man2/Makefile | 9 +- bsd/man/man2/clonefile.2 | 15 +- bsd/man/man2/connectx.2 | 15 - bsd/man/man2/exchangedata.2 | 4 +- bsd/man/man2/fcntl.2 | 8 +- bsd/man/man2/fs_snapshot_create.2 | 201 + bsd/man/man2/fs_snapshot_delete.2 | 1 + bsd/man/man2/fs_snapshot_list.2 | 1 + bsd/man/man2/fs_snapshot_rename.2 | 1 + bsd/man/man2/fsgetpath.2 | 126 + bsd/man/man2/futimens.2 | 1 + bsd/man/man2/getattrlist.2 | 83 +- bsd/man/man2/kqueue.2 | 70 +- bsd/man/man2/mount.2 | 23 +- bsd/man/man2/peeloff.2 | 99 - bsd/man/man2/posix_spawn.2 | 3 + bsd/man/man2/profil.2 | 144 - bsd/man/man2/readlink.2 | 1 + bsd/man/man2/setattrlist.2 | 60 +- bsd/man/man2/setattrlistat.2 | 1 + bsd/man/man2/utimensat.2 | 256 + bsd/man/man9/monotonic.9 | 81 + bsd/miscfs/devfs/devfs.h | 1 + bsd/miscfs/devfs/devfs_fdesc_support.c | 8 +- bsd/miscfs/devfs/devfs_tree.c | 2 +- bsd/miscfs/devfs/devfs_vnops.c | 52 +- bsd/miscfs/devfs/devfsdefs.h | 2 - bsd/miscfs/fifofs/fifo_vnops.c | 2 +- bsd/miscfs/nullfs/nullfs.h | 2 +- bsd/miscfs/routefs/routefs_ops.c | 2 +- bsd/miscfs/specfs/spec_vnops.c | 421 +- bsd/miscfs/specfs/specdev.h | 2 +- bsd/net/Makefile | 2 + bsd/net/altq/altq.h | 9 +- bsd/net/altq/altq_cbq.c | 272 - bsd/net/altq/altq_cbq.h | 19 +- bsd/net/altq/altq_fairq.c | 304 - bsd/net/altq/altq_fairq.h | 18 +- bsd/net/altq/altq_hfsc.c | 290 - bsd/net/altq/altq_hfsc.h | 18 +- bsd/net/altq/altq_priq.c | 266 - bsd/net/altq/altq_priq.h | 18 +- bsd/net/altq/altq_qfq.c | 240 - bsd/net/altq/altq_qfq.h | 18 +- bsd/net/altq/altq_subr.c | 487 - bsd/net/altq/altq_var.h | 95 - bsd/net/altq/if_altq.h | 168 - bsd/net/bpf.c | 346 +- bsd/net/bpf.h | 29 +- bsd/net/bpf_filter.c | 296 +- bsd/net/bridgestp.c | 2 +- bsd/net/classq/classq.c | 271 +- bsd/net/classq/classq.h | 44 +- bsd/net/classq/classq_blue.c | 385 - bsd/net/classq/classq_blue.h | 47 +- bsd/net/classq/classq_fq_codel.c | 257 +- bsd/net/classq/classq_fq_codel.h | 31 +- bsd/net/classq/classq_red.c | 630 - bsd/net/classq/classq_red.h | 78 +- bsd/net/classq/classq_rio.c | 554 - bsd/net/classq/classq_rio.h | 61 +- bsd/net/classq/classq_sfb.c | 267 +- bsd/net/classq/classq_sfb.h | 8 +- bsd/net/classq/classq_subr.c | 447 +- bsd/net/classq/if_classq.h | 140 +- bsd/net/content_filter.c | 265 +- bsd/net/content_filter.h | 81 +- bsd/net/dlil.c | 1067 +- bsd/net/dlil.h | 85 +- bsd/net/ether_if_module.c | 13 +- bsd/net/ethernet.h | 12 + bsd/net/flowadv.c | 7 +- bsd/net/flowadv.h | 5 +- bsd/net/flowhash.c | 63 +- bsd/net/if.c | 357 +- bsd/net/if.h | 67 +- bsd/net/if_bond.c | 4 +- bsd/net/if_bridge.c | 603 +- bsd/net/if_dl.h | 6 + bsd/net/if_fake.c | 1029 + bsd/net/if_fake_var.h | 76 + bsd/net/if_gif.c | 9 +- bsd/net/if_gif.h | 2 +- bsd/net/if_ipsec.c | 2491 +- bsd/net/if_ipsec.h | 14 +- bsd/net/if_llatbl.c | 860 + bsd/net/if_llatbl.h | 302 + bsd/net/if_llreach.c | 2 +- bsd/net/if_llreach.h | 4 +- bsd/net/if_loop.c | 27 +- bsd/net/if_pflog.c | 19 +- bsd/net/if_stf.c | 9 +- bsd/net/if_utun.c | 2198 +- bsd/net/if_utun.h | 15 +- bsd/net/if_var.h | 342 +- bsd/net/if_vlan.c | 92 +- bsd/net/iptap.c | 42 +- bsd/net/kpi_interface.c | 312 +- bsd/net/kpi_interface.h | 135 +- bsd/net/kpi_interfacefilter.c | 22 +- bsd/net/kpi_interfacefilter.h | 10 +- bsd/net/kpi_protocol.c | 2 +- bsd/net/ndrv.c | 10 +- bsd/net/necp.c | 1525 +- bsd/net/necp.h | 373 +- bsd/net/necp_client.c | 3371 +- bsd/net/net_api_stats.h | 160 + bsd/net/net_kev.h | 17 +- bsd/net/net_stubs.c | 12 +- bsd/net/netsrc.c | 381 +- bsd/net/netsrc.h | 58 +- bsd/net/network_agent.c | 304 +- bsd/net/network_agent.h | 65 +- bsd/net/ntstat.c | 1686 +- bsd/net/ntstat.h | 450 +- bsd/net/nwk_wq.c | 137 + bsd/net/nwk_wq.h | 45 + bsd/net/packet_mangler.c | 2 +- bsd/net/pf.c | 1482 +- bsd/net/pf_if.c | 18 +- bsd/net/pf_ioctl.c | 619 +- bsd/net/pf_norm.c | 435 +- bsd/net/pf_osfp.c | 8 +- bsd/net/pf_pbuf.c | 410 + bsd/net/pf_pbuf.h | 106 + bsd/net/pf_ruleset.c | 18 +- bsd/net/pf_table.c | 66 +- bsd/net/pfkeyv2.h | 3 + bsd/net/pfvar.h | 78 +- bsd/net/pktap.c | 59 +- bsd/net/pktap.h | 6 +- bsd/net/pktsched/pktsched.c | 224 +- bsd/net/pktsched/pktsched.h | 42 +- bsd/net/pktsched/pktsched_cbq.c | 705 - bsd/net/pktsched/pktsched_cbq.h | 47 +- bsd/net/pktsched/pktsched_fairq.c | 1300 - bsd/net/pktsched/pktsched_fairq.h | 93 +- bsd/net/pktsched/pktsched_fq_codel.c | 577 +- bsd/net/pktsched/pktsched_fq_codel.h | 35 +- bsd/net/pktsched/pktsched_hfsc.c | 2065 - bsd/net/pktsched/pktsched_hfsc.h | 177 +- bsd/net/pktsched/pktsched_priq.c | 1309 - bsd/net/pktsched/pktsched_priq.h | 69 +- bsd/net/pktsched/pktsched_qfq.c | 528 +- bsd/net/pktsched/pktsched_qfq.h | 22 +- bsd/net/pktsched/pktsched_rmclass.c | 1852 - bsd/net/pktsched/pktsched_rmclass.h | 221 +- bsd/net/pktsched/pktsched_rmclass_debug.h | 140 - bsd/net/pktsched/pktsched_tcq.c | 512 +- bsd/net/pktsched/pktsched_tcq.h | 25 +- bsd/net/radix.h | 8 + bsd/net/raw_cb.h | 2 +- bsd/net/raw_usrreq.c | 11 +- bsd/net/route.c | 388 +- bsd/net/route.h | 107 +- bsd/net/rtsock.c | 113 +- bsd/net/skmem_sysctl.c | 30 + bsd/netinet/Makefile | 3 +- .../{cpu_in_cksum.c => cpu_in_cksum_gen.c} | 176 +- bsd/netinet/flow_divert.c | 41 +- bsd/netinet/flow_divert.h | 3 +- bsd/netinet/flow_divert_proto.h | 4 +- bsd/netinet/icmp6.h | 2 +- bsd/netinet/igmp.c | 6 +- bsd/netinet/igmp_var.h | 4 +- bsd/netinet/in.c | 390 +- bsd/netinet/in.h | 35 +- bsd/netinet/in_arp.c | 158 +- bsd/netinet/in_arp.h | 1 + bsd/netinet/in_cksum.c | 644 +- bsd/netinet/in_mcast.c | 198 +- bsd/netinet/in_pcb.c | 158 +- bsd/netinet/in_pcb.h | 22 +- bsd/netinet/in_pcblist.c | 28 +- bsd/netinet/in_rmx.c | 19 +- bsd/netinet/in_stat.c | 99 + bsd/netinet/in_stat.h | 49 + bsd/netinet/in_tclass.c | 89 +- bsd/netinet/in_tclass.h | 25 +- bsd/netinet/in_var.h | 12 +- bsd/netinet/ip_compat.h | 6 +- bsd/netinet/ip_divert.c | 10 +- bsd/netinet/ip_dummynet.c | 255 +- bsd/netinet/ip_dummynet.h | 66 +- bsd/netinet/ip_fw2.c | 1 - bsd/netinet/ip_icmp.c | 8 +- bsd/netinet/ip_id.c | 8 +- bsd/netinet/ip_input.c | 192 +- bsd/netinet/ip_output.c | 41 +- bsd/netinet/ip_var.h | 14 +- bsd/netinet/kpi_ipfilter.c | 187 +- bsd/netinet/kpi_ipfilter.h | 18 +- bsd/netinet/mp_pcb.c | 134 +- bsd/netinet/mp_pcb.h | 49 +- bsd/netinet/mp_proto.c | 25 +- bsd/netinet/mptcp.c | 1374 +- bsd/netinet/mptcp.h | 6 +- bsd/netinet/mptcp_opt.c | 934 +- bsd/netinet/mptcp_opt.h | 20 +- bsd/netinet/mptcp_seq.h | 2 +- bsd/netinet/mptcp_subr.c | 5903 +- bsd/netinet/mptcp_timer.c | 27 +- bsd/netinet/mptcp_timer.h | 2 +- bsd/netinet/mptcp_usrreq.c | 1229 +- bsd/netinet/mptcp_var.h | 518 +- bsd/netinet/raw_ip.c | 38 +- bsd/netinet/tcp.h | 66 +- bsd/netinet/tcp_cache.c | 711 +- bsd/netinet/tcp_cache.h | 15 +- bsd/netinet/tcp_cc.c | 24 +- bsd/netinet/tcp_cc.h | 2 + bsd/netinet/tcp_cubic.c | 28 +- bsd/netinet/tcp_fsm.h | 4 + bsd/netinet/tcp_input.c | 510 +- bsd/netinet/tcp_ledbat.c | 21 +- bsd/netinet/tcp_lro.c | 1 + bsd/netinet/tcp_output.c | 269 +- bsd/netinet/tcp_sack.c | 15 +- bsd/netinet/tcp_seq.h | 6 +- bsd/netinet/tcp_subr.c | 827 +- bsd/netinet/tcp_timer.c | 371 +- bsd/netinet/tcp_timer.h | 3 + bsd/netinet/tcp_usrreq.c | 288 +- bsd/netinet/tcp_var.h | 150 +- bsd/netinet/udp_usrreq.c | 187 +- bsd/netinet/udp_var.h | 2 +- bsd/netinet6/Makefile | 6 +- bsd/netinet6/ah_input.c | 2 +- bsd/netinet6/esp.h | 1 + bsd/netinet6/esp6.h | 3 +- bsd/netinet6/esp_chachapoly.c | 481 + bsd/netinet6/esp_chachapoly.h | 53 + bsd/netinet6/esp_core.c | 32 +- bsd/netinet6/esp_input.c | 92 +- bsd/netinet6/esp_output.c | 4 +- bsd/netinet6/esp_rijndael.c | 66 +- bsd/netinet6/frag6.c | 47 +- bsd/netinet6/icmp6.c | 17 +- bsd/netinet6/in6.c | 591 +- bsd/netinet6/in6.h | 63 +- bsd/netinet6/in6_cga.c | 5 +- bsd/netinet6/in6_cksum.c | 44 + bsd/netinet6/in6_ifattach.c | 14 +- bsd/netinet6/in6_mcast.c | 224 +- bsd/netinet6/in6_pcb.c | 70 +- bsd/netinet6/in6_pcb.h | 3 +- bsd/netinet6/in6_proto.c | 9 +- bsd/netinet6/in6_rmx.c | 21 +- bsd/netinet6/in6_src.c | 475 +- bsd/netinet6/in6_var.h | 25 +- bsd/netinet6/ip6_fw.c | 22 - bsd/netinet6/ip6_input.c | 130 +- bsd/netinet6/ip6_output.c | 84 +- bsd/netinet6/ip6_var.h | 24 +- bsd/netinet6/ip6protosw.h | 8 +- bsd/netinet6/ipsec.c | 58 +- bsd/netinet6/ipsec.h | 5 + bsd/netinet6/mld6.c | 108 +- bsd/netinet6/mld6_var.h | 7 +- bsd/netinet6/nd6.c | 191 +- bsd/netinet6/nd6.h | 32 +- bsd/netinet6/nd6_nbr.c | 122 +- bsd/netinet6/nd6_prproxy.c | 12 +- bsd/netinet6/nd6_rtr.c | 58 +- bsd/netinet6/nd6_send.c | 8 +- bsd/netinet6/raw_ip6.c | 15 +- bsd/netinet6/tcp6_var.h | 2 +- bsd/netinet6/udp6_output.c | 14 +- bsd/netinet6/udp6_usrreq.c | 70 +- bsd/netinet6/udp6_var.h | 2 +- bsd/netkey/key.c | 182 +- bsd/netkey/key.h | 3 + bsd/netkey/keydb.c | 12 +- bsd/nfs/gss/gss_krb5_mech.c | 8 +- bsd/nfs/nfs.h | 20 +- bsd/nfs/nfs4_subs.c | 862 +- bsd/nfs/nfs4_vnops.c | 4 +- bsd/nfs/nfs_bio.c | 4 +- bsd/nfs/nfs_gss.c | 31 +- bsd/nfs/nfs_ioctl.h | 18 +- bsd/nfs/nfs_lock.c | 2 +- bsd/nfs/nfs_socket.c | 10 +- bsd/nfs/nfs_subs.c | 2 +- bsd/nfs/nfs_syscalls.c | 29 +- bsd/nfs/nfs_vfsops.c | 220 +- bsd/nfs/nfs_vnops.c | 53 +- bsd/nfs/nfsmount.h | 2 + bsd/pgo/profile_runtime.c | 24 +- bsd/security/audit/audit.h | 5 - bsd/security/audit/audit_bsm.c | 15 + bsd/sys/Makefile | 18 +- bsd/sys/_types/Makefile | 8 + bsd/sys/_types/_blkcnt_t.h | 1 + bsd/sys/_types/_blksize_t.h | 1 + bsd/sys/_types/_caddr_t.h | 31 + bsd/sys/_types/_clock_t.h | 1 + bsd/sys/_types/_ct_rune_t.h | 1 + bsd/sys/_types/_dev_t.h | 1 + bsd/sys/_types/_fd_def.h | 3 + bsd/sys/_types/_fsblkcnt_t.h | 1 + bsd/sys/_types/_fsfilcnt_t.h | 1 + bsd/sys/_types/_fsid_t.h | 1 + bsd/sys/_types/_fsobj_id_t.h | 2 + bsd/sys/_types/_gid_t.h | 1 + bsd/sys/_types/_id_t.h | 1 + bsd/sys/_types/_in_addr_t.h | 1 + bsd/sys/_types/_in_port_t.h | 1 + bsd/sys/_types/_ino64_t.h | 1 + bsd/sys/_types/_ino_t.h | 1 + bsd/sys/_types/_intptr_t.h | 2 + bsd/sys/_types/_iovec_t.h | 1 + bsd/sys/_types/_key_t.h | 1 + bsd/sys/_types/_mach_port_t.h | 1 + bsd/sys/_types/_mbstate_t.h | 1 + bsd/sys/_types/_mode_t.h | 1 + bsd/sys/_types/_nlink_t.h | 1 + bsd/sys/_types/_null.h | 3 +- bsd/sys/_types/_off_t.h | 1 + bsd/sys/_types/_pid_t.h | 1 + bsd/sys/_types/_ptrdiff_t.h | 1 + bsd/sys/_types/_rsize_t.h | 1 + bsd/sys/_types/_rune_t.h | 1 + bsd/sys/_types/_sa_family_t.h | 1 + bsd/sys/_types/_seek_set.h | 4 + bsd/sys/_types/_sigaltstack.h | 6 + bsd/sys/_types/_sigset_t.h | 1 + bsd/sys/_types/_size_t.h | 1 + bsd/sys/_types/_socklen_t.h | 1 + bsd/sys/_types/_ssize_t.h | 1 + bsd/sys/_types/_suseconds_t.h | 1 + bsd/sys/_types/_time_t.h | 1 + bsd/sys/_types/_timespec.h | 3 + bsd/sys/_types/_timeval.h | 4 + bsd/sys/_types/_timeval32.h | 3 + bsd/sys/_types/_timeval64.h | 3 + bsd/sys/_types/_u_char.h | 31 + bsd/sys/_types/_u_int.h | 31 + bsd/sys/_types/_u_short.h | 31 + bsd/sys/_types/_ucontext.h | 8 + bsd/sys/_types/_ucontext64.h | 8 + bsd/sys/_types/_uid_t.h | 1 + bsd/sys/_types/_useconds_t.h | 1 + bsd/sys/_types/_user32_ntptimeval.h | 41 + bsd/sys/_types/_user32_timex.h | 54 + bsd/sys/_types/_user64_ntptimeval.h | 41 + bsd/sys/_types/_user64_timex.h | 54 + bsd/sys/_types/_uuid_t.h | 1 + bsd/sys/_types/_va_list.h | 1 + bsd/sys/_types/_wchar_t.h | 1 + bsd/sys/_types/_wint_t.h | 1 + bsd/sys/acct.h | 4 + bsd/sys/attr.h | 9 +- bsd/sys/bitstring.h | 46 +- bsd/sys/buf_internal.h | 1 + bsd/sys/cdefs.h | 8 + bsd/sys/clonefile.h | 5 +- bsd/sys/coalition.h | 2 + bsd/sys/codesign.h | 150 +- bsd/sys/commpage.h | 41 + bsd/sys/conf.h | 6 +- bsd/sys/csr.h | 4 +- bsd/sys/decmpfs.h | 2 +- bsd/sys/disk.h | 11 + bsd/sys/domain.h | 3 + bsd/sys/dtrace.h | 21 + bsd/sys/dtrace_glue.h | 52 +- bsd/sys/dtrace_impl.h | 5 +- bsd/sys/dtrace_ptss.h | 3 + bsd/sys/event.h | 348 +- bsd/sys/eventhandler.h | 221 + bsd/sys/eventvar.h | 198 +- bsd/sys/fasttrap.h | 7 + bsd/sys/fasttrap_impl.h | 13 + bsd/sys/fbt.h | 11 + bsd/sys/fcntl.h | 2 +- bsd/sys/file_internal.h | 7 +- bsd/sys/filedesc.h | 12 +- bsd/sys/fsctl.h | 24 +- bsd/sys/fsevents.h | 4 +- bsd/sys/fsgetpath.h | 40 +- bsd/sys/guarded.h | 68 +- bsd/sys/imgact.h | 4 + bsd/sys/kauth.h | 2 + bsd/sys/kdebug.h | 267 +- bsd/sys/kern_control.h | 3 + bsd/sys/kern_memorystatus.h | 73 +- bsd/sys/kpi_mbuf.h | 8 +- bsd/sys/kpi_socket.h | 36 +- bsd/sys/kpi_socketfilter.h | 10 +- bsd/sys/ktrace.h | 8 +- bsd/sys/lctx.h | 2 + bsd/sys/linker_set.h | 10 +- bsd/sys/malloc.h | 30 +- bsd/sys/mbuf.h | 47 +- bsd/sys/mcache.h | 15 +- bsd/sys/mman.h | 3 + bsd/sys/monotonic.h | 149 + bsd/sys/mount.h | 23 +- bsd/sys/mount_internal.h | 2 + bsd/sys/munge.h | 53 + bsd/sys/netport.h | 2 + bsd/sys/persona.h | 2 +- bsd/sys/pgo.h | 5 + bsd/sys/pipe.h | 4 +- bsd/sys/priv.h | 19 +- bsd/sys/proc.h | 11 +- bsd/sys/proc_info.h | 58 +- bsd/sys/proc_internal.h | 16 +- bsd/sys/process_policy.h | 18 + bsd/sys/protosw.h | 107 +- bsd/sys/pthread_internal.h | 1 + bsd/sys/pthread_shims.h | 104 +- bsd/sys/quota.h | 1 + bsd/sys/reason.h | 26 +- bsd/sys/resource.h | 52 +- bsd/sys/resourcevar.h | 1 + bsd/sys/sdt_impl.h | 4 + bsd/sys/sem.h | 1 + bsd/sys/signal.h | 4 +- bsd/sys/snapshot.h | 2 + bsd/sys/socket.h | 13 +- bsd/sys/socketvar.h | 84 +- bsd/sys/sockio.h | 26 +- bsd/sys/spawn.h | 3 + bsd/sys/stat.h | 22 +- bsd/sys/subr_prf.h | 2 +- bsd/sys/sysctl.h | 28 + bsd/sys/sysent.h | 2 + bsd/sys/syslog.h | 6 +- bsd/sys/systm.h | 5 +- bsd/sys/systrace_args.h | 36 + bsd/sys/time.h | 4 + bsd/sys/timex.h | 212 + bsd/sys/tprintf.h | 2 +- bsd/sys/tty.h | 12 +- bsd/sys/types.h | 9 +- bsd/sys/ubc.h | 7 + bsd/sys/ubc_internal.h | 5 + bsd/sys/unistd.h | 6 +- bsd/sys/unpcb.h | 3 + bsd/sys/user.h | 12 +- bsd/sys/vm.h | 4 + bsd/sys/vnode.h | 76 +- bsd/sys/vnode_if.h | 6 +- bsd/sys/vnode_internal.h | 4 +- bsd/sys/work_interval.h | 145 +- bsd/vfs/Makefile | 4 +- bsd/vfs/kpi_vfs.c | 92 +- bsd/vfs/vfs_attrlist.c | 103 +- bsd/vfs/vfs_bio.c | 35 +- bsd/vfs/vfs_cache.c | 56 +- bsd/vfs/vfs_cluster.c | 83 +- bsd/vfs/vfs_disk_conditioner.c | 235 + bsd/vfs/vfs_disk_conditioner.h | 42 + bsd/vfs/vfs_fsevents.c | 79 +- bsd/vfs/vfs_lookup.c | 32 +- bsd/vfs/vfs_subr.c | 201 +- bsd/vfs/vfs_syscalls.c | 279 +- bsd/vfs/vfs_vnops.c | 30 +- bsd/vfs/vfs_xattr.c | 24 +- bsd/vm/vm_compressor_backing_file.c | 18 +- bsd/vm/vm_unix.c | 314 +- bsd/vm/vnode_pager.c | 14 +- config/BSDKernel.arm.exports | 16 + config/BSDKernel.arm64.exports | 16 + config/BSDKernel.exports | 3 +- config/IOKit.arm.exports | 309 + config/IOKit.arm64.exports | 230 + config/IOKit.exports | 10 +- config/IOKit.x86_64.exports | 7 +- config/Libkern.arm.exports | 4 + config/Libkern.arm64.exports | 5 + config/Libkern.exports | 10 + config/MACFramework.arm.exports | 0 config/MACFramework.arm64.exports | 0 config/MASTER | 50 +- config/MASTER.arm | 86 + config/MASTER.arm64 | 92 + config/MASTER.x86_64 | 26 +- config/Mach.arm.exports | 2 + config/Mach.arm64.exports | 1 + config/Mach.exports | 5 + config/Makefile | 4 + config/MasterVersion | 2 +- config/Private.arm.exports | 21 + config/Private.arm64.exports | 34 + config/Private.exports | 51 +- config/Private.x86_64.exports | 12 + .../System.kext/PlugIns/Kasan.kext/Info.plist | 34 + config/Unsupported.arm.exports | 24 + config/Unsupported.arm64.exports | 40 + config/Unsupported.exports | 13 +- config/Unsupported.x86_64.exports | 2 + iokit/IOKit/IOCPU.h | 14 +- iokit/IOKit/IODeviceTreeSupport.h | 5 + iokit/IOKit/IOEventSource.h | 19 +- iokit/IOKit/IOHibernatePrivate.h | 18 +- iokit/IOKit/IOInterruptAccounting.h | 2 + iokit/IOKit/IOInterruptController.h | 10 + iokit/IOKit/IOKernelReportStructs.h | 212 +- iokit/IOKit/IOKernelReporters.h | 18 +- iokit/IOKit/IOKitDebug.h | 35 +- iokit/IOKit/IOKitKeys.h | 1 + iokit/IOKit/IOLib.h | 11 +- iokit/IOKit/IOMapper.h | 4 +- iokit/IOKit/IOMemoryDescriptor.h | 54 +- iokit/IOKit/IOPlatformExpert.h | 5 +- iokit/IOKit/IOPolledInterface.h | 5 +- iokit/IOKit/IOReportTypes.h | 219 +- iokit/IOKit/IOReturn.h | 11 +- iokit/IOKit/IOService.h | 20 +- iokit/IOKit/IOTimeStamp.h | 1 + iokit/IOKit/IOTimerEventSource.h | 98 +- iokit/IOKit/IOTypes.h | 4 +- iokit/IOKit/Makefile | 19 +- iokit/IOKit/pwr_mgt/IOPM.h | 20 + iokit/IOKit/pwr_mgt/IOPMPrivate.h | 24 +- iokit/IOKit/pwr_mgt/IOPMlog.h | 2 + iokit/IOKit/pwr_mgt/RootDomain.h | 14 +- iokit/Kernel/IOBufferMemoryDescriptor.cpp | 21 +- iokit/Kernel/IOCPU.cpp | 241 +- iokit/Kernel/IOCommandGate.cpp | 8 +- iokit/Kernel/IOCommandQueue.cpp | 4 +- iokit/Kernel/IODMACommand.cpp | 136 +- iokit/Kernel/IODeviceTreeSupport.cpp | 340 +- iokit/Kernel/IOFilterInterruptEventSource.cpp | 13 +- iokit/Kernel/IOHibernateIO.cpp | 77 +- iokit/Kernel/IOHibernateRestoreKernel.c | 18 +- iokit/Kernel/IOHistogramReporter.cpp | 53 +- iokit/Kernel/IOInterruptAccounting.cpp | 2 + iokit/Kernel/IOInterruptController.cpp | 85 +- iokit/Kernel/IOInterruptEventSource.cpp | 20 +- iokit/Kernel/IOKitDebug.cpp | 99 +- iokit/Kernel/IOKitKernelInternal.h | 11 +- iokit/Kernel/IOLib.cpp | 42 +- iokit/Kernel/IOMapper.cpp | 38 +- iokit/Kernel/IOMemoryDescriptor.cpp | 550 +- iokit/Kernel/IONVRAM.cpp | 51 +- iokit/Kernel/IOPMrootDomain.cpp | 527 +- iokit/Kernel/IOPlatformExpert.cpp | 119 +- iokit/Kernel/IOPolledInterface.cpp | 110 +- iokit/Kernel/IOReporter.cpp | 8 +- iokit/Kernel/IOService.cpp | 371 +- iokit/Kernel/IOServicePM.cpp | 162 +- iokit/Kernel/IOServicePMPrivate.h | 8 +- iokit/Kernel/IOServicePrivate.h | 2 + iokit/Kernel/IOSimpleReporter.cpp | 4 +- iokit/Kernel/IOStartIOKit.cpp | 9 +- iokit/Kernel/IOStateReporter.cpp | 4 +- iokit/Kernel/IOStatistics.cpp | 7 +- iokit/Kernel/IOStringFuncs.c | 14 - iokit/Kernel/IOTimerEventSource.cpp | 221 +- iokit/Kernel/IOUserClient.cpp | 143 +- iokit/Kernel/IOWorkLoop.cpp | 17 +- iokit/Kernel/RootDomainUserClient.cpp | 9 +- iokit/Kernel/RootDomainUserClient.h | 2 +- iokit/Kernel/i386/IOKeyStoreHelper.cpp | 56 + iokit/Tests/TestIOMemoryDescriptor.cpp | 238 +- iokit/Tests/Tests.cpp | 50 +- iokit/bsddev/DINetBootHook.cpp | 65 +- iokit/bsddev/DINetBootHook.h | 3 +- iokit/bsddev/IOKitBSDInit.cpp | 76 +- iokit/conf/Makefile.arm | 18 + iokit/conf/Makefile.arm64 | 18 + iokit/conf/Makefile.template | 1 + iokit/conf/files.arm | 4 + iokit/conf/files.arm64 | 4 + libkdd/kcdata.h | 1140 +- libkdd/kcdtypes.c | 63 + libkdd/kdd.xcodeproj/project.pbxproj | 42 +- libkdd/tests/Tests.swift | 643 +- libkdd/tests/stackshot-sample-coalitions | Bin 0 -> 14448 bytes .../stackshot-sample-coalitions.plist.gz | Bin 0 -> 6794 bytes libkdd/tests/stackshot-sample-instrs-cycles | Bin 0 -> 625 bytes .../stackshot-sample-instrs-cycles.plist.gz | Bin 0 -> 1630 bytes libkdd/tests/stackshot-sample-thread-groups | Bin 0 -> 6784 bytes .../stackshot-sample-thread-groups.plist.gz | Bin 0 -> 4312 bytes libkdd/tests/stackshot-sample-thread-policy | Bin 0 -> 1274 bytes .../stackshot-sample-thread-policy.plist.gz | Bin 0 -> 2631 bytes libkern/OSKextLib.cpp | 10 +- libkern/c++/OSData.cpp | 4 +- libkern/c++/OSKext.cpp | 209 +- libkern/c++/OSMetaClass.cpp | 83 +- libkern/c++/OSUnserializeXML.cpp | 10 +- .../TestSerialization/test1/test1_main.cpp | 0 libkern/conf/Makefile.arm | 20 + libkern/conf/Makefile.arm64 | 20 + libkern/conf/Makefile.template | 1 + libkern/conf/files | 2 +- libkern/conf/files.arm | 3 + libkern/crypto/corecrypto_chacha20poly1305.c | 86 + libkern/firehose/chunk_private.h | 3 +- libkern/firehose/firehose_types_private.h | 42 +- libkern/gen/OSDebug.cpp | 43 +- libkern/kmod/libkmodtest/libkmodtest.h | 2 +- libkern/kxld/kxld_object.c | 32 +- libkern/kxld/kxld_util.c | 3 +- libkern/kxld/kxld_util.h | 2 +- libkern/kxld/kxld_versionmin.c | 31 +- libkern/kxld/kxld_versionmin.h | 5 +- libkern/kxld/tests/loadtest.py | 0 libkern/libkern/OSAtomic.h | 10 +- libkern/libkern/OSByteOrder.h | 2 + libkern/libkern/OSKextLib.h | 12 +- libkern/libkern/OSKextLibPrivate.h | 2 +- libkern/libkern/OSMalloc.h | 6 +- libkern/libkern/_OSByteOrder.h | 3 + libkern/libkern/arm/Makefile | 21 + libkern/libkern/arm/OSByteOrder.h | 147 + libkern/libkern/c++/OSData.h | 2 +- libkern/libkern/c++/OSKext.h | 14 +- libkern/libkern/c++/OSMetaClass.h | 92 +- libkern/libkern/crypto/Makefile | 2 +- libkern/libkern/crypto/chacha20poly1305.h | 55 + libkern/libkern/crypto/register_crypto.h | 18 +- libkern/net/inet_aton.c | 2 +- libkern/os/Makefile | 6 +- libkern/os/log.c | 19 +- libkern/os/log.h | 14 +- libkern/os/overflow.h | 11 + libkern/os/reason_private.h | 59 + libkern/zlib/gzio.c | 1031 - libsa/bootstrap.cpp | 29 + libsa/conf/Makefile.arm | 10 + libsa/conf/Makefile.arm64 | 10 + libsa/conf/Makefile.template | 1 + libsa/conf/files.arm | 0 libsa/conf/files.arm64 | 0 libsa/lastkerneldataconst.c | 5 + libsyscall/Libsyscall.xcconfig | 6 +- .../Libsyscall.xcodeproj/project.pbxproj | 83 +- libsyscall/Platforms/iPhoneOS/arm/syscall.map | 77 + .../Platforms/iPhoneOS/arm64/syscall.map | 67 + libsyscall/custom/SYS.h | 311 + libsyscall/custom/__fork.s | 53 + libsyscall/custom/__getpid.s | 71 + libsyscall/custom/__gettimeofday.s | 24 + libsyscall/custom/__kdebug_trace_string.s | 8 + libsyscall/custom/__lseek.s | 8 + libsyscall/custom/__pipe.s | 19 + libsyscall/custom/__ptrace.s | 19 + libsyscall/custom/__sigaltstack.s | 8 + libsyscall/custom/__sigreturn.s | 8 + libsyscall/custom/__syscall.s | 18 + libsyscall/custom/__thread_selfid.s | 8 + libsyscall/custom/__thread_selfusage.s | 8 + libsyscall/custom/__vfork.s | 101 + libsyscall/custom/custom.s | 20 + libsyscall/mach/err_libkern.sub | 1 + libsyscall/mach/host.c | 17 + libsyscall/mach/mach_init.c | 12 + libsyscall/mach/string.h | 4 +- libsyscall/os/tsd.h | 46 +- libsyscall/wrappers/__commpage_gettimeofday.c | 119 +- libsyscall/wrappers/__commpage_gettimeofday.s | 131 - libsyscall/wrappers/__get_cpu_capabilities.s | 25 + libsyscall/wrappers/_libkernel_init.c | 10 + libsyscall/wrappers/coalition.c | 11 + libsyscall/wrappers/init_cpu_capabilities.c | 12 + libsyscall/wrappers/libproc/libproc.c | 223 +- libsyscall/wrappers/libproc/libproc.h | 4 + .../wrappers/libproc/libproc_internal.h | 47 + libsyscall/wrappers/mach_absolute_time.s | 124 + libsyscall/wrappers/mach_approximate_time.s | 40 +- libsyscall/wrappers/mach_continuous_time.c | 41 +- libsyscall/wrappers/mach_get_times.c | 4 + libsyscall/wrappers/pid_shutdown_networking.c | 33 + libsyscall/wrappers/quota_obsolete.c | 2 + libsyscall/wrappers/remove-counter.c | 11 + libsyscall/wrappers/spawn/posix_spawn.c | 35 + libsyscall/wrappers/spawn/spawn_private.h | 4 + libsyscall/wrappers/thread_register_state.c | 38 +- libsyscall/wrappers/utimensat.c | 134 + libsyscall/wrappers/varargs_wrappers.s | 124 + libsyscall/wrappers/work_interval.c | 177 +- libsyscall/xcodescripts/create-syscalls.pl | 4 + libsyscall/xcodescripts/mach_install_mig.sh | 8 +- makedefs/MakeInc.cmd | 4 +- makedefs/MakeInc.def | 147 +- makedefs/MakeInc.top | 16 +- osfmk/arm/Makefile | 49 + osfmk/arm/WKdmCompress_new.s | 710 + osfmk/arm/WKdmData_new.s | 289 + osfmk/arm/WKdmDecompress_new.s | 427 + osfmk/arm/arch.h | 67 + osfmk/arm/arm_init.c | 531 + osfmk/arm/arm_timer.c | 279 + osfmk/arm/arm_vm_init.c | 537 + osfmk/arm/asm.h | 320 + osfmk/arm/atomic.h | 261 + osfmk/arm/bcopy.s | 402 + osfmk/arm/bsd_arm.c | 71 + osfmk/arm/bzero.s | 173 + osfmk/arm/caches.c | 753 + osfmk/arm/caches_asm.s | 362 + osfmk/arm/caches_internal.h | 101 + osfmk/arm/commpage/commpage.c | 432 + osfmk/arm/commpage/commpage.h | 49 + osfmk/arm/commpage/commpage_sigs.h | 111 + osfmk/arm/conf.c | 83 + osfmk/arm/cpu.c | 604 + osfmk/arm/cpu_affinity.h | 45 + osfmk/arm/cpu_capabilities.h | 212 + osfmk/arm/cpu_common.c | 577 + osfmk/arm/cpu_data.h | 92 + osfmk/arm/cpu_data_internal.h | 319 + osfmk/arm/cpu_internal.h | 79 + osfmk/arm/cpu_number.h | 80 + osfmk/arm/cpuid.c | 314 + osfmk/arm/cpuid.h | 224 + osfmk/arm/cpuid_internal.h | 55 + osfmk/arm/cswitch.s | 290 + osfmk/arm/data.s | 117 + osfmk/arm/dbgwrap.c | 53 + osfmk/arm/dbgwrap.h | 63 + osfmk/arm/exception.h | 77 + osfmk/arm/genassym.c | 368 + osfmk/arm/globals_asm.h | 40 + osfmk/arm/hw_lock_types.h | 73 + osfmk/arm/io_map.c | 112 + osfmk/arm/io_map_entries.h | 49 + osfmk/arm/kpc_arm.c | 986 + osfmk/arm/lock.h | 71 + osfmk/arm/locks.h | 332 + osfmk/arm/locks_arm.c | 2882 + osfmk/arm/locore.s | 2041 + osfmk/arm/loose_ends.c | 665 + osfmk/arm/lowglobals.h | 78 + osfmk/arm/lowmem_vectors.c | 91 + osfmk/arm/lz4_decode_armv7NEON.s | 348 + osfmk/arm/lz4_encode_armv7.s | 429 + osfmk/arm/machdep_call.c | 92 + osfmk/arm/machdep_call.h | 65 + osfmk/arm/machine_cpu.h | 60 + osfmk/arm/machine_cpuid.c | 163 + osfmk/arm/machine_cpuid.h | 107 + osfmk/arm/machine_kpc.h | 60 + osfmk/arm/machine_routines.c | 1176 + osfmk/arm/machine_routines.h | 884 + osfmk/arm/machine_routines_asm.s | 1131 + osfmk/arm/machine_routines_common.c | 614 + osfmk/arm/machine_task.c | 179 + osfmk/arm/machlimits.h | 98 + osfmk/arm/machparam.h | 64 + osfmk/arm/misc_protos.h | 98 + osfmk/arm/model_dep.c | 868 + osfmk/arm/monotonic.h | 37 + osfmk/arm/monotonic_arm.c | 51 + osfmk/arm/pal_routines.c | 62 + osfmk/arm/pal_routines.h | 69 + osfmk/arm/pcb.c | 406 + osfmk/arm/pmap.c | 10555 +++ osfmk/arm/pmap.h | 516 + osfmk/arm/proc_reg.h | 1084 + osfmk/arm/rtclock.c | 495 + osfmk/arm/rtclock.h | 97 + osfmk/arm/sched_param.h | 67 + osfmk/arm/setjmp.h | 69 + osfmk/arm/simple_lock.h | 194 + osfmk/arm/smp.h | 37 + osfmk/arm/start.s | 434 + osfmk/arm/status.c | 873 + osfmk/arm/status_shared.c | 81 + osfmk/arm/strlcpy.c | 42 + osfmk/arm/strlen.s | 119 + osfmk/arm/strncmp.s | 159 + osfmk/arm/strncpy.c | 42 + osfmk/arm/strnlen.s | 154 + osfmk/arm/task.h | 65 + osfmk/arm/thread.h | 219 + osfmk/arm/trap.c | 897 + osfmk/arm/trap.h | 284 + osfmk/arm/vm_tuning.h | 67 + osfmk/arm/xpr.h | 36 + osfmk/arm64/Makefile | 31 + osfmk/arm64/WKdmCompress_16k.s | 634 + osfmk/arm64/WKdmCompress_4k.s | 632 + osfmk/arm64/WKdmData.s | 330 + osfmk/arm64/WKdmDecompress_16k.s | 428 + osfmk/arm64/WKdmDecompress_4k.s | 428 + osfmk/arm64/alternate_debugger.c | 175 + osfmk/arm64/alternate_debugger.h | 45 + osfmk/arm64/alternate_debugger_asm.s | 65 + osfmk/arm64/arm_vm_init.c | 1203 + osfmk/arm64/asm.h | 189 + osfmk/arm64/bcopy.s | 296 + osfmk/arm64/bsd_arm64.c | 227 + osfmk/arm64/bzero.s | 153 + osfmk/arm64/caches_asm.s | 369 + osfmk/arm64/copyio.c | 311 + osfmk/arm64/cpu.c | 864 + osfmk/arm64/cswitch.s | 239 + osfmk/arm64/dbgwrap.c | 283 + osfmk/arm64/genassym.c | 430 + osfmk/arm64/kpc.c | 1135 + osfmk/arm64/locore.s | 868 + osfmk/arm64/loose_ends.c | 699 + osfmk/arm64/lowglobals.h | 87 + osfmk/arm64/lowmem_vectors.c | 98 + osfmk/arm64/lz4_decode_arm64.s | 333 + osfmk/arm64/lz4_encode_arm64.s | 406 + osfmk/arm64/machine_cpuid.h | 70 + osfmk/arm64/machine_kpc.h | 58 + osfmk/arm64/machine_machdep.h | 42 + osfmk/arm64/machine_routines.c | 2048 + osfmk/arm64/machine_routines_asm.s | 970 + osfmk/arm64/machine_task.c | 251 + osfmk/arm64/monotonic.h | 58 + osfmk/arm64/monotonic_arm64.c | 391 + osfmk/arm64/pcb.c | 878 + osfmk/arm64/pgtrace.c | 594 + osfmk/arm64/pgtrace.h | 163 + osfmk/arm64/pgtrace_decoder.c | 1551 + osfmk/arm64/pgtrace_decoder.h | 40 + osfmk/arm64/pinst.s | 127 + osfmk/arm64/platform_tests.c | 1087 + osfmk/arm64/proc_reg.h | 1401 + osfmk/arm64/sleh.c | 1456 + osfmk/arm64/start.s | 898 + osfmk/arm64/status.c | 1455 + osfmk/arm64/strncmp.s | 187 + osfmk/arm64/strnlen.s | 198 + osfmk/atm/atm.c | 6 +- osfmk/bank/bank.c | 265 +- osfmk/bank/bank_internal.h | 13 +- osfmk/chud/chud_thread.c | 2 + osfmk/chud/chud_xnu_glue.h | 2 + osfmk/chud/chud_xnu_private.h | 2 + osfmk/chud/i386/chud_osfmk_callback_i386.c | 2 + osfmk/conf/Makefile.arm | 10 + osfmk/conf/Makefile.arm64 | 13 + osfmk/conf/Makefile.template | 5 + osfmk/conf/files | 25 +- osfmk/conf/files.arm | 80 + osfmk/conf/files.arm64 | 92 + osfmk/conf/files.x86_64 | 2 + osfmk/console/Makefile | 3 +- osfmk/console/art/scalegear.c | 27 + osfmk/console/serial_console.c | 87 +- osfmk/console/serial_general.c | 57 +- osfmk/console/serial_protos.h | 42 + osfmk/console/video_console.c | 69 +- osfmk/console/video_console.h | 2 + osfmk/console/video_scroll.c | 2 - osfmk/corecrypto/cc/src/cc_abort.c | 36 - osfmk/corecrypto/cc/src/cc_clear.c | 27 + osfmk/corecrypto/cc/src/cc_cmp_safe.c | 24 + osfmk/corecrypto/cc/src/cc_try_abort.c | 60 + osfmk/corecrypto/ccaes/src/aes_tab.c | 1061 + .../ccaes/src/ccaes_ltc_ecb_encrypt_mode.c | 421 + .../ccaes/src/ccaes_private_types.h | 48 + osfmk/corecrypto/ccdbrg/src/ccdrbg_nisthmac.c | 112 +- osfmk/corecrypto/ccdigest/src/ccdigest_init.c | 24 + .../corecrypto/ccdigest/src/ccdigest_update.c | 24 + osfmk/corecrypto/cchmac/src/cchmac.c | 28 + osfmk/corecrypto/cchmac/src/cchmac_final.c | 24 + osfmk/corecrypto/cchmac/src/cchmac_init.c | 24 + osfmk/corecrypto/cchmac/src/cchmac_update.c | 24 + .../corecrypto/ccmode/src/ccmode_ctr_crypt.c | 72 + osfmk/corecrypto/ccmode/src/ccmode_ctr_init.c | 49 + .../corecrypto/ccmode/src/ccmode_ctr_setctr.c | 43 + .../ccmode/src/ccmode_factory_ctr_crypt.c | 41 + osfmk/corecrypto/ccmode/src/ccmode_internal.h | 297 + osfmk/corecrypto/ccn/src/ccn_set.c | 24 + .../ccsha1/src/ccdigest_final_64be.c | 24 + osfmk/corecrypto/ccsha1/src/ccsha1_eay.c | 24 + .../ccsha1/src/ccsha1_initial_state.c | 24 + osfmk/corecrypto/ccsha2/src/ccsha256_K.c | 53 + osfmk/corecrypto/ccsha2/src/ccsha256_di.c | 59 + .../ccsha2/src/ccsha256_initial_state.c | 46 + .../ccsha2/src/ccsha256_ltc_compress.c | 152 + osfmk/corecrypto/ccsha2/src/ccsha256_ltc_di.c | 48 + osfmk/corecrypto/ccsha2/src/ccsha2_internal.h | 63 + osfmk/corpses/corpse.c | 249 +- osfmk/corpses/task_corpse.h | 32 +- osfmk/device/device_port.h | 1 + osfmk/device/iokit_rpc.c | 22 +- osfmk/device/subrs.c | 154 +- osfmk/i386/AT386/conf.c | 5 +- osfmk/i386/AT386/model_dep.c | 385 +- osfmk/i386/Diagnostics.c | 27 +- osfmk/i386/Makefile | 1 + osfmk/i386/acpi.c | 54 +- osfmk/i386/bsd_i386.c | 25 +- osfmk/i386/bsd_i386_native.c | 1 - osfmk/i386/commpage/commpage.c | 116 +- osfmk/i386/commpage/commpage.h | 5 +- osfmk/i386/cpu.c | 4 +- osfmk/i386/cpu_capabilities.h | 11 + osfmk/i386/cpu_data.h | 30 +- osfmk/i386/cpu_threads.c | 32 +- osfmk/i386/cpu_threads.h | 2 + osfmk/i386/cpuid.c | 34 +- osfmk/i386/cpuid.h | 18 + osfmk/i386/fpu.c | 882 +- osfmk/i386/fpu.h | 27 +- osfmk/i386/genassym.c | 25 +- osfmk/i386/hibernate_i386.c | 53 +- osfmk/i386/hibernate_restore.c | 2 +- osfmk/i386/i386_init.c | 111 +- osfmk/i386/i386_lock.s | 517 +- osfmk/i386/i386_timer.c | 44 +- osfmk/i386/i386_vm_init.c | 131 +- osfmk/i386/io_map.c | 4 + osfmk/i386/lapic_native.c | 22 +- osfmk/i386/locks.h | 63 +- osfmk/i386/locks_i386.c | 475 +- osfmk/i386/machine_check.c | 18 +- osfmk/i386/machine_routines.c | 70 +- osfmk/i386/machine_routines.h | 23 +- osfmk/i386/machine_task.c | 22 + osfmk/i386/misc_protos.h | 2 + osfmk/i386/mp.c | 278 +- osfmk/i386/mp.h | 11 +- osfmk/i386/mp_desc.c | 48 +- osfmk/i386/mp_desc.h | 4 +- osfmk/i386/pcb.c | 145 +- osfmk/i386/pcb_native.c | 3 +- osfmk/i386/phys.c | 1 - osfmk/i386/pmCPU.c | 29 +- osfmk/i386/pmap.h | 14 +- osfmk/i386/pmap_common.c | 14 +- osfmk/i386/pmap_internal.h | 23 +- osfmk/i386/pmap_x86_common.c | 108 +- osfmk/i386/postcode.h | 45 +- osfmk/i386/proc_reg.h | 24 +- osfmk/i386/rtclock.c | 16 +- osfmk/i386/rtclock_asm.h | 4 +- osfmk/i386/seg.h | 6 +- osfmk/i386/task.h | 14 +- osfmk/i386/thread.h | 21 +- osfmk/i386/trap.c | 81 +- osfmk/i386/trap.h | 5 +- osfmk/i386/tsc.c | 1 + osfmk/i386/ucode.c | 29 +- osfmk/i386/ucode.h | 27 + osfmk/ipc/flipc.c | 2 +- osfmk/ipc/ipc_importance.c | 102 +- osfmk/ipc/ipc_init.c | 13 +- osfmk/ipc/ipc_kmsg.c | 130 +- osfmk/ipc/ipc_kmsg.h | 14 +- osfmk/ipc/ipc_notify.c | 2 + osfmk/ipc/ipc_object.c | 4 +- osfmk/ipc/ipc_object.h | 2 - osfmk/ipc/ipc_port.c | 488 +- osfmk/ipc/ipc_port.h | 81 +- osfmk/ipc/ipc_pset.c | 51 +- osfmk/ipc/ipc_right.c | 57 +- osfmk/ipc/ipc_types.h | 1 + osfmk/ipc/mach_kernelrpc.c | 16 +- osfmk/ipc/mach_msg.c | 105 +- osfmk/ipc/mach_port.c | 78 +- osfmk/kdp/Makefile | 4 +- osfmk/kdp/kdp.c | 3 +- osfmk/kdp/kdp_core.c | 1033 +- osfmk/kdp/kdp_core.h | 38 +- osfmk/kdp/kdp_dyld.h | 25 +- osfmk/kdp/kdp_internal.h | 10 +- osfmk/kdp/kdp_udp.c | 246 +- osfmk/kdp/ml/arm/kdp_machdep.c | 727 + osfmk/kdp/ml/arm/kdp_vm.c | 355 + osfmk/kdp/ml/i386/kdp_x86_common.c | 13 +- osfmk/kdp/ml/x86_64/kdp_machdep.c | 94 +- osfmk/kdp/ml/x86_64/kdp_vm.c | 9 +- osfmk/kdp/processor_core.c | 738 + osfmk/kdp/processor_core.h | 191 + osfmk/kern/Makefile | 12 +- osfmk/kern/affinity.c | 5 + osfmk/kern/arithmetic_128.h | 102 + osfmk/kern/assert.h | 5 - osfmk/kern/ast.c | 424 +- osfmk/kern/ast.h | 48 +- osfmk/kern/backtrace.c | 104 +- osfmk/kern/bits.h | 24 + osfmk/kern/block_hint.h | 20 + osfmk/kern/bsd_kern.c | 104 +- osfmk/kern/build_config.h | 5 + osfmk/kern/call_entry.h | 8 +- osfmk/kern/clock.c | 1095 +- osfmk/kern/clock.h | 23 +- osfmk/kern/clock_oldops.c | 5 +- osfmk/kern/coalition.c | 248 +- osfmk/kern/coalition.h | 38 +- osfmk/kern/copyout_shim.c | 99 + osfmk/kern/copyout_shim.h | 90 + osfmk/kern/cpu_number.h | 4 - osfmk/kern/cs_blobs.h | 213 + osfmk/kern/debug.c | 1231 +- osfmk/kern/debug.h | 393 +- osfmk/kern/exc_guard.h | 142 + osfmk/kern/exception.c | 104 +- osfmk/kern/gzalloc.c | 175 +- osfmk/kern/host.c | 53 +- osfmk/kern/host.h | 4 + osfmk/kern/ipc_host.c | 75 +- osfmk/kern/ipc_kobject.c | 16 +- osfmk/kern/ipc_kobject.h | 4 +- osfmk/kern/ipc_mig.c | 6 +- osfmk/kern/ipc_tt.c | 266 +- osfmk/kern/kalloc.c | 129 +- osfmk/kern/kalloc.h | 67 +- osfmk/kern/kcdata.h | 63 + osfmk/kern/kern_cdata.c | 15 +- osfmk/kern/kern_cdata.h | 4 +- osfmk/kern/kern_monotonic.c | 523 + osfmk/kern/kern_stackshot.c | 365 +- osfmk/kern/kern_types.h | 60 + osfmk/kern/kext_alloc.c | 14 +- osfmk/kern/kpc.h | 11 + osfmk/kern/kpc_common.c | 54 +- osfmk/kern/kpc_thread.c | 5 + osfmk/kern/ledger.c | 103 +- osfmk/kern/ledger.h | 9 +- osfmk/kern/locks.c | 110 +- osfmk/kern/locks.h | 15 +- osfmk/kern/ltable.c | 6 +- osfmk/kern/machine.c | 18 +- osfmk/kern/machine.h | 29 +- osfmk/kern/memset_s.c | 63 + osfmk/kern/misc_protos.h | 34 +- osfmk/kern/monotonic.h | 116 + osfmk/kern/policy_internal.h | 36 +- osfmk/kern/printf.c | 108 +- osfmk/kern/priority.c | 108 +- osfmk/kern/processor.c | 95 +- osfmk/kern/processor.h | 56 +- osfmk/kern/processor_data.c | 3 + osfmk/kern/processor_data.h | 12 + osfmk/kern/queue.h | 11 + osfmk/kern/sched.h | 17 +- osfmk/kern/sched_dualq.c | 17 +- osfmk/kern/sched_grrr.c | 14 + osfmk/kern/sched_multiq.c | 19 +- osfmk/kern/sched_prim.c | 1478 +- osfmk/kern/sched_prim.h | 140 +- osfmk/kern/sched_proto.c | 20 +- osfmk/kern/sched_traditional.c | 34 +- osfmk/kern/sfi.c | 17 +- osfmk/kern/stack.c | 52 +- osfmk/kern/startup.c | 88 +- osfmk/kern/sync_sema.c | 6 - osfmk/kern/syscall_subr.c | 24 +- osfmk/kern/syscall_sw.c | 4 +- osfmk/kern/syscall_sw.h | 2 + osfmk/kern/task.c | 606 +- osfmk/kern/task.h | 71 +- osfmk/kern/task_policy.c | 445 +- osfmk/kern/telemetry.c | 17 +- osfmk/kern/telemetry.h | 2 +- osfmk/kern/thread.c | 311 +- osfmk/kern/thread.h | 135 +- osfmk/kern/thread_act.c | 49 +- osfmk/kern/thread_call.c | 1259 +- osfmk/kern/thread_call.h | 104 +- osfmk/kern/thread_group.c | 51 + osfmk/kern/thread_group.h | 42 + osfmk/kern/thread_kernel_state.h | 46 + osfmk/kern/thread_policy.c | 284 +- osfmk/kern/timer.c | 4 + osfmk/kern/timer_call.c | 228 +- osfmk/kern/timer_call.h | 9 + osfmk/kern/timer_queue.h | 5 + osfmk/kern/waitq.c | 162 +- osfmk/kern/waitq.h | 33 +- osfmk/kern/work_interval.c | 459 + osfmk/kern/work_interval.h | 86 + osfmk/kern/zalloc.c | 1617 +- osfmk/kern/zalloc.h | 100 +- osfmk/kperf/action.c | 53 +- osfmk/kperf/action.h | 12 +- .../spl.c => osfmk/kperf/arm/kperf_meminfo.c | 122 +- osfmk/kperf/arm/kperf_mp.c | 100 + osfmk/kperf/buffer.h | 43 +- osfmk/kperf/callstack.c | 95 +- osfmk/kperf/kperf.c | 87 +- osfmk/kperf/kperf.h | 5 + osfmk/kperf/kperf_arch.h | 4 +- osfmk/kperf/kperf_timer.c | 71 +- osfmk/kperf/kperf_timer.h | 25 + osfmk/kperf/kperfbsd.c | 141 +- osfmk/kperf/pet.c | 3 +- osfmk/kperf/sample.h | 2 +- osfmk/kperf/thread_samplers.c | 91 +- osfmk/kperf/thread_samplers.h | 13 +- osfmk/kperf/x86_64/kperf_mp.c | 50 +- osfmk/libsa/arm/types.h | 75 + osfmk/libsa/machine/types.h | 2 + osfmk/libsa/string.h | 50 + osfmk/mach/Makefile | 2 + osfmk/mach/arm/Makefile | 32 + osfmk/mach/arm/_structs.h | 323 + osfmk/mach/arm/boolean.h | 70 + osfmk/mach/arm/exception.h | 66 + osfmk/mach/arm/kern_return.h | 74 + osfmk/mach/arm/ndr_def.h | 45 + osfmk/mach/arm/processor_info.h | 57 + osfmk/mach/arm/rpc.h | 35 + osfmk/mach/arm/sdt_isa.h | 440 + osfmk/mach/arm/syscall_sw.h | 123 + .../spl.h => osfmk/mach/arm/thread_state.h | 53 +- osfmk/mach/arm/thread_status.h | 708 + osfmk/mach/arm/vm_param.h | 219 + osfmk/mach/arm/vm_types.h | 161 + osfmk/mach/arm64/Makefile | 28 + osfmk/mach/coalition.h | 52 +- osfmk/mach/error.h | 2 +- osfmk/mach/exc.defs | 16 - osfmk/mach/exception_types.h | 6 +- osfmk/mach/host_priv.defs | 30 +- osfmk/mach/host_special_ports.h | 9 +- osfmk/mach/i386/_structs.h | 465 +- osfmk/mach/i386/fp_reg.h | 84 +- osfmk/mach/i386/thread_state.h | 4 + osfmk/mach/i386/thread_status.h | 71 +- osfmk/mach/i386/vm_param.h | 20 +- osfmk/mach/mach_exc.defs | 16 - osfmk/mach/mach_host.defs | 16 +- osfmk/mach/mach_traps.h | 12 + osfmk/mach/mach_types.defs | 6 +- osfmk/mach/mach_types.h | 1 + osfmk/mach/mach_vm.defs | 32 +- osfmk/mach/machine.h | 4 +- osfmk/mach/machine/Makefile | 2 +- osfmk/mach/machine/_structs.h | 40 + osfmk/mach/machine/asm.h | 4 + osfmk/mach/machine/boolean.h | 2 + osfmk/mach/machine/exception.h | 2 + osfmk/mach/machine/kern_return.h | 2 + osfmk/mach/machine/ndr_def.h | 2 + osfmk/mach/machine/processor_info.h | 2 + osfmk/mach/machine/rpc.h | 2 + osfmk/mach/machine/sdt_isa.h | 2 + osfmk/mach/machine/syscall_sw.h | 2 + osfmk/mach/machine/thread_state.h | 2 + osfmk/mach/machine/thread_status.h | 2 + osfmk/mach/machine/vm_param.h | 2 + osfmk/mach/machine/vm_types.h | 2 + osfmk/mach/memory_object_control.defs | 6 +- osfmk/mach/memory_object_types.h | 47 +- osfmk/mach/message.h | 77 +- osfmk/mach/mig.h | 2 +- osfmk/mach/mig_strncpy_zerofill_support.h | 27 + osfmk/mach/mig_voucher_support.h | 27 + osfmk/mach/port.h | 4 +- osfmk/mach/shared_memory_server.h | 14 + osfmk/mach/shared_region.h | 22 +- osfmk/mach/syscall_sw.h | 1 + osfmk/mach/task.defs | 12 + osfmk/mach/task_info.h | 74 +- osfmk/mach/task_inspect.h | 54 + osfmk/mach/thread_act.defs | 1 - osfmk/mach/thread_policy.h | 3 +- osfmk/mach/thread_status.h | 1 + osfmk/mach/vm_map.defs | 17 +- osfmk/mach/vm_param.h | 127 +- osfmk/mach/vm_prot.h | 6 - osfmk/mach/vm_purgable.h | 23 + osfmk/mach/vm_statistics.h | 83 +- osfmk/mach/vm_types.h | 60 +- osfmk/mach_debug/hash_info.h | 2 + osfmk/mach_debug/mach_debug_types.defs | 2 +- osfmk/mach_debug/mach_debug_types.h | 17 +- osfmk/mach_debug/zone_info.h | 24 +- osfmk/machine/Makefile | 6 + osfmk/machine/asm.h | 4 + osfmk/machine/atomic.h | 2 + osfmk/machine/commpage.h | 3 + osfmk/machine/config.h | 40 + osfmk/machine/cpu_affinity.h | 2 + osfmk/machine/cpu_capabilities.h | 4 + osfmk/machine/cpu_data.h | 2 + osfmk/machine/cpu_number.h | 2 + osfmk/machine/endian.h | 2 + osfmk/machine/io_map_entries.h | 2 + osfmk/machine/lock.h | 2 + osfmk/machine/locks.h | 2 + osfmk/machine/lowglobals.h | 4 + osfmk/machine/machine_cpu.h | 2 + osfmk/machine/machine_cpuid.h | 6 + osfmk/machine/machine_kpc.h | 4 + osfmk/machine/machine_routines.h | 2 + osfmk/machine/machine_rpc.h | 2 + osfmk/machine/machlimits.h | 2 + osfmk/machine/machparam.h | 2 + osfmk/machine/monotonic.h | 70 + osfmk/machine/pal_hibernate.h | 2 + osfmk/machine/pal_routines.h | 2 + osfmk/machine/pmap.h | 2 + osfmk/machine/sched_param.h | 2 + osfmk/machine/setjmp.h | 2 + osfmk/machine/simple_lock.h | 2 + osfmk/machine/smp.h | 2 + osfmk/machine/task.h | 4 + osfmk/machine/thread.h | 2 + osfmk/machine/trap.h | 2 + osfmk/machine/vm_tuning.h | 2 + osfmk/machine/xpr.h | 2 + osfmk/prng/random.c | 388 +- osfmk/prng/random.h | 33 +- osfmk/profiling/machine/profile-md.h | 2 + osfmk/vm/WKdm_new.h | 25 + osfmk/vm/bsd_vm.c | 70 +- osfmk/vm/device_vm.c | 179 +- osfmk/vm/lz4.c | 1 + osfmk/vm/lz4.h | 2 + osfmk/vm/lz4_assembly_select.h | 5 +- osfmk/vm/memory_object.c | 196 +- osfmk/vm/memory_object.h | 8 +- osfmk/vm/pmap.h | 97 +- osfmk/vm/vm32_user.c | 6 +- osfmk/vm/vm_apple_protect.c | 89 +- osfmk/vm/vm_compressor.c | 468 +- osfmk/vm/vm_compressor.h | 143 +- osfmk/vm/vm_compressor_algorithms.c | 130 +- osfmk/vm/vm_compressor_algorithms.h | 19 +- osfmk/vm/vm_compressor_backing_store.c | 396 +- osfmk/vm/vm_compressor_backing_store.h | 12 +- osfmk/vm/vm_compressor_pager.c | 56 +- osfmk/vm/vm_debug.c | 23 +- osfmk/vm/vm_fault.c | 719 +- osfmk/vm/vm_fault.h | 15 + osfmk/vm/vm_fourk_pager.c | 95 +- osfmk/vm/vm_init.c | 31 +- osfmk/vm/vm_init.h | 1 - osfmk/vm/vm_kern.c | 356 +- osfmk/vm/vm_kern.h | 211 +- osfmk/vm/vm_map.c | 3197 +- osfmk/vm/vm_map.h | 160 +- osfmk/vm/vm_map_store_rb.c | 9 + osfmk/vm/vm_object.c | 1288 +- osfmk/vm/vm_object.h | 148 +- osfmk/vm/vm_page.h | 235 +- osfmk/vm/vm_pageout.c | 3166 +- osfmk/vm/vm_pageout.h | 84 +- osfmk/vm/vm_phantom_cache.c | 12 + osfmk/vm/vm_protos.h | 75 +- osfmk/vm/vm_purgeable.c | 6 + osfmk/vm/vm_resident.c | 1432 +- osfmk/vm/vm_shared_region.c | 140 +- osfmk/vm/vm_shared_region.h | 2 + osfmk/vm/vm_swapfile_pager.c | 85 +- osfmk/vm/vm_user.c | 768 +- osfmk/x86_64/Makefile | 14 +- osfmk/x86_64/bzero.s | 7 + osfmk/x86_64/copyio.c | 22 +- osfmk/x86_64/idt64.s | 10 +- osfmk/x86_64/kpc_x86.c | 3 - osfmk/x86_64/locore.s | 12 +- osfmk/x86_64/loose_ends.c | 48 +- osfmk/x86_64/lz4_decode_x86_64.s | 27 + osfmk/x86_64/monotonic.h | 42 + osfmk/x86_64/monotonic_x86_64.c | 272 + osfmk/x86_64/pmap.c | 138 +- pexpert/arm/pe_bootargs.c | 12 + pexpert/arm/pe_consistent_debug.c | 68 + pexpert/arm/pe_identify_machine.c | 670 + pexpert/arm/pe_init.c | 661 + pexpert/arm/pe_kprintf.c | 141 + pexpert/arm/pe_serial.c | 831 + pexpert/conf/Makefile.arm | 7 + pexpert/conf/Makefile.arm64 | 7 + pexpert/conf/Makefile.template | 1 + pexpert/conf/files.arm | 8 + pexpert/conf/files.arm64 | 8 + pexpert/gen/bootargs.c | 13 + pexpert/gen/device_tree.c | 99 +- pexpert/gen/pe_gen.c | 7 + pexpert/i386/pe_init.c | 47 +- pexpert/pexpert/arm/AIC.h | 88 + pexpert/pexpert/arm/Makefile | 29 + pexpert/pexpert/arm/PL192_VIC.h | 33 + pexpert/pexpert/arm/S3cUART.h | 64 + pexpert/pexpert/arm/S7002.h | 41 + pexpert/pexpert/arm/T8002.h | 45 + pexpert/pexpert/arm/board_config.h | 40 + pexpert/pexpert/arm/boot.h | 67 + pexpert/pexpert/arm/consistent_debug.h | 117 + pexpert/pexpert/arm/protos.h | 33 + pexpert/pexpert/arm64/AIC.h | 32 + pexpert/pexpert/arm64/AMCC.h | 21 + pexpert/pexpert/arm64/Makefile | 34 + pexpert/pexpert/arm64/S3c2410x.h | 544 + pexpert/pexpert/arm64/S5L8960X.h | 19 + pexpert/pexpert/arm64/S8000.h | 17 + pexpert/pexpert/arm64/T7000.h | 19 + pexpert/pexpert/arm64/T8010.h | 42 + pexpert/pexpert/arm64/arm64_common.h | 157 + pexpert/pexpert/arm64/board_config.h | 110 + pexpert/pexpert/arm64/boot.h | 78 + pexpert/pexpert/arm64/cyclone.h | 25 + pexpert/pexpert/arm64/hurricane.h | 26 + pexpert/pexpert/arm64/twister.h | 16 + pexpert/pexpert/arm64/typhoon.h | 16 + pexpert/pexpert/device_tree.h | 81 +- pexpert/pexpert/i386/boot.h | 6 +- pexpert/pexpert/machine/boot.h | 4 + pexpert/pexpert/machine/protos.h | 2 + pexpert/pexpert/pexpert.h | 43 + san/Kasan.exports | 2 + san/Kasan_kasan.exports | 112 + san/Makefile | 93 + san/conf/Makefile | 43 + san/conf/Makefile.template | 91 + san/conf/Makefile.x86_64 | 7 + san/conf/files | 5 + san/conf/files.x86_64 | 5 + san/kasan-arm64.c | 322 + san/kasan-blacklist | 24 + san/kasan-blacklist-arm64 | 10 + san/kasan-blacklist-x86_64 | 71 + san/kasan-fakestack.c | 339 + san/kasan-memintrinsics.c | 167 + san/kasan-test-arm64.s | 58 + san/kasan-test-x86_64.s | 117 + san/kasan-test.c | 624 + san/kasan-x86_64.c | 342 + san/kasan.c | 1241 + san/kasan.h | 244 + san/kasan_dynamic_blacklist.c | 473 + san/kasan_internal.h | 148 + san/memintrinsics.h | 82 + san/tools/generate_dynamic_blacklist.py | 46 + san/tools/kasan_install | 159 + security/Makefile | 40 +- security/_label.h | 8 + security/conf/Makefile.arm | 7 + security/conf/Makefile.arm64 | 7 + security/conf/Makefile.template | 1 + security/conf/files | 1 + security/conf/files.arm | 0 security/conf/files.arm64 | 0 security/mac.h | 21 - security/mac_alloc.c | 4 +- security/mac_audit.c | 38 +- security/mac_base.c | 44 +- security/mac_framework.h | 11 +- security/mac_internal.h | 44 +- security/mac_kext.c | 28 + security/mac_mach.c | 115 +- security/mac_mach_internal.h | 22 +- security/mac_policy.h | 205 +- security/mac_priv.c | 6 + security/mac_process.c | 59 +- security/mac_pty.c | 28 + security/mac_skywalk.c | 52 + security/mac_socket.c | 61 +- security/mac_vfs.c | 390 +- tools/lldbmacros/Makefile | 4 + tools/lldbmacros/README.md | 2 + tools/lldbmacros/apic.py | 0 tools/lldbmacros/atm.py | 0 tools/lldbmacros/bank.py | 0 tools/lldbmacros/core/__init__.py | 0 tools/lldbmacros/core/caching.py | 0 tools/lldbmacros/core/configuration.py | 0 tools/lldbmacros/core/cvalue.py | 28 +- tools/lldbmacros/core/kernelcore.py | 54 +- tools/lldbmacros/core/lazytarget.py | 0 tools/lldbmacros/core/operating_system.py | 17 +- tools/lldbmacros/core/standard.py | 0 tools/lldbmacros/core/xnu_lldb_init.py | 3 + tools/lldbmacros/ioreg.py | 17 +- tools/lldbmacros/ipc.py | 31 +- tools/lldbmacros/ipcimportancedetail.py | 0 tools/lldbmacros/kasan.py | 345 + tools/lldbmacros/kauth.py | 0 tools/lldbmacros/kcdata.py | 236 +- tools/lldbmacros/kdp.py | 2 +- tools/lldbmacros/kevent.py | 381 + tools/lldbmacros/ktrace.py | 6 +- tools/lldbmacros/macho.py | 0 tools/lldbmacros/mbufdefines.py | 2 +- tools/lldbmacros/mbufs.py | 2 +- tools/lldbmacros/memory.py | 485 +- tools/lldbmacros/misc.py | 420 +- tools/lldbmacros/net.py | 2 - tools/lldbmacros/netdefines.py | 0 tools/lldbmacros/ntstat.py | 175 + tools/lldbmacros/pci.py | 0 tools/lldbmacros/pgtrace.py | 0 tools/lldbmacros/plugins/__init__.py | 0 tools/lldbmacros/plugins/iosspeedtracer.py | 0 tools/lldbmacros/plugins/speedtracer.py | 0 tools/lldbmacros/plugins/zprint_perf_log.py | 0 tools/lldbmacros/pmap.py | 33 +- tools/lldbmacros/process.py | 272 +- tools/lldbmacros/routedefines.py | 0 tools/lldbmacros/scheduler.py | 368 +- tools/lldbmacros/structanalyze.py | 0 tools/lldbmacros/userspace.py | 44 +- .../lldbmacros/usertaskdebugging/__init__.py | 0 .../lldbmacros/usertaskdebugging/gdbserver.py | 2 +- .../lldbmacros/usertaskdebugging/interface.py | 0 .../usertaskdebugging/rsprotocol.py | 0 tools/lldbmacros/usertaskdebugging/target.py | 2 +- .../usertaskdebugging/userprocess.py | 18 +- tools/lldbmacros/usertaskgdbserver.py | 0 tools/lldbmacros/utils.py | 0 tools/lldbmacros/waitq.py | 0 tools/lldbmacros/xnu.py | 161 +- tools/lldbmacros/xnudefines.py | 100 +- tools/lldbmacros/xnutriage.py | 2 +- tools/stackshot/Makefile | 24 - tools/stackshot/stackshot.c | 192 - tools/tests/MPMMTest/MPMMtest_run.sh | 24 +- tools/tests/Makefile | 4 +- tools/tests/Makefile.common | 4 +- tools/tests/darwintests/Makefile | 83 +- tools/tests/darwintests/avx.c | 736 + tools/tests/darwintests/contextswitch.c | 285 + tools/tests/darwintests/cpucount.c | 266 + tools/tests/darwintests/data_protection.c | 2 +- .../disk_mount_conditioner-entitlements.plist | 8 + .../darwintests/disk_mount_conditioner.c | 388 + .../tests/darwintests/gettimeofday_29192647.c | 47 + tools/tests/darwintests/ioperf.c | 256 + tools/tests/darwintests/kdebug.c | 139 +- tools/tests/darwintests/kevent_pty.c | 259 + tools/tests/darwintests/kevent_qos.c | 908 + tools/tests/darwintests/kpc.c | 68 + tools/tests/darwintests/kperf.c | 707 +- tools/tests/darwintests/kperf_backtracing.c | 35 +- .../darwintests/kqueue_add_and_trigger.c | 37 + tools/tests/darwintests/kqueue_close.c | 77 + .../tests/darwintests/kqueue_fifo_18776047.c | 4 +- .../kqueue_file_tests.c | 488 +- tools/tests/darwintests/kqueue_timer_tests.c | 437 + .../com.apple.xnu.test.kevent_qos.plist | 24 + .../tests/darwintests/mach_continuous_time.c | 209 +- .../mach_port_deallocate_21692215.c | 38 + tools/tests/darwintests/mach_port_mod_refs.c | 92 + .../darwintests/memorystatus_zone_test.c | 393 + tools/tests/darwintests/monotonic_core.c | 236 + tools/tests/darwintests/netbsd_utimensat.c | 191 + .../tests/darwintests/ntp_adjtime_29192647.c | 371 + tools/tests/darwintests/perf_compressor.c | 9 +- tools/tests/darwintests/perf_exit.c | 97 +- tools/tests/darwintests/perf_kdebug.c | 6 +- .../poll_select_kevent_paired_fds.c | 158 +- .../darwintests/private_entitlement.plist | 8 + tools/tests/darwintests/proc_info.c | 322 + .../tests/darwintests/settimeofday_29193041.c | 229 + .../settimeofday_29193041.entitlements | 8 + .../settimeofday_29193041_entitled.c | 214 + tools/tests/darwintests/sigchld_return.c | 50 + tools/tests/darwintests/sigcont_return.c | 28 + tools/tests/darwintests/stackshot.m | 422 + .../task_for_pid_entitlement.plist | 10 + tools/tests/darwintests/task_info.c | 907 + tools/tests/darwintests/task_info_28439149.c | 81 + tools/tests/darwintests/task_inspect.c | 19 +- .../darwintests/thread_group_set_32261625.c | 62 + tools/tests/darwintests/utimensat.c | 77 + tools/tests/darwintests/work_interval_test.c | 122 + .../work_interval_test.entitlements | 8 + tools/tests/darwintests/xnu_quick_test.c | 74 +- .../darwintests/xnu_quick_test_helpers.c | 114 + .../darwintests/xnu_quick_test_helpers.h | 16 + tools/tests/execperf/exit-asm.S | 18 + tools/tests/execperf/exit.c | 2 + tools/tests/jitter/Makefile | 12 +- tools/tests/jitter/cpu_number.s | 33 - tools/tests/jitter/timer_jitter.c | 8 +- tools/tests/kqueue_tests/Makefile | 31 - tools/tests/kqueue_tests/kqueue_timer_tests.c | 255 - tools/tests/libMicro/Makefile.com.Darwin | 0 .../perf_index/PerfIndex_COPS_Module/PITest.h | 2 +- .../PerfIndex.xcodeproj/project.pbxproj | 4 - tools/tests/perf_index/test_controller.py | 0 tools/tests/perf_index/test_fault_helper.c | 4 + tools/tests/zero-to-n/Makefile | 2 +- tools/tests/zero-to-n/zero-to-n.c | 43 +- tools/trace/kqtrace.lua | 339 + tools/trace/parse_ipc_trace.py | 28 +- 1695 files changed, 278930 insertions(+), 53551 deletions(-) create mode 100644 EXTERNAL_HEADERS/architecture/arm/Makefile create mode 100644 EXTERNAL_HEADERS/architecture/arm/arm_neon.h create mode 100644 EXTERNAL_HEADERS/corecrypto/cc_runtime_config.h create mode 100644 EXTERNAL_HEADERS/corecrypto/ccchacha20poly1305.h create mode 100644 EXTERNAL_HEADERS/corecrypto/fipspost_trace.h create mode 100644 EXTERNAL_HEADERS/mach-o/arm/reloc.h create mode 100644 EXTERNAL_HEADERS/mach-o/arm64/reloc.h create mode 100644 bsd/arm/Makefile create mode 100644 bsd/arm/_limits.h create mode 100644 bsd/arm/_mcontext.h create mode 100644 bsd/arm/_param.h create mode 100644 bsd/arm/_types.h create mode 100644 bsd/arm/disklabel.h create mode 100644 bsd/arm/endian.h create mode 100644 bsd/arm/exec.h create mode 100644 bsd/arm/fasttrap_isa.h create mode 100644 bsd/arm/limits.h create mode 100644 bsd/arm/param.h create mode 100644 bsd/arm/profile.h create mode 100644 bsd/arm/psl.h create mode 100644 bsd/arm/ptrace.h create mode 100644 bsd/arm/reboot.h create mode 100644 bsd/arm/reg.h create mode 100644 bsd/arm/signal.h create mode 100644 bsd/arm/types.h create mode 100644 bsd/arm/vmparam.h create mode 100644 bsd/conf/Makefile.arm create mode 100644 bsd/conf/Makefile.arm64 create mode 100644 bsd/conf/files.arm create mode 100644 bsd/conf/files.arm64 create mode 100644 bsd/dev/arm/conf.c create mode 100644 bsd/dev/arm/cons.c create mode 100644 bsd/dev/arm/cpu_in_cksum.s create mode 100644 bsd/dev/arm/disassembler.c create mode 100644 bsd/dev/arm/dtrace_isa.c create mode 100644 bsd/dev/arm/dtrace_subr_arm.c create mode 100644 bsd/dev/arm/fasttrap_isa.c create mode 100644 bsd/dev/arm/fbt_arm.c create mode 100644 bsd/dev/arm/kern_machdep.c create mode 100644 bsd/dev/arm/km.c create mode 100644 bsd/dev/arm/munge.c create mode 100644 bsd/dev/arm/pci_device.h create mode 100644 bsd/dev/arm/pio.h create mode 100644 bsd/dev/arm/sdt_arm.c create mode 100644 bsd/dev/arm/stubs.c create mode 100644 bsd/dev/arm/sysctl.c create mode 100644 bsd/dev/arm/systemcalls.c create mode 100644 bsd/dev/arm/table_inline.h create mode 100644 bsd/dev/arm/unix_signal.c create mode 100644 bsd/dev/arm64/conf.c create mode 100644 bsd/dev/arm64/cpu_in_cksum.s create mode 100644 bsd/dev/arm64/disassembler.c create mode 100644 bsd/dev/arm64/dtrace_isa.c create mode 100644 bsd/dev/arm64/dtrace_subr_arm.c create mode 100644 bsd/dev/arm64/fasttrap_isa.c create mode 100644 bsd/dev/arm64/fbt_arm.c create mode 100644 bsd/dev/arm64/sdt_arm.c create mode 100644 bsd/dev/arm64/sysctl.c create mode 100644 bsd/dev/dtrace/scripts/vm_map_delete_permanent.d create mode 100644 bsd/dev/monotonic.c create mode 100644 bsd/kern/kern_ntptime.c create mode 100644 bsd/kern/subr_eventhandler.c create mode 100644 bsd/machine/smp.h create mode 100644 bsd/man/man2/fs_snapshot_create.2 create mode 100644 bsd/man/man2/fs_snapshot_delete.2 create mode 100644 bsd/man/man2/fs_snapshot_list.2 create mode 100644 bsd/man/man2/fs_snapshot_rename.2 create mode 100644 bsd/man/man2/fsgetpath.2 create mode 100644 bsd/man/man2/futimens.2 delete mode 100644 bsd/man/man2/peeloff.2 delete mode 100644 bsd/man/man2/profil.2 create mode 100644 bsd/man/man2/setattrlistat.2 create mode 100644 bsd/man/man2/utimensat.2 create mode 100644 bsd/man/man9/monotonic.9 delete mode 100644 bsd/net/altq/altq_cbq.c delete mode 100644 bsd/net/altq/altq_fairq.c delete mode 100644 bsd/net/altq/altq_hfsc.c delete mode 100644 bsd/net/altq/altq_priq.c delete mode 100644 bsd/net/altq/altq_qfq.c delete mode 100644 bsd/net/altq/altq_subr.c delete mode 100644 bsd/net/altq/altq_var.h delete mode 100644 bsd/net/altq/if_altq.h delete mode 100644 bsd/net/classq/classq_blue.c delete mode 100644 bsd/net/classq/classq_red.c delete mode 100644 bsd/net/classq/classq_rio.c create mode 100644 bsd/net/if_fake.c create mode 100644 bsd/net/if_fake_var.h create mode 100644 bsd/net/if_llatbl.c create mode 100644 bsd/net/if_llatbl.h create mode 100644 bsd/net/net_api_stats.h create mode 100644 bsd/net/nwk_wq.c create mode 100644 bsd/net/nwk_wq.h create mode 100644 bsd/net/pf_pbuf.c create mode 100644 bsd/net/pf_pbuf.h delete mode 100644 bsd/net/pktsched/pktsched_cbq.c delete mode 100644 bsd/net/pktsched/pktsched_fairq.c delete mode 100644 bsd/net/pktsched/pktsched_hfsc.c delete mode 100644 bsd/net/pktsched/pktsched_priq.c delete mode 100644 bsd/net/pktsched/pktsched_rmclass.c delete mode 100644 bsd/net/pktsched/pktsched_rmclass_debug.h create mode 100644 bsd/net/skmem_sysctl.c rename bsd/netinet/{cpu_in_cksum.c => cpu_in_cksum_gen.c} (71%) create mode 100644 bsd/netinet/in_stat.c create mode 100644 bsd/netinet/in_stat.h create mode 100644 bsd/netinet6/esp_chachapoly.c create mode 100644 bsd/netinet6/esp_chachapoly.h create mode 100644 bsd/sys/_types/_caddr_t.h create mode 100644 bsd/sys/_types/_u_char.h create mode 100644 bsd/sys/_types/_u_int.h create mode 100644 bsd/sys/_types/_u_short.h create mode 100644 bsd/sys/_types/_user32_ntptimeval.h create mode 100644 bsd/sys/_types/_user32_timex.h create mode 100644 bsd/sys/_types/_user64_ntptimeval.h create mode 100644 bsd/sys/_types/_user64_timex.h create mode 100644 bsd/sys/commpage.h create mode 100644 bsd/sys/eventhandler.h create mode 100644 bsd/sys/monotonic.h create mode 100644 bsd/sys/systrace_args.h create mode 100644 bsd/sys/timex.h create mode 100644 bsd/vfs/vfs_disk_conditioner.c create mode 100644 bsd/vfs/vfs_disk_conditioner.h create mode 100644 config/BSDKernel.arm.exports create mode 100644 config/BSDKernel.arm64.exports create mode 100644 config/IOKit.arm.exports create mode 100644 config/IOKit.arm64.exports create mode 100644 config/Libkern.arm.exports create mode 100644 config/Libkern.arm64.exports create mode 100644 config/MACFramework.arm.exports create mode 100644 config/MACFramework.arm64.exports create mode 100644 config/MASTER.arm create mode 100644 config/MASTER.arm64 create mode 100644 config/Mach.arm.exports create mode 100644 config/Mach.arm64.exports create mode 100644 config/Private.arm.exports create mode 100644 config/Private.arm64.exports create mode 100644 config/System.kext/PlugIns/Kasan.kext/Info.plist create mode 100644 config/Unsupported.arm.exports create mode 100644 config/Unsupported.arm64.exports create mode 100644 iokit/conf/Makefile.arm create mode 100644 iokit/conf/Makefile.arm64 create mode 100644 iokit/conf/files.arm create mode 100644 iokit/conf/files.arm64 mode change 120000 => 100644 libkdd/kcdata.h create mode 100644 libkdd/tests/stackshot-sample-coalitions create mode 100644 libkdd/tests/stackshot-sample-coalitions.plist.gz create mode 100644 libkdd/tests/stackshot-sample-instrs-cycles create mode 100644 libkdd/tests/stackshot-sample-instrs-cycles.plist.gz create mode 100644 libkdd/tests/stackshot-sample-thread-groups create mode 100644 libkdd/tests/stackshot-sample-thread-groups.plist.gz create mode 100644 libkdd/tests/stackshot-sample-thread-policy create mode 100644 libkdd/tests/stackshot-sample-thread-policy.plist.gz mode change 100755 => 100644 libkern/c++/Tests/TestSerialization/test1/test1_main.cpp create mode 100644 libkern/conf/Makefile.arm create mode 100644 libkern/conf/Makefile.arm64 create mode 100644 libkern/conf/files.arm create mode 100644 libkern/crypto/corecrypto_chacha20poly1305.c mode change 100644 => 100755 libkern/kxld/tests/loadtest.py create mode 100644 libkern/libkern/arm/Makefile create mode 100644 libkern/libkern/arm/OSByteOrder.h create mode 100644 libkern/libkern/crypto/chacha20poly1305.h create mode 100644 libkern/os/reason_private.h delete mode 100644 libkern/zlib/gzio.c create mode 100644 libsa/conf/Makefile.arm create mode 100644 libsa/conf/Makefile.arm64 create mode 100644 libsa/conf/files.arm create mode 100644 libsa/conf/files.arm64 create mode 100644 libsyscall/Platforms/iPhoneOS/arm/syscall.map create mode 100644 libsyscall/Platforms/iPhoneOS/arm64/syscall.map delete mode 100644 libsyscall/wrappers/__commpage_gettimeofday.s create mode 100644 libsyscall/wrappers/pid_shutdown_networking.c create mode 100644 libsyscall/wrappers/utimensat.c create mode 100644 osfmk/arm/Makefile create mode 100644 osfmk/arm/WKdmCompress_new.s create mode 100644 osfmk/arm/WKdmData_new.s create mode 100644 osfmk/arm/WKdmDecompress_new.s create mode 100644 osfmk/arm/arch.h create mode 100644 osfmk/arm/arm_init.c create mode 100644 osfmk/arm/arm_timer.c create mode 100644 osfmk/arm/arm_vm_init.c create mode 100644 osfmk/arm/asm.h create mode 100644 osfmk/arm/atomic.h create mode 100644 osfmk/arm/bcopy.s create mode 100644 osfmk/arm/bsd_arm.c create mode 100644 osfmk/arm/bzero.s create mode 100644 osfmk/arm/caches.c create mode 100644 osfmk/arm/caches_asm.s create mode 100644 osfmk/arm/caches_internal.h create mode 100644 osfmk/arm/commpage/commpage.c create mode 100644 osfmk/arm/commpage/commpage.h create mode 100644 osfmk/arm/commpage/commpage_sigs.h create mode 100644 osfmk/arm/conf.c create mode 100644 osfmk/arm/cpu.c create mode 100644 osfmk/arm/cpu_affinity.h create mode 100644 osfmk/arm/cpu_capabilities.h create mode 100644 osfmk/arm/cpu_common.c create mode 100644 osfmk/arm/cpu_data.h create mode 100644 osfmk/arm/cpu_data_internal.h create mode 100644 osfmk/arm/cpu_internal.h create mode 100644 osfmk/arm/cpu_number.h create mode 100644 osfmk/arm/cpuid.c create mode 100644 osfmk/arm/cpuid.h create mode 100644 osfmk/arm/cpuid_internal.h create mode 100644 osfmk/arm/cswitch.s create mode 100644 osfmk/arm/data.s create mode 100644 osfmk/arm/dbgwrap.c create mode 100644 osfmk/arm/dbgwrap.h create mode 100644 osfmk/arm/exception.h create mode 100644 osfmk/arm/genassym.c create mode 100644 osfmk/arm/globals_asm.h create mode 100644 osfmk/arm/hw_lock_types.h create mode 100644 osfmk/arm/io_map.c create mode 100644 osfmk/arm/io_map_entries.h create mode 100644 osfmk/arm/kpc_arm.c create mode 100644 osfmk/arm/lock.h create mode 100644 osfmk/arm/locks.h create mode 100644 osfmk/arm/locks_arm.c create mode 100644 osfmk/arm/locore.s create mode 100644 osfmk/arm/loose_ends.c create mode 100644 osfmk/arm/lowglobals.h create mode 100644 osfmk/arm/lowmem_vectors.c create mode 100644 osfmk/arm/lz4_decode_armv7NEON.s create mode 100644 osfmk/arm/lz4_encode_armv7.s create mode 100644 osfmk/arm/machdep_call.c create mode 100644 osfmk/arm/machdep_call.h create mode 100644 osfmk/arm/machine_cpu.h create mode 100644 osfmk/arm/machine_cpuid.c create mode 100644 osfmk/arm/machine_cpuid.h create mode 100644 osfmk/arm/machine_kpc.h create mode 100644 osfmk/arm/machine_routines.c create mode 100644 osfmk/arm/machine_routines.h create mode 100644 osfmk/arm/machine_routines_asm.s create mode 100644 osfmk/arm/machine_routines_common.c create mode 100644 osfmk/arm/machine_task.c create mode 100644 osfmk/arm/machlimits.h create mode 100644 osfmk/arm/machparam.h create mode 100644 osfmk/arm/misc_protos.h create mode 100644 osfmk/arm/model_dep.c create mode 100644 osfmk/arm/monotonic.h create mode 100644 osfmk/arm/monotonic_arm.c create mode 100644 osfmk/arm/pal_routines.c create mode 100644 osfmk/arm/pal_routines.h create mode 100644 osfmk/arm/pcb.c create mode 100644 osfmk/arm/pmap.c create mode 100644 osfmk/arm/pmap.h create mode 100644 osfmk/arm/proc_reg.h create mode 100644 osfmk/arm/rtclock.c create mode 100644 osfmk/arm/rtclock.h create mode 100644 osfmk/arm/sched_param.h create mode 100644 osfmk/arm/setjmp.h create mode 100644 osfmk/arm/simple_lock.h create mode 100644 osfmk/arm/smp.h create mode 100644 osfmk/arm/start.s create mode 100644 osfmk/arm/status.c create mode 100644 osfmk/arm/status_shared.c create mode 100644 osfmk/arm/strlcpy.c create mode 100644 osfmk/arm/strlen.s create mode 100644 osfmk/arm/strncmp.s create mode 100644 osfmk/arm/strncpy.c create mode 100644 osfmk/arm/strnlen.s create mode 100644 osfmk/arm/task.h create mode 100644 osfmk/arm/thread.h create mode 100644 osfmk/arm/trap.c create mode 100644 osfmk/arm/trap.h create mode 100644 osfmk/arm/vm_tuning.h create mode 100644 osfmk/arm/xpr.h create mode 100644 osfmk/arm64/Makefile create mode 100644 osfmk/arm64/WKdmCompress_16k.s create mode 100644 osfmk/arm64/WKdmCompress_4k.s create mode 100644 osfmk/arm64/WKdmData.s create mode 100644 osfmk/arm64/WKdmDecompress_16k.s create mode 100644 osfmk/arm64/WKdmDecompress_4k.s create mode 100644 osfmk/arm64/alternate_debugger.c create mode 100644 osfmk/arm64/alternate_debugger.h create mode 100644 osfmk/arm64/alternate_debugger_asm.s create mode 100644 osfmk/arm64/arm_vm_init.c create mode 100644 osfmk/arm64/asm.h create mode 100644 osfmk/arm64/bcopy.s create mode 100644 osfmk/arm64/bsd_arm64.c create mode 100644 osfmk/arm64/bzero.s create mode 100644 osfmk/arm64/caches_asm.s create mode 100644 osfmk/arm64/copyio.c create mode 100644 osfmk/arm64/cpu.c create mode 100644 osfmk/arm64/cswitch.s create mode 100644 osfmk/arm64/dbgwrap.c create mode 100644 osfmk/arm64/genassym.c create mode 100644 osfmk/arm64/kpc.c create mode 100644 osfmk/arm64/locore.s create mode 100644 osfmk/arm64/loose_ends.c create mode 100644 osfmk/arm64/lowglobals.h create mode 100644 osfmk/arm64/lowmem_vectors.c create mode 100644 osfmk/arm64/lz4_decode_arm64.s create mode 100644 osfmk/arm64/lz4_encode_arm64.s create mode 100644 osfmk/arm64/machine_cpuid.h create mode 100644 osfmk/arm64/machine_kpc.h create mode 100644 osfmk/arm64/machine_machdep.h create mode 100644 osfmk/arm64/machine_routines.c create mode 100644 osfmk/arm64/machine_routines_asm.s create mode 100644 osfmk/arm64/machine_task.c create mode 100644 osfmk/arm64/monotonic.h create mode 100644 osfmk/arm64/monotonic_arm64.c create mode 100644 osfmk/arm64/pcb.c create mode 100644 osfmk/arm64/pgtrace.c create mode 100644 osfmk/arm64/pgtrace.h create mode 100644 osfmk/arm64/pgtrace_decoder.c create mode 100644 osfmk/arm64/pgtrace_decoder.h create mode 100644 osfmk/arm64/pinst.s create mode 100644 osfmk/arm64/platform_tests.c create mode 100644 osfmk/arm64/proc_reg.h create mode 100644 osfmk/arm64/sleh.c create mode 100644 osfmk/arm64/start.s create mode 100644 osfmk/arm64/status.c create mode 100644 osfmk/arm64/strncmp.s create mode 100644 osfmk/arm64/strnlen.s create mode 100644 osfmk/conf/Makefile.arm create mode 100644 osfmk/conf/Makefile.arm64 create mode 100644 osfmk/conf/files.arm create mode 100644 osfmk/conf/files.arm64 delete mode 100644 osfmk/corecrypto/cc/src/cc_abort.c create mode 100644 osfmk/corecrypto/cc/src/cc_try_abort.c create mode 100644 osfmk/corecrypto/ccaes/src/aes_tab.c create mode 100644 osfmk/corecrypto/ccaes/src/ccaes_ltc_ecb_encrypt_mode.c create mode 100644 osfmk/corecrypto/ccaes/src/ccaes_private_types.h create mode 100644 osfmk/corecrypto/ccmode/src/ccmode_ctr_crypt.c create mode 100644 osfmk/corecrypto/ccmode/src/ccmode_ctr_init.c create mode 100644 osfmk/corecrypto/ccmode/src/ccmode_ctr_setctr.c create mode 100644 osfmk/corecrypto/ccmode/src/ccmode_factory_ctr_crypt.c create mode 100644 osfmk/corecrypto/ccmode/src/ccmode_internal.h create mode 100644 osfmk/corecrypto/ccsha2/src/ccsha256_K.c create mode 100644 osfmk/corecrypto/ccsha2/src/ccsha256_di.c create mode 100644 osfmk/corecrypto/ccsha2/src/ccsha256_initial_state.c create mode 100644 osfmk/corecrypto/ccsha2/src/ccsha256_ltc_compress.c create mode 100644 osfmk/corecrypto/ccsha2/src/ccsha256_ltc_di.c create mode 100644 osfmk/corecrypto/ccsha2/src/ccsha2_internal.h create mode 100644 osfmk/kdp/ml/arm/kdp_machdep.c create mode 100644 osfmk/kdp/ml/arm/kdp_vm.c create mode 100644 osfmk/kdp/processor_core.c create mode 100644 osfmk/kdp/processor_core.h create mode 100644 osfmk/kern/arithmetic_128.h create mode 100644 osfmk/kern/copyout_shim.c create mode 100644 osfmk/kern/copyout_shim.h create mode 100644 osfmk/kern/cs_blobs.h create mode 100644 osfmk/kern/exc_guard.h create mode 100644 osfmk/kern/kern_monotonic.c create mode 100644 osfmk/kern/memset_s.c create mode 100644 osfmk/kern/monotonic.h create mode 100644 osfmk/kern/thread_group.c create mode 100644 osfmk/kern/thread_group.h create mode 100644 osfmk/kern/thread_kernel_state.h create mode 100644 osfmk/kern/work_interval.c create mode 100644 osfmk/kern/work_interval.h rename bsd/kern/spl.c => osfmk/kperf/arm/kperf_meminfo.c (59%) create mode 100644 osfmk/kperf/arm/kperf_mp.c create mode 100644 osfmk/libsa/arm/types.h create mode 100644 osfmk/mach/arm/Makefile create mode 100644 osfmk/mach/arm/_structs.h create mode 100644 osfmk/mach/arm/boolean.h create mode 100644 osfmk/mach/arm/exception.h create mode 100644 osfmk/mach/arm/kern_return.h create mode 100644 osfmk/mach/arm/ndr_def.h create mode 100644 osfmk/mach/arm/processor_info.h create mode 100644 osfmk/mach/arm/rpc.h create mode 100644 osfmk/mach/arm/sdt_isa.h create mode 100644 osfmk/mach/arm/syscall_sw.h rename bsd/machine/spl.h => osfmk/mach/arm/thread_state.h (58%) create mode 100644 osfmk/mach/arm/thread_status.h create mode 100644 osfmk/mach/arm/vm_param.h create mode 100644 osfmk/mach/arm/vm_types.h create mode 100644 osfmk/mach/arm64/Makefile create mode 100644 osfmk/mach/machine/_structs.h create mode 100644 osfmk/mach/task_inspect.h create mode 100644 osfmk/machine/config.h create mode 100644 osfmk/machine/monotonic.h create mode 100644 osfmk/x86_64/monotonic.h create mode 100644 osfmk/x86_64/monotonic_x86_64.c create mode 100644 pexpert/arm/pe_bootargs.c create mode 100644 pexpert/arm/pe_consistent_debug.c create mode 100644 pexpert/arm/pe_identify_machine.c create mode 100644 pexpert/arm/pe_init.c create mode 100644 pexpert/arm/pe_kprintf.c create mode 100644 pexpert/arm/pe_serial.c create mode 100644 pexpert/conf/Makefile.arm create mode 100644 pexpert/conf/Makefile.arm64 create mode 100644 pexpert/conf/files.arm create mode 100644 pexpert/conf/files.arm64 create mode 100644 pexpert/pexpert/arm/AIC.h create mode 100644 pexpert/pexpert/arm/Makefile create mode 100644 pexpert/pexpert/arm/PL192_VIC.h create mode 100644 pexpert/pexpert/arm/S3cUART.h create mode 100644 pexpert/pexpert/arm/S7002.h create mode 100644 pexpert/pexpert/arm/T8002.h create mode 100644 pexpert/pexpert/arm/board_config.h create mode 100644 pexpert/pexpert/arm/boot.h create mode 100644 pexpert/pexpert/arm/consistent_debug.h create mode 100644 pexpert/pexpert/arm/protos.h create mode 100644 pexpert/pexpert/arm64/AIC.h create mode 100644 pexpert/pexpert/arm64/AMCC.h create mode 100644 pexpert/pexpert/arm64/Makefile create mode 100644 pexpert/pexpert/arm64/S3c2410x.h create mode 100644 pexpert/pexpert/arm64/S5L8960X.h create mode 100644 pexpert/pexpert/arm64/S8000.h create mode 100644 pexpert/pexpert/arm64/T7000.h create mode 100644 pexpert/pexpert/arm64/T8010.h create mode 100644 pexpert/pexpert/arm64/arm64_common.h create mode 100644 pexpert/pexpert/arm64/board_config.h create mode 100644 pexpert/pexpert/arm64/boot.h create mode 100644 pexpert/pexpert/arm64/cyclone.h create mode 100644 pexpert/pexpert/arm64/hurricane.h create mode 100644 pexpert/pexpert/arm64/twister.h create mode 100644 pexpert/pexpert/arm64/typhoon.h create mode 100644 san/Kasan.exports create mode 100644 san/Kasan_kasan.exports create mode 100644 san/Makefile create mode 100644 san/conf/Makefile create mode 100644 san/conf/Makefile.template create mode 100644 san/conf/Makefile.x86_64 create mode 100644 san/conf/files create mode 100644 san/conf/files.x86_64 create mode 100644 san/kasan-arm64.c create mode 100644 san/kasan-blacklist create mode 100644 san/kasan-blacklist-arm64 create mode 100644 san/kasan-blacklist-x86_64 create mode 100644 san/kasan-fakestack.c create mode 100644 san/kasan-memintrinsics.c create mode 100644 san/kasan-test-arm64.s create mode 100644 san/kasan-test-x86_64.s create mode 100644 san/kasan-test.c create mode 100644 san/kasan-x86_64.c create mode 100644 san/kasan.c create mode 100644 san/kasan.h create mode 100644 san/kasan_dynamic_blacklist.c create mode 100644 san/kasan_internal.h create mode 100644 san/memintrinsics.h create mode 100755 san/tools/generate_dynamic_blacklist.py create mode 100755 san/tools/kasan_install create mode 100644 security/conf/Makefile.arm create mode 100644 security/conf/Makefile.arm64 create mode 100644 security/conf/files.arm create mode 100644 security/conf/files.arm64 create mode 100644 security/mac_skywalk.c mode change 100644 => 100755 tools/lldbmacros/apic.py mode change 100644 => 100755 tools/lldbmacros/atm.py mode change 100644 => 100755 tools/lldbmacros/bank.py mode change 100644 => 100755 tools/lldbmacros/core/__init__.py mode change 100644 => 100755 tools/lldbmacros/core/caching.py mode change 100644 => 100755 tools/lldbmacros/core/configuration.py mode change 100644 => 100755 tools/lldbmacros/core/cvalue.py mode change 100644 => 100755 tools/lldbmacros/core/kernelcore.py mode change 100644 => 100755 tools/lldbmacros/core/lazytarget.py mode change 100644 => 100755 tools/lldbmacros/core/operating_system.py mode change 100644 => 100755 tools/lldbmacros/core/standard.py mode change 100644 => 100755 tools/lldbmacros/core/xnu_lldb_init.py mode change 100644 => 100755 tools/lldbmacros/ioreg.py mode change 100644 => 100755 tools/lldbmacros/ipc.py mode change 100644 => 100755 tools/lldbmacros/ipcimportancedetail.py create mode 100755 tools/lldbmacros/kasan.py mode change 100644 => 100755 tools/lldbmacros/kauth.py mode change 100644 => 100755 tools/lldbmacros/kdp.py create mode 100755 tools/lldbmacros/kevent.py mode change 100644 => 100755 tools/lldbmacros/ktrace.py mode change 100644 => 100755 tools/lldbmacros/macho.py mode change 100644 => 100755 tools/lldbmacros/mbufdefines.py mode change 100644 => 100755 tools/lldbmacros/mbufs.py mode change 100644 => 100755 tools/lldbmacros/memory.py mode change 100644 => 100755 tools/lldbmacros/misc.py mode change 100644 => 100755 tools/lldbmacros/net.py mode change 100644 => 100755 tools/lldbmacros/netdefines.py create mode 100755 tools/lldbmacros/ntstat.py mode change 100644 => 100755 tools/lldbmacros/pci.py mode change 100644 => 100755 tools/lldbmacros/pgtrace.py mode change 100644 => 100755 tools/lldbmacros/plugins/__init__.py mode change 100644 => 100755 tools/lldbmacros/plugins/iosspeedtracer.py mode change 100644 => 100755 tools/lldbmacros/plugins/speedtracer.py mode change 100644 => 100755 tools/lldbmacros/plugins/zprint_perf_log.py mode change 100644 => 100755 tools/lldbmacros/pmap.py mode change 100644 => 100755 tools/lldbmacros/process.py mode change 100644 => 100755 tools/lldbmacros/routedefines.py mode change 100644 => 100755 tools/lldbmacros/scheduler.py mode change 100644 => 100755 tools/lldbmacros/structanalyze.py mode change 100644 => 100755 tools/lldbmacros/userspace.py mode change 100644 => 100755 tools/lldbmacros/usertaskdebugging/__init__.py mode change 100644 => 100755 tools/lldbmacros/usertaskdebugging/gdbserver.py mode change 100644 => 100755 tools/lldbmacros/usertaskdebugging/interface.py mode change 100644 => 100755 tools/lldbmacros/usertaskdebugging/rsprotocol.py mode change 100644 => 100755 tools/lldbmacros/usertaskdebugging/target.py mode change 100644 => 100755 tools/lldbmacros/usertaskdebugging/userprocess.py mode change 100644 => 100755 tools/lldbmacros/usertaskgdbserver.py mode change 100644 => 100755 tools/lldbmacros/utils.py mode change 100644 => 100755 tools/lldbmacros/waitq.py mode change 100644 => 100755 tools/lldbmacros/xnu.py mode change 100644 => 100755 tools/lldbmacros/xnudefines.py mode change 100644 => 100755 tools/lldbmacros/xnutriage.py delete mode 100644 tools/stackshot/Makefile delete mode 100644 tools/stackshot/stackshot.c create mode 100644 tools/tests/darwintests/avx.c create mode 100644 tools/tests/darwintests/contextswitch.c create mode 100644 tools/tests/darwintests/cpucount.c create mode 100644 tools/tests/darwintests/disk_mount_conditioner-entitlements.plist create mode 100644 tools/tests/darwintests/disk_mount_conditioner.c create mode 100644 tools/tests/darwintests/gettimeofday_29192647.c create mode 100644 tools/tests/darwintests/ioperf.c create mode 100644 tools/tests/darwintests/kevent_pty.c create mode 100644 tools/tests/darwintests/kevent_qos.c create mode 100644 tools/tests/darwintests/kpc.c create mode 100644 tools/tests/darwintests/kqueue_add_and_trigger.c create mode 100644 tools/tests/darwintests/kqueue_close.c rename tools/tests/{kqueue_tests => darwintests}/kqueue_file_tests.c (84%) create mode 100644 tools/tests/darwintests/kqueue_timer_tests.c create mode 100644 tools/tests/darwintests/launchd_plists/com.apple.xnu.test.kevent_qos.plist create mode 100644 tools/tests/darwintests/mach_port_deallocate_21692215.c create mode 100644 tools/tests/darwintests/mach_port_mod_refs.c create mode 100644 tools/tests/darwintests/memorystatus_zone_test.c create mode 100644 tools/tests/darwintests/monotonic_core.c create mode 100644 tools/tests/darwintests/netbsd_utimensat.c create mode 100644 tools/tests/darwintests/ntp_adjtime_29192647.c create mode 100644 tools/tests/darwintests/private_entitlement.plist create mode 100644 tools/tests/darwintests/proc_info.c create mode 100644 tools/tests/darwintests/settimeofday_29193041.c create mode 100644 tools/tests/darwintests/settimeofday_29193041.entitlements create mode 100644 tools/tests/darwintests/settimeofday_29193041_entitled.c create mode 100644 tools/tests/darwintests/sigchld_return.c create mode 100644 tools/tests/darwintests/sigcont_return.c create mode 100644 tools/tests/darwintests/stackshot.m create mode 100644 tools/tests/darwintests/task_for_pid_entitlement.plist create mode 100644 tools/tests/darwintests/task_info.c create mode 100644 tools/tests/darwintests/task_info_28439149.c create mode 100644 tools/tests/darwintests/thread_group_set_32261625.c create mode 100644 tools/tests/darwintests/utimensat.c create mode 100644 tools/tests/darwintests/work_interval_test.c create mode 100644 tools/tests/darwintests/work_interval_test.entitlements create mode 100644 tools/tests/darwintests/xnu_quick_test_helpers.c create mode 100644 tools/tests/darwintests/xnu_quick_test_helpers.h delete mode 100644 tools/tests/jitter/cpu_number.s delete mode 100755 tools/tests/kqueue_tests/Makefile delete mode 100644 tools/tests/kqueue_tests/kqueue_timer_tests.c mode change 100755 => 100644 tools/tests/libMicro/Makefile.com.Darwin mode change 100644 => 100755 tools/tests/perf_index/test_controller.py create mode 100755 tools/trace/kqtrace.lua mode change 100644 => 100755 tools/trace/parse_ipc_trace.py diff --git a/.gitignore b/.gitignore index b502c3239..f5ad2c6fd 100644 --- a/.gitignore +++ b/.gitignore @@ -3,10 +3,13 @@ BUILD/ build/ .DS_Store -# vim turds +# vim swap files *~ *.swp +# JSON compilation definitions (for the YouCompleteMe vim plugin) +compile_commands.json + # / /.remotebuild_credential /cscope.* diff --git a/.upstream_base_commits b/.upstream_base_commits index d9ee6e9d6..0343ee6c1 100644 --- a/.upstream_base_commits +++ b/.upstream_base_commits @@ -1,3 +1,5 @@ #freebsd = https://github.com/freebsd/freebsd.git bsd/man/man2/access.2 freebsd lib/libc/sys/access.2 5b882020081a138285227631c46a406c08e17bc8 bsd/man/man7/sticky.7 freebsd share/man/man7/sticky.7 5b882020081a138285227631c46a406c08e17bc8 +bsd/man/man2/utimensat.2 freebsd lib/libc/sys/utimensat.2 89c1fcc0d088065021703b658ef547f46b5481f0 +tools/tests/darwintests/netbsd_utimensat.c freebsd contrib/netbsd-tests/lib/libc/c063/t_utimensat.c 89c1fcc0d088065021703b658ef547f46b5481f0 diff --git a/EXTERNAL_HEADERS/AvailabilityMacros.h b/EXTERNAL_HEADERS/AvailabilityMacros.h index a9027a58a..6728fc446 100644 --- a/EXTERNAL_HEADERS/AvailabilityMacros.h +++ b/EXTERNAL_HEADERS/AvailabilityMacros.h @@ -124,6 +124,8 @@ #else #if __i386__ || __x86_64__ #define MAC_OS_X_VERSION_MIN_REQUIRED MAC_OS_X_VERSION_10_4 + #elif __arm__ || __arm64__ + #define MAC_OS_X_VERSION_MIN_REQUIRED MAC_OS_X_VERSION_10_5 #else #define MAC_OS_X_VERSION_MIN_REQUIRED MAC_OS_X_VERSION_10_1 #endif diff --git a/EXTERNAL_HEADERS/architecture/arm/Makefile b/EXTERNAL_HEADERS/architecture/arm/Makefile new file mode 100644 index 000000000..08f41e365 --- /dev/null +++ b/EXTERNAL_HEADERS/architecture/arm/Makefile @@ -0,0 +1,21 @@ +export MakeInc_cmd=${SRCROOT}/makedefs/MakeInc.cmd +export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def +export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule +export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir + +include $(MakeInc_cmd) +include $(MakeInc_def) + +EXPORT_FILES = \ + arm_neon.h \ + +INSTALL_MD_LIST = + +INSTALL_MD_DIR = + +EXPORT_MD_LIST = ${EXPORT_FILES} + +EXPORT_MD_DIR = architecture/arm + +include $(MakeInc_rule) +include $(MakeInc_dir) diff --git a/EXTERNAL_HEADERS/architecture/arm/arm_neon.h b/EXTERNAL_HEADERS/architecture/arm/arm_neon.h new file mode 100644 index 000000000..e294bd96c --- /dev/null +++ b/EXTERNAL_HEADERS/architecture/arm/arm_neon.h @@ -0,0 +1,74267 @@ +/*===---- arm_neon.h - ARM Neon intrinsics ---------------------------------=== + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + *===-----------------------------------------------------------------------=== + */ + +#ifndef __ARM_NEON_H +#define __ARM_NEON_H + +#if !defined(__ARM_NEON) +#error "NEON support not enabled" +#endif + +#include <stdint.h> + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wfloat-equal" +#pragma clang diagnostic ignored "-Wvector-conversion" + +typedef float float32_t; +typedef __fp16 float16_t; +#ifdef __aarch64__ +typedef double float64_t; +#endif + +#ifdef __aarch64__ +typedef uint8_t poly8_t; +typedef uint16_t poly16_t; +typedef uint64_t poly64_t; +typedef __uint128_t poly128_t; +#else +typedef int8_t poly8_t; +typedef int16_t poly16_t; +#endif +typedef __attribute__((neon_vector_type(8))) int8_t int8x8_t; +typedef __attribute__((neon_vector_type(16))) int8_t int8x16_t; +typedef __attribute__((neon_vector_type(4))) int16_t int16x4_t; +typedef __attribute__((neon_vector_type(8))) int16_t int16x8_t; +typedef __attribute__((neon_vector_type(2))) int32_t int32x2_t; +typedef __attribute__((neon_vector_type(4))) int32_t int32x4_t; +typedef __attribute__((neon_vector_type(1))) int64_t int64x1_t; +typedef __attribute__((neon_vector_type(2))) int64_t int64x2_t; +typedef __attribute__((neon_vector_type(8))) uint8_t uint8x8_t; +typedef __attribute__((neon_vector_type(16))) uint8_t uint8x16_t; +typedef __attribute__((neon_vector_type(4))) uint16_t uint16x4_t; +typedef __attribute__((neon_vector_type(8))) uint16_t uint16x8_t; +typedef __attribute__((neon_vector_type(2))) uint32_t uint32x2_t; +typedef __attribute__((neon_vector_type(4))) uint32_t uint32x4_t; +typedef __attribute__((neon_vector_type(1))) uint64_t uint64x1_t; +typedef __attribute__((neon_vector_type(2))) uint64_t uint64x2_t; +typedef __attribute__((neon_vector_type(4))) float16_t float16x4_t; +typedef __attribute__((neon_vector_type(8))) float16_t float16x8_t; +typedef __attribute__((neon_vector_type(2))) float32_t float32x2_t; +typedef __attribute__((neon_vector_type(4))) float32_t float32x4_t; +#ifdef __aarch64__ +typedef __attribute__((neon_vector_type(1))) float64_t float64x1_t; +typedef __attribute__((neon_vector_type(2))) float64_t float64x2_t; +#endif +typedef __attribute__((neon_polyvector_type(8))) poly8_t poly8x8_t; +typedef __attribute__((neon_polyvector_type(16))) poly8_t poly8x16_t; +typedef __attribute__((neon_polyvector_type(4))) poly16_t poly16x4_t; +typedef __attribute__((neon_polyvector_type(8))) poly16_t poly16x8_t; +#ifdef __aarch64__ +typedef __attribute__((neon_polyvector_type(1))) poly64_t poly64x1_t; +typedef __attribute__((neon_polyvector_type(2))) poly64_t poly64x2_t; +#endif + +typedef struct int8x8x2_t { + int8x8_t val[2]; +} int8x8x2_t; + +typedef struct int8x16x2_t { + int8x16_t val[2]; +} int8x16x2_t; + +typedef struct int16x4x2_t { + int16x4_t val[2]; +} int16x4x2_t; + +typedef struct int16x8x2_t { + int16x8_t val[2]; +} int16x8x2_t; + +typedef struct int32x2x2_t { + int32x2_t val[2]; +} int32x2x2_t; + +typedef struct int32x4x2_t { + int32x4_t val[2]; +} int32x4x2_t; + +typedef struct int64x1x2_t { + int64x1_t val[2]; +} int64x1x2_t; + +typedef struct int64x2x2_t { + int64x2_t val[2]; +} int64x2x2_t; + +typedef struct uint8x8x2_t { + uint8x8_t val[2]; +} uint8x8x2_t; + +typedef struct uint8x16x2_t { + uint8x16_t val[2]; +} uint8x16x2_t; + +typedef struct uint16x4x2_t { + uint16x4_t val[2]; +} uint16x4x2_t; + +typedef struct uint16x8x2_t { + uint16x8_t val[2]; +} uint16x8x2_t; + +typedef struct uint32x2x2_t { + uint32x2_t val[2]; +} uint32x2x2_t; + +typedef struct uint32x4x2_t { + uint32x4_t val[2]; +} uint32x4x2_t; + +typedef struct uint64x1x2_t { + uint64x1_t val[2]; +} uint64x1x2_t; + +typedef struct uint64x2x2_t { + uint64x2_t val[2]; +} uint64x2x2_t; + +typedef struct float16x4x2_t { + float16x4_t val[2]; +} float16x4x2_t; + +typedef struct float16x8x2_t { + float16x8_t val[2]; +} float16x8x2_t; + +typedef struct float32x2x2_t { + float32x2_t val[2]; +} float32x2x2_t; + +typedef struct float32x4x2_t { + float32x4_t val[2]; +} float32x4x2_t; + +#ifdef __aarch64__ +typedef struct float64x1x2_t { + float64x1_t val[2]; +} float64x1x2_t; + +typedef struct float64x2x2_t { + float64x2_t val[2]; +} float64x2x2_t; + +#endif +typedef struct poly8x8x2_t { + poly8x8_t val[2]; +} poly8x8x2_t; + +typedef struct poly8x16x2_t { + poly8x16_t val[2]; +} poly8x16x2_t; + +typedef struct poly16x4x2_t { + poly16x4_t val[2]; +} poly16x4x2_t; + +typedef struct poly16x8x2_t { + poly16x8_t val[2]; +} poly16x8x2_t; + +#ifdef __aarch64__ +typedef struct poly64x1x2_t { + poly64x1_t val[2]; +} poly64x1x2_t; + +typedef struct poly64x2x2_t { + poly64x2_t val[2]; +} poly64x2x2_t; + +#endif +typedef struct int8x8x3_t { + int8x8_t val[3]; +} int8x8x3_t; + +typedef struct int8x16x3_t { + int8x16_t val[3]; +} int8x16x3_t; + +typedef struct int16x4x3_t { + int16x4_t val[3]; +} int16x4x3_t; + +typedef struct int16x8x3_t { + int16x8_t val[3]; +} int16x8x3_t; + +typedef struct int32x2x3_t { + int32x2_t val[3]; +} int32x2x3_t; + +typedef struct int32x4x3_t { + int32x4_t val[3]; +} int32x4x3_t; + +typedef struct int64x1x3_t { + int64x1_t val[3]; +} int64x1x3_t; + +typedef struct int64x2x3_t { + int64x2_t val[3]; +} int64x2x3_t; + +typedef struct uint8x8x3_t { + uint8x8_t val[3]; +} uint8x8x3_t; + +typedef struct uint8x16x3_t { + uint8x16_t val[3]; +} uint8x16x3_t; + +typedef struct uint16x4x3_t { + uint16x4_t val[3]; +} uint16x4x3_t; + +typedef struct uint16x8x3_t { + uint16x8_t val[3]; +} uint16x8x3_t; + +typedef struct uint32x2x3_t { + uint32x2_t val[3]; +} uint32x2x3_t; + +typedef struct uint32x4x3_t { + uint32x4_t val[3]; +} uint32x4x3_t; + +typedef struct uint64x1x3_t { + uint64x1_t val[3]; +} uint64x1x3_t; + +typedef struct uint64x2x3_t { + uint64x2_t val[3]; +} uint64x2x3_t; + +typedef struct float16x4x3_t { + float16x4_t val[3]; +} float16x4x3_t; + +typedef struct float16x8x3_t { + float16x8_t val[3]; +} float16x8x3_t; + +typedef struct float32x2x3_t { + float32x2_t val[3]; +} float32x2x3_t; + +typedef struct float32x4x3_t { + float32x4_t val[3]; +} float32x4x3_t; + +#ifdef __aarch64__ +typedef struct float64x1x3_t { + float64x1_t val[3]; +} float64x1x3_t; + +typedef struct float64x2x3_t { + float64x2_t val[3]; +} float64x2x3_t; + +#endif +typedef struct poly8x8x3_t { + poly8x8_t val[3]; +} poly8x8x3_t; + +typedef struct poly8x16x3_t { + poly8x16_t val[3]; +} poly8x16x3_t; + +typedef struct poly16x4x3_t { + poly16x4_t val[3]; +} poly16x4x3_t; + +typedef struct poly16x8x3_t { + poly16x8_t val[3]; +} poly16x8x3_t; + +#ifdef __aarch64__ +typedef struct poly64x1x3_t { + poly64x1_t val[3]; +} poly64x1x3_t; + +typedef struct poly64x2x3_t { + poly64x2_t val[3]; +} poly64x2x3_t; + +#endif +typedef struct int8x8x4_t { + int8x8_t val[4]; +} int8x8x4_t; + +typedef struct int8x16x4_t { + int8x16_t val[4]; +} int8x16x4_t; + +typedef struct int16x4x4_t { + int16x4_t val[4]; +} int16x4x4_t; + +typedef struct int16x8x4_t { + int16x8_t val[4]; +} int16x8x4_t; + +typedef struct int32x2x4_t { + int32x2_t val[4]; +} int32x2x4_t; + +typedef struct int32x4x4_t { + int32x4_t val[4]; +} int32x4x4_t; + +typedef struct int64x1x4_t { + int64x1_t val[4]; +} int64x1x4_t; + +typedef struct int64x2x4_t { + int64x2_t val[4]; +} int64x2x4_t; + +typedef struct uint8x8x4_t { + uint8x8_t val[4]; +} uint8x8x4_t; + +typedef struct uint8x16x4_t { + uint8x16_t val[4]; +} uint8x16x4_t; + +typedef struct uint16x4x4_t { + uint16x4_t val[4]; +} uint16x4x4_t; + +typedef struct uint16x8x4_t { + uint16x8_t val[4]; +} uint16x8x4_t; + +typedef struct uint32x2x4_t { + uint32x2_t val[4]; +} uint32x2x4_t; + +typedef struct uint32x4x4_t { + uint32x4_t val[4]; +} uint32x4x4_t; + +typedef struct uint64x1x4_t { + uint64x1_t val[4]; +} uint64x1x4_t; + +typedef struct uint64x2x4_t { + uint64x2_t val[4]; +} uint64x2x4_t; + +typedef struct float16x4x4_t { + float16x4_t val[4]; +} float16x4x4_t; + +typedef struct float16x8x4_t { + float16x8_t val[4]; +} float16x8x4_t; + +typedef struct float32x2x4_t { + float32x2_t val[4]; +} float32x2x4_t; + +typedef struct float32x4x4_t { + float32x4_t val[4]; +} float32x4x4_t; + +#ifdef __aarch64__ +typedef struct float64x1x4_t { + float64x1_t val[4]; +} float64x1x4_t; + +typedef struct float64x2x4_t { + float64x2_t val[4]; +} float64x2x4_t; + +#endif +typedef struct poly8x8x4_t { + poly8x8_t val[4]; +} poly8x8x4_t; + +typedef struct poly8x16x4_t { + poly8x16_t val[4]; +} poly8x16x4_t; + +typedef struct poly16x4x4_t { + poly16x4_t val[4]; +} poly16x4x4_t; + +typedef struct poly16x8x4_t { + poly16x8_t val[4]; +} poly16x8x4_t; + +#ifdef __aarch64__ +typedef struct poly64x1x4_t { + poly64x1_t val[4]; +} poly64x1x4_t; + +typedef struct poly64x2x4_t { + poly64x2_t val[4]; +} poly64x2x4_t; + +#endif + +#define __ai static inline __attribute__((__always_inline__, __nodebug__)) + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vabdq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vabdq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vabdq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint8x16_t __noswap_vabdq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vabdq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vabdq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vabdq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint32x4_t __noswap_vabdq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vabdq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vabdq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vabdq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint16x8_t __noswap_vabdq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vabdq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vabdq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vabdq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int8x16_t __noswap_vabdq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vabdq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4_t vabdq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vabdq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vabdq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vabdq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vabdq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vabdq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vabdq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vabdq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vabdq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int16x8_t __noswap_vabdq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vabd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vabd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vabd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint8x8_t __noswap_vabd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vabd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vabd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vabd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint32x2_t __noswap_vabd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vabd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vabd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vabd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint16x4_t __noswap_vabd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vabd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vabd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vabd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int8x8_t __noswap_vabd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vabd_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2_t vabd_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vabd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vabd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vabd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vabd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int32x2_t __noswap_vabd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vabd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vabd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vabd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int16x4_t __noswap_vabd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vabsq_s8(int8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vabsq_v((int8x16_t)__p0, 32); + return __ret; +} +#else +__ai int8x16_t vabsq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vabsq_v((int8x16_t)__rev0, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vabsq_f32(float32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vabsq_v((int8x16_t)__p0, 41); + return __ret; +} +#else +__ai float32x4_t vabsq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vabsq_v((int8x16_t)__rev0, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vabsq_s32(int32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vabsq_v((int8x16_t)__p0, 34); + return __ret; +} +#else +__ai int32x4_t vabsq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vabsq_v((int8x16_t)__rev0, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vabsq_s16(int16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vabsq_v((int8x16_t)__p0, 33); + return __ret; +} +#else +__ai int16x8_t vabsq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vabsq_v((int8x16_t)__rev0, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vabs_s8(int8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vabs_v((int8x8_t)__p0, 0); + return __ret; +} +#else +__ai int8x8_t vabs_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vabs_v((int8x8_t)__rev0, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vabs_f32(float32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vabs_v((int8x8_t)__p0, 9); + return __ret; +} +#else +__ai float32x2_t vabs_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vabs_v((int8x8_t)__rev0, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vabs_s32(int32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vabs_v((int8x8_t)__p0, 2); + return __ret; +} +#else +__ai int32x2_t vabs_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vabs_v((int8x8_t)__rev0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vabs_s16(int16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vabs_v((int8x8_t)__p0, 1); + return __ret; +} +#else +__ai int16x4_t vabs_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vabs_v((int8x8_t)__rev0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vaddq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai uint8x16_t vaddq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vaddq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai uint32x4_t vaddq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vaddq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai uint64x2_t vaddq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vaddq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai uint16x8_t vaddq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vaddq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai int8x16_t vaddq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vaddq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai float32x4_t vaddq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vaddq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai int32x4_t vaddq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vaddq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai int64x2_t vaddq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vaddq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai int16x8_t vaddq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vadd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai uint8x8_t vadd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vadd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai uint32x2_t vadd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vadd_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai uint64x1_t vadd_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vadd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai uint16x4_t vadd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vadd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai int8x8_t vadd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vadd_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai float32x2_t vadd_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vadd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai int32x2_t vadd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vadd_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai int64x1_t vadd_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vadd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai int16x4_t vadd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vaddhn_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vaddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vaddhn_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vaddhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint16x4_t __noswap_vaddhn_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vaddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 17); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vaddhn_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vaddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vaddhn_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vaddhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint32x2_t __noswap_vaddhn_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vaddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 18); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vaddhn_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vaddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vaddhn_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vaddhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint8x8_t __noswap_vaddhn_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vaddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 16); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vaddhn_s32(int32x4_t __p0, int32x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vaddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vaddhn_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vaddhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int16x4_t __noswap_vaddhn_s32(int32x4_t __p0, int32x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vaddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vaddhn_s64(int64x2_t __p0, int64x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vaddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vaddhn_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vaddhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int32x2_t __noswap_vaddhn_s64(int64x2_t __p0, int64x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vaddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vaddhn_s16(int16x8_t __p0, int16x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vaddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vaddhn_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vaddhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int8x8_t __noswap_vaddhn_s16(int16x8_t __p0, int16x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vaddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vandq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai uint8x16_t vandq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __rev0 & __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vandq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai uint32x4_t vandq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 & __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vandq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai uint64x2_t vandq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __rev0 & __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vandq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai uint16x8_t vandq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 & __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vandq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai int8x16_t vandq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __rev0 & __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vandq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai int32x4_t vandq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 & __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vandq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai int64x2_t vandq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __rev0 & __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vandq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai int16x8_t vandq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 & __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vand_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai uint8x8_t vand_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __rev0 & __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vand_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai uint32x2_t vand_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __rev0 & __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vand_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai uint64x1_t vand_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vand_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai uint16x4_t vand_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __rev0 & __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vand_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai int8x8_t vand_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __rev0 & __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vand_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai int32x2_t vand_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __rev0 & __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vand_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai int64x1_t vand_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vand_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = __p0 & __p1; + return __ret; +} +#else +__ai int16x4_t vand_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __rev0 & __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vbicq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai uint8x16_t vbicq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __rev0 & ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vbicq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai uint32x4_t vbicq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 & ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vbicq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai uint64x2_t vbicq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __rev0 & ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vbicq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai uint16x8_t vbicq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 & ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vbicq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai int8x16_t vbicq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __rev0 & ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vbicq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai int32x4_t vbicq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 & ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vbicq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai int64x2_t vbicq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __rev0 & ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vbicq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai int16x8_t vbicq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 & ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vbic_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai uint8x8_t vbic_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __rev0 & ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vbic_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai uint32x2_t vbic_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __rev0 & ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vbic_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai uint64x1_t vbic_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vbic_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai uint16x4_t vbic_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __rev0 & ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vbic_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai int8x8_t vbic_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __rev0 & ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vbic_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai int32x2_t vbic_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __rev0 & ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vbic_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai int64x1_t vbic_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vbic_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = __p0 & ~__p1; + return __ret; +} +#else +__ai int16x4_t vbic_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __rev0 & ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vbsl_p8(uint8x8_t __p0, poly8x8_t __p1, poly8x8_t __p2) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 4); + return __ret; +} +#else +__ai poly8x8_t vbsl_p8(uint8x8_t __p0, poly8x8_t __p1, poly8x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vbsl_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vbsl_p16(uint16x4_t __p0, poly16x4_t __p1, poly16x4_t __p2) { + poly16x4_t __ret; + __ret = (poly16x4_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 5); + return __ret; +} +#else +__ai poly16x4_t vbsl_p16(uint16x4_t __p0, poly16x4_t __p1, poly16x4_t __p2) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + poly16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + poly16x4_t __ret; + __ret = (poly16x4_t) __builtin_neon_vbsl_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 5); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vbslq_p8(uint8x16_t __p0, poly8x16_t __p1, poly8x16_t __p2) { + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vbslq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 36); + return __ret; +} +#else +__ai poly8x16_t vbslq_p8(uint8x16_t __p0, poly8x16_t __p1, poly8x16_t __p2) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vbslq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 36); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vbslq_p16(uint16x8_t __p0, poly16x8_t __p1, poly16x8_t __p2) { + poly16x8_t __ret; + __ret = (poly16x8_t) __builtin_neon_vbslq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 37); + return __ret; +} +#else +__ai poly16x8_t vbslq_p16(uint16x8_t __p0, poly16x8_t __p1, poly16x8_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __ret; + __ret = (poly16x8_t) __builtin_neon_vbslq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 37); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vbslq_u8(uint8x16_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vbslq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 48); + return __ret; +} +#else +__ai uint8x16_t vbslq_u8(uint8x16_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vbslq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vbslq_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vbslq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 50); + return __ret; +} +#else +__ai uint32x4_t vbslq_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vbslq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vbslq_u64(uint64x2_t __p0, uint64x2_t __p1, uint64x2_t __p2) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vbslq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 51); + return __ret; +} +#else +__ai uint64x2_t vbslq_u64(uint64x2_t __p0, uint64x2_t __p1, uint64x2_t __p2) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vbslq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vbslq_u16(uint16x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vbslq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 49); + return __ret; +} +#else +__ai uint16x8_t vbslq_u16(uint16x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vbslq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vbslq_s8(uint8x16_t __p0, int8x16_t __p1, int8x16_t __p2) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vbslq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 32); + return __ret; +} +#else +__ai int8x16_t vbslq_s8(uint8x16_t __p0, int8x16_t __p1, int8x16_t __p2) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vbslq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vbslq_f32(uint32x4_t __p0, float32x4_t __p1, float32x4_t __p2) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vbslq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41); + return __ret; +} +#else +__ai float32x4_t vbslq_f32(uint32x4_t __p0, float32x4_t __p1, float32x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vbslq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vbslq_s32(uint32x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vbslq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 34); + return __ret; +} +#else +__ai int32x4_t vbslq_s32(uint32x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vbslq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vbslq_s64(uint64x2_t __p0, int64x2_t __p1, int64x2_t __p2) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vbslq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 35); + return __ret; +} +#else +__ai int64x2_t vbslq_s64(uint64x2_t __p0, int64x2_t __p1, int64x2_t __p2) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vbslq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vbslq_s16(uint16x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vbslq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 33); + return __ret; +} +#else +__ai int16x8_t vbslq_s16(uint16x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vbslq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vbsl_u8(uint8x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 16); + return __ret; +} +#else +__ai uint8x8_t vbsl_u8(uint8x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vbsl_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vbsl_u32(uint32x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 18); + return __ret; +} +#else +__ai uint32x2_t vbsl_u32(uint32x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vbsl_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vbsl_u64(uint64x1_t __p0, uint64x1_t __p1, uint64x1_t __p2) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 19); + return __ret; +} +#else +__ai uint64x1_t vbsl_u64(uint64x1_t __p0, uint64x1_t __p1, uint64x1_t __p2) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vbsl_u16(uint16x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 17); + return __ret; +} +#else +__ai uint16x4_t vbsl_u16(uint16x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vbsl_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vbsl_s8(uint8x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 0); + return __ret; +} +#else +__ai int8x8_t vbsl_s8(uint8x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vbsl_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vbsl_f32(uint32x2_t __p0, float32x2_t __p1, float32x2_t __p2) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9); + return __ret; +} +#else +__ai float32x2_t vbsl_f32(uint32x2_t __p0, float32x2_t __p1, float32x2_t __p2) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vbsl_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vbsl_s32(uint32x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 2); + return __ret; +} +#else +__ai int32x2_t vbsl_s32(uint32x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vbsl_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vbsl_s64(uint64x1_t __p0, int64x1_t __p1, int64x1_t __p2) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 3); + return __ret; +} +#else +__ai int64x1_t vbsl_s64(uint64x1_t __p0, int64x1_t __p1, int64x1_t __p2) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vbsl_s16(uint16x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 1); + return __ret; +} +#else +__ai int16x4_t vbsl_s16(uint16x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vbsl_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcageq_f32(float32x4_t __p0, float32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcageq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vcageq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcageq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcage_f32(float32x2_t __p0, float32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcage_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vcage_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcage_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcagtq_f32(float32x4_t __p0, float32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcagtq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vcagtq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcagtq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcagt_f32(float32x2_t __p0, float32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcagt_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vcagt_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcagt_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcaleq_f32(float32x4_t __p0, float32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcaleq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vcaleq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcaleq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcale_f32(float32x2_t __p0, float32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcale_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vcale_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcale_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcaltq_f32(float32x4_t __p0, float32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcaltq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vcaltq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcaltq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcalt_f32(float32x2_t __p0, float32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcalt_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vcalt_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcalt_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vceq_p8(poly8x8_t __p0, poly8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint8x8_t vceq_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vceqq_p8(poly8x16_t __p0, poly8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint8x16_t vceqq_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vceqq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint8x16_t vceqq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vceqq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint32x4_t vceqq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vceqq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint16x8_t vceqq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vceqq_s8(int8x16_t __p0, int8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint8x16_t vceqq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vceqq_f32(float32x4_t __p0, float32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint32x4_t vceqq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vceqq_s32(int32x4_t __p0, int32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint32x4_t vceqq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vceqq_s16(int16x8_t __p0, int16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint16x8_t vceqq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vceq_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint8x8_t vceq_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vceq_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint32x2_t vceq_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vceq_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint16x4_t vceq_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vceq_s8(int8x8_t __p0, int8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint8x8_t vceq_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vceq_f32(float32x2_t __p0, float32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint32x2_t vceq_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vceq_s32(int32x2_t __p0, int32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint32x2_t vceq_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vceq_s16(int16x4_t __p0, int16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint16x4_t vceq_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vcgeq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint8x16_t vcgeq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcgeq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint32x4_t vcgeq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcgeq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint16x8_t vcgeq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vcgeq_s8(int8x16_t __p0, int8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint8x16_t vcgeq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcgeq_f32(float32x4_t __p0, float32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint32x4_t vcgeq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcgeq_s32(int32x4_t __p0, int32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint32x4_t vcgeq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcgeq_s16(int16x8_t __p0, int16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint16x8_t vcgeq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vcge_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint8x8_t vcge_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcge_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint32x2_t vcge_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcge_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint16x4_t vcge_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vcge_s8(int8x8_t __p0, int8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint8x8_t vcge_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcge_f32(float32x2_t __p0, float32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint32x2_t vcge_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcge_s32(int32x2_t __p0, int32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint32x2_t vcge_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcge_s16(int16x4_t __p0, int16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint16x4_t vcge_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vcgtq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint8x16_t vcgtq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcgtq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint32x4_t vcgtq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcgtq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint16x8_t vcgtq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vcgtq_s8(int8x16_t __p0, int8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint8x16_t vcgtq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcgtq_f32(float32x4_t __p0, float32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint32x4_t vcgtq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcgtq_s32(int32x4_t __p0, int32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint32x4_t vcgtq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcgtq_s16(int16x8_t __p0, int16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint16x8_t vcgtq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vcgt_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint8x8_t vcgt_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcgt_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint32x2_t vcgt_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcgt_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint16x4_t vcgt_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vcgt_s8(int8x8_t __p0, int8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint8x8_t vcgt_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcgt_f32(float32x2_t __p0, float32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint32x2_t vcgt_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcgt_s32(int32x2_t __p0, int32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint32x2_t vcgt_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcgt_s16(int16x4_t __p0, int16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint16x4_t vcgt_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vcleq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint8x16_t vcleq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcleq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint32x4_t vcleq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcleq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint16x8_t vcleq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vcleq_s8(int8x16_t __p0, int8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint8x16_t vcleq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcleq_f32(float32x4_t __p0, float32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint32x4_t vcleq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcleq_s32(int32x4_t __p0, int32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint32x4_t vcleq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcleq_s16(int16x8_t __p0, int16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint16x8_t vcleq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vcle_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint8x8_t vcle_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcle_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint32x2_t vcle_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcle_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint16x4_t vcle_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vcle_s8(int8x8_t __p0, int8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint8x8_t vcle_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcle_f32(float32x2_t __p0, float32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint32x2_t vcle_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcle_s32(int32x2_t __p0, int32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint32x2_t vcle_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcle_s16(int16x4_t __p0, int16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint16x4_t vcle_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vclsq_s8(int8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vclsq_v((int8x16_t)__p0, 32); + return __ret; +} +#else +__ai int8x16_t vclsq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vclsq_v((int8x16_t)__rev0, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vclsq_s32(int32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vclsq_v((int8x16_t)__p0, 34); + return __ret; +} +#else +__ai int32x4_t vclsq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vclsq_v((int8x16_t)__rev0, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vclsq_s16(int16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vclsq_v((int8x16_t)__p0, 33); + return __ret; +} +#else +__ai int16x8_t vclsq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vclsq_v((int8x16_t)__rev0, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vcls_s8(int8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vcls_v((int8x8_t)__p0, 0); + return __ret; +} +#else +__ai int8x8_t vcls_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vcls_v((int8x8_t)__rev0, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vcls_s32(int32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vcls_v((int8x8_t)__p0, 2); + return __ret; +} +#else +__ai int32x2_t vcls_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vcls_v((int8x8_t)__rev0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vcls_s16(int16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vcls_v((int8x8_t)__p0, 1); + return __ret; +} +#else +__ai int16x4_t vcls_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vcls_v((int8x8_t)__rev0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vcltq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint8x16_t vcltq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcltq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint32x4_t vcltq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcltq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint16x8_t vcltq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vcltq_s8(int8x16_t __p0, int8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint8x16_t vcltq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcltq_f32(float32x4_t __p0, float32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint32x4_t vcltq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcltq_s32(int32x4_t __p0, int32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint32x4_t vcltq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcltq_s16(int16x8_t __p0, int16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint16x8_t vcltq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vclt_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint8x8_t vclt_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vclt_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint32x2_t vclt_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vclt_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint16x4_t vclt_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vclt_s8(int8x8_t __p0, int8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint8x8_t vclt_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vclt_f32(float32x2_t __p0, float32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint32x2_t vclt_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vclt_s32(int32x2_t __p0, int32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint32x2_t vclt_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vclt_s16(int16x4_t __p0, int16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint16x4_t vclt_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vclzq_u8(uint8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vclzq_v((int8x16_t)__p0, 48); + return __ret; +} +#else +__ai uint8x16_t vclzq_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vclzq_v((int8x16_t)__rev0, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vclzq_u32(uint32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vclzq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vclzq_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vclzq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vclzq_u16(uint16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vclzq_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vclzq_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vclzq_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vclzq_s8(int8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vclzq_v((int8x16_t)__p0, 32); + return __ret; +} +#else +__ai int8x16_t vclzq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vclzq_v((int8x16_t)__rev0, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vclzq_s32(int32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vclzq_v((int8x16_t)__p0, 34); + return __ret; +} +#else +__ai int32x4_t vclzq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vclzq_v((int8x16_t)__rev0, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vclzq_s16(int16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vclzq_v((int8x16_t)__p0, 33); + return __ret; +} +#else +__ai int16x8_t vclzq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vclzq_v((int8x16_t)__rev0, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vclz_u8(uint8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vclz_v((int8x8_t)__p0, 16); + return __ret; +} +#else +__ai uint8x8_t vclz_u8(uint8x8_t __p0) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vclz_v((int8x8_t)__rev0, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vclz_u32(uint32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vclz_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vclz_u32(uint32x2_t __p0) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vclz_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vclz_u16(uint16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vclz_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vclz_u16(uint16x4_t __p0) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vclz_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vclz_s8(int8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vclz_v((int8x8_t)__p0, 0); + return __ret; +} +#else +__ai int8x8_t vclz_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vclz_v((int8x8_t)__rev0, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vclz_s32(int32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vclz_v((int8x8_t)__p0, 2); + return __ret; +} +#else +__ai int32x2_t vclz_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vclz_v((int8x8_t)__rev0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vclz_s16(int16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vclz_v((int8x8_t)__p0, 1); + return __ret; +} +#else +__ai int16x4_t vclz_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vclz_v((int8x8_t)__rev0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vcnt_p8(poly8x8_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vcnt_v((int8x8_t)__p0, 4); + return __ret; +} +#else +__ai poly8x8_t vcnt_p8(poly8x8_t __p0) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vcnt_v((int8x8_t)__rev0, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vcntq_p8(poly8x16_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vcntq_v((int8x16_t)__p0, 36); + return __ret; +} +#else +__ai poly8x16_t vcntq_p8(poly8x16_t __p0) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vcntq_v((int8x16_t)__rev0, 36); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vcntq_u8(uint8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vcntq_v((int8x16_t)__p0, 48); + return __ret; +} +#else +__ai uint8x16_t vcntq_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vcntq_v((int8x16_t)__rev0, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vcntq_s8(int8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vcntq_v((int8x16_t)__p0, 32); + return __ret; +} +#else +__ai int8x16_t vcntq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vcntq_v((int8x16_t)__rev0, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vcnt_u8(uint8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vcnt_v((int8x8_t)__p0, 16); + return __ret; +} +#else +__ai uint8x8_t vcnt_u8(uint8x8_t __p0) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vcnt_v((int8x8_t)__rev0, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vcnt_s8(int8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vcnt_v((int8x8_t)__p0, 0); + return __ret; +} +#else +__ai int8x8_t vcnt_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vcnt_v((int8x8_t)__rev0, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vcombine_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + return __ret; +} +#else +__ai poly8x16_t vcombine_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vcombine_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3, 4, 5, 6, 7); + return __ret; +} +#else +__ai poly16x8_t vcombine_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + poly16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 1, 2, 3, 4, 5, 6, 7); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vcombine_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + return __ret; +} +#else +__ai uint8x16_t vcombine_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint8x16_t __noswap_vcombine_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcombine_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3); + return __ret; +} +#else +__ai uint32x4_t vcombine_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 1, 2, 3); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint32x4_t __noswap_vcombine_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcombine_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1); + return __ret; +} +#else +__ai uint64x2_t vcombine_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcombine_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3, 4, 5, 6, 7); + return __ret; +} +#else +__ai uint16x8_t vcombine_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 1, 2, 3, 4, 5, 6, 7); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint16x8_t __noswap_vcombine_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3, 4, 5, 6, 7); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vcombine_s8(int8x8_t __p0, int8x8_t __p1) { + int8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + return __ret; +} +#else +__ai int8x16_t vcombine_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int8x16_t __noswap_vcombine_s8(int8x8_t __p0, int8x8_t __p1) { + int8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vcombine_f32(float32x2_t __p0, float32x2_t __p1) { + float32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3); + return __ret; +} +#else +__ai float32x4_t vcombine_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 1, 2, 3); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai float32x4_t __noswap_vcombine_f32(float32x2_t __p0, float32x2_t __p1) { + float32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vcombine_f16(float16x4_t __p0, float16x4_t __p1) { + float16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3, 4, 5, 6, 7); + return __ret; +} +#else +__ai float16x8_t vcombine_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 1, 2, 3, 4, 5, 6, 7); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai float16x8_t __noswap_vcombine_f16(float16x4_t __p0, float16x4_t __p1) { + float16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3, 4, 5, 6, 7); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vcombine_s32(int32x2_t __p0, int32x2_t __p1) { + int32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3); + return __ret; +} +#else +__ai int32x4_t vcombine_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 1, 2, 3); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vcombine_s32(int32x2_t __p0, int32x2_t __p1) { + int32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vcombine_s64(int64x1_t __p0, int64x1_t __p1) { + int64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1); + return __ret; +} +#else +__ai int64x2_t vcombine_s64(int64x1_t __p0, int64x1_t __p1) { + int64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vcombine_s16(int16x4_t __p0, int16x4_t __p1) { + int16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3, 4, 5, 6, 7); + return __ret; +} +#else +__ai int16x8_t vcombine_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 1, 2, 3, 4, 5, 6, 7); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int16x8_t __noswap_vcombine_s16(int16x4_t __p0, int16x4_t __p1) { + int16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1, 2, 3, 4, 5, 6, 7); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vcreate_p8(uint64_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vcreate_p8(uint64_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vcreate_p16(uint64_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vcreate_p16(uint64_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vcreate_u8(uint64_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vcreate_u8(uint64_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcreate_u32(uint64_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vcreate_u32(uint64_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcreate_u64(uint64_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vcreate_u64(uint64_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcreate_u16(uint64_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vcreate_u16(uint64_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vcreate_s8(uint64_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vcreate_s8(uint64_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vcreate_f32(uint64_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vcreate_f32(uint64_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vcreate_f16(uint64_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vcreate_f16(uint64_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vcreate_s32(uint64_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vcreate_s32(uint64_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vcreate_s64(uint64_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vcreate_s64(uint64_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vcreate_s16(uint64_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vcreate_s16(uint64_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vcvtq_f32_u32(uint32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vcvtq_f32_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai float32x4_t vcvtq_f32_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vcvtq_f32_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vcvtq_f32_s32(int32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vcvtq_f32_v((int8x16_t)__p0, 34); + return __ret; +} +#else +__ai float32x4_t vcvtq_f32_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vcvtq_f32_v((int8x16_t)__rev0, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vcvt_f32_u32(uint32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vcvt_f32_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai float32x2_t vcvt_f32_u32(uint32x2_t __p0) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vcvt_f32_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vcvt_f32_s32(int32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vcvt_f32_v((int8x8_t)__p0, 2); + return __ret; +} +#else +__ai float32x2_t vcvt_f32_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vcvt_f32_v((int8x8_t)__rev0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtq_n_f32_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vcvtq_n_f32_v((int8x16_t)__s0, __p1, 50); \ + __ret; \ +}) +#else +#define vcvtq_n_f32_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vcvtq_n_f32_v((int8x16_t)__rev0, __p1, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtq_n_f32_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vcvtq_n_f32_v((int8x16_t)__s0, __p1, 34); \ + __ret; \ +}) +#else +#define vcvtq_n_f32_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vcvtq_n_f32_v((int8x16_t)__rev0, __p1, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvt_n_f32_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vcvt_n_f32_v((int8x8_t)__s0, __p1, 18); \ + __ret; \ +}) +#else +#define vcvt_n_f32_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vcvt_n_f32_v((int8x8_t)__rev0, __p1, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvt_n_f32_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vcvt_n_f32_v((int8x8_t)__s0, __p1, 2); \ + __ret; \ +}) +#else +#define vcvt_n_f32_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vcvt_n_f32_v((int8x8_t)__rev0, __p1, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtq_n_s32_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vcvtq_n_s32_v((int8x16_t)__s0, __p1, 34); \ + __ret; \ +}) +#else +#define vcvtq_n_s32_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vcvtq_n_s32_v((int8x16_t)__rev0, __p1, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvt_n_s32_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vcvt_n_s32_v((int8x8_t)__s0, __p1, 2); \ + __ret; \ +}) +#else +#define vcvt_n_s32_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vcvt_n_s32_v((int8x8_t)__rev0, __p1, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtq_n_u32_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vcvtq_n_u32_v((int8x16_t)__s0, __p1, 50); \ + __ret; \ +}) +#else +#define vcvtq_n_u32_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vcvtq_n_u32_v((int8x16_t)__rev0, __p1, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvt_n_u32_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vcvt_n_u32_v((int8x8_t)__s0, __p1, 18); \ + __ret; \ +}) +#else +#define vcvt_n_u32_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vcvt_n_u32_v((int8x8_t)__rev0, __p1, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vcvtq_s32_f32(float32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vcvtq_s32_v((int8x16_t)__p0, 34); + return __ret; +} +#else +__ai int32x4_t vcvtq_s32_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vcvtq_s32_v((int8x16_t)__rev0, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vcvt_s32_f32(float32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vcvt_s32_v((int8x8_t)__p0, 2); + return __ret; +} +#else +__ai int32x2_t vcvt_s32_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vcvt_s32_v((int8x8_t)__rev0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcvtq_u32_f32(float32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcvtq_u32_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vcvtq_u32_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcvtq_u32_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcvt_u32_f32(float32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcvt_u32_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vcvt_u32_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcvt_u32_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_p8(__p0, __p1) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8x8_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_lane_p8(__p0, __p1) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_p16(__p0, __p1) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16x4_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_lane_p16(__p0, __p1) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + poly16x4_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_p8(__p0, __p1) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8x16_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_lane_p8(__p0, __p1) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_p16(__p0, __p1) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16x8_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_lane_p16(__p0, __p1) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + poly16x8_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x16_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_lane_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x4_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_lane_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x4_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_lane_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x8_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_lane_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x16_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_lane_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x4_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_lane_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x4_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x4_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_lane_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x4_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_lane_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x8_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_lane_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_lane_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_lane_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1); \ + __ret; \ +}) +#else +#define vdup_lane_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_lane_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_lane_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_lane_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x2_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_lane_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1); \ + __ret; \ +}) +#else +#define vdup_lane_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_lane_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vdup_n_p8(poly8_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai poly8x8_t vdup_n_p8(poly8_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vdup_n_p16(poly16_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t) {__p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai poly16x4_t vdup_n_p16(poly16_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t) {__p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vdupq_n_p8(poly8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai poly8x16_t vdupq_n_p8(poly8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vdupq_n_p16(poly16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai poly16x8_t vdupq_n_p16(poly16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vdupq_n_u8(uint8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai uint8x16_t vdupq_n_u8(uint8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vdupq_n_u32(uint32_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) {__p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai uint32x4_t vdupq_n_u32(uint32_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) {__p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vdupq_n_u64(uint64_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) {__p0, __p0}; + return __ret; +} +#else +__ai uint64x2_t vdupq_n_u64(uint64_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) {__p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vdupq_n_u16(uint16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai uint16x8_t vdupq_n_u16(uint16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vdupq_n_s8(int8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai int8x16_t vdupq_n_s8(int8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vdupq_n_f32(float32_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) {__p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai float32x4_t vdupq_n_f32(float32_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) {__p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_n_f16(__p0) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16x8_t __ret; \ + __ret = (float16x8_t) {__s0, __s0, __s0, __s0, __s0, __s0, __s0, __s0}; \ + __ret; \ +}) +#else +#define vdupq_n_f16(__p0) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16x8_t __ret; \ + __ret = (float16x8_t) {__s0, __s0, __s0, __s0, __s0, __s0, __s0, __s0}; \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vdupq_n_s32(int32_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) {__p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai int32x4_t vdupq_n_s32(int32_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) {__p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vdupq_n_s64(int64_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) {__p0, __p0}; + return __ret; +} +#else +__ai int64x2_t vdupq_n_s64(int64_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) {__p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vdupq_n_s16(int16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai int16x8_t vdupq_n_s16(int16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vdup_n_u8(uint8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai uint8x8_t vdup_n_u8(uint8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vdup_n_u32(uint32_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) {__p0, __p0}; + return __ret; +} +#else +__ai uint32x2_t vdup_n_u32(uint32_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) {__p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vdup_n_u64(uint64_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) {__p0}; + return __ret; +} +#else +__ai uint64x1_t vdup_n_u64(uint64_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) {__p0}; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vdup_n_u16(uint16_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) {__p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai uint16x4_t vdup_n_u16(uint16_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) {__p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vdup_n_s8(int8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai int8x8_t vdup_n_s8(int8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vdup_n_f32(float32_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) {__p0, __p0}; + return __ret; +} +#else +__ai float32x2_t vdup_n_f32(float32_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) {__p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_n_f16(__p0) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16x4_t __ret; \ + __ret = (float16x4_t) {__s0, __s0, __s0, __s0}; \ + __ret; \ +}) +#else +#define vdup_n_f16(__p0) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16x4_t __ret; \ + __ret = (float16x4_t) {__s0, __s0, __s0, __s0}; \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vdup_n_s32(int32_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) {__p0, __p0}; + return __ret; +} +#else +__ai int32x2_t vdup_n_s32(int32_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) {__p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vdup_n_s64(int64_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) {__p0}; + return __ret; +} +#else +__ai int64x1_t vdup_n_s64(int64_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) {__p0}; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vdup_n_s16(int16_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) {__p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai int16x4_t vdup_n_s16(int16_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) {__p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t veorq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai uint8x16_t veorq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __rev0 ^ __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t veorq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai uint32x4_t veorq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 ^ __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t veorq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai uint64x2_t veorq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __rev0 ^ __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t veorq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai uint16x8_t veorq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 ^ __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t veorq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai int8x16_t veorq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __rev0 ^ __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t veorq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai int32x4_t veorq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 ^ __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t veorq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai int64x2_t veorq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __rev0 ^ __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t veorq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai int16x8_t veorq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 ^ __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t veor_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai uint8x8_t veor_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __rev0 ^ __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t veor_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai uint32x2_t veor_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __rev0 ^ __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t veor_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai uint64x1_t veor_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t veor_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai uint16x4_t veor_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __rev0 ^ __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t veor_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai int8x8_t veor_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __rev0 ^ __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t veor_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai int32x2_t veor_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __rev0 ^ __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t veor_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai int64x1_t veor_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t veor_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = __p0 ^ __p1; + return __ret; +} +#else +__ai int16x4_t veor_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __rev0 ^ __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vext_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8x8_t __s1 = __p1; \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 4); \ + __ret; \ +}) +#else +#define vext_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8x8_t __s1 = __p1; \ + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vext_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 4); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vext_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16x4_t __s1 = __p1; \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 5); \ + __ret; \ +}) +#else +#define vext_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16x4_t __s1 = __p1; \ + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vext_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 5); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vextq_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8x16_t __s1 = __p1; \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vextq_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 36); \ + __ret; \ +}) +#else +#define vextq_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8x16_t __s1 = __p1; \ + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vextq_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 36); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vextq_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16x8_t __s1 = __p1; \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vextq_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 37); \ + __ret; \ +}) +#else +#define vextq_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16x8_t __s1 = __p1; \ + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vextq_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 37); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vextq_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vextq_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 48); \ + __ret; \ +}) +#else +#define vextq_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vextq_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 48); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vextq_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vextq_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 50); \ + __ret; \ +}) +#else +#define vextq_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vextq_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vextq_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vextq_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 51); \ + __ret; \ +}) +#else +#define vextq_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vextq_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vextq_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vextq_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 49); \ + __ret; \ +}) +#else +#define vextq_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vextq_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vextq_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __s1 = __p1; \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vextq_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 32); \ + __ret; \ +}) +#else +#define vextq_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __s1 = __p1; \ + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vextq_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 32); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vextq_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vextq_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 41); \ + __ret; \ +}) +#else +#define vextq_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vextq_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 41); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vextq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vextq_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 34); \ + __ret; \ +}) +#else +#define vextq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vextq_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vextq_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vextq_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 35); \ + __ret; \ +}) +#else +#define vextq_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __s1 = __p1; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vextq_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vextq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vextq_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 33); \ + __ret; \ +}) +#else +#define vextq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vextq_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vext_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 16); \ + __ret; \ +}) +#else +#define vext_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vext_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vext_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 18); \ + __ret; \ +}) +#else +#define vext_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vext_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vext_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 19); \ + __ret; \ +}) +#else +#define vext_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vext_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 17); \ + __ret; \ +}) +#else +#define vext_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vext_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vext_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __s1 = __p1; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 0); \ + __ret; \ +}) +#else +#define vext_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __s1 = __p1; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vext_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vext_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 9); \ + __ret; \ +}) +#else +#define vext_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vext_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 9); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vext_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 2); \ + __ret; \ +}) +#else +#define vext_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vext_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vext_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 3); \ + __ret; \ +}) +#else +#define vext_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vext_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 1); \ + __ret; \ +}) +#else +#define vext_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vext_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vget_high_p8(poly8x16_t __p0) { + poly8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 8, 9, 10, 11, 12, 13, 14, 15); + return __ret; +} +#else +__ai poly8x8_t vget_high_p8(poly8x16_t __p0) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 8, 9, 10, 11, 12, 13, 14, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai poly8x8_t __noswap_vget_high_p8(poly8x16_t __p0) { + poly8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 8, 9, 10, 11, 12, 13, 14, 15); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vget_high_p16(poly16x8_t __p0) { + poly16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 4, 5, 6, 7); + return __ret; +} +#else +__ai poly16x4_t vget_high_p16(poly16x8_t __p0) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 4, 5, 6, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vget_high_u8(uint8x16_t __p0) { + uint8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 8, 9, 10, 11, 12, 13, 14, 15); + return __ret; +} +#else +__ai uint8x8_t vget_high_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 8, 9, 10, 11, 12, 13, 14, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint8x8_t __noswap_vget_high_u8(uint8x16_t __p0) { + uint8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 8, 9, 10, 11, 12, 13, 14, 15); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vget_high_u32(uint32x4_t __p0) { + uint32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 2, 3); + return __ret; +} +#else +__ai uint32x2_t vget_high_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 2, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint32x2_t __noswap_vget_high_u32(uint32x4_t __p0) { + uint32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 2, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vget_high_u64(uint64x2_t __p0) { + uint64x1_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1); + return __ret; +} +#else +__ai uint64x1_t vget_high_u64(uint64x2_t __p0) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x1_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vget_high_u16(uint16x8_t __p0) { + uint16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 4, 5, 6, 7); + return __ret; +} +#else +__ai uint16x4_t vget_high_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 4, 5, 6, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint16x4_t __noswap_vget_high_u16(uint16x8_t __p0) { + uint16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 4, 5, 6, 7); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vget_high_s8(int8x16_t __p0) { + int8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 8, 9, 10, 11, 12, 13, 14, 15); + return __ret; +} +#else +__ai int8x8_t vget_high_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 8, 9, 10, 11, 12, 13, 14, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int8x8_t __noswap_vget_high_s8(int8x16_t __p0) { + int8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 8, 9, 10, 11, 12, 13, 14, 15); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vget_high_f32(float32x4_t __p0) { + float32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 2, 3); + return __ret; +} +#else +__ai float32x2_t vget_high_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 2, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai float32x2_t __noswap_vget_high_f32(float32x4_t __p0) { + float32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 2, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vget_high_f16(float16x8_t __p0) { + float16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 4, 5, 6, 7); + return __ret; +} +#else +__ai float16x4_t vget_high_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 4, 5, 6, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai float16x4_t __noswap_vget_high_f16(float16x8_t __p0) { + float16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 4, 5, 6, 7); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vget_high_s32(int32x4_t __p0) { + int32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 2, 3); + return __ret; +} +#else +__ai int32x2_t vget_high_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 2, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int32x2_t __noswap_vget_high_s32(int32x4_t __p0) { + int32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 2, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vget_high_s64(int64x2_t __p0) { + int64x1_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1); + return __ret; +} +#else +__ai int64x1_t vget_high_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x1_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vget_high_s16(int16x8_t __p0) { + int16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 4, 5, 6, 7); + return __ret; +} +#else +__ai int16x4_t vget_high_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 4, 5, 6, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int16x4_t __noswap_vget_high_s16(int16x8_t __p0) { + int16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 4, 5, 6, 7); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vget_lane_p8(__p0, __p1) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8_t __ret; \ + __ret = (poly8_t) __builtin_neon_vget_lane_i8((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vget_lane_p8(__p0, __p1) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8_t __ret; \ + __ret = (poly8_t) __builtin_neon_vget_lane_i8((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vget_lane_p8(__p0, __p1) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8_t __ret; \ + __ret = (poly8_t) __builtin_neon_vget_lane_i8((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vget_lane_p16(__p0, __p1) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16_t __ret; \ + __ret = (poly16_t) __builtin_neon_vget_lane_i16((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vget_lane_p16(__p0, __p1) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + poly16_t __ret; \ + __ret = (poly16_t) __builtin_neon_vget_lane_i16((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vget_lane_p16(__p0, __p1) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16_t __ret; \ + __ret = (poly16_t) __builtin_neon_vget_lane_i16((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vgetq_lane_p8(__p0, __p1) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8_t __ret; \ + __ret = (poly8_t) __builtin_neon_vgetq_lane_i8((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vgetq_lane_p8(__p0, __p1) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8_t __ret; \ + __ret = (poly8_t) __builtin_neon_vgetq_lane_i8((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vgetq_lane_p8(__p0, __p1) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8_t __ret; \ + __ret = (poly8_t) __builtin_neon_vgetq_lane_i8((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vgetq_lane_p16(__p0, __p1) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16_t __ret; \ + __ret = (poly16_t) __builtin_neon_vgetq_lane_i16((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vgetq_lane_p16(__p0, __p1) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16_t __ret; \ + __ret = (poly16_t) __builtin_neon_vgetq_lane_i16((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vgetq_lane_p16(__p0, __p1) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16_t __ret; \ + __ret = (poly16_t) __builtin_neon_vgetq_lane_i16((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vgetq_lane_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vgetq_lane_i8((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vgetq_lane_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vgetq_lane_i8((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vgetq_lane_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vgetq_lane_i8((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vgetq_lane_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vgetq_lane_i32((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vgetq_lane_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vgetq_lane_i32((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vgetq_lane_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vgetq_lane_i32((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vgetq_lane_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vgetq_lane_i64((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vgetq_lane_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vgetq_lane_i64((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vgetq_lane_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vgetq_lane_i64((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vgetq_lane_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vgetq_lane_i16((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vgetq_lane_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vgetq_lane_i16((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vgetq_lane_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vgetq_lane_i16((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vgetq_lane_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vgetq_lane_i8((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vgetq_lane_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vgetq_lane_i8((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vgetq_lane_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vgetq_lane_i8((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vgetq_lane_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vgetq_lane_f32((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vgetq_lane_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vgetq_lane_f32((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vgetq_lane_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vgetq_lane_f32((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vgetq_lane_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vgetq_lane_f16((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vgetq_lane_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vgetq_lane_f16((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vgetq_lane_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vgetq_lane_f16((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vgetq_lane_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vgetq_lane_i32((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vgetq_lane_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vgetq_lane_i32((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vgetq_lane_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vgetq_lane_i32((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vgetq_lane_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vgetq_lane_i64((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vgetq_lane_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vgetq_lane_i64((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vgetq_lane_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vgetq_lane_i64((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vgetq_lane_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vgetq_lane_i16((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vgetq_lane_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vgetq_lane_i16((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vgetq_lane_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vgetq_lane_i16((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vget_lane_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vget_lane_i8((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vget_lane_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vget_lane_i8((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vget_lane_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vget_lane_i8((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vget_lane_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vget_lane_i32((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vget_lane_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vget_lane_i32((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vget_lane_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vget_lane_i32((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vget_lane_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vget_lane_i64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vget_lane_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vget_lane_i64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#define __noswap_vget_lane_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vget_lane_i64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vget_lane_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vget_lane_i16((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vget_lane_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vget_lane_i16((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vget_lane_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vget_lane_i16((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vget_lane_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vget_lane_i8((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vget_lane_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vget_lane_i8((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vget_lane_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vget_lane_i8((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vget_lane_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vget_lane_f32((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vget_lane_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vget_lane_f32((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vget_lane_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vget_lane_f32((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vget_lane_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vget_lane_f16((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vget_lane_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vget_lane_f16((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vget_lane_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vget_lane_f16((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vget_lane_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vget_lane_i32((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vget_lane_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vget_lane_i32((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vget_lane_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vget_lane_i32((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vget_lane_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vget_lane_i64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vget_lane_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vget_lane_i64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#define __noswap_vget_lane_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vget_lane_i64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vget_lane_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vget_lane_i16((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vget_lane_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vget_lane_i16((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vget_lane_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vget_lane_i16((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vget_low_p8(poly8x16_t __p0) { + poly8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0, 1, 2, 3, 4, 5, 6, 7); + return __ret; +} +#else +__ai poly8x8_t vget_low_p8(poly8x16_t __p0) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 0, 1, 2, 3, 4, 5, 6, 7); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vget_low_p16(poly16x8_t __p0) { + poly16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0, 1, 2, 3); + return __ret; +} +#else +__ai poly16x4_t vget_low_p16(poly16x8_t __p0) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 0, 1, 2, 3); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vget_low_u8(uint8x16_t __p0) { + uint8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0, 1, 2, 3, 4, 5, 6, 7); + return __ret; +} +#else +__ai uint8x8_t vget_low_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 0, 1, 2, 3, 4, 5, 6, 7); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vget_low_u32(uint32x4_t __p0) { + uint32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0, 1); + return __ret; +} +#else +__ai uint32x2_t vget_low_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vget_low_u64(uint64x2_t __p0) { + uint64x1_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0); + return __ret; +} +#else +__ai uint64x1_t vget_low_u64(uint64x2_t __p0) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x1_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vget_low_u16(uint16x8_t __p0) { + uint16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0, 1, 2, 3); + return __ret; +} +#else +__ai uint16x4_t vget_low_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 0, 1, 2, 3); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vget_low_s8(int8x16_t __p0) { + int8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0, 1, 2, 3, 4, 5, 6, 7); + return __ret; +} +#else +__ai int8x8_t vget_low_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 0, 1, 2, 3, 4, 5, 6, 7); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vget_low_f32(float32x4_t __p0) { + float32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0, 1); + return __ret; +} +#else +__ai float32x2_t vget_low_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vget_low_f16(float16x8_t __p0) { + float16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0, 1, 2, 3); + return __ret; +} +#else +__ai float16x4_t vget_low_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 0, 1, 2, 3); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vget_low_s32(int32x4_t __p0) { + int32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0, 1); + return __ret; +} +#else +__ai int32x2_t vget_low_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vget_low_s64(int64x2_t __p0) { + int64x1_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0); + return __ret; +} +#else +__ai int64x1_t vget_low_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x1_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vget_low_s16(int16x8_t __p0) { + int16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0, 1, 2, 3); + return __ret; +} +#else +__ai int16x4_t vget_low_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 0, 1, 2, 3); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vhaddq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vhaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vhaddq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vhaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vhaddq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vhaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vhaddq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vhaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vhaddq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vhaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vhaddq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vhaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vhaddq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vhaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vhaddq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vhaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vhaddq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vhaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vhaddq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vhaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vhaddq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vhaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vhaddq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vhaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vhadd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vhadd_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vhadd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vhadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vhadd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vhadd_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vhadd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vhadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vhadd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vhadd_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vhadd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vhadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vhadd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vhadd_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vhadd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vhadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vhadd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vhadd_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vhadd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vhadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vhadd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vhadd_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vhadd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vhadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vhsubq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vhsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vhsubq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vhsubq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vhsubq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vhsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vhsubq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vhsubq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vhsubq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vhsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vhsubq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vhsubq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vhsubq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vhsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vhsubq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vhsubq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vhsubq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vhsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vhsubq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vhsubq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vhsubq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vhsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vhsubq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vhsubq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vhsub_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vhsub_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vhsub_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vhsub_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vhsub_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vhsub_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vhsub_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vhsub_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vhsub_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vhsub_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vhsub_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vhsub_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vhsub_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vhsub_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vhsub_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vhsub_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vhsub_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vhsub_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vhsub_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vhsub_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vhsub_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vhsub_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vhsub_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_p8(__p0) __extension__ ({ \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vld1_v(__p0, 4); \ + __ret; \ +}) +#else +#define vld1_p8(__p0) __extension__ ({ \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vld1_v(__p0, 4); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_p16(__p0) __extension__ ({ \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vld1_v(__p0, 5); \ + __ret; \ +}) +#else +#define vld1_p16(__p0) __extension__ ({ \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vld1_v(__p0, 5); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_p8(__p0) __extension__ ({ \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vld1q_v(__p0, 36); \ + __ret; \ +}) +#else +#define vld1q_p8(__p0) __extension__ ({ \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vld1q_v(__p0, 36); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_p16(__p0) __extension__ ({ \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vld1q_v(__p0, 37); \ + __ret; \ +}) +#else +#define vld1q_p16(__p0) __extension__ ({ \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vld1q_v(__p0, 37); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u8(__p0) __extension__ ({ \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vld1q_v(__p0, 48); \ + __ret; \ +}) +#else +#define vld1q_u8(__p0) __extension__ ({ \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vld1q_v(__p0, 48); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u32(__p0) __extension__ ({ \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vld1q_v(__p0, 50); \ + __ret; \ +}) +#else +#define vld1q_u32(__p0) __extension__ ({ \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vld1q_v(__p0, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u64(__p0) __extension__ ({ \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vld1q_v(__p0, 51); \ + __ret; \ +}) +#else +#define vld1q_u64(__p0) __extension__ ({ \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vld1q_v(__p0, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u16(__p0) __extension__ ({ \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vld1q_v(__p0, 49); \ + __ret; \ +}) +#else +#define vld1q_u16(__p0) __extension__ ({ \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vld1q_v(__p0, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s8(__p0) __extension__ ({ \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vld1q_v(__p0, 32); \ + __ret; \ +}) +#else +#define vld1q_s8(__p0) __extension__ ({ \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vld1q_v(__p0, 32); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_f32(__p0) __extension__ ({ \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vld1q_v(__p0, 41); \ + __ret; \ +}) +#else +#define vld1q_f32(__p0) __extension__ ({ \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vld1q_v(__p0, 41); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_f16(__p0) __extension__ ({ \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vld1q_v(__p0, 40); \ + __ret; \ +}) +#else +#define vld1q_f16(__p0) __extension__ ({ \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vld1q_v(__p0, 40); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s32(__p0) __extension__ ({ \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vld1q_v(__p0, 34); \ + __ret; \ +}) +#else +#define vld1q_s32(__p0) __extension__ ({ \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vld1q_v(__p0, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s64(__p0) __extension__ ({ \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vld1q_v(__p0, 35); \ + __ret; \ +}) +#else +#define vld1q_s64(__p0) __extension__ ({ \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vld1q_v(__p0, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s16(__p0) __extension__ ({ \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vld1q_v(__p0, 33); \ + __ret; \ +}) +#else +#define vld1q_s16(__p0) __extension__ ({ \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vld1q_v(__p0, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u8(__p0) __extension__ ({ \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vld1_v(__p0, 16); \ + __ret; \ +}) +#else +#define vld1_u8(__p0) __extension__ ({ \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vld1_v(__p0, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u32(__p0) __extension__ ({ \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vld1_v(__p0, 18); \ + __ret; \ +}) +#else +#define vld1_u32(__p0) __extension__ ({ \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vld1_v(__p0, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u64(__p0) __extension__ ({ \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vld1_v(__p0, 19); \ + __ret; \ +}) +#else +#define vld1_u64(__p0) __extension__ ({ \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vld1_v(__p0, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u16(__p0) __extension__ ({ \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vld1_v(__p0, 17); \ + __ret; \ +}) +#else +#define vld1_u16(__p0) __extension__ ({ \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vld1_v(__p0, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s8(__p0) __extension__ ({ \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vld1_v(__p0, 0); \ + __ret; \ +}) +#else +#define vld1_s8(__p0) __extension__ ({ \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vld1_v(__p0, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_f32(__p0) __extension__ ({ \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vld1_v(__p0, 9); \ + __ret; \ +}) +#else +#define vld1_f32(__p0) __extension__ ({ \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vld1_v(__p0, 9); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_f16(__p0) __extension__ ({ \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vld1_v(__p0, 8); \ + __ret; \ +}) +#else +#define vld1_f16(__p0) __extension__ ({ \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vld1_v(__p0, 8); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s32(__p0) __extension__ ({ \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vld1_v(__p0, 2); \ + __ret; \ +}) +#else +#define vld1_s32(__p0) __extension__ ({ \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vld1_v(__p0, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s64(__p0) __extension__ ({ \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vld1_v(__p0, 3); \ + __ret; \ +}) +#else +#define vld1_s64(__p0) __extension__ ({ \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vld1_v(__p0, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s16(__p0) __extension__ ({ \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vld1_v(__p0, 1); \ + __ret; \ +}) +#else +#define vld1_s16(__p0) __extension__ ({ \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vld1_v(__p0, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_dup_p8(__p0) __extension__ ({ \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vld1_dup_v(__p0, 4); \ + __ret; \ +}) +#else +#define vld1_dup_p8(__p0) __extension__ ({ \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vld1_dup_v(__p0, 4); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_dup_p16(__p0) __extension__ ({ \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vld1_dup_v(__p0, 5); \ + __ret; \ +}) +#else +#define vld1_dup_p16(__p0) __extension__ ({ \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vld1_dup_v(__p0, 5); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_dup_p8(__p0) __extension__ ({ \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vld1q_dup_v(__p0, 36); \ + __ret; \ +}) +#else +#define vld1q_dup_p8(__p0) __extension__ ({ \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vld1q_dup_v(__p0, 36); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_dup_p16(__p0) __extension__ ({ \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vld1q_dup_v(__p0, 37); \ + __ret; \ +}) +#else +#define vld1q_dup_p16(__p0) __extension__ ({ \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vld1q_dup_v(__p0, 37); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_dup_u8(__p0) __extension__ ({ \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vld1q_dup_v(__p0, 48); \ + __ret; \ +}) +#else +#define vld1q_dup_u8(__p0) __extension__ ({ \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vld1q_dup_v(__p0, 48); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_dup_u32(__p0) __extension__ ({ \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vld1q_dup_v(__p0, 50); \ + __ret; \ +}) +#else +#define vld1q_dup_u32(__p0) __extension__ ({ \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vld1q_dup_v(__p0, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_dup_u64(__p0) __extension__ ({ \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vld1q_dup_v(__p0, 51); \ + __ret; \ +}) +#else +#define vld1q_dup_u64(__p0) __extension__ ({ \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vld1q_dup_v(__p0, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_dup_u16(__p0) __extension__ ({ \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vld1q_dup_v(__p0, 49); \ + __ret; \ +}) +#else +#define vld1q_dup_u16(__p0) __extension__ ({ \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vld1q_dup_v(__p0, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_dup_s8(__p0) __extension__ ({ \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vld1q_dup_v(__p0, 32); \ + __ret; \ +}) +#else +#define vld1q_dup_s8(__p0) __extension__ ({ \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vld1q_dup_v(__p0, 32); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_dup_f32(__p0) __extension__ ({ \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vld1q_dup_v(__p0, 41); \ + __ret; \ +}) +#else +#define vld1q_dup_f32(__p0) __extension__ ({ \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vld1q_dup_v(__p0, 41); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_dup_f16(__p0) __extension__ ({ \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vld1q_dup_v(__p0, 40); \ + __ret; \ +}) +#else +#define vld1q_dup_f16(__p0) __extension__ ({ \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vld1q_dup_v(__p0, 40); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_dup_s32(__p0) __extension__ ({ \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vld1q_dup_v(__p0, 34); \ + __ret; \ +}) +#else +#define vld1q_dup_s32(__p0) __extension__ ({ \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vld1q_dup_v(__p0, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_dup_s64(__p0) __extension__ ({ \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vld1q_dup_v(__p0, 35); \ + __ret; \ +}) +#else +#define vld1q_dup_s64(__p0) __extension__ ({ \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vld1q_dup_v(__p0, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_dup_s16(__p0) __extension__ ({ \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vld1q_dup_v(__p0, 33); \ + __ret; \ +}) +#else +#define vld1q_dup_s16(__p0) __extension__ ({ \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vld1q_dup_v(__p0, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_dup_u8(__p0) __extension__ ({ \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vld1_dup_v(__p0, 16); \ + __ret; \ +}) +#else +#define vld1_dup_u8(__p0) __extension__ ({ \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vld1_dup_v(__p0, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_dup_u32(__p0) __extension__ ({ \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vld1_dup_v(__p0, 18); \ + __ret; \ +}) +#else +#define vld1_dup_u32(__p0) __extension__ ({ \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vld1_dup_v(__p0, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_dup_u64(__p0) __extension__ ({ \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vld1_dup_v(__p0, 19); \ + __ret; \ +}) +#else +#define vld1_dup_u64(__p0) __extension__ ({ \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vld1_dup_v(__p0, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_dup_u16(__p0) __extension__ ({ \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vld1_dup_v(__p0, 17); \ + __ret; \ +}) +#else +#define vld1_dup_u16(__p0) __extension__ ({ \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vld1_dup_v(__p0, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_dup_s8(__p0) __extension__ ({ \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vld1_dup_v(__p0, 0); \ + __ret; \ +}) +#else +#define vld1_dup_s8(__p0) __extension__ ({ \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vld1_dup_v(__p0, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_dup_f32(__p0) __extension__ ({ \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vld1_dup_v(__p0, 9); \ + __ret; \ +}) +#else +#define vld1_dup_f32(__p0) __extension__ ({ \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vld1_dup_v(__p0, 9); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_dup_f16(__p0) __extension__ ({ \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vld1_dup_v(__p0, 8); \ + __ret; \ +}) +#else +#define vld1_dup_f16(__p0) __extension__ ({ \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vld1_dup_v(__p0, 8); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_dup_s32(__p0) __extension__ ({ \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vld1_dup_v(__p0, 2); \ + __ret; \ +}) +#else +#define vld1_dup_s32(__p0) __extension__ ({ \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vld1_dup_v(__p0, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_dup_s64(__p0) __extension__ ({ \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vld1_dup_v(__p0, 3); \ + __ret; \ +}) +#else +#define vld1_dup_s64(__p0) __extension__ ({ \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vld1_dup_v(__p0, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_dup_s16(__p0) __extension__ ({ \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vld1_dup_v(__p0, 1); \ + __ret; \ +}) +#else +#define vld1_dup_s16(__p0) __extension__ ({ \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vld1_dup_v(__p0, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8_t __s1 = __p1; \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 4); \ + __ret; \ +}) +#else +#define vld1_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8_t __s1 = __p1; \ + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__rev1, __p2, 4); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4_t __s1 = __p1; \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 5); \ + __ret; \ +}) +#else +#define vld1_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4_t __s1 = __p1; \ + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__rev1, __p2, 5); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16_t __s1 = __p1; \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 36); \ + __ret; \ +}) +#else +#define vld1q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16_t __s1 = __p1; \ + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 36); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8_t __s1 = __p1; \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 37); \ + __ret; \ +}) +#else +#define vld1q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8_t __s1 = __p1; \ + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 37); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 48); \ + __ret; \ +}) +#else +#define vld1q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 48); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 50); \ + __ret; \ +}) +#else +#define vld1q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 51); \ + __ret; \ +}) +#else +#define vld1q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 49); \ + __ret; \ +}) +#else +#define vld1q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16_t __s1 = __p1; \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 32); \ + __ret; \ +}) +#else +#define vld1q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16_t __s1 = __p1; \ + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 32); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4_t __s1 = __p1; \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 41); \ + __ret; \ +}) +#else +#define vld1q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4_t __s1 = __p1; \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 41); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __s1 = __p1; \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 40); \ + __ret; \ +}) +#else +#define vld1q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __s1 = __p1; \ + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 40); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 34); \ + __ret; \ +}) +#else +#define vld1q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s1 = __p1; \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 35); \ + __ret; \ +}) +#else +#define vld1q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2_t __s1 = __p1; \ + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s1 = __p1; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 33); \ + __ret; \ +}) +#else +#define vld1q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s1 = __p1; \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 16); \ + __ret; \ +}) +#else +#define vld1_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__rev1, __p2, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 18); \ + __ret; \ +}) +#else +#define vld1_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__rev1, __p2, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 19); \ + __ret; \ +}) +#else +#define vld1_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 17); \ + __ret; \ +}) +#else +#define vld1_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__rev1, __p2, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8_t __s1 = __p1; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 0); \ + __ret; \ +}) +#else +#define vld1_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8_t __s1 = __p1; \ + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__rev1, __p2, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2_t __s1 = __p1; \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 9); \ + __ret; \ +}) +#else +#define vld1_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2_t __s1 = __p1; \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__rev1, __p2, 9); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __s1 = __p1; \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 8); \ + __ret; \ +}) +#else +#define vld1_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __s1 = __p1; \ + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__rev1, __p2, 8); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s1 = __p1; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 2); \ + __ret; \ +}) +#else +#define vld1_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s1 = __p1; \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__rev1, __p2, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 3); \ + __ret; \ +}) +#else +#define vld1_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s1 = __p1; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 1); \ + __ret; \ +}) +#else +#define vld1_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s1 = __p1; \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__rev1, __p2, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_p8(__p0) __extension__ ({ \ + poly8x8x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 4); \ + __ret; \ +}) +#else +#define vld2_p8(__p0) __extension__ ({ \ + poly8x8x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 4); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_p16(__p0) __extension__ ({ \ + poly16x4x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 5); \ + __ret; \ +}) +#else +#define vld2_p16(__p0) __extension__ ({ \ + poly16x4x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 5); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_p8(__p0) __extension__ ({ \ + poly8x16x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 36); \ + __ret; \ +}) +#else +#define vld2q_p8(__p0) __extension__ ({ \ + poly8x16x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 36); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_p16(__p0) __extension__ ({ \ + poly16x8x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 37); \ + __ret; \ +}) +#else +#define vld2q_p16(__p0) __extension__ ({ \ + poly16x8x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 37); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_u8(__p0) __extension__ ({ \ + uint8x16x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 48); \ + __ret; \ +}) +#else +#define vld2q_u8(__p0) __extension__ ({ \ + uint8x16x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 48); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_u32(__p0) __extension__ ({ \ + uint32x4x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 50); \ + __ret; \ +}) +#else +#define vld2q_u32(__p0) __extension__ ({ \ + uint32x4x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 50); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_u16(__p0) __extension__ ({ \ + uint16x8x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 49); \ + __ret; \ +}) +#else +#define vld2q_u16(__p0) __extension__ ({ \ + uint16x8x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 49); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_s8(__p0) __extension__ ({ \ + int8x16x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 32); \ + __ret; \ +}) +#else +#define vld2q_s8(__p0) __extension__ ({ \ + int8x16x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 32); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_f32(__p0) __extension__ ({ \ + float32x4x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 41); \ + __ret; \ +}) +#else +#define vld2q_f32(__p0) __extension__ ({ \ + float32x4x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 41); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_f16(__p0) __extension__ ({ \ + float16x8x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 40); \ + __ret; \ +}) +#else +#define vld2q_f16(__p0) __extension__ ({ \ + float16x8x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 40); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_s32(__p0) __extension__ ({ \ + int32x4x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 34); \ + __ret; \ +}) +#else +#define vld2q_s32(__p0) __extension__ ({ \ + int32x4x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 34); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_s16(__p0) __extension__ ({ \ + int16x8x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 33); \ + __ret; \ +}) +#else +#define vld2q_s16(__p0) __extension__ ({ \ + int16x8x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 33); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_u8(__p0) __extension__ ({ \ + uint8x8x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 16); \ + __ret; \ +}) +#else +#define vld2_u8(__p0) __extension__ ({ \ + uint8x8x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 16); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_u32(__p0) __extension__ ({ \ + uint32x2x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 18); \ + __ret; \ +}) +#else +#define vld2_u32(__p0) __extension__ ({ \ + uint32x2x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 18); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_u64(__p0) __extension__ ({ \ + uint64x1x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 19); \ + __ret; \ +}) +#else +#define vld2_u64(__p0) __extension__ ({ \ + uint64x1x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_u16(__p0) __extension__ ({ \ + uint16x4x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 17); \ + __ret; \ +}) +#else +#define vld2_u16(__p0) __extension__ ({ \ + uint16x4x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 17); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_s8(__p0) __extension__ ({ \ + int8x8x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 0); \ + __ret; \ +}) +#else +#define vld2_s8(__p0) __extension__ ({ \ + int8x8x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 0); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_f32(__p0) __extension__ ({ \ + float32x2x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 9); \ + __ret; \ +}) +#else +#define vld2_f32(__p0) __extension__ ({ \ + float32x2x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 9); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_f16(__p0) __extension__ ({ \ + float16x4x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 8); \ + __ret; \ +}) +#else +#define vld2_f16(__p0) __extension__ ({ \ + float16x4x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 8); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_s32(__p0) __extension__ ({ \ + int32x2x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 2); \ + __ret; \ +}) +#else +#define vld2_s32(__p0) __extension__ ({ \ + int32x2x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 2); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_s64(__p0) __extension__ ({ \ + int64x1x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 3); \ + __ret; \ +}) +#else +#define vld2_s64(__p0) __extension__ ({ \ + int64x1x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_s16(__p0) __extension__ ({ \ + int16x4x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 1); \ + __ret; \ +}) +#else +#define vld2_s16(__p0) __extension__ ({ \ + int16x4x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 1); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_dup_p8(__p0) __extension__ ({ \ + poly8x8x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 4); \ + __ret; \ +}) +#else +#define vld2_dup_p8(__p0) __extension__ ({ \ + poly8x8x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 4); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_dup_p16(__p0) __extension__ ({ \ + poly16x4x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 5); \ + __ret; \ +}) +#else +#define vld2_dup_p16(__p0) __extension__ ({ \ + poly16x4x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 5); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_dup_u8(__p0) __extension__ ({ \ + uint8x8x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 16); \ + __ret; \ +}) +#else +#define vld2_dup_u8(__p0) __extension__ ({ \ + uint8x8x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 16); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_dup_u32(__p0) __extension__ ({ \ + uint32x2x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 18); \ + __ret; \ +}) +#else +#define vld2_dup_u32(__p0) __extension__ ({ \ + uint32x2x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 18); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_dup_u64(__p0) __extension__ ({ \ + uint64x1x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 19); \ + __ret; \ +}) +#else +#define vld2_dup_u64(__p0) __extension__ ({ \ + uint64x1x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_dup_u16(__p0) __extension__ ({ \ + uint16x4x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 17); \ + __ret; \ +}) +#else +#define vld2_dup_u16(__p0) __extension__ ({ \ + uint16x4x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 17); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_dup_s8(__p0) __extension__ ({ \ + int8x8x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 0); \ + __ret; \ +}) +#else +#define vld2_dup_s8(__p0) __extension__ ({ \ + int8x8x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 0); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_dup_f32(__p0) __extension__ ({ \ + float32x2x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 9); \ + __ret; \ +}) +#else +#define vld2_dup_f32(__p0) __extension__ ({ \ + float32x2x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 9); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_dup_f16(__p0) __extension__ ({ \ + float16x4x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 8); \ + __ret; \ +}) +#else +#define vld2_dup_f16(__p0) __extension__ ({ \ + float16x4x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 8); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_dup_s32(__p0) __extension__ ({ \ + int32x2x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 2); \ + __ret; \ +}) +#else +#define vld2_dup_s32(__p0) __extension__ ({ \ + int32x2x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 2); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_dup_s64(__p0) __extension__ ({ \ + int64x1x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 3); \ + __ret; \ +}) +#else +#define vld2_dup_s64(__p0) __extension__ ({ \ + int64x1x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_dup_s16(__p0) __extension__ ({ \ + int16x4x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 1); \ + __ret; \ +}) +#else +#define vld2_dup_s16(__p0) __extension__ ({ \ + int16x4x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 1); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8x2_t __s1 = __p1; \ + poly8x8x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 4); \ + __ret; \ +}) +#else +#define vld2_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8x2_t __s1 = __p1; \ + poly8x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], __p2, 4); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4x2_t __s1 = __p1; \ + poly16x4x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 5); \ + __ret; \ +}) +#else +#define vld2_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4x2_t __s1 = __p1; \ + poly16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + poly16x4x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], __p2, 5); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8x2_t __s1 = __p1; \ + poly16x8x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 37); \ + __ret; \ +}) +#else +#define vld2q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8x2_t __s1 = __p1; \ + poly16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x8x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 37); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4x2_t __s1 = __p1; \ + uint32x4x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 50); \ + __ret; \ +}) +#else +#define vld2q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4x2_t __s1 = __p1; \ + uint32x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + uint32x4x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 50); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8x2_t __s1 = __p1; \ + uint16x8x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 49); \ + __ret; \ +}) +#else +#define vld2q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8x2_t __s1 = __p1; \ + uint16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 49); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4x2_t __s1 = __p1; \ + float32x4x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 41); \ + __ret; \ +}) +#else +#define vld2q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4x2_t __s1 = __p1; \ + float32x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + float32x4x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __p2, 41); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8x2_t __s1 = __p1; \ + float16x8x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 40); \ + __ret; \ +}) +#else +#define vld2q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8x2_t __s1 = __p1; \ + float16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __p2, 40); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4x2_t __s1 = __p1; \ + int32x4x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 34); \ + __ret; \ +}) +#else +#define vld2q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4x2_t __s1 = __p1; \ + int32x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + int32x4x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __p2, 34); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8x2_t __s1 = __p1; \ + int16x8x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 33); \ + __ret; \ +}) +#else +#define vld2q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8x2_t __s1 = __p1; \ + int16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __p2, 33); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8x2_t __s1 = __p1; \ + uint8x8x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 16); \ + __ret; \ +}) +#else +#define vld2_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8x2_t __s1 = __p1; \ + uint8x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], __p2, 16); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2x2_t __s1 = __p1; \ + uint32x2x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 18); \ + __ret; \ +}) +#else +#define vld2_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2x2_t __s1 = __p1; \ + uint32x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + uint32x2x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], __p2, 18); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4x2_t __s1 = __p1; \ + uint16x4x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 17); \ + __ret; \ +}) +#else +#define vld2_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4x2_t __s1 = __p1; \ + uint16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + uint16x4x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], __p2, 17); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8x2_t __s1 = __p1; \ + int8x8x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 0); \ + __ret; \ +}) +#else +#define vld2_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8x2_t __s1 = __p1; \ + int8x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], __p2, 0); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2x2_t __s1 = __p1; \ + float32x2x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 9); \ + __ret; \ +}) +#else +#define vld2_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2x2_t __s1 = __p1; \ + float32x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + float32x2x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __p2, 9); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4x2_t __s1 = __p1; \ + float16x4x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 8); \ + __ret; \ +}) +#else +#define vld2_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4x2_t __s1 = __p1; \ + float16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + float16x4x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __p2, 8); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2x2_t __s1 = __p1; \ + int32x2x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 2); \ + __ret; \ +}) +#else +#define vld2_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2x2_t __s1 = __p1; \ + int32x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + int32x2x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __p2, 2); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4x2_t __s1 = __p1; \ + int16x4x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 1); \ + __ret; \ +}) +#else +#define vld2_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4x2_t __s1 = __p1; \ + int16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + int16x4x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __p2, 1); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_p8(__p0) __extension__ ({ \ + poly8x8x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 4); \ + __ret; \ +}) +#else +#define vld3_p8(__p0) __extension__ ({ \ + poly8x8x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 4); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_p16(__p0) __extension__ ({ \ + poly16x4x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 5); \ + __ret; \ +}) +#else +#define vld3_p16(__p0) __extension__ ({ \ + poly16x4x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 5); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_p8(__p0) __extension__ ({ \ + poly8x16x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 36); \ + __ret; \ +}) +#else +#define vld3q_p8(__p0) __extension__ ({ \ + poly8x16x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 36); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_p16(__p0) __extension__ ({ \ + poly16x8x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 37); \ + __ret; \ +}) +#else +#define vld3q_p16(__p0) __extension__ ({ \ + poly16x8x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 37); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_u8(__p0) __extension__ ({ \ + uint8x16x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 48); \ + __ret; \ +}) +#else +#define vld3q_u8(__p0) __extension__ ({ \ + uint8x16x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 48); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_u32(__p0) __extension__ ({ \ + uint32x4x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 50); \ + __ret; \ +}) +#else +#define vld3q_u32(__p0) __extension__ ({ \ + uint32x4x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 50); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_u16(__p0) __extension__ ({ \ + uint16x8x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 49); \ + __ret; \ +}) +#else +#define vld3q_u16(__p0) __extension__ ({ \ + uint16x8x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 49); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_s8(__p0) __extension__ ({ \ + int8x16x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 32); \ + __ret; \ +}) +#else +#define vld3q_s8(__p0) __extension__ ({ \ + int8x16x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 32); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_f32(__p0) __extension__ ({ \ + float32x4x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 41); \ + __ret; \ +}) +#else +#define vld3q_f32(__p0) __extension__ ({ \ + float32x4x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 41); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_f16(__p0) __extension__ ({ \ + float16x8x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 40); \ + __ret; \ +}) +#else +#define vld3q_f16(__p0) __extension__ ({ \ + float16x8x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 40); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_s32(__p0) __extension__ ({ \ + int32x4x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 34); \ + __ret; \ +}) +#else +#define vld3q_s32(__p0) __extension__ ({ \ + int32x4x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 34); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_s16(__p0) __extension__ ({ \ + int16x8x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 33); \ + __ret; \ +}) +#else +#define vld3q_s16(__p0) __extension__ ({ \ + int16x8x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 33); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_u8(__p0) __extension__ ({ \ + uint8x8x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 16); \ + __ret; \ +}) +#else +#define vld3_u8(__p0) __extension__ ({ \ + uint8x8x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 16); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_u32(__p0) __extension__ ({ \ + uint32x2x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 18); \ + __ret; \ +}) +#else +#define vld3_u32(__p0) __extension__ ({ \ + uint32x2x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 18); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_u64(__p0) __extension__ ({ \ + uint64x1x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 19); \ + __ret; \ +}) +#else +#define vld3_u64(__p0) __extension__ ({ \ + uint64x1x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_u16(__p0) __extension__ ({ \ + uint16x4x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 17); \ + __ret; \ +}) +#else +#define vld3_u16(__p0) __extension__ ({ \ + uint16x4x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 17); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_s8(__p0) __extension__ ({ \ + int8x8x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 0); \ + __ret; \ +}) +#else +#define vld3_s8(__p0) __extension__ ({ \ + int8x8x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 0); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_f32(__p0) __extension__ ({ \ + float32x2x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 9); \ + __ret; \ +}) +#else +#define vld3_f32(__p0) __extension__ ({ \ + float32x2x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 9); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_f16(__p0) __extension__ ({ \ + float16x4x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 8); \ + __ret; \ +}) +#else +#define vld3_f16(__p0) __extension__ ({ \ + float16x4x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 8); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_s32(__p0) __extension__ ({ \ + int32x2x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 2); \ + __ret; \ +}) +#else +#define vld3_s32(__p0) __extension__ ({ \ + int32x2x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 2); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_s64(__p0) __extension__ ({ \ + int64x1x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 3); \ + __ret; \ +}) +#else +#define vld3_s64(__p0) __extension__ ({ \ + int64x1x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_s16(__p0) __extension__ ({ \ + int16x4x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 1); \ + __ret; \ +}) +#else +#define vld3_s16(__p0) __extension__ ({ \ + int16x4x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 1); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_dup_p8(__p0) __extension__ ({ \ + poly8x8x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 4); \ + __ret; \ +}) +#else +#define vld3_dup_p8(__p0) __extension__ ({ \ + poly8x8x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 4); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_dup_p16(__p0) __extension__ ({ \ + poly16x4x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 5); \ + __ret; \ +}) +#else +#define vld3_dup_p16(__p0) __extension__ ({ \ + poly16x4x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 5); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_dup_u8(__p0) __extension__ ({ \ + uint8x8x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 16); \ + __ret; \ +}) +#else +#define vld3_dup_u8(__p0) __extension__ ({ \ + uint8x8x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 16); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_dup_u32(__p0) __extension__ ({ \ + uint32x2x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 18); \ + __ret; \ +}) +#else +#define vld3_dup_u32(__p0) __extension__ ({ \ + uint32x2x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 18); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_dup_u64(__p0) __extension__ ({ \ + uint64x1x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 19); \ + __ret; \ +}) +#else +#define vld3_dup_u64(__p0) __extension__ ({ \ + uint64x1x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_dup_u16(__p0) __extension__ ({ \ + uint16x4x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 17); \ + __ret; \ +}) +#else +#define vld3_dup_u16(__p0) __extension__ ({ \ + uint16x4x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 17); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_dup_s8(__p0) __extension__ ({ \ + int8x8x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 0); \ + __ret; \ +}) +#else +#define vld3_dup_s8(__p0) __extension__ ({ \ + int8x8x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 0); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_dup_f32(__p0) __extension__ ({ \ + float32x2x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 9); \ + __ret; \ +}) +#else +#define vld3_dup_f32(__p0) __extension__ ({ \ + float32x2x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 9); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_dup_f16(__p0) __extension__ ({ \ + float16x4x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 8); \ + __ret; \ +}) +#else +#define vld3_dup_f16(__p0) __extension__ ({ \ + float16x4x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 8); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_dup_s32(__p0) __extension__ ({ \ + int32x2x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 2); \ + __ret; \ +}) +#else +#define vld3_dup_s32(__p0) __extension__ ({ \ + int32x2x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 2); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_dup_s64(__p0) __extension__ ({ \ + int64x1x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 3); \ + __ret; \ +}) +#else +#define vld3_dup_s64(__p0) __extension__ ({ \ + int64x1x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_dup_s16(__p0) __extension__ ({ \ + int16x4x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 1); \ + __ret; \ +}) +#else +#define vld3_dup_s16(__p0) __extension__ ({ \ + int16x4x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 1); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8x3_t __s1 = __p1; \ + poly8x8x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 4); \ + __ret; \ +}) +#else +#define vld3_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8x3_t __s1 = __p1; \ + poly8x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], __p2, 4); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4x3_t __s1 = __p1; \ + poly16x4x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 5); \ + __ret; \ +}) +#else +#define vld3_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4x3_t __s1 = __p1; \ + poly16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + poly16x4x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], __p2, 5); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8x3_t __s1 = __p1; \ + poly16x8x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 37); \ + __ret; \ +}) +#else +#define vld3q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8x3_t __s1 = __p1; \ + poly16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x8x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 37); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4x3_t __s1 = __p1; \ + uint32x4x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 50); \ + __ret; \ +}) +#else +#define vld3q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4x3_t __s1 = __p1; \ + uint32x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + uint32x4x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 50); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8x3_t __s1 = __p1; \ + uint16x8x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 49); \ + __ret; \ +}) +#else +#define vld3q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8x3_t __s1 = __p1; \ + uint16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 49); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4x3_t __s1 = __p1; \ + float32x4x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 41); \ + __ret; \ +}) +#else +#define vld3q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4x3_t __s1 = __p1; \ + float32x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + float32x4x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 41); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8x3_t __s1 = __p1; \ + float16x8x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 40); \ + __ret; \ +}) +#else +#define vld3q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8x3_t __s1 = __p1; \ + float16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 40); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4x3_t __s1 = __p1; \ + int32x4x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 34); \ + __ret; \ +}) +#else +#define vld3q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4x3_t __s1 = __p1; \ + int32x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + int32x4x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 34); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8x3_t __s1 = __p1; \ + int16x8x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 33); \ + __ret; \ +}) +#else +#define vld3q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8x3_t __s1 = __p1; \ + int16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 33); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8x3_t __s1 = __p1; \ + uint8x8x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 16); \ + __ret; \ +}) +#else +#define vld3_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8x3_t __s1 = __p1; \ + uint8x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], __p2, 16); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2x3_t __s1 = __p1; \ + uint32x2x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 18); \ + __ret; \ +}) +#else +#define vld3_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2x3_t __s1 = __p1; \ + uint32x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + uint32x2x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], __p2, 18); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4x3_t __s1 = __p1; \ + uint16x4x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 17); \ + __ret; \ +}) +#else +#define vld3_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4x3_t __s1 = __p1; \ + uint16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + uint16x4x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], __p2, 17); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8x3_t __s1 = __p1; \ + int8x8x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 0); \ + __ret; \ +}) +#else +#define vld3_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8x3_t __s1 = __p1; \ + int8x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], __p2, 0); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2x3_t __s1 = __p1; \ + float32x2x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 9); \ + __ret; \ +}) +#else +#define vld3_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2x3_t __s1 = __p1; \ + float32x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + float32x2x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 9); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4x3_t __s1 = __p1; \ + float16x4x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 8); \ + __ret; \ +}) +#else +#define vld3_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4x3_t __s1 = __p1; \ + float16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + float16x4x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 8); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2x3_t __s1 = __p1; \ + int32x2x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 2); \ + __ret; \ +}) +#else +#define vld3_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2x3_t __s1 = __p1; \ + int32x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + int32x2x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 2); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4x3_t __s1 = __p1; \ + int16x4x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 1); \ + __ret; \ +}) +#else +#define vld3_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4x3_t __s1 = __p1; \ + int16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + int16x4x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 1); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_p8(__p0) __extension__ ({ \ + poly8x8x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 4); \ + __ret; \ +}) +#else +#define vld4_p8(__p0) __extension__ ({ \ + poly8x8x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 4); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_p16(__p0) __extension__ ({ \ + poly16x4x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 5); \ + __ret; \ +}) +#else +#define vld4_p16(__p0) __extension__ ({ \ + poly16x4x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 5); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_p8(__p0) __extension__ ({ \ + poly8x16x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 36); \ + __ret; \ +}) +#else +#define vld4q_p8(__p0) __extension__ ({ \ + poly8x16x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 36); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_p16(__p0) __extension__ ({ \ + poly16x8x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 37); \ + __ret; \ +}) +#else +#define vld4q_p16(__p0) __extension__ ({ \ + poly16x8x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 37); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_u8(__p0) __extension__ ({ \ + uint8x16x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 48); \ + __ret; \ +}) +#else +#define vld4q_u8(__p0) __extension__ ({ \ + uint8x16x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 48); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_u32(__p0) __extension__ ({ \ + uint32x4x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 50); \ + __ret; \ +}) +#else +#define vld4q_u32(__p0) __extension__ ({ \ + uint32x4x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 50); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_u16(__p0) __extension__ ({ \ + uint16x8x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 49); \ + __ret; \ +}) +#else +#define vld4q_u16(__p0) __extension__ ({ \ + uint16x8x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 49); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_s8(__p0) __extension__ ({ \ + int8x16x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 32); \ + __ret; \ +}) +#else +#define vld4q_s8(__p0) __extension__ ({ \ + int8x16x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 32); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_f32(__p0) __extension__ ({ \ + float32x4x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 41); \ + __ret; \ +}) +#else +#define vld4q_f32(__p0) __extension__ ({ \ + float32x4x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 41); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_f16(__p0) __extension__ ({ \ + float16x8x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 40); \ + __ret; \ +}) +#else +#define vld4q_f16(__p0) __extension__ ({ \ + float16x8x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 40); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_s32(__p0) __extension__ ({ \ + int32x4x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 34); \ + __ret; \ +}) +#else +#define vld4q_s32(__p0) __extension__ ({ \ + int32x4x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 34); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_s16(__p0) __extension__ ({ \ + int16x8x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 33); \ + __ret; \ +}) +#else +#define vld4q_s16(__p0) __extension__ ({ \ + int16x8x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 33); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_u8(__p0) __extension__ ({ \ + uint8x8x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 16); \ + __ret; \ +}) +#else +#define vld4_u8(__p0) __extension__ ({ \ + uint8x8x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 16); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_u32(__p0) __extension__ ({ \ + uint32x2x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 18); \ + __ret; \ +}) +#else +#define vld4_u32(__p0) __extension__ ({ \ + uint32x2x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 18); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_u64(__p0) __extension__ ({ \ + uint64x1x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 19); \ + __ret; \ +}) +#else +#define vld4_u64(__p0) __extension__ ({ \ + uint64x1x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_u16(__p0) __extension__ ({ \ + uint16x4x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 17); \ + __ret; \ +}) +#else +#define vld4_u16(__p0) __extension__ ({ \ + uint16x4x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 17); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_s8(__p0) __extension__ ({ \ + int8x8x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 0); \ + __ret; \ +}) +#else +#define vld4_s8(__p0) __extension__ ({ \ + int8x8x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 0); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_f32(__p0) __extension__ ({ \ + float32x2x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 9); \ + __ret; \ +}) +#else +#define vld4_f32(__p0) __extension__ ({ \ + float32x2x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 9); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_f16(__p0) __extension__ ({ \ + float16x4x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 8); \ + __ret; \ +}) +#else +#define vld4_f16(__p0) __extension__ ({ \ + float16x4x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 8); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_s32(__p0) __extension__ ({ \ + int32x2x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 2); \ + __ret; \ +}) +#else +#define vld4_s32(__p0) __extension__ ({ \ + int32x2x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 2); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_s64(__p0) __extension__ ({ \ + int64x1x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 3); \ + __ret; \ +}) +#else +#define vld4_s64(__p0) __extension__ ({ \ + int64x1x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_s16(__p0) __extension__ ({ \ + int16x4x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 1); \ + __ret; \ +}) +#else +#define vld4_s16(__p0) __extension__ ({ \ + int16x4x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 1); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_dup_p8(__p0) __extension__ ({ \ + poly8x8x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 4); \ + __ret; \ +}) +#else +#define vld4_dup_p8(__p0) __extension__ ({ \ + poly8x8x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 4); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_dup_p16(__p0) __extension__ ({ \ + poly16x4x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 5); \ + __ret; \ +}) +#else +#define vld4_dup_p16(__p0) __extension__ ({ \ + poly16x4x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 5); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_dup_u8(__p0) __extension__ ({ \ + uint8x8x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 16); \ + __ret; \ +}) +#else +#define vld4_dup_u8(__p0) __extension__ ({ \ + uint8x8x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 16); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_dup_u32(__p0) __extension__ ({ \ + uint32x2x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 18); \ + __ret; \ +}) +#else +#define vld4_dup_u32(__p0) __extension__ ({ \ + uint32x2x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 18); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_dup_u64(__p0) __extension__ ({ \ + uint64x1x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 19); \ + __ret; \ +}) +#else +#define vld4_dup_u64(__p0) __extension__ ({ \ + uint64x1x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_dup_u16(__p0) __extension__ ({ \ + uint16x4x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 17); \ + __ret; \ +}) +#else +#define vld4_dup_u16(__p0) __extension__ ({ \ + uint16x4x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 17); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_dup_s8(__p0) __extension__ ({ \ + int8x8x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 0); \ + __ret; \ +}) +#else +#define vld4_dup_s8(__p0) __extension__ ({ \ + int8x8x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 0); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_dup_f32(__p0) __extension__ ({ \ + float32x2x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 9); \ + __ret; \ +}) +#else +#define vld4_dup_f32(__p0) __extension__ ({ \ + float32x2x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 9); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_dup_f16(__p0) __extension__ ({ \ + float16x4x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 8); \ + __ret; \ +}) +#else +#define vld4_dup_f16(__p0) __extension__ ({ \ + float16x4x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 8); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_dup_s32(__p0) __extension__ ({ \ + int32x2x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 2); \ + __ret; \ +}) +#else +#define vld4_dup_s32(__p0) __extension__ ({ \ + int32x2x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 2); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_dup_s64(__p0) __extension__ ({ \ + int64x1x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 3); \ + __ret; \ +}) +#else +#define vld4_dup_s64(__p0) __extension__ ({ \ + int64x1x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_dup_s16(__p0) __extension__ ({ \ + int16x4x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 1); \ + __ret; \ +}) +#else +#define vld4_dup_s16(__p0) __extension__ ({ \ + int16x4x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 1); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8x4_t __s1 = __p1; \ + poly8x8x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 4); \ + __ret; \ +}) +#else +#define vld4_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8x4_t __s1 = __p1; \ + poly8x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], __p2, 4); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4x4_t __s1 = __p1; \ + poly16x4x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 5); \ + __ret; \ +}) +#else +#define vld4_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4x4_t __s1 = __p1; \ + poly16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + poly16x4x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], __p2, 5); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8x4_t __s1 = __p1; \ + poly16x8x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 37); \ + __ret; \ +}) +#else +#define vld4q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8x4_t __s1 = __p1; \ + poly16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x8x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 37); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4x4_t __s1 = __p1; \ + uint32x4x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 50); \ + __ret; \ +}) +#else +#define vld4q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4x4_t __s1 = __p1; \ + uint32x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + uint32x4x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 50); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8x4_t __s1 = __p1; \ + uint16x8x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 49); \ + __ret; \ +}) +#else +#define vld4q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8x4_t __s1 = __p1; \ + uint16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 49); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4x4_t __s1 = __p1; \ + float32x4x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 41); \ + __ret; \ +}) +#else +#define vld4q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4x4_t __s1 = __p1; \ + float32x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + float32x4x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 41); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8x4_t __s1 = __p1; \ + float16x8x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 40); \ + __ret; \ +}) +#else +#define vld4q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8x4_t __s1 = __p1; \ + float16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 40); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4x4_t __s1 = __p1; \ + int32x4x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 34); \ + __ret; \ +}) +#else +#define vld4q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4x4_t __s1 = __p1; \ + int32x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + int32x4x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 34); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8x4_t __s1 = __p1; \ + int16x8x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 33); \ + __ret; \ +}) +#else +#define vld4q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8x4_t __s1 = __p1; \ + int16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 33); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8x4_t __s1 = __p1; \ + uint8x8x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 16); \ + __ret; \ +}) +#else +#define vld4_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8x4_t __s1 = __p1; \ + uint8x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], __p2, 16); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2x4_t __s1 = __p1; \ + uint32x2x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 18); \ + __ret; \ +}) +#else +#define vld4_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2x4_t __s1 = __p1; \ + uint32x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + uint32x2x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], __p2, 18); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4x4_t __s1 = __p1; \ + uint16x4x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 17); \ + __ret; \ +}) +#else +#define vld4_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4x4_t __s1 = __p1; \ + uint16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + uint16x4x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], __p2, 17); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8x4_t __s1 = __p1; \ + int8x8x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 0); \ + __ret; \ +}) +#else +#define vld4_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8x4_t __s1 = __p1; \ + int8x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], __p2, 0); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2x4_t __s1 = __p1; \ + float32x2x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 9); \ + __ret; \ +}) +#else +#define vld4_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2x4_t __s1 = __p1; \ + float32x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + float32x2x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 9); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4x4_t __s1 = __p1; \ + float16x4x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 8); \ + __ret; \ +}) +#else +#define vld4_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4x4_t __s1 = __p1; \ + float16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + float16x4x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 8); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2x4_t __s1 = __p1; \ + int32x2x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 2); \ + __ret; \ +}) +#else +#define vld4_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2x4_t __s1 = __p1; \ + int32x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + int32x2x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 2); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4x4_t __s1 = __p1; \ + int16x4x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 1); \ + __ret; \ +}) +#else +#define vld4_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4x4_t __s1 = __p1; \ + int16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + int16x4x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 1); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vmaxq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vmaxq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmaxq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vmaxq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmaxq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vmaxq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vmaxq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vmaxq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vmaxq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4_t vmaxq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmaxq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vmaxq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmaxq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vmaxq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vmax_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vmax_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vmax_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vmax_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vmax_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vmax_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vmax_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vmax_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vmax_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vmax_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vmax_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vmax_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vmax_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vmax_v((int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2_t vmax_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vmax_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vmax_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vmax_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vmax_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vmax_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vmax_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vminq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vminq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vminq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vminq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vminq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vminq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vminq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vminq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vminq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vminq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vminq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vminq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vminq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vminq_v((int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4_t vminq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vminq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vminq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vminq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vminq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vminq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vminq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vmin_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vmin_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vmin_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vmin_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vmin_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vmin_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vmin_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vmin_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vmin_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vmin_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vmin_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vmin_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vmin_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vmin_v((int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2_t vmin_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vmin_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vmin_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vmin_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vmin_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vmin_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vmin_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vmlaq_u8(uint8x16_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint8x16_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai uint8x16_t vmlaq_u8(uint8x16_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmlaq_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai uint32x4_t vmlaq_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmlaq_u16(uint16x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint16x8_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai uint16x8_t vmlaq_u16(uint16x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vmlaq_s8(int8x16_t __p0, int8x16_t __p1, int8x16_t __p2) { + int8x16_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai int8x16_t vmlaq_s8(int8x16_t __p0, int8x16_t __p1, int8x16_t __p2) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vmlaq_f32(float32x4_t __p0, float32x4_t __p1, float32x4_t __p2) { + float32x4_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai float32x4_t vmlaq_f32(float32x4_t __p0, float32x4_t __p1, float32x4_t __p2) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmlaq_s32(int32x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int32x4_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai int32x4_t vmlaq_s32(int32x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmlaq_s16(int16x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int16x8_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai int16x8_t vmlaq_s16(int16x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vmla_u8(uint8x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint8x8_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai uint8x8_t vmla_u8(uint8x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vmla_u32(uint32x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint32x2_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai uint32x2_t vmla_u32(uint32x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + uint32x2_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vmla_u16(uint16x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint16x4_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai uint16x4_t vmla_u16(uint16x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vmla_s8(int8x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int8x8_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai int8x8_t vmla_s8(int8x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vmla_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2) { + float32x2_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai float32x2_t vmla_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + float32x2_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vmla_s32(int32x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int32x2_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai int32x2_t vmla_s32(int32x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int32x2_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vmla_s16(int16x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int16x4_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai int16x4_t vmla_s16(int16x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlaq_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint32x4_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlaq_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlaq_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint16x8_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlaq_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlaq_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x4_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlaq_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + float32x4_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlaq_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlaq_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int32x4_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlaq_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x8_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlaq_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmla_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint32x2_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3); \ + __ret; \ +}) +#else +#define vmla_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + uint32x2_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmla_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint16x4_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmla_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmla_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x2_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3); \ + __ret; \ +}) +#else +#define vmla_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + float32x2_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmla_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x2_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3); \ + __ret; \ +}) +#else +#define vmla_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int32x2_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmla_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x4_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmla_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmlaq_n_u32(uint32x4_t __p0, uint32x4_t __p1, uint32_t __p2) { + uint32x4_t __ret; + __ret = __p0 + __p1 * (uint32x4_t) {__p2, __p2, __p2, __p2}; + return __ret; +} +#else +__ai uint32x4_t vmlaq_n_u32(uint32x4_t __p0, uint32x4_t __p1, uint32_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 + __rev1 * (uint32x4_t) {__p2, __p2, __p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmlaq_n_u16(uint16x8_t __p0, uint16x8_t __p1, uint16_t __p2) { + uint16x8_t __ret; + __ret = __p0 + __p1 * (uint16x8_t) {__p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2}; + return __ret; +} +#else +__ai uint16x8_t vmlaq_n_u16(uint16x8_t __p0, uint16x8_t __p1, uint16_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 + __rev1 * (uint16x8_t) {__p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vmlaq_n_f32(float32x4_t __p0, float32x4_t __p1, float32_t __p2) { + float32x4_t __ret; + __ret = __p0 + __p1 * (float32x4_t) {__p2, __p2, __p2, __p2}; + return __ret; +} +#else +__ai float32x4_t vmlaq_n_f32(float32x4_t __p0, float32x4_t __p1, float32_t __p2) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __rev0 + __rev1 * (float32x4_t) {__p2, __p2, __p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmlaq_n_s32(int32x4_t __p0, int32x4_t __p1, int32_t __p2) { + int32x4_t __ret; + __ret = __p0 + __p1 * (int32x4_t) {__p2, __p2, __p2, __p2}; + return __ret; +} +#else +__ai int32x4_t vmlaq_n_s32(int32x4_t __p0, int32x4_t __p1, int32_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 + __rev1 * (int32x4_t) {__p2, __p2, __p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmlaq_n_s16(int16x8_t __p0, int16x8_t __p1, int16_t __p2) { + int16x8_t __ret; + __ret = __p0 + __p1 * (int16x8_t) {__p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2}; + return __ret; +} +#else +__ai int16x8_t vmlaq_n_s16(int16x8_t __p0, int16x8_t __p1, int16_t __p2) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 + __rev1 * (int16x8_t) {__p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vmla_n_u32(uint32x2_t __p0, uint32x2_t __p1, uint32_t __p2) { + uint32x2_t __ret; + __ret = __p0 + __p1 * (uint32x2_t) {__p2, __p2}; + return __ret; +} +#else +__ai uint32x2_t vmla_n_u32(uint32x2_t __p0, uint32x2_t __p1, uint32_t __p2) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __rev0 + __rev1 * (uint32x2_t) {__p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vmla_n_u16(uint16x4_t __p0, uint16x4_t __p1, uint16_t __p2) { + uint16x4_t __ret; + __ret = __p0 + __p1 * (uint16x4_t) {__p2, __p2, __p2, __p2}; + return __ret; +} +#else +__ai uint16x4_t vmla_n_u16(uint16x4_t __p0, uint16x4_t __p1, uint16_t __p2) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __rev0 + __rev1 * (uint16x4_t) {__p2, __p2, __p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vmla_n_f32(float32x2_t __p0, float32x2_t __p1, float32_t __p2) { + float32x2_t __ret; + __ret = __p0 + __p1 * (float32x2_t) {__p2, __p2}; + return __ret; +} +#else +__ai float32x2_t vmla_n_f32(float32x2_t __p0, float32x2_t __p1, float32_t __p2) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = __rev0 + __rev1 * (float32x2_t) {__p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vmla_n_s32(int32x2_t __p0, int32x2_t __p1, int32_t __p2) { + int32x2_t __ret; + __ret = __p0 + __p1 * (int32x2_t) {__p2, __p2}; + return __ret; +} +#else +__ai int32x2_t vmla_n_s32(int32x2_t __p0, int32x2_t __p1, int32_t __p2) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __rev0 + __rev1 * (int32x2_t) {__p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vmla_n_s16(int16x4_t __p0, int16x4_t __p1, int16_t __p2) { + int16x4_t __ret; + __ret = __p0 + __p1 * (int16x4_t) {__p2, __p2, __p2, __p2}; + return __ret; +} +#else +__ai int16x4_t vmla_n_s16(int16x4_t __p0, int16x4_t __p1, int16_t __p2) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __rev0 + __rev1 * (int16x4_t) {__p2, __p2, __p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vmlsq_u8(uint8x16_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint8x16_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai uint8x16_t vmlsq_u8(uint8x16_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmlsq_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai uint32x4_t vmlsq_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmlsq_u16(uint16x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint16x8_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai uint16x8_t vmlsq_u16(uint16x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vmlsq_s8(int8x16_t __p0, int8x16_t __p1, int8x16_t __p2) { + int8x16_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai int8x16_t vmlsq_s8(int8x16_t __p0, int8x16_t __p1, int8x16_t __p2) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vmlsq_f32(float32x4_t __p0, float32x4_t __p1, float32x4_t __p2) { + float32x4_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai float32x4_t vmlsq_f32(float32x4_t __p0, float32x4_t __p1, float32x4_t __p2) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmlsq_s32(int32x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int32x4_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai int32x4_t vmlsq_s32(int32x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmlsq_s16(int16x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int16x8_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai int16x8_t vmlsq_s16(int16x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vmls_u8(uint8x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint8x8_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai uint8x8_t vmls_u8(uint8x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vmls_u32(uint32x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint32x2_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai uint32x2_t vmls_u32(uint32x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + uint32x2_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vmls_u16(uint16x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint16x4_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai uint16x4_t vmls_u16(uint16x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vmls_s8(int8x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int8x8_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai int8x8_t vmls_s8(int8x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vmls_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2) { + float32x2_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai float32x2_t vmls_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + float32x2_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vmls_s32(int32x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int32x2_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai int32x2_t vmls_s32(int32x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int32x2_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vmls_s16(int16x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int16x4_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai int16x4_t vmls_s16(int16x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsq_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint32x4_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlsq_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsq_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint16x8_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlsq_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsq_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x4_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlsq_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + float32x4_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsq_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlsq_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int32x4_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsq_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x8_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlsq_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmls_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint32x2_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3); \ + __ret; \ +}) +#else +#define vmls_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + uint32x2_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmls_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint16x4_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmls_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmls_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x2_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3); \ + __ret; \ +}) +#else +#define vmls_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + float32x2_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmls_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x2_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3); \ + __ret; \ +}) +#else +#define vmls_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int32x2_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmls_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x4_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmls_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmlsq_n_u32(uint32x4_t __p0, uint32x4_t __p1, uint32_t __p2) { + uint32x4_t __ret; + __ret = __p0 - __p1 * (uint32x4_t) {__p2, __p2, __p2, __p2}; + return __ret; +} +#else +__ai uint32x4_t vmlsq_n_u32(uint32x4_t __p0, uint32x4_t __p1, uint32_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 - __rev1 * (uint32x4_t) {__p2, __p2, __p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmlsq_n_u16(uint16x8_t __p0, uint16x8_t __p1, uint16_t __p2) { + uint16x8_t __ret; + __ret = __p0 - __p1 * (uint16x8_t) {__p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2}; + return __ret; +} +#else +__ai uint16x8_t vmlsq_n_u16(uint16x8_t __p0, uint16x8_t __p1, uint16_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 - __rev1 * (uint16x8_t) {__p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vmlsq_n_f32(float32x4_t __p0, float32x4_t __p1, float32_t __p2) { + float32x4_t __ret; + __ret = __p0 - __p1 * (float32x4_t) {__p2, __p2, __p2, __p2}; + return __ret; +} +#else +__ai float32x4_t vmlsq_n_f32(float32x4_t __p0, float32x4_t __p1, float32_t __p2) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __rev0 - __rev1 * (float32x4_t) {__p2, __p2, __p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmlsq_n_s32(int32x4_t __p0, int32x4_t __p1, int32_t __p2) { + int32x4_t __ret; + __ret = __p0 - __p1 * (int32x4_t) {__p2, __p2, __p2, __p2}; + return __ret; +} +#else +__ai int32x4_t vmlsq_n_s32(int32x4_t __p0, int32x4_t __p1, int32_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 - __rev1 * (int32x4_t) {__p2, __p2, __p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmlsq_n_s16(int16x8_t __p0, int16x8_t __p1, int16_t __p2) { + int16x8_t __ret; + __ret = __p0 - __p1 * (int16x8_t) {__p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2}; + return __ret; +} +#else +__ai int16x8_t vmlsq_n_s16(int16x8_t __p0, int16x8_t __p1, int16_t __p2) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 - __rev1 * (int16x8_t) {__p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vmls_n_u32(uint32x2_t __p0, uint32x2_t __p1, uint32_t __p2) { + uint32x2_t __ret; + __ret = __p0 - __p1 * (uint32x2_t) {__p2, __p2}; + return __ret; +} +#else +__ai uint32x2_t vmls_n_u32(uint32x2_t __p0, uint32x2_t __p1, uint32_t __p2) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __rev0 - __rev1 * (uint32x2_t) {__p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vmls_n_u16(uint16x4_t __p0, uint16x4_t __p1, uint16_t __p2) { + uint16x4_t __ret; + __ret = __p0 - __p1 * (uint16x4_t) {__p2, __p2, __p2, __p2}; + return __ret; +} +#else +__ai uint16x4_t vmls_n_u16(uint16x4_t __p0, uint16x4_t __p1, uint16_t __p2) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __rev0 - __rev1 * (uint16x4_t) {__p2, __p2, __p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vmls_n_f32(float32x2_t __p0, float32x2_t __p1, float32_t __p2) { + float32x2_t __ret; + __ret = __p0 - __p1 * (float32x2_t) {__p2, __p2}; + return __ret; +} +#else +__ai float32x2_t vmls_n_f32(float32x2_t __p0, float32x2_t __p1, float32_t __p2) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = __rev0 - __rev1 * (float32x2_t) {__p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vmls_n_s32(int32x2_t __p0, int32x2_t __p1, int32_t __p2) { + int32x2_t __ret; + __ret = __p0 - __p1 * (int32x2_t) {__p2, __p2}; + return __ret; +} +#else +__ai int32x2_t vmls_n_s32(int32x2_t __p0, int32x2_t __p1, int32_t __p2) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __rev0 - __rev1 * (int32x2_t) {__p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vmls_n_s16(int16x4_t __p0, int16x4_t __p1, int16_t __p2) { + int16x4_t __ret; + __ret = __p0 - __p1 * (int16x4_t) {__p2, __p2, __p2, __p2}; + return __ret; +} +#else +__ai int16x4_t vmls_n_s16(int16x4_t __p0, int16x4_t __p1, int16_t __p2) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __rev0 - __rev1 * (int16x4_t) {__p2, __p2, __p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vmov_n_p8(poly8_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai poly8x8_t vmov_n_p8(poly8_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vmov_n_p16(poly16_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t) {__p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai poly16x4_t vmov_n_p16(poly16_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t) {__p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vmovq_n_p8(poly8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai poly8x16_t vmovq_n_p8(poly8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vmovq_n_p16(poly16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai poly16x8_t vmovq_n_p16(poly16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vmovq_n_u8(uint8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai uint8x16_t vmovq_n_u8(uint8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmovq_n_u32(uint32_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) {__p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai uint32x4_t vmovq_n_u32(uint32_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) {__p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmovq_n_u64(uint64_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) {__p0, __p0}; + return __ret; +} +#else +__ai uint64x2_t vmovq_n_u64(uint64_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) {__p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmovq_n_u16(uint16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai uint16x8_t vmovq_n_u16(uint16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vmovq_n_s8(int8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai int8x16_t vmovq_n_s8(int8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vmovq_n_f32(float32_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) {__p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai float32x4_t vmovq_n_f32(float32_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) {__p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmovq_n_f16(__p0) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16x8_t __ret; \ + __ret = (float16x8_t) {__s0, __s0, __s0, __s0, __s0, __s0, __s0, __s0}; \ + __ret; \ +}) +#else +#define vmovq_n_f16(__p0) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16x8_t __ret; \ + __ret = (float16x8_t) {__s0, __s0, __s0, __s0, __s0, __s0, __s0, __s0}; \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmovq_n_s32(int32_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) {__p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai int32x4_t vmovq_n_s32(int32_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) {__p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmovq_n_s64(int64_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) {__p0, __p0}; + return __ret; +} +#else +__ai int64x2_t vmovq_n_s64(int64_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) {__p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmovq_n_s16(int16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai int16x8_t vmovq_n_s16(int16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vmov_n_u8(uint8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai uint8x8_t vmov_n_u8(uint8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vmov_n_u32(uint32_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) {__p0, __p0}; + return __ret; +} +#else +__ai uint32x2_t vmov_n_u32(uint32_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) {__p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vmov_n_u64(uint64_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) {__p0}; + return __ret; +} +#else +__ai uint64x1_t vmov_n_u64(uint64_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) {__p0}; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vmov_n_u16(uint16_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) {__p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai uint16x4_t vmov_n_u16(uint16_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) {__p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vmov_n_s8(int8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai int8x8_t vmov_n_s8(int8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) {__p0, __p0, __p0, __p0, __p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vmov_n_f32(float32_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) {__p0, __p0}; + return __ret; +} +#else +__ai float32x2_t vmov_n_f32(float32_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) {__p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmov_n_f16(__p0) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16x4_t __ret; \ + __ret = (float16x4_t) {__s0, __s0, __s0, __s0}; \ + __ret; \ +}) +#else +#define vmov_n_f16(__p0) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16x4_t __ret; \ + __ret = (float16x4_t) {__s0, __s0, __s0, __s0}; \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vmov_n_s32(int32_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) {__p0, __p0}; + return __ret; +} +#else +__ai int32x2_t vmov_n_s32(int32_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) {__p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vmov_n_s64(int64_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) {__p0}; + return __ret; +} +#else +__ai int64x1_t vmov_n_s64(int64_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) {__p0}; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vmov_n_s16(int16_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) {__p0, __p0, __p0, __p0}; + return __ret; +} +#else +__ai int16x4_t vmov_n_s16(int16_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) {__p0, __p0, __p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmovl_u8(uint8x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vmovl_v((int8x8_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vmovl_u8(uint8x8_t __p0) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vmovl_v((int8x8_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint16x8_t __noswap_vmovl_u8(uint8x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vmovl_v((int8x8_t)__p0, 49); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmovl_u32(uint32x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vmovl_v((int8x8_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vmovl_u32(uint32x2_t __p0) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vmovl_v((int8x8_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint64x2_t __noswap_vmovl_u32(uint32x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vmovl_v((int8x8_t)__p0, 51); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmovl_u16(uint16x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vmovl_v((int8x8_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vmovl_u16(uint16x4_t __p0) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vmovl_v((int8x8_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint32x4_t __noswap_vmovl_u16(uint16x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vmovl_v((int8x8_t)__p0, 50); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmovl_s8(int8x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vmovl_v((int8x8_t)__p0, 33); + return __ret; +} +#else +__ai int16x8_t vmovl_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vmovl_v((int8x8_t)__rev0, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int16x8_t __noswap_vmovl_s8(int8x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vmovl_v((int8x8_t)__p0, 33); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmovl_s32(int32x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vmovl_v((int8x8_t)__p0, 35); + return __ret; +} +#else +__ai int64x2_t vmovl_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vmovl_v((int8x8_t)__rev0, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vmovl_s32(int32x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vmovl_v((int8x8_t)__p0, 35); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmovl_s16(int16x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vmovl_v((int8x8_t)__p0, 34); + return __ret; +} +#else +__ai int32x4_t vmovl_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vmovl_v((int8x8_t)__rev0, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vmovl_s16(int16x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vmovl_v((int8x8_t)__p0, 34); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vmovn_u32(uint32x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vmovn_v((int8x16_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vmovn_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vmovn_v((int8x16_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint16x4_t __noswap_vmovn_u32(uint32x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vmovn_v((int8x16_t)__p0, 17); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vmovn_u64(uint64x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vmovn_v((int8x16_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vmovn_u64(uint64x2_t __p0) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vmovn_v((int8x16_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint32x2_t __noswap_vmovn_u64(uint64x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vmovn_v((int8x16_t)__p0, 18); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vmovn_u16(uint16x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vmovn_v((int8x16_t)__p0, 16); + return __ret; +} +#else +__ai uint8x8_t vmovn_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vmovn_v((int8x16_t)__rev0, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint8x8_t __noswap_vmovn_u16(uint16x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vmovn_v((int8x16_t)__p0, 16); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vmovn_s32(int32x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vmovn_v((int8x16_t)__p0, 1); + return __ret; +} +#else +__ai int16x4_t vmovn_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vmovn_v((int8x16_t)__rev0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int16x4_t __noswap_vmovn_s32(int32x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vmovn_v((int8x16_t)__p0, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vmovn_s64(int64x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vmovn_v((int8x16_t)__p0, 2); + return __ret; +} +#else +__ai int32x2_t vmovn_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vmovn_v((int8x16_t)__rev0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int32x2_t __noswap_vmovn_s64(int64x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vmovn_v((int8x16_t)__p0, 2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vmovn_s16(int16x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vmovn_v((int8x16_t)__p0, 0); + return __ret; +} +#else +__ai int8x8_t vmovn_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vmovn_v((int8x16_t)__rev0, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int8x8_t __noswap_vmovn_s16(int16x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vmovn_v((int8x16_t)__p0, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vmulq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai uint8x16_t vmulq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmulq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai uint32x4_t vmulq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmulq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai uint16x8_t vmulq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vmulq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai int8x16_t vmulq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vmulq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai float32x4_t vmulq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmulq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai int32x4_t vmulq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmulq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai int16x8_t vmulq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vmul_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai uint8x8_t vmul_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vmul_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai uint32x2_t vmul_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vmul_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai uint16x4_t vmul_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vmul_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai int8x8_t vmul_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vmul_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai float32x2_t vmul_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vmul_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai int32x2_t vmul_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vmul_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai int16x4_t vmul_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vmul_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vmul_v((int8x8_t)__p0, (int8x8_t)__p1, 4); + return __ret; +} +#else +__ai poly8x8_t vmul_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vmul_v((int8x8_t)__rev0, (int8x8_t)__rev1, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vmulq_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vmulq_v((int8x16_t)__p0, (int8x16_t)__p1, 36); + return __ret; +} +#else +__ai poly8x16_t vmulq_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vmulq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 36); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulq_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x4_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmulq_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x4_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulq_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x8_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmulq_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulq_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x4_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmulq_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float32x4_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulq_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmulq_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x4_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulq_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmulq_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmul_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2); \ + __ret; \ +}) +#else +#define vmul_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x2_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmul_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmul_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmul_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2); \ + __ret; \ +}) +#else +#define vmul_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float32x2_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmul_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2); \ + __ret; \ +}) +#else +#define vmul_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmul_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmul_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmulq_n_u32(uint32x4_t __p0, uint32_t __p1) { + uint32x4_t __ret; + __ret = __p0 * (uint32x4_t) {__p1, __p1, __p1, __p1}; + return __ret; +} +#else +__ai uint32x4_t vmulq_n_u32(uint32x4_t __p0, uint32_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 * (uint32x4_t) {__p1, __p1, __p1, __p1}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmulq_n_u16(uint16x8_t __p0, uint16_t __p1) { + uint16x8_t __ret; + __ret = __p0 * (uint16x8_t) {__p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1}; + return __ret; +} +#else +__ai uint16x8_t vmulq_n_u16(uint16x8_t __p0, uint16_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 * (uint16x8_t) {__p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vmulq_n_f32(float32x4_t __p0, float32_t __p1) { + float32x4_t __ret; + __ret = __p0 * (float32x4_t) {__p1, __p1, __p1, __p1}; + return __ret; +} +#else +__ai float32x4_t vmulq_n_f32(float32x4_t __p0, float32_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __rev0 * (float32x4_t) {__p1, __p1, __p1, __p1}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmulq_n_s32(int32x4_t __p0, int32_t __p1) { + int32x4_t __ret; + __ret = __p0 * (int32x4_t) {__p1, __p1, __p1, __p1}; + return __ret; +} +#else +__ai int32x4_t vmulq_n_s32(int32x4_t __p0, int32_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 * (int32x4_t) {__p1, __p1, __p1, __p1}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmulq_n_s16(int16x8_t __p0, int16_t __p1) { + int16x8_t __ret; + __ret = __p0 * (int16x8_t) {__p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1}; + return __ret; +} +#else +__ai int16x8_t vmulq_n_s16(int16x8_t __p0, int16_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 * (int16x8_t) {__p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vmul_n_u32(uint32x2_t __p0, uint32_t __p1) { + uint32x2_t __ret; + __ret = __p0 * (uint32x2_t) {__p1, __p1}; + return __ret; +} +#else +__ai uint32x2_t vmul_n_u32(uint32x2_t __p0, uint32_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = __rev0 * (uint32x2_t) {__p1, __p1}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vmul_n_u16(uint16x4_t __p0, uint16_t __p1) { + uint16x4_t __ret; + __ret = __p0 * (uint16x4_t) {__p1, __p1, __p1, __p1}; + return __ret; +} +#else +__ai uint16x4_t vmul_n_u16(uint16x4_t __p0, uint16_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __rev0 * (uint16x4_t) {__p1, __p1, __p1, __p1}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vmul_n_f32(float32x2_t __p0, float32_t __p1) { + float32x2_t __ret; + __ret = __p0 * (float32x2_t) {__p1, __p1}; + return __ret; +} +#else +__ai float32x2_t vmul_n_f32(float32x2_t __p0, float32_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = __rev0 * (float32x2_t) {__p1, __p1}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vmul_n_s32(int32x2_t __p0, int32_t __p1) { + int32x2_t __ret; + __ret = __p0 * (int32x2_t) {__p1, __p1}; + return __ret; +} +#else +__ai int32x2_t vmul_n_s32(int32x2_t __p0, int32_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = __rev0 * (int32x2_t) {__p1, __p1}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vmul_n_s16(int16x4_t __p0, int16_t __p1) { + int16x4_t __ret; + __ret = __p0 * (int16x4_t) {__p1, __p1, __p1, __p1}; + return __ret; +} +#else +__ai int16x4_t vmul_n_s16(int16x4_t __p0, int16_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __rev0 * (int16x4_t) {__p1, __p1, __p1, __p1}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vmull_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly16x8_t __ret; + __ret = (poly16x8_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)__p1, 37); + return __ret; +} +#else +__ai poly16x8_t vmull_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __ret; + __ret = (poly16x8_t) __builtin_neon_vmull_v((int8x8_t)__rev0, (int8x8_t)__rev1, 37); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai poly16x8_t __noswap_vmull_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly16x8_t __ret; + __ret = (poly16x8_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)__p1, 37); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmull_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vmull_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vmull_v((int8x8_t)__rev0, (int8x8_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint16x8_t __noswap_vmull_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)__p1, 49); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmull_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vmull_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vmull_v((int8x8_t)__rev0, (int8x8_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint64x2_t __noswap_vmull_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)__p1, 51); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmull_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vmull_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vmull_v((int8x8_t)__rev0, (int8x8_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint32x4_t __noswap_vmull_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)__p1, 50); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmull_s8(int8x8_t __p0, int8x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vmull_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vmull_v((int8x8_t)__rev0, (int8x8_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int16x8_t __noswap_vmull_s8(int8x8_t __p0, int8x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)__p1, 33); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmull_s32(int32x2_t __p0, int32x2_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)__p1, 35); + return __ret; +} +#else +__ai int64x2_t vmull_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vmull_v((int8x8_t)__rev0, (int8x8_t)__rev1, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vmull_s32(int32x2_t __p0, int32x2_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)__p1, 35); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmull_s16(int16x4_t __p0, int16x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vmull_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vmull_v((int8x8_t)__rev0, (int8x8_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vmull_s16(int16x4_t __p0, int16x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)__p1, 34); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint64x2_t __ret; \ + __ret = vmull_u32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint64x2_t __ret; \ + __ret = __noswap_vmull_u32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint32x4_t __ret; \ + __ret = vmull_u16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __noswap_vmull_u16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = vmull_s32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vmull_s32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = vmull_s16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vmull_s16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmull_n_u32(uint32x2_t __p0, uint32_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)(uint32x2_t) {__p1, __p1}, 51); + return __ret; +} +#else +__ai uint64x2_t vmull_n_u32(uint32x2_t __p0, uint32_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vmull_v((int8x8_t)__rev0, (int8x8_t)(uint32x2_t) {__p1, __p1}, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint64x2_t __noswap_vmull_n_u32(uint32x2_t __p0, uint32_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)(uint32x2_t) {__p1, __p1}, 51); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmull_n_u16(uint16x4_t __p0, uint16_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)(uint16x4_t) {__p1, __p1, __p1, __p1}, 50); + return __ret; +} +#else +__ai uint32x4_t vmull_n_u16(uint16x4_t __p0, uint16_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vmull_v((int8x8_t)__rev0, (int8x8_t)(uint16x4_t) {__p1, __p1, __p1, __p1}, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint32x4_t __noswap_vmull_n_u16(uint16x4_t __p0, uint16_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)(uint16x4_t) {__p1, __p1, __p1, __p1}, 50); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmull_n_s32(int32x2_t __p0, int32_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)(int32x2_t) {__p1, __p1}, 35); + return __ret; +} +#else +__ai int64x2_t vmull_n_s32(int32x2_t __p0, int32_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vmull_v((int8x8_t)__rev0, (int8x8_t)(int32x2_t) {__p1, __p1}, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vmull_n_s32(int32x2_t __p0, int32_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)(int32x2_t) {__p1, __p1}, 35); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmull_n_s16(int16x4_t __p0, int16_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)(int16x4_t) {__p1, __p1, __p1, __p1}, 34); + return __ret; +} +#else +__ai int32x4_t vmull_n_s16(int16x4_t __p0, int16_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vmull_v((int8x8_t)__rev0, (int8x8_t)(int16x4_t) {__p1, __p1, __p1, __p1}, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vmull_n_s16(int16x4_t __p0, int16_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vmull_v((int8x8_t)__p0, (int8x8_t)(int16x4_t) {__p1, __p1, __p1, __p1}, 34); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vmvn_p8(poly8x8_t __p0) { + poly8x8_t __ret; + __ret = ~__p0; + return __ret; +} +#else +__ai poly8x8_t vmvn_p8(poly8x8_t __p0) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = ~__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vmvnq_p8(poly8x16_t __p0) { + poly8x16_t __ret; + __ret = ~__p0; + return __ret; +} +#else +__ai poly8x16_t vmvnq_p8(poly8x16_t __p0) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = ~__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vmvnq_u8(uint8x16_t __p0) { + uint8x16_t __ret; + __ret = ~__p0; + return __ret; +} +#else +__ai uint8x16_t vmvnq_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = ~__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmvnq_u32(uint32x4_t __p0) { + uint32x4_t __ret; + __ret = ~__p0; + return __ret; +} +#else +__ai uint32x4_t vmvnq_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = ~__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmvnq_u16(uint16x8_t __p0) { + uint16x8_t __ret; + __ret = ~__p0; + return __ret; +} +#else +__ai uint16x8_t vmvnq_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = ~__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vmvnq_s8(int8x16_t __p0) { + int8x16_t __ret; + __ret = ~__p0; + return __ret; +} +#else +__ai int8x16_t vmvnq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = ~__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmvnq_s32(int32x4_t __p0) { + int32x4_t __ret; + __ret = ~__p0; + return __ret; +} +#else +__ai int32x4_t vmvnq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = ~__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmvnq_s16(int16x8_t __p0) { + int16x8_t __ret; + __ret = ~__p0; + return __ret; +} +#else +__ai int16x8_t vmvnq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = ~__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vmvn_u8(uint8x8_t __p0) { + uint8x8_t __ret; + __ret = ~__p0; + return __ret; +} +#else +__ai uint8x8_t vmvn_u8(uint8x8_t __p0) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = ~__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vmvn_u32(uint32x2_t __p0) { + uint32x2_t __ret; + __ret = ~__p0; + return __ret; +} +#else +__ai uint32x2_t vmvn_u32(uint32x2_t __p0) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = ~__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vmvn_u16(uint16x4_t __p0) { + uint16x4_t __ret; + __ret = ~__p0; + return __ret; +} +#else +__ai uint16x4_t vmvn_u16(uint16x4_t __p0) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = ~__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vmvn_s8(int8x8_t __p0) { + int8x8_t __ret; + __ret = ~__p0; + return __ret; +} +#else +__ai int8x8_t vmvn_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = ~__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vmvn_s32(int32x2_t __p0) { + int32x2_t __ret; + __ret = ~__p0; + return __ret; +} +#else +__ai int32x2_t vmvn_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = ~__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vmvn_s16(int16x4_t __p0) { + int16x4_t __ret; + __ret = ~__p0; + return __ret; +} +#else +__ai int16x4_t vmvn_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = ~__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vnegq_s8(int8x16_t __p0) { + int8x16_t __ret; + __ret = -__p0; + return __ret; +} +#else +__ai int8x16_t vnegq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = -__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vnegq_f32(float32x4_t __p0) { + float32x4_t __ret; + __ret = -__p0; + return __ret; +} +#else +__ai float32x4_t vnegq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = -__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vnegq_s32(int32x4_t __p0) { + int32x4_t __ret; + __ret = -__p0; + return __ret; +} +#else +__ai int32x4_t vnegq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = -__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vnegq_s16(int16x8_t __p0) { + int16x8_t __ret; + __ret = -__p0; + return __ret; +} +#else +__ai int16x8_t vnegq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = -__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vneg_s8(int8x8_t __p0) { + int8x8_t __ret; + __ret = -__p0; + return __ret; +} +#else +__ai int8x8_t vneg_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = -__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vneg_f32(float32x2_t __p0) { + float32x2_t __ret; + __ret = -__p0; + return __ret; +} +#else +__ai float32x2_t vneg_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = -__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vneg_s32(int32x2_t __p0) { + int32x2_t __ret; + __ret = -__p0; + return __ret; +} +#else +__ai int32x2_t vneg_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = -__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vneg_s16(int16x4_t __p0) { + int16x4_t __ret; + __ret = -__p0; + return __ret; +} +#else +__ai int16x4_t vneg_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = -__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vornq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai uint8x16_t vornq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __rev0 | ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vornq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai uint32x4_t vornq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 | ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vornq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai uint64x2_t vornq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __rev0 | ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vornq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai uint16x8_t vornq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 | ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vornq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai int8x16_t vornq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __rev0 | ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vornq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai int32x4_t vornq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 | ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vornq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai int64x2_t vornq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __rev0 | ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vornq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai int16x8_t vornq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 | ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vorn_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai uint8x8_t vorn_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __rev0 | ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vorn_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai uint32x2_t vorn_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __rev0 | ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vorn_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai uint64x1_t vorn_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vorn_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai uint16x4_t vorn_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __rev0 | ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vorn_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai int8x8_t vorn_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __rev0 | ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vorn_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai int32x2_t vorn_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __rev0 | ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vorn_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai int64x1_t vorn_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vorn_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = __p0 | ~__p1; + return __ret; +} +#else +__ai int16x4_t vorn_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __rev0 | ~__rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vorrq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai uint8x16_t vorrq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __rev0 | __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vorrq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai uint32x4_t vorrq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 | __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vorrq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai uint64x2_t vorrq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __rev0 | __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vorrq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai uint16x8_t vorrq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 | __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vorrq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai int8x16_t vorrq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __rev0 | __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vorrq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai int32x4_t vorrq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 | __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vorrq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai int64x2_t vorrq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __rev0 | __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vorrq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai int16x8_t vorrq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 | __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vorr_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai uint8x8_t vorr_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __rev0 | __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vorr_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai uint32x2_t vorr_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __rev0 | __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vorr_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai uint64x1_t vorr_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vorr_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai uint16x4_t vorr_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __rev0 | __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vorr_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai int8x8_t vorr_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __rev0 | __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vorr_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai int32x2_t vorr_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __rev0 | __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vorr_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai int64x1_t vorr_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vorr_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = __p0 | __p1; + return __ret; +} +#else +__ai int16x4_t vorr_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __rev0 | __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vpadalq_u8(uint16x8_t __p0, uint8x16_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vpadalq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vpadalq_u8(uint16x8_t __p0, uint8x16_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vpadalq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vpadalq_u32(uint64x2_t __p0, uint32x4_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vpadalq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vpadalq_u32(uint64x2_t __p0, uint32x4_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vpadalq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vpadalq_u16(uint32x4_t __p0, uint16x8_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vpadalq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vpadalq_u16(uint32x4_t __p0, uint16x8_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vpadalq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vpadalq_s8(int16x8_t __p0, int8x16_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vpadalq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vpadalq_s8(int16x8_t __p0, int8x16_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vpadalq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vpadalq_s32(int64x2_t __p0, int32x4_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vpadalq_v((int8x16_t)__p0, (int8x16_t)__p1, 35); + return __ret; +} +#else +__ai int64x2_t vpadalq_s32(int64x2_t __p0, int32x4_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vpadalq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vpadalq_s16(int32x4_t __p0, int16x8_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vpadalq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vpadalq_s16(int32x4_t __p0, int16x8_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vpadalq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vpadal_u8(uint16x4_t __p0, uint8x8_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vpadal_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vpadal_u8(uint16x4_t __p0, uint8x8_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vpadal_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vpadal_u32(uint64x1_t __p0, uint32x2_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vpadal_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vpadal_u32(uint64x1_t __p0, uint32x2_t __p1) { + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vpadal_v((int8x8_t)__p0, (int8x8_t)__rev1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vpadal_u16(uint32x2_t __p0, uint16x4_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vpadal_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vpadal_u16(uint32x2_t __p0, uint16x4_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vpadal_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vpadal_s8(int16x4_t __p0, int8x8_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vpadal_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vpadal_s8(int16x4_t __p0, int8x8_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vpadal_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vpadal_s32(int64x1_t __p0, int32x2_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vpadal_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#else +__ai int64x1_t vpadal_s32(int64x1_t __p0, int32x2_t __p1) { + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vpadal_v((int8x8_t)__p0, (int8x8_t)__rev1, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vpadal_s16(int32x2_t __p0, int16x4_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vpadal_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vpadal_s16(int32x2_t __p0, int16x4_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vpadal_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vpadd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vpadd_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vpadd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vpadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vpadd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vpadd_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vpadd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vpadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vpadd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vpadd_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vpadd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vpadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vpadd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vpadd_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vpadd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vpadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vpadd_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vpadd_v((int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2_t vpadd_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vpadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vpadd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vpadd_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vpadd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vpadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vpadd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vpadd_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vpadd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vpadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vpaddlq_u8(uint8x16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vpaddlq_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vpaddlq_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vpaddlq_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vpaddlq_u32(uint32x4_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vpaddlq_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vpaddlq_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vpaddlq_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vpaddlq_u16(uint16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vpaddlq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vpaddlq_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vpaddlq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vpaddlq_s8(int8x16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vpaddlq_v((int8x16_t)__p0, 33); + return __ret; +} +#else +__ai int16x8_t vpaddlq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vpaddlq_v((int8x16_t)__rev0, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vpaddlq_s32(int32x4_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vpaddlq_v((int8x16_t)__p0, 35); + return __ret; +} +#else +__ai int64x2_t vpaddlq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vpaddlq_v((int8x16_t)__rev0, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vpaddlq_s16(int16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vpaddlq_v((int8x16_t)__p0, 34); + return __ret; +} +#else +__ai int32x4_t vpaddlq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vpaddlq_v((int8x16_t)__rev0, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vpaddl_u8(uint8x8_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vpaddl_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vpaddl_u8(uint8x8_t __p0) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vpaddl_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vpaddl_u32(uint32x2_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vpaddl_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vpaddl_u32(uint32x2_t __p0) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vpaddl_v((int8x8_t)__rev0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vpaddl_u16(uint16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vpaddl_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vpaddl_u16(uint16x4_t __p0) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vpaddl_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vpaddl_s8(int8x8_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vpaddl_v((int8x8_t)__p0, 1); + return __ret; +} +#else +__ai int16x4_t vpaddl_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vpaddl_v((int8x8_t)__rev0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vpaddl_s32(int32x2_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vpaddl_v((int8x8_t)__p0, 3); + return __ret; +} +#else +__ai int64x1_t vpaddl_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vpaddl_v((int8x8_t)__rev0, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vpaddl_s16(int16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vpaddl_v((int8x8_t)__p0, 2); + return __ret; +} +#else +__ai int32x2_t vpaddl_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vpaddl_v((int8x8_t)__rev0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vpmax_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vpmax_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vpmax_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vpmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vpmax_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vpmax_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vpmax_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vpmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vpmax_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vpmax_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vpmax_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vpmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vpmax_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vpmax_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vpmax_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vpmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vpmax_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vpmax_v((int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2_t vpmax_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vpmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vpmax_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vpmax_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vpmax_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vpmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vpmax_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vpmax_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vpmax_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vpmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vpmin_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vpmin_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vpmin_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vpmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vpmin_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vpmin_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vpmin_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vpmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vpmin_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vpmin_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vpmin_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vpmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vpmin_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vpmin_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vpmin_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vpmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vpmin_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vpmin_v((int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2_t vpmin_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vpmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vpmin_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vpmin_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vpmin_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vpmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vpmin_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vpmin_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vpmin_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vpmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqabsq_s8(int8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqabsq_v((int8x16_t)__p0, 32); + return __ret; +} +#else +__ai int8x16_t vqabsq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqabsq_v((int8x16_t)__rev0, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqabsq_s32(int32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqabsq_v((int8x16_t)__p0, 34); + return __ret; +} +#else +__ai int32x4_t vqabsq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqabsq_v((int8x16_t)__rev0, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vqabsq_s16(int16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqabsq_v((int8x16_t)__p0, 33); + return __ret; +} +#else +__ai int16x8_t vqabsq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqabsq_v((int8x16_t)__rev0, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqabs_s8(int8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqabs_v((int8x8_t)__p0, 0); + return __ret; +} +#else +__ai int8x8_t vqabs_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqabs_v((int8x8_t)__rev0, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vqabs_s32(int32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqabs_v((int8x8_t)__p0, 2); + return __ret; +} +#else +__ai int32x2_t vqabs_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqabs_v((int8x8_t)__rev0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vqabs_s16(int16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqabs_v((int8x8_t)__p0, 1); + return __ret; +} +#else +__ai int16x4_t vqabs_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqabs_v((int8x8_t)__rev0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vqaddq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vqaddq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vqaddq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vqaddq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vqaddq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vqaddq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vqaddq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vqaddq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqaddq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vqaddq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqaddq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vqaddq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vqaddq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqaddq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 35); + return __ret; +} +#else +__ai int64x2_t vqaddq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vqaddq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vqaddq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int16x8_t __noswap_vqaddq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vqadd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vqadd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vqadd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vqadd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vqadd_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vqadd_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vqadd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vqadd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqadd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vqadd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vqadd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vqadd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int32x2_t __noswap_vqadd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vqadd_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#else +__ai int64x1_t vqadd_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vqadd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vqadd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int16x4_t __noswap_vqadd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqdmlal_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmlal_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 35); + return __ret; +} +#else +__ai int64x2_t vqdmlal_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmlal_v((int8x16_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vqdmlal_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmlal_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 35); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqdmlal_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmlal_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 34); + return __ret; +} +#else +__ai int32x4_t vqdmlal_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmlal_v((int8x16_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vqdmlal_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmlal_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 34); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlal_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = vqdmlal_s32(__s0, __s1, __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlal_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vqdmlal_s32(__rev0, __rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlal_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = vqdmlal_s16(__s0, __s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlal_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqdmlal_s16(__rev0, __rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqdmlal_n_s32(int64x2_t __p0, int32x2_t __p1, int32_t __p2) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmlal_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)(int32x2_t) {__p2, __p2}, 35); + return __ret; +} +#else +__ai int64x2_t vqdmlal_n_s32(int64x2_t __p0, int32x2_t __p1, int32_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmlal_v((int8x16_t)__rev0, (int8x8_t)__rev1, (int8x8_t)(int32x2_t) {__p2, __p2}, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vqdmlal_n_s32(int64x2_t __p0, int32x2_t __p1, int32_t __p2) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmlal_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)(int32x2_t) {__p2, __p2}, 35); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqdmlal_n_s16(int32x4_t __p0, int16x4_t __p1, int16_t __p2) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmlal_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)(int16x4_t) {__p2, __p2, __p2, __p2}, 34); + return __ret; +} +#else +__ai int32x4_t vqdmlal_n_s16(int32x4_t __p0, int16x4_t __p1, int16_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmlal_v((int8x16_t)__rev0, (int8x8_t)__rev1, (int8x8_t)(int16x4_t) {__p2, __p2, __p2, __p2}, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vqdmlal_n_s16(int32x4_t __p0, int16x4_t __p1, int16_t __p2) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmlal_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)(int16x4_t) {__p2, __p2, __p2, __p2}, 34); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqdmlsl_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmlsl_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 35); + return __ret; +} +#else +__ai int64x2_t vqdmlsl_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmlsl_v((int8x16_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vqdmlsl_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmlsl_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 35); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqdmlsl_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmlsl_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 34); + return __ret; +} +#else +__ai int32x4_t vqdmlsl_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmlsl_v((int8x16_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vqdmlsl_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmlsl_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 34); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlsl_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = vqdmlsl_s32(__s0, __s1, __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlsl_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vqdmlsl_s32(__rev0, __rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlsl_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = vqdmlsl_s16(__s0, __s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlsl_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqdmlsl_s16(__rev0, __rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqdmlsl_n_s32(int64x2_t __p0, int32x2_t __p1, int32_t __p2) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmlsl_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)(int32x2_t) {__p2, __p2}, 35); + return __ret; +} +#else +__ai int64x2_t vqdmlsl_n_s32(int64x2_t __p0, int32x2_t __p1, int32_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmlsl_v((int8x16_t)__rev0, (int8x8_t)__rev1, (int8x8_t)(int32x2_t) {__p2, __p2}, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vqdmlsl_n_s32(int64x2_t __p0, int32x2_t __p1, int32_t __p2) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmlsl_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)(int32x2_t) {__p2, __p2}, 35); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqdmlsl_n_s16(int32x4_t __p0, int16x4_t __p1, int16_t __p2) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmlsl_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)(int16x4_t) {__p2, __p2, __p2, __p2}, 34); + return __ret; +} +#else +__ai int32x4_t vqdmlsl_n_s16(int32x4_t __p0, int16x4_t __p1, int16_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmlsl_v((int8x16_t)__rev0, (int8x8_t)__rev1, (int8x8_t)(int16x4_t) {__p2, __p2, __p2, __p2}, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vqdmlsl_n_s16(int32x4_t __p0, int16x4_t __p1, int16_t __p2) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmlsl_v((int8x16_t)__p0, (int8x8_t)__p1, (int8x8_t)(int16x4_t) {__p2, __p2, __p2, __p2}, 34); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqdmulhq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmulhq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vqdmulhq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmulhq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vqdmulhq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmulhq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vqdmulhq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqdmulhq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vqdmulhq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqdmulhq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int16x8_t __noswap_vqdmulhq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqdmulhq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vqdmulh_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqdmulh_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vqdmulh_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqdmulh_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int32x2_t __noswap_vqdmulh_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqdmulh_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vqdmulh_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqdmulh_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vqdmulh_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqdmulh_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int16x4_t __noswap_vqdmulh_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqdmulh_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmulhq_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = vqdmulhq_s32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmulhq_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqdmulhq_s32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmulhq_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __ret; \ + __ret = vqdmulhq_s16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmulhq_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __noswap_vqdmulhq_s16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmulh_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __ret; \ + __ret = vqdmulh_s32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmulh_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __ret; \ + __ret = __noswap_vqdmulh_s32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmulh_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __ret; \ + __ret = vqdmulh_s16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmulh_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __noswap_vqdmulh_s16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqdmulhq_n_s32(int32x4_t __p0, int32_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmulhq_v((int8x16_t)__p0, (int8x16_t)(int32x4_t) {__p1, __p1, __p1, __p1}, 34); + return __ret; +} +#else +__ai int32x4_t vqdmulhq_n_s32(int32x4_t __p0, int32_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmulhq_v((int8x16_t)__rev0, (int8x16_t)(int32x4_t) {__p1, __p1, __p1, __p1}, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vqdmulhq_n_s16(int16x8_t __p0, int16_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqdmulhq_v((int8x16_t)__p0, (int8x16_t)(int16x8_t) {__p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1}, 33); + return __ret; +} +#else +__ai int16x8_t vqdmulhq_n_s16(int16x8_t __p0, int16_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqdmulhq_v((int8x16_t)__rev0, (int8x16_t)(int16x8_t) {__p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1}, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vqdmulh_n_s32(int32x2_t __p0, int32_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqdmulh_v((int8x8_t)__p0, (int8x8_t)(int32x2_t) {__p1, __p1}, 2); + return __ret; +} +#else +__ai int32x2_t vqdmulh_n_s32(int32x2_t __p0, int32_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqdmulh_v((int8x8_t)__rev0, (int8x8_t)(int32x2_t) {__p1, __p1}, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vqdmulh_n_s16(int16x4_t __p0, int16_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqdmulh_v((int8x8_t)__p0, (int8x8_t)(int16x4_t) {__p1, __p1, __p1, __p1}, 1); + return __ret; +} +#else +__ai int16x4_t vqdmulh_n_s16(int16x4_t __p0, int16_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqdmulh_v((int8x8_t)__rev0, (int8x8_t)(int16x4_t) {__p1, __p1, __p1, __p1}, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqdmull_s32(int32x2_t __p0, int32x2_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmull_v((int8x8_t)__p0, (int8x8_t)__p1, 35); + return __ret; +} +#else +__ai int64x2_t vqdmull_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmull_v((int8x8_t)__rev0, (int8x8_t)__rev1, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vqdmull_s32(int32x2_t __p0, int32x2_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmull_v((int8x8_t)__p0, (int8x8_t)__p1, 35); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqdmull_s16(int16x4_t __p0, int16x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmull_v((int8x8_t)__p0, (int8x8_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vqdmull_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmull_v((int8x8_t)__rev0, (int8x8_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vqdmull_s16(int16x4_t __p0, int16x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmull_v((int8x8_t)__p0, (int8x8_t)__p1, 34); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmull_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = vqdmull_s32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmull_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vqdmull_s32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmull_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = vqdmull_s16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmull_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqdmull_s16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqdmull_n_s32(int32x2_t __p0, int32_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmull_v((int8x8_t)__p0, (int8x8_t)(int32x2_t) {__p1, __p1}, 35); + return __ret; +} +#else +__ai int64x2_t vqdmull_n_s32(int32x2_t __p0, int32_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmull_v((int8x8_t)__rev0, (int8x8_t)(int32x2_t) {__p1, __p1}, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vqdmull_n_s32(int32x2_t __p0, int32_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqdmull_v((int8x8_t)__p0, (int8x8_t)(int32x2_t) {__p1, __p1}, 35); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqdmull_n_s16(int16x4_t __p0, int16_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmull_v((int8x8_t)__p0, (int8x8_t)(int16x4_t) {__p1, __p1, __p1, __p1}, 34); + return __ret; +} +#else +__ai int32x4_t vqdmull_n_s16(int16x4_t __p0, int16_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmull_v((int8x8_t)__rev0, (int8x8_t)(int16x4_t) {__p1, __p1, __p1, __p1}, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vqdmull_n_s16(int16x4_t __p0, int16_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqdmull_v((int8x8_t)__p0, (int8x8_t)(int16x4_t) {__p1, __p1, __p1, __p1}, 34); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vqmovn_u32(uint32x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vqmovn_v((int8x16_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vqmovn_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vqmovn_v((int8x16_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint16x4_t __noswap_vqmovn_u32(uint32x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vqmovn_v((int8x16_t)__p0, 17); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vqmovn_u64(uint64x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vqmovn_v((int8x16_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vqmovn_u64(uint64x2_t __p0) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vqmovn_v((int8x16_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint32x2_t __noswap_vqmovn_u64(uint64x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vqmovn_v((int8x16_t)__p0, 18); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vqmovn_u16(uint16x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqmovn_v((int8x16_t)__p0, 16); + return __ret; +} +#else +__ai uint8x8_t vqmovn_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqmovn_v((int8x16_t)__rev0, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint8x8_t __noswap_vqmovn_u16(uint16x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqmovn_v((int8x16_t)__p0, 16); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vqmovn_s32(int32x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqmovn_v((int8x16_t)__p0, 1); + return __ret; +} +#else +__ai int16x4_t vqmovn_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqmovn_v((int8x16_t)__rev0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int16x4_t __noswap_vqmovn_s32(int32x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqmovn_v((int8x16_t)__p0, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vqmovn_s64(int64x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqmovn_v((int8x16_t)__p0, 2); + return __ret; +} +#else +__ai int32x2_t vqmovn_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqmovn_v((int8x16_t)__rev0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int32x2_t __noswap_vqmovn_s64(int64x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqmovn_v((int8x16_t)__p0, 2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqmovn_s16(int16x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqmovn_v((int8x16_t)__p0, 0); + return __ret; +} +#else +__ai int8x8_t vqmovn_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqmovn_v((int8x16_t)__rev0, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int8x8_t __noswap_vqmovn_s16(int16x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqmovn_v((int8x16_t)__p0, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vqmovun_s32(int32x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vqmovun_v((int8x16_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vqmovun_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vqmovun_v((int8x16_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint16x4_t __noswap_vqmovun_s32(int32x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vqmovun_v((int8x16_t)__p0, 17); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vqmovun_s64(int64x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vqmovun_v((int8x16_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vqmovun_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vqmovun_v((int8x16_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint32x2_t __noswap_vqmovun_s64(int64x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vqmovun_v((int8x16_t)__p0, 18); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vqmovun_s16(int16x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqmovun_v((int8x16_t)__p0, 16); + return __ret; +} +#else +__ai uint8x8_t vqmovun_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqmovun_v((int8x16_t)__rev0, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint8x8_t __noswap_vqmovun_s16(int16x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqmovun_v((int8x16_t)__p0, 16); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqnegq_s8(int8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqnegq_v((int8x16_t)__p0, 32); + return __ret; +} +#else +__ai int8x16_t vqnegq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqnegq_v((int8x16_t)__rev0, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqnegq_s32(int32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqnegq_v((int8x16_t)__p0, 34); + return __ret; +} +#else +__ai int32x4_t vqnegq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqnegq_v((int8x16_t)__rev0, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vqnegq_s16(int16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqnegq_v((int8x16_t)__p0, 33); + return __ret; +} +#else +__ai int16x8_t vqnegq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqnegq_v((int8x16_t)__rev0, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqneg_s8(int8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqneg_v((int8x8_t)__p0, 0); + return __ret; +} +#else +__ai int8x8_t vqneg_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqneg_v((int8x8_t)__rev0, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vqneg_s32(int32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqneg_v((int8x8_t)__p0, 2); + return __ret; +} +#else +__ai int32x2_t vqneg_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqneg_v((int8x8_t)__rev0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vqneg_s16(int16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqneg_v((int8x8_t)__p0, 1); + return __ret; +} +#else +__ai int16x4_t vqneg_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqneg_v((int8x8_t)__rev0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqrdmulhq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqrdmulhq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vqrdmulhq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqrdmulhq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vqrdmulhq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqrdmulhq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vqrdmulhq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqrdmulhq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vqrdmulhq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqrdmulhq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int16x8_t __noswap_vqrdmulhq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqrdmulhq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vqrdmulh_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqrdmulh_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vqrdmulh_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqrdmulh_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int32x2_t __noswap_vqrdmulh_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqrdmulh_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vqrdmulh_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqrdmulh_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vqrdmulh_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqrdmulh_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int16x4_t __noswap_vqrdmulh_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqrdmulh_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmulhq_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = vqrdmulhq_s32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqrdmulhq_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqrdmulhq_s32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmulhq_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __ret; \ + __ret = vqrdmulhq_s16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqrdmulhq_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __noswap_vqrdmulhq_s16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmulh_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __ret; \ + __ret = vqrdmulh_s32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqrdmulh_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __ret; \ + __ret = __noswap_vqrdmulh_s32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmulh_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __ret; \ + __ret = vqrdmulh_s16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqrdmulh_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __noswap_vqrdmulh_s16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqrdmulhq_n_s32(int32x4_t __p0, int32_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqrdmulhq_v((int8x16_t)__p0, (int8x16_t)(int32x4_t) {__p1, __p1, __p1, __p1}, 34); + return __ret; +} +#else +__ai int32x4_t vqrdmulhq_n_s32(int32x4_t __p0, int32_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqrdmulhq_v((int8x16_t)__rev0, (int8x16_t)(int32x4_t) {__p1, __p1, __p1, __p1}, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vqrdmulhq_n_s16(int16x8_t __p0, int16_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqrdmulhq_v((int8x16_t)__p0, (int8x16_t)(int16x8_t) {__p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1}, 33); + return __ret; +} +#else +__ai int16x8_t vqrdmulhq_n_s16(int16x8_t __p0, int16_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqrdmulhq_v((int8x16_t)__rev0, (int8x16_t)(int16x8_t) {__p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1}, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vqrdmulh_n_s32(int32x2_t __p0, int32_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqrdmulh_v((int8x8_t)__p0, (int8x8_t)(int32x2_t) {__p1, __p1}, 2); + return __ret; +} +#else +__ai int32x2_t vqrdmulh_n_s32(int32x2_t __p0, int32_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqrdmulh_v((int8x8_t)__rev0, (int8x8_t)(int32x2_t) {__p1, __p1}, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vqrdmulh_n_s16(int16x4_t __p0, int16_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqrdmulh_v((int8x8_t)__p0, (int8x8_t)(int16x4_t) {__p1, __p1, __p1, __p1}, 1); + return __ret; +} +#else +__ai int16x4_t vqrdmulh_n_s16(int16x4_t __p0, int16_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqrdmulh_v((int8x8_t)__rev0, (int8x8_t)(int16x4_t) {__p1, __p1, __p1, __p1}, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vqrshlq_u8(uint8x16_t __p0, int8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vqrshlq_u8(uint8x16_t __p0, int8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vqrshlq_u32(uint32x4_t __p0, int32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vqrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vqrshlq_u32(uint32x4_t __p0, int32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vqrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vqrshlq_u64(uint64x2_t __p0, int64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vqrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vqrshlq_u64(uint64x2_t __p0, int64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vqrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vqrshlq_u16(uint16x8_t __p0, int16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vqrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vqrshlq_u16(uint16x8_t __p0, int16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vqrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqrshlq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vqrshlq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqrshlq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vqrshlq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqrshlq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 35); + return __ret; +} +#else +__ai int64x2_t vqrshlq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vqrshlq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vqrshlq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vqrshl_u8(uint8x8_t __p0, int8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vqrshl_u8(uint8x8_t __p0, int8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqrshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vqrshl_u32(uint32x2_t __p0, int32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vqrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vqrshl_u32(uint32x2_t __p0, int32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vqrshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vqrshl_u64(uint64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vqrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vqrshl_u64(uint64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vqrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vqrshl_u16(uint16x4_t __p0, int16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vqrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vqrshl_u16(uint16x4_t __p0, int16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vqrshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqrshl_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vqrshl_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqrshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vqrshl_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vqrshl_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqrshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vqrshl_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vqrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#else +__ai int64x1_t vqrshl_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vqrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vqrshl_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vqrshl_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqrshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrn_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__s0, __p1, 17); \ + __ret; \ +}) +#else +#define vqrshrn_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__rev0, __p1, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vqrshrn_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__s0, __p1, 17); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrn_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__s0, __p1, 18); \ + __ret; \ +}) +#else +#define vqrshrn_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__rev0, __p1, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vqrshrn_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__s0, __p1, 18); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrn_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__s0, __p1, 16); \ + __ret; \ +}) +#else +#define vqrshrn_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__rev0, __p1, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vqrshrn_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__s0, __p1, 16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrn_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__s0, __p1, 1); \ + __ret; \ +}) +#else +#define vqrshrn_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__rev0, __p1, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vqrshrn_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__s0, __p1, 1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrn_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__s0, __p1, 2); \ + __ret; \ +}) +#else +#define vqrshrn_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__rev0, __p1, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vqrshrn_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__s0, __p1, 2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrn_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__s0, __p1, 0); \ + __ret; \ +}) +#else +#define vqrshrn_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__rev0, __p1, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vqrshrn_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vqrshrn_n_v((int8x16_t)__s0, __p1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrun_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqrshrun_n_v((int8x16_t)__s0, __p1, 17); \ + __ret; \ +}) +#else +#define vqrshrun_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqrshrun_n_v((int8x16_t)__rev0, __p1, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vqrshrun_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqrshrun_n_v((int8x16_t)__s0, __p1, 17); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrun_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqrshrun_n_v((int8x16_t)__s0, __p1, 18); \ + __ret; \ +}) +#else +#define vqrshrun_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqrshrun_n_v((int8x16_t)__rev0, __p1, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vqrshrun_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqrshrun_n_v((int8x16_t)__s0, __p1, 18); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrun_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqrshrun_n_v((int8x16_t)__s0, __p1, 16); \ + __ret; \ +}) +#else +#define vqrshrun_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqrshrun_n_v((int8x16_t)__rev0, __p1, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vqrshrun_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqrshrun_n_v((int8x16_t)__s0, __p1, 16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vqshlq_u8(uint8x16_t __p0, int8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vqshlq_u8(uint8x16_t __p0, int8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vqshlq_u32(uint32x4_t __p0, int32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vqshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vqshlq_u32(uint32x4_t __p0, int32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vqshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vqshlq_u64(uint64x2_t __p0, int64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vqshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vqshlq_u64(uint64x2_t __p0, int64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vqshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vqshlq_u16(uint16x8_t __p0, int16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vqshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vqshlq_u16(uint16x8_t __p0, int16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vqshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqshlq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vqshlq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqshlq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vqshlq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqshlq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 35); + return __ret; +} +#else +__ai int64x2_t vqshlq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vqshlq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vqshlq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vqshl_u8(uint8x8_t __p0, int8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqshl_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vqshl_u8(uint8x8_t __p0, int8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vqshl_u32(uint32x2_t __p0, int32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vqshl_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vqshl_u32(uint32x2_t __p0, int32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vqshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vqshl_u64(uint64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vqshl_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vqshl_u64(uint64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vqshl_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vqshl_u16(uint16x4_t __p0, int16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vqshl_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vqshl_u16(uint16x4_t __p0, int16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vqshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqshl_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqshl_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vqshl_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vqshl_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqshl_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vqshl_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vqshl_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vqshl_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#else +__ai int64x1_t vqshl_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vqshl_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vqshl_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqshl_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vqshl_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlq_n_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vqshlq_n_v((int8x16_t)__s0, __p1, 48); \ + __ret; \ +}) +#else +#define vqshlq_n_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vqshlq_n_v((int8x16_t)__rev0, __p1, 48); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlq_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vqshlq_n_v((int8x16_t)__s0, __p1, 50); \ + __ret; \ +}) +#else +#define vqshlq_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vqshlq_n_v((int8x16_t)__rev0, __p1, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlq_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vqshlq_n_v((int8x16_t)__s0, __p1, 51); \ + __ret; \ +}) +#else +#define vqshlq_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vqshlq_n_v((int8x16_t)__rev0, __p1, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlq_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vqshlq_n_v((int8x16_t)__s0, __p1, 49); \ + __ret; \ +}) +#else +#define vqshlq_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vqshlq_n_v((int8x16_t)__rev0, __p1, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlq_n_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vqshlq_n_v((int8x16_t)__s0, __p1, 32); \ + __ret; \ +}) +#else +#define vqshlq_n_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vqshlq_n_v((int8x16_t)__rev0, __p1, 32); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlq_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vqshlq_n_v((int8x16_t)__s0, __p1, 34); \ + __ret; \ +}) +#else +#define vqshlq_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vqshlq_n_v((int8x16_t)__rev0, __p1, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlq_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vqshlq_n_v((int8x16_t)__s0, __p1, 35); \ + __ret; \ +}) +#else +#define vqshlq_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vqshlq_n_v((int8x16_t)__rev0, __p1, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlq_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vqshlq_n_v((int8x16_t)__s0, __p1, 33); \ + __ret; \ +}) +#else +#define vqshlq_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vqshlq_n_v((int8x16_t)__rev0, __p1, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshl_n_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqshl_n_v((int8x8_t)__s0, __p1, 16); \ + __ret; \ +}) +#else +#define vqshl_n_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqshl_n_v((int8x8_t)__rev0, __p1, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshl_n_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqshl_n_v((int8x8_t)__s0, __p1, 18); \ + __ret; \ +}) +#else +#define vqshl_n_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqshl_n_v((int8x8_t)__rev0, __p1, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshl_n_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vqshl_n_v((int8x8_t)__s0, __p1, 19); \ + __ret; \ +}) +#else +#define vqshl_n_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vqshl_n_v((int8x8_t)__s0, __p1, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshl_n_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqshl_n_v((int8x8_t)__s0, __p1, 17); \ + __ret; \ +}) +#else +#define vqshl_n_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqshl_n_v((int8x8_t)__rev0, __p1, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshl_n_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vqshl_n_v((int8x8_t)__s0, __p1, 0); \ + __ret; \ +}) +#else +#define vqshl_n_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vqshl_n_v((int8x8_t)__rev0, __p1, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshl_n_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vqshl_n_v((int8x8_t)__s0, __p1, 2); \ + __ret; \ +}) +#else +#define vqshl_n_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vqshl_n_v((int8x8_t)__rev0, __p1, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshl_n_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vqshl_n_v((int8x8_t)__s0, __p1, 3); \ + __ret; \ +}) +#else +#define vqshl_n_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vqshl_n_v((int8x8_t)__s0, __p1, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshl_n_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vqshl_n_v((int8x8_t)__s0, __p1, 1); \ + __ret; \ +}) +#else +#define vqshl_n_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vqshl_n_v((int8x8_t)__rev0, __p1, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshluq_n_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vqshluq_n_v((int8x16_t)__s0, __p1, 48); \ + __ret; \ +}) +#else +#define vqshluq_n_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vqshluq_n_v((int8x16_t)__rev0, __p1, 48); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshluq_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vqshluq_n_v((int8x16_t)__s0, __p1, 50); \ + __ret; \ +}) +#else +#define vqshluq_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vqshluq_n_v((int8x16_t)__rev0, __p1, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshluq_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vqshluq_n_v((int8x16_t)__s0, __p1, 51); \ + __ret; \ +}) +#else +#define vqshluq_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vqshluq_n_v((int8x16_t)__rev0, __p1, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshluq_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vqshluq_n_v((int8x16_t)__s0, __p1, 49); \ + __ret; \ +}) +#else +#define vqshluq_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vqshluq_n_v((int8x16_t)__rev0, __p1, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlu_n_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqshlu_n_v((int8x8_t)__s0, __p1, 16); \ + __ret; \ +}) +#else +#define vqshlu_n_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqshlu_n_v((int8x8_t)__rev0, __p1, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlu_n_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqshlu_n_v((int8x8_t)__s0, __p1, 18); \ + __ret; \ +}) +#else +#define vqshlu_n_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqshlu_n_v((int8x8_t)__rev0, __p1, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlu_n_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vqshlu_n_v((int8x8_t)__s0, __p1, 19); \ + __ret; \ +}) +#else +#define vqshlu_n_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vqshlu_n_v((int8x8_t)__s0, __p1, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlu_n_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqshlu_n_v((int8x8_t)__s0, __p1, 17); \ + __ret; \ +}) +#else +#define vqshlu_n_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqshlu_n_v((int8x8_t)__rev0, __p1, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrn_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqshrn_n_v((int8x16_t)__s0, __p1, 17); \ + __ret; \ +}) +#else +#define vqshrn_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqshrn_n_v((int8x16_t)__rev0, __p1, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vqshrn_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqshrn_n_v((int8x16_t)__s0, __p1, 17); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrn_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqshrn_n_v((int8x16_t)__s0, __p1, 18); \ + __ret; \ +}) +#else +#define vqshrn_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqshrn_n_v((int8x16_t)__rev0, __p1, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vqshrn_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqshrn_n_v((int8x16_t)__s0, __p1, 18); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrn_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqshrn_n_v((int8x16_t)__s0, __p1, 16); \ + __ret; \ +}) +#else +#define vqshrn_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqshrn_n_v((int8x16_t)__rev0, __p1, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vqshrn_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqshrn_n_v((int8x16_t)__s0, __p1, 16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrn_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vqshrn_n_v((int8x16_t)__s0, __p1, 1); \ + __ret; \ +}) +#else +#define vqshrn_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vqshrn_n_v((int8x16_t)__rev0, __p1, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vqshrn_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vqshrn_n_v((int8x16_t)__s0, __p1, 1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrn_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vqshrn_n_v((int8x16_t)__s0, __p1, 2); \ + __ret; \ +}) +#else +#define vqshrn_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vqshrn_n_v((int8x16_t)__rev0, __p1, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vqshrn_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vqshrn_n_v((int8x16_t)__s0, __p1, 2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrn_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vqshrn_n_v((int8x16_t)__s0, __p1, 0); \ + __ret; \ +}) +#else +#define vqshrn_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vqshrn_n_v((int8x16_t)__rev0, __p1, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vqshrn_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vqshrn_n_v((int8x16_t)__s0, __p1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrun_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqshrun_n_v((int8x16_t)__s0, __p1, 17); \ + __ret; \ +}) +#else +#define vqshrun_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqshrun_n_v((int8x16_t)__rev0, __p1, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vqshrun_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vqshrun_n_v((int8x16_t)__s0, __p1, 17); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrun_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqshrun_n_v((int8x16_t)__s0, __p1, 18); \ + __ret; \ +}) +#else +#define vqshrun_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqshrun_n_v((int8x16_t)__rev0, __p1, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vqshrun_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vqshrun_n_v((int8x16_t)__s0, __p1, 18); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrun_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqshrun_n_v((int8x16_t)__s0, __p1, 16); \ + __ret; \ +}) +#else +#define vqshrun_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqshrun_n_v((int8x16_t)__rev0, __p1, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vqshrun_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vqshrun_n_v((int8x16_t)__s0, __p1, 16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vqsubq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vqsubq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqsubq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vqsubq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vqsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vqsubq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vqsubq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vqsubq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vqsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vqsubq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vqsubq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vqsubq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vqsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vqsubq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vqsubq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqsubq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vqsubq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqsubq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqsubq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vqsubq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqsubq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vqsubq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vqsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqsubq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 35); + return __ret; +} +#else +__ai int64x2_t vqsubq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqsubq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vqsubq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vqsubq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqsubq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int16x8_t __noswap_vqsubq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vqsubq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vqsub_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqsub_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vqsub_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqsub_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vqsub_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vqsub_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vqsub_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vqsub_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vqsub_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vqsub_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vqsub_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vqsub_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vqsub_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vqsub_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vqsub_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vqsub_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqsub_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqsub_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vqsub_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqsub_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vqsub_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqsub_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vqsub_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqsub_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int32x2_t __noswap_vqsub_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vqsub_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vqsub_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vqsub_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#else +__ai int64x1_t vqsub_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vqsub_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vqsub_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqsub_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vqsub_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqsub_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int16x4_t __noswap_vqsub_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vqsub_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vraddhn_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vraddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vraddhn_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vraddhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint16x4_t __noswap_vraddhn_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vraddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 17); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vraddhn_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vraddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vraddhn_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vraddhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint32x2_t __noswap_vraddhn_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vraddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 18); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vraddhn_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vraddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vraddhn_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vraddhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint8x8_t __noswap_vraddhn_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vraddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 16); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vraddhn_s32(int32x4_t __p0, int32x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vraddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vraddhn_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vraddhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int16x4_t __noswap_vraddhn_s32(int32x4_t __p0, int32x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vraddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vraddhn_s64(int64x2_t __p0, int64x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vraddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vraddhn_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vraddhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int32x2_t __noswap_vraddhn_s64(int64x2_t __p0, int64x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vraddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vraddhn_s16(int16x8_t __p0, int16x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vraddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vraddhn_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vraddhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int8x8_t __noswap_vraddhn_s16(int16x8_t __p0, int16x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vraddhn_v((int8x16_t)__p0, (int8x16_t)__p1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vrecpeq_u32(uint32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vrecpeq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vrecpeq_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vrecpeq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vrecpeq_f32(float32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrecpeq_v((int8x16_t)__p0, 41); + return __ret; +} +#else +__ai float32x4_t vrecpeq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrecpeq_v((int8x16_t)__rev0, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vrecpe_u32(uint32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vrecpe_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vrecpe_u32(uint32x2_t __p0) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vrecpe_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vrecpe_f32(float32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrecpe_v((int8x8_t)__p0, 9); + return __ret; +} +#else +__ai float32x2_t vrecpe_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrecpe_v((int8x8_t)__rev0, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vrecpsq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrecpsq_v((int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4_t vrecpsq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrecpsq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vrecps_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrecps_v((int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2_t vrecps_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrecps_v((int8x8_t)__rev0, (int8x8_t)__rev1, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vrev16_p8(poly8x8_t __p0) { + poly8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2, 5, 4, 7, 6); + return __ret; +} +#else +__ai poly8x8_t vrev16_p8(poly8x8_t __p0) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2, 5, 4, 7, 6); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vrev16q_p8(poly8x16_t __p0) { + poly8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); + return __ret; +} +#else +__ai poly8x16_t vrev16q_p8(poly8x16_t __p0) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vrev16q_u8(uint8x16_t __p0) { + uint8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); + return __ret; +} +#else +__ai uint8x16_t vrev16q_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vrev16q_s8(int8x16_t __p0) { + int8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); + return __ret; +} +#else +__ai int8x16_t vrev16q_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vrev16_u8(uint8x8_t __p0) { + uint8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2, 5, 4, 7, 6); + return __ret; +} +#else +__ai uint8x8_t vrev16_u8(uint8x8_t __p0) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2, 5, 4, 7, 6); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vrev16_s8(int8x8_t __p0) { + int8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2, 5, 4, 7, 6); + return __ret; +} +#else +__ai int8x8_t vrev16_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2, 5, 4, 7, 6); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vrev32_p8(poly8x8_t __p0) { + poly8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0, 7, 6, 5, 4); + return __ret; +} +#else +__ai poly8x8_t vrev32_p8(poly8x8_t __p0) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 3, 2, 1, 0, 7, 6, 5, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vrev32_p16(poly16x4_t __p0) { + poly16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2); + return __ret; +} +#else +__ai poly16x4_t vrev32_p16(poly16x4_t __p0) { + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + poly16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vrev32q_p8(poly8x16_t __p0) { + poly8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12); + return __ret; +} +#else +__ai poly8x16_t vrev32q_p8(poly8x16_t __p0) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vrev32q_p16(poly16x8_t __p0) { + poly16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2, 5, 4, 7, 6); + return __ret; +} +#else +__ai poly16x8_t vrev32q_p16(poly16x8_t __p0) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2, 5, 4, 7, 6); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vrev32q_u8(uint8x16_t __p0) { + uint8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12); + return __ret; +} +#else +__ai uint8x16_t vrev32q_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vrev32q_u16(uint16x8_t __p0) { + uint16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2, 5, 4, 7, 6); + return __ret; +} +#else +__ai uint16x8_t vrev32q_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2, 5, 4, 7, 6); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vrev32q_s8(int8x16_t __p0) { + int8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12); + return __ret; +} +#else +__ai int8x16_t vrev32q_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vrev32q_s16(int16x8_t __p0) { + int16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2, 5, 4, 7, 6); + return __ret; +} +#else +__ai int16x8_t vrev32q_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2, 5, 4, 7, 6); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vrev32_u8(uint8x8_t __p0) { + uint8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0, 7, 6, 5, 4); + return __ret; +} +#else +__ai uint8x8_t vrev32_u8(uint8x8_t __p0) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 3, 2, 1, 0, 7, 6, 5, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vrev32_u16(uint16x4_t __p0) { + uint16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2); + return __ret; +} +#else +__ai uint16x4_t vrev32_u16(uint16x4_t __p0) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vrev32_s8(int8x8_t __p0) { + int8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0, 7, 6, 5, 4); + return __ret; +} +#else +__ai int8x8_t vrev32_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 3, 2, 1, 0, 7, 6, 5, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vrev32_s16(int16x4_t __p0) { + int16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2); + return __ret; +} +#else +__ai int16x4_t vrev32_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vrev64_p8(poly8x8_t __p0) { + poly8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#else +__ai poly8x8_t vrev64_p8(poly8x8_t __p0) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 7, 6, 5, 4, 3, 2, 1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vrev64_p16(poly16x4_t __p0) { + poly16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + return __ret; +} +#else +__ai poly16x4_t vrev64_p16(poly16x4_t __p0) { + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + poly16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 3, 2, 1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vrev64q_p8(poly8x16_t __p0) { + poly8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8); + return __ret; +} +#else +__ai poly8x16_t vrev64q_p8(poly8x16_t __p0) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vrev64q_p16(poly16x8_t __p0) { + poly16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0, 7, 6, 5, 4); + return __ret; +} +#else +__ai poly16x8_t vrev64q_p16(poly16x8_t __p0) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 3, 2, 1, 0, 7, 6, 5, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vrev64q_u8(uint8x16_t __p0) { + uint8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8); + return __ret; +} +#else +__ai uint8x16_t vrev64q_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vrev64q_u32(uint32x4_t __p0) { + uint32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2); + return __ret; +} +#else +__ai uint32x4_t vrev64q_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vrev64q_u16(uint16x8_t __p0) { + uint16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0, 7, 6, 5, 4); + return __ret; +} +#else +__ai uint16x8_t vrev64q_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 3, 2, 1, 0, 7, 6, 5, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vrev64q_s8(int8x16_t __p0) { + int8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8); + return __ret; +} +#else +__ai int8x16_t vrev64q_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vrev64q_f32(float32x4_t __p0) { + float32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2); + return __ret; +} +#else +__ai float32x4_t vrev64q_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vrev64q_s32(int32x4_t __p0) { + int32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0, 3, 2); + return __ret; +} +#else +__ai int32x4_t vrev64q_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0, 3, 2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vrev64q_s16(int16x8_t __p0) { + int16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0, 7, 6, 5, 4); + return __ret; +} +#else +__ai int16x8_t vrev64q_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 3, 2, 1, 0, 7, 6, 5, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vrev64_u8(uint8x8_t __p0) { + uint8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#else +__ai uint8x8_t vrev64_u8(uint8x8_t __p0) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 7, 6, 5, 4, 3, 2, 1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vrev64_u32(uint32x2_t __p0) { + uint32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0); + return __ret; +} +#else +__ai uint32x2_t vrev64_u32(uint32x2_t __p0) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vrev64_u16(uint16x4_t __p0) { + uint16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + return __ret; +} +#else +__ai uint16x4_t vrev64_u16(uint16x4_t __p0) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 3, 2, 1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vrev64_s8(int8x8_t __p0) { + int8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#else +__ai int8x8_t vrev64_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 7, 6, 5, 4, 3, 2, 1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vrev64_f32(float32x2_t __p0) { + float32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0); + return __ret; +} +#else +__ai float32x2_t vrev64_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vrev64_s32(int32x2_t __p0) { + int32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1, 0); + return __ret; +} +#else +__ai int32x2_t vrev64_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vrev64_s16(int16x4_t __p0) { + int16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + return __ret; +} +#else +__ai int16x4_t vrev64_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 3, 2, 1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vrhaddq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vrhaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vrhaddq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vrhaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vrhaddq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vrhaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vrhaddq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vrhaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vrhaddq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vrhaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vrhaddq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vrhaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vrhaddq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vrhaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vrhaddq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vrhaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vrhaddq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vrhaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vrhaddq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vrhaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vrhaddq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vrhaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vrhaddq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vrhaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vrhadd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vrhadd_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vrhadd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vrhadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vrhadd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vrhadd_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vrhadd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vrhadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vrhadd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vrhadd_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vrhadd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vrhadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vrhadd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vrhadd_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vrhadd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vrhadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vrhadd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vrhadd_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vrhadd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vrhadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vrhadd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vrhadd_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vrhadd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vrhadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vrshlq_u8(uint8x16_t __p0, int8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vrshlq_u8(uint8x16_t __p0, int8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vrshlq_u32(uint32x4_t __p0, int32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vrshlq_u32(uint32x4_t __p0, int32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vrshlq_u64(uint64x2_t __p0, int64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vrshlq_u64(uint64x2_t __p0, int64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vrshlq_u16(uint16x8_t __p0, int16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vrshlq_u16(uint16x8_t __p0, int16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vrshlq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vrshlq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vrshlq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vrshlq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vrshlq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 35); + return __ret; +} +#else +__ai int64x2_t vrshlq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vrshlq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vrshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vrshlq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vrshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vrshl_u8(uint8x8_t __p0, int8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vrshl_u8(uint8x8_t __p0, int8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vrshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vrshl_u32(uint32x2_t __p0, int32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vrshl_u32(uint32x2_t __p0, int32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vrshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vrshl_u64(uint64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vrshl_u64(uint64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vrshl_u16(uint16x4_t __p0, int16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vrshl_u16(uint16x4_t __p0, int16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vrshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vrshl_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vrshl_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vrshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vrshl_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vrshl_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vrshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vrshl_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#else +__ai int64x1_t vrshl_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vrshl_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vrshl_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vrshl_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vrshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrq_n_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vrshrq_n_v((int8x16_t)__s0, __p1, 48); \ + __ret; \ +}) +#else +#define vrshrq_n_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vrshrq_n_v((int8x16_t)__rev0, __p1, 48); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrq_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vrshrq_n_v((int8x16_t)__s0, __p1, 50); \ + __ret; \ +}) +#else +#define vrshrq_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vrshrq_n_v((int8x16_t)__rev0, __p1, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrq_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vrshrq_n_v((int8x16_t)__s0, __p1, 51); \ + __ret; \ +}) +#else +#define vrshrq_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vrshrq_n_v((int8x16_t)__rev0, __p1, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrq_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vrshrq_n_v((int8x16_t)__s0, __p1, 49); \ + __ret; \ +}) +#else +#define vrshrq_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vrshrq_n_v((int8x16_t)__rev0, __p1, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrq_n_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vrshrq_n_v((int8x16_t)__s0, __p1, 32); \ + __ret; \ +}) +#else +#define vrshrq_n_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vrshrq_n_v((int8x16_t)__rev0, __p1, 32); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrq_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vrshrq_n_v((int8x16_t)__s0, __p1, 34); \ + __ret; \ +}) +#else +#define vrshrq_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vrshrq_n_v((int8x16_t)__rev0, __p1, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrq_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vrshrq_n_v((int8x16_t)__s0, __p1, 35); \ + __ret; \ +}) +#else +#define vrshrq_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vrshrq_n_v((int8x16_t)__rev0, __p1, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrq_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vrshrq_n_v((int8x16_t)__s0, __p1, 33); \ + __ret; \ +}) +#else +#define vrshrq_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vrshrq_n_v((int8x16_t)__rev0, __p1, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshr_n_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vrshr_n_v((int8x8_t)__s0, __p1, 16); \ + __ret; \ +}) +#else +#define vrshr_n_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vrshr_n_v((int8x8_t)__rev0, __p1, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshr_n_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vrshr_n_v((int8x8_t)__s0, __p1, 18); \ + __ret; \ +}) +#else +#define vrshr_n_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vrshr_n_v((int8x8_t)__rev0, __p1, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshr_n_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vrshr_n_v((int8x8_t)__s0, __p1, 19); \ + __ret; \ +}) +#else +#define vrshr_n_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vrshr_n_v((int8x8_t)__s0, __p1, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshr_n_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vrshr_n_v((int8x8_t)__s0, __p1, 17); \ + __ret; \ +}) +#else +#define vrshr_n_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vrshr_n_v((int8x8_t)__rev0, __p1, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshr_n_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vrshr_n_v((int8x8_t)__s0, __p1, 0); \ + __ret; \ +}) +#else +#define vrshr_n_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vrshr_n_v((int8x8_t)__rev0, __p1, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshr_n_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vrshr_n_v((int8x8_t)__s0, __p1, 2); \ + __ret; \ +}) +#else +#define vrshr_n_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vrshr_n_v((int8x8_t)__rev0, __p1, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshr_n_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vrshr_n_v((int8x8_t)__s0, __p1, 3); \ + __ret; \ +}) +#else +#define vrshr_n_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vrshr_n_v((int8x8_t)__s0, __p1, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshr_n_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vrshr_n_v((int8x8_t)__s0, __p1, 1); \ + __ret; \ +}) +#else +#define vrshr_n_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vrshr_n_v((int8x8_t)__rev0, __p1, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrn_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vrshrn_n_v((int8x16_t)__s0, __p1, 17); \ + __ret; \ +}) +#else +#define vrshrn_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vrshrn_n_v((int8x16_t)__rev0, __p1, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vrshrn_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vrshrn_n_v((int8x16_t)__s0, __p1, 17); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrn_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vrshrn_n_v((int8x16_t)__s0, __p1, 18); \ + __ret; \ +}) +#else +#define vrshrn_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vrshrn_n_v((int8x16_t)__rev0, __p1, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vrshrn_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vrshrn_n_v((int8x16_t)__s0, __p1, 18); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrn_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vrshrn_n_v((int8x16_t)__s0, __p1, 16); \ + __ret; \ +}) +#else +#define vrshrn_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vrshrn_n_v((int8x16_t)__rev0, __p1, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vrshrn_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vrshrn_n_v((int8x16_t)__s0, __p1, 16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrn_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vrshrn_n_v((int8x16_t)__s0, __p1, 1); \ + __ret; \ +}) +#else +#define vrshrn_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vrshrn_n_v((int8x16_t)__rev0, __p1, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vrshrn_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vrshrn_n_v((int8x16_t)__s0, __p1, 1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrn_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vrshrn_n_v((int8x16_t)__s0, __p1, 2); \ + __ret; \ +}) +#else +#define vrshrn_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vrshrn_n_v((int8x16_t)__rev0, __p1, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vrshrn_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vrshrn_n_v((int8x16_t)__s0, __p1, 2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrn_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vrshrn_n_v((int8x16_t)__s0, __p1, 0); \ + __ret; \ +}) +#else +#define vrshrn_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vrshrn_n_v((int8x16_t)__rev0, __p1, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vrshrn_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vrshrn_n_v((int8x16_t)__s0, __p1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vrsqrteq_u32(uint32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vrsqrteq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vrsqrteq_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vrsqrteq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vrsqrteq_f32(float32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrsqrteq_v((int8x16_t)__p0, 41); + return __ret; +} +#else +__ai float32x4_t vrsqrteq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrsqrteq_v((int8x16_t)__rev0, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vrsqrte_u32(uint32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vrsqrte_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vrsqrte_u32(uint32x2_t __p0) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vrsqrte_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vrsqrte_f32(float32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrsqrte_v((int8x8_t)__p0, 9); + return __ret; +} +#else +__ai float32x2_t vrsqrte_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrsqrte_v((int8x8_t)__rev0, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vrsqrtsq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrsqrtsq_v((int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4_t vrsqrtsq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrsqrtsq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vrsqrts_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrsqrts_v((int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2_t vrsqrts_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrsqrts_v((int8x8_t)__rev0, (int8x8_t)__rev1, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsraq_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vrsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 48); \ + __ret; \ +}) +#else +#define vrsraq_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vrsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 48); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsraq_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vrsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 50); \ + __ret; \ +}) +#else +#define vrsraq_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vrsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsraq_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vrsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 51); \ + __ret; \ +}) +#else +#define vrsraq_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vrsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsraq_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vrsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 49); \ + __ret; \ +}) +#else +#define vrsraq_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vrsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsraq_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __s1 = __p1; \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vrsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 32); \ + __ret; \ +}) +#else +#define vrsraq_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __s1 = __p1; \ + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vrsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 32); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsraq_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vrsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 34); \ + __ret; \ +}) +#else +#define vrsraq_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vrsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsraq_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vrsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 35); \ + __ret; \ +}) +#else +#define vrsraq_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __s1 = __p1; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vrsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsraq_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vrsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 33); \ + __ret; \ +}) +#else +#define vrsraq_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vrsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsra_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vrsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 16); \ + __ret; \ +}) +#else +#define vrsra_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vrsra_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsra_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vrsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 18); \ + __ret; \ +}) +#else +#define vrsra_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vrsra_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsra_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vrsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 19); \ + __ret; \ +}) +#else +#define vrsra_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vrsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsra_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vrsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 17); \ + __ret; \ +}) +#else +#define vrsra_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vrsra_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsra_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __s1 = __p1; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vrsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 0); \ + __ret; \ +}) +#else +#define vrsra_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __s1 = __p1; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vrsra_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsra_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vrsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 2); \ + __ret; \ +}) +#else +#define vrsra_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vrsra_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsra_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vrsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 3); \ + __ret; \ +}) +#else +#define vrsra_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vrsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsra_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vrsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 1); \ + __ret; \ +}) +#else +#define vrsra_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vrsra_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vrsubhn_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vrsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vrsubhn_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vrsubhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint16x4_t __noswap_vrsubhn_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vrsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 17); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vrsubhn_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vrsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vrsubhn_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vrsubhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint32x2_t __noswap_vrsubhn_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vrsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 18); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vrsubhn_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vrsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vrsubhn_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vrsubhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint8x8_t __noswap_vrsubhn_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vrsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 16); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vrsubhn_s32(int32x4_t __p0, int32x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vrsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vrsubhn_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vrsubhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int16x4_t __noswap_vrsubhn_s32(int32x4_t __p0, int32x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vrsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vrsubhn_s64(int64x2_t __p0, int64x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vrsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vrsubhn_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vrsubhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int32x2_t __noswap_vrsubhn_s64(int64x2_t __p0, int64x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vrsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vrsubhn_s16(int16x8_t __p0, int16x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vrsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vrsubhn_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vrsubhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int8x8_t __noswap_vrsubhn_s16(int16x8_t __p0, int16x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vrsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vset_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8_t __s0 = __p0; \ + poly8x8_t __s1 = __p1; \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vset_lane_i8(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vset_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8_t __s0 = __p0; \ + poly8x8_t __s1 = __p1; \ + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vset_lane_i8(__s0, (int8x8_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vset_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8_t __s0 = __p0; \ + poly8x8_t __s1 = __p1; \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vset_lane_i8(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vset_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16_t __s0 = __p0; \ + poly16x4_t __s1 = __p1; \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vset_lane_i16(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vset_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16_t __s0 = __p0; \ + poly16x4_t __s1 = __p1; \ + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vset_lane_i16(__s0, (int8x8_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vset_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16_t __s0 = __p0; \ + poly16x4_t __s1 = __p1; \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vset_lane_i16(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsetq_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8_t __s0 = __p0; \ + poly8x16_t __s1 = __p1; \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vsetq_lane_i8(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vsetq_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8_t __s0 = __p0; \ + poly8x16_t __s1 = __p1; \ + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vsetq_lane_i8(__s0, (int8x16_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vsetq_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8_t __s0 = __p0; \ + poly8x16_t __s1 = __p1; \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vsetq_lane_i8(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsetq_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16_t __s0 = __p0; \ + poly16x8_t __s1 = __p1; \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vsetq_lane_i16(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vsetq_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16_t __s0 = __p0; \ + poly16x8_t __s1 = __p1; \ + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vsetq_lane_i16(__s0, (int8x16_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vsetq_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16_t __s0 = __p0; \ + poly16x8_t __s1 = __p1; \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vsetq_lane_i16(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsetq_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vsetq_lane_i8(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vsetq_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vsetq_lane_i8(__s0, (int8x16_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vsetq_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vsetq_lane_i8(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsetq_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vsetq_lane_i32(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vsetq_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vsetq_lane_i32(__s0, (int8x16_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vsetq_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vsetq_lane_i32(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsetq_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vsetq_lane_i64(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vsetq_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vsetq_lane_i64(__s0, (int8x16_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vsetq_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vsetq_lane_i64(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsetq_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vsetq_lane_i16(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vsetq_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vsetq_lane_i16(__s0, (int8x16_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vsetq_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vsetq_lane_i16(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsetq_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8_t __s0 = __p0; \ + int8x16_t __s1 = __p1; \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vsetq_lane_i8(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vsetq_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8_t __s0 = __p0; \ + int8x16_t __s1 = __p1; \ + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vsetq_lane_i8(__s0, (int8x16_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vsetq_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8_t __s0 = __p0; \ + int8x16_t __s1 = __p1; \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vsetq_lane_i8(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsetq_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vsetq_lane_f32(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vsetq_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vsetq_lane_f32(__s0, (int8x16_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vsetq_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vsetq_lane_f32(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsetq_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vsetq_lane_f16(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vsetq_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vsetq_lane_f16(__s0, (int8x16_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsetq_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vsetq_lane_i32(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vsetq_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vsetq_lane_i32(__s0, (int8x16_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vsetq_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vsetq_lane_i32(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsetq_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64x2_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vsetq_lane_i64(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vsetq_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64x2_t __s1 = __p1; \ + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vsetq_lane_i64(__s0, (int8x16_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vsetq_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64x2_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vsetq_lane_i64(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsetq_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vsetq_lane_i16(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vsetq_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vsetq_lane_i16(__s0, (int8x16_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vsetq_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vsetq_lane_i16(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vset_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vset_lane_i8(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vset_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vset_lane_i8(__s0, (int8x8_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vset_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vset_lane_i8(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vset_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vset_lane_i32(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vset_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vset_lane_i32(__s0, (int8x8_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vset_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vset_lane_i32(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vset_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vset_lane_i64(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vset_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vset_lane_i64(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#define __noswap_vset_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vset_lane_i64(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vset_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vset_lane_i16(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vset_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vset_lane_i16(__s0, (int8x8_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vset_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vset_lane_i16(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vset_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8_t __s0 = __p0; \ + int8x8_t __s1 = __p1; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vset_lane_i8(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vset_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8_t __s0 = __p0; \ + int8x8_t __s1 = __p1; \ + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vset_lane_i8(__s0, (int8x8_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vset_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8_t __s0 = __p0; \ + int8x8_t __s1 = __p1; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vset_lane_i8(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vset_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vset_lane_f32(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vset_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vset_lane_f32(__s0, (int8x8_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vset_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vset_lane_f32(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vset_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vset_lane_f16(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vset_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vset_lane_f16(__s0, (int8x8_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vset_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vset_lane_i32(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vset_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vset_lane_i32(__s0, (int8x8_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vset_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vset_lane_i32(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vset_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vset_lane_i64(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vset_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vset_lane_i64(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#define __noswap_vset_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vset_lane_i64(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vset_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vset_lane_i16(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vset_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vset_lane_i16(__s0, (int8x8_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vset_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vset_lane_i16(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vshlq_u8(uint8x16_t __p0, int8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vshlq_u8(uint8x16_t __p0, int8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vshlq_u32(uint32x4_t __p0, int32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vshlq_u32(uint32x4_t __p0, int32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vshlq_u64(uint64x2_t __p0, int64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vshlq_u64(uint64x2_t __p0, int64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vshlq_u16(uint16x8_t __p0, int16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vshlq_u16(uint16x8_t __p0, int16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vshlq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vshlq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vshlq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vshlq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vshlq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 35); + return __ret; +} +#else +__ai int64x2_t vshlq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vshlq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vshlq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vshlq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vshlq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vshl_u8(uint8x8_t __p0, int8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vshl_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vshl_u8(uint8x8_t __p0, int8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vshl_u32(uint32x2_t __p0, int32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vshl_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vshl_u32(uint32x2_t __p0, int32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vshl_u64(uint64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vshl_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vshl_u64(uint64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vshl_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vshl_u16(uint16x4_t __p0, int16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vshl_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vshl_u16(uint16x4_t __p0, int16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vshl_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vshl_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vshl_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vshl_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vshl_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vshl_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vshl_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vshl_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#else +__ai int64x1_t vshl_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vshl_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vshl_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vshl_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vshl_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshlq_n_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vshlq_n_v((int8x16_t)__s0, __p1, 48); \ + __ret; \ +}) +#else +#define vshlq_n_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vshlq_n_v((int8x16_t)__rev0, __p1, 48); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshlq_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vshlq_n_v((int8x16_t)__s0, __p1, 50); \ + __ret; \ +}) +#else +#define vshlq_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vshlq_n_v((int8x16_t)__rev0, __p1, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshlq_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vshlq_n_v((int8x16_t)__s0, __p1, 51); \ + __ret; \ +}) +#else +#define vshlq_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vshlq_n_v((int8x16_t)__rev0, __p1, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshlq_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vshlq_n_v((int8x16_t)__s0, __p1, 49); \ + __ret; \ +}) +#else +#define vshlq_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vshlq_n_v((int8x16_t)__rev0, __p1, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshlq_n_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vshlq_n_v((int8x16_t)__s0, __p1, 32); \ + __ret; \ +}) +#else +#define vshlq_n_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vshlq_n_v((int8x16_t)__rev0, __p1, 32); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshlq_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vshlq_n_v((int8x16_t)__s0, __p1, 34); \ + __ret; \ +}) +#else +#define vshlq_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vshlq_n_v((int8x16_t)__rev0, __p1, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshlq_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vshlq_n_v((int8x16_t)__s0, __p1, 35); \ + __ret; \ +}) +#else +#define vshlq_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vshlq_n_v((int8x16_t)__rev0, __p1, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshlq_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vshlq_n_v((int8x16_t)__s0, __p1, 33); \ + __ret; \ +}) +#else +#define vshlq_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vshlq_n_v((int8x16_t)__rev0, __p1, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshl_n_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vshl_n_v((int8x8_t)__s0, __p1, 16); \ + __ret; \ +}) +#else +#define vshl_n_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vshl_n_v((int8x8_t)__rev0, __p1, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshl_n_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vshl_n_v((int8x8_t)__s0, __p1, 18); \ + __ret; \ +}) +#else +#define vshl_n_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vshl_n_v((int8x8_t)__rev0, __p1, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshl_n_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vshl_n_v((int8x8_t)__s0, __p1, 19); \ + __ret; \ +}) +#else +#define vshl_n_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vshl_n_v((int8x8_t)__s0, __p1, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshl_n_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vshl_n_v((int8x8_t)__s0, __p1, 17); \ + __ret; \ +}) +#else +#define vshl_n_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vshl_n_v((int8x8_t)__rev0, __p1, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshl_n_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vshl_n_v((int8x8_t)__s0, __p1, 0); \ + __ret; \ +}) +#else +#define vshl_n_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vshl_n_v((int8x8_t)__rev0, __p1, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshl_n_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vshl_n_v((int8x8_t)__s0, __p1, 2); \ + __ret; \ +}) +#else +#define vshl_n_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vshl_n_v((int8x8_t)__rev0, __p1, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshl_n_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vshl_n_v((int8x8_t)__s0, __p1, 3); \ + __ret; \ +}) +#else +#define vshl_n_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vshl_n_v((int8x8_t)__s0, __p1, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshl_n_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vshl_n_v((int8x8_t)__s0, __p1, 1); \ + __ret; \ +}) +#else +#define vshl_n_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vshl_n_v((int8x8_t)__rev0, __p1, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshll_n_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vshll_n_v((int8x8_t)__s0, __p1, 49); \ + __ret; \ +}) +#else +#define vshll_n_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vshll_n_v((int8x8_t)__rev0, __p1, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vshll_n_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vshll_n_v((int8x8_t)__s0, __p1, 49); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshll_n_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vshll_n_v((int8x8_t)__s0, __p1, 51); \ + __ret; \ +}) +#else +#define vshll_n_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vshll_n_v((int8x8_t)__rev0, __p1, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vshll_n_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vshll_n_v((int8x8_t)__s0, __p1, 51); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshll_n_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vshll_n_v((int8x8_t)__s0, __p1, 50); \ + __ret; \ +}) +#else +#define vshll_n_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vshll_n_v((int8x8_t)__rev0, __p1, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vshll_n_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vshll_n_v((int8x8_t)__s0, __p1, 50); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshll_n_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vshll_n_v((int8x8_t)__s0, __p1, 33); \ + __ret; \ +}) +#else +#define vshll_n_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vshll_n_v((int8x8_t)__rev0, __p1, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vshll_n_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vshll_n_v((int8x8_t)__s0, __p1, 33); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshll_n_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vshll_n_v((int8x8_t)__s0, __p1, 35); \ + __ret; \ +}) +#else +#define vshll_n_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vshll_n_v((int8x8_t)__rev0, __p1, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vshll_n_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vshll_n_v((int8x8_t)__s0, __p1, 35); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshll_n_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vshll_n_v((int8x8_t)__s0, __p1, 34); \ + __ret; \ +}) +#else +#define vshll_n_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vshll_n_v((int8x8_t)__rev0, __p1, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vshll_n_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vshll_n_v((int8x8_t)__s0, __p1, 34); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrq_n_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vshrq_n_v((int8x16_t)__s0, __p1, 48); \ + __ret; \ +}) +#else +#define vshrq_n_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vshrq_n_v((int8x16_t)__rev0, __p1, 48); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrq_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vshrq_n_v((int8x16_t)__s0, __p1, 50); \ + __ret; \ +}) +#else +#define vshrq_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vshrq_n_v((int8x16_t)__rev0, __p1, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrq_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vshrq_n_v((int8x16_t)__s0, __p1, 51); \ + __ret; \ +}) +#else +#define vshrq_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vshrq_n_v((int8x16_t)__rev0, __p1, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrq_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vshrq_n_v((int8x16_t)__s0, __p1, 49); \ + __ret; \ +}) +#else +#define vshrq_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vshrq_n_v((int8x16_t)__rev0, __p1, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrq_n_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vshrq_n_v((int8x16_t)__s0, __p1, 32); \ + __ret; \ +}) +#else +#define vshrq_n_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vshrq_n_v((int8x16_t)__rev0, __p1, 32); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrq_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vshrq_n_v((int8x16_t)__s0, __p1, 34); \ + __ret; \ +}) +#else +#define vshrq_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vshrq_n_v((int8x16_t)__rev0, __p1, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrq_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vshrq_n_v((int8x16_t)__s0, __p1, 35); \ + __ret; \ +}) +#else +#define vshrq_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vshrq_n_v((int8x16_t)__rev0, __p1, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrq_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vshrq_n_v((int8x16_t)__s0, __p1, 33); \ + __ret; \ +}) +#else +#define vshrq_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vshrq_n_v((int8x16_t)__rev0, __p1, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshr_n_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vshr_n_v((int8x8_t)__s0, __p1, 16); \ + __ret; \ +}) +#else +#define vshr_n_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vshr_n_v((int8x8_t)__rev0, __p1, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshr_n_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vshr_n_v((int8x8_t)__s0, __p1, 18); \ + __ret; \ +}) +#else +#define vshr_n_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vshr_n_v((int8x8_t)__rev0, __p1, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshr_n_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vshr_n_v((int8x8_t)__s0, __p1, 19); \ + __ret; \ +}) +#else +#define vshr_n_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vshr_n_v((int8x8_t)__s0, __p1, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshr_n_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vshr_n_v((int8x8_t)__s0, __p1, 17); \ + __ret; \ +}) +#else +#define vshr_n_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vshr_n_v((int8x8_t)__rev0, __p1, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshr_n_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vshr_n_v((int8x8_t)__s0, __p1, 0); \ + __ret; \ +}) +#else +#define vshr_n_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vshr_n_v((int8x8_t)__rev0, __p1, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshr_n_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vshr_n_v((int8x8_t)__s0, __p1, 2); \ + __ret; \ +}) +#else +#define vshr_n_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vshr_n_v((int8x8_t)__rev0, __p1, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshr_n_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vshr_n_v((int8x8_t)__s0, __p1, 3); \ + __ret; \ +}) +#else +#define vshr_n_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vshr_n_v((int8x8_t)__s0, __p1, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshr_n_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vshr_n_v((int8x8_t)__s0, __p1, 1); \ + __ret; \ +}) +#else +#define vshr_n_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vshr_n_v((int8x8_t)__rev0, __p1, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrn_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vshrn_n_v((int8x16_t)__s0, __p1, 17); \ + __ret; \ +}) +#else +#define vshrn_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vshrn_n_v((int8x16_t)__rev0, __p1, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vshrn_n_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vshrn_n_v((int8x16_t)__s0, __p1, 17); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrn_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vshrn_n_v((int8x16_t)__s0, __p1, 18); \ + __ret; \ +}) +#else +#define vshrn_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vshrn_n_v((int8x16_t)__rev0, __p1, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vshrn_n_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vshrn_n_v((int8x16_t)__s0, __p1, 18); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrn_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vshrn_n_v((int8x16_t)__s0, __p1, 16); \ + __ret; \ +}) +#else +#define vshrn_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vshrn_n_v((int8x16_t)__rev0, __p1, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vshrn_n_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vshrn_n_v((int8x16_t)__s0, __p1, 16); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrn_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vshrn_n_v((int8x16_t)__s0, __p1, 1); \ + __ret; \ +}) +#else +#define vshrn_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vshrn_n_v((int8x16_t)__rev0, __p1, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vshrn_n_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vshrn_n_v((int8x16_t)__s0, __p1, 1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrn_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vshrn_n_v((int8x16_t)__s0, __p1, 2); \ + __ret; \ +}) +#else +#define vshrn_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vshrn_n_v((int8x16_t)__rev0, __p1, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vshrn_n_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vshrn_n_v((int8x16_t)__s0, __p1, 2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrn_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vshrn_n_v((int8x16_t)__s0, __p1, 0); \ + __ret; \ +}) +#else +#define vshrn_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vshrn_n_v((int8x16_t)__rev0, __p1, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vshrn_n_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vshrn_n_v((int8x16_t)__s0, __p1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsli_n_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8x8_t __s1 = __p1; \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vsli_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 4); \ + __ret; \ +}) +#else +#define vsli_n_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8x8_t __s1 = __p1; \ + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vsli_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 4); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsli_n_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16x4_t __s1 = __p1; \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vsli_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 5); \ + __ret; \ +}) +#else +#define vsli_n_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16x4_t __s1 = __p1; \ + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vsli_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 5); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsliq_n_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8x16_t __s1 = __p1; \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vsliq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 36); \ + __ret; \ +}) +#else +#define vsliq_n_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8x16_t __s1 = __p1; \ + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vsliq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 36); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsliq_n_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16x8_t __s1 = __p1; \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vsliq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 37); \ + __ret; \ +}) +#else +#define vsliq_n_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16x8_t __s1 = __p1; \ + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vsliq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 37); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsliq_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vsliq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 48); \ + __ret; \ +}) +#else +#define vsliq_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vsliq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 48); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsliq_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vsliq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 50); \ + __ret; \ +}) +#else +#define vsliq_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vsliq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsliq_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vsliq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 51); \ + __ret; \ +}) +#else +#define vsliq_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vsliq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsliq_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vsliq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 49); \ + __ret; \ +}) +#else +#define vsliq_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vsliq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsliq_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __s1 = __p1; \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vsliq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 32); \ + __ret; \ +}) +#else +#define vsliq_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __s1 = __p1; \ + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vsliq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 32); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsliq_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vsliq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 34); \ + __ret; \ +}) +#else +#define vsliq_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vsliq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsliq_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vsliq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 35); \ + __ret; \ +}) +#else +#define vsliq_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __s1 = __p1; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vsliq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsliq_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vsliq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 33); \ + __ret; \ +}) +#else +#define vsliq_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vsliq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsli_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vsli_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 16); \ + __ret; \ +}) +#else +#define vsli_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vsli_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsli_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vsli_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 18); \ + __ret; \ +}) +#else +#define vsli_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vsli_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsli_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vsli_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 19); \ + __ret; \ +}) +#else +#define vsli_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vsli_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsli_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vsli_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 17); \ + __ret; \ +}) +#else +#define vsli_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vsli_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsli_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __s1 = __p1; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vsli_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 0); \ + __ret; \ +}) +#else +#define vsli_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __s1 = __p1; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vsli_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsli_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vsli_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 2); \ + __ret; \ +}) +#else +#define vsli_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vsli_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsli_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vsli_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 3); \ + __ret; \ +}) +#else +#define vsli_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vsli_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsli_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vsli_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 1); \ + __ret; \ +}) +#else +#define vsli_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vsli_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsraq_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 48); \ + __ret; \ +}) +#else +#define vsraq_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 48); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsraq_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 50); \ + __ret; \ +}) +#else +#define vsraq_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsraq_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 51); \ + __ret; \ +}) +#else +#define vsraq_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsraq_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 49); \ + __ret; \ +}) +#else +#define vsraq_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsraq_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __s1 = __p1; \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 32); \ + __ret; \ +}) +#else +#define vsraq_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __s1 = __p1; \ + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 32); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsraq_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 34); \ + __ret; \ +}) +#else +#define vsraq_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsraq_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 35); \ + __ret; \ +}) +#else +#define vsraq_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __s1 = __p1; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsraq_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vsraq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 33); \ + __ret; \ +}) +#else +#define vsraq_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vsraq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsra_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 16); \ + __ret; \ +}) +#else +#define vsra_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vsra_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsra_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 18); \ + __ret; \ +}) +#else +#define vsra_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vsra_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsra_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 19); \ + __ret; \ +}) +#else +#define vsra_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsra_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 17); \ + __ret; \ +}) +#else +#define vsra_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vsra_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsra_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __s1 = __p1; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 0); \ + __ret; \ +}) +#else +#define vsra_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __s1 = __p1; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vsra_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsra_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 2); \ + __ret; \ +}) +#else +#define vsra_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vsra_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsra_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 3); \ + __ret; \ +}) +#else +#define vsra_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsra_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vsra_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 1); \ + __ret; \ +}) +#else +#define vsra_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vsra_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsri_n_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8x8_t __s1 = __p1; \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vsri_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 4); \ + __ret; \ +}) +#else +#define vsri_n_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8x8_t __s1 = __p1; \ + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8_t __ret; \ + __ret = (poly8x8_t) __builtin_neon_vsri_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 4); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsri_n_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16x4_t __s1 = __p1; \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vsri_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 5); \ + __ret; \ +}) +#else +#define vsri_n_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16x4_t __s1 = __p1; \ + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + poly16x4_t __ret; \ + __ret = (poly16x4_t) __builtin_neon_vsri_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 5); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsriq_n_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8x16_t __s1 = __p1; \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vsriq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 36); \ + __ret; \ +}) +#else +#define vsriq_n_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8x16_t __s1 = __p1; \ + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16_t __ret; \ + __ret = (poly8x16_t) __builtin_neon_vsriq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 36); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsriq_n_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16x8_t __s1 = __p1; \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vsriq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 37); \ + __ret; \ +}) +#else +#define vsriq_n_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16x8_t __s1 = __p1; \ + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x8_t __ret; \ + __ret = (poly16x8_t) __builtin_neon_vsriq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 37); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsriq_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vsriq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 48); \ + __ret; \ +}) +#else +#define vsriq_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret; \ + __ret = (uint8x16_t) __builtin_neon_vsriq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 48); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsriq_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vsriq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 50); \ + __ret; \ +}) +#else +#define vsriq_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = (uint32x4_t) __builtin_neon_vsriq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 50); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsriq_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vsriq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 51); \ + __ret; \ +}) +#else +#define vsriq_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vsriq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsriq_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vsriq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 49); \ + __ret; \ +}) +#else +#define vsriq_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vsriq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsriq_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __s1 = __p1; \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vsriq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 32); \ + __ret; \ +}) +#else +#define vsriq_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __s1 = __p1; \ + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret; \ + __ret = (int8x16_t) __builtin_neon_vsriq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 32); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsriq_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vsriq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 34); \ + __ret; \ +}) +#else +#define vsriq_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = (int32x4_t) __builtin_neon_vsriq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 34); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsriq_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vsriq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 35); \ + __ret; \ +}) +#else +#define vsriq_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __s1 = __p1; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vsriq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsriq_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vsriq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 33); \ + __ret; \ +}) +#else +#define vsriq_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vsriq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsri_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vsri_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 16); \ + __ret; \ +}) +#else +#define vsri_n_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = (uint8x8_t) __builtin_neon_vsri_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 16); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsri_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vsri_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 18); \ + __ret; \ +}) +#else +#define vsri_n_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x2_t __ret; \ + __ret = (uint32x2_t) __builtin_neon_vsri_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 18); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsri_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vsri_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 19); \ + __ret; \ +}) +#else +#define vsri_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64x1_t __s1 = __p1; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vsri_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsri_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vsri_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 17); \ + __ret; \ +}) +#else +#define vsri_n_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vsri_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsri_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __s1 = __p1; \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vsri_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 0); \ + __ret; \ +}) +#else +#define vsri_n_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __s1 = __p1; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = (int8x8_t) __builtin_neon_vsri_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 0); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsri_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vsri_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 2); \ + __ret; \ +}) +#else +#define vsri_n_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __ret; \ + __ret = (int32x2_t) __builtin_neon_vsri_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsri_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vsri_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 3); \ + __ret; \ +}) +#else +#define vsri_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64x1_t __s1 = __p1; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vsri_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsri_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vsri_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 1); \ + __ret; \ +}) +#else +#define vsri_n_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vsri_n_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_p8(__p0, __p1) __extension__ ({ \ + poly8x8_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 4); \ +}) +#else +#define vst1_p8(__p0, __p1) __extension__ ({ \ + poly8x8_t __s1 = __p1; \ + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__rev1, 4); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_p16(__p0, __p1) __extension__ ({ \ + poly16x4_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 5); \ +}) +#else +#define vst1_p16(__p0, __p1) __extension__ ({ \ + poly16x4_t __s1 = __p1; \ + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__rev1, 5); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_p8(__p0, __p1) __extension__ ({ \ + poly8x16_t __s1 = __p1; \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 36); \ +}) +#else +#define vst1q_p8(__p0, __p1) __extension__ ({ \ + poly8x16_t __s1 = __p1; \ + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 36); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_p16(__p0, __p1) __extension__ ({ \ + poly16x8_t __s1 = __p1; \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 37); \ +}) +#else +#define vst1q_p16(__p0, __p1) __extension__ ({ \ + poly16x8_t __s1 = __p1; \ + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 37); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s1 = __p1; \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 48); \ +}) +#else +#define vst1q_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 48); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s1 = __p1; \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 50); \ +}) +#else +#define vst1q_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 50); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s1 = __p1; \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 51); \ +}) +#else +#define vst1q_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 51); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s1 = __p1; \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 49); \ +}) +#else +#define vst1q_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 49); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s1 = __p1; \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 32); \ +}) +#else +#define vst1q_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s1 = __p1; \ + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 32); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s1 = __p1; \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 41); \ +}) +#else +#define vst1q_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s1 = __p1; \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 41); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s1 = __p1; \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 40); \ +}) +#else +#define vst1q_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s1 = __p1; \ + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 40); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s1 = __p1; \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 34); \ +}) +#else +#define vst1q_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s1 = __p1; \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 34); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s1 = __p1; \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 35); \ +}) +#else +#define vst1q_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s1 = __p1; \ + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 35); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s1 = __p1; \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 33); \ +}) +#else +#define vst1q_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s1 = __p1; \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 33); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 16); \ +}) +#else +#define vst1_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__rev1, 16); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 18); \ +}) +#else +#define vst1_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__rev1, 18); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 19); \ +}) +#else +#define vst1_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 19); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 17); \ +}) +#else +#define vst1_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__rev1, 17); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 0); \ +}) +#else +#define vst1_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s1 = __p1; \ + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__rev1, 0); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 9); \ +}) +#else +#define vst1_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s1 = __p1; \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__rev1, 9); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 8); \ +}) +#else +#define vst1_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s1 = __p1; \ + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__rev1, 8); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 2); \ +}) +#else +#define vst1_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s1 = __p1; \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__rev1, 2); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 3); \ +}) +#else +#define vst1_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 3); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 1); \ +}) +#else +#define vst1_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s1 = __p1; \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__rev1, 1); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 4); \ +}) +#else +#define vst1_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8_t __s1 = __p1; \ + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__rev1, __p2, 4); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 5); \ +}) +#else +#define vst1_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4_t __s1 = __p1; \ + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__rev1, __p2, 5); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16_t __s1 = __p1; \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 36); \ +}) +#else +#define vst1q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16_t __s1 = __p1; \ + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 36); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8_t __s1 = __p1; \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 37); \ +}) +#else +#define vst1q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8_t __s1 = __p1; \ + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 37); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16_t __s1 = __p1; \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 48); \ +}) +#else +#define vst1q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16_t __s1 = __p1; \ + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 48); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s1 = __p1; \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 50); \ +}) +#else +#define vst1q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 50); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2_t __s1 = __p1; \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 51); \ +}) +#else +#define vst1q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2_t __s1 = __p1; \ + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 51); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s1 = __p1; \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 49); \ +}) +#else +#define vst1q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 49); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16_t __s1 = __p1; \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 32); \ +}) +#else +#define vst1q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16_t __s1 = __p1; \ + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 32); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4_t __s1 = __p1; \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 41); \ +}) +#else +#define vst1q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4_t __s1 = __p1; \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 41); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __s1 = __p1; \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 40); \ +}) +#else +#define vst1q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __s1 = __p1; \ + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 40); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s1 = __p1; \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 34); \ +}) +#else +#define vst1q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s1 = __p1; \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 34); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2_t __s1 = __p1; \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 35); \ +}) +#else +#define vst1q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2_t __s1 = __p1; \ + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 35); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s1 = __p1; \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 33); \ +}) +#else +#define vst1q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s1 = __p1; \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 33); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 16); \ +}) +#else +#define vst1_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8_t __s1 = __p1; \ + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__rev1, __p2, 16); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 18); \ +}) +#else +#define vst1_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__rev1, __p2, 18); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 19); \ +}) +#else +#define vst1_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 19); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 17); \ +}) +#else +#define vst1_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__rev1, __p2, 17); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 0); \ +}) +#else +#define vst1_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8_t __s1 = __p1; \ + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__rev1, __p2, 0); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 9); \ +}) +#else +#define vst1_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2_t __s1 = __p1; \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__rev1, __p2, 9); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 8); \ +}) +#else +#define vst1_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __s1 = __p1; \ + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__rev1, __p2, 8); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 2); \ +}) +#else +#define vst1_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s1 = __p1; \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__rev1, __p2, 2); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 3); \ +}) +#else +#define vst1_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 3); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 1); \ +}) +#else +#define vst1_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s1 = __p1; \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__rev1, __p2, 1); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_p8(__p0, __p1) __extension__ ({ \ + poly8x8x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 4); \ +}) +#else +#define vst2_p8(__p0, __p1) __extension__ ({ \ + poly8x8x2_t __s1 = __p1; \ + poly8x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], 4); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_p16(__p0, __p1) __extension__ ({ \ + poly16x4x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 5); \ +}) +#else +#define vst2_p16(__p0, __p1) __extension__ ({ \ + poly16x4x2_t __s1 = __p1; \ + poly16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], 5); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_p8(__p0, __p1) __extension__ ({ \ + poly8x16x2_t __s1 = __p1; \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 36); \ +}) +#else +#define vst2q_p8(__p0, __p1) __extension__ ({ \ + poly8x16x2_t __s1 = __p1; \ + poly8x16x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 36); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_p16(__p0, __p1) __extension__ ({ \ + poly16x8x2_t __s1 = __p1; \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 37); \ +}) +#else +#define vst2q_p16(__p0, __p1) __extension__ ({ \ + poly16x8x2_t __s1 = __p1; \ + poly16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 37); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_u8(__p0, __p1) __extension__ ({ \ + uint8x16x2_t __s1 = __p1; \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 48); \ +}) +#else +#define vst2q_u8(__p0, __p1) __extension__ ({ \ + uint8x16x2_t __s1 = __p1; \ + uint8x16x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 48); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_u32(__p0, __p1) __extension__ ({ \ + uint32x4x2_t __s1 = __p1; \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 50); \ +}) +#else +#define vst2q_u32(__p0, __p1) __extension__ ({ \ + uint32x4x2_t __s1 = __p1; \ + uint32x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 50); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_u16(__p0, __p1) __extension__ ({ \ + uint16x8x2_t __s1 = __p1; \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 49); \ +}) +#else +#define vst2q_u16(__p0, __p1) __extension__ ({ \ + uint16x8x2_t __s1 = __p1; \ + uint16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 49); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_s8(__p0, __p1) __extension__ ({ \ + int8x16x2_t __s1 = __p1; \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 32); \ +}) +#else +#define vst2q_s8(__p0, __p1) __extension__ ({ \ + int8x16x2_t __s1 = __p1; \ + int8x16x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 32); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_f32(__p0, __p1) __extension__ ({ \ + float32x4x2_t __s1 = __p1; \ + __builtin_neon_vst2q_v(__p0, __s1.val[0], __s1.val[1], 41); \ +}) +#else +#define vst2q_f32(__p0, __p1) __extension__ ({ \ + float32x4x2_t __s1 = __p1; \ + float32x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst2q_v(__p0, __rev1.val[0], __rev1.val[1], 41); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_f16(__p0, __p1) __extension__ ({ \ + float16x8x2_t __s1 = __p1; \ + __builtin_neon_vst2q_v(__p0, __s1.val[0], __s1.val[1], 40); \ +}) +#else +#define vst2q_f16(__p0, __p1) __extension__ ({ \ + float16x8x2_t __s1 = __p1; \ + float16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2q_v(__p0, __rev1.val[0], __rev1.val[1], 40); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_s32(__p0, __p1) __extension__ ({ \ + int32x4x2_t __s1 = __p1; \ + __builtin_neon_vst2q_v(__p0, __s1.val[0], __s1.val[1], 34); \ +}) +#else +#define vst2q_s32(__p0, __p1) __extension__ ({ \ + int32x4x2_t __s1 = __p1; \ + int32x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst2q_v(__p0, __rev1.val[0], __rev1.val[1], 34); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_s16(__p0, __p1) __extension__ ({ \ + int16x8x2_t __s1 = __p1; \ + __builtin_neon_vst2q_v(__p0, __s1.val[0], __s1.val[1], 33); \ +}) +#else +#define vst2q_s16(__p0, __p1) __extension__ ({ \ + int16x8x2_t __s1 = __p1; \ + int16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2q_v(__p0, __rev1.val[0], __rev1.val[1], 33); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_u8(__p0, __p1) __extension__ ({ \ + uint8x8x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 16); \ +}) +#else +#define vst2_u8(__p0, __p1) __extension__ ({ \ + uint8x8x2_t __s1 = __p1; \ + uint8x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], 16); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_u32(__p0, __p1) __extension__ ({ \ + uint32x2x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 18); \ +}) +#else +#define vst2_u32(__p0, __p1) __extension__ ({ \ + uint32x2x2_t __s1 = __p1; \ + uint32x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], 18); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_u64(__p0, __p1) __extension__ ({ \ + uint64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 19); \ +}) +#else +#define vst2_u64(__p0, __p1) __extension__ ({ \ + uint64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 19); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_u16(__p0, __p1) __extension__ ({ \ + uint16x4x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 17); \ +}) +#else +#define vst2_u16(__p0, __p1) __extension__ ({ \ + uint16x4x2_t __s1 = __p1; \ + uint16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], 17); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_s8(__p0, __p1) __extension__ ({ \ + int8x8x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 0); \ +}) +#else +#define vst2_s8(__p0, __p1) __extension__ ({ \ + int8x8x2_t __s1 = __p1; \ + int8x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], 0); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_f32(__p0, __p1) __extension__ ({ \ + float32x2x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, __s1.val[0], __s1.val[1], 9); \ +}) +#else +#define vst2_f32(__p0, __p1) __extension__ ({ \ + float32x2x2_t __s1 = __p1; \ + float32x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst2_v(__p0, __rev1.val[0], __rev1.val[1], 9); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_f16(__p0, __p1) __extension__ ({ \ + float16x4x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, __s1.val[0], __s1.val[1], 8); \ +}) +#else +#define vst2_f16(__p0, __p1) __extension__ ({ \ + float16x4x2_t __s1 = __p1; \ + float16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst2_v(__p0, __rev1.val[0], __rev1.val[1], 8); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_s32(__p0, __p1) __extension__ ({ \ + int32x2x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, __s1.val[0], __s1.val[1], 2); \ +}) +#else +#define vst2_s32(__p0, __p1) __extension__ ({ \ + int32x2x2_t __s1 = __p1; \ + int32x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst2_v(__p0, __rev1.val[0], __rev1.val[1], 2); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_s64(__p0, __p1) __extension__ ({ \ + int64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, __s1.val[0], __s1.val[1], 3); \ +}) +#else +#define vst2_s64(__p0, __p1) __extension__ ({ \ + int64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, __s1.val[0], __s1.val[1], 3); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_s16(__p0, __p1) __extension__ ({ \ + int16x4x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, __s1.val[0], __s1.val[1], 1); \ +}) +#else +#define vst2_s16(__p0, __p1) __extension__ ({ \ + int16x4x2_t __s1 = __p1; \ + int16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst2_v(__p0, __rev1.val[0], __rev1.val[1], 1); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 4); \ +}) +#else +#define vst2_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8x2_t __s1 = __p1; \ + poly8x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], __p2, 4); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 5); \ +}) +#else +#define vst2_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4x2_t __s1 = __p1; \ + poly16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], __p2, 5); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8x2_t __s1 = __p1; \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 37); \ +}) +#else +#define vst2q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8x2_t __s1 = __p1; \ + poly16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 37); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4x2_t __s1 = __p1; \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 50); \ +}) +#else +#define vst2q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4x2_t __s1 = __p1; \ + uint32x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 50); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8x2_t __s1 = __p1; \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 49); \ +}) +#else +#define vst2q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8x2_t __s1 = __p1; \ + uint16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 49); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4x2_t __s1 = __p1; \ + __builtin_neon_vst2q_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 41); \ +}) +#else +#define vst2q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4x2_t __s1 = __p1; \ + float32x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst2q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __p2, 41); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8x2_t __s1 = __p1; \ + __builtin_neon_vst2q_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 40); \ +}) +#else +#define vst2q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8x2_t __s1 = __p1; \ + float16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __p2, 40); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4x2_t __s1 = __p1; \ + __builtin_neon_vst2q_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 34); \ +}) +#else +#define vst2q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4x2_t __s1 = __p1; \ + int32x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst2q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __p2, 34); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8x2_t __s1 = __p1; \ + __builtin_neon_vst2q_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 33); \ +}) +#else +#define vst2q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8x2_t __s1 = __p1; \ + int16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __p2, 33); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 16); \ +}) +#else +#define vst2_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8x2_t __s1 = __p1; \ + uint8x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], __p2, 16); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 18); \ +}) +#else +#define vst2_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2x2_t __s1 = __p1; \ + uint32x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], __p2, 18); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 17); \ +}) +#else +#define vst2_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4x2_t __s1 = __p1; \ + uint16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], __p2, 17); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 0); \ +}) +#else +#define vst2_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8x2_t __s1 = __p1; \ + int8x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], __p2, 0); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 9); \ +}) +#else +#define vst2_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2x2_t __s1 = __p1; \ + float32x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst2_lane_v(__p0, __rev1.val[0], __rev1.val[1], __p2, 9); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 8); \ +}) +#else +#define vst2_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4x2_t __s1 = __p1; \ + float16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst2_lane_v(__p0, __rev1.val[0], __rev1.val[1], __p2, 8); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 2); \ +}) +#else +#define vst2_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2x2_t __s1 = __p1; \ + int32x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst2_lane_v(__p0, __rev1.val[0], __rev1.val[1], __p2, 2); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 1); \ +}) +#else +#define vst2_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4x2_t __s1 = __p1; \ + int16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst2_lane_v(__p0, __rev1.val[0], __rev1.val[1], __p2, 1); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_p8(__p0, __p1) __extension__ ({ \ + poly8x8x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 4); \ +}) +#else +#define vst3_p8(__p0, __p1) __extension__ ({ \ + poly8x8x3_t __s1 = __p1; \ + poly8x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], 4); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_p16(__p0, __p1) __extension__ ({ \ + poly16x4x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 5); \ +}) +#else +#define vst3_p16(__p0, __p1) __extension__ ({ \ + poly16x4x3_t __s1 = __p1; \ + poly16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], 5); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_p8(__p0, __p1) __extension__ ({ \ + poly8x16x3_t __s1 = __p1; \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 36); \ +}) +#else +#define vst3q_p8(__p0, __p1) __extension__ ({ \ + poly8x16x3_t __s1 = __p1; \ + poly8x16x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 36); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_p16(__p0, __p1) __extension__ ({ \ + poly16x8x3_t __s1 = __p1; \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 37); \ +}) +#else +#define vst3q_p16(__p0, __p1) __extension__ ({ \ + poly16x8x3_t __s1 = __p1; \ + poly16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 37); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_u8(__p0, __p1) __extension__ ({ \ + uint8x16x3_t __s1 = __p1; \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 48); \ +}) +#else +#define vst3q_u8(__p0, __p1) __extension__ ({ \ + uint8x16x3_t __s1 = __p1; \ + uint8x16x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 48); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_u32(__p0, __p1) __extension__ ({ \ + uint32x4x3_t __s1 = __p1; \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 50); \ +}) +#else +#define vst3q_u32(__p0, __p1) __extension__ ({ \ + uint32x4x3_t __s1 = __p1; \ + uint32x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 50); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_u16(__p0, __p1) __extension__ ({ \ + uint16x8x3_t __s1 = __p1; \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 49); \ +}) +#else +#define vst3q_u16(__p0, __p1) __extension__ ({ \ + uint16x8x3_t __s1 = __p1; \ + uint16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 49); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_s8(__p0, __p1) __extension__ ({ \ + int8x16x3_t __s1 = __p1; \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 32); \ +}) +#else +#define vst3q_s8(__p0, __p1) __extension__ ({ \ + int8x16x3_t __s1 = __p1; \ + int8x16x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 32); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_f32(__p0, __p1) __extension__ ({ \ + float32x4x3_t __s1 = __p1; \ + __builtin_neon_vst3q_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 41); \ +}) +#else +#define vst3q_f32(__p0, __p1) __extension__ ({ \ + float32x4x3_t __s1 = __p1; \ + float32x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst3q_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 41); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_f16(__p0, __p1) __extension__ ({ \ + float16x8x3_t __s1 = __p1; \ + __builtin_neon_vst3q_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 40); \ +}) +#else +#define vst3q_f16(__p0, __p1) __extension__ ({ \ + float16x8x3_t __s1 = __p1; \ + float16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3q_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 40); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_s32(__p0, __p1) __extension__ ({ \ + int32x4x3_t __s1 = __p1; \ + __builtin_neon_vst3q_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 34); \ +}) +#else +#define vst3q_s32(__p0, __p1) __extension__ ({ \ + int32x4x3_t __s1 = __p1; \ + int32x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst3q_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 34); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_s16(__p0, __p1) __extension__ ({ \ + int16x8x3_t __s1 = __p1; \ + __builtin_neon_vst3q_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 33); \ +}) +#else +#define vst3q_s16(__p0, __p1) __extension__ ({ \ + int16x8x3_t __s1 = __p1; \ + int16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3q_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 33); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_u8(__p0, __p1) __extension__ ({ \ + uint8x8x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 16); \ +}) +#else +#define vst3_u8(__p0, __p1) __extension__ ({ \ + uint8x8x3_t __s1 = __p1; \ + uint8x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], 16); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_u32(__p0, __p1) __extension__ ({ \ + uint32x2x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 18); \ +}) +#else +#define vst3_u32(__p0, __p1) __extension__ ({ \ + uint32x2x3_t __s1 = __p1; \ + uint32x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], 18); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_u64(__p0, __p1) __extension__ ({ \ + uint64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 19); \ +}) +#else +#define vst3_u64(__p0, __p1) __extension__ ({ \ + uint64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 19); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_u16(__p0, __p1) __extension__ ({ \ + uint16x4x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 17); \ +}) +#else +#define vst3_u16(__p0, __p1) __extension__ ({ \ + uint16x4x3_t __s1 = __p1; \ + uint16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], 17); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_s8(__p0, __p1) __extension__ ({ \ + int8x8x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 0); \ +}) +#else +#define vst3_s8(__p0, __p1) __extension__ ({ \ + int8x8x3_t __s1 = __p1; \ + int8x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], 0); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_f32(__p0, __p1) __extension__ ({ \ + float32x2x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 9); \ +}) +#else +#define vst3_f32(__p0, __p1) __extension__ ({ \ + float32x2x3_t __s1 = __p1; \ + float32x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 9); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_f16(__p0, __p1) __extension__ ({ \ + float16x4x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 8); \ +}) +#else +#define vst3_f16(__p0, __p1) __extension__ ({ \ + float16x4x3_t __s1 = __p1; \ + float16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 8); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_s32(__p0, __p1) __extension__ ({ \ + int32x2x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 2); \ +}) +#else +#define vst3_s32(__p0, __p1) __extension__ ({ \ + int32x2x3_t __s1 = __p1; \ + int32x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 2); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_s64(__p0, __p1) __extension__ ({ \ + int64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 3); \ +}) +#else +#define vst3_s64(__p0, __p1) __extension__ ({ \ + int64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 3); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_s16(__p0, __p1) __extension__ ({ \ + int16x4x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 1); \ +}) +#else +#define vst3_s16(__p0, __p1) __extension__ ({ \ + int16x4x3_t __s1 = __p1; \ + int16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 1); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 4); \ +}) +#else +#define vst3_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8x3_t __s1 = __p1; \ + poly8x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], __p2, 4); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 5); \ +}) +#else +#define vst3_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4x3_t __s1 = __p1; \ + poly16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], __p2, 5); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8x3_t __s1 = __p1; \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 37); \ +}) +#else +#define vst3q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8x3_t __s1 = __p1; \ + poly16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 37); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4x3_t __s1 = __p1; \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 50); \ +}) +#else +#define vst3q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4x3_t __s1 = __p1; \ + uint32x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 50); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8x3_t __s1 = __p1; \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 49); \ +}) +#else +#define vst3q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8x3_t __s1 = __p1; \ + uint16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 49); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4x3_t __s1 = __p1; \ + __builtin_neon_vst3q_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 41); \ +}) +#else +#define vst3q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4x3_t __s1 = __p1; \ + float32x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst3q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 41); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8x3_t __s1 = __p1; \ + __builtin_neon_vst3q_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 40); \ +}) +#else +#define vst3q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8x3_t __s1 = __p1; \ + float16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 40); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4x3_t __s1 = __p1; \ + __builtin_neon_vst3q_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 34); \ +}) +#else +#define vst3q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4x3_t __s1 = __p1; \ + int32x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst3q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 34); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8x3_t __s1 = __p1; \ + __builtin_neon_vst3q_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 33); \ +}) +#else +#define vst3q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8x3_t __s1 = __p1; \ + int16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 33); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 16); \ +}) +#else +#define vst3_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8x3_t __s1 = __p1; \ + uint8x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], __p2, 16); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 18); \ +}) +#else +#define vst3_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2x3_t __s1 = __p1; \ + uint32x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], __p2, 18); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 17); \ +}) +#else +#define vst3_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4x3_t __s1 = __p1; \ + uint16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], __p2, 17); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 0); \ +}) +#else +#define vst3_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8x3_t __s1 = __p1; \ + int8x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], __p2, 0); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 9); \ +}) +#else +#define vst3_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2x3_t __s1 = __p1; \ + float32x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst3_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 9); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 8); \ +}) +#else +#define vst3_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4x3_t __s1 = __p1; \ + float16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst3_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 8); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 2); \ +}) +#else +#define vst3_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2x3_t __s1 = __p1; \ + int32x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst3_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 2); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 1); \ +}) +#else +#define vst3_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4x3_t __s1 = __p1; \ + int16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst3_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 1); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_p8(__p0, __p1) __extension__ ({ \ + poly8x8x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 4); \ +}) +#else +#define vst4_p8(__p0, __p1) __extension__ ({ \ + poly8x8x4_t __s1 = __p1; \ + poly8x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], 4); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_p16(__p0, __p1) __extension__ ({ \ + poly16x4x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 5); \ +}) +#else +#define vst4_p16(__p0, __p1) __extension__ ({ \ + poly16x4x4_t __s1 = __p1; \ + poly16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], 5); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_p8(__p0, __p1) __extension__ ({ \ + poly8x16x4_t __s1 = __p1; \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 36); \ +}) +#else +#define vst4q_p8(__p0, __p1) __extension__ ({ \ + poly8x16x4_t __s1 = __p1; \ + poly8x16x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 36); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_p16(__p0, __p1) __extension__ ({ \ + poly16x8x4_t __s1 = __p1; \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 37); \ +}) +#else +#define vst4q_p16(__p0, __p1) __extension__ ({ \ + poly16x8x4_t __s1 = __p1; \ + poly16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 37); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_u8(__p0, __p1) __extension__ ({ \ + uint8x16x4_t __s1 = __p1; \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 48); \ +}) +#else +#define vst4q_u8(__p0, __p1) __extension__ ({ \ + uint8x16x4_t __s1 = __p1; \ + uint8x16x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 48); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_u32(__p0, __p1) __extension__ ({ \ + uint32x4x4_t __s1 = __p1; \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 50); \ +}) +#else +#define vst4q_u32(__p0, __p1) __extension__ ({ \ + uint32x4x4_t __s1 = __p1; \ + uint32x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 50); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_u16(__p0, __p1) __extension__ ({ \ + uint16x8x4_t __s1 = __p1; \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 49); \ +}) +#else +#define vst4q_u16(__p0, __p1) __extension__ ({ \ + uint16x8x4_t __s1 = __p1; \ + uint16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 49); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_s8(__p0, __p1) __extension__ ({ \ + int8x16x4_t __s1 = __p1; \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 32); \ +}) +#else +#define vst4q_s8(__p0, __p1) __extension__ ({ \ + int8x16x4_t __s1 = __p1; \ + int8x16x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 32); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_f32(__p0, __p1) __extension__ ({ \ + float32x4x4_t __s1 = __p1; \ + __builtin_neon_vst4q_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 41); \ +}) +#else +#define vst4q_f32(__p0, __p1) __extension__ ({ \ + float32x4x4_t __s1 = __p1; \ + float32x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst4q_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 41); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_f16(__p0, __p1) __extension__ ({ \ + float16x8x4_t __s1 = __p1; \ + __builtin_neon_vst4q_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 40); \ +}) +#else +#define vst4q_f16(__p0, __p1) __extension__ ({ \ + float16x8x4_t __s1 = __p1; \ + float16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4q_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 40); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_s32(__p0, __p1) __extension__ ({ \ + int32x4x4_t __s1 = __p1; \ + __builtin_neon_vst4q_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 34); \ +}) +#else +#define vst4q_s32(__p0, __p1) __extension__ ({ \ + int32x4x4_t __s1 = __p1; \ + int32x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst4q_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 34); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_s16(__p0, __p1) __extension__ ({ \ + int16x8x4_t __s1 = __p1; \ + __builtin_neon_vst4q_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 33); \ +}) +#else +#define vst4q_s16(__p0, __p1) __extension__ ({ \ + int16x8x4_t __s1 = __p1; \ + int16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4q_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 33); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_u8(__p0, __p1) __extension__ ({ \ + uint8x8x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 16); \ +}) +#else +#define vst4_u8(__p0, __p1) __extension__ ({ \ + uint8x8x4_t __s1 = __p1; \ + uint8x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], 16); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_u32(__p0, __p1) __extension__ ({ \ + uint32x2x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 18); \ +}) +#else +#define vst4_u32(__p0, __p1) __extension__ ({ \ + uint32x2x4_t __s1 = __p1; \ + uint32x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], 18); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_u64(__p0, __p1) __extension__ ({ \ + uint64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 19); \ +}) +#else +#define vst4_u64(__p0, __p1) __extension__ ({ \ + uint64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 19); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_u16(__p0, __p1) __extension__ ({ \ + uint16x4x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 17); \ +}) +#else +#define vst4_u16(__p0, __p1) __extension__ ({ \ + uint16x4x4_t __s1 = __p1; \ + uint16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], 17); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_s8(__p0, __p1) __extension__ ({ \ + int8x8x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 0); \ +}) +#else +#define vst4_s8(__p0, __p1) __extension__ ({ \ + int8x8x4_t __s1 = __p1; \ + int8x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], 0); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_f32(__p0, __p1) __extension__ ({ \ + float32x2x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 9); \ +}) +#else +#define vst4_f32(__p0, __p1) __extension__ ({ \ + float32x2x4_t __s1 = __p1; \ + float32x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 9); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_f16(__p0, __p1) __extension__ ({ \ + float16x4x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 8); \ +}) +#else +#define vst4_f16(__p0, __p1) __extension__ ({ \ + float16x4x4_t __s1 = __p1; \ + float16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 8); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_s32(__p0, __p1) __extension__ ({ \ + int32x2x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 2); \ +}) +#else +#define vst4_s32(__p0, __p1) __extension__ ({ \ + int32x2x4_t __s1 = __p1; \ + int32x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 2); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_s64(__p0, __p1) __extension__ ({ \ + int64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 3); \ +}) +#else +#define vst4_s64(__p0, __p1) __extension__ ({ \ + int64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 3); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_s16(__p0, __p1) __extension__ ({ \ + int16x4x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 1); \ +}) +#else +#define vst4_s16(__p0, __p1) __extension__ ({ \ + int16x4x4_t __s1 = __p1; \ + int16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 1); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 4); \ +}) +#else +#define vst4_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x8x4_t __s1 = __p1; \ + poly8x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], __p2, 4); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 5); \ +}) +#else +#define vst4_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x4x4_t __s1 = __p1; \ + poly16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], __p2, 5); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8x4_t __s1 = __p1; \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 37); \ +}) +#else +#define vst4q_lane_p16(__p0, __p1, __p2) __extension__ ({ \ + poly16x8x4_t __s1 = __p1; \ + poly16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 37); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4x4_t __s1 = __p1; \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 50); \ +}) +#else +#define vst4q_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4x4_t __s1 = __p1; \ + uint32x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 50); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8x4_t __s1 = __p1; \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 49); \ +}) +#else +#define vst4q_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8x4_t __s1 = __p1; \ + uint16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 49); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4x4_t __s1 = __p1; \ + __builtin_neon_vst4q_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 41); \ +}) +#else +#define vst4q_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4x4_t __s1 = __p1; \ + float32x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst4q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 41); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8x4_t __s1 = __p1; \ + __builtin_neon_vst4q_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 40); \ +}) +#else +#define vst4q_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8x4_t __s1 = __p1; \ + float16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 40); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4x4_t __s1 = __p1; \ + __builtin_neon_vst4q_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 34); \ +}) +#else +#define vst4q_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4x4_t __s1 = __p1; \ + int32x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst4q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 34); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8x4_t __s1 = __p1; \ + __builtin_neon_vst4q_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 33); \ +}) +#else +#define vst4q_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8x4_t __s1 = __p1; \ + int16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 33); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 16); \ +}) +#else +#define vst4_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x8x4_t __s1 = __p1; \ + uint8x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], __p2, 16); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 18); \ +}) +#else +#define vst4_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2x4_t __s1 = __p1; \ + uint32x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], __p2, 18); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 17); \ +}) +#else +#define vst4_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4x4_t __s1 = __p1; \ + uint16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], __p2, 17); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 0); \ +}) +#else +#define vst4_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x8x4_t __s1 = __p1; \ + int8x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], __p2, 0); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 9); \ +}) +#else +#define vst4_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2x4_t __s1 = __p1; \ + float32x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst4_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 9); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 8); \ +}) +#else +#define vst4_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4x4_t __s1 = __p1; \ + float16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst4_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 8); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 2); \ +}) +#else +#define vst4_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2x4_t __s1 = __p1; \ + int32x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst4_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 2); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 1); \ +}) +#else +#define vst4_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4x4_t __s1 = __p1; \ + int16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst4_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 1); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vsubq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai uint8x16_t vsubq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsubq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai uint32x4_t vsubq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vsubq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai uint64x2_t vsubq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vsubq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai uint16x8_t vsubq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vsubq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai int8x16_t vsubq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vsubq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai float32x4_t vsubq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vsubq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai int32x4_t vsubq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vsubq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai int64x2_t vsubq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vsubq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai int16x8_t vsubq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vsub_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai uint8x8_t vsub_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vsub_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai uint32x2_t vsub_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vsub_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai uint64x1_t vsub_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vsub_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai uint16x4_t vsub_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vsub_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai int8x8_t vsub_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vsub_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai float32x2_t vsub_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vsub_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai int32x2_t vsub_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vsub_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai int64x1_t vsub_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vsub_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai int16x4_t vsub_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vsubhn_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vsubhn_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vsubhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint16x4_t __noswap_vsubhn_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 17); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vsubhn_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vsubhn_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vsubhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint32x2_t __noswap_vsubhn_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 18); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vsubhn_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vsubhn_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vsubhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint8x8_t __noswap_vsubhn_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 16); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vsubhn_s32(int32x4_t __p0, int32x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vsubhn_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vsubhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int16x4_t __noswap_vsubhn_s32(int32x4_t __p0, int32x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vsubhn_s64(int64x2_t __p0, int64x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vsubhn_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vsubhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int32x2_t __noswap_vsubhn_s64(int64x2_t __p0, int64x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vsubhn_s16(int16x8_t __p0, int16x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vsubhn_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vsubhn_v((int8x16_t)__rev0, (int8x16_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int8x8_t __noswap_vsubhn_s16(int16x8_t __p0, int16x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vsubhn_v((int8x16_t)__p0, (int8x16_t)__p1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vsubl_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint16x8_t __ret; + __ret = vmovl_u8(__p0) - vmovl_u8(__p1); + return __ret; +} +#else +__ai uint16x8_t vsubl_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vmovl_u8(__rev0) - __noswap_vmovl_u8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vsubl_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint64x2_t __ret; + __ret = vmovl_u32(__p0) - vmovl_u32(__p1); + return __ret; +} +#else +__ai uint64x2_t vsubl_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __noswap_vmovl_u32(__rev0) - __noswap_vmovl_u32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsubl_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint32x4_t __ret; + __ret = vmovl_u16(__p0) - vmovl_u16(__p1); + return __ret; +} +#else +__ai uint32x4_t vsubl_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vmovl_u16(__rev0) - __noswap_vmovl_u16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vsubl_s8(int8x8_t __p0, int8x8_t __p1) { + int16x8_t __ret; + __ret = vmovl_s8(__p0) - vmovl_s8(__p1); + return __ret; +} +#else +__ai int16x8_t vsubl_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vmovl_s8(__rev0) - __noswap_vmovl_s8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vsubl_s32(int32x2_t __p0, int32x2_t __p1) { + int64x2_t __ret; + __ret = vmovl_s32(__p0) - vmovl_s32(__p1); + return __ret; +} +#else +__ai int64x2_t vsubl_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __noswap_vmovl_s32(__rev0) - __noswap_vmovl_s32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vsubl_s16(int16x4_t __p0, int16x4_t __p1) { + int32x4_t __ret; + __ret = vmovl_s16(__p0) - vmovl_s16(__p1); + return __ret; +} +#else +__ai int32x4_t vsubl_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vmovl_s16(__rev0) - __noswap_vmovl_s16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vsubw_u8(uint16x8_t __p0, uint8x8_t __p1) { + uint16x8_t __ret; + __ret = __p0 - vmovl_u8(__p1); + return __ret; +} +#else +__ai uint16x8_t vsubw_u8(uint16x8_t __p0, uint8x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 - __noswap_vmovl_u8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vsubw_u32(uint64x2_t __p0, uint32x2_t __p1) { + uint64x2_t __ret; + __ret = __p0 - vmovl_u32(__p1); + return __ret; +} +#else +__ai uint64x2_t vsubw_u32(uint64x2_t __p0, uint32x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __rev0 - __noswap_vmovl_u32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsubw_u16(uint32x4_t __p0, uint16x4_t __p1) { + uint32x4_t __ret; + __ret = __p0 - vmovl_u16(__p1); + return __ret; +} +#else +__ai uint32x4_t vsubw_u16(uint32x4_t __p0, uint16x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 - __noswap_vmovl_u16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vsubw_s8(int16x8_t __p0, int8x8_t __p1) { + int16x8_t __ret; + __ret = __p0 - vmovl_s8(__p1); + return __ret; +} +#else +__ai int16x8_t vsubw_s8(int16x8_t __p0, int8x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 - __noswap_vmovl_s8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vsubw_s32(int64x2_t __p0, int32x2_t __p1) { + int64x2_t __ret; + __ret = __p0 - vmovl_s32(__p1); + return __ret; +} +#else +__ai int64x2_t vsubw_s32(int64x2_t __p0, int32x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __rev0 - __noswap_vmovl_s32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vsubw_s16(int32x4_t __p0, int16x4_t __p1) { + int32x4_t __ret; + __ret = __p0 - vmovl_s16(__p1); + return __ret; +} +#else +__ai int32x4_t vsubw_s16(int32x4_t __p0, int16x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 - __noswap_vmovl_s16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vtbl1_p8(poly8x8_t __p0, uint8x8_t __p1) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbl1_v((int8x8_t)__p0, (int8x8_t)__p1, 4); + return __ret; +} +#else +__ai poly8x8_t vtbl1_p8(poly8x8_t __p0, uint8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbl1_v((int8x8_t)__rev0, (int8x8_t)__rev1, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vtbl1_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbl1_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vtbl1_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbl1_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vtbl1_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbl1_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vtbl1_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbl1_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vtbl2_p8(poly8x8x2_t __p0, uint8x8_t __p1) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbl2_v((int8x8_t)__p0.val[0], (int8x8_t)__p0.val[1], (int8x8_t)__p1, 4); + return __ret; +} +#else +__ai poly8x8_t vtbl2_p8(poly8x8x2_t __p0, uint8x8_t __p1) { + poly8x8x2_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbl2_v((int8x8_t)__rev0.val[0], (int8x8_t)__rev0.val[1], (int8x8_t)__rev1, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vtbl2_u8(uint8x8x2_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbl2_v((int8x8_t)__p0.val[0], (int8x8_t)__p0.val[1], (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vtbl2_u8(uint8x8x2_t __p0, uint8x8_t __p1) { + uint8x8x2_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbl2_v((int8x8_t)__rev0.val[0], (int8x8_t)__rev0.val[1], (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vtbl2_s8(int8x8x2_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbl2_v((int8x8_t)__p0.val[0], (int8x8_t)__p0.val[1], (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vtbl2_s8(int8x8x2_t __p0, int8x8_t __p1) { + int8x8x2_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbl2_v((int8x8_t)__rev0.val[0], (int8x8_t)__rev0.val[1], (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vtbl3_p8(poly8x8x3_t __p0, uint8x8_t __p1) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbl3_v((int8x8_t)__p0.val[0], (int8x8_t)__p0.val[1], (int8x8_t)__p0.val[2], (int8x8_t)__p1, 4); + return __ret; +} +#else +__ai poly8x8_t vtbl3_p8(poly8x8x3_t __p0, uint8x8_t __p1) { + poly8x8x3_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbl3_v((int8x8_t)__rev0.val[0], (int8x8_t)__rev0.val[1], (int8x8_t)__rev0.val[2], (int8x8_t)__rev1, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vtbl3_u8(uint8x8x3_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbl3_v((int8x8_t)__p0.val[0], (int8x8_t)__p0.val[1], (int8x8_t)__p0.val[2], (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vtbl3_u8(uint8x8x3_t __p0, uint8x8_t __p1) { + uint8x8x3_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbl3_v((int8x8_t)__rev0.val[0], (int8x8_t)__rev0.val[1], (int8x8_t)__rev0.val[2], (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vtbl3_s8(int8x8x3_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbl3_v((int8x8_t)__p0.val[0], (int8x8_t)__p0.val[1], (int8x8_t)__p0.val[2], (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vtbl3_s8(int8x8x3_t __p0, int8x8_t __p1) { + int8x8x3_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbl3_v((int8x8_t)__rev0.val[0], (int8x8_t)__rev0.val[1], (int8x8_t)__rev0.val[2], (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vtbl4_p8(poly8x8x4_t __p0, uint8x8_t __p1) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbl4_v((int8x8_t)__p0.val[0], (int8x8_t)__p0.val[1], (int8x8_t)__p0.val[2], (int8x8_t)__p0.val[3], (int8x8_t)__p1, 4); + return __ret; +} +#else +__ai poly8x8_t vtbl4_p8(poly8x8x4_t __p0, uint8x8_t __p1) { + poly8x8x4_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[3] = __builtin_shufflevector(__p0.val[3], __p0.val[3], 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbl4_v((int8x8_t)__rev0.val[0], (int8x8_t)__rev0.val[1], (int8x8_t)__rev0.val[2], (int8x8_t)__rev0.val[3], (int8x8_t)__rev1, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vtbl4_u8(uint8x8x4_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbl4_v((int8x8_t)__p0.val[0], (int8x8_t)__p0.val[1], (int8x8_t)__p0.val[2], (int8x8_t)__p0.val[3], (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vtbl4_u8(uint8x8x4_t __p0, uint8x8_t __p1) { + uint8x8x4_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[3] = __builtin_shufflevector(__p0.val[3], __p0.val[3], 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbl4_v((int8x8_t)__rev0.val[0], (int8x8_t)__rev0.val[1], (int8x8_t)__rev0.val[2], (int8x8_t)__rev0.val[3], (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vtbl4_s8(int8x8x4_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbl4_v((int8x8_t)__p0.val[0], (int8x8_t)__p0.val[1], (int8x8_t)__p0.val[2], (int8x8_t)__p0.val[3], (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vtbl4_s8(int8x8x4_t __p0, int8x8_t __p1) { + int8x8x4_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[3] = __builtin_shufflevector(__p0.val[3], __p0.val[3], 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbl4_v((int8x8_t)__rev0.val[0], (int8x8_t)__rev0.val[1], (int8x8_t)__rev0.val[2], (int8x8_t)__rev0.val[3], (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vtbx1_p8(poly8x8_t __p0, poly8x8_t __p1, uint8x8_t __p2) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbx1_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 4); + return __ret; +} +#else +__ai poly8x8_t vtbx1_p8(poly8x8_t __p0, poly8x8_t __p1, uint8x8_t __p2) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbx1_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vtbx1_u8(uint8x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbx1_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 16); + return __ret; +} +#else +__ai uint8x8_t vtbx1_u8(uint8x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbx1_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vtbx1_s8(int8x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbx1_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 0); + return __ret; +} +#else +__ai int8x8_t vtbx1_s8(int8x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbx1_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vtbx2_p8(poly8x8_t __p0, poly8x8x2_t __p1, uint8x8_t __p2) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbx2_v((int8x8_t)__p0, (int8x8_t)__p1.val[0], (int8x8_t)__p1.val[1], (int8x8_t)__p2, 4); + return __ret; +} +#else +__ai poly8x8_t vtbx2_p8(poly8x8_t __p0, poly8x8x2_t __p1, uint8x8_t __p2) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8x2_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbx2_v((int8x8_t)__rev0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev2, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vtbx2_u8(uint8x8_t __p0, uint8x8x2_t __p1, uint8x8_t __p2) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbx2_v((int8x8_t)__p0, (int8x8_t)__p1.val[0], (int8x8_t)__p1.val[1], (int8x8_t)__p2, 16); + return __ret; +} +#else +__ai uint8x8_t vtbx2_u8(uint8x8_t __p0, uint8x8x2_t __p1, uint8x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8x2_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbx2_v((int8x8_t)__rev0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev2, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vtbx2_s8(int8x8_t __p0, int8x8x2_t __p1, int8x8_t __p2) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbx2_v((int8x8_t)__p0, (int8x8_t)__p1.val[0], (int8x8_t)__p1.val[1], (int8x8_t)__p2, 0); + return __ret; +} +#else +__ai int8x8_t vtbx2_s8(int8x8_t __p0, int8x8x2_t __p1, int8x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8x2_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbx2_v((int8x8_t)__rev0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev2, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vtbx3_p8(poly8x8_t __p0, poly8x8x3_t __p1, uint8x8_t __p2) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbx3_v((int8x8_t)__p0, (int8x8_t)__p1.val[0], (int8x8_t)__p1.val[1], (int8x8_t)__p1.val[2], (int8x8_t)__p2, 4); + return __ret; +} +#else +__ai poly8x8_t vtbx3_p8(poly8x8_t __p0, poly8x8x3_t __p1, uint8x8_t __p2) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8x3_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbx3_v((int8x8_t)__rev0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev2, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vtbx3_u8(uint8x8_t __p0, uint8x8x3_t __p1, uint8x8_t __p2) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbx3_v((int8x8_t)__p0, (int8x8_t)__p1.val[0], (int8x8_t)__p1.val[1], (int8x8_t)__p1.val[2], (int8x8_t)__p2, 16); + return __ret; +} +#else +__ai uint8x8_t vtbx3_u8(uint8x8_t __p0, uint8x8x3_t __p1, uint8x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8x3_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbx3_v((int8x8_t)__rev0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev2, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vtbx3_s8(int8x8_t __p0, int8x8x3_t __p1, int8x8_t __p2) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbx3_v((int8x8_t)__p0, (int8x8_t)__p1.val[0], (int8x8_t)__p1.val[1], (int8x8_t)__p1.val[2], (int8x8_t)__p2, 0); + return __ret; +} +#else +__ai int8x8_t vtbx3_s8(int8x8_t __p0, int8x8x3_t __p1, int8x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8x3_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbx3_v((int8x8_t)__rev0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev2, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vtbx4_p8(poly8x8_t __p0, poly8x8x4_t __p1, uint8x8_t __p2) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbx4_v((int8x8_t)__p0, (int8x8_t)__p1.val[0], (int8x8_t)__p1.val[1], (int8x8_t)__p1.val[2], (int8x8_t)__p1.val[3], (int8x8_t)__p2, 4); + return __ret; +} +#else +__ai poly8x8_t vtbx4_p8(poly8x8_t __p0, poly8x8x4_t __p1, uint8x8_t __p2) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8x4_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[3] = __builtin_shufflevector(__p1.val[3], __p1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vtbx4_v((int8x8_t)__rev0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], (int8x8_t)__rev2, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vtbx4_u8(uint8x8_t __p0, uint8x8x4_t __p1, uint8x8_t __p2) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbx4_v((int8x8_t)__p0, (int8x8_t)__p1.val[0], (int8x8_t)__p1.val[1], (int8x8_t)__p1.val[2], (int8x8_t)__p1.val[3], (int8x8_t)__p2, 16); + return __ret; +} +#else +__ai uint8x8_t vtbx4_u8(uint8x8_t __p0, uint8x8x4_t __p1, uint8x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8x4_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[3] = __builtin_shufflevector(__p1.val[3], __p1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtbx4_v((int8x8_t)__rev0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], (int8x8_t)__rev2, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vtbx4_s8(int8x8_t __p0, int8x8x4_t __p1, int8x8_t __p2) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbx4_v((int8x8_t)__p0, (int8x8_t)__p1.val[0], (int8x8_t)__p1.val[1], (int8x8_t)__p1.val[2], (int8x8_t)__p1.val[3], (int8x8_t)__p2, 0); + return __ret; +} +#else +__ai int8x8_t vtbx4_s8(int8x8_t __p0, int8x8x4_t __p1, int8x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8x4_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[3] = __builtin_shufflevector(__p1.val[3], __p1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vtbx4_v((int8x8_t)__rev0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], (int8x8_t)__rev2, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8x2_t vtrn_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 4); + return __ret; +} +#else +__ai poly8x8x2_t vtrn_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 4); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4x2_t vtrn_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 5); + return __ret; +} +#else +__ai poly16x4x2_t vtrn_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + poly16x4x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 5); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16x2_t vtrnq_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 36); + return __ret; +} +#else +__ai poly8x16x2_t vtrnq_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 36); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8x2_t vtrnq_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 37); + return __ret; +} +#else +__ai poly16x8x2_t vtrnq_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 37); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16x2_t vtrnq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16x2_t vtrnq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 48); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4x2_t vtrnq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4x2_t vtrnq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 50); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8x2_t vtrnq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8x2_t vtrnq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 49); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16x2_t vtrnq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16x2_t vtrnq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 32); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4x2_t vtrnq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4x2_t vtrnq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 41); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4x2_t vtrnq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4x2_t vtrnq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 34); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8x2_t vtrnq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8x2_t vtrnq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 33); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8x2_t vtrn_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8x2_t vtrn_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 16); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2x2_t vtrn_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2x2_t vtrn_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 18); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4x2_t vtrn_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4x2_t vtrn_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 17); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8x2_t vtrn_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8x2_t vtrn_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 0); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2x2_t vtrn_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2x2_t vtrn_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 9); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2x2_t vtrn_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2x2_t vtrn_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 2); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4x2_t vtrn_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4x2_t vtrn_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 1); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vtst_p8(poly8x8_t __p0, poly8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtst_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vtst_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtst_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vtst_p16(poly16x4_t __p0, poly16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vtst_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vtst_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vtst_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vtstq_p8(poly8x16_t __p0, poly8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vtstq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vtstq_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vtstq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vtstq_p16(poly16x8_t __p0, poly16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vtstq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vtstq_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vtstq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vtstq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vtstq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vtstq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vtstq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vtstq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vtstq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vtstq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vtstq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vtstq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vtstq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vtstq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vtstq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vtstq_s8(int8x16_t __p0, int8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vtstq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vtstq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vtstq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vtstq_s32(int32x4_t __p0, int32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vtstq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vtstq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vtstq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vtstq_s16(int16x8_t __p0, int16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vtstq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vtstq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vtstq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vtst_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtst_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vtst_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtst_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vtst_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vtst_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vtst_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vtst_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vtst_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vtst_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vtst_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vtst_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vtst_s8(int8x8_t __p0, int8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtst_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vtst_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vtst_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vtst_s32(int32x2_t __p0, int32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vtst_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vtst_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vtst_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vtst_s16(int16x4_t __p0, int16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vtst_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vtst_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vtst_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8x2_t vuzp_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 4); + return __ret; +} +#else +__ai poly8x8x2_t vuzp_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 4); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4x2_t vuzp_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 5); + return __ret; +} +#else +__ai poly16x4x2_t vuzp_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + poly16x4x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 5); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16x2_t vuzpq_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 36); + return __ret; +} +#else +__ai poly8x16x2_t vuzpq_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 36); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8x2_t vuzpq_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 37); + return __ret; +} +#else +__ai poly16x8x2_t vuzpq_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 37); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16x2_t vuzpq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16x2_t vuzpq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 48); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4x2_t vuzpq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4x2_t vuzpq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 50); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8x2_t vuzpq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8x2_t vuzpq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 49); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16x2_t vuzpq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16x2_t vuzpq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 32); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4x2_t vuzpq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4x2_t vuzpq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 41); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4x2_t vuzpq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4x2_t vuzpq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 34); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8x2_t vuzpq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8x2_t vuzpq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 33); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8x2_t vuzp_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8x2_t vuzp_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 16); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2x2_t vuzp_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2x2_t vuzp_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 18); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4x2_t vuzp_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4x2_t vuzp_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 17); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8x2_t vuzp_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8x2_t vuzp_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 0); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2x2_t vuzp_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2x2_t vuzp_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 9); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2x2_t vuzp_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2x2_t vuzp_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 2); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4x2_t vuzp_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4x2_t vuzp_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 1); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8x2_t vzip_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 4); + return __ret; +} +#else +__ai poly8x8x2_t vzip_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 4); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4x2_t vzip_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 5); + return __ret; +} +#else +__ai poly16x4x2_t vzip_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + poly16x4x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 5); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16x2_t vzipq_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 36); + return __ret; +} +#else +__ai poly8x16x2_t vzipq_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 36); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8x2_t vzipq_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 37); + return __ret; +} +#else +__ai poly16x8x2_t vzipq_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 37); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16x2_t vzipq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16x2_t vzipq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 48); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4x2_t vzipq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4x2_t vzipq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 50); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8x2_t vzipq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8x2_t vzipq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 49); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16x2_t vzipq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16x2_t vzipq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 32); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4x2_t vzipq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4x2_t vzipq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 41); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4x2_t vzipq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4x2_t vzipq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 34); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8x2_t vzipq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8x2_t vzipq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 33); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8x2_t vzip_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8x2_t vzip_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 16); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2x2_t vzip_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2x2_t vzip_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 18); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4x2_t vzip_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4x2_t vzip_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 17); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8x2_t vzip_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8x2_t vzip_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 0); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2x2_t vzip_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2x2_t vzip_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 9); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2x2_t vzip_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2x2_t vzip_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 2); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4x2_t vzip_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4x2_t vzip_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 1); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#if !defined(__aarch64__) +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_p16(poly16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_p16(poly16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_u8(uint8x8_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_u8(uint8x8_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_u32(uint32x2_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_u32(uint32x2_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_u64(uint64x1_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_u64(uint64x1_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_u16(uint16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_u16(uint16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_s8(int8x8_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_s8(int8x8_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_f32(float32x2_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_f32(float32x2_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_f16(float16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_f16(float16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_s32(int32x2_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_s32(int32x2_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_s64(int64x1_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_s64(int64x1_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_s16(int16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_s16(int16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_p8(poly8x8_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_p8(poly8x8_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_u8(uint8x8_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_u8(uint8x8_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_u32(uint32x2_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_u32(uint32x2_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_u64(uint64x1_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_u64(uint64x1_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_u16(uint16x4_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_u16(uint16x4_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_s8(int8x8_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_s8(int8x8_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_f32(float32x2_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_f32(float32x2_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_f16(float16x4_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_f16(float16x4_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_s32(int32x2_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_s32(int32x2_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_s64(int64x1_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_s64(int64x1_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_s16(int16x4_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_s16(int16x4_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_p16(poly16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_p16(poly16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_u8(uint8x16_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_u8(uint8x16_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_u32(uint32x4_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_u32(uint32x4_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_u64(uint64x2_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_u64(uint64x2_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_u16(uint16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_u16(uint16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_s8(int8x16_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_s8(int8x16_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_f32(float32x4_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_f32(float32x4_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_f16(float16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_f16(float16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_s32(int32x4_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_s32(int32x4_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_s64(int64x2_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_s64(int64x2_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_s16(int16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_s16(int16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_p8(poly8x16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_p8(poly8x16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_u8(uint8x16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_u8(uint8x16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_u32(uint32x4_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_u32(uint32x4_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_u64(uint64x2_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_u64(uint64x2_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_u16(uint16x8_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_u16(uint16x8_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_s8(int8x16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_s8(int8x16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_f32(float32x4_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_f32(float32x4_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_f16(float16x8_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_f16(float16x8_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_s32(int32x4_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_s32(int32x4_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_s64(int64x2_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_s64(int64x2_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_s16(int16x8_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_s16(int16x8_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_p8(poly8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_p8(poly8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_p16(poly16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_p16(poly16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_u32(uint32x4_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_u32(uint32x4_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_u64(uint64x2_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_u64(uint64x2_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_u16(uint16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_u16(uint16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_s8(int8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_s8(int8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_f32(float32x4_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_f32(float32x4_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_f16(float16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_f16(float16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_s32(int32x4_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_s32(int32x4_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_s64(int64x2_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_s64(int64x2_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_s16(int16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_s16(int16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_p8(poly8x16_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_p8(poly8x16_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_p16(poly16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_p16(poly16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_u8(uint8x16_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_u8(uint8x16_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_u64(uint64x2_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_u64(uint64x2_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_u16(uint16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_u16(uint16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_s8(int8x16_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_s8(int8x16_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_f32(float32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_f32(float32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_f16(float16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_f16(float16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_s32(int32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_s32(int32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_s64(int64x2_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_s64(int64x2_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_s16(int16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_s16(int16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_p8(poly8x16_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_p8(poly8x16_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_p16(poly16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_p16(poly16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_u8(uint8x16_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_u8(uint8x16_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_u32(uint32x4_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_u32(uint32x4_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_u16(uint16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_u16(uint16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_s8(int8x16_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_s8(int8x16_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_f32(float32x4_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_f32(float32x4_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_f16(float16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_f16(float16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_s32(int32x4_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_s32(int32x4_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_s64(int64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_s64(int64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_s16(int16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_s16(int16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_p8(poly8x16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_p8(poly8x16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_p16(poly16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_p16(poly16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_u8(uint8x16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_u8(uint8x16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_u32(uint32x4_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_u32(uint32x4_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_u64(uint64x2_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_u64(uint64x2_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_s8(int8x16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_s8(int8x16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_f32(float32x4_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_f32(float32x4_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_f16(float16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_f16(float16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_s32(int32x4_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_s32(int32x4_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_s64(int64x2_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_s64(int64x2_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_s16(int16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_s16(int16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_p8(poly8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_p8(poly8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_p16(poly16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_p16(poly16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_u8(uint8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_u8(uint8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_u32(uint32x4_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_u32(uint32x4_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_u64(uint64x2_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_u64(uint64x2_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_u16(uint16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_u16(uint16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_f32(float32x4_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_f32(float32x4_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_f16(float16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_f16(float16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_s32(int32x4_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_s32(int32x4_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_s64(int64x2_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_s64(int64x2_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_s16(int16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_s16(int16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_p8(poly8x16_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_p8(poly8x16_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_p16(poly16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_p16(poly16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_u8(uint8x16_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_u8(uint8x16_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_u32(uint32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_u32(uint32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_u64(uint64x2_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_u64(uint64x2_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_u16(uint16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_u16(uint16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_s8(int8x16_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_s8(int8x16_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_f16(float16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_f16(float16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_s32(int32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_s32(int32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_s64(int64x2_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_s64(int64x2_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_s16(int16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_s16(int16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_p8(poly8x16_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_p8(poly8x16_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_p16(poly16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_p16(poly16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_u8(uint8x16_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_u8(uint8x16_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_u32(uint32x4_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_u32(uint32x4_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_u64(uint64x2_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_u64(uint64x2_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_u16(uint16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_u16(uint16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_s8(int8x16_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_s8(int8x16_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_f32(float32x4_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_f32(float32x4_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_s32(int32x4_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_s32(int32x4_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_s64(int64x2_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_s64(int64x2_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_s16(int16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_s16(int16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_p8(poly8x16_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_p8(poly8x16_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_p16(poly16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_p16(poly16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_u8(uint8x16_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_u8(uint8x16_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_u32(uint32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_u32(uint32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_u64(uint64x2_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_u64(uint64x2_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_u16(uint16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_u16(uint16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_s8(int8x16_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_s8(int8x16_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_f32(float32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_f32(float32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_f16(float16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_f16(float16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_s64(int64x2_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_s64(int64x2_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_s16(int16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_s16(int16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_p8(poly8x16_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_p8(poly8x16_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_p16(poly16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_p16(poly16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_u8(uint8x16_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_u8(uint8x16_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_u32(uint32x4_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_u32(uint32x4_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_u64(uint64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_u64(uint64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_u16(uint16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_u16(uint16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_s8(int8x16_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_s8(int8x16_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_f32(float32x4_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_f32(float32x4_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_f16(float16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_f16(float16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_s32(int32x4_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_s32(int32x4_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_s16(int16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_s16(int16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_p8(poly8x16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_p8(poly8x16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_p16(poly16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_p16(poly16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_u8(uint8x16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_u8(uint8x16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_u32(uint32x4_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_u32(uint32x4_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_u64(uint64x2_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_u64(uint64x2_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_u16(uint16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_u16(uint16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_s8(int8x16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_s8(int8x16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_f32(float32x4_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_f32(float32x4_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_f16(float16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_f16(float16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_s32(int32x4_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_s32(int32x4_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_s64(int64x2_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_s64(int64x2_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_p8(poly8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_p8(poly8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_p16(poly16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_p16(poly16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_u32(uint32x2_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_u32(uint32x2_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_u64(uint64x1_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_u64(uint64x1_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_u16(uint16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_u16(uint16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_s8(int8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_s8(int8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_f32(float32x2_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_f32(float32x2_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_f16(float16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_f16(float16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_s32(int32x2_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_s32(int32x2_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_s64(int64x1_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_s64(int64x1_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_s16(int16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_s16(int16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_p8(poly8x8_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_p8(poly8x8_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_p16(poly16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_p16(poly16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_u8(uint8x8_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_u8(uint8x8_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_u64(uint64x1_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_u64(uint64x1_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_u16(uint16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_u16(uint16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_s8(int8x8_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_s8(int8x8_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_f32(float32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_f32(float32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_f16(float16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_f16(float16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_s32(int32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_s32(int32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_s64(int64x1_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_s64(int64x1_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_s16(int16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_s16(int16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_p8(poly8x8_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_p8(poly8x8_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_p16(poly16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_p16(poly16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_u8(uint8x8_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_u8(uint8x8_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_u32(uint32x2_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_u32(uint32x2_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_u16(uint16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_u16(uint16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_s8(int8x8_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_s8(int8x8_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_f32(float32x2_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_f32(float32x2_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_f16(float16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_f16(float16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_s32(int32x2_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_s32(int32x2_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_s64(int64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_s64(int64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_s16(int16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_s16(int16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_p8(poly8x8_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_p8(poly8x8_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_p16(poly16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_p16(poly16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_u8(uint8x8_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_u8(uint8x8_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_u32(uint32x2_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_u32(uint32x2_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_u64(uint64x1_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_u64(uint64x1_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_s8(int8x8_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_s8(int8x8_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_f32(float32x2_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_f32(float32x2_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_f16(float16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_f16(float16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_s32(int32x2_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_s32(int32x2_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_s64(int64x1_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_s64(int64x1_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_s16(int16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_s16(int16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_p8(poly8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_p8(poly8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_p16(poly16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_p16(poly16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_u8(uint8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_u8(uint8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_u32(uint32x2_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_u32(uint32x2_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_u64(uint64x1_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_u64(uint64x1_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_u16(uint16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_u16(uint16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_f32(float32x2_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_f32(float32x2_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_f16(float16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_f16(float16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_s32(int32x2_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_s32(int32x2_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_s64(int64x1_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_s64(int64x1_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_s16(int16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_s16(int16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_p8(poly8x8_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_p8(poly8x8_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_p16(poly16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_p16(poly16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_u8(uint8x8_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_u8(uint8x8_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_u32(uint32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_u32(uint32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_u64(uint64x1_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_u64(uint64x1_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_u16(uint16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_u16(uint16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_s8(int8x8_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_s8(int8x8_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_f16(float16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_f16(float16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_s32(int32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_s32(int32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_s64(int64x1_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_s64(int64x1_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_s16(int16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_s16(int16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_p8(poly8x8_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_p8(poly8x8_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_p16(poly16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_p16(poly16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_u8(uint8x8_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_u8(uint8x8_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_u32(uint32x2_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_u32(uint32x2_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_u64(uint64x1_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_u64(uint64x1_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_u16(uint16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_u16(uint16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_s8(int8x8_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_s8(int8x8_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_f32(float32x2_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_f32(float32x2_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_s32(int32x2_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_s32(int32x2_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_s64(int64x1_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_s64(int64x1_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_s16(int16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_s16(int16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_p8(poly8x8_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_p8(poly8x8_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_p16(poly16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_p16(poly16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_u8(uint8x8_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_u8(uint8x8_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_u32(uint32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_u32(uint32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_u64(uint64x1_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_u64(uint64x1_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_u16(uint16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_u16(uint16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_s8(int8x8_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_s8(int8x8_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_f32(float32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_f32(float32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_f16(float16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_f16(float16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_s64(int64x1_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_s64(int64x1_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_s16(int16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_s16(int16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_p8(poly8x8_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_p8(poly8x8_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_p16(poly16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_p16(poly16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_u8(uint8x8_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_u8(uint8x8_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_u32(uint32x2_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_u32(uint32x2_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_u64(uint64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_u64(uint64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_u16(uint16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_u16(uint16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_s8(int8x8_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_s8(int8x8_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_f32(float32x2_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_f32(float32x2_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_f16(float16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_f16(float16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_s32(int32x2_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_s32(int32x2_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_s16(int16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_s16(int16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_p8(poly8x8_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_p8(poly8x8_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_p16(poly16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_p16(poly16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_u8(uint8x8_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_u8(uint8x8_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_u32(uint32x2_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_u32(uint32x2_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_u64(uint64x1_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_u64(uint64x1_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_u16(uint16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_u16(uint16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_s8(int8x8_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_s8(int8x8_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_f32(float32x2_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_f32(float32x2_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_f16(float16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_f16(float16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_s32(int32x2_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_s32(int32x2_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_s64(int64x1_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_s64(int64x1_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#endif +#if (__ARM_FP & 2) +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vcvt_f16_f32(float32x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vcvt_f16_f32((int8x16_t)__p0, 8); + return __ret; +} +#else +__ai float16x4_t vcvt_f16_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vcvt_f16_f32((int8x16_t)__rev0, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai float16x4_t __noswap_vcvt_f16_f32(float32x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vcvt_f16_f32((int8x16_t)__p0, 8); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vcvt_f32_f16(float16x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vcvt_f32_f16((int8x8_t)__p0, 41); + return __ret; +} +#else +__ai float32x4_t vcvt_f32_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vcvt_f32_f16((int8x8_t)__rev0, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai float32x4_t __noswap_vcvt_f32_f16(float16x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vcvt_f32_f16((int8x8_t)__p0, 41); + return __ret; +} +#endif + +#endif +#if __ARM_ARCH >= 8 +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vcvtaq_s32_f32(float32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vcvtaq_s32_v((int8x16_t)__p0, 34); + return __ret; +} +#else +__ai int32x4_t vcvtaq_s32_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vcvtaq_s32_v((int8x16_t)__rev0, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vcvta_s32_f32(float32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vcvta_s32_v((int8x8_t)__p0, 2); + return __ret; +} +#else +__ai int32x2_t vcvta_s32_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vcvta_s32_v((int8x8_t)__rev0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcvtaq_u32_f32(float32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcvtaq_u32_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vcvtaq_u32_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcvtaq_u32_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcvta_u32_f32(float32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcvta_u32_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vcvta_u32_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcvta_u32_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vcvtmq_s32_f32(float32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vcvtmq_s32_v((int8x16_t)__p0, 34); + return __ret; +} +#else +__ai int32x4_t vcvtmq_s32_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vcvtmq_s32_v((int8x16_t)__rev0, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vcvtm_s32_f32(float32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vcvtm_s32_v((int8x8_t)__p0, 2); + return __ret; +} +#else +__ai int32x2_t vcvtm_s32_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vcvtm_s32_v((int8x8_t)__rev0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcvtmq_u32_f32(float32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcvtmq_u32_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vcvtmq_u32_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcvtmq_u32_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcvtm_u32_f32(float32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcvtm_u32_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vcvtm_u32_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcvtm_u32_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vcvtnq_s32_f32(float32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vcvtnq_s32_v((int8x16_t)__p0, 34); + return __ret; +} +#else +__ai int32x4_t vcvtnq_s32_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vcvtnq_s32_v((int8x16_t)__rev0, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vcvtn_s32_f32(float32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vcvtn_s32_v((int8x8_t)__p0, 2); + return __ret; +} +#else +__ai int32x2_t vcvtn_s32_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vcvtn_s32_v((int8x8_t)__rev0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcvtnq_u32_f32(float32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcvtnq_u32_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vcvtnq_u32_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcvtnq_u32_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcvtn_u32_f32(float32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcvtn_u32_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vcvtn_u32_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcvtn_u32_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vcvtpq_s32_f32(float32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vcvtpq_s32_v((int8x16_t)__p0, 34); + return __ret; +} +#else +__ai int32x4_t vcvtpq_s32_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vcvtpq_s32_v((int8x16_t)__rev0, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vcvtp_s32_f32(float32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vcvtp_s32_v((int8x8_t)__p0, 2); + return __ret; +} +#else +__ai int32x2_t vcvtp_s32_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vcvtp_s32_v((int8x8_t)__rev0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcvtpq_u32_f32(float32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcvtpq_u32_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vcvtpq_u32_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcvtpq_u32_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcvtp_u32_f32(float32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcvtp_u32_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vcvtp_u32_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcvtp_u32_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#endif +#if __ARM_ARCH >= 8 && defined(__ARM_FEATURE_DIRECTED_ROUNDING) +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vrndq_f32(float32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrndq_v((int8x16_t)__p0, 41); + return __ret; +} +#else +__ai float32x4_t vrndq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrndq_v((int8x16_t)__rev0, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vrnd_f32(float32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrnd_v((int8x8_t)__p0, 9); + return __ret; +} +#else +__ai float32x2_t vrnd_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrnd_v((int8x8_t)__rev0, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vrndaq_f32(float32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrndaq_v((int8x16_t)__p0, 41); + return __ret; +} +#else +__ai float32x4_t vrndaq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrndaq_v((int8x16_t)__rev0, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vrnda_f32(float32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrnda_v((int8x8_t)__p0, 9); + return __ret; +} +#else +__ai float32x2_t vrnda_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrnda_v((int8x8_t)__rev0, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vrndmq_f32(float32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrndmq_v((int8x16_t)__p0, 41); + return __ret; +} +#else +__ai float32x4_t vrndmq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrndmq_v((int8x16_t)__rev0, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vrndm_f32(float32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrndm_v((int8x8_t)__p0, 9); + return __ret; +} +#else +__ai float32x2_t vrndm_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrndm_v((int8x8_t)__rev0, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vrndnq_f32(float32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrndnq_v((int8x16_t)__p0, 41); + return __ret; +} +#else +__ai float32x4_t vrndnq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrndnq_v((int8x16_t)__rev0, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vrndn_f32(float32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrndn_v((int8x8_t)__p0, 9); + return __ret; +} +#else +__ai float32x2_t vrndn_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrndn_v((int8x8_t)__rev0, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vrndpq_f32(float32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrndpq_v((int8x16_t)__p0, 41); + return __ret; +} +#else +__ai float32x4_t vrndpq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrndpq_v((int8x16_t)__rev0, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vrndp_f32(float32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrndp_v((int8x8_t)__p0, 9); + return __ret; +} +#else +__ai float32x2_t vrndp_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrndp_v((int8x8_t)__rev0, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vrndxq_f32(float32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrndxq_v((int8x16_t)__p0, 41); + return __ret; +} +#else +__ai float32x4_t vrndxq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrndxq_v((int8x16_t)__rev0, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vrndx_f32(float32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrndx_v((int8x8_t)__p0, 9); + return __ret; +} +#else +__ai float32x2_t vrndx_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrndx_v((int8x8_t)__rev0, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#endif +#if __ARM_ARCH >= 8 && defined(__ARM_FEATURE_NUMERIC_MAXMIN) +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vmaxnmq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vmaxnmq_v((int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4_t vmaxnmq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vmaxnmq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vmaxnm_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vmaxnm_v((int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2_t vmaxnm_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vmaxnm_v((int8x8_t)__rev0, (int8x8_t)__rev1, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vminnmq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vminnmq_v((int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4_t vminnmq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vminnmq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vminnm_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vminnm_v((int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2_t vminnm_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vminnm_v((int8x8_t)__rev0, (int8x8_t)__rev1, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#endif +#if __ARM_ARCH >= 8 && defined(__aarch64__) +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vcvtaq_s64_f64(float64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vcvtaq_s64_v((int8x16_t)__p0, 35); + return __ret; +} +#else +__ai int64x2_t vcvtaq_s64_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vcvtaq_s64_v((int8x16_t)__rev0, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vcvta_s64_f64(float64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vcvta_s64_v((int8x8_t)__p0, 3); + return __ret; +} +#else +__ai int64x1_t vcvta_s64_f64(float64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vcvta_s64_v((int8x8_t)__p0, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcvtaq_u64_f64(float64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcvtaq_u64_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vcvtaq_u64_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcvtaq_u64_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcvta_u64_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcvta_u64_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vcvta_u64_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcvta_u64_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vcvtmq_s64_f64(float64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vcvtmq_s64_v((int8x16_t)__p0, 35); + return __ret; +} +#else +__ai int64x2_t vcvtmq_s64_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vcvtmq_s64_v((int8x16_t)__rev0, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vcvtm_s64_f64(float64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vcvtm_s64_v((int8x8_t)__p0, 3); + return __ret; +} +#else +__ai int64x1_t vcvtm_s64_f64(float64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vcvtm_s64_v((int8x8_t)__p0, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcvtmq_u64_f64(float64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcvtmq_u64_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vcvtmq_u64_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcvtmq_u64_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcvtm_u64_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcvtm_u64_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vcvtm_u64_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcvtm_u64_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vcvtnq_s64_f64(float64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vcvtnq_s64_v((int8x16_t)__p0, 35); + return __ret; +} +#else +__ai int64x2_t vcvtnq_s64_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vcvtnq_s64_v((int8x16_t)__rev0, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vcvtn_s64_f64(float64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vcvtn_s64_v((int8x8_t)__p0, 3); + return __ret; +} +#else +__ai int64x1_t vcvtn_s64_f64(float64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vcvtn_s64_v((int8x8_t)__p0, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcvtnq_u64_f64(float64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcvtnq_u64_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vcvtnq_u64_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcvtnq_u64_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcvtn_u64_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcvtn_u64_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vcvtn_u64_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcvtn_u64_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vcvtpq_s64_f64(float64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vcvtpq_s64_v((int8x16_t)__p0, 35); + return __ret; +} +#else +__ai int64x2_t vcvtpq_s64_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vcvtpq_s64_v((int8x16_t)__rev0, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vcvtp_s64_f64(float64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vcvtp_s64_v((int8x8_t)__p0, 3); + return __ret; +} +#else +__ai int64x1_t vcvtp_s64_f64(float64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vcvtp_s64_v((int8x8_t)__p0, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcvtpq_u64_f64(float64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcvtpq_u64_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vcvtpq_u64_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcvtpq_u64_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcvtp_u64_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcvtp_u64_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vcvtp_u64_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcvtp_u64_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_p64(poly64x1_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_p64(poly64x1_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_p16(poly16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_p16(poly16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_u8(uint8x8_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_u8(uint8x8_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_u32(uint32x2_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_u32(uint32x2_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_u64(uint64x1_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_u64(uint64x1_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_u16(uint16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_u16(uint16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_s8(int8x8_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_s8(int8x8_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_f64(float64x1_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_f64(float64x1_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_f32(float32x2_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_f32(float32x2_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_f16(float16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_f16(float16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_s32(int32x2_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_s32(int32x2_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_s64(int64x1_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_s64(int64x1_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vreinterpret_p8_s16(int16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#else +__ai poly8x8_t vreinterpret_p8_s16(int16x4_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vreinterpret_p64_p8(poly8x8_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#else +__ai poly64x1_t vreinterpret_p64_p8(poly8x8_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vreinterpret_p64_p16(poly16x4_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#else +__ai poly64x1_t vreinterpret_p64_p16(poly16x4_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vreinterpret_p64_u8(uint8x8_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#else +__ai poly64x1_t vreinterpret_p64_u8(uint8x8_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vreinterpret_p64_u32(uint32x2_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#else +__ai poly64x1_t vreinterpret_p64_u32(uint32x2_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vreinterpret_p64_u64(uint64x1_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#else +__ai poly64x1_t vreinterpret_p64_u64(uint64x1_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vreinterpret_p64_u16(uint16x4_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#else +__ai poly64x1_t vreinterpret_p64_u16(uint16x4_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vreinterpret_p64_s8(int8x8_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#else +__ai poly64x1_t vreinterpret_p64_s8(int8x8_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vreinterpret_p64_f64(float64x1_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#else +__ai poly64x1_t vreinterpret_p64_f64(float64x1_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vreinterpret_p64_f32(float32x2_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#else +__ai poly64x1_t vreinterpret_p64_f32(float32x2_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vreinterpret_p64_f16(float16x4_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#else +__ai poly64x1_t vreinterpret_p64_f16(float16x4_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vreinterpret_p64_s32(int32x2_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#else +__ai poly64x1_t vreinterpret_p64_s32(int32x2_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vreinterpret_p64_s64(int64x1_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#else +__ai poly64x1_t vreinterpret_p64_s64(int64x1_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vreinterpret_p64_s16(int16x4_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#else +__ai poly64x1_t vreinterpret_p64_s16(int16x4_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_p8(poly8x8_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_p8(poly8x8_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_p64(poly64x1_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_p64(poly64x1_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_u8(uint8x8_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_u8(uint8x8_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_u32(uint32x2_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_u32(uint32x2_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_u64(uint64x1_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_u64(uint64x1_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_u16(uint16x4_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_u16(uint16x4_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_s8(int8x8_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_s8(int8x8_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_f64(float64x1_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_f64(float64x1_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_f32(float32x2_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_f32(float32x2_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_f16(float16x4_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_f16(float16x4_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_s32(int32x2_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_s32(int32x2_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_s64(int64x1_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_s64(int64x1_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vreinterpret_p16_s16(int16x4_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#else +__ai poly16x4_t vreinterpret_p16_s16(int16x4_t __p0) { + poly16x4_t __ret; + __ret = (poly16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_p128(poly128_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_p128(poly128_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_p64(poly64x2_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_p64(poly64x2_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_p16(poly16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_p16(poly16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_u8(uint8x16_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_u8(uint8x16_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_u32(uint32x4_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_u32(uint32x4_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_u64(uint64x2_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_u64(uint64x2_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_u16(uint16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_u16(uint16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_s8(int8x16_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_s8(int8x16_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_f64(float64x2_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_f64(float64x2_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_f32(float32x4_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_f32(float32x4_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_f16(float16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_f16(float16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_s32(int32x4_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_s32(int32x4_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_s64(int64x2_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_s64(int64x2_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vreinterpretq_p8_s16(int16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#else +__ai poly8x16_t vreinterpretq_p8_s16(int16x8_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vreinterpretq_p128_p8(poly8x16_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#else +__ai poly128_t vreinterpretq_p128_p8(poly8x16_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vreinterpretq_p128_p64(poly64x2_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#else +__ai poly128_t vreinterpretq_p128_p64(poly64x2_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vreinterpretq_p128_p16(poly16x8_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#else +__ai poly128_t vreinterpretq_p128_p16(poly16x8_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vreinterpretq_p128_u8(uint8x16_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#else +__ai poly128_t vreinterpretq_p128_u8(uint8x16_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vreinterpretq_p128_u32(uint32x4_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#else +__ai poly128_t vreinterpretq_p128_u32(uint32x4_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vreinterpretq_p128_u64(uint64x2_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#else +__ai poly128_t vreinterpretq_p128_u64(uint64x2_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vreinterpretq_p128_u16(uint16x8_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#else +__ai poly128_t vreinterpretq_p128_u16(uint16x8_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vreinterpretq_p128_s8(int8x16_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#else +__ai poly128_t vreinterpretq_p128_s8(int8x16_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vreinterpretq_p128_f64(float64x2_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#else +__ai poly128_t vreinterpretq_p128_f64(float64x2_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vreinterpretq_p128_f32(float32x4_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#else +__ai poly128_t vreinterpretq_p128_f32(float32x4_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vreinterpretq_p128_f16(float16x8_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#else +__ai poly128_t vreinterpretq_p128_f16(float16x8_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vreinterpretq_p128_s32(int32x4_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#else +__ai poly128_t vreinterpretq_p128_s32(int32x4_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vreinterpretq_p128_s64(int64x2_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#else +__ai poly128_t vreinterpretq_p128_s64(int64x2_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vreinterpretq_p128_s16(int16x8_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#else +__ai poly128_t vreinterpretq_p128_s16(int16x8_t __p0) { + poly128_t __ret; + __ret = (poly128_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vreinterpretq_p64_p8(poly8x16_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#else +__ai poly64x2_t vreinterpretq_p64_p8(poly8x16_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vreinterpretq_p64_p128(poly128_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#else +__ai poly64x2_t vreinterpretq_p64_p128(poly128_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vreinterpretq_p64_p16(poly16x8_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#else +__ai poly64x2_t vreinterpretq_p64_p16(poly16x8_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vreinterpretq_p64_u8(uint8x16_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#else +__ai poly64x2_t vreinterpretq_p64_u8(uint8x16_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vreinterpretq_p64_u32(uint32x4_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#else +__ai poly64x2_t vreinterpretq_p64_u32(uint32x4_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vreinterpretq_p64_u64(uint64x2_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#else +__ai poly64x2_t vreinterpretq_p64_u64(uint64x2_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vreinterpretq_p64_u16(uint16x8_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#else +__ai poly64x2_t vreinterpretq_p64_u16(uint16x8_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vreinterpretq_p64_s8(int8x16_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#else +__ai poly64x2_t vreinterpretq_p64_s8(int8x16_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vreinterpretq_p64_f64(float64x2_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#else +__ai poly64x2_t vreinterpretq_p64_f64(float64x2_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vreinterpretq_p64_f32(float32x4_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#else +__ai poly64x2_t vreinterpretq_p64_f32(float32x4_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vreinterpretq_p64_f16(float16x8_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#else +__ai poly64x2_t vreinterpretq_p64_f16(float16x8_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vreinterpretq_p64_s32(int32x4_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#else +__ai poly64x2_t vreinterpretq_p64_s32(int32x4_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vreinterpretq_p64_s64(int64x2_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#else +__ai poly64x2_t vreinterpretq_p64_s64(int64x2_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vreinterpretq_p64_s16(int16x8_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#else +__ai poly64x2_t vreinterpretq_p64_s16(int16x8_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_p8(poly8x16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_p8(poly8x16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_p128(poly128_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_p128(poly128_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_p64(poly64x2_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_p64(poly64x2_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_u8(uint8x16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_u8(uint8x16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_u32(uint32x4_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_u32(uint32x4_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_u64(uint64x2_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_u64(uint64x2_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_u16(uint16x8_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_u16(uint16x8_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_s8(int8x16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_s8(int8x16_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_f64(float64x2_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_f64(float64x2_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_f32(float32x4_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_f32(float32x4_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_f16(float16x8_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_f16(float16x8_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_s32(int32x4_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_s32(int32x4_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_s64(int64x2_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_s64(int64x2_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vreinterpretq_p16_s16(int16x8_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#else +__ai poly16x8_t vreinterpretq_p16_s16(int16x8_t __p0) { + poly16x8_t __ret; + __ret = (poly16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_p8(poly8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_p8(poly8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_p128(poly128_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_p128(poly128_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_p64(poly64x2_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_p64(poly64x2_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_p16(poly16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_p16(poly16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_u32(uint32x4_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_u32(uint32x4_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_u64(uint64x2_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_u64(uint64x2_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_u16(uint16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_u16(uint16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_s8(int8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_s8(int8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_f64(float64x2_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_f64(float64x2_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_f32(float32x4_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_f32(float32x4_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_f16(float16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_f16(float16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_s32(int32x4_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_s32(int32x4_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_s64(int64x2_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_s64(int64x2_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vreinterpretq_u8_s16(int16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#else +__ai uint8x16_t vreinterpretq_u8_s16(int16x8_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_p8(poly8x16_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_p8(poly8x16_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_p128(poly128_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_p128(poly128_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_p64(poly64x2_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_p64(poly64x2_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_p16(poly16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_p16(poly16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_u8(uint8x16_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_u8(uint8x16_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_u64(uint64x2_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_u64(uint64x2_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_u16(uint16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_u16(uint16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_s8(int8x16_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_s8(int8x16_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_f64(float64x2_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_f64(float64x2_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_f32(float32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_f32(float32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_f16(float16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_f16(float16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_s32(int32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_s32(int32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_s64(int64x2_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_s64(int64x2_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vreinterpretq_u32_s16(int16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#else +__ai uint32x4_t vreinterpretq_u32_s16(int16x8_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_p8(poly8x16_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_p8(poly8x16_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_p128(poly128_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_p128(poly128_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_p64(poly64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_p64(poly64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_p16(poly16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_p16(poly16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_u8(uint8x16_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_u8(uint8x16_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_u32(uint32x4_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_u32(uint32x4_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_u16(uint16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_u16(uint16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_s8(int8x16_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_s8(int8x16_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_f64(float64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_f64(float64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_f32(float32x4_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_f32(float32x4_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_f16(float16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_f16(float16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_s32(int32x4_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_s32(int32x4_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_s64(int64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_s64(int64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vreinterpretq_u64_s16(int16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#else +__ai uint64x2_t vreinterpretq_u64_s16(int16x8_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_p8(poly8x16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_p8(poly8x16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_p128(poly128_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_p128(poly128_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_p64(poly64x2_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_p64(poly64x2_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_p16(poly16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_p16(poly16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_u8(uint8x16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_u8(uint8x16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_u32(uint32x4_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_u32(uint32x4_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_u64(uint64x2_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_u64(uint64x2_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_s8(int8x16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_s8(int8x16_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_f64(float64x2_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_f64(float64x2_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_f32(float32x4_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_f32(float32x4_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_f16(float16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_f16(float16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_s32(int32x4_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_s32(int32x4_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_s64(int64x2_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_s64(int64x2_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vreinterpretq_u16_s16(int16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#else +__ai uint16x8_t vreinterpretq_u16_s16(int16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_p8(poly8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_p8(poly8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_p128(poly128_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_p128(poly128_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_p64(poly64x2_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_p64(poly64x2_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_p16(poly16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_p16(poly16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_u8(uint8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_u8(uint8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_u32(uint32x4_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_u32(uint32x4_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_u64(uint64x2_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_u64(uint64x2_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_u16(uint16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_u16(uint16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_f64(float64x2_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_f64(float64x2_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_f32(float32x4_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_f32(float32x4_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_f16(float16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_f16(float16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_s32(int32x4_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_s32(int32x4_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_s64(int64x2_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_s64(int64x2_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vreinterpretq_s8_s16(int16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#else +__ai int8x16_t vreinterpretq_s8_s16(int16x8_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vreinterpretq_f64_p8(poly8x16_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#else +__ai float64x2_t vreinterpretq_f64_p8(poly8x16_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vreinterpretq_f64_p128(poly128_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#else +__ai float64x2_t vreinterpretq_f64_p128(poly128_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vreinterpretq_f64_p64(poly64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#else +__ai float64x2_t vreinterpretq_f64_p64(poly64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vreinterpretq_f64_p16(poly16x8_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#else +__ai float64x2_t vreinterpretq_f64_p16(poly16x8_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vreinterpretq_f64_u8(uint8x16_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#else +__ai float64x2_t vreinterpretq_f64_u8(uint8x16_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vreinterpretq_f64_u32(uint32x4_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#else +__ai float64x2_t vreinterpretq_f64_u32(uint32x4_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vreinterpretq_f64_u64(uint64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#else +__ai float64x2_t vreinterpretq_f64_u64(uint64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vreinterpretq_f64_u16(uint16x8_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#else +__ai float64x2_t vreinterpretq_f64_u16(uint16x8_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vreinterpretq_f64_s8(int8x16_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#else +__ai float64x2_t vreinterpretq_f64_s8(int8x16_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vreinterpretq_f64_f32(float32x4_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#else +__ai float64x2_t vreinterpretq_f64_f32(float32x4_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vreinterpretq_f64_f16(float16x8_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#else +__ai float64x2_t vreinterpretq_f64_f16(float16x8_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vreinterpretq_f64_s32(int32x4_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#else +__ai float64x2_t vreinterpretq_f64_s32(int32x4_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vreinterpretq_f64_s64(int64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#else +__ai float64x2_t vreinterpretq_f64_s64(int64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vreinterpretq_f64_s16(int16x8_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#else +__ai float64x2_t vreinterpretq_f64_s16(int16x8_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_p8(poly8x16_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_p8(poly8x16_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_p128(poly128_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_p128(poly128_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_p64(poly64x2_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_p64(poly64x2_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_p16(poly16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_p16(poly16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_u8(uint8x16_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_u8(uint8x16_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_u32(uint32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_u32(uint32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_u64(uint64x2_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_u64(uint64x2_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_u16(uint16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_u16(uint16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_s8(int8x16_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_s8(int8x16_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_f64(float64x2_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_f64(float64x2_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_f16(float16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_f16(float16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_s32(int32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_s32(int32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_s64(int64x2_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_s64(int64x2_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vreinterpretq_f32_s16(int16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#else +__ai float32x4_t vreinterpretq_f32_s16(int16x8_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_p8(poly8x16_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_p8(poly8x16_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_p128(poly128_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_p128(poly128_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_p64(poly64x2_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_p64(poly64x2_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_p16(poly16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_p16(poly16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_u8(uint8x16_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_u8(uint8x16_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_u32(uint32x4_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_u32(uint32x4_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_u64(uint64x2_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_u64(uint64x2_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_u16(uint16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_u16(uint16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_s8(int8x16_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_s8(int8x16_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_f64(float64x2_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_f64(float64x2_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_f32(float32x4_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_f32(float32x4_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_s32(int32x4_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_s32(int32x4_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_s64(int64x2_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_s64(int64x2_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vreinterpretq_f16_s16(int16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#else +__ai float16x8_t vreinterpretq_f16_s16(int16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_p8(poly8x16_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_p8(poly8x16_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_p128(poly128_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_p128(poly128_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_p64(poly64x2_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_p64(poly64x2_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_p16(poly16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_p16(poly16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_u8(uint8x16_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_u8(uint8x16_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_u32(uint32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_u32(uint32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_u64(uint64x2_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_u64(uint64x2_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_u16(uint16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_u16(uint16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_s8(int8x16_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_s8(int8x16_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_f64(float64x2_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_f64(float64x2_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_f32(float32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_f32(float32x4_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_f16(float16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_f16(float16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_s64(int64x2_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_s64(int64x2_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vreinterpretq_s32_s16(int16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#else +__ai int32x4_t vreinterpretq_s32_s16(int16x8_t __p0) { + int32x4_t __ret; + __ret = (int32x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_p8(poly8x16_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_p8(poly8x16_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_p128(poly128_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_p128(poly128_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_p64(poly64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_p64(poly64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_p16(poly16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_p16(poly16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_u8(uint8x16_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_u8(uint8x16_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_u32(uint32x4_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_u32(uint32x4_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_u64(uint64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_u64(uint64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_u16(uint16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_u16(uint16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_s8(int8x16_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_s8(int8x16_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_f64(float64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_f64(float64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_f32(float32x4_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_f32(float32x4_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_f16(float16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_f16(float16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_s32(int32x4_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_s32(int32x4_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vreinterpretq_s64_s16(int16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#else +__ai int64x2_t vreinterpretq_s64_s16(int16x8_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_p8(poly8x16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_p8(poly8x16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_p128(poly128_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_p128(poly128_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_p64(poly64x2_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_p64(poly64x2_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_p16(poly16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_p16(poly16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_u8(uint8x16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_u8(uint8x16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_u32(uint32x4_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_u32(uint32x4_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_u64(uint64x2_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_u64(uint64x2_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_u16(uint16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_u16(uint16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_s8(int8x16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_s8(int8x16_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_f64(float64x2_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_f64(float64x2_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_f32(float32x4_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_f32(float32x4_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_f16(float16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_f16(float16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_s32(int32x4_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_s32(int32x4_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vreinterpretq_s16_s64(int64x2_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#else +__ai int16x8_t vreinterpretq_s16_s64(int64x2_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_p8(poly8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_p8(poly8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_p64(poly64x1_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_p64(poly64x1_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_p16(poly16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_p16(poly16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_u32(uint32x2_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_u32(uint32x2_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_u64(uint64x1_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_u64(uint64x1_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_u16(uint16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_u16(uint16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_s8(int8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_s8(int8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_f64(float64x1_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_f64(float64x1_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_f32(float32x2_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_f32(float32x2_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_f16(float16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_f16(float16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_s32(int32x2_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_s32(int32x2_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_s64(int64x1_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_s64(int64x1_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vreinterpret_u8_s16(int16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#else +__ai uint8x8_t vreinterpret_u8_s16(int16x4_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_p8(poly8x8_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_p8(poly8x8_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_p64(poly64x1_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_p64(poly64x1_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_p16(poly16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_p16(poly16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_u8(uint8x8_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_u8(uint8x8_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_u64(uint64x1_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_u64(uint64x1_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_u16(uint16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_u16(uint16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_s8(int8x8_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_s8(int8x8_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_f64(float64x1_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_f64(float64x1_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_f32(float32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_f32(float32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_f16(float16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_f16(float16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_s32(int32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_s32(int32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_s64(int64x1_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_s64(int64x1_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vreinterpret_u32_s16(int16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#else +__ai uint32x2_t vreinterpret_u32_s16(int16x4_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_p8(poly8x8_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_p8(poly8x8_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_p64(poly64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_p64(poly64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_p16(poly16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_p16(poly16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_u8(uint8x8_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_u8(uint8x8_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_u32(uint32x2_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_u32(uint32x2_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_u16(uint16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_u16(uint16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_s8(int8x8_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_s8(int8x8_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_f32(float32x2_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_f32(float32x2_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_f16(float16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_f16(float16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_s32(int32x2_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_s32(int32x2_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_s64(int64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_s64(int64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vreinterpret_u64_s16(int16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#else +__ai uint64x1_t vreinterpret_u64_s16(int16x4_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_p8(poly8x8_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_p8(poly8x8_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_p64(poly64x1_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_p64(poly64x1_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_p16(poly16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_p16(poly16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_u8(uint8x8_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_u8(uint8x8_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_u32(uint32x2_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_u32(uint32x2_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_u64(uint64x1_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_u64(uint64x1_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_s8(int8x8_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_s8(int8x8_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_f64(float64x1_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_f64(float64x1_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_f32(float32x2_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_f32(float32x2_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_f16(float16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_f16(float16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_s32(int32x2_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_s32(int32x2_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_s64(int64x1_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_s64(int64x1_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vreinterpret_u16_s16(int16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#else +__ai uint16x4_t vreinterpret_u16_s16(int16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_p8(poly8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_p8(poly8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_p64(poly64x1_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_p64(poly64x1_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_p16(poly16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_p16(poly16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_u8(uint8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_u8(uint8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_u32(uint32x2_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_u32(uint32x2_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_u64(uint64x1_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_u64(uint64x1_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_u16(uint16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_u16(uint16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_f64(float64x1_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_f64(float64x1_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_f32(float32x2_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_f32(float32x2_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_f16(float16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_f16(float16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_s32(int32x2_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_s32(int32x2_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_s64(int64x1_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_s64(int64x1_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vreinterpret_s8_s16(int16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#else +__ai int8x8_t vreinterpret_s8_s16(int16x4_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vreinterpret_f64_p8(poly8x8_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#else +__ai float64x1_t vreinterpret_f64_p8(poly8x8_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vreinterpret_f64_p64(poly64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#else +__ai float64x1_t vreinterpret_f64_p64(poly64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vreinterpret_f64_p16(poly16x4_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#else +__ai float64x1_t vreinterpret_f64_p16(poly16x4_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vreinterpret_f64_u8(uint8x8_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#else +__ai float64x1_t vreinterpret_f64_u8(uint8x8_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vreinterpret_f64_u32(uint32x2_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#else +__ai float64x1_t vreinterpret_f64_u32(uint32x2_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vreinterpret_f64_u64(uint64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#else +__ai float64x1_t vreinterpret_f64_u64(uint64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vreinterpret_f64_u16(uint16x4_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#else +__ai float64x1_t vreinterpret_f64_u16(uint16x4_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vreinterpret_f64_s8(int8x8_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#else +__ai float64x1_t vreinterpret_f64_s8(int8x8_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vreinterpret_f64_f32(float32x2_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#else +__ai float64x1_t vreinterpret_f64_f32(float32x2_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vreinterpret_f64_f16(float16x4_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#else +__ai float64x1_t vreinterpret_f64_f16(float16x4_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vreinterpret_f64_s32(int32x2_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#else +__ai float64x1_t vreinterpret_f64_s32(int32x2_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vreinterpret_f64_s64(int64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#else +__ai float64x1_t vreinterpret_f64_s64(int64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vreinterpret_f64_s16(int16x4_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#else +__ai float64x1_t vreinterpret_f64_s16(int16x4_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_p8(poly8x8_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_p8(poly8x8_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_p64(poly64x1_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_p64(poly64x1_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_p16(poly16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_p16(poly16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_u8(uint8x8_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_u8(uint8x8_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_u32(uint32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_u32(uint32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_u64(uint64x1_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_u64(uint64x1_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_u16(uint16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_u16(uint16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_s8(int8x8_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_s8(int8x8_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_f64(float64x1_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_f64(float64x1_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_f16(float16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_f16(float16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_s32(int32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_s32(int32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_s64(int64x1_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_s64(int64x1_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vreinterpret_f32_s16(int16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#else +__ai float32x2_t vreinterpret_f32_s16(int16x4_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_p8(poly8x8_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_p8(poly8x8_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_p64(poly64x1_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_p64(poly64x1_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_p16(poly16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_p16(poly16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_u8(uint8x8_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_u8(uint8x8_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_u32(uint32x2_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_u32(uint32x2_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_u64(uint64x1_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_u64(uint64x1_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_u16(uint16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_u16(uint16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_s8(int8x8_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_s8(int8x8_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_f64(float64x1_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_f64(float64x1_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_f32(float32x2_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_f32(float32x2_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_s32(int32x2_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_s32(int32x2_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_s64(int64x1_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_s64(int64x1_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vreinterpret_f16_s16(int16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#else +__ai float16x4_t vreinterpret_f16_s16(int16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_p8(poly8x8_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_p8(poly8x8_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_p64(poly64x1_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_p64(poly64x1_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_p16(poly16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_p16(poly16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_u8(uint8x8_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_u8(uint8x8_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_u32(uint32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_u32(uint32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_u64(uint64x1_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_u64(uint64x1_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_u16(uint16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_u16(uint16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_s8(int8x8_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_s8(int8x8_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_f64(float64x1_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_f64(float64x1_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_f32(float32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_f32(float32x2_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_f16(float16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_f16(float16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_s64(int64x1_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_s64(int64x1_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vreinterpret_s32_s16(int16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#else +__ai int32x2_t vreinterpret_s32_s16(int16x4_t __p0) { + int32x2_t __ret; + __ret = (int32x2_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_p8(poly8x8_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_p8(poly8x8_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_p64(poly64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_p64(poly64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_p16(poly16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_p16(poly16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_u8(uint8x8_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_u8(uint8x8_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_u32(uint32x2_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_u32(uint32x2_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_u64(uint64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_u64(uint64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_u16(uint16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_u16(uint16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_s8(int8x8_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_s8(int8x8_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_f64(float64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_f64(float64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_f32(float32x2_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_f32(float32x2_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_f16(float16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_f16(float16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_s32(int32x2_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_s32(int32x2_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vreinterpret_s64_s16(int16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#else +__ai int64x1_t vreinterpret_s64_s16(int16x4_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_p8(poly8x8_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_p8(poly8x8_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_p64(poly64x1_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_p64(poly64x1_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_p16(poly16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_p16(poly16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_u8(uint8x8_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_u8(uint8x8_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_u32(uint32x2_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_u32(uint32x2_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_u64(uint64x1_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_u64(uint64x1_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_u16(uint16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_u16(uint16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_s8(int8x8_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_s8(int8x8_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_f64(float64x1_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_f64(float64x1_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_f32(float32x2_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_f32(float32x2_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_f16(float16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_f16(float16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_s32(int32x2_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_s32(int32x2_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vreinterpret_s16_s64(int64x1_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#else +__ai int16x4_t vreinterpret_s16_s64(int64x1_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t)(__p0); + return __ret; +} +#endif + +#endif +#if __ARM_ARCH >= 8 && defined(__aarch64__) && defined(__ARM_FEATURE_DIRECTED_ROUNDING) +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vrndq_f64(float64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrndq_v((int8x16_t)__p0, 42); + return __ret; +} +#else +__ai float64x2_t vrndq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrndq_v((int8x16_t)__rev0, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vrnd_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrnd_v((int8x8_t)__p0, 10); + return __ret; +} +#else +__ai float64x1_t vrnd_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrnd_v((int8x8_t)__p0, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vrndaq_f64(float64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrndaq_v((int8x16_t)__p0, 42); + return __ret; +} +#else +__ai float64x2_t vrndaq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrndaq_v((int8x16_t)__rev0, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vrnda_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrnda_v((int8x8_t)__p0, 10); + return __ret; +} +#else +__ai float64x1_t vrnda_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrnda_v((int8x8_t)__p0, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vrndiq_f64(float64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrndiq_v((int8x16_t)__p0, 42); + return __ret; +} +#else +__ai float64x2_t vrndiq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrndiq_v((int8x16_t)__rev0, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vrndiq_f32(float32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrndiq_v((int8x16_t)__p0, 41); + return __ret; +} +#else +__ai float32x4_t vrndiq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vrndiq_v((int8x16_t)__rev0, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vrndi_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrndi_v((int8x8_t)__p0, 10); + return __ret; +} +#else +__ai float64x1_t vrndi_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrndi_v((int8x8_t)__p0, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vrndi_f32(float32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrndi_v((int8x8_t)__p0, 9); + return __ret; +} +#else +__ai float32x2_t vrndi_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vrndi_v((int8x8_t)__rev0, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vrndmq_f64(float64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrndmq_v((int8x16_t)__p0, 42); + return __ret; +} +#else +__ai float64x2_t vrndmq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrndmq_v((int8x16_t)__rev0, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vrndm_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrndm_v((int8x8_t)__p0, 10); + return __ret; +} +#else +__ai float64x1_t vrndm_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrndm_v((int8x8_t)__p0, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vrndnq_f64(float64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrndnq_v((int8x16_t)__p0, 42); + return __ret; +} +#else +__ai float64x2_t vrndnq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrndnq_v((int8x16_t)__rev0, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vrndn_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrndn_v((int8x8_t)__p0, 10); + return __ret; +} +#else +__ai float64x1_t vrndn_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrndn_v((int8x8_t)__p0, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vrndpq_f64(float64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrndpq_v((int8x16_t)__p0, 42); + return __ret; +} +#else +__ai float64x2_t vrndpq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrndpq_v((int8x16_t)__rev0, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vrndp_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrndp_v((int8x8_t)__p0, 10); + return __ret; +} +#else +__ai float64x1_t vrndp_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrndp_v((int8x8_t)__p0, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vrndxq_f64(float64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrndxq_v((int8x16_t)__p0, 42); + return __ret; +} +#else +__ai float64x2_t vrndxq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrndxq_v((int8x16_t)__rev0, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vrndx_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrndx_v((int8x8_t)__p0, 10); + return __ret; +} +#else +__ai float64x1_t vrndx_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrndx_v((int8x8_t)__p0, 10); + return __ret; +} +#endif + +#endif +#if __ARM_ARCH >= 8 && defined(__aarch64__) && defined(__ARM_FEATURE_NUMERIC_MAXMIN) +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vmaxnmq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vmaxnmq_v((int8x16_t)__p0, (int8x16_t)__p1, 42); + return __ret; +} +#else +__ai float64x2_t vmaxnmq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vmaxnmq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vmaxnm_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vmaxnm_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#else +__ai float64x1_t vmaxnm_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vmaxnm_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vminnmq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vminnmq_v((int8x16_t)__p0, (int8x16_t)__p1, 42); + return __ret; +} +#else +__ai float64x2_t vminnmq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vminnmq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vminnm_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vminnm_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#else +__ai float64x1_t vminnm_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vminnm_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#endif + +#endif +#if __ARM_FEATURE_CRYPTO +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vaesdq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vaesdq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vaesdq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vaesdq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vaeseq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vaeseq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vaeseq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vaeseq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vaesimcq_u8(uint8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vaesimcq_v((int8x16_t)__p0, 48); + return __ret; +} +#else +__ai uint8x16_t vaesimcq_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vaesimcq_v((int8x16_t)__rev0, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vaesmcq_u8(uint8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vaesmcq_v((int8x16_t)__p0, 48); + return __ret; +} +#else +__ai uint8x16_t vaesmcq_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vaesmcq_v((int8x16_t)__rev0, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsha1cq_u32(uint32x4_t __p0, uint32_t __p1, uint32x4_t __p2) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha1cq_u32((int8x16_t)__p0, __p1, (int8x16_t)__p2); + return __ret; +} +#else +__ai uint32x4_t vsha1cq_u32(uint32x4_t __p0, uint32_t __p1, uint32x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha1cq_u32((int8x16_t)__rev0, __p1, (int8x16_t)__rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vsha1h_u32(uint32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vsha1h_u32(__p0); + return __ret; +} +#else +__ai uint32_t vsha1h_u32(uint32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vsha1h_u32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsha1mq_u32(uint32x4_t __p0, uint32_t __p1, uint32x4_t __p2) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha1mq_u32((int8x16_t)__p0, __p1, (int8x16_t)__p2); + return __ret; +} +#else +__ai uint32x4_t vsha1mq_u32(uint32x4_t __p0, uint32_t __p1, uint32x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha1mq_u32((int8x16_t)__rev0, __p1, (int8x16_t)__rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsha1pq_u32(uint32x4_t __p0, uint32_t __p1, uint32x4_t __p2) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha1pq_u32((int8x16_t)__p0, __p1, (int8x16_t)__p2); + return __ret; +} +#else +__ai uint32x4_t vsha1pq_u32(uint32x4_t __p0, uint32_t __p1, uint32x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha1pq_u32((int8x16_t)__rev0, __p1, (int8x16_t)__rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsha1su0q_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha1su0q_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 50); + return __ret; +} +#else +__ai uint32x4_t vsha1su0q_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha1su0q_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsha1su1q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha1su1q_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vsha1su1q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha1su1q_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsha256hq_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha256hq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 50); + return __ret; +} +#else +__ai uint32x4_t vsha256hq_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha256hq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsha256h2q_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha256h2q_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 50); + return __ret; +} +#else +__ai uint32x4_t vsha256h2q_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha256h2q_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsha256su0q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha256su0q_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vsha256su0q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha256su0q_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsha256su1q_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha256su1q_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 50); + return __ret; +} +#else +__ai uint32x4_t vsha256su1q_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsha256su1q_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#endif +#if defined(__ARM_FEATURE_FMA) +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vfmaq_f32(float32x4_t __p0, float32x4_t __p1, float32x4_t __p2) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vfmaq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41); + return __ret; +} +#else +__ai float32x4_t vfmaq_f32(float32x4_t __p0, float32x4_t __p1, float32x4_t __p2) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vfmaq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai float32x4_t __noswap_vfmaq_f32(float32x4_t __p0, float32x4_t __p1, float32x4_t __p2) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vfmaq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 41); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vfma_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vfma_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9); + return __ret; +} +#else +__ai float32x2_t vfma_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vfma_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai float32x2_t __noswap_vfma_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vfma_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 9); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vfmsq_f32(float32x4_t __p0, float32x4_t __p1, float32x4_t __p2) { + float32x4_t __ret; + __ret = vfmaq_f32(__p0, -__p1, __p2); + return __ret; +} +#else +__ai float32x4_t vfmsq_f32(float32x4_t __p0, float32x4_t __p1, float32x4_t __p2) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __noswap_vfmaq_f32(__rev0, -__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vfms_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2) { + float32x2_t __ret; + __ret = vfma_f32(__p0, -__p1, __p2); + return __ret; +} +#else +__ai float32x2_t vfms_f32(float32x2_t __p0, float32x2_t __p1, float32x2_t __p2) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + float32x2_t __ret; + __ret = __noswap_vfma_f32(__rev0, -__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#endif +#if defined(__ARM_FEATURE_FP16_SCALAR_ARITHMETIC) +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vabdh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vabdh_f16(__p0, __p1); + return __ret; +} +#else +__ai float16_t vabdh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vabdh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vabsh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vabsh_f16(__p0); + return __ret; +} +#else +__ai float16_t vabsh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vabsh_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vaddh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vaddh_f16(__p0, __p1); + return __ret; +} +#else +__ai float16_t vaddh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vaddh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcageh_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcageh_f16(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vcageh_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcageh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcagth_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcagth_f16(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vcagth_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcagth_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcaleh_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcaleh_f16(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vcaleh_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcaleh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcalth_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcalth_f16(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vcalth_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcalth_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vceqh_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vceqh_f16(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vceqh_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vceqh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vceqzh_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vceqzh_f16(__p0); + return __ret; +} +#else +__ai uint32_t vceqzh_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vceqzh_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcgeh_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcgeh_f16(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vcgeh_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcgeh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcgezh_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcgezh_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcgezh_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcgezh_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcgth_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcgth_f16(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vcgth_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcgth_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcgtzh_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcgtzh_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcgtzh_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcgtzh_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcleh_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcleh_f16(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vcleh_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcleh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vclezh_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vclezh_f16(__p0); + return __ret; +} +#else +__ai uint32_t vclezh_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vclezh_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vclth_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vclth_f16(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vclth_f16(float16_t __p0, float16_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vclth_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcltzh_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcltzh_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcltzh_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcltzh_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvth_s16_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvth_s16_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvth_s16_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvth_s16_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvth_s32_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvth_s32_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvth_s32_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvth_s32_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvth_s64_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvth_s64_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvth_s64_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvth_s64_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvth_u16_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvth_u16_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvth_u16_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvth_u16_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvth_u32_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvth_u32_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvth_u32_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvth_u32_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvth_u64_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvth_u64_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvth_u64_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvth_u64_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtah_s16_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtah_s16_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvtah_s16_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtah_s16_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtah_s32_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtah_s32_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvtah_s32_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtah_s32_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtah_s64_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtah_s64_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvtah_s64_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtah_s64_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtah_u16_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtah_u16_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvtah_u16_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtah_u16_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtah_u32_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtah_u32_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvtah_u32_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtah_u32_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtah_u64_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtah_u64_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvtah_u64_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtah_u64_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vcvth_f16_s32(int32_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vcvth_f16_s32(__p0); + return __ret; +} +#else +__ai float16_t vcvth_f16_s32(int32_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vcvth_f16_s32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vcvth_f16_s64(int64_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vcvth_f16_s64(__p0); + return __ret; +} +#else +__ai float16_t vcvth_f16_s64(int64_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vcvth_f16_s64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vcvth_f16_s16(int16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vcvth_f16_s16(__p0); + return __ret; +} +#else +__ai float16_t vcvth_f16_s16(int16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vcvth_f16_s16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vcvth_f16_u32(uint32_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vcvth_f16_u32(__p0); + return __ret; +} +#else +__ai float16_t vcvth_f16_u32(uint32_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vcvth_f16_u32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vcvth_f16_u64(uint64_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vcvth_f16_u64(__p0); + return __ret; +} +#else +__ai float16_t vcvth_f16_u64(uint64_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vcvth_f16_u64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vcvth_f16_u16(uint16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vcvth_f16_u16(__p0); + return __ret; +} +#else +__ai float16_t vcvth_f16_u16(uint16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vcvth_f16_u16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtmh_s16_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtmh_s16_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvtmh_s16_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtmh_s16_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtmh_s32_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtmh_s32_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvtmh_s32_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtmh_s32_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtmh_s64_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtmh_s64_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvtmh_s64_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtmh_s64_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtmh_u16_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtmh_u16_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvtmh_u16_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtmh_u16_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtmh_u32_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtmh_u32_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvtmh_u32_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtmh_u32_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtmh_u64_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtmh_u64_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvtmh_u64_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtmh_u64_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtnh_s16_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtnh_s16_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvtnh_s16_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtnh_s16_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtnh_s32_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtnh_s32_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvtnh_s32_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtnh_s32_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtnh_s64_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtnh_s64_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvtnh_s64_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtnh_s64_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtnh_u16_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtnh_u16_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvtnh_u16_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtnh_u16_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtnh_u32_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtnh_u32_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvtnh_u32_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtnh_u32_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtnh_u64_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtnh_u64_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvtnh_u64_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtnh_u64_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtph_s16_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtph_s16_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvtph_s16_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtph_s16_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtph_s32_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtph_s32_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvtph_s32_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtph_s32_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtph_s64_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtph_s64_f16(__p0); + return __ret; +} +#else +__ai int32_t vcvtph_s64_f16(float16_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtph_s64_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtph_u16_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtph_u16_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvtph_u16_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtph_u16_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtph_u32_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtph_u32_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvtph_u32_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtph_u32_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtph_u64_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtph_u64_f16(__p0); + return __ret; +} +#else +__ai uint32_t vcvtph_u64_f16(float16_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtph_u64_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vfmah_f16(float16_t __p0, float16_t __p1, float16_t __p2) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vfmah_f16(__p0, __p1, __p2); + return __ret; +} +#else +__ai float16_t vfmah_f16(float16_t __p0, float16_t __p1, float16_t __p2) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vfmah_f16(__p0, __p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmah_lane_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16_t __s1 = __p1; \ + float16x4_t __s2 = __p2; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vfmah_lane_f16(__s0, __s1, (int8x8_t)__s2, __p3); \ + __ret; \ +}) +#else +#define vfmah_lane_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16_t __s1 = __p1; \ + float16x4_t __s2 = __p2; \ + float16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vfmah_lane_f16(__s0, __s1, (int8x8_t)__rev2, __p3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmah_laneq_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16_t __s1 = __p1; \ + float16x8_t __s2 = __p2; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vfmah_laneq_f16(__s0, __s1, (int8x16_t)__s2, __p3); \ + __ret; \ +}) +#else +#define vfmah_laneq_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16_t __s0 = __p0; \ + float16_t __s1 = __p1; \ + float16x8_t __s2 = __p2; \ + float16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vfmah_laneq_f16(__s0, __s1, (int8x16_t)__rev2, __p3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vmaxh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmaxh_f16(__p0, __p1); + return __ret; +} +#else +__ai float16_t vmaxh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmaxh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vmaxnmh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmaxnmh_f16(__p0, __p1); + return __ret; +} +#else +__ai float16_t vmaxnmh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmaxnmh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vminh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vminh_f16(__p0, __p1); + return __ret; +} +#else +__ai float16_t vminh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vminh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vminnmh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vminnmh_f16(__p0, __p1); + return __ret; +} +#else +__ai float16_t vminnmh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vminnmh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vmulh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmulh_f16(__p0, __p1); + return __ret; +} +#else +__ai float16_t vmulh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmulh_f16(__p0, __p1); + return __ret; +} +__ai float16_t __noswap_vmulh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmulh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulh_lane_f16(__p0_0, __p1_0, __p2_0) __extension__ ({ \ + float16_t __s0_0 = __p0_0; \ + float16x4_t __s1_0 = __p1_0; \ + float16_t __ret_0; \ + __ret_0 = vmulh_f16(__s0_0, vget_lane_f16(__s1_0, __p2_0)); \ + __ret_0; \ +}) +#else +#define vmulh_lane_f16(__p0_1, __p1_1, __p2_1) __extension__ ({ \ + float16_t __s0_1 = __p0_1; \ + float16x4_t __s1_1 = __p1_1; \ + float16x4_t __rev1_1; __rev1_1 = __builtin_shufflevector(__s1_1, __s1_1, 3, 2, 1, 0); \ + float16_t __ret_1; \ + __ret_1 = __noswap_vmulh_f16(__s0_1, __noswap_vget_lane_f16(__rev1_1, __p2_1)); \ + __ret_1; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulh_laneq_f16(__p0_2, __p1_2, __p2_2) __extension__ ({ \ + float16_t __s0_2 = __p0_2; \ + float16x8_t __s1_2 = __p1_2; \ + float16_t __ret_2; \ + __ret_2 = vmulh_f16(__s0_2, vgetq_lane_f16(__s1_2, __p2_2)); \ + __ret_2; \ +}) +#else +#define vmulh_laneq_f16(__p0_3, __p1_3, __p2_3) __extension__ ({ \ + float16_t __s0_3 = __p0_3; \ + float16x8_t __s1_3 = __p1_3; \ + float16x8_t __rev1_3; __rev1_3 = __builtin_shufflevector(__s1_3, __s1_3, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16_t __ret_3; \ + __ret_3 = __noswap_vmulh_f16(__s0_3, __noswap_vgetq_lane_f16(__rev1_3, __p2_3)); \ + __ret_3; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vmulxh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmulxh_f16(__p0, __p1); + return __ret; +} +#else +__ai float16_t vmulxh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmulxh_f16(__p0, __p1); + return __ret; +} +__ai float16_t __noswap_vmulxh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmulxh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulxh_lane_f16(__p0_4, __p1_4, __p2_4) __extension__ ({ \ + float16_t __s0_4 = __p0_4; \ + float16x4_t __s1_4 = __p1_4; \ + float16_t __ret_4; \ + __ret_4 = vmulxh_f16(__s0_4, vget_lane_f16(__s1_4, __p2_4)); \ + __ret_4; \ +}) +#else +#define vmulxh_lane_f16(__p0_5, __p1_5, __p2_5) __extension__ ({ \ + float16_t __s0_5 = __p0_5; \ + float16x4_t __s1_5 = __p1_5; \ + float16x4_t __rev1_5; __rev1_5 = __builtin_shufflevector(__s1_5, __s1_5, 3, 2, 1, 0); \ + float16_t __ret_5; \ + __ret_5 = __noswap_vmulxh_f16(__s0_5, __noswap_vget_lane_f16(__rev1_5, __p2_5)); \ + __ret_5; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulxh_laneq_f16(__p0_6, __p1_6, __p2_6) __extension__ ({ \ + float16_t __s0_6 = __p0_6; \ + float16x8_t __s1_6 = __p1_6; \ + float16_t __ret_6; \ + __ret_6 = vmulxh_f16(__s0_6, vgetq_lane_f16(__s1_6, __p2_6)); \ + __ret_6; \ +}) +#else +#define vmulxh_laneq_f16(__p0_7, __p1_7, __p2_7) __extension__ ({ \ + float16_t __s0_7 = __p0_7; \ + float16x8_t __s1_7 = __p1_7; \ + float16x8_t __rev1_7; __rev1_7 = __builtin_shufflevector(__s1_7, __s1_7, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16_t __ret_7; \ + __ret_7 = __noswap_vmulxh_f16(__s0_7, __noswap_vgetq_lane_f16(__rev1_7, __p2_7)); \ + __ret_7; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vnegh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vnegh_f16(__p0); + return __ret; +} +#else +__ai float16_t vnegh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vnegh_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vrecpeh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrecpeh_f16(__p0); + return __ret; +} +#else +__ai float16_t vrecpeh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrecpeh_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vrecpsh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrecpsh_f16(__p0, __p1); + return __ret; +} +#else +__ai float16_t vrecpsh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrecpsh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vrecpxh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrecpxh_f16(__p0); + return __ret; +} +#else +__ai float16_t vrecpxh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrecpxh_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vrndh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrndh_f16(__p0); + return __ret; +} +#else +__ai float16_t vrndh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrndh_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vrndah_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrndah_f16(__p0); + return __ret; +} +#else +__ai float16_t vrndah_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrndah_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vrndih_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrndih_f16(__p0); + return __ret; +} +#else +__ai float16_t vrndih_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrndih_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vrndmh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrndmh_f16(__p0); + return __ret; +} +#else +__ai float16_t vrndmh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrndmh_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vrndnh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrndnh_f16(__p0); + return __ret; +} +#else +__ai float16_t vrndnh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrndnh_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vrndph_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrndph_f16(__p0); + return __ret; +} +#else +__ai float16_t vrndph_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrndph_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vrndxh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrndxh_f16(__p0); + return __ret; +} +#else +__ai float16_t vrndxh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrndxh_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vrsqrteh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrsqrteh_f16(__p0); + return __ret; +} +#else +__ai float16_t vrsqrteh_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrsqrteh_f16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vrsqrtsh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrsqrtsh_f16(__p0, __p1); + return __ret; +} +#else +__ai float16_t vrsqrtsh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vrsqrtsh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vsubh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vsubh_f16(__p0, __p1); + return __ret; +} +#else +__ai float16_t vsubh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vsubh_f16(__p0, __p1); + return __ret; +} +#endif + +#endif +#if defined(__ARM_FEATURE_FP16_SCALAR_ARITHMETIC) && defined(__aarch64__) +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_f16_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vcvth_n_f16_s32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_f16_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vcvth_n_f16_s32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_f16_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vcvth_n_f16_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_f16_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vcvth_n_f16_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_f16_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vcvth_n_f16_s16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_f16_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vcvth_n_f16_s16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_f16_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vcvth_n_f16_u32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_f16_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vcvth_n_f16_u32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_f16_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vcvth_n_f16_u64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_f16_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vcvth_n_f16_u64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_f16_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vcvth_n_f16_u16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_f16_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vcvth_n_f16_u16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s32_f16(__p0, __p1) __extension__ ({ \ + float16_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vcvth_n_s32_f16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s32_f16(__p0, __p1) __extension__ ({ \ + float16_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vcvth_n_s32_f16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s32_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vcvth_n_s32_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s32_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vcvth_n_s32_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s32_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vcvth_n_s32_s16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s32_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vcvth_n_s32_s16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s32_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vcvth_n_s32_u32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s32_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vcvth_n_s32_u32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s32_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vcvth_n_s32_u64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s32_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vcvth_n_s32_u64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s32_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vcvth_n_s32_u16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s32_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vcvth_n_s32_u16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s64_f16(__p0, __p1) __extension__ ({ \ + float16_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vcvth_n_s64_f16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s64_f16(__p0, __p1) __extension__ ({ \ + float16_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vcvth_n_s64_f16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s64_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vcvth_n_s64_s32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s64_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vcvth_n_s64_s32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s64_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vcvth_n_s64_s16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s64_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vcvth_n_s64_s16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s64_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vcvth_n_s64_u32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s64_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vcvth_n_s64_u32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s64_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vcvth_n_s64_u64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s64_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vcvth_n_s64_u64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s64_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vcvth_n_s64_u16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s64_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vcvth_n_s64_u16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s16_f16(__p0, __p1) __extension__ ({ \ + float16_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vcvth_n_s16_f16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s16_f16(__p0, __p1) __extension__ ({ \ + float16_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vcvth_n_s16_f16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s16_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vcvth_n_s16_s32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s16_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vcvth_n_s16_s32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s16_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vcvth_n_s16_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s16_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vcvth_n_s16_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s16_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vcvth_n_s16_u32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s16_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vcvth_n_s16_u32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s16_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vcvth_n_s16_u64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s16_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vcvth_n_s16_u64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_s16_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vcvth_n_s16_u16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_s16_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vcvth_n_s16_u16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u32_f16(__p0, __p1) __extension__ ({ \ + float16_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vcvth_n_u32_f16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u32_f16(__p0, __p1) __extension__ ({ \ + float16_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vcvth_n_u32_f16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u32_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vcvth_n_u32_s32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u32_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vcvth_n_u32_s32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u32_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vcvth_n_u32_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u32_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vcvth_n_u32_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u32_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vcvth_n_u32_s16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u32_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vcvth_n_u32_s16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u32_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vcvth_n_u32_u64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u32_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vcvth_n_u32_u64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u32_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vcvth_n_u32_u16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u32_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vcvth_n_u32_u16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u64_f16(__p0, __p1) __extension__ ({ \ + float16_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vcvth_n_u64_f16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u64_f16(__p0, __p1) __extension__ ({ \ + float16_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vcvth_n_u64_f16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u64_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vcvth_n_u64_s32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u64_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vcvth_n_u64_s32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u64_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vcvth_n_u64_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u64_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vcvth_n_u64_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u64_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vcvth_n_u64_s16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u64_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vcvth_n_u64_s16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u64_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vcvth_n_u64_u32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u64_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vcvth_n_u64_u32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u64_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vcvth_n_u64_u16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u64_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vcvth_n_u64_u16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u16_f16(__p0, __p1) __extension__ ({ \ + float16_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vcvth_n_u16_f16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u16_f16(__p0, __p1) __extension__ ({ \ + float16_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vcvth_n_u16_f16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u16_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vcvth_n_u16_s32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u16_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vcvth_n_u16_s32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u16_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vcvth_n_u16_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u16_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vcvth_n_u16_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u16_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vcvth_n_u16_s16(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u16_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vcvth_n_u16_s16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u16_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vcvth_n_u16_u32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u16_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vcvth_n_u16_u32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvth_n_u16_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vcvth_n_u16_u64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvth_n_u16_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vcvth_n_u16_u64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vdivh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vdivh_f16(__p0, __p1); + return __ret; +} +#else +__ai float16_t vdivh_f16(float16_t __p0, float16_t __p1) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vdivh_f16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vfmsh_f16(float16_t __p0, float16_t __p1, float16_t __p2) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vfmsh_f16(__p0, __p1, __p2); + return __ret; +} +#else +__ai float16_t vfmsh_f16(float16_t __p0, float16_t __p1, float16_t __p2) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vfmsh_f16(__p0, __p1, __p2); + return __ret; +} +__ai float16_t __noswap_vfmsh_f16(float16_t __p0, float16_t __p1, float16_t __p2) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vfmsh_f16(__p0, __p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vsqrth_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vsqrth_f16(__p0); + return __ret; +} +#else +__ai float16_t vsqrth_f16(float16_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vsqrth_f16(__p0); + return __ret; +} +#endif + +#endif +#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vabdq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 40); + return __ret; +} +#else +__ai float16x8_t vabdq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vabdq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vabd_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4_t vabd_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vabd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vabsq_f16(float16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vabsq_v((int8x16_t)__p0, 40); + return __ret; +} +#else +__ai float16x8_t vabsq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vabsq_v((int8x16_t)__rev0, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vabs_f16(float16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vabs_v((int8x8_t)__p0, 8); + return __ret; +} +#else +__ai float16x4_t vabs_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vabs_v((int8x8_t)__rev0, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vaddq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai float16x8_t vaddq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vadd_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai float16x4_t vadd_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vbslq_f16(uint16x8_t __p0, float16x8_t __p1, float16x8_t __p2) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vbslq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 40); + return __ret; +} +#else +__ai float16x8_t vbslq_f16(uint16x8_t __p0, float16x8_t __p1, float16x8_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vbslq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vbsl_f16(uint16x4_t __p0, float16x4_t __p1, float16x4_t __p2) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 8); + return __ret; +} +#else +__ai float16x4_t vbsl_f16(uint16x4_t __p0, float16x4_t __p1, float16x4_t __p2) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vbsl_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcageq_f16(float16x8_t __p0, float16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcageq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vcageq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcageq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcage_f16(float16x4_t __p0, float16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcage_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vcage_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcage_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcagtq_f16(float16x8_t __p0, float16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcagtq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vcagtq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcagtq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcagt_f16(float16x4_t __p0, float16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcagt_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vcagt_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcagt_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcaleq_f16(float16x8_t __p0, float16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcaleq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vcaleq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcaleq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcale_f16(float16x4_t __p0, float16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcale_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vcale_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcale_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcaltq_f16(float16x8_t __p0, float16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcaltq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vcaltq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcaltq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcalt_f16(float16x4_t __p0, float16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcalt_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vcalt_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcalt_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vceqq_f16(float16x8_t __p0, float16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint16x8_t vceqq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vceq_f16(float16x4_t __p0, float16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint16x4_t vceq_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcgeq_f16(float16x8_t __p0, float16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint16x8_t vcgeq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcge_f16(float16x4_t __p0, float16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint16x4_t vcge_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcgtq_f16(float16x8_t __p0, float16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint16x8_t vcgtq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcgt_f16(float16x4_t __p0, float16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint16x4_t vcgt_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcleq_f16(float16x8_t __p0, float16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint16x8_t vcleq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcle_f16(float16x4_t __p0, float16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint16x4_t vcle_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcltq_f16(float16x8_t __p0, float16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint16x8_t vcltq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vclt_f16(float16x4_t __p0, float16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint16x4_t vclt_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vcvtq_f16_u16(uint16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vcvtq_f16_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai float16x8_t vcvtq_f16_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vcvtq_f16_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vcvtq_f16_s16(int16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vcvtq_f16_v((int8x16_t)__p0, 33); + return __ret; +} +#else +__ai float16x8_t vcvtq_f16_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vcvtq_f16_v((int8x16_t)__rev0, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vcvt_f16_u16(uint16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vcvt_f16_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai float16x4_t vcvt_f16_u16(uint16x4_t __p0) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vcvt_f16_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vcvt_f16_s16(int16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vcvt_f16_v((int8x8_t)__p0, 1); + return __ret; +} +#else +__ai float16x4_t vcvt_f16_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vcvt_f16_v((int8x8_t)__rev0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtq_n_f16_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vcvtq_n_f16_v((int8x16_t)__s0, __p1, 49); \ + __ret; \ +}) +#else +#define vcvtq_n_f16_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vcvtq_n_f16_v((int8x16_t)__rev0, __p1, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtq_n_f16_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vcvtq_n_f16_v((int8x16_t)__s0, __p1, 33); \ + __ret; \ +}) +#else +#define vcvtq_n_f16_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vcvtq_n_f16_v((int8x16_t)__rev0, __p1, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvt_n_f16_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vcvt_n_f16_v((int8x8_t)__s0, __p1, 17); \ + __ret; \ +}) +#else +#define vcvt_n_f16_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vcvt_n_f16_v((int8x8_t)__rev0, __p1, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvt_n_f16_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vcvt_n_f16_v((int8x8_t)__s0, __p1, 1); \ + __ret; \ +}) +#else +#define vcvt_n_f16_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vcvt_n_f16_v((int8x8_t)__rev0, __p1, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtq_n_s16_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vcvtq_n_s16_v((int8x16_t)__s0, __p1, 33); \ + __ret; \ +}) +#else +#define vcvtq_n_s16_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = (int16x8_t) __builtin_neon_vcvtq_n_s16_v((int8x16_t)__rev0, __p1, 33); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvt_n_s16_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vcvt_n_s16_v((int8x8_t)__s0, __p1, 1); \ + __ret; \ +}) +#else +#define vcvt_n_s16_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = (int16x4_t) __builtin_neon_vcvt_n_s16_v((int8x8_t)__rev0, __p1, 1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtq_n_u16_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vcvtq_n_u16_v((int8x16_t)__s0, __p1, 49); \ + __ret; \ +}) +#else +#define vcvtq_n_u16_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = (uint16x8_t) __builtin_neon_vcvtq_n_u16_v((int8x16_t)__rev0, __p1, 49); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvt_n_u16_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vcvt_n_u16_v((int8x8_t)__s0, __p1, 17); \ + __ret; \ +}) +#else +#define vcvt_n_u16_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = (uint16x4_t) __builtin_neon_vcvt_n_u16_v((int8x8_t)__rev0, __p1, 17); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vcvtq_s16_f16(float16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vcvtq_s16_v((int8x16_t)__p0, 33); + return __ret; +} +#else +__ai int16x8_t vcvtq_s16_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vcvtq_s16_v((int8x16_t)__rev0, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vcvt_s16_f16(float16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vcvt_s16_v((int8x8_t)__p0, 1); + return __ret; +} +#else +__ai int16x4_t vcvt_s16_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vcvt_s16_v((int8x8_t)__rev0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcvtq_u16_f16(float16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcvtq_u16_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vcvtq_u16_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcvtq_u16_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcvt_u16_f16(float16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcvt_u16_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vcvt_u16_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcvt_u16_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vcvtaq_s16_f16(float16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vcvtaq_s16_v((int8x16_t)__p0, 33); + return __ret; +} +#else +__ai int16x8_t vcvtaq_s16_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vcvtaq_s16_v((int8x16_t)__rev0, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vcvta_s16_f16(float16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vcvta_s16_v((int8x8_t)__p0, 1); + return __ret; +} +#else +__ai int16x4_t vcvta_s16_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vcvta_s16_v((int8x8_t)__rev0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcvtaq_u16_f16(float16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcvtaq_u16_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vcvtaq_u16_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcvtaq_u16_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcvta_u16_f16(float16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcvta_u16_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vcvta_u16_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcvta_u16_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vcvtmq_s16_f16(float16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vcvtmq_s16_v((int8x16_t)__p0, 33); + return __ret; +} +#else +__ai int16x8_t vcvtmq_s16_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vcvtmq_s16_v((int8x16_t)__rev0, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vcvtm_s16_f16(float16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vcvtm_s16_v((int8x8_t)__p0, 1); + return __ret; +} +#else +__ai int16x4_t vcvtm_s16_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vcvtm_s16_v((int8x8_t)__rev0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcvtmq_u16_f16(float16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcvtmq_u16_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vcvtmq_u16_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcvtmq_u16_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcvtm_u16_f16(float16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcvtm_u16_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vcvtm_u16_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcvtm_u16_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vcvtnq_s16_f16(float16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vcvtnq_s16_v((int8x16_t)__p0, 33); + return __ret; +} +#else +__ai int16x8_t vcvtnq_s16_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vcvtnq_s16_v((int8x16_t)__rev0, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vcvtn_s16_f16(float16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vcvtn_s16_v((int8x8_t)__p0, 1); + return __ret; +} +#else +__ai int16x4_t vcvtn_s16_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vcvtn_s16_v((int8x8_t)__rev0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcvtnq_u16_f16(float16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcvtnq_u16_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vcvtnq_u16_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcvtnq_u16_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcvtn_u16_f16(float16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcvtn_u16_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vcvtn_u16_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcvtn_u16_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vcvtpq_s16_f16(float16x8_t __p0) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vcvtpq_s16_v((int8x16_t)__p0, 33); + return __ret; +} +#else +__ai int16x8_t vcvtpq_s16_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vcvtpq_s16_v((int8x16_t)__rev0, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vcvtp_s16_f16(float16x4_t __p0) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vcvtp_s16_v((int8x8_t)__p0, 1); + return __ret; +} +#else +__ai int16x4_t vcvtp_s16_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vcvtp_s16_v((int8x8_t)__rev0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcvtpq_u16_f16(float16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcvtpq_u16_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vcvtpq_u16_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcvtpq_u16_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcvtp_u16_f16(float16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcvtp_u16_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vcvtp_u16_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcvtp_u16_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vduph_lane_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vduph_lane_f16((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vduph_lane_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vduph_lane_f16((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vduph_laneq_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vduph_laneq_f16((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vduph_laneq_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16_t __ret; \ + __ret = (float16_t) __builtin_neon_vduph_laneq_f16((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vextq_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vextq_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 40); \ + __ret; \ +}) +#else +#define vextq_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vextq_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 40); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vext_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 8); \ + __ret; \ +}) +#else +#define vext_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vext_v((int8x8_t)__rev0, (int8x8_t)__rev1, __p2, 8); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vfmaq_f16(float16x8_t __p0, float16x8_t __p1, float16x8_t __p2) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vfmaq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 40); + return __ret; +} +#else +__ai float16x8_t vfmaq_f16(float16x8_t __p0, float16x8_t __p1, float16x8_t __p2) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vfmaq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai float16x8_t __noswap_vfmaq_f16(float16x8_t __p0, float16x8_t __p1, float16x8_t __p2) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vfmaq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 40); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vfma_f16(float16x4_t __p0, float16x4_t __p1, float16x4_t __p2) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vfma_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 8); + return __ret; +} +#else +__ai float16x4_t vfma_f16(float16x4_t __p0, float16x4_t __p1, float16x4_t __p2) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vfma_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai float16x4_t __noswap_vfma_f16(float16x4_t __p0, float16x4_t __p1, float16x4_t __p2) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vfma_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 8); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vmaxq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 40); + return __ret; +} +#else +__ai float16x8_t vmaxq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vmax_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vmax_v((int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4_t vmax_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vmaxnmq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vmaxnmq_v((int8x16_t)__p0, (int8x16_t)__p1, 40); + return __ret; +} +#else +__ai float16x8_t vmaxnmq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vmaxnmq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vmaxnm_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vmaxnm_v((int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4_t vmaxnm_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vmaxnm_v((int8x8_t)__rev0, (int8x8_t)__rev1, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vminq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vminq_v((int8x16_t)__p0, (int8x16_t)__p1, 40); + return __ret; +} +#else +__ai float16x8_t vminq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vmin_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vmin_v((int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4_t vmin_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vminnmq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vminnmq_v((int8x16_t)__p0, (int8x16_t)__p1, 40); + return __ret; +} +#else +__ai float16x8_t vminnmq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vminnmq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vminnm_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vminnm_v((int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4_t vminnm_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vminnm_v((int8x8_t)__rev0, (int8x8_t)__rev1, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vmulq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai float16x8_t vmulq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vmul_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai float16x4_t vmul_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulq_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x8_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmulq_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float16x8_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmul_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x4_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmul_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float16x4_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulq_laneq_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x8_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmulq_laneq_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmul_laneq_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x4_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmul_laneq_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x4_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vmulq_n_f16(float16x8_t __p0, float16_t __p1) { + float16x8_t __ret; + __ret = __p0 * (float16x8_t) {__p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1}; + return __ret; +} +#else +__ai float16x8_t vmulq_n_f16(float16x8_t __p0, float16_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __rev0 * (float16x8_t) {__p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1}; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vmul_n_f16(float16x4_t __p0, float16_t __p1) { + float16x4_t __ret; + __ret = __p0 * (float16x4_t) {__p1, __p1, __p1, __p1}; + return __ret; +} +#else +__ai float16x4_t vmul_n_f16(float16x4_t __p0, float16_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __rev0 * (float16x4_t) {__p1, __p1, __p1, __p1}; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vnegq_f16(float16x8_t __p0) { + float16x8_t __ret; + __ret = -__p0; + return __ret; +} +#else +__ai float16x8_t vnegq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = -__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vneg_f16(float16x4_t __p0) { + float16x4_t __ret; + __ret = -__p0; + return __ret; +} +#else +__ai float16x4_t vneg_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = -__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vpadd_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vpadd_v((int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4_t vpadd_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vpadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vpmax_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vpmax_v((int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4_t vpmax_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vpmax_v((int8x8_t)__rev0, (int8x8_t)__rev1, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vpmin_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vpmin_v((int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4_t vpmin_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vpmin_v((int8x8_t)__rev0, (int8x8_t)__rev1, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vrecpeq_f16(float16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrecpeq_v((int8x16_t)__p0, 40); + return __ret; +} +#else +__ai float16x8_t vrecpeq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrecpeq_v((int8x16_t)__rev0, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vrecpe_f16(float16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrecpe_v((int8x8_t)__p0, 8); + return __ret; +} +#else +__ai float16x4_t vrecpe_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrecpe_v((int8x8_t)__rev0, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vrecpsq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrecpsq_v((int8x16_t)__p0, (int8x16_t)__p1, 40); + return __ret; +} +#else +__ai float16x8_t vrecpsq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrecpsq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vrecps_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrecps_v((int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4_t vrecps_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrecps_v((int8x8_t)__rev0, (int8x8_t)__rev1, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vrev64q_f16(float16x8_t __p0) { + float16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0, 7, 6, 5, 4); + return __ret; +} +#else +__ai float16x8_t vrev64q_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 3, 2, 1, 0, 7, 6, 5, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vrev64_f16(float16x4_t __p0) { + float16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + return __ret; +} +#else +__ai float16x4_t vrev64_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 3, 2, 1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vrndq_f16(float16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrndq_v((int8x16_t)__p0, 40); + return __ret; +} +#else +__ai float16x8_t vrndq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrndq_v((int8x16_t)__rev0, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vrnd_f16(float16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrnd_v((int8x8_t)__p0, 8); + return __ret; +} +#else +__ai float16x4_t vrnd_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrnd_v((int8x8_t)__rev0, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vrndaq_f16(float16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrndaq_v((int8x16_t)__p0, 40); + return __ret; +} +#else +__ai float16x8_t vrndaq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrndaq_v((int8x16_t)__rev0, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vrnda_f16(float16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrnda_v((int8x8_t)__p0, 8); + return __ret; +} +#else +__ai float16x4_t vrnda_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrnda_v((int8x8_t)__rev0, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vrndiq_f16(float16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrndiq_v((int8x16_t)__p0, 40); + return __ret; +} +#else +__ai float16x8_t vrndiq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrndiq_v((int8x16_t)__rev0, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vrndi_f16(float16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrndi_v((int8x8_t)__p0, 8); + return __ret; +} +#else +__ai float16x4_t vrndi_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrndi_v((int8x8_t)__rev0, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vrndmq_f16(float16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrndmq_v((int8x16_t)__p0, 40); + return __ret; +} +#else +__ai float16x8_t vrndmq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrndmq_v((int8x16_t)__rev0, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vrndm_f16(float16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrndm_v((int8x8_t)__p0, 8); + return __ret; +} +#else +__ai float16x4_t vrndm_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrndm_v((int8x8_t)__rev0, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vrndnq_f16(float16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrndnq_v((int8x16_t)__p0, 40); + return __ret; +} +#else +__ai float16x8_t vrndnq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrndnq_v((int8x16_t)__rev0, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vrndn_f16(float16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrndn_v((int8x8_t)__p0, 8); + return __ret; +} +#else +__ai float16x4_t vrndn_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrndn_v((int8x8_t)__rev0, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vrndpq_f16(float16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrndpq_v((int8x16_t)__p0, 40); + return __ret; +} +#else +__ai float16x8_t vrndpq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrndpq_v((int8x16_t)__rev0, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vrndp_f16(float16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrndp_v((int8x8_t)__p0, 8); + return __ret; +} +#else +__ai float16x4_t vrndp_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrndp_v((int8x8_t)__rev0, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vrndxq_f16(float16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrndxq_v((int8x16_t)__p0, 40); + return __ret; +} +#else +__ai float16x8_t vrndxq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrndxq_v((int8x16_t)__rev0, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vrndx_f16(float16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrndx_v((int8x8_t)__p0, 8); + return __ret; +} +#else +__ai float16x4_t vrndx_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrndx_v((int8x8_t)__rev0, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vrsqrteq_f16(float16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrsqrteq_v((int8x16_t)__p0, 40); + return __ret; +} +#else +__ai float16x8_t vrsqrteq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrsqrteq_v((int8x16_t)__rev0, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vrsqrte_f16(float16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrsqrte_v((int8x8_t)__p0, 8); + return __ret; +} +#else +__ai float16x4_t vrsqrte_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrsqrte_v((int8x8_t)__rev0, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vrsqrtsq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrsqrtsq_v((int8x16_t)__p0, (int8x16_t)__p1, 40); + return __ret; +} +#else +__ai float16x8_t vrsqrtsq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vrsqrtsq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vrsqrts_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrsqrts_v((int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4_t vrsqrts_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vrsqrts_v((int8x8_t)__rev0, (int8x8_t)__rev1, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vsubq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai float16x8_t vsubq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vsub_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai float16x4_t vsub_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8x2_t vtrnq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 40); + return __ret; +} +#else +__ai float16x8x2_t vtrnq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8x2_t __ret; + __builtin_neon_vtrnq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 40); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4x2_t vtrn_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4x2_t vtrn_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4x2_t __ret; + __builtin_neon_vtrn_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 8); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vtrn1q_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 8, 2, 10, 4, 12, 6, 14); + return __ret; +} +#else +__ai float16x8_t vtrn1q_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 8, 2, 10, 4, 12, 6, 14); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vtrn1_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 4, 2, 6); + return __ret; +} +#else +__ai float16x4_t vtrn1_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 4, 2, 6); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vtrn2q_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 9, 3, 11, 5, 13, 7, 15); + return __ret; +} +#else +__ai float16x8_t vtrn2q_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 9, 3, 11, 5, 13, 7, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vtrn2_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 5, 3, 7); + return __ret; +} +#else +__ai float16x4_t vtrn2_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 5, 3, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8x2_t vuzpq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 40); + return __ret; +} +#else +__ai float16x8x2_t vuzpq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8x2_t __ret; + __builtin_neon_vuzpq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 40); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4x2_t vuzp_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4x2_t vuzp_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4x2_t __ret; + __builtin_neon_vuzp_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 8); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vuzp1q_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6, 8, 10, 12, 14); + return __ret; +} +#else +__ai float16x8_t vuzp1q_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6, 8, 10, 12, 14); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vuzp1_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6); + return __ret; +} +#else +__ai float16x4_t vuzp1_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vuzp2q_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7, 9, 11, 13, 15); + return __ret; +} +#else +__ai float16x8_t vuzp2q_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7, 9, 11, 13, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vuzp2_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7); + return __ret; +} +#else +__ai float16x4_t vuzp2_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8x2_t vzipq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__p0, (int8x16_t)__p1, 40); + return __ret; +} +#else +__ai float16x8x2_t vzipq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8x2_t __ret; + __builtin_neon_vzipq_v(&__ret, (int8x16_t)__rev0, (int8x16_t)__rev1, 40); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4x2_t vzip_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4x2_t vzip_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4x2_t __ret; + __builtin_neon_vzip_v(&__ret, (int8x8_t)__rev0, (int8x8_t)__rev1, 8); + + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vzip1q_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 8, 1, 9, 2, 10, 3, 11); + return __ret; +} +#else +__ai float16x8_t vzip1q_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 8, 1, 9, 2, 10, 3, 11); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vzip1_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 4, 1, 5); + return __ret; +} +#else +__ai float16x4_t vzip1_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 4, 1, 5); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vzip2q_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 4, 12, 5, 13, 6, 14, 7, 15); + return __ret; +} +#else +__ai float16x8_t vzip2q_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 4, 12, 5, 13, 6, 14, 7, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vzip2_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 2, 6, 3, 7); + return __ret; +} +#else +__ai float16x4_t vzip2_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 2, 6, 3, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#endif +#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(__aarch64__) +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vceqzq_f16(float16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vceqzq_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vceqzq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vceqzq_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vceqz_f16(float16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vceqz_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vceqz_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcgezq_f16(float16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcgezq_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vcgezq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcgezq_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcgez_f16(float16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcgez_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vcgez_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcgez_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcgtzq_f16(float16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcgtzq_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vcgtzq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcgtzq_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcgtz_f16(float16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcgtz_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vcgtz_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcgtz_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vclezq_f16(float16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vclezq_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vclezq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vclezq_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vclez_f16(float16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vclez_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vclez_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vclez_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcltzq_f16(float16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcltzq_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vcltzq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcltzq_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcltz_f16(float16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcltz_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vcltz_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcltz_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vdivq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = __p0 / __p1; + return __ret; +} +#else +__ai float16x8_t vdivq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __rev0 / __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vdiv_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = __p0 / __p1; + return __ret; +} +#else +__ai float16x4_t vdiv_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __rev0 / __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmaq_lane_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x4_t __s2 = __p2; \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vfmaq_lane_v((int8x16_t)__s0, (int8x16_t)__s1, (int8x8_t)__s2, __p3, 40); \ + __ret; \ +}) +#else +#define vfmaq_lane_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x4_t __s2 = __p2; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vfmaq_lane_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x8_t)__rev2, __p3, 40); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vfmaq_lane_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x4_t __s2 = __p2; \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vfmaq_lane_v((int8x16_t)__s0, (int8x16_t)__s1, (int8x8_t)__s2, __p3, 40); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfma_lane_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x4_t __s2 = __p2; \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vfma_lane_v((int8x8_t)__s0, (int8x8_t)__s1, (int8x8_t)__s2, __p3, 8); \ + __ret; \ +}) +#else +#define vfma_lane_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x4_t __s2 = __p2; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vfma_lane_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, __p3, 8); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vfma_lane_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x4_t __s2 = __p2; \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vfma_lane_v((int8x8_t)__s0, (int8x8_t)__s1, (int8x8_t)__s2, __p3, 8); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmaq_laneq_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x8_t __s2 = __p2; \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vfmaq_laneq_v((int8x16_t)__s0, (int8x16_t)__s1, (int8x16_t)__s2, __p3, 40); \ + __ret; \ +}) +#else +#define vfmaq_laneq_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x8_t __s2 = __p2; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vfmaq_laneq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, __p3, 40); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vfmaq_laneq_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x8_t __s2 = __p2; \ + float16x8_t __ret; \ + __ret = (float16x8_t) __builtin_neon_vfmaq_laneq_v((int8x16_t)__s0, (int8x16_t)__s1, (int8x16_t)__s2, __p3, 40); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfma_laneq_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x8_t __s2 = __p2; \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vfma_laneq_v((int8x8_t)__s0, (int8x8_t)__s1, (int8x16_t)__s2, __p3, 8); \ + __ret; \ +}) +#else +#define vfma_laneq_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x8_t __s2 = __p2; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vfma_laneq_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x16_t)__rev2, __p3, 8); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vfma_laneq_f16(__p0, __p1, __p2, __p3) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x8_t __s2 = __p2; \ + float16x4_t __ret; \ + __ret = (float16x4_t) __builtin_neon_vfma_laneq_v((int8x8_t)__s0, (int8x8_t)__s1, (int8x16_t)__s2, __p3, 8); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vfmaq_n_f16(float16x8_t __p0, float16x8_t __p1, float16_t __p2) { + float16x8_t __ret; + __ret = vfmaq_f16(__p0, __p1, (float16x8_t) {__p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2}); + return __ret; +} +#else +__ai float16x8_t vfmaq_n_f16(float16x8_t __p0, float16x8_t __p1, float16_t __p2) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __noswap_vfmaq_f16(__rev0, __rev1, (float16x8_t) {__p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vfma_n_f16(float16x4_t __p0, float16x4_t __p1, float16_t __p2) { + float16x4_t __ret; + __ret = vfma_f16(__p0, __p1, (float16x4_t) {__p2, __p2, __p2, __p2}); + return __ret; +} +#else +__ai float16x4_t vfma_n_f16(float16x4_t __p0, float16x4_t __p1, float16_t __p2) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __noswap_vfma_f16(__rev0, __rev1, (float16x4_t) {__p2, __p2, __p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vfmsq_f16(float16x8_t __p0, float16x8_t __p1, float16x8_t __p2) { + float16x8_t __ret; + __ret = vfmaq_f16(__p0, -__p1, __p2); + return __ret; +} +#else +__ai float16x8_t vfmsq_f16(float16x8_t __p0, float16x8_t __p1, float16x8_t __p2) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __noswap_vfmaq_f16(__rev0, -__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vfms_f16(float16x4_t __p0, float16x4_t __p1, float16x4_t __p2) { + float16x4_t __ret; + __ret = vfma_f16(__p0, -__p1, __p2); + return __ret; +} +#else +__ai float16x4_t vfms_f16(float16x4_t __p0, float16x4_t __p1, float16x4_t __p2) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __noswap_vfma_f16(__rev0, -__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmsq_lane_f16(__p0_8, __p1_8, __p2_8, __p3_8) __extension__ ({ \ + float16x8_t __s0_8 = __p0_8; \ + float16x8_t __s1_8 = __p1_8; \ + float16x4_t __s2_8 = __p2_8; \ + float16x8_t __ret_8; \ + __ret_8 = vfmaq_lane_f16(__s0_8, -__s1_8, __s2_8, __p3_8); \ + __ret_8; \ +}) +#else +#define vfmsq_lane_f16(__p0_9, __p1_9, __p2_9, __p3_9) __extension__ ({ \ + float16x8_t __s0_9 = __p0_9; \ + float16x8_t __s1_9 = __p1_9; \ + float16x4_t __s2_9 = __p2_9; \ + float16x8_t __rev0_9; __rev0_9 = __builtin_shufflevector(__s0_9, __s0_9, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __rev1_9; __rev1_9 = __builtin_shufflevector(__s1_9, __s1_9, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x4_t __rev2_9; __rev2_9 = __builtin_shufflevector(__s2_9, __s2_9, 3, 2, 1, 0); \ + float16x8_t __ret_9; \ + __ret_9 = __noswap_vfmaq_lane_f16(__rev0_9, -__rev1_9, __rev2_9, __p3_9); \ + __ret_9 = __builtin_shufflevector(__ret_9, __ret_9, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_9; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfms_lane_f16(__p0_10, __p1_10, __p2_10, __p3_10) __extension__ ({ \ + float16x4_t __s0_10 = __p0_10; \ + float16x4_t __s1_10 = __p1_10; \ + float16x4_t __s2_10 = __p2_10; \ + float16x4_t __ret_10; \ + __ret_10 = vfma_lane_f16(__s0_10, -__s1_10, __s2_10, __p3_10); \ + __ret_10; \ +}) +#else +#define vfms_lane_f16(__p0_11, __p1_11, __p2_11, __p3_11) __extension__ ({ \ + float16x4_t __s0_11 = __p0_11; \ + float16x4_t __s1_11 = __p1_11; \ + float16x4_t __s2_11 = __p2_11; \ + float16x4_t __rev0_11; __rev0_11 = __builtin_shufflevector(__s0_11, __s0_11, 3, 2, 1, 0); \ + float16x4_t __rev1_11; __rev1_11 = __builtin_shufflevector(__s1_11, __s1_11, 3, 2, 1, 0); \ + float16x4_t __rev2_11; __rev2_11 = __builtin_shufflevector(__s2_11, __s2_11, 3, 2, 1, 0); \ + float16x4_t __ret_11; \ + __ret_11 = __noswap_vfma_lane_f16(__rev0_11, -__rev1_11, __rev2_11, __p3_11); \ + __ret_11 = __builtin_shufflevector(__ret_11, __ret_11, 3, 2, 1, 0); \ + __ret_11; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmsq_laneq_f16(__p0_12, __p1_12, __p2_12, __p3_12) __extension__ ({ \ + float16x8_t __s0_12 = __p0_12; \ + float16x8_t __s1_12 = __p1_12; \ + float16x8_t __s2_12 = __p2_12; \ + float16x8_t __ret_12; \ + __ret_12 = vfmaq_laneq_f16(__s0_12, -__s1_12, __s2_12, __p3_12); \ + __ret_12; \ +}) +#else +#define vfmsq_laneq_f16(__p0_13, __p1_13, __p2_13, __p3_13) __extension__ ({ \ + float16x8_t __s0_13 = __p0_13; \ + float16x8_t __s1_13 = __p1_13; \ + float16x8_t __s2_13 = __p2_13; \ + float16x8_t __rev0_13; __rev0_13 = __builtin_shufflevector(__s0_13, __s0_13, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __rev1_13; __rev1_13 = __builtin_shufflevector(__s1_13, __s1_13, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __rev2_13; __rev2_13 = __builtin_shufflevector(__s2_13, __s2_13, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __ret_13; \ + __ret_13 = __noswap_vfmaq_laneq_f16(__rev0_13, -__rev1_13, __rev2_13, __p3_13); \ + __ret_13 = __builtin_shufflevector(__ret_13, __ret_13, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_13; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfms_laneq_f16(__p0_14, __p1_14, __p2_14, __p3_14) __extension__ ({ \ + float16x4_t __s0_14 = __p0_14; \ + float16x4_t __s1_14 = __p1_14; \ + float16x8_t __s2_14 = __p2_14; \ + float16x4_t __ret_14; \ + __ret_14 = vfma_laneq_f16(__s0_14, -__s1_14, __s2_14, __p3_14); \ + __ret_14; \ +}) +#else +#define vfms_laneq_f16(__p0_15, __p1_15, __p2_15, __p3_15) __extension__ ({ \ + float16x4_t __s0_15 = __p0_15; \ + float16x4_t __s1_15 = __p1_15; \ + float16x8_t __s2_15 = __p2_15; \ + float16x4_t __rev0_15; __rev0_15 = __builtin_shufflevector(__s0_15, __s0_15, 3, 2, 1, 0); \ + float16x4_t __rev1_15; __rev1_15 = __builtin_shufflevector(__s1_15, __s1_15, 3, 2, 1, 0); \ + float16x8_t __rev2_15; __rev2_15 = __builtin_shufflevector(__s2_15, __s2_15, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x4_t __ret_15; \ + __ret_15 = __noswap_vfma_laneq_f16(__rev0_15, -__rev1_15, __rev2_15, __p3_15); \ + __ret_15 = __builtin_shufflevector(__ret_15, __ret_15, 3, 2, 1, 0); \ + __ret_15; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vfmsq_n_f16(float16x8_t __p0, float16x8_t __p1, float16_t __p2) { + float16x8_t __ret; + __ret = vfmaq_f16(__p0, -__p1, (float16x8_t) {__p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2}); + return __ret; +} +#else +__ai float16x8_t vfmsq_n_f16(float16x8_t __p0, float16x8_t __p1, float16_t __p2) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __noswap_vfmaq_f16(__rev0, -__rev1, (float16x8_t) {__p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vfms_n_f16(float16x4_t __p0, float16x4_t __p1, float16_t __p2) { + float16x4_t __ret; + __ret = vfma_f16(__p0, -__p1, (float16x4_t) {__p2, __p2, __p2, __p2}); + return __ret; +} +#else +__ai float16x4_t vfms_n_f16(float16x4_t __p0, float16x4_t __p1, float16_t __p2) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __noswap_vfma_f16(__rev0, -__rev1, (float16x4_t) {__p2, __p2, __p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vmaxnmvq_f16(float16x8_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmaxnmvq_f16((int8x16_t)__p0); + return __ret; +} +#else +__ai float16_t vmaxnmvq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmaxnmvq_f16((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vmaxnmv_f16(float16x4_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmaxnmv_f16((int8x8_t)__p0); + return __ret; +} +#else +__ai float16_t vmaxnmv_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmaxnmv_f16((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vmaxvq_f16(float16x8_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmaxvq_f16((int8x16_t)__p0); + return __ret; +} +#else +__ai float16_t vmaxvq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmaxvq_f16((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vmaxv_f16(float16x4_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmaxv_f16((int8x8_t)__p0); + return __ret; +} +#else +__ai float16_t vmaxv_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16_t __ret; + __ret = (float16_t) __builtin_neon_vmaxv_f16((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vminnmvq_f16(float16x8_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vminnmvq_f16((int8x16_t)__p0); + return __ret; +} +#else +__ai float16_t vminnmvq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16_t __ret; + __ret = (float16_t) __builtin_neon_vminnmvq_f16((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vminnmv_f16(float16x4_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vminnmv_f16((int8x8_t)__p0); + return __ret; +} +#else +__ai float16_t vminnmv_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16_t __ret; + __ret = (float16_t) __builtin_neon_vminnmv_f16((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vminvq_f16(float16x8_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vminvq_f16((int8x16_t)__p0); + return __ret; +} +#else +__ai float16_t vminvq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16_t __ret; + __ret = (float16_t) __builtin_neon_vminvq_f16((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16_t vminv_f16(float16x4_t __p0) { + float16_t __ret; + __ret = (float16_t) __builtin_neon_vminv_f16((int8x8_t)__p0); + return __ret; +} +#else +__ai float16_t vminv_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16_t __ret; + __ret = (float16_t) __builtin_neon_vminv_f16((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vmulxq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vmulxq_v((int8x16_t)__p0, (int8x16_t)__p1, 40); + return __ret; +} +#else +__ai float16x8_t vmulxq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vmulxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai float16x8_t __noswap_vmulxq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vmulxq_v((int8x16_t)__p0, (int8x16_t)__p1, 40); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vmulx_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vmulx_v((int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4_t vmulx_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vmulx_v((int8x8_t)__rev0, (int8x8_t)__rev1, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai float16x4_t __noswap_vmulx_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vmulx_v((int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulxq_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x8_t __ret; \ + __ret = vmulxq_f16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmulxq_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float16x8_t __ret; \ + __ret = __noswap_vmulxq_f16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulx_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x4_t __ret; \ + __ret = vmulx_f16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmulx_lane_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __s1 = __p1; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float16x4_t __ret; \ + __ret = __noswap_vmulx_f16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulxq_laneq_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x8_t __ret; \ + __ret = vmulxq_f16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmulxq_laneq_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __ret; \ + __ret = __noswap_vmulxq_f16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulx_laneq_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x4_t __ret; \ + __ret = vmulx_f16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmulx_laneq_f16(__p0, __p1, __p2) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x8_t __s1 = __p1; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x4_t __ret; \ + __ret = __noswap_vmulx_f16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vmulxq_n_f16(float16x8_t __p0, float16_t __p1) { + float16x8_t __ret; + __ret = vmulxq_f16(__p0, (float16x8_t) {__p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1}); + return __ret; +} +#else +__ai float16x8_t vmulxq_n_f16(float16x8_t __p0, float16_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __noswap_vmulxq_f16(__rev0, (float16x8_t) {__p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1}); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vmulx_n_f16(float16x4_t __p0, float16_t __p1) { + float16x4_t __ret; + __ret = vmulx_f16(__p0, (float16x4_t) {__p1, __p1, __p1, __p1}); + return __ret; +} +#else +__ai float16x4_t vmulx_n_f16(float16x4_t __p0, float16_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = __noswap_vmulx_f16(__rev0, (float16x4_t) {__p1, __p1, __p1, __p1}); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vpmaxnmq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vpmaxnmq_v((int8x16_t)__p0, (int8x16_t)__p1, 40); + return __ret; +} +#else +__ai float16x8_t vpmaxnmq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vpmaxnmq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vpmaxnm_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vpmaxnm_v((int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4_t vpmaxnm_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vpmaxnm_v((int8x8_t)__rev0, (int8x8_t)__rev1, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vpminnmq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vpminnmq_v((int8x16_t)__p0, (int8x16_t)__p1, 40); + return __ret; +} +#else +__ai float16x8_t vpminnmq_f16(float16x8_t __p0, float16x8_t __p1) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vpminnmq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vpminnm_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vpminnm_v((int8x8_t)__p0, (int8x8_t)__p1, 8); + return __ret; +} +#else +__ai float16x4_t vpminnm_f16(float16x4_t __p0, float16x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vpminnm_v((int8x8_t)__rev0, (int8x8_t)__rev1, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vsqrtq_f16(float16x8_t __p0) { + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vsqrtq_v((int8x16_t)__p0, 40); + return __ret; +} +#else +__ai float16x8_t vsqrtq_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float16x8_t __ret; + __ret = (float16x8_t) __builtin_neon_vsqrtq_v((int8x16_t)__rev0, 40); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x4_t vsqrt_f16(float16x4_t __p0) { + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vsqrt_v((int8x8_t)__p0, 8); + return __ret; +} +#else +__ai float16x4_t vsqrt_f16(float16x4_t __p0) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float16x4_t __ret; + __ret = (float16x4_t) __builtin_neon_vsqrt_v((int8x8_t)__rev0, 8); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#endif +#if defined(__ARM_FEATURE_QRDMX) +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqrdmlahq_s32(int32x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int32x4_t __ret; + __ret = vqaddq_s32(__p0, vqrdmulhq_s32(__p1, __p2)); + return __ret; +} +#else +__ai int32x4_t vqrdmlahq_s32(int32x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vqaddq_s32(__rev0, __noswap_vqrdmulhq_s32(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vqrdmlahq_s16(int16x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int16x8_t __ret; + __ret = vqaddq_s16(__p0, vqrdmulhq_s16(__p1, __p2)); + return __ret; +} +#else +__ai int16x8_t vqrdmlahq_s16(int16x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vqaddq_s16(__rev0, __noswap_vqrdmulhq_s16(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vqrdmlah_s32(int32x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int32x2_t __ret; + __ret = vqadd_s32(__p0, vqrdmulh_s32(__p1, __p2)); + return __ret; +} +#else +__ai int32x2_t vqrdmlah_s32(int32x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int32x2_t __ret; + __ret = __noswap_vqadd_s32(__rev0, __noswap_vqrdmulh_s32(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vqrdmlah_s16(int16x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int16x4_t __ret; + __ret = vqadd_s16(__p0, vqrdmulh_s16(__p1, __p2)); + return __ret; +} +#else +__ai int16x4_t vqrdmlah_s16(int16x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __noswap_vqadd_s16(__rev0, __noswap_vqrdmulh_s16(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlahq_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = vqaddq_s32(__s0, vqrdmulhq_s32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlahq_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqaddq_s32(__rev0, __noswap_vqrdmulhq_s32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlahq_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x8_t __ret; \ + __ret = vqaddq_s16(__s0, vqrdmulhq_s16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlahq_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __noswap_vqaddq_s16(__rev0, __noswap_vqrdmulhq_s16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlah_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x2_t __ret; \ + __ret = vqadd_s32(__s0, vqrdmulh_s32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlah_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int32x2_t __ret; \ + __ret = __noswap_vqadd_s32(__rev0, __noswap_vqrdmulh_s32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlah_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x4_t __ret; \ + __ret = vqadd_s16(__s0, vqrdmulh_s16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlah_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __noswap_vqadd_s16(__rev0, __noswap_vqrdmulh_s16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqrdmlshq_s32(int32x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int32x4_t __ret; + __ret = vqsubq_s32(__p0, vqrdmulhq_s32(__p1, __p2)); + return __ret; +} +#else +__ai int32x4_t vqrdmlshq_s32(int32x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vqsubq_s32(__rev0, __noswap_vqrdmulhq_s32(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vqrdmlshq_s16(int16x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int16x8_t __ret; + __ret = vqsubq_s16(__p0, vqrdmulhq_s16(__p1, __p2)); + return __ret; +} +#else +__ai int16x8_t vqrdmlshq_s16(int16x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vqsubq_s16(__rev0, __noswap_vqrdmulhq_s16(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vqrdmlsh_s32(int32x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int32x2_t __ret; + __ret = vqsub_s32(__p0, vqrdmulh_s32(__p1, __p2)); + return __ret; +} +#else +__ai int32x2_t vqrdmlsh_s32(int32x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int32x2_t __ret; + __ret = __noswap_vqsub_s32(__rev0, __noswap_vqrdmulh_s32(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vqrdmlsh_s16(int16x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int16x4_t __ret; + __ret = vqsub_s16(__p0, vqrdmulh_s16(__p1, __p2)); + return __ret; +} +#else +__ai int16x4_t vqrdmlsh_s16(int16x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __noswap_vqsub_s16(__rev0, __noswap_vqrdmulh_s16(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlshq_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = vqsubq_s32(__s0, vqrdmulhq_s32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlshq_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqsubq_s32(__rev0, __noswap_vqrdmulhq_s32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlshq_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x8_t __ret; \ + __ret = vqsubq_s16(__s0, vqrdmulhq_s16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlshq_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __noswap_vqsubq_s16(__rev0, __noswap_vqrdmulhq_s16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlsh_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x2_t __ret; \ + __ret = vqsub_s32(__s0, vqrdmulh_s32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlsh_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int32x2_t __ret; \ + __ret = __noswap_vqsub_s32(__rev0, __noswap_vqrdmulh_s32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlsh_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x4_t __ret; \ + __ret = vqsub_s16(__s0, vqrdmulh_s16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlsh_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __noswap_vqsub_s16(__rev0, __noswap_vqrdmulh_s16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#endif +#if defined(__ARM_FEATURE_QRDMX) && defined(__aarch64__) +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlahq_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = vqaddq_s32(__s0, vqrdmulhq_s32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlahq_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqaddq_s32(__rev0, __noswap_vqrdmulhq_s32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlahq_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x8_t __ret; \ + __ret = vqaddq_s16(__s0, vqrdmulhq_s16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlahq_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __noswap_vqaddq_s16(__rev0, __noswap_vqrdmulhq_s16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlah_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x2_t __ret; \ + __ret = vqadd_s32(__s0, vqrdmulh_s32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlah_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x2_t __ret; \ + __ret = __noswap_vqadd_s32(__rev0, __noswap_vqrdmulh_s32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlah_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x4_t __ret; \ + __ret = vqadd_s16(__s0, vqrdmulh_s16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlah_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __noswap_vqadd_s16(__rev0, __noswap_vqrdmulh_s16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlshq_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = vqsubq_s32(__s0, vqrdmulhq_s32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlshq_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqsubq_s32(__rev0, __noswap_vqrdmulhq_s32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlshq_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x8_t __ret; \ + __ret = vqsubq_s16(__s0, vqrdmulhq_s16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlshq_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __noswap_vqsubq_s16(__rev0, __noswap_vqrdmulhq_s16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlsh_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x2_t __ret; \ + __ret = vqsub_s32(__s0, vqrdmulh_s32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlsh_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x2_t __ret; \ + __ret = __noswap_vqsub_s32(__rev0, __noswap_vqrdmulh_s32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlsh_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x4_t __ret; \ + __ret = vqsub_s16(__s0, vqrdmulh_s16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3))); \ + __ret; \ +}) +#else +#define vqrdmlsh_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __noswap_vqsub_s16(__rev0, __noswap_vqrdmulh_s16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3))); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#endif +#if defined(__aarch64__) +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vabdq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vabdq_v((int8x16_t)__p0, (int8x16_t)__p1, 42); + return __ret; +} +#else +__ai float64x2_t vabdq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vabdq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vabd_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#else +__ai float64x1_t vabd_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vabd_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vabdd_f64(float64_t __p0, float64_t __p1) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vabdd_f64(__p0, __p1); + return __ret; +} +#else +__ai float64_t vabdd_f64(float64_t __p0, float64_t __p1) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vabdd_f64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vabds_f32(float32_t __p0, float32_t __p1) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vabds_f32(__p0, __p1); + return __ret; +} +#else +__ai float32_t vabds_f32(float32_t __p0, float32_t __p1) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vabds_f32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vabsq_f64(float64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vabsq_v((int8x16_t)__p0, 42); + return __ret; +} +#else +__ai float64x2_t vabsq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vabsq_v((int8x16_t)__rev0, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vabsq_s64(int64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vabsq_v((int8x16_t)__p0, 35); + return __ret; +} +#else +__ai int64x2_t vabsq_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vabsq_v((int8x16_t)__rev0, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vabs_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vabs_v((int8x8_t)__p0, 10); + return __ret; +} +#else +__ai float64x1_t vabs_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vabs_v((int8x8_t)__p0, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vabs_s64(int64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vabs_v((int8x8_t)__p0, 3); + return __ret; +} +#else +__ai int64x1_t vabs_s64(int64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vabs_v((int8x8_t)__p0, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vabsd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vabsd_s64(__p0); + return __ret; +} +#else +__ai int64_t vabsd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vabsd_s64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vaddq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai float64x2_t vaddq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = __rev0 + __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vadd_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#else +__ai float64x1_t vadd_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = __p0 + __p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vaddd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vaddd_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vaddd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vaddd_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vaddd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vaddd_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vaddd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vaddd_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vaddhn_high_u32(uint16x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint16x8_t __ret; + __ret = vcombine_u16(__p0, vaddhn_u32(__p1, __p2)); + return __ret; +} +#else +__ai uint16x8_t vaddhn_high_u32(uint16x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vcombine_u16(__rev0, __noswap_vaddhn_u32(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vaddhn_high_u64(uint32x2_t __p0, uint64x2_t __p1, uint64x2_t __p2) { + uint32x4_t __ret; + __ret = vcombine_u32(__p0, vaddhn_u64(__p1, __p2)); + return __ret; +} +#else +__ai uint32x4_t vaddhn_high_u64(uint32x2_t __p0, uint64x2_t __p1, uint64x2_t __p2) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vcombine_u32(__rev0, __noswap_vaddhn_u64(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vaddhn_high_u16(uint8x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint8x16_t __ret; + __ret = vcombine_u8(__p0, vaddhn_u16(__p1, __p2)); + return __ret; +} +#else +__ai uint8x16_t vaddhn_high_u16(uint8x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __noswap_vcombine_u8(__rev0, __noswap_vaddhn_u16(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vaddhn_high_s32(int16x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int16x8_t __ret; + __ret = vcombine_s16(__p0, vaddhn_s32(__p1, __p2)); + return __ret; +} +#else +__ai int16x8_t vaddhn_high_s32(int16x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vcombine_s16(__rev0, __noswap_vaddhn_s32(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vaddhn_high_s64(int32x2_t __p0, int64x2_t __p1, int64x2_t __p2) { + int32x4_t __ret; + __ret = vcombine_s32(__p0, vaddhn_s64(__p1, __p2)); + return __ret; +} +#else +__ai int32x4_t vaddhn_high_s64(int32x2_t __p0, int64x2_t __p1, int64x2_t __p2) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vcombine_s32(__rev0, __noswap_vaddhn_s64(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vaddhn_high_s16(int8x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int8x16_t __ret; + __ret = vcombine_s8(__p0, vaddhn_s16(__p1, __p2)); + return __ret; +} +#else +__ai int8x16_t vaddhn_high_s16(int8x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __noswap_vcombine_s8(__rev0, __noswap_vaddhn_s16(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16_t vaddlvq_u8(uint8x16_t __p0) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vaddlvq_u8((int8x16_t)__p0); + return __ret; +} +#else +__ai uint16_t vaddlvq_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vaddlvq_u8((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vaddlvq_u32(uint32x4_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vaddlvq_u32((int8x16_t)__p0); + return __ret; +} +#else +__ai uint64_t vaddlvq_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vaddlvq_u32((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vaddlvq_u16(uint16x8_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vaddlvq_u16((int8x16_t)__p0); + return __ret; +} +#else +__ai uint32_t vaddlvq_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vaddlvq_u16((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vaddlvq_s8(int8x16_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vaddlvq_s8((int8x16_t)__p0); + return __ret; +} +#else +__ai int16_t vaddlvq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int16_t __ret; + __ret = (int16_t) __builtin_neon_vaddlvq_s8((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vaddlvq_s32(int32x4_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vaddlvq_s32((int8x16_t)__p0); + return __ret; +} +#else +__ai int64_t vaddlvq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int64_t __ret; + __ret = (int64_t) __builtin_neon_vaddlvq_s32((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vaddlvq_s16(int16x8_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vaddlvq_s16((int8x16_t)__p0); + return __ret; +} +#else +__ai int32_t vaddlvq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int32_t __ret; + __ret = (int32_t) __builtin_neon_vaddlvq_s16((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16_t vaddlv_u8(uint8x8_t __p0) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vaddlv_u8((int8x8_t)__p0); + return __ret; +} +#else +__ai uint16_t vaddlv_u8(uint8x8_t __p0) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vaddlv_u8((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vaddlv_u32(uint32x2_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vaddlv_u32((int8x8_t)__p0); + return __ret; +} +#else +__ai uint64_t vaddlv_u32(uint32x2_t __p0) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vaddlv_u32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vaddlv_u16(uint16x4_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vaddlv_u16((int8x8_t)__p0); + return __ret; +} +#else +__ai uint32_t vaddlv_u16(uint16x4_t __p0) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vaddlv_u16((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vaddlv_s8(int8x8_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vaddlv_s8((int8x8_t)__p0); + return __ret; +} +#else +__ai int16_t vaddlv_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16_t __ret; + __ret = (int16_t) __builtin_neon_vaddlv_s8((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vaddlv_s32(int32x2_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vaddlv_s32((int8x8_t)__p0); + return __ret; +} +#else +__ai int64_t vaddlv_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64_t __ret; + __ret = (int64_t) __builtin_neon_vaddlv_s32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vaddlv_s16(int16x4_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vaddlv_s16((int8x8_t)__p0); + return __ret; +} +#else +__ai int32_t vaddlv_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32_t __ret; + __ret = (int32_t) __builtin_neon_vaddlv_s16((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8_t vaddvq_u8(uint8x16_t __p0) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vaddvq_u8((int8x16_t)__p0); + return __ret; +} +#else +__ai uint8_t vaddvq_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vaddvq_u8((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vaddvq_u32(uint32x4_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vaddvq_u32((int8x16_t)__p0); + return __ret; +} +#else +__ai uint32_t vaddvq_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vaddvq_u32((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vaddvq_u64(uint64x2_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vaddvq_u64((int8x16_t)__p0); + return __ret; +} +#else +__ai uint64_t vaddvq_u64(uint64x2_t __p0) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vaddvq_u64((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16_t vaddvq_u16(uint16x8_t __p0) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vaddvq_u16((int8x16_t)__p0); + return __ret; +} +#else +__ai uint16_t vaddvq_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vaddvq_u16((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vaddvq_s8(int8x16_t __p0) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vaddvq_s8((int8x16_t)__p0); + return __ret; +} +#else +__ai int8_t vaddvq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8_t __ret; + __ret = (int8_t) __builtin_neon_vaddvq_s8((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vaddvq_f64(float64x2_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vaddvq_f64((int8x16_t)__p0); + return __ret; +} +#else +__ai float64_t vaddvq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64_t __ret; + __ret = (float64_t) __builtin_neon_vaddvq_f64((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vaddvq_f32(float32x4_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vaddvq_f32((int8x16_t)__p0); + return __ret; +} +#else +__ai float32_t vaddvq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vaddvq_f32((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vaddvq_s32(int32x4_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vaddvq_s32((int8x16_t)__p0); + return __ret; +} +#else +__ai int32_t vaddvq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32_t __ret; + __ret = (int32_t) __builtin_neon_vaddvq_s32((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vaddvq_s64(int64x2_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vaddvq_s64((int8x16_t)__p0); + return __ret; +} +#else +__ai int64_t vaddvq_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64_t __ret; + __ret = (int64_t) __builtin_neon_vaddvq_s64((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vaddvq_s16(int16x8_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vaddvq_s16((int8x16_t)__p0); + return __ret; +} +#else +__ai int16_t vaddvq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16_t __ret; + __ret = (int16_t) __builtin_neon_vaddvq_s16((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8_t vaddv_u8(uint8x8_t __p0) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vaddv_u8((int8x8_t)__p0); + return __ret; +} +#else +__ai uint8_t vaddv_u8(uint8x8_t __p0) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vaddv_u8((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vaddv_u32(uint32x2_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vaddv_u32((int8x8_t)__p0); + return __ret; +} +#else +__ai uint32_t vaddv_u32(uint32x2_t __p0) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vaddv_u32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16_t vaddv_u16(uint16x4_t __p0) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vaddv_u16((int8x8_t)__p0); + return __ret; +} +#else +__ai uint16_t vaddv_u16(uint16x4_t __p0) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vaddv_u16((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vaddv_s8(int8x8_t __p0) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vaddv_s8((int8x8_t)__p0); + return __ret; +} +#else +__ai int8_t vaddv_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8_t __ret; + __ret = (int8_t) __builtin_neon_vaddv_s8((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vaddv_f32(float32x2_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vaddv_f32((int8x8_t)__p0); + return __ret; +} +#else +__ai float32_t vaddv_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vaddv_f32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vaddv_s32(int32x2_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vaddv_s32((int8x8_t)__p0); + return __ret; +} +#else +__ai int32_t vaddv_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32_t __ret; + __ret = (int32_t) __builtin_neon_vaddv_s32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vaddv_s16(int16x4_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vaddv_s16((int8x8_t)__p0); + return __ret; +} +#else +__ai int16_t vaddv_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16_t __ret; + __ret = (int16_t) __builtin_neon_vaddv_s16((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vbsl_p64(uint64x1_t __p0, poly64x1_t __p1, poly64x1_t __p2) { + poly64x1_t __ret; + __ret = (poly64x1_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 6); + return __ret; +} +#else +__ai poly64x1_t vbsl_p64(uint64x1_t __p0, poly64x1_t __p1, poly64x1_t __p2) { + poly64x1_t __ret; + __ret = (poly64x1_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 6); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vbslq_p64(uint64x2_t __p0, poly64x2_t __p1, poly64x2_t __p2) { + poly64x2_t __ret; + __ret = (poly64x2_t) __builtin_neon_vbslq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 38); + return __ret; +} +#else +__ai poly64x2_t vbslq_p64(uint64x2_t __p0, poly64x2_t __p1, poly64x2_t __p2) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + poly64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + poly64x2_t __ret; + __ret = (poly64x2_t) __builtin_neon_vbslq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 38); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vbslq_f64(uint64x2_t __p0, float64x2_t __p1, float64x2_t __p2) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vbslq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 42); + return __ret; +} +#else +__ai float64x2_t vbslq_f64(uint64x2_t __p0, float64x2_t __p1, float64x2_t __p2) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vbslq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vbsl_f64(uint64x1_t __p0, float64x1_t __p1, float64x1_t __p2) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 10); + return __ret; +} +#else +__ai float64x1_t vbsl_f64(uint64x1_t __p0, float64x1_t __p1, float64x1_t __p2) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vbsl_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcageq_f64(float64x2_t __p0, float64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcageq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vcageq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcageq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcage_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcage_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vcage_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcage_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcaged_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcaged_f64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vcaged_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcaged_f64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcages_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcages_f32(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vcages_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcages_f32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcagtq_f64(float64x2_t __p0, float64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcagtq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vcagtq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcagtq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcagt_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcagt_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vcagt_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcagt_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcagtd_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcagtd_f64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vcagtd_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcagtd_f64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcagts_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcagts_f32(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vcagts_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcagts_f32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcaleq_f64(float64x2_t __p0, float64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcaleq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vcaleq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcaleq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcale_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcale_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vcale_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcale_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcaled_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcaled_f64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vcaled_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcaled_f64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcales_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcales_f32(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vcales_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcales_f32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcaltq_f64(float64x2_t __p0, float64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcaltq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vcaltq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcaltq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcalt_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcalt_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vcalt_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcalt_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcaltd_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcaltd_f64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vcaltd_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcaltd_f64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcalts_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcalts_f32(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vcalts_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcalts_f32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vceq_p64(poly64x1_t __p0, poly64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint64x1_t vceq_p64(poly64x1_t __p0, poly64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 == __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vceqq_p64(poly64x2_t __p0, poly64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint64x2_t vceqq_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vceqq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint64x2_t vceqq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vceqq_f64(float64x2_t __p0, float64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint64x2_t vceqq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vceqq_s64(int64x2_t __p0, int64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint64x2_t vceqq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 == __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vceq_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint64x1_t vceq_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 == __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vceq_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint64x1_t vceq_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 == __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vceq_s64(int64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 == __p1); + return __ret; +} +#else +__ai uint64x1_t vceq_s64(int64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 == __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vceqd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vceqd_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vceqd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vceqd_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vceqd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vceqd_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vceqd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vceqd_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vceqd_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vceqd_f64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vceqd_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vceqd_f64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vceqs_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vceqs_f32(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vceqs_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vceqs_f32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vceqz_p8(poly8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 16); + return __ret; +} +#else +__ai uint8x8_t vceqz_p8(poly8x8_t __p0) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vceqz_v((int8x8_t)__rev0, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vceqz_p64(poly64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vceqz_p64(poly64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vceqz_p16(poly16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vceqz_p16(poly16x4_t __p0) { + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vceqz_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vceqzq_p8(poly8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vceqzq_v((int8x16_t)__p0, 48); + return __ret; +} +#else +__ai uint8x16_t vceqzq_p8(poly8x16_t __p0) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vceqzq_v((int8x16_t)__rev0, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vceqzq_p64(poly64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vceqzq_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vceqzq_p64(poly64x2_t __p0) { + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vceqzq_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vceqzq_p16(poly16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vceqzq_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vceqzq_p16(poly16x8_t __p0) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vceqzq_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vceqzq_u8(uint8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vceqzq_v((int8x16_t)__p0, 48); + return __ret; +} +#else +__ai uint8x16_t vceqzq_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vceqzq_v((int8x16_t)__rev0, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vceqzq_u32(uint32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vceqzq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vceqzq_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vceqzq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vceqzq_u64(uint64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vceqzq_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vceqzq_u64(uint64x2_t __p0) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vceqzq_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vceqzq_u16(uint16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vceqzq_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vceqzq_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vceqzq_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vceqzq_s8(int8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vceqzq_v((int8x16_t)__p0, 48); + return __ret; +} +#else +__ai uint8x16_t vceqzq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vceqzq_v((int8x16_t)__rev0, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vceqzq_f64(float64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vceqzq_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vceqzq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vceqzq_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vceqzq_f32(float32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vceqzq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vceqzq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vceqzq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vceqzq_s32(int32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vceqzq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vceqzq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vceqzq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vceqzq_s64(int64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vceqzq_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vceqzq_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vceqzq_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vceqzq_s16(int16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vceqzq_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vceqzq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vceqzq_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vceqz_u8(uint8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 16); + return __ret; +} +#else +__ai uint8x8_t vceqz_u8(uint8x8_t __p0) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vceqz_v((int8x8_t)__rev0, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vceqz_u32(uint32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vceqz_u32(uint32x2_t __p0) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vceqz_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vceqz_u64(uint64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vceqz_u64(uint64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vceqz_u16(uint16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vceqz_u16(uint16x4_t __p0) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vceqz_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vceqz_s8(int8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 16); + return __ret; +} +#else +__ai uint8x8_t vceqz_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vceqz_v((int8x8_t)__rev0, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vceqz_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vceqz_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vceqz_f32(float32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vceqz_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vceqz_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vceqz_s32(int32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vceqz_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vceqz_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vceqz_s64(int64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vceqz_s64(int64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vceqz_s16(int16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vceqz_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vceqz_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vceqz_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vceqzd_u64(uint64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vceqzd_u64(__p0); + return __ret; +} +#else +__ai uint64_t vceqzd_u64(uint64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vceqzd_u64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vceqzd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vceqzd_s64(__p0); + return __ret; +} +#else +__ai int64_t vceqzd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vceqzd_s64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vceqzd_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vceqzd_f64(__p0); + return __ret; +} +#else +__ai uint64_t vceqzd_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vceqzd_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vceqzs_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vceqzs_f32(__p0); + return __ret; +} +#else +__ai uint32_t vceqzs_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vceqzs_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcgeq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint64x2_t vcgeq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcgeq_f64(float64x2_t __p0, float64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint64x2_t vcgeq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcgeq_s64(int64x2_t __p0, int64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint64x2_t vcgeq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 >= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcge_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint64x1_t vcge_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 >= __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcge_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint64x1_t vcge_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 >= __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcge_s64(int64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 >= __p1); + return __ret; +} +#else +__ai uint64x1_t vcge_s64(int64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 >= __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vcged_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcged_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vcged_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcged_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcged_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcged_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vcged_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcged_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcged_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcged_f64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vcged_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcged_f64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcges_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcges_f32(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vcges_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcges_f32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vcgezq_s8(int8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vcgezq_v((int8x16_t)__p0, 48); + return __ret; +} +#else +__ai uint8x16_t vcgezq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vcgezq_v((int8x16_t)__rev0, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcgezq_f64(float64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcgezq_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vcgezq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcgezq_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcgezq_f32(float32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcgezq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vcgezq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcgezq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcgezq_s32(int32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcgezq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vcgezq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcgezq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcgezq_s64(int64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcgezq_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vcgezq_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcgezq_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcgezq_s16(int16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcgezq_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vcgezq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcgezq_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vcgez_s8(int8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vcgez_v((int8x8_t)__p0, 16); + return __ret; +} +#else +__ai uint8x8_t vcgez_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vcgez_v((int8x8_t)__rev0, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcgez_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcgez_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vcgez_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcgez_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcgez_f32(float32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcgez_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vcgez_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcgez_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcgez_s32(int32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcgez_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vcgez_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcgez_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcgez_s64(int64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcgez_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vcgez_s64(int64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcgez_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcgez_s16(int16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcgez_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vcgez_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcgez_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vcgezd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcgezd_s64(__p0); + return __ret; +} +#else +__ai int64_t vcgezd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcgezd_s64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcgezd_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcgezd_f64(__p0); + return __ret; +} +#else +__ai uint64_t vcgezd_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcgezd_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcgezs_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcgezs_f32(__p0); + return __ret; +} +#else +__ai uint32_t vcgezs_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcgezs_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcgtq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint64x2_t vcgtq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcgtq_f64(float64x2_t __p0, float64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint64x2_t vcgtq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcgtq_s64(int64x2_t __p0, int64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint64x2_t vcgtq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 > __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcgt_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint64x1_t vcgt_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 > __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcgt_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint64x1_t vcgt_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 > __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcgt_s64(int64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 > __p1); + return __ret; +} +#else +__ai uint64x1_t vcgt_s64(int64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 > __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vcgtd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcgtd_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vcgtd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcgtd_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcgtd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcgtd_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vcgtd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcgtd_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcgtd_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcgtd_f64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vcgtd_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcgtd_f64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcgts_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcgts_f32(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vcgts_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcgts_f32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vcgtzq_s8(int8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vcgtzq_v((int8x16_t)__p0, 48); + return __ret; +} +#else +__ai uint8x16_t vcgtzq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vcgtzq_v((int8x16_t)__rev0, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcgtzq_f64(float64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcgtzq_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vcgtzq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcgtzq_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcgtzq_f32(float32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcgtzq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vcgtzq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcgtzq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcgtzq_s32(int32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcgtzq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vcgtzq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcgtzq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcgtzq_s64(int64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcgtzq_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vcgtzq_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcgtzq_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcgtzq_s16(int16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcgtzq_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vcgtzq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcgtzq_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vcgtz_s8(int8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vcgtz_v((int8x8_t)__p0, 16); + return __ret; +} +#else +__ai uint8x8_t vcgtz_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vcgtz_v((int8x8_t)__rev0, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcgtz_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcgtz_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vcgtz_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcgtz_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcgtz_f32(float32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcgtz_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vcgtz_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcgtz_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcgtz_s32(int32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcgtz_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vcgtz_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcgtz_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcgtz_s64(int64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcgtz_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vcgtz_s64(int64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcgtz_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcgtz_s16(int16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcgtz_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vcgtz_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcgtz_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vcgtzd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcgtzd_s64(__p0); + return __ret; +} +#else +__ai int64_t vcgtzd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcgtzd_s64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcgtzd_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcgtzd_f64(__p0); + return __ret; +} +#else +__ai uint64_t vcgtzd_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcgtzd_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcgtzs_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcgtzs_f32(__p0); + return __ret; +} +#else +__ai uint32_t vcgtzs_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcgtzs_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcleq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint64x2_t vcleq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcleq_f64(float64x2_t __p0, float64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint64x2_t vcleq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcleq_s64(int64x2_t __p0, int64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint64x2_t vcleq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 <= __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcle_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint64x1_t vcle_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 <= __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcle_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint64x1_t vcle_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 <= __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcle_s64(int64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 <= __p1); + return __ret; +} +#else +__ai uint64x1_t vcle_s64(int64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 <= __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcled_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcled_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vcled_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcled_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vcled_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcled_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vcled_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcled_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcled_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcled_f64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vcled_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcled_f64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcles_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcles_f32(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vcles_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcles_f32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vclezq_s8(int8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vclezq_v((int8x16_t)__p0, 48); + return __ret; +} +#else +__ai uint8x16_t vclezq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vclezq_v((int8x16_t)__rev0, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vclezq_f64(float64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vclezq_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vclezq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vclezq_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vclezq_f32(float32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vclezq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vclezq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vclezq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vclezq_s32(int32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vclezq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vclezq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vclezq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vclezq_s64(int64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vclezq_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vclezq_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vclezq_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vclezq_s16(int16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vclezq_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vclezq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vclezq_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vclez_s8(int8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vclez_v((int8x8_t)__p0, 16); + return __ret; +} +#else +__ai uint8x8_t vclez_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vclez_v((int8x8_t)__rev0, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vclez_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vclez_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vclez_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vclez_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vclez_f32(float32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vclez_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vclez_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vclez_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vclez_s32(int32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vclez_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vclez_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vclez_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vclez_s64(int64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vclez_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vclez_s64(int64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vclez_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vclez_s16(int16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vclez_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vclez_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vclez_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vclezd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vclezd_s64(__p0); + return __ret; +} +#else +__ai int64_t vclezd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vclezd_s64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vclezd_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vclezd_f64(__p0); + return __ret; +} +#else +__ai uint64_t vclezd_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vclezd_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vclezs_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vclezs_f32(__p0); + return __ret; +} +#else +__ai uint32_t vclezs_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vclezs_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcltq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint64x2_t vcltq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcltq_f64(float64x2_t __p0, float64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint64x2_t vcltq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcltq_s64(int64x2_t __p0, int64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint64x2_t vcltq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__rev0 < __rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vclt_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint64x1_t vclt_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 < __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vclt_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint64x1_t vclt_f64(float64x1_t __p0, float64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 < __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vclt_s64(int64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 < __p1); + return __ret; +} +#else +__ai uint64x1_t vclt_s64(int64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t)(__p0 < __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcltd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcltd_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vcltd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcltd_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vcltd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcltd_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vcltd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcltd_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcltd_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcltd_f64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vcltd_f64(float64_t __p0, float64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcltd_f64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vclts_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vclts_f32(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vclts_f32(float32_t __p0, float32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vclts_f32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vcltzq_s8(int8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vcltzq_v((int8x16_t)__p0, 48); + return __ret; +} +#else +__ai uint8x16_t vcltzq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vcltzq_v((int8x16_t)__rev0, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcltzq_f64(float64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcltzq_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vcltzq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcltzq_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcltzq_f32(float32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcltzq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vcltzq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcltzq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vcltzq_s32(int32x4_t __p0) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcltzq_v((int8x16_t)__p0, 50); + return __ret; +} +#else +__ai uint32x4_t vcltzq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vcltzq_v((int8x16_t)__rev0, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcltzq_s64(int64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcltzq_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vcltzq_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcltzq_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vcltzq_s16(int16x8_t __p0) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcltzq_v((int8x16_t)__p0, 49); + return __ret; +} +#else +__ai uint16x8_t vcltzq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vcltzq_v((int8x16_t)__rev0, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vcltz_s8(int8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vcltz_v((int8x8_t)__p0, 16); + return __ret; +} +#else +__ai uint8x8_t vcltz_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vcltz_v((int8x8_t)__rev0, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcltz_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcltz_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vcltz_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcltz_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcltz_f32(float32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcltz_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vcltz_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcltz_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vcltz_s32(int32x2_t __p0) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcltz_v((int8x8_t)__p0, 18); + return __ret; +} +#else +__ai uint32x2_t vcltz_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vcltz_v((int8x8_t)__rev0, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcltz_s64(int64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcltz_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vcltz_s64(int64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcltz_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vcltz_s16(int16x4_t __p0) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcltz_v((int8x8_t)__p0, 17); + return __ret; +} +#else +__ai uint16x4_t vcltz_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vcltz_v((int8x8_t)__rev0, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vcltzd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcltzd_s64(__p0); + return __ret; +} +#else +__ai int64_t vcltzd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcltzd_s64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcltzd_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcltzd_f64(__p0); + return __ret; +} +#else +__ai uint64_t vcltzd_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcltzd_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcltzs_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcltzs_f32(__p0); + return __ret; +} +#else +__ai uint32_t vcltzs_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcltzs_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vcombine_p64(poly64x1_t __p0, poly64x1_t __p1) { + poly64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1); + return __ret; +} +#else +__ai poly64x2_t vcombine_p64(poly64x1_t __p0, poly64x1_t __p1) { + poly64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vcombine_f64(float64x1_t __p0, float64x1_t __p1) { + float64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1); + return __ret; +} +#else +__ai float64x2_t vcombine_f64(float64x1_t __p0, float64x1_t __p1) { + float64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_p8(__p0_16, __p1_16, __p2_16, __p3_16) __extension__ ({ \ + poly8x16_t __s0_16 = __p0_16; \ + poly8x8_t __s2_16 = __p2_16; \ + poly8x16_t __ret_16; \ + __ret_16 = vsetq_lane_p8(vget_lane_p8(__s2_16, __p3_16), __s0_16, __p1_16); \ + __ret_16; \ +}) +#else +#define vcopyq_lane_p8(__p0_17, __p1_17, __p2_17, __p3_17) __extension__ ({ \ + poly8x16_t __s0_17 = __p0_17; \ + poly8x8_t __s2_17 = __p2_17; \ + poly8x16_t __rev0_17; __rev0_17 = __builtin_shufflevector(__s0_17, __s0_17, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8_t __rev2_17; __rev2_17 = __builtin_shufflevector(__s2_17, __s2_17, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16_t __ret_17; \ + __ret_17 = __noswap_vsetq_lane_p8(__noswap_vget_lane_p8(__rev2_17, __p3_17), __rev0_17, __p1_17); \ + __ret_17 = __builtin_shufflevector(__ret_17, __ret_17, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_17; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_p16(__p0_18, __p1_18, __p2_18, __p3_18) __extension__ ({ \ + poly16x8_t __s0_18 = __p0_18; \ + poly16x4_t __s2_18 = __p2_18; \ + poly16x8_t __ret_18; \ + __ret_18 = vsetq_lane_p16(vget_lane_p16(__s2_18, __p3_18), __s0_18, __p1_18); \ + __ret_18; \ +}) +#else +#define vcopyq_lane_p16(__p0_19, __p1_19, __p2_19, __p3_19) __extension__ ({ \ + poly16x8_t __s0_19 = __p0_19; \ + poly16x4_t __s2_19 = __p2_19; \ + poly16x8_t __rev0_19; __rev0_19 = __builtin_shufflevector(__s0_19, __s0_19, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x4_t __rev2_19; __rev2_19 = __builtin_shufflevector(__s2_19, __s2_19, 3, 2, 1, 0); \ + poly16x8_t __ret_19; \ + __ret_19 = __noswap_vsetq_lane_p16(__noswap_vget_lane_p16(__rev2_19, __p3_19), __rev0_19, __p1_19); \ + __ret_19 = __builtin_shufflevector(__ret_19, __ret_19, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_19; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_u8(__p0_20, __p1_20, __p2_20, __p3_20) __extension__ ({ \ + uint8x16_t __s0_20 = __p0_20; \ + uint8x8_t __s2_20 = __p2_20; \ + uint8x16_t __ret_20; \ + __ret_20 = vsetq_lane_u8(vget_lane_u8(__s2_20, __p3_20), __s0_20, __p1_20); \ + __ret_20; \ +}) +#else +#define vcopyq_lane_u8(__p0_21, __p1_21, __p2_21, __p3_21) __extension__ ({ \ + uint8x16_t __s0_21 = __p0_21; \ + uint8x8_t __s2_21 = __p2_21; \ + uint8x16_t __rev0_21; __rev0_21 = __builtin_shufflevector(__s0_21, __s0_21, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __rev2_21; __rev2_21 = __builtin_shufflevector(__s2_21, __s2_21, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret_21; \ + __ret_21 = __noswap_vsetq_lane_u8(__noswap_vget_lane_u8(__rev2_21, __p3_21), __rev0_21, __p1_21); \ + __ret_21 = __builtin_shufflevector(__ret_21, __ret_21, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_21; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_u32(__p0_22, __p1_22, __p2_22, __p3_22) __extension__ ({ \ + uint32x4_t __s0_22 = __p0_22; \ + uint32x2_t __s2_22 = __p2_22; \ + uint32x4_t __ret_22; \ + __ret_22 = vsetq_lane_u32(vget_lane_u32(__s2_22, __p3_22), __s0_22, __p1_22); \ + __ret_22; \ +}) +#else +#define vcopyq_lane_u32(__p0_23, __p1_23, __p2_23, __p3_23) __extension__ ({ \ + uint32x4_t __s0_23 = __p0_23; \ + uint32x2_t __s2_23 = __p2_23; \ + uint32x4_t __rev0_23; __rev0_23 = __builtin_shufflevector(__s0_23, __s0_23, 3, 2, 1, 0); \ + uint32x2_t __rev2_23; __rev2_23 = __builtin_shufflevector(__s2_23, __s2_23, 1, 0); \ + uint32x4_t __ret_23; \ + __ret_23 = __noswap_vsetq_lane_u32(__noswap_vget_lane_u32(__rev2_23, __p3_23), __rev0_23, __p1_23); \ + __ret_23 = __builtin_shufflevector(__ret_23, __ret_23, 3, 2, 1, 0); \ + __ret_23; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_u64(__p0_24, __p1_24, __p2_24, __p3_24) __extension__ ({ \ + uint64x2_t __s0_24 = __p0_24; \ + uint64x1_t __s2_24 = __p2_24; \ + uint64x2_t __ret_24; \ + __ret_24 = vsetq_lane_u64(vget_lane_u64(__s2_24, __p3_24), __s0_24, __p1_24); \ + __ret_24; \ +}) +#else +#define vcopyq_lane_u64(__p0_25, __p1_25, __p2_25, __p3_25) __extension__ ({ \ + uint64x2_t __s0_25 = __p0_25; \ + uint64x1_t __s2_25 = __p2_25; \ + uint64x2_t __rev0_25; __rev0_25 = __builtin_shufflevector(__s0_25, __s0_25, 1, 0); \ + uint64x2_t __ret_25; \ + __ret_25 = __noswap_vsetq_lane_u64(__noswap_vget_lane_u64(__s2_25, __p3_25), __rev0_25, __p1_25); \ + __ret_25 = __builtin_shufflevector(__ret_25, __ret_25, 1, 0); \ + __ret_25; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_u16(__p0_26, __p1_26, __p2_26, __p3_26) __extension__ ({ \ + uint16x8_t __s0_26 = __p0_26; \ + uint16x4_t __s2_26 = __p2_26; \ + uint16x8_t __ret_26; \ + __ret_26 = vsetq_lane_u16(vget_lane_u16(__s2_26, __p3_26), __s0_26, __p1_26); \ + __ret_26; \ +}) +#else +#define vcopyq_lane_u16(__p0_27, __p1_27, __p2_27, __p3_27) __extension__ ({ \ + uint16x8_t __s0_27 = __p0_27; \ + uint16x4_t __s2_27 = __p2_27; \ + uint16x8_t __rev0_27; __rev0_27 = __builtin_shufflevector(__s0_27, __s0_27, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x4_t __rev2_27; __rev2_27 = __builtin_shufflevector(__s2_27, __s2_27, 3, 2, 1, 0); \ + uint16x8_t __ret_27; \ + __ret_27 = __noswap_vsetq_lane_u16(__noswap_vget_lane_u16(__rev2_27, __p3_27), __rev0_27, __p1_27); \ + __ret_27 = __builtin_shufflevector(__ret_27, __ret_27, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_27; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_s8(__p0_28, __p1_28, __p2_28, __p3_28) __extension__ ({ \ + int8x16_t __s0_28 = __p0_28; \ + int8x8_t __s2_28 = __p2_28; \ + int8x16_t __ret_28; \ + __ret_28 = vsetq_lane_s8(vget_lane_s8(__s2_28, __p3_28), __s0_28, __p1_28); \ + __ret_28; \ +}) +#else +#define vcopyq_lane_s8(__p0_29, __p1_29, __p2_29, __p3_29) __extension__ ({ \ + int8x16_t __s0_29 = __p0_29; \ + int8x8_t __s2_29 = __p2_29; \ + int8x16_t __rev0_29; __rev0_29 = __builtin_shufflevector(__s0_29, __s0_29, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __rev2_29; __rev2_29 = __builtin_shufflevector(__s2_29, __s2_29, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret_29; \ + __ret_29 = __noswap_vsetq_lane_s8(__noswap_vget_lane_s8(__rev2_29, __p3_29), __rev0_29, __p1_29); \ + __ret_29 = __builtin_shufflevector(__ret_29, __ret_29, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_29; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_f32(__p0_30, __p1_30, __p2_30, __p3_30) __extension__ ({ \ + float32x4_t __s0_30 = __p0_30; \ + float32x2_t __s2_30 = __p2_30; \ + float32x4_t __ret_30; \ + __ret_30 = vsetq_lane_f32(vget_lane_f32(__s2_30, __p3_30), __s0_30, __p1_30); \ + __ret_30; \ +}) +#else +#define vcopyq_lane_f32(__p0_31, __p1_31, __p2_31, __p3_31) __extension__ ({ \ + float32x4_t __s0_31 = __p0_31; \ + float32x2_t __s2_31 = __p2_31; \ + float32x4_t __rev0_31; __rev0_31 = __builtin_shufflevector(__s0_31, __s0_31, 3, 2, 1, 0); \ + float32x2_t __rev2_31; __rev2_31 = __builtin_shufflevector(__s2_31, __s2_31, 1, 0); \ + float32x4_t __ret_31; \ + __ret_31 = __noswap_vsetq_lane_f32(__noswap_vget_lane_f32(__rev2_31, __p3_31), __rev0_31, __p1_31); \ + __ret_31 = __builtin_shufflevector(__ret_31, __ret_31, 3, 2, 1, 0); \ + __ret_31; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_s32(__p0_32, __p1_32, __p2_32, __p3_32) __extension__ ({ \ + int32x4_t __s0_32 = __p0_32; \ + int32x2_t __s2_32 = __p2_32; \ + int32x4_t __ret_32; \ + __ret_32 = vsetq_lane_s32(vget_lane_s32(__s2_32, __p3_32), __s0_32, __p1_32); \ + __ret_32; \ +}) +#else +#define vcopyq_lane_s32(__p0_33, __p1_33, __p2_33, __p3_33) __extension__ ({ \ + int32x4_t __s0_33 = __p0_33; \ + int32x2_t __s2_33 = __p2_33; \ + int32x4_t __rev0_33; __rev0_33 = __builtin_shufflevector(__s0_33, __s0_33, 3, 2, 1, 0); \ + int32x2_t __rev2_33; __rev2_33 = __builtin_shufflevector(__s2_33, __s2_33, 1, 0); \ + int32x4_t __ret_33; \ + __ret_33 = __noswap_vsetq_lane_s32(__noswap_vget_lane_s32(__rev2_33, __p3_33), __rev0_33, __p1_33); \ + __ret_33 = __builtin_shufflevector(__ret_33, __ret_33, 3, 2, 1, 0); \ + __ret_33; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_s64(__p0_34, __p1_34, __p2_34, __p3_34) __extension__ ({ \ + int64x2_t __s0_34 = __p0_34; \ + int64x1_t __s2_34 = __p2_34; \ + int64x2_t __ret_34; \ + __ret_34 = vsetq_lane_s64(vget_lane_s64(__s2_34, __p3_34), __s0_34, __p1_34); \ + __ret_34; \ +}) +#else +#define vcopyq_lane_s64(__p0_35, __p1_35, __p2_35, __p3_35) __extension__ ({ \ + int64x2_t __s0_35 = __p0_35; \ + int64x1_t __s2_35 = __p2_35; \ + int64x2_t __rev0_35; __rev0_35 = __builtin_shufflevector(__s0_35, __s0_35, 1, 0); \ + int64x2_t __ret_35; \ + __ret_35 = __noswap_vsetq_lane_s64(__noswap_vget_lane_s64(__s2_35, __p3_35), __rev0_35, __p1_35); \ + __ret_35 = __builtin_shufflevector(__ret_35, __ret_35, 1, 0); \ + __ret_35; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_s16(__p0_36, __p1_36, __p2_36, __p3_36) __extension__ ({ \ + int16x8_t __s0_36 = __p0_36; \ + int16x4_t __s2_36 = __p2_36; \ + int16x8_t __ret_36; \ + __ret_36 = vsetq_lane_s16(vget_lane_s16(__s2_36, __p3_36), __s0_36, __p1_36); \ + __ret_36; \ +}) +#else +#define vcopyq_lane_s16(__p0_37, __p1_37, __p2_37, __p3_37) __extension__ ({ \ + int16x8_t __s0_37 = __p0_37; \ + int16x4_t __s2_37 = __p2_37; \ + int16x8_t __rev0_37; __rev0_37 = __builtin_shufflevector(__s0_37, __s0_37, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __rev2_37; __rev2_37 = __builtin_shufflevector(__s2_37, __s2_37, 3, 2, 1, 0); \ + int16x8_t __ret_37; \ + __ret_37 = __noswap_vsetq_lane_s16(__noswap_vget_lane_s16(__rev2_37, __p3_37), __rev0_37, __p1_37); \ + __ret_37 = __builtin_shufflevector(__ret_37, __ret_37, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_37; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_p8(__p0_38, __p1_38, __p2_38, __p3_38) __extension__ ({ \ + poly8x8_t __s0_38 = __p0_38; \ + poly8x8_t __s2_38 = __p2_38; \ + poly8x8_t __ret_38; \ + __ret_38 = vset_lane_p8(vget_lane_p8(__s2_38, __p3_38), __s0_38, __p1_38); \ + __ret_38; \ +}) +#else +#define vcopy_lane_p8(__p0_39, __p1_39, __p2_39, __p3_39) __extension__ ({ \ + poly8x8_t __s0_39 = __p0_39; \ + poly8x8_t __s2_39 = __p2_39; \ + poly8x8_t __rev0_39; __rev0_39 = __builtin_shufflevector(__s0_39, __s0_39, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8_t __rev2_39; __rev2_39 = __builtin_shufflevector(__s2_39, __s2_39, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8_t __ret_39; \ + __ret_39 = __noswap_vset_lane_p8(__noswap_vget_lane_p8(__rev2_39, __p3_39), __rev0_39, __p1_39); \ + __ret_39 = __builtin_shufflevector(__ret_39, __ret_39, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_39; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_p16(__p0_40, __p1_40, __p2_40, __p3_40) __extension__ ({ \ + poly16x4_t __s0_40 = __p0_40; \ + poly16x4_t __s2_40 = __p2_40; \ + poly16x4_t __ret_40; \ + __ret_40 = vset_lane_p16(vget_lane_p16(__s2_40, __p3_40), __s0_40, __p1_40); \ + __ret_40; \ +}) +#else +#define vcopy_lane_p16(__p0_41, __p1_41, __p2_41, __p3_41) __extension__ ({ \ + poly16x4_t __s0_41 = __p0_41; \ + poly16x4_t __s2_41 = __p2_41; \ + poly16x4_t __rev0_41; __rev0_41 = __builtin_shufflevector(__s0_41, __s0_41, 3, 2, 1, 0); \ + poly16x4_t __rev2_41; __rev2_41 = __builtin_shufflevector(__s2_41, __s2_41, 3, 2, 1, 0); \ + poly16x4_t __ret_41; \ + __ret_41 = __noswap_vset_lane_p16(__noswap_vget_lane_p16(__rev2_41, __p3_41), __rev0_41, __p1_41); \ + __ret_41 = __builtin_shufflevector(__ret_41, __ret_41, 3, 2, 1, 0); \ + __ret_41; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_u8(__p0_42, __p1_42, __p2_42, __p3_42) __extension__ ({ \ + uint8x8_t __s0_42 = __p0_42; \ + uint8x8_t __s2_42 = __p2_42; \ + uint8x8_t __ret_42; \ + __ret_42 = vset_lane_u8(vget_lane_u8(__s2_42, __p3_42), __s0_42, __p1_42); \ + __ret_42; \ +}) +#else +#define vcopy_lane_u8(__p0_43, __p1_43, __p2_43, __p3_43) __extension__ ({ \ + uint8x8_t __s0_43 = __p0_43; \ + uint8x8_t __s2_43 = __p2_43; \ + uint8x8_t __rev0_43; __rev0_43 = __builtin_shufflevector(__s0_43, __s0_43, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __rev2_43; __rev2_43 = __builtin_shufflevector(__s2_43, __s2_43, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret_43; \ + __ret_43 = __noswap_vset_lane_u8(__noswap_vget_lane_u8(__rev2_43, __p3_43), __rev0_43, __p1_43); \ + __ret_43 = __builtin_shufflevector(__ret_43, __ret_43, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_43; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_u32(__p0_44, __p1_44, __p2_44, __p3_44) __extension__ ({ \ + uint32x2_t __s0_44 = __p0_44; \ + uint32x2_t __s2_44 = __p2_44; \ + uint32x2_t __ret_44; \ + __ret_44 = vset_lane_u32(vget_lane_u32(__s2_44, __p3_44), __s0_44, __p1_44); \ + __ret_44; \ +}) +#else +#define vcopy_lane_u32(__p0_45, __p1_45, __p2_45, __p3_45) __extension__ ({ \ + uint32x2_t __s0_45 = __p0_45; \ + uint32x2_t __s2_45 = __p2_45; \ + uint32x2_t __rev0_45; __rev0_45 = __builtin_shufflevector(__s0_45, __s0_45, 1, 0); \ + uint32x2_t __rev2_45; __rev2_45 = __builtin_shufflevector(__s2_45, __s2_45, 1, 0); \ + uint32x2_t __ret_45; \ + __ret_45 = __noswap_vset_lane_u32(__noswap_vget_lane_u32(__rev2_45, __p3_45), __rev0_45, __p1_45); \ + __ret_45 = __builtin_shufflevector(__ret_45, __ret_45, 1, 0); \ + __ret_45; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_u64(__p0_46, __p1_46, __p2_46, __p3_46) __extension__ ({ \ + uint64x1_t __s0_46 = __p0_46; \ + uint64x1_t __s2_46 = __p2_46; \ + uint64x1_t __ret_46; \ + __ret_46 = vset_lane_u64(vget_lane_u64(__s2_46, __p3_46), __s0_46, __p1_46); \ + __ret_46; \ +}) +#else +#define vcopy_lane_u64(__p0_47, __p1_47, __p2_47, __p3_47) __extension__ ({ \ + uint64x1_t __s0_47 = __p0_47; \ + uint64x1_t __s2_47 = __p2_47; \ + uint64x1_t __ret_47; \ + __ret_47 = __noswap_vset_lane_u64(__noswap_vget_lane_u64(__s2_47, __p3_47), __s0_47, __p1_47); \ + __ret_47; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_u16(__p0_48, __p1_48, __p2_48, __p3_48) __extension__ ({ \ + uint16x4_t __s0_48 = __p0_48; \ + uint16x4_t __s2_48 = __p2_48; \ + uint16x4_t __ret_48; \ + __ret_48 = vset_lane_u16(vget_lane_u16(__s2_48, __p3_48), __s0_48, __p1_48); \ + __ret_48; \ +}) +#else +#define vcopy_lane_u16(__p0_49, __p1_49, __p2_49, __p3_49) __extension__ ({ \ + uint16x4_t __s0_49 = __p0_49; \ + uint16x4_t __s2_49 = __p2_49; \ + uint16x4_t __rev0_49; __rev0_49 = __builtin_shufflevector(__s0_49, __s0_49, 3, 2, 1, 0); \ + uint16x4_t __rev2_49; __rev2_49 = __builtin_shufflevector(__s2_49, __s2_49, 3, 2, 1, 0); \ + uint16x4_t __ret_49; \ + __ret_49 = __noswap_vset_lane_u16(__noswap_vget_lane_u16(__rev2_49, __p3_49), __rev0_49, __p1_49); \ + __ret_49 = __builtin_shufflevector(__ret_49, __ret_49, 3, 2, 1, 0); \ + __ret_49; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_s8(__p0_50, __p1_50, __p2_50, __p3_50) __extension__ ({ \ + int8x8_t __s0_50 = __p0_50; \ + int8x8_t __s2_50 = __p2_50; \ + int8x8_t __ret_50; \ + __ret_50 = vset_lane_s8(vget_lane_s8(__s2_50, __p3_50), __s0_50, __p1_50); \ + __ret_50; \ +}) +#else +#define vcopy_lane_s8(__p0_51, __p1_51, __p2_51, __p3_51) __extension__ ({ \ + int8x8_t __s0_51 = __p0_51; \ + int8x8_t __s2_51 = __p2_51; \ + int8x8_t __rev0_51; __rev0_51 = __builtin_shufflevector(__s0_51, __s0_51, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __rev2_51; __rev2_51 = __builtin_shufflevector(__s2_51, __s2_51, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret_51; \ + __ret_51 = __noswap_vset_lane_s8(__noswap_vget_lane_s8(__rev2_51, __p3_51), __rev0_51, __p1_51); \ + __ret_51 = __builtin_shufflevector(__ret_51, __ret_51, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_51; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_f32(__p0_52, __p1_52, __p2_52, __p3_52) __extension__ ({ \ + float32x2_t __s0_52 = __p0_52; \ + float32x2_t __s2_52 = __p2_52; \ + float32x2_t __ret_52; \ + __ret_52 = vset_lane_f32(vget_lane_f32(__s2_52, __p3_52), __s0_52, __p1_52); \ + __ret_52; \ +}) +#else +#define vcopy_lane_f32(__p0_53, __p1_53, __p2_53, __p3_53) __extension__ ({ \ + float32x2_t __s0_53 = __p0_53; \ + float32x2_t __s2_53 = __p2_53; \ + float32x2_t __rev0_53; __rev0_53 = __builtin_shufflevector(__s0_53, __s0_53, 1, 0); \ + float32x2_t __rev2_53; __rev2_53 = __builtin_shufflevector(__s2_53, __s2_53, 1, 0); \ + float32x2_t __ret_53; \ + __ret_53 = __noswap_vset_lane_f32(__noswap_vget_lane_f32(__rev2_53, __p3_53), __rev0_53, __p1_53); \ + __ret_53 = __builtin_shufflevector(__ret_53, __ret_53, 1, 0); \ + __ret_53; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_s32(__p0_54, __p1_54, __p2_54, __p3_54) __extension__ ({ \ + int32x2_t __s0_54 = __p0_54; \ + int32x2_t __s2_54 = __p2_54; \ + int32x2_t __ret_54; \ + __ret_54 = vset_lane_s32(vget_lane_s32(__s2_54, __p3_54), __s0_54, __p1_54); \ + __ret_54; \ +}) +#else +#define vcopy_lane_s32(__p0_55, __p1_55, __p2_55, __p3_55) __extension__ ({ \ + int32x2_t __s0_55 = __p0_55; \ + int32x2_t __s2_55 = __p2_55; \ + int32x2_t __rev0_55; __rev0_55 = __builtin_shufflevector(__s0_55, __s0_55, 1, 0); \ + int32x2_t __rev2_55; __rev2_55 = __builtin_shufflevector(__s2_55, __s2_55, 1, 0); \ + int32x2_t __ret_55; \ + __ret_55 = __noswap_vset_lane_s32(__noswap_vget_lane_s32(__rev2_55, __p3_55), __rev0_55, __p1_55); \ + __ret_55 = __builtin_shufflevector(__ret_55, __ret_55, 1, 0); \ + __ret_55; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_s64(__p0_56, __p1_56, __p2_56, __p3_56) __extension__ ({ \ + int64x1_t __s0_56 = __p0_56; \ + int64x1_t __s2_56 = __p2_56; \ + int64x1_t __ret_56; \ + __ret_56 = vset_lane_s64(vget_lane_s64(__s2_56, __p3_56), __s0_56, __p1_56); \ + __ret_56; \ +}) +#else +#define vcopy_lane_s64(__p0_57, __p1_57, __p2_57, __p3_57) __extension__ ({ \ + int64x1_t __s0_57 = __p0_57; \ + int64x1_t __s2_57 = __p2_57; \ + int64x1_t __ret_57; \ + __ret_57 = __noswap_vset_lane_s64(__noswap_vget_lane_s64(__s2_57, __p3_57), __s0_57, __p1_57); \ + __ret_57; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_s16(__p0_58, __p1_58, __p2_58, __p3_58) __extension__ ({ \ + int16x4_t __s0_58 = __p0_58; \ + int16x4_t __s2_58 = __p2_58; \ + int16x4_t __ret_58; \ + __ret_58 = vset_lane_s16(vget_lane_s16(__s2_58, __p3_58), __s0_58, __p1_58); \ + __ret_58; \ +}) +#else +#define vcopy_lane_s16(__p0_59, __p1_59, __p2_59, __p3_59) __extension__ ({ \ + int16x4_t __s0_59 = __p0_59; \ + int16x4_t __s2_59 = __p2_59; \ + int16x4_t __rev0_59; __rev0_59 = __builtin_shufflevector(__s0_59, __s0_59, 3, 2, 1, 0); \ + int16x4_t __rev2_59; __rev2_59 = __builtin_shufflevector(__s2_59, __s2_59, 3, 2, 1, 0); \ + int16x4_t __ret_59; \ + __ret_59 = __noswap_vset_lane_s16(__noswap_vget_lane_s16(__rev2_59, __p3_59), __rev0_59, __p1_59); \ + __ret_59 = __builtin_shufflevector(__ret_59, __ret_59, 3, 2, 1, 0); \ + __ret_59; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_laneq_p8(__p0_60, __p1_60, __p2_60, __p3_60) __extension__ ({ \ + poly8x16_t __s0_60 = __p0_60; \ + poly8x16_t __s2_60 = __p2_60; \ + poly8x16_t __ret_60; \ + __ret_60 = vsetq_lane_p8(vgetq_lane_p8(__s2_60, __p3_60), __s0_60, __p1_60); \ + __ret_60; \ +}) +#else +#define vcopyq_laneq_p8(__p0_61, __p1_61, __p2_61, __p3_61) __extension__ ({ \ + poly8x16_t __s0_61 = __p0_61; \ + poly8x16_t __s2_61 = __p2_61; \ + poly8x16_t __rev0_61; __rev0_61 = __builtin_shufflevector(__s0_61, __s0_61, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16_t __rev2_61; __rev2_61 = __builtin_shufflevector(__s2_61, __s2_61, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16_t __ret_61; \ + __ret_61 = __noswap_vsetq_lane_p8(__noswap_vgetq_lane_p8(__rev2_61, __p3_61), __rev0_61, __p1_61); \ + __ret_61 = __builtin_shufflevector(__ret_61, __ret_61, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_61; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_laneq_p16(__p0_62, __p1_62, __p2_62, __p3_62) __extension__ ({ \ + poly16x8_t __s0_62 = __p0_62; \ + poly16x8_t __s2_62 = __p2_62; \ + poly16x8_t __ret_62; \ + __ret_62 = vsetq_lane_p16(vgetq_lane_p16(__s2_62, __p3_62), __s0_62, __p1_62); \ + __ret_62; \ +}) +#else +#define vcopyq_laneq_p16(__p0_63, __p1_63, __p2_63, __p3_63) __extension__ ({ \ + poly16x8_t __s0_63 = __p0_63; \ + poly16x8_t __s2_63 = __p2_63; \ + poly16x8_t __rev0_63; __rev0_63 = __builtin_shufflevector(__s0_63, __s0_63, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x8_t __rev2_63; __rev2_63 = __builtin_shufflevector(__s2_63, __s2_63, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x8_t __ret_63; \ + __ret_63 = __noswap_vsetq_lane_p16(__noswap_vgetq_lane_p16(__rev2_63, __p3_63), __rev0_63, __p1_63); \ + __ret_63 = __builtin_shufflevector(__ret_63, __ret_63, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_63; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_laneq_u8(__p0_64, __p1_64, __p2_64, __p3_64) __extension__ ({ \ + uint8x16_t __s0_64 = __p0_64; \ + uint8x16_t __s2_64 = __p2_64; \ + uint8x16_t __ret_64; \ + __ret_64 = vsetq_lane_u8(vgetq_lane_u8(__s2_64, __p3_64), __s0_64, __p1_64); \ + __ret_64; \ +}) +#else +#define vcopyq_laneq_u8(__p0_65, __p1_65, __p2_65, __p3_65) __extension__ ({ \ + uint8x16_t __s0_65 = __p0_65; \ + uint8x16_t __s2_65 = __p2_65; \ + uint8x16_t __rev0_65; __rev0_65 = __builtin_shufflevector(__s0_65, __s0_65, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __rev2_65; __rev2_65 = __builtin_shufflevector(__s2_65, __s2_65, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret_65; \ + __ret_65 = __noswap_vsetq_lane_u8(__noswap_vgetq_lane_u8(__rev2_65, __p3_65), __rev0_65, __p1_65); \ + __ret_65 = __builtin_shufflevector(__ret_65, __ret_65, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_65; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_laneq_u32(__p0_66, __p1_66, __p2_66, __p3_66) __extension__ ({ \ + uint32x4_t __s0_66 = __p0_66; \ + uint32x4_t __s2_66 = __p2_66; \ + uint32x4_t __ret_66; \ + __ret_66 = vsetq_lane_u32(vgetq_lane_u32(__s2_66, __p3_66), __s0_66, __p1_66); \ + __ret_66; \ +}) +#else +#define vcopyq_laneq_u32(__p0_67, __p1_67, __p2_67, __p3_67) __extension__ ({ \ + uint32x4_t __s0_67 = __p0_67; \ + uint32x4_t __s2_67 = __p2_67; \ + uint32x4_t __rev0_67; __rev0_67 = __builtin_shufflevector(__s0_67, __s0_67, 3, 2, 1, 0); \ + uint32x4_t __rev2_67; __rev2_67 = __builtin_shufflevector(__s2_67, __s2_67, 3, 2, 1, 0); \ + uint32x4_t __ret_67; \ + __ret_67 = __noswap_vsetq_lane_u32(__noswap_vgetq_lane_u32(__rev2_67, __p3_67), __rev0_67, __p1_67); \ + __ret_67 = __builtin_shufflevector(__ret_67, __ret_67, 3, 2, 1, 0); \ + __ret_67; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_laneq_u64(__p0_68, __p1_68, __p2_68, __p3_68) __extension__ ({ \ + uint64x2_t __s0_68 = __p0_68; \ + uint64x2_t __s2_68 = __p2_68; \ + uint64x2_t __ret_68; \ + __ret_68 = vsetq_lane_u64(vgetq_lane_u64(__s2_68, __p3_68), __s0_68, __p1_68); \ + __ret_68; \ +}) +#else +#define vcopyq_laneq_u64(__p0_69, __p1_69, __p2_69, __p3_69) __extension__ ({ \ + uint64x2_t __s0_69 = __p0_69; \ + uint64x2_t __s2_69 = __p2_69; \ + uint64x2_t __rev0_69; __rev0_69 = __builtin_shufflevector(__s0_69, __s0_69, 1, 0); \ + uint64x2_t __rev2_69; __rev2_69 = __builtin_shufflevector(__s2_69, __s2_69, 1, 0); \ + uint64x2_t __ret_69; \ + __ret_69 = __noswap_vsetq_lane_u64(__noswap_vgetq_lane_u64(__rev2_69, __p3_69), __rev0_69, __p1_69); \ + __ret_69 = __builtin_shufflevector(__ret_69, __ret_69, 1, 0); \ + __ret_69; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_laneq_u16(__p0_70, __p1_70, __p2_70, __p3_70) __extension__ ({ \ + uint16x8_t __s0_70 = __p0_70; \ + uint16x8_t __s2_70 = __p2_70; \ + uint16x8_t __ret_70; \ + __ret_70 = vsetq_lane_u16(vgetq_lane_u16(__s2_70, __p3_70), __s0_70, __p1_70); \ + __ret_70; \ +}) +#else +#define vcopyq_laneq_u16(__p0_71, __p1_71, __p2_71, __p3_71) __extension__ ({ \ + uint16x8_t __s0_71 = __p0_71; \ + uint16x8_t __s2_71 = __p2_71; \ + uint16x8_t __rev0_71; __rev0_71 = __builtin_shufflevector(__s0_71, __s0_71, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev2_71; __rev2_71 = __builtin_shufflevector(__s2_71, __s2_71, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret_71; \ + __ret_71 = __noswap_vsetq_lane_u16(__noswap_vgetq_lane_u16(__rev2_71, __p3_71), __rev0_71, __p1_71); \ + __ret_71 = __builtin_shufflevector(__ret_71, __ret_71, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_71; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_laneq_s8(__p0_72, __p1_72, __p2_72, __p3_72) __extension__ ({ \ + int8x16_t __s0_72 = __p0_72; \ + int8x16_t __s2_72 = __p2_72; \ + int8x16_t __ret_72; \ + __ret_72 = vsetq_lane_s8(vgetq_lane_s8(__s2_72, __p3_72), __s0_72, __p1_72); \ + __ret_72; \ +}) +#else +#define vcopyq_laneq_s8(__p0_73, __p1_73, __p2_73, __p3_73) __extension__ ({ \ + int8x16_t __s0_73 = __p0_73; \ + int8x16_t __s2_73 = __p2_73; \ + int8x16_t __rev0_73; __rev0_73 = __builtin_shufflevector(__s0_73, __s0_73, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __rev2_73; __rev2_73 = __builtin_shufflevector(__s2_73, __s2_73, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret_73; \ + __ret_73 = __noswap_vsetq_lane_s8(__noswap_vgetq_lane_s8(__rev2_73, __p3_73), __rev0_73, __p1_73); \ + __ret_73 = __builtin_shufflevector(__ret_73, __ret_73, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_73; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_laneq_f32(__p0_74, __p1_74, __p2_74, __p3_74) __extension__ ({ \ + float32x4_t __s0_74 = __p0_74; \ + float32x4_t __s2_74 = __p2_74; \ + float32x4_t __ret_74; \ + __ret_74 = vsetq_lane_f32(vgetq_lane_f32(__s2_74, __p3_74), __s0_74, __p1_74); \ + __ret_74; \ +}) +#else +#define vcopyq_laneq_f32(__p0_75, __p1_75, __p2_75, __p3_75) __extension__ ({ \ + float32x4_t __s0_75 = __p0_75; \ + float32x4_t __s2_75 = __p2_75; \ + float32x4_t __rev0_75; __rev0_75 = __builtin_shufflevector(__s0_75, __s0_75, 3, 2, 1, 0); \ + float32x4_t __rev2_75; __rev2_75 = __builtin_shufflevector(__s2_75, __s2_75, 3, 2, 1, 0); \ + float32x4_t __ret_75; \ + __ret_75 = __noswap_vsetq_lane_f32(__noswap_vgetq_lane_f32(__rev2_75, __p3_75), __rev0_75, __p1_75); \ + __ret_75 = __builtin_shufflevector(__ret_75, __ret_75, 3, 2, 1, 0); \ + __ret_75; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_laneq_s32(__p0_76, __p1_76, __p2_76, __p3_76) __extension__ ({ \ + int32x4_t __s0_76 = __p0_76; \ + int32x4_t __s2_76 = __p2_76; \ + int32x4_t __ret_76; \ + __ret_76 = vsetq_lane_s32(vgetq_lane_s32(__s2_76, __p3_76), __s0_76, __p1_76); \ + __ret_76; \ +}) +#else +#define vcopyq_laneq_s32(__p0_77, __p1_77, __p2_77, __p3_77) __extension__ ({ \ + int32x4_t __s0_77 = __p0_77; \ + int32x4_t __s2_77 = __p2_77; \ + int32x4_t __rev0_77; __rev0_77 = __builtin_shufflevector(__s0_77, __s0_77, 3, 2, 1, 0); \ + int32x4_t __rev2_77; __rev2_77 = __builtin_shufflevector(__s2_77, __s2_77, 3, 2, 1, 0); \ + int32x4_t __ret_77; \ + __ret_77 = __noswap_vsetq_lane_s32(__noswap_vgetq_lane_s32(__rev2_77, __p3_77), __rev0_77, __p1_77); \ + __ret_77 = __builtin_shufflevector(__ret_77, __ret_77, 3, 2, 1, 0); \ + __ret_77; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_laneq_s64(__p0_78, __p1_78, __p2_78, __p3_78) __extension__ ({ \ + int64x2_t __s0_78 = __p0_78; \ + int64x2_t __s2_78 = __p2_78; \ + int64x2_t __ret_78; \ + __ret_78 = vsetq_lane_s64(vgetq_lane_s64(__s2_78, __p3_78), __s0_78, __p1_78); \ + __ret_78; \ +}) +#else +#define vcopyq_laneq_s64(__p0_79, __p1_79, __p2_79, __p3_79) __extension__ ({ \ + int64x2_t __s0_79 = __p0_79; \ + int64x2_t __s2_79 = __p2_79; \ + int64x2_t __rev0_79; __rev0_79 = __builtin_shufflevector(__s0_79, __s0_79, 1, 0); \ + int64x2_t __rev2_79; __rev2_79 = __builtin_shufflevector(__s2_79, __s2_79, 1, 0); \ + int64x2_t __ret_79; \ + __ret_79 = __noswap_vsetq_lane_s64(__noswap_vgetq_lane_s64(__rev2_79, __p3_79), __rev0_79, __p1_79); \ + __ret_79 = __builtin_shufflevector(__ret_79, __ret_79, 1, 0); \ + __ret_79; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_laneq_s16(__p0_80, __p1_80, __p2_80, __p3_80) __extension__ ({ \ + int16x8_t __s0_80 = __p0_80; \ + int16x8_t __s2_80 = __p2_80; \ + int16x8_t __ret_80; \ + __ret_80 = vsetq_lane_s16(vgetq_lane_s16(__s2_80, __p3_80), __s0_80, __p1_80); \ + __ret_80; \ +}) +#else +#define vcopyq_laneq_s16(__p0_81, __p1_81, __p2_81, __p3_81) __extension__ ({ \ + int16x8_t __s0_81 = __p0_81; \ + int16x8_t __s2_81 = __p2_81; \ + int16x8_t __rev0_81; __rev0_81 = __builtin_shufflevector(__s0_81, __s0_81, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev2_81; __rev2_81 = __builtin_shufflevector(__s2_81, __s2_81, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret_81; \ + __ret_81 = __noswap_vsetq_lane_s16(__noswap_vgetq_lane_s16(__rev2_81, __p3_81), __rev0_81, __p1_81); \ + __ret_81 = __builtin_shufflevector(__ret_81, __ret_81, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_81; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_laneq_p8(__p0_82, __p1_82, __p2_82, __p3_82) __extension__ ({ \ + poly8x8_t __s0_82 = __p0_82; \ + poly8x16_t __s2_82 = __p2_82; \ + poly8x8_t __ret_82; \ + __ret_82 = vset_lane_p8(vgetq_lane_p8(__s2_82, __p3_82), __s0_82, __p1_82); \ + __ret_82; \ +}) +#else +#define vcopy_laneq_p8(__p0_83, __p1_83, __p2_83, __p3_83) __extension__ ({ \ + poly8x8_t __s0_83 = __p0_83; \ + poly8x16_t __s2_83 = __p2_83; \ + poly8x8_t __rev0_83; __rev0_83 = __builtin_shufflevector(__s0_83, __s0_83, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16_t __rev2_83; __rev2_83 = __builtin_shufflevector(__s2_83, __s2_83, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8_t __ret_83; \ + __ret_83 = __noswap_vset_lane_p8(__noswap_vgetq_lane_p8(__rev2_83, __p3_83), __rev0_83, __p1_83); \ + __ret_83 = __builtin_shufflevector(__ret_83, __ret_83, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_83; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_laneq_p16(__p0_84, __p1_84, __p2_84, __p3_84) __extension__ ({ \ + poly16x4_t __s0_84 = __p0_84; \ + poly16x8_t __s2_84 = __p2_84; \ + poly16x4_t __ret_84; \ + __ret_84 = vset_lane_p16(vgetq_lane_p16(__s2_84, __p3_84), __s0_84, __p1_84); \ + __ret_84; \ +}) +#else +#define vcopy_laneq_p16(__p0_85, __p1_85, __p2_85, __p3_85) __extension__ ({ \ + poly16x4_t __s0_85 = __p0_85; \ + poly16x8_t __s2_85 = __p2_85; \ + poly16x4_t __rev0_85; __rev0_85 = __builtin_shufflevector(__s0_85, __s0_85, 3, 2, 1, 0); \ + poly16x8_t __rev2_85; __rev2_85 = __builtin_shufflevector(__s2_85, __s2_85, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x4_t __ret_85; \ + __ret_85 = __noswap_vset_lane_p16(__noswap_vgetq_lane_p16(__rev2_85, __p3_85), __rev0_85, __p1_85); \ + __ret_85 = __builtin_shufflevector(__ret_85, __ret_85, 3, 2, 1, 0); \ + __ret_85; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_laneq_u8(__p0_86, __p1_86, __p2_86, __p3_86) __extension__ ({ \ + uint8x8_t __s0_86 = __p0_86; \ + uint8x16_t __s2_86 = __p2_86; \ + uint8x8_t __ret_86; \ + __ret_86 = vset_lane_u8(vgetq_lane_u8(__s2_86, __p3_86), __s0_86, __p1_86); \ + __ret_86; \ +}) +#else +#define vcopy_laneq_u8(__p0_87, __p1_87, __p2_87, __p3_87) __extension__ ({ \ + uint8x8_t __s0_87 = __p0_87; \ + uint8x16_t __s2_87 = __p2_87; \ + uint8x8_t __rev0_87; __rev0_87 = __builtin_shufflevector(__s0_87, __s0_87, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __rev2_87; __rev2_87 = __builtin_shufflevector(__s2_87, __s2_87, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret_87; \ + __ret_87 = __noswap_vset_lane_u8(__noswap_vgetq_lane_u8(__rev2_87, __p3_87), __rev0_87, __p1_87); \ + __ret_87 = __builtin_shufflevector(__ret_87, __ret_87, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_87; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_laneq_u32(__p0_88, __p1_88, __p2_88, __p3_88) __extension__ ({ \ + uint32x2_t __s0_88 = __p0_88; \ + uint32x4_t __s2_88 = __p2_88; \ + uint32x2_t __ret_88; \ + __ret_88 = vset_lane_u32(vgetq_lane_u32(__s2_88, __p3_88), __s0_88, __p1_88); \ + __ret_88; \ +}) +#else +#define vcopy_laneq_u32(__p0_89, __p1_89, __p2_89, __p3_89) __extension__ ({ \ + uint32x2_t __s0_89 = __p0_89; \ + uint32x4_t __s2_89 = __p2_89; \ + uint32x2_t __rev0_89; __rev0_89 = __builtin_shufflevector(__s0_89, __s0_89, 1, 0); \ + uint32x4_t __rev2_89; __rev2_89 = __builtin_shufflevector(__s2_89, __s2_89, 3, 2, 1, 0); \ + uint32x2_t __ret_89; \ + __ret_89 = __noswap_vset_lane_u32(__noswap_vgetq_lane_u32(__rev2_89, __p3_89), __rev0_89, __p1_89); \ + __ret_89 = __builtin_shufflevector(__ret_89, __ret_89, 1, 0); \ + __ret_89; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_laneq_u64(__p0_90, __p1_90, __p2_90, __p3_90) __extension__ ({ \ + uint64x1_t __s0_90 = __p0_90; \ + uint64x2_t __s2_90 = __p2_90; \ + uint64x1_t __ret_90; \ + __ret_90 = vset_lane_u64(vgetq_lane_u64(__s2_90, __p3_90), __s0_90, __p1_90); \ + __ret_90; \ +}) +#else +#define vcopy_laneq_u64(__p0_91, __p1_91, __p2_91, __p3_91) __extension__ ({ \ + uint64x1_t __s0_91 = __p0_91; \ + uint64x2_t __s2_91 = __p2_91; \ + uint64x2_t __rev2_91; __rev2_91 = __builtin_shufflevector(__s2_91, __s2_91, 1, 0); \ + uint64x1_t __ret_91; \ + __ret_91 = __noswap_vset_lane_u64(__noswap_vgetq_lane_u64(__rev2_91, __p3_91), __s0_91, __p1_91); \ + __ret_91; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_laneq_u16(__p0_92, __p1_92, __p2_92, __p3_92) __extension__ ({ \ + uint16x4_t __s0_92 = __p0_92; \ + uint16x8_t __s2_92 = __p2_92; \ + uint16x4_t __ret_92; \ + __ret_92 = vset_lane_u16(vgetq_lane_u16(__s2_92, __p3_92), __s0_92, __p1_92); \ + __ret_92; \ +}) +#else +#define vcopy_laneq_u16(__p0_93, __p1_93, __p2_93, __p3_93) __extension__ ({ \ + uint16x4_t __s0_93 = __p0_93; \ + uint16x8_t __s2_93 = __p2_93; \ + uint16x4_t __rev0_93; __rev0_93 = __builtin_shufflevector(__s0_93, __s0_93, 3, 2, 1, 0); \ + uint16x8_t __rev2_93; __rev2_93 = __builtin_shufflevector(__s2_93, __s2_93, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x4_t __ret_93; \ + __ret_93 = __noswap_vset_lane_u16(__noswap_vgetq_lane_u16(__rev2_93, __p3_93), __rev0_93, __p1_93); \ + __ret_93 = __builtin_shufflevector(__ret_93, __ret_93, 3, 2, 1, 0); \ + __ret_93; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_laneq_s8(__p0_94, __p1_94, __p2_94, __p3_94) __extension__ ({ \ + int8x8_t __s0_94 = __p0_94; \ + int8x16_t __s2_94 = __p2_94; \ + int8x8_t __ret_94; \ + __ret_94 = vset_lane_s8(vgetq_lane_s8(__s2_94, __p3_94), __s0_94, __p1_94); \ + __ret_94; \ +}) +#else +#define vcopy_laneq_s8(__p0_95, __p1_95, __p2_95, __p3_95) __extension__ ({ \ + int8x8_t __s0_95 = __p0_95; \ + int8x16_t __s2_95 = __p2_95; \ + int8x8_t __rev0_95; __rev0_95 = __builtin_shufflevector(__s0_95, __s0_95, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __rev2_95; __rev2_95 = __builtin_shufflevector(__s2_95, __s2_95, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret_95; \ + __ret_95 = __noswap_vset_lane_s8(__noswap_vgetq_lane_s8(__rev2_95, __p3_95), __rev0_95, __p1_95); \ + __ret_95 = __builtin_shufflevector(__ret_95, __ret_95, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_95; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_laneq_f32(__p0_96, __p1_96, __p2_96, __p3_96) __extension__ ({ \ + float32x2_t __s0_96 = __p0_96; \ + float32x4_t __s2_96 = __p2_96; \ + float32x2_t __ret_96; \ + __ret_96 = vset_lane_f32(vgetq_lane_f32(__s2_96, __p3_96), __s0_96, __p1_96); \ + __ret_96; \ +}) +#else +#define vcopy_laneq_f32(__p0_97, __p1_97, __p2_97, __p3_97) __extension__ ({ \ + float32x2_t __s0_97 = __p0_97; \ + float32x4_t __s2_97 = __p2_97; \ + float32x2_t __rev0_97; __rev0_97 = __builtin_shufflevector(__s0_97, __s0_97, 1, 0); \ + float32x4_t __rev2_97; __rev2_97 = __builtin_shufflevector(__s2_97, __s2_97, 3, 2, 1, 0); \ + float32x2_t __ret_97; \ + __ret_97 = __noswap_vset_lane_f32(__noswap_vgetq_lane_f32(__rev2_97, __p3_97), __rev0_97, __p1_97); \ + __ret_97 = __builtin_shufflevector(__ret_97, __ret_97, 1, 0); \ + __ret_97; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_laneq_s32(__p0_98, __p1_98, __p2_98, __p3_98) __extension__ ({ \ + int32x2_t __s0_98 = __p0_98; \ + int32x4_t __s2_98 = __p2_98; \ + int32x2_t __ret_98; \ + __ret_98 = vset_lane_s32(vgetq_lane_s32(__s2_98, __p3_98), __s0_98, __p1_98); \ + __ret_98; \ +}) +#else +#define vcopy_laneq_s32(__p0_99, __p1_99, __p2_99, __p3_99) __extension__ ({ \ + int32x2_t __s0_99 = __p0_99; \ + int32x4_t __s2_99 = __p2_99; \ + int32x2_t __rev0_99; __rev0_99 = __builtin_shufflevector(__s0_99, __s0_99, 1, 0); \ + int32x4_t __rev2_99; __rev2_99 = __builtin_shufflevector(__s2_99, __s2_99, 3, 2, 1, 0); \ + int32x2_t __ret_99; \ + __ret_99 = __noswap_vset_lane_s32(__noswap_vgetq_lane_s32(__rev2_99, __p3_99), __rev0_99, __p1_99); \ + __ret_99 = __builtin_shufflevector(__ret_99, __ret_99, 1, 0); \ + __ret_99; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_laneq_s64(__p0_100, __p1_100, __p2_100, __p3_100) __extension__ ({ \ + int64x1_t __s0_100 = __p0_100; \ + int64x2_t __s2_100 = __p2_100; \ + int64x1_t __ret_100; \ + __ret_100 = vset_lane_s64(vgetq_lane_s64(__s2_100, __p3_100), __s0_100, __p1_100); \ + __ret_100; \ +}) +#else +#define vcopy_laneq_s64(__p0_101, __p1_101, __p2_101, __p3_101) __extension__ ({ \ + int64x1_t __s0_101 = __p0_101; \ + int64x2_t __s2_101 = __p2_101; \ + int64x2_t __rev2_101; __rev2_101 = __builtin_shufflevector(__s2_101, __s2_101, 1, 0); \ + int64x1_t __ret_101; \ + __ret_101 = __noswap_vset_lane_s64(__noswap_vgetq_lane_s64(__rev2_101, __p3_101), __s0_101, __p1_101); \ + __ret_101; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_laneq_s16(__p0_102, __p1_102, __p2_102, __p3_102) __extension__ ({ \ + int16x4_t __s0_102 = __p0_102; \ + int16x8_t __s2_102 = __p2_102; \ + int16x4_t __ret_102; \ + __ret_102 = vset_lane_s16(vgetq_lane_s16(__s2_102, __p3_102), __s0_102, __p1_102); \ + __ret_102; \ +}) +#else +#define vcopy_laneq_s16(__p0_103, __p1_103, __p2_103, __p3_103) __extension__ ({ \ + int16x4_t __s0_103 = __p0_103; \ + int16x8_t __s2_103 = __p2_103; \ + int16x4_t __rev0_103; __rev0_103 = __builtin_shufflevector(__s0_103, __s0_103, 3, 2, 1, 0); \ + int16x8_t __rev2_103; __rev2_103 = __builtin_shufflevector(__s2_103, __s2_103, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __ret_103; \ + __ret_103 = __noswap_vset_lane_s16(__noswap_vgetq_lane_s16(__rev2_103, __p3_103), __rev0_103, __p1_103); \ + __ret_103 = __builtin_shufflevector(__ret_103, __ret_103, 3, 2, 1, 0); \ + __ret_103; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vcreate_p64(uint64_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#else +__ai poly64x1_t vcreate_p64(uint64_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vcreate_f64(uint64_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#else +__ai float64x1_t vcreate_f64(uint64_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t)(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vcvts_f32_s32(int32_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vcvts_f32_s32(__p0); + return __ret; +} +#else +__ai float32_t vcvts_f32_s32(int32_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vcvts_f32_s32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vcvts_f32_u32(uint32_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vcvts_f32_u32(__p0); + return __ret; +} +#else +__ai float32_t vcvts_f32_u32(uint32_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vcvts_f32_u32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vcvt_f32_f64(float64x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vcvt_f32_f64((int8x16_t)__p0, 9); + return __ret; +} +#else +__ai float32x2_t vcvt_f32_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vcvt_f32_f64((int8x16_t)__rev0, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai float32x2_t __noswap_vcvt_f32_f64(float64x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vcvt_f32_f64((int8x16_t)__p0, 9); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vcvtd_f64_s64(int64_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vcvtd_f64_s64(__p0); + return __ret; +} +#else +__ai float64_t vcvtd_f64_s64(int64_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vcvtd_f64_s64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vcvtd_f64_u64(uint64_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vcvtd_f64_u64(__p0); + return __ret; +} +#else +__ai float64_t vcvtd_f64_u64(uint64_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vcvtd_f64_u64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vcvtq_f64_u64(uint64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vcvtq_f64_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai float64x2_t vcvtq_f64_u64(uint64x2_t __p0) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vcvtq_f64_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vcvtq_f64_s64(int64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vcvtq_f64_v((int8x16_t)__p0, 35); + return __ret; +} +#else +__ai float64x2_t vcvtq_f64_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vcvtq_f64_v((int8x16_t)__rev0, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vcvt_f64_u64(uint64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vcvt_f64_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai float64x1_t vcvt_f64_u64(uint64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vcvt_f64_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vcvt_f64_s64(int64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vcvt_f64_v((int8x8_t)__p0, 3); + return __ret; +} +#else +__ai float64x1_t vcvt_f64_s64(int64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vcvt_f64_v((int8x8_t)__p0, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vcvt_f64_f32(float32x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vcvt_f64_f32((int8x8_t)__p0, 42); + return __ret; +} +#else +__ai float64x2_t vcvt_f64_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vcvt_f64_f32((int8x8_t)__rev0, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai float64x2_t __noswap_vcvt_f64_f32(float32x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vcvt_f64_f32((int8x8_t)__p0, 42); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float16x8_t vcvt_high_f16_f32(float16x4_t __p0, float32x4_t __p1) { + float16x8_t __ret; + __ret = vcombine_f16(__p0, vcvt_f16_f32(__p1)); + return __ret; +} +#else +__ai float16x8_t vcvt_high_f16_f32(float16x4_t __p0, float32x4_t __p1) { + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float16x8_t __ret; + __ret = __noswap_vcombine_f16(__rev0, __noswap_vcvt_f16_f32(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vcvt_high_f32_f16(float16x8_t __p0) { + float32x4_t __ret; + __ret = vcvt_f32_f16(vget_high_f16(__p0)); + return __ret; +} +#else +__ai float32x4_t vcvt_high_f32_f16(float16x8_t __p0) { + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __noswap_vcvt_f32_f16(__noswap_vget_high_f16(__rev0)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vcvt_high_f32_f64(float32x2_t __p0, float64x2_t __p1) { + float32x4_t __ret; + __ret = vcombine_f32(__p0, vcvt_f32_f64(__p1)); + return __ret; +} +#else +__ai float32x4_t vcvt_high_f32_f64(float32x2_t __p0, float64x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x4_t __ret; + __ret = __noswap_vcombine_f32(__rev0, __noswap_vcvt_f32_f64(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vcvt_high_f64_f32(float32x4_t __p0) { + float64x2_t __ret; + __ret = vcvt_f64_f32(vget_high_f32(__p0)); + return __ret; +} +#else +__ai float64x2_t vcvt_high_f64_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float64x2_t __ret; + __ret = __noswap_vcvt_f64_f32(__noswap_vget_high_f32(__rev0)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvts_n_f32_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vcvts_n_f32_u32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvts_n_f32_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vcvts_n_f32_u32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvts_n_f32_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vcvts_n_f32_s32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvts_n_f32_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vcvts_n_f32_s32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtq_n_f64_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vcvtq_n_f64_v((int8x16_t)__s0, __p1, 51); \ + __ret; \ +}) +#else +#define vcvtq_n_f64_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vcvtq_n_f64_v((int8x16_t)__rev0, __p1, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtq_n_f64_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vcvtq_n_f64_v((int8x16_t)__s0, __p1, 35); \ + __ret; \ +}) +#else +#define vcvtq_n_f64_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vcvtq_n_f64_v((int8x16_t)__rev0, __p1, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvt_n_f64_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vcvt_n_f64_v((int8x8_t)__s0, __p1, 19); \ + __ret; \ +}) +#else +#define vcvt_n_f64_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vcvt_n_f64_v((int8x8_t)__s0, __p1, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvt_n_f64_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vcvt_n_f64_v((int8x8_t)__s0, __p1, 3); \ + __ret; \ +}) +#else +#define vcvt_n_f64_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vcvt_n_f64_v((int8x8_t)__s0, __p1, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtd_n_f64_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vcvtd_n_f64_u64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvtd_n_f64_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vcvtd_n_f64_u64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtd_n_f64_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vcvtd_n_f64_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvtd_n_f64_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vcvtd_n_f64_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvts_n_s32_f32(__p0, __p1) __extension__ ({ \ + float32_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vcvts_n_s32_f32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvts_n_s32_f32(__p0, __p1) __extension__ ({ \ + float32_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vcvts_n_s32_f32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtq_n_s64_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vcvtq_n_s64_v((int8x16_t)__s0, __p1, 35); \ + __ret; \ +}) +#else +#define vcvtq_n_s64_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64x2_t __ret; \ + __ret = (int64x2_t) __builtin_neon_vcvtq_n_s64_v((int8x16_t)__rev0, __p1, 35); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvt_n_s64_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vcvt_n_s64_v((int8x8_t)__s0, __p1, 3); \ + __ret; \ +}) +#else +#define vcvt_n_s64_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + int64x1_t __ret; \ + __ret = (int64x1_t) __builtin_neon_vcvt_n_s64_v((int8x8_t)__s0, __p1, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtd_n_s64_f64(__p0, __p1) __extension__ ({ \ + float64_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vcvtd_n_s64_f64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvtd_n_s64_f64(__p0, __p1) __extension__ ({ \ + float64_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vcvtd_n_s64_f64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvts_n_u32_f32(__p0, __p1) __extension__ ({ \ + float32_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vcvts_n_u32_f32(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvts_n_u32_f32(__p0, __p1) __extension__ ({ \ + float32_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vcvts_n_u32_f32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtq_n_u64_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vcvtq_n_u64_v((int8x16_t)__s0, __p1, 51); \ + __ret; \ +}) +#else +#define vcvtq_n_u64_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64x2_t __ret; \ + __ret = (uint64x2_t) __builtin_neon_vcvtq_n_u64_v((int8x16_t)__rev0, __p1, 51); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvt_n_u64_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vcvt_n_u64_v((int8x8_t)__s0, __p1, 19); \ + __ret; \ +}) +#else +#define vcvt_n_u64_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = (uint64x1_t) __builtin_neon_vcvt_n_u64_v((int8x8_t)__s0, __p1, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcvtd_n_u64_f64(__p0, __p1) __extension__ ({ \ + float64_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vcvtd_n_u64_f64(__s0, __p1); \ + __ret; \ +}) +#else +#define vcvtd_n_u64_f64(__p0, __p1) __extension__ ({ \ + float64_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vcvtd_n_u64_f64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvts_s32_f32(float32_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvts_s32_f32(__p0); + return __ret; +} +#else +__ai int32_t vcvts_s32_f32(float32_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvts_s32_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vcvtd_s64_f64(float64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcvtd_s64_f64(__p0); + return __ret; +} +#else +__ai int64_t vcvtd_s64_f64(float64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcvtd_s64_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vcvtq_s64_f64(float64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vcvtq_s64_v((int8x16_t)__p0, 35); + return __ret; +} +#else +__ai int64x2_t vcvtq_s64_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vcvtq_s64_v((int8x16_t)__rev0, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vcvt_s64_f64(float64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vcvt_s64_v((int8x8_t)__p0, 3); + return __ret; +} +#else +__ai int64x1_t vcvt_s64_f64(float64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vcvt_s64_v((int8x8_t)__p0, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvts_u32_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvts_u32_f32(__p0); + return __ret; +} +#else +__ai uint32_t vcvts_u32_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvts_u32_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcvtd_u64_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcvtd_u64_f64(__p0); + return __ret; +} +#else +__ai uint64_t vcvtd_u64_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcvtd_u64_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vcvtq_u64_f64(float64x2_t __p0) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcvtq_u64_v((int8x16_t)__p0, 51); + return __ret; +} +#else +__ai uint64x2_t vcvtq_u64_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vcvtq_u64_v((int8x16_t)__rev0, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vcvt_u64_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcvt_u64_v((int8x8_t)__p0, 19); + return __ret; +} +#else +__ai uint64x1_t vcvt_u64_f64(float64x1_t __p0) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vcvt_u64_v((int8x8_t)__p0, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtas_s32_f32(float32_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtas_s32_f32(__p0); + return __ret; +} +#else +__ai int32_t vcvtas_s32_f32(float32_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtas_s32_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vcvtad_s64_f64(float64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcvtad_s64_f64(__p0); + return __ret; +} +#else +__ai int64_t vcvtad_s64_f64(float64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcvtad_s64_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtas_u32_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtas_u32_f32(__p0); + return __ret; +} +#else +__ai uint32_t vcvtas_u32_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtas_u32_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcvtad_u64_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcvtad_u64_f64(__p0); + return __ret; +} +#else +__ai uint64_t vcvtad_u64_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcvtad_u64_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtms_s32_f32(float32_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtms_s32_f32(__p0); + return __ret; +} +#else +__ai int32_t vcvtms_s32_f32(float32_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtms_s32_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vcvtmd_s64_f64(float64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcvtmd_s64_f64(__p0); + return __ret; +} +#else +__ai int64_t vcvtmd_s64_f64(float64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcvtmd_s64_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtms_u32_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtms_u32_f32(__p0); + return __ret; +} +#else +__ai uint32_t vcvtms_u32_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtms_u32_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcvtmd_u64_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcvtmd_u64_f64(__p0); + return __ret; +} +#else +__ai uint64_t vcvtmd_u64_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcvtmd_u64_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtns_s32_f32(float32_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtns_s32_f32(__p0); + return __ret; +} +#else +__ai int32_t vcvtns_s32_f32(float32_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtns_s32_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vcvtnd_s64_f64(float64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcvtnd_s64_f64(__p0); + return __ret; +} +#else +__ai int64_t vcvtnd_s64_f64(float64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcvtnd_s64_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtns_u32_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtns_u32_f32(__p0); + return __ret; +} +#else +__ai uint32_t vcvtns_u32_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtns_u32_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcvtnd_u64_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcvtnd_u64_f64(__p0); + return __ret; +} +#else +__ai uint64_t vcvtnd_u64_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcvtnd_u64_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vcvtps_s32_f32(float32_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtps_s32_f32(__p0); + return __ret; +} +#else +__ai int32_t vcvtps_s32_f32(float32_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vcvtps_s32_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vcvtpd_s64_f64(float64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcvtpd_s64_f64(__p0); + return __ret; +} +#else +__ai int64_t vcvtpd_s64_f64(float64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vcvtpd_s64_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vcvtps_u32_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtps_u32_f32(__p0); + return __ret; +} +#else +__ai uint32_t vcvtps_u32_f32(float32_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vcvtps_u32_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vcvtpd_u64_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcvtpd_u64_f64(__p0); + return __ret; +} +#else +__ai uint64_t vcvtpd_u64_f64(float64_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vcvtpd_u64_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vcvtxd_f32_f64(float64_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vcvtxd_f32_f64(__p0); + return __ret; +} +#else +__ai float32_t vcvtxd_f32_f64(float64_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vcvtxd_f32_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vcvtx_f32_f64(float64x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vcvtx_f32_v((int8x16_t)__p0, 42); + return __ret; +} +#else +__ai float32x2_t vcvtx_f32_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vcvtx_f32_v((int8x16_t)__rev0, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai float32x2_t __noswap_vcvtx_f32_f64(float64x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vcvtx_f32_v((int8x16_t)__p0, 42); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vcvtx_high_f32_f64(float32x2_t __p0, float64x2_t __p1) { + float32x4_t __ret; + __ret = vcombine_f32(__p0, vcvtx_f32_f64(__p1)); + return __ret; +} +#else +__ai float32x4_t vcvtx_high_f32_f64(float32x2_t __p0, float64x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x4_t __ret; + __ret = __noswap_vcombine_f32(__rev0, __noswap_vcvtx_f32_f64(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vdivq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = __p0 / __p1; + return __ret; +} +#else +__ai float64x2_t vdivq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = __rev0 / __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vdivq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = __p0 / __p1; + return __ret; +} +#else +__ai float32x4_t vdivq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __rev0 / __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vdiv_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = __p0 / __p1; + return __ret; +} +#else +__ai float64x1_t vdiv_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = __p0 / __p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vdiv_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = __p0 / __p1; + return __ret; +} +#else +__ai float32x2_t vdiv_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = __rev0 / __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupb_lane_p8(__p0, __p1) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8_t __ret; \ + __ret = (poly8_t) __builtin_neon_vdupb_lane_i8((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdupb_lane_p8(__p0, __p1) __extension__ ({ \ + poly8x8_t __s0 = __p0; \ + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8_t __ret; \ + __ret = (poly8_t) __builtin_neon_vdupb_lane_i8((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vduph_lane_p16(__p0, __p1) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16_t __ret; \ + __ret = (poly16_t) __builtin_neon_vduph_lane_i16((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vduph_lane_p16(__p0, __p1) __extension__ ({ \ + poly16x4_t __s0 = __p0; \ + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + poly16_t __ret; \ + __ret = (poly16_t) __builtin_neon_vduph_lane_i16((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupb_lane_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vdupb_lane_i8((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdupb_lane_u8(__p0, __p1) __extension__ ({ \ + uint8x8_t __s0 = __p0; \ + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vdupb_lane_i8((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdups_lane_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vdups_lane_i32((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdups_lane_u32(__p0, __p1) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vdups_lane_i32((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupd_lane_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vdupd_lane_i64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdupd_lane_u64(__p0, __p1) __extension__ ({ \ + uint64x1_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vdupd_lane_i64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vduph_lane_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vduph_lane_i16((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vduph_lane_u16(__p0, __p1) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vduph_lane_i16((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupb_lane_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vdupb_lane_i8((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdupb_lane_s8(__p0, __p1) __extension__ ({ \ + int8x8_t __s0 = __p0; \ + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vdupb_lane_i8((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupd_lane_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vdupd_lane_f64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdupd_lane_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vdupd_lane_f64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdups_lane_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vdups_lane_f32((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdups_lane_f32(__p0, __p1) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vdups_lane_f32((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdups_lane_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vdups_lane_i32((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdups_lane_s32(__p0, __p1) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vdups_lane_i32((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupd_lane_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vdupd_lane_i64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdupd_lane_s64(__p0, __p1) __extension__ ({ \ + int64x1_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vdupd_lane_i64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vduph_lane_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vduph_lane_i16((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vduph_lane_s16(__p0, __p1) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vduph_lane_i16((int8x8_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_p64(__p0, __p1) __extension__ ({ \ + poly64x1_t __s0 = __p0; \ + poly64x1_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1); \ + __ret; \ +}) +#else +#define vdup_lane_p64(__p0, __p1) __extension__ ({ \ + poly64x1_t __s0 = __p0; \ + poly64x1_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_p64(__p0, __p1) __extension__ ({ \ + poly64x1_t __s0 = __p0; \ + poly64x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_lane_p64(__p0, __p1) __extension__ ({ \ + poly64x1_t __s0 = __p0; \ + poly64x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_lane_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_lane_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x8_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_lane_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float16x8_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x1_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1); \ + __ret; \ +}) +#else +#define vdup_lane_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x1_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_lane_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_lane_f16(__p0, __p1) __extension__ ({ \ + float16x4_t __s0 = __p0; \ + float16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float16x4_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupb_laneq_p8(__p0, __p1) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8_t __ret; \ + __ret = (poly8_t) __builtin_neon_vdupb_laneq_i8((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdupb_laneq_p8(__p0, __p1) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8_t __ret; \ + __ret = (poly8_t) __builtin_neon_vdupb_laneq_i8((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vduph_laneq_p16(__p0, __p1) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16_t __ret; \ + __ret = (poly16_t) __builtin_neon_vduph_laneq_i16((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vduph_laneq_p16(__p0, __p1) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16_t __ret; \ + __ret = (poly16_t) __builtin_neon_vduph_laneq_i16((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupb_laneq_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vdupb_laneq_i8((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdupb_laneq_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vdupb_laneq_i8((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdups_laneq_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vdups_laneq_i32((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdups_laneq_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vdups_laneq_i32((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupd_laneq_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vdupd_laneq_i64((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdupd_laneq_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vdupd_laneq_i64((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vduph_laneq_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vduph_laneq_i16((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vduph_laneq_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vduph_laneq_i16((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupb_laneq_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vdupb_laneq_i8((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdupb_laneq_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vdupb_laneq_i8((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupd_laneq_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vdupd_laneq_f64((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdupd_laneq_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vdupd_laneq_f64((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdups_laneq_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vdups_laneq_f32((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdups_laneq_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vdups_laneq_f32((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdups_laneq_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vdups_laneq_i32((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdups_laneq_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vdups_laneq_i32((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupd_laneq_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vdupd_laneq_i64((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vdupd_laneq_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vdupd_laneq_i64((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vduph_laneq_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vduph_laneq_i16((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vduph_laneq_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vduph_laneq_i16((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_laneq_p8(__p0, __p1) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8x8_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_laneq_p8(__p0, __p1) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x8_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_laneq_p64(__p0, __p1) __extension__ ({ \ + poly64x2_t __s0 = __p0; \ + poly64x1_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1); \ + __ret; \ +}) +#else +#define vdup_laneq_p64(__p0, __p1) __extension__ ({ \ + poly64x2_t __s0 = __p0; \ + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + poly64x1_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_laneq_p16(__p0, __p1) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16x4_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_laneq_p16(__p0, __p1) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x4_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_laneq_p8(__p0, __p1) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8x16_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_laneq_p8(__p0, __p1) __extension__ ({ \ + poly8x16_t __s0 = __p0; \ + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_laneq_p64(__p0, __p1) __extension__ ({ \ + poly64x2_t __s0 = __p0; \ + poly64x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_laneq_p64(__p0, __p1) __extension__ ({ \ + poly64x2_t __s0 = __p0; \ + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + poly64x2_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_laneq_p16(__p0, __p1) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16x8_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_laneq_p16(__p0, __p1) __extension__ ({ \ + poly16x8_t __s0 = __p0; \ + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly16x8_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_laneq_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_laneq_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_laneq_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_laneq_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_laneq_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_laneq_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64x2_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_laneq_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_laneq_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_laneq_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_laneq_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_laneq_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_laneq_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float64x2_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_laneq_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_laneq_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x4_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_laneq_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_laneq_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x8_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_laneq_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_laneq_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_laneq_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_laneq_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64x2_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdupq_laneq_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdupq_laneq_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_laneq_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x8_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_laneq_u8(__p0, __p1) __extension__ ({ \ + uint8x16_t __s0 = __p0; \ + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x8_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_laneq_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_laneq_u32(__p0, __p1) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x2_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_laneq_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x1_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1); \ + __ret; \ +}) +#else +#define vdup_laneq_u64(__p0, __p1) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint64x1_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_laneq_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x4_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_laneq_u16(__p0, __p1) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_laneq_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x8_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_laneq_s8(__p0, __p1) __extension__ ({ \ + int8x16_t __s0 = __p0; \ + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x8_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_laneq_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x1_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1); \ + __ret; \ +}) +#else +#define vdup_laneq_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float64x1_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_laneq_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_laneq_f32(__p0, __p1) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x2_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_laneq_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x4_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_laneq_f16(__p0, __p1) __extension__ ({ \ + float16x8_t __s0 = __p0; \ + float16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16x4_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_laneq_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x2_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_laneq_s32(__p0, __p1) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x2_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_laneq_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x1_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1); \ + __ret; \ +}) +#else +#define vdup_laneq_s64(__p0, __p1) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int64x1_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vdup_laneq_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x4_t __ret; \ + __ret = __builtin_shufflevector(__s0, __s0, __p1, __p1, __p1, __p1); \ + __ret; \ +}) +#else +#define vdup_laneq_s16(__p0, __p1) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __builtin_shufflevector(__rev0, __rev0, __p1, __p1, __p1, __p1); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vdup_n_p64(poly64_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t) {__p0}; + return __ret; +} +#else +__ai poly64x1_t vdup_n_p64(poly64_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t) {__p0}; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vdupq_n_p64(poly64_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t) {__p0, __p0}; + return __ret; +} +#else +__ai poly64x2_t vdupq_n_p64(poly64_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t) {__p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vdupq_n_f64(float64_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) {__p0, __p0}; + return __ret; +} +#else +__ai float64x2_t vdupq_n_f64(float64_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) {__p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vdup_n_f64(float64_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) {__p0}; + return __ret; +} +#else +__ai float64x1_t vdup_n_f64(float64_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) {__p0}; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vext_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1_t __s0 = __p0; \ + poly64x1_t __s1 = __p1; \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 6); \ + __ret; \ +}) +#else +#define vext_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1_t __s0 = __p0; \ + poly64x1_t __s1 = __p1; \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vextq_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2_t __s0 = __p0; \ + poly64x2_t __s1 = __p1; \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vextq_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 38); \ + __ret; \ +}) +#else +#define vextq_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2_t __s0 = __p0; \ + poly64x2_t __s1 = __p1; \ + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vextq_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 38); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vextq_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vextq_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 42); \ + __ret; \ +}) +#else +#define vextq_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vextq_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 42); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vext_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 10); \ + __ret; \ +}) +#else +#define vext_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vext_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vfmaq_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vfmaq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 42); + return __ret; +} +#else +__ai float64x2_t vfmaq_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vfmaq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai float64x2_t __noswap_vfmaq_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vfmaq_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 42); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vfma_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vfma_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 10); + return __ret; +} +#else +__ai float64x1_t vfma_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vfma_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 10); + return __ret; +} +__ai float64x1_t __noswap_vfma_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vfma_v((int8x8_t)__p0, (int8x8_t)__p1, (int8x8_t)__p2, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmad_lane_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64_t __s0 = __p0; \ + float64_t __s1 = __p1; \ + float64x1_t __s2 = __p2; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vfmad_lane_f64(__s0, __s1, (int8x8_t)__s2, __p3); \ + __ret; \ +}) +#else +#define vfmad_lane_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64_t __s0 = __p0; \ + float64_t __s1 = __p1; \ + float64x1_t __s2 = __p2; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vfmad_lane_f64(__s0, __s1, (int8x8_t)__s2, __p3); \ + __ret; \ +}) +#define __noswap_vfmad_lane_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64_t __s0 = __p0; \ + float64_t __s1 = __p1; \ + float64x1_t __s2 = __p2; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vfmad_lane_f64(__s0, __s1, (int8x8_t)__s2, __p3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmas_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32_t __s0 = __p0; \ + float32_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vfmas_lane_f32(__s0, __s1, (int8x8_t)__s2, __p3); \ + __ret; \ +}) +#else +#define vfmas_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32_t __s0 = __p0; \ + float32_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vfmas_lane_f32(__s0, __s1, (int8x8_t)__rev2, __p3); \ + __ret; \ +}) +#define __noswap_vfmas_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32_t __s0 = __p0; \ + float32_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vfmas_lane_f32(__s0, __s1, (int8x8_t)__s2, __p3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmaq_lane_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x1_t __s2 = __p2; \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vfmaq_lane_v((int8x16_t)__s0, (int8x16_t)__s1, (int8x8_t)__s2, __p3, 42); \ + __ret; \ +}) +#else +#define vfmaq_lane_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x1_t __s2 = __p2; \ + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vfmaq_lane_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x8_t)__s2, __p3, 42); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vfmaq_lane_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x1_t __s2 = __p2; \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vfmaq_lane_v((int8x16_t)__s0, (int8x16_t)__s1, (int8x8_t)__s2, __p3, 42); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmaq_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vfmaq_lane_v((int8x16_t)__s0, (int8x16_t)__s1, (int8x8_t)__s2, __p3, 41); \ + __ret; \ +}) +#else +#define vfmaq_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vfmaq_lane_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x8_t)__rev2, __p3, 41); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vfmaq_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vfmaq_lane_v((int8x16_t)__s0, (int8x16_t)__s1, (int8x8_t)__s2, __p3, 41); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfma_lane_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x1_t __s2 = __p2; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vfma_lane_v((int8x8_t)__s0, (int8x8_t)__s1, (int8x8_t)__s2, __p3, 10); \ + __ret; \ +}) +#else +#define vfma_lane_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x1_t __s2 = __p2; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vfma_lane_v((int8x8_t)__s0, (int8x8_t)__s1, (int8x8_t)__s2, __p3, 10); \ + __ret; \ +}) +#define __noswap_vfma_lane_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x1_t __s2 = __p2; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vfma_lane_v((int8x8_t)__s0, (int8x8_t)__s1, (int8x8_t)__s2, __p3, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfma_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vfma_lane_v((int8x8_t)__s0, (int8x8_t)__s1, (int8x8_t)__s2, __p3, 9); \ + __ret; \ +}) +#else +#define vfma_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vfma_lane_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x8_t)__rev2, __p3, 9); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vfma_lane_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __s2 = __p2; \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vfma_lane_v((int8x8_t)__s0, (int8x8_t)__s1, (int8x8_t)__s2, __p3, 9); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmad_laneq_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64_t __s0 = __p0; \ + float64_t __s1 = __p1; \ + float64x2_t __s2 = __p2; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vfmad_laneq_f64(__s0, __s1, (int8x16_t)__s2, __p3); \ + __ret; \ +}) +#else +#define vfmad_laneq_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64_t __s0 = __p0; \ + float64_t __s1 = __p1; \ + float64x2_t __s2 = __p2; \ + float64x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vfmad_laneq_f64(__s0, __s1, (int8x16_t)__rev2, __p3); \ + __ret; \ +}) +#define __noswap_vfmad_laneq_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64_t __s0 = __p0; \ + float64_t __s1 = __p1; \ + float64x2_t __s2 = __p2; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vfmad_laneq_f64(__s0, __s1, (int8x16_t)__s2, __p3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmas_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32_t __s0 = __p0; \ + float32_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vfmas_laneq_f32(__s0, __s1, (int8x16_t)__s2, __p3); \ + __ret; \ +}) +#else +#define vfmas_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32_t __s0 = __p0; \ + float32_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vfmas_laneq_f32(__s0, __s1, (int8x16_t)__rev2, __p3); \ + __ret; \ +}) +#define __noswap_vfmas_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32_t __s0 = __p0; \ + float32_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32_t __ret; \ + __ret = (float32_t) __builtin_neon_vfmas_laneq_f32(__s0, __s1, (int8x16_t)__s2, __p3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmaq_laneq_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x2_t __s2 = __p2; \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vfmaq_laneq_v((int8x16_t)__s0, (int8x16_t)__s1, (int8x16_t)__s2, __p3, 42); \ + __ret; \ +}) +#else +#define vfmaq_laneq_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x2_t __s2 = __p2; \ + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float64x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vfmaq_laneq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, __p3, 42); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vfmaq_laneq_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x2_t __s2 = __p2; \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vfmaq_laneq_v((int8x16_t)__s0, (int8x16_t)__s1, (int8x16_t)__s2, __p3, 42); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmaq_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vfmaq_laneq_v((int8x16_t)__s0, (int8x16_t)__s1, (int8x16_t)__s2, __p3, 41); \ + __ret; \ +}) +#else +#define vfmaq_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vfmaq_laneq_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, __p3, 41); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#define __noswap_vfmaq_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x4_t __ret; \ + __ret = (float32x4_t) __builtin_neon_vfmaq_laneq_v((int8x16_t)__s0, (int8x16_t)__s1, (int8x16_t)__s2, __p3, 41); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfma_laneq_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x2_t __s2 = __p2; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vfma_laneq_v((int8x8_t)__s0, (int8x8_t)__s1, (int8x16_t)__s2, __p3, 10); \ + __ret; \ +}) +#else +#define vfma_laneq_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x2_t __s2 = __p2; \ + float64x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vfma_laneq_v((int8x8_t)__s0, (int8x8_t)__s1, (int8x16_t)__rev2, __p3, 10); \ + __ret; \ +}) +#define __noswap_vfma_laneq_f64(__p0, __p1, __p2, __p3) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x2_t __s2 = __p2; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vfma_laneq_v((int8x8_t)__s0, (int8x8_t)__s1, (int8x16_t)__s2, __p3, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfma_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vfma_laneq_v((int8x8_t)__s0, (int8x8_t)__s1, (int8x16_t)__s2, __p3, 9); \ + __ret; \ +}) +#else +#define vfma_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vfma_laneq_v((int8x8_t)__rev0, (int8x8_t)__rev1, (int8x16_t)__rev2, __p3, 9); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vfma_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x2_t __ret; \ + __ret = (float32x2_t) __builtin_neon_vfma_laneq_v((int8x8_t)__s0, (int8x8_t)__s1, (int8x16_t)__s2, __p3, 9); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vfmaq_n_f64(float64x2_t __p0, float64x2_t __p1, float64_t __p2) { + float64x2_t __ret; + __ret = vfmaq_f64(__p0, __p1, (float64x2_t) {__p2, __p2}); + return __ret; +} +#else +__ai float64x2_t vfmaq_n_f64(float64x2_t __p0, float64x2_t __p1, float64_t __p2) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = __noswap_vfmaq_f64(__rev0, __rev1, (float64x2_t) {__p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vfmaq_n_f32(float32x4_t __p0, float32x4_t __p1, float32_t __p2) { + float32x4_t __ret; + __ret = vfmaq_f32(__p0, __p1, (float32x4_t) {__p2, __p2, __p2, __p2}); + return __ret; +} +#else +__ai float32x4_t vfmaq_n_f32(float32x4_t __p0, float32x4_t __p1, float32_t __p2) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __noswap_vfmaq_f32(__rev0, __rev1, (float32x4_t) {__p2, __p2, __p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vfma_n_f32(float32x2_t __p0, float32x2_t __p1, float32_t __p2) { + float32x2_t __ret; + __ret = vfma_f32(__p0, __p1, (float32x2_t) {__p2, __p2}); + return __ret; +} +#else +__ai float32x2_t vfma_n_f32(float32x2_t __p0, float32x2_t __p1, float32_t __p2) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = __noswap_vfma_f32(__rev0, __rev1, (float32x2_t) {__p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vfmsq_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) { + float64x2_t __ret; + __ret = vfmaq_f64(__p0, -__p1, __p2); + return __ret; +} +#else +__ai float64x2_t vfmsq_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + float64x2_t __ret; + __ret = __noswap_vfmaq_f64(__rev0, -__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vfms_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) { + float64x1_t __ret; + __ret = vfma_f64(__p0, -__p1, __p2); + return __ret; +} +#else +__ai float64x1_t vfms_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) { + float64x1_t __ret; + __ret = __noswap_vfma_f64(__p0, -__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmsd_lane_f64(__p0_104, __p1_104, __p2_104, __p3_104) __extension__ ({ \ + float64_t __s0_104 = __p0_104; \ + float64_t __s1_104 = __p1_104; \ + float64x1_t __s2_104 = __p2_104; \ + float64_t __ret_104; \ + __ret_104 = vfmad_lane_f64(__s0_104, -__s1_104, __s2_104, __p3_104); \ + __ret_104; \ +}) +#else +#define vfmsd_lane_f64(__p0_105, __p1_105, __p2_105, __p3_105) __extension__ ({ \ + float64_t __s0_105 = __p0_105; \ + float64_t __s1_105 = __p1_105; \ + float64x1_t __s2_105 = __p2_105; \ + float64_t __ret_105; \ + __ret_105 = __noswap_vfmad_lane_f64(__s0_105, -__s1_105, __s2_105, __p3_105); \ + __ret_105; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmss_lane_f32(__p0_106, __p1_106, __p2_106, __p3_106) __extension__ ({ \ + float32_t __s0_106 = __p0_106; \ + float32_t __s1_106 = __p1_106; \ + float32x2_t __s2_106 = __p2_106; \ + float32_t __ret_106; \ + __ret_106 = vfmas_lane_f32(__s0_106, -__s1_106, __s2_106, __p3_106); \ + __ret_106; \ +}) +#else +#define vfmss_lane_f32(__p0_107, __p1_107, __p2_107, __p3_107) __extension__ ({ \ + float32_t __s0_107 = __p0_107; \ + float32_t __s1_107 = __p1_107; \ + float32x2_t __s2_107 = __p2_107; \ + float32x2_t __rev2_107; __rev2_107 = __builtin_shufflevector(__s2_107, __s2_107, 1, 0); \ + float32_t __ret_107; \ + __ret_107 = __noswap_vfmas_lane_f32(__s0_107, -__s1_107, __rev2_107, __p3_107); \ + __ret_107; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmsq_lane_f64(__p0_108, __p1_108, __p2_108, __p3_108) __extension__ ({ \ + float64x2_t __s0_108 = __p0_108; \ + float64x2_t __s1_108 = __p1_108; \ + float64x1_t __s2_108 = __p2_108; \ + float64x2_t __ret_108; \ + __ret_108 = vfmaq_lane_f64(__s0_108, -__s1_108, __s2_108, __p3_108); \ + __ret_108; \ +}) +#else +#define vfmsq_lane_f64(__p0_109, __p1_109, __p2_109, __p3_109) __extension__ ({ \ + float64x2_t __s0_109 = __p0_109; \ + float64x2_t __s1_109 = __p1_109; \ + float64x1_t __s2_109 = __p2_109; \ + float64x2_t __rev0_109; __rev0_109 = __builtin_shufflevector(__s0_109, __s0_109, 1, 0); \ + float64x2_t __rev1_109; __rev1_109 = __builtin_shufflevector(__s1_109, __s1_109, 1, 0); \ + float64x2_t __ret_109; \ + __ret_109 = __noswap_vfmaq_lane_f64(__rev0_109, -__rev1_109, __s2_109, __p3_109); \ + __ret_109 = __builtin_shufflevector(__ret_109, __ret_109, 1, 0); \ + __ret_109; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmsq_lane_f32(__p0_110, __p1_110, __p2_110, __p3_110) __extension__ ({ \ + float32x4_t __s0_110 = __p0_110; \ + float32x4_t __s1_110 = __p1_110; \ + float32x2_t __s2_110 = __p2_110; \ + float32x4_t __ret_110; \ + __ret_110 = vfmaq_lane_f32(__s0_110, -__s1_110, __s2_110, __p3_110); \ + __ret_110; \ +}) +#else +#define vfmsq_lane_f32(__p0_111, __p1_111, __p2_111, __p3_111) __extension__ ({ \ + float32x4_t __s0_111 = __p0_111; \ + float32x4_t __s1_111 = __p1_111; \ + float32x2_t __s2_111 = __p2_111; \ + float32x4_t __rev0_111; __rev0_111 = __builtin_shufflevector(__s0_111, __s0_111, 3, 2, 1, 0); \ + float32x4_t __rev1_111; __rev1_111 = __builtin_shufflevector(__s1_111, __s1_111, 3, 2, 1, 0); \ + float32x2_t __rev2_111; __rev2_111 = __builtin_shufflevector(__s2_111, __s2_111, 1, 0); \ + float32x4_t __ret_111; \ + __ret_111 = __noswap_vfmaq_lane_f32(__rev0_111, -__rev1_111, __rev2_111, __p3_111); \ + __ret_111 = __builtin_shufflevector(__ret_111, __ret_111, 3, 2, 1, 0); \ + __ret_111; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfms_lane_f64(__p0_112, __p1_112, __p2_112, __p3_112) __extension__ ({ \ + float64x1_t __s0_112 = __p0_112; \ + float64x1_t __s1_112 = __p1_112; \ + float64x1_t __s2_112 = __p2_112; \ + float64x1_t __ret_112; \ + __ret_112 = vfma_lane_f64(__s0_112, -__s1_112, __s2_112, __p3_112); \ + __ret_112; \ +}) +#else +#define vfms_lane_f64(__p0_113, __p1_113, __p2_113, __p3_113) __extension__ ({ \ + float64x1_t __s0_113 = __p0_113; \ + float64x1_t __s1_113 = __p1_113; \ + float64x1_t __s2_113 = __p2_113; \ + float64x1_t __ret_113; \ + __ret_113 = __noswap_vfma_lane_f64(__s0_113, -__s1_113, __s2_113, __p3_113); \ + __ret_113; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfms_lane_f32(__p0_114, __p1_114, __p2_114, __p3_114) __extension__ ({ \ + float32x2_t __s0_114 = __p0_114; \ + float32x2_t __s1_114 = __p1_114; \ + float32x2_t __s2_114 = __p2_114; \ + float32x2_t __ret_114; \ + __ret_114 = vfma_lane_f32(__s0_114, -__s1_114, __s2_114, __p3_114); \ + __ret_114; \ +}) +#else +#define vfms_lane_f32(__p0_115, __p1_115, __p2_115, __p3_115) __extension__ ({ \ + float32x2_t __s0_115 = __p0_115; \ + float32x2_t __s1_115 = __p1_115; \ + float32x2_t __s2_115 = __p2_115; \ + float32x2_t __rev0_115; __rev0_115 = __builtin_shufflevector(__s0_115, __s0_115, 1, 0); \ + float32x2_t __rev1_115; __rev1_115 = __builtin_shufflevector(__s1_115, __s1_115, 1, 0); \ + float32x2_t __rev2_115; __rev2_115 = __builtin_shufflevector(__s2_115, __s2_115, 1, 0); \ + float32x2_t __ret_115; \ + __ret_115 = __noswap_vfma_lane_f32(__rev0_115, -__rev1_115, __rev2_115, __p3_115); \ + __ret_115 = __builtin_shufflevector(__ret_115, __ret_115, 1, 0); \ + __ret_115; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmsd_laneq_f64(__p0_116, __p1_116, __p2_116, __p3_116) __extension__ ({ \ + float64_t __s0_116 = __p0_116; \ + float64_t __s1_116 = __p1_116; \ + float64x2_t __s2_116 = __p2_116; \ + float64_t __ret_116; \ + __ret_116 = vfmad_laneq_f64(__s0_116, -__s1_116, __s2_116, __p3_116); \ + __ret_116; \ +}) +#else +#define vfmsd_laneq_f64(__p0_117, __p1_117, __p2_117, __p3_117) __extension__ ({ \ + float64_t __s0_117 = __p0_117; \ + float64_t __s1_117 = __p1_117; \ + float64x2_t __s2_117 = __p2_117; \ + float64x2_t __rev2_117; __rev2_117 = __builtin_shufflevector(__s2_117, __s2_117, 1, 0); \ + float64_t __ret_117; \ + __ret_117 = __noswap_vfmad_laneq_f64(__s0_117, -__s1_117, __rev2_117, __p3_117); \ + __ret_117; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmss_laneq_f32(__p0_118, __p1_118, __p2_118, __p3_118) __extension__ ({ \ + float32_t __s0_118 = __p0_118; \ + float32_t __s1_118 = __p1_118; \ + float32x4_t __s2_118 = __p2_118; \ + float32_t __ret_118; \ + __ret_118 = vfmas_laneq_f32(__s0_118, -__s1_118, __s2_118, __p3_118); \ + __ret_118; \ +}) +#else +#define vfmss_laneq_f32(__p0_119, __p1_119, __p2_119, __p3_119) __extension__ ({ \ + float32_t __s0_119 = __p0_119; \ + float32_t __s1_119 = __p1_119; \ + float32x4_t __s2_119 = __p2_119; \ + float32x4_t __rev2_119; __rev2_119 = __builtin_shufflevector(__s2_119, __s2_119, 3, 2, 1, 0); \ + float32_t __ret_119; \ + __ret_119 = __noswap_vfmas_laneq_f32(__s0_119, -__s1_119, __rev2_119, __p3_119); \ + __ret_119; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmsq_laneq_f64(__p0_120, __p1_120, __p2_120, __p3_120) __extension__ ({ \ + float64x2_t __s0_120 = __p0_120; \ + float64x2_t __s1_120 = __p1_120; \ + float64x2_t __s2_120 = __p2_120; \ + float64x2_t __ret_120; \ + __ret_120 = vfmaq_laneq_f64(__s0_120, -__s1_120, __s2_120, __p3_120); \ + __ret_120; \ +}) +#else +#define vfmsq_laneq_f64(__p0_121, __p1_121, __p2_121, __p3_121) __extension__ ({ \ + float64x2_t __s0_121 = __p0_121; \ + float64x2_t __s1_121 = __p1_121; \ + float64x2_t __s2_121 = __p2_121; \ + float64x2_t __rev0_121; __rev0_121 = __builtin_shufflevector(__s0_121, __s0_121, 1, 0); \ + float64x2_t __rev1_121; __rev1_121 = __builtin_shufflevector(__s1_121, __s1_121, 1, 0); \ + float64x2_t __rev2_121; __rev2_121 = __builtin_shufflevector(__s2_121, __s2_121, 1, 0); \ + float64x2_t __ret_121; \ + __ret_121 = __noswap_vfmaq_laneq_f64(__rev0_121, -__rev1_121, __rev2_121, __p3_121); \ + __ret_121 = __builtin_shufflevector(__ret_121, __ret_121, 1, 0); \ + __ret_121; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmsq_laneq_f32(__p0_122, __p1_122, __p2_122, __p3_122) __extension__ ({ \ + float32x4_t __s0_122 = __p0_122; \ + float32x4_t __s1_122 = __p1_122; \ + float32x4_t __s2_122 = __p2_122; \ + float32x4_t __ret_122; \ + __ret_122 = vfmaq_laneq_f32(__s0_122, -__s1_122, __s2_122, __p3_122); \ + __ret_122; \ +}) +#else +#define vfmsq_laneq_f32(__p0_123, __p1_123, __p2_123, __p3_123) __extension__ ({ \ + float32x4_t __s0_123 = __p0_123; \ + float32x4_t __s1_123 = __p1_123; \ + float32x4_t __s2_123 = __p2_123; \ + float32x4_t __rev0_123; __rev0_123 = __builtin_shufflevector(__s0_123, __s0_123, 3, 2, 1, 0); \ + float32x4_t __rev1_123; __rev1_123 = __builtin_shufflevector(__s1_123, __s1_123, 3, 2, 1, 0); \ + float32x4_t __rev2_123; __rev2_123 = __builtin_shufflevector(__s2_123, __s2_123, 3, 2, 1, 0); \ + float32x4_t __ret_123; \ + __ret_123 = __noswap_vfmaq_laneq_f32(__rev0_123, -__rev1_123, __rev2_123, __p3_123); \ + __ret_123 = __builtin_shufflevector(__ret_123, __ret_123, 3, 2, 1, 0); \ + __ret_123; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfms_laneq_f64(__p0_124, __p1_124, __p2_124, __p3_124) __extension__ ({ \ + float64x1_t __s0_124 = __p0_124; \ + float64x1_t __s1_124 = __p1_124; \ + float64x2_t __s2_124 = __p2_124; \ + float64x1_t __ret_124; \ + __ret_124 = vfma_laneq_f64(__s0_124, -__s1_124, __s2_124, __p3_124); \ + __ret_124; \ +}) +#else +#define vfms_laneq_f64(__p0_125, __p1_125, __p2_125, __p3_125) __extension__ ({ \ + float64x1_t __s0_125 = __p0_125; \ + float64x1_t __s1_125 = __p1_125; \ + float64x2_t __s2_125 = __p2_125; \ + float64x2_t __rev2_125; __rev2_125 = __builtin_shufflevector(__s2_125, __s2_125, 1, 0); \ + float64x1_t __ret_125; \ + __ret_125 = __noswap_vfma_laneq_f64(__s0_125, -__s1_125, __rev2_125, __p3_125); \ + __ret_125; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfms_laneq_f32(__p0_126, __p1_126, __p2_126, __p3_126) __extension__ ({ \ + float32x2_t __s0_126 = __p0_126; \ + float32x2_t __s1_126 = __p1_126; \ + float32x4_t __s2_126 = __p2_126; \ + float32x2_t __ret_126; \ + __ret_126 = vfma_laneq_f32(__s0_126, -__s1_126, __s2_126, __p3_126); \ + __ret_126; \ +}) +#else +#define vfms_laneq_f32(__p0_127, __p1_127, __p2_127, __p3_127) __extension__ ({ \ + float32x2_t __s0_127 = __p0_127; \ + float32x2_t __s1_127 = __p1_127; \ + float32x4_t __s2_127 = __p2_127; \ + float32x2_t __rev0_127; __rev0_127 = __builtin_shufflevector(__s0_127, __s0_127, 1, 0); \ + float32x2_t __rev1_127; __rev1_127 = __builtin_shufflevector(__s1_127, __s1_127, 1, 0); \ + float32x4_t __rev2_127; __rev2_127 = __builtin_shufflevector(__s2_127, __s2_127, 3, 2, 1, 0); \ + float32x2_t __ret_127; \ + __ret_127 = __noswap_vfma_laneq_f32(__rev0_127, -__rev1_127, __rev2_127, __p3_127); \ + __ret_127 = __builtin_shufflevector(__ret_127, __ret_127, 1, 0); \ + __ret_127; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vfmsq_n_f64(float64x2_t __p0, float64x2_t __p1, float64_t __p2) { + float64x2_t __ret; + __ret = vfmaq_f64(__p0, -__p1, (float64x2_t) {__p2, __p2}); + return __ret; +} +#else +__ai float64x2_t vfmsq_n_f64(float64x2_t __p0, float64x2_t __p1, float64_t __p2) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = __noswap_vfmaq_f64(__rev0, -__rev1, (float64x2_t) {__p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vfmsq_n_f32(float32x4_t __p0, float32x4_t __p1, float32_t __p2) { + float32x4_t __ret; + __ret = vfmaq_f32(__p0, -__p1, (float32x4_t) {__p2, __p2, __p2, __p2}); + return __ret; +} +#else +__ai float32x4_t vfmsq_n_f32(float32x4_t __p0, float32x4_t __p1, float32_t __p2) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __noswap_vfmaq_f32(__rev0, -__rev1, (float32x4_t) {__p2, __p2, __p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vfms_n_f32(float32x2_t __p0, float32x2_t __p1, float32_t __p2) { + float32x2_t __ret; + __ret = vfma_f32(__p0, -__p1, (float32x2_t) {__p2, __p2}); + return __ret; +} +#else +__ai float32x2_t vfms_n_f32(float32x2_t __p0, float32x2_t __p1, float32_t __p2) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = __noswap_vfma_f32(__rev0, -__rev1, (float32x2_t) {__p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vget_high_p64(poly64x2_t __p0) { + poly64x1_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1); + return __ret; +} +#else +__ai poly64x1_t vget_high_p64(poly64x2_t __p0) { + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + poly64x1_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1); + return __ret; +} +__ai poly64x1_t __noswap_vget_high_p64(poly64x2_t __p0) { + poly64x1_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vget_high_f64(float64x2_t __p0) { + float64x1_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 1); + return __ret; +} +#else +__ai float64x1_t vget_high_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x1_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vget_lane_p64(__p0, __p1) __extension__ ({ \ + poly64x1_t __s0 = __p0; \ + poly64_t __ret; \ + __ret = (poly64_t) __builtin_neon_vget_lane_i64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vget_lane_p64(__p0, __p1) __extension__ ({ \ + poly64x1_t __s0 = __p0; \ + poly64_t __ret; \ + __ret = (poly64_t) __builtin_neon_vget_lane_i64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#define __noswap_vget_lane_p64(__p0, __p1) __extension__ ({ \ + poly64x1_t __s0 = __p0; \ + poly64_t __ret; \ + __ret = (poly64_t) __builtin_neon_vget_lane_i64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vgetq_lane_p64(__p0, __p1) __extension__ ({ \ + poly64x2_t __s0 = __p0; \ + poly64_t __ret; \ + __ret = (poly64_t) __builtin_neon_vgetq_lane_i64((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vgetq_lane_p64(__p0, __p1) __extension__ ({ \ + poly64x2_t __s0 = __p0; \ + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + poly64_t __ret; \ + __ret = (poly64_t) __builtin_neon_vgetq_lane_i64((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vgetq_lane_p64(__p0, __p1) __extension__ ({ \ + poly64x2_t __s0 = __p0; \ + poly64_t __ret; \ + __ret = (poly64_t) __builtin_neon_vgetq_lane_i64((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vgetq_lane_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vgetq_lane_f64((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vgetq_lane_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vgetq_lane_f64((int8x16_t)__rev0, __p1); \ + __ret; \ +}) +#define __noswap_vgetq_lane_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vgetq_lane_f64((int8x16_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vget_lane_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vget_lane_f64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#else +#define vget_lane_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vget_lane_f64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#define __noswap_vget_lane_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64_t __ret; \ + __ret = (float64_t) __builtin_neon_vget_lane_f64((int8x8_t)__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vget_low_p64(poly64x2_t __p0) { + poly64x1_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0); + return __ret; +} +#else +__ai poly64x1_t vget_low_p64(poly64x2_t __p0) { + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + poly64x1_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vget_low_f64(float64x2_t __p0) { + float64x1_t __ret; + __ret = __builtin_shufflevector(__p0, __p0, 0); + return __ret; +} +#else +__ai float64x1_t vget_low_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x1_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev0, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_p64(__p0) __extension__ ({ \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vld1_v(__p0, 6); \ + __ret; \ +}) +#else +#define vld1_p64(__p0) __extension__ ({ \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vld1_v(__p0, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_p64(__p0) __extension__ ({ \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vld1q_v(__p0, 38); \ + __ret; \ +}) +#else +#define vld1q_p64(__p0) __extension__ ({ \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vld1q_v(__p0, 38); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_f64(__p0) __extension__ ({ \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vld1q_v(__p0, 42); \ + __ret; \ +}) +#else +#define vld1q_f64(__p0) __extension__ ({ \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vld1q_v(__p0, 42); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_f64(__p0) __extension__ ({ \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vld1_v(__p0, 10); \ + __ret; \ +}) +#else +#define vld1_f64(__p0) __extension__ ({ \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vld1_v(__p0, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_dup_p64(__p0) __extension__ ({ \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vld1_dup_v(__p0, 6); \ + __ret; \ +}) +#else +#define vld1_dup_p64(__p0) __extension__ ({ \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vld1_dup_v(__p0, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_dup_p64(__p0) __extension__ ({ \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vld1q_dup_v(__p0, 38); \ + __ret; \ +}) +#else +#define vld1q_dup_p64(__p0) __extension__ ({ \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vld1q_dup_v(__p0, 38); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_dup_f64(__p0) __extension__ ({ \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vld1q_dup_v(__p0, 42); \ + __ret; \ +}) +#else +#define vld1q_dup_f64(__p0) __extension__ ({ \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vld1q_dup_v(__p0, 42); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_dup_f64(__p0) __extension__ ({ \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vld1_dup_v(__p0, 10); \ + __ret; \ +}) +#else +#define vld1_dup_f64(__p0) __extension__ ({ \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vld1_dup_v(__p0, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1_t __s1 = __p1; \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 6); \ + __ret; \ +}) +#else +#define vld1_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1_t __s1 = __p1; \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2_t __s1 = __p1; \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 38); \ + __ret; \ +}) +#else +#define vld1q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2_t __s1 = __p1; \ + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 38); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2_t __s1 = __p1; \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__s1, __p2, 42); \ + __ret; \ +}) +#else +#define vld1q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2_t __s1 = __p1; \ + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vld1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 42); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1_t __s1 = __p1; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 10); \ + __ret; \ +}) +#else +#define vld1_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1_t __s1 = __p1; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vld1_lane_v(__p0, (int8x8_t)__s1, __p2, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_p8_x2(__p0) __extension__ ({ \ + poly8x8x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 4); \ + __ret; \ +}) +#else +#define vld1_p8_x2(__p0) __extension__ ({ \ + poly8x8x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 4); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_p64_x2(__p0) __extension__ ({ \ + poly64x1x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 6); \ + __ret; \ +}) +#else +#define vld1_p64_x2(__p0) __extension__ ({ \ + poly64x1x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_p16_x2(__p0) __extension__ ({ \ + poly16x4x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 5); \ + __ret; \ +}) +#else +#define vld1_p16_x2(__p0) __extension__ ({ \ + poly16x4x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 5); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_p8_x2(__p0) __extension__ ({ \ + poly8x16x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 36); \ + __ret; \ +}) +#else +#define vld1q_p8_x2(__p0) __extension__ ({ \ + poly8x16x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 36); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_p64_x2(__p0) __extension__ ({ \ + poly64x2x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 38); \ + __ret; \ +}) +#else +#define vld1q_p64_x2(__p0) __extension__ ({ \ + poly64x2x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 38); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_p16_x2(__p0) __extension__ ({ \ + poly16x8x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 37); \ + __ret; \ +}) +#else +#define vld1q_p16_x2(__p0) __extension__ ({ \ + poly16x8x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 37); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u8_x2(__p0) __extension__ ({ \ + uint8x16x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 48); \ + __ret; \ +}) +#else +#define vld1q_u8_x2(__p0) __extension__ ({ \ + uint8x16x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 48); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u32_x2(__p0) __extension__ ({ \ + uint32x4x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 50); \ + __ret; \ +}) +#else +#define vld1q_u32_x2(__p0) __extension__ ({ \ + uint32x4x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 50); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u64_x2(__p0) __extension__ ({ \ + uint64x2x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 51); \ + __ret; \ +}) +#else +#define vld1q_u64_x2(__p0) __extension__ ({ \ + uint64x2x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 51); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u16_x2(__p0) __extension__ ({ \ + uint16x8x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 49); \ + __ret; \ +}) +#else +#define vld1q_u16_x2(__p0) __extension__ ({ \ + uint16x8x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 49); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s8_x2(__p0) __extension__ ({ \ + int8x16x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 32); \ + __ret; \ +}) +#else +#define vld1q_s8_x2(__p0) __extension__ ({ \ + int8x16x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 32); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_f64_x2(__p0) __extension__ ({ \ + float64x2x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 42); \ + __ret; \ +}) +#else +#define vld1q_f64_x2(__p0) __extension__ ({ \ + float64x2x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 42); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_f32_x2(__p0) __extension__ ({ \ + float32x4x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 41); \ + __ret; \ +}) +#else +#define vld1q_f32_x2(__p0) __extension__ ({ \ + float32x4x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 41); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_f16_x2(__p0) __extension__ ({ \ + float16x8x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 40); \ + __ret; \ +}) +#else +#define vld1q_f16_x2(__p0) __extension__ ({ \ + float16x8x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 40); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s32_x2(__p0) __extension__ ({ \ + int32x4x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 34); \ + __ret; \ +}) +#else +#define vld1q_s32_x2(__p0) __extension__ ({ \ + int32x4x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 34); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s64_x2(__p0) __extension__ ({ \ + int64x2x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 35); \ + __ret; \ +}) +#else +#define vld1q_s64_x2(__p0) __extension__ ({ \ + int64x2x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 35); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s16_x2(__p0) __extension__ ({ \ + int16x8x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 33); \ + __ret; \ +}) +#else +#define vld1q_s16_x2(__p0) __extension__ ({ \ + int16x8x2_t __ret; \ + __builtin_neon_vld1q_x2_v(&__ret, __p0, 33); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u8_x2(__p0) __extension__ ({ \ + uint8x8x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 16); \ + __ret; \ +}) +#else +#define vld1_u8_x2(__p0) __extension__ ({ \ + uint8x8x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 16); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u32_x2(__p0) __extension__ ({ \ + uint32x2x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 18); \ + __ret; \ +}) +#else +#define vld1_u32_x2(__p0) __extension__ ({ \ + uint32x2x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 18); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u64_x2(__p0) __extension__ ({ \ + uint64x1x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 19); \ + __ret; \ +}) +#else +#define vld1_u64_x2(__p0) __extension__ ({ \ + uint64x1x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u16_x2(__p0) __extension__ ({ \ + uint16x4x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 17); \ + __ret; \ +}) +#else +#define vld1_u16_x2(__p0) __extension__ ({ \ + uint16x4x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 17); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s8_x2(__p0) __extension__ ({ \ + int8x8x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 0); \ + __ret; \ +}) +#else +#define vld1_s8_x2(__p0) __extension__ ({ \ + int8x8x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 0); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_f64_x2(__p0) __extension__ ({ \ + float64x1x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 10); \ + __ret; \ +}) +#else +#define vld1_f64_x2(__p0) __extension__ ({ \ + float64x1x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_f32_x2(__p0) __extension__ ({ \ + float32x2x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 9); \ + __ret; \ +}) +#else +#define vld1_f32_x2(__p0) __extension__ ({ \ + float32x2x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 9); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_f16_x2(__p0) __extension__ ({ \ + float16x4x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 8); \ + __ret; \ +}) +#else +#define vld1_f16_x2(__p0) __extension__ ({ \ + float16x4x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 8); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s32_x2(__p0) __extension__ ({ \ + int32x2x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 2); \ + __ret; \ +}) +#else +#define vld1_s32_x2(__p0) __extension__ ({ \ + int32x2x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 2); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s64_x2(__p0) __extension__ ({ \ + int64x1x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 3); \ + __ret; \ +}) +#else +#define vld1_s64_x2(__p0) __extension__ ({ \ + int64x1x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s16_x2(__p0) __extension__ ({ \ + int16x4x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 1); \ + __ret; \ +}) +#else +#define vld1_s16_x2(__p0) __extension__ ({ \ + int16x4x2_t __ret; \ + __builtin_neon_vld1_x2_v(&__ret, __p0, 1); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_p8_x3(__p0) __extension__ ({ \ + poly8x8x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 4); \ + __ret; \ +}) +#else +#define vld1_p8_x3(__p0) __extension__ ({ \ + poly8x8x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 4); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_p64_x3(__p0) __extension__ ({ \ + poly64x1x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 6); \ + __ret; \ +}) +#else +#define vld1_p64_x3(__p0) __extension__ ({ \ + poly64x1x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_p16_x3(__p0) __extension__ ({ \ + poly16x4x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 5); \ + __ret; \ +}) +#else +#define vld1_p16_x3(__p0) __extension__ ({ \ + poly16x4x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 5); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_p8_x3(__p0) __extension__ ({ \ + poly8x16x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 36); \ + __ret; \ +}) +#else +#define vld1q_p8_x3(__p0) __extension__ ({ \ + poly8x16x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 36); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_p64_x3(__p0) __extension__ ({ \ + poly64x2x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 38); \ + __ret; \ +}) +#else +#define vld1q_p64_x3(__p0) __extension__ ({ \ + poly64x2x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 38); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_p16_x3(__p0) __extension__ ({ \ + poly16x8x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 37); \ + __ret; \ +}) +#else +#define vld1q_p16_x3(__p0) __extension__ ({ \ + poly16x8x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 37); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u8_x3(__p0) __extension__ ({ \ + uint8x16x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 48); \ + __ret; \ +}) +#else +#define vld1q_u8_x3(__p0) __extension__ ({ \ + uint8x16x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 48); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u32_x3(__p0) __extension__ ({ \ + uint32x4x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 50); \ + __ret; \ +}) +#else +#define vld1q_u32_x3(__p0) __extension__ ({ \ + uint32x4x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 50); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u64_x3(__p0) __extension__ ({ \ + uint64x2x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 51); \ + __ret; \ +}) +#else +#define vld1q_u64_x3(__p0) __extension__ ({ \ + uint64x2x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 51); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u16_x3(__p0) __extension__ ({ \ + uint16x8x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 49); \ + __ret; \ +}) +#else +#define vld1q_u16_x3(__p0) __extension__ ({ \ + uint16x8x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 49); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s8_x3(__p0) __extension__ ({ \ + int8x16x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 32); \ + __ret; \ +}) +#else +#define vld1q_s8_x3(__p0) __extension__ ({ \ + int8x16x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 32); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_f64_x3(__p0) __extension__ ({ \ + float64x2x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 42); \ + __ret; \ +}) +#else +#define vld1q_f64_x3(__p0) __extension__ ({ \ + float64x2x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 42); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_f32_x3(__p0) __extension__ ({ \ + float32x4x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 41); \ + __ret; \ +}) +#else +#define vld1q_f32_x3(__p0) __extension__ ({ \ + float32x4x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 41); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_f16_x3(__p0) __extension__ ({ \ + float16x8x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 40); \ + __ret; \ +}) +#else +#define vld1q_f16_x3(__p0) __extension__ ({ \ + float16x8x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 40); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s32_x3(__p0) __extension__ ({ \ + int32x4x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 34); \ + __ret; \ +}) +#else +#define vld1q_s32_x3(__p0) __extension__ ({ \ + int32x4x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 34); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s64_x3(__p0) __extension__ ({ \ + int64x2x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 35); \ + __ret; \ +}) +#else +#define vld1q_s64_x3(__p0) __extension__ ({ \ + int64x2x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 35); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s16_x3(__p0) __extension__ ({ \ + int16x8x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 33); \ + __ret; \ +}) +#else +#define vld1q_s16_x3(__p0) __extension__ ({ \ + int16x8x3_t __ret; \ + __builtin_neon_vld1q_x3_v(&__ret, __p0, 33); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u8_x3(__p0) __extension__ ({ \ + uint8x8x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 16); \ + __ret; \ +}) +#else +#define vld1_u8_x3(__p0) __extension__ ({ \ + uint8x8x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 16); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u32_x3(__p0) __extension__ ({ \ + uint32x2x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 18); \ + __ret; \ +}) +#else +#define vld1_u32_x3(__p0) __extension__ ({ \ + uint32x2x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 18); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u64_x3(__p0) __extension__ ({ \ + uint64x1x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 19); \ + __ret; \ +}) +#else +#define vld1_u64_x3(__p0) __extension__ ({ \ + uint64x1x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u16_x3(__p0) __extension__ ({ \ + uint16x4x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 17); \ + __ret; \ +}) +#else +#define vld1_u16_x3(__p0) __extension__ ({ \ + uint16x4x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 17); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s8_x3(__p0) __extension__ ({ \ + int8x8x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 0); \ + __ret; \ +}) +#else +#define vld1_s8_x3(__p0) __extension__ ({ \ + int8x8x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 0); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_f64_x3(__p0) __extension__ ({ \ + float64x1x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 10); \ + __ret; \ +}) +#else +#define vld1_f64_x3(__p0) __extension__ ({ \ + float64x1x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_f32_x3(__p0) __extension__ ({ \ + float32x2x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 9); \ + __ret; \ +}) +#else +#define vld1_f32_x3(__p0) __extension__ ({ \ + float32x2x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 9); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_f16_x3(__p0) __extension__ ({ \ + float16x4x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 8); \ + __ret; \ +}) +#else +#define vld1_f16_x3(__p0) __extension__ ({ \ + float16x4x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 8); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s32_x3(__p0) __extension__ ({ \ + int32x2x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 2); \ + __ret; \ +}) +#else +#define vld1_s32_x3(__p0) __extension__ ({ \ + int32x2x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 2); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s64_x3(__p0) __extension__ ({ \ + int64x1x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 3); \ + __ret; \ +}) +#else +#define vld1_s64_x3(__p0) __extension__ ({ \ + int64x1x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s16_x3(__p0) __extension__ ({ \ + int16x4x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 1); \ + __ret; \ +}) +#else +#define vld1_s16_x3(__p0) __extension__ ({ \ + int16x4x3_t __ret; \ + __builtin_neon_vld1_x3_v(&__ret, __p0, 1); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_p8_x4(__p0) __extension__ ({ \ + poly8x8x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 4); \ + __ret; \ +}) +#else +#define vld1_p8_x4(__p0) __extension__ ({ \ + poly8x8x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 4); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_p64_x4(__p0) __extension__ ({ \ + poly64x1x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 6); \ + __ret; \ +}) +#else +#define vld1_p64_x4(__p0) __extension__ ({ \ + poly64x1x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_p16_x4(__p0) __extension__ ({ \ + poly16x4x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 5); \ + __ret; \ +}) +#else +#define vld1_p16_x4(__p0) __extension__ ({ \ + poly16x4x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 5); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_p8_x4(__p0) __extension__ ({ \ + poly8x16x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 36); \ + __ret; \ +}) +#else +#define vld1q_p8_x4(__p0) __extension__ ({ \ + poly8x16x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 36); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_p64_x4(__p0) __extension__ ({ \ + poly64x2x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 38); \ + __ret; \ +}) +#else +#define vld1q_p64_x4(__p0) __extension__ ({ \ + poly64x2x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 38); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_p16_x4(__p0) __extension__ ({ \ + poly16x8x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 37); \ + __ret; \ +}) +#else +#define vld1q_p16_x4(__p0) __extension__ ({ \ + poly16x8x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 37); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u8_x4(__p0) __extension__ ({ \ + uint8x16x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 48); \ + __ret; \ +}) +#else +#define vld1q_u8_x4(__p0) __extension__ ({ \ + uint8x16x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 48); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u32_x4(__p0) __extension__ ({ \ + uint32x4x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 50); \ + __ret; \ +}) +#else +#define vld1q_u32_x4(__p0) __extension__ ({ \ + uint32x4x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 50); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u64_x4(__p0) __extension__ ({ \ + uint64x2x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 51); \ + __ret; \ +}) +#else +#define vld1q_u64_x4(__p0) __extension__ ({ \ + uint64x2x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 51); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_u16_x4(__p0) __extension__ ({ \ + uint16x8x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 49); \ + __ret; \ +}) +#else +#define vld1q_u16_x4(__p0) __extension__ ({ \ + uint16x8x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 49); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s8_x4(__p0) __extension__ ({ \ + int8x16x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 32); \ + __ret; \ +}) +#else +#define vld1q_s8_x4(__p0) __extension__ ({ \ + int8x16x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 32); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_f64_x4(__p0) __extension__ ({ \ + float64x2x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 42); \ + __ret; \ +}) +#else +#define vld1q_f64_x4(__p0) __extension__ ({ \ + float64x2x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 42); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_f32_x4(__p0) __extension__ ({ \ + float32x4x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 41); \ + __ret; \ +}) +#else +#define vld1q_f32_x4(__p0) __extension__ ({ \ + float32x4x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 41); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_f16_x4(__p0) __extension__ ({ \ + float16x8x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 40); \ + __ret; \ +}) +#else +#define vld1q_f16_x4(__p0) __extension__ ({ \ + float16x8x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 40); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s32_x4(__p0) __extension__ ({ \ + int32x4x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 34); \ + __ret; \ +}) +#else +#define vld1q_s32_x4(__p0) __extension__ ({ \ + int32x4x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 34); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s64_x4(__p0) __extension__ ({ \ + int64x2x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 35); \ + __ret; \ +}) +#else +#define vld1q_s64_x4(__p0) __extension__ ({ \ + int64x2x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 35); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1q_s16_x4(__p0) __extension__ ({ \ + int16x8x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 33); \ + __ret; \ +}) +#else +#define vld1q_s16_x4(__p0) __extension__ ({ \ + int16x8x4_t __ret; \ + __builtin_neon_vld1q_x4_v(&__ret, __p0, 33); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u8_x4(__p0) __extension__ ({ \ + uint8x8x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 16); \ + __ret; \ +}) +#else +#define vld1_u8_x4(__p0) __extension__ ({ \ + uint8x8x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 16); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u32_x4(__p0) __extension__ ({ \ + uint32x2x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 18); \ + __ret; \ +}) +#else +#define vld1_u32_x4(__p0) __extension__ ({ \ + uint32x2x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 18); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u64_x4(__p0) __extension__ ({ \ + uint64x1x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 19); \ + __ret; \ +}) +#else +#define vld1_u64_x4(__p0) __extension__ ({ \ + uint64x1x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_u16_x4(__p0) __extension__ ({ \ + uint16x4x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 17); \ + __ret; \ +}) +#else +#define vld1_u16_x4(__p0) __extension__ ({ \ + uint16x4x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 17); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s8_x4(__p0) __extension__ ({ \ + int8x8x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 0); \ + __ret; \ +}) +#else +#define vld1_s8_x4(__p0) __extension__ ({ \ + int8x8x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 0); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_f64_x4(__p0) __extension__ ({ \ + float64x1x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 10); \ + __ret; \ +}) +#else +#define vld1_f64_x4(__p0) __extension__ ({ \ + float64x1x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_f32_x4(__p0) __extension__ ({ \ + float32x2x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 9); \ + __ret; \ +}) +#else +#define vld1_f32_x4(__p0) __extension__ ({ \ + float32x2x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 9); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_f16_x4(__p0) __extension__ ({ \ + float16x4x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 8); \ + __ret; \ +}) +#else +#define vld1_f16_x4(__p0) __extension__ ({ \ + float16x4x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 8); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s32_x4(__p0) __extension__ ({ \ + int32x2x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 2); \ + __ret; \ +}) +#else +#define vld1_s32_x4(__p0) __extension__ ({ \ + int32x2x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 2); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s64_x4(__p0) __extension__ ({ \ + int64x1x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 3); \ + __ret; \ +}) +#else +#define vld1_s64_x4(__p0) __extension__ ({ \ + int64x1x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld1_s16_x4(__p0) __extension__ ({ \ + int16x4x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 1); \ + __ret; \ +}) +#else +#define vld1_s16_x4(__p0) __extension__ ({ \ + int16x4x4_t __ret; \ + __builtin_neon_vld1_x4_v(&__ret, __p0, 1); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_p64(__p0) __extension__ ({ \ + poly64x1x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 6); \ + __ret; \ +}) +#else +#define vld2_p64(__p0) __extension__ ({ \ + poly64x1x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_p64(__p0) __extension__ ({ \ + poly64x2x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 38); \ + __ret; \ +}) +#else +#define vld2q_p64(__p0) __extension__ ({ \ + poly64x2x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 38); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_u64(__p0) __extension__ ({ \ + uint64x2x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 51); \ + __ret; \ +}) +#else +#define vld2q_u64(__p0) __extension__ ({ \ + uint64x2x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 51); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_f64(__p0) __extension__ ({ \ + float64x2x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 42); \ + __ret; \ +}) +#else +#define vld2q_f64(__p0) __extension__ ({ \ + float64x2x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 42); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_s64(__p0) __extension__ ({ \ + int64x2x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 35); \ + __ret; \ +}) +#else +#define vld2q_s64(__p0) __extension__ ({ \ + int64x2x2_t __ret; \ + __builtin_neon_vld2q_v(&__ret, __p0, 35); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_f64(__p0) __extension__ ({ \ + float64x1x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 10); \ + __ret; \ +}) +#else +#define vld2_f64(__p0) __extension__ ({ \ + float64x1x2_t __ret; \ + __builtin_neon_vld2_v(&__ret, __p0, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_dup_p64(__p0) __extension__ ({ \ + poly64x1x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 6); \ + __ret; \ +}) +#else +#define vld2_dup_p64(__p0) __extension__ ({ \ + poly64x1x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_dup_p8(__p0) __extension__ ({ \ + poly8x16x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 36); \ + __ret; \ +}) +#else +#define vld2q_dup_p8(__p0) __extension__ ({ \ + poly8x16x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 36); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_dup_p64(__p0) __extension__ ({ \ + poly64x2x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 38); \ + __ret; \ +}) +#else +#define vld2q_dup_p64(__p0) __extension__ ({ \ + poly64x2x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 38); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_dup_p16(__p0) __extension__ ({ \ + poly16x8x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 37); \ + __ret; \ +}) +#else +#define vld2q_dup_p16(__p0) __extension__ ({ \ + poly16x8x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 37); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_dup_u8(__p0) __extension__ ({ \ + uint8x16x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 48); \ + __ret; \ +}) +#else +#define vld2q_dup_u8(__p0) __extension__ ({ \ + uint8x16x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 48); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_dup_u32(__p0) __extension__ ({ \ + uint32x4x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 50); \ + __ret; \ +}) +#else +#define vld2q_dup_u32(__p0) __extension__ ({ \ + uint32x4x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 50); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_dup_u64(__p0) __extension__ ({ \ + uint64x2x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 51); \ + __ret; \ +}) +#else +#define vld2q_dup_u64(__p0) __extension__ ({ \ + uint64x2x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 51); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_dup_u16(__p0) __extension__ ({ \ + uint16x8x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 49); \ + __ret; \ +}) +#else +#define vld2q_dup_u16(__p0) __extension__ ({ \ + uint16x8x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 49); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_dup_s8(__p0) __extension__ ({ \ + int8x16x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 32); \ + __ret; \ +}) +#else +#define vld2q_dup_s8(__p0) __extension__ ({ \ + int8x16x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 32); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_dup_f64(__p0) __extension__ ({ \ + float64x2x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 42); \ + __ret; \ +}) +#else +#define vld2q_dup_f64(__p0) __extension__ ({ \ + float64x2x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 42); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_dup_f32(__p0) __extension__ ({ \ + float32x4x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 41); \ + __ret; \ +}) +#else +#define vld2q_dup_f32(__p0) __extension__ ({ \ + float32x4x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 41); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_dup_f16(__p0) __extension__ ({ \ + float16x8x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 40); \ + __ret; \ +}) +#else +#define vld2q_dup_f16(__p0) __extension__ ({ \ + float16x8x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 40); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_dup_s32(__p0) __extension__ ({ \ + int32x4x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 34); \ + __ret; \ +}) +#else +#define vld2q_dup_s32(__p0) __extension__ ({ \ + int32x4x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 34); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_dup_s64(__p0) __extension__ ({ \ + int64x2x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 35); \ + __ret; \ +}) +#else +#define vld2q_dup_s64(__p0) __extension__ ({ \ + int64x2x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 35); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_dup_s16(__p0) __extension__ ({ \ + int16x8x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 33); \ + __ret; \ +}) +#else +#define vld2q_dup_s16(__p0) __extension__ ({ \ + int16x8x2_t __ret; \ + __builtin_neon_vld2q_dup_v(&__ret, __p0, 33); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_dup_f64(__p0) __extension__ ({ \ + float64x1x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 10); \ + __ret; \ +}) +#else +#define vld2_dup_f64(__p0) __extension__ ({ \ + float64x1x2_t __ret; \ + __builtin_neon_vld2_dup_v(&__ret, __p0, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1x2_t __s1 = __p1; \ + poly64x1x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 6); \ + __ret; \ +}) +#else +#define vld2_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1x2_t __s1 = __p1; \ + poly64x1x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16x2_t __s1 = __p1; \ + poly8x16x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 36); \ + __ret; \ +}) +#else +#define vld2q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16x2_t __s1 = __p1; \ + poly8x16x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 36); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2x2_t __s1 = __p1; \ + poly64x2x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 38); \ + __ret; \ +}) +#else +#define vld2q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2x2_t __s1 = __p1; \ + poly64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + poly64x2x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 38); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16x2_t __s1 = __p1; \ + uint8x16x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 48); \ + __ret; \ +}) +#else +#define vld2q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16x2_t __s1 = __p1; \ + uint8x16x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 48); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2x2_t __s1 = __p1; \ + uint64x2x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 51); \ + __ret; \ +}) +#else +#define vld2q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2x2_t __s1 = __p1; \ + uint64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + uint64x2x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 51); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16x2_t __s1 = __p1; \ + int8x16x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 32); \ + __ret; \ +}) +#else +#define vld2q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16x2_t __s1 = __p1; \ + int8x16x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 32); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2x2_t __s1 = __p1; \ + float64x2x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 42); \ + __ret; \ +}) +#else +#define vld2q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2x2_t __s1 = __p1; \ + float64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + float64x2x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __p2, 42); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2x2_t __s1 = __p1; \ + int64x2x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 35); \ + __ret; \ +}) +#else +#define vld2q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2x2_t __s1 = __p1; \ + int64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + int64x2x2_t __ret; \ + __builtin_neon_vld2q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __p2, 35); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1x2_t __s1 = __p1; \ + uint64x1x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 19); \ + __ret; \ +}) +#else +#define vld2_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1x2_t __s1 = __p1; \ + uint64x1x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1x2_t __s1 = __p1; \ + float64x1x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 10); \ + __ret; \ +}) +#else +#define vld2_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1x2_t __s1 = __p1; \ + float64x1x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld2_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1x2_t __s1 = __p1; \ + int64x1x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 3); \ + __ret; \ +}) +#else +#define vld2_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1x2_t __s1 = __p1; \ + int64x1x2_t __ret; \ + __builtin_neon_vld2_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __p2, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_p64(__p0) __extension__ ({ \ + poly64x1x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 6); \ + __ret; \ +}) +#else +#define vld3_p64(__p0) __extension__ ({ \ + poly64x1x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_p64(__p0) __extension__ ({ \ + poly64x2x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 38); \ + __ret; \ +}) +#else +#define vld3q_p64(__p0) __extension__ ({ \ + poly64x2x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 38); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_u64(__p0) __extension__ ({ \ + uint64x2x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 51); \ + __ret; \ +}) +#else +#define vld3q_u64(__p0) __extension__ ({ \ + uint64x2x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 51); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_f64(__p0) __extension__ ({ \ + float64x2x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 42); \ + __ret; \ +}) +#else +#define vld3q_f64(__p0) __extension__ ({ \ + float64x2x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 42); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_s64(__p0) __extension__ ({ \ + int64x2x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 35); \ + __ret; \ +}) +#else +#define vld3q_s64(__p0) __extension__ ({ \ + int64x2x3_t __ret; \ + __builtin_neon_vld3q_v(&__ret, __p0, 35); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_f64(__p0) __extension__ ({ \ + float64x1x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 10); \ + __ret; \ +}) +#else +#define vld3_f64(__p0) __extension__ ({ \ + float64x1x3_t __ret; \ + __builtin_neon_vld3_v(&__ret, __p0, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_dup_p64(__p0) __extension__ ({ \ + poly64x1x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 6); \ + __ret; \ +}) +#else +#define vld3_dup_p64(__p0) __extension__ ({ \ + poly64x1x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_dup_p8(__p0) __extension__ ({ \ + poly8x16x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 36); \ + __ret; \ +}) +#else +#define vld3q_dup_p8(__p0) __extension__ ({ \ + poly8x16x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 36); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_dup_p64(__p0) __extension__ ({ \ + poly64x2x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 38); \ + __ret; \ +}) +#else +#define vld3q_dup_p64(__p0) __extension__ ({ \ + poly64x2x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 38); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_dup_p16(__p0) __extension__ ({ \ + poly16x8x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 37); \ + __ret; \ +}) +#else +#define vld3q_dup_p16(__p0) __extension__ ({ \ + poly16x8x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 37); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_dup_u8(__p0) __extension__ ({ \ + uint8x16x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 48); \ + __ret; \ +}) +#else +#define vld3q_dup_u8(__p0) __extension__ ({ \ + uint8x16x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 48); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_dup_u32(__p0) __extension__ ({ \ + uint32x4x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 50); \ + __ret; \ +}) +#else +#define vld3q_dup_u32(__p0) __extension__ ({ \ + uint32x4x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 50); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_dup_u64(__p0) __extension__ ({ \ + uint64x2x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 51); \ + __ret; \ +}) +#else +#define vld3q_dup_u64(__p0) __extension__ ({ \ + uint64x2x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 51); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_dup_u16(__p0) __extension__ ({ \ + uint16x8x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 49); \ + __ret; \ +}) +#else +#define vld3q_dup_u16(__p0) __extension__ ({ \ + uint16x8x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 49); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_dup_s8(__p0) __extension__ ({ \ + int8x16x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 32); \ + __ret; \ +}) +#else +#define vld3q_dup_s8(__p0) __extension__ ({ \ + int8x16x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 32); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_dup_f64(__p0) __extension__ ({ \ + float64x2x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 42); \ + __ret; \ +}) +#else +#define vld3q_dup_f64(__p0) __extension__ ({ \ + float64x2x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 42); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_dup_f32(__p0) __extension__ ({ \ + float32x4x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 41); \ + __ret; \ +}) +#else +#define vld3q_dup_f32(__p0) __extension__ ({ \ + float32x4x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 41); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_dup_f16(__p0) __extension__ ({ \ + float16x8x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 40); \ + __ret; \ +}) +#else +#define vld3q_dup_f16(__p0) __extension__ ({ \ + float16x8x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 40); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_dup_s32(__p0) __extension__ ({ \ + int32x4x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 34); \ + __ret; \ +}) +#else +#define vld3q_dup_s32(__p0) __extension__ ({ \ + int32x4x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 34); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_dup_s64(__p0) __extension__ ({ \ + int64x2x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 35); \ + __ret; \ +}) +#else +#define vld3q_dup_s64(__p0) __extension__ ({ \ + int64x2x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 35); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_dup_s16(__p0) __extension__ ({ \ + int16x8x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 33); \ + __ret; \ +}) +#else +#define vld3q_dup_s16(__p0) __extension__ ({ \ + int16x8x3_t __ret; \ + __builtin_neon_vld3q_dup_v(&__ret, __p0, 33); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_dup_f64(__p0) __extension__ ({ \ + float64x1x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 10); \ + __ret; \ +}) +#else +#define vld3_dup_f64(__p0) __extension__ ({ \ + float64x1x3_t __ret; \ + __builtin_neon_vld3_dup_v(&__ret, __p0, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1x3_t __s1 = __p1; \ + poly64x1x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 6); \ + __ret; \ +}) +#else +#define vld3_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1x3_t __s1 = __p1; \ + poly64x1x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16x3_t __s1 = __p1; \ + poly8x16x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 36); \ + __ret; \ +}) +#else +#define vld3q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16x3_t __s1 = __p1; \ + poly8x16x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 36); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2x3_t __s1 = __p1; \ + poly64x2x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 38); \ + __ret; \ +}) +#else +#define vld3q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2x3_t __s1 = __p1; \ + poly64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + poly64x2x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 38); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16x3_t __s1 = __p1; \ + uint8x16x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 48); \ + __ret; \ +}) +#else +#define vld3q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16x3_t __s1 = __p1; \ + uint8x16x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 48); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2x3_t __s1 = __p1; \ + uint64x2x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 51); \ + __ret; \ +}) +#else +#define vld3q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2x3_t __s1 = __p1; \ + uint64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + uint64x2x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 51); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16x3_t __s1 = __p1; \ + int8x16x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 32); \ + __ret; \ +}) +#else +#define vld3q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16x3_t __s1 = __p1; \ + int8x16x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 32); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2x3_t __s1 = __p1; \ + float64x2x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 42); \ + __ret; \ +}) +#else +#define vld3q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2x3_t __s1 = __p1; \ + float64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + float64x2x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 42); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2x3_t __s1 = __p1; \ + int64x2x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 35); \ + __ret; \ +}) +#else +#define vld3q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2x3_t __s1 = __p1; \ + int64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + int64x2x3_t __ret; \ + __builtin_neon_vld3q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 35); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1x3_t __s1 = __p1; \ + uint64x1x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 19); \ + __ret; \ +}) +#else +#define vld3_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1x3_t __s1 = __p1; \ + uint64x1x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1x3_t __s1 = __p1; \ + float64x1x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 10); \ + __ret; \ +}) +#else +#define vld3_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1x3_t __s1 = __p1; \ + float64x1x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld3_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1x3_t __s1 = __p1; \ + int64x1x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 3); \ + __ret; \ +}) +#else +#define vld3_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1x3_t __s1 = __p1; \ + int64x1x3_t __ret; \ + __builtin_neon_vld3_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_p64(__p0) __extension__ ({ \ + poly64x1x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 6); \ + __ret; \ +}) +#else +#define vld4_p64(__p0) __extension__ ({ \ + poly64x1x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_p64(__p0) __extension__ ({ \ + poly64x2x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 38); \ + __ret; \ +}) +#else +#define vld4q_p64(__p0) __extension__ ({ \ + poly64x2x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 38); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_u64(__p0) __extension__ ({ \ + uint64x2x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 51); \ + __ret; \ +}) +#else +#define vld4q_u64(__p0) __extension__ ({ \ + uint64x2x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 51); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_f64(__p0) __extension__ ({ \ + float64x2x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 42); \ + __ret; \ +}) +#else +#define vld4q_f64(__p0) __extension__ ({ \ + float64x2x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 42); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_s64(__p0) __extension__ ({ \ + int64x2x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 35); \ + __ret; \ +}) +#else +#define vld4q_s64(__p0) __extension__ ({ \ + int64x2x4_t __ret; \ + __builtin_neon_vld4q_v(&__ret, __p0, 35); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_f64(__p0) __extension__ ({ \ + float64x1x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 10); \ + __ret; \ +}) +#else +#define vld4_f64(__p0) __extension__ ({ \ + float64x1x4_t __ret; \ + __builtin_neon_vld4_v(&__ret, __p0, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_dup_p64(__p0) __extension__ ({ \ + poly64x1x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 6); \ + __ret; \ +}) +#else +#define vld4_dup_p64(__p0) __extension__ ({ \ + poly64x1x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_dup_p8(__p0) __extension__ ({ \ + poly8x16x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 36); \ + __ret; \ +}) +#else +#define vld4q_dup_p8(__p0) __extension__ ({ \ + poly8x16x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 36); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_dup_p64(__p0) __extension__ ({ \ + poly64x2x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 38); \ + __ret; \ +}) +#else +#define vld4q_dup_p64(__p0) __extension__ ({ \ + poly64x2x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 38); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_dup_p16(__p0) __extension__ ({ \ + poly16x8x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 37); \ + __ret; \ +}) +#else +#define vld4q_dup_p16(__p0) __extension__ ({ \ + poly16x8x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 37); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_dup_u8(__p0) __extension__ ({ \ + uint8x16x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 48); \ + __ret; \ +}) +#else +#define vld4q_dup_u8(__p0) __extension__ ({ \ + uint8x16x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 48); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_dup_u32(__p0) __extension__ ({ \ + uint32x4x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 50); \ + __ret; \ +}) +#else +#define vld4q_dup_u32(__p0) __extension__ ({ \ + uint32x4x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 50); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_dup_u64(__p0) __extension__ ({ \ + uint64x2x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 51); \ + __ret; \ +}) +#else +#define vld4q_dup_u64(__p0) __extension__ ({ \ + uint64x2x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 51); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_dup_u16(__p0) __extension__ ({ \ + uint16x8x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 49); \ + __ret; \ +}) +#else +#define vld4q_dup_u16(__p0) __extension__ ({ \ + uint16x8x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 49); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_dup_s8(__p0) __extension__ ({ \ + int8x16x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 32); \ + __ret; \ +}) +#else +#define vld4q_dup_s8(__p0) __extension__ ({ \ + int8x16x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 32); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_dup_f64(__p0) __extension__ ({ \ + float64x2x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 42); \ + __ret; \ +}) +#else +#define vld4q_dup_f64(__p0) __extension__ ({ \ + float64x2x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 42); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_dup_f32(__p0) __extension__ ({ \ + float32x4x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 41); \ + __ret; \ +}) +#else +#define vld4q_dup_f32(__p0) __extension__ ({ \ + float32x4x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 41); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_dup_f16(__p0) __extension__ ({ \ + float16x8x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 40); \ + __ret; \ +}) +#else +#define vld4q_dup_f16(__p0) __extension__ ({ \ + float16x8x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 40); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_dup_s32(__p0) __extension__ ({ \ + int32x4x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 34); \ + __ret; \ +}) +#else +#define vld4q_dup_s32(__p0) __extension__ ({ \ + int32x4x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 34); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_dup_s64(__p0) __extension__ ({ \ + int64x2x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 35); \ + __ret; \ +}) +#else +#define vld4q_dup_s64(__p0) __extension__ ({ \ + int64x2x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 35); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_dup_s16(__p0) __extension__ ({ \ + int16x8x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 33); \ + __ret; \ +}) +#else +#define vld4q_dup_s16(__p0) __extension__ ({ \ + int16x8x4_t __ret; \ + __builtin_neon_vld4q_dup_v(&__ret, __p0, 33); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_dup_f64(__p0) __extension__ ({ \ + float64x1x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 10); \ + __ret; \ +}) +#else +#define vld4_dup_f64(__p0) __extension__ ({ \ + float64x1x4_t __ret; \ + __builtin_neon_vld4_dup_v(&__ret, __p0, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1x4_t __s1 = __p1; \ + poly64x1x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 6); \ + __ret; \ +}) +#else +#define vld4_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1x4_t __s1 = __p1; \ + poly64x1x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16x4_t __s1 = __p1; \ + poly8x16x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 36); \ + __ret; \ +}) +#else +#define vld4q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16x4_t __s1 = __p1; \ + poly8x16x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + poly8x16x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 36); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2x4_t __s1 = __p1; \ + poly64x2x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 38); \ + __ret; \ +}) +#else +#define vld4q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2x4_t __s1 = __p1; \ + poly64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + poly64x2x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 38); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16x4_t __s1 = __p1; \ + uint8x16x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 48); \ + __ret; \ +}) +#else +#define vld4q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16x4_t __s1 = __p1; \ + uint8x16x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 48); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2x4_t __s1 = __p1; \ + uint64x2x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 51); \ + __ret; \ +}) +#else +#define vld4q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2x4_t __s1 = __p1; \ + uint64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + uint64x2x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 51); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16x4_t __s1 = __p1; \ + int8x16x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 32); \ + __ret; \ +}) +#else +#define vld4q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16x4_t __s1 = __p1; \ + int8x16x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 32); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2x4_t __s1 = __p1; \ + float64x2x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 42); \ + __ret; \ +}) +#else +#define vld4q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2x4_t __s1 = __p1; \ + float64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + float64x2x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 42); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2x4_t __s1 = __p1; \ + int64x2x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 35); \ + __ret; \ +}) +#else +#define vld4q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2x4_t __s1 = __p1; \ + int64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + int64x2x4_t __ret; \ + __builtin_neon_vld4q_lane_v(&__ret, __p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 35); \ + \ + __ret.val[0] = __builtin_shufflevector(__ret.val[0], __ret.val[0], 1, 0); \ + __ret.val[1] = __builtin_shufflevector(__ret.val[1], __ret.val[1], 1, 0); \ + __ret.val[2] = __builtin_shufflevector(__ret.val[2], __ret.val[2], 1, 0); \ + __ret.val[3] = __builtin_shufflevector(__ret.val[3], __ret.val[3], 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1x4_t __s1 = __p1; \ + uint64x1x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 19); \ + __ret; \ +}) +#else +#define vld4_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1x4_t __s1 = __p1; \ + uint64x1x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 19); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1x4_t __s1 = __p1; \ + float64x1x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 10); \ + __ret; \ +}) +#else +#define vld4_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1x4_t __s1 = __p1; \ + float64x1x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vld4_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1x4_t __s1 = __p1; \ + int64x1x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 3); \ + __ret; \ +}) +#else +#define vld4_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1x4_t __s1 = __p1; \ + int64x1x4_t __ret; \ + __builtin_neon_vld4_lane_v(&__ret, __p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vldrq_p128(__p0) __extension__ ({ \ + poly128_t __ret; \ + __ret = (poly128_t) __builtin_neon_vldrq_p128(__p0); \ + __ret; \ +}) +#else +#define vldrq_p128(__p0) __extension__ ({ \ + poly128_t __ret; \ + __ret = (poly128_t) __builtin_neon_vldrq_p128(__p0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vmaxq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 42); + return __ret; +} +#else +__ai float64x2_t vmaxq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vmax_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vmax_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#else +__ai float64x1_t vmax_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vmax_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vmaxnmvq_f64(float64x2_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vmaxnmvq_f64((int8x16_t)__p0); + return __ret; +} +#else +__ai float64_t vmaxnmvq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64_t __ret; + __ret = (float64_t) __builtin_neon_vmaxnmvq_f64((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vmaxnmvq_f32(float32x4_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vmaxnmvq_f32((int8x16_t)__p0); + return __ret; +} +#else +__ai float32_t vmaxnmvq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vmaxnmvq_f32((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vmaxnmv_f32(float32x2_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vmaxnmv_f32((int8x8_t)__p0); + return __ret; +} +#else +__ai float32_t vmaxnmv_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vmaxnmv_f32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8_t vmaxvq_u8(uint8x16_t __p0) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vmaxvq_u8((int8x16_t)__p0); + return __ret; +} +#else +__ai uint8_t vmaxvq_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vmaxvq_u8((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vmaxvq_u32(uint32x4_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vmaxvq_u32((int8x16_t)__p0); + return __ret; +} +#else +__ai uint32_t vmaxvq_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vmaxvq_u32((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16_t vmaxvq_u16(uint16x8_t __p0) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vmaxvq_u16((int8x16_t)__p0); + return __ret; +} +#else +__ai uint16_t vmaxvq_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vmaxvq_u16((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vmaxvq_s8(int8x16_t __p0) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vmaxvq_s8((int8x16_t)__p0); + return __ret; +} +#else +__ai int8_t vmaxvq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8_t __ret; + __ret = (int8_t) __builtin_neon_vmaxvq_s8((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vmaxvq_f64(float64x2_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vmaxvq_f64((int8x16_t)__p0); + return __ret; +} +#else +__ai float64_t vmaxvq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64_t __ret; + __ret = (float64_t) __builtin_neon_vmaxvq_f64((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vmaxvq_f32(float32x4_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vmaxvq_f32((int8x16_t)__p0); + return __ret; +} +#else +__ai float32_t vmaxvq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vmaxvq_f32((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vmaxvq_s32(int32x4_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vmaxvq_s32((int8x16_t)__p0); + return __ret; +} +#else +__ai int32_t vmaxvq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32_t __ret; + __ret = (int32_t) __builtin_neon_vmaxvq_s32((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vmaxvq_s16(int16x8_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vmaxvq_s16((int8x16_t)__p0); + return __ret; +} +#else +__ai int16_t vmaxvq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16_t __ret; + __ret = (int16_t) __builtin_neon_vmaxvq_s16((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8_t vmaxv_u8(uint8x8_t __p0) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vmaxv_u8((int8x8_t)__p0); + return __ret; +} +#else +__ai uint8_t vmaxv_u8(uint8x8_t __p0) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vmaxv_u8((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vmaxv_u32(uint32x2_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vmaxv_u32((int8x8_t)__p0); + return __ret; +} +#else +__ai uint32_t vmaxv_u32(uint32x2_t __p0) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vmaxv_u32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16_t vmaxv_u16(uint16x4_t __p0) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vmaxv_u16((int8x8_t)__p0); + return __ret; +} +#else +__ai uint16_t vmaxv_u16(uint16x4_t __p0) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vmaxv_u16((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vmaxv_s8(int8x8_t __p0) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vmaxv_s8((int8x8_t)__p0); + return __ret; +} +#else +__ai int8_t vmaxv_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8_t __ret; + __ret = (int8_t) __builtin_neon_vmaxv_s8((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vmaxv_f32(float32x2_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vmaxv_f32((int8x8_t)__p0); + return __ret; +} +#else +__ai float32_t vmaxv_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vmaxv_f32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vmaxv_s32(int32x2_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vmaxv_s32((int8x8_t)__p0); + return __ret; +} +#else +__ai int32_t vmaxv_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32_t __ret; + __ret = (int32_t) __builtin_neon_vmaxv_s32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vmaxv_s16(int16x4_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vmaxv_s16((int8x8_t)__p0); + return __ret; +} +#else +__ai int16_t vmaxv_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16_t __ret; + __ret = (int16_t) __builtin_neon_vmaxv_s16((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vminq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vminq_v((int8x16_t)__p0, (int8x16_t)__p1, 42); + return __ret; +} +#else +__ai float64x2_t vminq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vmin_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vmin_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#else +__ai float64x1_t vmin_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vmin_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vminnmvq_f64(float64x2_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vminnmvq_f64((int8x16_t)__p0); + return __ret; +} +#else +__ai float64_t vminnmvq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64_t __ret; + __ret = (float64_t) __builtin_neon_vminnmvq_f64((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vminnmvq_f32(float32x4_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vminnmvq_f32((int8x16_t)__p0); + return __ret; +} +#else +__ai float32_t vminnmvq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vminnmvq_f32((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vminnmv_f32(float32x2_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vminnmv_f32((int8x8_t)__p0); + return __ret; +} +#else +__ai float32_t vminnmv_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vminnmv_f32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8_t vminvq_u8(uint8x16_t __p0) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vminvq_u8((int8x16_t)__p0); + return __ret; +} +#else +__ai uint8_t vminvq_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vminvq_u8((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vminvq_u32(uint32x4_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vminvq_u32((int8x16_t)__p0); + return __ret; +} +#else +__ai uint32_t vminvq_u32(uint32x4_t __p0) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vminvq_u32((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16_t vminvq_u16(uint16x8_t __p0) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vminvq_u16((int8x16_t)__p0); + return __ret; +} +#else +__ai uint16_t vminvq_u16(uint16x8_t __p0) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vminvq_u16((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vminvq_s8(int8x16_t __p0) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vminvq_s8((int8x16_t)__p0); + return __ret; +} +#else +__ai int8_t vminvq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8_t __ret; + __ret = (int8_t) __builtin_neon_vminvq_s8((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vminvq_f64(float64x2_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vminvq_f64((int8x16_t)__p0); + return __ret; +} +#else +__ai float64_t vminvq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64_t __ret; + __ret = (float64_t) __builtin_neon_vminvq_f64((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vminvq_f32(float32x4_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vminvq_f32((int8x16_t)__p0); + return __ret; +} +#else +__ai float32_t vminvq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vminvq_f32((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vminvq_s32(int32x4_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vminvq_s32((int8x16_t)__p0); + return __ret; +} +#else +__ai int32_t vminvq_s32(int32x4_t __p0) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32_t __ret; + __ret = (int32_t) __builtin_neon_vminvq_s32((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vminvq_s16(int16x8_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vminvq_s16((int8x16_t)__p0); + return __ret; +} +#else +__ai int16_t vminvq_s16(int16x8_t __p0) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16_t __ret; + __ret = (int16_t) __builtin_neon_vminvq_s16((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8_t vminv_u8(uint8x8_t __p0) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vminv_u8((int8x8_t)__p0); + return __ret; +} +#else +__ai uint8_t vminv_u8(uint8x8_t __p0) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vminv_u8((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vminv_u32(uint32x2_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vminv_u32((int8x8_t)__p0); + return __ret; +} +#else +__ai uint32_t vminv_u32(uint32x2_t __p0) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vminv_u32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16_t vminv_u16(uint16x4_t __p0) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vminv_u16((int8x8_t)__p0); + return __ret; +} +#else +__ai uint16_t vminv_u16(uint16x4_t __p0) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vminv_u16((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vminv_s8(int8x8_t __p0) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vminv_s8((int8x8_t)__p0); + return __ret; +} +#else +__ai int8_t vminv_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8_t __ret; + __ret = (int8_t) __builtin_neon_vminv_s8((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vminv_f32(float32x2_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vminv_f32((int8x8_t)__p0); + return __ret; +} +#else +__ai float32_t vminv_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vminv_f32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vminv_s32(int32x2_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vminv_s32((int8x8_t)__p0); + return __ret; +} +#else +__ai int32_t vminv_s32(int32x2_t __p0) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32_t __ret; + __ret = (int32_t) __builtin_neon_vminv_s32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vminv_s16(int16x4_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vminv_s16((int8x8_t)__p0); + return __ret; +} +#else +__ai int16_t vminv_s16(int16x4_t __p0) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16_t __ret; + __ret = (int16_t) __builtin_neon_vminv_s16((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vmlaq_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) { + float64x2_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai float64x2_t vmlaq_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + float64x2_t __ret; + __ret = __rev0 + __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vmla_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) { + float64x1_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#else +__ai float64x1_t vmla_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) { + float64x1_t __ret; + __ret = __p0 + __p1 * __p2; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlaq_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint32x4_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlaq_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlaq_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint16x8_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlaq_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlaq_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x4_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlaq_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + float32x4_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlaq_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlaq_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlaq_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x8_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlaq_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmla_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint32x2_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3); \ + __ret; \ +}) +#else +#define vmla_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint32x2_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmla_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint16x4_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmla_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmla_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x2_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3); \ + __ret; \ +}) +#else +#define vmla_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + float32x2_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmla_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x2_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3); \ + __ret; \ +}) +#else +#define vmla_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x2_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmla_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x4_t __ret; \ + __ret = __s0 + __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmla_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __rev0 + __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vmlaq_n_f64(float64x2_t __p0, float64x2_t __p1, float64_t __p2) { + float64x2_t __ret; + __ret = __p0 + __p1 * (float64x2_t) {__p2, __p2}; + return __ret; +} +#else +__ai float64x2_t vmlaq_n_f64(float64x2_t __p0, float64x2_t __p1, float64_t __p2) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = __rev0 + __rev1 * (float64x2_t) {__p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_high_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint64x2_t __ret; \ + __ret = __s0 + vmull_u32(vget_high_u32(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_high_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + uint64x2_t __ret; \ + __ret = __rev0 + __noswap_vmull_u32(__noswap_vget_high_u32(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_high_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint32x4_t __ret; \ + __ret = __s0 + vmull_u16(vget_high_u16(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_high_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __rev0 + __noswap_vmull_u16(__noswap_vget_high_u16(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_high_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = __s0 + vmull_s32(vget_high_s32(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_high_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int64x2_t __ret; \ + __ret = __rev0 + __noswap_vmull_s32(__noswap_vget_high_s32(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_high_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = __s0 + vmull_s16(vget_high_s16(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_high_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __rev0 + __noswap_vmull_s16(__noswap_vget_high_s16(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_high_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint64x2_t __ret; \ + __ret = __s0 + vmull_u32(vget_high_u32(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_high_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint64x2_t __ret; \ + __ret = __rev0 + __noswap_vmull_u32(__noswap_vget_high_u32(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_high_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint32x4_t __ret; \ + __ret = __s0 + vmull_u16(vget_high_u16(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_high_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __rev0 + __noswap_vmull_u16(__noswap_vget_high_u16(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_high_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = __s0 + vmull_s32(vget_high_s32(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_high_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int64x2_t __ret; \ + __ret = __rev0 + __noswap_vmull_s32(__noswap_vget_high_s32(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_high_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = __s0 + vmull_s16(vget_high_s16(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_high_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __rev0 + __noswap_vmull_s16(__noswap_vget_high_s16(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint64x2_t __ret; \ + __ret = __s0 + vmull_u32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint64x2_t __ret; \ + __ret = __rev0 + __noswap_vmull_u32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint32x4_t __ret; \ + __ret = __s0 + vmull_u16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __rev0 + __noswap_vmull_u16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = __s0 + vmull_s32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int64x2_t __ret; \ + __ret = __rev0 + __noswap_vmull_s32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = __s0 + vmull_s16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __rev0 + __noswap_vmull_s16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vmlsq_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) { + float64x2_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai float64x2_t vmlsq_f64(float64x2_t __p0, float64x2_t __p1, float64x2_t __p2) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + float64x2_t __ret; + __ret = __rev0 - __rev1 * __rev2; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vmls_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) { + float64x1_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#else +__ai float64x1_t vmls_f64(float64x1_t __p0, float64x1_t __p1, float64x1_t __p2) { + float64x1_t __ret; + __ret = __p0 - __p1 * __p2; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsq_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint32x4_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlsq_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsq_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint16x8_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlsq_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsq_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x4_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlsq_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + float32x4_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsq_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlsq_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsq_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x8_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmlsq_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmls_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint32x2_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3); \ + __ret; \ +}) +#else +#define vmls_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint32x2_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmls_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint16x4_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmls_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmls_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x2_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3); \ + __ret; \ +}) +#else +#define vmls_laneq_f32(__p0, __p1, __p2, __p3) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x4_t __s2 = __p2; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + float32x2_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmls_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x2_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3); \ + __ret; \ +}) +#else +#define vmls_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x2_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmls_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x4_t __ret; \ + __ret = __s0 - __s1 * __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3); \ + __ret; \ +}) +#else +#define vmls_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __rev0 - __rev1 * __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vmlsq_n_f64(float64x2_t __p0, float64x2_t __p1, float64_t __p2) { + float64x2_t __ret; + __ret = __p0 - __p1 * (float64x2_t) {__p2, __p2}; + return __ret; +} +#else +__ai float64x2_t vmlsq_n_f64(float64x2_t __p0, float64x2_t __p1, float64_t __p2) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = __rev0 - __rev1 * (float64x2_t) {__p2, __p2}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_high_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint64x2_t __ret; \ + __ret = __s0 - vmull_u32(vget_high_u32(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_high_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + uint64x2_t __ret; \ + __ret = __rev0 - __noswap_vmull_u32(__noswap_vget_high_u32(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_high_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint32x4_t __ret; \ + __ret = __s0 - vmull_u16(vget_high_u16(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_high_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __rev0 - __noswap_vmull_u16(__noswap_vget_high_u16(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_high_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = __s0 - vmull_s32(vget_high_s32(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_high_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int64x2_t __ret; \ + __ret = __rev0 - __noswap_vmull_s32(__noswap_vget_high_s32(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_high_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = __s0 - vmull_s16(vget_high_s16(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_high_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __rev0 - __noswap_vmull_s16(__noswap_vget_high_s16(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_high_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint64x2_t __ret; \ + __ret = __s0 - vmull_u32(vget_high_u32(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_high_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint64x2_t __ret; \ + __ret = __rev0 - __noswap_vmull_u32(__noswap_vget_high_u32(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_high_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint32x4_t __ret; \ + __ret = __s0 - vmull_u16(vget_high_u16(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_high_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __rev0 - __noswap_vmull_u16(__noswap_vget_high_u16(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_high_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = __s0 - vmull_s32(vget_high_s32(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_high_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int64x2_t __ret; \ + __ret = __rev0 - __noswap_vmull_s32(__noswap_vget_high_s32(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_high_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = __s0 - vmull_s16(vget_high_s16(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_high_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __rev0 - __noswap_vmull_s16(__noswap_vget_high_s16(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint64x2_t __ret; \ + __ret = __s0 - vmull_u32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_laneq_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x4_t __s2 = __p2; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint64x2_t __ret; \ + __ret = __rev0 - __noswap_vmull_u32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint32x4_t __ret; \ + __ret = __s0 - vmull_u16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_laneq_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x8_t __s2 = __p2; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __rev0 - __noswap_vmull_u16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = __s0 - vmull_s32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int64x2_t __ret; \ + __ret = __rev0 - __noswap_vmull_s32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = __s0 - vmull_s16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __rev0 - __noswap_vmull_s16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x1_t vmov_n_p64(poly64_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t) {__p0}; + return __ret; +} +#else +__ai poly64x1_t vmov_n_p64(poly64_t __p0) { + poly64x1_t __ret; + __ret = (poly64x1_t) {__p0}; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vmovq_n_p64(poly64_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t) {__p0, __p0}; + return __ret; +} +#else +__ai poly64x2_t vmovq_n_p64(poly64_t __p0) { + poly64x2_t __ret; + __ret = (poly64x2_t) {__p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vmovq_n_f64(float64_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) {__p0, __p0}; + return __ret; +} +#else +__ai float64x2_t vmovq_n_f64(float64_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) {__p0, __p0}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vmov_n_f64(float64_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) {__p0}; + return __ret; +} +#else +__ai float64x1_t vmov_n_f64(float64_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) {__p0}; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmovl_high_u8(uint8x16_t __p0_128) { + uint16x8_t __ret_128; + uint8x8_t __a1_128 = vget_high_u8(__p0_128); + __ret_128 = (uint16x8_t)(vshll_n_u8(__a1_128, 0)); + return __ret_128; +} +#else +__ai uint16x8_t vmovl_high_u8(uint8x16_t __p0_129) { + uint8x16_t __rev0_129; __rev0_129 = __builtin_shufflevector(__p0_129, __p0_129, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret_129; + uint8x8_t __a1_129 = __noswap_vget_high_u8(__rev0_129); + __ret_129 = (uint16x8_t)(__noswap_vshll_n_u8(__a1_129, 0)); + __ret_129 = __builtin_shufflevector(__ret_129, __ret_129, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret_129; +} +__ai uint16x8_t __noswap_vmovl_high_u8(uint8x16_t __p0_130) { + uint16x8_t __ret_130; + uint8x8_t __a1_130 = __noswap_vget_high_u8(__p0_130); + __ret_130 = (uint16x8_t)(__noswap_vshll_n_u8(__a1_130, 0)); + return __ret_130; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmovl_high_u32(uint32x4_t __p0_131) { + uint64x2_t __ret_131; + uint32x2_t __a1_131 = vget_high_u32(__p0_131); + __ret_131 = (uint64x2_t)(vshll_n_u32(__a1_131, 0)); + return __ret_131; +} +#else +__ai uint64x2_t vmovl_high_u32(uint32x4_t __p0_132) { + uint32x4_t __rev0_132; __rev0_132 = __builtin_shufflevector(__p0_132, __p0_132, 3, 2, 1, 0); + uint64x2_t __ret_132; + uint32x2_t __a1_132 = __noswap_vget_high_u32(__rev0_132); + __ret_132 = (uint64x2_t)(__noswap_vshll_n_u32(__a1_132, 0)); + __ret_132 = __builtin_shufflevector(__ret_132, __ret_132, 1, 0); + return __ret_132; +} +__ai uint64x2_t __noswap_vmovl_high_u32(uint32x4_t __p0_133) { + uint64x2_t __ret_133; + uint32x2_t __a1_133 = __noswap_vget_high_u32(__p0_133); + __ret_133 = (uint64x2_t)(__noswap_vshll_n_u32(__a1_133, 0)); + return __ret_133; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmovl_high_u16(uint16x8_t __p0_134) { + uint32x4_t __ret_134; + uint16x4_t __a1_134 = vget_high_u16(__p0_134); + __ret_134 = (uint32x4_t)(vshll_n_u16(__a1_134, 0)); + return __ret_134; +} +#else +__ai uint32x4_t vmovl_high_u16(uint16x8_t __p0_135) { + uint16x8_t __rev0_135; __rev0_135 = __builtin_shufflevector(__p0_135, __p0_135, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret_135; + uint16x4_t __a1_135 = __noswap_vget_high_u16(__rev0_135); + __ret_135 = (uint32x4_t)(__noswap_vshll_n_u16(__a1_135, 0)); + __ret_135 = __builtin_shufflevector(__ret_135, __ret_135, 3, 2, 1, 0); + return __ret_135; +} +__ai uint32x4_t __noswap_vmovl_high_u16(uint16x8_t __p0_136) { + uint32x4_t __ret_136; + uint16x4_t __a1_136 = __noswap_vget_high_u16(__p0_136); + __ret_136 = (uint32x4_t)(__noswap_vshll_n_u16(__a1_136, 0)); + return __ret_136; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmovl_high_s8(int8x16_t __p0_137) { + int16x8_t __ret_137; + int8x8_t __a1_137 = vget_high_s8(__p0_137); + __ret_137 = (int16x8_t)(vshll_n_s8(__a1_137, 0)); + return __ret_137; +} +#else +__ai int16x8_t vmovl_high_s8(int8x16_t __p0_138) { + int8x16_t __rev0_138; __rev0_138 = __builtin_shufflevector(__p0_138, __p0_138, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret_138; + int8x8_t __a1_138 = __noswap_vget_high_s8(__rev0_138); + __ret_138 = (int16x8_t)(__noswap_vshll_n_s8(__a1_138, 0)); + __ret_138 = __builtin_shufflevector(__ret_138, __ret_138, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret_138; +} +__ai int16x8_t __noswap_vmovl_high_s8(int8x16_t __p0_139) { + int16x8_t __ret_139; + int8x8_t __a1_139 = __noswap_vget_high_s8(__p0_139); + __ret_139 = (int16x8_t)(__noswap_vshll_n_s8(__a1_139, 0)); + return __ret_139; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmovl_high_s32(int32x4_t __p0_140) { + int64x2_t __ret_140; + int32x2_t __a1_140 = vget_high_s32(__p0_140); + __ret_140 = (int64x2_t)(vshll_n_s32(__a1_140, 0)); + return __ret_140; +} +#else +__ai int64x2_t vmovl_high_s32(int32x4_t __p0_141) { + int32x4_t __rev0_141; __rev0_141 = __builtin_shufflevector(__p0_141, __p0_141, 3, 2, 1, 0); + int64x2_t __ret_141; + int32x2_t __a1_141 = __noswap_vget_high_s32(__rev0_141); + __ret_141 = (int64x2_t)(__noswap_vshll_n_s32(__a1_141, 0)); + __ret_141 = __builtin_shufflevector(__ret_141, __ret_141, 1, 0); + return __ret_141; +} +__ai int64x2_t __noswap_vmovl_high_s32(int32x4_t __p0_142) { + int64x2_t __ret_142; + int32x2_t __a1_142 = __noswap_vget_high_s32(__p0_142); + __ret_142 = (int64x2_t)(__noswap_vshll_n_s32(__a1_142, 0)); + return __ret_142; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmovl_high_s16(int16x8_t __p0_143) { + int32x4_t __ret_143; + int16x4_t __a1_143 = vget_high_s16(__p0_143); + __ret_143 = (int32x4_t)(vshll_n_s16(__a1_143, 0)); + return __ret_143; +} +#else +__ai int32x4_t vmovl_high_s16(int16x8_t __p0_144) { + int16x8_t __rev0_144; __rev0_144 = __builtin_shufflevector(__p0_144, __p0_144, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret_144; + int16x4_t __a1_144 = __noswap_vget_high_s16(__rev0_144); + __ret_144 = (int32x4_t)(__noswap_vshll_n_s16(__a1_144, 0)); + __ret_144 = __builtin_shufflevector(__ret_144, __ret_144, 3, 2, 1, 0); + return __ret_144; +} +__ai int32x4_t __noswap_vmovl_high_s16(int16x8_t __p0_145) { + int32x4_t __ret_145; + int16x4_t __a1_145 = __noswap_vget_high_s16(__p0_145); + __ret_145 = (int32x4_t)(__noswap_vshll_n_s16(__a1_145, 0)); + return __ret_145; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmovn_high_u32(uint16x4_t __p0, uint32x4_t __p1) { + uint16x8_t __ret; + __ret = vcombine_u16(__p0, vmovn_u32(__p1)); + return __ret; +} +#else +__ai uint16x8_t vmovn_high_u32(uint16x4_t __p0, uint32x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vcombine_u16(__rev0, __noswap_vmovn_u32(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmovn_high_u64(uint32x2_t __p0, uint64x2_t __p1) { + uint32x4_t __ret; + __ret = vcombine_u32(__p0, vmovn_u64(__p1)); + return __ret; +} +#else +__ai uint32x4_t vmovn_high_u64(uint32x2_t __p0, uint64x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vcombine_u32(__rev0, __noswap_vmovn_u64(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vmovn_high_u16(uint8x8_t __p0, uint16x8_t __p1) { + uint8x16_t __ret; + __ret = vcombine_u8(__p0, vmovn_u16(__p1)); + return __ret; +} +#else +__ai uint8x16_t vmovn_high_u16(uint8x8_t __p0, uint16x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __noswap_vcombine_u8(__rev0, __noswap_vmovn_u16(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmovn_high_s32(int16x4_t __p0, int32x4_t __p1) { + int16x8_t __ret; + __ret = vcombine_s16(__p0, vmovn_s32(__p1)); + return __ret; +} +#else +__ai int16x8_t vmovn_high_s32(int16x4_t __p0, int32x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vcombine_s16(__rev0, __noswap_vmovn_s32(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmovn_high_s64(int32x2_t __p0, int64x2_t __p1) { + int32x4_t __ret; + __ret = vcombine_s32(__p0, vmovn_s64(__p1)); + return __ret; +} +#else +__ai int32x4_t vmovn_high_s64(int32x2_t __p0, int64x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x4_t __ret; + __ret = __noswap_vcombine_s32(__rev0, __noswap_vmovn_s64(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vmovn_high_s16(int8x8_t __p0, int16x8_t __p1) { + int8x16_t __ret; + __ret = vcombine_s8(__p0, vmovn_s16(__p1)); + return __ret; +} +#else +__ai int8x16_t vmovn_high_s16(int8x8_t __p0, int16x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __noswap_vcombine_s8(__rev0, __noswap_vmovn_s16(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vmulq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai float64x2_t vmulq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = __rev0 * __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vmul_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#else +__ai float64x1_t vmul_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = __p0 * __p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmuld_lane_f64(__p0_146, __p1_146, __p2_146) __extension__ ({ \ + float64_t __s0_146 = __p0_146; \ + float64x1_t __s1_146 = __p1_146; \ + float64_t __ret_146; \ + __ret_146 = __s0_146 * vget_lane_f64(__s1_146, __p2_146); \ + __ret_146; \ +}) +#else +#define vmuld_lane_f64(__p0_147, __p1_147, __p2_147) __extension__ ({ \ + float64_t __s0_147 = __p0_147; \ + float64x1_t __s1_147 = __p1_147; \ + float64_t __ret_147; \ + __ret_147 = __s0_147 * __noswap_vget_lane_f64(__s1_147, __p2_147); \ + __ret_147; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmuls_lane_f32(__p0_148, __p1_148, __p2_148) __extension__ ({ \ + float32_t __s0_148 = __p0_148; \ + float32x2_t __s1_148 = __p1_148; \ + float32_t __ret_148; \ + __ret_148 = __s0_148 * vget_lane_f32(__s1_148, __p2_148); \ + __ret_148; \ +}) +#else +#define vmuls_lane_f32(__p0_149, __p1_149, __p2_149) __extension__ ({ \ + float32_t __s0_149 = __p0_149; \ + float32x2_t __s1_149 = __p1_149; \ + float32x2_t __rev1_149; __rev1_149 = __builtin_shufflevector(__s1_149, __s1_149, 1, 0); \ + float32_t __ret_149; \ + __ret_149 = __s0_149 * __noswap_vget_lane_f32(__rev1_149, __p2_149); \ + __ret_149; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmul_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vmul_lane_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 10); \ + __ret; \ +}) +#else +#define vmul_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vmul_lane_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulq_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x2_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2); \ + __ret; \ +}) +#else +#define vmulq_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float64x2_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__s1, __s1, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmuld_laneq_f64(__p0_150, __p1_150, __p2_150) __extension__ ({ \ + float64_t __s0_150 = __p0_150; \ + float64x2_t __s1_150 = __p1_150; \ + float64_t __ret_150; \ + __ret_150 = __s0_150 * vgetq_lane_f64(__s1_150, __p2_150); \ + __ret_150; \ +}) +#else +#define vmuld_laneq_f64(__p0_151, __p1_151, __p2_151) __extension__ ({ \ + float64_t __s0_151 = __p0_151; \ + float64x2_t __s1_151 = __p1_151; \ + float64x2_t __rev1_151; __rev1_151 = __builtin_shufflevector(__s1_151, __s1_151, 1, 0); \ + float64_t __ret_151; \ + __ret_151 = __s0_151 * __noswap_vgetq_lane_f64(__rev1_151, __p2_151); \ + __ret_151; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmuls_laneq_f32(__p0_152, __p1_152, __p2_152) __extension__ ({ \ + float32_t __s0_152 = __p0_152; \ + float32x4_t __s1_152 = __p1_152; \ + float32_t __ret_152; \ + __ret_152 = __s0_152 * vgetq_lane_f32(__s1_152, __p2_152); \ + __ret_152; \ +}) +#else +#define vmuls_laneq_f32(__p0_153, __p1_153, __p2_153) __extension__ ({ \ + float32_t __s0_153 = __p0_153; \ + float32x4_t __s1_153 = __p1_153; \ + float32x4_t __rev1_153; __rev1_153 = __builtin_shufflevector(__s1_153, __s1_153, 3, 2, 1, 0); \ + float32_t __ret_153; \ + __ret_153 = __s0_153 * __noswap_vgetq_lane_f32(__rev1_153, __p2_153); \ + __ret_153; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmul_laneq_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vmul_laneq_v((int8x8_t)__s0, (int8x16_t)__s1, __p2, 10); \ + __ret; \ +}) +#else +#define vmul_laneq_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vmul_laneq_v((int8x8_t)__s0, (int8x16_t)__rev1, __p2, 10); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulq_laneq_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmulq_laneq_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulq_laneq_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmulq_laneq_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulq_laneq_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x2_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2); \ + __ret; \ +}) +#else +#define vmulq_laneq_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float64x2_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulq_laneq_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmulq_laneq_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float32x4_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulq_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmulq_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulq_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmulq_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmul_laneq_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x2_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2); \ + __ret; \ +}) +#else +#define vmul_laneq_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x2_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmul_laneq_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x4_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmul_laneq_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x4_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmul_laneq_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x2_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2); \ + __ret; \ +}) +#else +#define vmul_laneq_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float32x2_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmul_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2); \ + __ret; \ +}) +#else +#define vmul_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x2_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmul_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __ret; \ + __ret = __s0 * __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2); \ + __ret; \ +}) +#else +#define vmul_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __rev0 * __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vmul_n_f64(float64x1_t __p0, float64_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vmul_n_f64((int8x8_t)__p0, __p1); + return __ret; +} +#else +__ai float64x1_t vmul_n_f64(float64x1_t __p0, float64_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vmul_n_f64((int8x8_t)__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vmulq_n_f64(float64x2_t __p0, float64_t __p1) { + float64x2_t __ret; + __ret = __p0 * (float64x2_t) {__p1, __p1}; + return __ret; +} +#else +__ai float64x2_t vmulq_n_f64(float64x2_t __p0, float64_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = __rev0 * (float64x2_t) {__p1, __p1}; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vmull_p64(poly64_t __p0, poly64_t __p1) { + poly128_t __ret; + __ret = (poly128_t) __builtin_neon_vmull_p64(__p0, __p1); + return __ret; +} +#else +__ai poly128_t vmull_p64(poly64_t __p0, poly64_t __p1) { + poly128_t __ret; + __ret = (poly128_t) __builtin_neon_vmull_p64(__p0, __p1); + return __ret; +} +__ai poly128_t __noswap_vmull_p64(poly64_t __p0, poly64_t __p1) { + poly128_t __ret; + __ret = (poly128_t) __builtin_neon_vmull_p64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vmull_high_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly16x8_t __ret; + __ret = vmull_p8(vget_high_p8(__p0), vget_high_p8(__p1)); + return __ret; +} +#else +__ai poly16x8_t vmull_high_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __ret; + __ret = __noswap_vmull_p8(__noswap_vget_high_p8(__rev0), __noswap_vget_high_p8(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmull_high_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint16x8_t __ret; + __ret = vmull_u8(vget_high_u8(__p0), vget_high_u8(__p1)); + return __ret; +} +#else +__ai uint16x8_t vmull_high_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vmull_u8(__noswap_vget_high_u8(__rev0), __noswap_vget_high_u8(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmull_high_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint64x2_t __ret; + __ret = vmull_u32(vget_high_u32(__p0), vget_high_u32(__p1)); + return __ret; +} +#else +__ai uint64x2_t vmull_high_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint64x2_t __ret; + __ret = __noswap_vmull_u32(__noswap_vget_high_u32(__rev0), __noswap_vget_high_u32(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmull_high_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint32x4_t __ret; + __ret = vmull_u16(vget_high_u16(__p0), vget_high_u16(__p1)); + return __ret; +} +#else +__ai uint32x4_t vmull_high_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vmull_u16(__noswap_vget_high_u16(__rev0), __noswap_vget_high_u16(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmull_high_s8(int8x16_t __p0, int8x16_t __p1) { + int16x8_t __ret; + __ret = vmull_s8(vget_high_s8(__p0), vget_high_s8(__p1)); + return __ret; +} +#else +__ai int16x8_t vmull_high_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vmull_s8(__noswap_vget_high_s8(__rev0), __noswap_vget_high_s8(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmull_high_s32(int32x4_t __p0, int32x4_t __p1) { + int64x2_t __ret; + __ret = vmull_s32(vget_high_s32(__p0), vget_high_s32(__p1)); + return __ret; +} +#else +__ai int64x2_t vmull_high_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vmull_s32(__noswap_vget_high_s32(__rev0), __noswap_vget_high_s32(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmull_high_s16(int16x8_t __p0, int16x8_t __p1) { + int32x4_t __ret; + __ret = vmull_s16(vget_high_s16(__p0), vget_high_s16(__p1)); + return __ret; +} +#else +__ai int32x4_t vmull_high_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vmull_s16(__noswap_vget_high_s16(__rev0), __noswap_vget_high_s16(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly128_t vmull_high_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly128_t __ret; + __ret = vmull_p64((poly64_t)(vget_high_p64(__p0)), (poly64_t)(vget_high_p64(__p1))); + return __ret; +} +#else +__ai poly128_t vmull_high_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + poly128_t __ret; + __ret = __noswap_vmull_p64((poly64_t)(__noswap_vget_high_p64(__rev0)), (poly64_t)(__noswap_vget_high_p64(__rev1))); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_high_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint64x2_t __ret; \ + __ret = vmull_u32(vget_high_u32(__s0), __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_high_lane_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint64x2_t __ret; \ + __ret = __noswap_vmull_u32(__noswap_vget_high_u32(__rev0), __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_high_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint32x4_t __ret; \ + __ret = vmull_u16(vget_high_u16(__s0), __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_high_lane_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __noswap_vmull_u16(__noswap_vget_high_u16(__rev0), __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_high_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = vmull_s32(vget_high_s32(__s0), __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_high_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vmull_s32(__noswap_vget_high_s32(__rev0), __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_high_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = vmull_s16(vget_high_s16(__s0), __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_high_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vmull_s16(__noswap_vget_high_s16(__rev0), __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_high_laneq_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint64x2_t __ret; \ + __ret = vmull_u32(vget_high_u32(__s0), __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_high_laneq_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint64x2_t __ret; \ + __ret = __noswap_vmull_u32(__noswap_vget_high_u32(__rev0), __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_high_laneq_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint32x4_t __ret; \ + __ret = vmull_u16(vget_high_u16(__s0), __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_high_laneq_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x8_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __noswap_vmull_u16(__noswap_vget_high_u16(__rev0), __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_high_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = vmull_s32(vget_high_s32(__s0), __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_high_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vmull_s32(__noswap_vget_high_s32(__rev0), __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_high_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = vmull_s16(vget_high_s16(__s0), __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_high_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vmull_s16(__noswap_vget_high_s16(__rev0), __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmull_high_n_u32(uint32x4_t __p0, uint32_t __p1) { + uint64x2_t __ret; + __ret = vmull_n_u32(vget_high_u32(__p0), __p1); + return __ret; +} +#else +__ai uint64x2_t vmull_high_n_u32(uint32x4_t __p0, uint32_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint64x2_t __ret; + __ret = __noswap_vmull_n_u32(__noswap_vget_high_u32(__rev0), __p1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmull_high_n_u16(uint16x8_t __p0, uint16_t __p1) { + uint32x4_t __ret; + __ret = vmull_n_u16(vget_high_u16(__p0), __p1); + return __ret; +} +#else +__ai uint32x4_t vmull_high_n_u16(uint16x8_t __p0, uint16_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vmull_n_u16(__noswap_vget_high_u16(__rev0), __p1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmull_high_n_s32(int32x4_t __p0, int32_t __p1) { + int64x2_t __ret; + __ret = vmull_n_s32(vget_high_s32(__p0), __p1); + return __ret; +} +#else +__ai int64x2_t vmull_high_n_s32(int32x4_t __p0, int32_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vmull_n_s32(__noswap_vget_high_s32(__rev0), __p1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmull_high_n_s16(int16x8_t __p0, int16_t __p1) { + int32x4_t __ret; + __ret = vmull_n_s16(vget_high_s16(__p0), __p1); + return __ret; +} +#else +__ai int32x4_t vmull_high_n_s16(int16x8_t __p0, int16_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vmull_n_s16(__noswap_vget_high_s16(__rev0), __p1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_laneq_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint64x2_t __ret; \ + __ret = vmull_u32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_laneq_u32(__p0, __p1, __p2) __extension__ ({ \ + uint32x2_t __s0 = __p0; \ + uint32x4_t __s1 = __p1; \ + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint64x2_t __ret; \ + __ret = __noswap_vmull_u32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_laneq_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint32x4_t __ret; \ + __ret = vmull_u16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_laneq_u16(__p0, __p1, __p2) __extension__ ({ \ + uint16x4_t __s0 = __p0; \ + uint16x8_t __s1 = __p1; \ + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __noswap_vmull_u16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = vmull_s32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vmull_s32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmull_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = vmull_s16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmull_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vmull_s16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vmulxq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vmulxq_v((int8x16_t)__p0, (int8x16_t)__p1, 42); + return __ret; +} +#else +__ai float64x2_t vmulxq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vmulxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai float64x2_t __noswap_vmulxq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vmulxq_v((int8x16_t)__p0, (int8x16_t)__p1, 42); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vmulxq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vmulxq_v((int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4_t vmulxq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vmulxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai float32x4_t __noswap_vmulxq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vmulxq_v((int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vmulx_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vmulx_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#else +__ai float64x1_t vmulx_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vmulx_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vmulx_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vmulx_v((int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2_t vmulx_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vmulx_v((int8x8_t)__rev0, (int8x8_t)__rev1, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai float32x2_t __noswap_vmulx_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vmulx_v((int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vmulxd_f64(float64_t __p0, float64_t __p1) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vmulxd_f64(__p0, __p1); + return __ret; +} +#else +__ai float64_t vmulxd_f64(float64_t __p0, float64_t __p1) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vmulxd_f64(__p0, __p1); + return __ret; +} +__ai float64_t __noswap_vmulxd_f64(float64_t __p0, float64_t __p1) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vmulxd_f64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vmulxs_f32(float32_t __p0, float32_t __p1) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vmulxs_f32(__p0, __p1); + return __ret; +} +#else +__ai float32_t vmulxs_f32(float32_t __p0, float32_t __p1) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vmulxs_f32(__p0, __p1); + return __ret; +} +__ai float32_t __noswap_vmulxs_f32(float32_t __p0, float32_t __p1) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vmulxs_f32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulxd_lane_f64(__p0_154, __p1_154, __p2_154) __extension__ ({ \ + float64_t __s0_154 = __p0_154; \ + float64x1_t __s1_154 = __p1_154; \ + float64_t __ret_154; \ + __ret_154 = vmulxd_f64(__s0_154, vget_lane_f64(__s1_154, __p2_154)); \ + __ret_154; \ +}) +#else +#define vmulxd_lane_f64(__p0_155, __p1_155, __p2_155) __extension__ ({ \ + float64_t __s0_155 = __p0_155; \ + float64x1_t __s1_155 = __p1_155; \ + float64_t __ret_155; \ + __ret_155 = __noswap_vmulxd_f64(__s0_155, __noswap_vget_lane_f64(__s1_155, __p2_155)); \ + __ret_155; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulxs_lane_f32(__p0_156, __p1_156, __p2_156) __extension__ ({ \ + float32_t __s0_156 = __p0_156; \ + float32x2_t __s1_156 = __p1_156; \ + float32_t __ret_156; \ + __ret_156 = vmulxs_f32(__s0_156, vget_lane_f32(__s1_156, __p2_156)); \ + __ret_156; \ +}) +#else +#define vmulxs_lane_f32(__p0_157, __p1_157, __p2_157) __extension__ ({ \ + float32_t __s0_157 = __p0_157; \ + float32x2_t __s1_157 = __p1_157; \ + float32x2_t __rev1_157; __rev1_157 = __builtin_shufflevector(__s1_157, __s1_157, 1, 0); \ + float32_t __ret_157; \ + __ret_157 = __noswap_vmulxs_f32(__s0_157, __noswap_vget_lane_f32(__rev1_157, __p2_157)); \ + __ret_157; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulxq_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x2_t __ret; \ + __ret = vmulxq_f64(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmulxq_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float64x2_t __ret; \ + __ret = __noswap_vmulxq_f64(__rev0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulxq_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x4_t __ret; \ + __ret = vmulxq_f32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmulxq_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float32x4_t __ret; \ + __ret = __noswap_vmulxq_f32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulx_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __ret; \ + __ret = vmulx_f32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmulx_lane_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x2_t __s1 = __p1; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float32x2_t __ret; \ + __ret = __noswap_vmulx_f32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulxd_laneq_f64(__p0_158, __p1_158, __p2_158) __extension__ ({ \ + float64_t __s0_158 = __p0_158; \ + float64x2_t __s1_158 = __p1_158; \ + float64_t __ret_158; \ + __ret_158 = vmulxd_f64(__s0_158, vgetq_lane_f64(__s1_158, __p2_158)); \ + __ret_158; \ +}) +#else +#define vmulxd_laneq_f64(__p0_159, __p1_159, __p2_159) __extension__ ({ \ + float64_t __s0_159 = __p0_159; \ + float64x2_t __s1_159 = __p1_159; \ + float64x2_t __rev1_159; __rev1_159 = __builtin_shufflevector(__s1_159, __s1_159, 1, 0); \ + float64_t __ret_159; \ + __ret_159 = __noswap_vmulxd_f64(__s0_159, __noswap_vgetq_lane_f64(__rev1_159, __p2_159)); \ + __ret_159; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulxs_laneq_f32(__p0_160, __p1_160, __p2_160) __extension__ ({ \ + float32_t __s0_160 = __p0_160; \ + float32x4_t __s1_160 = __p1_160; \ + float32_t __ret_160; \ + __ret_160 = vmulxs_f32(__s0_160, vgetq_lane_f32(__s1_160, __p2_160)); \ + __ret_160; \ +}) +#else +#define vmulxs_laneq_f32(__p0_161, __p1_161, __p2_161) __extension__ ({ \ + float32_t __s0_161 = __p0_161; \ + float32x4_t __s1_161 = __p1_161; \ + float32x4_t __rev1_161; __rev1_161 = __builtin_shufflevector(__s1_161, __s1_161, 3, 2, 1, 0); \ + float32_t __ret_161; \ + __ret_161 = __noswap_vmulxs_f32(__s0_161, __noswap_vgetq_lane_f32(__rev1_161, __p2_161)); \ + __ret_161; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulxq_laneq_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x2_t __ret; \ + __ret = vmulxq_f64(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmulxq_laneq_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float64x2_t __ret; \ + __ret = __noswap_vmulxq_f64(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulxq_laneq_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __ret; \ + __ret = vmulxq_f32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmulxq_laneq_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x4_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float32x4_t __ret; \ + __ret = __noswap_vmulxq_f32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulx_laneq_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x2_t __ret; \ + __ret = vmulx_f32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vmulx_laneq_f32(__p0, __p1, __p2) __extension__ ({ \ + float32x2_t __s0 = __p0; \ + float32x4_t __s1 = __p1; \ + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + float32x2_t __ret; \ + __ret = __noswap_vmulx_f32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vnegq_f64(float64x2_t __p0) { + float64x2_t __ret; + __ret = -__p0; + return __ret; +} +#else +__ai float64x2_t vnegq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = -__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vnegq_s64(int64x2_t __p0) { + int64x2_t __ret; + __ret = -__p0; + return __ret; +} +#else +__ai int64x2_t vnegq_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __ret; + __ret = -__rev0; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vneg_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = -__p0; + return __ret; +} +#else +__ai float64x1_t vneg_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = -__p0; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vneg_s64(int64x1_t __p0) { + int64x1_t __ret; + __ret = -__p0; + return __ret; +} +#else +__ai int64x1_t vneg_s64(int64x1_t __p0) { + int64x1_t __ret; + __ret = -__p0; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vnegd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vnegd_s64(__p0); + return __ret; +} +#else +__ai int64_t vnegd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vnegd_s64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vpaddq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vpaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vpaddq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vpaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vpaddq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vpaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vpaddq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vpaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vpaddq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vpaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vpaddq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vpaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vpaddq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vpaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vpaddq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vpaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vpaddq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vpaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vpaddq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vpaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vpaddq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vpaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 42); + return __ret; +} +#else +__ai float64x2_t vpaddq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vpaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vpaddq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vpaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4_t vpaddq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vpaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vpaddq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vpaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vpaddq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vpaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vpaddq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vpaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 35); + return __ret; +} +#else +__ai int64x2_t vpaddq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vpaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vpaddq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vpaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vpaddq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vpaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vpaddd_u64(uint64x2_t __p0) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vpaddd_u64((int8x16_t)__p0); + return __ret; +} +#else +__ai uint64_t vpaddd_u64(uint64x2_t __p0) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vpaddd_u64((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vpaddd_f64(float64x2_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vpaddd_f64((int8x16_t)__p0); + return __ret; +} +#else +__ai float64_t vpaddd_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64_t __ret; + __ret = (float64_t) __builtin_neon_vpaddd_f64((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vpaddd_s64(int64x2_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vpaddd_s64((int8x16_t)__p0); + return __ret; +} +#else +__ai int64_t vpaddd_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64_t __ret; + __ret = (int64_t) __builtin_neon_vpaddd_s64((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vpadds_f32(float32x2_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vpadds_f32((int8x8_t)__p0); + return __ret; +} +#else +__ai float32_t vpadds_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vpadds_f32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vpmaxq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vpmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vpmaxq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vpmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vpmaxq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vpmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vpmaxq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vpmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vpmaxq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vpmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vpmaxq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vpmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vpmaxq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vpmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vpmaxq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vpmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vpmaxq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vpmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 42); + return __ret; +} +#else +__ai float64x2_t vpmaxq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vpmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vpmaxq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vpmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4_t vpmaxq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vpmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vpmaxq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vpmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vpmaxq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vpmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vpmaxq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vpmaxq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vpmaxq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vpmaxq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vpmaxqd_f64(float64x2_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vpmaxqd_f64((int8x16_t)__p0); + return __ret; +} +#else +__ai float64_t vpmaxqd_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64_t __ret; + __ret = (float64_t) __builtin_neon_vpmaxqd_f64((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vpmaxs_f32(float32x2_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vpmaxs_f32((int8x8_t)__p0); + return __ret; +} +#else +__ai float32_t vpmaxs_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vpmaxs_f32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vpmaxnmq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vpmaxnmq_v((int8x16_t)__p0, (int8x16_t)__p1, 42); + return __ret; +} +#else +__ai float64x2_t vpmaxnmq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vpmaxnmq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vpmaxnmq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vpmaxnmq_v((int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4_t vpmaxnmq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vpmaxnmq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vpmaxnm_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vpmaxnm_v((int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2_t vpmaxnm_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vpmaxnm_v((int8x8_t)__rev0, (int8x8_t)__rev1, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vpmaxnmqd_f64(float64x2_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vpmaxnmqd_f64((int8x16_t)__p0); + return __ret; +} +#else +__ai float64_t vpmaxnmqd_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64_t __ret; + __ret = (float64_t) __builtin_neon_vpmaxnmqd_f64((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vpmaxnms_f32(float32x2_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vpmaxnms_f32((int8x8_t)__p0); + return __ret; +} +#else +__ai float32_t vpmaxnms_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vpmaxnms_f32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vpminq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vpminq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vpminq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vpminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vpminq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vpminq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vpminq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vpminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vpminq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vpminq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vpminq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vpminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vpminq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vpminq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vpminq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vpminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vpminq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vpminq_v((int8x16_t)__p0, (int8x16_t)__p1, 42); + return __ret; +} +#else +__ai float64x2_t vpminq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vpminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vpminq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vpminq_v((int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4_t vpminq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vpminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vpminq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vpminq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vpminq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vpminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vpminq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vpminq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vpminq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vpminq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vpminqd_f64(float64x2_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vpminqd_f64((int8x16_t)__p0); + return __ret; +} +#else +__ai float64_t vpminqd_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64_t __ret; + __ret = (float64_t) __builtin_neon_vpminqd_f64((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vpmins_f32(float32x2_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vpmins_f32((int8x8_t)__p0); + return __ret; +} +#else +__ai float32_t vpmins_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vpmins_f32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vpminnmq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vpminnmq_v((int8x16_t)__p0, (int8x16_t)__p1, 42); + return __ret; +} +#else +__ai float64x2_t vpminnmq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vpminnmq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vpminnmq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vpminnmq_v((int8x16_t)__p0, (int8x16_t)__p1, 41); + return __ret; +} +#else +__ai float32x4_t vpminnmq_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vpminnmq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vpminnm_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vpminnm_v((int8x8_t)__p0, (int8x8_t)__p1, 9); + return __ret; +} +#else +__ai float32x2_t vpminnm_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vpminnm_v((int8x8_t)__rev0, (int8x8_t)__rev1, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vpminnmqd_f64(float64x2_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vpminnmqd_f64((int8x16_t)__p0); + return __ret; +} +#else +__ai float64_t vpminnmqd_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64_t __ret; + __ret = (float64_t) __builtin_neon_vpminnmqd_f64((int8x16_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vpminnms_f32(float32x2_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vpminnms_f32((int8x8_t)__p0); + return __ret; +} +#else +__ai float32_t vpminnms_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32_t __ret; + __ret = (float32_t) __builtin_neon_vpminnms_f32((int8x8_t)__rev0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqabsq_s64(int64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqabsq_v((int8x16_t)__p0, 35); + return __ret; +} +#else +__ai int64x2_t vqabsq_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqabsq_v((int8x16_t)__rev0, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vqabs_s64(int64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vqabs_v((int8x8_t)__p0, 3); + return __ret; +} +#else +__ai int64x1_t vqabs_s64(int64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vqabs_v((int8x8_t)__p0, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vqabsb_s8(int8_t __p0) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqabsb_s8(__p0); + return __ret; +} +#else +__ai int8_t vqabsb_s8(int8_t __p0) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqabsb_s8(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqabss_s32(int32_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqabss_s32(__p0); + return __ret; +} +#else +__ai int32_t vqabss_s32(int32_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqabss_s32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vqabsd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqabsd_s64(__p0); + return __ret; +} +#else +__ai int64_t vqabsd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqabsd_s64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vqabsh_s16(int16_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqabsh_s16(__p0); + return __ret; +} +#else +__ai int16_t vqabsh_s16(int16_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqabsh_s16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8_t vqaddb_u8(uint8_t __p0, uint8_t __p1) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vqaddb_u8(__p0, __p1); + return __ret; +} +#else +__ai uint8_t vqaddb_u8(uint8_t __p0, uint8_t __p1) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vqaddb_u8(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vqadds_u32(uint32_t __p0, uint32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vqadds_u32(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vqadds_u32(uint32_t __p0, uint32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vqadds_u32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vqaddd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vqaddd_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vqaddd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vqaddd_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16_t vqaddh_u16(uint16_t __p0, uint16_t __p1) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vqaddh_u16(__p0, __p1); + return __ret; +} +#else +__ai uint16_t vqaddh_u16(uint16_t __p0, uint16_t __p1) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vqaddh_u16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vqaddb_s8(int8_t __p0, int8_t __p1) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqaddb_s8(__p0, __p1); + return __ret; +} +#else +__ai int8_t vqaddb_s8(int8_t __p0, int8_t __p1) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqaddb_s8(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqadds_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqadds_s32(__p0, __p1); + return __ret; +} +#else +__ai int32_t vqadds_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqadds_s32(__p0, __p1); + return __ret; +} +__ai int32_t __noswap_vqadds_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqadds_s32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vqaddd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqaddd_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vqaddd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqaddd_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vqaddh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqaddh_s16(__p0, __p1); + return __ret; +} +#else +__ai int16_t vqaddh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqaddh_s16(__p0, __p1); + return __ret; +} +__ai int16_t __noswap_vqaddh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqaddh_s16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vqdmlals_s32(int64_t __p0, int32_t __p1, int32_t __p2) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqdmlals_s32(__p0, __p1, __p2); + return __ret; +} +#else +__ai int64_t vqdmlals_s32(int64_t __p0, int32_t __p1, int32_t __p2) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqdmlals_s32(__p0, __p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqdmlalh_s16(int32_t __p0, int16_t __p1, int16_t __p2) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqdmlalh_s16(__p0, __p1, __p2); + return __ret; +} +#else +__ai int32_t vqdmlalh_s16(int32_t __p0, int16_t __p1, int16_t __p2) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqdmlalh_s16(__p0, __p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqdmlal_high_s32(int64x2_t __p0, int32x4_t __p1, int32x4_t __p2) { + int64x2_t __ret; + __ret = vqdmlal_s32(__p0, vget_high_s32(__p1), vget_high_s32(__p2)); + return __ret; +} +#else +__ai int64x2_t vqdmlal_high_s32(int64x2_t __p0, int32x4_t __p1, int32x4_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vqdmlal_s32(__rev0, __noswap_vget_high_s32(__rev1), __noswap_vget_high_s32(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqdmlal_high_s16(int32x4_t __p0, int16x8_t __p1, int16x8_t __p2) { + int32x4_t __ret; + __ret = vqdmlal_s16(__p0, vget_high_s16(__p1), vget_high_s16(__p2)); + return __ret; +} +#else +__ai int32x4_t vqdmlal_high_s16(int32x4_t __p0, int16x8_t __p1, int16x8_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vqdmlal_s16(__rev0, __noswap_vget_high_s16(__rev1), __noswap_vget_high_s16(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlal_high_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = vqdmlal_s32(__s0, vget_high_s32(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlal_high_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vqdmlal_s32(__rev0, __noswap_vget_high_s32(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlal_high_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = vqdmlal_s16(__s0, vget_high_s16(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlal_high_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqdmlal_s16(__rev0, __noswap_vget_high_s16(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlal_high_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = vqdmlal_s32(__s0, vget_high_s32(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlal_high_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vqdmlal_s32(__rev0, __noswap_vget_high_s32(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlal_high_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = vqdmlal_s16(__s0, vget_high_s16(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlal_high_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqdmlal_s16(__rev0, __noswap_vget_high_s16(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqdmlal_high_n_s32(int64x2_t __p0, int32x4_t __p1, int32_t __p2) { + int64x2_t __ret; + __ret = vqdmlal_n_s32(__p0, vget_high_s32(__p1), __p2); + return __ret; +} +#else +__ai int64x2_t vqdmlal_high_n_s32(int64x2_t __p0, int32x4_t __p1, int32_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vqdmlal_n_s32(__rev0, __noswap_vget_high_s32(__rev1), __p2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqdmlal_high_n_s16(int32x4_t __p0, int16x8_t __p1, int16_t __p2) { + int32x4_t __ret; + __ret = vqdmlal_n_s16(__p0, vget_high_s16(__p1), __p2); + return __ret; +} +#else +__ai int32x4_t vqdmlal_high_n_s16(int32x4_t __p0, int16x8_t __p1, int16_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vqdmlal_n_s16(__rev0, __noswap_vget_high_s16(__rev1), __p2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlals_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vqdmlals_lane_s32(__s0, __s1, (int8x8_t)__s2, __p3); \ + __ret; \ +}) +#else +#define vqdmlals_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vqdmlals_lane_s32(__s0, __s1, (int8x8_t)__rev2, __p3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlalh_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqdmlalh_lane_s16(__s0, __s1, (int8x8_t)__s2, __p3); \ + __ret; \ +}) +#else +#define vqdmlalh_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqdmlalh_lane_s16(__s0, __s1, (int8x8_t)__rev2, __p3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlals_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vqdmlals_laneq_s32(__s0, __s1, (int8x16_t)__s2, __p3); \ + __ret; \ +}) +#else +#define vqdmlals_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vqdmlals_laneq_s32(__s0, __s1, (int8x16_t)__rev2, __p3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlalh_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqdmlalh_laneq_s16(__s0, __s1, (int8x16_t)__s2, __p3); \ + __ret; \ +}) +#else +#define vqdmlalh_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqdmlalh_laneq_s16(__s0, __s1, (int8x16_t)__rev2, __p3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlal_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = vqdmlal_s32(__s0, __s1, __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlal_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vqdmlal_s32(__rev0, __rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlal_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = vqdmlal_s16(__s0, __s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlal_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqdmlal_s16(__rev0, __rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vqdmlsls_s32(int64_t __p0, int32_t __p1, int32_t __p2) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqdmlsls_s32(__p0, __p1, __p2); + return __ret; +} +#else +__ai int64_t vqdmlsls_s32(int64_t __p0, int32_t __p1, int32_t __p2) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqdmlsls_s32(__p0, __p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqdmlslh_s16(int32_t __p0, int16_t __p1, int16_t __p2) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqdmlslh_s16(__p0, __p1, __p2); + return __ret; +} +#else +__ai int32_t vqdmlslh_s16(int32_t __p0, int16_t __p1, int16_t __p2) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqdmlslh_s16(__p0, __p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqdmlsl_high_s32(int64x2_t __p0, int32x4_t __p1, int32x4_t __p2) { + int64x2_t __ret; + __ret = vqdmlsl_s32(__p0, vget_high_s32(__p1), vget_high_s32(__p2)); + return __ret; +} +#else +__ai int64x2_t vqdmlsl_high_s32(int64x2_t __p0, int32x4_t __p1, int32x4_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vqdmlsl_s32(__rev0, __noswap_vget_high_s32(__rev1), __noswap_vget_high_s32(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqdmlsl_high_s16(int32x4_t __p0, int16x8_t __p1, int16x8_t __p2) { + int32x4_t __ret; + __ret = vqdmlsl_s16(__p0, vget_high_s16(__p1), vget_high_s16(__p2)); + return __ret; +} +#else +__ai int32x4_t vqdmlsl_high_s16(int32x4_t __p0, int16x8_t __p1, int16x8_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vqdmlsl_s16(__rev0, __noswap_vget_high_s16(__rev1), __noswap_vget_high_s16(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlsl_high_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = vqdmlsl_s32(__s0, vget_high_s32(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlsl_high_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vqdmlsl_s32(__rev0, __noswap_vget_high_s32(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlsl_high_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = vqdmlsl_s16(__s0, vget_high_s16(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlsl_high_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqdmlsl_s16(__rev0, __noswap_vget_high_s16(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlsl_high_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = vqdmlsl_s32(__s0, vget_high_s32(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlsl_high_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vqdmlsl_s32(__rev0, __noswap_vget_high_s32(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlsl_high_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = vqdmlsl_s16(__s0, vget_high_s16(__s1), __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlsl_high_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqdmlsl_s16(__rev0, __noswap_vget_high_s16(__rev1), __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqdmlsl_high_n_s32(int64x2_t __p0, int32x4_t __p1, int32_t __p2) { + int64x2_t __ret; + __ret = vqdmlsl_n_s32(__p0, vget_high_s32(__p1), __p2); + return __ret; +} +#else +__ai int64x2_t vqdmlsl_high_n_s32(int64x2_t __p0, int32x4_t __p1, int32_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vqdmlsl_n_s32(__rev0, __noswap_vget_high_s32(__rev1), __p2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqdmlsl_high_n_s16(int32x4_t __p0, int16x8_t __p1, int16_t __p2) { + int32x4_t __ret; + __ret = vqdmlsl_n_s16(__p0, vget_high_s16(__p1), __p2); + return __ret; +} +#else +__ai int32x4_t vqdmlsl_high_n_s16(int32x4_t __p0, int16x8_t __p1, int16_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vqdmlsl_n_s16(__rev0, __noswap_vget_high_s16(__rev1), __p2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlsls_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vqdmlsls_lane_s32(__s0, __s1, (int8x8_t)__s2, __p3); \ + __ret; \ +}) +#else +#define vqdmlsls_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vqdmlsls_lane_s32(__s0, __s1, (int8x8_t)__rev2, __p3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlslh_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqdmlslh_lane_s16(__s0, __s1, (int8x8_t)__s2, __p3); \ + __ret; \ +}) +#else +#define vqdmlslh_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqdmlslh_lane_s16(__s0, __s1, (int8x8_t)__rev2, __p3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlsls_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vqdmlsls_laneq_s32(__s0, __s1, (int8x16_t)__s2, __p3); \ + __ret; \ +}) +#else +#define vqdmlsls_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vqdmlsls_laneq_s32(__s0, __s1, (int8x16_t)__rev2, __p3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlslh_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqdmlslh_laneq_s16(__s0, __s1, (int8x16_t)__s2, __p3); \ + __ret; \ +}) +#else +#define vqdmlslh_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqdmlslh_laneq_s16(__s0, __s1, (int8x16_t)__rev2, __p3); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlsl_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = vqdmlsl_s32(__s0, __s1, __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlsl_laneq_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vqdmlsl_s32(__rev0, __rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmlsl_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = vqdmlsl_s16(__s0, __s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vqdmlsl_laneq_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqdmlsl_s16(__rev0, __rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqdmulhs_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqdmulhs_s32(__p0, __p1); + return __ret; +} +#else +__ai int32_t vqdmulhs_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqdmulhs_s32(__p0, __p1); + return __ret; +} +__ai int32_t __noswap_vqdmulhs_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqdmulhs_s32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vqdmulhh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqdmulhh_s16(__p0, __p1); + return __ret; +} +#else +__ai int16_t vqdmulhh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqdmulhh_s16(__p0, __p1); + return __ret; +} +__ai int16_t __noswap_vqdmulhh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqdmulhh_s16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmulhs_lane_s32(__p0_162, __p1_162, __p2_162) __extension__ ({ \ + int32_t __s0_162 = __p0_162; \ + int32x2_t __s1_162 = __p1_162; \ + int32_t __ret_162; \ + __ret_162 = vqdmulhs_s32(__s0_162, vget_lane_s32(__s1_162, __p2_162)); \ + __ret_162; \ +}) +#else +#define vqdmulhs_lane_s32(__p0_163, __p1_163, __p2_163) __extension__ ({ \ + int32_t __s0_163 = __p0_163; \ + int32x2_t __s1_163 = __p1_163; \ + int32x2_t __rev1_163; __rev1_163 = __builtin_shufflevector(__s1_163, __s1_163, 1, 0); \ + int32_t __ret_163; \ + __ret_163 = __noswap_vqdmulhs_s32(__s0_163, __noswap_vget_lane_s32(__rev1_163, __p2_163)); \ + __ret_163; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmulhh_lane_s16(__p0_164, __p1_164, __p2_164) __extension__ ({ \ + int16_t __s0_164 = __p0_164; \ + int16x4_t __s1_164 = __p1_164; \ + int16_t __ret_164; \ + __ret_164 = vqdmulhh_s16(__s0_164, vget_lane_s16(__s1_164, __p2_164)); \ + __ret_164; \ +}) +#else +#define vqdmulhh_lane_s16(__p0_165, __p1_165, __p2_165) __extension__ ({ \ + int16_t __s0_165 = __p0_165; \ + int16x4_t __s1_165 = __p1_165; \ + int16x4_t __rev1_165; __rev1_165 = __builtin_shufflevector(__s1_165, __s1_165, 3, 2, 1, 0); \ + int16_t __ret_165; \ + __ret_165 = __noswap_vqdmulhh_s16(__s0_165, __noswap_vget_lane_s16(__rev1_165, __p2_165)); \ + __ret_165; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmulhs_laneq_s32(__p0_166, __p1_166, __p2_166) __extension__ ({ \ + int32_t __s0_166 = __p0_166; \ + int32x4_t __s1_166 = __p1_166; \ + int32_t __ret_166; \ + __ret_166 = vqdmulhs_s32(__s0_166, vgetq_lane_s32(__s1_166, __p2_166)); \ + __ret_166; \ +}) +#else +#define vqdmulhs_laneq_s32(__p0_167, __p1_167, __p2_167) __extension__ ({ \ + int32_t __s0_167 = __p0_167; \ + int32x4_t __s1_167 = __p1_167; \ + int32x4_t __rev1_167; __rev1_167 = __builtin_shufflevector(__s1_167, __s1_167, 3, 2, 1, 0); \ + int32_t __ret_167; \ + __ret_167 = __noswap_vqdmulhs_s32(__s0_167, __noswap_vgetq_lane_s32(__rev1_167, __p2_167)); \ + __ret_167; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmulhh_laneq_s16(__p0_168, __p1_168, __p2_168) __extension__ ({ \ + int16_t __s0_168 = __p0_168; \ + int16x8_t __s1_168 = __p1_168; \ + int16_t __ret_168; \ + __ret_168 = vqdmulhh_s16(__s0_168, vgetq_lane_s16(__s1_168, __p2_168)); \ + __ret_168; \ +}) +#else +#define vqdmulhh_laneq_s16(__p0_169, __p1_169, __p2_169) __extension__ ({ \ + int16_t __s0_169 = __p0_169; \ + int16x8_t __s1_169 = __p1_169; \ + int16x8_t __rev1_169; __rev1_169 = __builtin_shufflevector(__s1_169, __s1_169, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16_t __ret_169; \ + __ret_169 = __noswap_vqdmulhh_s16(__s0_169, __noswap_vgetq_lane_s16(__rev1_169, __p2_169)); \ + __ret_169; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmulhq_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = vqdmulhq_s32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmulhq_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqdmulhq_s32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmulhq_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __ret; \ + __ret = vqdmulhq_s16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmulhq_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __noswap_vqdmulhq_s16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmulh_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __ret; \ + __ret = vqdmulh_s32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmulh_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x2_t __ret; \ + __ret = __noswap_vqdmulh_s32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmulh_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __ret; \ + __ret = vqdmulh_s16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmulh_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __noswap_vqdmulh_s16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vqdmulls_s32(int32_t __p0, int32_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqdmulls_s32(__p0, __p1); + return __ret; +} +#else +__ai int64_t vqdmulls_s32(int32_t __p0, int32_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqdmulls_s32(__p0, __p1); + return __ret; +} +__ai int64_t __noswap_vqdmulls_s32(int32_t __p0, int32_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqdmulls_s32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqdmullh_s16(int16_t __p0, int16_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqdmullh_s16(__p0, __p1); + return __ret; +} +#else +__ai int32_t vqdmullh_s16(int16_t __p0, int16_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqdmullh_s16(__p0, __p1); + return __ret; +} +__ai int32_t __noswap_vqdmullh_s16(int16_t __p0, int16_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqdmullh_s16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqdmull_high_s32(int32x4_t __p0, int32x4_t __p1) { + int64x2_t __ret; + __ret = vqdmull_s32(vget_high_s32(__p0), vget_high_s32(__p1)); + return __ret; +} +#else +__ai int64x2_t vqdmull_high_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vqdmull_s32(__noswap_vget_high_s32(__rev0), __noswap_vget_high_s32(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqdmull_high_s16(int16x8_t __p0, int16x8_t __p1) { + int32x4_t __ret; + __ret = vqdmull_s16(vget_high_s16(__p0), vget_high_s16(__p1)); + return __ret; +} +#else +__ai int32x4_t vqdmull_high_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vqdmull_s16(__noswap_vget_high_s16(__rev0), __noswap_vget_high_s16(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmull_high_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = vqdmull_s32(vget_high_s32(__s0), __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmull_high_lane_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vqdmull_s32(__noswap_vget_high_s32(__rev0), __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmull_high_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = vqdmull_s16(vget_high_s16(__s0), __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmull_high_lane_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqdmull_s16(__noswap_vget_high_s16(__rev0), __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmull_high_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = vqdmull_s32(vget_high_s32(__s0), __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmull_high_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vqdmull_s32(__noswap_vget_high_s32(__rev0), __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmull_high_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = vqdmull_s16(vget_high_s16(__s0), __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmull_high_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqdmull_s16(__noswap_vget_high_s16(__rev0), __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqdmull_high_n_s32(int32x4_t __p0, int32_t __p1) { + int64x2_t __ret; + __ret = vqdmull_n_s32(vget_high_s32(__p0), __p1); + return __ret; +} +#else +__ai int64x2_t vqdmull_high_n_s32(int32x4_t __p0, int32_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vqdmull_n_s32(__noswap_vget_high_s32(__rev0), __p1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqdmull_high_n_s16(int16x8_t __p0, int16_t __p1) { + int32x4_t __ret; + __ret = vqdmull_n_s16(vget_high_s16(__p0), __p1); + return __ret; +} +#else +__ai int32x4_t vqdmull_high_n_s16(int16x8_t __p0, int16_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vqdmull_n_s16(__noswap_vget_high_s16(__rev0), __p1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmulls_lane_s32(__p0_170, __p1_170, __p2_170) __extension__ ({ \ + int32_t __s0_170 = __p0_170; \ + int32x2_t __s1_170 = __p1_170; \ + int64_t __ret_170; \ + __ret_170 = vqdmulls_s32(__s0_170, vget_lane_s32(__s1_170, __p2_170)); \ + __ret_170; \ +}) +#else +#define vqdmulls_lane_s32(__p0_171, __p1_171, __p2_171) __extension__ ({ \ + int32_t __s0_171 = __p0_171; \ + int32x2_t __s1_171 = __p1_171; \ + int32x2_t __rev1_171; __rev1_171 = __builtin_shufflevector(__s1_171, __s1_171, 1, 0); \ + int64_t __ret_171; \ + __ret_171 = __noswap_vqdmulls_s32(__s0_171, __noswap_vget_lane_s32(__rev1_171, __p2_171)); \ + __ret_171; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmullh_lane_s16(__p0_172, __p1_172, __p2_172) __extension__ ({ \ + int16_t __s0_172 = __p0_172; \ + int16x4_t __s1_172 = __p1_172; \ + int32_t __ret_172; \ + __ret_172 = vqdmullh_s16(__s0_172, vget_lane_s16(__s1_172, __p2_172)); \ + __ret_172; \ +}) +#else +#define vqdmullh_lane_s16(__p0_173, __p1_173, __p2_173) __extension__ ({ \ + int16_t __s0_173 = __p0_173; \ + int16x4_t __s1_173 = __p1_173; \ + int16x4_t __rev1_173; __rev1_173 = __builtin_shufflevector(__s1_173, __s1_173, 3, 2, 1, 0); \ + int32_t __ret_173; \ + __ret_173 = __noswap_vqdmullh_s16(__s0_173, __noswap_vget_lane_s16(__rev1_173, __p2_173)); \ + __ret_173; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmulls_laneq_s32(__p0_174, __p1_174, __p2_174) __extension__ ({ \ + int32_t __s0_174 = __p0_174; \ + int32x4_t __s1_174 = __p1_174; \ + int64_t __ret_174; \ + __ret_174 = vqdmulls_s32(__s0_174, vgetq_lane_s32(__s1_174, __p2_174)); \ + __ret_174; \ +}) +#else +#define vqdmulls_laneq_s32(__p0_175, __p1_175, __p2_175) __extension__ ({ \ + int32_t __s0_175 = __p0_175; \ + int32x4_t __s1_175 = __p1_175; \ + int32x4_t __rev1_175; __rev1_175 = __builtin_shufflevector(__s1_175, __s1_175, 3, 2, 1, 0); \ + int64_t __ret_175; \ + __ret_175 = __noswap_vqdmulls_s32(__s0_175, __noswap_vgetq_lane_s32(__rev1_175, __p2_175)); \ + __ret_175; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmullh_laneq_s16(__p0_176, __p1_176, __p2_176) __extension__ ({ \ + int16_t __s0_176 = __p0_176; \ + int16x8_t __s1_176 = __p1_176; \ + int32_t __ret_176; \ + __ret_176 = vqdmullh_s16(__s0_176, vgetq_lane_s16(__s1_176, __p2_176)); \ + __ret_176; \ +}) +#else +#define vqdmullh_laneq_s16(__p0_177, __p1_177, __p2_177) __extension__ ({ \ + int16_t __s0_177 = __p0_177; \ + int16x8_t __s1_177 = __p1_177; \ + int16x8_t __rev1_177; __rev1_177 = __builtin_shufflevector(__s1_177, __s1_177, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32_t __ret_177; \ + __ret_177 = __noswap_vqdmullh_s16(__s0_177, __noswap_vgetq_lane_s16(__rev1_177, __p2_177)); \ + __ret_177; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmull_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int64x2_t __ret; \ + __ret = vqdmull_s32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmull_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int64x2_t __ret; \ + __ret = __noswap_vqdmull_s32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqdmull_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = vqdmull_s16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqdmull_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqdmull_s16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vqmovns_s32(int32_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqmovns_s32(__p0); + return __ret; +} +#else +__ai int16_t vqmovns_s32(int32_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqmovns_s32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqmovnd_s64(int64_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqmovnd_s64(__p0); + return __ret; +} +#else +__ai int32_t vqmovnd_s64(int64_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqmovnd_s64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vqmovnh_s16(int16_t __p0) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqmovnh_s16(__p0); + return __ret; +} +#else +__ai int8_t vqmovnh_s16(int16_t __p0) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqmovnh_s16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16_t vqmovns_u32(uint32_t __p0) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vqmovns_u32(__p0); + return __ret; +} +#else +__ai uint16_t vqmovns_u32(uint32_t __p0) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vqmovns_u32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vqmovnd_u64(uint64_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vqmovnd_u64(__p0); + return __ret; +} +#else +__ai uint32_t vqmovnd_u64(uint64_t __p0) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vqmovnd_u64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8_t vqmovnh_u16(uint16_t __p0) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vqmovnh_u16(__p0); + return __ret; +} +#else +__ai uint8_t vqmovnh_u16(uint16_t __p0) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vqmovnh_u16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vqmovn_high_u32(uint16x4_t __p0, uint32x4_t __p1) { + uint16x8_t __ret; + __ret = vcombine_u16(__p0, vqmovn_u32(__p1)); + return __ret; +} +#else +__ai uint16x8_t vqmovn_high_u32(uint16x4_t __p0, uint32x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vcombine_u16(__rev0, __noswap_vqmovn_u32(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vqmovn_high_u64(uint32x2_t __p0, uint64x2_t __p1) { + uint32x4_t __ret; + __ret = vcombine_u32(__p0, vqmovn_u64(__p1)); + return __ret; +} +#else +__ai uint32x4_t vqmovn_high_u64(uint32x2_t __p0, uint64x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vcombine_u32(__rev0, __noswap_vqmovn_u64(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vqmovn_high_u16(uint8x8_t __p0, uint16x8_t __p1) { + uint8x16_t __ret; + __ret = vcombine_u8(__p0, vqmovn_u16(__p1)); + return __ret; +} +#else +__ai uint8x16_t vqmovn_high_u16(uint8x8_t __p0, uint16x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __noswap_vcombine_u8(__rev0, __noswap_vqmovn_u16(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vqmovn_high_s32(int16x4_t __p0, int32x4_t __p1) { + int16x8_t __ret; + __ret = vcombine_s16(__p0, vqmovn_s32(__p1)); + return __ret; +} +#else +__ai int16x8_t vqmovn_high_s32(int16x4_t __p0, int32x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vcombine_s16(__rev0, __noswap_vqmovn_s32(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vqmovn_high_s64(int32x2_t __p0, int64x2_t __p1) { + int32x4_t __ret; + __ret = vcombine_s32(__p0, vqmovn_s64(__p1)); + return __ret; +} +#else +__ai int32x4_t vqmovn_high_s64(int32x2_t __p0, int64x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x4_t __ret; + __ret = __noswap_vcombine_s32(__rev0, __noswap_vqmovn_s64(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqmovn_high_s16(int8x8_t __p0, int16x8_t __p1) { + int8x16_t __ret; + __ret = vcombine_s8(__p0, vqmovn_s16(__p1)); + return __ret; +} +#else +__ai int8x16_t vqmovn_high_s16(int8x8_t __p0, int16x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __noswap_vcombine_s8(__rev0, __noswap_vqmovn_s16(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vqmovuns_s32(int32_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqmovuns_s32(__p0); + return __ret; +} +#else +__ai int16_t vqmovuns_s32(int32_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqmovuns_s32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqmovund_s64(int64_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqmovund_s64(__p0); + return __ret; +} +#else +__ai int32_t vqmovund_s64(int64_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqmovund_s64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vqmovunh_s16(int16_t __p0) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqmovunh_s16(__p0); + return __ret; +} +#else +__ai int8_t vqmovunh_s16(int16_t __p0) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqmovunh_s16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vqmovun_high_s32(int16x4_t __p0, int32x4_t __p1) { + uint16x8_t __ret; + __ret = vcombine_u16((uint16x4_t)(__p0), vqmovun_s32(__p1)); + return __ret; +} +#else +__ai uint16x8_t vqmovun_high_s32(int16x4_t __p0, int32x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vcombine_u16((uint16x4_t)(__rev0), __noswap_vqmovun_s32(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vqmovun_high_s64(int32x2_t __p0, int64x2_t __p1) { + uint32x4_t __ret; + __ret = vcombine_u32((uint32x2_t)(__p0), vqmovun_s64(__p1)); + return __ret; +} +#else +__ai uint32x4_t vqmovun_high_s64(int32x2_t __p0, int64x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vcombine_u32((uint32x2_t)(__rev0), __noswap_vqmovun_s64(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vqmovun_high_s16(int8x8_t __p0, int16x8_t __p1) { + uint8x16_t __ret; + __ret = vcombine_u8((uint8x8_t)(__p0), vqmovun_s16(__p1)); + return __ret; +} +#else +__ai uint8x16_t vqmovun_high_s16(int8x8_t __p0, int16x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __noswap_vcombine_u8((uint8x8_t)(__rev0), __noswap_vqmovun_s16(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vqnegq_s64(int64x2_t __p0) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqnegq_v((int8x16_t)__p0, 35); + return __ret; +} +#else +__ai int64x2_t vqnegq_s64(int64x2_t __p0) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vqnegq_v((int8x16_t)__rev0, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vqneg_s64(int64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vqneg_v((int8x8_t)__p0, 3); + return __ret; +} +#else +__ai int64x1_t vqneg_s64(int64x1_t __p0) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vqneg_v((int8x8_t)__p0, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vqnegb_s8(int8_t __p0) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqnegb_s8(__p0); + return __ret; +} +#else +__ai int8_t vqnegb_s8(int8_t __p0) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqnegb_s8(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqnegs_s32(int32_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqnegs_s32(__p0); + return __ret; +} +#else +__ai int32_t vqnegs_s32(int32_t __p0) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqnegs_s32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vqnegd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqnegd_s64(__p0); + return __ret; +} +#else +__ai int64_t vqnegd_s64(int64_t __p0) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqnegd_s64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vqnegh_s16(int16_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqnegh_s16(__p0); + return __ret; +} +#else +__ai int16_t vqnegh_s16(int16_t __p0) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqnegh_s16(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqrdmulhs_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqrdmulhs_s32(__p0, __p1); + return __ret; +} +#else +__ai int32_t vqrdmulhs_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqrdmulhs_s32(__p0, __p1); + return __ret; +} +__ai int32_t __noswap_vqrdmulhs_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqrdmulhs_s32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vqrdmulhh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqrdmulhh_s16(__p0, __p1); + return __ret; +} +#else +__ai int16_t vqrdmulhh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqrdmulhh_s16(__p0, __p1); + return __ret; +} +__ai int16_t __noswap_vqrdmulhh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqrdmulhh_s16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmulhs_lane_s32(__p0_178, __p1_178, __p2_178) __extension__ ({ \ + int32_t __s0_178 = __p0_178; \ + int32x2_t __s1_178 = __p1_178; \ + int32_t __ret_178; \ + __ret_178 = vqrdmulhs_s32(__s0_178, vget_lane_s32(__s1_178, __p2_178)); \ + __ret_178; \ +}) +#else +#define vqrdmulhs_lane_s32(__p0_179, __p1_179, __p2_179) __extension__ ({ \ + int32_t __s0_179 = __p0_179; \ + int32x2_t __s1_179 = __p1_179; \ + int32x2_t __rev1_179; __rev1_179 = __builtin_shufflevector(__s1_179, __s1_179, 1, 0); \ + int32_t __ret_179; \ + __ret_179 = __noswap_vqrdmulhs_s32(__s0_179, __noswap_vget_lane_s32(__rev1_179, __p2_179)); \ + __ret_179; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmulhh_lane_s16(__p0_180, __p1_180, __p2_180) __extension__ ({ \ + int16_t __s0_180 = __p0_180; \ + int16x4_t __s1_180 = __p1_180; \ + int16_t __ret_180; \ + __ret_180 = vqrdmulhh_s16(__s0_180, vget_lane_s16(__s1_180, __p2_180)); \ + __ret_180; \ +}) +#else +#define vqrdmulhh_lane_s16(__p0_181, __p1_181, __p2_181) __extension__ ({ \ + int16_t __s0_181 = __p0_181; \ + int16x4_t __s1_181 = __p1_181; \ + int16x4_t __rev1_181; __rev1_181 = __builtin_shufflevector(__s1_181, __s1_181, 3, 2, 1, 0); \ + int16_t __ret_181; \ + __ret_181 = __noswap_vqrdmulhh_s16(__s0_181, __noswap_vget_lane_s16(__rev1_181, __p2_181)); \ + __ret_181; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmulhs_laneq_s32(__p0_182, __p1_182, __p2_182) __extension__ ({ \ + int32_t __s0_182 = __p0_182; \ + int32x4_t __s1_182 = __p1_182; \ + int32_t __ret_182; \ + __ret_182 = vqrdmulhs_s32(__s0_182, vgetq_lane_s32(__s1_182, __p2_182)); \ + __ret_182; \ +}) +#else +#define vqrdmulhs_laneq_s32(__p0_183, __p1_183, __p2_183) __extension__ ({ \ + int32_t __s0_183 = __p0_183; \ + int32x4_t __s1_183 = __p1_183; \ + int32x4_t __rev1_183; __rev1_183 = __builtin_shufflevector(__s1_183, __s1_183, 3, 2, 1, 0); \ + int32_t __ret_183; \ + __ret_183 = __noswap_vqrdmulhs_s32(__s0_183, __noswap_vgetq_lane_s32(__rev1_183, __p2_183)); \ + __ret_183; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmulhh_laneq_s16(__p0_184, __p1_184, __p2_184) __extension__ ({ \ + int16_t __s0_184 = __p0_184; \ + int16x8_t __s1_184 = __p1_184; \ + int16_t __ret_184; \ + __ret_184 = vqrdmulhh_s16(__s0_184, vgetq_lane_s16(__s1_184, __p2_184)); \ + __ret_184; \ +}) +#else +#define vqrdmulhh_laneq_s16(__p0_185, __p1_185, __p2_185) __extension__ ({ \ + int16_t __s0_185 = __p0_185; \ + int16x8_t __s1_185 = __p1_185; \ + int16x8_t __rev1_185; __rev1_185 = __builtin_shufflevector(__s1_185, __s1_185, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16_t __ret_185; \ + __ret_185 = __noswap_vqrdmulhh_s16(__s0_185, __noswap_vgetq_lane_s16(__rev1_185, __p2_185)); \ + __ret_185; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmulhq_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __ret; \ + __ret = vqrdmulhq_s32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqrdmulhq_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __noswap_vqrdmulhq_s32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmulhq_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __ret; \ + __ret = vqrdmulhq_s16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqrdmulhq_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x8_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret; \ + __ret = __noswap_vqrdmulhq_s16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmulh_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __ret; \ + __ret = vqrdmulh_s32(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqrdmulh_laneq_s32(__p0, __p1, __p2) __extension__ ({ \ + int32x2_t __s0 = __p0; \ + int32x4_t __s1 = __p1; \ + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int32x2_t __ret; \ + __ret = __noswap_vqrdmulh_s32(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmulh_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __ret; \ + __ret = vqrdmulh_s16(__s0, __builtin_shufflevector(__s1, __s1, __p2, __p2, __p2, __p2)); \ + __ret; \ +}) +#else +#define vqrdmulh_laneq_s16(__p0, __p1, __p2) __extension__ ({ \ + int16x4_t __s0 = __p0; \ + int16x8_t __s1 = __p1; \ + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x4_t __ret; \ + __ret = __noswap_vqrdmulh_s16(__rev0, __builtin_shufflevector(__rev1, __rev1, __p2, __p2, __p2, __p2)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8_t vqrshlb_u8(uint8_t __p0, uint8_t __p1) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vqrshlb_u8(__p0, __p1); + return __ret; +} +#else +__ai uint8_t vqrshlb_u8(uint8_t __p0, uint8_t __p1) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vqrshlb_u8(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vqrshls_u32(uint32_t __p0, uint32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vqrshls_u32(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vqrshls_u32(uint32_t __p0, uint32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vqrshls_u32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vqrshld_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vqrshld_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vqrshld_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vqrshld_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16_t vqrshlh_u16(uint16_t __p0, uint16_t __p1) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vqrshlh_u16(__p0, __p1); + return __ret; +} +#else +__ai uint16_t vqrshlh_u16(uint16_t __p0, uint16_t __p1) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vqrshlh_u16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vqrshlb_s8(int8_t __p0, int8_t __p1) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqrshlb_s8(__p0, __p1); + return __ret; +} +#else +__ai int8_t vqrshlb_s8(int8_t __p0, int8_t __p1) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqrshlb_s8(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqrshls_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqrshls_s32(__p0, __p1); + return __ret; +} +#else +__ai int32_t vqrshls_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqrshls_s32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vqrshld_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqrshld_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vqrshld_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqrshld_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vqrshlh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqrshlh_s16(__p0, __p1); + return __ret; +} +#else +__ai int16_t vqrshlh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqrshlh_s16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrn_high_n_u32(__p0_186, __p1_186, __p2_186) __extension__ ({ \ + uint16x4_t __s0_186 = __p0_186; \ + uint32x4_t __s1_186 = __p1_186; \ + uint16x8_t __ret_186; \ + __ret_186 = (uint16x8_t)(vcombine_u16((uint16x4_t)(__s0_186), (uint16x4_t)(vqrshrn_n_u32(__s1_186, __p2_186)))); \ + __ret_186; \ +}) +#else +#define vqrshrn_high_n_u32(__p0_187, __p1_187, __p2_187) __extension__ ({ \ + uint16x4_t __s0_187 = __p0_187; \ + uint32x4_t __s1_187 = __p1_187; \ + uint16x4_t __rev0_187; __rev0_187 = __builtin_shufflevector(__s0_187, __s0_187, 3, 2, 1, 0); \ + uint32x4_t __rev1_187; __rev1_187 = __builtin_shufflevector(__s1_187, __s1_187, 3, 2, 1, 0); \ + uint16x8_t __ret_187; \ + __ret_187 = (uint16x8_t)(__noswap_vcombine_u16((uint16x4_t)(__rev0_187), (uint16x4_t)(__noswap_vqrshrn_n_u32(__rev1_187, __p2_187)))); \ + __ret_187 = __builtin_shufflevector(__ret_187, __ret_187, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_187; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrn_high_n_u64(__p0_188, __p1_188, __p2_188) __extension__ ({ \ + uint32x2_t __s0_188 = __p0_188; \ + uint64x2_t __s1_188 = __p1_188; \ + uint32x4_t __ret_188; \ + __ret_188 = (uint32x4_t)(vcombine_u32((uint32x2_t)(__s0_188), (uint32x2_t)(vqrshrn_n_u64(__s1_188, __p2_188)))); \ + __ret_188; \ +}) +#else +#define vqrshrn_high_n_u64(__p0_189, __p1_189, __p2_189) __extension__ ({ \ + uint32x2_t __s0_189 = __p0_189; \ + uint64x2_t __s1_189 = __p1_189; \ + uint32x2_t __rev0_189; __rev0_189 = __builtin_shufflevector(__s0_189, __s0_189, 1, 0); \ + uint64x2_t __rev1_189; __rev1_189 = __builtin_shufflevector(__s1_189, __s1_189, 1, 0); \ + uint32x4_t __ret_189; \ + __ret_189 = (uint32x4_t)(__noswap_vcombine_u32((uint32x2_t)(__rev0_189), (uint32x2_t)(__noswap_vqrshrn_n_u64(__rev1_189, __p2_189)))); \ + __ret_189 = __builtin_shufflevector(__ret_189, __ret_189, 3, 2, 1, 0); \ + __ret_189; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrn_high_n_u16(__p0_190, __p1_190, __p2_190) __extension__ ({ \ + uint8x8_t __s0_190 = __p0_190; \ + uint16x8_t __s1_190 = __p1_190; \ + uint8x16_t __ret_190; \ + __ret_190 = (uint8x16_t)(vcombine_u8((uint8x8_t)(__s0_190), (uint8x8_t)(vqrshrn_n_u16(__s1_190, __p2_190)))); \ + __ret_190; \ +}) +#else +#define vqrshrn_high_n_u16(__p0_191, __p1_191, __p2_191) __extension__ ({ \ + uint8x8_t __s0_191 = __p0_191; \ + uint16x8_t __s1_191 = __p1_191; \ + uint8x8_t __rev0_191; __rev0_191 = __builtin_shufflevector(__s0_191, __s0_191, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1_191; __rev1_191 = __builtin_shufflevector(__s1_191, __s1_191, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret_191; \ + __ret_191 = (uint8x16_t)(__noswap_vcombine_u8((uint8x8_t)(__rev0_191), (uint8x8_t)(__noswap_vqrshrn_n_u16(__rev1_191, __p2_191)))); \ + __ret_191 = __builtin_shufflevector(__ret_191, __ret_191, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_191; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrn_high_n_s32(__p0_192, __p1_192, __p2_192) __extension__ ({ \ + int16x4_t __s0_192 = __p0_192; \ + int32x4_t __s1_192 = __p1_192; \ + int16x8_t __ret_192; \ + __ret_192 = (int16x8_t)(vcombine_s16((int16x4_t)(__s0_192), (int16x4_t)(vqrshrn_n_s32(__s1_192, __p2_192)))); \ + __ret_192; \ +}) +#else +#define vqrshrn_high_n_s32(__p0_193, __p1_193, __p2_193) __extension__ ({ \ + int16x4_t __s0_193 = __p0_193; \ + int32x4_t __s1_193 = __p1_193; \ + int16x4_t __rev0_193; __rev0_193 = __builtin_shufflevector(__s0_193, __s0_193, 3, 2, 1, 0); \ + int32x4_t __rev1_193; __rev1_193 = __builtin_shufflevector(__s1_193, __s1_193, 3, 2, 1, 0); \ + int16x8_t __ret_193; \ + __ret_193 = (int16x8_t)(__noswap_vcombine_s16((int16x4_t)(__rev0_193), (int16x4_t)(__noswap_vqrshrn_n_s32(__rev1_193, __p2_193)))); \ + __ret_193 = __builtin_shufflevector(__ret_193, __ret_193, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_193; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrn_high_n_s64(__p0_194, __p1_194, __p2_194) __extension__ ({ \ + int32x2_t __s0_194 = __p0_194; \ + int64x2_t __s1_194 = __p1_194; \ + int32x4_t __ret_194; \ + __ret_194 = (int32x4_t)(vcombine_s32((int32x2_t)(__s0_194), (int32x2_t)(vqrshrn_n_s64(__s1_194, __p2_194)))); \ + __ret_194; \ +}) +#else +#define vqrshrn_high_n_s64(__p0_195, __p1_195, __p2_195) __extension__ ({ \ + int32x2_t __s0_195 = __p0_195; \ + int64x2_t __s1_195 = __p1_195; \ + int32x2_t __rev0_195; __rev0_195 = __builtin_shufflevector(__s0_195, __s0_195, 1, 0); \ + int64x2_t __rev1_195; __rev1_195 = __builtin_shufflevector(__s1_195, __s1_195, 1, 0); \ + int32x4_t __ret_195; \ + __ret_195 = (int32x4_t)(__noswap_vcombine_s32((int32x2_t)(__rev0_195), (int32x2_t)(__noswap_vqrshrn_n_s64(__rev1_195, __p2_195)))); \ + __ret_195 = __builtin_shufflevector(__ret_195, __ret_195, 3, 2, 1, 0); \ + __ret_195; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrn_high_n_s16(__p0_196, __p1_196, __p2_196) __extension__ ({ \ + int8x8_t __s0_196 = __p0_196; \ + int16x8_t __s1_196 = __p1_196; \ + int8x16_t __ret_196; \ + __ret_196 = (int8x16_t)(vcombine_s8((int8x8_t)(__s0_196), (int8x8_t)(vqrshrn_n_s16(__s1_196, __p2_196)))); \ + __ret_196; \ +}) +#else +#define vqrshrn_high_n_s16(__p0_197, __p1_197, __p2_197) __extension__ ({ \ + int8x8_t __s0_197 = __p0_197; \ + int16x8_t __s1_197 = __p1_197; \ + int8x8_t __rev0_197; __rev0_197 = __builtin_shufflevector(__s0_197, __s0_197, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1_197; __rev1_197 = __builtin_shufflevector(__s1_197, __s1_197, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret_197; \ + __ret_197 = (int8x16_t)(__noswap_vcombine_s8((int8x8_t)(__rev0_197), (int8x8_t)(__noswap_vqrshrn_n_s16(__rev1_197, __p2_197)))); \ + __ret_197 = __builtin_shufflevector(__ret_197, __ret_197, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_197; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrns_n_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vqrshrns_n_u32(__s0, __p1); \ + __ret; \ +}) +#else +#define vqrshrns_n_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vqrshrns_n_u32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrnd_n_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vqrshrnd_n_u64(__s0, __p1); \ + __ret; \ +}) +#else +#define vqrshrnd_n_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vqrshrnd_n_u64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrnh_n_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vqrshrnh_n_u16(__s0, __p1); \ + __ret; \ +}) +#else +#define vqrshrnh_n_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vqrshrnh_n_u16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrns_n_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vqrshrns_n_s32(__s0, __p1); \ + __ret; \ +}) +#else +#define vqrshrns_n_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vqrshrns_n_s32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrnd_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqrshrnd_n_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vqrshrnd_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqrshrnd_n_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrnh_n_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vqrshrnh_n_s16(__s0, __p1); \ + __ret; \ +}) +#else +#define vqrshrnh_n_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vqrshrnh_n_s16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrun_high_n_s32(__p0_198, __p1_198, __p2_198) __extension__ ({ \ + int16x4_t __s0_198 = __p0_198; \ + int32x4_t __s1_198 = __p1_198; \ + int16x8_t __ret_198; \ + __ret_198 = (int16x8_t)(vcombine_s16((int16x4_t)(__s0_198), (int16x4_t)(vqrshrun_n_s32(__s1_198, __p2_198)))); \ + __ret_198; \ +}) +#else +#define vqrshrun_high_n_s32(__p0_199, __p1_199, __p2_199) __extension__ ({ \ + int16x4_t __s0_199 = __p0_199; \ + int32x4_t __s1_199 = __p1_199; \ + int16x4_t __rev0_199; __rev0_199 = __builtin_shufflevector(__s0_199, __s0_199, 3, 2, 1, 0); \ + int32x4_t __rev1_199; __rev1_199 = __builtin_shufflevector(__s1_199, __s1_199, 3, 2, 1, 0); \ + int16x8_t __ret_199; \ + __ret_199 = (int16x8_t)(__noswap_vcombine_s16((int16x4_t)(__rev0_199), (int16x4_t)(__noswap_vqrshrun_n_s32(__rev1_199, __p2_199)))); \ + __ret_199 = __builtin_shufflevector(__ret_199, __ret_199, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_199; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrun_high_n_s64(__p0_200, __p1_200, __p2_200) __extension__ ({ \ + int32x2_t __s0_200 = __p0_200; \ + int64x2_t __s1_200 = __p1_200; \ + int32x4_t __ret_200; \ + __ret_200 = (int32x4_t)(vcombine_s32((int32x2_t)(__s0_200), (int32x2_t)(vqrshrun_n_s64(__s1_200, __p2_200)))); \ + __ret_200; \ +}) +#else +#define vqrshrun_high_n_s64(__p0_201, __p1_201, __p2_201) __extension__ ({ \ + int32x2_t __s0_201 = __p0_201; \ + int64x2_t __s1_201 = __p1_201; \ + int32x2_t __rev0_201; __rev0_201 = __builtin_shufflevector(__s0_201, __s0_201, 1, 0); \ + int64x2_t __rev1_201; __rev1_201 = __builtin_shufflevector(__s1_201, __s1_201, 1, 0); \ + int32x4_t __ret_201; \ + __ret_201 = (int32x4_t)(__noswap_vcombine_s32((int32x2_t)(__rev0_201), (int32x2_t)(__noswap_vqrshrun_n_s64(__rev1_201, __p2_201)))); \ + __ret_201 = __builtin_shufflevector(__ret_201, __ret_201, 3, 2, 1, 0); \ + __ret_201; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrun_high_n_s16(__p0_202, __p1_202, __p2_202) __extension__ ({ \ + int8x8_t __s0_202 = __p0_202; \ + int16x8_t __s1_202 = __p1_202; \ + int8x16_t __ret_202; \ + __ret_202 = (int8x16_t)(vcombine_s8((int8x8_t)(__s0_202), (int8x8_t)(vqrshrun_n_s16(__s1_202, __p2_202)))); \ + __ret_202; \ +}) +#else +#define vqrshrun_high_n_s16(__p0_203, __p1_203, __p2_203) __extension__ ({ \ + int8x8_t __s0_203 = __p0_203; \ + int16x8_t __s1_203 = __p1_203; \ + int8x8_t __rev0_203; __rev0_203 = __builtin_shufflevector(__s0_203, __s0_203, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1_203; __rev1_203 = __builtin_shufflevector(__s1_203, __s1_203, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret_203; \ + __ret_203 = (int8x16_t)(__noswap_vcombine_s8((int8x8_t)(__rev0_203), (int8x8_t)(__noswap_vqrshrun_n_s16(__rev1_203, __p2_203)))); \ + __ret_203 = __builtin_shufflevector(__ret_203, __ret_203, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_203; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshruns_n_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vqrshruns_n_s32(__s0, __p1); \ + __ret; \ +}) +#else +#define vqrshruns_n_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vqrshruns_n_s32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrund_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqrshrund_n_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vqrshrund_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqrshrund_n_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrshrunh_n_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vqrshrunh_n_s16(__s0, __p1); \ + __ret; \ +}) +#else +#define vqrshrunh_n_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vqrshrunh_n_s16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8_t vqshlb_u8(uint8_t __p0, uint8_t __p1) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vqshlb_u8(__p0, __p1); + return __ret; +} +#else +__ai uint8_t vqshlb_u8(uint8_t __p0, uint8_t __p1) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vqshlb_u8(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vqshls_u32(uint32_t __p0, uint32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vqshls_u32(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vqshls_u32(uint32_t __p0, uint32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vqshls_u32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vqshld_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vqshld_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vqshld_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vqshld_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16_t vqshlh_u16(uint16_t __p0, uint16_t __p1) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vqshlh_u16(__p0, __p1); + return __ret; +} +#else +__ai uint16_t vqshlh_u16(uint16_t __p0, uint16_t __p1) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vqshlh_u16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vqshlb_s8(int8_t __p0, int8_t __p1) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqshlb_s8(__p0, __p1); + return __ret; +} +#else +__ai int8_t vqshlb_s8(int8_t __p0, int8_t __p1) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqshlb_s8(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqshls_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqshls_s32(__p0, __p1); + return __ret; +} +#else +__ai int32_t vqshls_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqshls_s32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vqshld_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqshld_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vqshld_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqshld_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vqshlh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqshlh_s16(__p0, __p1); + return __ret; +} +#else +__ai int16_t vqshlh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqshlh_s16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlb_n_u8(__p0, __p1) __extension__ ({ \ + uint8_t __s0 = __p0; \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vqshlb_n_u8(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshlb_n_u8(__p0, __p1) __extension__ ({ \ + uint8_t __s0 = __p0; \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vqshlb_n_u8(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshls_n_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vqshls_n_u32(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshls_n_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vqshls_n_u32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshld_n_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vqshld_n_u64(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshld_n_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vqshld_n_u64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlh_n_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vqshlh_n_u16(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshlh_n_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vqshlh_n_u16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlb_n_s8(__p0, __p1) __extension__ ({ \ + int8_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vqshlb_n_s8(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshlb_n_s8(__p0, __p1) __extension__ ({ \ + int8_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vqshlb_n_s8(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshls_n_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqshls_n_s32(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshls_n_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqshls_n_s32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshld_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vqshld_n_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshld_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vqshld_n_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlh_n_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vqshlh_n_s16(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshlh_n_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vqshlh_n_s16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlub_n_s8(__p0, __p1) __extension__ ({ \ + int8_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vqshlub_n_s8(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshlub_n_s8(__p0, __p1) __extension__ ({ \ + int8_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vqshlub_n_s8(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlus_n_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqshlus_n_s32(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshlus_n_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqshlus_n_s32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshlud_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vqshlud_n_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshlud_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vqshlud_n_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshluh_n_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vqshluh_n_s16(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshluh_n_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vqshluh_n_s16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrn_high_n_u32(__p0_204, __p1_204, __p2_204) __extension__ ({ \ + uint16x4_t __s0_204 = __p0_204; \ + uint32x4_t __s1_204 = __p1_204; \ + uint16x8_t __ret_204; \ + __ret_204 = (uint16x8_t)(vcombine_u16((uint16x4_t)(__s0_204), (uint16x4_t)(vqshrn_n_u32(__s1_204, __p2_204)))); \ + __ret_204; \ +}) +#else +#define vqshrn_high_n_u32(__p0_205, __p1_205, __p2_205) __extension__ ({ \ + uint16x4_t __s0_205 = __p0_205; \ + uint32x4_t __s1_205 = __p1_205; \ + uint16x4_t __rev0_205; __rev0_205 = __builtin_shufflevector(__s0_205, __s0_205, 3, 2, 1, 0); \ + uint32x4_t __rev1_205; __rev1_205 = __builtin_shufflevector(__s1_205, __s1_205, 3, 2, 1, 0); \ + uint16x8_t __ret_205; \ + __ret_205 = (uint16x8_t)(__noswap_vcombine_u16((uint16x4_t)(__rev0_205), (uint16x4_t)(__noswap_vqshrn_n_u32(__rev1_205, __p2_205)))); \ + __ret_205 = __builtin_shufflevector(__ret_205, __ret_205, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_205; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrn_high_n_u64(__p0_206, __p1_206, __p2_206) __extension__ ({ \ + uint32x2_t __s0_206 = __p0_206; \ + uint64x2_t __s1_206 = __p1_206; \ + uint32x4_t __ret_206; \ + __ret_206 = (uint32x4_t)(vcombine_u32((uint32x2_t)(__s0_206), (uint32x2_t)(vqshrn_n_u64(__s1_206, __p2_206)))); \ + __ret_206; \ +}) +#else +#define vqshrn_high_n_u64(__p0_207, __p1_207, __p2_207) __extension__ ({ \ + uint32x2_t __s0_207 = __p0_207; \ + uint64x2_t __s1_207 = __p1_207; \ + uint32x2_t __rev0_207; __rev0_207 = __builtin_shufflevector(__s0_207, __s0_207, 1, 0); \ + uint64x2_t __rev1_207; __rev1_207 = __builtin_shufflevector(__s1_207, __s1_207, 1, 0); \ + uint32x4_t __ret_207; \ + __ret_207 = (uint32x4_t)(__noswap_vcombine_u32((uint32x2_t)(__rev0_207), (uint32x2_t)(__noswap_vqshrn_n_u64(__rev1_207, __p2_207)))); \ + __ret_207 = __builtin_shufflevector(__ret_207, __ret_207, 3, 2, 1, 0); \ + __ret_207; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrn_high_n_u16(__p0_208, __p1_208, __p2_208) __extension__ ({ \ + uint8x8_t __s0_208 = __p0_208; \ + uint16x8_t __s1_208 = __p1_208; \ + uint8x16_t __ret_208; \ + __ret_208 = (uint8x16_t)(vcombine_u8((uint8x8_t)(__s0_208), (uint8x8_t)(vqshrn_n_u16(__s1_208, __p2_208)))); \ + __ret_208; \ +}) +#else +#define vqshrn_high_n_u16(__p0_209, __p1_209, __p2_209) __extension__ ({ \ + uint8x8_t __s0_209 = __p0_209; \ + uint16x8_t __s1_209 = __p1_209; \ + uint8x8_t __rev0_209; __rev0_209 = __builtin_shufflevector(__s0_209, __s0_209, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1_209; __rev1_209 = __builtin_shufflevector(__s1_209, __s1_209, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret_209; \ + __ret_209 = (uint8x16_t)(__noswap_vcombine_u8((uint8x8_t)(__rev0_209), (uint8x8_t)(__noswap_vqshrn_n_u16(__rev1_209, __p2_209)))); \ + __ret_209 = __builtin_shufflevector(__ret_209, __ret_209, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_209; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrn_high_n_s32(__p0_210, __p1_210, __p2_210) __extension__ ({ \ + int16x4_t __s0_210 = __p0_210; \ + int32x4_t __s1_210 = __p1_210; \ + int16x8_t __ret_210; \ + __ret_210 = (int16x8_t)(vcombine_s16((int16x4_t)(__s0_210), (int16x4_t)(vqshrn_n_s32(__s1_210, __p2_210)))); \ + __ret_210; \ +}) +#else +#define vqshrn_high_n_s32(__p0_211, __p1_211, __p2_211) __extension__ ({ \ + int16x4_t __s0_211 = __p0_211; \ + int32x4_t __s1_211 = __p1_211; \ + int16x4_t __rev0_211; __rev0_211 = __builtin_shufflevector(__s0_211, __s0_211, 3, 2, 1, 0); \ + int32x4_t __rev1_211; __rev1_211 = __builtin_shufflevector(__s1_211, __s1_211, 3, 2, 1, 0); \ + int16x8_t __ret_211; \ + __ret_211 = (int16x8_t)(__noswap_vcombine_s16((int16x4_t)(__rev0_211), (int16x4_t)(__noswap_vqshrn_n_s32(__rev1_211, __p2_211)))); \ + __ret_211 = __builtin_shufflevector(__ret_211, __ret_211, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_211; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrn_high_n_s64(__p0_212, __p1_212, __p2_212) __extension__ ({ \ + int32x2_t __s0_212 = __p0_212; \ + int64x2_t __s1_212 = __p1_212; \ + int32x4_t __ret_212; \ + __ret_212 = (int32x4_t)(vcombine_s32((int32x2_t)(__s0_212), (int32x2_t)(vqshrn_n_s64(__s1_212, __p2_212)))); \ + __ret_212; \ +}) +#else +#define vqshrn_high_n_s64(__p0_213, __p1_213, __p2_213) __extension__ ({ \ + int32x2_t __s0_213 = __p0_213; \ + int64x2_t __s1_213 = __p1_213; \ + int32x2_t __rev0_213; __rev0_213 = __builtin_shufflevector(__s0_213, __s0_213, 1, 0); \ + int64x2_t __rev1_213; __rev1_213 = __builtin_shufflevector(__s1_213, __s1_213, 1, 0); \ + int32x4_t __ret_213; \ + __ret_213 = (int32x4_t)(__noswap_vcombine_s32((int32x2_t)(__rev0_213), (int32x2_t)(__noswap_vqshrn_n_s64(__rev1_213, __p2_213)))); \ + __ret_213 = __builtin_shufflevector(__ret_213, __ret_213, 3, 2, 1, 0); \ + __ret_213; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrn_high_n_s16(__p0_214, __p1_214, __p2_214) __extension__ ({ \ + int8x8_t __s0_214 = __p0_214; \ + int16x8_t __s1_214 = __p1_214; \ + int8x16_t __ret_214; \ + __ret_214 = (int8x16_t)(vcombine_s8((int8x8_t)(__s0_214), (int8x8_t)(vqshrn_n_s16(__s1_214, __p2_214)))); \ + __ret_214; \ +}) +#else +#define vqshrn_high_n_s16(__p0_215, __p1_215, __p2_215) __extension__ ({ \ + int8x8_t __s0_215 = __p0_215; \ + int16x8_t __s1_215 = __p1_215; \ + int8x8_t __rev0_215; __rev0_215 = __builtin_shufflevector(__s0_215, __s0_215, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1_215; __rev1_215 = __builtin_shufflevector(__s1_215, __s1_215, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret_215; \ + __ret_215 = (int8x16_t)(__noswap_vcombine_s8((int8x8_t)(__rev0_215), (int8x8_t)(__noswap_vqshrn_n_s16(__rev1_215, __p2_215)))); \ + __ret_215 = __builtin_shufflevector(__ret_215, __ret_215, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_215; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrns_n_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vqshrns_n_u32(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshrns_n_u32(__p0, __p1) __extension__ ({ \ + uint32_t __s0 = __p0; \ + uint16_t __ret; \ + __ret = (uint16_t) __builtin_neon_vqshrns_n_u32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrnd_n_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vqshrnd_n_u64(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshrnd_n_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint32_t __ret; \ + __ret = (uint32_t) __builtin_neon_vqshrnd_n_u64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrnh_n_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vqshrnh_n_u16(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshrnh_n_u16(__p0, __p1) __extension__ ({ \ + uint16_t __s0 = __p0; \ + uint8_t __ret; \ + __ret = (uint8_t) __builtin_neon_vqshrnh_n_u16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrns_n_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vqshrns_n_s32(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshrns_n_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vqshrns_n_s32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrnd_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqshrnd_n_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshrnd_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqshrnd_n_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrnh_n_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vqshrnh_n_s16(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshrnh_n_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vqshrnh_n_s16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrun_high_n_s32(__p0_216, __p1_216, __p2_216) __extension__ ({ \ + int16x4_t __s0_216 = __p0_216; \ + int32x4_t __s1_216 = __p1_216; \ + int16x8_t __ret_216; \ + __ret_216 = (int16x8_t)(vcombine_s16((int16x4_t)(__s0_216), (int16x4_t)(vqshrun_n_s32(__s1_216, __p2_216)))); \ + __ret_216; \ +}) +#else +#define vqshrun_high_n_s32(__p0_217, __p1_217, __p2_217) __extension__ ({ \ + int16x4_t __s0_217 = __p0_217; \ + int32x4_t __s1_217 = __p1_217; \ + int16x4_t __rev0_217; __rev0_217 = __builtin_shufflevector(__s0_217, __s0_217, 3, 2, 1, 0); \ + int32x4_t __rev1_217; __rev1_217 = __builtin_shufflevector(__s1_217, __s1_217, 3, 2, 1, 0); \ + int16x8_t __ret_217; \ + __ret_217 = (int16x8_t)(__noswap_vcombine_s16((int16x4_t)(__rev0_217), (int16x4_t)(__noswap_vqshrun_n_s32(__rev1_217, __p2_217)))); \ + __ret_217 = __builtin_shufflevector(__ret_217, __ret_217, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_217; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrun_high_n_s64(__p0_218, __p1_218, __p2_218) __extension__ ({ \ + int32x2_t __s0_218 = __p0_218; \ + int64x2_t __s1_218 = __p1_218; \ + int32x4_t __ret_218; \ + __ret_218 = (int32x4_t)(vcombine_s32((int32x2_t)(__s0_218), (int32x2_t)(vqshrun_n_s64(__s1_218, __p2_218)))); \ + __ret_218; \ +}) +#else +#define vqshrun_high_n_s64(__p0_219, __p1_219, __p2_219) __extension__ ({ \ + int32x2_t __s0_219 = __p0_219; \ + int64x2_t __s1_219 = __p1_219; \ + int32x2_t __rev0_219; __rev0_219 = __builtin_shufflevector(__s0_219, __s0_219, 1, 0); \ + int64x2_t __rev1_219; __rev1_219 = __builtin_shufflevector(__s1_219, __s1_219, 1, 0); \ + int32x4_t __ret_219; \ + __ret_219 = (int32x4_t)(__noswap_vcombine_s32((int32x2_t)(__rev0_219), (int32x2_t)(__noswap_vqshrun_n_s64(__rev1_219, __p2_219)))); \ + __ret_219 = __builtin_shufflevector(__ret_219, __ret_219, 3, 2, 1, 0); \ + __ret_219; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrun_high_n_s16(__p0_220, __p1_220, __p2_220) __extension__ ({ \ + int8x8_t __s0_220 = __p0_220; \ + int16x8_t __s1_220 = __p1_220; \ + int8x16_t __ret_220; \ + __ret_220 = (int8x16_t)(vcombine_s8((int8x8_t)(__s0_220), (int8x8_t)(vqshrun_n_s16(__s1_220, __p2_220)))); \ + __ret_220; \ +}) +#else +#define vqshrun_high_n_s16(__p0_221, __p1_221, __p2_221) __extension__ ({ \ + int8x8_t __s0_221 = __p0_221; \ + int16x8_t __s1_221 = __p1_221; \ + int8x8_t __rev0_221; __rev0_221 = __builtin_shufflevector(__s0_221, __s0_221, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1_221; __rev1_221 = __builtin_shufflevector(__s1_221, __s1_221, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret_221; \ + __ret_221 = (int8x16_t)(__noswap_vcombine_s8((int8x8_t)(__rev0_221), (int8x8_t)(__noswap_vqshrun_n_s16(__rev1_221, __p2_221)))); \ + __ret_221 = __builtin_shufflevector(__ret_221, __ret_221, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_221; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshruns_n_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vqshruns_n_s32(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshruns_n_s32(__p0, __p1) __extension__ ({ \ + int32_t __s0 = __p0; \ + int16_t __ret; \ + __ret = (int16_t) __builtin_neon_vqshruns_n_s32(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrund_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqshrund_n_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshrund_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int32_t __ret; \ + __ret = (int32_t) __builtin_neon_vqshrund_n_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqshrunh_n_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vqshrunh_n_s16(__s0, __p1); \ + __ret; \ +}) +#else +#define vqshrunh_n_s16(__p0, __p1) __extension__ ({ \ + int16_t __s0 = __p0; \ + int8_t __ret; \ + __ret = (int8_t) __builtin_neon_vqshrunh_n_s16(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8_t vqsubb_u8(uint8_t __p0, uint8_t __p1) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vqsubb_u8(__p0, __p1); + return __ret; +} +#else +__ai uint8_t vqsubb_u8(uint8_t __p0, uint8_t __p1) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vqsubb_u8(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vqsubs_u32(uint32_t __p0, uint32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vqsubs_u32(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vqsubs_u32(uint32_t __p0, uint32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vqsubs_u32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vqsubd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vqsubd_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vqsubd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vqsubd_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16_t vqsubh_u16(uint16_t __p0, uint16_t __p1) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vqsubh_u16(__p0, __p1); + return __ret; +} +#else +__ai uint16_t vqsubh_u16(uint16_t __p0, uint16_t __p1) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vqsubh_u16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vqsubb_s8(int8_t __p0, int8_t __p1) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqsubb_s8(__p0, __p1); + return __ret; +} +#else +__ai int8_t vqsubb_s8(int8_t __p0, int8_t __p1) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vqsubb_s8(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqsubs_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqsubs_s32(__p0, __p1); + return __ret; +} +#else +__ai int32_t vqsubs_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqsubs_s32(__p0, __p1); + return __ret; +} +__ai int32_t __noswap_vqsubs_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vqsubs_s32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vqsubd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqsubd_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vqsubd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vqsubd_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vqsubh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqsubh_s16(__p0, __p1); + return __ret; +} +#else +__ai int16_t vqsubh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqsubh_s16(__p0, __p1); + return __ret; +} +__ai int16_t __noswap_vqsubh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vqsubh_s16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vqtbl1_p8(poly8x16_t __p0, uint8x8_t __p1) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbl1_v((int8x16_t)__p0, (int8x8_t)__p1, 4); + return __ret; +} +#else +__ai poly8x8_t vqtbl1_p8(poly8x16_t __p0, uint8x8_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbl1_v((int8x16_t)__rev0, (int8x8_t)__rev1, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vqtbl1q_p8(poly8x16_t __p0, uint8x16_t __p1) { + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbl1q_v((int8x16_t)__p0, (int8x16_t)__p1, 36); + return __ret; +} +#else +__ai poly8x16_t vqtbl1q_p8(poly8x16_t __p0, uint8x16_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbl1q_v((int8x16_t)__rev0, (int8x16_t)__rev1, 36); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vqtbl1q_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbl1q_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vqtbl1q_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbl1q_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqtbl1q_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbl1q_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vqtbl1q_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbl1q_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vqtbl1_u8(uint8x16_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbl1_v((int8x16_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vqtbl1_u8(uint8x16_t __p0, uint8x8_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbl1_v((int8x16_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqtbl1_s8(int8x16_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbl1_v((int8x16_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vqtbl1_s8(int8x16_t __p0, int8x8_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbl1_v((int8x16_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vqtbl2_p8(poly8x16x2_t __p0, uint8x8_t __p1) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbl2_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x8_t)__p1, 4); + return __ret; +} +#else +__ai poly8x8_t vqtbl2_p8(poly8x16x2_t __p0, uint8x8_t __p1) { + poly8x16x2_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbl2_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x8_t)__rev1, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vqtbl2q_p8(poly8x16x2_t __p0, uint8x16_t __p1) { + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbl2q_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p1, 36); + return __ret; +} +#else +__ai poly8x16_t vqtbl2q_p8(poly8x16x2_t __p0, uint8x16_t __p1) { + poly8x16x2_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbl2q_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev1, 36); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vqtbl2q_u8(uint8x16x2_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbl2q_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vqtbl2q_u8(uint8x16x2_t __p0, uint8x16_t __p1) { + uint8x16x2_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbl2q_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqtbl2q_s8(int8x16x2_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbl2q_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vqtbl2q_s8(int8x16x2_t __p0, int8x16_t __p1) { + int8x16x2_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbl2q_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vqtbl2_u8(uint8x16x2_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbl2_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vqtbl2_u8(uint8x16x2_t __p0, uint8x8_t __p1) { + uint8x16x2_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbl2_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqtbl2_s8(int8x16x2_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbl2_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vqtbl2_s8(int8x16x2_t __p0, int8x8_t __p1) { + int8x16x2_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbl2_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vqtbl3_p8(poly8x16x3_t __p0, uint8x8_t __p1) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbl3_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x8_t)__p1, 4); + return __ret; +} +#else +__ai poly8x8_t vqtbl3_p8(poly8x16x3_t __p0, uint8x8_t __p1) { + poly8x16x3_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbl3_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x8_t)__rev1, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vqtbl3q_p8(poly8x16x3_t __p0, uint8x16_t __p1) { + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbl3q_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x16_t)__p1, 36); + return __ret; +} +#else +__ai poly8x16_t vqtbl3q_p8(poly8x16x3_t __p0, uint8x16_t __p1) { + poly8x16x3_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbl3q_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x16_t)__rev1, 36); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vqtbl3q_u8(uint8x16x3_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbl3q_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vqtbl3q_u8(uint8x16x3_t __p0, uint8x16_t __p1) { + uint8x16x3_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbl3q_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqtbl3q_s8(int8x16x3_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbl3q_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vqtbl3q_s8(int8x16x3_t __p0, int8x16_t __p1) { + int8x16x3_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbl3q_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vqtbl3_u8(uint8x16x3_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbl3_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vqtbl3_u8(uint8x16x3_t __p0, uint8x8_t __p1) { + uint8x16x3_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbl3_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqtbl3_s8(int8x16x3_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbl3_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vqtbl3_s8(int8x16x3_t __p0, int8x8_t __p1) { + int8x16x3_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbl3_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vqtbl4_p8(poly8x16x4_t __p0, uint8x8_t __p1) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbl4_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x16_t)__p0.val[3], (int8x8_t)__p1, 4); + return __ret; +} +#else +__ai poly8x8_t vqtbl4_p8(poly8x16x4_t __p0, uint8x8_t __p1) { + poly8x16x4_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[3] = __builtin_shufflevector(__p0.val[3], __p0.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbl4_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x16_t)__rev0.val[3], (int8x8_t)__rev1, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vqtbl4q_p8(poly8x16x4_t __p0, uint8x16_t __p1) { + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbl4q_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x16_t)__p0.val[3], (int8x16_t)__p1, 36); + return __ret; +} +#else +__ai poly8x16_t vqtbl4q_p8(poly8x16x4_t __p0, uint8x16_t __p1) { + poly8x16x4_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[3] = __builtin_shufflevector(__p0.val[3], __p0.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbl4q_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x16_t)__rev0.val[3], (int8x16_t)__rev1, 36); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vqtbl4q_u8(uint8x16x4_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbl4q_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x16_t)__p0.val[3], (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vqtbl4q_u8(uint8x16x4_t __p0, uint8x16_t __p1) { + uint8x16x4_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[3] = __builtin_shufflevector(__p0.val[3], __p0.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbl4q_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x16_t)__rev0.val[3], (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqtbl4q_s8(int8x16x4_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbl4q_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x16_t)__p0.val[3], (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vqtbl4q_s8(int8x16x4_t __p0, int8x16_t __p1) { + int8x16x4_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[3] = __builtin_shufflevector(__p0.val[3], __p0.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbl4q_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x16_t)__rev0.val[3], (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vqtbl4_u8(uint8x16x4_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbl4_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x16_t)__p0.val[3], (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vqtbl4_u8(uint8x16x4_t __p0, uint8x8_t __p1) { + uint8x16x4_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[3] = __builtin_shufflevector(__p0.val[3], __p0.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbl4_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x16_t)__rev0.val[3], (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqtbl4_s8(int8x16x4_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbl4_v((int8x16_t)__p0.val[0], (int8x16_t)__p0.val[1], (int8x16_t)__p0.val[2], (int8x16_t)__p0.val[3], (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vqtbl4_s8(int8x16x4_t __p0, int8x8_t __p1) { + int8x16x4_t __rev0; + __rev0.val[0] = __builtin_shufflevector(__p0.val[0], __p0.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[1] = __builtin_shufflevector(__p0.val[1], __p0.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[2] = __builtin_shufflevector(__p0.val[2], __p0.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev0.val[3] = __builtin_shufflevector(__p0.val[3], __p0.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbl4_v((int8x16_t)__rev0.val[0], (int8x16_t)__rev0.val[1], (int8x16_t)__rev0.val[2], (int8x16_t)__rev0.val[3], (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vqtbx1_p8(poly8x8_t __p0, poly8x16_t __p1, uint8x8_t __p2) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbx1_v((int8x8_t)__p0, (int8x16_t)__p1, (int8x8_t)__p2, 4); + return __ret; +} +#else +__ai poly8x8_t vqtbx1_p8(poly8x8_t __p0, poly8x16_t __p1, uint8x8_t __p2) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbx1_v((int8x8_t)__rev0, (int8x16_t)__rev1, (int8x8_t)__rev2, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vqtbx1q_p8(poly8x16_t __p0, poly8x16_t __p1, uint8x16_t __p2) { + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbx1q_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 36); + return __ret; +} +#else +__ai poly8x16_t vqtbx1q_p8(poly8x16_t __p0, poly8x16_t __p1, uint8x16_t __p2) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbx1q_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 36); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vqtbx1q_u8(uint8x16_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbx1q_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 48); + return __ret; +} +#else +__ai uint8x16_t vqtbx1q_u8(uint8x16_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbx1q_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqtbx1q_s8(int8x16_t __p0, int8x16_t __p1, int8x16_t __p2) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbx1q_v((int8x16_t)__p0, (int8x16_t)__p1, (int8x16_t)__p2, 32); + return __ret; +} +#else +__ai int8x16_t vqtbx1q_s8(int8x16_t __p0, int8x16_t __p1, int8x16_t __p2) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbx1q_v((int8x16_t)__rev0, (int8x16_t)__rev1, (int8x16_t)__rev2, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vqtbx1_u8(uint8x8_t __p0, uint8x16_t __p1, uint8x8_t __p2) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbx1_v((int8x8_t)__p0, (int8x16_t)__p1, (int8x8_t)__p2, 16); + return __ret; +} +#else +__ai uint8x8_t vqtbx1_u8(uint8x8_t __p0, uint8x16_t __p1, uint8x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbx1_v((int8x8_t)__rev0, (int8x16_t)__rev1, (int8x8_t)__rev2, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqtbx1_s8(int8x8_t __p0, int8x16_t __p1, int8x8_t __p2) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbx1_v((int8x8_t)__p0, (int8x16_t)__p1, (int8x8_t)__p2, 0); + return __ret; +} +#else +__ai int8x8_t vqtbx1_s8(int8x8_t __p0, int8x16_t __p1, int8x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbx1_v((int8x8_t)__rev0, (int8x16_t)__rev1, (int8x8_t)__rev2, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vqtbx2_p8(poly8x8_t __p0, poly8x16x2_t __p1, uint8x8_t __p2) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbx2_v((int8x8_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x8_t)__p2, 4); + return __ret; +} +#else +__ai poly8x8_t vqtbx2_p8(poly8x8_t __p0, poly8x16x2_t __p1, uint8x8_t __p2) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16x2_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbx2_v((int8x8_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x8_t)__rev2, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vqtbx2q_p8(poly8x16_t __p0, poly8x16x2_t __p1, uint8x16_t __p2) { + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbx2q_v((int8x16_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p2, 36); + return __ret; +} +#else +__ai poly8x16_t vqtbx2q_p8(poly8x16_t __p0, poly8x16x2_t __p1, uint8x16_t __p2) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16x2_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbx2q_v((int8x16_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev2, 36); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vqtbx2q_u8(uint8x16_t __p0, uint8x16x2_t __p1, uint8x16_t __p2) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbx2q_v((int8x16_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p2, 48); + return __ret; +} +#else +__ai uint8x16_t vqtbx2q_u8(uint8x16_t __p0, uint8x16x2_t __p1, uint8x16_t __p2) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16x2_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbx2q_v((int8x16_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev2, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqtbx2q_s8(int8x16_t __p0, int8x16x2_t __p1, int8x16_t __p2) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbx2q_v((int8x16_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p2, 32); + return __ret; +} +#else +__ai int8x16_t vqtbx2q_s8(int8x16_t __p0, int8x16x2_t __p1, int8x16_t __p2) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16x2_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbx2q_v((int8x16_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev2, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vqtbx2_u8(uint8x8_t __p0, uint8x16x2_t __p1, uint8x8_t __p2) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbx2_v((int8x8_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x8_t)__p2, 16); + return __ret; +} +#else +__ai uint8x8_t vqtbx2_u8(uint8x8_t __p0, uint8x16x2_t __p1, uint8x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16x2_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbx2_v((int8x8_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x8_t)__rev2, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqtbx2_s8(int8x8_t __p0, int8x16x2_t __p1, int8x8_t __p2) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbx2_v((int8x8_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x8_t)__p2, 0); + return __ret; +} +#else +__ai int8x8_t vqtbx2_s8(int8x8_t __p0, int8x16x2_t __p1, int8x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16x2_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbx2_v((int8x8_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x8_t)__rev2, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vqtbx3_p8(poly8x8_t __p0, poly8x16x3_t __p1, uint8x8_t __p2) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbx3_v((int8x8_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x8_t)__p2, 4); + return __ret; +} +#else +__ai poly8x8_t vqtbx3_p8(poly8x8_t __p0, poly8x16x3_t __p1, uint8x8_t __p2) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16x3_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbx3_v((int8x8_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x8_t)__rev2, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vqtbx3q_p8(poly8x16_t __p0, poly8x16x3_t __p1, uint8x16_t __p2) { + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbx3q_v((int8x16_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x16_t)__p2, 36); + return __ret; +} +#else +__ai poly8x16_t vqtbx3q_p8(poly8x16_t __p0, poly8x16x3_t __p1, uint8x16_t __p2) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16x3_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbx3q_v((int8x16_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev2, 36); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vqtbx3q_u8(uint8x16_t __p0, uint8x16x3_t __p1, uint8x16_t __p2) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbx3q_v((int8x16_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x16_t)__p2, 48); + return __ret; +} +#else +__ai uint8x16_t vqtbx3q_u8(uint8x16_t __p0, uint8x16x3_t __p1, uint8x16_t __p2) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16x3_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbx3q_v((int8x16_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev2, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqtbx3q_s8(int8x16_t __p0, int8x16x3_t __p1, int8x16_t __p2) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbx3q_v((int8x16_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x16_t)__p2, 32); + return __ret; +} +#else +__ai int8x16_t vqtbx3q_s8(int8x16_t __p0, int8x16x3_t __p1, int8x16_t __p2) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16x3_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbx3q_v((int8x16_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev2, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vqtbx3_u8(uint8x8_t __p0, uint8x16x3_t __p1, uint8x8_t __p2) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbx3_v((int8x8_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x8_t)__p2, 16); + return __ret; +} +#else +__ai uint8x8_t vqtbx3_u8(uint8x8_t __p0, uint8x16x3_t __p1, uint8x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16x3_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbx3_v((int8x8_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x8_t)__rev2, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqtbx3_s8(int8x8_t __p0, int8x16x3_t __p1, int8x8_t __p2) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbx3_v((int8x8_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x8_t)__p2, 0); + return __ret; +} +#else +__ai int8x8_t vqtbx3_s8(int8x8_t __p0, int8x16x3_t __p1, int8x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16x3_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbx3_v((int8x8_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x8_t)__rev2, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vqtbx4_p8(poly8x8_t __p0, poly8x16x4_t __p1, uint8x8_t __p2) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbx4_v((int8x8_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x16_t)__p1.val[3], (int8x8_t)__p2, 4); + return __ret; +} +#else +__ai poly8x8_t vqtbx4_p8(poly8x8_t __p0, poly8x16x4_t __p1, uint8x8_t __p2) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16x4_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[3] = __builtin_shufflevector(__p1.val[3], __p1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vqtbx4_v((int8x8_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], (int8x8_t)__rev2, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vqtbx4q_p8(poly8x16_t __p0, poly8x16x4_t __p1, uint8x16_t __p2) { + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbx4q_v((int8x16_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x16_t)__p1.val[3], (int8x16_t)__p2, 36); + return __ret; +} +#else +__ai poly8x16_t vqtbx4q_p8(poly8x16_t __p0, poly8x16x4_t __p1, uint8x16_t __p2) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16x4_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[3] = __builtin_shufflevector(__p1.val[3], __p1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vqtbx4q_v((int8x16_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], (int8x16_t)__rev2, 36); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vqtbx4q_u8(uint8x16_t __p0, uint8x16x4_t __p1, uint8x16_t __p2) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbx4q_v((int8x16_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x16_t)__p1.val[3], (int8x16_t)__p2, 48); + return __ret; +} +#else +__ai uint8x16_t vqtbx4q_u8(uint8x16_t __p0, uint8x16x4_t __p1, uint8x16_t __p2) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16x4_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[3] = __builtin_shufflevector(__p1.val[3], __p1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vqtbx4q_v((int8x16_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], (int8x16_t)__rev2, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vqtbx4q_s8(int8x16_t __p0, int8x16x4_t __p1, int8x16_t __p2) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbx4q_v((int8x16_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x16_t)__p1.val[3], (int8x16_t)__p2, 32); + return __ret; +} +#else +__ai int8x16_t vqtbx4q_s8(int8x16_t __p0, int8x16x4_t __p1, int8x16_t __p2) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16x4_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[3] = __builtin_shufflevector(__p1.val[3], __p1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vqtbx4q_v((int8x16_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], (int8x16_t)__rev2, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vqtbx4_u8(uint8x8_t __p0, uint8x16x4_t __p1, uint8x8_t __p2) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbx4_v((int8x8_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x16_t)__p1.val[3], (int8x8_t)__p2, 16); + return __ret; +} +#else +__ai uint8x8_t vqtbx4_u8(uint8x8_t __p0, uint8x16x4_t __p1, uint8x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16x4_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[3] = __builtin_shufflevector(__p1.val[3], __p1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vqtbx4_v((int8x8_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], (int8x8_t)__rev2, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vqtbx4_s8(int8x8_t __p0, int8x16x4_t __p1, int8x8_t __p2) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbx4_v((int8x8_t)__p0, (int8x16_t)__p1.val[0], (int8x16_t)__p1.val[1], (int8x16_t)__p1.val[2], (int8x16_t)__p1.val[3], (int8x8_t)__p2, 0); + return __ret; +} +#else +__ai int8x8_t vqtbx4_s8(int8x8_t __p0, int8x16x4_t __p1, int8x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16x4_t __rev1; + __rev1.val[0] = __builtin_shufflevector(__p1.val[0], __p1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[1] = __builtin_shufflevector(__p1.val[1], __p1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[2] = __builtin_shufflevector(__p1.val[2], __p1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + __rev1.val[3] = __builtin_shufflevector(__p1.val[3], __p1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vqtbx4_v((int8x8_t)__rev0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], (int8x8_t)__rev2, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vraddhn_high_u32(uint16x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint16x8_t __ret; + __ret = vcombine_u16(__p0, vraddhn_u32(__p1, __p2)); + return __ret; +} +#else +__ai uint16x8_t vraddhn_high_u32(uint16x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vcombine_u16(__rev0, __noswap_vraddhn_u32(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vraddhn_high_u64(uint32x2_t __p0, uint64x2_t __p1, uint64x2_t __p2) { + uint32x4_t __ret; + __ret = vcombine_u32(__p0, vraddhn_u64(__p1, __p2)); + return __ret; +} +#else +__ai uint32x4_t vraddhn_high_u64(uint32x2_t __p0, uint64x2_t __p1, uint64x2_t __p2) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vcombine_u32(__rev0, __noswap_vraddhn_u64(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vraddhn_high_u16(uint8x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint8x16_t __ret; + __ret = vcombine_u8(__p0, vraddhn_u16(__p1, __p2)); + return __ret; +} +#else +__ai uint8x16_t vraddhn_high_u16(uint8x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __noswap_vcombine_u8(__rev0, __noswap_vraddhn_u16(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vraddhn_high_s32(int16x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int16x8_t __ret; + __ret = vcombine_s16(__p0, vraddhn_s32(__p1, __p2)); + return __ret; +} +#else +__ai int16x8_t vraddhn_high_s32(int16x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vcombine_s16(__rev0, __noswap_vraddhn_s32(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vraddhn_high_s64(int32x2_t __p0, int64x2_t __p1, int64x2_t __p2) { + int32x4_t __ret; + __ret = vcombine_s32(__p0, vraddhn_s64(__p1, __p2)); + return __ret; +} +#else +__ai int32x4_t vraddhn_high_s64(int32x2_t __p0, int64x2_t __p1, int64x2_t __p2) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vcombine_s32(__rev0, __noswap_vraddhn_s64(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vraddhn_high_s16(int8x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int8x16_t __ret; + __ret = vcombine_s8(__p0, vraddhn_s16(__p1, __p2)); + return __ret; +} +#else +__ai int8x16_t vraddhn_high_s16(int8x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __noswap_vcombine_s8(__rev0, __noswap_vraddhn_s16(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vrbit_p8(poly8x8_t __p0) { + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vrbit_v((int8x8_t)__p0, 4); + return __ret; +} +#else +__ai poly8x8_t vrbit_p8(poly8x8_t __p0) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = (poly8x8_t) __builtin_neon_vrbit_v((int8x8_t)__rev0, 4); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vrbitq_p8(poly8x16_t __p0) { + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vrbitq_v((int8x16_t)__p0, 36); + return __ret; +} +#else +__ai poly8x16_t vrbitq_p8(poly8x16_t __p0) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = (poly8x16_t) __builtin_neon_vrbitq_v((int8x16_t)__rev0, 36); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vrbitq_u8(uint8x16_t __p0) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vrbitq_v((int8x16_t)__p0, 48); + return __ret; +} +#else +__ai uint8x16_t vrbitq_u8(uint8x16_t __p0) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vrbitq_v((int8x16_t)__rev0, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vrbitq_s8(int8x16_t __p0) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vrbitq_v((int8x16_t)__p0, 32); + return __ret; +} +#else +__ai int8x16_t vrbitq_s8(int8x16_t __p0) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vrbitq_v((int8x16_t)__rev0, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vrbit_u8(uint8x8_t __p0) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vrbit_v((int8x8_t)__p0, 16); + return __ret; +} +#else +__ai uint8x8_t vrbit_u8(uint8x8_t __p0) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vrbit_v((int8x8_t)__rev0, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vrbit_s8(int8x8_t __p0) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vrbit_v((int8x8_t)__p0, 0); + return __ret; +} +#else +__ai int8x8_t vrbit_s8(int8x8_t __p0) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vrbit_v((int8x8_t)__rev0, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vrecpeq_f64(float64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrecpeq_v((int8x16_t)__p0, 42); + return __ret; +} +#else +__ai float64x2_t vrecpeq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrecpeq_v((int8x16_t)__rev0, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vrecpe_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrecpe_v((int8x8_t)__p0, 10); + return __ret; +} +#else +__ai float64x1_t vrecpe_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrecpe_v((int8x8_t)__p0, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vrecped_f64(float64_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vrecped_f64(__p0); + return __ret; +} +#else +__ai float64_t vrecped_f64(float64_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vrecped_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vrecpes_f32(float32_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vrecpes_f32(__p0); + return __ret; +} +#else +__ai float32_t vrecpes_f32(float32_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vrecpes_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vrecpsq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrecpsq_v((int8x16_t)__p0, (int8x16_t)__p1, 42); + return __ret; +} +#else +__ai float64x2_t vrecpsq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrecpsq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vrecps_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrecps_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#else +__ai float64x1_t vrecps_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrecps_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vrecpsd_f64(float64_t __p0, float64_t __p1) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vrecpsd_f64(__p0, __p1); + return __ret; +} +#else +__ai float64_t vrecpsd_f64(float64_t __p0, float64_t __p1) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vrecpsd_f64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vrecpss_f32(float32_t __p0, float32_t __p1) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vrecpss_f32(__p0, __p1); + return __ret; +} +#else +__ai float32_t vrecpss_f32(float32_t __p0, float32_t __p1) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vrecpss_f32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vrecpxd_f64(float64_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vrecpxd_f64(__p0); + return __ret; +} +#else +__ai float64_t vrecpxd_f64(float64_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vrecpxd_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vrecpxs_f32(float32_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vrecpxs_f32(__p0); + return __ret; +} +#else +__ai float32_t vrecpxs_f32(float32_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vrecpxs_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vrshld_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vrshld_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vrshld_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vrshld_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vrshld_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vrshld_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vrshld_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vrshld_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrd_n_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vrshrd_n_u64(__s0, __p1); \ + __ret; \ +}) +#else +#define vrshrd_n_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vrshrd_n_u64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrd_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vrshrd_n_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vrshrd_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vrshrd_n_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrn_high_n_u32(__p0_222, __p1_222, __p2_222) __extension__ ({ \ + uint16x4_t __s0_222 = __p0_222; \ + uint32x4_t __s1_222 = __p1_222; \ + uint16x8_t __ret_222; \ + __ret_222 = (uint16x8_t)(vcombine_u16((uint16x4_t)(__s0_222), (uint16x4_t)(vrshrn_n_u32(__s1_222, __p2_222)))); \ + __ret_222; \ +}) +#else +#define vrshrn_high_n_u32(__p0_223, __p1_223, __p2_223) __extension__ ({ \ + uint16x4_t __s0_223 = __p0_223; \ + uint32x4_t __s1_223 = __p1_223; \ + uint16x4_t __rev0_223; __rev0_223 = __builtin_shufflevector(__s0_223, __s0_223, 3, 2, 1, 0); \ + uint32x4_t __rev1_223; __rev1_223 = __builtin_shufflevector(__s1_223, __s1_223, 3, 2, 1, 0); \ + uint16x8_t __ret_223; \ + __ret_223 = (uint16x8_t)(__noswap_vcombine_u16((uint16x4_t)(__rev0_223), (uint16x4_t)(__noswap_vrshrn_n_u32(__rev1_223, __p2_223)))); \ + __ret_223 = __builtin_shufflevector(__ret_223, __ret_223, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_223; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrn_high_n_u64(__p0_224, __p1_224, __p2_224) __extension__ ({ \ + uint32x2_t __s0_224 = __p0_224; \ + uint64x2_t __s1_224 = __p1_224; \ + uint32x4_t __ret_224; \ + __ret_224 = (uint32x4_t)(vcombine_u32((uint32x2_t)(__s0_224), (uint32x2_t)(vrshrn_n_u64(__s1_224, __p2_224)))); \ + __ret_224; \ +}) +#else +#define vrshrn_high_n_u64(__p0_225, __p1_225, __p2_225) __extension__ ({ \ + uint32x2_t __s0_225 = __p0_225; \ + uint64x2_t __s1_225 = __p1_225; \ + uint32x2_t __rev0_225; __rev0_225 = __builtin_shufflevector(__s0_225, __s0_225, 1, 0); \ + uint64x2_t __rev1_225; __rev1_225 = __builtin_shufflevector(__s1_225, __s1_225, 1, 0); \ + uint32x4_t __ret_225; \ + __ret_225 = (uint32x4_t)(__noswap_vcombine_u32((uint32x2_t)(__rev0_225), (uint32x2_t)(__noswap_vrshrn_n_u64(__rev1_225, __p2_225)))); \ + __ret_225 = __builtin_shufflevector(__ret_225, __ret_225, 3, 2, 1, 0); \ + __ret_225; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrn_high_n_u16(__p0_226, __p1_226, __p2_226) __extension__ ({ \ + uint8x8_t __s0_226 = __p0_226; \ + uint16x8_t __s1_226 = __p1_226; \ + uint8x16_t __ret_226; \ + __ret_226 = (uint8x16_t)(vcombine_u8((uint8x8_t)(__s0_226), (uint8x8_t)(vrshrn_n_u16(__s1_226, __p2_226)))); \ + __ret_226; \ +}) +#else +#define vrshrn_high_n_u16(__p0_227, __p1_227, __p2_227) __extension__ ({ \ + uint8x8_t __s0_227 = __p0_227; \ + uint16x8_t __s1_227 = __p1_227; \ + uint8x8_t __rev0_227; __rev0_227 = __builtin_shufflevector(__s0_227, __s0_227, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1_227; __rev1_227 = __builtin_shufflevector(__s1_227, __s1_227, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret_227; \ + __ret_227 = (uint8x16_t)(__noswap_vcombine_u8((uint8x8_t)(__rev0_227), (uint8x8_t)(__noswap_vrshrn_n_u16(__rev1_227, __p2_227)))); \ + __ret_227 = __builtin_shufflevector(__ret_227, __ret_227, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_227; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrn_high_n_s32(__p0_228, __p1_228, __p2_228) __extension__ ({ \ + int16x4_t __s0_228 = __p0_228; \ + int32x4_t __s1_228 = __p1_228; \ + int16x8_t __ret_228; \ + __ret_228 = (int16x8_t)(vcombine_s16((int16x4_t)(__s0_228), (int16x4_t)(vrshrn_n_s32(__s1_228, __p2_228)))); \ + __ret_228; \ +}) +#else +#define vrshrn_high_n_s32(__p0_229, __p1_229, __p2_229) __extension__ ({ \ + int16x4_t __s0_229 = __p0_229; \ + int32x4_t __s1_229 = __p1_229; \ + int16x4_t __rev0_229; __rev0_229 = __builtin_shufflevector(__s0_229, __s0_229, 3, 2, 1, 0); \ + int32x4_t __rev1_229; __rev1_229 = __builtin_shufflevector(__s1_229, __s1_229, 3, 2, 1, 0); \ + int16x8_t __ret_229; \ + __ret_229 = (int16x8_t)(__noswap_vcombine_s16((int16x4_t)(__rev0_229), (int16x4_t)(__noswap_vrshrn_n_s32(__rev1_229, __p2_229)))); \ + __ret_229 = __builtin_shufflevector(__ret_229, __ret_229, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_229; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrn_high_n_s64(__p0_230, __p1_230, __p2_230) __extension__ ({ \ + int32x2_t __s0_230 = __p0_230; \ + int64x2_t __s1_230 = __p1_230; \ + int32x4_t __ret_230; \ + __ret_230 = (int32x4_t)(vcombine_s32((int32x2_t)(__s0_230), (int32x2_t)(vrshrn_n_s64(__s1_230, __p2_230)))); \ + __ret_230; \ +}) +#else +#define vrshrn_high_n_s64(__p0_231, __p1_231, __p2_231) __extension__ ({ \ + int32x2_t __s0_231 = __p0_231; \ + int64x2_t __s1_231 = __p1_231; \ + int32x2_t __rev0_231; __rev0_231 = __builtin_shufflevector(__s0_231, __s0_231, 1, 0); \ + int64x2_t __rev1_231; __rev1_231 = __builtin_shufflevector(__s1_231, __s1_231, 1, 0); \ + int32x4_t __ret_231; \ + __ret_231 = (int32x4_t)(__noswap_vcombine_s32((int32x2_t)(__rev0_231), (int32x2_t)(__noswap_vrshrn_n_s64(__rev1_231, __p2_231)))); \ + __ret_231 = __builtin_shufflevector(__ret_231, __ret_231, 3, 2, 1, 0); \ + __ret_231; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrshrn_high_n_s16(__p0_232, __p1_232, __p2_232) __extension__ ({ \ + int8x8_t __s0_232 = __p0_232; \ + int16x8_t __s1_232 = __p1_232; \ + int8x16_t __ret_232; \ + __ret_232 = (int8x16_t)(vcombine_s8((int8x8_t)(__s0_232), (int8x8_t)(vrshrn_n_s16(__s1_232, __p2_232)))); \ + __ret_232; \ +}) +#else +#define vrshrn_high_n_s16(__p0_233, __p1_233, __p2_233) __extension__ ({ \ + int8x8_t __s0_233 = __p0_233; \ + int16x8_t __s1_233 = __p1_233; \ + int8x8_t __rev0_233; __rev0_233 = __builtin_shufflevector(__s0_233, __s0_233, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1_233; __rev1_233 = __builtin_shufflevector(__s1_233, __s1_233, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret_233; \ + __ret_233 = (int8x16_t)(__noswap_vcombine_s8((int8x8_t)(__rev0_233), (int8x8_t)(__noswap_vrshrn_n_s16(__rev1_233, __p2_233)))); \ + __ret_233 = __builtin_shufflevector(__ret_233, __ret_233, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_233; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vrsqrteq_f64(float64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrsqrteq_v((int8x16_t)__p0, 42); + return __ret; +} +#else +__ai float64x2_t vrsqrteq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrsqrteq_v((int8x16_t)__rev0, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vrsqrte_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrsqrte_v((int8x8_t)__p0, 10); + return __ret; +} +#else +__ai float64x1_t vrsqrte_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrsqrte_v((int8x8_t)__p0, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vrsqrted_f64(float64_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vrsqrted_f64(__p0); + return __ret; +} +#else +__ai float64_t vrsqrted_f64(float64_t __p0) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vrsqrted_f64(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vrsqrtes_f32(float32_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vrsqrtes_f32(__p0); + return __ret; +} +#else +__ai float32_t vrsqrtes_f32(float32_t __p0) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vrsqrtes_f32(__p0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vrsqrtsq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrsqrtsq_v((int8x16_t)__p0, (int8x16_t)__p1, 42); + return __ret; +} +#else +__ai float64x2_t vrsqrtsq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vrsqrtsq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vrsqrts_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrsqrts_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#else +__ai float64x1_t vrsqrts_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vrsqrts_v((int8x8_t)__p0, (int8x8_t)__p1, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64_t vrsqrtsd_f64(float64_t __p0, float64_t __p1) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vrsqrtsd_f64(__p0, __p1); + return __ret; +} +#else +__ai float64_t vrsqrtsd_f64(float64_t __p0, float64_t __p1) { + float64_t __ret; + __ret = (float64_t) __builtin_neon_vrsqrtsd_f64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32_t vrsqrtss_f32(float32_t __p0, float32_t __p1) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vrsqrtss_f32(__p0, __p1); + return __ret; +} +#else +__ai float32_t vrsqrtss_f32(float32_t __p0, float32_t __p1) { + float32_t __ret; + __ret = (float32_t) __builtin_neon_vrsqrtss_f32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsrad_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __s1 = __p1; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vrsrad_n_u64(__s0, __s1, __p2); \ + __ret; \ +}) +#else +#define vrsrad_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __s1 = __p1; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vrsrad_n_u64(__s0, __s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vrsrad_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __s1 = __p1; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vrsrad_n_s64(__s0, __s1, __p2); \ + __ret; \ +}) +#else +#define vrsrad_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __s1 = __p1; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vrsrad_n_s64(__s0, __s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vrsubhn_high_u32(uint16x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint16x8_t __ret; + __ret = vcombine_u16(__p0, vrsubhn_u32(__p1, __p2)); + return __ret; +} +#else +__ai uint16x8_t vrsubhn_high_u32(uint16x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vcombine_u16(__rev0, __noswap_vrsubhn_u32(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vrsubhn_high_u64(uint32x2_t __p0, uint64x2_t __p1, uint64x2_t __p2) { + uint32x4_t __ret; + __ret = vcombine_u32(__p0, vrsubhn_u64(__p1, __p2)); + return __ret; +} +#else +__ai uint32x4_t vrsubhn_high_u64(uint32x2_t __p0, uint64x2_t __p1, uint64x2_t __p2) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vcombine_u32(__rev0, __noswap_vrsubhn_u64(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vrsubhn_high_u16(uint8x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint8x16_t __ret; + __ret = vcombine_u8(__p0, vrsubhn_u16(__p1, __p2)); + return __ret; +} +#else +__ai uint8x16_t vrsubhn_high_u16(uint8x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __noswap_vcombine_u8(__rev0, __noswap_vrsubhn_u16(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vrsubhn_high_s32(int16x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int16x8_t __ret; + __ret = vcombine_s16(__p0, vrsubhn_s32(__p1, __p2)); + return __ret; +} +#else +__ai int16x8_t vrsubhn_high_s32(int16x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vcombine_s16(__rev0, __noswap_vrsubhn_s32(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vrsubhn_high_s64(int32x2_t __p0, int64x2_t __p1, int64x2_t __p2) { + int32x4_t __ret; + __ret = vcombine_s32(__p0, vrsubhn_s64(__p1, __p2)); + return __ret; +} +#else +__ai int32x4_t vrsubhn_high_s64(int32x2_t __p0, int64x2_t __p1, int64x2_t __p2) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vcombine_s32(__rev0, __noswap_vrsubhn_s64(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vrsubhn_high_s16(int8x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int8x16_t __ret; + __ret = vcombine_s8(__p0, vrsubhn_s16(__p1, __p2)); + return __ret; +} +#else +__ai int8x16_t vrsubhn_high_s16(int8x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __noswap_vcombine_s8(__rev0, __noswap_vrsubhn_s16(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vset_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64_t __s0 = __p0; \ + poly64x1_t __s1 = __p1; \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vset_lane_i64(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vset_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64_t __s0 = __p0; \ + poly64x1_t __s1 = __p1; \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vset_lane_i64(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#define __noswap_vset_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64_t __s0 = __p0; \ + poly64x1_t __s1 = __p1; \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vset_lane_i64(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsetq_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64_t __s0 = __p0; \ + poly64x2_t __s1 = __p1; \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vsetq_lane_i64(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vsetq_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64_t __s0 = __p0; \ + poly64x2_t __s1 = __p1; \ + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vsetq_lane_i64(__s0, (int8x16_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vsetq_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64_t __s0 = __p0; \ + poly64x2_t __s1 = __p1; \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vsetq_lane_i64(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsetq_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vsetq_lane_f64(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vsetq_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vsetq_lane_f64(__s0, (int8x16_t)__rev1, __p2); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#define __noswap_vsetq_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64_t __s0 = __p0; \ + float64x2_t __s1 = __p1; \ + float64x2_t __ret; \ + __ret = (float64x2_t) __builtin_neon_vsetq_lane_f64(__s0, (int8x16_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vset_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vset_lane_f64(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#else +#define vset_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vset_lane_f64(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#define __noswap_vset_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64_t __s0 = __p0; \ + float64x1_t __s1 = __p1; \ + float64x1_t __ret; \ + __ret = (float64x1_t) __builtin_neon_vset_lane_f64(__s0, (int8x8_t)__s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vshld_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vshld_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vshld_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vshld_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vshld_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vshld_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vshld_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vshld_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshld_n_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vshld_n_u64(__s0, __p1); \ + __ret; \ +}) +#else +#define vshld_n_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vshld_n_u64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshld_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vshld_n_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vshld_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vshld_n_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshll_high_n_u8(__p0_234, __p1_234) __extension__ ({ \ + uint8x16_t __s0_234 = __p0_234; \ + uint16x8_t __ret_234; \ + __ret_234 = (uint16x8_t)(vshll_n_u8(vget_high_u8(__s0_234), __p1_234)); \ + __ret_234; \ +}) +#else +#define vshll_high_n_u8(__p0_235, __p1_235) __extension__ ({ \ + uint8x16_t __s0_235 = __p0_235; \ + uint8x16_t __rev0_235; __rev0_235 = __builtin_shufflevector(__s0_235, __s0_235, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __ret_235; \ + __ret_235 = (uint16x8_t)(__noswap_vshll_n_u8(__noswap_vget_high_u8(__rev0_235), __p1_235)); \ + __ret_235 = __builtin_shufflevector(__ret_235, __ret_235, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_235; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshll_high_n_u32(__p0_236, __p1_236) __extension__ ({ \ + uint32x4_t __s0_236 = __p0_236; \ + uint64x2_t __ret_236; \ + __ret_236 = (uint64x2_t)(vshll_n_u32(vget_high_u32(__s0_236), __p1_236)); \ + __ret_236; \ +}) +#else +#define vshll_high_n_u32(__p0_237, __p1_237) __extension__ ({ \ + uint32x4_t __s0_237 = __p0_237; \ + uint32x4_t __rev0_237; __rev0_237 = __builtin_shufflevector(__s0_237, __s0_237, 3, 2, 1, 0); \ + uint64x2_t __ret_237; \ + __ret_237 = (uint64x2_t)(__noswap_vshll_n_u32(__noswap_vget_high_u32(__rev0_237), __p1_237)); \ + __ret_237 = __builtin_shufflevector(__ret_237, __ret_237, 1, 0); \ + __ret_237; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshll_high_n_u16(__p0_238, __p1_238) __extension__ ({ \ + uint16x8_t __s0_238 = __p0_238; \ + uint32x4_t __ret_238; \ + __ret_238 = (uint32x4_t)(vshll_n_u16(vget_high_u16(__s0_238), __p1_238)); \ + __ret_238; \ +}) +#else +#define vshll_high_n_u16(__p0_239, __p1_239) __extension__ ({ \ + uint16x8_t __s0_239 = __p0_239; \ + uint16x8_t __rev0_239; __rev0_239 = __builtin_shufflevector(__s0_239, __s0_239, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint32x4_t __ret_239; \ + __ret_239 = (uint32x4_t)(__noswap_vshll_n_u16(__noswap_vget_high_u16(__rev0_239), __p1_239)); \ + __ret_239 = __builtin_shufflevector(__ret_239, __ret_239, 3, 2, 1, 0); \ + __ret_239; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshll_high_n_s8(__p0_240, __p1_240) __extension__ ({ \ + int8x16_t __s0_240 = __p0_240; \ + int16x8_t __ret_240; \ + __ret_240 = (int16x8_t)(vshll_n_s8(vget_high_s8(__s0_240), __p1_240)); \ + __ret_240; \ +}) +#else +#define vshll_high_n_s8(__p0_241, __p1_241) __extension__ ({ \ + int8x16_t __s0_241 = __p0_241; \ + int8x16_t __rev0_241; __rev0_241 = __builtin_shufflevector(__s0_241, __s0_241, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __ret_241; \ + __ret_241 = (int16x8_t)(__noswap_vshll_n_s8(__noswap_vget_high_s8(__rev0_241), __p1_241)); \ + __ret_241 = __builtin_shufflevector(__ret_241, __ret_241, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_241; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshll_high_n_s32(__p0_242, __p1_242) __extension__ ({ \ + int32x4_t __s0_242 = __p0_242; \ + int64x2_t __ret_242; \ + __ret_242 = (int64x2_t)(vshll_n_s32(vget_high_s32(__s0_242), __p1_242)); \ + __ret_242; \ +}) +#else +#define vshll_high_n_s32(__p0_243, __p1_243) __extension__ ({ \ + int32x4_t __s0_243 = __p0_243; \ + int32x4_t __rev0_243; __rev0_243 = __builtin_shufflevector(__s0_243, __s0_243, 3, 2, 1, 0); \ + int64x2_t __ret_243; \ + __ret_243 = (int64x2_t)(__noswap_vshll_n_s32(__noswap_vget_high_s32(__rev0_243), __p1_243)); \ + __ret_243 = __builtin_shufflevector(__ret_243, __ret_243, 1, 0); \ + __ret_243; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshll_high_n_s16(__p0_244, __p1_244) __extension__ ({ \ + int16x8_t __s0_244 = __p0_244; \ + int32x4_t __ret_244; \ + __ret_244 = (int32x4_t)(vshll_n_s16(vget_high_s16(__s0_244), __p1_244)); \ + __ret_244; \ +}) +#else +#define vshll_high_n_s16(__p0_245, __p1_245) __extension__ ({ \ + int16x8_t __s0_245 = __p0_245; \ + int16x8_t __rev0_245; __rev0_245 = __builtin_shufflevector(__s0_245, __s0_245, 7, 6, 5, 4, 3, 2, 1, 0); \ + int32x4_t __ret_245; \ + __ret_245 = (int32x4_t)(__noswap_vshll_n_s16(__noswap_vget_high_s16(__rev0_245), __p1_245)); \ + __ret_245 = __builtin_shufflevector(__ret_245, __ret_245, 3, 2, 1, 0); \ + __ret_245; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrd_n_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vshrd_n_u64(__s0, __p1); \ + __ret; \ +}) +#else +#define vshrd_n_u64(__p0, __p1) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vshrd_n_u64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrd_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vshrd_n_s64(__s0, __p1); \ + __ret; \ +}) +#else +#define vshrd_n_s64(__p0, __p1) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vshrd_n_s64(__s0, __p1); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrn_high_n_u32(__p0_246, __p1_246, __p2_246) __extension__ ({ \ + uint16x4_t __s0_246 = __p0_246; \ + uint32x4_t __s1_246 = __p1_246; \ + uint16x8_t __ret_246; \ + __ret_246 = (uint16x8_t)(vcombine_u16((uint16x4_t)(__s0_246), (uint16x4_t)(vshrn_n_u32(__s1_246, __p2_246)))); \ + __ret_246; \ +}) +#else +#define vshrn_high_n_u32(__p0_247, __p1_247, __p2_247) __extension__ ({ \ + uint16x4_t __s0_247 = __p0_247; \ + uint32x4_t __s1_247 = __p1_247; \ + uint16x4_t __rev0_247; __rev0_247 = __builtin_shufflevector(__s0_247, __s0_247, 3, 2, 1, 0); \ + uint32x4_t __rev1_247; __rev1_247 = __builtin_shufflevector(__s1_247, __s1_247, 3, 2, 1, 0); \ + uint16x8_t __ret_247; \ + __ret_247 = (uint16x8_t)(__noswap_vcombine_u16((uint16x4_t)(__rev0_247), (uint16x4_t)(__noswap_vshrn_n_u32(__rev1_247, __p2_247)))); \ + __ret_247 = __builtin_shufflevector(__ret_247, __ret_247, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_247; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrn_high_n_u64(__p0_248, __p1_248, __p2_248) __extension__ ({ \ + uint32x2_t __s0_248 = __p0_248; \ + uint64x2_t __s1_248 = __p1_248; \ + uint32x4_t __ret_248; \ + __ret_248 = (uint32x4_t)(vcombine_u32((uint32x2_t)(__s0_248), (uint32x2_t)(vshrn_n_u64(__s1_248, __p2_248)))); \ + __ret_248; \ +}) +#else +#define vshrn_high_n_u64(__p0_249, __p1_249, __p2_249) __extension__ ({ \ + uint32x2_t __s0_249 = __p0_249; \ + uint64x2_t __s1_249 = __p1_249; \ + uint32x2_t __rev0_249; __rev0_249 = __builtin_shufflevector(__s0_249, __s0_249, 1, 0); \ + uint64x2_t __rev1_249; __rev1_249 = __builtin_shufflevector(__s1_249, __s1_249, 1, 0); \ + uint32x4_t __ret_249; \ + __ret_249 = (uint32x4_t)(__noswap_vcombine_u32((uint32x2_t)(__rev0_249), (uint32x2_t)(__noswap_vshrn_n_u64(__rev1_249, __p2_249)))); \ + __ret_249 = __builtin_shufflevector(__ret_249, __ret_249, 3, 2, 1, 0); \ + __ret_249; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrn_high_n_u16(__p0_250, __p1_250, __p2_250) __extension__ ({ \ + uint8x8_t __s0_250 = __p0_250; \ + uint16x8_t __s1_250 = __p1_250; \ + uint8x16_t __ret_250; \ + __ret_250 = (uint8x16_t)(vcombine_u8((uint8x8_t)(__s0_250), (uint8x8_t)(vshrn_n_u16(__s1_250, __p2_250)))); \ + __ret_250; \ +}) +#else +#define vshrn_high_n_u16(__p0_251, __p1_251, __p2_251) __extension__ ({ \ + uint8x8_t __s0_251 = __p0_251; \ + uint16x8_t __s1_251 = __p1_251; \ + uint8x8_t __rev0_251; __rev0_251 = __builtin_shufflevector(__s0_251, __s0_251, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint16x8_t __rev1_251; __rev1_251 = __builtin_shufflevector(__s1_251, __s1_251, 7, 6, 5, 4, 3, 2, 1, 0); \ + uint8x16_t __ret_251; \ + __ret_251 = (uint8x16_t)(__noswap_vcombine_u8((uint8x8_t)(__rev0_251), (uint8x8_t)(__noswap_vshrn_n_u16(__rev1_251, __p2_251)))); \ + __ret_251 = __builtin_shufflevector(__ret_251, __ret_251, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_251; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrn_high_n_s32(__p0_252, __p1_252, __p2_252) __extension__ ({ \ + int16x4_t __s0_252 = __p0_252; \ + int32x4_t __s1_252 = __p1_252; \ + int16x8_t __ret_252; \ + __ret_252 = (int16x8_t)(vcombine_s16((int16x4_t)(__s0_252), (int16x4_t)(vshrn_n_s32(__s1_252, __p2_252)))); \ + __ret_252; \ +}) +#else +#define vshrn_high_n_s32(__p0_253, __p1_253, __p2_253) __extension__ ({ \ + int16x4_t __s0_253 = __p0_253; \ + int32x4_t __s1_253 = __p1_253; \ + int16x4_t __rev0_253; __rev0_253 = __builtin_shufflevector(__s0_253, __s0_253, 3, 2, 1, 0); \ + int32x4_t __rev1_253; __rev1_253 = __builtin_shufflevector(__s1_253, __s1_253, 3, 2, 1, 0); \ + int16x8_t __ret_253; \ + __ret_253 = (int16x8_t)(__noswap_vcombine_s16((int16x4_t)(__rev0_253), (int16x4_t)(__noswap_vshrn_n_s32(__rev1_253, __p2_253)))); \ + __ret_253 = __builtin_shufflevector(__ret_253, __ret_253, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_253; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrn_high_n_s64(__p0_254, __p1_254, __p2_254) __extension__ ({ \ + int32x2_t __s0_254 = __p0_254; \ + int64x2_t __s1_254 = __p1_254; \ + int32x4_t __ret_254; \ + __ret_254 = (int32x4_t)(vcombine_s32((int32x2_t)(__s0_254), (int32x2_t)(vshrn_n_s64(__s1_254, __p2_254)))); \ + __ret_254; \ +}) +#else +#define vshrn_high_n_s64(__p0_255, __p1_255, __p2_255) __extension__ ({ \ + int32x2_t __s0_255 = __p0_255; \ + int64x2_t __s1_255 = __p1_255; \ + int32x2_t __rev0_255; __rev0_255 = __builtin_shufflevector(__s0_255, __s0_255, 1, 0); \ + int64x2_t __rev1_255; __rev1_255 = __builtin_shufflevector(__s1_255, __s1_255, 1, 0); \ + int32x4_t __ret_255; \ + __ret_255 = (int32x4_t)(__noswap_vcombine_s32((int32x2_t)(__rev0_255), (int32x2_t)(__noswap_vshrn_n_s64(__rev1_255, __p2_255)))); \ + __ret_255 = __builtin_shufflevector(__ret_255, __ret_255, 3, 2, 1, 0); \ + __ret_255; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vshrn_high_n_s16(__p0_256, __p1_256, __p2_256) __extension__ ({ \ + int8x8_t __s0_256 = __p0_256; \ + int16x8_t __s1_256 = __p1_256; \ + int8x16_t __ret_256; \ + __ret_256 = (int8x16_t)(vcombine_s8((int8x8_t)(__s0_256), (int8x8_t)(vshrn_n_s16(__s1_256, __p2_256)))); \ + __ret_256; \ +}) +#else +#define vshrn_high_n_s16(__p0_257, __p1_257, __p2_257) __extension__ ({ \ + int8x8_t __s0_257 = __p0_257; \ + int16x8_t __s1_257 = __p1_257; \ + int8x8_t __rev0_257; __rev0_257 = __builtin_shufflevector(__s0_257, __s0_257, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16x8_t __rev1_257; __rev1_257 = __builtin_shufflevector(__s1_257, __s1_257, 7, 6, 5, 4, 3, 2, 1, 0); \ + int8x16_t __ret_257; \ + __ret_257 = (int8x16_t)(__noswap_vcombine_s8((int8x8_t)(__rev0_257), (int8x8_t)(__noswap_vshrn_n_s16(__rev1_257, __p2_257)))); \ + __ret_257 = __builtin_shufflevector(__ret_257, __ret_257, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __ret_257; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vslid_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __s1 = __p1; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vslid_n_u64(__s0, __s1, __p2); \ + __ret; \ +}) +#else +#define vslid_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __s1 = __p1; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vslid_n_u64(__s0, __s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vslid_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __s1 = __p1; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vslid_n_s64(__s0, __s1, __p2); \ + __ret; \ +}) +#else +#define vslid_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __s1 = __p1; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vslid_n_s64(__s0, __s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsli_n_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1_t __s0 = __p0; \ + poly64x1_t __s1 = __p1; \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vsli_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 6); \ + __ret; \ +}) +#else +#define vsli_n_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1_t __s0 = __p0; \ + poly64x1_t __s1 = __p1; \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vsli_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsliq_n_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2_t __s0 = __p0; \ + poly64x2_t __s1 = __p1; \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vsliq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 38); \ + __ret; \ +}) +#else +#define vsliq_n_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2_t __s0 = __p0; \ + poly64x2_t __s1 = __p1; \ + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vsliq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 38); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8_t vsqaddb_u8(uint8_t __p0, uint8_t __p1) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vsqaddb_u8(__p0, __p1); + return __ret; +} +#else +__ai uint8_t vsqaddb_u8(uint8_t __p0, uint8_t __p1) { + uint8_t __ret; + __ret = (uint8_t) __builtin_neon_vsqaddb_u8(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32_t vsqadds_u32(uint32_t __p0, uint32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vsqadds_u32(__p0, __p1); + return __ret; +} +#else +__ai uint32_t vsqadds_u32(uint32_t __p0, uint32_t __p1) { + uint32_t __ret; + __ret = (uint32_t) __builtin_neon_vsqadds_u32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vsqaddd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vsqaddd_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vsqaddd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vsqaddd_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16_t vsqaddh_u16(uint16_t __p0, uint16_t __p1) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vsqaddh_u16(__p0, __p1); + return __ret; +} +#else +__ai uint16_t vsqaddh_u16(uint16_t __p0, uint16_t __p1) { + uint16_t __ret; + __ret = (uint16_t) __builtin_neon_vsqaddh_u16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vsqaddq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vsqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 48); + return __ret; +} +#else +__ai uint8x16_t vsqaddq_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = (uint8x16_t) __builtin_neon_vsqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 48); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsqaddq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 50); + return __ret; +} +#else +__ai uint32x4_t vsqaddq_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t) __builtin_neon_vsqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 50); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vsqaddq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vsqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vsqaddq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vsqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vsqaddq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vsqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 49); + return __ret; +} +#else +__ai uint16x8_t vsqaddq_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t) __builtin_neon_vsqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 49); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vsqadd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vsqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 16); + return __ret; +} +#else +__ai uint8x8_t vsqadd_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = (uint8x8_t) __builtin_neon_vsqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 16); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vsqadd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vsqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 18); + return __ret; +} +#else +__ai uint32x2_t vsqadd_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = (uint32x2_t) __builtin_neon_vsqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 18); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vsqadd_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vsqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vsqadd_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vsqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vsqadd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vsqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 17); + return __ret; +} +#else +__ai uint16x4_t vsqadd_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = (uint16x4_t) __builtin_neon_vsqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 17); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vsqrtq_f64(float64x2_t __p0) { + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vsqrtq_v((int8x16_t)__p0, 42); + return __ret; +} +#else +__ai float64x2_t vsqrtq_f64(float64x2_t __p0) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __ret; + __ret = (float64x2_t) __builtin_neon_vsqrtq_v((int8x16_t)__rev0, 42); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vsqrtq_f32(float32x4_t __p0) { + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vsqrtq_v((int8x16_t)__p0, 41); + return __ret; +} +#else +__ai float32x4_t vsqrtq_f32(float32x4_t __p0) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __ret; + __ret = (float32x4_t) __builtin_neon_vsqrtq_v((int8x16_t)__rev0, 41); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vsqrt_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vsqrt_v((int8x8_t)__p0, 10); + return __ret; +} +#else +__ai float64x1_t vsqrt_f64(float64x1_t __p0) { + float64x1_t __ret; + __ret = (float64x1_t) __builtin_neon_vsqrt_v((int8x8_t)__p0, 10); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vsqrt_f32(float32x2_t __p0) { + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vsqrt_v((int8x8_t)__p0, 9); + return __ret; +} +#else +__ai float32x2_t vsqrt_f32(float32x2_t __p0) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __ret; + __ret = (float32x2_t) __builtin_neon_vsqrt_v((int8x8_t)__rev0, 9); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsrad_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __s1 = __p1; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vsrad_n_u64(__s0, __s1, __p2); \ + __ret; \ +}) +#else +#define vsrad_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __s1 = __p1; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vsrad_n_u64(__s0, __s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsrad_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __s1 = __p1; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vsrad_n_s64(__s0, __s1, __p2); \ + __ret; \ +}) +#else +#define vsrad_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __s1 = __p1; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vsrad_n_s64(__s0, __s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsrid_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __s1 = __p1; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vsrid_n_u64(__s0, __s1, __p2); \ + __ret; \ +}) +#else +#define vsrid_n_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64_t __s0 = __p0; \ + uint64_t __s1 = __p1; \ + uint64_t __ret; \ + __ret = (uint64_t) __builtin_neon_vsrid_n_u64(__s0, __s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsrid_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __s1 = __p1; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vsrid_n_s64(__s0, __s1, __p2); \ + __ret; \ +}) +#else +#define vsrid_n_s64(__p0, __p1, __p2) __extension__ ({ \ + int64_t __s0 = __p0; \ + int64_t __s1 = __p1; \ + int64_t __ret; \ + __ret = (int64_t) __builtin_neon_vsrid_n_s64(__s0, __s1, __p2); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsri_n_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1_t __s0 = __p0; \ + poly64x1_t __s1 = __p1; \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vsri_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 6); \ + __ret; \ +}) +#else +#define vsri_n_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1_t __s0 = __p0; \ + poly64x1_t __s1 = __p1; \ + poly64x1_t __ret; \ + __ret = (poly64x1_t) __builtin_neon_vsri_n_v((int8x8_t)__s0, (int8x8_t)__s1, __p2, 6); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vsriq_n_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2_t __s0 = __p0; \ + poly64x2_t __s1 = __p1; \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vsriq_n_v((int8x16_t)__s0, (int8x16_t)__s1, __p2, 38); \ + __ret; \ +}) +#else +#define vsriq_n_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2_t __s0 = __p0; \ + poly64x2_t __s1 = __p1; \ + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + poly64x2_t __ret; \ + __ret = (poly64x2_t) __builtin_neon_vsriq_n_v((int8x16_t)__rev0, (int8x16_t)__rev1, __p2, 38); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_p64(__p0, __p1) __extension__ ({ \ + poly64x1_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 6); \ +}) +#else +#define vst1_p64(__p0, __p1) __extension__ ({ \ + poly64x1_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 6); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_p64(__p0, __p1) __extension__ ({ \ + poly64x2_t __s1 = __p1; \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 38); \ +}) +#else +#define vst1q_p64(__p0, __p1) __extension__ ({ \ + poly64x2_t __s1 = __p1; \ + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 38); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s1 = __p1; \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__s1, 42); \ +}) +#else +#define vst1q_f64(__p0, __p1) __extension__ ({ \ + float64x2_t __s1 = __p1; \ + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + __builtin_neon_vst1q_v(__p0, (int8x16_t)__rev1, 42); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 10); \ +}) +#else +#define vst1_f64(__p0, __p1) __extension__ ({ \ + float64x1_t __s1 = __p1; \ + __builtin_neon_vst1_v(__p0, (int8x8_t)__s1, 10); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 6); \ +}) +#else +#define vst1_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 6); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2_t __s1 = __p1; \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 38); \ +}) +#else +#define vst1q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2_t __s1 = __p1; \ + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 38); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2_t __s1 = __p1; \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__s1, __p2, 42); \ +}) +#else +#define vst1q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2_t __s1 = __p1; \ + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + __builtin_neon_vst1q_lane_v(__p0, (int8x16_t)__rev1, __p2, 42); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 10); \ +}) +#else +#define vst1_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1_t __s1 = __p1; \ + __builtin_neon_vst1_lane_v(__p0, (int8x8_t)__s1, __p2, 10); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_p8_x2(__p0, __p1) __extension__ ({ \ + poly8x8x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 4); \ +}) +#else +#define vst1_p8_x2(__p0, __p1) __extension__ ({ \ + poly8x8x2_t __s1 = __p1; \ + poly8x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], 4); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_p64_x2(__p0, __p1) __extension__ ({ \ + poly64x1x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 6); \ +}) +#else +#define vst1_p64_x2(__p0, __p1) __extension__ ({ \ + poly64x1x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 6); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_p16_x2(__p0, __p1) __extension__ ({ \ + poly16x4x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 5); \ +}) +#else +#define vst1_p16_x2(__p0, __p1) __extension__ ({ \ + poly16x4x2_t __s1 = __p1; \ + poly16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], 5); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_p8_x2(__p0, __p1) __extension__ ({ \ + poly8x16x2_t __s1 = __p1; \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 36); \ +}) +#else +#define vst1q_p8_x2(__p0, __p1) __extension__ ({ \ + poly8x16x2_t __s1 = __p1; \ + poly8x16x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 36); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_p64_x2(__p0, __p1) __extension__ ({ \ + poly64x2x2_t __s1 = __p1; \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 38); \ +}) +#else +#define vst1q_p64_x2(__p0, __p1) __extension__ ({ \ + poly64x2x2_t __s1 = __p1; \ + poly64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 38); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_p16_x2(__p0, __p1) __extension__ ({ \ + poly16x8x2_t __s1 = __p1; \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 37); \ +}) +#else +#define vst1q_p16_x2(__p0, __p1) __extension__ ({ \ + poly16x8x2_t __s1 = __p1; \ + poly16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 37); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u8_x2(__p0, __p1) __extension__ ({ \ + uint8x16x2_t __s1 = __p1; \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 48); \ +}) +#else +#define vst1q_u8_x2(__p0, __p1) __extension__ ({ \ + uint8x16x2_t __s1 = __p1; \ + uint8x16x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 48); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u32_x2(__p0, __p1) __extension__ ({ \ + uint32x4x2_t __s1 = __p1; \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 50); \ +}) +#else +#define vst1q_u32_x2(__p0, __p1) __extension__ ({ \ + uint32x4x2_t __s1 = __p1; \ + uint32x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 50); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u64_x2(__p0, __p1) __extension__ ({ \ + uint64x2x2_t __s1 = __p1; \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 51); \ +}) +#else +#define vst1q_u64_x2(__p0, __p1) __extension__ ({ \ + uint64x2x2_t __s1 = __p1; \ + uint64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 51); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u16_x2(__p0, __p1) __extension__ ({ \ + uint16x8x2_t __s1 = __p1; \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 49); \ +}) +#else +#define vst1q_u16_x2(__p0, __p1) __extension__ ({ \ + uint16x8x2_t __s1 = __p1; \ + uint16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 49); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s8_x2(__p0, __p1) __extension__ ({ \ + int8x16x2_t __s1 = __p1; \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 32); \ +}) +#else +#define vst1q_s8_x2(__p0, __p1) __extension__ ({ \ + int8x16x2_t __s1 = __p1; \ + int8x16x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x2_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 32); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_f64_x2(__p0, __p1) __extension__ ({ \ + float64x2x2_t __s1 = __p1; \ + __builtin_neon_vst1q_x2_v(__p0, __s1.val[0], __s1.val[1], 42); \ +}) +#else +#define vst1q_f64_x2(__p0, __p1) __extension__ ({ \ + float64x2x2_t __s1 = __p1; \ + float64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst1q_x2_v(__p0, __rev1.val[0], __rev1.val[1], 42); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_f32_x2(__p0, __p1) __extension__ ({ \ + float32x4x2_t __s1 = __p1; \ + __builtin_neon_vst1q_x2_v(__p0, __s1.val[0], __s1.val[1], 41); \ +}) +#else +#define vst1q_f32_x2(__p0, __p1) __extension__ ({ \ + float32x4x2_t __s1 = __p1; \ + float32x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst1q_x2_v(__p0, __rev1.val[0], __rev1.val[1], 41); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_f16_x2(__p0, __p1) __extension__ ({ \ + float16x8x2_t __s1 = __p1; \ + __builtin_neon_vst1q_x2_v(__p0, __s1.val[0], __s1.val[1], 40); \ +}) +#else +#define vst1q_f16_x2(__p0, __p1) __extension__ ({ \ + float16x8x2_t __s1 = __p1; \ + float16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x2_v(__p0, __rev1.val[0], __rev1.val[1], 40); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s32_x2(__p0, __p1) __extension__ ({ \ + int32x4x2_t __s1 = __p1; \ + __builtin_neon_vst1q_x2_v(__p0, __s1.val[0], __s1.val[1], 34); \ +}) +#else +#define vst1q_s32_x2(__p0, __p1) __extension__ ({ \ + int32x4x2_t __s1 = __p1; \ + int32x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst1q_x2_v(__p0, __rev1.val[0], __rev1.val[1], 34); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s64_x2(__p0, __p1) __extension__ ({ \ + int64x2x2_t __s1 = __p1; \ + __builtin_neon_vst1q_x2_v(__p0, __s1.val[0], __s1.val[1], 35); \ +}) +#else +#define vst1q_s64_x2(__p0, __p1) __extension__ ({ \ + int64x2x2_t __s1 = __p1; \ + int64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst1q_x2_v(__p0, __rev1.val[0], __rev1.val[1], 35); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s16_x2(__p0, __p1) __extension__ ({ \ + int16x8x2_t __s1 = __p1; \ + __builtin_neon_vst1q_x2_v(__p0, __s1.val[0], __s1.val[1], 33); \ +}) +#else +#define vst1q_s16_x2(__p0, __p1) __extension__ ({ \ + int16x8x2_t __s1 = __p1; \ + int16x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x2_v(__p0, __rev1.val[0], __rev1.val[1], 33); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u8_x2(__p0, __p1) __extension__ ({ \ + uint8x8x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 16); \ +}) +#else +#define vst1_u8_x2(__p0, __p1) __extension__ ({ \ + uint8x8x2_t __s1 = __p1; \ + uint8x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], 16); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u32_x2(__p0, __p1) __extension__ ({ \ + uint32x2x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 18); \ +}) +#else +#define vst1_u32_x2(__p0, __p1) __extension__ ({ \ + uint32x2x2_t __s1 = __p1; \ + uint32x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], 18); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u64_x2(__p0, __p1) __extension__ ({ \ + uint64x1x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 19); \ +}) +#else +#define vst1_u64_x2(__p0, __p1) __extension__ ({ \ + uint64x1x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 19); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u16_x2(__p0, __p1) __extension__ ({ \ + uint16x4x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 17); \ +}) +#else +#define vst1_u16_x2(__p0, __p1) __extension__ ({ \ + uint16x4x2_t __s1 = __p1; \ + uint16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], 17); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s8_x2(__p0, __p1) __extension__ ({ \ + int8x8x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 0); \ +}) +#else +#define vst1_s8_x2(__p0, __p1) __extension__ ({ \ + int8x8x2_t __s1 = __p1; \ + int8x8x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_x2_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], 0); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_f64_x2(__p0, __p1) __extension__ ({ \ + float64x1x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, __s1.val[0], __s1.val[1], 10); \ +}) +#else +#define vst1_f64_x2(__p0, __p1) __extension__ ({ \ + float64x1x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, __s1.val[0], __s1.val[1], 10); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_f32_x2(__p0, __p1) __extension__ ({ \ + float32x2x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, __s1.val[0], __s1.val[1], 9); \ +}) +#else +#define vst1_f32_x2(__p0, __p1) __extension__ ({ \ + float32x2x2_t __s1 = __p1; \ + float32x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst1_x2_v(__p0, __rev1.val[0], __rev1.val[1], 9); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_f16_x2(__p0, __p1) __extension__ ({ \ + float16x4x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, __s1.val[0], __s1.val[1], 8); \ +}) +#else +#define vst1_f16_x2(__p0, __p1) __extension__ ({ \ + float16x4x2_t __s1 = __p1; \ + float16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst1_x2_v(__p0, __rev1.val[0], __rev1.val[1], 8); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s32_x2(__p0, __p1) __extension__ ({ \ + int32x2x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, __s1.val[0], __s1.val[1], 2); \ +}) +#else +#define vst1_s32_x2(__p0, __p1) __extension__ ({ \ + int32x2x2_t __s1 = __p1; \ + int32x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst1_x2_v(__p0, __rev1.val[0], __rev1.val[1], 2); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s64_x2(__p0, __p1) __extension__ ({ \ + int64x1x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, __s1.val[0], __s1.val[1], 3); \ +}) +#else +#define vst1_s64_x2(__p0, __p1) __extension__ ({ \ + int64x1x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, __s1.val[0], __s1.val[1], 3); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s16_x2(__p0, __p1) __extension__ ({ \ + int16x4x2_t __s1 = __p1; \ + __builtin_neon_vst1_x2_v(__p0, __s1.val[0], __s1.val[1], 1); \ +}) +#else +#define vst1_s16_x2(__p0, __p1) __extension__ ({ \ + int16x4x2_t __s1 = __p1; \ + int16x4x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __builtin_neon_vst1_x2_v(__p0, __rev1.val[0], __rev1.val[1], 1); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_p8_x3(__p0, __p1) __extension__ ({ \ + poly8x8x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 4); \ +}) +#else +#define vst1_p8_x3(__p0, __p1) __extension__ ({ \ + poly8x8x3_t __s1 = __p1; \ + poly8x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], 4); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_p64_x3(__p0, __p1) __extension__ ({ \ + poly64x1x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 6); \ +}) +#else +#define vst1_p64_x3(__p0, __p1) __extension__ ({ \ + poly64x1x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 6); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_p16_x3(__p0, __p1) __extension__ ({ \ + poly16x4x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 5); \ +}) +#else +#define vst1_p16_x3(__p0, __p1) __extension__ ({ \ + poly16x4x3_t __s1 = __p1; \ + poly16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], 5); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_p8_x3(__p0, __p1) __extension__ ({ \ + poly8x16x3_t __s1 = __p1; \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 36); \ +}) +#else +#define vst1q_p8_x3(__p0, __p1) __extension__ ({ \ + poly8x16x3_t __s1 = __p1; \ + poly8x16x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 36); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_p64_x3(__p0, __p1) __extension__ ({ \ + poly64x2x3_t __s1 = __p1; \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 38); \ +}) +#else +#define vst1q_p64_x3(__p0, __p1) __extension__ ({ \ + poly64x2x3_t __s1 = __p1; \ + poly64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 38); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_p16_x3(__p0, __p1) __extension__ ({ \ + poly16x8x3_t __s1 = __p1; \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 37); \ +}) +#else +#define vst1q_p16_x3(__p0, __p1) __extension__ ({ \ + poly16x8x3_t __s1 = __p1; \ + poly16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 37); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u8_x3(__p0, __p1) __extension__ ({ \ + uint8x16x3_t __s1 = __p1; \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 48); \ +}) +#else +#define vst1q_u8_x3(__p0, __p1) __extension__ ({ \ + uint8x16x3_t __s1 = __p1; \ + uint8x16x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 48); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u32_x3(__p0, __p1) __extension__ ({ \ + uint32x4x3_t __s1 = __p1; \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 50); \ +}) +#else +#define vst1q_u32_x3(__p0, __p1) __extension__ ({ \ + uint32x4x3_t __s1 = __p1; \ + uint32x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 50); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u64_x3(__p0, __p1) __extension__ ({ \ + uint64x2x3_t __s1 = __p1; \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 51); \ +}) +#else +#define vst1q_u64_x3(__p0, __p1) __extension__ ({ \ + uint64x2x3_t __s1 = __p1; \ + uint64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 51); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u16_x3(__p0, __p1) __extension__ ({ \ + uint16x8x3_t __s1 = __p1; \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 49); \ +}) +#else +#define vst1q_u16_x3(__p0, __p1) __extension__ ({ \ + uint16x8x3_t __s1 = __p1; \ + uint16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 49); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s8_x3(__p0, __p1) __extension__ ({ \ + int8x16x3_t __s1 = __p1; \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 32); \ +}) +#else +#define vst1q_s8_x3(__p0, __p1) __extension__ ({ \ + int8x16x3_t __s1 = __p1; \ + int8x16x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x3_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 32); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_f64_x3(__p0, __p1) __extension__ ({ \ + float64x2x3_t __s1 = __p1; \ + __builtin_neon_vst1q_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 42); \ +}) +#else +#define vst1q_f64_x3(__p0, __p1) __extension__ ({ \ + float64x2x3_t __s1 = __p1; \ + float64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst1q_x3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 42); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_f32_x3(__p0, __p1) __extension__ ({ \ + float32x4x3_t __s1 = __p1; \ + __builtin_neon_vst1q_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 41); \ +}) +#else +#define vst1q_f32_x3(__p0, __p1) __extension__ ({ \ + float32x4x3_t __s1 = __p1; \ + float32x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst1q_x3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 41); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_f16_x3(__p0, __p1) __extension__ ({ \ + float16x8x3_t __s1 = __p1; \ + __builtin_neon_vst1q_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 40); \ +}) +#else +#define vst1q_f16_x3(__p0, __p1) __extension__ ({ \ + float16x8x3_t __s1 = __p1; \ + float16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 40); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s32_x3(__p0, __p1) __extension__ ({ \ + int32x4x3_t __s1 = __p1; \ + __builtin_neon_vst1q_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 34); \ +}) +#else +#define vst1q_s32_x3(__p0, __p1) __extension__ ({ \ + int32x4x3_t __s1 = __p1; \ + int32x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst1q_x3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 34); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s64_x3(__p0, __p1) __extension__ ({ \ + int64x2x3_t __s1 = __p1; \ + __builtin_neon_vst1q_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 35); \ +}) +#else +#define vst1q_s64_x3(__p0, __p1) __extension__ ({ \ + int64x2x3_t __s1 = __p1; \ + int64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst1q_x3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 35); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s16_x3(__p0, __p1) __extension__ ({ \ + int16x8x3_t __s1 = __p1; \ + __builtin_neon_vst1q_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 33); \ +}) +#else +#define vst1q_s16_x3(__p0, __p1) __extension__ ({ \ + int16x8x3_t __s1 = __p1; \ + int16x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 33); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u8_x3(__p0, __p1) __extension__ ({ \ + uint8x8x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 16); \ +}) +#else +#define vst1_u8_x3(__p0, __p1) __extension__ ({ \ + uint8x8x3_t __s1 = __p1; \ + uint8x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], 16); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u32_x3(__p0, __p1) __extension__ ({ \ + uint32x2x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 18); \ +}) +#else +#define vst1_u32_x3(__p0, __p1) __extension__ ({ \ + uint32x2x3_t __s1 = __p1; \ + uint32x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], 18); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u64_x3(__p0, __p1) __extension__ ({ \ + uint64x1x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 19); \ +}) +#else +#define vst1_u64_x3(__p0, __p1) __extension__ ({ \ + uint64x1x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 19); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u16_x3(__p0, __p1) __extension__ ({ \ + uint16x4x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 17); \ +}) +#else +#define vst1_u16_x3(__p0, __p1) __extension__ ({ \ + uint16x4x3_t __s1 = __p1; \ + uint16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], 17); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s8_x3(__p0, __p1) __extension__ ({ \ + int8x8x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 0); \ +}) +#else +#define vst1_s8_x3(__p0, __p1) __extension__ ({ \ + int8x8x3_t __s1 = __p1; \ + int8x8x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_x3_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], 0); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_f64_x3(__p0, __p1) __extension__ ({ \ + float64x1x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 10); \ +}) +#else +#define vst1_f64_x3(__p0, __p1) __extension__ ({ \ + float64x1x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 10); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_f32_x3(__p0, __p1) __extension__ ({ \ + float32x2x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 9); \ +}) +#else +#define vst1_f32_x3(__p0, __p1) __extension__ ({ \ + float32x2x3_t __s1 = __p1; \ + float32x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst1_x3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 9); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_f16_x3(__p0, __p1) __extension__ ({ \ + float16x4x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 8); \ +}) +#else +#define vst1_f16_x3(__p0, __p1) __extension__ ({ \ + float16x4x3_t __s1 = __p1; \ + float16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst1_x3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 8); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s32_x3(__p0, __p1) __extension__ ({ \ + int32x2x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 2); \ +}) +#else +#define vst1_s32_x3(__p0, __p1) __extension__ ({ \ + int32x2x3_t __s1 = __p1; \ + int32x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst1_x3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 2); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s64_x3(__p0, __p1) __extension__ ({ \ + int64x1x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 3); \ +}) +#else +#define vst1_s64_x3(__p0, __p1) __extension__ ({ \ + int64x1x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 3); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s16_x3(__p0, __p1) __extension__ ({ \ + int16x4x3_t __s1 = __p1; \ + __builtin_neon_vst1_x3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 1); \ +}) +#else +#define vst1_s16_x3(__p0, __p1) __extension__ ({ \ + int16x4x3_t __s1 = __p1; \ + int16x4x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __builtin_neon_vst1_x3_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 1); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_p8_x4(__p0, __p1) __extension__ ({ \ + poly8x8x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 4); \ +}) +#else +#define vst1_p8_x4(__p0, __p1) __extension__ ({ \ + poly8x8x4_t __s1 = __p1; \ + poly8x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], 4); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_p64_x4(__p0, __p1) __extension__ ({ \ + poly64x1x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 6); \ +}) +#else +#define vst1_p64_x4(__p0, __p1) __extension__ ({ \ + poly64x1x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 6); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_p16_x4(__p0, __p1) __extension__ ({ \ + poly16x4x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 5); \ +}) +#else +#define vst1_p16_x4(__p0, __p1) __extension__ ({ \ + poly16x4x4_t __s1 = __p1; \ + poly16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], 5); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_p8_x4(__p0, __p1) __extension__ ({ \ + poly8x16x4_t __s1 = __p1; \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 36); \ +}) +#else +#define vst1q_p8_x4(__p0, __p1) __extension__ ({ \ + poly8x16x4_t __s1 = __p1; \ + poly8x16x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 36); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_p64_x4(__p0, __p1) __extension__ ({ \ + poly64x2x4_t __s1 = __p1; \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 38); \ +}) +#else +#define vst1q_p64_x4(__p0, __p1) __extension__ ({ \ + poly64x2x4_t __s1 = __p1; \ + poly64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 38); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_p16_x4(__p0, __p1) __extension__ ({ \ + poly16x8x4_t __s1 = __p1; \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 37); \ +}) +#else +#define vst1q_p16_x4(__p0, __p1) __extension__ ({ \ + poly16x8x4_t __s1 = __p1; \ + poly16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 37); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u8_x4(__p0, __p1) __extension__ ({ \ + uint8x16x4_t __s1 = __p1; \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 48); \ +}) +#else +#define vst1q_u8_x4(__p0, __p1) __extension__ ({ \ + uint8x16x4_t __s1 = __p1; \ + uint8x16x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 48); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u32_x4(__p0, __p1) __extension__ ({ \ + uint32x4x4_t __s1 = __p1; \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 50); \ +}) +#else +#define vst1q_u32_x4(__p0, __p1) __extension__ ({ \ + uint32x4x4_t __s1 = __p1; \ + uint32x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 50); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u64_x4(__p0, __p1) __extension__ ({ \ + uint64x2x4_t __s1 = __p1; \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 51); \ +}) +#else +#define vst1q_u64_x4(__p0, __p1) __extension__ ({ \ + uint64x2x4_t __s1 = __p1; \ + uint64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 51); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_u16_x4(__p0, __p1) __extension__ ({ \ + uint16x8x4_t __s1 = __p1; \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 49); \ +}) +#else +#define vst1q_u16_x4(__p0, __p1) __extension__ ({ \ + uint16x8x4_t __s1 = __p1; \ + uint16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 49); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s8_x4(__p0, __p1) __extension__ ({ \ + int8x16x4_t __s1 = __p1; \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 32); \ +}) +#else +#define vst1q_s8_x4(__p0, __p1) __extension__ ({ \ + int8x16x4_t __s1 = __p1; \ + int8x16x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x4_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 32); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_f64_x4(__p0, __p1) __extension__ ({ \ + float64x2x4_t __s1 = __p1; \ + __builtin_neon_vst1q_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 42); \ +}) +#else +#define vst1q_f64_x4(__p0, __p1) __extension__ ({ \ + float64x2x4_t __s1 = __p1; \ + float64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst1q_x4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 42); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_f32_x4(__p0, __p1) __extension__ ({ \ + float32x4x4_t __s1 = __p1; \ + __builtin_neon_vst1q_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 41); \ +}) +#else +#define vst1q_f32_x4(__p0, __p1) __extension__ ({ \ + float32x4x4_t __s1 = __p1; \ + float32x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst1q_x4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 41); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_f16_x4(__p0, __p1) __extension__ ({ \ + float16x8x4_t __s1 = __p1; \ + __builtin_neon_vst1q_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 40); \ +}) +#else +#define vst1q_f16_x4(__p0, __p1) __extension__ ({ \ + float16x8x4_t __s1 = __p1; \ + float16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 40); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s32_x4(__p0, __p1) __extension__ ({ \ + int32x4x4_t __s1 = __p1; \ + __builtin_neon_vst1q_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 34); \ +}) +#else +#define vst1q_s32_x4(__p0, __p1) __extension__ ({ \ + int32x4x4_t __s1 = __p1; \ + int32x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst1q_x4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 34); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s64_x4(__p0, __p1) __extension__ ({ \ + int64x2x4_t __s1 = __p1; \ + __builtin_neon_vst1q_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 35); \ +}) +#else +#define vst1q_s64_x4(__p0, __p1) __extension__ ({ \ + int64x2x4_t __s1 = __p1; \ + int64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst1q_x4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 35); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1q_s16_x4(__p0, __p1) __extension__ ({ \ + int16x8x4_t __s1 = __p1; \ + __builtin_neon_vst1q_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 33); \ +}) +#else +#define vst1q_s16_x4(__p0, __p1) __extension__ ({ \ + int16x8x4_t __s1 = __p1; \ + int16x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1q_x4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 33); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u8_x4(__p0, __p1) __extension__ ({ \ + uint8x8x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 16); \ +}) +#else +#define vst1_u8_x4(__p0, __p1) __extension__ ({ \ + uint8x8x4_t __s1 = __p1; \ + uint8x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], 16); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u32_x4(__p0, __p1) __extension__ ({ \ + uint32x2x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 18); \ +}) +#else +#define vst1_u32_x4(__p0, __p1) __extension__ ({ \ + uint32x2x4_t __s1 = __p1; \ + uint32x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], 18); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u64_x4(__p0, __p1) __extension__ ({ \ + uint64x1x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 19); \ +}) +#else +#define vst1_u64_x4(__p0, __p1) __extension__ ({ \ + uint64x1x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 19); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_u16_x4(__p0, __p1) __extension__ ({ \ + uint16x4x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 17); \ +}) +#else +#define vst1_u16_x4(__p0, __p1) __extension__ ({ \ + uint16x4x4_t __s1 = __p1; \ + uint16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], 17); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s8_x4(__p0, __p1) __extension__ ({ \ + int8x8x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 0); \ +}) +#else +#define vst1_s8_x4(__p0, __p1) __extension__ ({ \ + int8x8x4_t __s1 = __p1; \ + int8x8x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst1_x4_v(__p0, (int8x8_t)__rev1.val[0], (int8x8_t)__rev1.val[1], (int8x8_t)__rev1.val[2], (int8x8_t)__rev1.val[3], 0); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_f64_x4(__p0, __p1) __extension__ ({ \ + float64x1x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 10); \ +}) +#else +#define vst1_f64_x4(__p0, __p1) __extension__ ({ \ + float64x1x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 10); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_f32_x4(__p0, __p1) __extension__ ({ \ + float32x2x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 9); \ +}) +#else +#define vst1_f32_x4(__p0, __p1) __extension__ ({ \ + float32x2x4_t __s1 = __p1; \ + float32x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst1_x4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 9); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_f16_x4(__p0, __p1) __extension__ ({ \ + float16x4x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 8); \ +}) +#else +#define vst1_f16_x4(__p0, __p1) __extension__ ({ \ + float16x4x4_t __s1 = __p1; \ + float16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst1_x4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 8); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s32_x4(__p0, __p1) __extension__ ({ \ + int32x2x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 2); \ +}) +#else +#define vst1_s32_x4(__p0, __p1) __extension__ ({ \ + int32x2x4_t __s1 = __p1; \ + int32x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst1_x4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 2); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s64_x4(__p0, __p1) __extension__ ({ \ + int64x1x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 3); \ +}) +#else +#define vst1_s64_x4(__p0, __p1) __extension__ ({ \ + int64x1x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 3); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst1_s16_x4(__p0, __p1) __extension__ ({ \ + int16x4x4_t __s1 = __p1; \ + __builtin_neon_vst1_x4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 1); \ +}) +#else +#define vst1_s16_x4(__p0, __p1) __extension__ ({ \ + int16x4x4_t __s1 = __p1; \ + int16x4x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 3, 2, 1, 0); \ + __builtin_neon_vst1_x4_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 1); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_p64(__p0, __p1) __extension__ ({ \ + poly64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 6); \ +}) +#else +#define vst2_p64(__p0, __p1) __extension__ ({ \ + poly64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], 6); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_p64(__p0, __p1) __extension__ ({ \ + poly64x2x2_t __s1 = __p1; \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 38); \ +}) +#else +#define vst2q_p64(__p0, __p1) __extension__ ({ \ + poly64x2x2_t __s1 = __p1; \ + poly64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 38); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_u64(__p0, __p1) __extension__ ({ \ + uint64x2x2_t __s1 = __p1; \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], 51); \ +}) +#else +#define vst2q_u64(__p0, __p1) __extension__ ({ \ + uint64x2x2_t __s1 = __p1; \ + uint64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst2q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], 51); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_f64(__p0, __p1) __extension__ ({ \ + float64x2x2_t __s1 = __p1; \ + __builtin_neon_vst2q_v(__p0, __s1.val[0], __s1.val[1], 42); \ +}) +#else +#define vst2q_f64(__p0, __p1) __extension__ ({ \ + float64x2x2_t __s1 = __p1; \ + float64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst2q_v(__p0, __rev1.val[0], __rev1.val[1], 42); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_s64(__p0, __p1) __extension__ ({ \ + int64x2x2_t __s1 = __p1; \ + __builtin_neon_vst2q_v(__p0, __s1.val[0], __s1.val[1], 35); \ +}) +#else +#define vst2q_s64(__p0, __p1) __extension__ ({ \ + int64x2x2_t __s1 = __p1; \ + int64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst2q_v(__p0, __rev1.val[0], __rev1.val[1], 35); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_f64(__p0, __p1) __extension__ ({ \ + float64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, __s1.val[0], __s1.val[1], 10); \ +}) +#else +#define vst2_f64(__p0, __p1) __extension__ ({ \ + float64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_v(__p0, __s1.val[0], __s1.val[1], 10); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 6); \ +}) +#else +#define vst2_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 6); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16x2_t __s1 = __p1; \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 36); \ +}) +#else +#define vst2q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16x2_t __s1 = __p1; \ + poly8x16x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 36); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2x2_t __s1 = __p1; \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 38); \ +}) +#else +#define vst2q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2x2_t __s1 = __p1; \ + poly64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 38); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16x2_t __s1 = __p1; \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 48); \ +}) +#else +#define vst2q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16x2_t __s1 = __p1; \ + uint8x16x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 48); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2x2_t __s1 = __p1; \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 51); \ +}) +#else +#define vst2q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2x2_t __s1 = __p1; \ + uint64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 51); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16x2_t __s1 = __p1; \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], __p2, 32); \ +}) +#else +#define vst2q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16x2_t __s1 = __p1; \ + int8x16x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst2q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], __p2, 32); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2x2_t __s1 = __p1; \ + __builtin_neon_vst2q_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 42); \ +}) +#else +#define vst2q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2x2_t __s1 = __p1; \ + float64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst2q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __p2, 42); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2x2_t __s1 = __p1; \ + __builtin_neon_vst2q_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 35); \ +}) +#else +#define vst2q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2x2_t __s1 = __p1; \ + int64x2x2_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __builtin_neon_vst2q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __p2, 35); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 19); \ +}) +#else +#define vst2_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], __p2, 19); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 10); \ +}) +#else +#define vst2_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 10); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst2_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 3); \ +}) +#else +#define vst2_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1x2_t __s1 = __p1; \ + __builtin_neon_vst2_lane_v(__p0, __s1.val[0], __s1.val[1], __p2, 3); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_p64(__p0, __p1) __extension__ ({ \ + poly64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 6); \ +}) +#else +#define vst3_p64(__p0, __p1) __extension__ ({ \ + poly64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], 6); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_p64(__p0, __p1) __extension__ ({ \ + poly64x2x3_t __s1 = __p1; \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 38); \ +}) +#else +#define vst3q_p64(__p0, __p1) __extension__ ({ \ + poly64x2x3_t __s1 = __p1; \ + poly64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 38); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_u64(__p0, __p1) __extension__ ({ \ + uint64x2x3_t __s1 = __p1; \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], 51); \ +}) +#else +#define vst3q_u64(__p0, __p1) __extension__ ({ \ + uint64x2x3_t __s1 = __p1; \ + uint64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst3q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], 51); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_f64(__p0, __p1) __extension__ ({ \ + float64x2x3_t __s1 = __p1; \ + __builtin_neon_vst3q_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 42); \ +}) +#else +#define vst3q_f64(__p0, __p1) __extension__ ({ \ + float64x2x3_t __s1 = __p1; \ + float64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst3q_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 42); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_s64(__p0, __p1) __extension__ ({ \ + int64x2x3_t __s1 = __p1; \ + __builtin_neon_vst3q_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 35); \ +}) +#else +#define vst3q_s64(__p0, __p1) __extension__ ({ \ + int64x2x3_t __s1 = __p1; \ + int64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst3q_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], 35); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_f64(__p0, __p1) __extension__ ({ \ + float64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 10); \ +}) +#else +#define vst3_f64(__p0, __p1) __extension__ ({ \ + float64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], 10); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 6); \ +}) +#else +#define vst3_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 6); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16x3_t __s1 = __p1; \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 36); \ +}) +#else +#define vst3q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16x3_t __s1 = __p1; \ + poly8x16x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 36); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2x3_t __s1 = __p1; \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 38); \ +}) +#else +#define vst3q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2x3_t __s1 = __p1; \ + poly64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 38); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16x3_t __s1 = __p1; \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 48); \ +}) +#else +#define vst3q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16x3_t __s1 = __p1; \ + uint8x16x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 48); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2x3_t __s1 = __p1; \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 51); \ +}) +#else +#define vst3q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2x3_t __s1 = __p1; \ + uint64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 51); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16x3_t __s1 = __p1; \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], __p2, 32); \ +}) +#else +#define vst3q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16x3_t __s1 = __p1; \ + int8x16x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst3q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], __p2, 32); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2x3_t __s1 = __p1; \ + __builtin_neon_vst3q_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 42); \ +}) +#else +#define vst3q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2x3_t __s1 = __p1; \ + float64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst3q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 42); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2x3_t __s1 = __p1; \ + __builtin_neon_vst3q_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 35); \ +}) +#else +#define vst3q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2x3_t __s1 = __p1; \ + int64x2x3_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __builtin_neon_vst3q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __p2, 35); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 19); \ +}) +#else +#define vst3_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], __p2, 19); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 10); \ +}) +#else +#define vst3_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 10); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst3_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 3); \ +}) +#else +#define vst3_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1x3_t __s1 = __p1; \ + __builtin_neon_vst3_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __p2, 3); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_p64(__p0, __p1) __extension__ ({ \ + poly64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 6); \ +}) +#else +#define vst4_p64(__p0, __p1) __extension__ ({ \ + poly64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], 6); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_p64(__p0, __p1) __extension__ ({ \ + poly64x2x4_t __s1 = __p1; \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 38); \ +}) +#else +#define vst4q_p64(__p0, __p1) __extension__ ({ \ + poly64x2x4_t __s1 = __p1; \ + poly64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 38); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_u64(__p0, __p1) __extension__ ({ \ + uint64x2x4_t __s1 = __p1; \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], 51); \ +}) +#else +#define vst4q_u64(__p0, __p1) __extension__ ({ \ + uint64x2x4_t __s1 = __p1; \ + uint64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst4q_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], 51); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_f64(__p0, __p1) __extension__ ({ \ + float64x2x4_t __s1 = __p1; \ + __builtin_neon_vst4q_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 42); \ +}) +#else +#define vst4q_f64(__p0, __p1) __extension__ ({ \ + float64x2x4_t __s1 = __p1; \ + float64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst4q_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 42); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_s64(__p0, __p1) __extension__ ({ \ + int64x2x4_t __s1 = __p1; \ + __builtin_neon_vst4q_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 35); \ +}) +#else +#define vst4q_s64(__p0, __p1) __extension__ ({ \ + int64x2x4_t __s1 = __p1; \ + int64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst4q_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], 35); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_f64(__p0, __p1) __extension__ ({ \ + float64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 10); \ +}) +#else +#define vst4_f64(__p0, __p1) __extension__ ({ \ + float64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], 10); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 6); \ +}) +#else +#define vst4_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 6); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16x4_t __s1 = __p1; \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 36); \ +}) +#else +#define vst4q_lane_p8(__p0, __p1, __p2) __extension__ ({ \ + poly8x16x4_t __s1 = __p1; \ + poly8x16x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 36); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2x4_t __s1 = __p1; \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 38); \ +}) +#else +#define vst4q_lane_p64(__p0, __p1, __p2) __extension__ ({ \ + poly64x2x4_t __s1 = __p1; \ + poly64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 38); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16x4_t __s1 = __p1; \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 48); \ +}) +#else +#define vst4q_lane_u8(__p0, __p1, __p2) __extension__ ({ \ + uint8x16x4_t __s1 = __p1; \ + uint8x16x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 48); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2x4_t __s1 = __p1; \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 51); \ +}) +#else +#define vst4q_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x2x4_t __s1 = __p1; \ + uint64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 51); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16x4_t __s1 = __p1; \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__s1.val[0], (int8x16_t)__s1.val[1], (int8x16_t)__s1.val[2], (int8x16_t)__s1.val[3], __p2, 32); \ +}) +#else +#define vst4q_lane_s8(__p0, __p1, __p2) __extension__ ({ \ + int8x16x4_t __s1 = __p1; \ + int8x16x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); \ + __builtin_neon_vst4q_lane_v(__p0, (int8x16_t)__rev1.val[0], (int8x16_t)__rev1.val[1], (int8x16_t)__rev1.val[2], (int8x16_t)__rev1.val[3], __p2, 32); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2x4_t __s1 = __p1; \ + __builtin_neon_vst4q_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 42); \ +}) +#else +#define vst4q_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x2x4_t __s1 = __p1; \ + float64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst4q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 42); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2x4_t __s1 = __p1; \ + __builtin_neon_vst4q_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 35); \ +}) +#else +#define vst4q_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x2x4_t __s1 = __p1; \ + int64x2x4_t __rev1; \ + __rev1.val[0] = __builtin_shufflevector(__s1.val[0], __s1.val[0], 1, 0); \ + __rev1.val[1] = __builtin_shufflevector(__s1.val[1], __s1.val[1], 1, 0); \ + __rev1.val[2] = __builtin_shufflevector(__s1.val[2], __s1.val[2], 1, 0); \ + __rev1.val[3] = __builtin_shufflevector(__s1.val[3], __s1.val[3], 1, 0); \ + __builtin_neon_vst4q_lane_v(__p0, __rev1.val[0], __rev1.val[1], __rev1.val[2], __rev1.val[3], __p2, 35); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 19); \ +}) +#else +#define vst4_lane_u64(__p0, __p1, __p2) __extension__ ({ \ + uint64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, (int8x8_t)__s1.val[0], (int8x8_t)__s1.val[1], (int8x8_t)__s1.val[2], (int8x8_t)__s1.val[3], __p2, 19); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 10); \ +}) +#else +#define vst4_lane_f64(__p0, __p1, __p2) __extension__ ({ \ + float64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 10); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vst4_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 3); \ +}) +#else +#define vst4_lane_s64(__p0, __p1, __p2) __extension__ ({ \ + int64x1x4_t __s1 = __p1; \ + __builtin_neon_vst4_lane_v(__p0, __s1.val[0], __s1.val[1], __s1.val[2], __s1.val[3], __p2, 3); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vstrq_p128(__p0, __p1) __extension__ ({ \ + poly128_t __s1 = __p1; \ + __builtin_neon_vstrq_p128(__p0, __s1); \ +}) +#else +#define vstrq_p128(__p0, __p1) __extension__ ({ \ + poly128_t __s1 = __p1; \ + __builtin_neon_vstrq_p128(__p0, __s1); \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vsubd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vsubd_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vsubd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vsubd_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vsubd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vsubd_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vsubd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vsubd_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vsubq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai float64x2_t vsubq_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = __rev0 - __rev1; + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x1_t vsub_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#else +__ai float64x1_t vsub_f64(float64x1_t __p0, float64x1_t __p1) { + float64x1_t __ret; + __ret = __p0 - __p1; + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vsubhn_high_u32(uint16x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint16x8_t __ret; + __ret = vcombine_u16(__p0, vsubhn_u32(__p1, __p2)); + return __ret; +} +#else +__ai uint16x8_t vsubhn_high_u32(uint16x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vcombine_u16(__rev0, __noswap_vsubhn_u32(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsubhn_high_u64(uint32x2_t __p0, uint64x2_t __p1, uint64x2_t __p2) { + uint32x4_t __ret; + __ret = vcombine_u32(__p0, vsubhn_u64(__p1, __p2)); + return __ret; +} +#else +__ai uint32x4_t vsubhn_high_u64(uint32x2_t __p0, uint64x2_t __p1, uint64x2_t __p2) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vcombine_u32(__rev0, __noswap_vsubhn_u64(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vsubhn_high_u16(uint8x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint8x16_t __ret; + __ret = vcombine_u8(__p0, vsubhn_u16(__p1, __p2)); + return __ret; +} +#else +__ai uint8x16_t vsubhn_high_u16(uint8x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __noswap_vcombine_u8(__rev0, __noswap_vsubhn_u16(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vsubhn_high_s32(int16x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int16x8_t __ret; + __ret = vcombine_s16(__p0, vsubhn_s32(__p1, __p2)); + return __ret; +} +#else +__ai int16x8_t vsubhn_high_s32(int16x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vcombine_s16(__rev0, __noswap_vsubhn_s32(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vsubhn_high_s64(int32x2_t __p0, int64x2_t __p1, int64x2_t __p2) { + int32x4_t __ret; + __ret = vcombine_s32(__p0, vsubhn_s64(__p1, __p2)); + return __ret; +} +#else +__ai int32x4_t vsubhn_high_s64(int32x2_t __p0, int64x2_t __p1, int64x2_t __p2) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vcombine_s32(__rev0, __noswap_vsubhn_s64(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vsubhn_high_s16(int8x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int8x16_t __ret; + __ret = vcombine_s8(__p0, vsubhn_s16(__p1, __p2)); + return __ret; +} +#else +__ai int8x16_t vsubhn_high_s16(int8x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __noswap_vcombine_s8(__rev0, __noswap_vsubhn_s16(__rev1, __rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vsubl_high_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint16x8_t __ret; + __ret = vmovl_high_u8(__p0) - vmovl_high_u8(__p1); + return __ret; +} +#else +__ai uint16x8_t vsubl_high_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vmovl_high_u8(__rev0) - __noswap_vmovl_high_u8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vsubl_high_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint64x2_t __ret; + __ret = vmovl_high_u32(__p0) - vmovl_high_u32(__p1); + return __ret; +} +#else +__ai uint64x2_t vsubl_high_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint64x2_t __ret; + __ret = __noswap_vmovl_high_u32(__rev0) - __noswap_vmovl_high_u32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsubl_high_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint32x4_t __ret; + __ret = vmovl_high_u16(__p0) - vmovl_high_u16(__p1); + return __ret; +} +#else +__ai uint32x4_t vsubl_high_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vmovl_high_u16(__rev0) - __noswap_vmovl_high_u16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vsubl_high_s8(int8x16_t __p0, int8x16_t __p1) { + int16x8_t __ret; + __ret = vmovl_high_s8(__p0) - vmovl_high_s8(__p1); + return __ret; +} +#else +__ai int16x8_t vsubl_high_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vmovl_high_s8(__rev0) - __noswap_vmovl_high_s8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vsubl_high_s32(int32x4_t __p0, int32x4_t __p1) { + int64x2_t __ret; + __ret = vmovl_high_s32(__p0) - vmovl_high_s32(__p1); + return __ret; +} +#else +__ai int64x2_t vsubl_high_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vmovl_high_s32(__rev0) - __noswap_vmovl_high_s32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vsubl_high_s16(int16x8_t __p0, int16x8_t __p1) { + int32x4_t __ret; + __ret = vmovl_high_s16(__p0) - vmovl_high_s16(__p1); + return __ret; +} +#else +__ai int32x4_t vsubl_high_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vmovl_high_s16(__rev0) - __noswap_vmovl_high_s16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vsubw_high_u8(uint16x8_t __p0, uint8x16_t __p1) { + uint16x8_t __ret; + __ret = __p0 - vmovl_high_u8(__p1); + return __ret; +} +#else +__ai uint16x8_t vsubw_high_u8(uint16x8_t __p0, uint8x16_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 - __noswap_vmovl_high_u8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vsubw_high_u32(uint64x2_t __p0, uint32x4_t __p1) { + uint64x2_t __ret; + __ret = __p0 - vmovl_high_u32(__p1); + return __ret; +} +#else +__ai uint64x2_t vsubw_high_u32(uint64x2_t __p0, uint32x4_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint64x2_t __ret; + __ret = __rev0 - __noswap_vmovl_high_u32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vsubw_high_u16(uint32x4_t __p0, uint16x8_t __p1) { + uint32x4_t __ret; + __ret = __p0 - vmovl_high_u16(__p1); + return __ret; +} +#else +__ai uint32x4_t vsubw_high_u16(uint32x4_t __p0, uint16x8_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 - __noswap_vmovl_high_u16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vsubw_high_s8(int16x8_t __p0, int8x16_t __p1) { + int16x8_t __ret; + __ret = __p0 - vmovl_high_s8(__p1); + return __ret; +} +#else +__ai int16x8_t vsubw_high_s8(int16x8_t __p0, int8x16_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 - __noswap_vmovl_high_s8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vsubw_high_s32(int64x2_t __p0, int32x4_t __p1) { + int64x2_t __ret; + __ret = __p0 - vmovl_high_s32(__p1); + return __ret; +} +#else +__ai int64x2_t vsubw_high_s32(int64x2_t __p0, int32x4_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __rev0 - __noswap_vmovl_high_s32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vsubw_high_s16(int32x4_t __p0, int16x8_t __p1) { + int32x4_t __ret; + __ret = __p0 - vmovl_high_s16(__p1); + return __ret; +} +#else +__ai int32x4_t vsubw_high_s16(int32x4_t __p0, int16x8_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 - __noswap_vmovl_high_s16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vtrn1_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 8, 2, 10, 4, 12, 6, 14); + return __ret; +} +#else +__ai poly8x8_t vtrn1_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 8, 2, 10, 4, 12, 6, 14); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vtrn1_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 4, 2, 6); + return __ret; +} +#else +__ai poly16x4_t vtrn1_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + poly16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 4, 2, 6); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vtrn1q_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30); + return __ret; +} +#else +__ai poly8x16_t vtrn1q_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vtrn1q_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai poly64x2_t vtrn1q_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + poly64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vtrn1q_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 8, 2, 10, 4, 12, 6, 14); + return __ret; +} +#else +__ai poly16x8_t vtrn1q_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 8, 2, 10, 4, 12, 6, 14); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vtrn1q_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30); + return __ret; +} +#else +__ai uint8x16_t vtrn1q_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vtrn1q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 4, 2, 6); + return __ret; +} +#else +__ai uint32x4_t vtrn1q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 4, 2, 6); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vtrn1q_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai uint64x2_t vtrn1q_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vtrn1q_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 8, 2, 10, 4, 12, 6, 14); + return __ret; +} +#else +__ai uint16x8_t vtrn1q_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 8, 2, 10, 4, 12, 6, 14); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vtrn1q_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30); + return __ret; +} +#else +__ai int8x16_t vtrn1q_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vtrn1q_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai float64x2_t vtrn1q_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vtrn1q_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 4, 2, 6); + return __ret; +} +#else +__ai float32x4_t vtrn1q_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 4, 2, 6); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vtrn1q_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 4, 2, 6); + return __ret; +} +#else +__ai int32x4_t vtrn1q_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 4, 2, 6); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vtrn1q_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai int64x2_t vtrn1q_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vtrn1q_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 8, 2, 10, 4, 12, 6, 14); + return __ret; +} +#else +__ai int16x8_t vtrn1q_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 8, 2, 10, 4, 12, 6, 14); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vtrn1_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 8, 2, 10, 4, 12, 6, 14); + return __ret; +} +#else +__ai uint8x8_t vtrn1_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 8, 2, 10, 4, 12, 6, 14); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vtrn1_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai uint32x2_t vtrn1_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vtrn1_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 4, 2, 6); + return __ret; +} +#else +__ai uint16x4_t vtrn1_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 4, 2, 6); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vtrn1_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 8, 2, 10, 4, 12, 6, 14); + return __ret; +} +#else +__ai int8x8_t vtrn1_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 8, 2, 10, 4, 12, 6, 14); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vtrn1_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai float32x2_t vtrn1_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vtrn1_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai int32x2_t vtrn1_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vtrn1_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 4, 2, 6); + return __ret; +} +#else +__ai int16x4_t vtrn1_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 4, 2, 6); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vtrn2_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 9, 3, 11, 5, 13, 7, 15); + return __ret; +} +#else +__ai poly8x8_t vtrn2_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 9, 3, 11, 5, 13, 7, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vtrn2_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 5, 3, 7); + return __ret; +} +#else +__ai poly16x4_t vtrn2_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + poly16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 5, 3, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vtrn2q_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31); + return __ret; +} +#else +__ai poly8x16_t vtrn2q_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vtrn2q_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai poly64x2_t vtrn2q_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + poly64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vtrn2q_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 9, 3, 11, 5, 13, 7, 15); + return __ret; +} +#else +__ai poly16x8_t vtrn2q_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 9, 3, 11, 5, 13, 7, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vtrn2q_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31); + return __ret; +} +#else +__ai uint8x16_t vtrn2q_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vtrn2q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 5, 3, 7); + return __ret; +} +#else +__ai uint32x4_t vtrn2q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 5, 3, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vtrn2q_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai uint64x2_t vtrn2q_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vtrn2q_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 9, 3, 11, 5, 13, 7, 15); + return __ret; +} +#else +__ai uint16x8_t vtrn2q_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 9, 3, 11, 5, 13, 7, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vtrn2q_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31); + return __ret; +} +#else +__ai int8x16_t vtrn2q_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vtrn2q_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai float64x2_t vtrn2q_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vtrn2q_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 5, 3, 7); + return __ret; +} +#else +__ai float32x4_t vtrn2q_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 5, 3, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vtrn2q_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 5, 3, 7); + return __ret; +} +#else +__ai int32x4_t vtrn2q_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 5, 3, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vtrn2q_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai int64x2_t vtrn2q_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vtrn2q_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 9, 3, 11, 5, 13, 7, 15); + return __ret; +} +#else +__ai int16x8_t vtrn2q_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 9, 3, 11, 5, 13, 7, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vtrn2_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 9, 3, 11, 5, 13, 7, 15); + return __ret; +} +#else +__ai uint8x8_t vtrn2_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 9, 3, 11, 5, 13, 7, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vtrn2_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai uint32x2_t vtrn2_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vtrn2_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 5, 3, 7); + return __ret; +} +#else +__ai uint16x4_t vtrn2_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 5, 3, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vtrn2_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 9, 3, 11, 5, 13, 7, 15); + return __ret; +} +#else +__ai int8x8_t vtrn2_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 9, 3, 11, 5, 13, 7, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vtrn2_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai float32x2_t vtrn2_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vtrn2_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai int32x2_t vtrn2_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vtrn2_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 5, 3, 7); + return __ret; +} +#else +__ai int16x4_t vtrn2_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 5, 3, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vtst_p64(poly64x1_t __p0, poly64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vtst_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vtst_p64(poly64x1_t __p0, poly64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vtst_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vtstq_p64(poly64x2_t __p0, poly64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vtstq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vtstq_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vtstq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vtstq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vtstq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vtstq_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vtstq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vtstq_s64(int64x2_t __p0, int64x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vtstq_v((int8x16_t)__p0, (int8x16_t)__p1, 51); + return __ret; +} +#else +__ai uint64x2_t vtstq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t) __builtin_neon_vtstq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 51); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vtst_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vtst_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vtst_u64(uint64x1_t __p0, uint64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vtst_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x1_t vtst_s64(int64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vtst_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#else +__ai uint64x1_t vtst_s64(int64x1_t __p0, int64x1_t __p1) { + uint64x1_t __ret; + __ret = (uint64x1_t) __builtin_neon_vtst_v((int8x8_t)__p0, (int8x8_t)__p1, 19); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64_t vtstd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vtstd_u64(__p0, __p1); + return __ret; +} +#else +__ai uint64_t vtstd_u64(uint64_t __p0, uint64_t __p1) { + uint64_t __ret; + __ret = (uint64_t) __builtin_neon_vtstd_u64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vtstd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vtstd_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vtstd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vtstd_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8_t vuqaddb_s8(int8_t __p0, int8_t __p1) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vuqaddb_s8(__p0, __p1); + return __ret; +} +#else +__ai int8_t vuqaddb_s8(int8_t __p0, int8_t __p1) { + int8_t __ret; + __ret = (int8_t) __builtin_neon_vuqaddb_s8(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vuqadds_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vuqadds_s32(__p0, __p1); + return __ret; +} +#else +__ai int32_t vuqadds_s32(int32_t __p0, int32_t __p1) { + int32_t __ret; + __ret = (int32_t) __builtin_neon_vuqadds_s32(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64_t vuqaddd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vuqaddd_s64(__p0, __p1); + return __ret; +} +#else +__ai int64_t vuqaddd_s64(int64_t __p0, int64_t __p1) { + int64_t __ret; + __ret = (int64_t) __builtin_neon_vuqaddd_s64(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vuqaddh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vuqaddh_s16(__p0, __p1); + return __ret; +} +#else +__ai int16_t vuqaddh_s16(int16_t __p0, int16_t __p1) { + int16_t __ret; + __ret = (int16_t) __builtin_neon_vuqaddh_s16(__p0, __p1); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vuqaddq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vuqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 32); + return __ret; +} +#else +__ai int8x16_t vuqaddq_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = (int8x16_t) __builtin_neon_vuqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 32); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vuqaddq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vuqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 34); + return __ret; +} +#else +__ai int32x4_t vuqaddq_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t) __builtin_neon_vuqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 34); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vuqaddq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vuqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 35); + return __ret; +} +#else +__ai int64x2_t vuqaddq_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t) __builtin_neon_vuqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 35); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vuqaddq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vuqaddq_v((int8x16_t)__p0, (int8x16_t)__p1, 33); + return __ret; +} +#else +__ai int16x8_t vuqaddq_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t) __builtin_neon_vuqaddq_v((int8x16_t)__rev0, (int8x16_t)__rev1, 33); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vuqadd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vuqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 0); + return __ret; +} +#else +__ai int8x8_t vuqadd_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = (int8x8_t) __builtin_neon_vuqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 0); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vuqadd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vuqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 2); + return __ret; +} +#else +__ai int32x2_t vuqadd_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = (int32x2_t) __builtin_neon_vuqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x1_t vuqadd_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vuqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#else +__ai int64x1_t vuqadd_s64(int64x1_t __p0, int64x1_t __p1) { + int64x1_t __ret; + __ret = (int64x1_t) __builtin_neon_vuqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 3); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vuqadd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vuqadd_v((int8x8_t)__p0, (int8x8_t)__p1, 1); + return __ret; +} +#else +__ai int16x4_t vuqadd_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = (int16x4_t) __builtin_neon_vuqadd_v((int8x8_t)__rev0, (int8x8_t)__rev1, 1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vuzp1_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6, 8, 10, 12, 14); + return __ret; +} +#else +__ai poly8x8_t vuzp1_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6, 8, 10, 12, 14); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vuzp1_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6); + return __ret; +} +#else +__ai poly16x4_t vuzp1_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + poly16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vuzp1q_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + return __ret; +} +#else +__ai poly8x16_t vuzp1q_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vuzp1q_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai poly64x2_t vuzp1q_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + poly64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vuzp1q_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6, 8, 10, 12, 14); + return __ret; +} +#else +__ai poly16x8_t vuzp1q_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6, 8, 10, 12, 14); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vuzp1q_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + return __ret; +} +#else +__ai uint8x16_t vuzp1q_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vuzp1q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6); + return __ret; +} +#else +__ai uint32x4_t vuzp1q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vuzp1q_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai uint64x2_t vuzp1q_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vuzp1q_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6, 8, 10, 12, 14); + return __ret; +} +#else +__ai uint16x8_t vuzp1q_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6, 8, 10, 12, 14); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vuzp1q_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + return __ret; +} +#else +__ai int8x16_t vuzp1q_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vuzp1q_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai float64x2_t vuzp1q_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vuzp1q_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6); + return __ret; +} +#else +__ai float32x4_t vuzp1q_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vuzp1q_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6); + return __ret; +} +#else +__ai int32x4_t vuzp1q_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vuzp1q_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai int64x2_t vuzp1q_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vuzp1q_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6, 8, 10, 12, 14); + return __ret; +} +#else +__ai int16x8_t vuzp1q_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6, 8, 10, 12, 14); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vuzp1_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6, 8, 10, 12, 14); + return __ret; +} +#else +__ai uint8x8_t vuzp1_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6, 8, 10, 12, 14); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vuzp1_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai uint32x2_t vuzp1_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vuzp1_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6); + return __ret; +} +#else +__ai uint16x4_t vuzp1_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vuzp1_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6, 8, 10, 12, 14); + return __ret; +} +#else +__ai int8x8_t vuzp1_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6, 8, 10, 12, 14); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vuzp1_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai float32x2_t vuzp1_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vuzp1_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai int32x2_t vuzp1_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vuzp1_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2, 4, 6); + return __ret; +} +#else +__ai int16x4_t vuzp1_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2, 4, 6); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vuzp2_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7, 9, 11, 13, 15); + return __ret; +} +#else +__ai poly8x8_t vuzp2_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7, 9, 11, 13, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vuzp2_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7); + return __ret; +} +#else +__ai poly16x4_t vuzp2_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + poly16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vuzp2q_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + return __ret; +} +#else +__ai poly8x16_t vuzp2q_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vuzp2q_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai poly64x2_t vuzp2q_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + poly64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vuzp2q_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7, 9, 11, 13, 15); + return __ret; +} +#else +__ai poly16x8_t vuzp2q_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7, 9, 11, 13, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vuzp2q_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + return __ret; +} +#else +__ai uint8x16_t vuzp2q_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vuzp2q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7); + return __ret; +} +#else +__ai uint32x4_t vuzp2q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vuzp2q_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai uint64x2_t vuzp2q_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vuzp2q_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7, 9, 11, 13, 15); + return __ret; +} +#else +__ai uint16x8_t vuzp2q_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7, 9, 11, 13, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vuzp2q_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + return __ret; +} +#else +__ai int8x16_t vuzp2q_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vuzp2q_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai float64x2_t vuzp2q_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vuzp2q_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7); + return __ret; +} +#else +__ai float32x4_t vuzp2q_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vuzp2q_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7); + return __ret; +} +#else +__ai int32x4_t vuzp2q_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vuzp2q_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai int64x2_t vuzp2q_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vuzp2q_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7, 9, 11, 13, 15); + return __ret; +} +#else +__ai int16x8_t vuzp2q_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7, 9, 11, 13, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vuzp2_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7, 9, 11, 13, 15); + return __ret; +} +#else +__ai uint8x8_t vuzp2_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7, 9, 11, 13, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vuzp2_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai uint32x2_t vuzp2_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vuzp2_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7); + return __ret; +} +#else +__ai uint16x4_t vuzp2_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vuzp2_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7, 9, 11, 13, 15); + return __ret; +} +#else +__ai int8x8_t vuzp2_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7, 9, 11, 13, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vuzp2_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai float32x2_t vuzp2_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vuzp2_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai int32x2_t vuzp2_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vuzp2_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3, 5, 7); + return __ret; +} +#else +__ai int16x4_t vuzp2_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3, 5, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vzip1_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 8, 1, 9, 2, 10, 3, 11); + return __ret; +} +#else +__ai poly8x8_t vzip1_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 8, 1, 9, 2, 10, 3, 11); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vzip1_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 4, 1, 5); + return __ret; +} +#else +__ai poly16x4_t vzip1_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + poly16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 4, 1, 5); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vzip1q_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23); + return __ret; +} +#else +__ai poly8x16_t vzip1q_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vzip1q_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai poly64x2_t vzip1q_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + poly64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vzip1q_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 8, 1, 9, 2, 10, 3, 11); + return __ret; +} +#else +__ai poly16x8_t vzip1q_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 8, 1, 9, 2, 10, 3, 11); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vzip1q_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23); + return __ret; +} +#else +__ai uint8x16_t vzip1q_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vzip1q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 4, 1, 5); + return __ret; +} +#else +__ai uint32x4_t vzip1q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 4, 1, 5); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vzip1q_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai uint64x2_t vzip1q_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vzip1q_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 8, 1, 9, 2, 10, 3, 11); + return __ret; +} +#else +__ai uint16x8_t vzip1q_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 8, 1, 9, 2, 10, 3, 11); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vzip1q_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23); + return __ret; +} +#else +__ai int8x16_t vzip1q_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vzip1q_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai float64x2_t vzip1q_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vzip1q_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 4, 1, 5); + return __ret; +} +#else +__ai float32x4_t vzip1q_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 4, 1, 5); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vzip1q_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 4, 1, 5); + return __ret; +} +#else +__ai int32x4_t vzip1q_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 4, 1, 5); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vzip1q_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai int64x2_t vzip1q_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vzip1q_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 8, 1, 9, 2, 10, 3, 11); + return __ret; +} +#else +__ai int16x8_t vzip1q_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 8, 1, 9, 2, 10, 3, 11); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vzip1_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 8, 1, 9, 2, 10, 3, 11); + return __ret; +} +#else +__ai uint8x8_t vzip1_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 8, 1, 9, 2, 10, 3, 11); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vzip1_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai uint32x2_t vzip1_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vzip1_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 4, 1, 5); + return __ret; +} +#else +__ai uint16x4_t vzip1_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 4, 1, 5); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vzip1_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 8, 1, 9, 2, 10, 3, 11); + return __ret; +} +#else +__ai int8x8_t vzip1_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 8, 1, 9, 2, 10, 3, 11); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vzip1_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai float32x2_t vzip1_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vzip1_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 2); + return __ret; +} +#else +__ai int32x2_t vzip1_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vzip1_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 0, 4, 1, 5); + return __ret; +} +#else +__ai int16x4_t vzip1_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 0, 4, 1, 5); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x8_t vzip2_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 4, 12, 5, 13, 6, 14, 7, 15); + return __ret; +} +#else +__ai poly8x8_t vzip2_p8(poly8x8_t __p0, poly8x8_t __p1) { + poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 4, 12, 5, 13, 6, 14, 7, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x4_t vzip2_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 2, 6, 3, 7); + return __ret; +} +#else +__ai poly16x4_t vzip2_p16(poly16x4_t __p0, poly16x4_t __p1) { + poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + poly16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 2, 6, 3, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly8x16_t vzip2q_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31); + return __ret; +} +#else +__ai poly8x16_t vzip2q_p8(poly8x16_t __p0, poly8x16_t __p1) { + poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + poly8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly64x2_t vzip2q_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai poly64x2_t vzip2q_p64(poly64x2_t __p0, poly64x2_t __p1) { + poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + poly64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai poly16x8_t vzip2q_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 4, 12, 5, 13, 6, 14, 7, 15); + return __ret; +} +#else +__ai poly16x8_t vzip2q_p16(poly16x8_t __p0, poly16x8_t __p1) { + poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + poly16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 4, 12, 5, 13, 6, 14, 7, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vzip2q_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31); + return __ret; +} +#else +__ai uint8x16_t vzip2q_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vzip2q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 2, 6, 3, 7); + return __ret; +} +#else +__ai uint32x4_t vzip2q_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 2, 6, 3, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vzip2q_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai uint64x2_t vzip2q_u64(uint64x2_t __p0, uint64x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vzip2q_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 4, 12, 5, 13, 6, 14, 7, 15); + return __ret; +} +#else +__ai uint16x8_t vzip2q_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 4, 12, 5, 13, 6, 14, 7, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vzip2q_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31); + return __ret; +} +#else +__ai int8x16_t vzip2q_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float64x2_t vzip2q_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai float64x2_t vzip2q_f64(float64x2_t __p0, float64x2_t __p1) { + float64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x4_t vzip2q_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 2, 6, 3, 7); + return __ret; +} +#else +__ai float32x4_t vzip2q_f32(float32x4_t __p0, float32x4_t __p1) { + float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + float32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + float32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 2, 6, 3, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vzip2q_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 2, 6, 3, 7); + return __ret; +} +#else +__ai int32x4_t vzip2q_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 2, 6, 3, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vzip2q_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai int64x2_t vzip2q_s64(int64x2_t __p0, int64x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int64x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vzip2q_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 4, 12, 5, 13, 6, 14, 7, 15); + return __ret; +} +#else +__ai int16x8_t vzip2q_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 4, 12, 5, 13, 6, 14, 7, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vzip2_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 4, 12, 5, 13, 6, 14, 7, 15); + return __ret; +} +#else +__ai uint8x8_t vzip2_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 4, 12, 5, 13, 6, 14, 7, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vzip2_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai uint32x2_t vzip2_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vzip2_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 2, 6, 3, 7); + return __ret; +} +#else +__ai uint16x4_t vzip2_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 2, 6, 3, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vzip2_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 4, 12, 5, 13, 6, 14, 7, 15); + return __ret; +} +#else +__ai int8x8_t vzip2_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 4, 12, 5, 13, 6, 14, 7, 15); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai float32x2_t vzip2_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai float32x2_t vzip2_f32(float32x2_t __p0, float32x2_t __p1) { + float32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + float32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + float32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vzip2_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 1, 3); + return __ret; +} +#else +__ai int32x2_t vzip2_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 1, 3); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vzip2_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __ret; + __ret = __builtin_shufflevector(__p0, __p1, 2, 6, 3, 7); + return __ret; +} +#else +__ai int16x4_t vzip2_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __builtin_shufflevector(__rev0, __rev1, 2, 6, 3, 7); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#endif +#ifdef __LITTLE_ENDIAN__ +__ai uint8x16_t vabaq_u8(uint8x16_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint8x16_t __ret; + __ret = __p0 + vabdq_u8(__p1, __p2); + return __ret; +} +#else +__ai uint8x16_t vabaq_u8(uint8x16_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __ret; + __ret = __rev0 + __noswap_vabdq_u8(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vabaq_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __ret; + __ret = __p0 + vabdq_u32(__p1, __p2); + return __ret; +} +#else +__ai uint32x4_t vabaq_u32(uint32x4_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 + __noswap_vabdq_u32(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vabaq_u16(uint16x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint16x8_t __ret; + __ret = __p0 + vabdq_u16(__p1, __p2); + return __ret; +} +#else +__ai uint16x8_t vabaq_u16(uint16x8_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 + __noswap_vabdq_u16(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x16_t vabaq_s8(int8x16_t __p0, int8x16_t __p1, int8x16_t __p2) { + int8x16_t __ret; + __ret = __p0 + vabdq_s8(__p1, __p2); + return __ret; +} +#else +__ai int8x16_t vabaq_s8(int8x16_t __p0, int8x16_t __p1, int8x16_t __p2) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __ret; + __ret = __rev0 + __noswap_vabdq_s8(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vabaq_s32(int32x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int32x4_t __ret; + __ret = __p0 + vabdq_s32(__p1, __p2); + return __ret; +} +#else +__ai int32x4_t vabaq_s32(int32x4_t __p0, int32x4_t __p1, int32x4_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 + __noswap_vabdq_s32(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vabaq_s16(int16x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int16x8_t __ret; + __ret = __p0 + vabdq_s16(__p1, __p2); + return __ret; +} +#else +__ai int16x8_t vabaq_s16(int16x8_t __p0, int16x8_t __p1, int16x8_t __p2) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 + __noswap_vabdq_s16(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint8x8_t vaba_u8(uint8x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint8x8_t __ret; + __ret = __p0 + vabd_u8(__p1, __p2); + return __ret; +} +#else +__ai uint8x8_t vaba_u8(uint8x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __ret; + __ret = __rev0 + __noswap_vabd_u8(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x2_t vaba_u32(uint32x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint32x2_t __ret; + __ret = __p0 + vabd_u32(__p1, __p2); + return __ret; +} +#else +__ai uint32x2_t vaba_u32(uint32x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + uint32x2_t __ret; + __ret = __rev0 + __noswap_vabd_u32(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x4_t vaba_u16(uint16x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint16x4_t __ret; + __ret = __p0 + vabd_u16(__p1, __p2); + return __ret; +} +#else +__ai uint16x4_t vaba_u16(uint16x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint16x4_t __ret; + __ret = __rev0 + __noswap_vabd_u16(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int8x8_t vaba_s8(int8x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int8x8_t __ret; + __ret = __p0 + vabd_s8(__p1, __p2); + return __ret; +} +#else +__ai int8x8_t vaba_s8(int8x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __ret; + __ret = __rev0 + __noswap_vabd_s8(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x2_t vaba_s32(int32x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int32x2_t __ret; + __ret = __p0 + vabd_s32(__p1, __p2); + return __ret; +} +#else +__ai int32x2_t vaba_s32(int32x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int32x2_t __ret; + __ret = __rev0 + __noswap_vabd_s32(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x4_t vaba_s16(int16x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int16x4_t __ret; + __ret = __p0 + vabd_s16(__p1, __p2); + return __ret; +} +#else +__ai int16x4_t vaba_s16(int16x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int16x4_t __ret; + __ret = __rev0 + __noswap_vabd_s16(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vabdl_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(vmovl_u8((uint8x8_t)(vabd_u8(__p0, __p1)))); + return __ret; +} +#else +__ai uint16x8_t vabdl_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = (uint16x8_t)(__noswap_vmovl_u8((uint8x8_t)(__noswap_vabd_u8(__rev0, __rev1)))); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint16x8_t __noswap_vabdl_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint16x8_t __ret; + __ret = (uint16x8_t)(__noswap_vmovl_u8((uint8x8_t)(__noswap_vabd_u8(__p0, __p1)))); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vabdl_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(vmovl_u32((uint32x2_t)(vabd_u32(__p0, __p1)))); + return __ret; +} +#else +__ai uint64x2_t vabdl_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = (uint64x2_t)(__noswap_vmovl_u32((uint32x2_t)(__noswap_vabd_u32(__rev0, __rev1)))); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint64x2_t __noswap_vabdl_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint64x2_t __ret; + __ret = (uint64x2_t)(__noswap_vmovl_u32((uint32x2_t)(__noswap_vabd_u32(__p0, __p1)))); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vabdl_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(vmovl_u16((uint16x4_t)(vabd_u16(__p0, __p1)))); + return __ret; +} +#else +__ai uint32x4_t vabdl_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = (uint32x4_t)(__noswap_vmovl_u16((uint16x4_t)(__noswap_vabd_u16(__rev0, __rev1)))); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint32x4_t __noswap_vabdl_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint32x4_t __ret; + __ret = (uint32x4_t)(__noswap_vmovl_u16((uint16x4_t)(__noswap_vabd_u16(__p0, __p1)))); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vabdl_s8(int8x8_t __p0, int8x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t)(vmovl_u8((uint8x8_t)(vabd_s8(__p0, __p1)))); + return __ret; +} +#else +__ai int16x8_t vabdl_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = (int16x8_t)(__noswap_vmovl_u8((uint8x8_t)(__noswap_vabd_s8(__rev0, __rev1)))); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int16x8_t __noswap_vabdl_s8(int8x8_t __p0, int8x8_t __p1) { + int16x8_t __ret; + __ret = (int16x8_t)(__noswap_vmovl_u8((uint8x8_t)(__noswap_vabd_s8(__p0, __p1)))); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vabdl_s32(int32x2_t __p0, int32x2_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t)(vmovl_u32((uint32x2_t)(vabd_s32(__p0, __p1)))); + return __ret; +} +#else +__ai int64x2_t vabdl_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = (int64x2_t)(__noswap_vmovl_u32((uint32x2_t)(__noswap_vabd_s32(__rev0, __rev1)))); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vabdl_s32(int32x2_t __p0, int32x2_t __p1) { + int64x2_t __ret; + __ret = (int64x2_t)(__noswap_vmovl_u32((uint32x2_t)(__noswap_vabd_s32(__p0, __p1)))); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vabdl_s16(int16x4_t __p0, int16x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t)(vmovl_u16((uint16x4_t)(vabd_s16(__p0, __p1)))); + return __ret; +} +#else +__ai int32x4_t vabdl_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = (int32x4_t)(__noswap_vmovl_u16((uint16x4_t)(__noswap_vabd_s16(__rev0, __rev1)))); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vabdl_s16(int16x4_t __p0, int16x4_t __p1) { + int32x4_t __ret; + __ret = (int32x4_t)(__noswap_vmovl_u16((uint16x4_t)(__noswap_vabd_s16(__p0, __p1)))); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vaddl_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint16x8_t __ret; + __ret = vmovl_u8(__p0) + vmovl_u8(__p1); + return __ret; +} +#else +__ai uint16x8_t vaddl_u8(uint8x8_t __p0, uint8x8_t __p1) { + uint8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vmovl_u8(__rev0) + __noswap_vmovl_u8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vaddl_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint64x2_t __ret; + __ret = vmovl_u32(__p0) + vmovl_u32(__p1); + return __ret; +} +#else +__ai uint64x2_t vaddl_u32(uint32x2_t __p0, uint32x2_t __p1) { + uint32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __noswap_vmovl_u32(__rev0) + __noswap_vmovl_u32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vaddl_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint32x4_t __ret; + __ret = vmovl_u16(__p0) + vmovl_u16(__p1); + return __ret; +} +#else +__ai uint32x4_t vaddl_u16(uint16x4_t __p0, uint16x4_t __p1) { + uint16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vmovl_u16(__rev0) + __noswap_vmovl_u16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vaddl_s8(int8x8_t __p0, int8x8_t __p1) { + int16x8_t __ret; + __ret = vmovl_s8(__p0) + vmovl_s8(__p1); + return __ret; +} +#else +__ai int16x8_t vaddl_s8(int8x8_t __p0, int8x8_t __p1) { + int8x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vmovl_s8(__rev0) + __noswap_vmovl_s8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vaddl_s32(int32x2_t __p0, int32x2_t __p1) { + int64x2_t __ret; + __ret = vmovl_s32(__p0) + vmovl_s32(__p1); + return __ret; +} +#else +__ai int64x2_t vaddl_s32(int32x2_t __p0, int32x2_t __p1) { + int32x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __noswap_vmovl_s32(__rev0) + __noswap_vmovl_s32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vaddl_s16(int16x4_t __p0, int16x4_t __p1) { + int32x4_t __ret; + __ret = vmovl_s16(__p0) + vmovl_s16(__p1); + return __ret; +} +#else +__ai int32x4_t vaddl_s16(int16x4_t __p0, int16x4_t __p1) { + int16x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vmovl_s16(__rev0) + __noswap_vmovl_s16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vaddw_u8(uint16x8_t __p0, uint8x8_t __p1) { + uint16x8_t __ret; + __ret = __p0 + vmovl_u8(__p1); + return __ret; +} +#else +__ai uint16x8_t vaddw_u8(uint16x8_t __p0, uint8x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 + __noswap_vmovl_u8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vaddw_u32(uint64x2_t __p0, uint32x2_t __p1) { + uint64x2_t __ret; + __ret = __p0 + vmovl_u32(__p1); + return __ret; +} +#else +__ai uint64x2_t vaddw_u32(uint64x2_t __p0, uint32x2_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __rev0 + __noswap_vmovl_u32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vaddw_u16(uint32x4_t __p0, uint16x4_t __p1) { + uint32x4_t __ret; + __ret = __p0 + vmovl_u16(__p1); + return __ret; +} +#else +__ai uint32x4_t vaddw_u16(uint32x4_t __p0, uint16x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 + __noswap_vmovl_u16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vaddw_s8(int16x8_t __p0, int8x8_t __p1) { + int16x8_t __ret; + __ret = __p0 + vmovl_s8(__p1); + return __ret; +} +#else +__ai int16x8_t vaddw_s8(int16x8_t __p0, int8x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 + __noswap_vmovl_s8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vaddw_s32(int64x2_t __p0, int32x2_t __p1) { + int64x2_t __ret; + __ret = __p0 + vmovl_s32(__p1); + return __ret; +} +#else +__ai int64x2_t vaddw_s32(int64x2_t __p0, int32x2_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __rev0 + __noswap_vmovl_s32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vaddw_s16(int32x4_t __p0, int16x4_t __p1) { + int32x4_t __ret; + __ret = __p0 + vmovl_s16(__p1); + return __ret; +} +#else +__ai int32x4_t vaddw_s16(int32x4_t __p0, int16x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 + __noswap_vmovl_s16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmlal_u8(uint16x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint16x8_t __ret; + __ret = __p0 + vmull_u8(__p1, __p2); + return __ret; +} +#else +__ai uint16x8_t vmlal_u8(uint16x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 + __noswap_vmull_u8(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint16x8_t __noswap_vmlal_u8(uint16x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint16x8_t __ret; + __ret = __p0 + __noswap_vmull_u8(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmlal_u32(uint64x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint64x2_t __ret; + __ret = __p0 + vmull_u32(__p1, __p2); + return __ret; +} +#else +__ai uint64x2_t vmlal_u32(uint64x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + uint64x2_t __ret; + __ret = __rev0 + __noswap_vmull_u32(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint64x2_t __noswap_vmlal_u32(uint64x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint64x2_t __ret; + __ret = __p0 + __noswap_vmull_u32(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmlal_u16(uint32x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint32x4_t __ret; + __ret = __p0 + vmull_u16(__p1, __p2); + return __ret; +} +#else +__ai uint32x4_t vmlal_u16(uint32x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 + __noswap_vmull_u16(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint32x4_t __noswap_vmlal_u16(uint32x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint32x4_t __ret; + __ret = __p0 + __noswap_vmull_u16(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmlal_s8(int16x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int16x8_t __ret; + __ret = __p0 + vmull_s8(__p1, __p2); + return __ret; +} +#else +__ai int16x8_t vmlal_s8(int16x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 + __noswap_vmull_s8(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int16x8_t __noswap_vmlal_s8(int16x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int16x8_t __ret; + __ret = __p0 + __noswap_vmull_s8(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmlal_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __ret; + __ret = __p0 + vmull_s32(__p1, __p2); + return __ret; +} +#else +__ai int64x2_t vmlal_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int64x2_t __ret; + __ret = __rev0 + __noswap_vmull_s32(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vmlal_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __ret; + __ret = __p0 + __noswap_vmull_s32(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmlal_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __ret; + __ret = __p0 + vmull_s16(__p1, __p2); + return __ret; +} +#else +__ai int32x4_t vmlal_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 + __noswap_vmull_s16(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vmlal_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __ret; + __ret = __p0 + __noswap_vmull_s16(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint64x2_t __ret; \ + __ret = __s0 + vmull_u32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + uint64x2_t __ret; \ + __ret = __rev0 + __noswap_vmull_u32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint32x4_t __ret; \ + __ret = __s0 + vmull_u16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __rev0 + __noswap_vmull_u16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = __s0 + vmull_s32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int64x2_t __ret; \ + __ret = __rev0 + __noswap_vmull_s32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlal_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = __s0 + vmull_s16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlal_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __rev0 + __noswap_vmull_s16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmlal_n_u32(uint64x2_t __p0, uint32x2_t __p1, uint32_t __p2) { + uint64x2_t __ret; + __ret = __p0 + vmull_u32(__p1, (uint32x2_t) {__p2, __p2}); + return __ret; +} +#else +__ai uint64x2_t vmlal_n_u32(uint64x2_t __p0, uint32x2_t __p1, uint32_t __p2) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __rev0 + __noswap_vmull_u32(__rev1, (uint32x2_t) {__p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint64x2_t __noswap_vmlal_n_u32(uint64x2_t __p0, uint32x2_t __p1, uint32_t __p2) { + uint64x2_t __ret; + __ret = __p0 + __noswap_vmull_u32(__p1, (uint32x2_t) {__p2, __p2}); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmlal_n_u16(uint32x4_t __p0, uint16x4_t __p1, uint16_t __p2) { + uint32x4_t __ret; + __ret = __p0 + vmull_u16(__p1, (uint16x4_t) {__p2, __p2, __p2, __p2}); + return __ret; +} +#else +__ai uint32x4_t vmlal_n_u16(uint32x4_t __p0, uint16x4_t __p1, uint16_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 + __noswap_vmull_u16(__rev1, (uint16x4_t) {__p2, __p2, __p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint32x4_t __noswap_vmlal_n_u16(uint32x4_t __p0, uint16x4_t __p1, uint16_t __p2) { + uint32x4_t __ret; + __ret = __p0 + __noswap_vmull_u16(__p1, (uint16x4_t) {__p2, __p2, __p2, __p2}); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmlal_n_s32(int64x2_t __p0, int32x2_t __p1, int32_t __p2) { + int64x2_t __ret; + __ret = __p0 + vmull_s32(__p1, (int32x2_t) {__p2, __p2}); + return __ret; +} +#else +__ai int64x2_t vmlal_n_s32(int64x2_t __p0, int32x2_t __p1, int32_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __rev0 + __noswap_vmull_s32(__rev1, (int32x2_t) {__p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vmlal_n_s32(int64x2_t __p0, int32x2_t __p1, int32_t __p2) { + int64x2_t __ret; + __ret = __p0 + __noswap_vmull_s32(__p1, (int32x2_t) {__p2, __p2}); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmlal_n_s16(int32x4_t __p0, int16x4_t __p1, int16_t __p2) { + int32x4_t __ret; + __ret = __p0 + vmull_s16(__p1, (int16x4_t) {__p2, __p2, __p2, __p2}); + return __ret; +} +#else +__ai int32x4_t vmlal_n_s16(int32x4_t __p0, int16x4_t __p1, int16_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 + __noswap_vmull_s16(__rev1, (int16x4_t) {__p2, __p2, __p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vmlal_n_s16(int32x4_t __p0, int16x4_t __p1, int16_t __p2) { + int32x4_t __ret; + __ret = __p0 + __noswap_vmull_s16(__p1, (int16x4_t) {__p2, __p2, __p2, __p2}); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmlsl_u8(uint16x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint16x8_t __ret; + __ret = __p0 - vmull_u8(__p1, __p2); + return __ret; +} +#else +__ai uint16x8_t vmlsl_u8(uint16x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 - __noswap_vmull_u8(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint16x8_t __noswap_vmlsl_u8(uint16x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint16x8_t __ret; + __ret = __p0 - __noswap_vmull_u8(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmlsl_u32(uint64x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint64x2_t __ret; + __ret = __p0 - vmull_u32(__p1, __p2); + return __ret; +} +#else +__ai uint64x2_t vmlsl_u32(uint64x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + uint64x2_t __ret; + __ret = __rev0 - __noswap_vmull_u32(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint64x2_t __noswap_vmlsl_u32(uint64x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint64x2_t __ret; + __ret = __p0 - __noswap_vmull_u32(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmlsl_u16(uint32x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint32x4_t __ret; + __ret = __p0 - vmull_u16(__p1, __p2); + return __ret; +} +#else +__ai uint32x4_t vmlsl_u16(uint32x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 - __noswap_vmull_u16(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint32x4_t __noswap_vmlsl_u16(uint32x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint32x4_t __ret; + __ret = __p0 - __noswap_vmull_u16(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmlsl_s8(int16x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int16x8_t __ret; + __ret = __p0 - vmull_s8(__p1, __p2); + return __ret; +} +#else +__ai int16x8_t vmlsl_s8(int16x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 - __noswap_vmull_s8(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int16x8_t __noswap_vmlsl_s8(int16x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int16x8_t __ret; + __ret = __p0 - __noswap_vmull_s8(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmlsl_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __ret; + __ret = __p0 - vmull_s32(__p1, __p2); + return __ret; +} +#else +__ai int64x2_t vmlsl_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int64x2_t __ret; + __ret = __rev0 - __noswap_vmull_s32(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vmlsl_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __ret; + __ret = __p0 - __noswap_vmull_s32(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmlsl_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __ret; + __ret = __p0 - vmull_s16(__p1, __p2); + return __ret; +} +#else +__ai int32x4_t vmlsl_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 - __noswap_vmull_s16(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vmlsl_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __ret; + __ret = __p0 - __noswap_vmull_s16(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint64x2_t __ret; \ + __ret = __s0 - vmull_u32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_lane_u32(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint64x2_t __s0 = __p0; \ + uint32x2_t __s1 = __p1; \ + uint32x2_t __s2 = __p2; \ + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + uint64x2_t __ret; \ + __ret = __rev0 - __noswap_vmull_u32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint32x4_t __ret; \ + __ret = __s0 - vmull_u16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_lane_u16(__p0, __p1, __p2, __p3) __extension__ ({ \ + uint32x4_t __s0 = __p0; \ + uint16x4_t __s1 = __p1; \ + uint16x4_t __s2 = __p2; \ + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + uint32x4_t __ret; \ + __ret = __rev0 - __noswap_vmull_u16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __ret; \ + __ret = __s0 - vmull_s32(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_lane_s32(__p0, __p1, __p2, __p3) __extension__ ({ \ + int64x2_t __s0 = __p0; \ + int32x2_t __s1 = __p1; \ + int32x2_t __s2 = __p2; \ + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 1, 0); \ + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \ + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 1, 0); \ + int64x2_t __ret; \ + __ret = __rev0 - __noswap_vmull_s32(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmlsl_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __ret; \ + __ret = __s0 - vmull_s16(__s1, __builtin_shufflevector(__s2, __s2, __p3, __p3, __p3, __p3)); \ + __ret; \ +}) +#else +#define vmlsl_lane_s16(__p0, __p1, __p2, __p3) __extension__ ({ \ + int32x4_t __s0 = __p0; \ + int16x4_t __s1 = __p1; \ + int16x4_t __s2 = __p2; \ + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, 3, 2, 1, 0); \ + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, 3, 2, 1, 0); \ + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__s2, __s2, 3, 2, 1, 0); \ + int32x4_t __ret; \ + __ret = __rev0 - __noswap_vmull_s16(__rev1, __builtin_shufflevector(__rev2, __rev2, __p3, __p3, __p3, __p3)); \ + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); \ + __ret; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmlsl_n_u32(uint64x2_t __p0, uint32x2_t __p1, uint32_t __p2) { + uint64x2_t __ret; + __ret = __p0 - vmull_u32(__p1, (uint32x2_t) {__p2, __p2}); + return __ret; +} +#else +__ai uint64x2_t vmlsl_n_u32(uint64x2_t __p0, uint32x2_t __p1, uint32_t __p2) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint64x2_t __ret; + __ret = __rev0 - __noswap_vmull_u32(__rev1, (uint32x2_t) {__p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint64x2_t __noswap_vmlsl_n_u32(uint64x2_t __p0, uint32x2_t __p1, uint32_t __p2) { + uint64x2_t __ret; + __ret = __p0 - __noswap_vmull_u32(__p1, (uint32x2_t) {__p2, __p2}); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmlsl_n_u16(uint32x4_t __p0, uint16x4_t __p1, uint16_t __p2) { + uint32x4_t __ret; + __ret = __p0 - vmull_u16(__p1, (uint16x4_t) {__p2, __p2, __p2, __p2}); + return __ret; +} +#else +__ai uint32x4_t vmlsl_n_u16(uint32x4_t __p0, uint16x4_t __p1, uint16_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 - __noswap_vmull_u16(__rev1, (uint16x4_t) {__p2, __p2, __p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint32x4_t __noswap_vmlsl_n_u16(uint32x4_t __p0, uint16x4_t __p1, uint16_t __p2) { + uint32x4_t __ret; + __ret = __p0 - __noswap_vmull_u16(__p1, (uint16x4_t) {__p2, __p2, __p2, __p2}); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmlsl_n_s32(int64x2_t __p0, int32x2_t __p1, int32_t __p2) { + int64x2_t __ret; + __ret = __p0 - vmull_s32(__p1, (int32x2_t) {__p2, __p2}); + return __ret; +} +#else +__ai int64x2_t vmlsl_n_s32(int64x2_t __p0, int32x2_t __p1, int32_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int64x2_t __ret; + __ret = __rev0 - __noswap_vmull_s32(__rev1, (int32x2_t) {__p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vmlsl_n_s32(int64x2_t __p0, int32x2_t __p1, int32_t __p2) { + int64x2_t __ret; + __ret = __p0 - __noswap_vmull_s32(__p1, (int32x2_t) {__p2, __p2}); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmlsl_n_s16(int32x4_t __p0, int16x4_t __p1, int16_t __p2) { + int32x4_t __ret; + __ret = __p0 - vmull_s16(__p1, (int16x4_t) {__p2, __p2, __p2, __p2}); + return __ret; +} +#else +__ai int32x4_t vmlsl_n_s16(int32x4_t __p0, int16x4_t __p1, int16_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 - __noswap_vmull_s16(__rev1, (int16x4_t) {__p2, __p2, __p2, __p2}); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vmlsl_n_s16(int32x4_t __p0, int16x4_t __p1, int16_t __p2) { + int32x4_t __ret; + __ret = __p0 - __noswap_vmull_s16(__p1, (int16x4_t) {__p2, __p2, __p2, __p2}); + return __ret; +} +#endif + +#if defined(__ARM_FEATURE_FP16_SCALAR_ARITHMETIC) +#ifdef __LITTLE_ENDIAN__ +#define vfmsh_lane_f16(__p0_258, __p1_258, __p2_258, __p3_258) __extension__ ({ \ + float16_t __s0_258 = __p0_258; \ + float16_t __s1_258 = __p1_258; \ + float16x4_t __s2_258 = __p2_258; \ + float16_t __ret_258; \ + __ret_258 = vfmsh_f16(__s0_258, __s1_258, vget_lane_f16(__s2_258, __p3_258)); \ + __ret_258; \ +}) +#else +#define vfmsh_lane_f16(__p0_259, __p1_259, __p2_259, __p3_259) __extension__ ({ \ + float16_t __s0_259 = __p0_259; \ + float16_t __s1_259 = __p1_259; \ + float16x4_t __s2_259 = __p2_259; \ + float16x4_t __rev2_259; __rev2_259 = __builtin_shufflevector(__s2_259, __s2_259, 3, 2, 1, 0); \ + float16_t __ret_259; \ + __ret_259 = __noswap_vfmsh_f16(__s0_259, __s1_259, __noswap_vget_lane_f16(__rev2_259, __p3_259)); \ + __ret_259; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vfmsh_laneq_f16(__p0_260, __p1_260, __p2_260, __p3_260) __extension__ ({ \ + float16_t __s0_260 = __p0_260; \ + float16_t __s1_260 = __p1_260; \ + float16x8_t __s2_260 = __p2_260; \ + float16_t __ret_260; \ + __ret_260 = vfmsh_f16(__s0_260, __s1_260, vgetq_lane_f16(__s2_260, __p3_260)); \ + __ret_260; \ +}) +#else +#define vfmsh_laneq_f16(__p0_261, __p1_261, __p2_261, __p3_261) __extension__ ({ \ + float16_t __s0_261 = __p0_261; \ + float16_t __s1_261 = __p1_261; \ + float16x8_t __s2_261 = __p2_261; \ + float16x8_t __rev2_261; __rev2_261 = __builtin_shufflevector(__s2_261, __s2_261, 7, 6, 5, 4, 3, 2, 1, 0); \ + float16_t __ret_261; \ + __ret_261 = __noswap_vfmsh_f16(__s0_261, __s1_261, __noswap_vgetq_lane_f16(__rev2_261, __p3_261)); \ + __ret_261; \ +}) +#endif + +#endif +#if defined(__ARM_FEATURE_QRDMX) && defined(__aarch64__) +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqrdmlahs_s32(int32_t __p0, int32_t __p1, int32_t __p2) { + int32_t __ret; + __ret = vqadds_s32(__p0, vqrdmulhs_s32(__p1, __p2)); + return __ret; +} +#else +__ai int32_t vqrdmlahs_s32(int32_t __p0, int32_t __p1, int32_t __p2) { + int32_t __ret; + __ret = __noswap_vqadds_s32(__p0, __noswap_vqrdmulhs_s32(__p1, __p2)); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vqrdmlahh_s16(int16_t __p0, int16_t __p1, int16_t __p2) { + int16_t __ret; + __ret = vqaddh_s16(__p0, vqrdmulhh_s16(__p1, __p2)); + return __ret; +} +#else +__ai int16_t vqrdmlahh_s16(int16_t __p0, int16_t __p1, int16_t __p2) { + int16_t __ret; + __ret = __noswap_vqaddh_s16(__p0, __noswap_vqrdmulhh_s16(__p1, __p2)); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlahs_lane_s32(__p0_262, __p1_262, __p2_262, __p3_262) __extension__ ({ \ + int32_t __s0_262 = __p0_262; \ + int32_t __s1_262 = __p1_262; \ + int32x2_t __s2_262 = __p2_262; \ + int32_t __ret_262; \ + __ret_262 = vqadds_s32(__s0_262, vqrdmulhs_s32(__s1_262, vget_lane_s32(__s2_262, __p3_262))); \ + __ret_262; \ +}) +#else +#define vqrdmlahs_lane_s32(__p0_263, __p1_263, __p2_263, __p3_263) __extension__ ({ \ + int32_t __s0_263 = __p0_263; \ + int32_t __s1_263 = __p1_263; \ + int32x2_t __s2_263 = __p2_263; \ + int32x2_t __rev2_263; __rev2_263 = __builtin_shufflevector(__s2_263, __s2_263, 1, 0); \ + int32_t __ret_263; \ + __ret_263 = __noswap_vqadds_s32(__s0_263, __noswap_vqrdmulhs_s32(__s1_263, __noswap_vget_lane_s32(__rev2_263, __p3_263))); \ + __ret_263; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlahh_lane_s16(__p0_264, __p1_264, __p2_264, __p3_264) __extension__ ({ \ + int16_t __s0_264 = __p0_264; \ + int16_t __s1_264 = __p1_264; \ + int16x4_t __s2_264 = __p2_264; \ + int16_t __ret_264; \ + __ret_264 = vqaddh_s16(__s0_264, vqrdmulhh_s16(__s1_264, vget_lane_s16(__s2_264, __p3_264))); \ + __ret_264; \ +}) +#else +#define vqrdmlahh_lane_s16(__p0_265, __p1_265, __p2_265, __p3_265) __extension__ ({ \ + int16_t __s0_265 = __p0_265; \ + int16_t __s1_265 = __p1_265; \ + int16x4_t __s2_265 = __p2_265; \ + int16x4_t __rev2_265; __rev2_265 = __builtin_shufflevector(__s2_265, __s2_265, 3, 2, 1, 0); \ + int16_t __ret_265; \ + __ret_265 = __noswap_vqaddh_s16(__s0_265, __noswap_vqrdmulhh_s16(__s1_265, __noswap_vget_lane_s16(__rev2_265, __p3_265))); \ + __ret_265; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlahs_laneq_s32(__p0_266, __p1_266, __p2_266, __p3_266) __extension__ ({ \ + int32_t __s0_266 = __p0_266; \ + int32_t __s1_266 = __p1_266; \ + int32x4_t __s2_266 = __p2_266; \ + int32_t __ret_266; \ + __ret_266 = vqadds_s32(__s0_266, vqrdmulhs_s32(__s1_266, vgetq_lane_s32(__s2_266, __p3_266))); \ + __ret_266; \ +}) +#else +#define vqrdmlahs_laneq_s32(__p0_267, __p1_267, __p2_267, __p3_267) __extension__ ({ \ + int32_t __s0_267 = __p0_267; \ + int32_t __s1_267 = __p1_267; \ + int32x4_t __s2_267 = __p2_267; \ + int32x4_t __rev2_267; __rev2_267 = __builtin_shufflevector(__s2_267, __s2_267, 3, 2, 1, 0); \ + int32_t __ret_267; \ + __ret_267 = __noswap_vqadds_s32(__s0_267, __noswap_vqrdmulhs_s32(__s1_267, __noswap_vgetq_lane_s32(__rev2_267, __p3_267))); \ + __ret_267; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlahh_laneq_s16(__p0_268, __p1_268, __p2_268, __p3_268) __extension__ ({ \ + int16_t __s0_268 = __p0_268; \ + int16_t __s1_268 = __p1_268; \ + int16x8_t __s2_268 = __p2_268; \ + int16_t __ret_268; \ + __ret_268 = vqaddh_s16(__s0_268, vqrdmulhh_s16(__s1_268, vgetq_lane_s16(__s2_268, __p3_268))); \ + __ret_268; \ +}) +#else +#define vqrdmlahh_laneq_s16(__p0_269, __p1_269, __p2_269, __p3_269) __extension__ ({ \ + int16_t __s0_269 = __p0_269; \ + int16_t __s1_269 = __p1_269; \ + int16x8_t __s2_269 = __p2_269; \ + int16x8_t __rev2_269; __rev2_269 = __builtin_shufflevector(__s2_269, __s2_269, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16_t __ret_269; \ + __ret_269 = __noswap_vqaddh_s16(__s0_269, __noswap_vqrdmulhh_s16(__s1_269, __noswap_vgetq_lane_s16(__rev2_269, __p3_269))); \ + __ret_269; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32_t vqrdmlshs_s32(int32_t __p0, int32_t __p1, int32_t __p2) { + int32_t __ret; + __ret = vqsubs_s32(__p0, vqrdmulhs_s32(__p1, __p2)); + return __ret; +} +#else +__ai int32_t vqrdmlshs_s32(int32_t __p0, int32_t __p1, int32_t __p2) { + int32_t __ret; + __ret = __noswap_vqsubs_s32(__p0, __noswap_vqrdmulhs_s32(__p1, __p2)); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16_t vqrdmlshh_s16(int16_t __p0, int16_t __p1, int16_t __p2) { + int16_t __ret; + __ret = vqsubh_s16(__p0, vqrdmulhh_s16(__p1, __p2)); + return __ret; +} +#else +__ai int16_t vqrdmlshh_s16(int16_t __p0, int16_t __p1, int16_t __p2) { + int16_t __ret; + __ret = __noswap_vqsubh_s16(__p0, __noswap_vqrdmulhh_s16(__p1, __p2)); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlshs_lane_s32(__p0_270, __p1_270, __p2_270, __p3_270) __extension__ ({ \ + int32_t __s0_270 = __p0_270; \ + int32_t __s1_270 = __p1_270; \ + int32x2_t __s2_270 = __p2_270; \ + int32_t __ret_270; \ + __ret_270 = vqsubs_s32(__s0_270, vqrdmulhs_s32(__s1_270, vget_lane_s32(__s2_270, __p3_270))); \ + __ret_270; \ +}) +#else +#define vqrdmlshs_lane_s32(__p0_271, __p1_271, __p2_271, __p3_271) __extension__ ({ \ + int32_t __s0_271 = __p0_271; \ + int32_t __s1_271 = __p1_271; \ + int32x2_t __s2_271 = __p2_271; \ + int32x2_t __rev2_271; __rev2_271 = __builtin_shufflevector(__s2_271, __s2_271, 1, 0); \ + int32_t __ret_271; \ + __ret_271 = __noswap_vqsubs_s32(__s0_271, __noswap_vqrdmulhs_s32(__s1_271, __noswap_vget_lane_s32(__rev2_271, __p3_271))); \ + __ret_271; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlshh_lane_s16(__p0_272, __p1_272, __p2_272, __p3_272) __extension__ ({ \ + int16_t __s0_272 = __p0_272; \ + int16_t __s1_272 = __p1_272; \ + int16x4_t __s2_272 = __p2_272; \ + int16_t __ret_272; \ + __ret_272 = vqsubh_s16(__s0_272, vqrdmulhh_s16(__s1_272, vget_lane_s16(__s2_272, __p3_272))); \ + __ret_272; \ +}) +#else +#define vqrdmlshh_lane_s16(__p0_273, __p1_273, __p2_273, __p3_273) __extension__ ({ \ + int16_t __s0_273 = __p0_273; \ + int16_t __s1_273 = __p1_273; \ + int16x4_t __s2_273 = __p2_273; \ + int16x4_t __rev2_273; __rev2_273 = __builtin_shufflevector(__s2_273, __s2_273, 3, 2, 1, 0); \ + int16_t __ret_273; \ + __ret_273 = __noswap_vqsubh_s16(__s0_273, __noswap_vqrdmulhh_s16(__s1_273, __noswap_vget_lane_s16(__rev2_273, __p3_273))); \ + __ret_273; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlshs_laneq_s32(__p0_274, __p1_274, __p2_274, __p3_274) __extension__ ({ \ + int32_t __s0_274 = __p0_274; \ + int32_t __s1_274 = __p1_274; \ + int32x4_t __s2_274 = __p2_274; \ + int32_t __ret_274; \ + __ret_274 = vqsubs_s32(__s0_274, vqrdmulhs_s32(__s1_274, vgetq_lane_s32(__s2_274, __p3_274))); \ + __ret_274; \ +}) +#else +#define vqrdmlshs_laneq_s32(__p0_275, __p1_275, __p2_275, __p3_275) __extension__ ({ \ + int32_t __s0_275 = __p0_275; \ + int32_t __s1_275 = __p1_275; \ + int32x4_t __s2_275 = __p2_275; \ + int32x4_t __rev2_275; __rev2_275 = __builtin_shufflevector(__s2_275, __s2_275, 3, 2, 1, 0); \ + int32_t __ret_275; \ + __ret_275 = __noswap_vqsubs_s32(__s0_275, __noswap_vqrdmulhs_s32(__s1_275, __noswap_vgetq_lane_s32(__rev2_275, __p3_275))); \ + __ret_275; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vqrdmlshh_laneq_s16(__p0_276, __p1_276, __p2_276, __p3_276) __extension__ ({ \ + int16_t __s0_276 = __p0_276; \ + int16_t __s1_276 = __p1_276; \ + int16x8_t __s2_276 = __p2_276; \ + int16_t __ret_276; \ + __ret_276 = vqsubh_s16(__s0_276, vqrdmulhh_s16(__s1_276, vgetq_lane_s16(__s2_276, __p3_276))); \ + __ret_276; \ +}) +#else +#define vqrdmlshh_laneq_s16(__p0_277, __p1_277, __p2_277, __p3_277) __extension__ ({ \ + int16_t __s0_277 = __p0_277; \ + int16_t __s1_277 = __p1_277; \ + int16x8_t __s2_277 = __p2_277; \ + int16x8_t __rev2_277; __rev2_277 = __builtin_shufflevector(__s2_277, __s2_277, 7, 6, 5, 4, 3, 2, 1, 0); \ + int16_t __ret_277; \ + __ret_277 = __noswap_vqsubh_s16(__s0_277, __noswap_vqrdmulhh_s16(__s1_277, __noswap_vgetq_lane_s16(__rev2_277, __p3_277))); \ + __ret_277; \ +}) +#endif + +#endif +#if defined(__aarch64__) +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vabdl_high_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint16x8_t __ret; + __ret = vabdl_u8(vget_high_u8(__p0), vget_high_u8(__p1)); + return __ret; +} +#else +__ai uint16x8_t vabdl_high_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vabdl_u8(__noswap_vget_high_u8(__rev0), __noswap_vget_high_u8(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vabdl_high_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint64x2_t __ret; + __ret = vabdl_u32(vget_high_u32(__p0), vget_high_u32(__p1)); + return __ret; +} +#else +__ai uint64x2_t vabdl_high_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint64x2_t __ret; + __ret = __noswap_vabdl_u32(__noswap_vget_high_u32(__rev0), __noswap_vget_high_u32(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vabdl_high_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint32x4_t __ret; + __ret = vabdl_u16(vget_high_u16(__p0), vget_high_u16(__p1)); + return __ret; +} +#else +__ai uint32x4_t vabdl_high_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vabdl_u16(__noswap_vget_high_u16(__rev0), __noswap_vget_high_u16(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vabdl_high_s8(int8x16_t __p0, int8x16_t __p1) { + int16x8_t __ret; + __ret = vabdl_s8(vget_high_s8(__p0), vget_high_s8(__p1)); + return __ret; +} +#else +__ai int16x8_t vabdl_high_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vabdl_s8(__noswap_vget_high_s8(__rev0), __noswap_vget_high_s8(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vabdl_high_s32(int32x4_t __p0, int32x4_t __p1) { + int64x2_t __ret; + __ret = vabdl_s32(vget_high_s32(__p0), vget_high_s32(__p1)); + return __ret; +} +#else +__ai int64x2_t vabdl_high_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vabdl_s32(__noswap_vget_high_s32(__rev0), __noswap_vget_high_s32(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vabdl_high_s16(int16x8_t __p0, int16x8_t __p1) { + int32x4_t __ret; + __ret = vabdl_s16(vget_high_s16(__p0), vget_high_s16(__p1)); + return __ret; +} +#else +__ai int32x4_t vabdl_high_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vabdl_s16(__noswap_vget_high_s16(__rev0), __noswap_vget_high_s16(__rev1)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vaddl_high_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint16x8_t __ret; + __ret = vmovl_high_u8(__p0) + vmovl_high_u8(__p1); + return __ret; +} +#else +__ai uint16x8_t vaddl_high_u8(uint8x16_t __p0, uint8x16_t __p1) { + uint8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vmovl_high_u8(__rev0) + __noswap_vmovl_high_u8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vaddl_high_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint64x2_t __ret; + __ret = vmovl_high_u32(__p0) + vmovl_high_u32(__p1); + return __ret; +} +#else +__ai uint64x2_t vaddl_high_u32(uint32x4_t __p0, uint32x4_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint64x2_t __ret; + __ret = __noswap_vmovl_high_u32(__rev0) + __noswap_vmovl_high_u32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vaddl_high_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint32x4_t __ret; + __ret = vmovl_high_u16(__p0) + vmovl_high_u16(__p1); + return __ret; +} +#else +__ai uint32x4_t vaddl_high_u16(uint16x8_t __p0, uint16x8_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vmovl_high_u16(__rev0) + __noswap_vmovl_high_u16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vaddl_high_s8(int8x16_t __p0, int8x16_t __p1) { + int16x8_t __ret; + __ret = vmovl_high_s8(__p0) + vmovl_high_s8(__p1); + return __ret; +} +#else +__ai int16x8_t vaddl_high_s8(int8x16_t __p0, int8x16_t __p1) { + int8x16_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vmovl_high_s8(__rev0) + __noswap_vmovl_high_s8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vaddl_high_s32(int32x4_t __p0, int32x4_t __p1) { + int64x2_t __ret; + __ret = vmovl_high_s32(__p0) + vmovl_high_s32(__p1); + return __ret; +} +#else +__ai int64x2_t vaddl_high_s32(int32x4_t __p0, int32x4_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vmovl_high_s32(__rev0) + __noswap_vmovl_high_s32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vaddl_high_s16(int16x8_t __p0, int16x8_t __p1) { + int32x4_t __ret; + __ret = vmovl_high_s16(__p0) + vmovl_high_s16(__p1); + return __ret; +} +#else +__ai int32x4_t vaddl_high_s16(int16x8_t __p0, int16x8_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vmovl_high_s16(__rev0) + __noswap_vmovl_high_s16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vaddw_high_u8(uint16x8_t __p0, uint8x16_t __p1) { + uint16x8_t __ret; + __ret = __p0 + vmovl_high_u8(__p1); + return __ret; +} +#else +__ai uint16x8_t vaddw_high_u8(uint16x8_t __p0, uint8x16_t __p1) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 + __noswap_vmovl_high_u8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vaddw_high_u32(uint64x2_t __p0, uint32x4_t __p1) { + uint64x2_t __ret; + __ret = __p0 + vmovl_high_u32(__p1); + return __ret; +} +#else +__ai uint64x2_t vaddw_high_u32(uint64x2_t __p0, uint32x4_t __p1) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint64x2_t __ret; + __ret = __rev0 + __noswap_vmovl_high_u32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vaddw_high_u16(uint32x4_t __p0, uint16x8_t __p1) { + uint32x4_t __ret; + __ret = __p0 + vmovl_high_u16(__p1); + return __ret; +} +#else +__ai uint32x4_t vaddw_high_u16(uint32x4_t __p0, uint16x8_t __p1) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 + __noswap_vmovl_high_u16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vaddw_high_s8(int16x8_t __p0, int8x16_t __p1) { + int16x8_t __ret; + __ret = __p0 + vmovl_high_s8(__p1); + return __ret; +} +#else +__ai int16x8_t vaddw_high_s8(int16x8_t __p0, int8x16_t __p1) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 + __noswap_vmovl_high_s8(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vaddw_high_s32(int64x2_t __p0, int32x4_t __p1) { + int64x2_t __ret; + __ret = __p0 + vmovl_high_s32(__p1); + return __ret; +} +#else +__ai int64x2_t vaddw_high_s32(int64x2_t __p0, int32x4_t __p1) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __rev0 + __noswap_vmovl_high_s32(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vaddw_high_s16(int32x4_t __p0, int16x8_t __p1) { + int32x4_t __ret; + __ret = __p0 + vmovl_high_s16(__p1); + return __ret; +} +#else +__ai int32x4_t vaddw_high_s16(int32x4_t __p0, int16x8_t __p1) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 + __noswap_vmovl_high_s16(__rev1); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_p64(__p0_278, __p1_278, __p2_278, __p3_278) __extension__ ({ \ + poly64x2_t __s0_278 = __p0_278; \ + poly64x1_t __s2_278 = __p2_278; \ + poly64x2_t __ret_278; \ + __ret_278 = vsetq_lane_p64(vget_lane_p64(__s2_278, __p3_278), __s0_278, __p1_278); \ + __ret_278; \ +}) +#else +#define vcopyq_lane_p64(__p0_279, __p1_279, __p2_279, __p3_279) __extension__ ({ \ + poly64x2_t __s0_279 = __p0_279; \ + poly64x1_t __s2_279 = __p2_279; \ + poly64x2_t __rev0_279; __rev0_279 = __builtin_shufflevector(__s0_279, __s0_279, 1, 0); \ + poly64x2_t __ret_279; \ + __ret_279 = __noswap_vsetq_lane_p64(__noswap_vget_lane_p64(__s2_279, __p3_279), __rev0_279, __p1_279); \ + __ret_279 = __builtin_shufflevector(__ret_279, __ret_279, 1, 0); \ + __ret_279; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_lane_f64(__p0_280, __p1_280, __p2_280, __p3_280) __extension__ ({ \ + float64x2_t __s0_280 = __p0_280; \ + float64x1_t __s2_280 = __p2_280; \ + float64x2_t __ret_280; \ + __ret_280 = vsetq_lane_f64(vget_lane_f64(__s2_280, __p3_280), __s0_280, __p1_280); \ + __ret_280; \ +}) +#else +#define vcopyq_lane_f64(__p0_281, __p1_281, __p2_281, __p3_281) __extension__ ({ \ + float64x2_t __s0_281 = __p0_281; \ + float64x1_t __s2_281 = __p2_281; \ + float64x2_t __rev0_281; __rev0_281 = __builtin_shufflevector(__s0_281, __s0_281, 1, 0); \ + float64x2_t __ret_281; \ + __ret_281 = __noswap_vsetq_lane_f64(__noswap_vget_lane_f64(__s2_281, __p3_281), __rev0_281, __p1_281); \ + __ret_281 = __builtin_shufflevector(__ret_281, __ret_281, 1, 0); \ + __ret_281; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_p64(__p0_282, __p1_282, __p2_282, __p3_282) __extension__ ({ \ + poly64x1_t __s0_282 = __p0_282; \ + poly64x1_t __s2_282 = __p2_282; \ + poly64x1_t __ret_282; \ + __ret_282 = vset_lane_p64(vget_lane_p64(__s2_282, __p3_282), __s0_282, __p1_282); \ + __ret_282; \ +}) +#else +#define vcopy_lane_p64(__p0_283, __p1_283, __p2_283, __p3_283) __extension__ ({ \ + poly64x1_t __s0_283 = __p0_283; \ + poly64x1_t __s2_283 = __p2_283; \ + poly64x1_t __ret_283; \ + __ret_283 = __noswap_vset_lane_p64(__noswap_vget_lane_p64(__s2_283, __p3_283), __s0_283, __p1_283); \ + __ret_283; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_lane_f64(__p0_284, __p1_284, __p2_284, __p3_284) __extension__ ({ \ + float64x1_t __s0_284 = __p0_284; \ + float64x1_t __s2_284 = __p2_284; \ + float64x1_t __ret_284; \ + __ret_284 = vset_lane_f64(vget_lane_f64(__s2_284, __p3_284), __s0_284, __p1_284); \ + __ret_284; \ +}) +#else +#define vcopy_lane_f64(__p0_285, __p1_285, __p2_285, __p3_285) __extension__ ({ \ + float64x1_t __s0_285 = __p0_285; \ + float64x1_t __s2_285 = __p2_285; \ + float64x1_t __ret_285; \ + __ret_285 = __noswap_vset_lane_f64(__noswap_vget_lane_f64(__s2_285, __p3_285), __s0_285, __p1_285); \ + __ret_285; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_laneq_p64(__p0_286, __p1_286, __p2_286, __p3_286) __extension__ ({ \ + poly64x2_t __s0_286 = __p0_286; \ + poly64x2_t __s2_286 = __p2_286; \ + poly64x2_t __ret_286; \ + __ret_286 = vsetq_lane_p64(vgetq_lane_p64(__s2_286, __p3_286), __s0_286, __p1_286); \ + __ret_286; \ +}) +#else +#define vcopyq_laneq_p64(__p0_287, __p1_287, __p2_287, __p3_287) __extension__ ({ \ + poly64x2_t __s0_287 = __p0_287; \ + poly64x2_t __s2_287 = __p2_287; \ + poly64x2_t __rev0_287; __rev0_287 = __builtin_shufflevector(__s0_287, __s0_287, 1, 0); \ + poly64x2_t __rev2_287; __rev2_287 = __builtin_shufflevector(__s2_287, __s2_287, 1, 0); \ + poly64x2_t __ret_287; \ + __ret_287 = __noswap_vsetq_lane_p64(__noswap_vgetq_lane_p64(__rev2_287, __p3_287), __rev0_287, __p1_287); \ + __ret_287 = __builtin_shufflevector(__ret_287, __ret_287, 1, 0); \ + __ret_287; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopyq_laneq_f64(__p0_288, __p1_288, __p2_288, __p3_288) __extension__ ({ \ + float64x2_t __s0_288 = __p0_288; \ + float64x2_t __s2_288 = __p2_288; \ + float64x2_t __ret_288; \ + __ret_288 = vsetq_lane_f64(vgetq_lane_f64(__s2_288, __p3_288), __s0_288, __p1_288); \ + __ret_288; \ +}) +#else +#define vcopyq_laneq_f64(__p0_289, __p1_289, __p2_289, __p3_289) __extension__ ({ \ + float64x2_t __s0_289 = __p0_289; \ + float64x2_t __s2_289 = __p2_289; \ + float64x2_t __rev0_289; __rev0_289 = __builtin_shufflevector(__s0_289, __s0_289, 1, 0); \ + float64x2_t __rev2_289; __rev2_289 = __builtin_shufflevector(__s2_289, __s2_289, 1, 0); \ + float64x2_t __ret_289; \ + __ret_289 = __noswap_vsetq_lane_f64(__noswap_vgetq_lane_f64(__rev2_289, __p3_289), __rev0_289, __p1_289); \ + __ret_289 = __builtin_shufflevector(__ret_289, __ret_289, 1, 0); \ + __ret_289; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_laneq_p64(__p0_290, __p1_290, __p2_290, __p3_290) __extension__ ({ \ + poly64x1_t __s0_290 = __p0_290; \ + poly64x2_t __s2_290 = __p2_290; \ + poly64x1_t __ret_290; \ + __ret_290 = vset_lane_p64(vgetq_lane_p64(__s2_290, __p3_290), __s0_290, __p1_290); \ + __ret_290; \ +}) +#else +#define vcopy_laneq_p64(__p0_291, __p1_291, __p2_291, __p3_291) __extension__ ({ \ + poly64x1_t __s0_291 = __p0_291; \ + poly64x2_t __s2_291 = __p2_291; \ + poly64x2_t __rev2_291; __rev2_291 = __builtin_shufflevector(__s2_291, __s2_291, 1, 0); \ + poly64x1_t __ret_291; \ + __ret_291 = __noswap_vset_lane_p64(__noswap_vgetq_lane_p64(__rev2_291, __p3_291), __s0_291, __p1_291); \ + __ret_291; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vcopy_laneq_f64(__p0_292, __p1_292, __p2_292, __p3_292) __extension__ ({ \ + float64x1_t __s0_292 = __p0_292; \ + float64x2_t __s2_292 = __p2_292; \ + float64x1_t __ret_292; \ + __ret_292 = vset_lane_f64(vgetq_lane_f64(__s2_292, __p3_292), __s0_292, __p1_292); \ + __ret_292; \ +}) +#else +#define vcopy_laneq_f64(__p0_293, __p1_293, __p2_293, __p3_293) __extension__ ({ \ + float64x1_t __s0_293 = __p0_293; \ + float64x2_t __s2_293 = __p2_293; \ + float64x2_t __rev2_293; __rev2_293 = __builtin_shufflevector(__s2_293, __s2_293, 1, 0); \ + float64x1_t __ret_293; \ + __ret_293 = __noswap_vset_lane_f64(__noswap_vgetq_lane_f64(__rev2_293, __p3_293), __s0_293, __p1_293); \ + __ret_293; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmlal_high_u8(uint16x8_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint16x8_t __ret; + __ret = vmlal_u8(__p0, vget_high_u8(__p1), vget_high_u8(__p2)); + return __ret; +} +#else +__ai uint16x8_t vmlal_high_u8(uint16x8_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vmlal_u8(__rev0, __noswap_vget_high_u8(__rev1), __noswap_vget_high_u8(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmlal_high_u32(uint64x2_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint64x2_t __ret; + __ret = vmlal_u32(__p0, vget_high_u32(__p1), vget_high_u32(__p2)); + return __ret; +} +#else +__ai uint64x2_t vmlal_high_u32(uint64x2_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint64x2_t __ret; + __ret = __noswap_vmlal_u32(__rev0, __noswap_vget_high_u32(__rev1), __noswap_vget_high_u32(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmlal_high_u16(uint32x4_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint32x4_t __ret; + __ret = vmlal_u16(__p0, vget_high_u16(__p1), vget_high_u16(__p2)); + return __ret; +} +#else +__ai uint32x4_t vmlal_high_u16(uint32x4_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vmlal_u16(__rev0, __noswap_vget_high_u16(__rev1), __noswap_vget_high_u16(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmlal_high_s8(int16x8_t __p0, int8x16_t __p1, int8x16_t __p2) { + int16x8_t __ret; + __ret = vmlal_s8(__p0, vget_high_s8(__p1), vget_high_s8(__p2)); + return __ret; +} +#else +__ai int16x8_t vmlal_high_s8(int16x8_t __p0, int8x16_t __p1, int8x16_t __p2) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vmlal_s8(__rev0, __noswap_vget_high_s8(__rev1), __noswap_vget_high_s8(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmlal_high_s32(int64x2_t __p0, int32x4_t __p1, int32x4_t __p2) { + int64x2_t __ret; + __ret = vmlal_s32(__p0, vget_high_s32(__p1), vget_high_s32(__p2)); + return __ret; +} +#else +__ai int64x2_t vmlal_high_s32(int64x2_t __p0, int32x4_t __p1, int32x4_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vmlal_s32(__rev0, __noswap_vget_high_s32(__rev1), __noswap_vget_high_s32(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmlal_high_s16(int32x4_t __p0, int16x8_t __p1, int16x8_t __p2) { + int32x4_t __ret; + __ret = vmlal_s16(__p0, vget_high_s16(__p1), vget_high_s16(__p2)); + return __ret; +} +#else +__ai int32x4_t vmlal_high_s16(int32x4_t __p0, int16x8_t __p1, int16x8_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vmlal_s16(__rev0, __noswap_vget_high_s16(__rev1), __noswap_vget_high_s16(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmlal_high_n_u32(uint64x2_t __p0, uint32x4_t __p1, uint32_t __p2) { + uint64x2_t __ret; + __ret = vmlal_n_u32(__p0, vget_high_u32(__p1), __p2); + return __ret; +} +#else +__ai uint64x2_t vmlal_high_n_u32(uint64x2_t __p0, uint32x4_t __p1, uint32_t __p2) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint64x2_t __ret; + __ret = __noswap_vmlal_n_u32(__rev0, __noswap_vget_high_u32(__rev1), __p2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmlal_high_n_u16(uint32x4_t __p0, uint16x8_t __p1, uint16_t __p2) { + uint32x4_t __ret; + __ret = vmlal_n_u16(__p0, vget_high_u16(__p1), __p2); + return __ret; +} +#else +__ai uint32x4_t vmlal_high_n_u16(uint32x4_t __p0, uint16x8_t __p1, uint16_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vmlal_n_u16(__rev0, __noswap_vget_high_u16(__rev1), __p2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmlal_high_n_s32(int64x2_t __p0, int32x4_t __p1, int32_t __p2) { + int64x2_t __ret; + __ret = vmlal_n_s32(__p0, vget_high_s32(__p1), __p2); + return __ret; +} +#else +__ai int64x2_t vmlal_high_n_s32(int64x2_t __p0, int32x4_t __p1, int32_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vmlal_n_s32(__rev0, __noswap_vget_high_s32(__rev1), __p2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmlal_high_n_s16(int32x4_t __p0, int16x8_t __p1, int16_t __p2) { + int32x4_t __ret; + __ret = vmlal_n_s16(__p0, vget_high_s16(__p1), __p2); + return __ret; +} +#else +__ai int32x4_t vmlal_high_n_s16(int32x4_t __p0, int16x8_t __p1, int16_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vmlal_n_s16(__rev0, __noswap_vget_high_s16(__rev1), __p2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vmlsl_high_u8(uint16x8_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint16x8_t __ret; + __ret = vmlsl_u8(__p0, vget_high_u8(__p1), vget_high_u8(__p2)); + return __ret; +} +#else +__ai uint16x8_t vmlsl_high_u8(uint16x8_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vmlsl_u8(__rev0, __noswap_vget_high_u8(__rev1), __noswap_vget_high_u8(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmlsl_high_u32(uint64x2_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint64x2_t __ret; + __ret = vmlsl_u32(__p0, vget_high_u32(__p1), vget_high_u32(__p2)); + return __ret; +} +#else +__ai uint64x2_t vmlsl_high_u32(uint64x2_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint64x2_t __ret; + __ret = __noswap_vmlsl_u32(__rev0, __noswap_vget_high_u32(__rev1), __noswap_vget_high_u32(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmlsl_high_u16(uint32x4_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint32x4_t __ret; + __ret = vmlsl_u16(__p0, vget_high_u16(__p1), vget_high_u16(__p2)); + return __ret; +} +#else +__ai uint32x4_t vmlsl_high_u16(uint32x4_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vmlsl_u16(__rev0, __noswap_vget_high_u16(__rev1), __noswap_vget_high_u16(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vmlsl_high_s8(int16x8_t __p0, int8x16_t __p1, int8x16_t __p2) { + int16x8_t __ret; + __ret = vmlsl_s8(__p0, vget_high_s8(__p1), vget_high_s8(__p2)); + return __ret; +} +#else +__ai int16x8_t vmlsl_high_s8(int16x8_t __p0, int8x16_t __p1, int8x16_t __p2) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vmlsl_s8(__rev0, __noswap_vget_high_s8(__rev1), __noswap_vget_high_s8(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmlsl_high_s32(int64x2_t __p0, int32x4_t __p1, int32x4_t __p2) { + int64x2_t __ret; + __ret = vmlsl_s32(__p0, vget_high_s32(__p1), vget_high_s32(__p2)); + return __ret; +} +#else +__ai int64x2_t vmlsl_high_s32(int64x2_t __p0, int32x4_t __p1, int32x4_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vmlsl_s32(__rev0, __noswap_vget_high_s32(__rev1), __noswap_vget_high_s32(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmlsl_high_s16(int32x4_t __p0, int16x8_t __p1, int16x8_t __p2) { + int32x4_t __ret; + __ret = vmlsl_s16(__p0, vget_high_s16(__p1), vget_high_s16(__p2)); + return __ret; +} +#else +__ai int32x4_t vmlsl_high_s16(int32x4_t __p0, int16x8_t __p1, int16x8_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vmlsl_s16(__rev0, __noswap_vget_high_s16(__rev1), __noswap_vget_high_s16(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vmlsl_high_n_u32(uint64x2_t __p0, uint32x4_t __p1, uint32_t __p2) { + uint64x2_t __ret; + __ret = vmlsl_n_u32(__p0, vget_high_u32(__p1), __p2); + return __ret; +} +#else +__ai uint64x2_t vmlsl_high_n_u32(uint64x2_t __p0, uint32x4_t __p1, uint32_t __p2) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint64x2_t __ret; + __ret = __noswap_vmlsl_n_u32(__rev0, __noswap_vget_high_u32(__rev1), __p2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vmlsl_high_n_u16(uint32x4_t __p0, uint16x8_t __p1, uint16_t __p2) { + uint32x4_t __ret; + __ret = vmlsl_n_u16(__p0, vget_high_u16(__p1), __p2); + return __ret; +} +#else +__ai uint32x4_t vmlsl_high_n_u16(uint32x4_t __p0, uint16x8_t __p1, uint16_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vmlsl_n_u16(__rev0, __noswap_vget_high_u16(__rev1), __p2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vmlsl_high_n_s32(int64x2_t __p0, int32x4_t __p1, int32_t __p2) { + int64x2_t __ret; + __ret = vmlsl_n_s32(__p0, vget_high_s32(__p1), __p2); + return __ret; +} +#else +__ai int64x2_t vmlsl_high_n_s32(int64x2_t __p0, int32x4_t __p1, int32_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vmlsl_n_s32(__rev0, __noswap_vget_high_s32(__rev1), __p2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vmlsl_high_n_s16(int32x4_t __p0, int16x8_t __p1, int16_t __p2) { + int32x4_t __ret; + __ret = vmlsl_n_s16(__p0, vget_high_s16(__p1), __p2); + return __ret; +} +#else +__ai int32x4_t vmlsl_high_n_s16(int32x4_t __p0, int16x8_t __p1, int16_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vmlsl_n_s16(__rev0, __noswap_vget_high_s16(__rev1), __p2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulx_lane_f64(__p0_294, __p1_294, __p2_294) __extension__ ({ \ + float64x1_t __s0_294 = __p0_294; \ + float64x1_t __s1_294 = __p1_294; \ + float64x1_t __ret_294; \ + float64_t __x_294 = vget_lane_f64(__s0_294, 0); \ + float64_t __y_294 = vget_lane_f64(__s1_294, __p2_294); \ + float64_t __z_294 = vmulxd_f64(__x_294, __y_294); \ + __ret_294 = vset_lane_f64(__z_294, __s0_294, __p2_294); \ + __ret_294; \ +}) +#else +#define vmulx_lane_f64(__p0_295, __p1_295, __p2_295) __extension__ ({ \ + float64x1_t __s0_295 = __p0_295; \ + float64x1_t __s1_295 = __p1_295; \ + float64x1_t __ret_295; \ + float64_t __x_295 = __noswap_vget_lane_f64(__s0_295, 0); \ + float64_t __y_295 = __noswap_vget_lane_f64(__s1_295, __p2_295); \ + float64_t __z_295 = __noswap_vmulxd_f64(__x_295, __y_295); \ + __ret_295 = __noswap_vset_lane_f64(__z_295, __s0_295, __p2_295); \ + __ret_295; \ +}) +#endif + +#ifdef __LITTLE_ENDIAN__ +#define vmulx_laneq_f64(__p0_296, __p1_296, __p2_296) __extension__ ({ \ + float64x1_t __s0_296 = __p0_296; \ + float64x2_t __s1_296 = __p1_296; \ + float64x1_t __ret_296; \ + float64_t __x_296 = vget_lane_f64(__s0_296, 0); \ + float64_t __y_296 = vgetq_lane_f64(__s1_296, __p2_296); \ + float64_t __z_296 = vmulxd_f64(__x_296, __y_296); \ + __ret_296 = vset_lane_f64(__z_296, __s0_296, 0); \ + __ret_296; \ +}) +#else +#define vmulx_laneq_f64(__p0_297, __p1_297, __p2_297) __extension__ ({ \ + float64x1_t __s0_297 = __p0_297; \ + float64x2_t __s1_297 = __p1_297; \ + float64x2_t __rev1_297; __rev1_297 = __builtin_shufflevector(__s1_297, __s1_297, 1, 0); \ + float64x1_t __ret_297; \ + float64_t __x_297 = __noswap_vget_lane_f64(__s0_297, 0); \ + float64_t __y_297 = __noswap_vgetq_lane_f64(__rev1_297, __p2_297); \ + float64_t __z_297 = __noswap_vmulxd_f64(__x_297, __y_297); \ + __ret_297 = __noswap_vset_lane_f64(__z_297, __s0_297, 0); \ + __ret_297; \ +}) +#endif + +#endif +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vabal_u8(uint16x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint16x8_t __ret; + __ret = __p0 + vabdl_u8(__p1, __p2); + return __ret; +} +#else +__ai uint16x8_t vabal_u8(uint16x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __rev0 + __noswap_vabdl_u8(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai uint16x8_t __noswap_vabal_u8(uint16x8_t __p0, uint8x8_t __p1, uint8x8_t __p2) { + uint16x8_t __ret; + __ret = __p0 + __noswap_vabdl_u8(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vabal_u32(uint64x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint64x2_t __ret; + __ret = __p0 + vabdl_u32(__p1, __p2); + return __ret; +} +#else +__ai uint64x2_t vabal_u32(uint64x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + uint32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + uint64x2_t __ret; + __ret = __rev0 + __noswap_vabdl_u32(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai uint64x2_t __noswap_vabal_u32(uint64x2_t __p0, uint32x2_t __p1, uint32x2_t __p2) { + uint64x2_t __ret; + __ret = __p0 + __noswap_vabdl_u32(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vabal_u16(uint32x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint32x4_t __ret; + __ret = __p0 + vabdl_u16(__p1, __p2); + return __ret; +} +#else +__ai uint32x4_t vabal_u16(uint32x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __rev0 + __noswap_vabdl_u16(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai uint32x4_t __noswap_vabal_u16(uint32x4_t __p0, uint16x4_t __p1, uint16x4_t __p2) { + uint32x4_t __ret; + __ret = __p0 + __noswap_vabdl_u16(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vabal_s8(int16x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int16x8_t __ret; + __ret = __p0 + vabdl_s8(__p1, __p2); + return __ret; +} +#else +__ai int16x8_t vabal_s8(int16x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int8x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __rev0 + __noswap_vabdl_s8(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +__ai int16x8_t __noswap_vabal_s8(int16x8_t __p0, int8x8_t __p1, int8x8_t __p2) { + int16x8_t __ret; + __ret = __p0 + __noswap_vabdl_s8(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vabal_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __ret; + __ret = __p0 + vabdl_s32(__p1, __p2); + return __ret; +} +#else +__ai int64x2_t vabal_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x2_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 1, 0); + int32x2_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 1, 0); + int64x2_t __ret; + __ret = __rev0 + __noswap_vabdl_s32(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +__ai int64x2_t __noswap_vabal_s32(int64x2_t __p0, int32x2_t __p1, int32x2_t __p2) { + int64x2_t __ret; + __ret = __p0 + __noswap_vabdl_s32(__p1, __p2); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vabal_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __ret; + __ret = __p0 + vabdl_s16(__p1, __p2); + return __ret; +} +#else +__ai int32x4_t vabal_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int16x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __rev0 + __noswap_vabdl_s16(__rev1, __rev2); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +__ai int32x4_t __noswap_vabal_s16(int32x4_t __p0, int16x4_t __p1, int16x4_t __p2) { + int32x4_t __ret; + __ret = __p0 + __noswap_vabdl_s16(__p1, __p2); + return __ret; +} +#endif + +#if defined(__aarch64__) +#ifdef __LITTLE_ENDIAN__ +__ai uint16x8_t vabal_high_u8(uint16x8_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint16x8_t __ret; + __ret = vabal_u8(__p0, vget_high_u8(__p1), vget_high_u8(__p2)); + return __ret; +} +#else +__ai uint16x8_t vabal_high_u8(uint16x8_t __p0, uint8x16_t __p1, uint8x16_t __p2) { + uint16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __ret; + __ret = __noswap_vabal_u8(__rev0, __noswap_vget_high_u8(__rev1), __noswap_vget_high_u8(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint64x2_t vabal_high_u32(uint64x2_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint64x2_t __ret; + __ret = vabal_u32(__p0, vget_high_u32(__p1), vget_high_u32(__p2)); + return __ret; +} +#else +__ai uint64x2_t vabal_high_u32(uint64x2_t __p0, uint32x4_t __p1, uint32x4_t __p2) { + uint64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + uint32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + uint32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + uint64x2_t __ret; + __ret = __noswap_vabal_u32(__rev0, __noswap_vget_high_u32(__rev1), __noswap_vget_high_u32(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai uint32x4_t vabal_high_u16(uint32x4_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint32x4_t __ret; + __ret = vabal_u16(__p0, vget_high_u16(__p1), vget_high_u16(__p2)); + return __ret; +} +#else +__ai uint32x4_t vabal_high_u16(uint32x4_t __p0, uint16x8_t __p1, uint16x8_t __p2) { + uint32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + uint16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + uint16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + uint32x4_t __ret; + __ret = __noswap_vabal_u16(__rev0, __noswap_vget_high_u16(__rev1), __noswap_vget_high_u16(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int16x8_t vabal_high_s8(int16x8_t __p0, int8x16_t __p1, int8x16_t __p2) { + int16x8_t __ret; + __ret = vabal_s8(__p0, vget_high_s8(__p1), vget_high_s8(__p2)); + return __ret; +} +#else +__ai int16x8_t vabal_high_s8(int16x8_t __p0, int8x16_t __p1, int8x16_t __p2) { + int16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __ret; + __ret = __noswap_vabal_s8(__rev0, __noswap_vget_high_s8(__rev1), __noswap_vget_high_s8(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 7, 6, 5, 4, 3, 2, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int64x2_t vabal_high_s32(int64x2_t __p0, int32x4_t __p1, int32x4_t __p2) { + int64x2_t __ret; + __ret = vabal_s32(__p0, vget_high_s32(__p1), vget_high_s32(__p2)); + return __ret; +} +#else +__ai int64x2_t vabal_high_s32(int64x2_t __p0, int32x4_t __p1, int32x4_t __p2) { + int64x2_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 1, 0); + int32x4_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 3, 2, 1, 0); + int32x4_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 3, 2, 1, 0); + int64x2_t __ret; + __ret = __noswap_vabal_s32(__rev0, __noswap_vget_high_s32(__rev1), __noswap_vget_high_s32(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 1, 0); + return __ret; +} +#endif + +#ifdef __LITTLE_ENDIAN__ +__ai int32x4_t vabal_high_s16(int32x4_t __p0, int16x8_t __p1, int16x8_t __p2) { + int32x4_t __ret; + __ret = vabal_s16(__p0, vget_high_s16(__p1), vget_high_s16(__p2)); + return __ret; +} +#else +__ai int32x4_t vabal_high_s16(int32x4_t __p0, int16x8_t __p1, int16x8_t __p2) { + int32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, 3, 2, 1, 0); + int16x8_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, 7, 6, 5, 4, 3, 2, 1, 0); + int16x8_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, 7, 6, 5, 4, 3, 2, 1, 0); + int32x4_t __ret; + __ret = __noswap_vabal_s16(__rev0, __noswap_vget_high_s16(__rev1), __noswap_vget_high_s16(__rev2)); + __ret = __builtin_shufflevector(__ret, __ret, 3, 2, 1, 0); + return __ret; +} +#endif + +#endif + +#undef __ai + +#pragma clang diagnostic pop + +#endif /* __ARM_NEON_H */ diff --git a/EXTERNAL_HEADERS/corecrypto/cc.h b/EXTERNAL_HEADERS/corecrypto/cc.h index 6b01e33c1..7790a4faa 100644 --- a/EXTERNAL_HEADERS/corecrypto/cc.h +++ b/EXTERNAL_HEADERS/corecrypto/cc.h @@ -39,26 +39,28 @@ uint8_t b[_alignment_]; \ /* sizeof of a context declared with cc_ctx_decl */ #define cc_ctx_sizeof(_type_, _size_) sizeof(_type_[cc_ctx_n(_type_, _size_)]) -//- WARNING: The _MSC_VER version of cc_ctx_decl() is not compatible with the way *_decl macros are used in CommonCrypto, AppleKeyStore and SecurityFrameworks -// to observe the incompatibilities and errors, use below definition. Corecrypto itself, accepts both deinitions -// #define cc_ctx_decl(_type_, _size_, _name_) _type_ _name_ ## _array[cc_ctx_n(_type_, (_size_))]; _type_ *_name_ = _name_ ## _array -//- Never use sizeof() operator for the variables declared with cc_ctx_decl(), because it is not be compatible with the _MSC_VER version of cc_ctx_decl(). +/* + 1. _alloca cannot be removed becasue this header file is compiled with both MSVC++ and with clang. + 2. The _MSC_VER version of cc_ctx_decl() is not compatible with the way *_decl macros as used in CommonCrypto, AppleKeyStore and SecurityFrameworks. To observe the incompatibilities and errors, use below definition. Corecrypto itself, accepts both deinitions + #define cc_ctx_decl(_type_, _size_, _name_) _type_ _name_ ## _array[cc_ctx_n(_type_, (_size_))]; _type_ *_name_ = _name_ ## _array + 3. Never use sizeof() operator for the variables declared with cc_ctx_decl(), because it is not be compatible with the _MSC_VER version of cc_ctx_decl(). + */ #if defined(_MSC_VER) - #define UNIQUE_ARRAY(data_type, _var_, total_count) data_type* _var_ = (data_type*)_alloca(sizeof(data_type)*(total_count)); - #define cc_ctx_decl(_type_, _size_, _name_) UNIQUE_ARRAY(_type_, _name_,cc_ctx_n(_type_, (_size_))) +#define cc_ctx_decl(_type_, _size_, _name_) _type_ * _name_ = (_type_ *) _alloca(sizeof(_type_) * cc_ctx_n(_type_, _size_) ) #else - #define cc_ctx_decl(_type_, _size_, _name_) _type_ _name_ [cc_ctx_n(_type_, _size_)] +#define cc_ctx_decl(_type_, _size_, _name_) _type_ _name_ [cc_ctx_n(_type_, _size_)] #endif /* bzero is deprecated. memset is the way to go */ /* FWIW, L4, HEXAGON and ARMCC even with gnu compatibility mode don't have bzero */ #define cc_zero(_size_,_data_) memset((_data_),0 ,(_size_)) -/* cc_clear: - Set "len" bytes of memory to zero at address "dst". - cc_clear has been developed so that it won't be optimized out. - To be used to clear key buffers or sensitive data. -*/ +/*! + @brief cc_clear(len, dst) zeroizes array dst and it will not be optimized out. + @discussion It is used to clear sensitive data, particularly when the are defined in the stack + @param len number of bytes to be cleared in dst + @param dst input array + */ CC_NONNULL2 void cc_clear(size_t len, void *dst); diff --git a/EXTERNAL_HEADERS/corecrypto/cc_config.h b/EXTERNAL_HEADERS/corecrypto/cc_config.h index 464f32b18..044c8e168 100644 --- a/EXTERNAL_HEADERS/corecrypto/cc_config.h +++ b/EXTERNAL_HEADERS/corecrypto/cc_config.h @@ -45,15 +45,20 @@ */ -//Do not set these macros to 1, unless you are developing/testing for Windows +//Do not set this macros to 1, unless you are developing/testing for Linux under macOS +#define CORECRYPTO_SIMULATE_POSIX_ENVIRONMENT 0 + +//Do not set these macros to 1, unless you are developing/testing for Windows under macOS #define CORECRYPTO_SIMULATE_WINDOWS_ENVIRONMENT 0 -#define CORECRYPTO_HACK_FOR_WINDOWS_DEVELOPMENT 0 //to be removed after <rdar://problem/26585938> port corecrypto to Windows +#define CORECRYPTO_HACK_FOR_WINDOWS_DEVELOPMENT 0 //to be removed after <rdar://problem/27304763> port corecrypto to Windows //this macro is used to turn on/off usage of transparent union in corecrypto //it should be commented out in corecrypto and be used only by the software that use corecrypto //#define CORECRYPTO_DONOT_USE_TRANSPARENT_UNION -#ifdef CORECRYPTO_DONOT_USE_TRANSPARENT_UNION - #define CORECRYPTO_USE_TRANSPARENT_UNION 0 +#if defined(__cplusplus) +#define CORECRYPTO_USE_TRANSPARENT_UNION 0 +#elif defined(CORECRYPTO_DONOT_USE_TRANSPARENT_UNION) + #define CORECRYPTO_USE_TRANSPARENT_UNION !CORECRYPTO_DONOT_USE_TRANSPARENT_UNION #else #define CORECRYPTO_USE_TRANSPARENT_UNION 1 #endif @@ -76,9 +81,7 @@ #define CC_KERNEL 0 #endif -// LINUX_BUILD_TEST is for sanity check of the configuration -// > xcodebuild -scheme "corecrypto_test" OTHER_CFLAGS="$(values) -DLINUX_BUILD_TEST" -#if defined(__linux__) || defined(LINUX_BUILD_TEST) +#if defined(__linux__) || CORECRYPTO_SIMULATE_POSIX_ENVIRONMENT #define CC_LINUX 1 #else #define CC_LINUX 0 @@ -90,6 +93,12 @@ #define CC_USE_L4 0 #endif +#if defined(RTKIT) && (RTKIT) + #define CC_RTKIT 1 +#else + #define CC_RTKIT 0 +#endif + #if defined(USE_SEPROM) && (USE_SEPROM) #define CC_USE_SEPROM 1 #else @@ -120,20 +129,32 @@ #define CC_IBOOT 0 #endif -// BB configuration +// Defined by the XNU build scripts +// Applies to code embedded in XNU but NOT to the kext +#if defined(XNU_KERNEL_PRIVATE) + #define CC_XNU_KERNEL_PRIVATE 1 +#else + #define CC_XNU_KERNEL_PRIVATE 0 +#endif + +// handle unaligned data, if the cpu cannot. Currently for gladman AES and the C version of the SHA256 +#define CC_HANDLE_UNALIGNED_DATA CC_BASEBAND + +// BaseBand configuration #if CC_BASEBAND // -- ENDIANESS +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) #if defined(ENDIAN_LITTLE) || (defined(__arm__) && !defined(__BIG_ENDIAN)) #define __LITTLE_ENDIAN__ #elif !defined(ENDIAN_BIG) && !defined(__BIG_ENDIAN) #error Baseband endianess not defined. #endif #define AESOPT_ENDIAN_NO_FILE +#endif // -- Architecture #define CCN_UNIT_SIZE 4 // 32 bits - #define SAFE_IO // AES support for unaligned Input/Output // -- External function #define assert ASSERT // sanity @@ -143,21 +164,27 @@ // #1254-D: arithmetic on pointer to void or function type // #186-D: pointless comparison of unsigned integer with zero // #546-D: transfer of control bypasses initialization of - #if defined(__GNUC__) + #ifdef __arm__ + #pragma diag_suppress 186, 1254,546 + #elif defined(__GNUC__) // warning: pointer of type 'void *' used in arithmetic #pragma GCC diagnostic ignored "-Wpointer-arith" - #endif // arm or gnuc - + #endif // __arm__ #endif // CC_BASEBAND //CC_XNU_KERNEL_AVAILABLE indicates the availibity of XNU kernel functions, //like what we have on OSX, iOS, tvOS, Watch OS -#if defined(__APPLE__) && defined(__MACH__) +#if defined(__APPLE__) && defined(__MACH__) #define CC_XNU_KERNEL_AVAILABLE 1 #else #define CC_XNU_KERNEL_AVAILABLE 0 #endif +//arm arch64 definition for gcc +#if defined(__GNUC__) && defined(__aarch64__) && !defined(__arm64__) + #define __arm64__ +#endif + #if !defined(CCN_UNIT_SIZE) #if defined(__arm64__) || defined(__x86_64__) || defined(_WIN64) #define CCN_UNIT_SIZE 8 @@ -192,16 +219,35 @@ #endif #endif -#if __clang__ || CCN_UNIT_SIZE==8 - #define CC_ALIGNED(x) __attribute__ ((aligned(x))) -#elif _MSC_VER - #define CC_ALIGNED(x) __declspec(align(x)) +#if defined(_MSC_VER) + #if defined(__clang__) + #define CC_ALIGNED(x) __attribute__ ((aligned(x))) //clang compiler + #else + #define CC_ALIGNED(x) __declspec(align(x)) //MS complier + #endif #else - #define CC_ALIGNED(x) __attribute__ ((aligned((x)>8?8:(x)))) + #if __clang__ || CCN_UNIT_SIZE==8 + #define CC_ALIGNED(x) __attribute__ ((aligned(x))) + #else + #define CC_ALIGNED(x) __attribute__ ((aligned((x)>8?8:(x)))) + #endif #endif +#if defined(__arm__) +//this is copied from <arm/arch.h>, because <arm/arch.h> is not available on SEPROM environment + #if defined (__ARM_ARCH_7A__) || defined (__ARM_ARCH_7S__) || defined (__ARM_ARCH_7F__) || defined (__ARM_ARCH_7K__) + #define _ARM_ARCH_7 + #endif + + #if defined(__ARM_ARCH_6M__) || defined(__TARGET_ARCH_6S_M) || defined (__armv6m__) + #define _ARM_ARCH_6M + #endif +#endif -#if defined(__x86_64__) || defined(__i386__) +#if defined(__arm64__) || defined(__arm__) + #define CCN_IOS 1 + #define CCN_OSX 0 +#elif defined(__x86_64__) || defined(__i386__) #define CCN_IOS 0 #define CCN_OSX 1 #endif @@ -213,17 +259,15 @@ #endif #if !defined(CC_USE_HEAP_FOR_WORKSPACE) - #if CC_USE_L4 || CC_IBOOT || defined(_MSC_VER) - /* For L4, stack is too short, need to use HEAP for some computations */ - /* CC_USE_HEAP_FOR_WORKSPACE not supported for KERNEL! */ - #define CC_USE_HEAP_FOR_WORKSPACE 1 - #else + #if CC_USE_S3 || CC_USE_SEPROM || CC_RTKIT #define CC_USE_HEAP_FOR_WORKSPACE 0 + #else + #define CC_USE_HEAP_FOR_WORKSPACE 1 #endif #endif /* memset_s is only available in few target */ -#if CC_KERNEL || CC_USE_SEPROM || defined(__CC_ARM) \ +#if CC_USE_SEPROM || defined(__CC_ARM) \ || defined(__hexagon__) || CC_EFI #define CC_HAS_MEMSET_S 0 #else @@ -237,8 +281,7 @@ #endif /* __has_include(<TargetConditionals.h>) */ #endif /* defined(__has_include) */ -// Disable FIPS key gen algorithm on userland and kext so that related POST -// is skipped and boot time is reduced +// Disable RSA Keygen on iBridge #if defined(TARGET_OS_BRIDGE) && TARGET_OS_BRIDGE && CC_KERNEL #define CC_DISABLE_RSAKEYGEN 1 /* for iBridge */ #else @@ -247,13 +290,14 @@ //- functions implemented in assembly ------------------------------------------ //this the list of corecrypto clients that use assembly and the clang compiler -#if !(CC_XNU_KERNEL_AVAILABLE || CC_KERNEL || CC_USE_L4 || CC_IBOOT || CC_USE_SEPROM || CC_USE_S3) && !defined(_WIN32) && CORECRYPTO_DEBUG +#if !(CC_XNU_KERNEL_AVAILABLE || CC_KERNEL || CC_USE_L4 || CC_IBOOT || CC_RTKIT || CC_USE_SEPROM || CC_USE_S3) && !defined(_WIN32) && CORECRYPTO_DEBUG #warning "You are using the default corecrypto configuration, assembly optimizations may not be available for your platform" #endif -// use this macro to strictly disable assembly regardless of cpu/os/compiler/etc +// Use this macro to strictly disable assembly regardless of cpu/os/compiler/etc. +// Our assembly code is not gcc compatible. Clang defines the __GNUC__ macro as well. #if !defined(CC_USE_ASM) - #if defined(_MSC_VER) || CC_LINUX || CC_EFI || CC_BASEBAND + #if defined(_WIN32) || CC_EFI || CC_BASEBAND || CC_XNU_KERNEL_PRIVATE || (defined(__GNUC__) && !defined(__clang__)) || defined(__ANDROID_API__) #define CC_USE_ASM 0 #else #define CC_USE_ASM 1 @@ -277,7 +321,7 @@ #define CCN_SHIFT_RIGHT_ASM 1 #define CCAES_ARM_ASM 1 #define CCAES_INTEL_ASM 0 - #if CC_KERNEL || CC_USE_L4 || CC_IBOOT || CC_USE_SEPROM || CC_USE_S3 + #if CC_KERNEL || CC_USE_L4 || CC_IBOOT || CC_RTKIT || CC_USE_SEPROM || CC_USE_S3 #define CCAES_MUX 0 #else #define CCAES_MUX 1 @@ -296,6 +340,31 @@ #define CCSHA256_ARMV6M_ASM 0 //-(2) ARM 64 +#elif defined(__arm64__) && __clang__ && CC_USE_ASM + #define CCN_DEDICATED_SQR 1 + #define CCN_MUL_KARATSUBA 1 // 4*n CCN_UNIT extra memory required. + #define CCN_ADD_ASM 1 + #define CCN_SUB_ASM 1 + #define CCN_MUL_ASM 1 + #define CCN_ADDMUL1_ASM 0 + #define CCN_MUL1_ASM 0 + #define CCN_CMP_ASM 1 + #define CCN_ADD1_ASM 0 + #define CCN_SUB1_ASM 0 + #define CCN_N_ASM 1 + #define CCN_SET_ASM 0 + #define CCN_SHIFT_RIGHT_ASM 1 + #define CCAES_ARM_ASM 1 + #define CCAES_INTEL_ASM 0 + #define CCAES_MUX 0 // On 64bit SoC, asm is much faster than HW + #define CCN_USE_BUILTIN_CLZ 1 + #define CCSHA1_VNG_INTEL 0 + #define CCSHA2_VNG_INTEL 0 + #define CCSHA1_VNG_ARMV7NEON 1 // reused this to avoid making change to xcode project, put arm64 assembly code with armv7 code + #define CCSHA2_VNG_ARMV7NEON 1 + #define CCSHA256_ARMV6M_ASM 0 + +//-(3) Intel 32/64 #elif (defined(__x86_64__) || defined(__i386__)) && __clang__ && CC_USE_ASM #define CCN_DEDICATED_SQR 1 #define CCN_MUL_KARATSUBA 1 // 4*n CCN_UNIT extra memory required. @@ -431,5 +500,11 @@ #define CC_MALLOC #endif /* !__GNUC__ */ +// Enable FIPSPOST function tracing only when supported. */ +#ifdef CORECRYPTO_POST_TRACE +#define CC_FIPSPOST_TRACE 1 +#else +#define CC_FIPSPOST_TRACE 0 +#endif #endif /* _CORECRYPTO_CC_CONFIG_H_ */ diff --git a/EXTERNAL_HEADERS/corecrypto/cc_debug.h b/EXTERNAL_HEADERS/corecrypto/cc_debug.h index 5c8ebbdc7..80e61a7b3 100644 --- a/EXTERNAL_HEADERS/corecrypto/cc_debug.h +++ b/EXTERNAL_HEADERS/corecrypto/cc_debug.h @@ -21,15 +21,20 @@ // Printf for corecrypto // ======================== #if CC_KERNEL -#include <pexpert/pexpert.h> -#define cc_printf(x...) kprintf(x) -extern int printf(const char *format, ...) __printflike(1,2); -#elif CC_USE_S3 -#include <stdio.h> -#define cc_printf(x...) printf(x) + #include <pexpert/pexpert.h> + #define cc_printf(x...) kprintf(x) + #if !CONFIG_EMBEDDED + extern int printf(const char *format, ...) __printflike(1,2); + #endif +#elif CC_USE_S3 || CC_IBOOT || CC_RTKIT + #include <stdio.h> + #define cc_printf(x...) printf(x) +#elif defined(__ANDROID_API__) + #include <android/log.h> + #define cc_printf(x...) __android_log_print(ANDROID_LOG_DEBUG, "corecrypto", x); #else -#include <stdio.h> -#define cc_printf(x...) fprintf(stderr, x) + #include <stdio.h> + #define cc_printf(x...) fprintf(stderr, x) #endif // ======================== diff --git a/EXTERNAL_HEADERS/corecrypto/cc_priv.h b/EXTERNAL_HEADERS/corecrypto/cc_priv.h index 417d45c5c..55e0eb2b8 100644 --- a/EXTERNAL_HEADERS/corecrypto/cc_priv.h +++ b/EXTERNAL_HEADERS/corecrypto/cc_priv.h @@ -53,14 +53,9 @@ The following are not defined yet... define them if needed. CC_BSWAP64c : byte swap a 64 bits constant CC_READ_LE32 : read a 32 bits little endian value - CC_READ_LE64 : read a 64 bits little endian value - CC_READ_BE32 : read a 32 bits big endian value - CC_READ_BE64 : read a 64 bits big endian value CC_WRITE_LE32 : write a 32 bits little endian value CC_WRITE_LE64 : write a 64 bits little endian value - CC_WRITE_BE32 : write a 32 bits big endian value - CC_WRITE_BE64 : write a 64 bits big endian value CC_H2BE64 : convert a 64 bits value between host and big endian order CC_H2LE64 : convert a 64 bits value between host and little endian order @@ -360,6 +355,32 @@ CC_INLINE uint32_t CC_BSWAP(uint32_t x) #define CC_H2LE32(x) CC_BSWAP(x) #endif +#define CC_READ_LE32(ptr) \ +( (uint32_t)( \ +((uint32_t)((const uint8_t *)(ptr))[0]) | \ +(((uint32_t)((const uint8_t *)(ptr))[1]) << 8) | \ +(((uint32_t)((const uint8_t *)(ptr))[2]) << 16) | \ +(((uint32_t)((const uint8_t *)(ptr))[3]) << 24))) + +#define CC_WRITE_LE32(ptr, x) \ +do { \ +((uint8_t *)(ptr))[0] = (uint8_t)( (x) & 0xFF); \ +((uint8_t *)(ptr))[1] = (uint8_t)(((x) >> 8) & 0xFF); \ +((uint8_t *)(ptr))[2] = (uint8_t)(((x) >> 16) & 0xFF); \ +((uint8_t *)(ptr))[3] = (uint8_t)(((x) >> 24) & 0xFF); \ +} while(0) + +#define CC_WRITE_LE64(ptr, x) \ +do { \ +((uint8_t *)(ptr))[0] = (uint8_t)( (x) & 0xFF); \ +((uint8_t *)(ptr))[1] = (uint8_t)(((x) >> 8) & 0xFF); \ +((uint8_t *)(ptr))[2] = (uint8_t)(((x) >> 16) & 0xFF); \ +((uint8_t *)(ptr))[3] = (uint8_t)(((x) >> 24) & 0xFF); \ +((uint8_t *)(ptr))[4] = (uint8_t)(((x) >> 32) & 0xFF); \ +((uint8_t *)(ptr))[5] = (uint8_t)(((x) >> 40) & 0xFF); \ +((uint8_t *)(ptr))[6] = (uint8_t)(((x) >> 48) & 0xFF); \ +((uint8_t *)(ptr))[7] = (uint8_t)(((x) >> 56) & 0xFF); \ +} while(0) /* extract a byte portably */ #ifdef _MSC_VER @@ -413,8 +434,8 @@ CC_INLINE uint32_t CC_BSWAP(uint32_t x) #define cc_ceiling(a,b) (((a)+((b)-1))/(b)) #define CC_BITLEN_TO_BYTELEN(x) cc_ceiling((x), 8) -//cc_abort() is implemented to comply with FIPS 140-2. See radar 19129408 -void cc_abort(const char * msg , ...); +//cc_try_abort() is implemented to comply with FIPS 140-2. See radar 19129408 +void cc_try_abort(const char * msg , ...); /*! @brief cc_muxp(s, a, b) is equivalent to z = s ? a : b, but it executes in constant time diff --git a/EXTERNAL_HEADERS/corecrypto/cc_runtime_config.h b/EXTERNAL_HEADERS/corecrypto/cc_runtime_config.h new file mode 100644 index 000000000..0064c6ca6 --- /dev/null +++ b/EXTERNAL_HEADERS/corecrypto/cc_runtime_config.h @@ -0,0 +1,48 @@ +/* + * cc_runtime_config.h + * corecrypto + * + * Created on 09/18/2012 + * + * Copyright (c) 2012,2014,2015 Apple Inc. All rights reserved. + * + */ + +#ifndef CORECRYPTO_CC_RUNTIME_CONFIG_H_ +#define CORECRYPTO_CC_RUNTIME_CONFIG_H_ + +#include <corecrypto/cc_config.h> + +/* Only intel systems have these runtime switches today. */ +#if (CCSHA1_VNG_INTEL || CCSHA2_VNG_INTEL || CCAES_INTEL_ASM) \ + && (defined(__x86_64__) || defined(__i386__)) + +#if CC_KERNEL + #include <i386/cpuid.h> + #define CC_HAS_AESNI() ((cpuid_features() & CPUID_FEATURE_AES) != 0) + #define CC_HAS_SupplementalSSE3() ((cpuid_features() & CPUID_FEATURE_SSSE3) != 0) + #define CC_HAS_AVX1() ((cpuid_features() & CPUID_FEATURE_AVX1_0) != 0) + #define CC_HAS_AVX2() ((cpuid_info()->cpuid_leaf7_features & CPUID_LEAF7_FEATURE_AVX2) != 0) + +#elif CC_XNU_KERNEL_AVAILABLE + # include <System/i386/cpu_capabilities.h> + + #ifndef kHasAVX2_0 /* 10.8 doesn't have kHasAVX2_0 defined */ + #define kHasAVX2_0 0 + #endif + + extern int _cpu_capabilities; + #define CC_HAS_AESNI() (_cpu_capabilities & kHasAES) + #define CC_HAS_SupplementalSSE3() (_cpu_capabilities & kHasSupplementalSSE3) + #define CC_HAS_AVX1() (_cpu_capabilities & kHasAVX1_0) + #define CC_HAS_AVX2() (_cpu_capabilities & kHasAVX2_0) +#else + #define CC_HAS_AESNI() 0 + #define CC_HAS_SupplementalSSE3() 0 + #define CC_HAS_AVX1() 0 + #define CC_HAS_AVX2() 0 +#endif + +#endif /* !(defined(__x86_64__) || defined(__i386__)) */ + +#endif /* CORECRYPTO_CC_RUNTIME_CONFIG_H_ */ diff --git a/EXTERNAL_HEADERS/corecrypto/ccaes.h b/EXTERNAL_HEADERS/corecrypto/ccaes.h index 630cdd282..ec119b9b6 100644 --- a/EXTERNAL_HEADERS/corecrypto/ccaes.h +++ b/EXTERNAL_HEADERS/corecrypto/ccaes.h @@ -19,6 +19,8 @@ #define CCAES_KEY_SIZE_192 24 #define CCAES_KEY_SIZE_256 32 +#define CCAES_CTR_MAX_PARALLEL_NBLOCKS 8 + extern const struct ccmode_ecb ccaes_ltc_ecb_decrypt_mode; extern const struct ccmode_ecb ccaes_ltc_ecb_encrypt_mode; @@ -46,8 +48,13 @@ extern const struct ccmode_ofb ccaes_arm_ofb_crypt_mode; extern const struct ccmode_cbc ccaes_ios_hardware_cbc_encrypt_mode; extern const struct ccmode_cbc ccaes_ios_hardware_cbc_decrypt_mode; +extern const struct ccmode_ctr ccaes_ios_hardware_ctr_crypt_mode; + extern const struct ccmode_cbc *ccaes_ios_mux_cbc_encrypt_mode(void); extern const struct ccmode_cbc *ccaes_ios_mux_cbc_decrypt_mode(void); + +extern const struct ccmode_ctr *ccaes_ios_mux_ctr_crypt_mode(void); + #endif #if CCAES_INTEL_ASM diff --git a/EXTERNAL_HEADERS/corecrypto/ccasn1.h b/EXTERNAL_HEADERS/corecrypto/ccasn1.h index 7eb1182e6..28fba4eef 100644 --- a/EXTERNAL_HEADERS/corecrypto/ccasn1.h +++ b/EXTERNAL_HEADERS/corecrypto/ccasn1.h @@ -88,7 +88,7 @@ size_t ccoid_size(ccoid_t oid) { return 2 + CCOID(oid)[1]; } -CC_INLINE CC_PURE CC_NONNULL((1)) CC_NONNULL((2)) +CC_INLINE CC_PURE CC_NONNULL_TU((1)) CC_NONNULL_TU((2)) bool ccoid_equal(ccoid_t oid1, ccoid_t oid2) { return (ccoid_size(oid1) == ccoid_size(oid2) && memcmp(CCOID(oid1), CCOID(oid2), ccoid_size(oid1))== 0); diff --git a/EXTERNAL_HEADERS/corecrypto/ccchacha20poly1305.h b/EXTERNAL_HEADERS/corecrypto/ccchacha20poly1305.h new file mode 100644 index 000000000..3e76b81b4 --- /dev/null +++ b/EXTERNAL_HEADERS/corecrypto/ccchacha20poly1305.h @@ -0,0 +1,295 @@ +/* + ccchacha20poly1305.h + corecrypto + + Copyright 2014 Apple Inc. All rights reserved. +*/ + +#ifndef _CORECRYPTO_CCCHACHA20POLY1305_H_ +#define _CORECRYPTO_CCCHACHA20POLY1305_H_ + +#include <stdbool.h> +#include <stddef.h> +#include <stdint.h> + +#define CCCHACHA20_KEY_NBYTES 32 +#define CCCHACHA20_BLOCK_NBYTES 64 +#define CCCHACHA20_BLOCK_NBITS (CCCHACHA20_BLOCK_NBYTES * 8) +#define CCCHACHA20_NONCE_NBYTES 12 + +typedef struct { + uint32_t state[16]; + uint8_t buffer[CCCHACHA20_BLOCK_NBYTES]; + size_t leftover; +} ccchacha20_ctx; + +#define CCPOLY1305_TAG_NBYTES 16 + +typedef struct { + uint32_t r0, r1, r2, r3, r4; + uint32_t s1, s2, s3, s4; + uint32_t h0, h1, h2, h3, h4; + uint8_t buf[16]; + size_t buf_used; + uint8_t key[16]; +} ccpoly1305_ctx; + + +/*! + @group ccchacha20poly1305 + @abstract Encrypts and authenticates or decrypts and verifies data. + @discussion See RFC 7539 for details. + + @warning The key-nonce pair must be unique per encryption. + + @warning A single message can be at most (2^38 - 64) bytes in length. + + The correct sequence of calls to encrypt is: + + @code ccchacha20poly1305_init(...) + ccchacha20poly1305_setnonce(...) + ccchacha20poly1305_aad(...) (may be called zero or more times) + ccchacha20poly1305_encrypt(...) (may be called zero or more times) + ccchacha20poly1305_finalize(...) + + To reuse the context for additional encryptions, follow this sequence: + + @code ccchacha20poly1305_reset(...) + ccchacha20poly1305_setnonce(...) + ccchacha20poly1305_aad(...) (may be called zero or more times) + ccchacha20poly1305_encrypt(...) (may be called zero or more times) + ccchacha20poly1305_finalize(...) + + To decrypt, follow this call sequence: + + @code ccchacha20poly1305_init(...) + ccchacha20poly1305_setnonce(...) + ccchacha20poly1305_aad(...) (may be called zero or more times) + ccchacha20poly1305_decrypt(...) (may be called zero or more times) + ccchacha20poly1305_verify(...) (returns zero on successful decryption) + + To reuse the context for additional encryptions, follow this sequence: + + @code ccchacha20poly1305_reset(...) + ccchacha20poly1305_setnonce(...) + ccchacha20poly1305_aad(...) (may be called zero or more times) + ccchacha20poly1305_decrypt(...) (may be called zero or more times) + ccchacha20poly1305_verify(...) (returns zero on successful decryption) +*/ + +#define CCCHACHA20POLY1305_KEY_NBYTES (CCCHACHA20_KEY_NBYTES) +#define CCCHACHA20POLY1305_NONCE_NBYTES (CCCHACHA20_NONCE_NBYTES) +#define CCCHACHA20POLY1305_TAG_NBYTES (CCPOLY1305_TAG_NBYTES) + +/* (2^32 - 1) blocks */ +/* (2^38 - 64) bytes */ +/* (2^41 - 512) bits */ +/* Exceeding this figure breaks confidentiality and authenticity. */ +#define CCCHACHA20POLY1305_TEXT_MAX_NBYTES ((1ULL << 38) - 64ULL) + +#define CCCHACHA20POLY1305_STATE_SETNONCE 1 +#define CCCHACHA20POLY1305_STATE_AAD 2 +#define CCCHACHA20POLY1305_STATE_ENCRYPT 3 +#define CCCHACHA20POLY1305_STATE_DECRYPT 4 +#define CCCHACHA20POLY1305_STATE_FINAL 5 + +typedef struct { + ccchacha20_ctx chacha20_ctx; + ccpoly1305_ctx poly1305_ctx; + uint64_t aad_nbytes; + uint64_t text_nbytes; + uint8_t state; +} ccchacha20poly1305_ctx; + +// This is just a stub right now. +// Eventually we will optimize by platform. +struct ccchacha20poly1305_info { + +}; + +extern const struct ccchacha20poly1305_info ccchacha20poly1305_info_default; + +const struct ccchacha20poly1305_info *ccchacha20poly1305_info(void); + +/*! + @function ccchacha20poly1305_init + @abstract Initialize a chacha20poly1305 context. + + @param info Implementation descriptor + @param ctx Context for this instance + @param key Secret chacha20 key + + @result 0 iff successful. + + @discussion The key is 32 bytes in length. + + @warning The key-nonce pair must be unique per encryption. + */ +int ccchacha20poly1305_init(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, const uint8_t *key); + +/*! + @function ccchacha20poly1305_reset + @abstract Reset a chacha20poly1305 context for reuse. + + @param info Implementation descriptor + @param ctx Context for this instance + + @result 0 iff successful. + */ +int ccchacha20poly1305_reset(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx); + +/*! + @function ccchacha20poly1305_setnonce + @abstract Set the nonce for encryption or decryption. + + @param info Implementation descriptor + @param ctx Context for this instance + @param nonce Unique nonce per encryption + + @result 0 iff successful. + + @discussion The nonce is 12 bytes in length. + + @warning The key-nonce pair must be unique per encryption. + */ +int ccchacha20poly1305_setnonce(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, const uint8_t *nonce); +int ccchacha20poly1305_incnonce(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, uint8_t *nonce); + +/*! + @function ccchacha20poly1305_aad + @abstract Authenticate additional data. + + @param info Descriptor for the mode + @param ctx Context for this instance + @param nbytes Length of the additional data in bytes + @param aad Additional data to authenticate + + @result 0 iff successful. + + @discussion This is typically used to authenticate data that cannot be encrypted (e.g. packet headers). + + This function may be called zero or more times. + */ +int ccchacha20poly1305_aad(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, size_t nbytes, const void *aad); + +/*! + @function ccchacha20poly1305_encrypt + @abstract Encrypt data. + + @param info Descriptor for the mode + @param ctx Context for this instance + @param nbytes Length of the plaintext in bytes + @param ptext Input plaintext + @param ctext Output ciphertext + + @result 0 iff successful. + + @discussion In-place processing is supported. + + This function may be called zero or more times. + */ +int ccchacha20poly1305_encrypt(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, size_t nbytes, const void *ptext, void *ctext); + +/*! + @function ccchacha20poly1305_finalize + @abstract Finalize encryption. + + @param info Descriptor for the mode + @param ctx Context for this instance + @param tag Generated authentication tag + + @result 0 iff successful. + + @discussion The generated tag is 16 bytes in length. + */ +int ccchacha20poly1305_finalize(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, uint8_t *tag); + +/*! + @function ccchacha20poly1305_decrypt + @abstract Decrypt data. + + @param info Descriptor for the mode + @param ctx Context for this instance + @param nbytes Length of the ciphertext in bytes + @param ctext Input ciphertext + @param ptext Output plaintext + + @result 0 iff successful. + + @discussion In-place processing is supported. + + This function may be called zero or more times. + */ +int ccchacha20poly1305_decrypt(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, size_t nbytes, const void *ctext, void *ptext); + +/*! + @function ccchacha20poly1305_verify + @abstract Verify authenticity. + + @param info Descriptor for the mode + @param ctx Context for this instance + @param tag Expected authentication tag + + @result 0 iff authentic and otherwise successful. + + @discussion The expected tag is 16 bytes in length. + */ +int ccchacha20poly1305_verify(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, const uint8_t *tag); + +/*! + @function ccchacha20poly1305_encrypt_oneshot + @abstract Encrypt with chacha20poly1305. + + @param info Descriptor for the mode + @param key Secret chacha20 key + @param nonce Unique nonce per encryption + @param aad_nbytes Length of the additional data in bytes + @param aad Additional data to authenticate + @param ptext_nbytes Length of the plaintext in bytes + @param ptext Input plaintext + @param ctext Output ciphertext + @param tag Generated authentication tag + + @discussion See RFC 7539 for details. + + The key is 32 bytes in length. + + The nonce is 12 bytes in length. + + The generated tag is 16 bytes in length. + + In-place processing is supported. + + @warning The key-nonce pair must be unique per encryption. + + @warning A single message can be at most (2^38 - 64) bytes in length. + */ +int ccchacha20poly1305_encrypt_oneshot(const struct ccchacha20poly1305_info *info, const uint8_t *key, const uint8_t *nonce, size_t aad_nbytes, const void *aad, size_t ptext_nbytes, const void *ptext, void *ctext, uint8_t *tag); + +/*! + @function ccchacha20poly1305_decrypt_oneshot + @abstract Decrypt with chacha20poly1305. + + @param info Descriptor for the mode + @param key Secret chacha20 key + @param nonce Unique nonce per encryption + @param aad_nbytes Length of the additional data in bytes + @param aad Additional data to authenticate + @param ctext_nbytes Length of the ciphertext in bytes + @param ctext Input ciphertext + @param ptext Output plaintext + @param tag Expected authentication tag + + @discussion See RFC 7539 for details. + + The key is 32 bytes in length. + + The nonce is 12 bytes in length. + + The generated tag is 16 bytes in length. + + In-place processing is supported. + */ +int ccchacha20poly1305_decrypt_oneshot(const struct ccchacha20poly1305_info *info, const uint8_t *key, const uint8_t *nonce, size_t aad_nbytes, const void *aad, size_t ctext_nbytes, const void *ctext, void *ptext, const uint8_t *tag); + +#endif diff --git a/EXTERNAL_HEADERS/corecrypto/cccmac.h b/EXTERNAL_HEADERS/corecrypto/cccmac.h index 75918fe84..63a892fd6 100644 --- a/EXTERNAL_HEADERS/corecrypto/cccmac.h +++ b/EXTERNAL_HEADERS/corecrypto/cccmac.h @@ -89,16 +89,6 @@ typedef struct cccmac_ctx* cccmac_ctx_t; /* CMAC as defined in NIST SP800-38B - 2005 */ -/* HACK: - To change the prototype of cccmac_init (and preserve the name) we need to - proceed in steps: - 1) Make corecrypto change (23557380) - 2) Have all clients define "CC_CHANGEFUNCTION_28544056_cccmac_init" - 3) Remove CC_CHANGEFUNCTION_28544056_cccmac_init logic and old functions of corecrypto - 4) Clients can remove CC_CHANGEFUNCTION_28544056_cccmac_init at their leisure - - */ - /* ============================================================================= ONE SHOT @@ -169,25 +159,9 @@ int cccmac_one_shot_verify(const struct ccmode_cbc *cbc, @discussion Only supports CMAC_BLOCKSIZE block ciphers */ - - -#ifndef CC_CHANGEFUNCTION_28544056_cccmac_init -int cccmac_init(const struct ccmode_cbc *cbc, - cccmac_ctx_t ctx, - size_t key_nbytes, const void *key) -__attribute__((deprecated("see guidelines in corecrypto/cccmac.h for migration", "define 'CC_CHANGEFUNCTION_28544056_cccmac_init' and use new cccmac_init with parameter key_nbytes"))); -// If you see this deprecate warning -// Define CC_CHANGEFUNCTION_28544056_cccmac_init and use "cccmac_init(...,...,16,...)" -// This will be removed with 28544056 -#define cccmac_init(cbc,ctx,key) cccmac_init(cbc,ctx,16,key) - -#else - -// This is the authoritative prototype, which will be left after 28544056 int cccmac_init(const struct ccmode_cbc *cbc, cccmac_ctx_t ctx, size_t key_nbytes, const void *key); -#endif /*! @function cccmac_update @@ -235,70 +209,4 @@ int cccmac_final_generate(cccmac_ctx_t ctx, int cccmac_final_verify(cccmac_ctx_t ctx, size_t expected_mac_nbytes, const void *expected_mac); - -/* ============================================================================= - - Legacy - Please migrate to new functions above - - ==============================================================================*/ - -/* - Guidelines for switching to new CMAC functions - - Legacy New functions - cccmac_init -> cccmac_init w/ key kength in bytes - cccmac_block_update -> cccmac_update w/ size in bytes instead of blocks - cccmac_final -> cccmac_final_generate or cccmac_final_verify - depending the use case preceeded - by cccmac_update if any leftover bytes. - cccmac -> cccmac_one_shot_generate or cccmac_one_shot_verify - depending the use case - - */ - -/*! - @function cccmac_block_update - @abstract Process data - */ - -CC_INLINE void cccmac_block_update(CC_UNUSED const struct ccmode_cbc *cbc, cccmac_ctx_t ctx, - size_t nblocks, const void *data) -__attribute__((deprecated("see guidelines in corecrypto/cccmac.h for migration", "cccmac_update"))); - -CC_INLINE void cccmac_block_update(CC_UNUSED const struct ccmode_cbc *cbc, cccmac_ctx_t ctx, - size_t nblocks, const void *data) { - cccmac_update(ctx,(nblocks)*CMAC_BLOCKSIZE,data); -} - -/*! - @function cccmac_final - @abstract Finalize CMAC generation - */ -CC_INLINE void cccmac_final(CC_UNUSED const struct ccmode_cbc *cbc, cccmac_ctx_t ctx, - size_t nbytes, const void *in, void *out) -__attribute__((deprecated("see guidelines in corecrypto/cccmac.h for migration", "cccmac_final_generate or cccmac_final_verify"))); - -CC_INLINE void cccmac_final(CC_UNUSED const struct ccmode_cbc *cbc, cccmac_ctx_t ctx, - size_t nbytes, const void *in, void *out) { - cccmac_update(ctx, nbytes, in); - cccmac_final_generate(ctx,CMAC_BLOCKSIZE,out); -} - -/*! - @function cccmac - @abstract One shot CMAC generation with 128bit key - */ -CC_INLINE void cccmac(const struct ccmode_cbc *cbc, - const void *key, - size_t data_len, const void *data, void *mac) -__attribute__((deprecated("see guidelines in corecrypto/cccmac.h for migration", "cccmac_one_shot_generate or cccmac_one_shot_verify"))); - -CC_INLINE void cccmac(const struct ccmode_cbc *cbc, - const void *key, - size_t data_len, const void *data, void *mac) { - cccmac_one_shot_generate(cbc,16,key,data_len,data,16,mac); -} - - - #endif /* _CORECRYPTO_cccmac_H_ */ diff --git a/EXTERNAL_HEADERS/corecrypto/ccder.h b/EXTERNAL_HEADERS/corecrypto/ccder.h index f29140edf..6e2c504be 100644 --- a/EXTERNAL_HEADERS/corecrypto/ccder.h +++ b/EXTERNAL_HEADERS/corecrypto/ccder.h @@ -17,82 +17,76 @@ #define CCDER_MULTIBYTE_TAGS 1 #ifdef CCDER_MULTIBYTE_TAGS - #if defined(_MSC_VER) - //TODO related to rdar://problem/24868013 - typedef int ccder_tag; //MSVC forces enums to be ints - #else - typedef unsigned long ccder_tag; - #endif +typedef unsigned long ccder_tag; #else typedef uint8_t ccder_tag; #endif /* DER types to be used with ccder_decode and ccder_encode functions. */ -enum { - CCDER_EOL = CCASN1_EOL, - CCDER_BOOLEAN = CCASN1_BOOLEAN, - CCDER_INTEGER = CCASN1_INTEGER, - CCDER_BIT_STRING = CCASN1_BIT_STRING, - CCDER_OCTET_STRING = CCASN1_OCTET_STRING, - CCDER_NULL = CCASN1_NULL, - CCDER_OBJECT_IDENTIFIER = CCASN1_OBJECT_IDENTIFIER, - CCDER_OBJECT_DESCRIPTOR = CCASN1_OBJECT_DESCRIPTOR, +#define CCDER_EOL CCASN1_EOL +#define CCDER_BOOLEAN CCASN1_BOOLEAN +#define CCDER_INTEGER CCASN1_INTEGER +#define CCDER_BIT_STRING CCASN1_BIT_STRING +#define CCDER_OCTET_STRING CCASN1_OCTET_STRING +#define CCDER_NULL CCASN1_NULL +#define CCDER_OBJECT_IDENTIFIER CCASN1_OBJECT_IDENTIFIER +#define CCDER_OBJECT_DESCRIPTOR CCASN1_OBJECT_DESCRIPTOR /* External or instance-of 0x08 */ - CCDER_REAL = CCASN1_REAL, - CCDER_ENUMERATED = CCASN1_ENUMERATED, - CCDER_EMBEDDED_PDV = CCASN1_EMBEDDED_PDV, - CCDER_UTF8_STRING = CCASN1_UTF8_STRING, +#define CCDER_REAL CCASN1_REAL +#define CCDER_ENUMERATED CCASN1_ENUMERATED +#define CCDER_EMBEDDED_PDV CCASN1_EMBEDDED_PDV +#define CCDER_UTF8_STRING CCASN1_UTF8_STRING /* 0x0d */ /* 0x0e */ /* 0x0f */ - CCDER_SEQUENCE = CCASN1_SEQUENCE, - CCDER_SET = CCASN1_SET, - CCDER_NUMERIC_STRING = CCASN1_NUMERIC_STRING, - CCDER_PRINTABLE_STRING = CCASN1_PRINTABLE_STRING, - CCDER_T61_STRING = CCASN1_T61_STRING, - CCDER_VIDEOTEX_STRING = CCASN1_VIDEOTEX_STRING, - CCDER_IA5_STRING = CCASN1_IA5_STRING, - CCDER_UTC_TIME = CCASN1_UTC_TIME, - CCDER_GENERALIZED_TIME = CCASN1_GENERALIZED_TIME, - CCDER_GRAPHIC_STRING = CCASN1_GRAPHIC_STRING, - CCDER_VISIBLE_STRING = CCASN1_VISIBLE_STRING, - CCDER_GENERAL_STRING = CCASN1_GENERAL_STRING, - CCDER_UNIVERSAL_STRING = CCASN1_UNIVERSAL_STRING, +#define CCDER_SEQUENCE CCASN1_SEQUENCE +#define CCDER_SET CCASN1_SET +#define CCDER_NUMERIC_STRING CCASN1_NUMERIC_STRING +#define CCDER_PRINTABLE_STRING CCASN1_PRINTABLE_STRING +#define CCDER_T61_STRING CCASN1_T61_STRING +#define CCDER_VIDEOTEX_STRING CCASN1_VIDEOTEX_STRING +#define CCDER_IA5_STRING CCASN1_IA5_STRING +#define CCDER_UTC_TIME CCASN1_UTC_TIME +#define CCDER_GENERALIZED_TIME CCASN1_GENERALIZED_TIME +#define CCDER_GRAPHIC_STRING CCASN1_GRAPHIC_STRING +#define CCDER_VISIBLE_STRING CCASN1_VISIBLE_STRING +#define CCDER_GENERAL_STRING CCASN1_GENERAL_STRING +#define CCDER_UNIVERSAL_STRING CCASN1_UNIVERSAL_STRING /* 0x1d */ - CCDER_BMP_STRING = CCASN1_BMP_STRING, - CCDER_HIGH_TAG_NUMBER = CCASN1_HIGH_TAG_NUMBER, - CCDER_TELETEX_STRING = CCDER_T61_STRING, +#define CCDER_BMP_STRING CCASN1_BMP_STRING +#define CCDER_HIGH_TAG_NUMBER CCASN1_HIGH_TAG_NUMBER +#define CCDER_TELETEX_STRING CCDER_T61_STRING #ifdef CCDER_MULTIBYTE_TAGS - CCDER_TAG_MASK = ((ccder_tag)~0), - CCDER_TAGNUM_MASK = ((ccder_tag)~((ccder_tag)7 << (sizeof(ccder_tag) * 8 - 3))), - - CCDER_METHOD_MASK = ((ccder_tag)1 << (sizeof(ccder_tag) * 8 - 3)), - CCDER_PRIMITIVE = ((ccder_tag)0 << (sizeof(ccder_tag) * 8 - 3)), - CCDER_CONSTRUCTED = ((ccder_tag)1 << (sizeof(ccder_tag) * 8 - 3)), - - CCDER_CLASS_MASK = ((ccder_tag)3 << (sizeof(ccder_tag) * 8 - 2)), - CCDER_UNIVERSAL = ((ccder_tag)0 << (sizeof(ccder_tag) * 8 - 2)), - CCDER_APPLICATION = ((ccder_tag)1 << (sizeof(ccder_tag) * 8 - 2)), - CCDER_CONTEXT_SPECIFIC = ((ccder_tag)2 << (sizeof(ccder_tag) * 8 - 2)), - CCDER_PRIVATE = ((ccder_tag)3 << (sizeof(ccder_tag) * 8 - 2)), -#else - CCDER_TAG_MASK = CCASN1_TAG_MASK, - CCDER_TAGNUM_MASK = CCASN1_TAGNUM_MASK, - - CCDER_METHOD_MASK = CCASN1_METHOD_MASK, - CCDER_PRIMITIVE = CCASN1_PRIMITIVE, - CCDER_CONSTRUCTED = CCASN1_CONSTRUCTED, - - CCDER_CLASS_MASK = CCASN1_CLASS_MASK, - CCDER_UNIVERSAL = CCASN1_UNIVERSAL, - CCDER_APPLICATION = CCASN1_APPLICATION, - CCDER_CONTEXT_SPECIFIC = CCASN1_CONTEXT_SPECIFIC, - CCDER_PRIVATE = CCASN1_PRIVATE, -#endif - CCDER_CONSTRUCTED_SET = CCDER_SET | CCDER_CONSTRUCTED, - CCDER_CONSTRUCTED_SEQUENCE = CCDER_SEQUENCE | CCDER_CONSTRUCTED, -}; +#define CCDER_TAG_MASK ((ccder_tag)~0) +#define CCDER_TAGNUM_MASK ((ccder_tag)~((ccder_tag)7 << (sizeof(ccder_tag) * 8 - 3))) + +#define CCDER_METHOD_MASK ((ccder_tag)1 << (sizeof(ccder_tag) * 8 - 3)) +#define CCDER_PRIMITIVE ((ccder_tag)0 << (sizeof(ccder_tag) * 8 - 3)) +#define CCDER_CONSTRUCTED ((ccder_tag)1 << (sizeof(ccder_tag) * 8 - 3)) + +#define CCDER_CLASS_MASK ((ccder_tag)3 << (sizeof(ccder_tag) * 8 - 2)) +#define CCDER_UNIVERSAL ((ccder_tag)0 << (sizeof(ccder_tag) * 8 - 2)) +#define CCDER_APPLICATION ((ccder_tag)1 << (sizeof(ccder_tag) * 8 - 2)) +#define CCDER_CONTEXT_SPECIFIC ((ccder_tag)2 << (sizeof(ccder_tag) * 8 - 2)) +#define CCDER_PRIVATE ((ccder_tag)3 << (sizeof(ccder_tag) * 8 - 2)) +#else /* !CCDER_MULTIBYTE_TAGS */ +#define CCDER_TAG_MASK CCASN1_TAG_MASK +#define CCDER_TAGNUM_MASK CCASN1_TAGNUM_MASK + +#define CCDER_METHOD_MASK CCASN1_METHOD_MASK +#define CCDER_PRIMITIVE CCASN1_PRIMITIVE +#define CCDER_CONSTRUCTED CCASN1_CONSTRUCTED + +#define CCDER_CLASS_MASK CCASN1_CLASS_MASK +#define CCDER_UNIVERSAL CCASN1_UNIVERSAL +#define CCDER_APPLICATION CCASN1_APPLICATION +#define CCDER_CONTEXT_SPECIFIC CCASN1_CONTEXT_SPECIFIC +#define CCDER_PRIVATE CCASN1_PRIVATE +#endif /* !CCDER_MULTIBYTE_TAGS */ +#define CCDER_CONSTRUCTED_SET (CCDER_SET | CCDER_CONSTRUCTED) +#define CCDER_CONSTRUCTED_SEQUENCE (CCDER_SEQUENCE | CCDER_CONSTRUCTED) + // MARK: ccder_sizeof_ functions diff --git a/EXTERNAL_HEADERS/corecrypto/ccdrbg.h b/EXTERNAL_HEADERS/corecrypto/ccdrbg.h index 7ab4f491d..af5b010a9 100644 --- a/EXTERNAL_HEADERS/corecrypto/ccdrbg.h +++ b/EXTERNAL_HEADERS/corecrypto/ccdrbg.h @@ -41,7 +41,7 @@ #define CCDRBG_MAX_ENTROPY_SIZE ((uint32_t)1<<16) #define CCDRBG_MAX_ADDITIONALINPUT_SIZE ((uint32_t)1<<16) #define CCDRBG_MAX_PSINPUT_SIZE ((uint32_t)1<<16) -#define CCDRBG_MAX_REQUEST_SIZE ((uint32_t)1<<16) //this is the the absolute maximum in NIST 800-90A +#define CCDRBG_MAX_REQUEST_SIZE ((uint32_t)1<<16) //this is the absolute maximum in NIST 800-90A #define CCDRBG_RESEED_INTERVAL ((uint64_t)1<<30) // must be able to fit the NIST maximum of 2^48 @@ -95,10 +95,10 @@ CC_INLINE size_t ccdrbg_context_size(const struct ccdrbg_info *drbg) /* * NIST SP 800-90 CTR_DRBG - * the mximum security strengh of drbg equals to the block size of the corresponding ECB. + * the maximum security strengh of drbg equals to the block size of the corresponding ECB. */ struct ccdrbg_nistctr_custom { - const struct ccmode_ecb *ecb; + const struct ccmode_ctr *ctr_info; size_t keylen; int strictFIPS; int use_df; diff --git a/EXTERNAL_HEADERS/corecrypto/cchmac.h b/EXTERNAL_HEADERS/corecrypto/cchmac.h index c3427eaab..81c1ab835 100644 --- a/EXTERNAL_HEADERS/corecrypto/cchmac.h +++ b/EXTERNAL_HEADERS/corecrypto/cchmac.h @@ -19,10 +19,14 @@ struct cchmac_ctx { uint8_t b[8]; } CC_ALIGNED(8); +#if CORECRYPTO_USE_TRANSPARENT_UNION typedef union { struct cchmac_ctx *hdr; ccdigest_ctx_t digest; } cchmac_ctx_t __attribute__((transparent_union)); +#else +typedef struct cchmac_ctx* cchmac_ctx_t; +#endif #define cchmac_ctx_size(STATE_SIZE, BLOCK_SIZE) (ccdigest_ctx_size(STATE_SIZE, BLOCK_SIZE) + (STATE_SIZE)) #define cchmac_di_size(_di_) (cchmac_ctx_size((_di_)->state_size, (_di_)->block_size)) @@ -35,24 +39,43 @@ typedef union { #define cchmac_di_clear(_di_, _name_) cchmac_ctx_clear((_di_)->state_size, (_di_)->block_size, _name_) /* Return a ccdigest_ctx_t which can be accesed with the macros in ccdigest.h */ +#if CORECRYPTO_USE_TRANSPARENT_UNION #define cchmac_digest_ctx(_di_, HC) (((cchmac_ctx_t)(HC)).digest) +#else +#define cchmac_digest_ctx(_di_, HC) ((ccdigest_ctx_t)(HC)) +#endif /* Accesors for ostate fields, this is all cchmac_ctx_t adds to the ccdigest_ctx_t. */ +#if CORECRYPTO_USE_TRANSPARENT_UNION #define cchmac_ostate(_di_, HC) ((struct ccdigest_state *)(((cchmac_ctx_t)(HC)).hdr->b + ccdigest_di_size(_di_))) +#else +#define cchmac_ostate(_di_, HC) ((struct ccdigest_state *)(((cchmac_ctx_t)(HC))->b + ccdigest_di_size(_di_))) +#endif #define cchmac_ostate8(_di_, HC) (ccdigest_u8(cchmac_ostate(_di_, HC))) #define cchmac_ostate32(_di_, HC) (ccdigest_u32(cchmac_ostate(_di_, HC))) #define cchmac_ostate64(_di_, HC) (ccdigest_u64(cchmac_ostate(_di_, HC))) #define cchmac_ostateccn(_di_, HC) (ccdigest_ccn(cchmac_ostate(_di_, HC))) /* Convenience accessors for ccdigest_ctx_t fields. */ +#if CORECRYPTO_USE_TRANSPARENT_UNION #define cchmac_istate(_di_, HC) ccdigest_state(_di_, ((cchmac_ctx_t)(HC)).digest) +#else +#define cchmac_istate(_di_, HC) ccdigest_state(_di_, ((ccdigest_ctx_t)(HC))) +#endif #define cchmac_istate8(_di_, HC) ccdigest_u8(cchmac_istate(_di_, HC)) #define cchmac_istate32(_di_, HC) ccdigest_u32(cchmac_istate(_di_, HC)) #define cchmac_istate64(_di_, HC) ccdigest_u64(cchmac_istate(_di_, HC)) #define cchmac_istateccn(_di_, HC) ccdigest_ccn(cchmac_istate(_di_, HC)) + +#if CORECRYPTO_USE_TRANSPARENT_UNION #define cchmac_data(_di_, HC) ccdigest_data(_di_, ((cchmac_ctx_t)(HC)).digest) #define cchmac_num(_di_, HC) ccdigest_num(_di_, ((cchmac_ctx_t)(HC)).digest) #define cchmac_nbits(_di_, HC) ccdigest_nbits(_di_, ((cchmac_ctx_t)(HC)).digest) +#else +#define cchmac_data(_di_, HC) ccdigest_data(_di_, ((ccdigest_ctx_t)(HC))) +#define cchmac_num(_di_, HC) ccdigest_num(_di_, ((ccdigest_ctx_t)(HC))) +#define cchmac_nbits(_di_, HC) ccdigest_nbits(_di_, ((ccdigest_ctx_t)(HC))) +#endif void cchmac_init(const struct ccdigest_info *di, cchmac_ctx_t ctx, size_t key_len, const void *key); diff --git a/EXTERNAL_HEADERS/corecrypto/ccmode.h b/EXTERNAL_HEADERS/corecrypto/ccmode.h index eda253b4d..191460b9b 100644 --- a/EXTERNAL_HEADERS/corecrypto/ccmode.h +++ b/EXTERNAL_HEADERS/corecrypto/ccmode.h @@ -115,23 +115,10 @@ CC_INLINE int cccbc_update(const struct ccmode_cbc *mode, cccbc_ctx *ctx, return mode->cbc(ctx, iv, nblocks, in, out); } -CC_INLINE int cccbc_one_shot(const struct ccmode_cbc *mode, - size_t key_len, const void *key, - const void *iv, size_t nblocks, - const void *in, void *out) -{ - int rc; - cccbc_ctx_decl(mode->size, ctx); - cccbc_iv_decl(mode->block_size, iv_ctx); - rc = mode->init(mode, ctx, key_len, key); - if (iv) - cccbc_set_iv(mode, iv_ctx, iv); - else - cc_zero(mode->block_size, iv_ctx); - mode->cbc(ctx, iv_ctx, nblocks, in, out); - cccbc_ctx_clear(mode->size, ctx); - return rc; -} +int cccbc_one_shot(const struct ccmode_cbc *mode, + size_t key_len, const void *key, + const void *iv, size_t nblocks, + const void *in, void *out); /* CFB mode. */ @@ -256,7 +243,8 @@ CC_INLINE int ccctr_one_shot(const struct ccmode_ctr *mode, int rc; ccctr_ctx_decl(mode->size, ctx); rc = mode->init(mode, ctx, key_len, key, iv); - mode->ctr(ctx, nbytes, in, out); + if (rc) return rc; + rc = mode->ctr(ctx, nbytes, in, out); ccctr_ctx_clear(mode->size, ctx); return rc; } @@ -429,6 +417,12 @@ int ccxts_one_shot(const struct ccmode_xts *mode, #define CCGCM_IV_NBYTES 12 #define CCGCM_BLOCK_NBYTES 16 +/* (2^32 - 2) blocks */ +/* (2^36 - 32) bytes */ +/* (2^39 - 256) bits */ +/* Exceeding this figure breaks confidentiality and authenticity. */ +#define CCGCM_TEXT_MAX_NBYTES ((1ULL << 36) - 32ULL) + CC_INLINE size_t ccgcm_context_size(const struct ccmode_gcm *mode) { return mode->size; @@ -470,6 +464,7 @@ CC_INLINE size_t ccgcm_block_size(const struct ccmode_gcm *mode) @warning It is not permitted to call @p ccgcm_inc_iv after initializing the cipher via the @p ccgcm_init interface. Nonzero is returned in the event of an improper call sequence. + @warning This function is not FIPS-compliant. Use @p ccgcm_init_with_iv instead. */ CC_INLINE int ccgcm_init(const struct ccmode_gcm *mode, ccgcm_ctx *ctx, size_t key_nbytes, const void *key) @@ -536,6 +531,8 @@ int ccgcm_init_with_iv(const struct ccmode_gcm *mode, ccgcm_ctx *ctx, In stateless protocols, it is recommended to choose a 16-byte value using a cryptographically-secure pseudorandom number generator (e.g. @p ccrng). @warning This function may not be used after initializing the cipher via @p ccgcm_init_with_iv. Nonzero is returned in the event of an improper call sequence. + + @warning This function is not FIPS-compliant. Use @p ccgcm_init_with_iv instead. */ CC_INLINE int ccgcm_set_iv(const struct ccmode_gcm *mode, ccgcm_ctx *ctx, size_t iv_nbytes, const void *iv) @@ -653,9 +650,9 @@ CC_INLINE int ccgcm_update(const struct ccmode_gcm *mode, ccgcm_ctx *ctx, On encryption, @p tag is purely an output parameter. The generated tag is written to @p tag. - On decryption, @p tag is primarily an input parameter. The caller should provide the authentication tag generated during encryption. The function will return nonzero if the input tag does not match the generated tag. + On decryption, @p tag is both an input and an output parameter. Well-behaved callers should provide the authentication tag generated during encryption. The function will return nonzero if the input tag does not match the generated tag. The generated tag will be written into the @p tag buffer whether authentication succeeds or fails. - @warning To support legacy applications, @p tag is also an output parameter during decryption. The generated tag is written to @p tag. Legacy callers may choose to compare this to the tag generated during encryption. Do not follow this usage pattern in new applications. + @warning The generated tag is written to @p tag to support legacy applications that perform authentication manually. Do not follow this usage pattern in new applications. Rely on the function's error code to verify authenticity. */ CC_INLINE int ccgcm_finalize(const struct ccmode_gcm *mode, ccgcm_ctx *ctx, size_t tag_nbytes, void *tag) @@ -815,10 +812,10 @@ CC_INLINE int ccccm_reset(const struct ccmode_ccm *mode, ccccm_ctx *ctx, ccccm_n CC_INLINE int ccccm_one_shot(const struct ccmode_ccm *mode, size_t key_len, const void *key, - unsigned nonce_len, const void *nonce, + size_t nonce_len, const void *nonce, size_t nbytes, const void *in, void *out, - unsigned adata_len, const void* adata, - unsigned mac_size, void *mac) + size_t adata_len, const void* adata, + size_t mac_size, void *mac) { int rc=0; ccccm_ctx_decl(mode->size, ctx); @@ -829,7 +826,7 @@ CC_INLINE int ccccm_one_shot(const struct ccmode_ccm *mode, if(rc==0) rc=mode->ccm(ctx, nonce_ctx, nbytes, in, out); if(rc==0) rc=mode->finalize(ctx, nonce_ctx, mac); ccccm_ctx_clear(mode->size, ctx); - ccccm_nonce_clear(mode->size, nonce_ctx); + ccccm_nonce_clear(mode->nonce_size, nonce_ctx); return rc; } diff --git a/EXTERNAL_HEADERS/corecrypto/ccmode_factory.h b/EXTERNAL_HEADERS/corecrypto/ccmode_factory.h index c05518e27..668ea9d59 100644 --- a/EXTERNAL_HEADERS/corecrypto/ccmode_factory.h +++ b/EXTERNAL_HEADERS/corecrypto/ccmode_factory.h @@ -251,12 +251,13 @@ void ccmode_factory_cfb8_encrypt(struct ccmode_cfb8 *cfb8, int ccmode_ctr_init(const struct ccmode_ctr *ctr, ccctr_ctx *ctx, size_t rawkey_len, const void *rawkey, const void *iv); +int ccmode_ctr_setctr(const struct ccmode_ctr *mode, ccctr_ctx *ctx, const void *ctr); int ccmode_ctr_crypt(ccctr_ctx *ctx, size_t nbytes, const void *in, void *out); struct _ccmode_ctr_key { const struct ccmode_ecb *ecb; - size_t pad_len; + size_t pad_offset; cc_unit u[]; }; @@ -264,7 +265,9 @@ struct _ccmode_ctr_key { #define CCMODE_FACTORY_CTR_CRYPT(ECB_ENCRYPT) { \ .size = ccn_sizeof_size(sizeof(struct _ccmode_ctr_key)) + 2 * ccn_sizeof_size((ECB_ENCRYPT)->block_size) + ccn_sizeof_size((ECB_ENCRYPT)->size), \ .block_size = 1, \ +.ecb_block_size = (ECB_ENCRYPT)->block_size, \ .init = ccmode_ctr_init, \ +.setctr = ccmode_ctr_setctr, \ .ctr = ccmode_ctr_crypt, \ .custom = (ECB_ENCRYPT) \ } @@ -292,13 +295,13 @@ int ccmode_gcm_encrypt(ccgcm_ctx *ctx, size_t nbytes, const void *in, /*! @function ccmode_gcm_finalize() finalizes AES-GCM call sequence @param key encryption or decryption key - @param tag_size - @param tag + @param tag_nbytes length of tag in bytes + @param tag authentication tag @result 0=success or non zero= error @discussion For decryption, the tag parameter must be the expected-tag. A secure compare is performed between the provided expected-tag and the computed-tag. If they are the same, 0 is returned. Otherwise, non zero is returned. For encryption, tag is output and provides the authentication tag. */ -int ccmode_gcm_finalize(ccgcm_ctx *key, size_t tag_size, void *tag); +int ccmode_gcm_finalize(ccgcm_ctx *key, size_t tag_nbytes, void *tag); int ccmode_gcm_reset(ccgcm_ctx *key); #define CCGCM_FLAGS_INIT_WITH_IV 1 @@ -331,7 +334,7 @@ struct _ccmode_gcm_key { int encdec; //is it an encrypt or decrypt object // Buffer with ECB key and H table if applicable - unsigned char u[] __attribute__ ((aligned (16))); // ecb key + tables + CC_ALIGNED(16) unsigned char u[]; // ecb key + tables }; #define GCM_ECB_KEY_SIZE(ECB_ENCRYPT) \ diff --git a/EXTERNAL_HEADERS/corecrypto/ccmode_impl.h b/EXTERNAL_HEADERS/corecrypto/ccmode_impl.h index 1337e1467..795054161 100644 --- a/EXTERNAL_HEADERS/corecrypto/ccmode_impl.h +++ b/EXTERNAL_HEADERS/corecrypto/ccmode_impl.h @@ -101,9 +101,11 @@ cc_aligned_struct(16) ccctr_ctx; struct ccmode_ctr { size_t size; /* first argument to ccctr_ctx_decl(). */ - size_t block_size; - int (*init)(const struct ccmode_ctr *ctr, ccctr_ctx *ctx, + size_t block_size; /* for historical reasons, this is set to 1 */ + size_t ecb_block_size; /* the actual block size of the underlying cipher */ + int (*init)(const struct ccmode_ctr *mode, ccctr_ctx *ctx, size_t key_len, const void *key, const void *iv); + int (*setctr)(const struct ccmode_ctr *mode, ccctr_ctx *ctx, const void *ctr); int (*ctr)(ccctr_ctx *ctx, size_t nbytes, const void *in, void *out); const void *custom; }; @@ -125,15 +127,13 @@ cc_aligned_struct(16) ccxts_ctx; cc_aligned_struct(16) ccxts_tweak; struct ccmode_xts { - size_t size; /* first argument to ccxts_ctx_decl(). */ - size_t tweak_size; /* first argument to ccxts_tweak_decl(). */ + size_t size; /* first argument to ccxts_ctx_decl(). Size of the ctx data structure */ + size_t tweak_size; /* first argument to ccxts_tweak_decl(). Size of the tweak structure, not the expected tweak size */ size_t block_size; - /* Create a xts key from a xts mode object. The tweak_len here - determines how long the tweak is in bytes, for each subsequent call to - ccmode_xts->xts(). - key must point to at least 'size' cc_units of free storage. - tweak_key must point to at least 'tweak_size' cc_units of free storage. + /* Create a xts key from a xts mode object. + key must point to at least 'size' bytes of free storage. + tweak_key must point to at least 'tweak_size' bytes of free storage. key and tweak_key must differ. Returns nonzero on failure. */ diff --git a/EXTERNAL_HEADERS/corecrypto/ccmode_siv.h b/EXTERNAL_HEADERS/corecrypto/ccmode_siv.h index 69069bb3a..5186e1227 100644 --- a/EXTERNAL_HEADERS/corecrypto/ccmode_siv.h +++ b/EXTERNAL_HEADERS/corecrypto/ccmode_siv.h @@ -68,8 +68,7 @@ CC_INLINE size_t ccsiv_plaintext_size(const struct ccmode_siv *mode, return ciphertext_size-mode->cbc->block_size; } -// In theory, supported key sizes are 32, 48, 64 bytes -// In practice, we only support key size 32 bytes due to cmac limitation +// Supported key sizes are 32, 48, 64 bytes CC_INLINE int ccsiv_init(const struct ccmode_siv *mode, ccsiv_ctx *ctx, size_t key_byte_len, const uint8_t *key) { @@ -115,7 +114,8 @@ CC_INLINE int ccsiv_one_shot(const struct ccmode_siv *mode, { int rc; ccsiv_ctx_decl(mode->size, ctx); - ccsiv_init(mode, ctx, key_len, key); + rc=mode->init(mode, ctx, key_len, key); + if (rc) {return rc;} rc=mode->set_nonce(ctx, nonce_nbytes, nonce); if (rc) {return rc;} rc=mode->auth(ctx, adata_nbytes, adata); diff --git a/EXTERNAL_HEADERS/corecrypto/ccn.h b/EXTERNAL_HEADERS/corecrypto/ccn.h index 53c152c88..afaed41ae 100644 --- a/EXTERNAL_HEADERS/corecrypto/ccn.h +++ b/EXTERNAL_HEADERS/corecrypto/ccn.h @@ -402,8 +402,8 @@ void ccn_mul(cc_size n, cc_unit *r_2n, const cc_unit *s, const cc_unit *t); { n bit, n bit -> 2 * n bit } n = count * sizeof(cc_unit) * 8 { N bit, N bit -> 2N bit } N = ccn_bitsof(n) Provide a workspace for potential speedup */ -CC_NONNULL((2, 3, 4, 5)) -void ccn_mul_ws(cc_size count, cc_unit *r, const cc_unit *s, const cc_unit *t, cc_ws_t ws); +CC_NONNULL((1, 3, 4, 5)) +void ccn_mul_ws(cc_ws_t ws, cc_size count, cc_unit *r, const cc_unit *s, const cc_unit *t); /* s[0..n) * v -> r[0..n)+return value { N bit, sizeof(cc_unit) * 8 bit -> N + sizeof(cc_unit) * 8 bit } N = n * sizeof(cc_unit) * 8 */ @@ -500,8 +500,8 @@ void ccn_sqr(cc_size n, cc_unit *r, const cc_unit *s); /* s^2 -> r { n bit -> 2 * n bit } */ -CC_NONNULL((2, 3, 4)) -void ccn_sqr_ws(cc_size n, cc_unit *r, const cc_unit *s, cc_ws_t ws); +CC_NONNULL((1, 3, 4)) +void ccn_sqr_ws(cc_ws_t ws, cc_size n, cc_unit *r, const cc_unit *s); #else @@ -515,8 +515,8 @@ void ccn_sqr(cc_size n, cc_unit *r, const cc_unit *s) { /* s^2 -> r { n bit -> 2 * n bit } */ CC_INLINE CC_NONNULL((2, 3, 4)) -void ccn_sqr_ws(cc_size n, cc_unit *r, const cc_unit *s, cc_ws_t ws) { - ccn_mul_ws(n, r, s, s, ws); +void ccn_sqr_ws(cc_ws_t ws, cc_size n, cc_unit *r, const cc_unit *s) { + ccn_mul_ws(ws, n, r, s, s); } #endif @@ -639,7 +639,7 @@ int ccn_random_bits(cc_size nbits, cc_unit *r, struct ccrng_state *rng); @param d input number d */ CC_NONNULL((2, 3)) -void ccn_make_recip(cc_size nd, cc_unit *recip, const cc_unit *d); +int ccn_make_recip(cc_size nd, cc_unit *recip, const cc_unit *d); CC_NONNULL((6, 8)) int ccn_div_euclid(cc_size nq, cc_unit *q, cc_size nr, cc_unit *r, cc_size na, const cc_unit *a, cc_size nd, const cc_unit *d); @@ -647,22 +647,4 @@ int ccn_div_euclid(cc_size nq, cc_unit *q, cc_size nr, cc_unit *r, cc_size na, c #define ccn_div(nq, q, na, a, nd, d) ccn_div_euclid(nq, q, 0, NULL, na, a, nd, d) #define ccn_mod(nr, r, na, a, nd, d) ccn_div_euclid(0 , NULL, nr, r, na, a, nd, d) -/*! - @brief ccn_div_use_recip(nq, q, nr, r, na, a, nd, d) comutes q=a/d and r=a%d - @discussion q and rcan be NULL. Reads na from a and nd from d. Writes nq in q and nr in r. nq and nr must be large enough to accomodate results, otherwise error is retuned. Execution time depends on the size of a. Computation is perfomed on of fixedsize and the leadig zeros of a of q are are also used in the computation. - @param nq length of array q that hold the quotients. The maximum length of quotient is the actual length of dividend a - @param q returned quotient. If nq is larger than needed, it is filled with leading zeros. If it is smaller, error is returned. q can be set to NULL, if not needed. - @param nr length of array r that hold the remainder. The maximum length of remainder is the actual length of divisor d - @param r returned remainder. If nr is larger than needed, it is filled with leading zeros. Ifi is smaller error is returned. r can be set to NULL if not required. - @param na length of dividend. Dividend may have leading zeros. - @param a input Dividend - @param nd length of input divisor. Divisor may have leading zeros. - @param d input Divisor - @param recip_d The reciprocal of d, of length nd+1. - - @return returns 0 if successful, negative of error. - */ -CC_NONNULL((6, 8, 9)) -int ccn_div_use_recip(cc_size nq, cc_unit *q, cc_size nr, cc_unit *r, cc_size na, const cc_unit *a, cc_size nd, const cc_unit *d, const cc_unit *recip_d); - #endif /* _CORECRYPTO_CCN_H_ */ diff --git a/EXTERNAL_HEADERS/corecrypto/ccrng.h b/EXTERNAL_HEADERS/corecrypto/ccrng.h index f32922276..698f412ca 100644 --- a/EXTERNAL_HEADERS/corecrypto/ccrng.h +++ b/EXTERNAL_HEADERS/corecrypto/ccrng.h @@ -11,33 +11,50 @@ #ifndef _CORECRYPTO_CCRNG_H_ #define _CORECRYPTO_CCRNG_H_ -#include <stdint.h> - #include <corecrypto/cc.h> -#define CC_ERR_DEVICE -100 -#define CC_ERR_INTERUPTS -101 -#define CC_ERR_CRYPTO_CONFIG -102 -#define CC_ERR_PERMS -103 -#define CC_ERR_PARAMETER -104 -#define CC_ERR_MEMORY -105 -#define CC_ERR_FILEDESC -106 -#define CC_ERR_OUT_OF_ENTROPY -107 -#define CC_ERR_INTERNAL -108 -#define CC_ERR_ATFORK -109 -#define CC_ERR_OVERFLOW -110 +#define CCERR_DEVICE -100 +#define CCERR_INTERUPTS -101 +#define CCERR_CRYPTO_CONFIG -102 +#define CCERR_PERMS -103 +#define CCERR_PARAMETER -104 +#define CCERR_MEMORY -105 +#define CCERR_FILEDESC -106 +#define CCERR_OUT_OF_ENTROPY -107 +#define CCERR_INTERNAL -108 +#define CCERR_ATFORK -109 +#define CCERR_OVERFLOW -110 #define CCRNG_STATE_COMMON \ int (*generate)(struct ccrng_state *rng, size_t outlen, void *out); -/* Get a pointer to a ccrng has never been simpler! Just call this */ -struct ccrng_state *ccrng(int *error); - -/* default state structure - do not instantiate, instead use the specific one you need */ +/* default state structure. Do not instantiate, ccrng() returns a reference to this structure */ struct ccrng_state { CCRNG_STATE_COMMON }; -#define ccrng_generate(ctx, outlen, out) ((ctx)->generate((ctx), (outlen), (out))) +/*! + @function ccrng + @abstract initializes a AES-CTR mode cryptographic random number generator and returns the statically alocated rng object. + Getting a pointer to a ccrng has never been simpler! + Call this function, get an rng object and then pass the object to ccrng_generate() to generate randoms. + ccrng() may be called more than once. It returns pointer to the same object on all calls. + + @result a cryptographically secure random number generator or NULL if fails + + @discussion + - It is significantly faster than using the system /dev/random + - FIPS Compliant: NIST SP800-80A + FIPS 140-2 + - Seeded from the system entropy. + - Provides at least 128bit security if the system provide 2bit of entropy / byte. + - Entropy accumulation + - Backtracing resistance + - Prediction break with frequent (asynchronous) reseed + */ + +struct ccrng_state *ccrng(int *error); + +//call this macro with the rng argument set to output of the call to the ccrng() function +#define ccrng_generate(rng, outlen, out) ((rng)->generate((rng), (outlen), (out))) #endif /* _CORECRYPTO_CCRNG_H_ */ diff --git a/EXTERNAL_HEADERS/corecrypto/ccrsa.h b/EXTERNAL_HEADERS/corecrypto/ccrsa.h index 97a88529a..c821efc40 100644 --- a/EXTERNAL_HEADERS/corecrypto/ccrsa.h +++ b/EXTERNAL_HEADERS/corecrypto/ccrsa.h @@ -182,7 +182,7 @@ ccrsa_pubkeylength(ccrsa_pub_ctx_t pubk) { /* Initialize key based on modulus and e as cc_unit. key->zp.n must already be set. */ CC_NONNULL_TU((1)) CC_NONNULL((2, 3)) -void ccrsa_init_pub(ccrsa_pub_ctx_t key, const cc_unit *modulus, +int ccrsa_init_pub(ccrsa_pub_ctx_t key, const cc_unit *modulus, const cc_unit *e); /* Initialize key based on modulus and e as big endian byte array diff --git a/EXTERNAL_HEADERS/corecrypto/cczp.h b/EXTERNAL_HEADERS/corecrypto/cczp.h index f19891bd8..f06b96a9d 100644 --- a/EXTERNAL_HEADERS/corecrypto/cczp.h +++ b/EXTERNAL_HEADERS/corecrypto/cczp.h @@ -41,7 +41,7 @@ typedef union { typedef struct cczp* cczp_t; typedef const struct cczp* cczp_const_t; #endif -typedef void (*ccmod_func_t)(cczp_const_t zp, cc_unit *r, const cc_unit *s, cc_ws_t ws); +typedef void (*ccmod_func_t)(cc_ws_t ws, cczp_const_t zp, cc_unit *r, const cc_unit *s); // keep cczp_hd and cczp structures consistent // cczp_hd is typecasted to cczp to read EC curve params @@ -168,7 +168,7 @@ CC_INLINE size_t cczp_bitlen(cczp_const_t zp) { /* Ensure both cczp_mod_prime(zp) and cczp_recip(zp) are valid. cczp_n and cczp_prime must have been previously initialized. */ CC_NONNULL_TU((1)) -void cczp_init(cczp_t zp); +int cczp_init(cczp_t zp); /* Compute r = s2n mod cczp_prime(zp). Will write cczp_n(zp) units to r and reads 2 * cczp_n(zp) units units from s2n. If r and s2n are not @@ -176,7 +176,7 @@ void cczp_init(cczp_t zp); cczp_init(zp) must have been called or both CCZP_MOD_PRIME((cc_unit *)zp) and CCZP_RECIP((cc_unit *)zp) must be initialized some other way. */ CC_NONNULL_TU((1)) CC_NONNULL((2, 3)) -void cczp_mod(cczp_const_t zp, cc_unit *r, const cc_unit *s2n, cc_ws_t ws); +void cczp_mod(cc_ws_t ws, cczp_const_t zp, cc_unit *r, const cc_unit *s2n); /* Compute r = sn mod cczp_prime(zp), Will write cczp_n(zp) units to r and reads sn units units from s. If r and s are not @@ -184,7 +184,6 @@ void cczp_mod(cczp_const_t zp, cc_unit *r, const cc_unit *s2n, cc_ws_t ws); cczp_init(zp) must have been called or both CCZP_MOD_PRIME((cc_unit *)zp) and CCZP_RECIP((cc_unit *)zp) must be initialized some other way. */ CC_NONNULL_TU((1)) CC_NONNULL((2, 4)) - int cczp_modn(cczp_const_t zp, cc_unit *r, cc_size ns, const cc_unit *s); /* Compute r = x * y mod cczp_prime(zp). Will write cczp_n(zp) units to r @@ -197,7 +196,7 @@ CC_NONNULL_TU((1)) CC_NONNULL((2, 3, 4)) void cczp_mul(cczp_const_t zp, cc_unit *t, const cc_unit *x, const cc_unit *y); CC_NONNULL_TU((1)) CC_NONNULL((2, 3, 4, 5)) -void cczp_mul_ws(cczp_const_t zp, cc_unit *t, const cc_unit *x, const cc_unit *y, cc_ws_t ws); +void cczp_mul_ws(cc_ws_t ws, cczp_const_t zp, cc_unit *t, const cc_unit *x, const cc_unit *y); /* Compute r = x * x mod cczp_prime(zp). Will write cczp_n(zp) units to r and reads cczp_n(zp) units from x. If r and x are not identical they must @@ -208,7 +207,7 @@ CC_NONNULL_TU((1)) CC_NONNULL((2, 3)) void cczp_sqr(cczp_const_t zp, cc_unit *r, const cc_unit *x); CC_NONNULL_TU((1)) CC_NONNULL((2, 3, 4)) -void cczp_sqr_ws(cczp_const_t zp, cc_unit *r, const cc_unit *x, cc_ws_t ws); +void cczp_sqr_ws(cc_ws_t ws, cczp_const_t zp, cc_unit *r, const cc_unit *x); /* Compute r = x^(1/2) mod cczp_prime(zp). Will write cczp_n(zp) units to r and reads cczp_n(zp) units from x. If r and x are not identical they must @@ -229,8 +228,8 @@ int cczp_sqrt(cczp_const_t zp, cc_unit *r, const cc_unit *x); be initialized some other way. */ CC_NONNULL_TU((1)) CC_NONNULL((2, 3, 4)) -void cczp_power(cczp_const_t zp, cc_unit *r, const cc_unit *m, - const cc_unit *e); +int cczp_power(cczp_const_t zp, cc_unit *r, const cc_unit *m, + const cc_unit *e); /* Compute r = m ^ e mod cczp_prime(zp), using Square Square Multiply Always. - writes cczp_n(zp) units to r @@ -258,8 +257,8 @@ int cczp_power_ssma_ws(cc_ws_t ws, cczp_const_t zp, cc_unit *r, const cc_unit *s or both CCZP_MOD_PRIME((cc_unit *)zp) and CCZP_RECIP((cc_unit *)zp) must be initialized some other way. */ CC_NONNULL_TU((1)) CC_NONNULL((2, 3, 5)) -void cczp_powern(cczp_const_t zp, cc_unit *r, const cc_unit *s, - size_t ebitlen, const cc_unit *e); +int cczp_powern(cczp_const_t zp, cc_unit *r, const cc_unit *s, + size_t ebitlen, const cc_unit *e); /* Compute r = x + y mod cczp_prime(zp). Will write cczp_n(zp) units to r and reads cczp_n(zp) units units from x and y. If r and x are not identical @@ -270,8 +269,8 @@ void cczp_add(cczp_const_t zp, cc_unit *r, const cc_unit *x, const cc_unit *y); CC_NONNULL_TU((1)) CC_NONNULL((2, 3, 4, 5)) -void cczp_add_ws(cczp_const_t zp, cc_unit *r, const cc_unit *x, - const cc_unit *y, cc_ws_t ws); +void cczp_add_ws(cc_ws_t ws, cczp_const_t zp, cc_unit *r, const cc_unit *x, + const cc_unit *y); /* Compute r = x - y mod cczp_prime(zp). Will write cczp_n(zp) units to r and reads cczp_n(zp) units units from x and y. If r and x are not identical @@ -281,8 +280,8 @@ CC_NONNULL_TU((1)) CC_NONNULL((2, 3, 4)) void cczp_sub(cczp_const_t zp, cc_unit *r, const cc_unit *x, const cc_unit *y); CC_NONNULL_TU((1)) CC_NONNULL((2, 3, 4, 5)) -void cczp_sub_ws(cczp_const_t zp, cc_unit *r, const cc_unit *x, - const cc_unit *y, cc_ws_t ws); +void cczp_sub_ws(cc_ws_t ws, cczp_const_t zp, cc_unit *r, const cc_unit *x, + const cc_unit *y); /* Compute r = x / 2 mod cczp_prime(zp). Will write cczp_n(zp) units to r and reads cczp_n(zp) units units from x. If r and x are not identical diff --git a/EXTERNAL_HEADERS/corecrypto/fipspost_trace.h b/EXTERNAL_HEADERS/corecrypto/fipspost_trace.h new file mode 100644 index 000000000..c236bebd7 --- /dev/null +++ b/EXTERNAL_HEADERS/corecrypto/fipspost_trace.h @@ -0,0 +1,45 @@ +/* + * fipspost_trace.h + * corecrypto + * + * Created on 01/25/2017 + * + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + */ + +#ifndef _CORECRYPTO_FIPSPOST_TRACE_H_ +#define _CORECRYPTO_FIPSPOST_TRACE_H_ + +#if CC_FIPSPOST_TRACE + +/* + * Use this string to separate out tests. + */ +#define FIPSPOST_TRACE_TEST_STR "?" + +int fipspost_trace_is_active(void); +void fipspost_trace_call(const char *fname); + +/* Only trace when VERBOSE is set to avoid impacting normal boots. */ +#define FIPSPOST_TRACE_EVENT do { \ + if (fipspost_trace_is_active()) { \ + fipspost_trace_call(__FUNCTION__); \ + } \ +} while (0); + +#define FIPSPOST_TRACE_MESSAGE(MSG) do { \ + if (fipspost_trace_is_active()) { \ + fipspost_trace_call(MSG); \ + } \ +} while (0); + +#else + +/* Not building a CC_FIPSPOST_TRACE-enabled, no TRACE operations. */ +#define FIPSPOST_TRACE_EVENT +#define FIPSPOST_TRACE_MESSAGE(X) + +#endif + +#endif diff --git a/EXTERNAL_HEADERS/mach-o/arm/reloc.h b/EXTERNAL_HEADERS/mach-o/arm/reloc.h new file mode 100644 index 000000000..2447814fd --- /dev/null +++ b/EXTERNAL_HEADERS/mach-o/arm/reloc.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 1999 Apple Computer, Inc. All rights reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ +/* + * Relocation types used in the arm implementation. Relocation entries for + * things other than instructions use the same generic relocation as discribed + * in <mach-o/reloc.h> and their r_type is ARM_RELOC_VANILLA, one of the + * *_SECTDIFF or the *_PB_LA_PTR types. The rest of the relocation types are + * for instructions. Since they are for instructions the r_address field + * indicates the 32 bit instruction that the relocation is to be preformed on. + */ +enum reloc_type_arm +{ + ARM_RELOC_VANILLA, /* generic relocation as discribed above */ + ARM_RELOC_PAIR, /* the second relocation entry of a pair */ + ARM_RELOC_SECTDIFF, /* a PAIR follows with subtract symbol value */ + ARM_RELOC_LOCAL_SECTDIFF, /* like ARM_RELOC_SECTDIFF, but the symbol + referenced was local. */ + ARM_RELOC_PB_LA_PTR,/* prebound lazy pointer */ + ARM_RELOC_BR24, /* 24 bit branch displacement (to a word address) */ + ARM_THUMB_RELOC_BR22, /* 22 bit branch displacement (to a half-word + address) */ + ARM_THUMB_32BIT_BRANCH, /* obsolete - a thumb 32-bit branch instruction + possibly needing page-spanning branch workaround */ + + /* + * For these two r_type relocations they always have a pair following them + * and the r_length bits are used differently. The encoding of the + * r_length is as follows: + * low bit of r_length: + * 0 - :lower16: for movw instructions + * 1 - :upper16: for movt instructions + * high bit of r_length: + * 0 - arm instructions + * 1 - thumb instructions + * the other half of the relocated expression is in the following pair + * relocation entry in the the low 16 bits of r_address field. + */ + ARM_RELOC_HALF, + ARM_RELOC_HALF_SECTDIFF +}; diff --git a/EXTERNAL_HEADERS/mach-o/arm64/reloc.h b/EXTERNAL_HEADERS/mach-o/arm64/reloc.h new file mode 100644 index 000000000..0a98f18e7 --- /dev/null +++ b/EXTERNAL_HEADERS/mach-o/arm64/reloc.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2010 Apple Computer, Inc. All rights reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ +/* + * Relocation types used in the arm64 implementation. + */ +enum reloc_type_arm64 +{ + ARM64_RELOC_UNSIGNED, // for pointers + ARM64_RELOC_SUBTRACTOR, // must be followed by a ARM64_RELOC_UNSIGNED + ARM64_RELOC_BRANCH26, // a B/BL instruction with 26-bit displacement + ARM64_RELOC_PAGE21, // pc-rel distance to page of target + ARM64_RELOC_PAGEOFF12, // offset within page, scaled by r_length + ARM64_RELOC_GOT_LOAD_PAGE21, // pc-rel distance to page of GOT slot + ARM64_RELOC_GOT_LOAD_PAGEOFF12, // offset within page of GOT slot, + // scaled by r_length + ARM64_RELOC_POINTER_TO_GOT, // for pointers to GOT slots + ARM64_RELOC_TLVP_LOAD_PAGE21, // pc-rel distance to page of TLVP slot + ARM64_RELOC_TLVP_LOAD_PAGEOFF12, // offset within page of TLVP slot, + // scaled by r_length + ARM64_RELOC_ADDEND // must be followed by PAGE21 or PAGEOFF12 +}; diff --git a/EXTERNAL_HEADERS/mach-o/loader.h b/EXTERNAL_HEADERS/mach-o/loader.h index ffaf873d8..d6bc7e0cd 100644 --- a/EXTERNAL_HEADERS/mach-o/loader.h +++ b/EXTERNAL_HEADERS/mach-o/loader.h @@ -302,6 +302,8 @@ struct load_command { #define LC_LINKER_OPTIMIZATION_HINT 0x2E /* optimization hints in MH_OBJECT files */ #define LC_VERSION_MIN_TVOS 0x2F /* build for AppleTV min OS version */ #define LC_VERSION_MIN_WATCHOS 0x30 /* build for Watch min OS version */ +#define LC_NOTE 0x31 /* arbitrary data included within a Mach-O file */ +#define LC_BUILD_VERSION 0x32 /* build for platform min OS version */ /* * A variable length string in a load command is represented by an lc_str @@ -1203,13 +1205,45 @@ struct encryption_info_command_64 { */ struct version_min_command { uint32_t cmd; /* LC_VERSION_MIN_MACOSX or - LC_VERSION_MIN_IPHONEOS - LC_VERSION_MIN_WATCHOS */ + LC_VERSION_MIN_IPHONEOS or + LC_VERSION_MIN_WATCHOS or + LC_VERSION_MIN_TVOS */ uint32_t cmdsize; /* sizeof(struct min_version_command) */ uint32_t version; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */ uint32_t sdk; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */ }; +/* + * The build_version_command contains the min OS version on which this + * binary was built to run for its platform. The list of known platforms and + * tool values following it. + */ +struct build_version_command { + uint32_t cmd; /* LC_BUILD_VERSION */ + uint32_t cmdsize; /* sizeof(struct build_version_command) plus */ + /* ntools * sizeof(struct build_tool_version) */ + uint32_t platform; /* platform */ + uint32_t minos; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */ + uint32_t sdk; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */ + uint32_t ntools; /* number of tool entries following this */ +}; + +struct build_tool_version { + uint32_t tool; /* enum for the tool */ + uint32_t version; /* version number of the tool */ +}; + +/* Known values for the platform field above. */ +#define PLATFORM_MACOS 1 +#define PLATFORM_IOS 2 +#define PLATFORM_TVOS 3 +#define PLATFORM_WATCHOS 4 + +/* Known values for the tool field above. */ +#define TOOL_CLANG 1 +#define TOOL_SWIFT 2 +#define TOOL_LD 3 + /* * The dyld_info_command contains the file offsets and sizes of * the new compressed form of the information dyld needs to @@ -1489,4 +1523,16 @@ struct tlv_descriptor unsigned long offset; }; +/* + * LC_NOTE commands describe a region of arbitrary data included in a Mach-O + * file. Its initial use is to record extra data in MH_CORE files. + */ +struct note_command { + uint32_t cmd; /* LC_NOTE */ + uint32_t cmdsize; /* sizeof(struct note_command) */ + char data_owner[16]; /* owner name for this LC_NOTE */ + uint64_t offset; /* file offset of this data */ + uint64_t size; /* length of data region */ +}; + #endif /* _MACHO_LOADER_H_ */ diff --git a/Makefile b/Makefile index c4c0663ef..fa9f39132 100644 --- a/Makefile +++ b/Makefile @@ -196,16 +196,17 @@ include $(MakeInc_cmd) include $(MakeInc_def) ALL_SUBDIRS = \ + security \ bsd \ iokit \ osfmk \ pexpert \ libkern \ libsa \ - security \ - config + config \ + san -CONFIG_SUBDIRS = config tools +CONFIG_SUBDIRS = config tools san INSTINC_SUBDIRS = $(ALL_SUBDIRS) EXTERNAL_HEADERS INSTINC_SUBDIRS_X86_64 = $(INSTINC_SUBDIRS) @@ -219,7 +220,7 @@ EXPINC_SUBDIRS_X86_64H = $(EXPINC_SUBDIRS) EXPINC_SUBDIRS_ARM = $(EXPINC_SUBDIRS) EXPINC_SUBDIRS_ARM64 = $(EXPINC_SUBDIRS) -SETUP_SUBDIRS = SETUP +SETUP_SUBDIRS = SETUP san COMP_SUBDIRS_X86_64 = $(ALL_SUBDIRS) COMP_SUBDIRS_X86_64H = $(ALL_SUBDIRS) diff --git a/README.md b/README.md index 2d3ba49a8..b285c1a6a 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,8 @@ Other makefile options * $ make REMOTEBUILD=user@remotehost # perform build on remote host * $ make BUILD_JSON_COMPILATION_DATABASE=1 # Build Clang JSON Compilation Database +The XNU build system can optionally output color-formatted build output. To enable this, you can either +set the `XNU_LOGCOLORS` environment variable to `y`, or you can pass `LOGCOLORS=y` to the make command. Debug information formats @@ -244,6 +246,11 @@ member file lists and their default location are described below - Definition - INSTALL_KF_MI_LCL_LIST = ${KERNELFILES} ${PRIVATE_KERNELFILES} + e. `EXPORT_MI_LIST` : Exports header file to all of xnu (bsd/, osfmk/, etc.) + for compilation only. Does not install anything into the SDK. + Definition - + EXPORT_MI_LIST = ${KERNELFILES} ${PRIVATE_KERNELFILES} + If you want to install the header file in a sub-directory of the paths described in (1), specify the directory name using two variables `INSTALL_MI_DIR` and `EXPORT_MI_DIR` as follows - diff --git a/SETUP/kextsymboltool/kextsymboltool.c b/SETUP/kextsymboltool/kextsymboltool.c index 8bd2c293c..edb6dfaea 100644 --- a/SETUP/kextsymboltool/kextsymboltool.c +++ b/SETUP/kextsymboltool/kextsymboltool.c @@ -597,11 +597,6 @@ int main(int argc, char * argv[]) else num_export_syms += files[filenum].nsyms; } - if (!num_export_syms) - { - fprintf(stderr, "no export names\n"); - exit(1); - } import_symbols = calloc(num_import_syms, sizeof(struct symbol)); export_symbols = calloc(num_export_syms, sizeof(struct symbol)); diff --git a/bsd/arm/Makefile b/bsd/arm/Makefile new file mode 100644 index 000000000..f9ab9a989 --- /dev/null +++ b/bsd/arm/Makefile @@ -0,0 +1,34 @@ +export MakeInc_cmd=${SRCROOT}/makedefs/MakeInc.cmd +export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def +export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule +export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir + +include $(MakeInc_cmd) +include $(MakeInc_def) + +DATAFILES = \ + endian.h fasttrap_isa.h param.h \ + profile.h signal.h limits.h _limits.h \ + types.h vmparam.h _types.h _param.h \ + _mcontext.h + +PRIVATE_DATAFILES = \ + disklabel.h + +KERNELFILES = \ + endian.h param.h \ + profile.h signal.h limits.h _limits.h \ + types.h vmparam.h _types.h _param.h \ + _mcontext.h + +INSTALL_MD_LIST = ${DATAFILES} +INSTALL_MD_LCL_LIST = ${PRIVATE_DATAFILES} + +INSTALL_MD_DIR = arm + +EXPORT_MD_LIST = ${KERNELFILES} + +EXPORT_MD_DIR = arm + +include $(MakeInc_rule) +include $(MakeInc_dir) diff --git a/bsd/arm/_limits.h b/bsd/arm/_limits.h new file mode 100644 index 000000000..f3d3fcb2c --- /dev/null +++ b/bsd/arm/_limits.h @@ -0,0 +1,9 @@ +/* + * Copyright (c) 2004-2007 Apple Inc. All rights reserved. + */ +#ifndef _ARM__LIMITS_H_ +#define _ARM__LIMITS_H_ + +#define __DARWIN_CLK_TCK 100 /* ticks per second */ + +#endif /* _ARM__LIMITS_H_ */ diff --git a/bsd/arm/_mcontext.h b/bsd/arm/_mcontext.h new file mode 100644 index 000000000..5a2e735cf --- /dev/null +++ b/bsd/arm/_mcontext.h @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2003-2012 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef __ARM_MCONTEXT_H_ +#define __ARM_MCONTEXT_H_ + +#include <sys/cdefs.h> /* __DARWIN_UNIX03 */ +#include <sys/appleapiopts.h> +#include <mach/machine/_structs.h> + +#ifndef _STRUCT_MCONTEXT32 +#if __DARWIN_UNIX03 +#define _STRUCT_MCONTEXT32 struct __darwin_mcontext32 +_STRUCT_MCONTEXT32 +{ + _STRUCT_ARM_EXCEPTION_STATE __es; + _STRUCT_ARM_THREAD_STATE __ss; + _STRUCT_ARM_VFP_STATE __fs; +}; + +#else /* !__DARWIN_UNIX03 */ +#define _STRUCT_MCONTEXT32 struct mcontext32 +_STRUCT_MCONTEXT32 +{ + _STRUCT_ARM_EXCEPTION_STATE es; + _STRUCT_ARM_THREAD_STATE ss; + _STRUCT_ARM_VFP_STATE fs; +}; + +#endif /* __DARWIN_UNIX03 */ +#endif /* _STRUCT_MCONTEXT32 */ + + +#ifndef _STRUCT_MCONTEXT64 +#if __DARWIN_UNIX03 +#define _STRUCT_MCONTEXT64 struct __darwin_mcontext64 +_STRUCT_MCONTEXT64 +{ + _STRUCT_ARM_EXCEPTION_STATE64 __es; + _STRUCT_ARM_THREAD_STATE64 __ss; + _STRUCT_ARM_NEON_STATE64 __ns; +}; + +#else /* !__DARWIN_UNIX03 */ +#define _STRUCT_MCONTEXT64 struct mcontext64 +_STRUCT_MCONTEXT64 +{ + _STRUCT_ARM_EXCEPTION_STATE64 es; + _STRUCT_ARM_THREAD_STATE64 ss; + _STRUCT_ARM_NEON_STATE64 ns; +}; +#endif /* __DARWIN_UNIX03 */ +#endif /* _STRUCT_MCONTEXT32 */ + +#ifndef _MCONTEXT_T +#define _MCONTEXT_T +#if defined(__LP64__) +typedef _STRUCT_MCONTEXT64 *mcontext_t; +#define _STRUCT_MCONTEXT _STRUCT_MCONTEXT64 +#else +typedef _STRUCT_MCONTEXT32 *mcontext_t; +#define _STRUCT_MCONTEXT _STRUCT_MCONTEXT32 +#endif +#endif /* _MCONTEXT_T */ + +#endif /* __ARM_MCONTEXT_H_ */ diff --git a/bsd/arm/_param.h b/bsd/arm/_param.h new file mode 100644 index 000000000..2d1e03a96 --- /dev/null +++ b/bsd/arm/_param.h @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2006-2007 Apple Inc. All rights reserved. + */ + +#ifndef _ARM__PARAM_H_ +#define _ARM__PARAM_H_ + +#include <arm/_types.h> + +/* + * Round p (pointer or byte index) up to a correctly-aligned value for all + * data types (int, long, ...). The result is unsigned int and must be + * cast to any desired pointer type. + */ +#define __DARWIN_ALIGNBYTES (sizeof(__darwin_size_t) - 1) +#define __DARWIN_ALIGN(p) ((__darwin_size_t)((char *)(__darwin_size_t)(p) + __DARWIN_ALIGNBYTES) &~ __DARWIN_ALIGNBYTES) + +#define __DARWIN_ALIGNBYTES32 (sizeof(__uint32_t) - 1) +#define __DARWIN_ALIGN32(p) ((__darwin_size_t)((char *)(__darwin_size_t)(p) + __DARWIN_ALIGNBYTES32) &~ __DARWIN_ALIGNBYTES32) + + +#endif /* _ARM__PARAM_H_ */ diff --git a/bsd/arm/_types.h b/bsd/arm/_types.h new file mode 100644 index 000000000..d76d8a64b --- /dev/null +++ b/bsd/arm/_types.h @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +#ifndef _BSD_ARM__TYPES_H_ +#define _BSD_ARM__TYPES_H_ + +/* + * This header file contains integer types. It's intended to also contain + * flotaing point and other arithmetic types, as needed, later. + */ + +#ifdef __GNUC__ +typedef __signed char __int8_t; +#else /* !__GNUC__ */ +typedef char __int8_t; +#endif /* !__GNUC__ */ +typedef unsigned char __uint8_t; +typedef short __int16_t; +typedef unsigned short __uint16_t; +typedef int __int32_t; +typedef unsigned int __uint32_t; +typedef long long __int64_t; +typedef unsigned long long __uint64_t; + +typedef long __darwin_intptr_t; +typedef unsigned int __darwin_natural_t; + +/* + * The rune type below is declared to be an ``int'' instead of the more natural + * ``unsigned long'' or ``long''. Two things are happening here. It is not + * unsigned so that EOF (-1) can be naturally assigned to it and used. Also, + * it looks like 10646 will be a 31 bit standard. This means that if your + * ints cannot hold 32 bits, you will be in trouble. The reason an int was + * chosen over a long is that the is*() and to*() routines take ints (says + * ANSI C), but they use __darwin_ct_rune_t instead of int. By changing it + * here, you lose a bit of ANSI conformance, but your programs will still + * work. + * + * NOTE: rune_t is not covered by ANSI nor other standards, and should not + * be instantiated outside of lib/libc/locale. Use wchar_t. wchar_t and + * rune_t must be the same type. Also wint_t must be no narrower than + * wchar_t, and should also be able to hold all members of the largest + * character set plus one extra value (WEOF). wint_t must be at least 16 bits. + */ + +typedef int __darwin_ct_rune_t; /* ct_rune_t */ + +/* + * mbstate_t is an opaque object to keep conversion state, during multibyte + * stream conversions. The content must not be referenced by user programs. + */ +typedef union { + char __mbstate8[128]; + long long _mbstateL; /* for alignment */ +} __mbstate_t; + +typedef __mbstate_t __darwin_mbstate_t; /* mbstate_t */ + +#if defined(__PTRDIFF_TYPE__) +typedef __PTRDIFF_TYPE__ __darwin_ptrdiff_t; /* ptr1 - ptr2 */ +#elif defined(__LP64__) +typedef long __darwin_ptrdiff_t; /* ptr1 - ptr2 */ +#else +typedef int __darwin_ptrdiff_t; /* ptr1 - ptr2 */ +#endif /* __GNUC__ */ + +#if defined(__SIZE_TYPE__) +typedef __SIZE_TYPE__ __darwin_size_t; /* sizeof() */ +#else +typedef unsigned long __darwin_size_t; /* sizeof() */ +#endif + +#if (__GNUC__ > 2) +typedef __builtin_va_list __darwin_va_list; /* va_list */ +#else +typedef void * __darwin_va_list; /* va_list */ +#endif + +#if defined(__WCHAR_TYPE__) +typedef __WCHAR_TYPE__ __darwin_wchar_t; /* wchar_t */ +#else +typedef __darwin_ct_rune_t __darwin_wchar_t; /* wchar_t */ +#endif + +typedef __darwin_wchar_t __darwin_rune_t; /* rune_t */ + +#if defined(__WINT_TYPE__) +typedef __WINT_TYPE__ __darwin_wint_t; /* wint_t */ +#else +typedef __darwin_ct_rune_t __darwin_wint_t; /* wint_t */ +#endif + +typedef unsigned long __darwin_clock_t; /* clock() */ +typedef __uint32_t __darwin_socklen_t; /* socklen_t (duh) */ +typedef long __darwin_ssize_t; /* byte count or error */ +typedef long __darwin_time_t; /* time() */ + +#endif /* _BSD_ARM__TYPES_H_ */ diff --git a/bsd/arm/disklabel.h b/bsd/arm/disklabel.h new file mode 100644 index 000000000..966f66d50 --- /dev/null +++ b/bsd/arm/disklabel.h @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +#ifndef _MACHINE_DISKLABEL_H_ +#define _MACHINE_DISKLABEL_H_ + +#include <sys/appleapiopts.h> + +#ifdef __APPLE_API_OBSOLETE +#define LABELSECTOR (1024 / DEV_BSIZE) /* sector containing label */ +#define LABELOFFSET 0 /* offset of label in sector */ +#define MAXPARTITIONS 8 /* number of partitions */ +#define RAW_PART 2 /* raw partition: xx?c */ + +/* Just a dummy */ +struct cpu_disklabel { + int cd_dummy; /* must have one element. */ +}; +#endif /* __APPLE_API_OBSOLETE */ + +#endif /* _MACHINE_DISKLABEL_H_ */ diff --git a/bsd/arm/endian.h b/bsd/arm/endian.h new file mode 100644 index 000000000..6cd67268d --- /dev/null +++ b/bsd/arm/endian.h @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +/* + * Copyright 1995 NeXT Computer, Inc. All rights reserved. + */ +/* + * Copyright (c) 1987, 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)endian.h 8.1 (Berkeley) 6/11/93 + */ + +#ifndef _ARM__ENDIAN_H_ +#define _ARM__ENDIAN_H_ + +#include <sys/cdefs.h> +/* + * Define _NOQUAD if the compiler does NOT support 64-bit integers. + */ +/* #define _NOQUAD */ + +/* + * Define the order of 32-bit words in 64-bit words. + */ +#define _QUAD_HIGHWORD 1 +#define _QUAD_LOWWORD 0 + +/* + * Definitions for byte order, according to byte significance from low + * address to high. + */ +#define __DARWIN_LITTLE_ENDIAN 1234 /* LSB first: i386, vax */ +#define __DARWIN_BIG_ENDIAN 4321 /* MSB first: 68000, ibm, net */ +#define __DARWIN_PDP_ENDIAN 3412 /* LSB first in word, MSW first in long */ + +#define __DARWIN_BYTE_ORDER __DARWIN_LITTLE_ENDIAN + +#if defined(KERNEL) || (!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)) + +#define LITTLE_ENDIAN __DARWIN_LITTLE_ENDIAN +#define BIG_ENDIAN __DARWIN_BIG_ENDIAN +#define PDP_ENDIAN __DARWIN_PDP_ENDIAN + +#define BYTE_ORDER __DARWIN_BYTE_ORDER + +#include <sys/_endian.h> + +#endif /* defined(KERNEL) || (!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)) */ +#endif /* !_ARM__ENDIAN_H_ */ diff --git a/bsd/arm/exec.h b/bsd/arm/exec.h new file mode 100644 index 000000000..e1266aacd --- /dev/null +++ b/bsd/arm/exec.h @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +/*- + * Copyright (c) 1992, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)exec.h 8.1 (Berkeley) 6/11/93 + */ + +#ifndef _BSD_ARM_EXEC_H_ +#define _BSD_ARM_EXEC_H_ + + +#ifdef BSD_KERNEL_PRIVATE +/* Size of a page in an object file. */ +#define __LDPGSZ 4096 + +/* Valid magic number check. */ +#define N_BADMAG(ex) \ + ((ex).a_magic != NMAGIC && (ex).a_magic != OMAGIC && \ + (ex).a_magic != ZMAGIC) + +/* Address of the bottom of the text segment. */ +#define N_TXTADDR(X) 0 + +/* Address of the bottom of the data segment. */ +#define N_DATADDR(ex) \ + (N_TXTADDR(ex) + ((ex).a_magic == OMAGIC ? (ex).a_text \ + : __LDPGSZ + ((ex).a_text - 1 & ~(__LDPGSZ - 1)))) + +/* Text segment offset. */ +#define N_TXTOFF(ex) \ + ((ex).a_magic == ZMAGIC ? __LDPGSZ : sizeof(struct exec)) + +/* Data segment offset. */ +#define N_DATOFF(ex) \ + (N_TXTOFF(ex) + ((ex).a_magic != ZMAGIC ? (ex).a_text : \ + __LDPGSZ + ((ex).a_text - 1 & ~(__LDPGSZ - 1)))) + +/* Symbol table offset. */ +#define N_SYMOFF(ex) \ + (N_TXTOFF(ex) + (ex).a_text + (ex).a_data + (ex).a_trsize + \ + (ex).a_drsize) + +/* String table offset. */ +#define N_STROFF(ex) (N_SYMOFF(ex) + (ex).a_syms) + +/* Description of the object file header (a.out format). */ +struct exec { +#define OMAGIC 0407 /* old impure format */ +#define NMAGIC 0410 /* read-only text */ +#define ZMAGIC 0413 /* demand load format */ +#define QMAGIC 0314 /* demand load format. Header in text. */ + unsigned int a_magic; /* magic number */ + + unsigned int a_text; /* text segment size */ + unsigned int a_data; /* initialized data size */ + unsigned int a_bss; /* uninitialized data size */ + unsigned int a_syms; /* symbol table size */ + unsigned int a_entry; /* entry point */ + unsigned int a_trsize; /* text relocation size */ + unsigned int a_drsize; /* data relocation size */ +}; + +#endif /* BSD_KERNEL_PRIVATE */ + +#endif /* _BSD_ARM_EXEC_H_ */ diff --git a/bsd/arm/fasttrap_isa.h b/bsd/arm/fasttrap_isa.h new file mode 100644 index 000000000..eb577a43f --- /dev/null +++ b/bsd/arm/fasttrap_isa.h @@ -0,0 +1,230 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + */ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2004 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _FASTTRAP_ISA_H +#define _FASTTRAP_ISA_H + +/* #pragma ident "@(#)fasttrap_isa.h 1.4 05/06/08 SMI" */ + +#include <sys/types.h> +#include <stdint.h> + +#ifdef __cplusplus +extern "C" { +#endif + +typedef union { + uint32_t instr32; + struct { + uint16_t instr1; + uint16_t instr2; + } instr16; +} fasttrap_instr_t; + +typedef struct fasttrap_machtp { + fasttrap_instr_t ftmt_instr; /* original instruction */ + + uint8_t ftmt_fntype; /* One of the FASTTRAP_FN* constants defined below */ + /* Once the tracepoint is initialized, fntype will be FN_DONE_INIT and thumb will be 0 for ARM, 1 for Thumb */ + uint8_t ftmt_thumb; + + uint8_t ftmt_type; + uint8_t ftmt_installed:1; + uint8_t ftmt_retired:1; +} fasttrap_machtp_t; + +#define ftt_instr ftt_mtp.ftmt_instr.instr32 +#define ftt_instr1 ftt_mtp.ftmt_instr.instr16.instr1 +#define ftt_instr2 ftt_mtp.ftmt_instr.instr16.instr2 +#define ftt_fntype ftt_mtp.ftmt_fntype +#define ftt_thumb ftt_mtp.ftmt_thumb +#define ftt_type ftt_mtp.ftmt_type +#define ftt_installed ftt_mtp.ftmt_installed +#define ftt_retired ftt_mtp.ftmt_retired + +#define FASTTRAP_T_INV 1 +#define FASTTRAP_T_COMMON 2 +#define FASTTRAP_T_BLX 3 +#define FASTTRAP_T_B_COND 4 +#define FASTTRAP_T_B_UNCOND 5 +#define FASTTRAP_T_BX_REG 6 +#define FASTTRAP_T_PUSH_LR 7 +#define FASTTRAP_T_POP_PC 8 +#define FASTTRAP_T_STM_LR 9 +#define FASTTRAP_T_LDM_PC 10 +#define FASTTRAP_T_CPY_PC 11 +#define FASTTRAP_T_MOV_PC_REG 12 +#define FASTTRAP_T_LDR_PC_IMMED 13 +#define FASTTRAP_T_VLDR_PC_IMMED 14 +#define FASTTRAP_T_CB_N_Z 15 +#if defined(__arm64__) +#define FASTTRAP_T_ARM64_STANDARD_FUNCTION_ENTRY 16 /* stp fp, lr, [sp, #-16]! */ +#define FASTTRAP_T_ARM64_LDR_S_PC_REL 17 +#define FASTTRAP_T_ARM64_LDR_W_PC_REL 18 +#define FASTTRAP_T_ARM64_LDR_D_PC_REL 19 +#define FASTTRAP_T_ARM64_LDR_X_PC_REL 20 +#define FASTTRAP_T_ARM64_LDR_Q_PC_REL 21 +#define FASTTRAP_T_ARM64_LDRSW_PC_REL 22 +#define FASTTRAP_T_ARM64_B_COND 23 +#define FASTTRAP_T_ARM64_CBNZ_W 24 +#define FASTTRAP_T_ARM64_CBNZ_X 25 +#define FASTTRAP_T_ARM64_CBZ_W 26 +#define FASTTRAP_T_ARM64_CBZ_X 27 +#define FASTTRAP_T_ARM64_TBNZ 28 +#define FASTTRAP_T_ARM64_TBZ 29 +#define FASTTRAP_T_ARM64_B 30 +#define FASTTRAP_T_ARM64_BL 31 +#define FASTTRAP_T_ARM64_BLR 32 +#define FASTTRAP_T_ARM64_BR 33 +#define FASTTRAP_T_ARM64_RET 34 +#define FASTTRAP_T_ARM64_ADRP 35 +#define FASTTRAP_T_ARM64_ADR 36 +#define FASTTRAP_T_ARM64_PRFM 37 +#define FASTTRAP_T_ARM64_EXCLUSIVE_MEM 38 +#endif + +#if defined (__arm__) +#define FASTTRAP_ARM_INSTR 0xe7ffdefc +#define FASTTRAP_THUMB_INSTR 0xdefc +#define FASTTRAP_ARM_RET_INSTR 0xe7ffdefb +#define FASTTRAP_THUMB_RET_INSTR 0xdefb + +#elif defined (__arm64__) +#define FASTTRAP_ARM32_INSTR 0xe7ffdefc +#define FASTTRAP_THUMB32_INSTR 0xdefc +#define FASTTRAP_ARM64_INSTR 0xe7eeee7e + +#define FASTTRAP_ARM32_RET_INSTR 0xe7ffdefb +#define FASTTRAP_THUMB32_RET_INSTR 0xdefb +#define FASTTRAP_ARM64_RET_INSTR 0xe7eeee7d +#endif + +#define FASTTRAP_FN_DONE_INIT 255 +#define FASTTRAP_FN_UNKNOWN 0 +#define FASTTRAP_FN_ARM 1 +#define FASTTRAP_FN_THUMB 2 +#define FASTTRAP_FN_USDT 3 + +#define ARM_RM(x) ((x) & 0xF) +#define ARM_RS(x) (((x) >> 8) & 0xF) +#define ARM_RD(x) (((x) >> 12) & 0xF) +#define ARM_RN(x) (((x) >> 16) & 0xF) +#define ARM_CONDCODE(x) ((x) >> 28) + +#define THUMB16_HRM(x) (((x) >> 3) & 0xF) +#define THUMB16_HRD(x) (((x) & 0x7) | ((((x) >> 4) & 0x8))) + +#define THUMB32_RM(x,y) ((y) & 0xF) +#define THUMB32_RD(x,y) (((y) >> 8) & 0xF) +#define THUMB32_RT(x,y) (((y) >> 12) & 0xF) +#define THUMB32_RN(x,y) ((x) & 0xF) + +#define REG_SP 13 +#define REG_LR 14 +#define REG_PC 15 + +#define FASTTRAP_RETURN_AFRAMES 6 +#define FASTTRAP_ENTRY_AFRAMES 5 +#define FASTTRAP_OFFSET_AFRAMES 5 + +#if defined(__arm64__) +#define FASTTRAP_ARM64_OP_VALUE_FUNC_ENTRY 0xa9bf7bfd /* stp fp, lr, [sp, #-16]! */ + +#define FASTTRAP_ARM64_OP_MASK_LDR_S_PC_REL 0xff000000 /* Bits to check for ldr St, label */ +#define FASTTRAP_ARM64_OP_VALUE_LDR_S_PC_REL 0x1c000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_LDR_W_PC_REL 0xff000000 /* Bits to check for ldr Wt, label */ +#define FASTTRAP_ARM64_OP_VALUE_LDR_W_PC_REL 0x18000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_LDR_D_PC_REL 0xff000000 /* Bits to check for ldr Dt, label */ +#define FASTTRAP_ARM64_OP_VALUE_LDR_D_PC_REL 0x5c000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_LDR_X_PC_REL 0xff000000 /* Bits to check for ldr Xt, label */ +#define FASTTRAP_ARM64_OP_VALUE_LDR_X_PC_REL 0x58000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_LDR_Q_PC_REL 0xff000000 /* Bits to check for ldr Qt, label */ +#define FASTTRAP_ARM64_OP_VALUE_LDR_Q_PC_REL 0x9c000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_LRDSW_PC_REL 0xff000000 /* Bits to check for ldrsw <reg>, label */ +#define FASTTRAP_ARM64_OP_VALUE_LRDSW_PC_REL 0x98000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_B_COND_PC_REL 0xff000010 /* Bits to check for b.cond label */ +#define FASTTRAP_ARM64_OP_VALUE_B_COND_PC_REL 0x54000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_CBNZ_W_PC_REL 0xff000000 /* Bits to check for cbnz Wt, _label */ +#define FASTTRAP_ARM64_OP_VALUE_CBNZ_W_PC_REL 0x35000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_CBNZ_X_PC_REL 0xff000000 /* Bits to check for cbnz Xt, _label */ +#define FASTTRAP_ARM64_OP_VALUE_CBNZ_X_PC_REL 0xb5000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_CBZ_W_PC_REL 0xff000000 /* Bits to check for cbz Wt, _label */ +#define FASTTRAP_ARM64_OP_VALUE_CBZ_W_PC_REL 0x34000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_CBZ_X_PC_REL 0xff000000 /* Bits to check for cbz Xt, _label */ +#define FASTTRAP_ARM64_OP_VALUE_CBZ_X_PC_REL 0xb4000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_TBNZ_PC_REL 0x7f000000 /* Bits to check for tbnz Xt, _label */ +#define FASTTRAP_ARM64_OP_VALUE_TBNZ_PC_REL 0x37000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_TBZ_PC_REL 0x7f000000 /* Bits to check for tbz Xt, _label */ +#define FASTTRAP_ARM64_OP_VALUE_TBZ_PC_REL 0x36000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_B_PC_REL 0xfc000000 /* Bits to check for b _label */ +#define FASTTRAP_ARM64_OP_VALUE_B_PC_REL 0x14000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_BL_PC_REL 0xfc000000 /* Bits to check for bl _label */ +#define FASTTRAP_ARM64_OP_VALUE_BL_PC_REL 0x94000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_BLR 0xfffffe1f /* Bits to check for blr Xt */ +#define FASTTRAP_ARM64_OP_VALUE_BLR 0xd63f0000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_BR 0xfffffe1f /* Bits to check for br Xt */ +#define FASTTRAP_ARM64_OP_VALUE_BR 0xd61f0000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_RET 0xfffffc1f /* Bits to check for ret Rt */ +#define FASTTRAP_ARM64_OP_VALUE_RET 0xd65f0000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_ADRP 0x9f000000 /* Bits to check for adrp Xt, label*/ +#define FASTTRAP_ARM64_OP_VALUE_ADRP 0x90000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_ADR 0x9f000000 /* Bits to check for adr Xt, label*/ +#define FASTTRAP_ARM64_OP_VALUE_ADR 0x10000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_PRFM 0xff000000 /* Bits to check for adr Xt, label*/ +#define FASTTRAP_ARM64_OP_VALUE_PRFM 0xd8000000 /* Value to find */ + +#define FASTTRAP_ARM64_OP_MASK_EXCL_MEM 0x3f000000 /* Bits to check for exclusive memory operation */ +#define FASTTRAP_ARM64_OP_VALUE_EXCL_MEM 0x08000000 /* Value to find */ +#endif /* defined(__arm64__) */ + +#ifdef __cplusplus +} +#endif + +#endif /* _FASTTRAP_ISA_H */ diff --git a/bsd/arm/limits.h b/bsd/arm/limits.h new file mode 100644 index 000000000..32c8033b9 --- /dev/null +++ b/bsd/arm/limits.h @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +/* + * Copyright (c) 1988, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)limits.h 8.3 (Berkeley) 1/4/94 + */ + +#ifndef _ARM_LIMITS_H_ +#define _ARM_LIMITS_H_ + +#include <sys/cdefs.h> +#include <arm/_limits.h> + +#define CHAR_BIT 8 /* number of bits in a char */ +#define MB_LEN_MAX 6 /* Allow 31 bit UTF2 */ + +#if !defined(_ANSI_SOURCE) && (!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)) +#define CLK_TCK __DARWIN_CLK_TCK /* ticks per second */ +#endif /* !_ANSI_SOURCE && (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */ + +/* + * According to ANSI (section 2.2.4.2), the values below must be usable by + * #if preprocessing directives. Additionally, the expression must have the + * same type as would an expression that is an object of the corresponding + * type converted according to the integral promotions. The subtraction for + * INT_MIN and LONG_MIN is so the value is not unsigned; 2147483648 is an + * unsigned int for 32-bit two's complement ANSI compilers (section 3.1.3.2). + * These numbers work for pcc as well. The UINT_MAX and ULONG_MAX values + * are written as hex so that GCC will be quiet about large integer constants. + */ +#define SCHAR_MAX 127 /* min value for a signed char */ +#define SCHAR_MIN (-128) /* max value for a signed char */ + +#define UCHAR_MAX 255 /* max value for an unsigned char */ +#define CHAR_MAX 127 /* max value for a char */ +#define CHAR_MIN (-128) /* min value for a char */ + +#define USHRT_MAX 65535 /* max value for an unsigned short */ +#define SHRT_MAX 32767 /* max value for a short */ +#define SHRT_MIN (-32768) /* min value for a short */ + +#define UINT_MAX 0xffffffff /* max value for an unsigned int */ +#define INT_MAX 2147483647 /* max value for an int */ +#define INT_MIN (-2147483647-1) /* min value for an int */ + +#ifdef __LP64__ +#define ULONG_MAX 0xffffffffffffffffUL /* max unsigned long */ +#define LONG_MAX 0x7fffffffffffffffL /* max signed long */ +#define LONG_MIN (-0x7fffffffffffffffL-1) /* min signed long */ +#else /* !__LP64__ */ +#define ULONG_MAX 0xffffffffUL /* max unsigned long */ +#define LONG_MAX 2147483647L /* max signed long */ +#define LONG_MIN (-2147483647L-1) /* min signed long */ +#endif /* __LP64__ */ + +#define ULLONG_MAX 0xffffffffffffffffULL /* max unsigned long long */ +#define LLONG_MAX 0x7fffffffffffffffLL /* max signed long long */ +#define LLONG_MIN (-0x7fffffffffffffffLL-1) /* min signed long long */ + +#if !defined(_ANSI_SOURCE) +#ifdef __LP64__ +#define LONG_BIT 64 +#else /* !__LP64__ */ +#define LONG_BIT 32 +#endif /* __LP64__ */ +#define SSIZE_MAX LONG_MAX /* max value for a ssize_t */ +#define WORD_BIT 32 + +#if (!defined(_POSIX_C_SOURCE) && !defined(_XOPEN_SOURCE)) || defined(_DARWIN_C_SOURCE) +#define SIZE_T_MAX ULONG_MAX /* max value for a size_t */ + +#define UQUAD_MAX ULLONG_MAX +#define QUAD_MAX LLONG_MAX +#define QUAD_MIN LLONG_MIN + +#endif /* (!_POSIX_C_SOURCE && !_XOPEN_SOURCE) || _DARWIN_C_SOURCE */ +#endif /* !_ANSI_SOURCE */ + +#endif /* _ARM_LIMITS_H_ */ diff --git a/bsd/arm/param.h b/bsd/arm/param.h new file mode 100644 index 000000000..538e53418 --- /dev/null +++ b/bsd/arm/param.h @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2000-2010 Apple Inc. All rights reserved. + */ +/*- + * Copyright (c) 1990, 1993 + * The Regents of the University of California. All rights reserved. + * (c) UNIX System Laboratories, Inc. + * All or some portions of this file are derived from material licensed + * to the University of California by American Telephone and Telegraph + * Co. or Unix System Laboratories, Inc. and are reproduced herein with + * the permission of UNIX System Laboratories, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)param.h 8.1 (Berkeley) 4/4/95 + */ + +/* + * Machine dependent constants for ARM + */ + +#ifndef _ARM_PARAM_H_ +#define _ARM_PARAM_H_ + +#include <arm/_param.h> + +/* + * Round p (pointer or byte index) up to a correctly-aligned value for all + * data types (int, long, ...). The result is unsigned int and must be + * cast to any desired pointer type. + */ +#define ALIGNBYTES __DARWIN_ALIGNBYTES +#define ALIGN(p) __DARWIN_ALIGN(p) + +#define NBPG 4096 /* bytes/page */ +#define PGOFSET (NBPG-1) /* byte offset into page */ +#define PGSHIFT 12 /* LOG2(NBPG) */ + +#define DEV_BSIZE 512 +#define DEV_BSHIFT 9 /* log2(DEV_BSIZE) */ +#define BLKDEV_IOSIZE 2048 +#define MAXPHYS (64 * 1024) /* max raw I/O transfer size */ + +#define CLSIZE 1 +#define CLSIZELOG2 0 + +/* + * Constants related to network buffer management. + * MCLBYTES must be no larger than CLBYTES (the software page size), and, + * on machines that exchange pages of input or output buffers with mbuf + * clusters (MAPPED_MBUFS), MCLBYTES must also be an integral multiple + * of the hardware page size. + */ +#define MSIZESHIFT 8 /* 256 */ +#define MSIZE (1 << MSIZESHIFT) /* size of an mbuf */ +#define MCLSHIFT 11 /* 2048 */ +#define MCLBYTES (1 << MCLSHIFT) /* size of an mbuf cluster */ +#define MBIGCLSHIFT 12 /* 4096 */ +#define MBIGCLBYTES (1 << MBIGCLSHIFT) /* size of a big cluster */ +#define M16KCLSHIFT 14 /* 16384 */ +#define M16KCLBYTES (1 << M16KCLSHIFT) /* size of a jumbo cluster */ + +#define MCLOFSET (MCLBYTES - 1) +#ifndef NMBCLUSTERS +#define NMBCLUSTERS CONFIG_NMBCLUSTERS /* cl map size */ +#endif + +/* + * Some macros for units conversion + */ +/* Core clicks (NeXT_page_size bytes) to segments and vice versa */ +#define ctos(x) (x) +#define stoc(x) (x) + +/* Core clicks (4096 bytes) to disk blocks */ +#define ctod(x) ((x)<<(PGSHIFT-DEV_BSHIFT)) +#define dtoc(x) ((x)>>(PGSHIFT-DEV_BSHIFT)) +#define dtob(x) ((x)<<DEV_BSHIFT) + +/* clicks to bytes */ +#define ctob(x) ((x)<<PGSHIFT) + +/* bytes to clicks */ +#define btoc(x) (((unsigned)(x)+(NBPG-1))>>PGSHIFT) + +#ifdef __APPLE__ +#define btodb(bytes, devBlockSize) \ + ((unsigned)(bytes) / devBlockSize) +#define dbtob(db, devBlockSize) \ + ((unsigned)(db) * devBlockSize) +#else +#define btodb(bytes) /* calculates (bytes / DEV_BSIZE) */ \ + ((unsigned)(bytes) >> DEV_BSHIFT) +#define dbtob(db) /* calculates (db * DEV_BSIZE) */ \ + ((unsigned)(db) << DEV_BSHIFT) +#endif + +/* + * Map a ``block device block'' to a file system block. + * This should be device dependent, and will be if we + * add an entry to cdevsw/bdevsw for that purpose. + * For now though just use DEV_BSIZE. + */ +#define bdbtofsb(bn) ((bn) / (BLKDEV_IOSIZE/DEV_BSIZE)) + +/* + * Macros to decode (and encode) processor status word. + */ +#define STATUS_WORD(rpl, ipl) (((ipl) << 8) | (rpl)) +#define USERMODE(x) (((x) & 3) == 3) +#define BASEPRI(x) (((x) & (255 << 8)) == 0) + + +#if defined(KERNEL) || defined(STANDALONE) +#define DELAY(n) delay(n) + +#else /* defined(KERNEL) || defined(STANDALONE) */ +#define DELAY(n) { int N = (n); while (--N > 0); } +#endif /* defined(KERNEL) || defined(STANDALONE) */ + +#endif /* _ARM_PARAM_H_ */ diff --git a/bsd/arm/profile.h b/bsd/arm/profile.h new file mode 100644 index 000000000..728d3f99b --- /dev/null +++ b/bsd/arm/profile.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +/* + * Copyright (c) 1997, Apple Computer, Inc. All rights reserved. + * + */ + +#ifndef _BSD_ARM_PROFILE_H_ +#define _BSD_ARM_PROFILE_H_ + +#include <sys/appleapiopts.h> + +#ifdef KERNEL +#ifdef __APPLE_API_UNSTABLE + +/* + * Block interrupts during mcount so that those interrupts can also be + * counted (as soon as we get done with the current counting). On the + * arm platfom, can't do splhigh/splx as those are C routines and can + * recursively invoke mcount. + */ +#warning MCOUNT_* not implemented yet. + +#define MCOUNT_INIT +#define MCOUNT_ENTER /* s = splhigh(); */ /* XXX TODO */ +#define MCOUNT_EXIT /* (void) splx(s); */ /* XXX TODO */ + +#endif /* __APPLE_API_UNSTABLE */ +#endif /* KERNEL */ + +#endif /* _BSD_ARM_PROFILE_H_ */ diff --git a/bsd/arm/psl.h b/bsd/arm/psl.h new file mode 100644 index 000000000..313ba2d20 --- /dev/null +++ b/bsd/arm/psl.h @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +/* + * Copyright (c) 1992 NeXT Computer, Inc. + * + */ + +#if KERNEL_PRIVATE + +#ifndef _BSD_ARM_PSL_H_ +#define _BSD_ARM_PSL_H_ + +#endif /* _BSD_ARM_PSL_H_ */ + +#endif /* KERNEL_PRIVATE */ diff --git a/bsd/arm/ptrace.h b/bsd/arm/ptrace.h new file mode 100644 index 000000000..4bac00b86 --- /dev/null +++ b/bsd/arm/ptrace.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +/* + * Copyright (c) 1992, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)ptrace.h 8.1 (Berkeley) 6/11/93 + */ + +/* + * Machine dependent trace commands. + * + * None for the ARM at this time. + */ diff --git a/bsd/arm/reboot.h b/bsd/arm/reboot.h new file mode 100644 index 000000000..5d47728b4 --- /dev/null +++ b/bsd/arm/reboot.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ + +#ifndef _BSD_ARM_REBOOT_H_ +#define _BSD_ARM_REBOOT_H_ + +/* + * Empty file (publicly) + */ + +#include <sys/appleapiopts.h> + +#ifdef BSD_KERNEL_PRIVATE + +/* + * Use most significant 16 bits to avoid collisions with + * machine independent flags. + */ +#define RB_POWERDOWN 0x00010000 /* power down on halt */ +#define RB_NOBOOTRC 0x00020000 /* don't run '/etc/rc.boot' */ +#define RB_DEBUG 0x00040000 /* drop into mini monitor on panic */ +#define RB_EJECT 0x00080000 /* eject disks on halt */ +#define RB_COMMAND 0x00100000 /* new boot command specified */ +#define RB_NOFP 0x00200000 /* don't use floating point */ +#define RB_BOOTNEXT 0x00400000 /* reboot into NeXT */ +#define RB_BOOTDOS 0x00800000 /* reboot into DOS */ +#define RB_PRETTY 0x01000000 /* shutdown with pretty graphics */ + +#endif /* BSD_KERNEL_PRIVATE */ + +#endif /* _BSD_ARM_REBOOT_H_ */ diff --git a/bsd/arm/reg.h b/bsd/arm/reg.h new file mode 100644 index 000000000..bffce0700 --- /dev/null +++ b/bsd/arm/reg.h @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +/* + * Copyright (c) 1992 NeXT Computer, Inc. + * + */ + +#ifdef KERNEL_PRIVATE + +#ifndef _BSD_ARM_REG_H_ +#define _BSD_ARM_REG_H_ + +#endif /* _BSD_ARM_REG_H_ */ + +#endif /* KERNEL_PRIVATE */ diff --git a/bsd/arm/signal.h b/bsd/arm/signal.h new file mode 100644 index 000000000..e6ed0e24c --- /dev/null +++ b/bsd/arm/signal.h @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2000-2009 Apple, Inc. All rights reserved. + */ +/* + * Copyright (c) 1992 NeXT Computer, Inc. + * + */ + +#ifndef _ARM_SIGNAL_ +#define _ARM_SIGNAL_ 1 + +#include <sys/cdefs.h> + +#ifndef _ANSI_SOURCE +typedef int sig_atomic_t; +#endif /* ! _ANSI_SOURCE */ + +#endif /* _ARM_SIGNAL_ */ + diff --git a/bsd/arm/types.h b/bsd/arm/types.h new file mode 100644 index 000000000..18906141c --- /dev/null +++ b/bsd/arm/types.h @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2000-2008 Apple Inc. All rights reserved. + */ +/* + * Copyright 1995 NeXT Computer, Inc. All rights reserved. + */ +/* + * Copyright (c) 1990, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)types.h 8.3 (Berkeley) 1/5/94 + */ + +#ifndef _MACHTYPES_H_ +#define _MACHTYPES_H_ + +#ifndef __ASSEMBLER__ +#include <arm/_types.h> +#include <sys/cdefs.h> +/* + * Basic integral types. Omit the typedef if + * not possible for a machine/compiler combination. + */ +#include <sys/_types/_int8_t.h> +#include <sys/_types/_int16_t.h> +#include <sys/_types/_int32_t.h> +#include <sys/_types/_int64_t.h> + +#include <sys/_types/_u_int8_t.h> +#include <sys/_types/_u_int16_t.h> +#include <sys/_types/_u_int32_t.h> +#include <sys/_types/_u_int64_t.h> + +#if __LP64__ +typedef int64_t register_t; +#else +typedef int32_t register_t; +#endif + +#include <sys/_types/_intptr_t.h> +#include <sys/_types/_uintptr_t.h> + +#if !defined(_ANSI_SOURCE) && (!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)) +/* These types are used for reserving the largest possible size. */ +#ifdef __arm64__ +typedef u_int64_t user_addr_t; +typedef u_int64_t user_size_t; +typedef int64_t user_ssize_t; +typedef int64_t user_long_t; +typedef u_int64_t user_ulong_t; +typedef int64_t user_time_t; +typedef int64_t user_off_t; +#else +typedef u_int32_t user_addr_t; +typedef u_int32_t user_size_t; +typedef int32_t user_ssize_t; +typedef int32_t user_long_t; +typedef u_int32_t user_ulong_t; +typedef int32_t user_time_t; +typedef int64_t user_off_t; +#endif + +#define USER_ADDR_NULL ((user_addr_t) 0) +#define CAST_USER_ADDR_T(a_ptr) ((user_addr_t)((uintptr_t)(a_ptr))) + +#ifdef KERNEL + +/* + * These types are used when you know the word size of the target + * user process. They can be used to create struct layouts independent + * of the types and alignment requirements of the current running + * kernel. + */ + +/* + * The user64_ types are not used on the ARM platform, but exist + * so that APIs that conditionalize their behavior based on the + * size of an input structure (like many ioctl(2) implementations) + * can differentiate those structures without a duplicate case + * value. + */ + +/* + * The default ABI for the ARM platform aligns fundamental integral + * data types to their natural boundaries, with a maximum alignment + * of 4, even for 8-byte quantites. + */ + +typedef __uint64_t user64_addr_t; +typedef __uint64_t user64_size_t; +typedef __int64_t user64_ssize_t; +typedef __int64_t user64_long_t; +typedef __uint64_t user64_ulong_t; +typedef __int64_t user64_time_t; +typedef __int64_t user64_off_t; + +typedef __uint32_t user32_addr_t; +typedef __uint32_t user32_size_t; +typedef __int32_t user32_ssize_t; +typedef __int32_t user32_long_t; +typedef __uint32_t user32_ulong_t; +typedef __int32_t user32_time_t; +#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4) +typedef __int64_t user32_off_t; +#else +typedef __int64_t user32_off_t __attribute__((aligned(4))); +#endif + +#endif /* KERNEL */ + +#endif /* !_ANSI_SOURCE && (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */ + +/* This defines the size of syscall arguments after copying into the kernel: */ +#if defined(__arm__) +typedef u_int32_t syscall_arg_t; +#elif defined(__arm64__) +typedef u_int64_t syscall_arg_t; +#else +#error Unknown architecture. +#endif + +#endif /* __ASSEMBLER__ */ +#endif /* _MACHTYPES_H_ */ diff --git a/bsd/arm/vmparam.h b/bsd/arm/vmparam.h new file mode 100644 index 000000000..dbee6526d --- /dev/null +++ b/bsd/arm/vmparam.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ + +#ifndef _BSD_ARM_VMPARAM_H_ +#define _BSD_ARM_VMPARAM_H_ 1 + +#include <sys/resource.h> + +#define USRSTACK (0x27E00000) /* ASLR slides stack down by up to 1MB */ +#define USRSTACK64 (0x000000016FE00000ULL) + +/* + * Virtual memory related constants, all in bytes + */ +#ifndef DFLDSIZ +#define DFLDSIZ (RLIM_INFINITY) /* initial data size limit */ +#endif +#ifndef MAXDSIZ +#define MAXDSIZ (RLIM_INFINITY) /* max data size */ +#endif +#ifndef DFLSSIZ +#define DFLSSIZ (1024*1024 - 16*1024) /* initial stack size limit */ +#endif +#ifndef MAXSSIZ +#define MAXSSIZ (1024*1024) /* max stack size */ +#endif +#ifndef DFLCSIZ +#define DFLCSIZ (0) /* initial core size limit */ +#endif +#ifndef MAXCSIZ +#define MAXCSIZ (RLIM_INFINITY) /* max core size */ +#endif /* MAXCSIZ */ + +#endif /* _BSD_ARM_VMPARAM_H_ */ diff --git a/bsd/bsm/audit_kevents.h b/bsd/bsm/audit_kevents.h index fff152e65..cd7142d60 100644 --- a/bsd/bsm/audit_kevents.h +++ b/bsd/bsm/audit_kevents.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2005-2016 Apple Inc. + * Copyright (c) 2005-2017 Apple Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -612,6 +612,8 @@ #define AUE_OPENBYID_RWT 43209 /* Darwin. */ #define AUE_CLONEFILEAT 43210 /* Darwin. */ #define AUE_FCLONEFILEAT 43211 /* Darwin. */ +#define AUE_SETATTRLISTAT 43212 /* Darwin. */ +#define AUE_FMOUNT 43213 /* Darwin. */ #define AUE_SESSION_START 44901 /* Darwin. */ #define AUE_SESSION_UPDATE 44902 /* Darwin. */ @@ -817,5 +819,6 @@ #define AUE_USRCTL AUE_NULL #define AUE_NEXUS AUE_NULL #define AUE_CHANNEL AUE_NULL +#define AUE_NET AUE_NULL #endif /* !_BSM_AUDIT_KEVENTS_H_ */ diff --git a/bsd/bsm/audit_record.h b/bsd/bsm/audit_record.h index f8dced869..2b6ae891a 100644 --- a/bsd/bsm/audit_record.h +++ b/bsd/bsm/audit_record.h @@ -32,6 +32,7 @@ #ifndef _BSM_AUDIT_RECORD_H_ #define _BSM_AUDIT_RECORD_H_ +#include <bsm/audit.h> /* token_t */ #include <sys/time.h> /* struct timeval */ /* diff --git a/bsd/conf/Makefile.arm b/bsd/conf/Makefile.arm new file mode 100644 index 000000000..9141afe61 --- /dev/null +++ b/bsd/conf/Makefile.arm @@ -0,0 +1,13 @@ +###################################################################### +#BEGIN Machine dependent Makefile fragment for arm +###################################################################### + +# Files that currently violate cast alignment checks at build time +fbt_arm.o_CFLAGS_ADD += -Wno-cast-qual + +# Inline assembly doesn't interact well with LTO +fbt_arm.o_CFLAGS_ADD += $(CFLAGS_NOLTO_FLAG) + +###################################################################### +#END Machine dependent Makefile fragment for arm +###################################################################### diff --git a/bsd/conf/Makefile.arm64 b/bsd/conf/Makefile.arm64 new file mode 100644 index 000000000..c22cdd613 --- /dev/null +++ b/bsd/conf/Makefile.arm64 @@ -0,0 +1,10 @@ +###################################################################### +#BEGIN Machine dependent Makefile fragment for arm +###################################################################### + +# Inline assembly doesn't interact well with LTO +fbt_arm.o_CFLAGS_ADD += $(CFLAGS_NOLTO_FLAG) + +###################################################################### +#END Machine dependent Makefile fragment for arm +###################################################################### diff --git a/bsd/conf/Makefile.template b/bsd/conf/Makefile.template index edfd0c767..afe23cf34 100644 --- a/bsd/conf/Makefile.template +++ b/bsd/conf/Makefile.template @@ -41,6 +41,7 @@ include $(MakeInc_def) CFLAGS+= -include meta_features.h -DDRIVER_PRIVATE \ -D_KERNEL_BUILD -DKERNEL_BUILD -DMACH_KERNEL -DBSD_BUILD \ -DBSD_KERNEL_PRIVATE -DLP64_DEBUG=0 +SFLAGS+= -include meta_features.h # # Directories for mig generated files @@ -82,8 +83,6 @@ vm_unix.o_CFLAGS_ADD += -Wshorten-64-to-32 pthread_synch.o_CFLAGS_ADD += -Wno-unused-parameter -Wno-missing-prototypes pthread_support.o_CFLAGS_ADD += -Wno-unused-parameter -Wno-missing-prototypes -ip_icmp.o_CFLFAGS_ADD += -O0 - # Objects that don't want -Wsign-compare OBJS_NO_SIGN_COMPARE = \ radix.o \ @@ -115,6 +114,7 @@ OBJS_NO_SIGN_COMPARE = \ esp_input.o \ esp_output.o \ esp_rijndael.o \ + esp_chachapoly.o \ ipsec.o \ dest6.o \ frag6.o \ @@ -238,6 +238,7 @@ OBJS_NO_PACKED_ADDRESS = \ ip6_forward.o \ ip6_input.o \ ip6_output.o \ + iptap.o \ ipsec.o \ mld6.o \ mptcp_opt.o \ @@ -256,7 +257,6 @@ OBJS_NO_PACKED_ADDRESS = \ udp6_usrreq.o \ udp_usrreq.o -$(foreach file,$(OBJS_NO_PACKED_ADDRESS),$(eval $(call add_perfile_cflags,$(file),-Wno-unknown-warning-option))) $(foreach file,$(OBJS_NO_PACKED_ADDRESS),$(eval $(call add_perfile_cflags,$(file),-Wno-address-of-packed-member))) # @@ -300,6 +300,10 @@ audit_kevents.c: $(SRCROOT)/bsd/kern/syscalls.master $(MAKESYSCALLS) @echo "[$(CMD_MC)] $(ColorH)GENERATING$(Color0) $(ColorLF)$@$(Color0) from $(ColorF)$<$(Color0)"; $(_v)$(MAKESYSCALLS) $< audit > /dev/null +systrace_args.c: $(SRCROOT)/bsd/kern/syscalls.master $(MAKESYSCALLS) + @echo "[$(CMD_MC)] $(ColorH)GENERATING$(Color0) $(ColorLF)$@$(Color0) from $(ColorF)$<$(Color0)"; + $(_v)$(MAKESYSCALLS) $< systrace > /dev/null + do_all: $(COMPONENT).filelist do_build_all:: do_all diff --git a/bsd/conf/files b/bsd/conf/files index 3eaa5a6a3..104c577ef 100644 --- a/bsd/conf/files +++ b/bsd/conf/files @@ -59,6 +59,7 @@ OPTIONS/ipv6send optional ipv6send OPTIONS/ether optional ether OPTIONS/vlan optional vlan OPTIONS/bond optional bond +OPTIONS/if_fake optional if_fake OPTIONS/bpfilter optional bpfilter OPTIONS/multipath optional multipath OPTIONS/mptcp optional mptcp @@ -70,14 +71,6 @@ OPTIONS/gif optional gif OPTIONS/sendfile optional sendfile OPTIONS/pf optional pf OPTIONS/pflog optional pflog pf -OPTIONS/pf_altq optional pf_altq pf -OPTIONS/classq_blue optional classq_blue -OPTIONS/classq_red optional classq_red -OPTIONS/classq_rio optional classq_rio -OPTIONS/pktsched_cbq optional pktsched_cbq -OPTIONS/pktsched_fairq optional pktsched_fairq -OPTIONS/pktsched_hfsc optional pktsched_hfsc -OPTIONS/pktsched_priq optional pktsched_priq OPTIONS/zlib optional zlib @@ -132,6 +125,7 @@ bsd/dev/dtrace/sdt_subr.c optional config_dtrace bsd/dev/dtrace/systrace.c optional config_dtrace bsd/dev/dtrace/profile_prvd.c optional config_dtrace bsd/dev/dtrace/fasttrap.c optional config_dtrace +./systrace_args.c optional config_dtrace bsd/dev/random/randomdev.c standard @@ -143,6 +137,8 @@ bsd/dev/unix_startup.c standard bsd/dev/vn/vn.c optional vndevice bsd/dev/vn/shadow.c optional vndevice +bsd/dev/monotonic.c optional monotonic + bsd/libkern/crc16.c standard bsd/libkern/crc32.c standard bsd/libkern/random.c standard @@ -173,6 +169,7 @@ bsd/vfs/kpi_vfs.c standard bsd/vfs/vfs_fsevents.c standard bsd/vfs/vfs_cprotect.c standard bsd/vfs/doc_tombstone.c standard +bsd/vfs/vfs_disk_conditioner.c standard bsd/miscfs/deadfs/dead_vnops.c standard bsd/miscfs/devfs/devfs_fdesc_support.c optional fdesc @@ -205,6 +202,7 @@ bsd/net/ether_inet6_pr_module.c optional ether inet6 bsd/net/if_loop.c optional loop bsd/net/if_mib.c optional networking bsd/net/if_vlan.c optional vlan +bsd/net/if_fake.c optional if_fake bsd/net/multicast_list.c optional networking bsd/net/if_bond.c optional bond bsd/net/devtimer.c optional bond @@ -234,6 +232,7 @@ bsd/net/pf_if.c optional pf bsd/net/pf_ioctl.c optional pf bsd/net/pf_norm.c optional pf bsd/net/pf_osfp.c optional pf +bsd/net/pf_pbuf.c optional pf bsd/net/pf_ruleset.c optional pf bsd/net/pf_table.c optional pf bsd/net/iptap.c optional networking @@ -243,33 +242,23 @@ bsd/net/flowhash.c optional networking bsd/net/flowadv.c optional networking bsd/net/content_filter.c optional content_filter bsd/net/packet_mangler.c optional packet_mangler +bsd/net/if_llatbl.c optional networking +bsd/net/nwk_wq.c optional networking +bsd/net/skmem_sysctl.c optional skywalk bsd/net/classq/classq.c optional networking -bsd/net/classq/classq_blue.c optional classq_blue -bsd/net/classq/classq_red.c optional classq_red -bsd/net/classq/classq_rio.c optional classq_rio bsd/net/classq/classq_sfb.c optional networking bsd/net/classq/classq_subr.c optional networking bsd/net/classq/classq_util.c optional networking bsd/net/classq/classq_fq_codel.c optional networking bsd/net/pktsched/pktsched.c optional networking -bsd/net/pktsched/pktsched_cbq.c optional pktsched_cbq -bsd/net/pktsched/pktsched_fairq.c optional pktsched_fairq -bsd/net/pktsched/pktsched_hfsc.c optional pktsched_hfsc -bsd/net/pktsched/pktsched_priq.c optional pktsched_priq bsd/net/pktsched/pktsched_qfq.c optional networking -bsd/net/pktsched/pktsched_rmclass.c optional pktsched_cbq bsd/net/pktsched/pktsched_tcq.c optional networking bsd/net/pktsched/pktsched_fq_codel.c optional networking -bsd/net/altq/altq_cbq.c optional pktsched_cbq pf_altq -bsd/net/altq/altq_fairq.c optional pktsched_fairq pf_altq -bsd/net/altq/altq_hfsc.c optional pktsched_hfsc pf_altq -bsd/net/altq/altq_priq.c optional pktsched_priq pf_altq -bsd/net/altq/altq_qfq.c optional pf_altq -bsd/net/altq/altq_subr.c optional pf_altq - +bsd/netinet/cpu_in_cksum_gen.c standard +bsd/netinet/in_cksum.c optional inet bsd/netinet/igmp.c optional inet bsd/netinet/in.c optional inet bsd/netinet/dhcp_options.c optional inet @@ -279,6 +268,7 @@ bsd/netinet/in_pcb.c optional inet bsd/netinet/in_pcblist.c optional inet bsd/netinet/in_proto.c optional inet bsd/netinet/in_rmx.c optional inet +bsd/netinet/in_stat.c optional inet bsd/netinet/in_tclass.c optional inet bsd/netinet/ip_dummynet.c optional dummynet bsd/netinet/ip_icmp.c optional inet @@ -320,6 +310,7 @@ bsd/netinet6/esp_core.c optional ipsec ipsec_esp bsd/netinet6/esp_input.c optional ipsec ipsec_esp bsd/netinet6/esp_output.c optional ipsec ipsec_esp bsd/netinet6/esp_rijndael.c optional ipsec ipsec_esp +bsd/netinet6/esp_chachapoly.c optional ipsec ipsec_esp bsd/netinet6/ipsec.c optional ipsec bsd/netinet6/dest6.c optional inet6 bsd/netinet6/frag6.c optional inet6 @@ -433,7 +424,6 @@ bsd/kern/kern_time.c standard bsd/kern/kern_xxx.c standard bsd/kern/mach_process.c standard bsd/kern/mcache.c optional sockets -bsd/kern/spl.c standard bsd/kern/stackshot.c standard bsd/kern/subr_log.c standard bsd/kern/subr_prf.c standard @@ -483,14 +473,15 @@ bsd/kern/proc_info.c standard bsd/kern/process_policy.c standard bsd/kern/kern_overrides.c standard bsd/kern/socket_info.c optional sockets +bsd/kern/subr_eventhandler.c standard bsd/kern/sys_reason.c standard bsd/vm/vnode_pager.c standard bsd/vm/vm_unix.c standard bsd/vm/dp_backing_file.c standard bsd/vm/vm_compressor_backing_file.c standard - bsd/kern/kern_ecc.c optional config_ecc_logging +bsd/kern/kern_ntptime.c standard bsd/uxkern/ux_exception.c standard diff --git a/bsd/conf/files.arm b/bsd/conf/files.arm new file mode 100644 index 000000000..0b1712ed9 --- /dev/null +++ b/bsd/conf/files.arm @@ -0,0 +1,20 @@ +bsd/dev/arm/conf.c standard +bsd/dev/arm/cons.c standard +bsd/dev/arm/km.c standard +bsd/dev/arm/kern_machdep.c standard +bsd/dev/arm/stubs.c standard +bsd/dev/arm/systemcalls.c standard +bsd/dev/arm/sysctl.c standard +bsd/dev/arm/unix_signal.c standard +bsd/dev/arm/cpu_in_cksum.s standard + +bsd/dev/arm/dtrace_isa.c optional config_dtrace +bsd/dev/arm/dtrace_subr_arm.c optional config_dtrace +bsd/dev/arm/fbt_arm.c optional config_dtrace +bsd/dev/arm/fasttrap_isa.c optional config_dtrace +bsd/dev/arm/disassembler.c optional config_dtrace +bsd/dev/arm/sdt_arm.c optional config_dtrace + +bsd/dev/arm/munge.c standard + +bsd/kern/bsd_stubs.c standard diff --git a/bsd/conf/files.arm64 b/bsd/conf/files.arm64 new file mode 100644 index 000000000..64009971c --- /dev/null +++ b/bsd/conf/files.arm64 @@ -0,0 +1,21 @@ +bsd/dev/arm64/conf.c standard +bsd/dev/arm/cons.c standard +bsd/dev/arm/km.c standard +bsd/dev/arm/kern_machdep.c standard +bsd/dev/arm/stubs.c standard +bsd/dev/arm/systemcalls.c standard +bsd/dev/arm64/sysctl.c standard +bsd/dev/arm/unix_signal.c standard + +bsd/dev/arm64/cpu_in_cksum.s standard + +bsd/dev/arm64/dtrace_isa.c optional config_dtrace +bsd/dev/arm64/dtrace_subr_arm.c optional config_dtrace +bsd/dev/arm64/fbt_arm.c optional config_dtrace +bsd/dev/arm64/fasttrap_isa.c optional config_dtrace +bsd/dev/arm64/disassembler.c optional config_dtrace +bsd/dev/arm64/sdt_arm.c optional config_dtrace + +bsd/dev/pgtrace/pgtrace_dev.c optional config_pgtrace_nonkext + +bsd/kern/bsd_stubs.c standard diff --git a/bsd/conf/files.x86_64 b/bsd/conf/files.x86_64 index 2fba68035..4eb06372f 100644 --- a/bsd/conf/files.x86_64 +++ b/bsd/conf/files.x86_64 @@ -21,5 +21,3 @@ bsd/dev/i386/dis_tables.c optional config_dtrace bsd/kern/policy_check.c optional config_macf bsd/kern/bsd_stubs.c standard -bsd/netinet/cpu_in_cksum.c standard -bsd/netinet/in_cksum.c optional inet diff --git a/bsd/conf/param.c b/bsd/conf/param.c index 00da0c590..d78d06c4a 100644 --- a/bsd/conf/param.c +++ b/bsd/conf/param.c @@ -82,15 +82,24 @@ struct timezone tz = { 0, 0 }; +#if CONFIG_EMBEDDED +#define NPROC 1000 /* Account for TOTAL_CORPSES_ALLOWED by making this slightly lower than we can. */ +#define NPROC_PER_UID 950 +#else #define NPROC (20 + 16 * 32) #define NPROC_PER_UID (NPROC/2) +#endif /* NOTE: maxproc and hard_maxproc values are subject to device specific scaling in bsd_scale_setup */ #define HNPROC 2500 /* based on thread_max */ int maxproc = NPROC; int maxprocperuid = NPROC_PER_UID; +#if CONFIG_EMBEDDED +int hard_maxproc = NPROC; /* hardcoded limit -- for embedded the number of processes is limited by the ASID space */ +#else int hard_maxproc = HNPROC; /* hardcoded limit */ +#endif int nprocs = 0; /* XXX */ diff --git a/bsd/dev/arm/conf.c b/bsd/dev/arm/conf.c new file mode 100644 index 000000000..ef78baad1 --- /dev/null +++ b/bsd/dev/arm/conf.c @@ -0,0 +1,306 @@ +/* + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. + */ +/* + * Copyright (c) 1997 by Apple Computer, Inc., all rights reserved + * Copyright (c) 1993 NeXT Computer, Inc. + * + * UNIX Device switch tables. + * + * HISTORY + * + * 30 July 1997 Umesh Vaishampayan (umeshv@apple.com) + * enabled file descriptor pseudo-device. + * 18 June 1993 ? at NeXT + * Cleaned up a lot of stuff in this file. + */ + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/ioctl.h> +#include <sys/tty.h> +#include <sys/conf.h> + +/* Prototypes that should be elsewhere: */ +extern dev_t chrtoblk(dev_t dev); +extern int chrtoblk_set(int cdev, int bdev); + +struct bdevsw bdevsw[] = +{ + /* + * For block devices, every other block of 8 slots is + * reserved to NeXT. The other slots are available for + * the user. This way we can both add new entries without + * running into each other. Be sure to fill in NeXT's + * 8 reserved slots when you jump over us -- we'll do the + * same for you. + */ + + /* 0 - 7 are reserved to NeXT */ + + NO_BDEVICE, /* 0 */ + NO_BDEVICE, /* 1 */ + NO_BDEVICE, /* 2 */ + NO_BDEVICE, /* 3 */ + NO_BDEVICE, /* 4 */ + NO_BDEVICE, /* 5 */ + NO_BDEVICE, /* 6 */ + NO_BDEVICE, /* 7 */ + + /* 8 - 15 are reserved to the user */ + NO_BDEVICE, /* 8 */ + NO_BDEVICE, /* 9 */ + NO_BDEVICE, /* 10 */ + NO_BDEVICE, /* 11 */ + NO_BDEVICE, /* 12 */ + NO_BDEVICE, /* 13 */ + NO_BDEVICE, /* 14 */ + NO_BDEVICE, /* 15 */ + + /* 16 - 23 are reserved to NeXT */ + NO_BDEVICE, /* 16 */ + NO_BDEVICE, /* 17 */ + NO_BDEVICE, /* 18 */ + NO_BDEVICE, /* 18 */ + NO_BDEVICE, /* 20 */ + NO_BDEVICE, /* 21 */ + NO_BDEVICE, /* 22 */ + NO_BDEVICE, /* 23 */ +}; + +const int nblkdev = sizeof(bdevsw) / sizeof(bdevsw[0]); + +extern struct tty *km_tty[]; +extern d_open_t cnopen; +extern d_close_t cnclose; +extern d_read_t cnread; +extern d_write_t cnwrite; +extern d_ioctl_t cnioctl; +extern d_select_t cnselect; +extern d_open_t kmopen; +extern d_close_t kmclose; +extern d_read_t kmread; +extern d_write_t kmwrite; +extern d_ioctl_t kmioctl; +extern d_open_t sgopen; +extern d_close_t sgclose; +extern d_ioctl_t sgioctl; + +#if NVOL > 0 +extern d_open_t volopen; +extern d_close_t volclose; +extern d_ioctl_t volioctl; +#else +#define volopen eno_opcl +#define volclose eno_opcl +#define volioctl eno_ioctl +#endif + +extern d_open_t cttyopen; +extern d_read_t cttyread; +extern d_write_t cttywrite; +extern d_ioctl_t cttyioctl; +extern d_select_t cttyselect; + +extern d_read_t mmread; +extern d_write_t mmwrite; +extern d_ioctl_t mmioctl; +#define mmselect (select_fcn_t *)seltrue +#define mmmmap eno_mmap + +#include <pty.h> +#if NPTY > 0 +extern d_open_t ptsopen; +extern d_close_t ptsclose; +extern d_read_t ptsread; +extern d_write_t ptswrite; +extern d_stop_t ptsstop; +extern d_select_t ptsselect; +extern d_open_t ptcopen; +extern d_close_t ptcclose; +extern d_read_t ptcread; +extern d_write_t ptcwrite; +extern d_select_t ptcselect; +extern d_ioctl_t ptyioctl; +#else +#define ptsopen eno_opcl +#define ptsclose eno_opcl +#define ptsread eno_rdwrt +#define ptswrite eno_rdwrt +#define ptsstop nulldev + +#define ptcopen eno_opcl +#define ptcclose eno_opcl +#define ptcread eno_rdwrt +#define ptcwrite eno_rdwrt +#define ptcselect eno_select +#define ptyioctl eno_ioctl +#endif + +extern d_open_t logopen; +extern d_close_t logclose; +extern d_read_t logread; +extern d_ioctl_t logioctl; +extern d_select_t logselect; + +extern d_open_t oslog_streamopen; +extern d_close_t oslog_streamclose; +extern d_read_t oslog_streamread; +extern d_ioctl_t oslog_streamioctl; +extern d_select_t oslog_streamselect; + +extern d_open_t oslogopen; +extern d_close_t oslogclose; +extern d_select_t oslogselect; +extern d_ioctl_t oslogioctl; + +#define nullopen (d_open_t *)&nulldev +#define nullclose (d_close_t *)&nulldev +#define nullread (d_read_t *)&nulldev +#define nullwrite (d_write_t *)&nulldev +#define nullioctl (d_ioctl_t *)&nulldev +#define nullselect (d_select_t *)&nulldev +#define nullstop (d_stop_t *)&nulldev +#define nullreset (d_reset_t *)&nulldev + +struct cdevsw cdevsw[] = { + /* + * To add character devices to this table dynamically, use cdevsw_add. + */ + + [0] = { + cnopen, cnclose, cnread, cnwrite, + cnioctl, nullstop, nullreset, 0, cnselect, + eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY + }, + [1] = NO_CDEVICE, + [2] = { + cttyopen, nullclose, cttyread, cttywrite, + cttyioctl, nullstop, nullreset, 0, cttyselect, + eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY + }, + [3] = { + nullopen, nullclose, mmread, mmwrite, + mmioctl, nullstop, nullreset, 0, mmselect, + mmmmap, eno_strat, eno_getc, eno_putc, D_DISK + }, + [PTC_MAJOR] = { + ptsopen, ptsclose, ptsread, ptswrite, + ptyioctl, ptsstop, nullreset, 0, ptsselect, + eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY + }, + [PTS_MAJOR] = { + ptcopen, ptcclose, ptcread, ptcwrite, + ptyioctl, nullstop, nullreset, 0, ptcselect, + eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY + }, + [6] = { + logopen, logclose, logread, eno_rdwrt, + logioctl, eno_stop, nullreset, 0, logselect, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + }, + [7] = { + oslogopen, oslogclose, eno_rdwrt, eno_rdwrt, + oslogioctl, eno_stop, nullreset, 0, oslogselect, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + }, + [8] = { + oslog_streamopen, oslog_streamclose, oslog_streamread, eno_rdwrt, + oslog_streamioctl, eno_stop, nullreset, 0, oslog_streamselect, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + }, + [9 ... 11] = NO_CDEVICE, + [12] = { + kmopen, kmclose, kmread, kmwrite, + kmioctl, nullstop, nullreset, km_tty, ttselect, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + }, + [13 ... 41] = NO_CDEVICE, + [42] = { + volopen, volclose, eno_rdwrt, eno_rdwrt, + volioctl, eno_stop, eno_reset, 0, (select_fcn_t *) seltrue, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + } +}; +const int nchrdev = sizeof(cdevsw) / sizeof(cdevsw[0]); + +uint64_t cdevsw_flags[sizeof(cdevsw) / sizeof(cdevsw[0])]; + +#include <sys/vnode.h> /* for VCHR and VBLK */ +/* + * return true if a disk + */ +int +isdisk(dev_t dev, int type) +{ + dev_t maj = major(dev); + + switch (type) { + case VCHR: + maj = chrtoblk(maj); + if (maj == NODEV) { + break; + } + /* FALL THROUGH */ + case VBLK: + if (bdevsw[maj].d_type == D_DISK) { + return (1); + } + break; + } + return (0); +} + +static int chrtoblktab[] = { + /* CHR *//* BLK *//* CHR *//* BLK */ + /* 0 */ NODEV, /* 1 */ NODEV, + /* 2 */ NODEV, /* 3 */ NODEV, + /* 4 */ NODEV, /* 5 */ NODEV, + /* 6 */ NODEV, /* 7 */ NODEV, + /* 8 */ NODEV, /* 9 */ NODEV, + /* 10 */ NODEV, /* 11 */ NODEV, + /* 12 */ NODEV, /* 13 */ NODEV, + /* 14 */ NODEV, /* 15 */ NODEV, + /* 16 */ NODEV, /* 17 */ NODEV, + /* 18 */ NODEV, /* 19 */ NODEV, + /* 20 */ NODEV, /* 21 */ NODEV, + /* 22 */ NODEV, /* 23 */ NODEV, + /* 24 */ NODEV, /* 25 */ NODEV, + /* 26 */ NODEV, /* 27 */ NODEV, + /* 28 */ NODEV, /* 29 */ NODEV, + /* 30 */ NODEV, /* 31 */ NODEV, + /* 32 */ NODEV, /* 33 */ NODEV, + /* 34 */ NODEV, /* 35 */ NODEV, + /* 36 */ NODEV, /* 37 */ NODEV, + /* 38 */ NODEV, /* 39 */ NODEV, + /* 40 */ NODEV, /* 41 */ NODEV, + /* 42 */ NODEV, /* 43 */ NODEV, + /* 44 */ NODEV, +}; + +/* + * convert chr dev to blk dev + */ +dev_t +chrtoblk(dev_t dev) +{ + int blkmaj; + + if (major(dev) >= nchrdev) + return (NODEV); + blkmaj = chrtoblktab[major(dev)]; + if (blkmaj == NODEV) + return (NODEV); + return (makedev(blkmaj, minor(dev))); +} + +int +chrtoblk_set(int cdev, int bdev) +{ + if (cdev >= nchrdev) + return (-1); + if (bdev != NODEV && bdev >= nblkdev) + return (-1); + chrtoblktab[cdev] = bdev; + return 0; +} diff --git a/bsd/dev/arm/cons.c b/bsd/dev/arm/cons.c new file mode 100644 index 000000000..910c15257 --- /dev/null +++ b/bsd/dev/arm/cons.c @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +/* + * Copyright (c) 1987, 1988 NeXT, Inc. + * + * HISTORY 7-Jan-93 Mac Gillon (mgillon) at NeXT Integrated POSIX support + * + * 12-Aug-87 John Seamons (jks) at NeXT Ported to NeXT. + */ + +/* + * Indirect driver for console. + */ +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/conf.h> +#include <sys/ioctl.h> +#include <sys/tty.h> +#include <sys/proc.h> +#include <sys/uio.h> + +struct tty *constty; /* current console device */ + +/* + * The km driver supplied the default console device for the systems + * (usually a raw frame buffer driver, but potentially a serial driver). + */ +extern struct tty *km_tty[1]; + +/* + * cdevsw[] entries for the console device driver + */ +int cnopen(__unused dev_t dev, int flag, int devtype, proc_t pp); +int cnclose(__unused dev_t dev, int flag, int mode, proc_t pp); +int cnread(__unused dev_t dev, struct uio *uio, int ioflag); +int cnwrite(__unused dev_t dev, struct uio *uio, int ioflag); +int cnioctl(__unused dev_t dev, u_long cmd, caddr_t addr, int flg, proc_t p); +int cnselect(__unused dev_t dev, int flag, void * wql, proc_t p); + +static dev_t +cndev(void) +{ + if (constty) + return constty->t_dev; + else + return km_tty[0]->t_dev; +} + +int +cnopen(__unused dev_t dev, int flag, int devtype, struct proc *pp) +{ + dev = cndev(); + return ((*cdevsw[major(dev)].d_open)(dev, flag, devtype, pp)); +} + + +int +cnclose(__unused dev_t dev, int flag, int mode, struct proc *pp) +{ + dev = cndev(); + return ((*cdevsw[major(dev)].d_close)(dev, flag, mode, pp)); +} + + +int +cnread(__unused dev_t dev, struct uio *uio, int ioflag) +{ + dev = cndev(); + return ((*cdevsw[major(dev)].d_read)(dev, uio, ioflag)); +} + + +int +cnwrite(__unused dev_t dev, struct uio *uio, int ioflag) +{ + dev = cndev(); + return ((*cdevsw[major(dev)].d_write)(dev, uio, ioflag)); +} + + +int +cnioctl(__unused dev_t dev, u_long cmd, caddr_t addr, int flag, struct proc *p) +{ + dev = cndev(); + + /* + * XXX This check prevents the cons.c code from being shared between + * XXX all architectures; it is probably not needed on ARM, either, + * XXX but I have no test platforms or ability to run a kernel. + * + * Superuser can always use this to wrest control of console + * output from the "virtual" console. + */ + if ((unsigned) cmd == TIOCCONS && constty) { + int error = proc_suser(p); + if (error) + return (error); + constty = NULL; + return (0); + } + return ((*cdevsw[major(dev)].d_ioctl)(dev, cmd, addr, flag, p)); +} + + +int +cnselect(__unused dev_t dev, int flag, void *wql, struct proc *p) +{ + dev = cndev(); + return ((*cdevsw[major(dev)].d_select)(dev, flag, wql, p)); +} diff --git a/bsd/dev/arm/cpu_in_cksum.s b/bsd/dev/arm/cpu_in_cksum.s new file mode 100644 index 000000000..28f648183 --- /dev/null +++ b/bsd/dev/arm/cpu_in_cksum.s @@ -0,0 +1,444 @@ +/* + * Copyright (c) 2009-2017 Apple Inc. All rights reserved. + * + * This document is the property of Apple Inc. + * It is considered confidential and proprietary. + * + * This document may not be reproduced or transmitted in any form, + * in whole or in part, without the express written permission of + * Apple Inc. + */ + +/* $NetBSD: cpu_in_cksum.S,v 1.2 2008/01/27 16:58:05 chris Exp $ */ + +/* + * Copyright 2003 Wasabi Systems, Inc. + * All rights reserved. + * + * Written by Steve C. Woodford for Wasabi Systems, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed for the NetBSD Project by + * Wasabi Systems, Inc. + * 4. The name of Wasabi Systems, Inc. may not be used to endorse + * or promote products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifdef KERNEL +#include "../../../osfmk/arm/arch.h" +#include "../../../osfmk/arm/proc_reg.h" + +#if __ARM_VFP__ < 3 +#error "Unsupported: __ARM_VFP__ < 3" +#endif /* __ARM_VFP__ < 3 */ +#define CKSUM_ERR _kprintf +#else /* !KERNEL */ +#ifndef LIBSYSCALL_INTERFACE +#error "LIBSYSCALL_INTERFACE not defined" +#endif /* !LIBSYSCALL_INTERFACE */ +#define CKSUM_ERR _fprintf_stderr +#define __ARM_VFP__ 3 +#endif /* !KERNEL */ + +/* + * The following default the implementation to little-endian architectures. + */ +#define LITTLE_ENDIAN 1 +#define BYTE_ORDER LITTLE_ENDIAN + +.syntax unified + +/* + * XXX: adi@apple.com: + * + * Ugly, but we have little choice, since relying on genassym and <assym.s> + * is not possible unless this code lives in osfmk. Note also that this + * routine expects "mbuf-like" argument, and it does not expect the mbuf to be + * authentic; it only cares about 3 fields. + */ +#define M_NEXT 0 +#define M_DATA 8 +#define M_LEN 12 + +/* + * APPLE MODIFICATION + * + * The use of R7 in this code as data register prevents + * the use of debugging or instrumentation tools, which is an acceptable + * tradeoff considering the potential gain in performance. + */ + +/* + * Hand-optimised implementations for ARM/Xscale + */ + + .macro EnableVFP +#ifdef KERNEL + push {r0, r1, r2, r12} + bl _enable_kernel_vfp_context + pop {r0, r1, r2, r12} +#endif /* KERNEL */ + .endm + + +/* + * uint32_t os_cpu_in_cksum_mbuf(struct mbuf *m, int len, int off, + * uint32_t initial_sum); + * + * Entry: + * r0 m + * r1 len + * r2 off + * r3 initial_sum + * + * Function wide register usage + * r8 accumulated sum + * r9 remaining length to parse + * ip pointer to next mbuf + * + * This function returns the partial 16-bit checksum accumulated in + * a 32-bit variable (withouth 1's complement); caller is responsible + * for folding the 32-bit sum into 16-bit and performinng the 1's + * complement if applicable + */ + .globl _os_cpu_in_cksum_mbuf + .text + .align 4 +_os_cpu_in_cksum_mbuf: + stmfd sp!, {r4-r11,lr} + + mov r8, r3 /* Accumulate sum in r8 */ + mov r9, r1 /* save len in r9 */ + mov ip, r0 /* set ip to the current mbuf */ + + cmp r9, #0 /* length is 0? */ + bne .Lin_cksum_skip_loop /* if not, proceed further */ + mov r0, r8 /* otherwise, return initial sum */ + + ldmfd sp!, {r4-r11, pc} + +.Lin_cksum_skip_loop: + ldr r1, [ip, #(M_LEN)] + ldr r0, [ip, #(M_DATA)] + ldr ip, [ip, #(M_NEXT)] +.Lin_cksum_skip_entry: + subs r2, r2, r1 /* offset = offset - mbuf length */ + blt .Lin_cksum_skip_done /* if offset has gone negative start with this mbuf */ + cmp ip, #0x00 + bne .Lin_cksum_skip_loop + b .Lin_cksum_whoops + +.Lin_cksum_skip_done: + add r0, r2, r0 /* data += offset (offset is < 0) */ + add r0, r0, r1 /* data += length of mbuf */ + /* data == start of data to cksum */ + rsb r1, r2, #0x00 /* length = remainder of mbuf to read */ + mov r10, #0x00 + b .Lin_cksum_entry + +.Lin_cksum_loop: + ldr r1, [ip, #(M_LEN)] + ldr r0, [ip, #(M_DATA)] + ldr ip, [ip, #(M_NEXT)] +.Lin_cksum_entry: + cmp r9, r1 + movlt r1, r9 + sub r9, r9, r1 + eor r11, r10, r0 + add r10, r10, r1 + adds r2, r1, #0x00 + + beq .Lin_cksum_next + +/* + * APPLE MODIFICATION + * + * Replace the 'blne _ASM_LABEL(L_cksumdata)' by bringing the called function + * inline. This results in slightly faster code, and also permits the whole + * function to be included in kernel profiling data. + */ + +/* + * The main in*_cksum() workhorse... + * + * Entry parameters: + * r0 Pointer to buffer + * r1 Buffer length + * lr Return address + * + * Returns: + * r2 Accumulated 32-bit sum + * + * Clobbers: + * r0-r7 + */ + mov r2, #0 + + /* We first have to word-align the buffer. */ + ands r7, r0, #0x03 + beq .Lcksumdata_wordaligned + rsb r7, r7, #0x04 + cmp r1, r7 /* Enough bytes left to make it? */ + blt .Lcksumdata_endgame + cmp r7, #0x02 + ldrb r4, [r0], #0x01 /* Fetch 1st byte */ + ldrbge r5, [r0], #0x01 /* Fetch 2nd byte */ + movlt r5, #0x00 + ldrbgt r6, [r0], #0x01 /* Fetch 3rd byte */ + movle r6, #0x00 + /* Combine the three bytes depending on endianness and alignment */ +#if BYTE_ORDER != LITTLE_ENDIAN + orreq r2, r5, r4, lsl #8 + orreq r2, r2, r6, lsl #24 + orrne r2, r4, r5, lsl #8 + orrne r2, r2, r6, lsl #16 +#else + orreq r2, r4, r5, lsl #8 + orreq r2, r2, r6, lsl #16 + orrne r2, r5, r4, lsl #8 + orrne r2, r2, r6, lsl #24 +#endif + subs r1, r1, r7 /* Update length */ + beq .Lin_cksum_next /* All done? */ + + /* Buffer is now word aligned */ +.Lcksumdata_wordaligned: + +#if __ARM_VFP__ >= 3 + + cmp r1, #512 // do this if r1 is at least 512 + blt 9f + + EnableVFP + + and r3, r1, #~0x3f + + vpush {q0-q7} + + // move r2 to s16 (q4) for neon computation + veor q4, q4, q4 + vld1.32 {q0-q1}, [r0]! + vmov s16, r2 + vld1.32 {q2-q3}, [r0]! + + // pre-decrement size by 64 + subs r3, r3, #0x80 + + vpadal.u32 q4, q0 + vld1.32 {q0}, [r0]! + vpaddl.u32 q5, q1 + vld1.32 {q1}, [r0]! + vpaddl.u32 q6, q2 + vld1.32 {q2}, [r0]! + vpaddl.u32 q7, q3 + vld1.32 {q3}, [r0]! + +0: + subs r3, r3, #0x40 // decrement size by 64 + + vpadal.u32 q4, q0 + vld1.32 {q0}, [r0]! + vpadal.u32 q5, q1 + vld1.32 {q1}, [r0]! + vpadal.u32 q6, q2 + vld1.32 {q2}, [r0]! + vpadal.u32 q7, q3 + vld1.32 {q3}, [r0]! + + bgt 0b + + vpadal.u32 q4, q0 + vpadal.u32 q5, q1 + vpadal.u32 q6, q2 + vpadal.u32 q7, q3 + + vpadal.u32 q4, q5 + vpadal.u32 q6, q7 + vpadal.u32 q4, q6 + vadd.i64 d8, d9 + + vpaddl.u32 d8, d8 + vpaddl.u32 d8, d8 + vpaddl.u32 d8, d8 + + vmov r2, s16 + + vpop {q0-q7} + + ands r1, r1, #0x3f // residual bytes + beq .Lin_cksum_next + +9: + +#endif /* __ARM_VFP__ >= 3 */ + + subs r1, r1, #0x40 + blt .Lcksumdata_bigloop_end + +.Lcksumdata_bigloop: + ldmia r0!, {r3, r4, r5, r6} + adds r2, r2, r3 + adcs r2, r2, r4 + adcs r2, r2, r5 + ldmia r0!, {r3, r4, r5, r7} + adcs r2, r2, r6 + adcs r2, r2, r3 + adcs r2, r2, r4 + adcs r2, r2, r5 + ldmia r0!, {r3, r4, r5, r6} + adcs r2, r2, r7 + adcs r2, r2, r3 + adcs r2, r2, r4 + adcs r2, r2, r5 + ldmia r0!, {r3, r4, r5, r7} + adcs r2, r2, r6 + adcs r2, r2, r3 + adcs r2, r2, r4 + adcs r2, r2, r5 + adcs r2, r2, r7 + adc r2, r2, #0x00 + subs r1, r1, #0x40 + bge .Lcksumdata_bigloop +.Lcksumdata_bigloop_end: + + adds r1, r1, #0x40 + beq .Lin_cksum_next + + cmp r1, #0x20 + + blt .Lcksumdata_less_than_32 + ldmia r0!, {r3, r4, r5, r6} + adds r2, r2, r3 + adcs r2, r2, r4 + adcs r2, r2, r5 + ldmia r0!, {r3, r4, r5, r7} + adcs r2, r2, r6 + adcs r2, r2, r3 + adcs r2, r2, r4 + adcs r2, r2, r5 + adcs r2, r2, r7 + adc r2, r2, #0x00 + subs r1, r1, #0x20 + beq .Lin_cksum_next + +.Lcksumdata_less_than_32: + /* There are less than 32 bytes left */ + and r3, r1, #0x18 + rsb r4, r3, #0x18 + sub r1, r1, r3 + adds r4, r4, r4, lsr #1 /* Side effect: Clear carry flag */ + addne pc, pc, r4 + +/* + * Note: We use ldm here, even on Xscale, since the combined issue/result + * latencies for ldm and ldrd are the same. Using ldm avoids needless #ifdefs. + */ + /* At least 24 bytes remaining... */ + ldmia r0!, {r4, r5} + nop + adcs r2, r2, r4 + adcs r2, r2, r5 + + /* At least 16 bytes remaining... */ + ldmia r0!, {r4, r5} + adcs r2, r2, r4 + adcs r2, r2, r5 + + /* At least 8 bytes remaining... */ + ldmia r0!, {r4, r5} + adcs r2, r2, r4 + adcs r2, r2, r5 + + /* Less than 8 bytes remaining... */ + adc r2, r2, #0x00 + subs r1, r1, #0x04 + blt .Lcksumdata_lessthan4 + + ldr r4, [r0], #0x04 + sub r1, r1, #0x04 + adds r2, r2, r4 + adc r2, r2, #0x00 + + /* Deal with < 4 bytes remaining */ +.Lcksumdata_lessthan4: + adds r1, r1, #0x04 + beq .Lin_cksum_next + + /* Deal with 1 to 3 remaining bytes, possibly misaligned */ +.Lcksumdata_endgame: + ldrb r3, [r0] /* Fetch first byte */ + cmp r1, #0x02 + ldrbge r4, [r0, #0x01] /* Fetch 2nd and 3rd as necessary */ + movlt r4, #0x00 + ldrbgt r5, [r0, #0x02] + movle r5, #0x00 + /* Combine the three bytes depending on endianness and alignment */ + tst r0, #0x01 +#if BYTE_ORDER != LITTLE_ENDIAN + orreq r3, r4, r3, lsl #8 + orreq r3, r3, r5, lsl #24 + orrne r3, r3, r4, lsl #8 + orrne r3, r3, r5, lsl #16 +#else + orreq r3, r3, r4, lsl #8 + orreq r3, r3, r5, lsl #16 + orrne r3, r4, r3, lsl #8 + orrne r3, r3, r5, lsl #24 +#endif + adds r2, r2, r3 + adc r2, r2, #0x00 + +.Lin_cksum_next: + tst r11, #0x01 + movne r2, r2, ror #8 + adds r8, r8, r2 + adc r8, r8, #0x00 + cmp ip, #00 + bne .Lin_cksum_loop + + mov r1, #0xff + orr r1, r1, #0xff00 + and r0, r8, r1 + add r0, r0, r8, lsr #16 + add r0, r0, r0, lsr #16 + and r0, r0, r1 + /* + * If we were to 1's complement it (XOR with 0xffff): + * + * eor r0, r0, r1 + */ + + ldmfd sp!, {r4-r11, pc} + +.Lin_cksum_whoops: + adr r0, .Lin_cksum_whoops_str + bl #CKSUM_ERR + mov r0, #-1 + + ldmfd sp!, {r4-r11, pc} + +.Lin_cksum_whoops_str: + .asciz "os_cpu_in_cksum_mbuf: out of data\n" + .align 5 diff --git a/bsd/dev/arm/disassembler.c b/bsd/dev/arm/disassembler.c new file mode 100644 index 000000000..a5db2033e --- /dev/null +++ b/bsd/dev/arm/disassembler.c @@ -0,0 +1,1097 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * Disassemblers for ARM (arm), Thumb (thumb16), and Thumb2 (thumb32). + * + * Each disassembly begins with a call to dtrace_decode_arm or dtrace_decode_thumb. The thumb + * decoder will then call dtrace_decode_thumb16 or dtrace_decode_thumb32 as appropriate. + * + * The respective disassembly functions are all of the form {arm,thumb16,thumb32}_type. They + * follow the ordering and breakdown in the ARMv7 Architecture Reference Manual. + */ + +#include <sys/fasttrap_isa.h> + +#define BITS(x,n,mask) (((x) >> (n)) & (mask)) + +static uint32_t thumb32_instword_to_arm(uint16_t hw1, uint16_t hw2) +{ + return (hw1 << 16) | hw2; +} + +int dtrace_decode_arm(uint32_t instr); +int dtrace_decode_thumb(uint32_t instr); + +/* + * VFP decoder - shared between ARM and THUMB32 mode + */ + +static +int vfp_struct_loadstore(uint32_t instr) +{ + if (ARM_RM(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int vfp_64transfer(uint32_t instr) +{ + /* These instructions all use RD and RN */ + if (ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int vfp_transfer(uint32_t instr) +{ + /* These instructions all use RD only */ + if (ARM_RD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int vfp_loadstore(uint32_t instr) +{ + int opcode = BITS(instr,20,0x1F); + + /* Instrument VLDR */ + if ((opcode & 0x13) == 0x11 && ARM_RN(instr) == REG_PC) + return FASTTRAP_T_VLDR_PC_IMMED; + + /* These instructions all use RN only */ + if (ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +/* + * ARM decoder + */ + +static +int arm_unconditional_misc(uint32_t instr) +{ + int op = BITS(instr,20,0x7F); + + if ((op & 0x60) == 0x20) { + /* VFP data processing uses its own registers */ + return FASTTRAP_T_COMMON; + } + + if ((op & 0x71) == 0x40) { + return vfp_struct_loadstore(instr); + } + + return FASTTRAP_T_INV; +} + +static +int arm_unconditional(uint32_t instr) +{ + if (BITS(instr,27,0x1) == 0) + return arm_unconditional_misc(instr); + + /* The rest are privileged or BL/BLX, do not instrument */ + + /* Do not need to instrument BL/BLX either, see comment in arm_misc(uint32_t) */ + + return FASTTRAP_T_INV; +} + +static +int arm_syscall_coproc(uint32_t instr) +{ + /* Instrument any VFP data processing instructions, ignore the rest */ + + int op1 = BITS(instr,20,0x3F), coproc = BITS(instr,8,0xF), op = BITS(instr,4,0x1); + + if ((op1 & 0x3E) == 0 || (op1 & 0x30) == 0x30) { + /* Undefined or swi */ + return FASTTRAP_T_INV; + } + + if ((coproc & 0xE) == 0xA) { + /* VFP instruction */ + + if ((op1 & 0x20) == 0 && (op1 & 0x3A) != 0) + return vfp_loadstore(instr); + + if ((op1 & 0x3E) == 0x04) + return vfp_64transfer(instr); + + if ((op1 & 0x30) == 0x20) { + /* VFP data processing or 8, 16, or 32 bit move between ARM reg and VFP reg */ + if (op == 0) { + /* VFP data processing uses its own registers */ + return FASTTRAP_T_COMMON; + } else { + return vfp_transfer(instr); + } + } + } + + return FASTTRAP_T_INV; +} + +static +int arm_branch_link_blockdata(uint32_t instr) +{ + int branch = BITS(instr,25,0x1), link = BITS(instr,24,0x1), op = BITS(instr,20,0x1F), uses_pc = BITS(instr,15,0x1), uses_lr = BITS(instr,14,0x1); + + if (branch == 1) { + if (link == 0) + return FASTTRAP_T_B_COND; + return FASTTRAP_T_INV; + } else { + /* Only emulate a use of the pc if it's a return from function: ldmia sp!, { ... pc } */ + if (op == 0x0B && ARM_RN(instr) == REG_SP && uses_pc == 1) + return FASTTRAP_T_LDM_PC; + + /* stmia sp!, { ... lr } doesn't touch the pc, but it is very common, so special case it */ + if (op == 0x12 && ARM_RN(instr) == REG_SP && uses_lr == 1) + return FASTTRAP_T_STM_LR; + + if (ARM_RN(instr) != REG_PC && uses_pc == 0) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int arm_signed_multiplies(uint32_t instr) +{ + int op1 = BITS(instr,20,0x7), op2 = BITS(instr,5,0x7); + + /* smlald, smlsld, smmls use RD in addition to RM, RS, and RN */ + if ((op1 == 0x4 && (op2 & 0x4) == 0) || (op1 == 0x5 && (op2 & 0x6) == 0x6)) { + if (ARM_RD(instr) == REG_PC) + return FASTTRAP_T_INV; + } + + if (ARM_RM(instr) != REG_PC && ARM_RS(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_pack_unpack_sat_reversal(uint32_t instr) +{ + int op1 = BITS(instr,20,0x7), op2 = BITS(instr,5,0x7); + + /* pkh, sel use RN in addition to RD and RM */ + if ((op1 == 0 && (op2 & 0x1) == 0) || (op1 == 0 && op2 == 0x5)) { + if (ARM_RN(instr) == REG_PC) + return FASTTRAP_T_INV; + } + + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_parallel_addsub_unsigned(uint32_t instr) +{ + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_parallel_addsub_signed(uint32_t instr) +{ + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_media(uint32_t instr) +{ + int op1 = BITS(instr,20,0x1F), op2 = BITS(instr,5,0x7); + + if ((op1 & 0x1C) == 0) + return arm_parallel_addsub_signed(instr); + + if ((op1 & 0x1C) == 0x04) + return arm_parallel_addsub_unsigned(instr); + + if ((op1 & 0x18) == 0x08) + return arm_pack_unpack_sat_reversal(instr); + + if ((op1 & 0x18) == 0x10) + return arm_signed_multiplies(instr); + + if (op1 == 0x1F && op2 == 0x7) { + /* Undefined instruction */ + return FASTTRAP_T_INV; + } + + if (op1 == 0x18 && op2 == 0) { + /* usad8 usada8 */ + /* The registers are named differently in the reference manual for this instruction + * but the following positions are correct */ + + if (ARM_RM(instr) != REG_PC && ARM_RS(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; + } + + if ((op1 & 0x1E) == 0x1C && (op2 & 0x3) == 0) { + /* bfc bfi */ + if (ARM_RD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; + } + + if (((op1 & 0x1E) == 0x1A || (op1 & 0x1E) == 0x1E) && ((op2 & 0x3) == 0x2)) { + /* sbfx ubfx */ + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; + } + + return FASTTRAP_T_INV; +} + +static +int arm_loadstore_wordbyte(uint32_t instr) +{ + /* Instrument PC relative load with immediate, ignore any other uses of the PC */ + int R = BITS(instr,25,0x1), L = BITS(instr,20,0x1); + + if (R == 1) { + /* Three register load/store */ + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + } else { + /* Immediate load/store, but still do not support ldr pc, [pc...] */ + if (L == 1 && ARM_RN(instr) == REG_PC && ARM_RD(instr) != REG_PC) + return FASTTRAP_T_LDR_PC_IMMED; + + if (ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int arm_saturating(uint32_t instr) +{ + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_misc(uint32_t instr) +{ + int op = BITS(instr,21,0x3), __unused op1 = BITS(instr,16,0xF), op2 = BITS(instr,4,0x7); + + if (op2 == 1 && op == 1) + return FASTTRAP_T_BX_REG; + + /* We do not need to emulate BLX for entry/return probes; if we eventually support full offset + * tracing, then we will. This is because BLX overwrites the link register, so a function that + * can execute this as its first instruction is a special function indeed. + */ + + if (op2 == 0x5) + return arm_saturating(instr); + + return FASTTRAP_T_INV; +} + +static +int arm_msr_hints(__unused uint32_t instr) +{ + /* These deal with the psr, not instrumented */ + + return FASTTRAP_T_INV; +} + +static +int arm_sync_primitive(__unused uint32_t instr) +{ + /* TODO will instrumenting these interfere with any kernel usage of these instructions? */ + /* Don't instrument for now */ + + return FASTTRAP_T_INV; +} + +static +int arm_extra_loadstore_unpriv(uint32_t instr) +{ + int op = BITS(instr,20,0x1), __unused op2 = BITS(instr,5,0x3), immed = BITS(instr,22,0x1); + + if (op == 0 && (op2 & 0x2) == 0x2) { + /* Unpredictable or undefined */ + return FASTTRAP_T_INV; + } + + if (immed == 1) { + if (ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + } else { + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int arm_extra_loadstore(uint32_t instr) +{ + int op1 = BITS(instr,20,0x1F); + + /* There are two variants, and we do not instrument either of them that use the PC */ + + if ((op1 & 0x4) == 0) { + /* Variant 1, register */ + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + } else { + /* Variant 2, immediate */ + if (ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int arm_halfword_multiply(uint32_t instr) +{ + /* Not all multiply instructions use all four registers. The ones that don't should have those + * register locations set to 0, so we can test them anyway. + */ + + if (ARM_RN(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RS(instr) != REG_PC && ARM_RM(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_multiply(uint32_t instr) +{ + /* Not all multiply instructions use all four registers. The ones that don't should have those + * register locations set to 0, so we can test them anyway. + */ + + if (ARM_RN(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RS(instr) != REG_PC && ARM_RM(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_dataproc_immed(uint32_t instr) +{ + /* All these instructions are either two registers, or one register and have 0 where the other reg would be used */ + if (ARM_RN(instr) != REG_PC && ARM_RD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_dataproc_regshift(uint32_t instr) +{ + /* All these instructions are either four registers, or three registers and have 0 where there last reg would be used */ + if (ARM_RN(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RS(instr) != REG_PC && ARM_RM(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_dataproc_reg(uint32_t instr) +{ + int op1 = BITS(instr,20,0x1F), op2 = BITS(instr,7,0x1F), op3 = BITS(instr,5,0x3); + + if (op1 == 0x11 || op1 == 0x13 || op1 == 0x15 || op1 == 0x17) { + /* These are comparison flag setting instructions and do not have RD */ + if (ARM_RN(instr) != REG_PC && ARM_RM(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; + } + + /* The rest can, in theory, write or use the PC. The only one we instrument is mov pc, reg. + * movs pc, reg is a privileged instruction so we don't instrument that variant. The s bit + * is bit 0 of op1 and should be zero. + */ + if (op1 == 0x1A && op2 == 0 && op3 == 0 && ARM_RD(instr) == REG_PC) + return FASTTRAP_T_MOV_PC_REG; + + /* Any instruction at this point is a three register instruction or two register instruction with RN = 0 */ + if (ARM_RN(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RM(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_dataproc_misc(uint32_t instr) +{ + int op = BITS(instr,25,0x1), op1 = BITS(instr,20,0x1F), op2 = BITS(instr,4,0xF); + + if (op == 0) { + if ((op1 & 0x19) != 0x10 && (op2 & 0x1) == 0) + return arm_dataproc_reg(instr); + + if ((op1 & 0x19) != 0x10 && (op2 & 0x9) == 0x1) + return arm_dataproc_regshift(instr); + + if ((op1 & 0x19) == 0x10 && (op2 & 0x8) == 0) + return arm_misc(instr); + + if ((op1 & 0x19) == 0x19 && (op2 & 0x9) == 0x8) + return arm_halfword_multiply(instr); + + if ((op1 & 0x10) == 0 && op2 == 0x9) + return arm_multiply(instr); + + if ((op1 & 0x10) == 0x10 && op2 == 0x9) + return arm_sync_primitive(instr); + + if ((op1 & 0x12) != 0x02 && (op2 == 0xB || (op2 & 0xD) == 0xD)) + return arm_extra_loadstore(instr); + + if ((op1 & 0x12) == 0x02 && (op2 == 0xB || (op2 & 0xD) == 0xD)) + return arm_extra_loadstore_unpriv(instr); + } else { + if ((op1 & 0x19) != 0x10) + return arm_dataproc_immed(instr); + + if (op1 == 0x10) { + /* 16 bit immediate load (mov (immed)) [encoding A2] */ + if (ARM_RD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; + } + + if (op1 == 0x14) { + /* high halfword 16 bit immediate load (movt) [encoding A1] */ + if (ARM_RD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; + } + + if ((op1 & 0x1B) == 0x12) + return arm_msr_hints(instr); + } + + return FASTTRAP_T_INV; +} + +int dtrace_decode_arm(uint32_t instr) +{ + int cond = BITS(instr,28,0xF), op1 = BITS(instr,25,0x7), op = BITS(instr,4,0x1); + + if (cond == 0xF) + return arm_unconditional(instr); + + if ((op1 & 0x6) == 0) + return arm_dataproc_misc(instr); + + if (op1 == 0x2) + return arm_loadstore_wordbyte(instr); + + if (op1 == 0x3 && op == 0) + return arm_loadstore_wordbyte(instr); + + if (op1 == 0x3 && op == 1) + return arm_media(instr); + + if ((op1 & 0x6) == 0x4) + return arm_branch_link_blockdata(instr); + + if ((op1 & 0x6) == 0x6) + return arm_syscall_coproc(instr); + + return FASTTRAP_T_INV; +} + + +/* + * Thumb 16-bit decoder + */ + +static +int thumb16_cond_supervisor(uint16_t instr) +{ + int opcode = BITS(instr,8,0xF); + + if ((opcode & 0xE) != 0xE) + return FASTTRAP_T_B_COND; + + return FASTTRAP_T_INV; +} + +static +int thumb16_misc(uint16_t instr) +{ + int opcode = BITS(instr,5,0x7F); + + if ((opcode & 0x70) == 0x30 || (opcode & 0x70) == 0x70) { + /* setend, cps, breakpoint, or if-then, not instrumentable */ + return FASTTRAP_T_INV; + } else if ((opcode & 0x78) == 0x28) { + /* Doesn't modify pc, but this happens a lot so make this a special case for emulation */ + return FASTTRAP_T_PUSH_LR; + } else if ((opcode & 0x78) == 0x68) { + return FASTTRAP_T_POP_PC; + } else if ((opcode & 0x28) == 0x08) { + return FASTTRAP_T_CB_N_Z; + } + + /* All other instructions work on low regs only and are instrumentable */ + return FASTTRAP_T_COMMON; +} + +static +int thumb16_loadstore_single(__unused uint16_t instr) +{ + /* These all access the low registers or SP only */ + return FASTTRAP_T_COMMON; +} + +static +int thumb16_data_special_and_branch(uint16_t instr) +{ + int opcode = BITS(instr,6,0xF); + + if (opcode == 0x4) { + /* Unpredictable */ + return FASTTRAP_T_INV; + } else if ((opcode & 0xC) == 0xC) { + /* bx or blx */ + /* Only instrument the bx */ + if ((opcode & 0x2) == 0) + return FASTTRAP_T_BX_REG; + return FASTTRAP_T_INV; + } else { + /* Data processing on high registers, only instrument mov pc, reg */ + if ((opcode & 0xC) == 0x8 && THUMB16_HRD(instr) == REG_PC) + return FASTTRAP_T_CPY_PC; + + if (THUMB16_HRM(instr) != REG_PC && THUMB16_HRD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int thumb16_data_proc(__unused uint16_t instr) +{ + /* These all access the low registers only */ + return FASTTRAP_T_COMMON; +} + +static +int thumb16_shift_addsub_move_compare(__unused uint16_t instr) +{ + /* These all access the low registers only */ + return FASTTRAP_T_COMMON; +} + +static +int dtrace_decode_thumb16(uint16_t instr) +{ + int opcode = BITS(instr,10,0x3F); + + if ((opcode & 0x30) == 0) + return thumb16_shift_addsub_move_compare(instr); + + if (opcode == 0x10) + return thumb16_data_proc(instr); + + if (opcode == 0x11) + return thumb16_data_special_and_branch(instr); + + if ((opcode & 0x3E) == 0x12) { + /* ldr (literal) */ + return FASTTRAP_T_LDR_PC_IMMED; + } + + if ((opcode & 0x3C) == 0x14 || (opcode & 0x38) == 0x18 || (opcode & 0x38) == 0x20) + return thumb16_loadstore_single(instr); + + if ((opcode & 0x3E) == 0x28) { + /* adr, uses the pc */ + return FASTTRAP_T_INV; + } + + if ((opcode & 0x3E) == 0x2A) { + /* add (sp plus immediate) */ + return FASTTRAP_T_COMMON; + } + + if ((opcode & 0x3C) == 0x2C) + return thumb16_misc(instr); + + if ((opcode & 0x3E) == 0x30) { + /* stm - can't access high registers */ + return FASTTRAP_T_COMMON; + } + + if ((opcode & 0x3E) == 0x32) { + /* ldm - can't access high registers */ + return FASTTRAP_T_COMMON; + } + + if ((opcode & 0x3C) == 0x34) { + return thumb16_cond_supervisor(instr); + } + + if ((opcode & 0x3E) == 0x38) { + /* b unconditional */ + return FASTTRAP_T_B_UNCOND; + } + + return FASTTRAP_T_INV; +} + +/* + * Thumb 32-bit decoder + */ + +static +int thumb32_coproc(uint16_t instr1, uint16_t instr2) +{ + /* Instrument any VFP data processing instructions, ignore the rest */ + + int op1 = BITS(instr1,4,0x3F), coproc = BITS(instr2,8,0xF), op = BITS(instr2,4,0x1); + + if ((op1 & 0x3E) == 0) { + /* Undefined */ + return FASTTRAP_T_INV; + } + + if ((coproc & 0xE) == 0xA || (op1 & 0x30) == 0x30) { + /* VFP instruction */ + uint32_t instr = thumb32_instword_to_arm(instr1,instr2); + + if ((op1 & 0x30) == 0x30) { + /* VFP data processing uses its own registers */ + return FASTTRAP_T_COMMON; + } + + if ((op1 & 0x3A) == 0x02 || (op1 & 0x38) == 0x08 || (op1 & 0x30) == 0x10) + return vfp_loadstore(instr); + + if ((op1 & 0x3E) == 0x04) + return vfp_64transfer(instr); + + if ((op1 & 0x30) == 0x20) { + /* VFP data processing or 8, 16, or 32 bit move between ARM reg and VFP reg */ + if (op == 0) { + /* VFP data processing uses its own registers */ + return FASTTRAP_T_COMMON; + } else { + return vfp_transfer(instr); + } + } + } + + return FASTTRAP_T_INV; +} + +static +int thumb32_longmultiply(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,4,0x7), op2 = BITS(instr2,4,0xF); + + if ((op1 == 1 && op2 == 0xF) || (op1 == 0x3 && op2 == 0xF)) { + /* Three register instruction */ + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } else { + /* Four register instruction */ + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && + THUMB32_RT(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int thumb32_multiply(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,4,0x7), op2 = BITS(instr2,4,0x3); + + if ((op1 == 0 && op2 == 1) || (op1 == 0x6 && (op2 & 0x2) == 0)) { + if (THUMB32_RT(instr1,instr2) == REG_PC) + return FASTTRAP_T_INV; + } + + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_misc(uint16_t instr1, uint16_t instr2) +{ + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_parallel_addsub_unsigned(uint16_t instr1, uint16_t instr2) +{ + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_parallel_addsub_signed(uint16_t instr1, uint16_t instr2) +{ + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_dataproc_reg(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,4,0xF), op2 = BITS(instr2,4,0xF); + + if (((0 <= op1) && (op1 <= 5)) && (op2 & 0x8) == 0x8) { + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } + + if ((op1 & 0x8) == 0 && op2 == 0) { + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } + + if ((op1 & 0x8) == 0x8 && (op2 & 0xC) == 0) + return thumb32_parallel_addsub_signed(instr1,instr2); + + if ((op1 & 0x8) == 0x8 && (op2 & 0xC) == 0x4) + return thumb32_parallel_addsub_unsigned(instr1,instr2); + + if ((op1 & 0xC) == 0x8 && (op2 & 0xC) == 0x8) + return thumb32_misc(instr1,instr2); + + return FASTTRAP_T_INV; +} + +static +int thumb32_dataproc_regshift(uint16_t instr1, uint16_t instr2) +{ + int op = BITS(instr1,5,0xF), S = BITS(instr1,4,0x1); + + if (op == 0 || op == 0x4 || op == 0x8 || op == 0xD) { + /* These become test instructions if S is 1 and Rd is PC, otherwise they are data instructions. */ + if (S == 1) { + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } else { + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && + THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } + } else if (op == 0x2 || op == 0x3) { + /* These become moves if RN is PC, otherwise they are data insts. We don't instrument mov pc, reg here */ + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } else { + /* Normal three register instruction */ + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int thumb32_store_single(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,5,0x7), op2 = BITS(instr2,6,0x3F); + + /* Do not support any use of the pc yet */ + if ((op1 == 0 || op1 == 1 || op1 == 2) && (op2 & 0x20) == 0) { + /* str (register) uses RM */ + if (THUMB32_RM(instr1,instr2) == REG_PC) + return FASTTRAP_T_INV; + } + + if (THUMB32_RT(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_loadbyte_memhint(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,7,0x3), __unused op2 = BITS(instr2,6,0x3F); + + /* Do not support any use of the pc yet */ + if ((op1 == 0 || op1 == 0x2) && THUMB32_RM(instr1,instr2) == REG_PC) + return FASTTRAP_T_INV; + + if (THUMB32_RT(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_loadhalfword_memhint(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,7,0x3), op2 = BITS(instr2,6,0x3F); + + /* Do not support any use of the PC yet */ + if (op1 == 0 && op2 == 0 && THUMB32_RM(inst1,instr2) == REG_PC) + return FASTTRAP_T_INV; + + if (THUMB32_RT(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_loadword(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,7,0x3), op2 = BITS(instr2,6,0x3F); + + if ((op1 & 0x2) == 0 && THUMB32_RN(instr1,instr2) == REG_PC && THUMB32_RT(instr1,instr2) != REG_PC) + return FASTTRAP_T_LDR_PC_IMMED; + + if (op1 == 0 && op2 == 0) { + /* ldr (register) uses an additional reg */ + if (THUMB32_RM(instr1,instr2) == REG_PC) + return FASTTRAP_T_INV; + } + + if (THUMB32_RT(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_loadstore_double_exclusive_table(__unused uint16_t instr1, __unused uint16_t instr2) +{ + /* Don't instrument any of these */ + + return FASTTRAP_T_INV; +} + +static +int thumb32_loadstore_multiple(uint16_t instr1, uint16_t instr2) +{ + int op = BITS(instr1,7,0x3), L = BITS(instr1,4,0x1), uses_pc = BITS(instr2,15,0x1), uses_lr = BITS(instr2,14,0x1); + + if (op == 0 || op == 0x3) { + /* Privileged instructions: srs, rfe */ + return FASTTRAP_T_INV; + } + + /* Only emulate a use of the pc if it's a return from function: ldmia sp!, { ... pc }, aka pop { ... pc } */ + if (op == 0x1 && L == 1 && THUMB32_RN(instr1,instr2) == REG_SP && uses_pc == 1) + return FASTTRAP_T_LDM_PC; + + /* stmia sp!, { ... lr }, aka push { ... lr } doesn't touch the pc, but it is very common, so special case it */ + if (op == 0x2 && L == 0 && THUMB32_RN(instr1,instr2) == REG_SP && uses_lr == 1) + return FASTTRAP_T_STM_LR; + + if (THUMB32_RN(instr1,instr2) != REG_PC && uses_pc == 0) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_misc_control(__unused uint16_t instr1, __unused uint16_t instr2) +{ + /* Privileged, and instructions dealing with ThumbEE */ + return FASTTRAP_T_INV; +} + +static +int thumb32_cps_hints(__unused uint16_t instr1, __unused uint16_t instr2) +{ + /* Privileged */ + return FASTTRAP_T_INV; +} + +static +int thumb32_b_misc_control(uint16_t instr1, uint16_t instr2) +{ + int op = BITS(instr1,4,0x7F), op1 = BITS(instr2,12,0x7), __unused op2 = BITS(instr2,8,0xF); + + if ((op1 & 0x5) == 0) { + if ((op & 0x38) != 0x38) + return FASTTRAP_T_B_COND; + + if (op == 0x3A) + return thumb32_cps_hints(instr1,instr2); + + if (op == 0x3B) + return thumb32_misc_control(instr1,instr2); + } + + if ((op1 & 0x5) == 1) + return FASTTRAP_T_B_UNCOND; + + return FASTTRAP_T_INV; +} + +static +int thumb32_dataproc_plain_immed(uint16_t instr1, uint16_t instr2) +{ + int op = BITS(instr1,4,0x1F); + + if (op == 0x04 || op == 0x0C || op == 0x16) { + /* mov, movt, bfi, bfc */ + /* These use only RD */ + if (THUMB32_RD(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } else { + if (THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int thumb32_dataproc_mod_immed(uint16_t instr1, uint16_t instr2) +{ + int op = BITS(instr1,5,0xF), S = BITS(instr1,4,0x1); + + if (op == 0x2 || op == 0x3) { + /* These allow REG_PC in RN, but it doesn't mean use the PC! */ + if (THUMB32_RD(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } + + if (op == 0 || op == 0x4 || op == 0x8 || op == 0xD) { + /* These are test instructions, if the sign bit is set and RD is the PC. */ + if (S && THUMB32_RD(instr1,instr2) == REG_PC) + return FASTTRAP_T_COMMON; + } + + if (THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int dtrace_decode_thumb32(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,11,0x3), op2 = BITS(instr1,4,0x7F), op = BITS(instr2,15,0x1); + + if (op1 == 0x1) { + if ((op2 & 0x64) == 0) + return thumb32_loadstore_multiple(instr1,instr2); + + if ((op2 & 0x64) == 0x04) + return thumb32_loadstore_double_exclusive_table(instr1,instr2); + + if ((op2 & 0x60) == 0x20) + return thumb32_dataproc_regshift(instr1,instr2); + + if ((op2 & 0x40) == 0x40) + return thumb32_coproc(instr1,instr2); + } + + if (op1 == 0x2) { + if ((op2 & 0x20) == 0 && op == 0) + return thumb32_dataproc_mod_immed(instr1,instr2); + + if ((op2 & 0x20) == 0x20 && op == 0) + return thumb32_dataproc_plain_immed(instr1,instr2); + + if (op == 1) + return thumb32_b_misc_control(instr1,instr2); + } + + if (op1 == 0x3) { + if ((op2 & 0x71) == 0) + return thumb32_store_single(instr1,instr2); + + if ((op2 & 0x71) == 0x10) { + return vfp_struct_loadstore(thumb32_instword_to_arm(instr1,instr2)); + } + + if ((op2 & 0x67) == 0x01) + return thumb32_loadbyte_memhint(instr1,instr2); + + if ((op2 & 0x67) == 0x03) + return thumb32_loadhalfword_memhint(instr1,instr2); + + if ((op2 & 0x67) == 0x05) + return thumb32_loadword(instr1,instr2); + + if ((op2 & 0x67) == 0x07) { + /* Undefined instruction */ + return FASTTRAP_T_INV; + } + + if ((op2 & 0x70) == 0x20) + return thumb32_dataproc_reg(instr1,instr2); + + if ((op2 & 0x78) == 0x30) + return thumb32_multiply(instr1,instr2); + + if ((op2 & 0x78) == 0x38) + return thumb32_longmultiply(instr1,instr2); + + if ((op2 & 0x40) == 0x40) + return thumb32_coproc(instr1,instr2); + } + + return FASTTRAP_T_INV; +} + +int dtrace_decode_thumb(uint32_t instr) +{ + uint16_t* pInstr = (uint16_t*) &instr; + uint16_t hw1 = pInstr[0], hw2 = pInstr[1]; + + int size = BITS(hw1,11,0x1F); + + if (size == 0x1D || size == 0x1E || size == 0x1F) + return dtrace_decode_thumb32(hw1,hw2); + else + return dtrace_decode_thumb16(hw1); +} + diff --git a/bsd/dev/arm/dtrace_isa.c b/bsd/dev/arm/dtrace_isa.c new file mode 100644 index 000000000..d38831ba3 --- /dev/null +++ b/bsd/dev/arm/dtrace_isa.c @@ -0,0 +1,631 @@ +/* + * Copyright (c) 2005-2008 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#define MACH__POSIX_C_SOURCE_PRIVATE 1 /* pulls in suitable savearea from + * mach/ppc/thread_status.h */ +#include <arm/proc_reg.h> + +#include <kern/thread.h> +#include <mach/thread_status.h> + +#include <stdarg.h> +#include <string.h> +#include <sys/malloc.h> +#include <sys/time.h> +#include <sys/systm.h> +#include <sys/proc.h> +#include <sys/proc_internal.h> +#include <sys/kauth.h> +#include <sys/dtrace.h> +#include <sys/dtrace_impl.h> +#include <libkern/OSAtomic.h> +#include <kern/simple_lock.h> +#include <kern/sched_prim.h> /* for thread_wakeup() */ +#include <kern/thread_call.h> +#include <kern/task.h> +#include <miscfs/devfs/devfs.h> +#include <mach/vm_param.h> + +extern struct arm_saved_state *find_kern_regs(thread_t); + +extern dtrace_id_t dtrace_probeid_error; /* special ERROR probe */ +typedef arm_saved_state_t savearea_t; + +extern lck_attr_t *dtrace_lck_attr; +extern lck_grp_t *dtrace_lck_grp; + +int dtrace_arm_condition_true(int condition, int cpsr); + +/* + * Atomicity and synchronization + */ +inline void +dtrace_membar_producer(void) +{ +#if __ARM_SMP__ + __asm__ volatile("dmb ish" : : : "memory"); +#else + __asm__ volatile("nop" : : : "memory"); +#endif +} + +inline void +dtrace_membar_consumer(void) +{ +#if __ARM_SMP__ + __asm__ volatile("dmb ish" : : : "memory"); +#else + __asm__ volatile("nop" : : : "memory"); +#endif +} + +/* + * Interrupt manipulation + * XXX dtrace_getipl() can be called from probe context. + */ +int +dtrace_getipl(void) +{ + /* + * XXX Drat, get_interrupt_level is MACH_KERNEL_PRIVATE + * in osfmk/kern/cpu_data.h + */ + /* return get_interrupt_level(); */ + return (ml_at_interrupt_context() ? 1 : 0); +} + +#if __ARM_SMP__ +/* + * MP coordination + */ + +decl_lck_mtx_data(static, dt_xc_lock); +static uint32_t dt_xc_sync; + +typedef struct xcArg { + processorid_t cpu; + dtrace_xcall_t f; + void *arg; +} xcArg_t; + +static void +xcRemote(void *foo) +{ + xcArg_t *pArg = (xcArg_t *) foo; + + if (pArg->cpu == CPU->cpu_id || pArg->cpu == DTRACE_CPUALL) + (pArg->f) (pArg->arg); + + if (hw_atomic_sub(&dt_xc_sync, 1) == 0) + thread_wakeup((event_t) &dt_xc_sync); +} +#endif + +/* + * dtrace_xcall() is not called from probe context. + */ +void +dtrace_xcall(processorid_t cpu, dtrace_xcall_t f, void *arg) +{ +#if __ARM_SMP__ + /* Only one dtrace_xcall in flight allowed */ + lck_mtx_lock(&dt_xc_lock); + + xcArg_t xcArg; + + xcArg.cpu = cpu; + xcArg.f = f; + xcArg.arg = arg; + + cpu_broadcast_xcall(&dt_xc_sync, TRUE, xcRemote, (void*) &xcArg); + + lck_mtx_unlock(&dt_xc_lock); + return; +#else +#pragma unused(cpu) + /* On uniprocessor systems, the cpu should always be either ourselves or all */ + ASSERT(cpu == CPU->cpu_id || cpu == DTRACE_CPUALL); + + (*f)(arg); + return; +#endif +} + +/* + * Initialization + */ +void +dtrace_isa_init(void) +{ +#if __ARM_SMP__ + lck_mtx_init(&dt_xc_lock, dtrace_lck_grp, dtrace_lck_attr); +#endif + return; +} + +/* + * Runtime and ABI + */ +uint64_t +dtrace_getreg(struct regs * savearea, uint_t reg) +{ + struct arm_saved_state *regs = (struct arm_saved_state *) savearea; + + /* beyond register limit? */ + if (reg > ARM_SAVED_STATE32_COUNT - 1) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); + return (0); + } + return (uint64_t) ((unsigned int *) (&(regs->r)))[reg]; +} + +#define RETURN_OFFSET 4 + +static int +dtrace_getustack_common(uint64_t * pcstack, int pcstack_limit, user_addr_t pc, + user_addr_t sp) +{ + int ret = 0; + + ASSERT(pcstack == NULL || pcstack_limit > 0); + + while (pc != 0) { + ret++; + if (pcstack != NULL) { + *pcstack++ = (uint64_t) pc; + pcstack_limit--; + if (pcstack_limit <= 0) + break; + } + + if (sp == 0) + break; + + pc = dtrace_fuword32((sp + RETURN_OFFSET)); + sp = dtrace_fuword32(sp); + } + + return (ret); +} + +void +dtrace_getupcstack(uint64_t * pcstack, int pcstack_limit) +{ + thread_t thread = current_thread(); + savearea_t *regs; + user_addr_t pc, sp; + volatile uint16_t *flags = (volatile uint16_t *) & cpu_core[CPU->cpu_id].cpuc_dtrace_flags; + int n; + + if (*flags & CPU_DTRACE_FAULT) + return; + + if (pcstack_limit <= 0) + return; + + /* + * If there's no user context we still need to zero the stack. + */ + if (thread == NULL) + goto zero; + + regs = (savearea_t *) find_user_regs(thread); + if (regs == NULL) + goto zero; + + *pcstack++ = (uint64_t)dtrace_proc_selfpid(); + pcstack_limit--; + + if (pcstack_limit <= 0) + return; + + pc = regs->pc; + sp = regs->sp; + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) { + *pcstack++ = (uint64_t) pc; + pcstack_limit--; + if (pcstack_limit <= 0) + return; + + pc = regs->lr; + } + + n = dtrace_getustack_common(pcstack, pcstack_limit, pc, regs->r[7]); + + ASSERT(n >= 0); + ASSERT(n <= pcstack_limit); + + pcstack += n; + pcstack_limit -= n; + +zero: + while (pcstack_limit-- > 0) + *pcstack++ = 0ULL; +} + +int +dtrace_getustackdepth(void) +{ + thread_t thread = current_thread(); + savearea_t *regs; + user_addr_t pc, sp; + int n = 0; + + if (thread == NULL) + return 0; + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT)) + return (-1); + + regs = (savearea_t *) find_user_regs(thread); + if (regs == NULL) + return 0; + + pc = regs->pc; + sp = regs->sp; + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) { + n++; + pc = regs->lr; + } + + /* + * Note that unlike ppc, the arm code does not use + * CPU_DTRACE_USTACK_FP. This is because arm always + * traces from the sp, even in syscall/profile/fbt + * providers. + */ + + n += dtrace_getustack_common(NULL, 0, pc, regs->r[7]); + + return (n); +} + +void +dtrace_getufpstack(uint64_t * pcstack, uint64_t * fpstack, int pcstack_limit) +{ + /* XXX ARMTODO 64vs32 */ + thread_t thread = current_thread(); + savearea_t *regs; + user_addr_t pc, sp; + + volatile uint16_t *flags = (volatile uint16_t *) & cpu_core[CPU->cpu_id].cpuc_dtrace_flags; + +#if 0 + uintptr_t oldcontext; + size_t s1, s2; +#endif + + if (*flags & CPU_DTRACE_FAULT) + return; + + if (pcstack_limit <= 0) + return; + + /* + * If there's no user context we still need to zero the stack. + */ + if (thread == NULL) + goto zero; + + regs = (savearea_t *) find_user_regs(thread); + if (regs == NULL) + goto zero; + + *pcstack++ = (uint64_t)dtrace_proc_selfpid(); + pcstack_limit--; + + if (pcstack_limit <= 0) + return; + + pc = regs->pc; + sp = regs->sp; + +#if 0 /* XXX signal stack crawl */ + oldcontext = lwp->lwp_oldcontext; + + if (p->p_model == DATAMODEL_NATIVE) { + s1 = sizeof(struct frame) + 2 * sizeof(long); + s2 = s1 + sizeof(siginfo_t); + } else { + s1 = sizeof(struct frame32) + 3 * sizeof(int); + s2 = s1 + sizeof(siginfo32_t); + } +#endif + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) { + *pcstack++ = (uint64_t) pc; + *fpstack++ = 0; + pcstack_limit--; + if (pcstack_limit <= 0) + return; + + pc = dtrace_fuword32(sp); + } + while (pc != 0 && sp != 0) { + *pcstack++ = (uint64_t) pc; + *fpstack++ = sp; + pcstack_limit--; + if (pcstack_limit <= 0) + break; + +#if 0 /* XXX signal stack crawl */ + if (oldcontext == sp + s1 || oldcontext == sp + s2) { + if (p->p_model == DATAMODEL_NATIVE) { + ucontext_t *ucp = (ucontext_t *) oldcontext; + greg_t *gregs = ucp->uc_mcontext.gregs; + + sp = dtrace_fulword(&gregs[REG_FP]); + pc = dtrace_fulword(&gregs[REG_PC]); + + oldcontext = dtrace_fulword(&ucp->uc_link); + } else { + ucontext_t *ucp = (ucontext_t *) oldcontext; + greg_t *gregs = ucp->uc_mcontext.gregs; + + sp = dtrace_fuword32(&gregs[EBP]); + pc = dtrace_fuword32(&gregs[EIP]); + + oldcontext = dtrace_fuword32(&ucp->uc_link); + } + } else +#endif + { + pc = dtrace_fuword32((sp + RETURN_OFFSET)); + sp = dtrace_fuword32(sp); + } + +#if 0 + /* XXX ARMTODO*/ + /* + * This is totally bogus: if we faulted, we're going to clear + * the fault and break. This is to deal with the apparently + * broken Java stacks on x86. + */ + if (*flags & CPU_DTRACE_FAULT) { + *flags &= ~CPU_DTRACE_FAULT; + break; + } +#endif + } + +zero: + while (pcstack_limit-- > 0) + *pcstack++ = 0ULL; +} + +void +dtrace_getpcstack(pc_t * pcstack, int pcstack_limit, int aframes, + uint32_t * intrpc) +{ + struct frame *fp = (struct frame *) __builtin_frame_address(0); + struct frame *nextfp, *minfp, *stacktop; + int depth = 0; + int on_intr; + int last = 0; + uintptr_t pc; + uintptr_t caller = CPU->cpu_dtrace_caller; + + if ((on_intr = CPU_ON_INTR(CPU)) != 0) + stacktop = (struct frame *) dtrace_get_cpu_int_stack_top(); + else + stacktop = (struct frame *) (dtrace_get_kernel_stack(current_thread()) + kernel_stack_size); + + minfp = fp; + + aframes++; + + if (intrpc != NULL && depth < pcstack_limit) + pcstack[depth++] = (pc_t) intrpc; + + while (depth < pcstack_limit) { + nextfp = *(struct frame **) fp; + pc = *(uintptr_t *) (((uint32_t) fp) + RETURN_OFFSET); + + if (nextfp <= minfp || nextfp >= stacktop) { + if (on_intr) { + /* + * Hop from interrupt stack to thread stack. + */ + arm_saved_state_t *arm_kern_regs = (arm_saved_state_t *) find_kern_regs(current_thread()); + if (arm_kern_regs) { + nextfp = (struct frame *)arm_kern_regs->r[7]; + + vm_offset_t kstack_base = dtrace_get_kernel_stack(current_thread()); + + minfp = (struct frame *)kstack_base; + stacktop = (struct frame *)(kstack_base + kernel_stack_size); + + on_intr = 0; + + if (nextfp <= minfp || nextfp >= stacktop) { + last = 1; + } + } else { + /* + * If this thread was on the interrupt stack, but did not + * take an interrupt (i.e, the idle thread), there is no + * explicit saved state for us to use. + */ + last = 1; + } + } else { + /* + * This is the last frame we can process; indicate + * that we should return after processing this frame. + */ + last = 1; + } + } + if (aframes > 0) { + if (--aframes == 0 && caller != (uintptr_t)NULL) { + /* + * We've just run out of artificial frames, + * and we have a valid caller -- fill it in + * now. + */ + ASSERT(depth < pcstack_limit); + pcstack[depth++] = (pc_t) caller; + caller = (uintptr_t)NULL; + } + } else { + if (depth < pcstack_limit) + pcstack[depth++] = (pc_t) pc; + } + + if (last) { + while (depth < pcstack_limit) + pcstack[depth++] = (pc_t) NULL; + return; + } + fp = nextfp; + minfp = fp; + } +} + +int +dtrace_instr_size(uint32_t instr, int thumb_mode) +{ + if (thumb_mode) { + uint16_t instr16 = *(uint16_t*) &instr; + if (((instr16 >> 11) & 0x1F) > 0x1C) + return 4; + else + return 2; + } else { + return 4; + } +} + +uint64_t +dtrace_getarg(int arg, int aframes, dtrace_mstate_t *mstate, dtrace_vstate_t *vstate) +{ +#pragma unused(arg, aframes, mstate, vstate) +#if 0 + /* XXX ARMTODO */ + uint64_t val; + uintptr_t *fp = (uintptr_t *)__builtin_frame_address(0); + uintptr_t *stack; + uintptr_t pc; + int i; + + for (i = 1; i <= aframes; i++) { + fp = fp[0]; + pc = fp[1]; + + if (dtrace_invop_callsite_pre != NULL + && pc > (uintptr_t)dtrace_invop_callsite_pre + && pc <= (uintptr_t)dtrace_invop_callsite_post) { + /* + * If we pass through the invalid op handler, we will + * use the pointer that it passed to the stack as the + * second argument to dtrace_invop() as the pointer to + * the frame we're hunting for. + */ + + stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */ + fp = (struct frame *)stack[1]; /* Grab *second* argument */ + stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */ + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); + val = (uint64_t)(stack[arg]); + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); + return val; + } + } + + /* + * Arrive here when provider has called dtrace_probe directly. + */ + stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */ + stack++; /* Advance past probeID */ + + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); + val = *(((uint64_t *)stack) + arg); /* dtrace_probe arguments arg0 .. arg4 are 64bits wide */ + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); + return (val); +#endif + return 0xfeedfacedeafbeadLL; +} + +void +dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which, + int fltoffs, int fault, uint64_t illval) +{ + /* XXX ARMTODO */ + /* + * For the case of the error probe firing lets + * stash away "illval" here, and special-case retrieving it in DIF_VARIABLE_ARG. + */ + state->dts_arg_error_illval = illval; + dtrace_probe( dtrace_probeid_error, (uint64_t)(uintptr_t)state, epid, which, fltoffs, fault ); +} + +void +dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit)) +{ + /* XXX ARMTODO check copied from ppc/x86*/ + /* + * "base" is the smallest toxic address in the range, "limit" is the first + * VALID address greater than "base". + */ + func(0x0, VM_MIN_KERNEL_ADDRESS); + if (VM_MAX_KERNEL_ADDRESS < ~(uintptr_t)0) + func(VM_MAX_KERNEL_ADDRESS + 1, ~(uintptr_t)0); +} + +int +dtrace_arm_condition_true(int cond, int cpsr) +{ + int taken = 0; + int zf = (cpsr & PSR_ZF) ? 1 : 0, + nf = (cpsr & PSR_NF) ? 1 : 0, + cf = (cpsr & PSR_CF) ? 1 : 0, + vf = (cpsr & PSR_VF) ? 1 : 0; + + switch(cond) { + case 0: taken = zf; break; + case 1: taken = !zf; break; + case 2: taken = cf; break; + case 3: taken = !cf; break; + case 4: taken = nf; break; + case 5: taken = !nf; break; + case 6: taken = vf; break; + case 7: taken = !vf; break; + case 8: taken = (cf && !zf); break; + case 9: taken = (!cf || zf); break; + case 10: taken = (nf == vf); break; + case 11: taken = (nf != vf); break; + case 12: taken = (!zf && (nf == vf)); break; + case 13: taken = (zf || (nf != vf)); break; + case 14: taken = 1; break; + case 15: taken = 1; break; /* always "true" for ARM, unpredictable for THUMB. */ + } + + return taken; +} diff --git a/bsd/dev/arm/dtrace_subr_arm.c b/bsd/dev/arm/dtrace_subr_arm.c new file mode 100644 index 000000000..ab0bc4820 --- /dev/null +++ b/bsd/dev/arm/dtrace_subr_arm.c @@ -0,0 +1,186 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + */ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* + * #pragma ident "@(#)dtrace_subr.c 1.12 05/06/08 SMI" + */ + +#include <sys/dtrace.h> +#include <sys/dtrace_glue.h> +#include <sys/dtrace_impl.h> +#include <sys/fasttrap.h> +#include <sys/vm.h> +#include <sys/user.h> +#include <sys/kauth.h> +#include <kern/debug.h> +#include <arm/proc_reg.h> + +int (*dtrace_pid_probe_ptr) (arm_saved_state_t *); +int (*dtrace_return_probe_ptr) (arm_saved_state_t *); + +kern_return_t +dtrace_user_probe(arm_saved_state_t *, unsigned int); + +kern_return_t +dtrace_user_probe(arm_saved_state_t *regs, unsigned int instr) +{ + /* + * FIXME + * + * The only call path into this method is always a user trap. + * We don't need to test for user trap, but should assert it. + */ + + lck_rw_t *rwp; + struct proc *p = current_proc(); + + uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread()); + + kauth_cred_uthread_update(uthread, p); + + if (((regs->cpsr & PSR_TF) && ((uint16_t) instr) == FASTTRAP_THUMB_RET_INSTR) || + ((uint32_t) instr == FASTTRAP_ARM_RET_INSTR)) { + uint8_t step = uthread->t_dtrace_step; + uint8_t ret = uthread->t_dtrace_ret; + user_addr_t npc = uthread->t_dtrace_npc; + + if (uthread->t_dtrace_ast) { + printf("dtrace_user_probe() should be calling aston()\n"); + // aston(thread); + // uthread->t_sig_check = 1; + } + + /* + * Clear all user tracing flags. + */ + uthread->t_dtrace_ft = 0; + + /* + * If we weren't expecting to take a return probe trap, kill + * the process as though it had just executed an unassigned + * trap instruction. + */ + if (step == 0) { + /* + * APPLE NOTE: We're returning KERN_FAILURE, which causes + * the generic signal handling code to take over, which will effectively + * deliver a EXC_BAD_INSTRUCTION to the user process. + */ + return KERN_FAILURE; + } + + /* + * If we hit this trap unrelated to a return probe, we're + * just here to reset the AST flag since we deferred a signal + * until after we logically single-stepped the instruction we + * copied out. + */ + if (ret == 0) { + regs->pc = npc; + return KERN_SUCCESS; + } + + /* + * We need to wait until after we've called the + * dtrace_return_probe_ptr function pointer to step the pc. + */ + rwp = &CPU->cpu_ft_lock; + lck_rw_lock_shared(rwp); + + if (dtrace_return_probe_ptr != NULL) + (void) (*dtrace_return_probe_ptr)(regs); + lck_rw_unlock_shared(rwp); + + regs->pc = npc; + + return KERN_SUCCESS; + } else { + rwp = &CPU->cpu_ft_lock; + + /* + * The DTrace fasttrap provider uses a trap, + * FASTTRAP_{ARM,THUMB}_INSTR. We let + * DTrace take the first crack at handling + * this trap; if it's not a probe that DTrace knows about, + * we call into the trap() routine to handle it like a + * breakpoint placed by a conventional debugger. + */ + + /* + * APPLE NOTE: I believe the purpose of the reader/writers lock + * is thus: There are times which dtrace needs to prevent calling + * dtrace_pid_probe_ptr(). Sun's original impl grabbed a plain + * mutex here. However, that serialized all probe calls, and + * destroyed MP behavior. So now they use a RW lock, with probes + * as readers, and the top level synchronization as a writer. + */ + lck_rw_lock_shared(rwp); + if (dtrace_pid_probe_ptr != NULL && + (*dtrace_pid_probe_ptr)(regs) == 0) { + lck_rw_unlock_shared(rwp); + return KERN_SUCCESS; + } + lck_rw_unlock_shared(rwp); + + /* + * If the instruction that caused the breakpoint trap doesn't + * look like our trap anymore, it may be that this tracepoint + * was removed just after the user thread executed it. In + * that case, return to user land to retry the instuction. + * + * Note that the PC points to the instruction that caused the fault. + */ + if (regs->cpsr & PSR_TF) { + uint16_t instr_check; + if (fuword16(regs->pc, &instr_check) == 0 && instr_check != FASTTRAP_THUMB_INSTR) { + return KERN_SUCCESS; + } + } else { + uint32_t instr_check; + if (fuword32(regs->pc, &instr_check) == 0 && instr_check != FASTTRAP_ARM_INSTR) { + return KERN_SUCCESS; + } + } + } + + return KERN_FAILURE; +} + +void +dtrace_safe_synchronous_signal(void) +{ + /* Not implemented */ +} + +int +dtrace_safe_defer_signal(void) +{ + /* Not implemented */ + return 0; +} diff --git a/bsd/dev/arm/fasttrap_isa.c b/bsd/dev/arm/fasttrap_isa.c new file mode 100644 index 000000000..d48b48a71 --- /dev/null +++ b/bsd/dev/arm/fasttrap_isa.c @@ -0,0 +1,1297 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + */ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* + * #pragma ident "@(#)fasttrap_isa.c 1.19 05/09/14 SMI" + */ + +#ifdef KERNEL +#ifndef _KERNEL +#define _KERNEL /* Solaris vs. Darwin */ +#endif +#endif + +#include <sys/fasttrap_isa.h> +#include <sys/fasttrap_impl.h> +#include <sys/dtrace.h> +#include <sys/dtrace_impl.h> +#include <kern/task.h> +#include <vm/pmap.h> +#include <vm/vm_map.h> +#include <mach/mach_vm.h> +#include <arm/proc_reg.h> +#include <arm/caches_internal.h> + +#include <sys/dtrace_ptss.h> +#include <kern/debug.h> + +#include <pexpert/pexpert.h> + +extern dtrace_id_t dtrace_probeid_error; + +/* Solaris proc_t is the struct. Darwin's proc_t is a pointer to it. */ +#define proc_t struct proc /* Steer clear of the Darwin typedef for proc_t */ + +extern int dtrace_decode_arm(uint32_t instr); +extern int dtrace_decode_thumb(uint32_t instr); + +/* + * Lossless User-Land Tracing on ARM + * --------------------------------- + * + * The details here will be fleshed out as more of this is implemented. The + * basic design will be the same way as tracing works in x86. + * + * Some ARM specific issues: + * + * We need to patch differently for ARM instructions and Thumb instructions. + * When we hit a probe, we check to see if the mode we're currently in is the + * same as the mode we're patching for. If not, we remove the tracepoint and + * abort. This ARM/Thumb information is pulled in from the arch specific + * information in the fasttrap probe. + * + * On ARM, any instruction that uses registers can also use the pc as a + * register. This presents problems during emulation because we have copied + * the instruction and thus the pc can be different. Currently we've emulated + * any instructions that use the pc if they can be used in a return probe. + * Eventually we will want to support all instructions that use the pc, but + * to do so requires disassembling the instruction and reconstituting it by + * substituting a different register. + * + */ + +#define THUMB_INSTR(x) (*(uint16_t*) &(x)) + +#define SIGNEXTEND(x,v) ((((int) (x)) << (32-(v))) >> (32-(v))) +#define ALIGNADDR(x,v) (((x) >> (v)) << (v)) +#define GETITSTATE(x) ((((x) >> 8) & 0xFC) | (((x) >> 25) & 0x3)) +#define ISLASTINIT(x) (((x) & 0xF) == 8) + +#define SET16(x,w) *((uint16_t*) (x)) = (w) +#define SET32(x,w) *((uint32_t*) (x)) = (w) + +#define IS_ARM_NOP(x) ((x) == 0xE1A00000) +/* Marker for is-enabled probes */ +#define IS_ARM_IS_ENABLED(x) ((x) == 0xE0200000) + +#define IS_THUMB_NOP(x) ((x) == 0x46C0) +/* Marker for is-enabled probes */ +#define IS_THUMB_IS_ENABLED(x) ((x) == 0x4040) + +#define ARM_LDM_UF (1 << 23) +#define ARM_LDM_PF (1 << 24) +#define ARM_LDM_WF (1 << 21) + +#define ARM_LDR_UF (1 << 23) +#define ARM_LDR_BF (1 << 22) + +extern int dtrace_arm_condition_true(int cond, int cpsr); + +static +void flush_caches(void) +{ + /* TODO There were some problems with flushing just the cache line that had been modified. + * For now, we'll flush the entire cache, until we figure out how to flush just the patched block. + */ + FlushPoU_Dcache(); + InvalidatePoU_Icache(); +} + +int +fasttrap_tracepoint_init(proc_t *p, fasttrap_tracepoint_t *tp, + user_addr_t pc, fasttrap_probe_type_t type) +{ +#pragma unused(type) + uint32_t instr; + + /* + * Read the instruction at the given address out of the process's + * address space. We don't have to worry about a debugger + * changing this instruction before we overwrite it with our trap + * instruction since P_PR_LOCK is set. Since instructions can span + * pages, we potentially read the instruction in two parts. If the + * second part fails, we just zero out that part of the instruction. + */ + /* + * APPLE NOTE: Of course, we do not have a P_PR_LOCK, so this is racey... + */ + + if (uread(p, &instr, 4, pc) != 0) + return (-1); + + /* We want &instr to always point to the saved instruction, so just copy the + * whole thing When cast to a pointer to a uint16_t, that will give us a + * pointer to the first two bytes, which is the thumb instruction. + */ + tp->ftt_instr = instr; + + if (tp->ftt_fntype != FASTTRAP_FN_DONE_INIT) { + switch(tp->ftt_fntype) { + case FASTTRAP_FN_UNKNOWN: + /* Can't instrument without any information. We can add some heuristics later if necessary. */ + return (-1); + + case FASTTRAP_FN_USDT: + if (IS_ARM_NOP(instr) || IS_ARM_IS_ENABLED(instr)) { + tp->ftt_thumb = 0; + } else if (IS_THUMB_NOP(THUMB_INSTR(instr)) || IS_THUMB_IS_ENABLED(THUMB_INSTR(instr))) { + tp->ftt_thumb = 1; + } else { + /* Shouldn't reach here - this means we don't recognize + * the instruction at one of the USDT probe locations + */ + return (-1); + } + tp->ftt_fntype = FASTTRAP_FN_DONE_INIT; + break; + + case FASTTRAP_FN_ARM: + tp->ftt_thumb = 0; + tp->ftt_fntype = FASTTRAP_FN_DONE_INIT; + break; + + case FASTTRAP_FN_THUMB: + tp->ftt_thumb = 1; + tp->ftt_fntype = FASTTRAP_FN_DONE_INIT; + break; + + default: + return (-1); + } + } + + if (tp->ftt_thumb) { + tp->ftt_type = dtrace_decode_thumb(instr); + } else { + tp->ftt_type = dtrace_decode_arm(instr); + } + + if (tp->ftt_type == FASTTRAP_T_INV) { + /* This is an instruction we either don't recognize or can't instrument */ + printf("dtrace: fasttrap: Unrecognized instruction: %08x at %08x\n", + (tp->ftt_thumb && dtrace_instr_size(tp->ftt_instr,tp->ftt_thumb) == 2) ? tp->ftt_instr1 : instr, pc); + return (-1); + } + + return (0); +} + +// These are not exported from vm_map.h. +extern kern_return_t vm_map_write_user(vm_map_t map, void *src_p, vm_map_address_t dst_addr, vm_size_t size); + +/* Patches the instructions. Almost like uwrite, but need special instructions on ARM to flush the caches. */ +static +int patchInst(proc_t *p, void *buf, user_size_t len, user_addr_t a) +{ + kern_return_t ret; + + ASSERT(p != NULL); + ASSERT(p->task != NULL); + + task_t task = p->task; + + /* + * Grab a reference to the task vm_map_t to make sure + * the map isn't pulled out from under us. + * + * Because the proc_lock is not held at all times on all code + * paths leading here, it is possible for the proc to have + * exited. If the map is null, fail. + */ + vm_map_t map = get_task_map_reference(task); + if (map) { + /* Find the memory permissions. */ + uint32_t nestingDepth=999999; + vm_region_submap_short_info_data_64_t info; + mach_msg_type_number_t count = VM_REGION_SUBMAP_SHORT_INFO_COUNT_64; + mach_vm_address_t address = (mach_vm_address_t)a; + mach_vm_size_t sizeOfRegion = (mach_vm_size_t)len; + + ret = mach_vm_region_recurse(map, &address, &sizeOfRegion, &nestingDepth, (vm_region_recurse_info_t)&info, &count); + if (ret != KERN_SUCCESS) + goto done; + + vm_prot_t reprotect; + + if (!(info.protection & VM_PROT_WRITE)) { + /* Save the original protection values for restoration later */ + reprotect = info.protection; + if (info.max_protection & VM_PROT_WRITE) { + /* The memory is not currently writable, but can be made writable. */ + /* Making it both writable and executable at the same time causes warning on embedded */ + ret = mach_vm_protect (map, (mach_vm_offset_t)a, (mach_vm_size_t)len, 0, (reprotect & ~VM_PROT_EXECUTE) | VM_PROT_WRITE); + } else { + /* + * The memory is not currently writable, and cannot be made writable. We need to COW this memory. + * + * Strange, we can't just say "reprotect | VM_PROT_COPY", that fails. + */ + ret = mach_vm_protect (map, (mach_vm_offset_t)a, (mach_vm_size_t)len, 0, VM_PROT_COPY | VM_PROT_READ | VM_PROT_WRITE); + } + + if (ret != KERN_SUCCESS) + goto done; + + } else { + /* The memory was already writable. */ + reprotect = VM_PROT_NONE; + } + + ret = vm_map_write_user( map, + buf, + (vm_map_address_t)a, + (vm_size_t)len); + + flush_caches(); + + if (ret != KERN_SUCCESS) + goto done; + + if (reprotect != VM_PROT_NONE) { + ASSERT(reprotect & VM_PROT_EXECUTE); + ret = mach_vm_protect (map, (mach_vm_offset_t)a, (mach_vm_size_t)len, 0, reprotect); + } + +done: + vm_map_deallocate(map); + } else + ret = KERN_TERMINATED; + + return (int)ret; +} + +int +fasttrap_tracepoint_install(proc_t *p, fasttrap_tracepoint_t *tp) +{ + /* The thumb patch is a 2 byte instruction regardless of the size of the original instruction */ + uint32_t instr; + int size = tp->ftt_thumb ? 2 : 4; + + if (tp->ftt_thumb) { + *((uint16_t*) &instr) = FASTTRAP_THUMB_INSTR; + } else { + instr = FASTTRAP_ARM_INSTR; + } + + if (patchInst(p, &instr, size, tp->ftt_pc) != 0) + return (-1); + + tp->ftt_installed = 1; + + return (0); +} + +int +fasttrap_tracepoint_remove(proc_t *p, fasttrap_tracepoint_t *tp) +{ + /* The thumb patch is a 2 byte instruction regardless of the size of the original instruction */ + uint32_t instr; + int size = tp->ftt_thumb ? 2 : 4; + + /* + * Distinguish between read or write failures and a changed + * instruction. + */ + if (uread(p, &instr, size, tp->ftt_pc) != 0) + goto end; + if (tp->ftt_thumb) { + if (*((uint16_t*) &instr) != FASTTRAP_THUMB_INSTR) + goto end; + } else { + if (instr != FASTTRAP_ARM_INSTR) + goto end; + } + if (patchInst(p, &tp->ftt_instr, size, tp->ftt_pc) != 0) + return (-1); + +end: + tp->ftt_installed = 0; + + return (0); +} + +static void +fasttrap_return_common(proc_t *p, arm_saved_state_t *regs, user_addr_t pc, user_addr_t new_pc) +{ + pid_t pid = p->p_pid; + fasttrap_tracepoint_t *tp; + fasttrap_bucket_t *bucket; + fasttrap_id_t *id; + lck_mtx_t *pid_mtx; + int retire_tp = 1; + + pid_mtx = &cpu_core[CPU->cpu_id].cpuc_pid_lock; + lck_mtx_lock(pid_mtx); + bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; + + for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { + if (pid == tp->ftt_pid && pc == tp->ftt_pc && + tp->ftt_proc->ftpc_acount != 0) + break; + } + + /* + * Don't sweat it if we can't find the tracepoint again; unlike + * when we're in fasttrap_pid_probe(), finding the tracepoint here + * is not essential to the correct execution of the process. + */ + if (tp == NULL) { + lck_mtx_unlock(pid_mtx); + return; + } + + for (id = tp->ftt_retids; id != NULL; id = id->fti_next) { + fasttrap_probe_t *probe = id->fti_probe; + /* + * If there's a branch that could act as a return site, we + * need to trace it, and check here if the program counter is + * external to the function. + */ + if (tp->ftt_type != FASTTRAP_T_LDM_PC && + tp->ftt_type != FASTTRAP_T_POP_PC && + new_pc - probe->ftp_faddr < probe->ftp_fsize) + continue; + + if (probe->ftp_prov->ftp_provider_type == DTFTP_PROVIDER_ONESHOT) { + uint8_t already_triggered = atomic_or_8(&probe->ftp_triggered, 1); + if (already_triggered) { + continue; + } + } + /* + * If we have at least one probe associated that + * is not a oneshot probe, don't remove the + * tracepoint + */ + else { + retire_tp = 0; + } +#ifndef CONFIG_EMBEDDED + if (ISSET(current_proc()->p_lflag, P_LNOATTACH)) { + dtrace_probe(dtrace_probeid_error, 0 /* state */, id->fti_probe->ftp_id, + 1 /* ndx */, -1 /* offset */, DTRACEFLT_UPRIV); +#else + if (FALSE) { +#endif + } else { + dtrace_probe(id->fti_probe->ftp_id, + pc - id->fti_probe->ftp_faddr, + regs->r[0], 0, 0, 0); + } + } + if (retire_tp) { + fasttrap_tracepoint_retire(p, tp); + } + + lck_mtx_unlock(pid_mtx); +} + +static void +fasttrap_sigsegv(proc_t *p, uthread_t t, user_addr_t addr, arm_saved_state_t *regs) +{ + /* TODO: This function isn't implemented yet. In debug mode, panic the system to + * find out why we're hitting this point. In other modes, kill the process. + */ +#if DEBUG +#pragma unused(p,t,addr,arm_saved_state) + panic("fasttrap: sigsegv not yet implemented"); +#else +#pragma unused(p,t,addr) + /* Kill the process */ + regs->pc = 0; +#endif + +#if 0 + proc_lock(p); + + /* Set fault address and mark signal */ + t->uu_code = addr; + t->uu_siglist |= sigmask(SIGSEGV); + + /* + * XXX These two line may be redundant; if not, then we need + * XXX to potentially set the data address in the machine + * XXX specific thread state structure to indicate the address. + */ + t->uu_exception = KERN_INVALID_ADDRESS; /* SIGSEGV */ + t->uu_subcode = 0; /* XXX pad */ + + proc_unlock(p); + + /* raise signal */ + signal_setast(t->uu_context.vc_thread); +#endif +} + +static void +fasttrap_usdt_args(fasttrap_probe_t *probe, arm_saved_state_t *regs, int argc, + uint32_t *argv) +{ + int i, x, cap = MIN(argc, probe->ftp_nargs); + + for (i = 0; i < cap; i++) { + x = probe->ftp_argmap[i]; + + if (x < 4) { + argv[i] = regs->r[x]; + } else { + fasttrap_fuword32_noerr(regs->sp + (x - 4) * sizeof(uint32_t), &argv[i]); + } + } + + for (; i < argc; i++) { + argv[i] = 0; + } +} + +static void set_thumb_flag(arm_saved_state_t *regs, user_addr_t pc) +{ + if (pc & 1) { + regs->cpsr |= PSR_TF; + } else { + regs->cpsr &= ~PSR_TF; + } +} + +int +fasttrap_pid_probe(arm_saved_state_t *regs) +{ + proc_t *p = current_proc(); + user_addr_t new_pc = 0; + fasttrap_bucket_t *bucket; + lck_mtx_t *pid_mtx; + fasttrap_tracepoint_t *tp, tp_local; + pid_t pid; + dtrace_icookie_t cookie; + uint_t is_enabled = 0; + int instr_size; + int was_simulated = 1, retire_tp = 1; + + user_addr_t pc = regs->pc; + + uthread_t uthread = (uthread_t) get_bsdthread_info(current_thread()); + + /* + * It's possible that a user (in a veritable orgy of bad planning) + * could redirect this thread's flow of control before it reached the + * return probe fasttrap. In this case we need to kill the process + * since it's in a unrecoverable state. + */ + if (uthread->t_dtrace_step) { + ASSERT(uthread->t_dtrace_on); + fasttrap_sigtrap(p, uthread, pc); + return (0); + } + + /* + * Clear all user tracing flags. + */ + uthread->t_dtrace_ft = 0; + uthread->t_dtrace_pc = 0; + uthread->t_dtrace_npc = 0; + uthread->t_dtrace_scrpc = 0; + uthread->t_dtrace_astpc = 0; + + /* + * Treat a child created by a call to vfork(2) as if it were its + * parent. We know that there's only one thread of control in such a + * process: this one. + */ + if (p->p_lflag & P_LINVFORK) { + proc_list_lock(); + while (p->p_lflag & P_LINVFORK) + p = p->p_pptr; + proc_list_unlock(); + } + + pid = p->p_pid; + pid_mtx = &cpu_core[CPU->cpu_id].cpuc_pid_lock; + lck_mtx_lock(pid_mtx); + bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid,pc)]; + + /* + * Lookup the tracepoint that the process just hit. + */ + for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { + if (pid == tp->ftt_pid && pc == tp->ftt_pc && + tp->ftt_proc->ftpc_acount != 0) + break; + } + + /* + * If we couldn't find a matching tracepoint, either a tracepoint has + * been inserted without using the pid<pid> ioctl interface (see + * fasttrap_ioctl), or somehow we have mislaid this tracepoint. + */ + if (tp == NULL) { + lck_mtx_unlock(pid_mtx); + return (-1); + } + + /* Default to always execute */ + int condition_code = 0xE; + if (tp->ftt_thumb) { + uint32_t itstate = GETITSTATE(regs->cpsr); + if (itstate != 0) { + /* In IT block, make sure it's the last statement in the block */ + if (ISLASTINIT(itstate)) { + condition_code = itstate >> 4; + } else { + printf("dtrace: fasttrap: Tried to trace instruction %08x at %08x but not at end of IT block\n", + (tp->ftt_thumb && dtrace_instr_size(tp->ftt_instr,tp->ftt_thumb) == 2) ? tp->ftt_instr1 : tp->ftt_instr, pc); + + fasttrap_tracepoint_remove(p, tp); + lck_mtx_unlock(pid_mtx); + return (-1); + } + } + } else { + condition_code = ARM_CONDCODE(tp->ftt_instr); + } + + if (!tp->ftt_thumb != !(regs->cpsr & PSR_TF)) { + /* The ARM/Thumb mode does not match what we expected for this probe. + * Remove this probe and bail. + */ + fasttrap_tracepoint_remove(p, tp); + lck_mtx_unlock(pid_mtx); + return (-1); + } + + if (tp->ftt_ids != NULL) { + fasttrap_id_t *id; + + uint32_t s4; + uint32_t *stack = (uint32_t *)regs->sp; + + /* First four parameters are passed in registers */ + fasttrap_fuword32_noerr((user_addr_t)(uint32_t)stack, &s4); + + for (id = tp->ftt_ids; id != NULL; id = id->fti_next) { + fasttrap_probe_t *probe = id->fti_probe; + +#ifndef CONFIG_EMBEDDED + if (ISSET(current_proc()->p_lflag, P_LNOATTACH)) { + dtrace_probe(dtrace_probeid_error, 0 /* state */, probe->ftp_id, + 1 /* ndx */, -1 /* offset */, DTRACEFLT_UPRIV); +#else + if (FALSE) { +#endif + } else { + if (probe->ftp_prov->ftp_provider_type == DTFTP_PROVIDER_ONESHOT) { + uint8_t already_triggered = atomic_or_8(&probe->ftp_triggered, 1); + if (already_triggered) { + continue; + } + } + /* + * If we have at least probe associated that + * is not a oneshot probe, don't remove the + * tracepoint + */ + else { + retire_tp = 0; + } + if (id->fti_ptype == DTFTP_ENTRY) { + /* + * We note that this was an entry + * probe to help ustack() find the + * first caller. + */ + cookie = dtrace_interrupt_disable(); + DTRACE_CPUFLAG_SET(CPU_DTRACE_ENTRY); + dtrace_probe(probe->ftp_id, regs->r[0], regs->r[1], regs->r[2], regs->r[3], s4); + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_ENTRY); + dtrace_interrupt_enable(cookie); + } else if (id->fti_ptype == DTFTP_IS_ENABLED) { + /* + * Note that in this case, we don't + * call dtrace_probe() since it's only + * an artificial probe meant to change + * the flow of control so that it + * encounters the true probe. + */ + is_enabled = 1; + } else if (probe->ftp_argmap == NULL) { + dtrace_probe(probe->ftp_id, regs->r[0], regs->r[1], regs->r[2], regs->r[3], s4); + } else { + uint32_t t[5]; + + fasttrap_usdt_args(probe, regs, 5, t); + dtrace_probe(probe->ftp_id, t[0], t[1], t[2], t[3], t[4]); + } + } + } + if (retire_tp) { + fasttrap_tracepoint_retire(p, tp); + } + } + /* + * We're about to do a bunch of work so we cache a local copy of + * the tracepoint to emulate the instruction, and then find the + * tracepoint again later if we need to light up any return probes. + */ + tp_local = *tp; + lck_mtx_unlock(pid_mtx); + tp = &tp_local; + + /* + * If there's an is-enabled probe connected to this tracepoint it + * means that there was a 'eor r0,r0,r0' + * instruction that was placed there by DTrace when the binary was + * linked. As this probe is, in fact, enabled, we need to stuff 1 + * into R0. Accordingly, we can bypass all the instruction + * emulation logic since we know the inevitable result. It's possible + * that a user could construct a scenario where the 'is-enabled' + * probe was on some other instruction, but that would be a rather + * exotic way to shoot oneself in the foot. + */ + + if (is_enabled) { + regs->r[0] = 1; + new_pc = regs->pc + (tp->ftt_thumb ? 2 : 4); + goto done; + } + + /* For USDT probes, bypass all the emulation logic for the nop instruction */ + if ((tp->ftt_thumb && IS_THUMB_NOP(THUMB_INSTR(tp->ftt_instr))) || + (!tp->ftt_thumb && IS_ARM_NOP(tp->ftt_instr))) { + new_pc = regs->pc + (tp->ftt_thumb ? 2 : 4); + goto done; + } + + instr_size = dtrace_instr_size(tp->ftt_instr,tp->ftt_thumb); + + switch (tp->ftt_type) { + case FASTTRAP_T_MOV_PC_REG: + case FASTTRAP_T_CPY_PC: + { + if (!dtrace_arm_condition_true(condition_code, regs->cpsr)) { + new_pc = pc + instr_size; + break; + } + + int rm; + if (tp->ftt_thumb) { + rm = THUMB16_HRM(tp->ftt_instr1); + } else { + rm = tp->ftt_instr & 0xF; + } + new_pc = regs->r[rm]; + + /* This instruction does not change the Thumb state */ + + break; + } + + case FASTTRAP_T_STM_LR: + case FASTTRAP_T_PUSH_LR: + { + /* + * This is a very common case, so we want to emulate this instruction if + * possible. However, on a push, it is possible that we might reach the end + * of a page and have to allocate a new page. Most of the time this will not + * happen, and we know that the push instruction can store at most 16 words, + * so check to see if we are far from the boundary, and if so, emulate. This + * can be made more aggressive by checking the actual number of words being + * pushed, but we won't do that for now. + * + * Some of the same issues that apply to POP_PC probably apply here also. + */ + + int reglist; + int ret; + uintptr_t* base; + + if (!dtrace_arm_condition_true(condition_code, regs->cpsr)) { + new_pc = pc + instr_size; + break; + } + + base = (uintptr_t*) regs->sp; + if (((((uintptr_t) base)-16*4) >> PAGE_SHIFT) != (((uintptr_t) base) >> PAGE_SHIFT)) { + /* Crosses the page boundary, go to emulation */ + goto instr_emulate; + } + + if (tp->ftt_thumb) { + if (instr_size == 4) { + /* We know we have to push lr, never push sp or pc */ + reglist = tp->ftt_instr2 & 0x1FFF; + } else { + reglist = tp->ftt_instr1 & 0xFF; + } + } else { + /* We know we have to push lr, never push sp or pc */ + reglist = tp->ftt_instr & 0x1FFF; + } + + /* Push the link register */ + base--; + ret = fasttrap_suword32((uint32_t) base, regs->lr); + if (ret == -1) { + fasttrap_sigsegv(p, uthread, (user_addr_t) base, regs); + new_pc = regs->pc; + break; + } + + /* Start pushing from $r12 */ + int regmask = 1 << 12; + int regnum = 12; + + while (regmask) { + if (reglist & regmask) { + base--; + ret = fasttrap_suword32((uint32_t) base, regs->r[regnum]); + if (ret == -1) { + fasttrap_sigsegv(p, uthread, (user_addr_t) base, regs); + new_pc = regs->pc; + break; + } + } + regmask >>= 1; + regnum--; + } + + regs->sp = (uintptr_t) base; + + new_pc = pc + instr_size; + + break; + } + + + case FASTTRAP_T_LDM_PC: + case FASTTRAP_T_POP_PC: + { + /* TODO Two issues that will eventually need to be resolved: + * + * 1. Understand what the hardware does if we have to segfault (data abort) in + * the middle of a load multiple. We currently don't have a working segfault + * handler anyway, and with no swapfile we should never segfault on this load. + * If we do, we'll just kill the process by setting the pc to 0. + * + * 2. The emulation is no longer atomic. We currently only emulate pop for + * function epilogues, and so we should never have a race here because one + * thread should never be trying to manipulate another thread's stack frames. + * That is almost certainly a bug in the program. + * + * This will need to be fixed if we ever: + * a. Ship dtrace externally, as this could be a potential attack vector + * b. Support instruction level tracing, as we might then pop/ldm non epilogues. + * + */ + + /* Assume ldmia! sp/pop ... pc */ + + int regnum = 0, reglist; + int ret; + uintptr_t* base; + + if (!dtrace_arm_condition_true(condition_code, regs->cpsr)) { + new_pc = pc + instr_size; + break; + } + + if (tp->ftt_thumb) { + if (instr_size == 4) { + /* We know we have to load the pc, don't do it twice */ + reglist = tp->ftt_instr2 & 0x7FFF; + } else { + reglist = tp->ftt_instr1 & 0xFF; + } + } else { + /* We know we have to load the pc, don't do it twice */ + reglist = tp->ftt_instr & 0x7FFF; + } + + base = (uintptr_t*) regs->sp; + while (reglist) { + if (reglist & 1) { + ret = fasttrap_fuword32((uint32_t) base, ®s->r[regnum]); + if (ret == -1) { + fasttrap_sigsegv(p, uthread, (user_addr_t) base, regs); + new_pc = regs->pc; + break; + } + base++; + } + reglist >>= 1; + regnum++; + } + + ret = fasttrap_fuword32((uint32_t) base, &new_pc); + if (ret == -1) { + fasttrap_sigsegv(p, uthread, (user_addr_t) base, regs); + new_pc = regs->pc; + break; + } + base++; + + regs->sp = (uintptr_t) base; + + set_thumb_flag(regs, new_pc); + + break; + } + + case FASTTRAP_T_CB_N_Z: + { + /* Thumb mode instruction, and not permitted in IT block, so skip the condition code check */ + int rn = tp->ftt_instr1 & 0x7; + int offset = (((tp->ftt_instr1 & 0x00F8) >> 2) | ((tp->ftt_instr1 & 0x0200) >> 3)) + 4; + int nonzero = tp->ftt_instr1 & 0x0800; + if (!nonzero != !(regs->r[rn] == 0)) { + new_pc = pc + offset; + } else { + new_pc = pc + instr_size; + } + break; + } + + case FASTTRAP_T_B_COND: + { + /* Use the condition code in the instruction and ignore the ITSTATE */ + + int code, offset; + if (tp->ftt_thumb) { + if (instr_size == 4) { + code = (tp->ftt_instr1 >> 6) & 0xF; + if (code == 14 || code == 15) { + panic("fasttrap: Emulation of invalid branch"); + } + int S = (tp->ftt_instr1 >> 10) & 1, + J1 = (tp->ftt_instr2 >> 13) & 1, + J2 = (tp->ftt_instr2 >> 11) & 1; + offset = 4 + SIGNEXTEND( + (S << 20) | (J2 << 19) | (J1 << 18) | + ((tp->ftt_instr1 & 0x003F) << 12) | + ((tp->ftt_instr2 & 0x07FF) << 1), + 21); + } else { + code = (tp->ftt_instr1 >> 8) & 0xF; + if (code == 14 || code == 15) { + panic("fasttrap: Emulation of invalid branch"); + } + offset = 4 + (SIGNEXTEND(tp->ftt_instr1 & 0xFF, 8) << 1); + } + } else { + code = ARM_CONDCODE(tp->ftt_instr); + if (code == 15) { + panic("fasttrap: Emulation of invalid branch"); + } + offset = 8 + (SIGNEXTEND(tp->ftt_instr & 0x00FFFFFF, 24) << 2); + } + + if (dtrace_arm_condition_true(code, regs->cpsr)) { + new_pc = pc + offset; + } else { + new_pc = pc + instr_size; + } + + break; + } + + case FASTTRAP_T_B_UNCOND: + { + int offset; + + /* Unconditional branches can only be taken from Thumb mode */ + /* (This is different from an ARM branch with condition code "always") */ + ASSERT(tp->ftt_thumb == 1); + + if (!dtrace_arm_condition_true(condition_code, regs->cpsr)) { + new_pc = pc + instr_size; + break; + } + + if (instr_size == 4) { + int S = (tp->ftt_instr1 >> 10) & 1, + J1 = (tp->ftt_instr2 >> 13) & 1, + J2 = (tp->ftt_instr2 >> 11) & 1; + int I1 = (J1 != S) ? 0 : 1, I2 = (J2 != S) ? 0 : 1; + offset = 4 + SIGNEXTEND( + (S << 24) | (I1 << 23) | (I2 << 22) | + ((tp->ftt_instr1 & 0x03FF) << 12) | + ((tp->ftt_instr2 & 0x07FF) << 1), + 25); + } else { + uint32_t instr1 = tp->ftt_instr1; + offset = 4 + (SIGNEXTEND(instr1 & 0x7FF, 11) << 1); + } + + new_pc = pc + offset; + + break; + } + + case FASTTRAP_T_BX_REG: + { + int reg; + + if (!dtrace_arm_condition_true(condition_code, regs->cpsr)) { + new_pc = pc + instr_size; + break; + } + + if (tp->ftt_thumb) { + reg = THUMB16_HRM(tp->ftt_instr1); + } else { + reg = ARM_RM(tp->ftt_instr); + } + new_pc = regs->r[reg]; + set_thumb_flag(regs, new_pc); + + break; + } + + case FASTTRAP_T_LDR_PC_IMMED: + case FASTTRAP_T_VLDR_PC_IMMED: + /* Handle these instructions by replacing the PC in the instruction with another + * register. They are common, so we'd like to support them, and this way we do so + * without any risk of having to simulate a segfault. + */ + + /* Fall through */ + + instr_emulate: + case FASTTRAP_T_COMMON: + { + user_addr_t addr; + uint8_t scratch[32]; + uint_t i = 0; + fasttrap_instr_t emul_instr; + emul_instr.instr32 = tp->ftt_instr; + int emul_instr_size; + + /* + * Unfortunately sometimes when we emulate the instruction and have to replace the + * PC, there is no longer a thumb mode equivalent. We end up having to run the + * modified instruction in ARM mode. We use this variable to keep track of which + * mode we should emulate in. We still use the original variable to determine + * what mode to return to. + */ + uint8_t emul_thumb = tp->ftt_thumb; + int save_reg = -1; + uint32_t save_val = 0; + + /* + * Dealing with condition codes and emulation: + * We can't just uniformly do a condition code check here because not all instructions + * have condition codes. We currently do not support an instruction by instruction trace, + * so we can assume that either: 1. We are executing a Thumb instruction, in which case + * we either are not in an IT block and should execute always, or we are last in an IT + * block. Either way, the traced instruction will run correctly, and we won't have any + * problems when we return to the original code, because we will no longer be in the IT + * block. 2. We are executing an ARM instruction, in which case we are ok as long as + * we don't attempt to change the condition code. + */ + if (tp->ftt_type == FASTTRAP_T_LDR_PC_IMMED) { + /* We know we always have a free register (the one we plan to write the + * result value to!). So we'll replace the pc with that one. + */ + int new_reg; + if (tp->ftt_thumb) { + /* Check to see if thumb or thumb2 */ + if (instr_size == 2) { + /* + * Sadness. We need to emulate this instruction in ARM mode + * because it has an 8 bit immediate offset. Instead of having + * to deal with condition codes in the ARM instruction, we'll + * just check the condition and abort if the condition is false. + */ + if (!dtrace_arm_condition_true(condition_code, regs->cpsr)) { + new_pc = pc + instr_size; + break; + } + + new_reg = (tp->ftt_instr1 >> 8) & 0x7; + regs->r[new_reg] = ALIGNADDR(regs->pc + 4, 2); + emul_thumb = 0; + emul_instr.instr32 = 0xE5900000 | (new_reg << 16) | (new_reg << 12) | ((tp->ftt_instr1 & 0xFF) << 2); + } else { + /* Thumb2. Just replace the register. */ + new_reg = (tp->ftt_instr2 >> 12) & 0xF; + regs->r[new_reg] = ALIGNADDR(regs->pc + 4, 2); + emul_instr.instr16.instr1 &= ~0x000F; + emul_instr.instr16.instr1 |= new_reg; + } + } else { + /* ARM. Just replace the register. */ + new_reg = (tp->ftt_instr >> 12) & 0xF; + regs->r[new_reg] = ALIGNADDR(regs->pc + 8,2); + emul_instr.instr32 &= ~0x000F0000; + emul_instr.instr32 |= new_reg << 16; + } + } else if (tp->ftt_type == FASTTRAP_T_VLDR_PC_IMMED) { + /* This instruction only uses one register, and if we're here, we know + * it must be the pc. So we'll just replace it with R0. + */ + save_reg = 0; + save_val = regs->r[0]; + regs->r[save_reg] = ALIGNADDR(regs->pc + (tp->ftt_thumb ? 4 : 8), 2); + if (tp->ftt_thumb) { + emul_instr.instr16.instr1 &= ~0x000F; + } else { + emul_instr.instr32 &= ~0x000F0000; + } + } + + emul_instr_size = dtrace_instr_size(emul_instr.instr32, emul_thumb); + + /* + * At this point: + * tp->ftt_thumb = thumb mode of original instruction + * emul_thumb = thumb mode for emulation + * emul_instr = instruction we are using to emulate original instruction + * emul_instr_size = size of emulating instruction + */ + + addr = uthread->t_dtrace_scratch->addr; + + if (addr == 0LL) { + fasttrap_sigtrap(p, uthread, pc); // Should be killing target proc + new_pc = pc; + break; + } + + uthread->t_dtrace_scrpc = addr; + if (emul_thumb) { + /* + * No way to do an unconditional branch in Thumb mode, shove the address + * onto the user stack and go to the next location with a pop. This can + * segfault if this push happens to cross a stack page, but that's ok, since + * we are running in userland, and the kernel knows how to handle userland + * stack expansions correctly. + * + * Layout of scratch space for Thumb mode: + * Emulated instruction + * ldr save_reg, [pc, #16] (if necessary, restore any register we clobbered) + * push { r0, r1 } + * ldr r0, [pc, #4] + * str r0, [sp, #4] + * pop { r0, pc } + * Location we should return to in original program + * Saved value of clobbered register (if necessary) + */ + + bcopy(&emul_instr, &scratch[i], emul_instr_size); i += emul_instr_size; + + if (save_reg != -1) { + uint16_t restore_inst = 0x4803; + restore_inst |= (save_reg & 0x7) << 8; + SET16(scratch+i, restore_inst); i += 2; // ldr reg, [pc , #16] + } + + SET16(scratch+i, 0xB403); i += 2; // push { r0, r1 } + SET16(scratch+i, 0x4801); i += 2; // ldr r0, [pc, #4] + SET16(scratch+i, 0x9001); i += 2; // str r0, [sp, #4] + SET16(scratch+i, 0xBD01); i += 2; // pop { r0, pc } + + if (i % 4) { + SET16(scratch+i, 0); i += 2; // padding - saved 32 bit words must be aligned + } + SET32(scratch+i, pc + instr_size + (tp->ftt_thumb ? 1 : 0)); i += 4; // Return address + if (save_reg != -1) { + SET32(scratch+i, save_val); i += 4; // saved value of clobbered register + } + + uthread->t_dtrace_astpc = addr + i; + bcopy(&emul_instr, &scratch[i], emul_instr_size); i += emul_instr_size; + SET16(scratch+i, FASTTRAP_THUMB_RET_INSTR); i += 2; + } else { + /* + * Layout of scratch space for ARM mode: + * Emulated instruction + * ldr save_reg, [pc, #12] (if necessary, restore any register we clobbered) + * ldr pc, [pc, #4] + * Location we should return to in original program + * Saved value of clobbered register (if necessary) + */ + + bcopy(&emul_instr, &scratch[i], emul_instr_size); i += emul_instr_size; + + if (save_reg != -1) { + uint32_t restore_inst = 0xE59F0004; + restore_inst |= save_reg << 12; + SET32(scratch+i, restore_inst); i += 4; // ldr reg, [pc, #12] + } + SET32(scratch+i, 0xE51FF004); i += 4; // ldr pc, [pc, #4] + + SET32(scratch+i, pc + instr_size + (tp->ftt_thumb ? 1 : 0)); i += 4; // Return address + if (save_reg != -1) { + SET32(scratch+i, save_val); i += 4; // Saved value of clobbered register + } + + uthread->t_dtrace_astpc = addr + i; + bcopy(&emul_instr, &scratch[i], emul_instr_size); i += emul_instr_size; + SET32(scratch+i, FASTTRAP_ARM_RET_INSTR); i += 4; + } + + if (patchInst(p, scratch, i, uthread->t_dtrace_scratch->write_addr) != KERN_SUCCESS) { + fasttrap_sigtrap(p, uthread, pc); + new_pc = pc; + break; + } + + if (tp->ftt_retids != NULL) { + uthread->t_dtrace_step = 1; + uthread->t_dtrace_ret = 1; + new_pc = uthread->t_dtrace_astpc + (emul_thumb ? 1 : 0); + } else { + new_pc = uthread->t_dtrace_scrpc + (emul_thumb ? 1 : 0); + } + + uthread->t_dtrace_pc = pc; + uthread->t_dtrace_npc = pc + instr_size; + uthread->t_dtrace_on = 1; + was_simulated = 0; + set_thumb_flag(regs, new_pc); + break; + } + + default: + panic("fasttrap: mishandled an instruction"); + } + +done: + /* + * APPLE NOTE: + * + * We're setting this earlier than Solaris does, to get a "correct" + * ustack() output. In the Sun code, a() -> b() -> c() -> d() is + * reported at: d, b, a. The new way gives c, b, a, which is closer + * to correct, as the return instruction has already exectued. + */ + regs->pc = new_pc; + + /* + * If there were no return probes when we first found the tracepoint, + * we should feel no obligation to honor any return probes that were + * subsequently enabled -- they'll just have to wait until the next + * time around. + */ + if (tp->ftt_retids != NULL) { + /* + * We need to wait until the results of the instruction are + * apparent before invoking any return probes. If this + * instruction was emulated we can just call + * fasttrap_return_common(); if it needs to be executed, we + * need to wait until the user thread returns to the kernel. + */ + /* + * It used to be that only common instructions were simulated. + * For performance reasons, we now simulate some instructions + * when safe and go back to userland otherwise. The was_simulated + * flag means we don't need to go back to userland. + */ + if (was_simulated) { + fasttrap_return_common(p, regs, pc, new_pc); + } else { + ASSERT(uthread->t_dtrace_ret != 0); + ASSERT(uthread->t_dtrace_pc == pc); + ASSERT(uthread->t_dtrace_scrpc != 0); + ASSERT(new_pc == uthread->t_dtrace_astpc); + } + } + + return (0); +} + +int +fasttrap_return_probe(arm_saved_state_t *regs) +{ + proc_t *p = current_proc(); + uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread()); + user_addr_t pc = uthread->t_dtrace_pc; + user_addr_t npc = uthread->t_dtrace_npc; + + uthread->t_dtrace_pc = 0; + uthread->t_dtrace_npc = 0; + uthread->t_dtrace_scrpc = 0; + uthread->t_dtrace_astpc = 0; + + /* + * Treat a child created by a call to vfork(2) as if it were its + * parent. We know that there's only one thread of control in such a + * process: this one. + */ + if (p->p_lflag & P_LINVFORK) { + proc_list_lock(); + while (p->p_lflag & P_LINVFORK) + p = p->p_pptr; + proc_list_unlock(); + } + + /* + * We set rp->r_pc to the address of the traced instruction so + * that it appears to dtrace_probe() that we're on the original + * instruction, and so that the user can't easily detect our + * complex web of lies. dtrace_return_probe() (our caller) + * will correctly set %pc after we return. + */ + regs->pc = pc; + + fasttrap_return_common(p, regs, pc, npc); + + return (0); +} + +uint64_t +fasttrap_pid_getarg(void *arg, dtrace_id_t id, void *parg, int argno, + int aframes) +{ +#pragma unused(arg, id, parg, aframes) + arm_saved_state_t* regs = find_user_regs(current_thread()); + + /* First four arguments are in registers */ + if (argno < 4) + return regs->r[argno]; + + /* Look on the stack for the rest */ + uint32_t value; + uint32_t* sp = (uint32_t*) regs->sp; + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); + value = dtrace_fuword32((user_addr_t) (sp+argno-4)); + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT | CPU_DTRACE_BADADDR); + + return value; +} + +uint64_t +fasttrap_usdt_getarg(void *arg, dtrace_id_t id, void *parg, int argno, int aframes) +{ +#pragma unused(arg, id, parg, argno, aframes) +#if 0 + return (fasttrap_anarg(ttolwp(curthread)->lwp_regs, 0, argno)); +#endif + + return 0; +} + diff --git a/bsd/dev/arm/fbt_arm.c b/bsd/dev/arm/fbt_arm.c new file mode 100644 index 000000000..c594f9c92 --- /dev/null +++ b/bsd/dev/arm/fbt_arm.c @@ -0,0 +1,681 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + */ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* #pragma ident "@(#)fbt.c 1.15 05/09/19 SMI" */ + +#ifdef KERNEL +#ifndef _KERNEL +#define _KERNEL /* Solaris vs. Darwin */ +#endif +#endif + +#define MACH__POSIX_C_SOURCE_PRIVATE 1 /* pulls in suitable savearea from + * mach/ppc/thread_status.h */ +#include <kern/thread.h> +#include <mach/thread_status.h> +#include <arm/proc_reg.h> +#include <arm/caches_internal.h> +#include <arm/thread.h> + +#include <mach-o/loader.h> +#include <mach-o/nlist.h> +#include <libkern/kernel_mach_header.h> + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/errno.h> +#include <sys/stat.h> +#include <sys/ioctl.h> +#include <sys/conf.h> +#include <sys/fcntl.h> +#include <miscfs/devfs/devfs.h> + +#include <sys/dtrace.h> +#include <sys/dtrace_impl.h> +#include <sys/fbt.h> + +#include <sys/dtrace_glue.h> + +#define DTRACE_INVOP_PUSH_LR 8 +#define DTRACE_INVOP_BL 9 +#define DTRACE_INVOP_POP_PC 10 + +#define DTRACE_INVOP_THUMB_NOP_SKIP 2 +#define DTRACE_INVOP_POP_PC_SKIP 2 +#define DTRACE_INVOP_THUMB_SET_R7_SKIP 2 +#define DTRACE_INVOP_THUMB_MOV_SP_TO_R7_SKIP 2 + +#define FBT_IS_THUMB_PUSH_LR(x) (((x) & 0x0000ff00) == 0x0000b500) +#define FBT_IS_THUMB_POP_R7(x) (((x) & 0x0000ff80) == 0x0000bc80) +#define FBT_IS_THUMB32_POP_R7LR(x,y) (((x) == 0x0000e8bd) && (((y) & 0x00004080) == 0x00004080)) +#define FBT_IS_THUMB_POP_PC(x) (((x) & 0x0000ff00) == 0x0000bd00) +#define FBT_IS_THUMB_SET_R7(x) (((x) & 0x0000ff00) == 0x0000af00) +#define FBT_IS_THUMB_MOV_SP_TO_R7(x) (((x) & 0x0000ffff) == 0x0000466f) +#define FBT_THUMB_SET_R7_OFFSET(x) (((x) & 0x000000ff) << 2) +#define FBT_IS_THUMB_LDR_PC(x) (((x) & 0x0000f800) == 0x00004800) +#define FBT_IS_THUMB32_LDR_PC(x,y) ((x) == 0x0000f8df) /* Only for positive offset PC relative loads */ +#define FBT_THUMB_STACK_REGS(x) ((x) & 0x00FF) +#define FBT_IS_THUMB_BX_REG(x) (((x) & 0x0000ff87) == 0x00004700) + +#define FBT_PATCHVAL 0xdefc +#define FBT_AFRAMES_ENTRY 8 +#define FBT_AFRAMES_RETURN 6 + +#define FBT_ENTRY "entry" +#define FBT_RETURN "return" +#define FBT_ADDR2NDX(addr) ((((uintptr_t)(addr)) >> 4) & fbt_probetab_mask) + +#define VFPSAVE_ALIGN_DTRACE 16 /* This value should come from VFPSAVE_ALIGN */ + +extern dtrace_provider_id_t fbt_id; +extern fbt_probe_t **fbt_probetab; +extern int fbt_probetab_mask; + +kern_return_t fbt_perfCallback(int, struct arm_saved_state *, __unused int, __unused int); + +static int fbt_uninstrumented_arm = 0; +static const int fbt_log_uninstrumented = 0; + +extern int dtrace_arm_condition_true(int cond, int cpsr); + + +/* Calculate the address of the ldr. (From the ARM Architecture reference) */ +/* Does not check to see if it's really a load instruction, caller must do that */ + +static uint32_t thumb_ldr_pc_address(uint32_t address) +{ + return (address & 0xFFFFFFFC) + (*(uint16_t*) address & 0xFF) * 4 + 4; +} + +static uint32_t thumb32_ldr_pc_address(uint32_t address) +{ + return (address & 0xFFFFFFFC) + (*(uint16_t*) (address+2) & 0xFFF) + 4; +} + +/* Extract the current ITSTATE from the CPSR */ +static uint32_t get_itstate(uint32_t cpsr) +{ + return + ((cpsr & 0x06000000) >> 25) | + ((cpsr & 0x0000FC00) >> 8); +} + +static void clear_itstate(uint32_t* cpsr) +{ + *cpsr &= ~0x0600FC00; +} + +int +fbt_invop(uintptr_t addr, uintptr_t * stack, uintptr_t rval) +{ + fbt_probe_t *fbt = fbt_probetab[FBT_ADDR2NDX(addr)]; + + for (; fbt != NULL; fbt = fbt->fbtp_hashnext) { + if ((uintptr_t) fbt->fbtp_patchpoint == addr) { + if (0 == CPU->cpu_dtrace_invop_underway) { + CPU->cpu_dtrace_invop_underway = 1; /* Race not possible on + * this per-cpu state */ + + struct arm_saved_state* regs = (struct arm_saved_state*) stack; + uintptr_t stack4 = *((uintptr_t*) regs->sp); + + if ((regs->cpsr & PSR_MODE_MASK) == PSR_FIQ_MODE) { + /* + * We do not handle probes firing from FIQ context. We used to + * try to undo the patch and rerun the instruction, but + * most of the time we can't do that successfully anyway. + * Instead, we just panic now so we fail fast. + */ + panic("dtrace: fbt: The probe at %08x was called from FIQ_MODE",(unsigned) addr); + } + + /* + * If we are not outside an IT block, and are not executing the last instruction of an IT block, + * then that is an instrumentation error or a code gen error. Either way, we panic. + */ + uint32_t itstate = get_itstate(regs->cpsr); + if ((itstate & 0x7) != 0) { + panic("dtrace: fbt: Instruction stream error: Middle of IT block at %08x",(unsigned) addr); + } + + if (fbt->fbtp_roffset == 0) { + /* + We need the frames to set up the backtrace, but we won't have the frame pointers + until after the instruction is emulated. So here we calculate the address of the + frame pointer from the saved instruction and put it in the stack. Yes, we end up + repeating this work again when we emulate the instruction. + + This assumes that the frame area is immediately after the saved reg storage! + */ + uint32_t offset = ((uint32_t) regs) + sizeof(struct arm_saved_state); +#if __ARM_VFP__ + /* Match the stack alignment required for arm_vfpsaved_state */ + offset &= ~(VFPSAVE_ALIGN_DTRACE - 1); + offset += VFPSAVE_ALIGN_DTRACE + sizeof(struct arm_vfpsaved_state); +#endif /* __ARM_VFP__ */ + if (FBT_IS_THUMB_SET_R7(fbt->fbtp_savedval)) + *((uint32_t*) offset) = regs->sp + FBT_THUMB_SET_R7_OFFSET(fbt->fbtp_savedval); + else + *((uint32_t*) offset) = regs->sp; + + CPU->cpu_dtrace_caller = regs->lr; + dtrace_probe(fbt->fbtp_id, regs->r[0], regs->r[1], regs->r[2], regs->r[3], stack4); + CPU->cpu_dtrace_caller = 0; + } else { + /* Check to see if we're in the middle of an IT block. */ + if (itstate != 0) { + /* + * We've already checked previously to see how far we are in the IT block. + * Here we must be getting ready to execute the last instruction. + */ + int condition_it = (itstate & 0xF0) >> 4; + + if (dtrace_arm_condition_true(condition_it, regs->cpsr) == 0) { + /* Condition wasn't true, so becomes a nop. */ + clear_itstate(®s->cpsr); + CPU->cpu_dtrace_invop_underway = 0; + return DTRACE_INVOP_NOP; + } + } + + dtrace_probe(fbt->fbtp_id, fbt->fbtp_roffset, rval, 0, 0, 0); + CPU->cpu_dtrace_caller = 0; + + /* The dtrace script may access cpsr, so make sure to clear only after probe fired. */ + clear_itstate(®s->cpsr); + } + CPU->cpu_dtrace_invop_underway = 0; + } + + /* + On other architectures, we return a DTRACE constant to let the callback function + know what was replaced. On the ARM, since the function prologue/epilogue machine code + can vary, we need the actual bytes of the instruction, so return the savedval instead. + */ + return (fbt->fbtp_savedval); + } + } + + return (0); +} + +#define IS_USER_TRAP(regs) (((regs)->cpsr & PSR_MODE_MASK) == PSR_USER_MODE) +#define T_INVALID_OPCODE EXC_BAD_INSTRUCTION +#define FBT_EXCEPTION_CODE T_INVALID_OPCODE + +kern_return_t +fbt_perfCallback( + int trapno, + struct arm_saved_state * regs, + __unused int unused1, + __unused int unused2) +{ +#pragma unused (unused1) +#pragma unused (unused2) + kern_return_t retval = KERN_FAILURE; + + if (FBT_EXCEPTION_CODE == trapno && !IS_USER_TRAP(regs)) { + boolean_t oldlevel = 0; + machine_inst_t emul = 0; + + oldlevel = ml_set_interrupts_enabled(FALSE); + + __asm__ volatile( + "Ldtrace_invop_callsite_pre_label:\n" + ".data\n" + ".private_extern _dtrace_invop_callsite_pre\n" + "_dtrace_invop_callsite_pre:\n" + " .long Ldtrace_invop_callsite_pre_label\n" + ".text\n" + ); + + emul = dtrace_invop(regs->pc, (uintptr_t*) regs, regs->r[0]); + + __asm__ volatile( + "Ldtrace_invop_callsite_post_label:\n" + ".data\n" + ".private_extern _dtrace_invop_callsite_post\n" + "_dtrace_invop_callsite_post:\n" + " .long Ldtrace_invop_callsite_post_label\n" + ".text\n" + ); + + /* + * The following emulation code does not execute properly if we are in the middle of + * an IT block. IT blocks need to be handled in the dtrace_invop function. If we do + * manage to get here and we are inside an IT block, then we missed a case somewhere + * prior to this point. + */ + uint32_t itstate = get_itstate(regs->cpsr); + if (itstate != 0) { + panic("dtrace: fbt: Not emulated: Middle of IT block at %08x",(unsigned) regs->pc); + } + + if (emul == DTRACE_INVOP_NOP) { + regs->pc += DTRACE_INVOP_THUMB_NOP_SKIP; + retval = KERN_SUCCESS; + } else if (FBT_IS_THUMB_SET_R7(emul)) { + regs->r[7] = regs->sp + FBT_THUMB_SET_R7_OFFSET(emul); + regs->pc += DTRACE_INVOP_THUMB_SET_R7_SKIP; + retval = KERN_SUCCESS; + } else if (FBT_IS_THUMB_MOV_SP_TO_R7(emul)) { + regs->r[7] = regs->sp; + regs->pc += DTRACE_INVOP_THUMB_MOV_SP_TO_R7_SKIP; + retval = KERN_SUCCESS; + } else if (FBT_IS_THUMB_POP_PC(emul)) { + uintptr_t* sp = (uintptr_t*) regs->sp; + + machine_inst_t mask = 0x0001; + int regnum = 0; + while (mask & 0x00ff) { + if (emul & mask) { + /* Pop this register */ + regs->r[regnum] = *sp++; + } + mask <<= 1; + regnum++; + } + + regs->pc = *sp++; + regs->sp = (uintptr_t) sp; + if (regs->pc & 1) { + regs->cpsr |= PSR_TF; + } else { + regs->cpsr &= ~PSR_TF; + } + + retval = KERN_SUCCESS; + } else if (FBT_IS_THUMB_BX_REG(emul)) { + regs->pc = regs->r[(emul >> 3) & 0xF]; + + if (regs->pc & 1) { + regs->cpsr |= PSR_TF; + } else { + regs->cpsr &= ~PSR_TF; + } + + retval = KERN_SUCCESS; + } else if (emul == FBT_PATCHVAL) { + /* Means we encountered an error but handled it, try same inst again */ + retval = KERN_SUCCESS; + } else { + retval = KERN_FAILURE; + } + + ml_set_interrupts_enabled(oldlevel); + } + + return retval; +} + +void +fbt_provide_probe(struct modctl *ctl, uintptr_t instrLow, uintptr_t instrHigh, char *modname, char* symbolName, machine_inst_t* symbolStart) +{ + unsigned int j; + int doenable = 0; + dtrace_id_t thisid; + + fbt_probe_t *newfbt, *retfbt, *entryfbt; + machine_inst_t *instr, *pushinstr = NULL, *limit, theInstr; + int foundPushLR, savedRegs; + + /* + * Guard against null symbols + */ + if (!symbolStart || !instrLow || !instrHigh) { + kprintf("dtrace: %s has an invalid address\n", symbolName); + return; + } + + /* + * Assume the compiler doesn't schedule instructions in the prologue. + */ + foundPushLR = 0; + savedRegs = -1; + limit = (machine_inst_t *)instrHigh; + for (j = 0, instr = symbolStart, theInstr = 0; + (j < 8) && ((uintptr_t)instr >= instrLow) && (instrHigh > (uintptr_t)(instr)); j++, instr++) + { + theInstr = *instr; + if (FBT_IS_THUMB_PUSH_LR(theInstr)) { + foundPushLR = 1; + /* Keep track of what registers we pushed. Compare this against the pop later. */ + savedRegs = FBT_THUMB_STACK_REGS(theInstr); + pushinstr = instr; + } + if (foundPushLR && (FBT_IS_THUMB_SET_R7(theInstr) || FBT_IS_THUMB_MOV_SP_TO_R7(theInstr))) + /* Guard against a random setting of r7 from sp, we make sure we found the push first */ + break; + if (FBT_IS_THUMB_BX_REG(theInstr)) /* We've gone too far, bail. */ + break; + if (FBT_IS_THUMB_POP_PC(theInstr)) /* We've gone too far, bail. */ + break; + + /* Check for 4 byte thumb instruction */ + if (dtrace_instr_size(theInstr,1) == 4) + instr++; + } + + if (!(foundPushLR && (FBT_IS_THUMB_SET_R7(theInstr) || FBT_IS_THUMB_MOV_SP_TO_R7(theInstr)))) { + return; + } + + thisid = dtrace_probe_lookup(fbt_id, modname, symbolName, FBT_ENTRY); + newfbt = kmem_zalloc(sizeof(fbt_probe_t), KM_SLEEP); + newfbt->fbtp_next = NULL; + strlcpy( (char *)&(newfbt->fbtp_name), symbolName, MAX_FBTP_NAME_CHARS ); + + if (thisid != 0) { + /* + * The dtrace_probe previously existed, so we have to hook + * the newfbt entry onto the end of the existing fbt's + * chain. + * If we find an fbt entry that was previously patched to + * fire, (as indicated by the current patched value), then + * we want to enable this newfbt on the spot. + */ + entryfbt = dtrace_probe_arg (fbt_id, thisid); + ASSERT (entryfbt != NULL); + for(; entryfbt != NULL; entryfbt = entryfbt->fbtp_next) { + if (entryfbt->fbtp_currentval == entryfbt->fbtp_patchval) + doenable++; + + if (entryfbt->fbtp_next == NULL) { + entryfbt->fbtp_next = newfbt; + newfbt->fbtp_id = entryfbt->fbtp_id; + break; + } + } + } + else { + /* + * The dtrace_probe did not previously exist, so we + * create it and hook in the newfbt. Since the probe is + * new, we obviously do not need to enable it on the spot. + */ + newfbt->fbtp_id = dtrace_probe_create(fbt_id, modname, symbolName, FBT_ENTRY, FBT_AFRAMES_ENTRY, newfbt); + doenable = 0; + } + + newfbt->fbtp_patchpoint = instr; + newfbt->fbtp_ctl = ctl; + newfbt->fbtp_loadcnt = ctl->mod_loadcnt; + newfbt->fbtp_rval = DTRACE_INVOP_PUSH_LR; + newfbt->fbtp_savedval = theInstr; + newfbt->fbtp_patchval = FBT_PATCHVAL; + newfbt->fbtp_currentval = 0; + newfbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; + fbt_probetab[FBT_ADDR2NDX(instr)] = newfbt; + + if (doenable) + fbt_enable(NULL, newfbt->fbtp_id, newfbt); + + /* + * The fbt entry chain is in place, one entry point per symbol. + * The fbt return chain can have multiple return points per + * symbol. + * Here we find the end of the fbt return chain. + */ + + doenable=0; + + thisid = dtrace_probe_lookup(fbt_id, modname, symbolName, FBT_RETURN); + + if (thisid != 0) { + /* The dtrace_probe previously existed, so we have to + * find the end of the existing fbt chain. If we find + * an fbt return that was previously patched to fire, + * (as indicated by the currrent patched value), then + * we want to enable any new fbts on the spot. + */ + retfbt = dtrace_probe_arg (fbt_id, thisid); + ASSERT(retfbt != NULL); + for (; retfbt != NULL; retfbt = retfbt->fbtp_next) { + if (retfbt->fbtp_currentval == retfbt->fbtp_patchval) + doenable++; + if(retfbt->fbtp_next == NULL) + break; + } + } + else { + doenable = 0; + retfbt = NULL; + } + + /* + * Go back to the start of the function, in case + * the compiler emitted pcrel data loads + * before R7 was adjusted. + */ + instr = pushinstr + 1; +again: + if (instr >= limit) + return; + + /* + * We (desperately) want to avoid erroneously instrumenting a + * jump table. To determine if we're looking at a true instruction + * or an inline jump table that happens to contain the same + * byte sequences, we resort to some heuristic sleeze: we + * treat this instruction as being contained within a pointer, + * and see if that pointer points to within the body of the + * function. If it does, we refuse to instrument it. + */ + if (((uintptr_t)instr & 0x3) == 0) { + machine_inst_t *ptr = *(machine_inst_t **)(void *)instr; + + if (ptr >= (machine_inst_t *)symbolStart && ptr < limit) { + /* kprintf("dtrace: fbt: Found jump table in %s, at %08x\n",symbolName,(unsigned)instr); */ + instr++; + goto again; + } + } + + /* + * OK, it's an instruction. + */ + theInstr = *instr; + + /* Walked onto the start of the next routine? If so, bail out from this function */ + if (FBT_IS_THUMB_PUSH_LR(theInstr)) { + if (!retfbt) + kprintf("dtrace: fbt: No return probe for %s, walked to next routine at %08x\n",symbolName,(unsigned)instr); + return; + } + + /* The PC relative data should be stored after the end of the function. If + * we see a PC relative load, assume the address to load from is the new end + * of the function. */ + if (FBT_IS_THUMB_LDR_PC(theInstr)) { + uint32_t newlimit = thumb_ldr_pc_address((uint32_t) instr); + if (newlimit < (uint32_t) limit) + limit = (machine_inst_t*) newlimit; + } + if ((instr+1) < limit && FBT_IS_THUMB32_LDR_PC(*instr,*(instr+1))) { + uint32_t newlimit = thumb32_ldr_pc_address((uint32_t) instr); + if (newlimit < (uint32_t) limit) + limit = (machine_inst_t*) newlimit; + } + + /* Look for the 1. pop { ..., pc } or 2. pop { ..., r7 } ... bx reg or 3. ldmia.w sp!, { ..., r7, lr } ... bx reg */ + if (!FBT_IS_THUMB_POP_PC(theInstr) && + !FBT_IS_THUMB_POP_R7(theInstr) && + !FBT_IS_THUMB32_POP_R7LR(theInstr,*(instr+1))) { + instr++; + if (dtrace_instr_size(theInstr,1) == 4) + instr++; + goto again; + } + + if (FBT_IS_THUMB_POP_PC(theInstr)) { + if (savedRegs != FBT_THUMB_STACK_REGS(theInstr)) { + /* What we're popping doesn't match what we're pushing, assume that we've + * gone too far in the function. Bail. + */ + kprintf("dtrace: fbt: No return probe for %s, popped regs don't match at %08x\n",symbolName,(unsigned)instr); + return; + } + } else { + /* Scan ahead for the bx */ + for (j = 0; (j < 4) && (instr < limit); j++, instr++) { + theInstr = *instr; + if (FBT_IS_THUMB_BX_REG(theInstr)) + break; + if (dtrace_instr_size(theInstr,1) == 4) + instr++; + } + + if (!FBT_IS_THUMB_BX_REG(theInstr)) + return; + } + + /* + * pop { ..., pc}, bx reg -- We have a winner! + */ + + newfbt = kmem_zalloc(sizeof(fbt_probe_t), KM_SLEEP); + newfbt->fbtp_next = NULL; + strlcpy( (char *)&(newfbt->fbtp_name), symbolName, MAX_FBTP_NAME_CHARS ); + + if (retfbt == NULL) { + newfbt->fbtp_id = dtrace_probe_create(fbt_id, modname, + symbolName, FBT_RETURN, FBT_AFRAMES_RETURN, newfbt); + } else { + retfbt->fbtp_next = newfbt; + newfbt->fbtp_id = retfbt->fbtp_id; + } + + retfbt = newfbt; + newfbt->fbtp_patchpoint = instr; + newfbt->fbtp_ctl = ctl; + newfbt->fbtp_loadcnt = ctl->mod_loadcnt; + + ASSERT(FBT_IS_THUMB_POP_PC(theInstr) || FBT_IS_THUMB_BX_REG(theInstr)); + newfbt->fbtp_rval = DTRACE_INVOP_POP_PC; + newfbt->fbtp_roffset = + (uintptr_t) ((uint8_t*) instr - (uint8_t *)symbolStart); + newfbt->fbtp_savedval = theInstr; + newfbt->fbtp_patchval = FBT_PATCHVAL; + newfbt->fbtp_currentval = 0; + newfbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; + fbt_probetab[FBT_ADDR2NDX(instr)] = newfbt; + + if (doenable) + fbt_enable(NULL, newfbt->fbtp_id, newfbt); + + instr++; + goto again; +} + +void +fbt_provide_module_kernel_syms(struct modctl *ctl) +{ + kernel_mach_header_t *mh; + struct load_command *cmd; + kernel_segment_command_t *orig_ts = NULL, *orig_le = NULL; + struct symtab_command *orig_st = NULL; + kernel_nlist_t *sym = NULL; + char *strings; + uintptr_t instrLow, instrHigh; + char *modname; + unsigned int i; + + mh = (kernel_mach_header_t *)(ctl->mod_address); + modname = ctl->mod_modname; + + /* + * Employees of dtrace and their families are ineligible. Void + * where prohibited. + */ + + if (mh->magic != MH_MAGIC_KERNEL) + return; + + cmd = (struct load_command *) & mh[1]; + for (i = 0; i < mh->ncmds; i++) { + if (cmd->cmd == LC_SEGMENT_KERNEL) { + kernel_segment_command_t *orig_sg = (kernel_segment_command_t *) cmd; + + if (LIT_STRNEQL(orig_sg->segname, SEG_TEXT)) + orig_ts = orig_sg; + else if (LIT_STRNEQL(orig_sg->segname, SEG_LINKEDIT)) + orig_le = orig_sg; + else if (LIT_STRNEQL(orig_sg->segname, "")) + orig_ts = orig_sg; /* kexts have a single + * unnamed segment */ + } else if (cmd->cmd == LC_SYMTAB) + orig_st = (struct symtab_command *) cmd; + + cmd = (struct load_command *) ((caddr_t) cmd + cmd->cmdsize); + } + + if ((orig_ts == NULL) || (orig_st == NULL) || (orig_le == NULL)) + return; + + sym = (kernel_nlist_t *)(orig_le->vmaddr + orig_st->symoff - orig_le->fileoff); + strings = (char *)(orig_le->vmaddr + orig_st->stroff - orig_le->fileoff); + + /* Find extent of the TEXT section */ + instrLow = (uintptr_t) orig_ts->vmaddr; + instrHigh = (uintptr_t) (orig_ts->vmaddr + orig_ts->vmsize); + + for (i = 0; i < orig_st->nsyms; i++) { + uint8_t n_type = sym[i].n_type & (N_TYPE | N_EXT); + char *name = strings + sym[i].n_un.n_strx; + + /* Check that the symbol is a global and that it has a name. */ + if (((N_SECT | N_EXT) != n_type && (N_ABS | N_EXT) != n_type)) + continue; + + if (0 == sym[i].n_un.n_strx) /* iff a null, "", name. */ + continue; + + /* Lop off omnipresent leading underscore. */ + if (*name == '_') + name += 1; + + + if (sym[i].n_sect == 1 && !(sym[i].n_desc & N_ARM_THUMB_DEF)) { + /* A function but not a Thumb function */ + fbt_uninstrumented_arm++; + if (fbt_log_uninstrumented) + kprintf("dtrace: fbt: Skipping ARM mode function %s at %08x\n",name,(unsigned)sym[i].n_value); + + continue; + } + + /* + * We're only blacklisting functions in the kernel for now. + */ + if (MOD_IS_MACH_KERNEL(ctl) && fbt_excluded(name)) + continue; + + fbt_provide_probe(ctl, instrLow, instrHigh, modname, name, (machine_inst_t*)sym[i].n_value); + } +} diff --git a/bsd/dev/arm/kern_machdep.c b/bsd/dev/arm/kern_machdep.c new file mode 100644 index 000000000..312952ac9 --- /dev/null +++ b/bsd/dev/arm/kern_machdep.c @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +/* + * Copyright (C) 1990, NeXT, Inc. + * + * File: next/kern_machdep.c + * Author: John Seamons + * + * Machine-specific kernel routines. + */ + +#include <sys/types.h> +#include <mach/machine.h> +#include <kern/cpu_number.h> +#include <machine/exec.h> + +#if __arm64__ +extern int bootarg_no64exec; /* bsd_init.c */ +static cpu_subtype_t cpu_subtype32(void); +#endif /* __arm64__ */ + +#if __arm64__ +/* + * When an arm64 CPU is executing an arm32 binary, we need to map from the + * host's 64-bit subtype to the appropriate 32-bit subtype. + */ +static cpu_subtype_t +cpu_subtype32() +{ + switch (cpu_subtype()) { + case CPU_SUBTYPE_ARM64_V8: + return CPU_SUBTYPE_ARM_V8; + default: + return 0; + } +} +#endif /* __arm64__*/ + +/********************************************************************** + * Routine: grade_binary() + * + * Function: Return a relative preference for exectypes and + * execsubtypes in fat executable files. The higher the + * grade, the higher the preference. A grade of 0 means + * not acceptable. + **********************************************************************/ +int +grade_binary(cpu_type_t exectype, cpu_subtype_t execsubtype) +{ +#if __arm64__ + cpu_subtype_t hostsubtype = (exectype & CPU_ARCH_ABI64) ? cpu_subtype() : cpu_subtype32(); +#else + cpu_subtype_t hostsubtype = cpu_subtype(); +#endif /* __arm64__ */ + + switch (exectype) { +#if __arm64__ + case CPU_TYPE_ARM64: + if (bootarg_no64exec) return 0; + + switch (hostsubtype) { + case CPU_SUBTYPE_ARM64_V8: + switch (execsubtype) { + case CPU_SUBTYPE_ARM64_V8: + return 9; + case CPU_SUBTYPE_ARM64_ALL: + return 8; + } + break; + + + break; +#else /* __arm64__ */ + + case CPU_TYPE_ARM: + switch (hostsubtype) { + /* + * For 32-bit ARMv8, try the ARMv8 slice before falling back to Swift. + */ + case CPU_SUBTYPE_ARM_V8: + switch (execsubtype) { + case CPU_SUBTYPE_ARM_V8: + return 7; + } + goto v7s; + + /* + * For Swift and later, we prefer to run a swift slice, but fall back + * to v7 as Cortex A9 errata should not apply + */ +v7s: + case CPU_SUBTYPE_ARM_V7S: + switch (execsubtype) { + case CPU_SUBTYPE_ARM_V7S: + return 6; + } + goto v7; + + /* + * For Cortex A7, accept v7k only due to differing ABI + */ + case CPU_SUBTYPE_ARM_V7K: + switch (execsubtype) { + case CPU_SUBTYPE_ARM_V7K: + return 6; + } + break; + + /* + * For Cortex A9, we prefer the A9 slice, but will run v7 albeit + * under the risk of hitting the NEON load/store errata + */ + case CPU_SUBTYPE_ARM_V7F: + switch (execsubtype) { + case CPU_SUBTYPE_ARM_V7F: + return 6; + } + goto v7; + +v7: + case CPU_SUBTYPE_ARM_V7: + switch (execsubtype) { + case CPU_SUBTYPE_ARM_V7: + return 5; + } + // fall through... + + case CPU_SUBTYPE_ARM_V6: + switch (execsubtype) { + case CPU_SUBTYPE_ARM_V6: + return 4; + } + // fall through... + + case CPU_SUBTYPE_ARM_V5TEJ: + switch (execsubtype) { + case CPU_SUBTYPE_ARM_V5TEJ: + return 3; + } + // fall through + + case CPU_SUBTYPE_ARM_V4T: + switch (execsubtype) { + case CPU_SUBTYPE_ARM_V4T: + return 2; + case CPU_SUBTYPE_ARM_ALL: + return 1; + } + break; + + case CPU_SUBTYPE_ARM_XSCALE: + switch (execsubtype) { + case CPU_SUBTYPE_ARM_XSCALE: + return 4; + case CPU_SUBTYPE_ARM_V5TEJ: + return 3; + case CPU_SUBTYPE_ARM_V4T: + return 2; + case CPU_SUBTYPE_ARM_ALL: + return 1; + } + break; + } +#endif /* __arm64__ */ + } + + return 0; +} + +boolean_t +pie_required(cpu_type_t exectype, cpu_subtype_t execsubtype) +{ + switch (exectype) { +#if __arm64__ + case CPU_TYPE_ARM64: + return TRUE; +#endif /* __arm64__ */ + + case CPU_TYPE_ARM: + switch (execsubtype) { + case CPU_SUBTYPE_ARM_V7K: + return TRUE; + } + break; + } + return FALSE; +} diff --git a/bsd/dev/arm/km.c b/bsd/dev/arm/km.c new file mode 100644 index 000000000..cd77c9fb1 --- /dev/null +++ b/bsd/dev/arm/km.c @@ -0,0 +1,403 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +/* + * Copyright (c) 1992 NeXT Computer, Inc. All rights reserved. + * + * km.m - kernel keyboard/monitor module, procedural interface. + * + * HISTORY + */ +#include <sys/param.h> +#include <sys/tty.h> + +#include <machine/cons.h> +#include <sys/conf.h> +#include <sys/systm.h> +#include <sys/uio.h> +#include <sys/fcntl.h> /* for kmopen */ +#include <sys/errno.h> +#include <sys/proc.h> /* for kmopen */ +#include <sys/msgbuf.h> +#include <sys/time.h> +#include <dev/kmreg_com.h> +#include <pexpert/pexpert.h> +#include <console/serial_protos.h> + +extern int hz; + +extern void cnputcusr(char); +extern void cnputsusr(char *, int); +extern int cngetc(void); + + +void kminit(void); +void cons_cinput(char ch); + +/* + * 'Global' variables, shared only by this file and conf.c. + */ +struct tty *km_tty[1] = { 0 }; + +/* + * this works early on, after initialize_screen() but before autoconf (and thus + * before we have a kmDevice). + */ +int disableConsoleOutput; + +/* + * 'Global' variables, shared only by this file and kmDevice.m. + */ +int initialized = 0; + +static int kmoutput(struct tty * tp); +static void kmstart(struct tty * tp); + +extern void KeyboardOpen(void); + +void +kminit(void) +{ + km_tty[0] = ttymalloc(); + km_tty[0]->t_dev = makedev(12, 0); + initialized = 1; +} + +/* + * cdevsw interface to km driver. + */ +int +kmopen(dev_t dev, int flag, __unused int devtype, proc_t pp) +{ + int unit; + struct tty *tp; + struct winsize *wp; + int ret; + + unit = minor(dev); + if (unit >= 1) + return (ENXIO); + + tp = km_tty[unit]; + + tty_lock(tp); + + tp->t_oproc = kmstart; + tp->t_param = NULL; + tp->t_dev = dev; + + if (!(tp->t_state & TS_ISOPEN)) { + tp->t_iflag = TTYDEF_IFLAG; + tp->t_oflag = TTYDEF_OFLAG; + tp->t_cflag = (CREAD | CS8 | CLOCAL); + tp->t_lflag = TTYDEF_LFLAG; + tp->t_ispeed = tp->t_ospeed = TTYDEF_SPEED; + termioschars(&tp->t_termios); + ttsetwater(tp); + } else if ((tp->t_state & TS_XCLUDE) && proc_suser(pp)) { + ret = EBUSY; + goto out; + } + + tp->t_state |= TS_CARR_ON; /* lie and say carrier exists and is + * on. */ + ret = ((*linesw[tp->t_line].l_open) (dev, tp)); + { + PE_Video video; + wp = &tp->t_winsize; + /* + * Magic numbers. These are CHARWIDTH and CHARHEIGHT from + * pexpert/i386/video_console.c + */ + wp->ws_xpixel = 8; + wp->ws_ypixel = 16; + + tty_unlock(tp); /* XXX race window */ + + if (flag & O_POPUP) + PE_initialize_console(0, kPETextScreen); + + bzero(&video, sizeof(video)); + PE_current_console(&video); + + tty_lock(tp); + + if (serialmode & SERIALMODE_OUTPUT) { + wp->ws_col = 80; + wp->ws_row = 24; + } else if (video.v_width != 0 && video.v_height != 0) { + wp->ws_col = video.v_width / wp->ws_xpixel; + wp->ws_row = video.v_height / wp->ws_ypixel; + } else { + wp->ws_col = 100; + wp->ws_row = 36; + } + } + +out: + tty_unlock(tp); + + return ret; +} + +int +kmclose(dev_t dev, int flag, __unused int mode, __unused proc_t p) +{ + int ret; + struct tty *tp = km_tty[minor(dev)]; + + tty_lock(tp); + ret = (*linesw[tp->t_line].l_close)(tp, flag); + ttyclose(tp); + tty_unlock(tp); + + return (ret); +} + +int +kmread(dev_t dev, struct uio * uio, int ioflag) +{ + int ret; + struct tty *tp = km_tty[minor(dev)]; + + tty_lock(tp); + ret = (*linesw[tp->t_line].l_read)(tp, uio, ioflag); + tty_unlock(tp); + + return (ret); +} + +int +kmwrite(dev_t dev, struct uio * uio, int ioflag) +{ + int ret; + struct tty *tp = km_tty[minor(dev)]; + + tty_lock(tp); + ret = (*linesw[tp->t_line].l_write)(tp, uio, ioflag); + tty_unlock(tp); + + return (ret); +} + +int +kmioctl(dev_t dev, u_long cmd, caddr_t data, int flag, proc_t p) +{ + int error = 0; + struct tty *tp = km_tty[minor(dev)]; + struct winsize *wp; + + tty_lock(tp); + + switch (cmd) { + case KMIOCSIZE: + wp = (struct winsize *) data; + *wp = tp->t_winsize; + break; + + case TIOCSWINSZ: + /* + * Prevent changing of console size -- this ensures that + * login doesn't revert to the termcap-defined size + */ + error = EINVAL; + break; + + /* Bodge in the CLOCAL flag as the km device is always local */ + case TIOCSETA_32: + case TIOCSETAW_32: + case TIOCSETAF_32: + { + struct termios32 *t = (struct termios32 *)data; + t->c_cflag |= CLOCAL; + /* No Break */ + } + goto fallthrough; + case TIOCSETA_64: + case TIOCSETAW_64: + case TIOCSETAF_64: + { + struct user_termios *t = (struct user_termios *)data; + t->c_cflag |= CLOCAL; + /* No Break */ + } +fallthrough: + default: + error = (*linesw[tp->t_line].l_ioctl) (tp, cmd, data, flag, p); + if (ENOTTY != error) + break; + error = ttioctl_locked(tp, cmd, data, flag, p); + break; + } + + tty_unlock(tp); + + return (error); +} + + +/* + * kmputc + * + * Output a character to the serial console driver via cnputcusr(), + * which is exported by that driver. + * + * Locks: Assumes tp in the calling tty driver code is locked on + * entry, remains locked on exit + * + * Notes: Called from kmoutput(); giving the locking output + * assumptions here, this routine should be static (and + * inlined, given there is only one call site). + */ +int +kmputc(__unused dev_t dev, char c) +{ + if(!disableConsoleOutput && initialized) { + /* OCRNL */ + if(c == '\n') + cnputcusr('\r'); + cnputcusr(c); + } + + return (0); +} + + +/* + * Callouts from linesw. + */ + +#define KM_LOWAT_DELAY ((ns_time_t)1000) + +/* + * t_oproc for this driver; called from within the line discipline + * + * Locks: Assumes tp is locked on entry, remains locked on exit + */ +static void +kmstart(struct tty *tp) +{ + if (tp->t_state & (TS_TIMEOUT | TS_BUSY | TS_TTSTOP)) + goto out; + if (tp->t_outq.c_cc == 0) + goto out; + tp->t_state |= TS_BUSY; + if (tp->t_outq.c_cc > tp->t_lowat) { + /* + * Start immediately. + */ + kmoutput(tp); + } else { + /* + * Wait a bit... + */ +#if 0 + /* FIXME */ + timeout(kmtimeout, tp, hz); +#else + kmoutput(tp); +#endif + } + return; + +out: + (*linesw[tp->t_line].l_start) (tp); + return; +} + +/* + * One-shot output retry timeout from kmoutput(); re-calls kmoutput() at + * intervals until the output queue for the tty is empty, at which point + * the timeout is not rescheduled by kmoutput() + * + * This function must take the tty_lock() around the kmoutput() call; it + * ignores the return value. + */ +static void +kmtimeout(void *arg) +{ + struct tty *tp = (struct tty *)arg; + + tty_lock(tp); + (void)kmoutput(tp); + tty_unlock(tp); +} + +/* + * kmoutput + * + * Locks: Assumes tp is locked on entry, remains locked on exit + * + * Notes: Called from kmstart() and kmtimeout(); kmtimeout() is a + * timer initiated by this routine to deal with pending + * output not yet flushed (output is flushed at a maximum + * of sizeof(buf) charatcers at a time before dropping into + * the timeout code). + */ +static int +kmoutput(struct tty * tp) +{ + unsigned char buf[80]; /* buffer; limits output per call */ + unsigned char *cp; + int cc = -1; + + /* While there is data available to be output... */ + while (tp->t_outq.c_cc > 0) { + cc = ndqb(&tp->t_outq, 0); + if (cc == 0) + break; + /* + * attempt to output as many characters as are available, + * up to the available transfer buffer size. + */ + cc = min(cc, sizeof(buf)); + /* copy the output queue contents to the buffer */ + (void) q_to_b(&tp->t_outq, buf, cc); + for (cp = buf; cp < &buf[cc]; cp++) { + /* output the buffer one charatcer at a time */ + *cp = *cp & 0x7f; + } + if (cc > 1) { + cnputsusr((char *)buf, cc); + } else { + kmputc(tp->t_dev, *buf); + } + } + /* + * XXX This is likely not necessary, as the tty output queue is not + * XXX writeable while we hold the tty_lock(). + */ + if (tp->t_outq.c_cc > 0) { + timeout(kmtimeout, tp, hz); + } + tp->t_state &= ~TS_BUSY; + /* Start the output processing for the line discipline */ + (*linesw[tp->t_line].l_start) (tp); + + return 0; +} + + +/* + * cons_cinput + * + * Driver character input from the polled mode serial console driver calls + * this routine to input a character from the serial driver into the tty + * line discipline specific input processing receiv interrupt routine, + * l_rint(). + * + * Locks: Assumes that the tty_lock() is NOT held on the tp, so a + * serial driver should NOT call this function as a result + * of being called from a function which already holds the + * lock; ECHOE will be handled at the line discipline, if + * output echo processing is going to occur. + */ +void +cons_cinput(char ch) +{ + struct tty *tp = km_tty[0]; /* XXX */ + + tty_lock(tp); + (*linesw[tp->t_line].l_rint) (ch, tp); + tty_unlock(tp); +} diff --git a/bsd/dev/arm/munge.c b/bsd/dev/arm/munge.c new file mode 100644 index 000000000..d98953ad2 --- /dev/null +++ b/bsd/dev/arm/munge.c @@ -0,0 +1,767 @@ +/* + * Coyright (c) 2005-2015 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +/* + * For arm32 ABI where 64-bit types are aligned to even registers and + * 64-bits on stack, we need to unpack registers differently. So + * we use the mungers for that. Currently this is just ARMv7k. + * + * Since arm32 has no need for munging otherwise, we don't include + * any of this for other arm32 ABIs + */ +#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4) + +#include <sys/munge.h> +#include <sys/param.h> +#include <mach/thread_status.h> +#include <libkern/libkern.h> +#include <stdint.h> + + +/* + * Userspace args are in r0-r6, then r8, then stack unless this is an + * indirect call in which case the syscall number is in r0 then args + * are in registers r1-r6, then r8, then stack. This is for mach and + * BSD style syscalls. + */ + + +#define SS_TO_STYLE(ss) ((ss->r[12] != 0) ? kDirect : kIndirect) +#define REGS_TO_STYLE(regs) (SS_TO_STYLE(((const arm_saved_state_t *)regs))) + +typedef enum { + kIndirect = 0, + kDirect +} style_t; + +#define DECLARE_AND_CAST(regs, args, ss, uu_args) const arm_saved_state_t *ss = (const arm_saved_state_t *)regs; \ + uint32_t *uu_args = (uint32_t *)args; + +/* + * We start 32 bytes after sp since 4 registers are pushed onto the stack + * in the userspace syscall handler, and the first 4 stack argumnets are moved + * into registers already + */ +#define ARG_SP_BYTE_OFFSET 32 + + +/* + * Marshal in arguments from userspace where no padding exists + */ + +static int +marshal_no_pad(const arm_saved_state_t *ss, uint32_t *args, const uint32_t word_count) +{ + int error = 0; + /* init assuming kDirect style */ + uint32_t copy_count, contiguous_reg_count = 7, contiguous_reg_start = 0; + style_t style = SS_TO_STYLE(ss); + + if (style == kIndirect) { + contiguous_reg_count--; + contiguous_reg_start++; + } + + /* r0 through r6 */ + copy_count = MIN(word_count, contiguous_reg_count); + memcpy(args, &(ss->r[contiguous_reg_start]), copy_count * sizeof(uint32_t)); + args += copy_count; + + if (word_count > copy_count) { + /* r8 */ + *args = ss->r[8]; + args++; + copy_count++; + + /* stack */ + if (word_count > copy_count) { + error = copyin(ss->sp + ARG_SP_BYTE_OFFSET, + args, (word_count - copy_count) * sizeof(uint32_t)); + if (error) + return error; + } + } + return error; +} + +/* + * Define mungers to marshal userspace data into argument structs + */ + +int +munge_w(const void *regs, void *args) +{ + return marshal_no_pad(regs, args, 1); +} + +int +munge_ww(const void *regs, void *args) +{ + return marshal_no_pad(regs, args, 2); +} + +int +munge_www(const void *regs, void *args) +{ + return marshal_no_pad(regs, args, 3); +} + +int +munge_wwww(const void *regs, void *args) +{ + return marshal_no_pad(regs, args, 4); +} + +int +munge_wwwww(const void *regs, void *args) +{ + return marshal_no_pad(regs, args, 5); +} + +int +munge_wwwwww(const void *regs, void *args) +{ + return marshal_no_pad(regs, args, 6); +} + +int +munge_wwwwwww(const void *regs, void *args) +{ + return marshal_no_pad(regs, args, 7); +} + +int +munge_wwwwwwww(const void *regs, void *args) +{ + return marshal_no_pad(regs, args, 8); +} + +int +munge_wwl(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 3); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + uu_args[0] = ss->r[1]; // w + uu_args[1] = ss->r[2]; // w + uu_args[2] = ss->r[4]; // l (longs are aligned to even registers for armv7k, so skip r3) + uu_args[3] = ss->r[5]; // + return 0; + } +} + +int +munge_wwlw(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 5); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + int error = munge_wwl(regs, args); // wwl + uu_args[4] = ss->r[6]; // w + return error; + } +} + +int +munge_wwlww(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + // the long-long here is aligned on an even register + // so there shouldn't be any padding + return marshal_no_pad(regs, args, 6); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + int error = munge_wwlw(regs, args); // wwlw + uu_args[5] = ss->r[8]; // w + return error; + } +} + +int +munge_wwlll(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 8); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + int error = munge_wwl(regs, args); // wwl + if (error) + return error; + uu_args[4] = ss->r[6]; // l + uu_args[5] = ss->r[8]; // + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // l + &(uu_args[6]), 2 * sizeof(uint32_t)); + } +} + +int +munge_wwllww(const void *regs, void *args) +{ + return munge_wwlll(regs, args); +} + +int +munge_wl(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + memcpy(args, regs, 4 * sizeof(uint32_t)); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + uu_args[0] = ss->r[1]; // w + uu_args[2] = ss->r[2]; // l + uu_args[3] = ss->r[3]; // + } + return 0; +} + +int +munge_wlw(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + memcpy(args, regs, 5 * sizeof(uint32_t)); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + uu_args[0] = ss->r[1]; // w + uu_args[2] = ss->r[2]; // l + uu_args[3] = ss->r[3]; // + uu_args[4] = ss->r[4]; // w + } + return 0; +} + +int +munge_wlww(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + memcpy(args, regs, 6 * sizeof(uint32_t)); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + uu_args[0] = ss->r[1]; // w + uu_args[2] = ss->r[2]; // l + uu_args[3] = ss->r[3]; // + uu_args[4] = ss->r[4]; // w + uu_args[5] = ss->r[5]; // w + } + return 0; +} + +int +munge_wlwwwll(const void *regs, void *args) +{ + DECLARE_AND_CAST(regs, args, ss, uu_args); + + if (REGS_TO_STYLE(regs) == kDirect) { + memcpy(args, regs, 7 * sizeof(uint32_t)); // wlwww + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // ll + uu_args + 8, 4 * sizeof(uint32_t)); + } + else { + uu_args[0] = ss->r[1]; // w + uu_args[2] = ss->r[2]; // l + uu_args[3] = ss->r[3]; // + uu_args[4] = ss->r[4]; // w + uu_args[5] = ss->r[5]; // w + uu_args[6] = ss->r[6]; // w + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // ll + uu_args + 8, 4 * sizeof(uint32_t)); + } +} + +int +munge_wlwwwllw(const void *regs, void *args) +{ + DECLARE_AND_CAST(regs, args, ss, uu_args); + + if (REGS_TO_STYLE(regs) == kDirect) { + memcpy(args, regs, 7 * sizeof(uint32_t)); // wlwww + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, + uu_args + 8, 5 * sizeof(uint32_t)); // ll + } + else { + uu_args[0] = ss->r[1]; // w + uu_args[2] = ss->r[2]; // l + uu_args[3] = ss->r[3]; // + uu_args[4] = ss->r[4]; // w + uu_args[5] = ss->r[5]; // w + uu_args[6] = ss->r[6]; // w + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // llw + uu_args + 8, 5 * sizeof(uint32_t)); + } +} + +int +munge_wlwwlwlw(const void *regs, void *args) +{ + DECLARE_AND_CAST(regs, args, ss, uu_args); + + if (REGS_TO_STYLE(regs) == kDirect) + uu_args[0] = ss->r[0]; // w + else + uu_args[0] = ss->r[1]; // w + + uu_args[2] = ss->r[2]; // l + uu_args[3] = ss->r[3]; // + uu_args[4] = ss->r[4]; // w + uu_args[5] = ss->r[5]; // w + uu_args[6] = ss->r[6]; // l + uu_args[7] = ss->r[8]; // + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // wlw + uu_args + 8, 5 * sizeof(uint32_t)); +} + +int +munge_wll(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + memcpy(args, regs, 6 * sizeof(uint32_t)); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + uu_args[0] = ss->r[1]; // w + uu_args[2] = ss->r[2]; // l + uu_args[3] = ss->r[3]; // + uu_args[4] = ss->r[4]; // l + uu_args[5] = ss->r[5]; // + } + return 0; +} + +int +munge_wlll(const void *regs, void *args) +{ + DECLARE_AND_CAST(regs, args, ss, uu_args); + + int error = munge_wll(regs, args); // wll + uu_args[6] = ss->r[6]; // l + uu_args[7] = ss->r[8]; // + return error; +} + +int +munge_wllll(const void *regs, void *args) +{ + DECLARE_AND_CAST(regs, args, ss, uu_args); + + munge_wlll(regs, args); // wlll + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // l + uu_args + 8, 2 * sizeof(uint32_t)); +} + +int +munge_wllww(const void *regs, void *args) +{ + return munge_wlll(regs, args); +} + +int +munge_wllwwll(const void *regs, void *args) +{ + DECLARE_AND_CAST(regs, args, ss, uu_args); + + int error = munge_wlll(regs, args); // wllww + if (error) + return error; + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // ll + uu_args + 8, 4 * sizeof(uint32_t)); +} + +int +munge_wwwlw(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + memcpy(args, regs, 7 * sizeof(uint32_t)); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + uu_args[0] = ss->r[1]; // w + uu_args[1] = ss->r[2]; // w + uu_args[2] = ss->r[3]; // w + uu_args[4] = ss->r[4]; // l + uu_args[5] = ss->r[5]; // + uu_args[6] = ss->r[6]; // w + } + return 0; +} + +int +munge_wwwlww(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return munge_wlll(regs, args); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + uu_args[0] = ss->r[1]; // w + uu_args[1] = ss->r[2]; // w + uu_args[2] = ss->r[3]; // w + uu_args[4] = ss->r[4]; // l + uu_args[5] = ss->r[5]; // + uu_args[6] = ss->r[6]; // w + uu_args[7] = ss->r[8]; // w + return 0; + } +} + +int +munge_wwwl(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return munge_wll(regs, args); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + uu_args[0] = ss->r[1]; // w + uu_args[1] = ss->r[2]; // w + uu_args[2] = ss->r[3]; // w + uu_args[4] = ss->r[4]; // l + uu_args[5] = ss->r[5]; // + return 0; + } +} + +int +munge_wwwwl(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 6); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + uu_args[0] = ss->r[1]; // w + uu_args[1] = ss->r[2]; // w + uu_args[2] = ss->r[3]; // w + uu_args[3] = ss->r[4]; // w + uu_args[4] = ss->r[6]; // l + uu_args[5] = ss->r[8]; // + return 0; + } +} + +int +munge_wwwwlw(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 7); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + int error = munge_wwwwl(regs, args); // wwwwl + if (error) + return error; + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // w + uu_args + 6, sizeof(uint32_t)); + } +} + +int +munge_wwwwwl(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return munge_wlll(regs, args); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + uu_args[0] = ss->r[1]; // w + uu_args[1] = ss->r[2]; // w + uu_args[2] = ss->r[3]; // w + uu_args[3] = ss->r[4]; // w + uu_args[4] = ss->r[5]; // w + uu_args[6] = ss->r[6]; // l + uu_args[7] = ss->r[8]; // + return 0; + } +} + +int +munge_wwwwwlww(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return munge_wllll(regs, args); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + int error = munge_wwwwwl(regs, args); // wwwwwl + if (error) + return error; + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // ww + uu_args + 8, 2 * sizeof(uint32_t)); + } +} + +int +munge_wwwwwllw(const void *regs, void *args) +{ + DECLARE_AND_CAST(regs, args, ss, uu_args); + + int error = munge_wwwwwl(regs, args); // wwwwwl + if (error) + return error; + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // lw + uu_args + 8, 3 * sizeof(uint32_t)); +} + +int +munge_wwwwwlll(const void *regs, void *args) +{ + DECLARE_AND_CAST(regs, args, ss, uu_args); + int error; + + if (REGS_TO_STYLE(regs) == kDirect) { + error = munge_wlll(regs, args); // wlll + if (error) + return error; + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // ll + uu_args + 8, 4 * sizeof(uint32_t)); + } + else { + error = munge_wwwwwl(regs, args); // wwwwwl + if (error) + return error; + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // ll + uu_args + 8, 4 * sizeof(uint32_t)); + } +} + +int +munge_wwwwwwl(const void *regs, void *args) +{ + munge_wwlll(regs, args); + + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 8); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + memcpy(args, &(ss->r[1]), 6 * sizeof(uint32_t)); // wwwwww + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // l + &(uu_args[6]), 2 * sizeof(uint32_t)); + } +} + +int +munge_wwwwwwlw(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 9); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + memcpy(args, &(ss->r[1]), 6 * sizeof(uint32_t)); // wwwwww + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // lw + &(uu_args[6]), 3 * sizeof(uint32_t)); + } +} + +int +munge_wwwwwwll(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 10); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + memcpy(args, &(ss->r[1]), 6 * sizeof(uint32_t)); // wwwwww + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // ll + &(uu_args[6]), 4 * sizeof(uint32_t)); + } +} + +int +munge_wsw(const void *regs, void *args) +{ + return munge_wlw(regs, args); +} + +int +munge_wws(const void *regs, void *args) +{ + return munge_wwl(regs, args); +} + +int +munge_wwws(const void *regs, void *args) +{ + return munge_wwwl(regs, args); +} + +int +munge_wwwsw(const void *regs, void *args) +{ + return munge_wwwlw(regs, args); +} + +int +munge_llllll(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 12); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + uu_args[0] = ss->r[2]; // l + uu_args[1] = ss->r[3]; // + uu_args[2] = ss->r[4]; // l + uu_args[3] = ss->r[5]; // + uu_args[4] = ss->r[6]; // l + uu_args[5] = ss->r[8]; // + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // lll + uu_args + 6, 6 * sizeof(uint32_t)); + } +} + +int +munge_ll(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 4); + else + memcpy(args, (const uint32_t*)regs + 2, 4 * sizeof(uint32_t)); + return 0; +} + +int +munge_l(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 2); + else + memcpy(args, (const uint32_t*)regs + 2, 2 * sizeof(uint32_t)); + return 0; +} + +int +munge_lw(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 3); + else + memcpy(args, (const uint32_t*)regs + 2, 3 * sizeof(uint32_t)); + return 0; +} + +int +munge_lwww(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 5); + else + memcpy(args, (const uint32_t*)regs + 2, 5 * sizeof(uint32_t)); + return 0; +} + +int +munge_lwwwwwww(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 9); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + uu_args[0] = ss->r[2]; // l + uu_args[1] = ss->r[3]; // + uu_args[2] = ss->r[4]; // w + uu_args[3] = ss->r[5]; // w + uu_args[4] = ss->r[6]; // w + uu_args[5] = ss->r[8]; // w + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // www + uu_args + 6, 3 * sizeof(uint32_t)); + } +} + +int +munge_wwlwww(const void *regs, void *args) +{ + if (REGS_TO_STYLE(regs) == kDirect) + return marshal_no_pad(regs, args, 7); + else { + DECLARE_AND_CAST(regs, args, ss, uu_args); + + uu_args[0] = ss->r[1]; // w + uu_args[1] = ss->r[2]; // w + uu_args[2] = ss->r[4]; // l + uu_args[3] = ss->r[5]; // + uu_args[4] = ss->r[6]; // w + uu_args[5] = ss->r[8]; // w + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // w + uu_args + 6, sizeof(uint32_t)); + } + +} + +int +munge_wlwwwl(const void *regs, void *args) +{ + DECLARE_AND_CAST(regs, args, ss, uu_args); + + if (REGS_TO_STYLE(regs) == kDirect) { + memcpy(args, regs, 7 * sizeof(uint32_t)); // wlwww + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // l + uu_args + 8, 2 * sizeof(uint32_t)); + } else { + uu_args[0] = ss->r[1]; // w + uu_args[2] = ss->r[2]; // l + uu_args[3] = ss->r[3]; // + uu_args[4] = ss->r[4]; // w + uu_args[5] = ss->r[5]; // w + uu_args[6] = ss->r[6]; // w + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // l + uu_args + 8, 2 * sizeof(uint32_t)); + } +} + +int +munge_wwlwwwl(const void *regs, void *args) +{ + DECLARE_AND_CAST(regs, args, ss, uu_args); + + if (REGS_TO_STYLE(regs) == kDirect) { + memcpy(args, regs, 7 * sizeof(uint32_t)); // wwlwww + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // l + uu_args + 8, 2 * sizeof(uint32_t)); + } else { + uu_args[0] = ss->r[1]; // w + uu_args[1] = ss->r[2]; // w + uu_args[2] = ss->r[4]; // l + uu_args[3] = ss->r[5]; // + uu_args[4] = ss->r[6]; // w + uu_args[5] = ss->r[8]; // w + return copyin(ss->sp + ARG_SP_BYTE_OFFSET, // wl + uu_args + 6, 4 * sizeof(uint32_t)); + } +} + +#endif // __arm__ && (__BIGGEST_ALIGNMENT__ > 4) diff --git a/bsd/dev/arm/pci_device.h b/bsd/dev/arm/pci_device.h new file mode 100644 index 000000000..32844c3ce --- /dev/null +++ b/bsd/dev/arm/pci_device.h @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +/* + * @OSF_FREE_COPYRIGHT@ + * + */ +/* + * HISTORY + * + * Revision 1.2 1998/09/30 21:20:44 wsanchez + * Merged in IntelMerge1 (mburg: Intel support) + * + * Revision 1.1.2.1 1998/09/30 18:18:50 mburg + * Changes for Intel port + * + * Revision 1.1.1.1 1998/03/07 02:25:45 wsanchez + * Import of OSF Mach kernel (~mburg) + * + * Revision 1.1.6.2 1995/12/15 10:52:14 bernadat + * Split dev and vendor ids. + * [95/11/15 bernadat] + * + * Revision 1.1.6.1 1995/02/23 17:22:27 alanl + * Taken from DIPC2_SHARED + * [1995/01/03 19:09:31 alanl] + * + * Revision 1.1.2.1 1994/10/11 18:24:42 rwd + * Created. + * [1994/10/11 18:15:31 rwd] + * + * $EndLog$ + */ +/* + * Taken from + * + * Copyright (c) 1994 Wolfgang Stanglmeier, Koeln, Germany + * <wolf@dentaro.GUN.de> + */ + +#ifndef __PCI_DEVICE_H__ +#define __PCI_DEVICE_H__ + +/*------------------------------------------------------------ + * + * Per driver structure. + * + *------------------------------------------------------------ +*/ + +typedef unsigned short pci_vendor_id_t; +typedef unsigned short pci_dev_id_t; + +typedef union { + unsigned long cfg1; + struct { + unsigned char enable; + unsigned char forward; + unsigned short port; + } cfg2; + } pcici_t; + +struct pci_driver { + int (*probe )(pcici_t pci_ident); /* test whether device + is present */ + int (*attach)(pcici_t pci_ident); /* setup driver for a + device */ + pci_vendor_id_t vendor_id; /* vendor pci id */ + pci_dev_id_t device_id; /* device pci id */ + char *name; /* device name */ + char *vendor; /* device long name */ + void (*intr)(int); /* interupt handler */ +}; + +/*----------------------------------------------------------- + * + * Per device structure. + * + * It is initialized by the config utility and should live in + * "ioconf.c". At the moment there is only one field. + * + * This is a first attempt to include the pci bus to 386bsd. + * So this structure may grow .. + * + *----------------------------------------------------------- +*/ + +struct pci_device { + struct pci_driver * pd_driver; +}; + +/*----------------------------------------------------------- + * + * This functions may be used by drivers to map devices + * to virtual and physical addresses. The va and pa + * addresses are "in/out" parameters. If they are 0 + * on entry, the mapping function assigns an address. + * + *----------------------------------------------------------- +*/ + +int pci_map_mem(pcici_t tag, + unsigned long entry, + vm_offset_t *va, + vm_offset_t *pa); +#endif /*__PCI_DEVICE_H__*/ diff --git a/bsd/dev/arm/pio.h b/bsd/dev/arm/pio.h new file mode 100644 index 000000000..fd9c1ecca --- /dev/null +++ b/bsd/dev/arm/pio.h @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2000-2007 AppleInc. All rights reserved. + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * HISTORY + * + * Revision 1.2 1998/09/30 21:20:45 wsanchez + * Merged in IntelMerge1 (mburg: Intel support) + * + * Revision 1.1.2.1 1998/09/30 18:18:50 mburg + * Changes for Intel port + * + * Revision 1.1.1.1 1998/03/07 02:25:38 wsanchez + * Import of OSF Mach kernel (~mburg) + * + * Revision 1.1.8.2 1996/07/31 09:46:36 paire + * Merged with nmk20b7_shared (1.1.11.2 -> 1.1.11.1) + * [96/06/10 paire] + * + * Revision 1.1.11.2 1996/06/13 12:38:25 bernadat + * Do not use inline macros when MACH_ASSERT is configured. + * [96/05/24 bernadat] + * + * Revision 1.1.11.1 1996/05/14 13:50:23 paire + * Added new linl and loutl __inline__. + * Added conditional compilation for [l]{in|oub}[bwl]() __inline__. + * [95/11/24 paire] + * + * Revision 1.1.8.1 1994/09/23 02:00:28 ezf + * change marker to not FREE + * [1994/09/22 21:25:52 ezf] + * + * Revision 1.1.4.5 1993/08/09 19:40:41 dswartz + * Add ANSI prototypes - CR#9523 + * [1993/08/06 17:45:57 dswartz] + * + * Revision 1.1.4.4 1993/06/11 15:17:37 jeffc + * CR9176 - ANSI C violations: inb/outb macros must be changed from + * ({ ... }) to inline functions, with proper type definitions. Callers + * must pass proper types to these functions: 386 I/O port addresses + * are unsigned shorts (not pointers). + * [1993/06/10 14:26:10 jeffc] + * + * Revision 1.1.4.3 1993/06/07 22:09:28 jeffc + * CR9176 - ANSI C violations: trailing tokens on CPP + * directives, extra semicolons after decl_ ..., asm keywords + * [1993/06/07 19:00:26 jeffc] + * + * Revision 1.1.4.2 1993/06/04 15:28:45 jeffc + * CR9176 - ANSI problems - + * Added casts to get macros to take caddr_t as an I/O space address. + * [1993/06/04 13:45:55 jeffc] + * + * Revision 1.1 1992/09/30 02:25:51 robert + * Initial revision + * + * $EndLog$ + */ +/* CMU_HIST */ +/* + * Revision 2.5 91/05/14 16:14:20 mrt + * Correcting copyright + * + * Revision 2.4 91/02/05 17:13:56 mrt + * Changed to new Mach copyright + * [91/02/01 17:37:08 mrt] + * + * Revision 2.3 90/12/20 16:36:37 jeffreyh + * changes for __STDC__ + * [90/12/07 jeffreyh] + * + * Revision 2.2 90/11/26 14:48:41 rvb + * Pulled from 2.5 + * [90/11/22 10:09:38 rvb] + * + * [90/08/14 mg32] + * + * Now we know how types are factor in. + * Cleaned up a bunch: eliminated ({ for output and flushed unused + * output variables. + * [90/08/14 rvb] + * + * This is how its done in gcc: + * Created. + * [90/03/26 rvb] + * + */ +/* CMU_ENDHIST */ +/* + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ +/* + */ +#ifndef ARM_PIO_H +#define ARM_PIO_H + +typedef unsigned short i386_ioport_t; + +/* read a longword */ +extern unsigned long inl( + i386_ioport_t port); +/* read a shortword */ +extern unsigned short inw( + i386_ioport_t port); +/* read a byte */ +extern unsigned char inb( + i386_ioport_t port); +/* write a longword */ +extern void outl( + i386_ioport_t port, + unsigned long datum); +/* write a word */ +extern void outw( + i386_ioport_t port, + unsigned short datum); +/* write a longword */ +extern void outb( + i386_ioport_t port, + unsigned char datum); + +/* input an array of longwords */ +extern void linl( + i386_ioport_t port, + int * data, + int count); +/* output an array of longwords */ +extern void loutl( + i386_ioport_t port, + int * data, + int count); + +/* input an array of words */ +extern void linw( + i386_ioport_t port, + int * data, + int count); +/* output an array of words */ +extern void loutw( + i386_ioport_t port, + int * data, + int count); + +/* input an array of bytes */ +extern void linb( + i386_ioport_t port, + char * data, + int count); +/* output an array of bytes */ +extern void loutb( + i386_ioport_t port, + char * data, + int count); + +extern __inline__ unsigned long inl( + i386_ioport_t port) +{ + unsigned long datum; + __asm__ volatile("inl %1, %0" : "=a" (datum) : "d" (port)); + return(datum); +} + +extern __inline__ unsigned short inw( + i386_ioport_t port) +{ + unsigned short datum; + __asm__ volatile(".byte 0x66; inl %1, %0" : "=a" (datum) : "d" (port)); + return(datum); +} + +extern __inline__ unsigned char inb( + i386_ioport_t port) +{ + unsigned char datum; + __asm__ volatile("inb %1, %0" : "=a" (datum) : "d" (port)); + return(datum); +} + +extern __inline__ void outl( + i386_ioport_t port, + unsigned long datum) +{ + __asm__ volatile("outl %0, %1" : : "a" (datum), "d" (port)); +} + +extern __inline__ void outw( + i386_ioport_t port, + unsigned short datum) +{ + __asm__ volatile(".byte 0x66; outl %0, %1" : : "a" (datum), "d" (port)); +} + +extern __inline__ void outb( + i386_ioport_t port, + unsigned char datum) +{ + __asm__ volatile("outb %0, %1" : : "a" (datum), "d" (port)); +} + +#endif /* ARM_PIO_H */ diff --git a/bsd/dev/arm/sdt_arm.c b/bsd/dev/arm/sdt_arm.c new file mode 100644 index 000000000..b8db51b52 --- /dev/null +++ b/bsd/dev/arm/sdt_arm.c @@ -0,0 +1,166 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2006 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* #pragma ident "@(#)sdt.c 1.6 06/03/24 SMI" */ + +#ifdef KERNEL +#ifndef _KERNEL +#define _KERNEL /* Solaris vs. Darwin */ +#endif +#endif + +#define MACH__POSIX_C_SOURCE_PRIVATE 1 /* pulls in suitable savearea from mach/ppc/thread_status.h */ +#include <kern/cpu_data.h> +#include <kern/thread.h> +#include <mach/thread_status.h> +#include <mach/vm_param.h> + +#include <sys/dtrace.h> +#include <sys/dtrace_impl.h> + +#include <sys/dtrace_glue.h> + +#include <sys/sdt_impl.h> + +extern sdt_probe_t **sdt_probetab; + +int +sdt_invop(__unused uintptr_t addr, __unused uintptr_t *stack, __unused uintptr_t eax) +{ +#pragma unused(eax) + sdt_probe_t *sdt = sdt_probetab[SDT_ADDR2NDX(addr)]; + + for (; sdt != NULL; sdt = sdt->sdp_hashnext) { + if ((uintptr_t) sdt->sdp_patchpoint == addr) { + struct arm_saved_state* regs = (struct arm_saved_state*) stack; + uintptr_t stack4 = *((uintptr_t*) regs->sp); + + dtrace_probe(sdt->sdp_id, regs->r[0], regs->r[1], regs->r[2], regs->r[3], stack4); + + return (DTRACE_INVOP_NOP); + } + } + + return (0); +} + +struct frame { + struct frame *backchain; + uintptr_t retaddr; +}; + +/*ARGSUSED*/ +uint64_t +sdt_getarg(void *arg, dtrace_id_t id, void *parg, int argno, int aframes) +{ +#pragma unused(arg,id,parg) /* __APPLE__ */ + uint64_t val = 0; + struct frame *fp = (struct frame *)__builtin_frame_address(0); + uintptr_t *stack; + uintptr_t pc; + int i; + + /* + * On ARM, up to four args are passed via registers; r0,r1,r2,r3 + * So coming into this function, arg >= 4 should be on the stack. + * e.g. arg==5 refers to the 6th arg passed to the probed function. + */ + int inreg = 4; + + for (i = 1; i <= aframes; i++) { + fp = fp->backchain; + pc = fp->retaddr; + + if (dtrace_invop_callsite_pre != NULL + && pc > (uintptr_t)dtrace_invop_callsite_pre + && pc <= (uintptr_t)dtrace_invop_callsite_post) { + + /* + * When we pass through the invalid op handler, + * we expect to find the save area structure, + * pushed on the stack where we took the trap. + * If the argument we seek is passed in a register, then + * we can load it directly from this saved area. + * If the argument we seek is passed on the stack, then + * we increment the frame pointer further, to find the + * pushed args + */ + + /* fp points to the dtrace_invop activation */ + fp = fp->backchain; /* to the fbt_perfCallback activation */ + fp = fp->backchain; /* to the sleh_undef activation */ + +#if __BIGGEST_ALIGNMENT__ > 4 + /** + * rdar://problem/24228656: On armv7k, the stack is realigned in sleh_undef2 to + * be 16-bytes aligned and the old value is pushed to + * the stack, so we retrieve it from here + */ + arm_saved_state_t *saved_state = (arm_saved_state_t *)(uintptr_t*)*((uintptr_t *)&fp[1]); +#else + arm_saved_state_t *saved_state = (arm_saved_state_t *)((uintptr_t *)&fp[1]); +#endif + if (argno <= inreg) { + /* For clarity only... should not get here */ + stack = (uintptr_t *)&saved_state->r[0]; + } else { + fp = (struct frame *)(saved_state->sp); + stack = (uintptr_t *)&fp[0]; /* Find marshalled arguments */ + argno -= inreg; + } + goto load; + } + } + + /* + * We know that we did not come through a trap to get into + * dtrace_probe() -- We arrive here when the provider has + * called dtrace_probe() directly. + * The probe ID is the first argument to dtrace_probe(). + * We must advance beyond that to get the argX. + */ + argno++; /* Advance past probeID */ + + if (argno <= inreg) { + /* + * This shouldn't happen. If the argument is passed in a + * register then it should have been, well, passed in a + * register... + */ + DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); + return (0); + } + + argno -= (inreg + 1); + stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */ + +load: + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); + /* dtrace_probe arguments arg0 .. arg4 are 64bits wide */ + val = (uint64_t)(*(((uintptr_t *)stack) + argno)); + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); + return (val); + +} diff --git a/bsd/dev/arm/stubs.c b/bsd/dev/arm/stubs.c new file mode 100644 index 000000000..644dae630 --- /dev/null +++ b/bsd/dev/arm/stubs.c @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +/* + * Copyright (c) 1997 by Apple Computer, Inc., all rights reserved + * Copyright (c) 1993 NeXT Computer, Inc. + * + */ + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/ioctl.h> +#include <sys/tty.h> +#include <sys/conf.h> +#include <sys/kauth.h> +#include <sys/ucred.h> +#include <sys/proc_internal.h> +#include <sys/user.h> +#include <kern/task.h> +#include <kern/thread.h> +#include <vm/vm_map.h> + +/* + * copy a null terminated string from the kernel address space into the user + * address space. - if the user is denied write access, return EFAULT. - if + * the end of string isn't found before maxlen bytes are copied, return + * ENAMETOOLONG, indicating an incomplete copy. - otherwise, return 0, + * indicating success. the number of bytes copied is always returned in + * lencopied. + */ +int +copyoutstr(const void *from, user_addr_t to, size_t maxlen, size_t * lencopied) +{ + size_t slen; + size_t len; + int error = 0; + + slen = strlen(from) + 1; + if (slen > maxlen) + error = ENAMETOOLONG; + + len = min(maxlen, slen); + if (copyout(from, to, len)) + error = EFAULT; + *lencopied = len; + + return error; +} + + +/* + * copy a null terminated string from one point to another in the kernel + * address space. - no access checks are performed. - if the end of string + * isn't found before maxlen bytes are copied, return ENAMETOOLONG, + * indicating an incomplete copy. - otherwise, return 0, indicating success. + * the number of bytes copied is always returned in lencopied. + */ +/* from ppc/fault_copy.c -Titan1T4 VERSION */ +int +copystr(const void *vfrom, void *vto, size_t maxlen, size_t * lencopied) +{ + size_t l; + char const *from = (char const *) vfrom; + char *to = (char *) vto; + + for (l = 0; l < maxlen; l++) { + if ((*to++ = *from++) == '\0') { + if (lencopied) + *lencopied = l + 1; + return 0; + } + } + if (lencopied) + *lencopied = maxlen; + return ENAMETOOLONG; +} + +int +copywithin(void *src, void *dst, size_t count) +{ + bcopy(src, dst, count); + return 0; +} diff --git a/bsd/dev/arm/sysctl.c b/bsd/dev/arm/sysctl.c new file mode 100644 index 000000000..a1ee66f16 --- /dev/null +++ b/bsd/dev/arm/sysctl.c @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2003-2007 Apple Inc. All rights reserved. + */ +#include <sys/param.h> +#include <sys/kernel.h> +#include <sys/sysctl.h> + +#include <machine/machine_routines.h> + +extern int trap_on_alignment_fault; +extern uint64_t wake_abstime; + +static +SYSCTL_INT(_machdep, OID_AUTO, alignmenttrap, + CTLFLAG_RW, &trap_on_alignment_fault, 0, + "trap on alignment faults (number of alignment faults per trap)"); + +static +SYSCTL_QUAD(_machdep, OID_AUTO, wake_abstime, + CTLFLAG_RD | CTLFLAG_KERN, &wake_abstime, + "Absolute Time at the last wakeup"); + +static int +sysctl_time_since_reset SYSCTL_HANDLER_ARGS +{ +#pragma unused(arg1, arg2, oidp) + int error = 0; + uint64_t return_value = 0; + + return_value = ml_get_time_since_reset(); + + SYSCTL_OUT(req, &return_value, sizeof(return_value)); + + return error; +} + +SYSCTL_PROC(_machdep, OID_AUTO, time_since_reset, + CTLFLAG_RD | CTLTYPE_QUAD | CTLFLAG_LOCKED, + 0, 0, sysctl_time_since_reset, "I", + "Continuous time since last SOC boot/wake started"); + +static int +sysctl_wake_conttime SYSCTL_HANDLER_ARGS +{ +#pragma unused(arg1, arg2, oidp) + int error = 0; + uint64_t return_value = 0; + + return_value = ml_get_conttime_wake_time(); + + SYSCTL_OUT(req, &return_value, sizeof(return_value)); + + return error; +} + +SYSCTL_PROC(_machdep, OID_AUTO, wake_conttime, + CTLFLAG_RD | CTLTYPE_QUAD | CTLFLAG_LOCKED, + 0, 0, sysctl_wake_conttime, "I", + "Continuous Time at the last wakeup"); + diff --git a/bsd/dev/arm/systemcalls.c b/bsd/dev/arm/systemcalls.c new file mode 100644 index 000000000..df9f22d09 --- /dev/null +++ b/bsd/dev/arm/systemcalls.c @@ -0,0 +1,651 @@ +/* + * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + */ + +#include <kern/task.h> +#include <kern/thread.h> +#include <kern/assert.h> +#include <kern/clock.h> +#include <kern/locks.h> +#include <kern/sched_prim.h> +#include <mach/machine/thread_status.h> +#include <mach/thread_act.h> +#include <arm/thread.h> +#include <arm/proc_reg.h> +#include <pexpert/pexpert.h> + +#include <sys/kernel.h> +#include <sys/vm.h> +#include <sys/proc_internal.h> +#include <sys/syscall.h> +#include <sys/systm.h> +#include <sys/user.h> +#include <sys/errno.h> +#include <sys/kdebug.h> +#include <sys/sysent.h> +#include <sys/sysproto.h> +#include <sys/kauth.h> + +#include <security/audit/audit.h> + +#if CONFIG_DTRACE +extern int32_t dtrace_systrace_syscall(struct proc *, void *, int *); +extern void dtrace_systrace_syscall_return(unsigned short, int, int *); +#endif /* CONFIG_DTRACE */ + +extern void +unix_syscall(struct arm_saved_state * regs, thread_t thread_act, + struct uthread * uthread, struct proc * proc); + +static int arm_get_syscall_args(uthread_t, struct arm_saved_state *, struct sysent *); +static int arm_get_u32_syscall_args(uthread_t, arm_saved_state32_t *, struct sysent *); +static void arm_prepare_u32_syscall_return(struct sysent *, arm_saved_state32_t *, uthread_t, int); +static void arm_prepare_syscall_return(struct sysent *, struct arm_saved_state *, uthread_t, int); +static int arm_get_syscall_number(struct arm_saved_state *); +static void arm_trace_unix_syscall(int, struct arm_saved_state *); +static void arm_clear_syscall_error(struct arm_saved_state *); +#define save_r0 r[0] +#define save_r1 r[1] +#define save_r2 r[2] +#define save_r3 r[3] +#define save_r4 r[4] +#define save_r5 r[5] +#define save_r6 r[6] +#define save_r7 r[7] +#define save_r8 r[8] +#define save_r9 r[9] +#define save_r10 r[10] +#define save_r11 r[11] +#define save_r12 r[12] +#define save_r13 r[13] + +#if COUNT_SYSCALLS +__XNU_PRIVATE_EXTERN int do_count_syscalls = 1; +__XNU_PRIVATE_EXTERN int syscalls_log[SYS_MAXSYSCALL]; +#endif + +#define code_is_kdebug_trace(code) (((code) == SYS_kdebug_trace) || \ + ((code) == SYS_kdebug_trace64) || \ + ((code) == SYS_kdebug_trace_string)) + +/* + * Function: unix_syscall + * + * Inputs: regs - pointer to Process Control Block + * + * Outputs: none + */ +#ifdef __arm__ +__attribute__((noreturn)) +#endif +void +unix_syscall( + struct arm_saved_state * state, + __unused thread_t thread_act, + struct uthread * uthread, + struct proc * proc) +{ + struct sysent *callp; + int error; + unsigned short code; + pid_t pid; + +#if defined(__arm__) + assert(is_saved_state32(state)); +#endif + + uthread_reset_proc_refcount(uthread); + + code = arm_get_syscall_number(state); + +#define unix_syscall_kprintf(x...) /* kprintf("unix_syscall: " x) */ + +#if (KDEBUG_LEVEL >= KDEBUG_LEVEL_IST) + if (kdebug_enable && !code_is_kdebug_trace(code)) { + arm_trace_unix_syscall(code, state); + } +#endif + + if ((uthread->uu_flag & UT_VFORK)) + proc = current_proc(); + + callp = (code >= nsysent) ? &sysent[SYS_invalid] : &sysent[code]; + + /* + * sy_narg is inaccurate on ARM if a 64 bit parameter is specified. Since user_addr_t + * is currently a 32 bit type, this is really a long word count. See rdar://problem/6104668. + */ + if (callp->sy_narg != 0) { + if (arm_get_syscall_args(uthread, state, callp) != 0) { + /* Too many arguments, or something failed */ + unix_syscall_kprintf("arm_get_syscall_args failed.\n"); + callp = &sysent[SYS_invalid]; + } + } + + uthread->uu_flag |= UT_NOTCANCELPT; + uthread->syscall_code = code; + + uthread->uu_rval[0] = 0; + + /* + * r4 is volatile, if we set it to regs->save_r4 here the child + * will have parents r4 after execve + */ + uthread->uu_rval[1] = 0; + + error = 0; + + /* + * ARM runtime will call cerror if the carry bit is set after a + * system call, so clear it here for the common case of success. + */ + arm_clear_syscall_error(state); + +#if COUNT_SYSCALLS + if (do_count_syscalls > 0) { + syscalls_log[code]++; + } +#endif + pid = proc_pid(proc); + +#ifdef JOE_DEBUG + uthread->uu_iocount = 0; + uthread->uu_vpindex = 0; +#endif + unix_syscall_kprintf("code %d (pid %d - %s, tid %lld)\n", code, + pid, proc->p_comm, thread_tid(current_thread())); + + AUDIT_SYSCALL_ENTER(code, proc, uthread); + error = (*(callp->sy_call)) (proc, &uthread->uu_arg[0], &(uthread->uu_rval[0])); + AUDIT_SYSCALL_EXIT(code, proc, uthread, error); + + unix_syscall_kprintf("code %d, error %d, results %x, %x (pid %d - %s, tid %lld)\n", code, error, + uthread->uu_rval[0], uthread->uu_rval[1], + pid, get_bsdtask_info(current_task()) ? proc->p_comm : "unknown" , thread_tid(current_thread())); + +#ifdef JOE_DEBUG + if (uthread->uu_iocount) { + printf("system call returned with uu_iocount != 0"); + } +#endif +#if CONFIG_DTRACE + uthread->t_dtrace_errno = error; +#endif /* CONFIG_DTRACE */ +#if DEBUG || DEVELOPMENT + kern_allocation_name_t + prior __assert_only = thread_set_allocation_name(NULL); + assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior)); +#endif /* DEBUG || DEVELOPMENT */ + + arm_prepare_syscall_return(callp, state, uthread, error); + + uthread->uu_flag &= ~UT_NOTCANCELPT; + + if (uthread->uu_lowpri_window) { + /* + * task is marked as a low priority I/O type + * and the I/O we issued while in this system call + * collided with normal I/O operations... we'll + * delay in order to mitigate the impact of this + * task on the normal operation of the system + */ + throttle_lowpri_io(1); + } +#if (KDEBUG_LEVEL >= KDEBUG_LEVEL_IST) + if (kdebug_enable && !code_is_kdebug_trace(code)) { + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_END, + error, uthread->uu_rval[0], uthread->uu_rval[1], pid, 0); + } +#endif + +#if PROC_REF_DEBUG + if (__improbable(uthread_get_proc_refcount(uthread) != 0)) { + panic("system call returned with uu_proc_refcount != 0"); + } +#endif + +#ifdef __arm__ + thread_exception_return(); +#endif +} + +void +unix_syscall_return(int error) +{ + thread_t thread_act; + struct uthread *uthread; + struct proc *proc; + struct arm_saved_state *regs; + unsigned short code; + struct sysent *callp; + +#define unix_syscall_return_kprintf(x...) /* kprintf("unix_syscall_retur + * n: " x) */ + + thread_act = current_thread(); + proc = current_proc(); + uthread = get_bsdthread_info(thread_act); + + regs = find_user_regs(thread_act); + code = uthread->syscall_code; + callp = (code >= nsysent) ? &sysent[SYS_invalid] : &sysent[code]; + +#if CONFIG_DTRACE + if (callp->sy_call == dtrace_systrace_syscall) + dtrace_systrace_syscall_return( code, error, uthread->uu_rval ); +#endif /* CONFIG_DTRACE */ +#if DEBUG || DEVELOPMENT + kern_allocation_name_t + prior __assert_only = thread_set_allocation_name(NULL); + assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior)); +#endif /* DEBUG || DEVELOPMENT */ + + AUDIT_SYSCALL_EXIT(code, proc, uthread, error); + + /* + * Get index into sysent table + */ + arm_prepare_syscall_return(callp, regs, uthread, error); + + uthread->uu_flag &= ~UT_NOTCANCELPT; + + if (uthread->uu_lowpri_window) { + /* + * task is marked as a low priority I/O type + * and the I/O we issued while in this system call + * collided with normal I/O operations... we'll + * delay in order to mitigate the impact of this + * task on the normal operation of the system + */ + throttle_lowpri_io(1); + } +#if (KDEBUG_LEVEL >= KDEBUG_LEVEL_IST) + if (kdebug_enable && !code_is_kdebug_trace(code)) { + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_END, + error, uthread->uu_rval[0], uthread->uu_rval[1], proc->p_pid, 0); + } +#endif + + thread_exception_return(); + /* NOTREACHED */ +} + +static void +arm_prepare_u32_syscall_return(struct sysent *callp, arm_saved_state32_t *regs, uthread_t uthread, int error) +{ + if (error == ERESTART) { + regs->pc -= 4; + } else if (error != EJUSTRETURN) { + if (error) { + regs->save_r0 = error; + regs->save_r1 = 0; + /* set the carry bit to execute cerror routine */ + regs->cpsr |= PSR_CF; + unix_syscall_return_kprintf("error: setting carry to trigger cerror call\n"); + } else { /* (not error) */ + switch (callp->sy_return_type) { + case _SYSCALL_RET_INT_T: + case _SYSCALL_RET_UINT_T: + case _SYSCALL_RET_OFF_T: + case _SYSCALL_RET_ADDR_T: + case _SYSCALL_RET_SIZE_T: + case _SYSCALL_RET_SSIZE_T: + case _SYSCALL_RET_UINT64_T: + regs->save_r0 = uthread->uu_rval[0]; + regs->save_r1 = uthread->uu_rval[1]; + break; + case _SYSCALL_RET_NONE: + regs->save_r0 = 0; + regs->save_r1 = 0; + break; + default: + panic("unix_syscall: unknown return type"); + break; + } + } + } + /* else (error == EJUSTRETURN) { nothing } */ + +} + +static void +arm_trace_u32_unix_syscall(int code, arm_saved_state32_t *regs) +{ + boolean_t indirect = (regs->save_r12 == 0); + if (indirect) + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START, + regs->save_r1, regs->save_r2, regs->save_r3, regs->save_r4, 0); + else + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START, + regs->save_r0, regs->save_r1, regs->save_r2, regs->save_r3, 0); +} + +static void +arm_clear_u32_syscall_error(arm_saved_state32_t *regs) +{ + regs->cpsr &= ~PSR_CF; +} + +#if defined(__arm__) + +static int +arm_get_syscall_args(uthread_t uthread, struct arm_saved_state *state, struct sysent *callp) +{ + assert(is_saved_state32(state)); + return arm_get_u32_syscall_args(uthread, saved_state32(state), callp); +} + +#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4) +/* + * For armv7k, the alignment constraints of the ABI mean we don't know how the userspace + * arguments are arranged without knowing the the prototype of the syscall. So we use mungers + * to marshal the userspace data into the uu_arg. This also means we need the same convention + * as mach syscalls. That means we use r8 to pass arguments in the BSD case as well. + */ +static int +arm_get_u32_syscall_args(uthread_t uthread, arm_saved_state32_t *regs, struct sysent *callp) +{ + sy_munge_t *munger; + + /* This check is probably not very useful since these both come from build-time */ + if (callp->sy_arg_bytes > sizeof(uthread->uu_arg)) + return -1; + + /* get the munger and use it to marshal in the data from userspace */ + munger = callp->sy_arg_munge32; + if (munger == NULL || (callp->sy_arg_bytes == 0)) + return 0; + + return munger(regs, uthread->uu_arg); +} +#else +/* + * For an AArch32 kernel, where we know that we have only AArch32 userland, + * we do not do any munging (which is a little confusing, as it is a contrast + * to the i386 kernel, where, like the x86_64 kernel, we always munge + * arguments from a 32-bit userland out to 64-bit. + */ +static int +arm_get_u32_syscall_args(uthread_t uthread, arm_saved_state32_t *regs, struct sysent *callp) +{ + int regparams; + int flavor = (regs->save_r12 == 0 ? 1 : 0); + + regparams = (7 - flavor); /* Indirect value consumes a register */ + + assert((unsigned) callp->sy_arg_bytes <= sizeof (uthread->uu_arg)); + + if (callp->sy_arg_bytes <= (sizeof(uint32_t) * regparams)) { + /* + * Seven arguments or less are passed in registers. + */ + memcpy(&uthread->uu_arg[0], ®s->r[flavor], callp->sy_arg_bytes); + } else if (callp->sy_arg_bytes <= sizeof(uthread->uu_arg)) { + /* + * In this case, we composite - take the first args from registers, + * the remainder from the stack (offset by the 7 regs therein). + */ + unix_syscall_kprintf("%s: spillover...\n", __FUNCTION__); + memcpy(&uthread->uu_arg[0] , ®s->r[flavor], regparams * sizeof(int)); + if (copyin((user_addr_t)regs->sp + 7 * sizeof(int), (int *)&uthread->uu_arg[0] + regparams, + (callp->sy_arg_bytes - (sizeof(uint32_t) * regparams))) != 0) { + return -1; + } + } else { + return -1; + } + + return 0; +} +#endif + +static int +arm_get_syscall_number(struct arm_saved_state *regs) +{ + if (regs->save_r12 != 0) { + return regs->save_r12; + } else { + return regs->save_r0; + } +} + +static void +arm_prepare_syscall_return(struct sysent *callp, struct arm_saved_state *state, uthread_t uthread, int error) +{ + assert(is_saved_state32(state)); + arm_prepare_u32_syscall_return(callp, state, uthread, error); +} + +static void +arm_trace_unix_syscall(int code, struct arm_saved_state *state) +{ + assert(is_saved_state32(state)); + arm_trace_u32_unix_syscall(code, saved_state32(state)); +} + +static void +arm_clear_syscall_error(struct arm_saved_state * state) +{ + assert(is_saved_state32(state)); + arm_clear_u32_syscall_error(saved_state32(state)); +} + +#elif defined(__arm64__) +static void arm_prepare_u64_syscall_return(struct sysent *, arm_saved_state64_t *, uthread_t, int); +static int arm_get_u64_syscall_args(uthread_t, arm_saved_state64_t *, struct sysent *); + +static int +arm_get_syscall_args(uthread_t uthread, struct arm_saved_state *state, struct sysent *callp) +{ + if (is_saved_state32(state)) { + return arm_get_u32_syscall_args(uthread, saved_state32(state), callp); + } else { + return arm_get_u64_syscall_args(uthread, saved_state64(state), callp); + } +} + +/* + * 64-bit: all arguments in registers. We're willing to use x9, a temporary + * register per the ABI, to pass an argument to the kernel for one case, + * an indirect syscall with 8 arguments. No munging required, as all arguments + * are in 64-bit wide registers already. + */ +static int +arm_get_u64_syscall_args(uthread_t uthread, arm_saved_state64_t *regs, struct sysent *callp) +{ + int indirect_offset, regparams; + + indirect_offset = (regs->x[ARM64_SYSCALL_CODE_REG_NUM] == 0) ? 1 : 0; + regparams = 9 - indirect_offset; + + /* + * Everything should fit in registers for now. + */ + assert(callp->sy_narg <= 8); + if (callp->sy_narg > regparams) { + return -1; + } + + memcpy(&uthread->uu_arg[0], ®s->x[indirect_offset], callp->sy_narg * sizeof(uint64_t)); + return 0; +} +/* + * When the kernel is running AArch64, munge arguments from 32-bit + * userland out to 64-bit. + * + * flavor == 1 indicates an indirect syscall. + */ +static int +arm_get_u32_syscall_args(uthread_t uthread, arm_saved_state32_t *regs, struct sysent *callp) +{ + int regparams; +#if CONFIG_REQUIRES_U32_MUNGING + sy_munge_t *mungerp; +#else +#error U32 syscalls on ARM64 kernel requires munging +#endif + int flavor = (regs->save_r12 == 0 ? 1 : 0); + + regparams = (7 - flavor); /* Indirect value consumes a register */ + + assert((unsigned) callp->sy_arg_bytes <= sizeof (uthread->uu_arg)); + + if (callp->sy_arg_bytes <= (sizeof(uint32_t) * regparams)) { + /* + * Seven arguments or less are passed in registers. + */ + memcpy(&uthread->uu_arg[0], ®s->r[flavor], callp->sy_arg_bytes); + } else if (callp->sy_arg_bytes <= sizeof(uthread->uu_arg)) { + /* + * In this case, we composite - take the first args from registers, + * the remainder from the stack (offset by the 7 regs therein). + */ + unix_syscall_kprintf("%s: spillover...\n", __FUNCTION__); + memcpy(&uthread->uu_arg[0] , ®s->r[flavor], regparams * sizeof(int)); + if (copyin((user_addr_t)regs->sp + 7 * sizeof(int), (int *)&uthread->uu_arg[0] + regparams, + (callp->sy_arg_bytes - (sizeof(uint32_t) * regparams))) != 0) { + return -1; + } + } else { + return -1; + } + +#if CONFIG_REQUIRES_U32_MUNGING + /* Munge here */ + mungerp = callp->sy_arg_munge32; + if (mungerp != NULL) { + (*mungerp)(&uthread->uu_arg[0]); + } +#endif + + return 0; + +} + +static int +arm_get_syscall_number(struct arm_saved_state *state) +{ + if (is_saved_state32(state)) { + if (saved_state32(state)->save_r12 != 0) { + return saved_state32(state)->save_r12; + } else { + return saved_state32(state)->save_r0; + } + } else { + if (saved_state64(state)->x[ARM64_SYSCALL_CODE_REG_NUM] != 0) { + return saved_state64(state)->x[ARM64_SYSCALL_CODE_REG_NUM]; + } else { + return saved_state64(state)->x[0]; + } + } + +} + +static void +arm_prepare_syscall_return(struct sysent *callp, struct arm_saved_state *state, uthread_t uthread, int error) +{ + if (is_saved_state32(state)) { + arm_prepare_u32_syscall_return(callp, saved_state32(state), uthread, error); + } else { + arm_prepare_u64_syscall_return(callp, saved_state64(state), uthread, error); + } +} + +static void +arm_prepare_u64_syscall_return(struct sysent *callp, arm_saved_state64_t *regs, uthread_t uthread, int error) +{ + if (error == ERESTART) { + regs->pc -= 4; + } else if (error != EJUSTRETURN) { + if (error) { + regs->x[0] = error; + regs->x[1] = 0; + /* + * Set the carry bit to execute cerror routine. + * ARM64_TODO: should we have a separate definition? + * The bits are the same. + */ + regs->cpsr |= PSR_CF; + unix_syscall_return_kprintf("error: setting carry to trigger cerror call\n"); + } else { /* (not error) */ + switch (callp->sy_return_type) { + case _SYSCALL_RET_INT_T: + regs->x[0] = uthread->uu_rval[0]; + regs->x[1] = uthread->uu_rval[1]; + break; + case _SYSCALL_RET_UINT_T: + regs->x[0] = (u_int)uthread->uu_rval[0]; + regs->x[1] = (u_int)uthread->uu_rval[1]; + break; + case _SYSCALL_RET_OFF_T: + case _SYSCALL_RET_ADDR_T: + case _SYSCALL_RET_SIZE_T: + case _SYSCALL_RET_SSIZE_T: + case _SYSCALL_RET_UINT64_T: + regs->x[0] = *((uint64_t *)(&uthread->uu_rval[0])); + regs->x[1] = 0; + break; + case _SYSCALL_RET_NONE: + break; + default: + panic("unix_syscall: unknown return type"); + break; + } + } + } + /* else (error == EJUSTRETURN) { nothing } */ + + +} +static void +arm_trace_u64_unix_syscall(int code, arm_saved_state64_t *regs) +{ + boolean_t indirect = (regs->x[ARM64_SYSCALL_CODE_REG_NUM] == 0); + if (indirect) + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START, + regs->x[1], regs->x[2], regs->x[3], regs->x[4], 0); + else + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START, + regs->x[0], regs->x[1], regs->x[2], regs->x[3], 0); +} + +static void +arm_trace_unix_syscall(int code, struct arm_saved_state *state) +{ + if (is_saved_state32(state)) { + arm_trace_u32_unix_syscall(code, saved_state32(state)); + } else { + arm_trace_u64_unix_syscall(code, saved_state64(state)); + } +} + +static void +arm_clear_u64_syscall_error(arm_saved_state64_t *regs) +{ + /* + * ARM64_TODO: should we have a separate definition? + * The bits are the same. + */ + regs->cpsr &= ~PSR_CF; +} + +static void +arm_clear_syscall_error(struct arm_saved_state * state) +{ + if (is_saved_state32(state)) { + arm_clear_u32_syscall_error(saved_state32(state)); + } else { + arm_clear_u64_syscall_error(saved_state64(state)); + } +} + +#else +#error Unknown architecture. +#endif diff --git a/bsd/dev/arm/table_inline.h b/bsd/dev/arm/table_inline.h new file mode 100644 index 000000000..8f358e423 --- /dev/null +++ b/bsd/dev/arm/table_inline.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ +/* + * Copyright (c) 1992 NeXT Computer, Inc. + * + * Intel386 Family: Selector based access to descriptor tables. + * + * HISTORY + * + * 2 April 1992 ? at NeXT + * Created. + */ + +#include <architecture/i386/table.h> + +#include <machdep/i386/gdt.h> +#include <machdep/i386/idt.h> + +static inline gdt_entry_t * +sel_to_gdt_entry(sel_t sel) +{ + return (&gdt[sel.index]); +} + +static inline idt_entry_t * +sel_to_idt_entry(sel_t sel) +{ + return (&idt[sel.index]); +} + +static inline ldt_entry_t * +sel_to_ldt_entry(ldt_t *tbl, sel_t sel) +{ + return (&tbl[sel.index]); +} diff --git a/bsd/dev/arm/unix_signal.c b/bsd/dev/arm/unix_signal.c new file mode 100644 index 000000000..51c4d7e48 --- /dev/null +++ b/bsd/dev/arm/unix_signal.c @@ -0,0 +1,737 @@ +/* + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + */ + +#include <mach/mach_types.h> +#include <mach/exception_types.h> + +#include <sys/param.h> +#include <sys/proc_internal.h> +#include <sys/user.h> +#include <sys/signal.h> +#include <sys/ucontext.h> +#include <sys/sysproto.h> +#include <sys/systm.h> +#include <sys/ux_exception.h> + +#include <arm/signal.h> +#include <sys/signalvar.h> +#include <sys/kdebug.h> +#include <sys/sdt.h> +#include <sys/wait.h> +#include <kern/thread.h> +#include <mach/arm/thread_status.h> +#include <arm/proc_reg.h> + +#include <kern/assert.h> +#include <pexpert/pexpert.h> + +extern struct arm_saved_state *get_user_regs(thread_t); +extern user_addr_t thread_get_cthread_self(void); +extern kern_return_t thread_getstatus(thread_t act, int flavor, + thread_state_t tstate, mach_msg_type_number_t *count); +extern kern_return_t thread_setstatus(thread_t thread, int flavor, + thread_state_t tstate, mach_msg_type_number_t count); +/* XXX Put these someplace smarter... */ +typedef struct mcontext32 mcontext32_t; +typedef struct mcontext64 mcontext64_t; + +/* Signal handler flavors supported */ +/* These defns should match the Libc implmn */ +#define UC_TRAD 1 +#define UC_FLAVOR 30 + +/* The following are valid mcontext sizes */ +#define UC_FLAVOR_SIZE32 ((ARM_THREAD_STATE_COUNT + ARM_EXCEPTION_STATE_COUNT + ARM_VFP_STATE_COUNT) * sizeof(int)) +#define UC_FLAVOR_SIZE64 ((ARM_THREAD_STATE64_COUNT + ARM_EXCEPTION_STATE64_COUNT + ARM_NEON_STATE64_COUNT) * sizeof(int)) + +#if __arm64__ +#define C_64_REDZONE_LEN 128 +#endif + +static int +sendsig_get_state32(thread_t th_act, mcontext32_t *mcp) +{ + void *tstate; + mach_msg_type_number_t state_count; + + assert(!proc_is64bit(current_proc())); + + tstate = (void *) &mcp->ss; + state_count = ARM_THREAD_STATE_COUNT; + if (thread_getstatus(th_act, ARM_THREAD_STATE, (thread_state_t) tstate, &state_count) != KERN_SUCCESS) + return EINVAL; + + tstate = (void *) &mcp->es; + state_count = ARM_EXCEPTION_STATE_COUNT; + if (thread_getstatus(th_act, ARM_EXCEPTION_STATE, (thread_state_t) tstate, &state_count) != KERN_SUCCESS) + return EINVAL; + + tstate = (void *) &mcp->fs; + state_count = ARM_VFP_STATE_COUNT; + if (thread_getstatus(th_act, ARM_VFP_STATE, (thread_state_t) tstate, &state_count) != KERN_SUCCESS) + return EINVAL; + + return 0; +} + +#if defined(__arm64__) +struct user_sigframe64 { + /* We can pass the last arg in a register for ARM64 */ + user64_siginfo_t sinfo; + struct user_ucontext64 uctx; + mcontext64_t mctx; +}; + +static int +sendsig_get_state64(thread_t th_act, mcontext64_t *mcp) +{ + void *tstate; + mach_msg_type_number_t state_count; + + assert(proc_is64bit(current_proc())); + + tstate = (void *) &mcp->ss; + state_count = ARM_THREAD_STATE64_COUNT; + if (thread_getstatus(th_act, ARM_THREAD_STATE64, (thread_state_t) tstate, &state_count) != KERN_SUCCESS) + return EINVAL; + + tstate = (void *) &mcp->es; + state_count = ARM_EXCEPTION_STATE64_COUNT; + if (thread_getstatus(th_act, ARM_EXCEPTION_STATE64, (thread_state_t) tstate, &state_count) != KERN_SUCCESS) + return EINVAL; + + tstate = (void *) &mcp->ns; + state_count = ARM_NEON_STATE64_COUNT; + if (thread_getstatus(th_act, ARM_NEON_STATE64, (thread_state_t) tstate, &state_count) != KERN_SUCCESS) + return EINVAL; + + return 0; +} + +static void +sendsig_fill_uctx64(user_ucontext64_t *uctx, int oonstack, int mask, user64_addr_t sp, user64_size_t stack_size, user64_addr_t p_mctx) +{ + bzero(uctx, sizeof(*uctx)); + uctx->uc_onstack = oonstack; + uctx->uc_sigmask = mask; + uctx->uc_stack.ss_sp = sp; + uctx->uc_stack.ss_size = stack_size; + if (oonstack) + uctx->uc_stack.ss_flags |= SS_ONSTACK; + uctx->uc_link = (user64_addr_t)0; + uctx->uc_mcsize = (user64_size_t) UC_FLAVOR_SIZE64; + uctx->uc_mcontext64 = (user64_addr_t) p_mctx; +} + +static kern_return_t +sendsig_set_thread_state64(arm_thread_state64_t *regs, + user64_addr_t catcher, int infostyle, int sig, user64_addr_t p_sinfo, + user64_addr_t p_uctx, user64_addr_t trampact, user64_addr_t sp, thread_t th_act) +{ + assert(proc_is64bit(current_proc())); + + regs->x[0] = catcher; + regs->x[1] = infostyle; + regs->x[2] = sig; + regs->x[3] = p_sinfo; + regs->x[4] = p_uctx; + regs->pc = trampact; + regs->cpsr = PSR64_USER64_DEFAULT; + regs->sp = sp; + + return thread_setstatus(th_act, ARM_THREAD_STATE64, (void *)regs, ARM_THREAD_STATE64_COUNT); +} +#endif /* defined(__arm64__) */ + +static void +sendsig_fill_uctx32(user_ucontext32_t *uctx, int oonstack, int mask, user_addr_t sp, user_size_t stack_size, user_addr_t p_mctx) +{ + bzero(uctx, sizeof(*uctx)); + uctx->uc_onstack = oonstack; + uctx->uc_sigmask = mask; + uctx->uc_stack.ss_sp = (user32_addr_t) sp; + uctx->uc_stack.ss_size = (user32_size_t) stack_size; + if (oonstack) + uctx->uc_stack.ss_flags |= SS_ONSTACK; + uctx->uc_link = (user32_addr_t)0; + uctx->uc_mcsize = (user32_size_t) UC_FLAVOR_SIZE32; + uctx->uc_mcontext = (user32_addr_t) p_mctx; +} + +static kern_return_t +sendsig_set_thread_state32(arm_thread_state_t *regs, + user32_addr_t catcher, int infostyle, int sig, user32_addr_t p_sinfo, + user32_addr_t trampact, user32_addr_t sp, thread_t th_act) +{ + + assert(!proc_is64bit(current_proc())); + + regs->r[0] = catcher; + regs->r[1] = infostyle; + regs->r[2] = sig; + regs->r[3] = p_sinfo; + if (trampact & 1) { + regs->pc = trampact & ~1; +#if defined(__arm64__) + regs->cpsr = PSR64_USER32_DEFAULT | PSR64_MODE_USER32_THUMB; +#elif defined(__arm__) + regs->cpsr = PSR_USERDFLT | PSR_TF; +#else +#error Unknown architeture. +#endif + } else { + regs->pc = trampact; + regs->cpsr = PSR_USERDFLT; + } + regs->sp = sp; + + return thread_setstatus(th_act, ARM_THREAD_STATE, (void *)regs, ARM_THREAD_STATE_COUNT); +} + +#if CONFIG_DTRACE +static void +sendsig_do_dtrace(uthread_t ut, user_siginfo_t *sinfo, int sig, user_addr_t catcher) +{ + bzero((caddr_t)&(ut->t_dtrace_siginfo), sizeof(ut->t_dtrace_siginfo)); + + ut->t_dtrace_siginfo.si_signo = sinfo->si_signo; + ut->t_dtrace_siginfo.si_code = sinfo->si_code; + ut->t_dtrace_siginfo.si_pid = sinfo->si_pid; + ut->t_dtrace_siginfo.si_uid = sinfo->si_uid; + ut->t_dtrace_siginfo.si_status = sinfo->si_status; + /* XXX truncates faulting address to void * */ + ut->t_dtrace_siginfo.si_addr = CAST_DOWN_EXPLICIT(void *, sinfo->si_addr); + + /* Fire DTrace proc:::fault probe when signal is generated by hardware. */ + switch (sig) { + case SIGILL: case SIGBUS: case SIGSEGV: case SIGFPE: case SIGTRAP: + DTRACE_PROC2(fault, int, (int)(ut->uu_code), siginfo_t *, &(ut->t_dtrace_siginfo)); + break; + default: + break; + } + + /* XXX truncates faulting address to uintptr_t */ + DTRACE_PROC3(signal__handle, int, sig, siginfo_t *, &(ut->t_dtrace_siginfo), + void (*)(void), CAST_DOWN(sig_t, catcher)); +} +#endif + +struct user_sigframe32 { + user32_addr_t puctx; + user32_siginfo_t sinfo; + struct user_ucontext32 uctx; + mcontext32_t mctx; +}; + +/* + * Send an interrupt to process. + * + */ +void +sendsig( + struct proc * p, + user_addr_t catcher, + int sig, + int mask, + __unused uint32_t code +) +{ + union { + struct user_sigframe32 uf32; +#if defined(__arm64__) + struct user_sigframe64 uf64; +#endif + } user_frame; + + user_siginfo_t sinfo; + user_addr_t sp = 0, trampact; + struct sigacts *ps = p->p_sigacts; + int oonstack, infostyle; + thread_t th_act; + struct uthread *ut; + user_size_t stack_size = 0; + + th_act = current_thread(); + ut = get_bsdthread_info(th_act); + + bzero(&user_frame, sizeof(user_frame)); + + if (p->p_sigacts->ps_siginfo & sigmask(sig)) + infostyle = UC_FLAVOR; + else + infostyle = UC_TRAD; + + trampact = ps->ps_trampact[sig]; + oonstack = ps->ps_sigstk.ss_flags & SA_ONSTACK; + + /* + * Get sundry thread state. + */ + if (proc_is64bit(p)) { +#ifdef __arm64__ + if (sendsig_get_state64(th_act, &user_frame.uf64.mctx) != 0) { + goto bad2; + } +#else + panic("Shouldn't have 64-bit thread states on a 32-bit kernel."); +#endif + } else { + if (sendsig_get_state32(th_act, &user_frame.uf32.mctx) != 0) { + goto bad2; + } + } + + /* + * Figure out where our new stack lives. + */ + if ((ps->ps_flags & SAS_ALTSTACK) && !oonstack && + (ps->ps_sigonstack & sigmask(sig))) { + sp = ps->ps_sigstk.ss_sp; + sp += ps->ps_sigstk.ss_size; + stack_size = ps->ps_sigstk.ss_size; + ps->ps_sigstk.ss_flags |= SA_ONSTACK; + } else { + /* + * Get stack pointer, and allocate enough space + * for signal handler data. + */ + if (proc_is64bit(p)) { +#if defined(__arm64__) + sp = CAST_USER_ADDR_T(user_frame.uf64.mctx.ss.sp); + sp = (sp - sizeof(user_frame.uf64) - C_64_REDZONE_LEN) & ~0xf; /* Make sure to align to 16 bytes and respect red zone */ +#else + panic("Shouldn't have 64-bit thread states on a 32-bit kernel."); +#endif + } else { + sp = CAST_USER_ADDR_T(user_frame.uf32.mctx.ss.sp); + sp -= sizeof(user_frame.uf32); +#if defined(__arm__) && (__BIGGEST_ALIGNMENT__ > 4) + sp &= ~0xf; /* Make sure to align to 16 bytes for armv7k */ +#endif + } + } + + proc_unlock(p); + + /* + * Fill in ucontext (points to mcontext, i.e. thread states). + */ + if (proc_is64bit(p)) { +#if defined(__arm64__) + sendsig_fill_uctx64(&user_frame.uf64.uctx, oonstack, mask, sp, (user64_size_t)stack_size, + (user64_addr_t)&((struct user_sigframe64*)sp)->mctx); +#else + panic("Shouldn't have 64-bit thread states on a 32-bit kernel."); +#endif + } else { + sendsig_fill_uctx32(&user_frame.uf32.uctx, oonstack, mask, sp, (user32_size_t)stack_size, + (user32_addr_t)&((struct user_sigframe32*)sp)->mctx); + } + + /* + * Setup siginfo. + */ + bzero((caddr_t) & sinfo, sizeof(sinfo)); + sinfo.si_signo = sig; + + if (proc_is64bit(p)) { +#if defined(__arm64__) + sinfo.si_addr = user_frame.uf64.mctx.ss.pc; + sinfo.pad[0] = user_frame.uf64.mctx.ss.sp; +#else + panic("Shouldn't have 64-bit thread states on a 32-bit kernel."); +#endif + } else { + sinfo.si_addr = user_frame.uf32.mctx.ss.pc; + sinfo.pad[0] = user_frame.uf32.mctx.ss.sp; + } + + switch (sig) { + case SIGILL: +#ifdef BER_XXX + if (mctx.ss.srr1 & (1 << (31 - SRR1_PRG_ILL_INS_BIT))) + sinfo.si_code = ILL_ILLOPC; + else if (mctx.ss.srr1 & (1 << (31 - SRR1_PRG_PRV_INS_BIT))) + sinfo.si_code = ILL_PRVOPC; + else if (mctx.ss.srr1 & (1 << (31 - SRR1_PRG_TRAP_BIT))) + sinfo.si_code = ILL_ILLTRP; + else + sinfo.si_code = ILL_NOOP; +#else + sinfo.si_code = ILL_ILLTRP; +#endif + break; + + case SIGFPE: + break; + + case SIGBUS: + if (proc_is64bit(p)) { +#if defined(__arm64__) + sinfo.si_addr = user_frame.uf64.mctx.es.far; +#else + panic("Shouldn't have 64-bit thread states on a 32-bit kernel."); +#endif + } else { + sinfo.si_addr = user_frame.uf32.mctx.es.far; + } + + sinfo.si_code = BUS_ADRALN; + break; + + case SIGSEGV: + if (proc_is64bit(p)) { +#if defined(__arm64__) + sinfo.si_addr = user_frame.uf64.mctx.es.far; +#else + panic("Shouldn't have 64-bit thread states on a 32-bit kernel."); +#endif + } else { + sinfo.si_addr = user_frame.uf32.mctx.es.far; + } + +#ifdef BER_XXX + /* First check in srr1 and then in dsisr */ + if (mctx.ss.srr1 & (1 << (31 - DSISR_PROT_BIT))) + sinfo.si_code = SEGV_ACCERR; + else if (mctx.es.dsisr & (1 << (31 - DSISR_PROT_BIT))) + sinfo.si_code = SEGV_ACCERR; + else + sinfo.si_code = SEGV_MAPERR; +#else + sinfo.si_code = SEGV_ACCERR; +#endif + break; + + default: + { + int status_and_exitcode; + + /* + * All other signals need to fill out a minimum set of + * information for the siginfo structure passed into + * the signal handler, if SA_SIGINFO was specified. + * + * p->si_status actually contains both the status and + * the exit code; we save it off in its own variable + * for later breakdown. + */ + proc_lock(p); + sinfo.si_pid = p->si_pid; + p->si_pid = 0; + status_and_exitcode = p->si_status; + p->si_status = 0; + sinfo.si_uid = p->si_uid; + p->si_uid = 0; + sinfo.si_code = p->si_code; + p->si_code = 0; + proc_unlock(p); + if (sinfo.si_code == CLD_EXITED) { + if (WIFEXITED(status_and_exitcode)) + sinfo.si_code = CLD_EXITED; + else if (WIFSIGNALED(status_and_exitcode)) { + if (WCOREDUMP(status_and_exitcode)) { + sinfo.si_code = CLD_DUMPED; + status_and_exitcode = W_EXITCODE(status_and_exitcode,status_and_exitcode); + } else { + sinfo.si_code = CLD_KILLED; + status_and_exitcode = W_EXITCODE(status_and_exitcode,status_and_exitcode); + } + } + } + /* + * The recorded status contains the exit code and the + * signal information, but the information to be passed + * in the siginfo to the handler is supposed to only + * contain the status, so we have to shift it out. + */ + sinfo.si_status = (WEXITSTATUS(status_and_exitcode) & 0x00FFFFFF) | (((uint32_t)(p->p_xhighbits) << 24) & 0xFF000000); + p->p_xhighbits = 0; + break; + } + } + +#if CONFIG_DTRACE + sendsig_do_dtrace(ut, &sinfo, sig, catcher); +#endif /* CONFIG_DTRACE */ + + /* + * Copy signal-handling frame out to user space, set thread state. + */ + if (proc_is64bit(p)) { +#if defined(__arm64__) + /* + * mctx filled in when we get state. uctx filled in by + * sendsig_fill_uctx64(). We fill in the sinfo now. + */ + siginfo_user_to_user64(&sinfo, &user_frame.uf64.sinfo); + + if (copyout(&user_frame.uf64, sp, sizeof(user_frame.uf64)) != 0) { + goto bad; + } + + if (sendsig_set_thread_state64(&user_frame.uf64.mctx.ss, + catcher, infostyle, sig, (user64_addr_t)&((struct user_sigframe64*)sp)->sinfo, + (user64_addr_t)&((struct user_sigframe64*)sp)->uctx, trampact, sp, th_act) != KERN_SUCCESS) + goto bad; + +#else + panic("Shouldn't have 64-bit thread states on a 32-bit kernel."); +#endif + } else { + /* + * mctx filled in when we get state. uctx filled in by + * sendsig_fill_uctx32(). We fill in the sinfo and *pointer* + * to uctx now. + */ + siginfo_user_to_user32(&sinfo, &user_frame.uf32.sinfo); + user_frame.uf32.puctx = (user32_addr_t) &((struct user_sigframe32*)sp)->uctx; + + if (copyout(&user_frame.uf32, sp, sizeof(user_frame.uf32)) != 0) { + goto bad; + } + + if (sendsig_set_thread_state32(&user_frame.uf32.mctx.ss, + CAST_DOWN_EXPLICIT(user32_addr_t, catcher), infostyle, sig, (user32_addr_t)&((struct user_sigframe32*)sp)->sinfo, + CAST_DOWN_EXPLICIT(user32_addr_t, trampact), CAST_DOWN_EXPLICIT(user32_addr_t, sp), th_act) != KERN_SUCCESS) + goto bad; + } + + proc_lock(p); + return; + +bad: + proc_lock(p); +bad2: + SIGACTION(p, SIGILL) = SIG_DFL; + sig = sigmask(SIGILL); + p->p_sigignore &= ~sig; + p->p_sigcatch &= ~sig; + ut->uu_sigmask &= ~sig; + /* sendsig is called with signal lock held */ + proc_unlock(p); + psignal_locked(p, SIGILL); + proc_lock(p); +} + +/* + * System call to cleanup state after a signal + * has been taken. Reset signal mask and + * stack state from context left by sendsig (above). + * Return to previous * context left by sendsig. + * Check carefully to * make sure that the user has not + * modified the * spr to gain improper priviledges. + */ + +static int +sigreturn_copyin_ctx32(struct user_ucontext32 *uctx, mcontext32_t *mctx, user_addr_t uctx_addr) +{ + int error; + + assert(!proc_is64bit(current_proc())); + + error = copyin(uctx_addr, uctx, sizeof(*uctx)); + if (error) { + return (error); + } + + /* validate the machine context size */ + switch (uctx->uc_mcsize) { + case UC_FLAVOR_SIZE32: + break; + default: + return (EINVAL); + } + + assert(uctx->uc_mcsize == sizeof(*mctx)); + error = copyin((user_addr_t)uctx->uc_mcontext, mctx, uctx->uc_mcsize); + if (error) { + return (error); + } + + return 0; +} + +static int +sigreturn_set_state32(thread_t th_act, mcontext32_t *mctx) +{ + assert(!proc_is64bit(current_proc())); + + /* validate the thread state, set/reset appropriate mode bits in cpsr */ +#if defined(__arm__) + mctx->ss.cpsr = (mctx->ss.cpsr & ~PSR_MODE_MASK) | PSR_USERDFLT; +#elif defined(__arm64__) + mctx->ss.cpsr = (mctx->ss.cpsr & ~PSR64_MODE_MASK) | PSR64_USER32_DEFAULT; +#else +#error Unknown architecture. +#endif + + if (thread_setstatus(th_act, ARM_THREAD_STATE, (void *)&mctx->ss, ARM_THREAD_STATE_COUNT) != KERN_SUCCESS) { + return (EINVAL); + } + if (thread_setstatus(th_act, ARM_VFP_STATE, (void *)&mctx->fs, ARM_VFP_STATE_COUNT) != KERN_SUCCESS) { + return (EINVAL); + } + + return 0; +} + +#if defined(__arm64__) +static int +sigreturn_copyin_ctx64(struct user_ucontext64 *uctx, mcontext64_t *mctx, user_addr_t uctx_addr) +{ + int error; + + assert(proc_is64bit(current_proc())); + + error = copyin(uctx_addr, uctx, sizeof(*uctx)); + if (error) { + return (error); + } + + /* validate the machine context size */ + switch (uctx->uc_mcsize) { + case UC_FLAVOR_SIZE64: + break; + default: + return (EINVAL); + } + + assert(uctx->uc_mcsize == sizeof(*mctx)); + error = copyin((user_addr_t)uctx->uc_mcontext64, mctx, uctx->uc_mcsize); + if (error) { + return (error); + } + + return 0; +} + +static int +sigreturn_set_state64(thread_t th_act, mcontext64_t *mctx) +{ + assert(proc_is64bit(current_proc())); + + /* validate the thread state, set/reset appropriate mode bits in cpsr */ + mctx->ss.cpsr = (mctx->ss.cpsr & ~PSR64_MODE_MASK) | PSR64_USER64_DEFAULT; + + if (thread_setstatus(th_act, ARM_THREAD_STATE64, (void *)&mctx->ss, ARM_THREAD_STATE64_COUNT) != KERN_SUCCESS) { + return (EINVAL); + } + if (thread_setstatus(th_act, ARM_NEON_STATE64, (void *)&mctx->ns, ARM_NEON_STATE64_COUNT) != KERN_SUCCESS) { + return (EINVAL); + } + + return 0; +} +#endif /* defined(__arm64__) */ + +/* ARGSUSED */ +int +sigreturn( + struct proc * p, + struct sigreturn_args * uap, + __unused int *retval) +{ + union { + user_ucontext32_t uc32; +#if defined(__arm64__) + user_ucontext64_t uc64; +#endif + } uctx; + + union { + mcontext32_t mc32; +#if defined(__arm64__) + mcontext64_t mc64; +#endif + } mctx; + + int error, sigmask = 0, onstack = 0; + thread_t th_act; + struct uthread *ut; + + th_act = current_thread(); + ut = (struct uthread *) get_bsdthread_info(th_act); + + if (proc_is64bit(p)) { +#if defined(__arm64__) + error = sigreturn_copyin_ctx64(&uctx.uc64, &mctx.mc64, uap->uctx); + if (error != 0) { + return error; + } + + onstack = uctx.uc64.uc_onstack; + sigmask = uctx.uc64.uc_sigmask; +#else + panic("Shouldn't have 64-bit thread states on a 32-bit kernel."); +#endif + } else { + error = sigreturn_copyin_ctx32(&uctx.uc32, &mctx.mc32, uap->uctx); + if (error != 0) { + return error; + } + + onstack = uctx.uc32.uc_onstack; + sigmask = uctx.uc32.uc_sigmask; + } + + if ((onstack & 01)) + p->p_sigacts->ps_sigstk.ss_flags |= SA_ONSTACK; + else + p->p_sigacts->ps_sigstk.ss_flags &= ~SA_ONSTACK; + + ut->uu_sigmask = sigmask & ~sigcantmask; + if (ut->uu_siglist & ~ut->uu_sigmask) + signal_setast(current_thread()); + + if (proc_is64bit(p)) { +#if defined(__arm64__) + error = sigreturn_set_state64(th_act, &mctx.mc64); + if (error != 0) { + return error; + } +#else + panic("Shouldn't have 64-bit thread states on a 32-bit kernel."); +#endif + } else { + error = sigreturn_set_state32(th_act, &mctx.mc32); + if (error != 0) { + return error; + } + } + + return (EJUSTRETURN); +} + +/* + * machine_exception() performs MD translation + * of a mach exception to a unix signal and code. + */ + +boolean_t +machine_exception( + int exception, + mach_exception_subcode_t code, + __unused mach_exception_subcode_t subcode, + int *unix_signal, + mach_exception_subcode_t * unix_code +) +{ + switch (exception) { + case EXC_BAD_INSTRUCTION: + *unix_signal = SIGILL; + *unix_code = code; + break; + + case EXC_ARITHMETIC: + *unix_signal = SIGFPE; + *unix_code = code; + break; + + default: + return (FALSE); + } + return (TRUE); +} diff --git a/bsd/dev/arm64/conf.c b/bsd/dev/arm64/conf.c new file mode 100644 index 000000000..761484db1 --- /dev/null +++ b/bsd/dev/arm64/conf.c @@ -0,0 +1,306 @@ +/* + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. + */ +/* + * Copyright (c) 1997 by Apple Computer, Inc., all rights reserved + * Copyright (c) 1993 NeXT Computer, Inc. + * + * UNIX Device switch tables. + * + * HISTORY + * + * 30 July 1997 Umesh Vaishampayan (umeshv@apple.com) + * enabled file descriptor pseudo-device. + * 18 June 1993 ? at NeXT + * Cleaned up a lot of stuff in this file. + */ + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/ioctl.h> +#include <sys/tty.h> +#include <sys/conf.h> + +/* Prototypes that should be elsewhere: */ +extern dev_t chrtoblk(dev_t dev); +extern int chrtoblk_set(int cdev, int bdev); + +struct bdevsw bdevsw[] = +{ + /* + * For block devices, every other block of 8 slots is reserved to Apple. + * The other slots are available for the user. This way we can both + * add new entries without running into each other. Be sure to fill in + * Apple's 8 reserved slots when you jump over us -- we'll do the same + * for you. + */ + + /* 0 - 7 are reserved to Apple */ + + NO_BDEVICE, /* 0 */ + NO_BDEVICE, /* 1 */ + NO_BDEVICE, /* 2 */ + NO_BDEVICE, /* 3 */ + NO_BDEVICE, /* 4 */ + NO_BDEVICE, /* 5 */ + NO_BDEVICE, /* 6 */ + NO_BDEVICE, /* 7 */ + + /* 8 - 15 are reserved to the user */ + NO_BDEVICE, /* 8 */ + NO_BDEVICE, /* 9 */ + NO_BDEVICE, /* 10 */ + NO_BDEVICE, /* 11 */ + NO_BDEVICE, /* 12 */ + NO_BDEVICE, /* 13 */ + NO_BDEVICE, /* 14 */ + NO_BDEVICE, /* 15 */ + + /* 16 - 23 are reserved to Apple */ + NO_BDEVICE, /* 16 */ + NO_BDEVICE, /* 17 */ + NO_BDEVICE, /* 18 */ + NO_BDEVICE, /* 18 */ + NO_BDEVICE, /* 20 */ + NO_BDEVICE, /* 21 */ + NO_BDEVICE, /* 22 */ + NO_BDEVICE, /* 23 */ +}; + +const int nblkdev = sizeof(bdevsw) / sizeof(bdevsw[0]); + +extern struct tty *km_tty[]; +extern d_open_t cnopen; +extern d_close_t cnclose; +extern d_read_t cnread; +extern d_write_t cnwrite; +extern d_ioctl_t cnioctl; +extern d_select_t cnselect; +extern d_open_t kmopen; +extern d_close_t kmclose; +extern d_read_t kmread; +extern d_write_t kmwrite; +extern d_ioctl_t kmioctl; +extern d_open_t sgopen; +extern d_close_t sgclose; +extern d_ioctl_t sgioctl; + +#if NVOL > 0 +extern d_open_t volopen; +extern d_close_t volclose; +extern d_ioctl_t volioctl; +#else +#define volopen eno_opcl +#define volclose eno_opcl +#define volioctl eno_ioctl +#endif + +extern d_open_t cttyopen; +extern d_read_t cttyread; +extern d_write_t cttywrite; +extern d_ioctl_t cttyioctl; +extern d_select_t cttyselect; + +extern d_read_t mmread; +extern d_write_t mmwrite; +extern d_ioctl_t mmioctl; +#define mmselect (select_fcn_t *)seltrue +#define mmmmap eno_mmap + +#include <pty.h> +#if NPTY > 0 +extern d_open_t ptsopen; +extern d_close_t ptsclose; +extern d_read_t ptsread; +extern d_write_t ptswrite; +extern d_select_t ptsselect; +extern d_stop_t ptsstop; +extern d_open_t ptcopen; +extern d_close_t ptcclose; +extern d_read_t ptcread; +extern d_write_t ptcwrite; +extern d_select_t ptcselect; +extern d_ioctl_t ptyioctl; +#else +#define ptsopen eno_opcl +#define ptsclose eno_opcl +#define ptsread eno_rdwrt +#define ptswrite eno_rdwrt +#define ptsstop nulldev + +#define ptcopen eno_opcl +#define ptcclose eno_opcl +#define ptcread eno_rdwrt +#define ptcwrite eno_rdwrt +#define ptcselect eno_select +#define ptyioctl eno_ioctl +#endif + +extern d_open_t logopen; +extern d_close_t logclose; +extern d_read_t logread; +extern d_ioctl_t logioctl; +extern d_select_t logselect; + +extern d_open_t oslog_streamopen; +extern d_close_t oslog_streamclose; +extern d_read_t oslog_streamread; +extern d_ioctl_t oslog_streamioctl; +extern d_select_t oslog_streamselect; + + +extern d_open_t oslogopen; +extern d_close_t oslogclose; +extern d_ioctl_t oslogioctl; +extern d_select_t oslogselect; + +#define nullopen (d_open_t *)&nulldev +#define nullclose (d_close_t *)&nulldev +#define nullread (d_read_t *)&nulldev +#define nullwrite (d_write_t *)&nulldev +#define nullioctl (d_ioctl_t *)&nulldev +#define nullselect (d_select_t *)&nulldev +#define nullstop (d_stop_t *)&nulldev +#define nullreset (d_reset_t *)&nulldev + +struct cdevsw cdevsw[] = { + /* + * To add character devices to this table dynamically, use cdevsw_add. + */ + + [0] = { + cnopen, cnclose, cnread, cnwrite, + cnioctl, nullstop, nullreset, 0, cnselect, + eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY + }, + [1] = NO_CDEVICE, + [2] = { + cttyopen, nullclose, cttyread, cttywrite, + cttyioctl, nullstop, nullreset, 0, cttyselect, + eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY + }, + [3] = { + nullopen, nullclose, mmread, mmwrite, + mmioctl, nullstop, nullreset, 0, mmselect, + mmmmap, eno_strat, eno_getc, eno_putc, D_DISK + }, + [PTC_MAJOR] = { + ptsopen, ptsclose, ptsread, ptswrite, + ptyioctl, ptsstop, nullreset, 0, ptsselect, + eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY + }, + [PTS_MAJOR] = { + ptcopen, ptcclose, ptcread, ptcwrite, + ptyioctl, nullstop, nullreset, 0, ptcselect, + eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY + }, + [6] = { + logopen, logclose, logread, eno_rdwrt, + logioctl, eno_stop, nullreset, 0, logselect, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + }, + [7] = { + oslogopen, oslogclose, eno_rdwrt, eno_rdwrt, + oslogioctl, eno_stop, nullreset, 0, oslogselect, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + }, + [8] = { + oslog_streamopen, oslog_streamclose, oslog_streamread, eno_rdwrt, + oslog_streamioctl, eno_stop, nullreset, 0, oslog_streamselect, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + }, + [9 ... 11] = NO_CDEVICE, + [12] = { + kmopen, kmclose, kmread, kmwrite, + kmioctl, nullstop, nullreset, km_tty, ttselect, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + }, + [13 ... 41] = NO_CDEVICE, + [42] = { + volopen, volclose, eno_rdwrt, eno_rdwrt, + volioctl, eno_stop, eno_reset, 0, (select_fcn_t *) seltrue, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + } +}; +const int nchrdev = sizeof(cdevsw) / sizeof(cdevsw[0]); + +uint64_t cdevsw_flags[sizeof(cdevsw) / sizeof(cdevsw[0])]; + +#include <sys/vnode.h> /* for VCHR and VBLK */ +/* + * return true if a disk + */ +int +isdisk(dev_t dev, int type) +{ + dev_t maj = major(dev); + + switch (type) { + case VCHR: + maj = chrtoblk(maj); + if (maj == NODEV) { + break; + } + /* FALL THROUGH */ + case VBLK: + if (bdevsw[maj].d_type == D_DISK) { + return (1); + } + break; + } + return (0); +} + +static int chrtoblktab[] = { + /* CHR *//* BLK *//* CHR *//* BLK */ + /* 0 */ NODEV, /* 1 */ NODEV, + /* 2 */ NODEV, /* 3 */ NODEV, + /* 4 */ NODEV, /* 5 */ NODEV, + /* 6 */ NODEV, /* 7 */ NODEV, + /* 8 */ NODEV, /* 9 */ NODEV, + /* 10 */ NODEV, /* 11 */ NODEV, + /* 12 */ NODEV, /* 13 */ NODEV, + /* 14 */ NODEV, /* 15 */ NODEV, + /* 16 */ NODEV, /* 17 */ NODEV, + /* 18 */ NODEV, /* 19 */ NODEV, + /* 20 */ NODEV, /* 21 */ NODEV, + /* 22 */ NODEV, /* 23 */ NODEV, + /* 24 */ NODEV, /* 25 */ NODEV, + /* 26 */ NODEV, /* 27 */ NODEV, + /* 28 */ NODEV, /* 29 */ NODEV, + /* 30 */ NODEV, /* 31 */ NODEV, + /* 32 */ NODEV, /* 33 */ NODEV, + /* 34 */ NODEV, /* 35 */ NODEV, + /* 36 */ NODEV, /* 37 */ NODEV, + /* 38 */ NODEV, /* 39 */ NODEV, + /* 40 */ NODEV, /* 41 */ NODEV, + /* 42 */ NODEV, /* 43 */ NODEV, + /* 44 */ NODEV, +}; + +/* + * convert chr dev to blk dev + */ +dev_t +chrtoblk(dev_t dev) +{ + int blkmaj; + + if (major(dev) >= nchrdev) + return (NODEV); + blkmaj = chrtoblktab[major(dev)]; + if (blkmaj == NODEV) + return (NODEV); + return (makedev(blkmaj, minor(dev))); +} + +int +chrtoblk_set(int cdev, int bdev) +{ + if (cdev >= nchrdev) + return (-1); + if (bdev != NODEV && bdev >= nblkdev) + return (-1); + chrtoblktab[cdev] = bdev; + return 0; +} diff --git a/bsd/dev/arm64/cpu_in_cksum.s b/bsd/dev/arm64/cpu_in_cksum.s new file mode 100644 index 000000000..b01b27172 --- /dev/null +++ b/bsd/dev/arm64/cpu_in_cksum.s @@ -0,0 +1,404 @@ +/* + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. + * + * This document is the property of Apple Inc. + * It is considered confidential and proprietary. + * + * This document may not be reproduced or transmitted in any form, + * in whole or in part, without the express written permission of + * Apple Inc. + */ + +/* + * This assembly was previously cloned from ../arm/cpu_in_cksum.s (__arm__) + * with __arm64__ tagged ARM64_TODO . This code revision is optimized based + * on the 64-bit part in netinet/cpu_in_cksum.c + * + * cclee - CoreOS - Vector & Numerics. 06/20/2012. + */ + +#ifdef KERNEL +#define CKSUM_ERR _kprintf +#else +#ifndef LIBSYSCALL_INTERFACE +#error "LIBSYSCALL_INTERFACE not defined" +#endif /* !LIBSYSCALL_INTERFACE */ +#define CKSUM_ERR _fprintf_stderr +#endif /* !KERNEL */ + +/* + * XXX: adi@apple.com: + * + * Ugly, but we have little choice, since relying on genassym and <assym.s> + * is not possible unless this code lives in osfmk. Note also that this + * routine expects "mbuf-like" argument, and it does not expect the mbuf to be + * authentic; it only cares about 3 fields. + */ +#define M_NEXT 0 +#define M_DATA 16 // 8-byte address, would be aligned to 8-byte boundary +#define M_LEN 24 + + .globl _os_cpu_in_cksum_mbuf + .text + .align 4 +_os_cpu_in_cksum_mbuf: + + +/* + * 64-bit version. + * + * This function returns the partial 16-bit checksum accumulated in + * a 32-bit variable (withouth 1's complement); caller is responsible + * for folding the 32-bit sum into 16-bit and performinng the 1's + * complement if applicable + */ + +/* + * uint32_t + * os_cpu_in_cksum_mbuf(struct mbuf *m, int len, int off, uint32_t initial_sum) + * { + * int mlen; + * uint64_t sum, partial; + * unsigned int final_acc; + * uint8_t *data; + * boolean_t needs_swap, started_on_odd; + * + * VERIFY(len >= 0); + * VERIFY(off >= 0); + * + * needs_swap = FALSE; + * started_on_odd = FALSE; + * sum = initial_sum; + */ + + #define m x0 + #define len x1 + #define off x2 + #define sum x3 + #define needs_swap x4 + #define started_on_odd x5 + #define mlen x6 + #define Wmlen w6 + #define t x7 + #define data x8 + + mov needs_swap, #0 // needs_swap = FALSE; + mov started_on_odd, #0 // started_on_odd = FALSE; + mov w3, w3 // clear higher half + + +/* + * for (;;) { + * if (PREDICT_FALSE(m == NULL)) { + * CKSUM_ERR("%s: out of data\n", __func__); + * return (-1); + * } + * mlen = m->m_len; + * if (mlen > off) { + * mlen -= off; + * data = mtod(m, uint8_t *) + off; + * goto post_initial_offset; + * } + * off -= mlen; + * if (len == 0) + * break; + * m = m->m_next; + * } + */ + +0: + cbz m, Lin_cksum_whoops // if (m == NULL) return -1; + ldr Wmlen, [m, #M_LEN] // mlen = m->m_len; + cmp mlen, off + b.le 1f + ldr data, [m, #M_DATA] // mtod(m, uint8_t *) + sub mlen, mlen, off // mlen -= off; + add data, data, off // data = mtod(m, uint8_t *) + off; + b L_post_initial_offset +1: + sub off, off, mlen + cbnz len, 2f + mov x0, x3 + ret lr +2: + ldr m, [m, #M_NEXT] + b 0b + +L_loop: // for (; len > 0; m = m->m_next) { +/* + * if (PREDICT_FALSE(m == NULL)) { + * CKSUM_ERR("%s: out of data\n", __func__); + * return (-1); + * } + * mlen = m->m_len; + * data = mtod(m, uint8_t *); + */ + cbz m, Lin_cksum_whoops // if (m == NULL) return -1; + ldr Wmlen, [m, #M_LEN] // mlen = m->m_len; + ldr data, [m, #M_DATA] // mtod(m, uint8_t *) + +L_post_initial_offset: +/* + * if (mlen == 0) continue; + * if (mlen > len) mlen = len; + * len -= mlen; + */ + + cbz mlen, L_continue + cmp mlen, len + csel mlen, mlen, len, le + sub len, len, mlen + +/* + * partial = 0; + * if ((uintptr_t)data & 1) { + * started_on_odd = !started_on_odd; + * partial = *data << 8; + * ++data; + * --mlen; + * } + * needs_swap = started_on_odd; + */ + + tst data, #1 + mov x7, #0 + mov x10, #0 + b.eq 1f + ldrb w7, [data], #1 + eor started_on_odd, started_on_odd, #1 + sub mlen, mlen, #1 + lsl w7, w7, #8 +1: + + +/* + * if ((uintptr_t)data & 2) { + * if (mlen < 2) + * goto trailing_bytes; + * partial += *(uint16_t *)(void *)data; + * data += 2; + * mlen -= 2; + * } + */ + tst data, #2 + mov needs_swap, started_on_odd + b.eq 1f + cmp mlen, #2 + b.lt L_trailing_bytes + ldrh w9, [data], #2 + sub mlen, mlen, #2 + add w7, w7, w9 +1: + +/* + * while (mlen >= 64) { + * __builtin_prefetch(data + 32); + * __builtin_prefetch(data + 64); + * partial += *(uint32_t *)(void *)data; + * partial += *(uint32_t *)(void *)(data + 4); + * partial += *(uint32_t *)(void *)(data + 8); + * partial += *(uint32_t *)(void *)(data + 12); + * partial += *(uint32_t *)(void *)(data + 16); + * partial += *(uint32_t *)(void *)(data + 20); + * partial += *(uint32_t *)(void *)(data + 24); + * partial += *(uint32_t *)(void *)(data + 28); + * partial += *(uint32_t *)(void *)(data + 32); + * partial += *(uint32_t *)(void *)(data + 36); + * partial += *(uint32_t *)(void *)(data + 40); + * partial += *(uint32_t *)(void *)(data + 44); + * partial += *(uint32_t *)(void *)(data + 48); + * partial += *(uint32_t *)(void *)(data + 52); + * partial += *(uint32_t *)(void *)(data + 56); + * partial += *(uint32_t *)(void *)(data + 60); + * data += 64; + * mlen -= 64; + * // if (PREDICT_FALSE(partial & (3ULL << 62))) { + * // if (needs_swap) + * // partial = (partial << 8) + + * // (partial >> 56); + * // sum += (partial >> 32); + * // sum += (partial & 0xffffffff); + * // partial = 0; + * // } + * } +*/ + + // pre-decrement mlen by 64, and if < 64 bytes, try 32 bytes next + subs mlen, mlen, #64 + b.lt L32_bytes + + // save used vector registers + sub sp, sp, #8*16 + mov x11, sp + st1.4s {v0, v1, v2, v3}, [x11], #4*16 + st1.4s {v4, v5, v6, v7}, [x11], #4*16 + + // spread partial into 8 8-byte registers in v0-v3 + fmov s3, w7 + eor.16b v0, v0, v0 + eor.16b v1, v1, v1 + eor.16b v2, v2, v2 + + // load the 1st 64 bytes (16 32-bit words) + ld1.4s {v4,v5,v6,v7},[data],#64 + + // branch to finish off if mlen<64 + subs mlen, mlen, #64 + b.lt L64_finishup + + /* + * loop for loading and accumulating 16 32-bit words into + * 8 8-byte accumulators per iteration. + */ +L64_loop: + subs mlen, mlen, #64 // mlen -= 64 + + uadalp.2d v0, v4 + ld1.4s {v4},[data], #16 + + uadalp.2d v1, v5 + ld1.4s {v5},[data], #16 + + uadalp.2d v2, v6 + ld1.4s {v6},[data], #16 + + uadalp.2d v3, v7 + ld1.4s {v7},[data], #16 + + b.ge L64_loop + +L64_finishup: + uadalp.2d v0, v4 + uadalp.2d v1, v5 + uadalp.2d v2, v6 + uadalp.2d v3, v7 + + add.2d v0, v0, v1 + add.2d v2, v2, v3 + addp.2d d0, v0 + addp.2d d2, v2 + add.2d v0, v0, v2 + fmov x7, d0 // partial in x7 now + + // restore used vector registers + ld1.4s {v0, v1, v2, v3}, [sp], #4*16 + ld1.4s {v4, v5, v6, v7}, [sp], #4*16 + +L32_bytes: + tst mlen, #32 + b.eq L16_bytes + ldp x9, x10, [data], #16 + ldp x11, x12, [data], #16 + adds x7, x7, x9 + mov x9, #0 + adcs x7, x7, x10 + adcs x7, x7, x11 + adcs x7, x7, x12 + adc x7, x7, x9 + +L16_bytes: + tst mlen, #16 + b.eq L8_bytes + ldp x9, x10, [data], #16 + adds x7, x7, x9 + mov x9, #0 + adcs x7, x7, x10 + adc x7, x7, x9 + +L8_bytes: + tst mlen, #8 + mov x10, #0 + b.eq L4_bytes + ldr x9,[data],#8 + adds x7, x7, x9 + adc x7, x7, x10 + +L4_bytes: + tst mlen, #4 + b.eq L2_bytes + ldr w9,[data],#4 + adds x7, x7, x9 + adc x7, x7, x10 + +L2_bytes: + tst mlen, #2 + b.eq L_trailing_bytes + ldrh w9,[data],#2 + adds x7, x7, x9 + adc x7, x7, x10 + +L_trailing_bytes: + tst mlen, #1 + b.eq L0_bytes + ldrb w9,[data],#1 + adds x7, x7, x9 + adc x7, x7, x10 + eor started_on_odd, started_on_odd, #1 + +L0_bytes: +/* + * if (needs_swap) + * partial = (partial << 8) + (partial >> 56); + */ + cbz needs_swap, 1f + ror x7, x7, #56 +1: +/* + * sum += (partial >> 32) + (partial & 0xffffffff); + * sum = (sum >> 32) + (sum & 0xffffffff); + * } + */ + + add x3, x3, x7, lsr #32 + mov w7, w7 + add x3, x3, x7 + mov w7, w3 + add x3, x7, x3, lsr #32 + +L_continue: + cmp len, #0 + ldr m, [m, #M_NEXT] // m = m->m_next + b.gt L_loop + +/* + * final_acc = (sum >> 48) + ((sum >> 32) & 0xffff) + + * ((sum >> 16) & 0xffff) + (sum & 0xffff); + * final_acc = (final_acc >> 16) + (final_acc & 0xffff); + * final_acc = (final_acc >> 16) + (final_acc & 0xffff); + * return (final_acc & 0xffff); + * } + */ + + mov w4, #0x00ffff + and x0, x4, x3, lsr #48 + and x1, x4, x3, lsr #32 + and x2, x4, x3, lsr #16 + and x3, x4, x3 + add w0, w0, w1 + add w2, w2, w3 + add w0, w0, w2 + and w1, w4, w0, lsr #16 + and w0, w4, w0 + add w0, w0, w1 + and w1, w4, w0, lsr #16 + and w0, w4, w0 + add w0, w0, w1 + /* + * If we were to 1's complement it (XOR with 0xffff): + * + * eor w0, w0, w4 + */ + and w0, w0, w4 + + ret lr + +Lin_cksum_whoops: + adrp x0, Lin_cksum_whoops_str@page + add x0, x0, Lin_cksum_whoops_str@pageoff + bl #CKSUM_ERR + mov x0, #-1 + ret lr + +Lin_cksum_whoops_str: + .asciz "os_cpu_in_cksum_mbuf: out of data\n" + .align 5 diff --git a/bsd/dev/arm64/disassembler.c b/bsd/dev/arm64/disassembler.c new file mode 100644 index 000000000..a00f8d0eb --- /dev/null +++ b/bsd/dev/arm64/disassembler.c @@ -0,0 +1,1146 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * Disassemblers for ARM (arm), Thumb (thumb16), and Thumb2 (thumb32). + * + * Each disassembly begins with a call to dtrace_decode_arm or dtrace_decode_thumb. The thumb + * decoder will then call dtrace_decode_thumb16 or dtrace_decode_thumb32 as appropriate. + * + * The respective disassembly functions are all of the form {arm,thumb16,thumb32}_type. They + * follow the ordering and breakdown in the ARMv7 Architecture Reference Manual. + */ + +#include <sys/fasttrap_isa.h> + +#define BITS(x,n,mask) (((x) >> (n)) & (mask)) + +static uint32_t thumb32_instword_to_arm(uint16_t hw1, uint16_t hw2) +{ + return (hw1 << 16) | hw2; +} + +int dtrace_decode_arm(uint32_t instr); +int dtrace_decode_arm64(uint32_t instr); +int dtrace_decode_thumb(uint32_t instr); + +/* + * VFP decoder - shared between ARM and THUMB32 mode + */ + +static +int vfp_struct_loadstore(uint32_t instr) +{ + if (ARM_RM(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int vfp_64transfer(uint32_t instr) +{ + /* These instructions all use RD and RN */ + if (ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int vfp_transfer(uint32_t instr) +{ + /* These instructions all use RD only */ + if (ARM_RD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int vfp_loadstore(uint32_t instr) +{ + int opcode = BITS(instr,20,0x1F); + + /* Instrument VLDR */ + if ((opcode & 0x13) == 0x11 && ARM_RN(instr) == REG_PC) + return FASTTRAP_T_VLDR_PC_IMMED; + + /* These instructions all use RN only */ + if (ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +/* + * ARM decoder + */ + +static +int arm_unconditional_misc(uint32_t instr) +{ + int op = BITS(instr,20,0x7F); + + if ((op & 0x60) == 0x20) { + /* VFP data processing uses its own registers */ + return FASTTRAP_T_COMMON; + } + + if ((op & 0x71) == 0x40) { + return vfp_struct_loadstore(instr); + } + + return FASTTRAP_T_INV; +} + +static +int arm_unconditional(uint32_t instr) +{ + if (BITS(instr,27,0x1) == 0) + return arm_unconditional_misc(instr); + + /* The rest are privileged or BL/BLX, do not instrument */ + + /* Do not need to instrument BL/BLX either, see comment in arm_misc(uint32_t) */ + + return FASTTRAP_T_INV; +} + +static +int arm_syscall_coproc(uint32_t instr) +{ + /* Instrument any VFP data processing instructions, ignore the rest */ + + int op1 = BITS(instr,20,0x3F), coproc = BITS(instr,8,0xF), op = BITS(instr,4,0x1); + + if ((op1 & 0x3E) == 0 || (op1 & 0x30) == 0x30) { + /* Undefined or swi */ + return FASTTRAP_T_INV; + } + + if ((coproc & 0xE) == 0xA) { + /* VFP instruction */ + + if ((op1 & 0x20) == 0 && (op1 & 0x3A) != 0) + return vfp_loadstore(instr); + + if ((op1 & 0x3E) == 0x04) + return vfp_64transfer(instr); + + if ((op1 & 0x30) == 0x20) { + /* VFP data processing or 8, 16, or 32 bit move between ARM reg and VFP reg */ + if (op == 0) { + /* VFP data processing uses its own registers */ + return FASTTRAP_T_COMMON; + } else { + return vfp_transfer(instr); + } + } + } + + return FASTTRAP_T_INV; +} + +static +int arm_branch_link_blockdata(uint32_t instr) +{ + int branch = BITS(instr,25,0x1), link = BITS(instr,24,0x1), op = BITS(instr,20,0x1F), uses_pc = BITS(instr,15,0x1), uses_lr = BITS(instr,14,0x1); + + if (branch == 1) { + if (link == 0) + return FASTTRAP_T_B_COND; + return FASTTRAP_T_INV; + } else { + /* Only emulate a use of the pc if it's a return from function: ldmia sp!, { ... pc } */ + if (op == 0x0B && ARM_RN(instr) == REG_SP && uses_pc == 1) + return FASTTRAP_T_LDM_PC; + + /* stmia sp!, { ... lr } doesn't touch the pc, but it is very common, so special case it */ + if (op == 0x12 && ARM_RN(instr) == REG_SP && uses_lr == 1) + return FASTTRAP_T_STM_LR; + + if (ARM_RN(instr) != REG_PC && uses_pc == 0) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int arm_signed_multiplies(uint32_t instr) +{ + int op1 = BITS(instr,20,0x7), op2 = BITS(instr,5,0x7); + + /* smlald, smlsld, smmls use RD in addition to RM, RS, and RN */ + if ((op1 == 0x4 && (op2 & 0x4) == 0) || (op1 == 0x5 && (op2 & 0x6) == 0x6)) { + if (ARM_RD(instr) == REG_PC) + return FASTTRAP_T_INV; + } + + if (ARM_RM(instr) != REG_PC && ARM_RS(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_pack_unpack_sat_reversal(uint32_t instr) +{ + int op1 = BITS(instr,20,0x7), op2 = BITS(instr,5,0x7); + + /* pkh, sel use RN in addition to RD and RM */ + if ((op1 == 0 && (op2 & 0x1) == 0) || (op1 == 0 && op2 == 0x5)) { + if (ARM_RN(instr) == REG_PC) + return FASTTRAP_T_INV; + } + + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_parallel_addsub_unsigned(uint32_t instr) +{ + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_parallel_addsub_signed(uint32_t instr) +{ + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_media(uint32_t instr) +{ + int op1 = BITS(instr,20,0x1F), op2 = BITS(instr,5,0x7); + + if ((op1 & 0x1C) == 0) + return arm_parallel_addsub_signed(instr); + + if ((op1 & 0x1C) == 0x04) + return arm_parallel_addsub_unsigned(instr); + + if ((op1 & 0x18) == 0x08) + return arm_pack_unpack_sat_reversal(instr); + + if ((op1 & 0x18) == 0x10) + return arm_signed_multiplies(instr); + + if (op1 == 0x1F && op2 == 0x7) { + /* Undefined instruction */ + return FASTTRAP_T_INV; + } + + if (op1 == 0x18 && op2 == 0) { + /* usad8 usada8 */ + /* The registers are named differently in the reference manual for this instruction + * but the following positions are correct */ + + if (ARM_RM(instr) != REG_PC && ARM_RS(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; + } + + if ((op1 & 0x1E) == 0x1C && (op2 & 0x3) == 0) { + /* bfc bfi */ + if (ARM_RD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; + } + + if (((op1 & 0x1E) == 0x1A || (op1 & 0x1E) == 0x1E) && ((op2 & 0x3) == 0x2)) { + /* sbfx ubfx */ + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; + } + + return FASTTRAP_T_INV; +} + +static +int arm_loadstore_wordbyte(uint32_t instr) +{ + /* Instrument PC relative load with immediate, ignore any other uses of the PC */ + int R = BITS(instr,25,0x1), L = BITS(instr,20,0x1); + + if (R == 1) { + /* Three register load/store */ + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + } else { + /* Immediate load/store, but still do not support ldr pc, [pc...] */ + if (L == 1 && ARM_RN(instr) == REG_PC && ARM_RD(instr) != REG_PC) + return FASTTRAP_T_LDR_PC_IMMED; + + if (ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int arm_saturating(uint32_t instr) +{ + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_misc(uint32_t instr) +{ + int op = BITS(instr,21,0x3), __unused op1 = BITS(instr,16,0xF), op2 = BITS(instr,4,0x7); + + if (op2 == 1 && op == 1) + return FASTTRAP_T_BX_REG; + + /* We do not need to emulate BLX for entry/return probes; if we eventually support full offset + * tracing, then we will. This is because BLX overwrites the link register, so a function that + * can execute this as its first instruction is a special function indeed. + */ + + if (op2 == 0x5) + return arm_saturating(instr); + + return FASTTRAP_T_INV; +} + +static +int arm_msr_hints(__unused uint32_t instr) +{ + /* These deal with the psr, not instrumented */ + + return FASTTRAP_T_INV; +} + +static +int arm_sync_primitive(__unused uint32_t instr) +{ + /* TODO will instrumenting these interfere with any kernel usage of these instructions? */ + /* Don't instrument for now */ + + return FASTTRAP_T_INV; +} + +static +int arm_extra_loadstore_unpriv(uint32_t instr) +{ + int op = BITS(instr,20,0x1), __unused op2 = BITS(instr,5,0x3), immed = BITS(instr,22,0x1); + + if (op == 0 && (op2 & 0x2) == 0x2) { + /* Unpredictable or undefined */ + return FASTTRAP_T_INV; + } + + if (immed == 1) { + if (ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + } else { + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int arm_extra_loadstore(uint32_t instr) +{ + int op1 = BITS(instr,20,0x1F); + + /* There are two variants, and we do not instrument either of them that use the PC */ + + if ((op1 & 0x4) == 0) { + /* Variant 1, register */ + if (ARM_RM(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + } else { + /* Variant 2, immediate */ + if (ARM_RD(instr) != REG_PC && ARM_RN(instr) != REG_PC) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int arm_halfword_multiply(uint32_t instr) +{ + /* Not all multiply instructions use all four registers. The ones that don't should have those + * register locations set to 0, so we can test them anyway. + */ + + if (ARM_RN(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RS(instr) != REG_PC && ARM_RM(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_multiply(uint32_t instr) +{ + /* Not all multiply instructions use all four registers. The ones that don't should have those + * register locations set to 0, so we can test them anyway. + */ + + if (ARM_RN(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RS(instr) != REG_PC && ARM_RM(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_dataproc_immed(uint32_t instr) +{ + /* All these instructions are either two registers, or one register and have 0 where the other reg would be used */ + if (ARM_RN(instr) != REG_PC && ARM_RD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_dataproc_regshift(uint32_t instr) +{ + /* All these instructions are either four registers, or three registers and have 0 where there last reg would be used */ + if (ARM_RN(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RS(instr) != REG_PC && ARM_RM(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_dataproc_reg(uint32_t instr) +{ + int op1 = BITS(instr,20,0x1F), op2 = BITS(instr,7,0x1F), op3 = BITS(instr,5,0x3); + + if (op1 == 0x11 || op1 == 0x13 || op1 == 0x15 || op1 == 0x17) { + /* These are comparison flag setting instructions and do not have RD */ + if (ARM_RN(instr) != REG_PC && ARM_RM(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; + } + + /* The rest can, in theory, write or use the PC. The only one we instrument is mov pc, reg. + * movs pc, reg is a privileged instruction so we don't instrument that variant. The s bit + * is bit 0 of op1 and should be zero. + */ + if (op1 == 0x1A && op2 == 0 && op3 == 0 && ARM_RD(instr) == REG_PC) + return FASTTRAP_T_MOV_PC_REG; + + /* Any instruction at this point is a three register instruction or two register instruction with RN = 0 */ + if (ARM_RN(instr) != REG_PC && ARM_RD(instr) != REG_PC && ARM_RM(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int arm_dataproc_misc(uint32_t instr) +{ + int op = BITS(instr,25,0x1), op1 = BITS(instr,20,0x1F), op2 = BITS(instr,4,0xF); + + if (op == 0) { + if ((op1 & 0x19) != 0x10 && (op2 & 0x1) == 0) + return arm_dataproc_reg(instr); + + if ((op1 & 0x19) != 0x10 && (op2 & 0x9) == 0x1) + return arm_dataproc_regshift(instr); + + if ((op1 & 0x19) == 0x10 && (op2 & 0x8) == 0) + return arm_misc(instr); + + if ((op1 & 0x19) == 0x19 && (op2 & 0x9) == 0x8) + return arm_halfword_multiply(instr); + + if ((op1 & 0x10) == 0 && op2 == 0x9) + return arm_multiply(instr); + + if ((op1 & 0x10) == 0x10 && op2 == 0x9) + return arm_sync_primitive(instr); + + if ((op1 & 0x12) != 0x02 && (op2 == 0xB || (op2 & 0xD) == 0xD)) + return arm_extra_loadstore(instr); + + if ((op1 & 0x12) == 0x02 && (op2 == 0xB || (op2 & 0xD) == 0xD)) + return arm_extra_loadstore_unpriv(instr); + } else { + if ((op1 & 0x19) != 0x10) + return arm_dataproc_immed(instr); + + if (op1 == 0x10) { + /* 16 bit immediate load (mov (immed)) [encoding A2] */ + if (ARM_RD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; + } + + if (op1 == 0x14) { + /* high halfword 16 bit immediate load (movt) [encoding A1] */ + if (ARM_RD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; + } + + if ((op1 & 0x1B) == 0x12) + return arm_msr_hints(instr); + } + + return FASTTRAP_T_INV; +} + +int dtrace_decode_arm(uint32_t instr) +{ + int cond = BITS(instr,28,0xF), op1 = BITS(instr,25,0x7), op = BITS(instr,4,0x1); + + if (cond == 0xF) + return arm_unconditional(instr); + + if ((op1 & 0x6) == 0) + return arm_dataproc_misc(instr); + + if (op1 == 0x2) + return arm_loadstore_wordbyte(instr); + + if (op1 == 0x3 && op == 0) + return arm_loadstore_wordbyte(instr); + + if (op1 == 0x3 && op == 1) + return arm_media(instr); + + if ((op1 & 0x6) == 0x4) + return arm_branch_link_blockdata(instr); + + if ((op1 & 0x6) == 0x6) + return arm_syscall_coproc(instr); + + return FASTTRAP_T_INV; +} + +/* + * Thumb 16-bit decoder + */ + +static +int thumb16_cond_supervisor(uint16_t instr) +{ + int opcode = BITS(instr,8,0xF); + + if ((opcode & 0xE) != 0xE) + return FASTTRAP_T_B_COND; + + return FASTTRAP_T_INV; +} + +static +int thumb16_misc(uint16_t instr) +{ + int opcode = BITS(instr,5,0x7F); + + if ((opcode & 0x70) == 0x30 || (opcode & 0x70) == 0x70) { + /* setend, cps, breakpoint, or if-then, not instrumentable */ + return FASTTRAP_T_INV; + } else if ((opcode & 0x78) == 0x28) { + /* Doesn't modify pc, but this happens a lot so make this a special case for emulation */ + return FASTTRAP_T_PUSH_LR; + } else if ((opcode & 0x78) == 0x68) { + return FASTTRAP_T_POP_PC; + } else if ((opcode & 0x28) == 0x08) { + return FASTTRAP_T_CB_N_Z; + } + + /* All other instructions work on low regs only and are instrumentable */ + return FASTTRAP_T_COMMON; +} + +static +int thumb16_loadstore_single(__unused uint16_t instr) +{ + /* These all access the low registers or SP only */ + return FASTTRAP_T_COMMON; +} + +static +int thumb16_data_special_and_branch(uint16_t instr) +{ + int opcode = BITS(instr,6,0xF); + + if (opcode == 0x4) { + /* Unpredictable */ + return FASTTRAP_T_INV; + } else if ((opcode & 0xC) == 0xC) { + /* bx or blx */ + /* Only instrument the bx */ + if ((opcode & 0x2) == 0) + return FASTTRAP_T_BX_REG; + return FASTTRAP_T_INV; + } else { + /* Data processing on high registers, only instrument mov pc, reg */ + if ((opcode & 0xC) == 0x8 && THUMB16_HRD(instr) == REG_PC) + return FASTTRAP_T_CPY_PC; + + if (THUMB16_HRM(instr) != REG_PC && THUMB16_HRD(instr) != REG_PC) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int thumb16_data_proc(__unused uint16_t instr) +{ + /* These all access the low registers only */ + return FASTTRAP_T_COMMON; +} + +static +int thumb16_shift_addsub_move_compare(__unused uint16_t instr) +{ + /* These all access the low registers only */ + return FASTTRAP_T_COMMON; +} + +static +int dtrace_decode_thumb16(uint16_t instr) +{ + int opcode = BITS(instr,10,0x3F); + + if ((opcode & 0x30) == 0) + return thumb16_shift_addsub_move_compare(instr); + + if (opcode == 0x10) + return thumb16_data_proc(instr); + + if (opcode == 0x11) + return thumb16_data_special_and_branch(instr); + + if ((opcode & 0x3E) == 0x12) { + /* ldr (literal) */ + return FASTTRAP_T_LDR_PC_IMMED; + } + + if ((opcode & 0x3C) == 0x14 || (opcode & 0x38) == 0x18 || (opcode & 0x38) == 0x20) + return thumb16_loadstore_single(instr); + + if ((opcode & 0x3E) == 0x28) { + /* adr, uses the pc */ + return FASTTRAP_T_INV; + } + + if ((opcode & 0x3E) == 0x2A) { + /* add (sp plus immediate) */ + return FASTTRAP_T_COMMON; + } + + if ((opcode & 0x3C) == 0x2C) + return thumb16_misc(instr); + + if ((opcode & 0x3E) == 0x30) { + /* stm - can't access high registers */ + return FASTTRAP_T_COMMON; + } + + if ((opcode & 0x3E) == 0x32) { + /* ldm - can't access high registers */ + return FASTTRAP_T_COMMON; + } + + if ((opcode & 0x3C) == 0x34) { + return thumb16_cond_supervisor(instr); + } + + if ((opcode & 0x3E) == 0x38) { + /* b unconditional */ + return FASTTRAP_T_B_UNCOND; + } + + return FASTTRAP_T_INV; +} + +/* + * Thumb 32-bit decoder + */ + +static +int thumb32_coproc(uint16_t instr1, uint16_t instr2) +{ + /* Instrument any VFP data processing instructions, ignore the rest */ + + int op1 = BITS(instr1,4,0x3F), coproc = BITS(instr2,8,0xF), op = BITS(instr2,4,0x1); + + if ((op1 & 0x3E) == 0) { + /* Undefined */ + return FASTTRAP_T_INV; + } + + if ((coproc & 0xE) == 0xA || (op1 & 0x30) == 0x30) { + /* VFP instruction */ + uint32_t instr = thumb32_instword_to_arm(instr1,instr2); + + if ((op1 & 0x30) == 0x30) { + /* VFP data processing uses its own registers */ + return FASTTRAP_T_COMMON; + } + + if ((op1 & 0x3A) == 0x02 || (op1 & 0x38) == 0x08 || (op1 & 0x30) == 0x10) + return vfp_loadstore(instr); + + if ((op1 & 0x3E) == 0x04) + return vfp_64transfer(instr); + + if ((op1 & 0x30) == 0x20) { + /* VFP data processing or 8, 16, or 32 bit move between ARM reg and VFP reg */ + if (op == 0) { + /* VFP data processing uses its own registers */ + return FASTTRAP_T_COMMON; + } else { + return vfp_transfer(instr); + } + } + } + + return FASTTRAP_T_INV; +} + +static +int thumb32_longmultiply(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,4,0x7), op2 = BITS(instr2,4,0xF); + + if ((op1 == 1 && op2 == 0xF) || (op1 == 0x3 && op2 == 0xF)) { + /* Three register instruction */ + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } else { + /* Four register instruction */ + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && + THUMB32_RT(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int thumb32_multiply(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,4,0x7), op2 = BITS(instr2,4,0x3); + + if ((op1 == 0 && op2 == 1) || (op1 == 0x6 && (op2 & 0x2) == 0)) { + if (THUMB32_RT(instr1,instr2) == REG_PC) + return FASTTRAP_T_INV; + } + + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_misc(uint16_t instr1, uint16_t instr2) +{ + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_parallel_addsub_unsigned(uint16_t instr1, uint16_t instr2) +{ + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_parallel_addsub_signed(uint16_t instr1, uint16_t instr2) +{ + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_dataproc_reg(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,4,0xF), op2 = BITS(instr2,4,0xF); + + if (((0 <= op1) && (op1 <= 5)) && (op2 & 0x8) == 0x8) { + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } + + if ((op1 & 0x8) == 0 && op2 == 0) { + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } + + if ((op1 & 0x8) == 0x8 && (op2 & 0xC) == 0) + return thumb32_parallel_addsub_signed(instr1,instr2); + + if ((op1 & 0x8) == 0x8 && (op2 & 0xC) == 0x4) + return thumb32_parallel_addsub_unsigned(instr1,instr2); + + if ((op1 & 0xC) == 0x8 && (op2 & 0xC) == 0x8) + return thumb32_misc(instr1,instr2); + + return FASTTRAP_T_INV; +} + +static +int thumb32_dataproc_regshift(uint16_t instr1, uint16_t instr2) +{ + int op = BITS(instr1,5,0xF), S = BITS(instr1,4,0x1); + + if (op == 0 || op == 0x4 || op == 0x8 || op == 0xD) { + /* These become test instructions if S is 1 and Rd is PC, otherwise they are data instructions. */ + if (S == 1) { + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } else { + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && + THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } + } else if (op == 0x2 || op == 0x3) { + /* These become moves if RN is PC, otherwise they are data insts. We don't instrument mov pc, reg here */ + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } else { + /* Normal three register instruction */ + if (THUMB32_RM(instr1,instr2) != REG_PC && THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int thumb32_store_single(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,5,0x7), op2 = BITS(instr2,6,0x3F); + + /* Do not support any use of the pc yet */ + if ((op1 == 0 || op1 == 1 || op1 == 2) && (op2 & 0x20) == 0) { + /* str (register) uses RM */ + if (THUMB32_RM(instr1,instr2) == REG_PC) + return FASTTRAP_T_INV; + } + + if (THUMB32_RT(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_loadbyte_memhint(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,7,0x3), __unused op2 = BITS(instr2,6,0x3F); + + /* Do not support any use of the pc yet */ + if ((op1 == 0 || op1 == 0x2) && THUMB32_RM(instr1,instr2) == REG_PC) + return FASTTRAP_T_INV; + + if (THUMB32_RT(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_loadhalfword_memhint(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,7,0x3), op2 = BITS(instr2,6,0x3F); + + /* Do not support any use of the PC yet */ + if (op1 == 0 && op2 == 0 && THUMB32_RM(inst1,instr2) == REG_PC) + return FASTTRAP_T_INV; + + if (THUMB32_RT(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_loadword(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,7,0x3), op2 = BITS(instr2,6,0x3F); + + if ((op1 & 0x2) == 0 && THUMB32_RN(instr1,instr2) == REG_PC && THUMB32_RT(instr1,instr2) != REG_PC) + return FASTTRAP_T_LDR_PC_IMMED; + + if (op1 == 0 && op2 == 0) { + /* ldr (register) uses an additional reg */ + if (THUMB32_RM(instr1,instr2) == REG_PC) + return FASTTRAP_T_INV; + } + + if (THUMB32_RT(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_loadstore_double_exclusive_table(__unused uint16_t instr1, __unused uint16_t instr2) +{ + /* Don't instrument any of these */ + + return FASTTRAP_T_INV; +} + +static +int thumb32_loadstore_multiple(uint16_t instr1, uint16_t instr2) +{ + int op = BITS(instr1,7,0x3), L = BITS(instr1,4,0x1), uses_pc = BITS(instr2,15,0x1), uses_lr = BITS(instr2,14,0x1); + + if (op == 0 || op == 0x3) { + /* Privileged instructions: srs, rfe */ + return FASTTRAP_T_INV; + } + + /* Only emulate a use of the pc if it's a return from function: ldmia sp!, { ... pc }, aka pop { ... pc } */ + if (op == 0x1 && L == 1 && THUMB32_RN(instr1,instr2) == REG_SP && uses_pc == 1) + return FASTTRAP_T_LDM_PC; + + /* stmia sp!, { ... lr }, aka push { ... lr } doesn't touch the pc, but it is very common, so special case it */ + if (op == 0x2 && L == 0 && THUMB32_RN(instr1,instr2) == REG_SP && uses_lr == 1) + return FASTTRAP_T_STM_LR; + + if (THUMB32_RN(instr1,instr2) != REG_PC && uses_pc == 0) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int thumb32_misc_control(__unused uint16_t instr1, __unused uint16_t instr2) +{ + /* Privileged, and instructions dealing with ThumbEE */ + return FASTTRAP_T_INV; +} + +static +int thumb32_cps_hints(__unused uint16_t instr1, __unused uint16_t instr2) +{ + /* Privileged */ + return FASTTRAP_T_INV; +} + +static +int thumb32_b_misc_control(uint16_t instr1, uint16_t instr2) +{ + int op = BITS(instr1,4,0x7F), op1 = BITS(instr2,12,0x7), __unused op2 = BITS(instr2,8,0xF); + + if ((op1 & 0x5) == 0) { + if ((op & 0x38) != 0x38) + return FASTTRAP_T_B_COND; + + if (op == 0x3A) + return thumb32_cps_hints(instr1,instr2); + + if (op == 0x3B) + return thumb32_misc_control(instr1,instr2); + } + + if ((op1 & 0x5) == 1) + return FASTTRAP_T_B_UNCOND; + + return FASTTRAP_T_INV; +} + +static +int thumb32_dataproc_plain_immed(uint16_t instr1, uint16_t instr2) +{ + int op = BITS(instr1,4,0x1F); + + if (op == 0x04 || op == 0x0C || op == 0x16) { + /* mov, movt, bfi, bfc */ + /* These use only RD */ + if (THUMB32_RD(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } else { + if (THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } + + return FASTTRAP_T_INV; +} + +static +int thumb32_dataproc_mod_immed(uint16_t instr1, uint16_t instr2) +{ + int op = BITS(instr1,5,0xF), S = BITS(instr1,4,0x1); + + if (op == 0x2 || op == 0x3) { + /* These allow REG_PC in RN, but it doesn't mean use the PC! */ + if (THUMB32_RD(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + } + + if (op == 0 || op == 0x4 || op == 0x8 || op == 0xD) { + /* These are test instructions, if the sign bit is set and RD is the PC. */ + if (S && THUMB32_RD(instr1,instr2) == REG_PC) + return FASTTRAP_T_COMMON; + } + + if (THUMB32_RD(instr1,instr2) != REG_PC && THUMB32_RN(instr1,instr2) != REG_PC) + return FASTTRAP_T_COMMON; + + return FASTTRAP_T_INV; +} + +static +int dtrace_decode_thumb32(uint16_t instr1, uint16_t instr2) +{ + int op1 = BITS(instr1,11,0x3), op2 = BITS(instr1,4,0x7F), op = BITS(instr2,15,0x1); + + if (op1 == 0x1) { + if ((op2 & 0x64) == 0) + return thumb32_loadstore_multiple(instr1,instr2); + + if ((op2 & 0x64) == 0x04) + return thumb32_loadstore_double_exclusive_table(instr1,instr2); + + if ((op2 & 0x60) == 0x20) + return thumb32_dataproc_regshift(instr1,instr2); + + if ((op2 & 0x40) == 0x40) + return thumb32_coproc(instr1,instr2); + } + + if (op1 == 0x2) { + if ((op2 & 0x20) == 0 && op == 0) + return thumb32_dataproc_mod_immed(instr1,instr2); + + if ((op2 & 0x20) == 0x20 && op == 0) + return thumb32_dataproc_plain_immed(instr1,instr2); + + if (op == 1) + return thumb32_b_misc_control(instr1,instr2); + } + + if (op1 == 0x3) { + if ((op2 & 0x71) == 0) + return thumb32_store_single(instr1,instr2); + + if ((op2 & 0x71) == 0x10) { + return vfp_struct_loadstore(thumb32_instword_to_arm(instr1,instr2)); + } + + if ((op2 & 0x67) == 0x01) + return thumb32_loadbyte_memhint(instr1,instr2); + + if ((op2 & 0x67) == 0x03) + return thumb32_loadhalfword_memhint(instr1,instr2); + + if ((op2 & 0x67) == 0x05) + return thumb32_loadword(instr1,instr2); + + if ((op2 & 0x67) == 0x07) { + /* Undefined instruction */ + return FASTTRAP_T_INV; + } + + if ((op2 & 0x70) == 0x20) + return thumb32_dataproc_reg(instr1,instr2); + + if ((op2 & 0x78) == 0x30) + return thumb32_multiply(instr1,instr2); + + if ((op2 & 0x78) == 0x38) + return thumb32_longmultiply(instr1,instr2); + + if ((op2 & 0x40) == 0x40) + return thumb32_coproc(instr1,instr2); + } + + return FASTTRAP_T_INV; +} + +int dtrace_decode_thumb(uint32_t instr) +{ + uint16_t* pInstr = (uint16_t*) &instr; + uint16_t hw1 = pInstr[0], hw2 = pInstr[1]; + + int size = BITS(hw1,11,0x1F); + + if (size == 0x1D || size == 0x1E || size == 0x1F) + return dtrace_decode_thumb32(hw1,hw2); + else + return dtrace_decode_thumb16(hw1); +} + +struct arm64_decode_entry { + uint32_t mask; + uint32_t value; + uint32_t type; +}; + +struct arm64_decode_entry arm64_decode_table[] = { + { .mask = 0xFFFFFFFF, .value = FASTTRAP_ARM64_OP_VALUE_FUNC_ENTRY, .type = FASTTRAP_T_ARM64_STANDARD_FUNCTION_ENTRY }, + { .mask = FASTTRAP_ARM64_OP_MASK_LDR_S_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_LDR_S_PC_REL, .type = FASTTRAP_T_ARM64_LDR_S_PC_REL }, + { .mask = FASTTRAP_ARM64_OP_MASK_LDR_W_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_LDR_W_PC_REL, .type = FASTTRAP_T_ARM64_LDR_W_PC_REL }, + { .mask = FASTTRAP_ARM64_OP_MASK_LDR_D_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_LDR_D_PC_REL, .type = FASTTRAP_T_ARM64_LDR_D_PC_REL }, + { .mask = FASTTRAP_ARM64_OP_MASK_LDR_X_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_LDR_X_PC_REL, .type = FASTTRAP_T_ARM64_LDR_X_PC_REL }, + { .mask = FASTTRAP_ARM64_OP_MASK_LDR_Q_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_LDR_Q_PC_REL, .type = FASTTRAP_T_ARM64_LDR_Q_PC_REL }, + { .mask = FASTTRAP_ARM64_OP_MASK_LRDSW_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_LRDSW_PC_REL, .type = FASTTRAP_T_ARM64_LDRSW_PC_REL }, + { .mask = FASTTRAP_ARM64_OP_MASK_B_COND_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_B_COND_PC_REL, .type = FASTTRAP_T_ARM64_B_COND }, + { .mask = FASTTRAP_ARM64_OP_MASK_CBNZ_W_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_CBNZ_W_PC_REL, .type = FASTTRAP_T_ARM64_CBNZ_W }, + { .mask = FASTTRAP_ARM64_OP_MASK_CBNZ_X_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_CBNZ_X_PC_REL, .type = FASTTRAP_T_ARM64_CBNZ_X }, + { .mask = FASTTRAP_ARM64_OP_MASK_CBZ_W_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_CBZ_W_PC_REL, .type = FASTTRAP_T_ARM64_CBZ_W }, + { .mask = FASTTRAP_ARM64_OP_MASK_CBZ_X_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_CBZ_X_PC_REL, .type = FASTTRAP_T_ARM64_CBZ_X }, + { .mask = FASTTRAP_ARM64_OP_MASK_TBNZ_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_TBNZ_PC_REL, .type = FASTTRAP_T_ARM64_TBNZ }, + { .mask = FASTTRAP_ARM64_OP_MASK_TBZ_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_TBZ_PC_REL, .type = FASTTRAP_T_ARM64_TBZ }, + { .mask = FASTTRAP_ARM64_OP_MASK_B_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_B_PC_REL, .type = FASTTRAP_T_ARM64_B }, + { .mask = FASTTRAP_ARM64_OP_MASK_BL_PC_REL, .value = FASTTRAP_ARM64_OP_VALUE_BL_PC_REL, .type = FASTTRAP_T_ARM64_BL }, + { .mask = FASTTRAP_ARM64_OP_MASK_BLR, .value = FASTTRAP_ARM64_OP_VALUE_BLR, .type = FASTTRAP_T_ARM64_BLR }, + { .mask = FASTTRAP_ARM64_OP_MASK_BR, .value = FASTTRAP_ARM64_OP_VALUE_BR, .type = FASTTRAP_T_ARM64_BR }, + { .mask = FASTTRAP_ARM64_OP_MASK_RET, .value = FASTTRAP_ARM64_OP_VALUE_RET, .type = FASTTRAP_T_ARM64_RET }, + { .mask = FASTTRAP_ARM64_OP_MASK_ADRP, .value = FASTTRAP_ARM64_OP_VALUE_ADRP, .type = FASTTRAP_T_ARM64_ADRP }, + { .mask = FASTTRAP_ARM64_OP_MASK_ADR, .value = FASTTRAP_ARM64_OP_VALUE_ADR, .type = FASTTRAP_T_ARM64_ADR }, + { .mask = FASTTRAP_ARM64_OP_MASK_PRFM, .value = FASTTRAP_ARM64_OP_VALUE_PRFM, .type = FASTTRAP_T_ARM64_PRFM }, + { .mask = FASTTRAP_ARM64_OP_MASK_EXCL_MEM, .value = FASTTRAP_ARM64_OP_VALUE_EXCL_MEM, .type = FASTTRAP_T_ARM64_EXCLUSIVE_MEM }}; + +#define NUM_DECODE_ENTRIES (sizeof(arm64_decode_table) / sizeof(struct arm64_decode_entry)) + + + +int dtrace_decode_arm64(uint32_t instr) +{ + unsigned i; + + for (i = 0; i < NUM_DECODE_ENTRIES; i++) { + if ((instr & arm64_decode_table[i].mask) == arm64_decode_table[i].value) { + return arm64_decode_table[i].type; + } + } + + return FASTTRAP_T_COMMON; +} + + diff --git a/bsd/dev/arm64/dtrace_isa.c b/bsd/dev/arm64/dtrace_isa.c new file mode 100644 index 000000000..3f81cb706 --- /dev/null +++ b/bsd/dev/arm64/dtrace_isa.c @@ -0,0 +1,696 @@ +/* + * Copyright (c) 2005-2008 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#define MACH__POSIX_C_SOURCE_PRIVATE 1 /* pulls in suitable savearea from + * mach/ppc/thread_status.h */ +#include <arm/proc_reg.h> + +#include <kern/thread.h> +#include <mach/thread_status.h> + +#include <stdarg.h> +#include <string.h> +#include <sys/malloc.h> +#include <sys/time.h> +#include <sys/systm.h> +#include <sys/proc.h> +#include <sys/proc_internal.h> +#include <sys/kauth.h> +#include <sys/dtrace.h> +#include <sys/dtrace_impl.h> +#include <libkern/OSAtomic.h> +#include <kern/simple_lock.h> +#include <kern/sched_prim.h> /* for thread_wakeup() */ +#include <kern/thread_call.h> +#include <kern/task.h> +#include <miscfs/devfs/devfs.h> +#include <mach/vm_param.h> + +extern struct arm_saved_state *find_kern_regs(thread_t); + +extern dtrace_id_t dtrace_probeid_error; /* special ERROR probe */ +typedef arm_saved_state_t savearea_t; + +extern lck_attr_t *dtrace_lck_attr; +extern lck_grp_t *dtrace_lck_grp; + + +struct frame { + struct frame *backchain; + uintptr_t retaddr; +}; + +/* + * Atomicity and synchronization + */ +inline void +dtrace_membar_producer(void) +{ +#if __ARM_SMP__ + __asm__ volatile("dmb ish" : : : "memory"); +#else + __asm__ volatile("nop" : : : "memory"); +#endif +} + +inline void +dtrace_membar_consumer(void) +{ +#if __ARM_SMP__ + __asm__ volatile("dmb ish" : : : "memory"); +#else + __asm__ volatile("nop" : : : "memory"); +#endif +} + +/* + * Interrupt manipulation + * XXX dtrace_getipl() can be called from probe context. + */ +int +dtrace_getipl(void) +{ + /* + * XXX Drat, get_interrupt_level is MACH_KERNEL_PRIVATE + * in osfmk/kern/cpu_data.h + */ + /* return get_interrupt_level(); */ + return (ml_at_interrupt_context() ? 1 : 0); +} + +#if __ARM_SMP__ +/* + * MP coordination + */ + +decl_lck_mtx_data(static, dt_xc_lock); +static uint32_t dt_xc_sync; + +typedef struct xcArg { + processorid_t cpu; + dtrace_xcall_t f; + void *arg; +} xcArg_t; + +static void +xcRemote(void *foo) +{ + xcArg_t *pArg = (xcArg_t *) foo; + + if (pArg->cpu == CPU->cpu_id || pArg->cpu == DTRACE_CPUALL) + (pArg->f) (pArg->arg); + + if (hw_atomic_sub(&dt_xc_sync, 1) == 0) + thread_wakeup((event_t) &dt_xc_sync); +} +#endif + +/* + * dtrace_xcall() is not called from probe context. + */ +void +dtrace_xcall(processorid_t cpu, dtrace_xcall_t f, void *arg) +{ +#if __ARM_SMP__ + /* Only one dtrace_xcall in flight allowed */ + lck_mtx_lock(&dt_xc_lock); + + xcArg_t xcArg; + + xcArg.cpu = cpu; + xcArg.f = f; + xcArg.arg = arg; + + cpu_broadcast_xcall(&dt_xc_sync, TRUE, xcRemote, (void*) &xcArg); + + lck_mtx_unlock(&dt_xc_lock); + return; +#else +#pragma unused(cpu) + /* On uniprocessor systems, the cpu should always be either ourselves or all */ + ASSERT(cpu == CPU->cpu_id || cpu == DTRACE_CPUALL); + + (*f)(arg); + return; +#endif +} + +/* + * Initialization + */ +void +dtrace_isa_init(void) +{ + lck_mtx_init(&dt_xc_lock, dtrace_lck_grp, dtrace_lck_attr); + return; +} + + +/** + * Register definitions + */ +#define ARM_FP 7 +#define ARM_SP 13 +#define ARM_LR 14 +#define ARM_PC 15 +#define ARM_CPSR 16 + +#define ARM64_FP 29 +#define ARM64_LR 30 +#define ARM64_SP 31 +#define ARM64_PC 32 +#define ARM64_CPSR 33 + +/* + * Runtime and ABI + */ +uint64_t +dtrace_getreg(struct regs * savearea, uint_t reg) +{ + struct arm_saved_state *regs = (struct arm_saved_state *) savearea; + + if (is_saved_state32(regs)) { + // Fix special registers if user is 32 bits + switch (reg) { + case ARM64_FP: + reg = ARM_FP; + break; + case ARM64_SP: + reg = ARM_SP; + break; + case ARM64_LR: + reg = ARM_LR; + break; + case ARM64_PC: + reg = ARM_PC; + break; + case ARM64_CPSR: + reg = ARM_CPSR; + break; + } + } + + if (!check_saved_state_reglimit(regs, reg)) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); + return (0); + } + + return ((uint64_t)get_saved_state_reg(regs, reg)); +} + +#define RETURN_OFFSET 4 +#define RETURN_OFFSET64 8 + +static int +dtrace_getustack_common(uint64_t * pcstack, int pcstack_limit, user_addr_t pc, + user_addr_t sp) +{ + int ret = 0; + boolean_t is64bit = proc_is64bit(current_proc()); + + ASSERT(pcstack == NULL || pcstack_limit > 0); + + while (pc != 0) { + ret++; + if (pcstack != NULL) { + *pcstack++ = (uint64_t) pc; + pcstack_limit--; + if (pcstack_limit <= 0) + break; + } + + if (sp == 0) + break; + + if (is64bit) { + pc = dtrace_fuword64((sp + RETURN_OFFSET64)); + sp = dtrace_fuword64(sp); + } else { + pc = dtrace_fuword32((sp + RETURN_OFFSET)); + sp = dtrace_fuword32(sp); + } + } + + return (ret); +} + +void +dtrace_getupcstack(uint64_t * pcstack, int pcstack_limit) +{ + thread_t thread = current_thread(); + savearea_t *regs; + user_addr_t pc, sp, fp; + volatile uint16_t *flags = (volatile uint16_t *) & cpu_core[CPU->cpu_id].cpuc_dtrace_flags; + int n; + + if (*flags & CPU_DTRACE_FAULT) + return; + + if (pcstack_limit <= 0) + return; + + /* + * If there's no user context we still need to zero the stack. + */ + if (thread == NULL) + goto zero; + + regs = (savearea_t *) find_user_regs(thread); + if (regs == NULL) + goto zero; + + *pcstack++ = (uint64_t)dtrace_proc_selfpid(); + pcstack_limit--; + + if (pcstack_limit <= 0) + return; + + pc = get_saved_state_pc(regs); + sp = get_saved_state_sp(regs); + fp = get_saved_state_fp(regs); + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) { + *pcstack++ = (uint64_t) pc; + pcstack_limit--; + if (pcstack_limit <= 0) + return; + + pc = get_saved_state_lr(regs); + } + + n = dtrace_getustack_common(pcstack, pcstack_limit, pc, fp); + + ASSERT(n >= 0); + ASSERT(n <= pcstack_limit); + + pcstack += n; + pcstack_limit -= n; + +zero: + while (pcstack_limit-- > 0) + *pcstack++ = 0ULL; +} + +int +dtrace_getustackdepth(void) +{ + thread_t thread = current_thread(); + savearea_t *regs; + user_addr_t pc, sp, fp; + int n = 0; + + if (thread == NULL) + return 0; + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT)) + return (-1); + + regs = (savearea_t *) find_user_regs(thread); + if (regs == NULL) + return 0; + + pc = get_saved_state_pc(regs); + sp = get_saved_state_sp(regs); + fp = get_saved_state_fp(regs); + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) { + n++; + pc = get_saved_state_lr(regs); + } + + /* + * Note that unlike ppc, the arm code does not use + * CPU_DTRACE_USTACK_FP. This is because arm always + * traces from the sp, even in syscall/profile/fbt + * providers. + */ + + n += dtrace_getustack_common(NULL, 0, pc, fp); + + return (n); +} + +void +dtrace_getufpstack(uint64_t * pcstack, uint64_t * fpstack, int pcstack_limit) +{ + thread_t thread = current_thread(); + boolean_t is64bit = proc_is64bit(current_proc()); + savearea_t *regs; + user_addr_t pc, sp; + volatile uint16_t *flags = (volatile uint16_t *) & cpu_core[CPU->cpu_id].cpuc_dtrace_flags; + +#if 0 + uintptr_t oldcontext; + size_t s1, s2; +#endif + + if (*flags & CPU_DTRACE_FAULT) + return; + + if (pcstack_limit <= 0) + return; + + /* + * If there's no user context we still need to zero the stack. + */ + if (thread == NULL) + goto zero; + + regs = (savearea_t *) find_user_regs(thread); + if (regs == NULL) + goto zero; + + *pcstack++ = (uint64_t)dtrace_proc_selfpid(); + pcstack_limit--; + + if (pcstack_limit <= 0) + return; + + pc = get_saved_state_pc(regs); + sp = get_saved_state_lr(regs); + +#if 0 /* XXX signal stack crawl */ + oldcontext = lwp->lwp_oldcontext; + + if (p->p_model == DATAMODEL_NATIVE) { + s1 = sizeof(struct frame) + 2 * sizeof(long); + s2 = s1 + sizeof(siginfo_t); + } else { + s1 = sizeof(struct frame32) + 3 * sizeof(int); + s2 = s1 + sizeof(siginfo32_t); + } +#endif + + if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) { + *pcstack++ = (uint64_t) pc; + *fpstack++ = 0; + pcstack_limit--; + if (pcstack_limit <= 0) + return; + + if (is64bit) + pc = dtrace_fuword64(sp); + else + pc = dtrace_fuword32(sp); + } + while (pc != 0 && sp != 0) { + *pcstack++ = (uint64_t) pc; + *fpstack++ = sp; + pcstack_limit--; + if (pcstack_limit <= 0) + break; + +#if 0 /* XXX signal stack crawl */ + if (oldcontext == sp + s1 || oldcontext == sp + s2) { + if (p->p_model == DATAMODEL_NATIVE) { + ucontext_t *ucp = (ucontext_t *) oldcontext; + greg_t *gregs = ucp->uc_mcontext.gregs; + + sp = dtrace_fulword(&gregs[REG_FP]); + pc = dtrace_fulword(&gregs[REG_PC]); + + oldcontext = dtrace_fulword(&ucp->uc_link); + } else { + ucontext_t *ucp = (ucontext_t *) oldcontext; + greg_t *gregs = ucp->uc_mcontext.gregs; + + sp = dtrace_fuword32(&gregs[EBP]); + pc = dtrace_fuword32(&gregs[EIP]); + + oldcontext = dtrace_fuword32(&ucp->uc_link); + } + } else +#endif + { + if (is64bit) { + pc = dtrace_fuword64((sp + RETURN_OFFSET64)); + sp = dtrace_fuword64(sp); + } else { + pc = dtrace_fuword32((sp + RETURN_OFFSET)); + sp = dtrace_fuword32(sp); + } + } + +#if 0 + /* XXX ARMTODO*/ + /* + * This is totally bogus: if we faulted, we're going to clear + * the fault and break. This is to deal with the apparently + * broken Java stacks on x86. + */ + if (*flags & CPU_DTRACE_FAULT) { + *flags &= ~CPU_DTRACE_FAULT; + break; + } +#endif + } + +zero: + while (pcstack_limit-- > 0) + *pcstack++ = 0ULL; +} + + +void +dtrace_getpcstack(pc_t * pcstack, int pcstack_limit, int aframes, + uint32_t * intrpc) +{ + struct frame *fp = (struct frame *) __builtin_frame_address(0); + struct frame *nextfp, *minfp, *stacktop; + int depth = 0; + int on_intr; + int last = 0; + uintptr_t pc; + uintptr_t caller = CPU->cpu_dtrace_caller; + + if ((on_intr = CPU_ON_INTR(CPU)) != 0) + stacktop = (struct frame *) dtrace_get_cpu_int_stack_top(); + else + stacktop = (struct frame *) (dtrace_get_kernel_stack(current_thread()) + kernel_stack_size); + + minfp = fp; + + aframes++; + + if (intrpc != NULL && depth < pcstack_limit) + pcstack[depth++] = (pc_t) intrpc; + + while (depth < pcstack_limit) { + nextfp = *(struct frame **) fp; + pc = *(uintptr_t *) (((uintptr_t) fp) + RETURN_OFFSET64); + + if (nextfp <= minfp || nextfp >= stacktop) { + if (on_intr) { + /* + * Hop from interrupt stack to thread stack. + */ + arm_saved_state_t *arm_kern_regs = (arm_saved_state_t *) find_kern_regs(current_thread()); + if (arm_kern_regs) { + nextfp = (struct frame *)(saved_state64(arm_kern_regs)->fp); + + { + vm_offset_t kstack_base = dtrace_get_kernel_stack(current_thread()); + + minfp = (struct frame *)kstack_base; + stacktop = (struct frame *)(kstack_base + kernel_stack_size); + } + + on_intr = 0; + + if (nextfp <= minfp || nextfp >= stacktop) { + last = 1; + } + } else { + /* + * If this thread was on the interrupt stack, but did not + * take an interrupt (i.e, the idle thread), there is no + * explicit saved state for us to use. + */ + last = 1; + } + } else { + { + /* + * This is the last frame we can process; indicate + * that we should return after processing this frame. + */ + last = 1; + } + } + } + if (aframes > 0) { + if (--aframes == 0 && caller != (uintptr_t)NULL) { + /* + * We've just run out of artificial frames, + * and we have a valid caller -- fill it in + * now. + */ + ASSERT(depth < pcstack_limit); + pcstack[depth++] = (pc_t) caller; + caller = (uintptr_t)NULL; + } + } else { + if (depth < pcstack_limit) + pcstack[depth++] = (pc_t) pc; + } + + if (last) { + while (depth < pcstack_limit) + pcstack[depth++] = (pc_t) NULL; + return; + } + fp = nextfp; + minfp = fp; + } +} + +/* + * On arm64, we support both 32bit and 64bit user processes. + * This routine is only called when handling 32bit processes + * where thumb_mode is pertinent. + * If this routine is called when handling 64bit processes + * thumb_mode should always be zero. + */ +int +dtrace_instr_size(uint32_t instr, int thumb_mode) +{ + if (thumb_mode) { + uint16_t instr16 = *(uint16_t*) &instr; + if (((instr16 >> 11) & 0x1F) > 0x1C) + return 4; + else + return 2; + } else { + return 4; + } +} + +uint64_t +dtrace_getarg(int arg, int aframes, dtrace_mstate_t *mstate, dtrace_vstate_t *vstate) +{ +#pragma unused(arg, aframes) + uint64_t val = 0; + struct frame *fp = (struct frame *)__builtin_frame_address(0); + uintptr_t *stack; + uintptr_t pc; + int i; + + /* + * A total of 8 arguments are passed via registers; any argument with + * index of 7 or lower is therefore in a register. + */ + int inreg = 7; + + for (i = 1; i <= aframes; ++i) { + fp = fp->backchain; + pc = fp->retaddr; + + if (dtrace_invop_callsite_pre != NULL + && pc > (uintptr_t) dtrace_invop_callsite_pre + && pc <= (uintptr_t) dtrace_invop_callsite_post) + { + /* fp points to frame of dtrace_invop() activation */ + fp = fp->backchain; /* to fbt_perfCallback activation */ + fp = fp->backchain; /* to sleh_synchronous activation */ + fp = fp->backchain; /* to fleh_synchronous activation */ + + arm_saved_state_t *tagged_regs = (arm_saved_state_t*) ((void*) &fp[1]); + arm_saved_state64_t *saved_state = saved_state64(tagged_regs); + + if (arg <= inreg) { + /* the argument will be found in a register */ + stack = (uintptr_t*) &saved_state->x[0]; + } else { + /* the argument will be found in the stack */ + fp = (struct frame*) saved_state->sp; + stack = (uintptr_t*) &fp[1]; + arg -= (inreg + 1); + } + + goto load; + } + } + + /* + * We know that we did not come through a trap to get into + * dtrace_probe() -- We arrive here when the provider has + * called dtrace_probe() directly. + * The probe ID is the first argument to dtrace_probe(). + * We must advance beyond that to get the argX. + */ + arg++; /* Advance past probeID */ + + if (arg <= inreg) { + /* + * This shouldn't happen. If the argument is passed in a + * register then it should have been, well, passed in a + * register... + */ + DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); + return (0); + } + + arg -= (inreg + 1); + stack = (uintptr_t*) &fp[1]; /* Find marshalled arguments */ + +load: + if (dtrace_canload((uint64_t)(stack + arg), sizeof(uint64_t), + mstate, vstate)) { + /* dtrace_probe arguments arg0 ... arg4 are 64bits wide */ + val = dtrace_load64((uint64_t)(stack + arg)); + } + + return (val); +} + +void +dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which, + int fltoffs, int fault, uint64_t illval) +{ + /* XXX ARMTODO */ + /* + * For the case of the error probe firing lets + * stash away "illval" here, and special-case retrieving it in DIF_VARIABLE_ARG. + */ + state->dts_arg_error_illval = illval; + dtrace_probe( dtrace_probeid_error, (uint64_t)(uintptr_t)state, epid, which, fltoffs, fault ); +} + +void +dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit)) +{ + /* XXX ARMTODO check copied from ppc/x86*/ + /* + * "base" is the smallest toxic address in the range, "limit" is the first + * VALID address greater than "base". + */ + func(0x0, VM_MIN_KERNEL_ADDRESS); + if (VM_MAX_KERNEL_ADDRESS < ~(uintptr_t)0) + func(VM_MAX_KERNEL_ADDRESS + 1, ~(uintptr_t)0); +} + diff --git a/bsd/dev/arm64/dtrace_subr_arm.c b/bsd/dev/arm64/dtrace_subr_arm.c new file mode 100644 index 000000000..efdbb54f5 --- /dev/null +++ b/bsd/dev/arm64/dtrace_subr_arm.c @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + */ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* + * #pragma ident "@(#)dtrace_subr.c 1.12 05/06/08 SMI" + */ + +#include <sys/dtrace.h> +#include <sys/dtrace_glue.h> +#include <sys/dtrace_impl.h> +#include <sys/fasttrap.h> +#include <sys/vm.h> +#include <sys/user.h> +#include <sys/kauth.h> +#include <kern/debug.h> +#include <arm/proc_reg.h> + +int (*dtrace_pid_probe_ptr) (arm_saved_state_t *); +int (*dtrace_return_probe_ptr) (arm_saved_state_t *); + +kern_return_t +dtrace_user_probe(arm_saved_state_t *); + +kern_return_t +dtrace_user_probe(arm_saved_state_t *regs) +{ + /* + * FIXME + * + * The only call path into this method is always a user trap. + * We don't need to test for user trap, but should assert it. + */ + + lck_rw_t *rwp; + struct proc *p = current_proc(); + int is_fasttrap = 0; + + uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread()); + + kauth_cred_uthread_update(uthread, p); + + if (is_saved_state32(regs)) { + if (saved_state32(regs)->cpsr & PSR_TF) { + uint16_t pc; + if (copyin((user_addr_t)saved_state32(regs)->pc, &pc, sizeof(uint16_t))) { + return KERN_FAILURE; + } + is_fasttrap = (pc == FASTTRAP_THUMB32_RET_INSTR); + } else { + uint32_t pc; + if (copyin((user_addr_t)saved_state32(regs)->pc, &pc, sizeof(uint32_t))) { + return KERN_FAILURE; + } + is_fasttrap = (pc == FASTTRAP_ARM32_RET_INSTR); + } + } else { + uint32_t pc; + if (copyin((user_addr_t)saved_state64(regs)->pc, &pc, sizeof(uint32_t))) { + return KERN_FAILURE; + } + is_fasttrap = (pc == FASTTRAP_ARM64_RET_INSTR); + } + + if (is_fasttrap) { + uint8_t step = uthread->t_dtrace_step; + uint8_t ret = uthread->t_dtrace_ret; + user_addr_t npc = uthread->t_dtrace_npc; + + if (uthread->t_dtrace_ast) { + printf("dtrace_user_probe() should be calling aston()\n"); + // aston(thread); + // uthread->t_sig_check = 1; + } + + /* + * Clear all user tracing flags. + */ + uthread->t_dtrace_ft = 0; + + /* + * If we weren't expecting a quick return to the kernel, just kill + * the process as though it had just executed an unassigned + * trap instruction. + */ + if (step == 0) { + /* + * APPLE NOTE: We're returning KERN_FAILURE, which causes + * the generic signal handling code to take over, which will effectively + * deliver a EXC_BAD_INSTRUCTION to the user process. + */ + return KERN_FAILURE; + } + + /* + * If we hit this trap unrelated to a return probe, we're + * here to either: + * + * 1. Reset the AST flag, since we deferred a signal + * until after we logically single-stepped the instruction we + * copied out. + * + * 2. Just return to normal execution (required for U64). + */ + if (ret == 0) { + set_saved_state_pc(regs, npc); + return KERN_SUCCESS; + } + + /* + * We need to wait until after we've called the + * dtrace_return_probe_ptr function pointer to step the pc. + */ + rwp = &CPU->cpu_ft_lock; + lck_rw_lock_shared(rwp); + + if (dtrace_return_probe_ptr != NULL) + (void) (*dtrace_return_probe_ptr)(regs); + lck_rw_unlock_shared(rwp); + + set_saved_state_pc(regs, npc); + + return KERN_SUCCESS; + } else { + rwp = &CPU->cpu_ft_lock; + + /* + * The DTrace fasttrap provider uses a trap, + * FASTTRAP_{ARM,THUMB}_INSTR. We let + * DTrace take the first crack at handling + * this trap; if it's not a probe that DTrace knows about, + * we call into the trap() routine to handle it like a + * breakpoint placed by a conventional debugger. + */ + + /* + * APPLE NOTE: I believe the purpose of the reader/writers lock + * is thus: There are times which dtrace needs to prevent calling + * dtrace_pid_probe_ptr(). Sun's original impl grabbed a plain + * mutex here. However, that serialized all probe calls, and + * destroyed MP behavior. So now they use a RW lock, with probes + * as readers, and the top level synchronization as a writer. + */ + lck_rw_lock_shared(rwp); + if (dtrace_pid_probe_ptr != NULL && + (*dtrace_pid_probe_ptr)(regs) == 0) { + lck_rw_unlock_shared(rwp); + return KERN_SUCCESS; + } + lck_rw_unlock_shared(rwp); + + /* + * If the instruction that caused the breakpoint trap doesn't + * look like our trap anymore, it may be that this tracepoint + * was removed just after the user thread executed it. In + * that case, return to user land to retry the instuction. + * + * Note that the PC points to the instruction that caused the fault. + */ + if (is_saved_state32(regs)) { + if (saved_state32(regs)->cpsr & PSR_TF) { + uint16_t instr; + if (fuword16(saved_state32(regs)->pc, &instr) == 0 && instr != FASTTRAP_THUMB32_INSTR) { + return KERN_SUCCESS; + } + } else { + uint32_t instr; + if (fuword32(saved_state32(regs)->pc, &instr) == 0 && instr != FASTTRAP_ARM32_INSTR) { + return KERN_SUCCESS; + } + } + } else { + uint32_t instr; + if (fuword32(saved_state64(regs)->pc, &instr) == 0 && instr != FASTTRAP_ARM64_INSTR) { + return KERN_SUCCESS; + } + } + } + + return KERN_FAILURE; +} + +void +dtrace_safe_synchronous_signal(void) +{ + /* Not implemented */ +} + +int +dtrace_safe_defer_signal(void) +{ + /* Not implemented */ + return 0; +} diff --git a/bsd/dev/arm64/fasttrap_isa.c b/bsd/dev/arm64/fasttrap_isa.c new file mode 100644 index 000000000..c0af6a9e2 --- /dev/null +++ b/bsd/dev/arm64/fasttrap_isa.c @@ -0,0 +1,2127 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + */ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* + * #pragma ident "@(#)fasttrap_isa.c 1.19 05/09/14 SMI" + */ + +#ifdef KERNEL +#ifndef _KERNEL +#define _KERNEL /* Solaris vs. Darwin */ +#endif +#endif + +#include <sys/fasttrap_isa.h> +#include <sys/fasttrap_impl.h> +#include <sys/dtrace.h> +#include <sys/dtrace_impl.h> +#include <kern/task.h> +#include <vm/pmap.h> +#include <vm/vm_map.h> +#include <mach/mach_vm.h> +#include <arm/proc_reg.h> +#include <arm/thread.h> +#include <arm/caches_internal.h> + +#include <sys/dtrace_ptss.h> +#include <kern/debug.h> + +#include <pexpert/pexpert.h> + +extern dtrace_id_t dtrace_probeid_error; + +/* Solaris proc_t is the struct. Darwin's proc_t is a pointer to it. */ +#define proc_t struct proc /* Steer clear of the Darwin typedef for proc_t */ + +extern int dtrace_decode_arm64(uint32_t instr); +extern int dtrace_decode_arm(uint32_t instr); +extern int dtrace_decode_thumb(uint32_t instr); + +/* + * Lossless User-Land Tracing on ARM + * --------------------------------- + * + * The details here will be fleshed out as more of this is implemented. The + * basic design will be the same way as tracing works in x86. + * + * Some ARM specific issues: + * + * We need to patch differently for ARM instructions and Thumb instructions. + * When we hit a probe, we check to see if the mode we're currently in is the + * same as the mode we're patching for. If not, we remove the tracepoint and + * abort. This ARM/Thumb information is pulled in from the arch specific + * information in the fasttrap probe. + * + * On ARM, any instruction that uses registers can also use the pc as a + * register. This presents problems during emulation because we have copied + * the instruction and thus the pc can be different. Currently we've emulated + * any instructions that use the pc if they can be used in a return probe. + * Eventually we will want to support all instructions that use the pc, but + * to do so requires disassembling the instruction and reconstituting it by + * substituting a different register. + * + */ + +#define THUMB_INSTR(x) (*(uint16_t*) &(x)) + +#define SIGNEXTEND(x,v) ((((int) (x)) << (32-(v))) >> (32-(v))) +#define ALIGNADDR(x,v) (((x) >> (v)) << (v)) +#define GETITSTATE(x) ((((x) >> 8) & 0xFC) | (((x) >> 25) & 0x3)) +#define ISLASTINIT(x) (((x) & 0xF) == 8) + +#define SET16(x,w) *((uint16_t*) (x)) = (w) +#define SET32(x,w) *((uint32_t*) (x)) = (w) + +#define IS_ARM32_NOP(x) ((x) == 0xE1A00000) +/* Marker for is-enabled probes */ +#define IS_ARM32_IS_ENABLED(x) ((x) == 0xE0200000) + +#define IS_ARM64_NOP(x) ((x) == 0xD503201F) +/* Marker for is-enabled probes */ +#define IS_ARM64_IS_ENABLED(x) ((x) == 0xD2800000) + +#define IS_THUMB32_NOP(x) ((x) == 0x46C0) +/* Marker for is-enabled probes */ +#define IS_THUMB32_IS_ENABLED(x) ((x) == 0x4040) + +#define ARM_LDM_UF (1 << 23) +#define ARM_LDM_PF (1 << 24) +#define ARM_LDM_WF (1 << 21) + +#define ARM_LDR_UF (1 << 23) +#define ARM_LDR_BF (1 << 22) + +static void +flush_caches(void) +{ + /* TODO There were some problems with flushing just the cache line that had been modified. + * For now, we'll flush the entire cache, until we figure out how to flush just the patched block. + */ + FlushPoU_Dcache(); + InvalidatePoU_Icache(); +} + + +static int fasttrap_tracepoint_init32 (proc_t *, fasttrap_tracepoint_t *, user_addr_t, fasttrap_probe_type_t); +static int fasttrap_tracepoint_init64 (proc_t *, fasttrap_tracepoint_t *, user_addr_t, fasttrap_probe_type_t); + +int +fasttrap_tracepoint_init(proc_t *p, fasttrap_tracepoint_t *tp, + user_addr_t pc, fasttrap_probe_type_t type) +{ + if (proc_is64bit(p)) { + return fasttrap_tracepoint_init64(p, tp, pc, type); + } else { + return fasttrap_tracepoint_init32(p, tp, pc, type); + } +} + +static int +fasttrap_tracepoint_init32(proc_t *p, fasttrap_tracepoint_t *tp, + user_addr_t pc, fasttrap_probe_type_t type) +{ +#pragma unused(type) + uint32_t instr; + + /* + * Read the instruction at the given address out of the process's + * address space. We don't have to worry about a debugger + * changing this instruction before we overwrite it with our trap + * instruction since P_PR_LOCK is set. Since instructions can span + * pages, we potentially read the instruction in two parts. If the + * second part fails, we just zero out that part of the instruction. + */ + /* + * APPLE NOTE: Of course, we do not have a P_PR_LOCK, so this is racey... + */ + + if (uread(p, &instr, 4, pc) != 0) + return (-1); + + /* We want &instr to always point to the saved instruction, so just copy the + * whole thing When cast to a pointer to a uint16_t, that will give us a + * pointer to the first two bytes, which is the thumb instruction. + */ + tp->ftt_instr = instr; + + if (tp->ftt_fntype != FASTTRAP_FN_DONE_INIT) { + switch(tp->ftt_fntype) { + case FASTTRAP_FN_UNKNOWN: + /* Can't instrument without any information. We can add some heuristics later if necessary. */ + return (-1); + + case FASTTRAP_FN_USDT: + if (IS_ARM32_NOP(instr) || IS_ARM32_IS_ENABLED(instr)) { + tp->ftt_thumb = 0; + } else if (IS_THUMB32_NOP(THUMB_INSTR(instr)) || IS_THUMB32_IS_ENABLED(THUMB_INSTR(instr))) { + tp->ftt_thumb = 1; + } else { + /* Shouldn't reach here - this means we don't recognize + * the instruction at one of the USDT probe locations + */ + return (-1); + } + tp->ftt_fntype = FASTTRAP_FN_DONE_INIT; + break; + + case FASTTRAP_FN_ARM: + tp->ftt_thumb = 0; + tp->ftt_fntype = FASTTRAP_FN_DONE_INIT; + break; + + case FASTTRAP_FN_THUMB: + tp->ftt_thumb = 1; + tp->ftt_fntype = FASTTRAP_FN_DONE_INIT; + break; + + default: + return (-1); + } + } + + if (tp->ftt_thumb) { + tp->ftt_type = dtrace_decode_thumb(instr); + } else { + tp->ftt_type = dtrace_decode_arm(instr); + } + + if (tp->ftt_type == FASTTRAP_T_INV) { + /* This is an instruction we either don't recognize or can't instrument */ + printf("dtrace: fasttrap init32: Unrecognized instruction: %08x at %08llx\n", + (tp->ftt_thumb && dtrace_instr_size(tp->ftt_instr,tp->ftt_thumb) == 2) ? tp->ftt_instr1 : instr, pc); + return (-1); + } + + return (0); +} + + +static int +fasttrap_tracepoint_init64(proc_t *p, fasttrap_tracepoint_t *tp, + user_addr_t pc, fasttrap_probe_type_t type) +{ +#pragma unused(type) + uint32_t instr = 0; + + /* + * Read the instruction at the given address out of the process's + * address space. We don't have to worry about a debugger + * changing this instruction before we overwrite it with our trap + * instruction since P_PR_LOCK is set. Since instructions can span + * pages, we potentially read the instruction in two parts. If the + * second part fails, we just zero out that part of the instruction. + */ + /* + * APPLE NOTE: Of course, we do not have a P_PR_LOCK, so this is racey... + */ + + if (uread(p, &instr, 4, pc) != 0) + return (-1); + + tp->ftt_instr = instr; + tp->ftt_thumb = 0; /* Always zero on 64bit */ + + if (tp->ftt_fntype != FASTTRAP_FN_DONE_INIT) { + switch(tp->ftt_fntype) { + case FASTTRAP_FN_UNKNOWN: + /* + * On arm64 there is no distinction between + * arm vs. thumb mode instruction types. + */ + tp->ftt_fntype = FASTTRAP_FN_DONE_INIT; + break; + + case FASTTRAP_FN_USDT: + if (IS_ARM64_NOP(instr) || IS_ARM64_IS_ENABLED(instr)) { + tp->ftt_fntype = FASTTRAP_FN_DONE_INIT; + } else { + /* + * Shouldn't reach here - this means we don't + * recognize the instruction at one of the + * USDT probe locations + */ + return (-1); + } + + break; + + case FASTTRAP_FN_ARM: + case FASTTRAP_FN_THUMB: + default: + /* + * If we get an arm or thumb mode type + * then we are clearly in the wrong path. + */ + return (-1); + } + } + + tp->ftt_type = dtrace_decode_arm64(instr); + + if (tp->ftt_type == FASTTRAP_T_ARM64_EXCLUSIVE_MEM) { + kprintf("Detected attempt to place DTrace probe on exclusive memory instruction (pc = 0x%llx); refusing to trace (or exclusive operation could never succeed).\n", pc); + tp->ftt_type = FASTTRAP_T_INV; + return (-1); + } + + if (tp->ftt_type == FASTTRAP_T_INV) { + /* This is an instruction we either don't recognize or can't instrument */ + printf("dtrace: fasttrap init64: Unrecognized instruction: %08x at %08llx\n", instr, pc); + return (-1); + } + + return (0); +} + +// These are not exported from vm_map.h. +extern kern_return_t vm_map_write_user(vm_map_t map, void *src_p, vm_map_address_t dst_addr, vm_size_t size); + +/* Patches the instructions. Almost like uwrite, but need special instructions on ARM to flush the caches. */ +static +int patchInst(proc_t *p, void *buf, user_size_t len, user_addr_t a) +{ + kern_return_t ret; + + ASSERT(p != NULL); + ASSERT(p->task != NULL); + + task_t task = p->task; + + /* + * Grab a reference to the task vm_map_t to make sure + * the map isn't pulled out from under us. + * + * Because the proc_lock is not held at all times on all code + * paths leading here, it is possible for the proc to have + * exited. If the map is null, fail. + */ + vm_map_t map = get_task_map_reference(task); + if (map) { + /* Find the memory permissions. */ + uint32_t nestingDepth=999999; + vm_region_submap_short_info_data_64_t info; + mach_msg_type_number_t count = VM_REGION_SUBMAP_SHORT_INFO_COUNT_64; + mach_vm_address_t address = (mach_vm_address_t)a; + mach_vm_size_t sizeOfRegion = (mach_vm_size_t)len; + + ret = mach_vm_region_recurse(map, &address, &sizeOfRegion, &nestingDepth, (vm_region_recurse_info_t)&info, &count); + if (ret != KERN_SUCCESS) + goto done; + + vm_prot_t reprotect; + + if (!(info.protection & VM_PROT_WRITE)) { + /* Save the original protection values for restoration later */ + reprotect = info.protection; + if (info.max_protection & VM_PROT_WRITE) { + /* The memory is not currently writable, but can be made writable. */ + /* Making it both writable and executable at the same time causes warning on embedded */ + ret = mach_vm_protect (map, (mach_vm_offset_t)a, (mach_vm_size_t)len, 0, (reprotect & ~VM_PROT_EXECUTE) | VM_PROT_WRITE); + } else { + /* + * The memory is not currently writable, and cannot be made writable. We need to COW this memory. + * + * Strange, we can't just say "reprotect | VM_PROT_COPY", that fails. + */ + ret = mach_vm_protect (map, (mach_vm_offset_t)a, (mach_vm_size_t)len, 0, VM_PROT_COPY | VM_PROT_READ | VM_PROT_WRITE); + } + + if (ret != KERN_SUCCESS) + goto done; + + } else { + /* The memory was already writable. */ + reprotect = VM_PROT_NONE; + } + + ret = vm_map_write_user( map, + buf, + (vm_map_address_t)a, + (vm_size_t)len); + + flush_caches(); + + if (ret != KERN_SUCCESS) + goto done; + + if (reprotect != VM_PROT_NONE) { + ASSERT(reprotect & VM_PROT_EXECUTE); + ret = mach_vm_protect (map, (mach_vm_offset_t)a, (mach_vm_size_t)len, 0, reprotect); + } + +done: + vm_map_deallocate(map); + } else + ret = KERN_TERMINATED; + + return (int)ret; +} + +int +fasttrap_tracepoint_install(proc_t *p, fasttrap_tracepoint_t *tp) +{ + /* The thumb patch is a 2 byte instruction regardless of the size of the original instruction */ + uint32_t instr; + int size; + + if (proc_is64bit(p)) { + size = 4; + instr = FASTTRAP_ARM64_INSTR; + } + else { + size = tp->ftt_thumb ? 2 : 4; + if (tp->ftt_thumb) { + *((uint16_t*) &instr) = FASTTRAP_THUMB32_INSTR; + } else { + instr = FASTTRAP_ARM32_INSTR; + } + } + + if (patchInst(p, &instr, size, tp->ftt_pc) != 0) + return (-1); + + tp->ftt_installed = 1; + + return (0); +} + +int +fasttrap_tracepoint_remove(proc_t *p, fasttrap_tracepoint_t *tp) +{ + /* The thumb patch is a 2 byte instruction regardless of the size of the original instruction */ + uint32_t instr; + int size; + + if (proc_is64bit(p)) { + /* + * Distinguish between read or write failures and a changed + * instruction. + */ + size = 4; + if (uread(p, &instr, size, tp->ftt_pc) != 0) + goto end; + + if (instr != FASTTRAP_ARM64_INSTR) + goto end; + } else { + /* + * Distinguish between read or write failures and a changed + * instruction. + */ + size = tp->ftt_thumb ? 2 : 4; + if (uread(p, &instr, size, tp->ftt_pc) != 0) + goto end; + + if (tp->ftt_thumb) { + if (*((uint16_t*) &instr) != FASTTRAP_THUMB32_INSTR) + goto end; + } else { + if (instr != FASTTRAP_ARM32_INSTR) + goto end; + } + } + + if (patchInst(p, &tp->ftt_instr, size, tp->ftt_pc) != 0) + return (-1); + +end: + tp->ftt_installed = 0; + + return (0); +} + +static void +fasttrap_return_common(proc_t *p, arm_saved_state_t *regs, user_addr_t pc, user_addr_t new_pc) +{ + pid_t pid = p->p_pid; + fasttrap_tracepoint_t *tp; + fasttrap_bucket_t *bucket; + fasttrap_id_t *id; + lck_mtx_t *pid_mtx; + int retire_tp = 1; + pid_mtx = &cpu_core[CPU->cpu_id].cpuc_pid_lock; + lck_mtx_lock(pid_mtx); + bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; + + for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { + if (pid == tp->ftt_pid && pc == tp->ftt_pc && + tp->ftt_proc->ftpc_acount != 0) + break; + } + + /* + * Don't sweat it if we can't find the tracepoint again; unlike + * when we're in fasttrap_pid_probe(), finding the tracepoint here + * is not essential to the correct execution of the process. + */ + if (tp == NULL) { + lck_mtx_unlock(pid_mtx); + return; + } + + for (id = tp->ftt_retids; id != NULL; id = id->fti_next) { + fasttrap_probe_t *probe = id->fti_probe; + /* + * If there's a branch that could act as a return site, we + * need to trace it, and check here if the program counter is + * external to the function. + */ + if (is_saved_state32(regs)) + { + if (tp->ftt_type != FASTTRAP_T_LDM_PC && + tp->ftt_type != FASTTRAP_T_POP_PC && + new_pc - probe->ftp_faddr < probe->ftp_fsize) + continue; + } + else { + /* ARM64_TODO - check for FASTTRAP_T_RET */ + if ((tp->ftt_type != FASTTRAP_T_ARM64_RET) && + new_pc - probe->ftp_faddr < probe->ftp_fsize) + continue; + } + if (probe->ftp_prov->ftp_provider_type == DTFTP_PROVIDER_ONESHOT) { + uint8_t already_triggered = atomic_or_8(&probe->ftp_triggered, 1); + if (already_triggered) { + continue; + } + } + /* + * If we have at least one probe associated that + * is not a oneshot probe, don't remove the + * tracepoint + */ + else { + retire_tp = 0; + } + +#ifndef CONFIG_EMBEDDED + if (ISSET(current_proc()->p_lflag, P_LNOATTACH)) { + dtrace_probe(dtrace_probeid_error, 0 /* state */, id->fti_probe->ftp_id, + 1 /* ndx */, -1 /* offset */, DTRACEFLT_UPRIV); +#else + if (FALSE) { +#endif + } else { + if (is_saved_state32(regs)) { + dtrace_probe(probe->ftp_id, + pc - id->fti_probe->ftp_faddr, + saved_state32(regs)->r[0], 0, 0, 0); + } else { + dtrace_probe(probe->ftp_id, + pc - id->fti_probe->ftp_faddr, + saved_state64(regs)->x[0], 0, 0, 0); + } + } + } + if (retire_tp) { + fasttrap_tracepoint_retire(p, tp); + } + + lck_mtx_unlock(pid_mtx); +} + +static void +fasttrap_sigsegv(proc_t *p, uthread_t t, user_addr_t addr, arm_saved_state_t *regs) +{ + /* TODO: This function isn't implemented yet. In debug mode, panic the system to + * find out why we're hitting this point. In other modes, kill the process. + */ +#if DEBUG +#pragma unused(p,t,addr,arm_saved_state) + panic("fasttrap: sigsegv not yet implemented"); +#else +#pragma unused(p,t,addr) + /* Kill the process */ + set_saved_state_pc(regs, 0); +#endif + +#if 0 + proc_lock(p); + + /* Set fault address and mark signal */ + t->uu_code = addr; + t->uu_siglist |= sigmask(SIGSEGV); + + /* + * XXX These two line may be redundant; if not, then we need + * XXX to potentially set the data address in the machine + * XXX specific thread state structure to indicate the address. + */ + t->uu_exception = KERN_INVALID_ADDRESS; /* SIGSEGV */ + t->uu_subcode = 0; /* XXX pad */ + + proc_unlock(p); + + /* raise signal */ + signal_setast(t->uu_context.vc_thread); +#endif +} + +static void +fasttrap_usdt_args32(fasttrap_probe_t *probe, arm_saved_state32_t *regs32, int argc, + uint64_t *argv) +{ + int i, x, cap = MIN(argc, probe->ftp_nargs); + + for (i = 0; i < cap; i++) { + x = probe->ftp_argmap[i]; + + /* Up to 4 args are passed in registers on arm */ + if (x < 4) { + argv[i] = regs32->r[x]; + } else { + uint32_t arg; + fasttrap_fuword32_noerr(regs32->sp + (x - 4) * sizeof(uint32_t), &arg); + + argv[i] = arg; + } + } + + for (; i < argc; i++) { + argv[i] = 0; + } +} + +static void +fasttrap_usdt_args64(fasttrap_probe_t *probe, arm_saved_state64_t *regs64, int argc, + uint64_t *argv) +{ + int i, x, cap = MIN(argc, probe->ftp_nargs); + + for (i = 0; i < cap; i++) { + x = probe->ftp_argmap[i]; + + /* Up to 8 args are passed in registers on arm64 */ + if (x < 8) { + argv[i] = regs64->x[x]; + } else { + fasttrap_fuword64_noerr(regs64->sp + (x - 8) * sizeof(uint64_t), &argv[i]); + } + } + + for (; i < argc; i++) { + argv[i] = 0; + } +} + +static int condition_true(int cond, int cpsr) +{ + int taken = 0; + int zf = (cpsr & PSR_ZF) ? 1 : 0, + nf = (cpsr & PSR_NF) ? 1 : 0, + cf = (cpsr & PSR_CF) ? 1 : 0, + vf = (cpsr & PSR_VF) ? 1 : 0; + + switch(cond) { + case 0: taken = zf; break; + case 1: taken = !zf; break; + case 2: taken = cf; break; + case 3: taken = !cf; break; + case 4: taken = nf; break; + case 5: taken = !nf; break; + case 6: taken = vf; break; + case 7: taken = !vf; break; + case 8: taken = (cf && !zf); break; + case 9: taken = (!cf || zf); break; + case 10: taken = (nf == vf); break; + case 11: taken = (nf != vf); break; + case 12: taken = (!zf && (nf == vf)); break; + case 13: taken = (zf || (nf != vf)); break; + case 14: taken = 1; break; + case 15: taken = 1; break; /* always "true" for ARM, unpredictable for THUMB. */ + } + + return taken; +} + +static void set_thumb_flag(arm_saved_state32_t *regs32, user_addr_t pc) +{ + if (pc & 1) { + regs32->cpsr |= PSR_TF; + } else { + regs32->cpsr &= ~PSR_TF; + } +} + +static int +fasttrap_pid_probe_thumb_state_valid(arm_saved_state32_t *state32, fasttrap_tracepoint_t *tp) +{ + uint32_t cpsr = state32->cpsr; + uint32_t itstate = GETITSTATE(cpsr); + + /* If in IT block, make sure it's the last statement in the block */ + if ((itstate != 0) && !ISLASTINIT(itstate)) { + printf("dtrace: fasttrap: Tried to trace instruction %08x at %08x but not at end of IT block\n", + (tp->ftt_thumb && dtrace_instr_size(tp->ftt_instr,tp->ftt_thumb) == 2) ? tp->ftt_instr1 : tp->ftt_instr, state32->pc); + return 0; + } + + if (!(cpsr & PSR_TF)) { + return 0; + } + + return 1; +} + +static int +fasttrap_get_condition_code(arm_saved_state32_t *regs32, fasttrap_tracepoint_t *tp) +{ + /* Default to always execute */ + int condition_code = 0xE; + if (tp->ftt_thumb) { + uint32_t itstate = GETITSTATE(regs32->cpsr); + if (itstate != 0) { + /* In IT block, make sure it's the last statement in the block */ + assert(ISLASTINIT(itstate)); + condition_code = itstate >> 4; + } + } else { + condition_code = ARM_CONDCODE(tp->ftt_instr); + } + + return condition_code; +} + +static void +fasttrap_pid_probe_handle_patched_instr32(arm_saved_state_t *state, fasttrap_tracepoint_t *tp, uthread_t uthread, + proc_t *p, uint_t is_enabled, int *was_simulated) +{ + arm_saved_state32_t *regs32 = saved_state32(state); + uint32_t new_pc = 0; + uint32_t pc = regs32->pc; + int instr_size; + int condition_code; + + *was_simulated = 1; + + /* + * If there's an is-enabled probe connected to this tracepoint it + * means that there was a 'eor r0,r0,r0' + * instruction that was placed there by DTrace when the binary was + * linked. As this probe is, in fact, enabled, we need to stuff 1 + * into R0. Accordingly, we can bypass all the instruction + * emulation logic since we know the inevitable result. It's possible + * that a user could construct a scenario where the 'is-enabled' + * probe was on some other instruction, but that would be a rather + * exotic way to shoot oneself in the foot. + */ + + if (is_enabled) { + regs32->r[0] = 1; + new_pc = regs32->pc + (tp->ftt_thumb ? 2 : 4); + goto done; + } + + /* For USDT probes, bypass all the emulation logic for the nop instruction */ + if ((tp->ftt_thumb && IS_THUMB32_NOP(THUMB_INSTR(tp->ftt_instr))) || + (!tp->ftt_thumb && IS_ARM32_NOP(tp->ftt_instr))) { + new_pc = regs32->pc + (tp->ftt_thumb ? 2 : 4); + goto done; + } + + condition_code = fasttrap_get_condition_code(regs32, tp); + instr_size = dtrace_instr_size(tp->ftt_instr,tp->ftt_thumb); + + switch (tp->ftt_type) { + case FASTTRAP_T_MOV_PC_REG: + case FASTTRAP_T_CPY_PC: + { + if (!condition_true(condition_code, regs32->cpsr)) { + new_pc = pc + instr_size; + break; + } + + int rm; + if (tp->ftt_thumb) { + rm = THUMB16_HRM(tp->ftt_instr1); + } else { + rm = tp->ftt_instr & 0xF; + } + new_pc = regs32->r[rm]; + + /* This instruction does not change the Thumb state */ + + break; + } + + case FASTTRAP_T_STM_LR: + case FASTTRAP_T_PUSH_LR: + { + /* + * This is a very common case, so we want to emulate this instruction if + * possible. However, on a push, it is possible that we might reach the end + * of a page and have to allocate a new page. Most of the time this will not + * happen, and we know that the push instruction can store at most 16 words, + * so check to see if we are far from the boundary, and if so, emulate. This + * can be made more aggressive by checking the actual number of words being + * pushed, but we won't do that for now. + * + * Some of the same issues that apply to POP_PC probably apply here also. + */ + + int reglist; + int ret; + uint32_t base; + + if (!condition_true(condition_code, regs32->cpsr)) { + new_pc = pc + instr_size; + break; + } + + base = regs32->sp; + if (((base-16*4) >> PAGE_SHIFT) != (base >> PAGE_SHIFT)) { + /* Crosses the page boundary, go to emulation */ + goto instr_emulate; + } + + if (tp->ftt_thumb) { + if (instr_size == 4) { + /* We know we have to push lr, never push sp or pc */ + reglist = tp->ftt_instr2 & 0x1FFF; + } else { + reglist = tp->ftt_instr1 & 0xFF; + } + } else { + /* We know we have to push lr, never push sp or pc */ + reglist = tp->ftt_instr & 0x1FFF; + } + + /* Push the link register */ + base -= 4; + ret = fasttrap_suword32(base, regs32->lr); + if (ret == -1) { + fasttrap_sigsegv(p, uthread, (user_addr_t) base, state); + new_pc = regs32->pc; + break; + } + + /* Start pushing from $r12 */ + int regmask = 1 << 12; + int regnum = 12; + + while (regmask) { + if (reglist & regmask) { + base -= 4; + ret = fasttrap_suword32(base, regs32->r[regnum]); + if (ret == -1) { + fasttrap_sigsegv(p, uthread, (user_addr_t) base, state); + new_pc = regs32->pc; + break; + } + } + regmask >>= 1; + regnum--; + } + + regs32->sp = base; + + new_pc = pc + instr_size; + + break; + } + + + case FASTTRAP_T_LDM_PC: + case FASTTRAP_T_POP_PC: + { + /* TODO Two issues that will eventually need to be resolved: + * + * 1. Understand what the hardware does if we have to segfault (data abort) in + * the middle of a load multiple. We currently don't have a working segfault + * handler anyway, and with no swapfile we should never segfault on this load. + * If we do, we'll just kill the process by setting the pc to 0. + * + * 2. The emulation is no longer atomic. We currently only emulate pop for + * function epilogues, and so we should never have a race here because one + * thread should never be trying to manipulate another thread's stack frames. + * That is almost certainly a bug in the program. + * + * This will need to be fixed if we ever: + * a. Ship dtrace externally, as this could be a potential attack vector + * b. Support instruction level tracing, as we might then pop/ldm non epilogues. + * + */ + + /* Assume ldmia! sp/pop ... pc */ + + int regnum = 0, reglist; + int ret; + uint32_t base; + + if (!condition_true(condition_code, regs32->cpsr)) { + new_pc = pc + instr_size; + break; + } + + if (tp->ftt_thumb) { + if (instr_size == 4) { + /* We know we have to load the pc, don't do it twice */ + reglist = tp->ftt_instr2 & 0x7FFF; + } else { + reglist = tp->ftt_instr1 & 0xFF; + } + } else { + /* We know we have to load the pc, don't do it twice */ + reglist = tp->ftt_instr & 0x7FFF; + } + + base = regs32->sp; + while (reglist) { + if (reglist & 1) { + ret = fasttrap_fuword32((user_addr_t)base, ®s32->r[regnum]); + if (ret == -1) { + fasttrap_sigsegv(p, uthread, (user_addr_t) base, state); + new_pc = regs32->pc; + break; + } + base += 4; + } + reglist >>= 1; + regnum++; + } + + ret = fasttrap_fuword32((user_addr_t)base, &new_pc); + if (ret == -1) { + fasttrap_sigsegv(p, uthread, (user_addr_t) base, state); + new_pc = regs32->pc; + break; + } + base += 4; + + regs32->sp = base; + + set_thumb_flag(regs32, new_pc); + + break; + } + + case FASTTRAP_T_CB_N_Z: + { + /* Thumb mode instruction, and not permitted in IT block, so skip the condition code check */ + int rn = tp->ftt_instr1 & 0x7; + int offset = (((tp->ftt_instr1 & 0x00F8) >> 2) | ((tp->ftt_instr1 & 0x0200) >> 3)) + 4; + int nonzero = tp->ftt_instr1 & 0x0800; + if (!nonzero != !(regs32->r[rn] == 0)) { + new_pc = pc + offset; + } else { + new_pc = pc + instr_size; + } + break; + } + + case FASTTRAP_T_B_COND: + { + /* Use the condition code in the instruction and ignore the ITSTATE */ + + int code, offset; + if (tp->ftt_thumb) { + if (instr_size == 4) { + code = (tp->ftt_instr1 >> 6) & 0xF; + if (code == 14 || code == 15) { + panic("fasttrap: Emulation of invalid branch"); + } + int S = (tp->ftt_instr1 >> 10) & 1, + J1 = (tp->ftt_instr2 >> 13) & 1, + J2 = (tp->ftt_instr2 >> 11) & 1; + offset = 4 + SIGNEXTEND( + (S << 20) | (J2 << 19) | (J1 << 18) | + ((tp->ftt_instr1 & 0x003F) << 12) | + ((tp->ftt_instr2 & 0x07FF) << 1), + 21); + } else { + code = (tp->ftt_instr1 >> 8) & 0xF; + if (code == 14 || code == 15) { + panic("fasttrap: Emulation of invalid branch"); + } + offset = 4 + (SIGNEXTEND(tp->ftt_instr1 & 0xFF, 8) << 1); + } + } else { + code = ARM_CONDCODE(tp->ftt_instr); + if (code == 15) { + panic("fasttrap: Emulation of invalid branch"); + } + offset = 8 + (SIGNEXTEND(tp->ftt_instr & 0x00FFFFFF, 24) << 2); + } + + if (condition_true(code, regs32->cpsr)) { + new_pc = pc + offset; + } else { + new_pc = pc + instr_size; + } + + break; + } + + case FASTTRAP_T_B_UNCOND: + { + int offset; + + /* Unconditional branches can only be taken from Thumb mode */ + /* (This is different from an ARM branch with condition code "always") */ + ASSERT(tp->ftt_thumb == 1); + + if (!condition_true(condition_code, regs32->cpsr)) { + new_pc = pc + instr_size; + break; + } + + if (instr_size == 4) { + int S = (tp->ftt_instr1 >> 10) & 1, + J1 = (tp->ftt_instr2 >> 13) & 1, + J2 = (tp->ftt_instr2 >> 11) & 1; + int I1 = (J1 != S) ? 0 : 1, I2 = (J2 != S) ? 0 : 1; + offset = 4 + SIGNEXTEND( + (S << 24) | (I1 << 23) | (I2 << 22) | + ((tp->ftt_instr1 & 0x03FF) << 12) | + ((tp->ftt_instr2 & 0x07FF) << 1), + 25); + } else { + uint32_t instr1 = tp->ftt_instr1; + offset = 4 + (SIGNEXTEND(instr1 & 0x7FF, 11) << 1); + } + + new_pc = pc + offset; + + break; + } + + case FASTTRAP_T_BX_REG: + { + int reg; + + if (!condition_true(condition_code, regs32->cpsr)) { + new_pc = pc + instr_size; + break; + } + + if (tp->ftt_thumb) { + reg = THUMB16_HRM(tp->ftt_instr1); + } else { + reg = ARM_RM(tp->ftt_instr); + } + new_pc = regs32->r[reg]; + set_thumb_flag(regs32, new_pc); + + break; + } + + case FASTTRAP_T_LDR_PC_IMMED: + case FASTTRAP_T_VLDR_PC_IMMED: + /* Handle these instructions by replacing the PC in the instruction with another + * register. They are common, so we'd like to support them, and this way we do so + * without any risk of having to simulate a segfault. + */ + + /* Fall through */ + + instr_emulate: + case FASTTRAP_T_COMMON: + { + user_addr_t addr; + uint8_t scratch[32]; + uint_t i = 0; + fasttrap_instr_t emul_instr; + emul_instr.instr32 = tp->ftt_instr; + int emul_instr_size; + + /* + * Unfortunately sometimes when we emulate the instruction and have to replace the + * PC, there is no longer a thumb mode equivalent. We end up having to run the + * modified instruction in ARM mode. We use this variable to keep track of which + * mode we should emulate in. We still use the original variable to determine + * what mode to return to. + */ + uint8_t emul_thumb = tp->ftt_thumb; + int save_reg = -1; + uint32_t save_val = 0; + + /* + * Dealing with condition codes and emulation: + * We can't just uniformly do a condition code check here because not all instructions + * have condition codes. We currently do not support an instruction by instruction trace, + * so we can assume that either: 1. We are executing a Thumb instruction, in which case + * we either are not in an IT block and should execute always, or we are last in an IT + * block. Either way, the traced instruction will run correctly, and we won't have any + * problems when we return to the original code, because we will no longer be in the IT + * block. 2. We are executing an ARM instruction, in which case we are ok as long as + * we don't attempt to change the condition code. + */ + if (tp->ftt_type == FASTTRAP_T_LDR_PC_IMMED) { + /* We know we always have a free register (the one we plan to write the + * result value to!). So we'll replace the pc with that one. + */ + int new_reg; + if (tp->ftt_thumb) { + /* Check to see if thumb or thumb2 */ + if (instr_size == 2) { + /* + * Sadness. We need to emulate this instruction in ARM mode + * because it has an 8 bit immediate offset. Instead of having + * to deal with condition codes in the ARM instruction, we'll + * just check the condition and abort if the condition is false. + */ + if (!condition_true(condition_code, regs32->cpsr)) { + new_pc = pc + instr_size; + break; + } + + new_reg = (tp->ftt_instr1 >> 8) & 0x7; + regs32->r[new_reg] = ALIGNADDR(regs32->pc + 4, 2); + emul_thumb = 0; + emul_instr.instr32 = 0xE5900000 | (new_reg << 16) | (new_reg << 12) | ((tp->ftt_instr1 & 0xFF) << 2); + } else { + /* Thumb2. Just replace the register. */ + new_reg = (tp->ftt_instr2 >> 12) & 0xF; + regs32->r[new_reg] = ALIGNADDR(regs32->pc + 4, 2); + emul_instr.instr16.instr1 &= ~0x000F; + emul_instr.instr16.instr1 |= new_reg; + } + } else { + /* ARM. Just replace the register. */ + new_reg = (tp->ftt_instr >> 12) & 0xF; + regs32->r[new_reg] = ALIGNADDR(regs32->pc + 8,2); + emul_instr.instr32 &= ~0x000F0000; + emul_instr.instr32 |= new_reg << 16; + } + } else if (tp->ftt_type == FASTTRAP_T_VLDR_PC_IMMED) { + /* This instruction only uses one register, and if we're here, we know + * it must be the pc. So we'll just replace it with R0. + */ + save_reg = 0; + save_val = regs32->r[0]; + regs32->r[save_reg] = ALIGNADDR(regs32->pc + (tp->ftt_thumb ? 4 : 8), 2); + if (tp->ftt_thumb) { + emul_instr.instr16.instr1 &= ~0x000F; + } else { + emul_instr.instr32 &= ~0x000F0000; + } + } + + emul_instr_size = dtrace_instr_size(emul_instr.instr32, emul_thumb); + + /* + * At this point: + * tp->ftt_thumb = thumb mode of original instruction + * emul_thumb = thumb mode for emulation + * emul_instr = instruction we are using to emulate original instruction + * emul_instr_size = size of emulating instruction + */ + + addr = uthread->t_dtrace_scratch->addr; + + if (addr == 0LL) { + fasttrap_sigtrap(p, uthread, pc); // Should be killing target proc + new_pc = pc; + break; + } + + uthread->t_dtrace_scrpc = addr; + if (emul_thumb) { + /* + * No way to do an unconditional branch in Thumb mode, shove the address + * onto the user stack and go to the next location with a pop. This can + * segfault if this push happens to cross a stack page, but that's ok, since + * we are running in userland, and the kernel knows how to handle userland + * stack expansions correctly. + * + * Layout of scratch space for Thumb mode: + * Emulated instruction + * ldr save_reg, [pc, #16] (if necessary, restore any register we clobbered) + * push { r0, r1 } + * ldr r0, [pc, #4] + * str r0, [sp, #4] + * pop { r0, pc } + * Location we should return to in original program + * Saved value of clobbered register (if necessary) + */ + + bcopy(&emul_instr, &scratch[i], emul_instr_size); i += emul_instr_size; + + if (save_reg != -1) { + uint16_t restore_inst = 0x4803; + restore_inst |= (save_reg & 0x7) << 8; + SET16(scratch+i, restore_inst); i += 2; // ldr reg, [pc , #16] + } + + SET16(scratch+i, 0xB403); i += 2; // push { r0, r1 } + SET16(scratch+i, 0x4801); i += 2; // ldr r0, [pc, #4] + SET16(scratch+i, 0x9001); i += 2; // str r0, [sp, #4] + SET16(scratch+i, 0xBD01); i += 2; // pop { r0, pc } + + if (i % 4) { + SET16(scratch+i, 0); i += 2; // padding - saved 32 bit words must be aligned + } + SET32(scratch+i, pc + instr_size + (tp->ftt_thumb ? 1 : 0)); i += 4; // Return address + if (save_reg != -1) { + SET32(scratch+i, save_val); i += 4; // saved value of clobbered register + } + + uthread->t_dtrace_astpc = addr + i; + bcopy(&emul_instr, &scratch[i], emul_instr_size); i += emul_instr_size; + SET16(scratch+i, FASTTRAP_THUMB32_RET_INSTR); i += 2; + } else { + /* + * Layout of scratch space for ARM mode: + * Emulated instruction + * ldr save_reg, [pc, #12] (if necessary, restore any register we clobbered) + * ldr pc, [pc, #4] + * Location we should return to in original program + * Saved value of clobbered register (if necessary) + */ + + bcopy(&emul_instr, &scratch[i], emul_instr_size); i += emul_instr_size; + + if (save_reg != -1) { + uint32_t restore_inst = 0xE59F0004; + restore_inst |= save_reg << 12; + SET32(scratch+i, restore_inst); i += 4; // ldr reg, [pc, #12] + } + SET32(scratch+i, 0xE51FF004); i += 4; // ldr pc, [pc, #4] + + SET32(scratch+i, pc + instr_size + (tp->ftt_thumb ? 1 : 0)); i += 4; // Return address + if (save_reg != -1) { + SET32(scratch+i, save_val); i += 4; // Saved value of clobbered register + } + + uthread->t_dtrace_astpc = addr + i; + bcopy(&emul_instr, &scratch[i], emul_instr_size); i += emul_instr_size; + SET32(scratch+i, FASTTRAP_ARM32_RET_INSTR); i += 4; + } + + if (patchInst(p, scratch, i, uthread->t_dtrace_scratch->write_addr) != KERN_SUCCESS) { + fasttrap_sigtrap(p, uthread, pc); + new_pc = pc; + break; + } + + if (tp->ftt_retids != NULL) { + uthread->t_dtrace_step = 1; + uthread->t_dtrace_ret = 1; + new_pc = uthread->t_dtrace_astpc + (emul_thumb ? 1 : 0); + } else { + new_pc = uthread->t_dtrace_scrpc + (emul_thumb ? 1 : 0); + } + + uthread->t_dtrace_pc = pc; + uthread->t_dtrace_npc = pc + instr_size; + uthread->t_dtrace_on = 1; + *was_simulated = 0; + set_thumb_flag(regs32, new_pc); + break; + } + + default: + panic("fasttrap: mishandled an instruction"); + } +done: + set_saved_state_pc(state, new_pc); + return; +} + +/* + * Copy out an instruction for execution in userland. + * Trap back to kernel to handle return to original flow of execution, because + * direct branches don't have sufficient range (+/- 128MB) and we + * cannot clobber a GPR. Note that we have to specially handle PC-rel loads/stores + * as well, which have range +/- 1MB (convert to an indirect load). Instruction buffer + * layout: + * + * [ Thunked instruction sequence ] + * [ Trap for return to original code and return probe handling ] + * + * This *does* make it impossible for an ldxr/stxr pair to succeed if we trace on or between + * them... may need to get fancy at some point. + */ +static void +fasttrap_pid_probe_thunk_instr64(arm_saved_state_t *state, fasttrap_tracepoint_t *tp, proc_t *p, uthread_t uthread, + const uint32_t *instructions, uint32_t num_instrs, user_addr_t *pc_out) +{ + uint32_t local_scratch[8]; + user_addr_t pc = get_saved_state_pc(state); + user_addr_t user_scratch_area; + + assert(num_instrs < 8); + + bcopy(instructions, local_scratch, num_instrs * sizeof(uint32_t)); + local_scratch[num_instrs] = FASTTRAP_ARM64_RET_INSTR; + + uthread->t_dtrace_astpc = uthread->t_dtrace_scrpc = uthread->t_dtrace_scratch->addr; + user_scratch_area = uthread->t_dtrace_scratch->write_addr; + + if (user_scratch_area == (user_addr_t)0) { + fasttrap_sigtrap(p, uthread, pc); // Should be killing target proc + *pc_out = pc; + return; + } + + if (patchInst(p, local_scratch, (num_instrs + 1) * sizeof(uint32_t), user_scratch_area) != KERN_SUCCESS) { + fasttrap_sigtrap(p, uthread, pc); + *pc_out = pc; + return; + } + + /* We're stepping (come back to kernel to adjust PC for return to regular code). */ + uthread->t_dtrace_step = 1; + + /* We may or may not be about to run a return probe (but we wouldn't thunk ret lr)*/ + uthread->t_dtrace_ret = (tp->ftt_retids != NULL); + assert(tp->ftt_type != FASTTRAP_T_ARM64_RET); + + /* Set address of instruction we've patched */ + uthread->t_dtrace_pc = pc; + + /* Any branch would be emulated, next instruction should be one ahead */ + uthread->t_dtrace_npc = pc + 4; + + /* We are certainly handling a probe */ + uthread->t_dtrace_on = 1; + + /* Let's jump to the scratch area */ + *pc_out = uthread->t_dtrace_scratch->addr; +} + +/* + * Sign-extend bit "sign_bit_index" out to bit 64. + */ +static int64_t +sign_extend(int64_t input, uint32_t sign_bit_index) +{ + assert(sign_bit_index < 63); + if (input & (1ULL << sign_bit_index)) { + /* All 1's & ~[1's from 0 to sign bit] */ + input |= ((~0ULL) & ~((1ULL << (sign_bit_index + 1)) - 1ULL)); + } + + return input; +} + +/* + * Handle xzr vs. sp, fp, lr, etc. Will *not* read the SP. + */ +static uint64_t +get_saved_state64_regno(arm_saved_state64_t *regs64, uint32_t regno, int use_xzr) +{ + /* Set PC to register value */ + switch (regno) { + case 29: + return regs64->fp; + case 30: + return regs64->lr; + case 31: + /* xzr */ + if (use_xzr) { + return 0; + } else { + return regs64->sp; + } + default: + return regs64->x[regno]; + } +} + +static void +set_saved_state64_regno(arm_saved_state64_t *regs64, uint32_t regno, int use_xzr, register_t value) +{ + /* Set PC to register value */ + switch (regno) { + case 29: + regs64->fp = value; + break; + case 30: + regs64->lr = value; + break; + case 31: + if (!use_xzr) { + regs64->sp = value; + } + break; + default: + regs64->x[regno] = value; + break; + } +} + +/* + * Common operation: extract sign-extended PC offset from instruction + * Left-shifts result by two bits. + */ +static uint64_t +extract_address_literal_sign_extended(uint32_t instr, uint32_t base, uint32_t numbits) +{ + uint64_t offset; + + offset = (instr >> base) & ((1 << numbits) - 1); + offset = sign_extend(offset, numbits - 1); + offset = offset << 2; + + return offset; +} + +static void +do_cbz_cnbz(arm_saved_state64_t *regs64, uint32_t regwidth, uint32_t instr, int is_cbz, user_addr_t *pc_out) +{ + uint32_t regno; + uint64_t regval; + uint64_t offset; + + /* Extract register */ + regno = (instr & 0x1f); + assert(regno <= 31); + regval = get_saved_state64_regno(regs64, regno, 1); + + /* Control for size */ + if (regwidth == 32) { + regval &= 0xFFFFFFFFULL; + } + + /* Extract offset */ + offset = extract_address_literal_sign_extended(instr, 5, 19); + + /* Do test */ + if ((is_cbz && regval == 0) || ((!is_cbz) && regval != 0)) { + /* Set PC from label */ + *pc_out = regs64->pc + offset; + } else { + /* Advance PC */ + *pc_out = regs64->pc + 4; + } +} + +static void +do_tbz_tbnz(arm_saved_state64_t *regs64, uint32_t instr, int is_tbz, user_addr_t *pc_out) +{ + uint64_t offset, regval; + uint32_t bit_index, b5, b40, regno, bit_set; + + /* Compute offset */ + offset = extract_address_literal_sign_extended(instr, 5, 14); + + /* Extract bit index */ + b5 = (instr >> 31); + b40 = ((instr >> 19) & 0x1f); + bit_index = (b5 << 5) | b40; + assert(bit_index <= 63); + + /* Extract register */ + regno = (instr & 0x1f); + assert(regno <= 31); + regval = get_saved_state64_regno(regs64, regno, 1); + + /* Test bit */ + bit_set = ((regval & (1 << bit_index)) != 0); + + if ((is_tbz && (!bit_set)) || ((!is_tbz) && bit_set)) { + /* Branch: unsigned addition so overflow defined */ + *pc_out = regs64->pc + offset; + } else { + /* Advance PC */ + *pc_out = regs64->pc + 4; + } +} + + +static void +fasttrap_pid_probe_handle_patched_instr64(arm_saved_state_t *state, fasttrap_tracepoint_t *tp __unused, uthread_t uthread, + proc_t *p, uint_t is_enabled, int *was_simulated) +{ + int res1, res2; + arm_saved_state64_t *regs64 = saved_state64(state); + uint32_t instr = tp->ftt_instr; + user_addr_t new_pc = 0; + + /* Neon state should be threaded throw, but hack it until we have better arm/arm64 integration */ + arm_neon_saved_state64_t *ns64 = &(get_user_neon_regs(uthread->uu_thread)->ns_64); + + /* is-enabled probe: set x0 to 1 and step forwards */ + if (is_enabled) { + regs64->x[0] = 1; + set_saved_state_pc(state, regs64->pc + 4); + return; + } + + /* For USDT probes, bypass all the emulation logic for the nop instruction */ + if (IS_ARM64_NOP(tp->ftt_instr)) { + set_saved_state_pc(state, regs64->pc + 4); + return; + } + + + /* Only one of many cases in the switch doesn't simulate */ + switch(tp->ftt_type) { + /* + * Function entry: emulate for speed. + * stp fp, lr, [sp, #-16]! + */ + case FASTTRAP_T_ARM64_STANDARD_FUNCTION_ENTRY: + { + /* Store values to stack */ + res1 = fasttrap_suword64(regs64->sp - 16, regs64->fp); + res2 = fasttrap_suword64(regs64->sp - 8, regs64->lr); + if (res1 != 0 || res2 != 0) { + fasttrap_sigsegv(p, uthread, regs64->sp - (res1 ? 16 : 8), state); + new_pc = regs64->pc; /* Bit of a hack */ + break; + } + + /* Move stack pointer */ + regs64->sp -= 16; + + /* Move PC forward */ + new_pc = regs64->pc + 4; + *was_simulated = 1; + break; + } + + /* + * PC-relative loads/stores: emulate for correctness. + * All loads are 32bits or greater (no need to handle byte or halfword accesses). + * LDR Wt, addr + * LDR Xt, addr + * LDRSW Xt, addr + * + * LDR St, addr + * LDR Dt, addr + * LDR Qt, addr + * PRFM label -> becomes a NOP + */ + case FASTTRAP_T_ARM64_LDR_S_PC_REL: + case FASTTRAP_T_ARM64_LDR_W_PC_REL: + case FASTTRAP_T_ARM64_LDR_D_PC_REL: + case FASTTRAP_T_ARM64_LDR_X_PC_REL: + case FASTTRAP_T_ARM64_LDR_Q_PC_REL: + case FASTTRAP_T_ARM64_LDRSW_PC_REL: + { + uint64_t offset; + uint32_t valsize, regno; + user_addr_t address; + union { + uint32_t val32; + uint64_t val64; + uint128_t val128; + } value; + + /* Extract 19-bit offset, add to pc */ + offset = extract_address_literal_sign_extended(instr, 5, 19); + address = regs64->pc + offset; + + /* Extract destination register */ + regno = (instr & 0x1f); + assert(regno <= 31); + + /* Read value of desired size from memory */ + switch (tp->ftt_type) { + case FASTTRAP_T_ARM64_LDR_S_PC_REL: + case FASTTRAP_T_ARM64_LDR_W_PC_REL: + case FASTTRAP_T_ARM64_LDRSW_PC_REL: + valsize = 4; + break; + case FASTTRAP_T_ARM64_LDR_D_PC_REL: + case FASTTRAP_T_ARM64_LDR_X_PC_REL: + valsize = 8; + break; + case FASTTRAP_T_ARM64_LDR_Q_PC_REL: + valsize = 16; + break; + default: + panic("Should never get here!"); + valsize = -1; + break; + } + + if (copyin(address, &value, valsize) != 0) { + fasttrap_sigsegv(p, uthread, address, state); + new_pc = regs64->pc; /* Bit of a hack, we know about update in fasttrap_sigsegv() */ + break; + } + + /* Stash in correct register slot */ + switch (tp->ftt_type) { + case FASTTRAP_T_ARM64_LDR_W_PC_REL: + set_saved_state64_regno(regs64, regno, 1, value.val32); + break; + case FASTTRAP_T_ARM64_LDRSW_PC_REL: + set_saved_state64_regno(regs64, regno, 1, sign_extend(value.val32, 31)); + break; + case FASTTRAP_T_ARM64_LDR_X_PC_REL: + set_saved_state64_regno(regs64, regno, 1, value.val64); + break; + case FASTTRAP_T_ARM64_LDR_S_PC_REL: + ns64->v.s[regno][0] = value.val32; + break; + case FASTTRAP_T_ARM64_LDR_D_PC_REL: + ns64->v.d[regno][0] = value.val64; + break; + case FASTTRAP_T_ARM64_LDR_Q_PC_REL: + ns64->v.q[regno] = value.val128; + break; + default: + panic("Should never get here!"); + } + + + /* Move PC forward */ + new_pc = regs64->pc + 4; + *was_simulated = 1; + break; + + } + + case FASTTRAP_T_ARM64_PRFM: + { + /* Becomes a NOP (architecturally permitted). Just move PC forward */ + new_pc = regs64->pc + 4; + *was_simulated = 1; + break; + } + + /* + * End explicit memory accesses. + */ + + /* + * Branches: parse condition codes if needed, emulate for correctness and + * in the case of the indirect branches, convenience + * B.cond + * CBNZ Wn, label + * CBNZ Xn, label + * CBZ Wn, label + * CBZ Xn, label + * TBNZ, Xn|Wn, #uimm16, label + * TBZ, Xn|Wn, #uimm16, label + * + * B label + * BL label + * + * BLR Xm + * BR Xm + * RET Xm + */ + case FASTTRAP_T_ARM64_B_COND: + { + int cond; + + /* Extract condition code */ + cond = (instr & 0xf); + + /* Determine if it passes */ + if (condition_true(cond, regs64->cpsr)) { + uint64_t offset; + + /* Extract 19-bit target offset, add to PC */ + offset = extract_address_literal_sign_extended(instr, 5, 19); + new_pc = regs64->pc + offset; + } else { + /* Move forwards */ + new_pc = regs64->pc + 4; + } + + *was_simulated = 1; + break; + } + + case FASTTRAP_T_ARM64_CBNZ_W: + { + do_cbz_cnbz(regs64, 32, instr, 0, &new_pc); + *was_simulated = 1; + break; + } + case FASTTRAP_T_ARM64_CBNZ_X: + { + do_cbz_cnbz(regs64, 64, instr, 0, &new_pc); + *was_simulated = 1; + break; + } + case FASTTRAP_T_ARM64_CBZ_W: + { + do_cbz_cnbz(regs64, 32, instr, 1, &new_pc); + *was_simulated = 1; + break; + } + case FASTTRAP_T_ARM64_CBZ_X: + { + do_cbz_cnbz(regs64, 64, instr, 1, &new_pc); + *was_simulated = 1; + break; + } + + case FASTTRAP_T_ARM64_TBNZ: + { + do_tbz_tbnz(regs64, instr, 0, &new_pc); + *was_simulated = 1; + break; + } + case FASTTRAP_T_ARM64_TBZ: + { + do_tbz_tbnz(regs64, instr, 1, &new_pc); + *was_simulated = 1; + break; + } + case FASTTRAP_T_ARM64_B: + case FASTTRAP_T_ARM64_BL: + { + uint64_t offset; + + /* Extract offset from instruction */ + offset = extract_address_literal_sign_extended(instr, 0, 26); + + /* Update LR if appropriate */ + if (tp->ftt_type == FASTTRAP_T_ARM64_BL) { + regs64->lr = regs64->pc + 4; + } + + /* Compute PC (unsigned addition for defined overflow) */ + new_pc = regs64->pc + offset; + *was_simulated = 1; + break; + } + + case FASTTRAP_T_ARM64_BLR: + case FASTTRAP_T_ARM64_BR: + { + uint32_t regno; + + /* Extract register from instruction */ + regno = ((instr >> 5) & 0x1f); + assert(regno <= 31); + + /* Update LR if appropriate */ + if (tp->ftt_type == FASTTRAP_T_ARM64_BLR) { + regs64->lr = regs64->pc + 4; + } + + /* Update PC in saved state */ + new_pc = get_saved_state64_regno(regs64, regno, 1); + *was_simulated = 1; + break; + } + + case FASTTRAP_T_ARM64_RET: + { + /* Extract register */ + unsigned regno = ((instr >> 5) & 0x1f); + assert(regno <= 31); + + /* Set PC to register value (xzr, not sp) */ + new_pc = get_saved_state64_regno(regs64, regno, 1); + *was_simulated = 1; + break; + } + + /* + * End branches. + */ + + /* + * Address calculations: emulate for correctness. + * + * ADRP Xd, label + * ADR Xd, label + */ + case FASTTRAP_T_ARM64_ADRP: + case FASTTRAP_T_ARM64_ADR: + { + uint64_t immhi, immlo, offset, result; + uint32_t regno; + + /* Extract destination register */ + regno = (instr & 0x1f); + assert(regno <= 31); + + /* Extract offset */ + immhi = ((instr & 0x00ffffe0) >> 5); /* bits [23,5]: 19 bits */ + immlo = ((instr & 0x60000000) >> 29); /* bits [30,29]: 2 bits */ + + /* Add to PC. Use unsigned addition so that overflow wraps (rather than being undefined). */ + if (tp->ftt_type == FASTTRAP_T_ARM64_ADRP) { + offset = (immhi << 14) | (immlo << 12); /* Concatenate bits into [32,12]*/ + offset = sign_extend(offset, 32); /* Sign extend from bit 32 */ + result = (regs64->pc & ~0xfffULL) + offset; /* And add to page of current pc */ + } else { + assert(tp->ftt_type == FASTTRAP_T_ARM64_ADR); + offset = (immhi << 2) | immlo; /* Concatenate bits into [20,0] */ + offset = sign_extend(offset, 20); /* Sign-extend */ + result = regs64->pc + offset; /* And add to page of current pc */ + } + + /* xzr, not sp */ + set_saved_state64_regno(regs64, regno, 1, result); + + /* Move PC forward */ + new_pc = regs64->pc + 4; + *was_simulated = 1; + break; + } + + /* + * End address calculations. + */ + + /* + * Everything else: thunk to userland + */ + case FASTTRAP_T_COMMON: + { + fasttrap_pid_probe_thunk_instr64(state, tp, p, uthread, &tp->ftt_instr, 1, &new_pc); + *was_simulated = 0; + break; + } + default: + { + panic("An instruction DTrace doesn't expect: %d\n", tp->ftt_type); + break; + } + } + + set_saved_state_pc(state, new_pc); + return; +} + +int +fasttrap_pid_probe(arm_saved_state_t *state) +{ + proc_t *p = current_proc(); + fasttrap_bucket_t *bucket; + lck_mtx_t *pid_mtx; + fasttrap_tracepoint_t *tp, tp_local; + pid_t pid; + dtrace_icookie_t cookie; + uint_t is_enabled = 0; + int was_simulated, retire_tp = 1; + int is_64_bit = is_saved_state64(state); + + uint64_t pc = get_saved_state_pc(state); + + assert(is_64_bit || (pc <= UINT32_MAX)); + + uthread_t uthread = (uthread_t) get_bsdthread_info(current_thread()); + + /* + * It's possible that a user (in a veritable orgy of bad planning) + * could redirect this thread's flow of control before it reached the + * return probe fasttrap. In this case we need to kill the process + * since it's in a unrecoverable state. + */ + if (uthread->t_dtrace_step) { + ASSERT(uthread->t_dtrace_on); + fasttrap_sigtrap(p, uthread, (user_addr_t)pc); + return (0); + } + + /* + * Clear all user tracing flags. + */ + uthread->t_dtrace_ft = 0; + uthread->t_dtrace_pc = 0; + uthread->t_dtrace_npc = 0; + uthread->t_dtrace_scrpc = 0; + uthread->t_dtrace_astpc = 0; + uthread->t_dtrace_reg = 0; + + /* + * Treat a child created by a call to vfork(2) as if it were its + * parent. We know that there's only one thread of control in such a + * process: this one. + */ + if (p->p_lflag & P_LINVFORK) { + proc_list_lock(); + while (p->p_lflag & P_LINVFORK) + p = p->p_pptr; + proc_list_unlock(); + } + + pid = p->p_pid; + pid_mtx = &cpu_core[CPU->cpu_id].cpuc_pid_lock; + lck_mtx_lock(pid_mtx); + bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid,pc)]; + + /* + * Lookup the tracepoint that the process just hit. + */ + for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { + if (pid == tp->ftt_pid && pc == tp->ftt_pc && + tp->ftt_proc->ftpc_acount != 0) + break; + } + + /* + * If we couldn't find a matching tracepoint, either a tracepoint has + * been inserted without using the pid<pid> ioctl interface (see + * fasttrap_ioctl), or somehow we have mislaid this tracepoint. + */ + if (tp == NULL) { + lck_mtx_unlock(pid_mtx); + return (-1); + } + + /* Validation of THUMB-related state */ + if (tp->ftt_thumb) { + if (!fasttrap_pid_probe_thumb_state_valid(saved_state32(state), tp)) { + fasttrap_tracepoint_remove(p, tp); + lck_mtx_unlock(pid_mtx); + return (-1); + } + } + + /* Execute the actual probe */ + if (tp->ftt_ids != NULL) { + fasttrap_id_t *id; + uint64_t arg4; + + if (is_saved_state64(state)) { + arg4 = get_saved_state_reg(state, 4); + } else { + uint32_t arg; + user_addr_t stack = (user_addr_t)get_saved_state_sp(state); + + fasttrap_fuword32_noerr(stack, &arg); + arg4 = arg; + } + + + /* First four parameters are passed in registers */ + + for (id = tp->ftt_ids; id != NULL; id = id->fti_next) { + fasttrap_probe_t *probe = id->fti_probe; + +#ifndef CONFIG_EMBEDDED + if (ISSET(current_proc()->p_lflag, P_LNOATTACH)) { + dtrace_probe(dtrace_probeid_error, 0 /* state */, probe->ftp_id, + 1 /* ndx */, -1 /* offset */, DTRACEFLT_UPRIV); +#else + if (FALSE) { +#endif + } else { + if (probe->ftp_prov->ftp_provider_type == DTFTP_PROVIDER_ONESHOT) { + uint8_t already_triggered = atomic_or_8(&probe->ftp_triggered, 1); + if (already_triggered) { + continue; + } + } + /* + * If we have at least one probe associated that + * is not a oneshot probe, don't remove the + * tracepoint + */ + else { + retire_tp = 0; + } + if (id->fti_ptype == DTFTP_ENTRY) { + /* + * We note that this was an entry + * probe to help ustack() find the + * first caller. + */ + cookie = dtrace_interrupt_disable(); + DTRACE_CPUFLAG_SET(CPU_DTRACE_ENTRY); + dtrace_probe(probe->ftp_id, + get_saved_state_reg(state, 0), + get_saved_state_reg(state, 1), + get_saved_state_reg(state, 2), + get_saved_state_reg(state, 3), + arg4); + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_ENTRY); + dtrace_interrupt_enable(cookie); + } else if (id->fti_ptype == DTFTP_IS_ENABLED) { + /* + * Note that in this case, we don't + * call dtrace_probe() since it's only + * an artificial probe meant to change + * the flow of control so that it + * encounters the true probe. + */ + is_enabled = 1; + } else if (probe->ftp_argmap == NULL) { + dtrace_probe(probe->ftp_id, + get_saved_state_reg(state, 0), + get_saved_state_reg(state, 1), + get_saved_state_reg(state, 2), + get_saved_state_reg(state, 3), + arg4); + + } else { + uint64_t t[5]; + + if (is_64_bit) { + fasttrap_usdt_args64(probe, saved_state64(state), 5, t); + } else { + fasttrap_usdt_args32(probe, saved_state32(state), 5, t); + } + dtrace_probe(probe->ftp_id, t[0], t[1], t[2], t[3], t[4]); + } + } + } + if (retire_tp) { + fasttrap_tracepoint_retire(p, tp); + } + } + /* + * We're about to do a bunch of work so we cache a local copy of + * the tracepoint to emulate the instruction, and then find the + * tracepoint again later if we need to light up any return probes. + */ + tp_local = *tp; + lck_mtx_unlock(pid_mtx); + tp = &tp_local; + + /* + * APPLE NOTE: + * + * Subroutines should update PC. + * We're setting this earlier than Solaris does, to get a "correct" + * ustack() output. In the Sun code, a() -> b() -> c() -> d() is + * reported at: d, b, a. The new way gives c, b, a, which is closer + * to correct, as the return instruction has already exectued. + */ + if (is_64_bit) { + fasttrap_pid_probe_handle_patched_instr64(state, tp, uthread, p, is_enabled, &was_simulated); + } else { + fasttrap_pid_probe_handle_patched_instr32(state, tp, uthread, p, is_enabled, &was_simulated); + } + + /* + * If there were no return probes when we first found the tracepoint, + * we should feel no obligation to honor any return probes that were + * subsequently enabled -- they'll just have to wait until the next + * time around. + */ + if (tp->ftt_retids != NULL) { + /* + * We need to wait until the results of the instruction are + * apparent before invoking any return probes. If this + * instruction was emulated we can just call + * fasttrap_return_common(); if it needs to be executed, we + * need to wait until the user thread returns to the kernel. + */ + /* + * It used to be that only common instructions were simulated. + * For performance reasons, we now simulate some instructions + * when safe and go back to userland otherwise. The was_simulated + * flag means we don't need to go back to userland. + */ + if (was_simulated) { + fasttrap_return_common(p, state, (user_addr_t)pc, (user_addr_t)get_saved_state_pc(state)); + } else { + ASSERT(uthread->t_dtrace_ret != 0); + ASSERT(uthread->t_dtrace_pc == pc); + ASSERT(uthread->t_dtrace_scrpc != 0); + ASSERT(((user_addr_t)get_saved_state_pc(state)) == uthread->t_dtrace_astpc); + } + } + + return (0); +} + +int +fasttrap_return_probe(arm_saved_state_t *regs) +{ + proc_t *p = current_proc(); + uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread()); + user_addr_t pc = uthread->t_dtrace_pc; + user_addr_t npc = uthread->t_dtrace_npc; + + uthread->t_dtrace_pc = 0; + uthread->t_dtrace_npc = 0; + uthread->t_dtrace_scrpc = 0; + uthread->t_dtrace_astpc = 0; + + /* + * Treat a child created by a call to vfork(2) as if it were its + * parent. We know that there's only one thread of control in such a + * process: this one. + */ + if (p->p_lflag & P_LINVFORK) { + proc_list_lock(); + while (p->p_lflag & P_LINVFORK) + p = p->p_pptr; + proc_list_unlock(); + } + + /* + * We set rp->r_pc to the address of the traced instruction so + * that it appears to dtrace_probe() that we're on the original + * instruction, and so that the user can't easily detect our + * complex web of lies. dtrace_return_probe() (our caller) + * will correctly set %pc after we return. + */ + set_saved_state_pc(regs, pc); + + fasttrap_return_common(p, regs, pc, npc); + + return (0); +} + +uint64_t +fasttrap_pid_getarg(void *arg, dtrace_id_t id, void *parg, int argno, + int aframes) +{ +#pragma unused(arg, id, parg, aframes) + arm_saved_state_t* regs = find_user_regs(current_thread()); + + if (is_saved_state32(regs)) { + /* First four arguments are in registers */ + if (argno < 4) + return saved_state32(regs)->r[argno]; + + /* Look on the stack for the rest */ + uint32_t value; + uint32_t* sp = (uint32_t*)(uintptr_t) saved_state32(regs)->sp; + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); + value = dtrace_fuword32((user_addr_t) (sp+argno-4)); + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT | CPU_DTRACE_BADADDR); + + return value; + } + else { + /* First eight arguments are in registers */ + if (argno < 8) + return saved_state64(regs)->x[argno]; + + /* Look on the stack for the rest */ + uint64_t value; + uint64_t* sp = (uint64_t*) saved_state64(regs)->sp; + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); + value = dtrace_fuword64((user_addr_t) (sp+argno-8)); + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT | CPU_DTRACE_BADADDR); + + return value; + } + +} + +uint64_t +fasttrap_usdt_getarg(void *arg, dtrace_id_t id, void *parg, int argno, int aframes) +{ +#pragma unused(arg, id, parg, argno, aframes) +#if 0 + return (fasttrap_anarg(ttolwp(curthread)->lwp_regs, 0, argno)); +#endif + + return 0; +} + diff --git a/bsd/dev/arm64/fbt_arm.c b/bsd/dev/arm64/fbt_arm.c new file mode 100644 index 000000000..c2f348f9a --- /dev/null +++ b/bsd/dev/arm64/fbt_arm.c @@ -0,0 +1,608 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + */ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* #pragma ident "@(#)fbt.c 1.15 05/09/19 SMI" */ + +#ifdef KERNEL +#ifndef _KERNEL +#define _KERNEL /* Solaris vs. Darwin */ +#endif +#endif + +#define MACH__POSIX_C_SOURCE_PRIVATE 1 /* pulls in suitable savearea from + * mach/ppc/thread_status.h */ +#include <kern/thread.h> +#include <mach/thread_status.h> +#include <arm/proc_reg.h> +#include <arm/caches_internal.h> + +#include <mach-o/loader.h> +#include <mach-o/nlist.h> +#include <libkern/kernel_mach_header.h> + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/errno.h> +#include <sys/stat.h> +#include <sys/ioctl.h> +#include <sys/conf.h> +#include <sys/fcntl.h> +#include <miscfs/devfs/devfs.h> + +#include <sys/dtrace.h> +#include <sys/dtrace_impl.h> +#include <sys/fbt.h> + +#include <sys/dtrace_glue.h> + +#define DTRACE_INVOP_PUSH_FRAME 11 + +#define DTRACE_INVOP_NOP_SKIP 4 +#define DTRACE_INVOP_ADD_FP_SP_SKIP 4 + +#define DTRACE_INVOP_POP_PC_SKIP 2 + +/* + * stp fp, lr, [sp, #val] + * stp fp, lr, [sp, #val]! + */ +#define FBT_IS_ARM64_FRAME_PUSH(x) \ + (((x) & 0xffc07fff) == 0xa9007bfd || ((x) & 0xffc07fff) == 0xa9807bfd) + +/* + * stp Xt1, Xt2, [sp, #val] + * stp Xt1, Xt2, [sp, #val]! + */ +#define FBT_IS_ARM64_PUSH(x) \ + (((x) & 0xffc003e0) == 0xa90003e0 || ((x) & 0xffc003e0) == 0xa98003e0) + +/* + * ldp fp, lr, [sp, #val] + * ldp fp, lr, [sp], #val + */ +#define FBT_IS_ARM64_FRAME_POP(x) \ + (((x) & 0xffc07fff) == 0xa9407bfd || ((x) & 0xffc07fff) == 0xa8c07bfd) + +#define FBT_IS_ARM64_ADD_FP_SP(x) (((x) & 0xffc003ff) == 0x910003fd) /* add fp, sp, #val (add fp, sp, #0 == mov fp, sp) */ +#define FBT_IS_ARM64_RET(x) ((x) == 0xd65f03c0) /* ret */ + + +#define FBT_B_MASK 0xff000000 +#define FBT_B_IMM_MASK 0x00ffffff +#define FBT_B_INSTR 0x14000000 + +#define FBT_IS_ARM64_B_INSTR(x) ((x & FBT_B_MASK) == FBT_B_INSTR) +#define FBT_GET_ARM64_B_IMM(x) ((x & FBT_B_IMM_MASK) << 2) + +#define FBT_PATCHVAL 0xe7eeee7e +#define FBT_AFRAMES_ENTRY 7 +#define FBT_AFRAMES_RETURN 7 + +#define FBT_ENTRY "entry" +#define FBT_RETURN "return" +#define FBT_ADDR2NDX(addr) ((((uintptr_t)(addr)) >> 4) & fbt_probetab_mask) + +extern dtrace_provider_id_t fbt_id; +extern fbt_probe_t **fbt_probetab; +extern int fbt_probetab_mask; + +kern_return_t fbt_perfCallback(int, struct arm_saved_state *, __unused int, __unused int); + +int +fbt_invop(uintptr_t addr, uintptr_t * stack, uintptr_t rval) +{ + fbt_probe_t *fbt = fbt_probetab[FBT_ADDR2NDX(addr)]; + + for (; fbt != NULL; fbt = fbt->fbtp_hashnext) { + if ((uintptr_t) fbt->fbtp_patchpoint == addr) { + if (0 == CPU->cpu_dtrace_invop_underway) { + CPU->cpu_dtrace_invop_underway = 1; /* Race not possible on + * this per-cpu state */ + + if (fbt->fbtp_roffset == 0) { + /* + * Stack looks like this: + * + * [Higher addresses] + * + * Frame of caller + * Extra args for callee + * ------------------------ + * Frame from traced function: <previous sp (e.g. 0x1000), return address> + * ------------------------ + * arm_context_t + * ------------------------ + * Frame from trap handler: <previous sp (e.g. 0x1000) , traced PC > + * The traced function never got to mov fp, sp, + * so there is no frame in the backtrace pointing + * to the frame on the stack containing the LR in the + * caller. + * ------------------------ + * | + * | + * | stack grows this way + * | + * | + * v + * [Lower addresses] + */ + + arm_saved_state_t *regs = (arm_saved_state_t *)(&((arm_context_t *)stack)->ss); + + /* + * cpu_dtrace_caller compensates for fact that the traced function never got to update its fp. + * When walking the stack, when we reach the frame where we extract a PC in the patched + * function, we put the cpu_dtrace_caller in the backtrace instead. The next frame we extract + * will be in the caller's caller, so we output a backtrace starting at the caller and going + * sequentially up the stack. + */ + CPU->cpu_dtrace_caller = get_saved_state_lr(regs); + dtrace_probe(fbt->fbtp_id, get_saved_state_reg(regs, 0), get_saved_state_reg(regs, 1), + get_saved_state_reg(regs, 2), get_saved_state_reg(regs, 3),get_saved_state_reg(regs, 4)); + CPU->cpu_dtrace_caller = 0; + } else { + /* + * When fbtp_roffset is non-zero, we know we are handling a return probe point. + * + * + * Stack looks like this, as we've already popped the frame in the traced callee, and + * we trap with lr set to the return address in the caller. + * [Higher addresses] + * + * Frame of caller + * Extra args for callee + * ------------------------ + * arm_context_t + * ------------------------ + * Frame from trap handler: <sp at time of trap, traced PC > + * ------------------------ + * | + * | + * | stack grows this way + * | + * | + * v + * [Lower addresses] + */ + arm_saved_state_t *regs = (arm_saved_state_t *)(&((arm_context_t *)stack)->ss); + + CPU->cpu_dtrace_caller = get_saved_state_lr(regs); + dtrace_probe(fbt->fbtp_id, fbt->fbtp_roffset, rval, 0, 0, 0); + CPU->cpu_dtrace_caller = 0; + } + CPU->cpu_dtrace_invop_underway = 0; + } + + /* + On other architectures, we return a DTRACE constant to let the callback function + know what was replaced. On the ARM, since the function prologue/epilogue machine code + can vary, we need the actual bytes of the instruction, so return the savedval instead. + */ + return (fbt->fbtp_savedval); + } + } + + return (0); +} + +#define IS_USER_TRAP(regs) (PSR64_IS_USER(get_saved_state_cpsr(regs))) +#define T_INVALID_OPCODE EXC_BAD_INSTRUCTION +#define FBT_EXCEPTION_CODE T_INVALID_OPCODE + +kern_return_t +fbt_perfCallback( + int trapno, + struct arm_saved_state * regs, + __unused int unused1, + __unused int unused2) +{ + kern_return_t retval = KERN_FAILURE; + + if (FBT_EXCEPTION_CODE == trapno && !IS_USER_TRAP(regs)) { + boolean_t oldlevel = 0; + machine_inst_t emul = 0; + uint64_t sp, pc, lr, imm; + + oldlevel = ml_set_interrupts_enabled(FALSE); + + __asm__ volatile( + "Ldtrace_invop_callsite_pre_label:\n" + ".data\n" + ".private_extern _dtrace_invop_callsite_pre\n" + "_dtrace_invop_callsite_pre:\n" + " .quad Ldtrace_invop_callsite_pre_label\n" + ".text\n" + ); + + emul = dtrace_invop(get_saved_state_pc(regs), (uintptr_t*) regs, get_saved_state_reg(regs,0)); + + __asm__ volatile( + "Ldtrace_invop_callsite_post_label:\n" + ".data\n" + ".private_extern _dtrace_invop_callsite_post\n" + "_dtrace_invop_callsite_post:\n" + " .quad Ldtrace_invop_callsite_post_label\n" + ".text\n" + ); + + if (emul == DTRACE_INVOP_NOP) { + /* + * Skip over the patched NOP planted by sdt + */ + pc = get_saved_state_pc(regs); + set_saved_state_pc(regs, pc + DTRACE_INVOP_NOP_SKIP); + retval = KERN_SUCCESS; + } else if (FBT_IS_ARM64_ADD_FP_SP(emul)) { + /* retrieve the value to add */ + uint64_t val = (emul >> 10) & 0xfff; + assert(val < 4096); + + /* retrieve sp */ + sp = get_saved_state_sp(regs); + + /* + * emulate the instruction: + * add fp, sp, #val + */ + assert(sp < (UINT64_MAX - val)); + set_saved_state_fp(regs, sp + val); + + /* skip over the bytes of the patched instruction */ + pc = get_saved_state_pc(regs); + set_saved_state_pc(regs, pc + DTRACE_INVOP_ADD_FP_SP_SKIP); + + retval = KERN_SUCCESS; + } else if (FBT_IS_ARM64_RET(emul)) { + lr = get_saved_state_lr(regs); + set_saved_state_pc(regs, lr); + retval = KERN_SUCCESS; + } else if (FBT_IS_ARM64_B_INSTR(emul)) { + pc = get_saved_state_pc(regs); + imm = FBT_GET_ARM64_B_IMM(emul); + set_saved_state_pc(regs, pc + imm); + retval = KERN_SUCCESS; + } else if (emul == FBT_PATCHVAL) { + /* Means we encountered an error but handled it, try same inst again */ + retval = KERN_SUCCESS; + } else { + retval = KERN_FAILURE; + } + + ml_set_interrupts_enabled(oldlevel); + } + + return retval; +} + +void +fbt_provide_probe(struct modctl *ctl, uintptr_t instrLow, uintptr_t instrHigh, char *modname, char* symbolName, machine_inst_t* symbolStart) +{ + unsigned int j; + int doenable = 0; + dtrace_id_t thisid; + + fbt_probe_t *newfbt, *retfbt, *entryfbt; + machine_inst_t *instr, *pushinstr = NULL, *limit, theInstr; + int foundPushLR, savedRegs; + + /* + * Guard against null symbols + */ + if (!symbolStart || !instrLow || !instrHigh) { + kprintf("dtrace: %s has an invalid address\n", symbolName); + return; + } + + /* + * Assume the compiler doesn't schedule instructions in the prologue. + */ + + foundPushLR = 0; + savedRegs = -1; + limit = (machine_inst_t *)instrHigh; + + assert(sizeof(*instr) == 4); + + for (j = 0, instr = symbolStart, theInstr = 0; + (j < 8) && ((uintptr_t)instr >= instrLow) && (instrHigh > (uintptr_t)(instr)); j++, instr++) + { + /* + * Count the number of time we pushed something onto the stack + * before hitting a frame push. That will give us an estimation + * of how many stack pops we should expect when looking for the + * RET instruction. + */ + theInstr = *instr; + if (FBT_IS_ARM64_FRAME_PUSH(theInstr)) { + foundPushLR = 1; + pushinstr = instr; + } + + if (foundPushLR && (FBT_IS_ARM64_ADD_FP_SP(theInstr))) + /* Guard against a random setting of fp from sp, we make sure we found the push first */ + break; + if (FBT_IS_ARM64_RET(theInstr)) /* We've gone too far, bail. */ + break; + if (FBT_IS_ARM64_FRAME_POP(theInstr)) /* We've gone too far, bail. */ + break; + } + + if (!(foundPushLR && (FBT_IS_ARM64_ADD_FP_SP(theInstr)))) { + return; + } + + thisid = dtrace_probe_lookup(fbt_id, modname, symbolName, FBT_ENTRY); + newfbt = kmem_zalloc(sizeof(fbt_probe_t), KM_SLEEP); + newfbt->fbtp_next = NULL; + strlcpy( (char *)&(newfbt->fbtp_name), symbolName, MAX_FBTP_NAME_CHARS ); + + if (thisid != 0) { + /* + * The dtrace_probe previously existed, so we have to hook + * the newfbt entry onto the end of the existing fbt's + * chain. + * If we find an fbt entry that was previously patched to + * fire, (as indicated by the current patched value), then + * we want to enable this newfbt on the spot. + */ + entryfbt = dtrace_probe_arg (fbt_id, thisid); + ASSERT (entryfbt != NULL); + for(; entryfbt != NULL; entryfbt = entryfbt->fbtp_next) { + if (entryfbt->fbtp_currentval == entryfbt->fbtp_patchval) + doenable++; + + if (entryfbt->fbtp_next == NULL) { + entryfbt->fbtp_next = newfbt; + newfbt->fbtp_id = entryfbt->fbtp_id; + break; + } + } + } + else { + /* + * The dtrace_probe did not previously exist, so we + * create it and hook in the newfbt. Since the probe is + * new, we obviously do not need to enable it on the spot. + */ + newfbt->fbtp_id = dtrace_probe_create(fbt_id, modname, symbolName, FBT_ENTRY, FBT_AFRAMES_ENTRY, newfbt); + doenable = 0; + } + + newfbt->fbtp_patchpoint = instr; + newfbt->fbtp_ctl = ctl; + newfbt->fbtp_loadcnt = ctl->mod_loadcnt; + newfbt->fbtp_rval = DTRACE_INVOP_PUSH_FRAME; + newfbt->fbtp_savedval = theInstr; + newfbt->fbtp_patchval = FBT_PATCHVAL; + newfbt->fbtp_currentval = 0; + newfbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; + fbt_probetab[FBT_ADDR2NDX(instr)] = newfbt; + + if (doenable) + fbt_enable(NULL, newfbt->fbtp_id, newfbt); + + /* + * The fbt entry chain is in place, one entry point per symbol. + * The fbt return chain can have multiple return points per + * symbol. + * Here we find the end of the fbt return chain. + */ + + doenable=0; + + thisid = dtrace_probe_lookup(fbt_id, modname, symbolName, FBT_RETURN); + + if (thisid != 0) { + /* The dtrace_probe previously existed, so we have to + * find the end of the existing fbt chain. If we find + * an fbt return that was previously patched to fire, + * (as indicated by the currrent patched value), then + * we want to enable any new fbts on the spot. + */ + retfbt = dtrace_probe_arg (fbt_id, thisid); + ASSERT(retfbt != NULL); + for (; retfbt != NULL; retfbt = retfbt->fbtp_next) { + if (retfbt->fbtp_currentval == retfbt->fbtp_patchval) + doenable++; + if(retfbt->fbtp_next == NULL) + break; + } + } + else { + doenable = 0; + retfbt = NULL; + } + + /* + * Go back to the start of the function, in case + * the compiler emitted pcrel data loads + * before FP was adjusted. + */ + instr = pushinstr + 1; +again: + if (instr >= limit) + return; + + /* XXX FIXME ... extra jump table detection? */ + + /* + * OK, it's an instruction. + */ + theInstr = *instr; + + /* Walked onto the start of the next routine? If so, bail out from this function */ + if (FBT_IS_ARM64_FRAME_PUSH(theInstr)) { + if (!retfbt) + kprintf("dtrace: fbt: No return probe for %s, walked to next routine at 0x%016llx\n",symbolName,(uint64_t)instr); + return; + } + + /* XXX fancy detection of end of function using PC-relative loads */ + + /* + * Look for: + * ldp fp, lr, [sp], #val + * ldp fp, lr, [sp, #val] + */ + if (!FBT_IS_ARM64_FRAME_POP(theInstr)) { + instr++; + goto again; + } + + /* go to the next instruction */ + instr++; + + /* Scan ahead for a ret or a branch outside the function */ + for (; instr < limit; instr++) { + theInstr = *instr; + if (FBT_IS_ARM64_RET(theInstr)) + break; + if (FBT_IS_ARM64_B_INSTR(theInstr)) { + machine_inst_t *dest = instr + FBT_GET_ARM64_B_IMM(theInstr); + /* + * Check whether the destination of the branch + * is outside of the function + */ + if (dest >= limit || dest < symbolStart) + break; + } + } + + if (!FBT_IS_ARM64_RET(theInstr) && !FBT_IS_ARM64_B_INSTR(theInstr)) + return; + + newfbt = kmem_zalloc(sizeof(fbt_probe_t), KM_SLEEP); + newfbt->fbtp_next = NULL; + strlcpy( (char *)&(newfbt->fbtp_name), symbolName, MAX_FBTP_NAME_CHARS ); + + if (retfbt == NULL) { + newfbt->fbtp_id = dtrace_probe_create(fbt_id, modname, + symbolName, FBT_RETURN, FBT_AFRAMES_RETURN, newfbt); + } else { + retfbt->fbtp_next = newfbt; + newfbt->fbtp_id = retfbt->fbtp_id; + } + + retfbt = newfbt; + newfbt->fbtp_patchpoint = instr; + newfbt->fbtp_ctl = ctl; + newfbt->fbtp_loadcnt = ctl->mod_loadcnt; + + ASSERT(FBT_IS_ARM64_RET(theInstr)); + newfbt->fbtp_rval = DTRACE_INVOP_RET; + newfbt->fbtp_roffset = (uintptr_t) ((uint8_t*) instr - (uint8_t *)symbolStart); + newfbt->fbtp_savedval = theInstr; + newfbt->fbtp_patchval = FBT_PATCHVAL; + newfbt->fbtp_currentval = 0; + newfbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; + fbt_probetab[FBT_ADDR2NDX(instr)] = newfbt; + + if (doenable) + fbt_enable(NULL, newfbt->fbtp_id, newfbt); + + instr++; + goto again; +} + +void +fbt_provide_module_kernel_syms(struct modctl *ctl) +{ + kernel_mach_header_t *mh; + struct load_command *cmd; + kernel_segment_command_t *orig_ts = NULL, *orig_le = NULL; + struct symtab_command *orig_st = NULL; + kernel_nlist_t *sym = NULL; + char *strings; + uintptr_t instrLow, instrHigh; + char *modname; + unsigned int i; + + mh = (kernel_mach_header_t *)(ctl->mod_address); + modname = ctl->mod_modname; + + /* + * Employees of dtrace and their families are ineligible. Void + * where prohibited. + */ + + if (mh->magic != MH_MAGIC_KERNEL) + return; + + cmd = (struct load_command *) & mh[1]; + for (i = 0; i < mh->ncmds; i++) { + if (cmd->cmd == LC_SEGMENT_KERNEL) { + kernel_segment_command_t *orig_sg = (kernel_segment_command_t *) cmd; + + if (LIT_STRNEQL(orig_sg->segname, SEG_TEXT)) + orig_ts = orig_sg; + else if (LIT_STRNEQL(orig_sg->segname, SEG_LINKEDIT)) + orig_le = orig_sg; + else if (LIT_STRNEQL(orig_sg->segname, "")) + orig_ts = orig_sg; /* kexts have a single + * unnamed segment */ + } else if (cmd->cmd == LC_SYMTAB) + orig_st = (struct symtab_command *) cmd; + + cmd = (struct load_command *) ((caddr_t) cmd + cmd->cmdsize); + } + + if ((orig_ts == NULL) || (orig_st == NULL) || (orig_le == NULL)) + return; + + sym = (kernel_nlist_t *)(orig_le->vmaddr + orig_st->symoff - orig_le->fileoff); + strings = (char *)(orig_le->vmaddr + orig_st->stroff - orig_le->fileoff); + + /* Find extent of the TEXT section */ + instrLow = (uintptr_t) orig_ts->vmaddr; + instrHigh = (uintptr_t) (orig_ts->vmaddr + orig_ts->vmsize); + + for (i = 0; i < orig_st->nsyms; i++) { + uint8_t n_type = sym[i].n_type & (N_TYPE | N_EXT); + char *name = strings + sym[i].n_un.n_strx; + + /* Check that the symbol is a global and that it has a name. */ + if (((N_SECT | N_EXT) != n_type && (N_ABS | N_EXT) != n_type)) + continue; + + if (0 == sym[i].n_un.n_strx) /* iff a null, "", name. */ + continue; + + /* Lop off omnipresent leading underscore. */ + if (*name == '_') + name += 1; + + /* + * We're only blacklisting functions in the kernel for now. + */ + if (MOD_IS_MACH_KERNEL(ctl) && fbt_excluded(name)) + continue; + + fbt_provide_probe(ctl, instrLow, instrHigh, modname, name, (machine_inst_t*)sym[i].n_value); + } +} diff --git a/bsd/dev/arm64/sdt_arm.c b/bsd/dev/arm64/sdt_arm.c new file mode 100644 index 000000000..17bb69327 --- /dev/null +++ b/bsd/dev/arm64/sdt_arm.c @@ -0,0 +1,162 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2006 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* #pragma ident "@(#)sdt.c 1.6 06/03/24 SMI" */ + +#ifdef KERNEL +#ifndef _KERNEL +#define _KERNEL /* Solaris vs. Darwin */ +#endif +#endif + +#include <kern/cpu_data.h> +#include <kern/debug.h> +#include <kern/thread.h> +#include <mach/thread_status.h> +#include <mach/vm_param.h> + +#include <sys/dtrace.h> +#include <sys/dtrace_impl.h> + +#include <sys/dtrace_glue.h> + +#include <sys/sdt_impl.h> + +extern sdt_probe_t **sdt_probetab; + +int +sdt_invop(__unused uintptr_t addr, __unused uintptr_t *stack, __unused uintptr_t eax) +{ +#pragma unused(eax) + sdt_probe_t *sdt = sdt_probetab[SDT_ADDR2NDX(addr)]; + + for (; sdt != NULL; sdt = sdt->sdp_hashnext) { + if ((uintptr_t) sdt->sdp_patchpoint == addr) { + struct arm_saved_state* regs = (struct arm_saved_state*) stack; + + dtrace_probe(sdt->sdp_id, get_saved_state_reg(regs, 0), get_saved_state_reg(regs, 1), + get_saved_state_reg(regs, 2), get_saved_state_reg(regs, 3),get_saved_state_reg(regs, 4)); + + return (DTRACE_INVOP_NOP); + } + } + + return (0); +} + +struct frame { + struct frame *backchain; + uintptr_t retaddr; +}; + +/*ARGSUSED*/ +uint64_t +sdt_getarg(void *arg, dtrace_id_t id, void *parg, int argno, int aframes) +{ + +#pragma unused(arg,id,parg) /* __APPLE__ */ + + uint64_t val = 0; + struct frame *fp = (struct frame *)__builtin_frame_address(0); + uintptr_t *stack; + uintptr_t pc; + int i; + + /* + * A total of eight arguments are passed via registers; any argument + * with an index of 7 or lower is therefore in a register. + */ + + int inreg = 7; + + for (i = 1; i <= aframes; i++) { + fp = fp->backchain; + pc = fp->retaddr; + + if (dtrace_invop_callsite_pre != NULL + && pc > (uintptr_t)dtrace_invop_callsite_pre + && pc <= (uintptr_t)dtrace_invop_callsite_post) { + + /* + * When we pass through the invalid op handler, + * we expect to find the save area structure, + * pushed on the stack where we took the trap. + * If the argument we seek is passed in a register, then + * we can load it directly from this saved area. + * If the argument we seek is passed on the stack, then + * we increment the frame pointer further, to find the + * pushed args + */ + + /* fp points to the dtrace_invop activation */ + fp = fp->backchain; /* fbt_perfCallback */ + fp = fp->backchain; /* sleh_synchronous */ + fp = fp->backchain; /* fleh_synchronous */ + + arm_saved_state_t *tagged_regs = (arm_saved_state_t *)((uintptr_t *)&fp[1]); + arm_saved_state64_t *saved_state = saved_state64(tagged_regs); + + if (argno <= inreg) { + /* The argument will be in a register */ + stack = (uintptr_t *)&saved_state->x[0]; + } else { + /* The argument will be found on the stack */ + fp = (struct frame *)(saved_state->sp); + stack = (uintptr_t *)&fp[0]; /* Find marshalled arguments */ + argno -= (inreg + 1); + } + goto load; + } + } + + /* + * We know that we did not come through a trap to get into + * dtrace_probe() -- We arrive here when the provider has + * called dtrace_probe() directly. + * The probe ID is the first argument to dtrace_probe(). + * We must advance beyond that to get the argX. + */ + argno++; /* Advance past probeID */ + + if (argno <= inreg) { + /* + * This shouldn't happen. If the argument is passed in a + * register then it should have been, well, passed in a + * register... + */ + DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); + return (0); + } + + argno -= (inreg + 1); + stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */ + +load: + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); + /* dtrace_probe arguments arg0 .. arg4 are 64bits wide */ + val = (uint64_t)(*(((uintptr_t *)stack) + argno)); + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); + return (val); +} diff --git a/bsd/dev/arm64/sysctl.c b/bsd/dev/arm64/sysctl.c new file mode 100644 index 000000000..22dcc12d7 --- /dev/null +++ b/bsd/dev/arm64/sysctl.c @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2003-2007 Apple Inc. All rights reserved. + */ +#include <sys/param.h> +#include <sys/kernel.h> +#include <sys/sysctl.h> + +#include <machine/machine_routines.h> + +extern uint64_t wake_abstime; + +static +SYSCTL_QUAD(_machdep, OID_AUTO, wake_abstime, + CTLFLAG_RD, &wake_abstime, + "Absolute Time at the last wakeup"); + +static int +sysctl_time_since_reset SYSCTL_HANDLER_ARGS +{ +#pragma unused(arg1, arg2, oidp) + int error = 0; + uint64_t return_value = 0; + + return_value = ml_get_time_since_reset(); + + SYSCTL_OUT(req, &return_value, sizeof(return_value)); + + return error; +} + +SYSCTL_PROC(_machdep, OID_AUTO, time_since_reset, + CTLFLAG_RD | CTLTYPE_QUAD | CTLFLAG_LOCKED, + 0, 0, sysctl_time_since_reset, "I", + "Continuous time since last SOC boot/wake started"); + +static int +sysctl_wake_conttime SYSCTL_HANDLER_ARGS +{ +#pragma unused(arg1, arg2, oidp) + int error = 0; + uint64_t return_value = 0; + + return_value = ml_get_conttime_wake_time(); + + SYSCTL_OUT(req, &return_value, sizeof(return_value)); + + return error; +} + +SYSCTL_PROC(_machdep, OID_AUTO, wake_conttime, + CTLFLAG_RD | CTLTYPE_QUAD | CTLFLAG_LOCKED, + 0, 0, sysctl_wake_conttime, "I", + "Continuous Time at the last wakeup"); + + diff --git a/bsd/dev/dtrace/dtrace.c b/bsd/dev/dtrace/dtrace.c index f1f32ea70..75fc4d38d 100644 --- a/bsd/dev/dtrace/dtrace.c +++ b/bsd/dev/dtrace/dtrace.c @@ -100,6 +100,11 @@ #include <libkern/sysctl.h> #include <sys/kdebug.h> +#if MONOTONIC +#include <kern/monotonic.h> +#include <machine/monotonic.h> +#endif /* MONOTONIC */ + #include <kern/cpu_data.h> extern uint32_t pmap_find_phys(void *, uint64_t); extern boolean_t pmap_valid_page(uint32_t); @@ -145,7 +150,7 @@ uint64_t dtrace_buffer_memory_inuse = 0; int dtrace_destructive_disallow = 0; dtrace_optval_t dtrace_nonroot_maxsize = (16 * 1024 * 1024); size_t dtrace_difo_maxsize = (256 * 1024); -dtrace_optval_t dtrace_dof_maxsize = (384 * 1024); +dtrace_optval_t dtrace_dof_maxsize = (512 * 1024); dtrace_optval_t dtrace_statvar_maxsize = (16 * 1024); dtrace_optval_t dtrace_statvar_maxsize_max = (16 * 10 * 1024); size_t dtrace_actions_max = (16 * 1024); @@ -289,7 +294,7 @@ static int dtrace_module_unloaded(struct kmod_info *kmod); * * ASSERT(MUTEX_HELD(&cpu_lock)); * becomes: - * lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED); + * LCK_MTX_ASSERT(&cpu_lock, LCK_MTX_ASSERT_OWNED); * */ static lck_mtx_t dtrace_lock; /* probe state lock */ @@ -352,6 +357,15 @@ int dtrace_helptrace_enabled = 1; int dtrace_helptrace_enabled = 0; #endif +#if defined (__arm64__) +/* + * The ioctl for adding helper DOF is based on the + * size of a user_addr_t. We need to recognize both + * U32 and U64 as the same action. + */ +#define DTRACEHIOC_ADDDOF_U32 _IOW('h', 4, user32_addr_t) +#define DTRACEHIOC_ADDDOF_U64 _IOW('h', 4, user64_addr_t) +#endif /* __arm64__ */ /* * DTrace Error Hashing @@ -418,6 +432,25 @@ static lck_mtx_t dtrace_errlock; (where) = ((thr + DIF_VARIABLE_MAX) & \ (((uint64_t)1 << 61) - 1)) | ((uint64_t)intr << 61); \ } +#elif defined(__arm__) +/* FIXME: three function calls!!! */ +#define DTRACE_TLS_THRKEY(where) { \ + uint_t intr = ml_at_interrupt_context(); /* Note: just one measly bit */ \ + uint64_t thr = (uintptr_t)current_thread(); \ + uint_t pid = (uint_t)dtrace_proc_selfpid(); \ + ASSERT(intr < (1 << 3)); \ + (where) = (((thr << 32 | pid) + DIF_VARIABLE_MAX) & \ + (((uint64_t)1 << 61) - 1)) | ((uint64_t)intr << 61); \ +} +#elif defined (__arm64__) +/* FIXME: two function calls!! */ +#define DTRACE_TLS_THRKEY(where) { \ + uint_t intr = ml_at_interrupt_context(); /* Note: just one measly bit */ \ + uint64_t thr = (uintptr_t)current_thread(); \ + ASSERT(intr < (1 << 3)); \ + (where) = ((thr + DIF_VARIABLE_MAX) & \ + (((uint64_t)1 << 61) - 1)) | ((uint64_t)intr << 61); \ +} #else #error Unknown architecture #endif @@ -786,13 +819,21 @@ sysctl_dtrace_provide_private_probes SYSCTL_HANDLER_ARGS if (error) return (error); - if (value != 0 && value != 1) - return (ERANGE); + if (req->newptr) { + if (value != 0 && value != 1) + return (ERANGE); - lck_mtx_lock(&dtrace_lock); - dtrace_provide_private_probes = value; - lck_mtx_unlock(&dtrace_lock); + /* + * We do not allow changing this back to zero, as private probes + * would still be left registered + */ + if (value != 1) + return (EPERM); + lck_mtx_lock(&dtrace_lock); + dtrace_provide_private_probes = value; + lck_mtx_unlock(&dtrace_lock); + } return (0); } @@ -1052,7 +1093,7 @@ dtrace_canstore_remains(uint64_t addr, size_t sz, size_t *remain, * DTrace subroutines (DIF_SUBR_*) should use this helper to implement * appropriate memory access protection. */ -static int +int dtrace_canload(uint64_t addr, size_t sz, dtrace_mstate_t *mstate, dtrace_vstate_t *vstate) { @@ -3180,6 +3221,7 @@ dtrace_dif_variable(dtrace_mstate_t *mstate, dtrace_state_t *state, uint64_t v, * APPLE NOTE: Account for introduction of __dtrace_probe() */ int aframes = mstate->dtms_probe->dtpr_aframes + 3; + dtrace_vstate_t *vstate = &state->dts_vstate; dtrace_provider_t *pv; uint64_t val; @@ -3194,7 +3236,7 @@ dtrace_dif_variable(dtrace_mstate_t *mstate, dtrace_state_t *state, uint64_t v, } else - val = dtrace_getarg(ndx, aframes); + val = dtrace_getarg(ndx, aframes, mstate, vstate); /* * This is regrettably required to keep the compiler @@ -3467,27 +3509,47 @@ dtrace_dif_variable(dtrace_mstate_t *mstate, dtrace_state_t *state, uint64_t v, case DIF_VAR_ZONENAME: - { - /* scratch_size is equal to length('global') + 1 for the null-terminator. */ - char *zname = (char *)mstate->dtms_scratch_ptr; - size_t scratch_size = 6 + 1; + { + /* scratch_size is equal to length('global') + 1 for the null-terminator. */ + char *zname = (char *)mstate->dtms_scratch_ptr; + size_t scratch_size = 6 + 1; if (!dtrace_priv_proc(state)) return (0); - /* The scratch allocation's lifetime is that of the clause. */ - if (!DTRACE_INSCRATCH(mstate, scratch_size)) { - DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); - return 0; - } + /* The scratch allocation's lifetime is that of the clause. */ + if (!DTRACE_INSCRATCH(mstate, scratch_size)) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); + return 0; + } + + mstate->dtms_scratch_ptr += scratch_size; - mstate->dtms_scratch_ptr += scratch_size; + /* The kernel does not provide zonename, it will always return 'global'. */ + strlcpy(zname, "global", scratch_size); - /* The kernel does not provide zonename, it will always return 'global'. */ - strlcpy(zname, "global", scratch_size); + return ((uint64_t)(uintptr_t)zname); + } - return ((uint64_t)(uintptr_t)zname); - } +#if MONOTONIC + case DIF_VAR_CPUINSTRS: + return mt_cur_cpu_instrs(); + + case DIF_VAR_CPUCYCLES: + return mt_cur_cpu_cycles(); + + case DIF_VAR_VINSTRS: + return mt_cur_thread_instrs(); + + case DIF_VAR_VCYCLES: + return mt_cur_thread_cycles(); +#else /* MONOTONIC */ + case DIF_VAR_CPUINSTRS: /* FALLTHROUGH */ + case DIF_VAR_CPUCYCLES: /* FALLTHROUGH */ + case DIF_VAR_VINSTRS: /* FALLTHROUGH */ + case DIF_VAR_VCYCLES: /* FALLTHROUGH */ + return 0; +#endif /* !MONOTONIC */ case DIF_VAR_UID: if (!dtrace_priv_proc_relaxed(state)) @@ -3896,7 +3958,7 @@ dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs, char c, target = (char)tupregs[1].dttk_value; if (!dtrace_strcanload(addr, size, &lim, mstate, vstate)) { - regs[rd] = NULL; + regs[rd] = 0; break; } addr_limit = addr + lim; @@ -4156,7 +4218,7 @@ dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs, */ regs[rd] = 0; mstate->dtms_strtok = 0; - mstate->dtms_strtok_limit = NULL; + mstate->dtms_strtok_limit = 0; break; } @@ -4305,9 +4367,20 @@ dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs, case DIF_SUBR_LLTOSTR: { int64_t i = (int64_t)tupregs[0].dttk_value; - int64_t val = i < 0 ? i * -1 : i; - uint64_t size = 22; /* enough room for 2^64 in decimal */ + uint64_t val, digit; + uint64_t size = 65; /* enough room for 2^64 in binary */ char *end = (char *)mstate->dtms_scratch_ptr + size - 1; + int base = 10; + + if (nargs > 1) { + if ((base = tupregs[1].dttk_value) <= 1 || + base > ('z' - 'a' + 1) + ('9' - '0' + 1)) { + *flags |= CPU_DTRACE_ILLOP; + break; + } + } + + val = (base == 10 && i < 0) ? i * -1 : i; if (!DTRACE_INSCRATCH(mstate, size)) { DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); @@ -4315,13 +4388,24 @@ dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs, break; } - for (*end-- = '\0'; val; val /= 10) - *end-- = '0' + (val % 10); + for (*end-- = '\0'; val; val /= base) { + if ((digit = val % base) <= '9' - '0') { + *end-- = '0' + digit; + } else { + *end-- = 'a' + (digit - ('9' - '0') - 1); + } + } - if (i == 0) + if (i == 0 && base == 16) *end-- = '0'; - if (i < 0) + if (base == 16) + *end-- = 'x'; + + if (i == 0 || base == 8 || base == 16) + *end-- = '0'; + + if (i < 0 && base == 10) *end-- = '-'; regs[rd] = (uintptr_t)end + 1; @@ -6458,7 +6542,7 @@ __dtrace_probe(dtrace_id_t id, uint64_t arg0, uint64_t arg1, if (pred != NULL) { dtrace_difo_t *dp = pred->dtp_difo; - int rval; + uint64_t rval; rval = dtrace_dif_emulate(dp, &mstate, vstate, state); @@ -7179,11 +7263,13 @@ dtrace_cred2priv(cred_t *cr, uint32_t *privp, uid_t *uidp, zoneid_t *zoneidp) if (cr == NULL || PRIV_POLICY_ONLY(cr, PRIV_ALL, B_FALSE)) { if (dtrace_is_restricted() && !dtrace_are_restrictions_relaxed()) { - priv = DTRACE_PRIV_USER | DTRACE_PRIV_PROC; + priv = DTRACE_PRIV_USER | DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER; } else { priv = DTRACE_PRIV_ALL; } + *uidp = 0; + *zoneidp = 0; } else { *uidp = crgetuid(cr); *zoneidp = crgetzoneid(cr); @@ -7471,7 +7557,7 @@ dtrace_match(const dtrace_probekey_t *pkp, uint32_t priv, uid_t uid, int len, rc, best = INT_MAX, nmatched = 0; dtrace_id_t i; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); /* * If the probe ID is specified in the key, just lookup by ID and @@ -7730,8 +7816,8 @@ dtrace_register(const char *name, const dtrace_pattr_t *pap, uint32_t priv, *idp = (dtrace_provider_id_t)provider; if (pops == &dtrace_provider_ops) { - lck_mtx_assert(&dtrace_provider_lock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_provider_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(dtrace_anon.dta_enabling == NULL); /* @@ -7801,8 +7887,8 @@ dtrace_unregister(dtrace_provider_id_t id) */ ASSERT(old == dtrace_provider); ASSERT(dtrace_devi != NULL); - lck_mtx_assert(&dtrace_provider_lock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_provider_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); self = 1; if (dtrace_provider->dtpv_next != NULL) { @@ -8035,7 +8121,7 @@ dtrace_probe_create(dtrace_provider_id_t prov, const char *mod, dtrace_id_t id; if (provider == dtrace_provider) { - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); } else { lck_mtx_lock(&dtrace_lock); } @@ -8108,7 +8194,7 @@ dtrace_probe_create(dtrace_provider_id_t prov, const char *mod, static dtrace_probe_t * dtrace_probe_lookup_id(dtrace_id_t id) { - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); if (id == 0 || id > (dtrace_id_t)dtrace_nprobes) return (NULL); @@ -8215,7 +8301,7 @@ dtrace_probe_provide(dtrace_probedesc_t *desc, dtrace_provider_t *prv) struct modctl *ctl; int all = 0; - lck_mtx_assert(&dtrace_provider_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_provider_lock, LCK_MTX_ASSERT_OWNED); if (prv == NULL) { all = 1; @@ -8295,7 +8381,7 @@ dtrace_probe_enable(const dtrace_probedesc_t *desc, dtrace_enabling_t *enab, dtr uid_t uid; zoneid_t zoneid; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); dtrace_ecb_create_cache = NULL; @@ -8451,7 +8537,7 @@ dtrace_helper_provide(dof_helper_t *dhp, proc_t *p) dof_hdr_t *dof = (dof_hdr_t *)daddr; uint32_t i; - lck_mtx_assert(&dtrace_meta_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_meta_lock, LCK_MTX_ASSERT_OWNED); for (i = 0; i < dof->dofh_secnum; i++) { dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr + @@ -8499,7 +8585,7 @@ dtrace_helper_provider_remove(dof_helper_t *dhp, proc_t *p) dof_hdr_t *dof = (dof_hdr_t *)daddr; uint32_t i; - lck_mtx_assert(&dtrace_meta_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_meta_lock, LCK_MTX_ASSERT_OWNED); for (i = 0; i < dof->dofh_secnum; i++) { dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr + @@ -9318,7 +9404,7 @@ dtrace_difo_hold(dtrace_difo_t *dp) { uint_t i; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); dp->dtdo_refcnt++; ASSERT(dp->dtdo_refcnt != 0); @@ -9502,7 +9588,7 @@ dtrace_difo_init(dtrace_difo_t *dp, dtrace_vstate_t *vstate) int oldsvars, osz, nsz, otlocals, ntlocals; uint_t i, id; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(dp->dtdo_buf != NULL && dp->dtdo_len != 0); for (i = 0; i < dp->dtdo_varlen; i++) { @@ -9724,7 +9810,7 @@ dtrace_difo_release(dtrace_difo_t *dp, dtrace_vstate_t *vstate) { uint_t i; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(dp->dtdo_refcnt != 0); for (i = 0; i < dp->dtdo_varlen; i++) { @@ -9837,7 +9923,7 @@ dtrace_predicate_create(dtrace_difo_t *dp) { dtrace_predicate_t *pred; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(dp->dtdo_refcnt != 0); pred = kmem_zalloc(sizeof (dtrace_predicate_t), KM_SLEEP); @@ -9867,7 +9953,7 @@ dtrace_predicate_create(dtrace_difo_t *dp) static void dtrace_predicate_hold(dtrace_predicate_t *pred) { - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(pred->dtp_difo != NULL && pred->dtp_difo->dtdo_refcnt != 0); ASSERT(pred->dtp_refcnt > 0); @@ -9880,7 +9966,7 @@ dtrace_predicate_release(dtrace_predicate_t *pred, dtrace_vstate_t *vstate) dtrace_difo_t *dp = pred->dtp_difo; #pragma unused(dp) /* __APPLE__ */ - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(dp != NULL && dp->dtdo_refcnt != 0); ASSERT(pred->dtp_refcnt > 0); @@ -9955,7 +10041,7 @@ dtrace_ecb_add(dtrace_state_t *state, dtrace_probe_t *probe) dtrace_ecb_t *ecb; dtrace_epid_t epid; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ecb = kmem_zalloc(sizeof (dtrace_ecb_t), KM_SLEEP); ecb->dte_predicate = NULL; @@ -10021,8 +10107,8 @@ dtrace_ecb_enable(dtrace_ecb_t *ecb) { dtrace_probe_t *probe = ecb->dte_probe; - lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&cpu_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(ecb->dte_next == NULL); if (probe == NULL) { @@ -10367,7 +10453,7 @@ dtrace_ecb_action_add(dtrace_ecb_t *ecb, dtrace_actdesc_t *desc) dtrace_optval_t nframes=0, strsize; uint64_t arg = desc->dtad_arg; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(ecb->dte_action == NULL || ecb->dte_action->dta_refcnt == 1); if (DTRACEACT_ISAGG(desc->dtad_kind)) { @@ -10646,7 +10732,7 @@ dtrace_ecb_disable(dtrace_ecb_t *ecb) dtrace_ecb_t *pecb, *prev = NULL; dtrace_probe_t *probe = ecb->dte_probe; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); if (probe == NULL) { /* @@ -10725,7 +10811,7 @@ dtrace_ecb_destroy(dtrace_ecb_t *ecb) dtrace_predicate_t *pred; dtrace_epid_t epid = ecb->dte_epid; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(ecb->dte_next == NULL); ASSERT(ecb->dte_probe == NULL || ecb->dte_probe->dtpr_ecb != ecb); @@ -10750,7 +10836,7 @@ dtrace_ecb_create(dtrace_state_t *state, dtrace_probe_t *probe, dtrace_provider_t *prov; dtrace_ecbdesc_t *desc = enab->dten_current; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(state != NULL); ecb = dtrace_ecb_add(state, probe); @@ -10862,7 +10948,7 @@ dtrace_epid2ecb(dtrace_state_t *state, dtrace_epid_t id) dtrace_ecb_t *ecb; #pragma unused(ecb) /* __APPLE__ */ - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); if (id == 0 || id > (dtrace_epid_t)state->dts_necbs) return (NULL); @@ -10879,7 +10965,7 @@ dtrace_aggid2agg(dtrace_state_t *state, dtrace_aggid_t id) dtrace_aggregation_t *agg; #pragma unused(agg) /* __APPLE__ */ - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); if (id == 0 || id > (dtrace_aggid_t)state->dts_naggregations) return (NULL); @@ -10982,8 +11068,8 @@ dtrace_buffer_alloc(dtrace_buffer_t *bufs, size_t limit, size_t size, int flags, dtrace_buffer_t *buf; size_t size_before_alloc = dtrace_buffer_memory_inuse; - lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&cpu_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); if (size > (size_t)dtrace_nonroot_maxsize && !PRIV_POLICY_CHOICE(CRED(), PRIV_ALL, B_FALSE)) @@ -11009,7 +11095,6 @@ dtrace_buffer_alloc(dtrace_buffer_t *bufs, size_t limit, size_t size, int flags, ASSERT(buf->dtb_xamot == NULL); - /* DTrace, please do not eat all the memory. */ if (dtrace_buffer_canalloc(size) == B_FALSE) goto err; @@ -11346,7 +11431,7 @@ static void dtrace_buffer_polish(dtrace_buffer_t *buf) { ASSERT(buf->dtb_flags & DTRACEBUF_RING); - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); if (!(buf->dtb_flags & DTRACEBUF_WRAPPED)) return; @@ -11523,7 +11608,7 @@ dtrace_enabling_destroy(dtrace_enabling_t *enab) dtrace_ecbdesc_t *ep; dtrace_vstate_t *vstate = enab->dten_vstate; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); for (i = 0; i < enab->dten_ndesc; i++) { dtrace_actdesc_t *act, *next; @@ -11583,7 +11668,7 @@ dtrace_enabling_retain(dtrace_enabling_t *enab) { dtrace_state_t *state; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(enab->dten_next == NULL && enab->dten_prev == NULL); ASSERT(enab->dten_vstate != NULL); @@ -11618,7 +11703,7 @@ dtrace_enabling_replicate(dtrace_state_t *state, dtrace_probedesc_t *match, dtrace_enabling_t *new, *enab; int found = 0, err = ENOENT; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(strlen(match->dtpd_provider) < DTRACE_PROVNAMELEN); ASSERT(strlen(match->dtpd_mod) < DTRACE_MODNAMELEN); ASSERT(strlen(match->dtpd_func) < DTRACE_FUNCNAMELEN); @@ -11685,7 +11770,7 @@ dtrace_enabling_retract(dtrace_state_t *state) { dtrace_enabling_t *enab, *next; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); /* * Iterate over all retained enablings, destroy the enablings retained @@ -11715,8 +11800,8 @@ dtrace_enabling_match(dtrace_enabling_t *enab, int *nmatched, dtrace_match_cond_ int i = 0; int total_matched = 0, matched = 0; - lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&cpu_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); for (i = 0; i < enab->dten_ndesc; i++) { dtrace_ecbdesc_t *ep = enab->dten_desc[i]; @@ -11867,8 +11952,8 @@ dtrace_enabling_provide(dtrace_provider_t *prv) dtrace_probedesc_t desc; dtrace_genid_t gen; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&dtrace_provider_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_provider_lock, LCK_MTX_ASSERT_OWNED); if (prv == NULL) { all = 1; @@ -11935,7 +12020,7 @@ dtrace_dof_create(dtrace_state_t *state) roundup(sizeof (dof_sec_t), sizeof (uint64_t)) + sizeof (dof_optdesc_t) * DTRACEOPT_MAX; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); dof = dt_kmem_zalloc_aligned(len, 8, KM_SLEEP); dof->dofh_ident[DOF_ID_MAG0] = DOF_MAG_MAG0; @@ -11988,7 +12073,7 @@ dtrace_dof_copyin(user_addr_t uarg, int *errp) { dof_hdr_t hdr, *dof; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_NOTOWNED); /* * First, we're going to copyin() the sizeof (dof_hdr_t). @@ -12032,7 +12117,7 @@ dtrace_dof_copyin_from_proc(proc_t* p, user_addr_t uarg, int *errp) { dof_hdr_t hdr, *dof; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_NOTOWNED); /* * First, we're going to copyin() the sizeof (dof_hdr_t). @@ -12638,7 +12723,7 @@ dtrace_dof_slurp(dof_hdr_t *dof, dtrace_vstate_t *vstate, cred_t *cr, dtrace_enabling_t *enab; uint_t i; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(dof->dofh_loadsz >= sizeof (dof_hdr_t)); /* @@ -12880,7 +12965,7 @@ dtrace_dstate_init(dtrace_dstate_t *dstate, size_t size) dtrace_dynvar_t *dvar, *next, *start; size_t i; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(dstate->dtds_base == NULL && dstate->dtds_percpu == NULL); bzero(dstate, sizeof (dtrace_dstate_t)); @@ -12979,7 +13064,7 @@ dtrace_dstate_init(dtrace_dstate_t *dstate, size_t size) static void dtrace_dstate_fini(dtrace_dstate_t *dstate) { - lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&cpu_lock, LCK_MTX_ASSERT_OWNED); if (dstate->dtds_base == NULL) return; @@ -13061,8 +13146,8 @@ dtrace_state_create(dev_t *devp, cred_t *cr, dtrace_state_t **new_state) dtrace_optval_t *opt; int bufsize = (int)NCPU * sizeof (dtrace_buffer_t), i; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&cpu_lock, LCK_MTX_ASSERT_OWNED); /* Cause restart */ *new_state = NULL; @@ -13291,8 +13376,8 @@ dtrace_state_buffer(dtrace_state_t *state, dtrace_buffer_t *buf, int which) size_t limit = buf->dtb_size; int flags = 0, rval; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&cpu_lock, LCK_MTX_ASSERT_OWNED); ASSERT(which < DTRACEOPT_MAX); ASSERT(state->dts_activity == DTRACE_ACTIVITY_INACTIVE || (state == dtrace_anon.dta_state && @@ -13690,7 +13775,7 @@ dtrace_state_stop(dtrace_state_t *state, processorid_t *cpu) { dtrace_icookie_t cookie; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE && state->dts_activity != DTRACE_ACTIVITY_DRAINING) @@ -13741,7 +13826,7 @@ static int dtrace_state_option(dtrace_state_t *state, dtrace_optid_t option, dtrace_optval_t val) { - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) return (EBUSY); @@ -13808,8 +13893,8 @@ dtrace_state_destroy(dtrace_state_t *state) int nspec = state->dts_nspeculations; uint32_t match; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&cpu_lock, LCK_MTX_ASSERT_OWNED); /* * First, retract any retained enablings for this state. @@ -13920,7 +14005,7 @@ dtrace_anon_grab(void) { dtrace_state_t *state; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); if ((state = dtrace_anon.dta_state) == NULL) { ASSERT(dtrace_anon.dta_enabling == NULL); @@ -13945,8 +14030,8 @@ dtrace_anon_property(void) dof_hdr_t *dof; char c[32]; /* enough for "dof-data-" + digits */ - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&cpu_lock, LCK_MTX_ASSERT_OWNED); for (i = 0; ; i++) { (void) snprintf(c, sizeof (c), "dof-data-%d", i); @@ -14217,7 +14302,7 @@ dtrace_helper_destroygen(proc_t* p, int gen) dtrace_vstate_t *vstate; uint_t i; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); if (help == NULL || gen > help->dthps_generation) return (EINVAL); @@ -14394,7 +14479,7 @@ static void dtrace_helper_provider_register(proc_t *p, dtrace_helpers_t *help, dof_helper_t *dofhp) { - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(&dtrace_meta_lock); lck_mtx_lock(&dtrace_lock); @@ -14456,7 +14541,7 @@ dtrace_helper_provider_add(proc_t* p, dof_helper_t *dofhp, int gen) dtrace_helper_provider_t *hprov, **tmp_provs; uint_t tmp_maxprovs, i; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); help = p->p_dtrace_helpers; ASSERT(help != NULL); @@ -14752,7 +14837,7 @@ dtrace_helper_slurp(proc_t* p, dof_hdr_t *dof, dof_helper_t *dhp) int i, gen, rv, nhelpers = 0, nprovs = 0, destroy = 1; uintptr_t daddr = (uintptr_t)dof; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); if ((help = p->p_dtrace_helpers) == NULL) help = dtrace_helpers_create(p); @@ -15199,9 +15284,9 @@ dtrace_lazy_dofs_proc_iterate_doit(proc_t *p, void* ignored) static int dtrace_lazy_dofs_duplicate(proc_t *parent, proc_t *child) { - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_NOTOWNED); - lck_mtx_assert(&parent->p_dtrace_sprlock, LCK_MTX_ASSERT_NOTOWNED); - lck_mtx_assert(&child->p_dtrace_sprlock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&parent->p_dtrace_sprlock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&child->p_dtrace_sprlock, LCK_MTX_ASSERT_NOTOWNED); lck_rw_lock_shared(&dtrace_dof_mode_lock); lck_mtx_lock(&parent->p_dtrace_sprlock); @@ -15254,7 +15339,7 @@ dtrace_helpers_create(proc_t *p) { dtrace_helpers_t *help; - lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dtrace_lock, LCK_MTX_ASSERT_OWNED); ASSERT(p->p_dtrace_helpers == NULL); help = kmem_zalloc(sizeof (dtrace_helpers_t), KM_SLEEP); @@ -15600,7 +15685,7 @@ dtrace_modctl_add(struct modctl * newctl) struct modctl *nextp, *prevp; ASSERT(newctl != NULL); - lck_mtx_assert(&mod_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&mod_lock, LCK_MTX_ASSERT_OWNED); // Insert new module at the front of the list, @@ -15646,7 +15731,7 @@ dtrace_modctl_add(struct modctl * newctl) static modctl_t * dtrace_modctl_lookup(struct kmod_info * kmod) { - lck_mtx_assert(&mod_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&mod_lock, LCK_MTX_ASSERT_OWNED); struct modctl * ctl; @@ -15666,7 +15751,7 @@ static void dtrace_modctl_remove(struct modctl * ctl) { ASSERT(ctl != NULL); - lck_mtx_assert(&mod_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&mod_lock, LCK_MTX_ASSERT_OWNED); modctl_t *prevp, *nextp, *curp; // Remove stale chain first @@ -16047,7 +16132,7 @@ dtrace_resume(void) static int dtrace_cpu_setup(cpu_setup_t what, processorid_t cpu) { - lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&cpu_lock, LCK_MTX_ASSERT_OWNED); lck_mtx_lock(&dtrace_lock); switch (what) { @@ -16179,7 +16264,7 @@ dtrace_attach(dev_info_t *devi, ddi_attach_cmd_t cmd) register_cpu_setup_func((cpu_setup_func_t *)dtrace_cpu_setup, NULL); - lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&cpu_lock, LCK_MTX_ASSERT_OWNED); dtrace_arena = vmem_create("dtrace", (void *)1, UINT32_MAX, 1, NULL, NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER); @@ -16190,7 +16275,7 @@ dtrace_attach(dev_info_t *devi, ddi_attach_cmd_t cmd) sizeof (dtrace_dstate_percpu_t) * (int)NCPU, DTRACE_STATE_ALIGN, NULL, NULL, NULL, NULL, NULL, 0); - lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&cpu_lock, LCK_MTX_ASSERT_OWNED); dtrace_bymod = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_mod), offsetof(dtrace_probe_t, dtpr_nextmod), @@ -16235,6 +16320,13 @@ dtrace_attach(dev_info_t *devi, ddi_attach_cmd_t cmd) dtrace_provider, NULL, NULL, "END", 0, NULL); dtrace_probeid_error = dtrace_probe_create((dtrace_provider_id_t) dtrace_provider, NULL, NULL, "ERROR", 3, NULL); +#elif (defined(__arm__) || defined(__arm64__)) + dtrace_probeid_begin = dtrace_probe_create((dtrace_provider_id_t) + dtrace_provider, NULL, NULL, "BEGIN", 2, NULL); + dtrace_probeid_end = dtrace_probe_create((dtrace_provider_id_t) + dtrace_provider, NULL, NULL, "END", 1, NULL); + dtrace_probeid_error = dtrace_probe_create((dtrace_provider_id_t) + dtrace_provider, NULL, NULL, "ERROR", 4, NULL); #else #error Unknown Architecture #endif @@ -16515,7 +16607,12 @@ dtrace_ioctl_helper(u_long cmd, caddr_t arg, int *rv) return KERN_SUCCESS; switch (cmd) { +#if defined (__arm64__) + case DTRACEHIOC_ADDDOF_U32: + case DTRACEHIOC_ADDDOF_U64: +#else case DTRACEHIOC_ADDDOF: +#endif /* __arm64__*/ { dof_helper_t *dhp = NULL; size_t dof_ioctl_data_size; @@ -16527,6 +16624,16 @@ dtrace_ioctl_helper(u_long cmd, caddr_t arg, int *rv) int multi_dof_claimed = 0; proc_t* p = current_proc(); + /* + * If this is a restricted process and dtrace is restricted, + * do not allow DOFs to be registered + */ + if (dtrace_is_restricted() && + !dtrace_are_restrictions_relaxed() && + !dtrace_can_attach_to_proc(current_proc())) { + return (EACCES); + } + /* * Read the number of DOF sections being passed in. */ @@ -16536,7 +16643,7 @@ dtrace_ioctl_helper(u_long cmd, caddr_t arg, int *rv) dtrace_dof_error(NULL, "failed to copyin dofiod_count"); return (EFAULT); } - + /* * Range check the count. */ @@ -17309,7 +17416,8 @@ dtrace_ioctl(dev_t dev, u_long cmd, user_addr_t arg, int md, cred_t *cr, int *rv /* * We have our snapshot; now copy it out. */ - if (copyout(buf->dtb_xamot, (user_addr_t)desc.dtbd_data, + if (dtrace_buffer_copyout(buf->dtb_xamot, + (user_addr_t)desc.dtbd_data, buf->dtb_xamot_offset) != 0) { lck_mtx_unlock(&dtrace_lock); return (EFAULT); @@ -18242,7 +18350,12 @@ dtrace_init( void ) * makes no sense... */ if (!PE_parse_boot_argn("dtrace_dof_mode", &dtrace_dof_mode, sizeof (dtrace_dof_mode))) { +#if CONFIG_EMBEDDED + /* Disable DOF mode by default for performance reasons */ + dtrace_dof_mode = DTRACE_DOF_MODE_NEVER; +#else dtrace_dof_mode = DTRACE_DOF_MODE_LAZY_ON; +#endif } /* @@ -18307,6 +18420,10 @@ dtrace_postinit(void) if (dtrace_module_loaded(&fake_kernel_kmod, 0) != 0) { printf("dtrace_postinit: Could not register mach_kernel modctl\n"); } + + if (!PE_parse_boot_argn("dtrace_provide_private_probes", &dtrace_provide_private_probes, sizeof (dtrace_provide_private_probes))) { + dtrace_provide_private_probes = 0; + } (void)OSKextRegisterKextsWithDTrace(); } diff --git a/bsd/dev/dtrace/dtrace_glue.c b/bsd/dev/dtrace/dtrace_glue.c index 7bd5500b0..57f0f3207 100644 --- a/bsd/dev/dtrace/dtrace_glue.c +++ b/bsd/dev/dtrace/dtrace_glue.c @@ -301,7 +301,11 @@ typedef struct wrap_timer_call { typedef struct cyc_list { cyc_omni_handler_t cyl_omni; wrap_timer_call_t cyl_wrap_by_cpus[]; +#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4) +} __attribute__ ((aligned (8))) cyc_list_t; +#else } cyc_list_t; +#endif /* CPU going online/offline notifications */ void (*dtrace_cpu_state_changed_hook)(int, boolean_t) = NULL; @@ -573,29 +577,6 @@ cyclic_remove(cyclic_id_t cyclic) } } -/* - * timeout / untimeout (converted to dtrace_timeout / dtrace_untimeout due to name collision) - */ - -thread_call_t -dtrace_timeout(void (*func)(void *, void *), void* arg, uint64_t nanos) -{ -#pragma unused(arg) - thread_call_t call = thread_call_allocate(func, NULL); - - nanoseconds_to_absolutetime(nanos, &nanos); - - /* - * This method does not use clock_deadline_for_periodic_event() because it is a one-shot, - * and clock drift on later invocations is not a worry. - */ - uint64_t deadline = mach_absolute_time() + nanos; - /* DRK: consider using a lower priority callout here */ - thread_call_enter_delayed(call, deadline); - - return call; -} - /* * ddi */ @@ -1249,6 +1230,30 @@ dtrace_copyoutstr(uintptr_t src, user_addr_t dst, size_t len, volatile uint16_t } } +extern const int copysize_limit_panic; + +int +dtrace_buffer_copyout(const void *kaddr, user_addr_t uaddr, vm_size_t nbytes) +{ + /* + * Partition the copyout in copysize_limit_panic-sized chunks + */ + while (nbytes >= (vm_size_t)copysize_limit_panic) { + if (copyout(kaddr, uaddr, copysize_limit_panic) != 0) + return (EFAULT); + + nbytes -= copysize_limit_panic; + uaddr += copysize_limit_panic; + kaddr += copysize_limit_panic; + } + if (nbytes > 0) { + if (copyout(kaddr, uaddr, nbytes) != 0) + return (EFAULT); + } + + return (0); +} + uint8_t dtrace_fuword8(user_addr_t uaddr) { @@ -1483,6 +1488,8 @@ strstr(const char *in, const char *str) { char c; size_t len; + if (!in || !str) + return in; c = *str++; if (!c) @@ -1502,6 +1509,26 @@ strstr(const char *in, const char *str) return (const char *) (in - 1); } +const void* +bsearch(const void *key, const void *base0, size_t nmemb, size_t size, int (*compar)(const void *, const void *)) +{ + const char *base = base0; + size_t lim; + int cmp; + const void *p; + for (lim = nmemb; lim != 0; lim >>= 1) { + p = base + (lim >> 1) * size; + cmp = (*compar)(key, p); + if (cmp == 0) + return p; + if (cmp > 0) { /* key > p: move right */ + base = (const char *)p + size; + lim--; + } /* else move left */ + } + return (NULL); +} + /* * Runtime and ABI */ diff --git a/bsd/dev/dtrace/dtrace_ptss.c b/bsd/dev/dtrace/dtrace_ptss.c index c09b8f32e..1ce9c28d8 100644 --- a/bsd/dev/dtrace/dtrace_ptss.c +++ b/bsd/dev/dtrace/dtrace_ptss.c @@ -49,7 +49,7 @@ */ struct dtrace_ptss_page_entry* dtrace_ptss_claim_entry_locked(struct proc* p) { - lck_mtx_assert(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED); struct dtrace_ptss_page_entry* entry = NULL; @@ -98,8 +98,8 @@ dtrace_ptss_claim_entry_locked(struct proc* p) { struct dtrace_ptss_page_entry* dtrace_ptss_claim_entry(struct proc* p) { // Verify no locks held on entry - lck_mtx_assert(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_NOTOWNED); - lck_mtx_assert(&p->p_mlock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&p->p_mlock, LCK_MTX_ASSERT_NOTOWNED); struct dtrace_ptss_page_entry* entry = NULL; @@ -167,17 +167,43 @@ dtrace_ptss_allocate_page(struct proc* p) mach_vm_size_t size = PAGE_MAX_SIZE; mach_vm_offset_t addr = 0; +#if CONFIG_EMBEDDED + mach_vm_offset_t write_addr = 0; + /* + * The embedded OS has extra permissions for writable and executable pages. + * To ensure correct permissions, we must set the page protections separately. + */ + vm_prot_t cur_protection = VM_PROT_READ|VM_PROT_EXECUTE; + vm_prot_t max_protection = VM_PROT_READ|VM_PROT_EXECUTE|VM_PROT_WRITE; +#else vm_prot_t cur_protection = VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE; vm_prot_t max_protection = VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE; +#endif /* CONFIG_EMBEDDED */ - kern_return_t kr = mach_vm_map(map, &addr, size, 0, VM_FLAGS_ANYWHERE, IPC_PORT_NULL, 0, FALSE, cur_protection, max_protection, VM_INHERIT_DEFAULT); + kern_return_t kr = mach_vm_map_kernel(map, &addr, size, 0, VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_NONE, IPC_PORT_NULL, 0, FALSE, cur_protection, max_protection, VM_INHERIT_DEFAULT); if (kr != KERN_SUCCESS) { goto err; } +#if CONFIG_EMBEDDED + /* + * If on embedded, remap the scratch space as writable at another + * virtual address + */ + kr = mach_vm_remap_kernel(map, &write_addr, size, 0, VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_NONE, map, addr, FALSE, &cur_protection, &max_protection, VM_INHERIT_DEFAULT); + if (kr != KERN_SUCCESS || !(max_protection & VM_PROT_WRITE)) + goto err; + + kr = mach_vm_protect (map, (mach_vm_offset_t)write_addr, (mach_vm_size_t)size, 0, VM_PROT_READ | VM_PROT_WRITE); + if (kr != KERN_SUCCESS) + goto err; +#endif // Chain the page entries. int i; for (i=0; i<DTRACE_PTSS_ENTRIES_PER_PAGE; i++) { ptss_page->entries[i].addr = addr + (i * DTRACE_PTSS_SCRATCH_SPACE_PER_THREAD); +#if CONFIG_EMBEDDED + ptss_page->entries[i].write_addr = write_addr + (i * DTRACE_PTSS_SCRATCH_SPACE_PER_THREAD); +#endif ptss_page->entries[i].next = &ptss_page->entries[i+1]; } @@ -217,6 +243,10 @@ dtrace_ptss_free_page(struct proc* p, struct dtrace_ptss_page* ptss_page) // Silent failures, no point in checking return code. mach_vm_deallocate(map, addr, size); +#ifdef CONFIG_EMBEDDED + mach_vm_address_t write_addr = ptss_page->entries[0].write_addr; + mach_vm_deallocate(map, write_addr, size); +#endif vm_map_deallocate(map); } @@ -227,8 +257,8 @@ dtrace_ptss_free_page(struct proc* p, struct dtrace_ptss_page* ptss_page) */ void dtrace_ptss_enable(struct proc* p) { - lck_mtx_assert(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&p->p_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&p->p_mlock, LCK_MTX_ASSERT_OWNED); struct uthread* uth; /* @@ -252,8 +282,8 @@ dtrace_ptss_exec_exit(struct proc* p) { * Should hold sprlock to touch the pages list. Must not * hold the proc lock to avoid deadlock. */ - lck_mtx_assert(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&p->p_mlock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&p->p_mlock, LCK_MTX_ASSERT_NOTOWNED); p->p_dtrace_ptss_free_list = NULL; @@ -295,10 +325,10 @@ dtrace_ptss_fork(struct proc* parent, struct proc* child) { * Finally, to prevent a deadlock with the fasttrap cleanup code, * neither the parent or child proc_lock should be held. */ - lck_mtx_assert(&parent->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&parent->p_mlock, LCK_MTX_ASSERT_NOTOWNED); - lck_mtx_assert(&child->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&child->p_mlock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&parent->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&parent->p_mlock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&child->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&child->p_mlock, LCK_MTX_ASSERT_NOTOWNED); // Get page list from *PARENT* struct dtrace_ptss_page* temp = parent->p_dtrace_ptss_pages; diff --git a/bsd/dev/dtrace/fasttrap.c b/bsd/dev/dtrace/fasttrap.c index d25f82dba..8425b98af 100644 --- a/bsd/dev/dtrace/fasttrap.c +++ b/bsd/dev/dtrace/fasttrap.c @@ -31,6 +31,7 @@ #include <sys/types.h> #include <sys/time.h> +#include <sys/codesign.h> #include <sys/errno.h> #include <sys/stat.h> #include <sys/conf.h> @@ -50,8 +51,14 @@ #include <sys/dtrace_glue.h> #include <sys/dtrace_ptss.h> +#include <kern/cs_blobs.h> +#include <kern/thread.h> #include <kern/zalloc.h> +#include <mach/thread_act.h> + +extern kern_return_t kernel_thread_start_priority(thread_continue_t continuation, void *parameter, integer_t priority, thread_t *new_thread); + /* Solaris proc_t is the struct. Darwin's proc_t is a pointer to it. */ #define proc_t struct proc /* Steer clear of the Darwin typedef for proc_t */ @@ -139,9 +146,15 @@ qsort(void *a, size_t n, size_t es, int (*cmp)(const void *, const void *)); static dev_info_t *fasttrap_devi; static dtrace_meta_provider_id_t fasttrap_meta_id; -static thread_call_t fasttrap_timeout; +static thread_t fasttrap_cleanup_thread; + static lck_mtx_t fasttrap_cleanup_mtx; -static uint_t fasttrap_cleanup_work; + + +#define FASTTRAP_CLEANUP_PROVIDER 0x1 +#define FASTTRAP_CLEANUP_TRACEPOINT 0x2 + +static uint32_t fasttrap_cleanup_work = 0; /* * Generation count on modifications to the global tracepoint lookup table. @@ -156,6 +169,7 @@ static volatile uint64_t fasttrap_mod_gen; */ static uint32_t fasttrap_max; +static uint32_t fasttrap_retired; static uint32_t fasttrap_total; @@ -312,139 +326,257 @@ fasttrap_mod_barrier(uint64_t gen) } } -/* - * This is the timeout's callback for cleaning up the providers and their - * probes. - */ -/*ARGSUSED*/ -static void -fasttrap_pid_cleanup_cb(void *ignored, void* ignored2) +static void fasttrap_pid_cleanup(uint32_t); + +static unsigned int +fasttrap_pid_cleanup_providers(void) { -#pragma unused(ignored, ignored2) fasttrap_provider_t **fpp, *fp; fasttrap_bucket_t *bucket; dtrace_provider_id_t provid; - unsigned int i, later = 0; + unsigned int later = 0, i; - static volatile int in = 0; - ASSERT(in == 0); - in = 1; + /* + * Iterate over all the providers trying to remove the marked + * ones. If a provider is marked but not retired, we just + * have to take a crack at removing it -- it's no big deal if + * we can't. + */ + for (i = 0; i < fasttrap_provs.fth_nent; i++) { + bucket = &fasttrap_provs.fth_table[i]; + lck_mtx_lock(&bucket->ftb_mtx); + fpp = (fasttrap_provider_t **)&bucket->ftb_data; - lck_mtx_lock(&fasttrap_cleanup_mtx); - while (fasttrap_cleanup_work) { - fasttrap_cleanup_work = 0; - lck_mtx_unlock(&fasttrap_cleanup_mtx); + while ((fp = *fpp) != NULL) { + if (!fp->ftp_marked) { + fpp = &fp->ftp_next; + continue; + } - later = 0; + lck_mtx_lock(&fp->ftp_mtx); - /* - * Iterate over all the providers trying to remove the marked - * ones. If a provider is marked but not retired, we just - * have to take a crack at removing it -- it's no big deal if - * we can't. - */ - for (i = 0; i < fasttrap_provs.fth_nent; i++) { - bucket = &fasttrap_provs.fth_table[i]; - lck_mtx_lock(&bucket->ftb_mtx); - fpp = (fasttrap_provider_t **)&bucket->ftb_data; + /* + * If this provider has consumers actively + * creating probes (ftp_ccount) or is a USDT + * provider (ftp_mcount), we can't unregister + * or even condense. + */ + if (fp->ftp_ccount != 0 || + fp->ftp_mcount != 0) { + fp->ftp_marked = 0; + lck_mtx_unlock(&fp->ftp_mtx); + continue; + } - while ((fp = *fpp) != NULL) { - if (!fp->ftp_marked) { - fpp = &fp->ftp_next; - continue; - } + if (!fp->ftp_retired || fp->ftp_rcount != 0) + fp->ftp_marked = 0; - lck_mtx_lock(&fp->ftp_mtx); + lck_mtx_unlock(&fp->ftp_mtx); - /* - * If this provider has consumers actively - * creating probes (ftp_ccount) or is a USDT - * provider (ftp_mcount), we can't unregister - * or even condense. - */ - if (fp->ftp_ccount != 0 || - fp->ftp_mcount != 0) { - fp->ftp_marked = 0; - lck_mtx_unlock(&fp->ftp_mtx); - continue; - } + /* + * If we successfully unregister this + * provider we can remove it from the hash + * chain and free the memory. If our attempt + * to unregister fails and this is a retired + * provider, increment our flag to try again + * pretty soon. If we've consumed more than + * half of our total permitted number of + * probes call dtrace_condense() to try to + * clean out the unenabled probes. + */ + provid = fp->ftp_provid; + if (dtrace_unregister(provid) != 0) { + if (fasttrap_total > fasttrap_max / 2) + (void) dtrace_condense(provid); + later += fp->ftp_marked; + fpp = &fp->ftp_next; + } else { + *fpp = fp->ftp_next; + fasttrap_provider_free(fp); + } + } + lck_mtx_unlock(&bucket->ftb_mtx); + } - if (!fp->ftp_retired || fp->ftp_rcount != 0) - fp->ftp_marked = 0; + return later; +} - lck_mtx_unlock(&fp->ftp_mtx); +#ifdef FASTTRAP_ASYNC_REMOVE +typedef struct fasttrap_tracepoint_spec { + pid_t fttps_pid; + user_addr_t fttps_pc; +} fasttrap_tracepoint_spec_t; - /* - * If we successfully unregister this - * provider we can remove it from the hash - * chain and free the memory. If our attempt - * to unregister fails and this is a retired - * provider, increment our flag to try again - * pretty soon. If we've consumed more than - * half of our total permitted number of - * probes call dtrace_condense() to try to - * clean out the unenabled probes. - */ - provid = fp->ftp_provid; - if (dtrace_unregister(provid) != 0) { - if (fasttrap_total > fasttrap_max / 2) - (void) dtrace_condense(provid); - later += fp->ftp_marked; - fpp = &fp->ftp_next; - } else { - *fpp = fp->ftp_next; - fasttrap_provider_free(fp); - } +static fasttrap_tracepoint_spec_t *fasttrap_retired_spec; +static size_t fasttrap_cur_retired = 0, fasttrap_retired_size; +static lck_mtx_t fasttrap_retired_mtx; + +#define DEFAULT_RETIRED_SIZE 256 + +static void +fasttrap_tracepoint_cleanup(void) +{ + size_t i; + pid_t pid = 0; + user_addr_t pc; + proc_t *p = PROC_NULL; + fasttrap_tracepoint_t *tp = NULL; + lck_mtx_lock(&fasttrap_retired_mtx); + fasttrap_bucket_t *bucket; + for (i = 0; i < fasttrap_cur_retired; i++) { + pc = fasttrap_retired_spec[i].fttps_pc; + if (fasttrap_retired_spec[i].fttps_pid != pid) { + pid = fasttrap_retired_spec[i].fttps_pid; + if (p != PROC_NULL) { + sprunlock(p); + } + if ((p = sprlock(pid)) == PROC_NULL) { + pid = 0; + continue; } + } + bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; + lck_mtx_lock(&bucket->ftb_mtx); + for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { + if (pid == tp->ftt_pid && pc == tp->ftt_pc && + tp->ftt_proc->ftpc_acount != 0) + break; + } + /* + * Check that the tracepoint is not gone or has not been + * re-activated for another probe + */ + if (tp == NULL || tp->ftt_retired == 0) { lck_mtx_unlock(&bucket->ftb_mtx); + continue; } + fasttrap_tracepoint_remove(p, tp); + lck_mtx_unlock(&bucket->ftb_mtx); + } + if (p != PROC_NULL) { + sprunlock(p); + } - lck_mtx_lock(&fasttrap_cleanup_mtx); + fasttrap_cur_retired = 0; + + lck_mtx_unlock(&fasttrap_retired_mtx); +} + +void +fasttrap_tracepoint_retire(proc_t *p, fasttrap_tracepoint_t *tp) +{ + if (tp->ftt_retired) + return; + lck_mtx_lock(&fasttrap_retired_mtx); + fasttrap_tracepoint_spec_t *s = &fasttrap_retired_spec[fasttrap_cur_retired++]; + s->fttps_pid = p->p_pid; + s->fttps_pc = tp->ftt_pc; + + if (fasttrap_cur_retired == fasttrap_retired_size) { + fasttrap_retired_size *= 2; + fasttrap_tracepoint_spec_t *new_retired = kmem_zalloc( + fasttrap_retired_size * + sizeof(fasttrap_tracepoint_t*), + KM_SLEEP); + memcpy(new_retired, fasttrap_retired_spec, sizeof(fasttrap_tracepoint_t*) * fasttrap_retired_size); + kmem_free(fasttrap_retired_spec, sizeof(fasttrap_tracepoint_t*) * (fasttrap_retired_size / 2)); + fasttrap_retired_spec = new_retired; } - ASSERT(fasttrap_timeout != 0); + lck_mtx_unlock(&fasttrap_retired_mtx); - /* - * APPLE NOTE: You must hold the fasttrap_cleanup_mtx to do this! - */ - if (fasttrap_timeout != (thread_call_t)1) - thread_call_free(fasttrap_timeout); + tp->ftt_retired = 1; - /* - * If we were unable to remove a retired provider, try again after - * a second. This situation can occur in certain circumstances where - * providers cannot be unregistered even though they have no probes - * enabled because of an execution of dtrace -l or something similar. - * If the timeout has been disabled (set to 1 because we're trying - * to detach), we set fasttrap_cleanup_work to ensure that we'll - * get a chance to do that work if and when the timeout is reenabled - * (if detach fails). - */ - if (later > 0 && fasttrap_timeout != (thread_call_t)1) - /* The time value passed to dtrace_timeout is in nanos */ - fasttrap_timeout = dtrace_timeout(&fasttrap_pid_cleanup_cb, NULL, NANOSEC / SEC); - else if (later > 0) - fasttrap_cleanup_work = 1; - else - fasttrap_timeout = 0; + fasttrap_pid_cleanup(FASTTRAP_CLEANUP_TRACEPOINT); +} +#else +void fasttrap_tracepoint_retire(proc_t *p, fasttrap_tracepoint_t *tp) +{ + if (tp->ftt_retired) + return; + + fasttrap_tracepoint_remove(p, tp); +} +#endif + +static void +fasttrap_pid_cleanup_compute_priority(void) +{ + if (fasttrap_total > (fasttrap_max / 100 * 90) || fasttrap_retired > fasttrap_max / 2) { + thread_precedence_policy_data_t precedence = {12 /* BASEPRI_PREEMPT_HIGH */}; + thread_policy_set(fasttrap_cleanup_thread, THREAD_PRECEDENCE_POLICY, (thread_policy_t) &precedence, THREAD_PRECEDENCE_POLICY_COUNT); + } + else { + thread_precedence_policy_data_t precedence = {-39 /* BASEPRI_USER_INITIATED */}; + thread_policy_set(fasttrap_cleanup_thread, THREAD_PRECEDENCE_POLICY, (thread_policy_t) &precedence, THREAD_PRECEDENCE_POLICY_COUNT); + + } +} + +/* + * This is the timeout's callback for cleaning up the providers and their + * probes. + */ +/*ARGSUSED*/ +__attribute__((noreturn)) +static void +fasttrap_pid_cleanup_cb(void) +{ + uint32_t work = 0; + lck_mtx_lock(&fasttrap_cleanup_mtx); + msleep(&fasttrap_pid_cleanup_cb, &fasttrap_cleanup_mtx, PRIBIO, "fasttrap_pid_cleanup_cb", NULL); + while (1) { + unsigned int later = 0; + + work = atomic_and_32(&fasttrap_cleanup_work, 0); + lck_mtx_unlock(&fasttrap_cleanup_mtx); + if (work & FASTTRAP_CLEANUP_PROVIDER) { + later = fasttrap_pid_cleanup_providers(); + } +#ifdef FASTTRAP_ASYNC_REMOVE + if (work & FASTTRAP_CLEANUP_TRACEPOINT) { + fasttrap_tracepoint_cleanup(); + } +#endif + lck_mtx_lock(&fasttrap_cleanup_mtx); + + fasttrap_pid_cleanup_compute_priority(); + if (!fasttrap_cleanup_work) { + /* + * If we were unable to remove a retired provider, try again after + * a second. This situation can occur in certain circumstances where + * providers cannot be unregistered even though they have no probes + * enabled because of an execution of dtrace -l or something similar. + * If the timeout has been disabled (set to 1 because we're trying + * to detach), we set fasttrap_cleanup_work to ensure that we'll + * get a chance to do that work if and when the timeout is reenabled + * (if detach fails). + */ + if (later > 0) { + struct timespec t = {1, 0}; + msleep(&fasttrap_pid_cleanup_cb, &fasttrap_cleanup_mtx, PRIBIO, "fasttrap_pid_cleanup_cb", &t); + } + else + msleep(&fasttrap_pid_cleanup_cb, &fasttrap_cleanup_mtx, PRIBIO, "fasttrap_pid_cleanup_cb", NULL); + } + } - lck_mtx_unlock(&fasttrap_cleanup_mtx); - in = 0; } /* * Activates the asynchronous cleanup mechanism. */ static void -fasttrap_pid_cleanup(void) +fasttrap_pid_cleanup(uint32_t work) { lck_mtx_lock(&fasttrap_cleanup_mtx); - fasttrap_cleanup_work = 1; - if (fasttrap_timeout == 0) - fasttrap_timeout = dtrace_timeout(&fasttrap_pid_cleanup_cb, NULL, NANOSEC / MILLISEC); + atomic_or_32(&fasttrap_cleanup_work, work); + fasttrap_pid_cleanup_compute_priority(); + wakeup(&fasttrap_pid_cleanup_cb); lck_mtx_unlock(&fasttrap_cleanup_mtx); } + /* * This is called from cfork() via dtrace_fasttrap_fork(). The child * process's address space is a (roughly) a copy of the parent process's so @@ -458,7 +590,7 @@ fasttrap_fork(proc_t *p, proc_t *cp) unsigned int i; ASSERT(current_proc() == p); - lck_mtx_assert(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED); ASSERT(p->p_dtrace_count > 0); ASSERT(cp->p_dtrace_count == 0); @@ -477,7 +609,7 @@ fasttrap_fork(proc_t *p, proc_t *cp) * because we're in fork(). */ if (cp != sprlock(cp->p_pid)) { - printf("fasttrap_fork: sprlock(%d) returned a differt proc\n", cp->p_pid); + printf("fasttrap_fork: sprlock(%d) returned a different proc\n", cp->p_pid); return; } proc_unlock(cp); @@ -527,8 +659,8 @@ static void fasttrap_exec_exit(proc_t *p) { ASSERT(p == current_proc()); - lck_mtx_assert(&p->p_mlock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&p->p_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_NOTOWNED); /* APPLE NOTE: Okay, the locking here is really odd and needs some @@ -618,6 +750,7 @@ fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index) again: lck_mtx_lock(&bucket->ftb_mtx); for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { + int rc = 0; /* * Note that it's safe to access the active count on the * associated proc structure because we know that at least one @@ -634,7 +767,10 @@ again: * enabled and the trap instruction hasn't been overwritten. * Since this is a little hairy, we'll punt for now. */ - + if (!tp->ftt_installed) { + if (fasttrap_tracepoint_install(p, tp) != 0) + rc = FASTTRAP_ENABLE_PARTIAL; + } /* * This can't be the first interested probe. We don't have * to worry about another thread being in the midst of @@ -666,6 +802,8 @@ again: ASSERT(0); } + tp->ftt_retired = 0; + lck_mtx_unlock(&bucket->ftb_mtx); if (new_tp != NULL) { @@ -673,7 +811,7 @@ again: new_tp->ftt_retids = NULL; } - return (0); + return rc; } /* @@ -697,7 +835,6 @@ again: */ if (fasttrap_tracepoint_install(p, new_tp) != 0) rc = FASTTRAP_ENABLE_PARTIAL; - /* * Increment the count of the number of tracepoints active in * the victim process. @@ -705,6 +842,7 @@ again: //ASSERT(p->p_proc_flag & P_PR_LOCK); p->p_dtrace_count++; + return (rc); } @@ -714,6 +852,7 @@ again: * Initialize the tracepoint that's been preallocated with the probe. */ new_tp = probe->ftp_tps[index].fit_tp; + new_tp->ftt_retired = 0; ASSERT(new_tp->ftt_pid == pid); ASSERT(new_tp->ftt_pc == pc); @@ -1148,8 +1287,9 @@ fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg) lck_mtx_unlock(&provider->ftp_mtx); } - if (whack) - fasttrap_pid_cleanup(); + if (whack) { + fasttrap_pid_cleanup(FASTTRAP_CLEANUP_PROVIDER); + } if (!probe->ftp_enabled) return; @@ -1213,6 +1353,7 @@ fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg) ASSERT(fasttrap_total >= probe->ftp_ntps); atomic_add_32(&fasttrap_total, -probe->ftp_ntps); + atomic_add_32(&fasttrap_retired, -probe->ftp_ntps); if (probe->ftp_gen + 1 >= fasttrap_mod_gen) fasttrap_mod_barrier(probe->ftp_gen); @@ -1579,7 +1720,6 @@ fasttrap_provider_retire(proc_t *p, const char *name, int mprov) fasttrap_provider_t *fp; fasttrap_bucket_t *bucket; dtrace_provider_id_t provid; - ASSERT(strlen(name) < sizeof (fp->ftp_name)); bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(p->p_pid, name)]; @@ -1622,6 +1762,13 @@ fasttrap_provider_retire(proc_t *p, const char *name, int mprov) atomic_add_64(&fp->ftp_proc->ftpc_acount, -1); ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount); + /* + * Add this provider probes to the retired count and + * make sure we don't add them twice + */ + atomic_add_32(&fasttrap_retired, fp->ftp_pcount); + fp->ftp_pcount = 0; + fp->ftp_retired = 1; fp->ftp_marked = 1; provid = fp->ftp_provid; @@ -1636,7 +1783,7 @@ fasttrap_provider_retire(proc_t *p, const char *name, int mprov) lck_mtx_unlock(&bucket->ftb_mtx); - fasttrap_pid_cleanup(); + fasttrap_pid_cleanup(FASTTRAP_CLEANUP_PROVIDER); } static int @@ -1703,9 +1850,21 @@ fasttrap_add_probe(fasttrap_probe_spec_t *pdata) if (p == PROC_NULL) return (ESRCH); + /* + * Set that the process is allowed to run modified code and + * bail if it is not allowed to + */ +#if CONFIG_EMBEDDED + if ((p->p_csflags & (CS_KILL|CS_HARD)) && !cs_allow_invalid(p)) { + proc_rele(p); + return (EPERM); + } +#endif if ((provider = fasttrap_provider_lookup(p, pdata->ftps_provider_type, - provider_name, &pid_attr)) == NULL) + provider_name, &pid_attr)) == NULL) { + proc_rele(p); return (ESRCH); + } proc_rele(p); /* @@ -1738,11 +1897,11 @@ fasttrap_add_probe(fasttrap_probe_spec_t *pdata) continue; atomic_add_32(&fasttrap_total, 1); - if (fasttrap_total > fasttrap_max) { atomic_add_32(&fasttrap_total, -1); goto no_mem; } + provider->ftp_pcount++; pp = zalloc(fasttrap_probe_t_zones[1]); bzero(pp, sizeof (fasttrap_probe_t)); @@ -1760,6 +1919,15 @@ fasttrap_add_probe(fasttrap_probe_spec_t *pdata) tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; tp->ftt_pid = pdata->ftps_pid; +#if defined(__arm__) || defined(__arm64__) + /* + * On arm the subinfo is used to distinguish between arm + * and thumb modes. On arm64 there is no thumb mode, so + * this field is simply initialized to 0 on its way + * into the kernel. + */ + tp->ftt_fntype = pdata->ftps_arch_subinfo; +#endif pp->ftp_tps[0].fit_tp = tp; pp->ftp_tps[0].fit_id.fti_probe = pp; @@ -1792,7 +1960,7 @@ fasttrap_add_probe(fasttrap_probe_spec_t *pdata) atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); goto no_mem; } - + provider->ftp_pcount += pdata->ftps_noffs; ASSERT(pdata->ftps_noffs > 0); if (pdata->ftps_noffs < FASTTRAP_PROBE_T_ZONE_MAX_TRACEPOINTS) { pp = zalloc(fasttrap_probe_t_zones[pdata->ftps_noffs]); @@ -1814,6 +1982,16 @@ fasttrap_add_probe(fasttrap_probe_spec_t *pdata) tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; tp->ftt_pid = pdata->ftps_pid; +#if defined(__arm__) || defined (__arm64__) + /* + * On arm the subinfo is used to distinguish between arm + * and thumb modes. On arm64 there is no thumb mode, so + * this field is simply initialized to 0 on its way + * into the kernel. + */ + + tp->ftt_fntype = pdata->ftps_arch_subinfo; +#endif pp->ftp_tps[i].fit_tp = tp; pp->ftp_tps[i].fit_id.fti_probe = pp; pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_probe_type; @@ -1837,7 +2015,7 @@ fasttrap_add_probe(fasttrap_probe_spec_t *pdata) lck_mtx_unlock(&provider->ftp_mtx); if (whack) - fasttrap_pid_cleanup(); + fasttrap_pid_cleanup(FASTTRAP_CLEANUP_PROVIDER); return (0); @@ -1854,7 +2032,7 @@ no_mem: provider->ftp_marked = 1; lck_mtx_unlock(&provider->ftp_mtx); - fasttrap_pid_cleanup(); + fasttrap_pid_cleanup(FASTTRAP_CLEANUP_PROVIDER); return (ENOMEM); } @@ -2025,6 +2203,8 @@ fasttrap_meta_create_probe(void *arg, void *parg, return; } + provider->ftp_pcount += ntps; + if (ntps < FASTTRAP_PROBE_T_ZONE_MAX_TRACEPOINTS) { pp = zalloc(fasttrap_probe_t_zones[ntps]); bzero(pp, offsetof(fasttrap_probe_t, ftp_tps[ntps])); @@ -2058,6 +2238,14 @@ fasttrap_meta_create_probe(void *arg, void *parg, * Both 32 & 64 bit want to go back one byte, to point at the first NOP */ tp->ftt_pc = dhpb->dthpb_base + (int64_t)dhpb->dthpb_offs[i] - 1; +#elif defined(__arm__) || defined(__arm64__) + /* + * All ARM and ARM64 probes are zero offset. We need to zero out the + * thumb bit because we still support 32bit user processes. + * On 64bit user processes, bit zero won't be set anyway. + */ + tp->ftt_pc = (dhpb->dthpb_base + (int64_t)dhpb->dthpb_offs[i]) & ~0x1UL; + tp->ftt_fntype = FASTTRAP_FN_USDT; #else #error "Architecture not supported" #endif @@ -2088,6 +2276,14 @@ fasttrap_meta_create_probe(void *arg, void *parg, * Both 32 & 64 bit want to go forward two bytes, to point at a single byte nop. */ tp->ftt_pc = dhpb->dthpb_base + (int64_t)dhpb->dthpb_enoffs[j] + 2; +#elif defined(__arm__) || defined(__arm64__) + /* + * All ARM and ARM64 probes are zero offset. We need to zero out the + * thumb bit because we still support 32bit user processes. + * On 64bit user processes, bit zero won't be set anyway. + */ + tp->ftt_pc = (dhpb->dthpb_base + (int64_t)dhpb->dthpb_enoffs[j]) & ~0x1UL; + tp->ftt_fntype = FASTTRAP_FN_USDT; #else #error "Architecture not supported" #endif @@ -2356,10 +2552,23 @@ fasttrap_attach(dev_info_t *devi, ddi_attach_cmd_t cmd) * Yes, this is a WAG. */ fasttrap_max = (sane_size >> 28) * 100000; + +#if CONFIG_EMBEDDED +#if defined(__LP64__) + /* + * On embedded, the zone map does not grow with the memory size over 1GB + * (see osfmk/vm/vm_init.c) + */ + if (fasttrap_max > 400000) { + fasttrap_max = 400000; + } +#endif +#endif if (fasttrap_max == 0) fasttrap_max = 50000; fasttrap_total = 0; + fasttrap_retired = 0; /* * Conjure up the tracepoints hashtable... @@ -2561,6 +2770,22 @@ fasttrap_init( void ) return; } + /* + * Start the fasttrap cleanup thread + */ + kern_return_t res = kernel_thread_start_priority((thread_continue_t)fasttrap_pid_cleanup_cb, NULL, 46 /* BASEPRI_BACKGROUND */, &fasttrap_cleanup_thread); + if (res != KERN_SUCCESS) { + panic("Could not create fasttrap_cleanup_thread"); + } + thread_set_thread_name(fasttrap_cleanup_thread, "dtrace_fasttrap_cleanup_thread"); + +#ifdef FASTTRAP_ASYNC_REMOVE + fasttrap_retired_size = DEFAULT_RETIRED_SIZE; + fasttrap_retired_spec = kmem_zalloc(fasttrap_retired_size * sizeof(fasttrap_tracepoint_t*), + KM_SLEEP); + lck_mtx_init(&fasttrap_retired_mtx, fasttrap_lck_grp, fasttrap_lck_attr); +#endif + gFasttrapInited = 1; } } diff --git a/bsd/dev/dtrace/fbt.c b/bsd/dev/dtrace/fbt.c index e05d5a922..25f052f1a 100644 --- a/bsd/dev/dtrace/fbt.c +++ b/bsd/dev/dtrace/fbt.c @@ -31,11 +31,12 @@ #endif #endif -#include <mach-o/loader.h> +#include <mach-o/loader.h> #include <libkern/kernel_mach_header.h> #include <sys/param.h> #include <sys/systm.h> +#include <sys/sysctl.h> #include <sys/errno.h> #include <sys/stat.h> #include <sys/ioctl.h> @@ -53,7 +54,11 @@ /* #include <machine/trap.h> */ struct savearea_t; /* Used anonymously */ -#if defined(__x86_64__) +#if defined(__arm__) || defined(__arm64__) +typedef kern_return_t (*perfCallback)(int, struct savearea_t *, __unused int, __unused int); +extern perfCallback tempDTraceTrapHook; +extern kern_return_t fbt_perfCallback(int, struct savearea_t *, __unused int, __unused int); +#elif defined(__x86_64__) typedef kern_return_t (*perfCallback)(int, struct savearea_t *, uintptr_t *, __unused int); extern perfCallback tempDTraceTrapHook; extern kern_return_t fbt_perfCallback(int, struct savearea_t *, uintptr_t *, __unused int); @@ -71,8 +76,407 @@ fbt_probe_t **fbt_probetab; int fbt_probetab_mask; static int fbt_verbose = 0; +int ignore_fbt_blacklist = 0; + +extern int dtrace_kernel_symbol_mode; + + void fbt_init( void ); +/* + * Critical routines that must not be probed. PR_5221096, PR_5379018. + * The blacklist must be kept in alphabetic order for purposes of bsearch(). + */ +static const char * critical_blacklist[] = +{ + "Call_DebuggerC", + "SysChoked", + "_ZN9IOService14newTemperatureElPS_", /* IOService::newTemperature */ + "_ZN9IOService26temperatureCriticalForZoneEPS_", /* IOService::temperatureCriticalForZone */ + "_ZNK6OSData14getBytesNoCopyEv", /* Data::getBytesNoCopy, IOHibernateSystemWake path */ + "_disable_preemption", + "_enable_preemption", + "bcopy_phys", + "console_cpu_alloc", + "console_cpu_free", + "cpu_IA32e_disable", + "cpu_IA32e_enable", + "cpu_NMI_interrupt", + "cpu_control", + "cpu_data_alloc", + "cpu_desc_init", + "cpu_desc_init64", + "cpu_desc_load", + "cpu_desc_load64", + "cpu_exit_wait", + "cpu_info", + "cpu_info_count", + "cpu_init", + "cpu_interrupt", + "cpu_machine_init", + "cpu_mode_init", + "cpu_processor_alloc", + "cpu_processor_free", + "cpu_signal_handler", + "cpu_sleep", + "cpu_start", + "cpu_subtype", + "cpu_thread_alloc", + "cpu_thread_halt", + "cpu_thread_init", + "cpu_threadtype", + "cpu_to_processor", + "cpu_topology_sort", + "cpu_topology_start_cpu", + "cpu_type", + "cpuid_cpu_display", + "cpuid_extfeatures", + "dtrace_invop", + "enter_lohandler", + "fbt_invop", + "fbt_perfCallback", + "get_threadtask", + "handle_pending_TLB_flushes", + "hw_compare_and_store", + "interrupt", + "kernel_trap", + "kprintf", + "lo_alltraps", + "lock_debugger", + "machine_idle_cstate", + "machine_thread_get_kern_state", + "mca_cpu_alloc", + "mca_cpu_init", + "ml_nofault_copy", + "nanoseconds_to_absolutetime", + "nanotime_to_absolutetime", + "packA", + "panic", + "pmKextRegister", + "pmMarkAllCPUsOff", + "pmSafeMode", + "pmTimerRestore", + "pmTimerSave", + "pmUnRegister", + "pmap_cpu_alloc", + "pmap_cpu_free", + "pmap_cpu_high_map_vaddr", + "pmap_cpu_high_shared_remap", + "pmap_cpu_init", + "power_management_init", + "preemption_underflow_panic", + "register_cpu_setup_func", + "sdt_invop", + "sprlock", + "sprunlock", + "t_invop", + "tmrCvt", + "uread", + "uwrite", + "unlock_debugger", + "unpackA", + "unregister_cpu_setup_func", + "vstart" +}; +#define CRITICAL_BLACKLIST_COUNT (sizeof(critical_blacklist)/sizeof(critical_blacklist[0])) + +/* + * The transitive closure of entry points that can be reached from probe context. + * (Apart from routines whose names begin with dtrace_). + */ +static const char * probe_ctx_closure[] = +{ + "ClearIdlePop", + "Debugger", + "IS_64BIT_PROCESS", + "OSCompareAndSwap", + "SetIdlePop", + "absolutetime_to_microtime", + "act_set_astbsd", + "arm_init_idle_cpu", + "ast_dtrace_on", + "ast_pending", + "clean_dcache", + "clean_mmu_dcache", + "clock_get_calendar_nanotime_nowait", + "copyin", + "copyin_kern", + "copyin_user", + "copyinstr", + "copyout", + "copyoutstr", + "cpu_number", + "current_proc", + "current_processor", + "current_task", + "current_thread", + "debug_enter", + "drain_write_buffer", + "find_user_regs", + "flush_dcache", + "flush_tlb64", + "get_bsdtask_info", + "get_bsdthread_info", + "hertz_tick", + "hw_atomic_and", + "invalidate_mmu_icache", + "kauth_cred_get", + "kauth_getgid", + "kauth_getuid", + "kernel_preempt_check", + "kvtophys", + "mach_absolute_time", + "max_valid_stack_address", + "memcpy", + "memmove", + "ml_at_interrupt_context", + "ml_phys_write_byte_64", + "ml_phys_write_half_64", + "ml_phys_write_word_64", + "ml_set_interrupts_enabled", + "mt_core_snap", + "mt_cur_cpu_cycles", + "mt_cur_cpu_instrs", + "mt_cur_thread_cycles", + "mt_cur_thread_instrs", + "mt_fixed_counts", + "mt_fixed_counts_internal", + "mt_mtc_update_count", + "mt_update_thread", + "ovbcopy", + "panic", + "pmap64_pde", + "pmap64_pdpt", + "pmap_find_phys", + "pmap_get_mapwindow", + "pmap_pde", + "pmap_pte", + "pmap_put_mapwindow", + "pmap_valid_page", + "prf", + "proc_is64bit", + "proc_selfname", + "psignal_lock", + "rtc_nanotime_load", + "rtc_nanotime_read", + "sdt_getargdesc", + "setPop", + "strlcpy", + "sync_iss_to_iks_unconditionally", + "systrace_stub", + "timer_grab" +}; +#define PROBE_CTX_CLOSURE_COUNT (sizeof(probe_ctx_closure)/sizeof(probe_ctx_closure[0])) + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wcast-qual" +static int _cmp(const void *a, const void *b) +{ + return strncmp((const char *)a, *(const char **)b, strlen((const char *)a) + 1); +} +#pragma clang diagnostic pop +/* + * Module validation + */ +int +fbt_module_excluded(struct modctl* ctl) +{ + ASSERT(!MOD_FBT_DONE(ctl)); + + if (ctl->mod_address == 0 || ctl->mod_size == 0) { + return TRUE; + } + + if (ctl->mod_loaded == 0) { + return TRUE; + } + + /* + * If the user sets this, trust they know what they are doing. + */ + if (ignore_fbt_blacklist) + return FALSE; + + /* + * These drivers control low level functions that when traced + * cause problems often in the sleep/wake paths as well as + * critical debug and panic paths. + * If somebody really wants to drill in on one of these kexts, then + * they can override blacklisting using the boot-arg above. + */ + +#ifdef __x86_64__ + if (strstr(ctl->mod_modname, "AppleACPIEC") != NULL) + return TRUE; + + if (strstr(ctl->mod_modname, "AppleACPIPlatform") != NULL) + return TRUE; + + if (strstr(ctl->mod_modname, "AppleRTC") != NULL) + return TRUE; + + if (strstr(ctl->mod_modname, "IOACPIFamily") != NULL) + return TRUE; + + if (strstr(ctl->mod_modname, "AppleIntelCPUPowerManagement") != NULL) + return TRUE; + + if (strstr(ctl->mod_modname, "AppleProfile") != NULL) + return TRUE; + + if (strstr(ctl->mod_modname, "AppleIntelProfile") != NULL) + return TRUE; + + if (strstr(ctl->mod_modname, "AppleEFI") != NULL) + return TRUE; + +#elif __arm__ || __arm64__ + if (LIT_STRNEQL(ctl->mod_modname, "com.apple.driver.AppleARMPlatform") || + LIT_STRNEQL(ctl->mod_modname, "com.apple.driver.AppleARMPL192VIC") || + LIT_STRNEQL(ctl->mod_modname, "com.apple.driver.AppleInterruptController")) + return TRUE; +#endif + + return FALSE; +} + +/* + * FBT probe name validation + */ +int +fbt_excluded(const char* name) +{ + /* + * If the user set this, trust they know what they are doing. + */ + if (ignore_fbt_blacklist) + return FALSE; + + if (LIT_STRNSTART(name, "dtrace_") && !LIT_STRNSTART(name, "dtrace_safe_")) { + /* + * Anything beginning with "dtrace_" may be called + * from probe context unless it explitly indicates + * that it won't be called from probe context by + * using the prefix "dtrace_safe_". + */ + return TRUE; + } + + /* + * Place no probes on critical routines (5221096) + */ + if (bsearch( name, critical_blacklist, CRITICAL_BLACKLIST_COUNT, sizeof(name), _cmp ) != NULL) + return TRUE; + + /* + * Place no probes that could be hit in probe context. + */ + if (bsearch( name, probe_ctx_closure, PROBE_CTX_CLOSURE_COUNT, sizeof(name), _cmp ) != NULL) { + return TRUE; + } + + /* + * Place no probes that could be hit in probe context. + * In the interests of safety, some of these may be overly cautious. + * Also exclude very low-level "firmware" class calls. + */ + if (LIT_STRNSTART(name, "cpu_") || /* Coarse */ + LIT_STRNSTART(name, "platform_") || /* Coarse */ + LIT_STRNSTART(name, "machine_") || /* Coarse */ + LIT_STRNSTART(name, "ml_") || /* Coarse */ + LIT_STRNSTART(name, "PE_") || /* Coarse */ + LIT_STRNSTART(name, "rtc_") || /* Coarse */ + LIT_STRNSTART(name, "_rtc_") || + LIT_STRNSTART(name, "rtclock_") || + LIT_STRNSTART(name, "clock_") || + LIT_STRNSTART(name, "bcopy") || + LIT_STRNSTART(name, "pmap_") || + LIT_STRNSTART(name, "hw_") || /* Coarse */ + LIT_STRNSTART(name, "lapic_") || /* Coarse */ + LIT_STRNSTART(name, "OSAdd") || + LIT_STRNSTART(name, "OSBit") || + LIT_STRNSTART(name, "OSDecrement") || + LIT_STRNSTART(name, "OSIncrement") || + LIT_STRNSTART(name, "OSCompareAndSwap") || + LIT_STRNSTART(name, "etimer_") || + LIT_STRNSTART(name, "dtxnu_kern_") || + LIT_STRNSTART(name, "flush_mmu_tlb_")) + return TRUE; + /* + * Fasttrap inner-workings we can't instrument + * on Intel (6230149) + */ + if (LIT_STRNSTART(name, "fasttrap_") || + LIT_STRNSTART(name, "fuword") || + LIT_STRNSTART(name, "suword")) + return TRUE; + + if (LIT_STRNSTART(name, "_dtrace")) + return TRUE; /* Shims in dtrace.c */ + + if (LIT_STRNSTART(name, "hibernate_")) + return TRUE; + + /* + * Place no probes in the exception handling path + */ +#if __arm__ || __arm64__ + if (LIT_STRNSTART(name, "fleh_") || + LIT_STRNSTART(name, "sleh_") || + LIT_STRNSTART(name, "timer_state_event") || + LIT_STRNEQL(name, "get_vfp_enabled")) + return TRUE; + + if (LIT_STRNSTART(name, "_ZNK15OSMetaClassBase8metaCastEPK11OSMetaClass") || + LIT_STRNSTART(name, "_ZN15OSMetaClassBase12safeMetaCastEPKS_PK11OSMetaClass") || + LIT_STRNSTART(name, "_ZNK11OSMetaClass13checkMetaCastEPK15OSMetaClassBase")) + return TRUE; +#endif + + +#ifdef __x86_64__ + if (LIT_STRNSTART(name, "machine_") || + LIT_STRNSTART(name, "mapping_") || + LIT_STRNSTART(name, "tsc_") || + LIT_STRNSTART(name, "pmCPU") || + LIT_STRNSTART(name, "pms") || + LIT_STRNSTART(name, "usimple_") || + LIT_STRNSTART(name, "lck_spin_lock") || + LIT_STRNSTART(name, "lck_spin_unlock") || + LIT_STRNSTART(name, "absolutetime_to_") || + LIT_STRNSTART(name, "commpage_") || + LIT_STRNSTART(name, "ml_") || + LIT_STRNSTART(name, "PE_") || + LIT_STRNSTART(name, "act_machine") || + LIT_STRNSTART(name, "acpi_") || + LIT_STRNSTART(name, "pal_")) { + return TRUE; + } + // Don't Steal Mac OS X + if (LIT_STRNSTART(name, "dsmos_")) + return TRUE; + +#endif + + /* + * Place no probes that could be hit on the way to the debugger. + */ + if (LIT_STRNSTART(name, "kdp_") || + LIT_STRNSTART(name, "kdb_") || + LIT_STRNSTART(name, "debug_")) { + return TRUE; + } + + /* + * Place no probes that could be hit on the way to a panic. + */ + if (NULL != strstr(name, "panic_")) + return TRUE; + + return FALSE; +} + + /*ARGSUSED*/ static void fbt_destroy(void *arg, dtrace_id_t id, void *parg) @@ -267,6 +671,13 @@ fbt_resume(void *arg, dtrace_id_t id, void *parg) (void)ml_nofault_copy( (vm_offset_t)&fbt->fbtp_patchval, (vm_offset_t)fbt->fbtp_patchpoint, sizeof(fbt->fbtp_patchval)); +#if CONFIG_EMBEDDED + /* + * Make the patched instruction visible via a data + instruction cache flush. + */ + flush_dcache((vm_offset_t)fbt->fbtp_patchpoint,(vm_size_t)sizeof(fbt->fbtp_patchval), 0); + invalidate_icache((vm_offset_t)fbt->fbtp_patchpoint,(vm_size_t)sizeof(fbt->fbtp_patchval), 0); +#endif fbt->fbtp_currentval = fbt->fbtp_patchval; } @@ -374,6 +785,85 @@ err: } #endif /* __APPLE__ */ +static void +fbt_provide_module_user_syms(struct modctl *ctl) +{ + unsigned int i; + char *modname = ctl->mod_modname; + + dtrace_module_symbols_t* module_symbols = ctl->mod_user_symbols; + if (module_symbols) { + for (i=0; i<module_symbols->dtmodsyms_count; i++) { + + /* + * symbol->dtsym_addr (the symbol address) passed in from + * user space, is already slid for both kexts and kernel. + */ + dtrace_symbol_t* symbol = &module_symbols->dtmodsyms_symbols[i]; + + char* name = symbol->dtsym_name; + + /* Lop off omnipresent leading underscore. */ + if (*name == '_') + name += 1; + + /* + * We're only blacklisting functions in the kernel for now. + */ + if (MOD_IS_MACH_KERNEL(ctl) && fbt_excluded(name)) + continue; + + /* + * Ignore symbols with a null address + */ + if (!symbol->dtsym_addr) + continue; + + fbt_provide_probe(ctl, (uintptr_t)symbol->dtsym_addr, (uintptr_t)(symbol->dtsym_addr + symbol->dtsym_size), modname, name, (machine_inst_t*)(uintptr_t)symbol->dtsym_addr); + } + } +} + + +void +fbt_provide_module(void *arg, struct modctl *ctl) +{ +#pragma unused(arg) + ASSERT(ctl != NULL); + ASSERT(dtrace_kernel_symbol_mode != DTRACE_KERNEL_SYMBOLS_NEVER); + LCK_MTX_ASSERT(&mod_lock, LCK_MTX_ASSERT_OWNED); + + // Update the "ignore blacklist" bit + if (ignore_fbt_blacklist) + ctl->mod_flags |= MODCTL_FBT_PROVIDE_BLACKLISTED_PROBES; + + if (MOD_FBT_DONE(ctl)) + return; + + if (fbt_module_excluded(ctl)) { + ctl->mod_flags |= MODCTL_FBT_INVALID; + return; + } + + if (MOD_HAS_KERNEL_SYMBOLS(ctl)) { + fbt_provide_module_kernel_syms(ctl); + ctl->mod_flags |= MODCTL_FBT_PROBES_PROVIDED; + if (MOD_FBT_PROVIDE_BLACKLISTED_PROBES(ctl)) + ctl->mod_flags |= MODCTL_FBT_BLACKLISTED_PROBES_PROVIDED; + return; + } + + if (MOD_HAS_USERSPACE_SYMBOLS(ctl)) { + fbt_provide_module_user_syms(ctl); + ctl->mod_flags |= MODCTL_FBT_PROBES_PROVIDED; + if (MOD_FBT_PROVIDE_PRIVATE_PROBES(ctl)) + ctl->mod_flags |= MODCTL_FBT_PRIVATE_PROBES_PROVIDED; + if (MOD_FBT_PROVIDE_BLACKLISTED_PROBES(ctl)) + ctl->mod_flags |= MODCTL_FBT_BLACKLISTED_PROBES_PROVIDED; + return; + } +} + static dtrace_pattr_t fbt_attr = { { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, @@ -451,6 +941,47 @@ _fbt_open(dev_t dev, int flags, int devtype, struct proc *p) #define FBT_MAJOR -24 /* let the kernel pick the device number */ +SYSCTL_DECL(_kern_dtrace); + +static int +sysctl_dtrace_ignore_fbt_blacklist SYSCTL_HANDLER_ARGS +{ +#pragma unused(oidp, arg2) + int err; + int value = *(int*)arg1; + + err = sysctl_io_number(req, value, sizeof(value), &value, NULL); + if (err) + return (err); + if (req->newptr) { + if (!(value == 0 || value == 1)) + return (ERANGE); + + /* + * We do not allow setting the blacklist back to on, as we have no way + * of knowing if those unsafe probes are still used. + * + * If we are using kernel symbols, we also do not allow any change, + * since the symbols are jettison'd after the first pass. + * + * We do not need to take any locks here because those symbol modes + * are permanent and do not change after boot. + */ + if (value != 1 || dtrace_kernel_symbol_mode == DTRACE_KERNEL_SYMBOLS_NEVER || + dtrace_kernel_symbol_mode == DTRACE_KERNEL_SYMBOLS_ALWAYS_FROM_KERNEL) + return (EPERM); + + ignore_fbt_blacklist = 1; + } + + return (0); +} + +SYSCTL_PROC(_kern_dtrace, OID_AUTO, ignore_fbt_blacklist, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, + &ignore_fbt_blacklist, 0, + sysctl_dtrace_ignore_fbt_blacklist, "I", "fbt provider ignore blacklist"); + /* * A struct describing which functions will get invoked for certain * actions. @@ -473,8 +1004,8 @@ static struct cdevsw fbt_cdevsw = 0 /* type */ }; -int gIgnoreFBTBlacklist = 0; -static int gFBTInited = 0; +static int fbt_inited = 0; + #undef kmem_alloc /* from its binding to dt_kmem_alloc glue */ #undef kmem_free /* from its binding to dt_kmem_free glue */ #include <vm/vm_kern.h> @@ -482,7 +1013,7 @@ static int gFBTInited = 0; void fbt_init( void ) { - if (0 == gFBTInited) + if (0 == fbt_inited) { int majdevno = cdevsw_add(FBT_MAJOR, &fbt_cdevsw); @@ -491,11 +1022,11 @@ fbt_init( void ) return; } - PE_parse_boot_argn("IgnoreFBTBlacklist", &gIgnoreFBTBlacklist, sizeof (gIgnoreFBTBlacklist)); + PE_parse_boot_argn("IgnoreFBTBlacklist", &ignore_fbt_blacklist, sizeof (ignore_fbt_blacklist)); fbt_attach( (dev_info_t *)(uintptr_t)majdevno, DDI_ATTACH ); - gFBTInited = 1; /* Ensure this initialization occurs just one time. */ + fbt_inited = 1; /* Ensure this initialization occurs just one time. */ } else panic("fbt_init: called twice!\n"); diff --git a/bsd/dev/dtrace/lockstat.c b/bsd/dev/dtrace/lockstat.c index ef1d9f1e7..2aebd0a1e 100644 --- a/bsd/dev/dtrace/lockstat.c +++ b/bsd/dev/dtrace/lockstat.c @@ -52,6 +52,18 @@ #define membar_producer dtrace_membar_producer +#define PROBE_ARGS0(a, b, c, d, e) "\000" +#define PROBE_ARGS1(a, b, c, d, e) a "\000" +#define PROBE_ARGS2(a, b, c, d, e) a "\000" b "\000" +#define PROBE_ARGS3(a, b, c, d, e) a "\000" b "\000" c "\000" +#define PROBE_ARGS4(a, b, c, d, e) a "\000" b "\000" c "\000" d "\000" +#define PROBE_ARGS5(a, b, c, d, e) a "\000" b "\000" c "\000" d "\000" e "\000" +#define PROBE_ARGS_(a, b, c, d, e, n, ...) PROBE_ARGS##n(a, b, c, d, e) +#define PROBE_ARGS(...) PROBE_ARGS_(__VA_ARGS__, 5, 4, 3, 2, 1, 0) + +#define LOCKSTAT_PROBE(func, name, probe, ...) \ + {func, name, probe, DTRACE_IDNONE, PROBE_ARGS(__VA_ARGS__)} + /* * Hot patch values, x86 */ @@ -59,6 +71,14 @@ #define NOP 0x90 #define RET 0xc3 #define LOCKSTAT_AFRAMES 1 +#elif defined(__arm__) +#define NOP 0xE1A00000 +#define BXLR 0xE12FFF1E +#define LOCKSTAT_AFRAMES 2 +#elif defined(__arm64__) +#define NOP 0xD503201F +#define RET 0xD65f03c0 +#define LOCKSTAT_AFRAMES 2 #else #error "not ported to this architecture" #endif @@ -68,52 +88,90 @@ typedef struct lockstat_probe { const char *lsp_name; int lsp_probe; dtrace_id_t lsp_id; + const char *lsp_args; } lockstat_probe_t; lockstat_probe_t lockstat_probes[] = { #if defined(__x86_64__) /* Only provide implemented probes for each architecture */ - { LS_LCK_MTX_LOCK, LSA_ACQUIRE, LS_LCK_MTX_LOCK_ACQUIRE, DTRACE_IDNONE }, - { LS_LCK_MTX_LOCK, LSA_SPIN, LS_LCK_MTX_LOCK_SPIN, DTRACE_IDNONE }, - { LS_LCK_MTX_LOCK, LSA_BLOCK, LS_LCK_MTX_LOCK_BLOCK, DTRACE_IDNONE }, - { LS_LCK_MTX_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_LOCK_ACQUIRE, DTRACE_IDNONE }, - { LS_LCK_MTX_TRY_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE, DTRACE_IDNONE }, - { LS_LCK_MTX_UNLOCK, LSA_RELEASE, LS_LCK_MTX_UNLOCK_RELEASE, DTRACE_IDNONE }, - { LS_LCK_MTX_EXT_LOCK, LSA_ACQUIRE, LS_LCK_MTX_EXT_LOCK_ACQUIRE, DTRACE_IDNONE }, - { LS_LCK_MTX_EXT_LOCK, LSA_SPIN, LS_LCK_MTX_EXT_LOCK_SPIN, DTRACE_IDNONE }, - { LS_LCK_MTX_EXT_LOCK, LSA_BLOCK, LS_LCK_MTX_EXT_LOCK_BLOCK, DTRACE_IDNONE }, -// { LS_LCK_MTX_EXT_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_EXT_LOCK_ACQUIRE, DTRACE_IDNONE }, - { LS_LCK_MTX_EXT_UNLOCK, LSA_RELEASE, LS_LCK_MTX_EXT_UNLOCK_RELEASE, DTRACE_IDNONE }, - { LS_LCK_MTX_LOCK_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_LOCK_SPIN_ACQUIRE, DTRACE_IDNONE }, - { LS_LCK_RW_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_LOCK_SHARED_ACQUIRE, DTRACE_IDNONE }, - { LS_LCK_RW_LOCK_SHARED, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_BLOCK, DTRACE_IDNONE }, - { LS_LCK_RW_LOCK_SHARED, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_SPIN, DTRACE_IDNONE }, - { LS_LCK_RW_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_LOCK_EXCL_ACQUIRE, DTRACE_IDNONE }, - { LS_LCK_RW_LOCK_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_EXCL_BLOCK, DTRACE_IDNONE }, - { LS_LCK_RW_LOCK_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_EXCL_SPIN, DTRACE_IDNONE }, - { LS_LCK_RW_DONE, LSR_RELEASE, LS_LCK_RW_DONE_RELEASE, DTRACE_IDNONE }, - { LS_LCK_RW_TRY_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE, DTRACE_IDNONE }, - { LS_LCK_RW_TRY_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE, DTRACE_IDNONE }, - { LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_UPGRADE, LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE, DTRACE_IDNONE }, - { LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN, DTRACE_IDNONE }, - { LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK, DTRACE_IDNONE }, - { LS_LCK_RW_LOCK_EXCL_TO_SHARED, LSR_DOWNGRADE, LS_LCK_RW_LOCK_EXCL_TO_SHARED_DOWNGRADE, DTRACE_IDNONE }, + LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_ACQUIRE, LS_LCK_MTX_LOCK_ACQUIRE, "lck_mtx_t"), + LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_SPIN, LS_LCK_MTX_LOCK_SPIN, "lck_mtx_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_BLOCK, LS_LCK_MTX_LOCK_BLOCK, "lck_mtx_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_MTX_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_LOCK_ACQUIRE, "lck_mtx_t"), + LOCKSTAT_PROBE(LS_LCK_MTX_TRY_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE, "lck_mtx_t"), + LOCKSTAT_PROBE(LS_LCK_MTX_UNLOCK, LSA_RELEASE, LS_LCK_MTX_UNLOCK_RELEASE, "lck_mtx_t"), + LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_ACQUIRE, LS_LCK_MTX_EXT_LOCK_ACQUIRE, "lck_mtx_t"), + LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_SPIN, LS_LCK_MTX_EXT_LOCK_SPIN, "lck_mtx_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_BLOCK, LS_LCK_MTX_EXT_LOCK_BLOCK, "lck_mtx_t", "uint64_t"), +// LOCKSTAT_PROBE(LS_LCK_MTX_EXT_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_EXT_LOCK_ACQUIRE) + LOCKSTAT_PROBE(LS_LCK_MTX_EXT_UNLOCK, LSA_RELEASE, LS_LCK_MTX_EXT_UNLOCK_RELEASE, "lck_mtx_t"), + LOCKSTAT_PROBE(LS_LCK_MTX_LOCK_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_LOCK_SPIN_ACQUIRE, "lck_mtx_t"), + // TODO: This should not be a uint64_t ! + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_LOCK_SHARED_ACQUIRE, "lck_rw_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_SPIN, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"), + // TODO: This should NOT be a uint64_t + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_LOCK_EXCL_ACQUIRE, "lck_rw_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_EXCL_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_EXCL_SPIN, "lck_rw_t", "uint64_t", "int"), + LOCKSTAT_PROBE(LS_LCK_RW_DONE, LSR_RELEASE, LS_LCK_RW_DONE_RELEASE, "lck_rw_t", "_Bool"), + // TODO : This should NOT be a uint64_t + LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE, "lck_rw_t", "uint64_t"), + // See above + LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE, "lck_rw_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_UPGRADE, LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE, "lck_rw_t", "_Bool"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN, "lck_rw_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL_TO_SHARED, LSR_DOWNGRADE, LS_LCK_RW_LOCK_EXCL_TO_SHARED_DOWNGRADE, "lck_rw_t"), + //TODO : Separate the probes for the hw_bit from the probe for the normal hw locks + LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK, LSS_ACQUIRE, LS_LCK_SPIN_LOCK_ACQUIRE, "hw_lock_t"), + LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK, LSS_SPIN, LS_LCK_SPIN_LOCK_SPIN, "hw_lock_t", "uint64_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_SPIN_UNLOCK, LSS_RELEASE, LS_LCK_SPIN_UNLOCK_RELEASE, "hw_lock_t"), +#elif defined(__arm__) || defined(__arm64__) + LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_ACQUIRE, LS_LCK_MTX_LOCK_ACQUIRE, "lck_mtx_t"), +// LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_SPIN, LS_LCK_MTX_LOCK_SPIN, "lck_mtx_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_BLOCK, LS_LCK_MTX_LOCK_BLOCK, "lck_mtx_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_MTX_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_LOCK_ACQUIRE, "lck_mtx_t"), +// LOCKSTAT_PROBE(LS_LCK_MTX_TRY_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE, "lck_mtx_t"), + LOCKSTAT_PROBE(LS_LCK_MTX_UNLOCK, LSA_RELEASE, LS_LCK_MTX_UNLOCK_RELEASE, "lck_mtx_t"), + LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_ACQUIRE, LS_LCK_MTX_EXT_LOCK_ACQUIRE, "lck_mtx_t"), +// LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_SPIN, LS_LCK_MTX_EXT_LOCK_SPIN, "lck_mtx_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_BLOCK, LS_LCK_MTX_EXT_LOCK_BLOCK, "lck_mtx_t", "uint64_t"), +// LOCKSTAT_PROBE(LS_LCK_MTX_EXT_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_EXT_LOCK_ACQUIRE) +// LOCKSTAT_PROBE(LS_LCK_MTX_EXT_UNLOCK, LSA_RELEASE, LS_LCK_MTX_EXT_UNLOCK_RELEASE, "lck_mtx_t"), +// LOCKSTAT_PROBE(LS_LCK_MTX_LOCK_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_LOCK_SPIN_ACQUIRE, "lck_mtx_t"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_LOCK_SHARED_ACQUIRE, "lck_rw_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_SPIN, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_LOCK_EXCL_ACQUIRE, "lck_rw_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_EXCL_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_EXCL_SPIN, "lck_rw_t", "uint64_t", "int"), + LOCKSTAT_PROBE(LS_LCK_RW_DONE, LSR_RELEASE, LS_LCK_RW_DONE_RELEASE, "lck_rw_t", "_Bool"), + // TODO : This should NOT be a uint64_t + LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE, "lck_rw_t", "uint64_t"), + // See above + LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE, "lck_rw_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_UPGRADE, LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE, "lck_rw_t", "_Bool"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN, "lck_rw_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK, "lck_rw_t", "uint64_t", "_Bool", "_Bool", "int"), + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL_TO_SHARED, LSR_DOWNGRADE, LS_LCK_RW_LOCK_EXCL_TO_SHARED_DOWNGRADE, "lck_rw_t"), + //TODO : Separate the probes for the hw_bit from the probe for the normal hw locks + LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK, LSS_ACQUIRE, LS_LCK_SPIN_LOCK_ACQUIRE, "hw_lock_t"), + LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK, LSS_SPIN, LS_LCK_SPIN_LOCK_SPIN, "hw_lock_t", "uint64_t", "uint64_t"), + LOCKSTAT_PROBE(LS_LCK_SPIN_UNLOCK, LSS_RELEASE, LS_LCK_SPIN_UNLOCK_RELEASE, "hw_lock_t"), #endif + /* Interlock measurements would be nice, but later */ + #ifdef LATER - /* Interlock and spinlock measurements would be nice, but later */ - { LS_LCK_SPIN_LOCK, LSS_ACQUIRE, LS_LCK_SPIN_LOCK_ACQUIRE, DTRACE_IDNONE }, - { LS_LCK_SPIN_LOCK, LSS_SPIN, LS_LCK_SPIN_LOCK_SPIN, DTRACE_IDNONE }, - { LS_LCK_SPIN_UNLOCK, LSS_RELEASE, LS_LCK_SPIN_UNLOCK_RELEASE, DTRACE_IDNONE }, - - { LS_LCK_RW_LOCK_EXCL_TO_SHARED, LSA_ILK_SPIN, LS_LCK_RW_LOCK_EXCL_TO_SHARED_ILK_SPIN, DTRACE_IDNONE }, - { LS_LCK_MTX_LOCK, LSA_ILK_SPIN, LS_LCK_MTX_LOCK_ILK_SPIN, DTRACE_IDNONE }, - { LS_LCK_MTX_EXT_LOCK, LSA_ILK_SPIN, LS_LCK_MTX_EXT_LOCK_ILK_SPIN, DTRACE_IDNONE }, - { LS_LCK_RW_TRY_LOCK_EXCL, LSA_ILK_SPIN, LS_LCK_RW_TRY_LOCK_EXCL_ILK_SPIN, DTRACE_IDNONE }, - { LS_LCK_RW_TRY_LOCK_SHARED, LSA_SPIN, LS_LCK_RW_TRY_LOCK_SHARED_SPIN, DTRACE_IDNONE }, + LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL_TO_SHARED, LSA_ILK_SPIN, LS_LCK_RW_LOCK_EXCL_TO_SHARED_ILK_SPIN), + LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_ILK_SPIN, LS_LCK_MTX_LOCK_ILK_SPIN), + LOCKSTAT_PROBE(LS_LCK_MTX_EXT_LOCK, LSA_ILK_SPIN, LS_LCK_MTX_EXT_LOCK_ILK_SPIN), + LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_EXCL, LSA_ILK_SPIN, LS_LCK_RW_TRY_LOCK_EXCL_ILK_SPIN), + LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_SHARED, LSA_SPIN, LS_LCK_RW_TRY_LOCK_SHARED_SPIN) #endif - { NULL, NULL, 0, 0 } + { NULL, NULL, 0, 0, NULL} }; dtrace_id_t lockstat_probemap[LS_NPROBES]; @@ -126,14 +184,15 @@ extern void lck_mtx_try_lock_spin_lockstat_patch_point(void); extern void lck_mtx_unlock_lockstat_patch_point(void); extern void lck_mtx_lock_ext_lockstat_patch_point(void); extern void lck_mtx_ext_unlock_lockstat_patch_point(void); -extern void lck_rw_lock_shared_lockstat_patch_point(void); -extern void lck_rw_lock_exclusive_lockstat_patch_point(void); -extern void lck_rw_lock_shared_to_exclusive_lockstat_patch_point(void); -extern void lck_rw_try_lock_shared_lockstat_patch_point(void); -extern void lck_rw_try_lock_exclusive_lockstat_patch_point(void); extern void lck_mtx_lock_spin_lockstat_patch_point(void); +#endif +#if defined (__arm__) + #endif +#if defined (__arm64__) + +#endif #endif /* CONFIG_DTRACE */ typedef struct lockstat_assembly_probe { @@ -158,11 +217,6 @@ typedef struct lockstat_assembly_probe { { LS_LCK_MTX_UNLOCK_RELEASE, (vm_offset_t *) lck_mtx_unlock_lockstat_patch_point }, { LS_LCK_MTX_EXT_LOCK_ACQUIRE, (vm_offset_t *) lck_mtx_lock_ext_lockstat_patch_point }, { LS_LCK_MTX_EXT_UNLOCK_RELEASE, (vm_offset_t *) lck_mtx_ext_unlock_lockstat_patch_point }, - { LS_LCK_RW_LOCK_SHARED_ACQUIRE, (vm_offset_t *) lck_rw_lock_shared_lockstat_patch_point }, - { LS_LCK_RW_LOCK_EXCL_ACQUIRE, (vm_offset_t *) lck_rw_lock_exclusive_lockstat_patch_point }, - { LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE,(vm_offset_t *) lck_rw_lock_shared_to_exclusive_lockstat_patch_point }, - { LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE, (vm_offset_t *) lck_rw_try_lock_shared_lockstat_patch_point }, - { LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE, (vm_offset_t *) lck_rw_try_lock_exclusive_lockstat_patch_point }, { LS_LCK_MTX_LOCK_SPIN_ACQUIRE, (vm_offset_t *) lck_mtx_lock_spin_lockstat_patch_point }, #endif /* No assembly patch points for ARM */ @@ -199,6 +253,20 @@ void lockstat_hot_patch(boolean_t active, int ls_probe) (void) ml_nofault_copy( (vm_offset_t)&instr, *(assembly_probes[i].lsap_patch_point), sizeof(instr)); } +#elif defined (__arm__) + { + uint32_t instr; + instr = (active ? NOP : BXLR ); + (void) ml_nofault_copy( (vm_offset_t)&instr, *(assembly_probes[i].lsap_patch_point), + sizeof(instr)); + } +#elif defined (__arm64__) + { + uint32_t instr; + instr = (active ? NOP : RET ); + (void) ml_nofault_copy( (vm_offset_t)&instr, *(assembly_probes[i].lsap_patch_point), + sizeof(instr)); + } #endif } /* for */ } @@ -310,6 +378,29 @@ lockstat_destroy(void *arg, dtrace_id_t id, void *parg) probe->lsp_id = 0; } +static void +lockstat_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc) +{ +#pragma unused(arg, id) + lockstat_probe_t *probe = parg; + const char* argdesc = probe->lsp_args; + int narg = 0; + + desc->dtargd_native[0] = '\0'; + desc->dtargd_xlate[0] = '\0'; + + while(argdesc[0] != '\0') { + if (narg == desc->dtargd_ndx) { + strlcpy(desc->dtargd_native, argdesc, DTRACE_ARGTYPELEN); + return; + } + argdesc += strlen(argdesc) + 1; + narg++; + } + + desc->dtargd_ndx = DTRACE_ARGNONE; +} + static dtrace_pattr_t lockstat_attr = { { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, @@ -325,7 +416,7 @@ static dtrace_pops_t lockstat_pops = { lockstat_disable, NULL, NULL, - NULL, + lockstat_getargdesc, NULL, NULL, lockstat_destroy diff --git a/bsd/dev/dtrace/profile_prvd.c b/bsd/dev/dtrace/profile_prvd.c index eb5ada1bb..259fab8bc 100644 --- a/bsd/dev/dtrace/profile_prvd.c +++ b/bsd/dev/dtrace/profile_prvd.c @@ -54,6 +54,8 @@ #if defined(__x86_64__) extern x86_saved_state_t *find_kern_regs(thread_t); +#elif defined (__arm__) || defined(__arm64__) +extern struct arm_saved_state *find_kern_regs(thread_t); #else #error Unknown architecture #endif @@ -98,6 +100,8 @@ static dtrace_provider_id_t profile_id; #if defined(__x86_64__) #define PROF_ARTIFICIAL_FRAMES 9 +#elif defined(__arm__) || defined(__arm64__) +#define PROF_ARTIFICIAL_FRAMES 8 #else #error Unknown architecture #endif @@ -186,6 +190,50 @@ profile_fire(void *arg) dtrace_probe(prof->prof_id, 0x0, regs->eip, late, 0, 0); } } +#elif defined(__arm__) + { + arm_saved_state_t *arm_kern_regs = (arm_saved_state_t *) find_kern_regs(current_thread()); + + // We should only come in here from interrupt context, so we should always have valid kernel regs + assert(NULL != arm_kern_regs); + + if (arm_kern_regs->cpsr & 0xF) { + /* Kernel was interrupted. */ + dtrace_probe(prof->prof_id, arm_kern_regs->pc, 0x0, late, 0, 0); + } else { + /* Possibly a user interrupt */ + arm_saved_state_t *arm_user_regs = (arm_saved_state_t *)find_user_regs(current_thread()); + + if (NULL == arm_user_regs) { + /* Too bad, so sad, no useful interrupt state. */ + dtrace_probe(prof->prof_id, 0xcafebabe, 0x0, late, 0, 0); /* XXX_BOGUS also see profile_usermode() below. */ + } else { + dtrace_probe(prof->prof_id, 0x0, arm_user_regs->pc, late, 0, 0); + } + } + } +#elif defined(__arm64__) + { + arm_saved_state_t *arm_kern_regs = (arm_saved_state_t *) find_kern_regs(current_thread()); + + // We should only come in here from interrupt context, so we should always have valid kernel regs + assert(NULL != arm_kern_regs); + + if (saved_state64(arm_kern_regs)->cpsr & 0xF) { + /* Kernel was interrupted. */ + dtrace_probe(prof->prof_id, saved_state64(arm_kern_regs)->pc, 0x0, late, 0, 0); + } else { + /* Possibly a user interrupt */ + arm_saved_state_t *arm_user_regs = (arm_saved_state_t *)find_user_regs(current_thread()); + + if (NULL == arm_user_regs) { + /* Too bad, so sad, no useful interrupt state. */ + dtrace_probe(prof->prof_id, 0xcafebabe, 0x0, late, 0, 0); /* XXX_BOGUS also see profile_usermode() below. */ + } else { + dtrace_probe(prof->prof_id, 0x0, get_saved_state_pc(arm_user_regs), late, 0, 0); + } + } + } #else #error Unknown architecture #endif @@ -221,6 +269,45 @@ profile_tick(void *arg) dtrace_probe(prof->prof_id, 0x0, regs->eip, 0, 0, 0); } } +#elif defined(__arm__) + { + arm_saved_state_t *arm_kern_regs = (arm_saved_state_t *) find_kern_regs(current_thread()); + + if (NULL != arm_kern_regs) { + /* Kernel was interrupted. */ + dtrace_probe(prof->prof_id, arm_kern_regs->pc, 0x0, 0, 0, 0); + } else { + /* Possibly a user interrupt */ + arm_saved_state_t *arm_user_regs = (arm_saved_state_t *)find_user_regs(current_thread()); + + if (NULL == arm_user_regs) { + /* Too bad, so sad, no useful interrupt state. */ + dtrace_probe(prof->prof_id, 0xcafebabe, 0x0, 0, 0, 0); /* XXX_BOGUS also see profile_usermode() below. */ + } else { + dtrace_probe(prof->prof_id, 0x0, arm_user_regs->pc, 0, 0, 0); + } + } + } +#elif defined(__arm64__) + { + arm_saved_state_t *arm_kern_regs = (arm_saved_state_t *) find_kern_regs(current_thread()); + + if (NULL != arm_kern_regs) { + /* Kernel was interrupted. */ + dtrace_probe(prof->prof_id, saved_state64(arm_kern_regs)->pc, 0x0, 0, 0, 0); + } else { + /* Possibly a user interrupt */ + arm_saved_state_t *arm_user_regs = (arm_saved_state_t *)find_user_regs(current_thread()); + + if (NULL == arm_user_regs) { + /* Too bad, so sad, no useful interrupt state. */ + dtrace_probe(prof->prof_id, 0xcafebabe, 0x0, 0, 0, 0); /* XXX_BOGUS also see profile_usermode() below. */ + } else { + dtrace_probe(prof->prof_id, 0x0, get_saved_state_pc(arm_user_regs), 0, 0, 0); + } + } + } + #else #error Unknown architecture #endif @@ -499,6 +586,46 @@ profile_disable(void *arg, dtrace_id_t id, void *parg) prof->prof_cyclic = CYCLIC_NONE; } +static uint64_t +profile_getarg(void *arg, dtrace_id_t id, void *parg, int argno, int aframes) +{ +#pragma unused(arg, id, parg, argno, aframes) + /* + * All the required arguments for the profile probe are passed directly + * to dtrace_probe, and we do not go through dtrace_getarg which doesn't + * know how to hop to the kernel stack from the interrupt stack like + * dtrace_getpcstack + */ + return 0; +} + +static void +profile_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc) +{ +#pragma unused(arg, id) + profile_probe_t *prof = parg; + const char *argdesc = NULL; + switch (desc->dtargd_ndx) { + case 0: + argdesc = "void*"; + break; + case 1: + argdesc = "user_addr_t"; + break; + case 2: + if (prof->prof_kind == PROF_PROFILE) { + argdesc = "hrtime_t"; + } + break; + } + if (argdesc) { + strlcpy(desc->dtargd_native, argdesc, DTRACE_ARGTYPELEN); + } + else { + desc->dtargd_ndx = DTRACE_ARGNONE; + } +} + /* * APPLE NOTE: profile_usermode call not supported. */ @@ -524,8 +651,8 @@ static dtrace_pops_t profile_pops = { profile_disable, NULL, NULL, - NULL, - NULL, + profile_getargdesc, + profile_getarg, profile_usermode, profile_destroy }; diff --git a/bsd/dev/dtrace/scripts/Makefile b/bsd/dev/dtrace/scripts/Makefile index a6f2527cd..1957fb2b0 100644 --- a/bsd/dev/dtrace/scripts/Makefile +++ b/bsd/dev/dtrace/scripts/Makefile @@ -18,7 +18,8 @@ INSTALL_DTRACE_SCRIPTS_LIST = \ unistd.d INSTALL_DTRACE_LIBEXEC_LIST = \ - log_unnest_badness.d + log_unnest_badness.d \ + vm_map_delete_permanent.d ifneq ($(filter $(SUPPORTED_EMBEDDED_PLATFORMS),$(PLATFORM)),) INSTALL_DTRACE_SCRIPTS_LIST += mptcp.d diff --git a/bsd/dev/dtrace/scripts/mptcp.d b/bsd/dev/dtrace/scripts/mptcp.d index 331f82928..479b531dc 100644 --- a/bsd/dev/dtrace/scripts/mptcp.d +++ b/bsd/dev/dtrace/scripts/mptcp.d @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2014 Apple Computer, Inc. All Rights Reserved. + * Copyright (c) 2013-2017 Apple Computer, Inc. All Rights Reserved. * * @APPLE_LICENSE_HEADER_START@ * @@ -48,14 +48,9 @@ inline int MPTCPS_FIN_WAIT_2 = 7; #pragma D binding "1.0" MPTCPS_FIN_WAIT_2 inline int MPTCPS_TIME_WAIT = 8; #pragma D binding "1.0" MPTCPS_TIME_WAIT -inline int MPTCPS_FASTCLOSE_WAIT = 9; -#pragma D binding "1.0" MPTCPS_FASTCLOSE_WAIT inline int MPTCPS_TERMINATE = 10; #pragma D binding "1.0" MPTCPS_TERMINATE -typedef uint64_t mptcp_key_t; -typedef uint32_t mptcp_token_t; - typedef struct mptsinfo { string state; uint32_t flags; @@ -76,7 +71,6 @@ typedef struct mptsinfo { uint64_t local_idsn; uint32_t sndwnd; uint64_t rcvnxt; - uint64_t rcvatmark; uint64_t remote_idsn; uint32_t rcvwnd; struct mptcb *mptcb; @@ -94,15 +88,13 @@ translator mptsinfo_t < struct mptcb *T > { T->mpt_state == MPTCPS_LAST_ACK ? "state-last-ack" : T->mpt_state == MPTCPS_FIN_WAIT_2 ? "state-fin-wait-2" : T->mpt_state == MPTCPS_TIME_WAIT ? "state-time-wait" : - T->mpt_state == MPTCPS_FASTCLOSE_WAIT ? - "state-fastclose-wait" : T->mpt_state == MPTCPS_TERMINATE ? "state-terminate" : "<unknown>"; flags = T->mpt_flags; vers = T->mpt_version; error = T->mpt_softerror; - localkey = T->mpt_localkey ? *T->mpt_localkey : 0; + localkey = T->mpt_localkey; remotekey = T->mpt_remotekey; localtoken = T->mpt_localtoken; remotetoken = T->mpt_remotetoken; @@ -117,7 +109,6 @@ translator mptsinfo_t < struct mptcb *T > { local_idsn = T->mpt_local_idsn; sndwnd = T->mpt_sndwnd; rcvnxt = T->mpt_rcvnxt; - rcvatmark = T->mpt_rcvatmark; remote_idsn = T->mpt_remote_idsn; rcvwnd = T->mpt_rcvwnd; mptcb = T; @@ -210,17 +201,12 @@ inline int MPTSF_ACTIVE = 0x40000; #pragma D binding "1.0" MPTSF_ACTIVE inline int MPTSF_MPCAP_CTRSET = 0x80000; #pragma D binding "1.0" MPTSF_MPCAP_CTRSET -inline int MPTSF_FASTJ_SEND = 0x100000; -#pragma D binding "1.0" MPTSF_FASTJ_SEND typedef struct mptsubinfo { uint32_t flags; uint32_t evctl; - uint32_t family; sae_connid_t connid; uint32_t rank; - int32_t error; - uint64_t sndnxt; struct mptsub *mptsub; } mptsubinfo_t; @@ -228,10 +214,6 @@ typedef struct mptsubinfo { translator mptsubinfo_t < struct mptsub *T > { flags = T->mpts_flags; evctl = T->mpts_evctl; - family = T->mpts_family; connid = T->mpts_connid; - rank = T->mpts_rank; - error = T->mpts_soerror; - sndnxt = T->mpts_sndnxt; mptsub = T; }; diff --git a/bsd/dev/dtrace/scripts/vm_map_delete_permanent.d b/bsd/dev/dtrace/scripts/vm_map_delete_permanent.d new file mode 100644 index 000000000..9adb1c65f --- /dev/null +++ b/bsd/dev/dtrace/scripts/vm_map_delete_permanent.d @@ -0,0 +1,14 @@ +#!/usr/sbin/dtrace -s + +vminfo::vm_map_delete_permanent: +{ + printf("%d[%s]: attempt to delete permanent mapping (0x%llx, 0x%llx) prot 0x%x/0x%x", + $pid, + execname, + (uint64_t) arg0, + (uint64_t) arg1, + arg2, + arg3); + stack(); + ustack(); +} diff --git a/bsd/dev/dtrace/sdt.c b/bsd/dev/dtrace/sdt.c index a157923ee..2923bf644 100644 --- a/bsd/dev/dtrace/sdt.c +++ b/bsd/dev/dtrace/sdt.c @@ -40,6 +40,9 @@ #include <sys/fcntl.h> #include <miscfs/devfs/devfs.h> +#if CONFIG_EMBEDDED +#include <arm/caches_internal.h> +#endif #include <sys/dtrace.h> #include <sys/dtrace_impl.h> @@ -52,7 +55,19 @@ extern int dtrace_kernel_symbol_mode; /* #include <machine/trap.h */ struct savearea_t; /* Used anonymously */ -#if defined(__x86_64__) +#if defined(__arm__) +typedef kern_return_t (*perfCallback)(int, struct savearea_t *, __unused int, __unused int); +extern perfCallback tempDTraceTrapHook; +extern kern_return_t fbt_perfCallback(int, struct savearea_t *, __unused int, __unused int); +#define SDT_PATCHVAL 0xdefc +#define SDT_AFRAMES 7 +#elif defined(__arm64__) +typedef kern_return_t (*perfCallback)(int, struct savearea_t *, __unused int, __unused int); +extern perfCallback tempDTraceTrapHook; +extern kern_return_t fbt_perfCallback(int, struct savearea_t *, __unused int, __unused int); +#define SDT_PATCHVAL 0xe7eeee7e +#define SDT_AFRAMES 7 +#elif defined(__x86_64__) typedef kern_return_t (*perfCallback)(int, struct savearea_t *, uintptr_t *, int); extern perfCallback tempDTraceTrapHook; extern kern_return_t fbt_perfCallback(int, struct savearea_t *, uintptr_t *, int); @@ -564,6 +579,12 @@ void sdt_init( void ) strncpy(sdpd->sdpd_func, prev_name, len); /* NUL termination is ensured. */ sdpd->sdpd_offset = *(unsigned long *)sym[i].n_value; +#if defined(__arm__) + /* PR8353094 - mask off thumb-bit */ + sdpd->sdpd_offset &= ~0x1U; +#elif defined(__arm64__) + sdpd->sdpd_offset &= ~0x1LU; +#endif /* __arm__ */ #if 0 printf("sdt_init: sdpd_offset=0x%lx, n_value=0x%lx, name=%s\n", @@ -594,7 +615,7 @@ sdt_provide_module(void *arg, struct modctl *ctl) #pragma unused(arg) ASSERT(ctl != NULL); ASSERT(dtrace_kernel_symbol_mode != DTRACE_KERNEL_SYMBOLS_NEVER); - lck_mtx_assert(&mod_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&mod_lock, LCK_MTX_ASSERT_OWNED); if (MOD_SDT_DONE(ctl)) return; diff --git a/bsd/dev/dtrace/sdt_subr.c b/bsd/dev/dtrace/sdt_subr.c index 18c16e074..ad71d1ffe 100644 --- a/bsd/dev/dtrace/sdt_subr.c +++ b/bsd/dev/dtrace/sdt_subr.c @@ -887,11 +887,7 @@ sdt_argdesc_t sdt_args[] = { { "mptcp", "checksum-result", 0, 0, "struct tcpcb *", "tcpsinfo_t *" }, { "mptcp", "checksum-result", 1, 1, "struct mbuf *", "pktinfo_t *" }, { "mptcp", "checksum-result", 2, 2, "uint32_t", "uint32_t" }, - { "mptcp", "session-create", 0, 0, "struct socket *", "socketinfo_t *"}, - { "mptcp", "session-create", 1, 1, "struct sockbuf *", "socketbuf_t *"}, - { "mptcp", "session-create", 2, 2, "struct sockbuf *", "socketbuf_t *"}, - { "mptcp", "session-create", 3, 3, "struct mppcb *", "mppsinfo_t *" }, - { "mptcp", "session-create", 4, 4, "int", "int" }, + { "mptcp", "session-create", 0, 0, "struct mppcb *", "mppsinfo_t *" }, { "mptcp", "session-destroy", 0, 0, "struct mptses *", "mptsesinfo_t *" }, { "mptcp", "session-destroy", 1, 1, "struct mptcb *", "mptsinfo_t *" }, { "mptcp", "subflow-create", 0, 0, "struct mptses *", "mptsesinfo_t *"}, @@ -909,11 +905,6 @@ sdt_argdesc_t sdt_args[] = { { "mptcp", "subflow-receive", 0, 0, "struct socket *", "socketinfo_t *" }, { "mptcp", "subflow-receive", 1, 1, "struct sockbuf *", "socketbuf_t *" }, { "mptcp", "subflow-receive", 2, 2, "struct sockbuf *", "socketbuf_t *" }, - { "mptcp", "subflow-peeloff", 0, 0, "struct mptses *", "mptsesinfo_t *",}, - { "mptcp", "subflow-peeloff", 1, 1, "struct mptsub *", "mptsubinfo_t *",}, - { "mptcp", "subflow-peeloff", 2, 2, "struct socket *", "socketinfo_t *",}, - { "mptcp", "subflow-peeloff", 3, 3, "struct sockbuf *", "socketbuf_t *" }, - { "mptcp", "subflow-peeloff", 4, 4, "struct sockbuf *", "socketbuf_t *" }, { "mptcp", "subflow-input", 0, 0, "struct mptses *", "mptsesinfo_t *" }, { "mptcp", "subflow-input", 1, 1, "struct mptsub *", "mptsubinfo_t *" }, { "mptcp", "subflow-output", 0, 0, "struct mptses *", "mptsesinfo_t *"}, @@ -955,9 +946,6 @@ sdt_argdesc_t sdt_args[] = { { "mptcp", "disconnectx", 2, 2, "sae_connid_t", "sae_connid_t" }, { "mptcp", "disconnectx", 3, 3, "struct socket *", "sockinfo_t *" }, { "mptcp", "disconnectx", 4, 4, "struct mptcb *", "mptsinfo_t *" }, - { "mptcp", "peeloff", 0, 0, "struct mptses *", "mptsesinfo_t *" }, - { "mptcp", "peeloff", 1, 1, "sae_associd_t", "sae_associd_t" }, - { "mptcp", "peeloff", 2, 2, "struct socket *", "sockinfo_t *" }, { NULL, NULL, 0, 0, NULL, NULL } }; diff --git a/bsd/dev/dtrace/systrace.c b/bsd/dev/dtrace/systrace.c index 00ee62d29..9c0c34f63 100644 --- a/bsd/dev/dtrace/systrace.c +++ b/bsd/dev/dtrace/systrace.c @@ -54,6 +54,7 @@ #include <sys/dtrace.h> #include <sys/dtrace_impl.h> +#include <sys/systrace_args.h> #include "systrace.h" #include <sys/stat.h> #include <sys/systm.h> @@ -65,10 +66,15 @@ #if defined (__x86_64__) #define SYSTRACE_ARTIFICIAL_FRAMES 2 #define MACHTRACE_ARTIFICIAL_FRAMES 3 +#elif defined(__arm__) || defined(__arm64__) +#define SYSTRACE_ARTIFICIAL_FRAMES 2 +#define MACHTRACE_ARTIFICIAL_FRAMES 3 #else #error Unknown Architecture #endif +#define SYSTRACE_NARGS (int)(sizeof(((uthread_t)NULL)->uu_arg) / sizeof(((uthread_t)NULL)->uu_arg[0])) + #include <sys/sysent.h> #define sy_callc sy_call /* Map Solaris slot name to Darwin's */ #define NSYSCALL nsysent /* and is less than 500 or so */ @@ -89,7 +95,8 @@ static lck_mtx_t dtrace_systrace_lock; /* probe state lock */ systrace_sysent_t *systrace_sysent = NULL; void (*systrace_probe)(dtrace_id_t, uint64_t, uint64_t, uint64_t, uint64_t, uint64_t); -static uint64_t systrace_getarg(void *, dtrace_id_t, void *, int, int); +static uint64_t systrace_getargval(void *, dtrace_id_t, void *, int, int); +static void systrace_getargdesc(void *, dtrace_id_t, void *, dtrace_argdesc_t *); void systrace_stub(dtrace_id_t id, uint64_t arg0, uint64_t arg1, @@ -106,10 +113,8 @@ dtrace_systrace_syscall(struct proc *pp, void *uap, int *rv) systrace_sysent_t *sy; dtrace_id_t id; int32_t rval; -#if 0 /* XXX */ - proc_t *p; -#endif syscall_arg_t *ip = (syscall_arg_t *)uap; + uint64_t uargs[SYSTRACE_NARGS] = {0}; #if defined (__x86_64__) { @@ -135,6 +140,51 @@ dtrace_systrace_syscall(struct proc *pp, void *uap, int *rv) } } } +#elif defined(__arm__) + { + /* + * On arm, syscall numbers depend on a flavor (indirect or not) + * and can be in either r0 or r12 (always u32) + */ + + /* See bsd/dev/arm/systemcalls.c:arm_get_syscall_number */ + arm_saved_state_t *arm_regs = (arm_saved_state_t *) find_user_regs(current_thread()); + + /* Check for indirect system call */ + if (arm_regs->r[12] != 0) + code = arm_regs->r[12]; + else + code = arm_regs->r[0]; + } +#elif defined(__arm64__) + { + /* + * On arm64, syscall numbers depend on a flavor (indirect or not) + * ... and for u32 can be in either r0 or r12 + * ... and for u64 can be in either x0 or x16 + */ + + /* see bsd/dev/arm/systemcalls.c:arm_get_syscall_number */ + arm_saved_state_t *arm_regs = (arm_saved_state_t *) find_user_regs(current_thread()); + + if (is_saved_state32(arm_regs)) { + /* Check for indirect system call */ + if (saved_state32(arm_regs)->r[12] != 0) { + code = saved_state32(arm_regs)->r[12]; + } + else { + code = saved_state32(arm_regs)->r[0]; + } + } else { + /* Check for indirect system call */ + if (saved_state64(arm_regs)->x[ARM64_SYSCALL_CODE_REG_NUM] != 0 ) { + code = saved_state64(arm_regs)->x[ARM64_SYSCALL_CODE_REG_NUM]; + } + else { + code = saved_state64(arm_regs)->x[0]; + } + } + } #else #error Unknown Architecture #endif @@ -142,20 +192,22 @@ dtrace_systrace_syscall(struct proc *pp, void *uap, int *rv) // Bounds "check" the value of code a la unix_syscall sy = (code >= nsysent) ? &systrace_sysent[SYS_invalid] : &systrace_sysent[code]; + systrace_args(code, ip, uargs); + if ((id = sy->stsy_entry) != DTRACE_IDNONE) { uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread()); if (uthread) - uthread->t_dtrace_syscall_args = (void *)ip; + uthread->t_dtrace_syscall_args = uargs; - if (ip) - (*systrace_probe)(id, *ip, *(ip+1), *(ip+2), *(ip+3), *(ip+4)); - else - (*systrace_probe)(id, 0, 0, 0, 0, 0); + static_assert(SYSTRACE_NARGS >= 5, "not enough system call arguments"); + (*systrace_probe)(id, uargs[0], uargs[1], uargs[2], uargs[3], uargs[4]); if (uthread) - uthread->t_dtrace_syscall_args = (void *)0; + uthread->t_dtrace_syscall_args = NULL; } + + #if 0 /* XXX */ /* * APPLE NOTE: Not implemented. @@ -356,11 +408,6 @@ systrace_init(struct sysent *actual, systrace_sysent_t **interposed) if (a->sy_callc == dtrace_systrace_syscall) continue; -#ifdef _SYSCALL32_IMPL - if (a->sy_callc == dtrace_systrace_syscall32) - continue; -#endif - s->stsy_underlying = a->sy_callc; s->stsy_return_type = a->sy_return_type; } @@ -379,9 +426,6 @@ systrace_provide(void *arg, const dtrace_probedesc_t *desc) return; systrace_init(sysent, &systrace_sysent); -#ifdef _SYSCALL32_IMPL - systrace_init(sysent32, &systrace_sysent32); -#endif for (i = 0; i < NSYSCALL; i++) { if (systrace_sysent[i].stsy_underlying == NULL) @@ -400,10 +444,6 @@ systrace_provide(void *arg, const dtrace_probedesc_t *desc) systrace_sysent[i].stsy_entry = DTRACE_IDNONE; systrace_sysent[i].stsy_return = DTRACE_IDNONE; -#ifdef _SYSCALL32_IMPL - systrace_sysent32[i].stsy_entry = DTRACE_IDNONE; - systrace_sysent32[i].stsy_return = DTRACE_IDNONE; -#endif } } #undef systrace_init @@ -423,14 +463,8 @@ systrace_destroy(void *arg, dtrace_id_t id, void *parg) */ if (SYSTRACE_ISENTRY((uintptr_t)parg)) { ASSERT(systrace_sysent[sysnum].stsy_entry == DTRACE_IDNONE); -#ifdef _SYSCALL32_IMPL - ASSERT(systrace_sysent32[sysnum].stsy_entry == DTRACE_IDNONE); -#endif } else { ASSERT(systrace_sysent[sysnum].stsy_return == DTRACE_IDNONE); -#ifdef _SYSCALL32_IMPL - ASSERT(systrace_sysent32[sysnum].stsy_return == DTRACE_IDNONE); -#endif } } @@ -446,25 +480,14 @@ systrace_enable(void *arg, dtrace_id_t id, void *parg) if (SYSTRACE_ISENTRY((uintptr_t)parg)) { systrace_sysent[sysnum].stsy_entry = id; -#ifdef _SYSCALL32_IMPL - systrace_sysent32[sysnum].stsy_entry = id; -#endif } else { systrace_sysent[sysnum].stsy_return = id; -#ifdef _SYSCALL32_IMPL - systrace_sysent32[sysnum].stsy_return = id; -#endif } if (enabled) { ASSERT(sysent[sysnum].sy_callc == dtrace_systrace_syscall); return(0); } -#ifdef _SYSCALL32_IMPL - (void) casptr(&sysent32[sysnum].sy_callc, - (void *)systrace_sysent32[sysnum].stsy_underlying, - (void *)dtrace_systrace_syscall32); -#endif lck_mtx_lock(&dtrace_systrace_lock); if (sysent[sysnum].sy_callc == systrace_sysent[sysnum].stsy_underlying) { @@ -491,23 +514,12 @@ systrace_disable(void *arg, dtrace_id_t id, void *parg) ml_nofault_copy((vm_offset_t)&systrace_sysent[sysnum].stsy_underlying, (vm_offset_t)&sysent[sysnum].sy_callc, sizeof(systrace_sysent[sysnum].stsy_underlying)); lck_mtx_unlock(&dtrace_systrace_lock); -#ifdef _SYSCALL32_IMPL - (void) casptr(&sysent32[sysnum].sy_callc, - (void *)dtrace_systrace_syscall32, - (void *)systrace_sysent32[sysnum].stsy_underlying); -#endif } if (SYSTRACE_ISENTRY((uintptr_t)parg)) { systrace_sysent[sysnum].stsy_entry = DTRACE_IDNONE; -#ifdef _SYSCALL32_IMPL - systrace_sysent32[sysnum].stsy_entry = DTRACE_IDNONE; -#endif } else { systrace_sysent[sysnum].stsy_return = DTRACE_IDNONE; -#ifdef _SYSCALL32_IMPL - systrace_sysent32[sysnum].stsy_return = DTRACE_IDNONE; -#endif } } @@ -526,8 +538,8 @@ static dtrace_pops_t systrace_pops = { systrace_disable, NULL, NULL, - NULL, - systrace_getarg, + systrace_getargdesc, + systrace_getargval, NULL, systrace_destroy }; @@ -673,6 +685,28 @@ dtrace_machtrace_syscall(struct mach_call_args *args) code = -saved_state32(tagged_regs)->eax; } } +#elif defined(__arm__) + { + /* r12 has the machcall number, but it is -ve */ + arm_saved_state_t *arm_regs = (arm_saved_state_t *) find_user_regs(current_thread()); + code = (int)arm_regs->r[12]; + ASSERT(code < 0); /* Otherwise it would be a Unix syscall */ + code = -code; + } +#elif defined(__arm64__) + { + /* From arm/thread_status.h:get_saved_state_svc_number */ + arm_saved_state_t *arm_regs = (arm_saved_state_t *) find_user_regs(current_thread()); + if (is_saved_state32(arm_regs)) { + code = (int)saved_state32(arm_regs)->r[12]; + } else { + code = (int)saved_state64(arm_regs)->x[ARM64_SYSCALL_CODE_REG_NUM]; + } + + /* From bsd/arm64.c:mach_syscall */ + ASSERT(code < 0); /* Otherwise it would be a Unix syscall */ + code = -code; + } #else #error Unknown Architecture #endif @@ -966,27 +1000,55 @@ void systrace_init( void ) #undef SYSTRACE_MAJOR static uint64_t -systrace_getarg(void *arg, dtrace_id_t id, void *parg, int argno, int aframes) +systrace_getargval(void *arg, dtrace_id_t id, void *parg, int argno, int aframes) { #pragma unused(arg,id,parg,aframes) /* __APPLE__ */ uint64_t val = 0; - syscall_arg_t *stack = (syscall_arg_t *)NULL; + uint64_t *uargs = NULL; uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread()); if (uthread) - stack = (syscall_arg_t *)uthread->t_dtrace_syscall_args; - - if (!stack) + uargs = uthread->t_dtrace_syscall_args; + if (!uargs) + return(0); + if (argno < 0 || argno > SYSTRACE_NARGS) return(0); DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); - /* dtrace_probe arguments arg0 .. arg4 are 64bits wide */ - val = (uint64_t)*(stack+argno); + val = uargs[argno]; DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); return (val); } +static void +systrace_getargdesc(void *arg, dtrace_id_t id, void *parg, + dtrace_argdesc_t *desc) +{ +#pragma unused(arg, id) + int sysnum = SYSTRACE_SYSNUM(parg); + uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread()); + uint64_t *uargs = NULL; + + if (!uthread) { + desc->dtargd_ndx = DTRACE_ARGNONE; + return; + } + + uargs = uthread->t_dtrace_syscall_args; + + if (SYSTRACE_ISENTRY((uintptr_t)parg)) { + systrace_entry_setargdesc(sysnum, desc->dtargd_ndx, + desc->dtargd_native, sizeof(desc->dtargd_native)); + } + else { + systrace_return_setargdesc(sysnum, desc->dtargd_ndx, + desc->dtargd_native, sizeof(desc->dtargd_native)); + } + + if (desc->dtargd_native[0] == '\0') + desc->dtargd_ndx = DTRACE_ARGNONE; +} static uint64_t machtrace_getarg(void *arg, dtrace_id_t id, void *parg, int argno, int aframes) diff --git a/bsd/dev/dtrace/systrace.h b/bsd/dev/dtrace/systrace.h index e9af96363..c86f324a9 100644 --- a/bsd/dev/dtrace/systrace.h +++ b/bsd/dev/dtrace/systrace.h @@ -62,11 +62,6 @@ extern int32_t dtrace_systrace_syscall(struct proc *, void *, int *); extern void dtrace_systrace_syscall_return(unsigned short, int, int *); -#ifdef _SYSCALL32_IMPL -extern int64_t dtrace_systrace_syscall32(uintptr_t arg0, uintptr_t arg1, - uintptr_t arg2, uintptr_t arg3, uintptr_t arg4, uintptr_t arg5); -#endif - #endif /* _KERNEL */ #ifdef __cplusplus diff --git a/bsd/dev/i386/conf.c b/bsd/dev/i386/conf.c index f76413818..36de4f945 100644 --- a/bsd/dev/i386/conf.c +++ b/bsd/dev/i386/conf.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997-2012 Apple Computer, Inc. All rights reserved. + * Copyright (c) 1997-2017 Apple Computer, Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -165,11 +165,6 @@ extern d_close_t logclose; extern d_read_t logread; extern d_ioctl_t logioctl; extern d_select_t logselect; -extern d_open_t fdesc_open; -extern d_read_t fdesc_read; -extern d_write_t fdesc_write; -extern d_ioctl_t fdesc_ioctl; -extern d_select_t fdesc_select; extern d_open_t oslog_streamopen; extern d_close_t oslog_streamclose; @@ -191,118 +186,64 @@ extern d_ioctl_t oslogioctl; #define nullstop (d_stop_t *)&nulldev #define nullreset (d_reset_t *)&nulldev -struct cdevsw cdevsw[] = -{ +struct cdevsw cdevsw[] = { /* - * For character devices, every other block of 16 slots is - * reserved for Apple. The other slots are available for - * the user. This way we can both add new entries without - * running into each other. Be sure to fill in Apple's - * 16 reserved slots when you jump over us -- we'll do the - * same for you. + * To add character devices to this table dynamically, use cdevsw_add. */ - /* 0 - 15 are reserved for Apple */ - - { - cnopen, cnclose, cnread, cnwrite, /* 0*/ - cnioctl, nullstop, nullreset, 0, cnselect, - eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY - }, - NO_CDEVICE, /* 1*/ - { - cttyopen, nullclose, cttyread, cttywrite, /* 2*/ - cttyioctl, nullstop, nullreset, 0, cttyselect, - eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY - }, - { - nullopen, nullclose, mmread, mmwrite, /* 3*/ - mmioctl, nullstop, nullreset, 0, mmselect, - mmmmap, eno_strat, eno_getc, eno_putc, D_DISK - }, - { - ptsopen, ptsclose, ptsread, ptswrite, /* 4*/ - ptyioctl, ptsstop, nullreset, 0, ptsselect, - eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY - }, - { - ptcopen, ptcclose, ptcread, ptcwrite, /* 5*/ - ptyioctl, nullstop, nullreset, 0, ptcselect, - eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY - }, - { - logopen, logclose, logread, eno_rdwrt, /* 6*/ - logioctl, eno_stop, nullreset, 0, logselect, - eno_mmap, eno_strat, eno_getc, eno_putc, 0 - }, - { - oslogopen, oslogclose, eno_rdwrt, eno_rdwrt, /* 7*/ - oslogioctl, eno_stop, nullreset, 0, oslogselect, - eno_mmap, eno_strat, eno_getc, eno_putc, 0 - }, - { - oslog_streamopen, oslog_streamclose, oslog_streamread, eno_rdwrt, /* 8*/ - oslog_streamioctl, eno_stop, nullreset, 0, oslog_streamselect, - eno_mmap, eno_strat, eno_getc, eno_putc, 0 - }, - NO_CDEVICE, /* 9*/ - NO_CDEVICE, /*10*/ - NO_CDEVICE, /*11*/ - { - kmopen, kmclose, kmread, kmwrite, /*12*/ - kmioctl, nullstop, nullreset, km_tty, ttselect, - eno_mmap, eno_strat, eno_getc, eno_putc, 0 - }, - NO_CDEVICE, /*13*/ - NO_CDEVICE, /*14*/ - NO_CDEVICE, /*15*/ - - /* 16 - 31 are reserved to the user */ - NO_CDEVICE, /*16*/ - NO_CDEVICE, /*17*/ - NO_CDEVICE, /*18*/ - NO_CDEVICE, /*19*/ - NO_CDEVICE, /*20*/ - NO_CDEVICE, /*21*/ - NO_CDEVICE, /*22*/ - NO_CDEVICE, /*23*/ - NO_CDEVICE, /*24*/ - NO_CDEVICE, /*25*/ - NO_CDEVICE, /*26*/ - NO_CDEVICE, /*27*/ - NO_CDEVICE, /*28*/ - NO_CDEVICE, /*29*/ - NO_CDEVICE, /*30*/ - NO_CDEVICE, /*31*/ - - /* 32 - 47 are reserved to NeXT */ - { - fdesc_open, eno_opcl, fdesc_read, fdesc_write, /*32*/ - fdesc_ioctl, eno_stop, eno_reset, 0, fdesc_select, - eno_mmap, eno_strat, eno_getc, eno_putc, 0 - }, -#if 1 - NO_CDEVICE, -#else - { - sgopen, sgclose, eno_rdwrt, eno_rdwrt, /*33*/ - sgioctl, eno_stop, eno_reset, 0, eno_select, - eno_mmap, eno_strat, eno_getc, eno_putc, D_TAPE - }, -#endif - NO_CDEVICE, /*34*/ - NO_CDEVICE, /*35*/ - NO_CDEVICE, /*36*/ - NO_CDEVICE, /*37*/ - NO_CDEVICE, /*38*/ - NO_CDEVICE, /*39*/ - NO_CDEVICE, /*40*/ - NO_CDEVICE, /*41*/ - { - volopen, volclose, eno_rdwrt, eno_rdwrt, /*42*/ - volioctl, eno_stop, eno_reset, 0, (select_fcn_t *)seltrue, - eno_mmap, eno_strat, eno_getc, eno_putc, 0 - }, + [0] = { + cnopen, cnclose, cnread, cnwrite, + cnioctl, nullstop, nullreset, 0, cnselect, + eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY + }, + [1] = NO_CDEVICE, + [2] = { + cttyopen, nullclose, cttyread, cttywrite, + cttyioctl, nullstop, nullreset, 0, cttyselect, + eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY + }, + [3] = { + nullopen, nullclose, mmread, mmwrite, + mmioctl, nullstop, nullreset, 0, mmselect, + mmmmap, eno_strat, eno_getc, eno_putc, D_DISK + }, + [PTC_MAJOR] = { + ptsopen, ptsclose, ptsread, ptswrite, + ptyioctl, ptsstop, nullreset, 0, ptsselect, + eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY + }, + [PTS_MAJOR] = { + ptcopen, ptcclose, ptcread, ptcwrite, + ptyioctl, nullstop, nullreset, 0, ptcselect, + eno_mmap, eno_strat, eno_getc, eno_putc, D_TTY + }, + [6] = { + logopen, logclose, logread, eno_rdwrt, + logioctl, eno_stop, nullreset, 0, logselect, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + }, + [7] = { + oslogopen, oslogclose, eno_rdwrt, eno_rdwrt, + oslogioctl, eno_stop, nullreset, 0, oslogselect, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + }, + [8] = { + oslog_streamopen, oslog_streamclose, oslog_streamread, eno_rdwrt, + oslog_streamioctl, eno_stop, nullreset, 0, oslog_streamselect, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + }, + [9 ... 11] = NO_CDEVICE, + [12] = { + kmopen, kmclose, kmread, kmwrite, + kmioctl, nullstop, nullreset, km_tty, ttselect, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + }, + [13 ... 41] = NO_CDEVICE, + [42] = { + volopen, volclose, eno_rdwrt, eno_rdwrt, + volioctl, eno_stop, eno_reset, 0, (select_fcn_t *) seltrue, + eno_mmap, eno_strat, eno_getc, eno_putc, 0 + } }; const int nchrdev = sizeof(cdevsw) / sizeof(cdevsw[0]); diff --git a/bsd/dev/i386/dis_tables.c b/bsd/dev/i386/dis_tables.c index 03f3c6197..c67273b79 100644 --- a/bsd/dev/i386/dis_tables.c +++ b/bsd/dev/i386/dis_tables.c @@ -23,8 +23,6 @@ /* * Copyright (c) 2015, Joyent, Inc. * Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. - * - * Use is subject to license terms. */ /* @@ -35,9 +33,15 @@ /* Copyright (c) 1988 AT&T */ /* All Rights Reserved */ +/* + * APPLE NOTE: There is a copy of this file in userspace in + * dtrace:/disassembler/dis_tables.c + * + * It needs to be in sync with this file. + */ /* - * #pragma ident "@(#)dis_tables.c 1.18 08/05/24 SMI" + * #pragma ident "@(#)dis_tables.c 1.18 08/05/24 SMI" */ #include <sys/dtrace.h> #include <sys/dtrace_glue.h> @@ -68,6 +72,7 @@ #ifdef DIS_TEXT extern char *strncpy(char *, const char *, size_t); extern size_t strlen(const char *); +extern int strcmp(const char *, const char *); extern int strncmp(const char *, const char *, size_t); extern size_t strlcat(char *, const char *, size_t); #endif @@ -91,6 +96,8 @@ typedef struct instable { uint_t it_always64:1; /* 64 bit when in 64 bit mode */ uint_t it_invalid32:1; /* invalid in IA32 */ uint_t it_stackop:1; /* push/pop stack operation */ + uint_t it_vexwoxmm:1; /* VEX instructions that don't use XMM/YMM */ + uint_t it_avxsuf:1; /* AVX suffix required */ } instable_t; /* @@ -109,16 +116,18 @@ enum { Mv, Mw, M, /* register or memory */ + MG9, /* register or memory in group 9 (prefix optional) */ Mb, /* register or memory, always byte sized */ MO, /* memory only (no registers) */ PREF, - SWAPGS, + SWAPGS_RDTSCP, MONITOR_MWAIT, R, RA, SEG, MR, RM, + RM_66r, /* RM, but with a required 0x66 prefix */ IA, MA, SD, @@ -221,6 +230,7 @@ enum { VEX_NONE, /* VEX no operand */ VEX_MO, /* VEX mod_rm -> implicit reg */ VEX_RMrX, /* VEX VEX.vvvv, mod_rm -> mod_reg */ + VEX_VRMrX, /* VEX mod_rm, VEX.vvvv -> mod_rm */ VEX_RRX, /* VEX VEX.vvvv, mod_reg -> mod_rm */ VEX_RMRX, /* VEX VEX.vvvv, mod_rm, imm8[7:4] -> mod_reg */ VEX_MX, /* VEX mod_rm -> mod_reg */ @@ -232,9 +242,16 @@ enum { VEX_RR, /* VEX mod_rm -> mod_reg */ VEX_RRi, /* VEX mod_rm, imm8 -> mod_reg */ VEX_RM, /* VEX mod_reg -> mod_rm */ - VEX_RIM, /* VEX mod_reg, imm8 -> mod_rm */ + VEX_RIM, /* VEX mod_reg, imm8 -> mod_rm */ VEX_RRM, /* VEX VEX.vvvv, mod_reg -> mod_rm */ VEX_RMX, /* VEX VEX.vvvv, mod_rm -> mod_reg */ + VEX_SbVM, /* VEX SIB, VEX.vvvv -> mod_rm */ + VMx, /* vmcall/vmlaunch/vmresume/vmxoff */ + VMxo, /* VMx instruction with optional prefix */ + SVM, /* AMD SVM instructions */ + BLS, /* BLSR, BLSMSK, BLSI */ + FMA, /* FMA instructions, all VEX_RMrX */ + ADX /* ADX instructions, support REX.w, mod_rm->mod_reg */ }; /* @@ -272,32 +289,36 @@ enum { * IND - indirect to another to another table * "T" - means to Terminate indirections (this is the final opcode) * "S" - means "operand length suffix required" + * "Sa" - means AVX2 suffix (d/q) required * "NS" - means "no suffix" which is the operand length suffix of the opcode * "Z" - means instruction size arg required * "u" - means the opcode is invalid in IA32 but valid in amd64 * "x" - means the opcode is invalid in amd64, but not IA32 * "y" - means the operand size is always 64 bits in 64 bit mode * "p" - means push/pop stack operation + * "vr" - means VEX instruction that operates on normal registers, not fpu */ #if defined(DIS_TEXT) && defined(DIS_MEM) -#define IND(table) {(instable_t *)table, 0, "", 0, 0, 0, 0, 0, 0} -#define INDx(table) {(instable_t *)table, 0, "", 0, 0, 1, 0, 0, 0} -#define TNS(name, amode) {TERM, amode, name, 0, 0, 0, 0, 0, 0} -#define TNSu(name, amode) {TERM, amode, name, 0, 0, 0, 0, 1, 0} -#define TNSx(name, amode) {TERM, amode, name, 0, 0, 1, 0, 0, 0} -#define TNSy(name, amode) {TERM, amode, name, 0, 0, 0, 1, 0, 0} -#define TNSyp(name, amode) {TERM, amode, name, 0, 0, 0, 1, 0, 1} -#define TNSZ(name, amode, sz) {TERM, amode, name, 0, sz, 0, 0, 0, 0} -#define TNSZy(name, amode, sz) {TERM, amode, name, 0, sz, 0, 1, 0, 0} -#define TS(name, amode) {TERM, amode, name, 1, 0, 0, 0, 0, 0} -#define TSx(name, amode) {TERM, amode, name, 1, 0, 1, 0, 0, 0} -#define TSy(name, amode) {TERM, amode, name, 1, 0, 0, 1, 0, 0} -#define TSp(name, amode) {TERM, amode, name, 1, 0, 0, 0, 0, 1} -#define TSZ(name, amode, sz) {TERM, amode, name, 1, sz, 0, 0, 0, 0} -#define TSZx(name, amode, sz) {TERM, amode, name, 1, sz, 1, 0, 0, 0} -#define TSZy(name, amode, sz) {TERM, amode, name, 1, sz, 0, 1, 0, 0} -#define INVALID {TERM, UNKNOWN, "", 0, 0, 0, 0, 0} +#define IND(table) {(instable_t *)table, 0, "", 0, 0, 0, 0, 0, 0, 0, 0} +#define INDx(table) {(instable_t *)table, 0, "", 0, 0, 1, 0, 0, 0, 0, 0} +#define TNS(name, amode) {TERM, amode, name, 0, 0, 0, 0, 0, 0, 0, 0} +#define TNSu(name, amode) {TERM, amode, name, 0, 0, 0, 0, 1, 0, 0, 0} +#define TNSx(name, amode) {TERM, amode, name, 0, 0, 1, 0, 0, 0, 0, 0} +#define TNSy(name, amode) {TERM, amode, name, 0, 0, 0, 1, 0, 0, 0, 0} +#define TNSyp(name, amode) {TERM, amode, name, 0, 0, 0, 1, 0, 1, 0, 0} +#define TNSZ(name, amode, sz) {TERM, amode, name, 0, sz, 0, 0, 0, 0, 0, 0} +#define TNSZy(name, amode, sz) {TERM, amode, name, 0, sz, 0, 1, 0, 0, 0, 0} +#define TNSZvr(name, amode, sz) {TERM, amode, name, 0, sz, 0, 0, 0, 0, 1, 0} +#define TS(name, amode) {TERM, amode, name, 1, 0, 0, 0, 0, 0, 0, 0} +#define TSx(name, amode) {TERM, amode, name, 1, 0, 1, 0, 0, 0, 0, 0} +#define TSy(name, amode) {TERM, amode, name, 1, 0, 0, 1, 0, 0, 0, 0} +#define TSp(name, amode) {TERM, amode, name, 1, 0, 0, 0, 0, 1, 0, 0} +#define TSZ(name, amode, sz) {TERM, amode, name, 1, sz, 0, 0, 0, 0, 0, 0} +#define TSaZ(name, amode, sz) {TERM, amode, name, 1, sz, 0, 0, 0, 0, 0, 1} +#define TSZx(name, amode, sz) {TERM, amode, name, 1, sz, 1, 0, 0, 0, 0, 0} +#define TSZy(name, amode, sz) {TERM, amode, name, 1, sz, 0, 1, 0, 0, 0, 0} +#define INVALID {TERM, UNKNOWN, "", 0, 0, 0, 0, 0, 0, 0, 0} #elif defined(DIS_TEXT) #define IND(table) {(instable_t *)table, 0, "", 0, 0, 0, 0, 0} #define INDx(table) {(instable_t *)table, 0, "", 0, 1, 0, 0, 0} @@ -308,50 +329,56 @@ enum { #define TNSyp(name, amode) {TERM, amode, name, 0, 0, 1, 0, 1} #define TNSZ(name, amode, sz) {TERM, amode, name, 0, 0, 0, 0, 0} #define TNSZy(name, amode, sz) {TERM, amode, name, 0, 0, 1, 0, 0} +#define TNSZvr(name, amode, sz) {TERM, amode, name, 0, 0, 0, 0, 0, 1} #define TS(name, amode) {TERM, amode, name, 1, 0, 0, 0, 0} #define TSx(name, amode) {TERM, amode, name, 1, 1, 0, 0, 0} #define TSy(name, amode) {TERM, amode, name, 1, 0, 1, 0, 0} #define TSp(name, amode) {TERM, amode, name, 1, 0, 0, 0, 1} #define TSZ(name, amode, sz) {TERM, amode, name, 1, 0, 0, 0, 0} +#define TSaZ(name, amode, sz) {TERM, amode, name, 1, 0, 0, 0, 0, 0, 1} #define TSZx(name, amode, sz) {TERM, amode, name, 1, 1, 0, 0, 0} #define TSZy(name, amode, sz) {TERM, amode, name, 1, 0, 1, 0, 0} #define INVALID {TERM, UNKNOWN, "", 0, 0, 0, 0, 0} #elif defined(DIS_MEM) #define IND(table) {(instable_t *)table, 0, 0, 0, 0, 0, 0} #define INDx(table) {(instable_t *)table, 0, 0, 1, 0, 0, 0} -#define TNS(name, amode) {TERM, amode, 0, 0, 0, 0, 0} -#define TNSu(name, amode) {TERM, amode, 0, 0, 0, 1, 0} -#define TNSy(name, amode) {TERM, amode, 0, 0, 1, 0, 0} -#define TNSyp(name, amode) {TERM, amode, 0, 0, 1, 0, 1} -#define TNSx(name, amode) {TERM, amode, 0, 1, 0, 0, 0} -#define TNSZ(name, amode, sz) {TERM, amode, sz, 0, 0, 0, 0} -#define TNSZy(name, amode, sz) {TERM, amode, sz, 0, 1, 0, 0} -#define TS(name, amode) {TERM, amode, 0, 0, 0, 0, 0} -#define TSx(name, amode) {TERM, amode, 0, 1, 0, 0, 0} -#define TSy(name, amode) {TERM, amode, 0, 0, 1, 0, 0} -#define TSp(name, amode) {TERM, amode, 0, 0, 0, 0, 1} -#define TSZ(name, amode, sz) {TERM, amode, sz, 0, 0, 0, 0} -#define TSZx(name, amode, sz) {TERM, amode, sz, 1, 0, 0, 0} -#define TSZy(name, amode, sz) {TERM, amode, sz, 0, 1, 0, 0} -#define INVALID {TERM, UNKNOWN, 0, 0, 0, 0, 0} +#define TNS(name, amode) {TERM, amode, 0, 0, 0, 0, 0, 0, 0} +#define TNSu(name, amode) {TERM, amode, 0, 0, 0, 1, 0, 0, 0} +#define TNSy(name, amode) {TERM, amode, 0, 0, 1, 0, 0, 0, 0} +#define TNSyp(name, amode) {TERM, amode, 0, 0, 1, 0, 1, 0, 0} +#define TNSx(name, amode) {TERM, amode, 0, 1, 0, 0, 0, 0, 0} +#define TNSZ(name, amode, sz) {TERM, amode, sz, 0, 0, 0, 0, 0, 0} +#define TNSZy(name, amode, sz) {TERM, amode, sz, 0, 1, 0, 0, 0, 0} +#define TNSZvr(name, amode, sz) {TERM, amode, sz, 0, 0, 0, 0, 1, 0} +#define TS(name, amode) {TERM, amode, 0, 0, 0, 0, 0, 0, 0} +#define TSx(name, amode) {TERM, amode, 0, 1, 0, 0, 0, 0, 0} +#define TSy(name, amode) {TERM, amode, 0, 0, 1, 0, 0, 0, 0} +#define TSp(name, amode) {TERM, amode, 0, 0, 0, 0, 1, 0, 0} +#define TSZ(name, amode, sz) {TERM, amode, sz, 0, 0, 0, 0, 0, 0} +#define TSaZ(name, amode, sz) {TERM, amode, sz, 0, 0, 0, 0, 0, 1} +#define TSZx(name, amode, sz) {TERM, amode, sz, 1, 0, 0, 0, 0 ,0} +#define TSZy(name, amode, sz) {TERM, amode, sz, 0, 1, 0, 0, 0, 0} +#define INVALID {TERM, UNKNOWN, 0, 0, 0, 0, 0, 0, 0} #else -#define IND(table) {(instable_t *)table, 0, 0, 0, 0, 0} -#define INDx(table) {(instable_t *)table, 0, 1, 0, 0, 0} -#define TNS(name, amode) {TERM, amode, 0, 0, 0, 0} -#define TNSu(name, amode) {TERM, amode, 0, 0, 1, 0} -#define TNSy(name, amode) {TERM, amode, 0, 1, 0, 0} -#define TNSyp(name, amode) {TERM, amode, 0, 1, 0, 1} -#define TNSx(name, amode) {TERM, amode, 1, 0, 0, 0} -#define TNSZ(name, amode, sz) {TERM, amode, 0, 0, 0, 0} -#define TNSZy(name, amode, sz) {TERM, amode, 0, 1, 0, 0} -#define TS(name, amode) {TERM, amode, 0, 0, 0, 0} -#define TSx(name, amode) {TERM, amode, 1, 0, 0, 0} -#define TSy(name, amode) {TERM, amode, 0, 1, 0, 0} -#define TSp(name, amode) {TERM, amode, 0, 0, 0, 1} -#define TSZ(name, amode, sz) {TERM, amode, 0, 0, 0, 0} -#define TSZx(name, amode, sz) {TERM, amode, 1, 0, 0, 0} -#define TSZy(name, amode, sz) {TERM, amode, 0, 1, 0, 0} -#define INVALID {TERM, UNKNOWN, 0, 0, 0, 0} +#define IND(table) {(instable_t *)table, 0, 0, 0, 0, 0, 0, 0} +#define INDx(table) {(instable_t *)table, 0, 1, 0, 0, 0, 0, 0} +#define TNS(name, amode) {TERM, amode, 0, 0, 0, 0, 0, 0} +#define TNSu(name, amode) {TERM, amode, 0, 0, 1, 0, 0, 0} +#define TNSy(name, amode) {TERM, amode, 0, 1, 0, 0, 0, 0} +#define TNSyp(name, amode) {TERM, amode, 0, 1, 0, 1, 0, 0} +#define TNSx(name, amode) {TERM, amode, 1, 0, 0, 0, 0, 0} +#define TNSZ(name, amode, sz) {TERM, amode, 0, 0, 0, 0, 0, 0} +#define TNSZy(name, amode, sz) {TERM, amode, 0, 1, 0, 0, 0, 0} +#define TNSZvr(name, amode, sz) {TERM, amode, 0, 0, 0, 0, 1, 0} +#define TS(name, amode) {TERM, amode, 0, 0, 0, 0, 0, 0} +#define TSx(name, amode) {TERM, amode, 1, 0, 0, 0, 0, 0} +#define TSy(name, amode) {TERM, amode, 0, 1, 0, 0, 0, 0} +#define TSp(name, amode) {TERM, amode, 0, 0, 0, 1, 0, 0} +#define TSZ(name, amode, sz) {TERM, amode, 0, 0, 0, 0, 0, 0} +#define TSaZ(name, amode, sz) {TERM, amode, 0, 0, 0, 0, 0, 1} +#define TSZx(name, amode, sz) {TERM, amode, 1, 0, 0, 0, 0, 0} +#define TSZy(name, amode, sz) {TERM, amode, 0, 1, 0, 0, 0, 0} +#define INVALID {TERM, UNKNOWN, 0, 0, 0, 0, 0, 0} #endif #ifdef DIS_TEXT @@ -398,6 +425,12 @@ const char *const dis_addr64_mode12[16] = { */ const char *const dis_scale_factor[4] = { ")", ",2)", ",4)", ",8)" }; +/* + * decode for scale from VSIB byte, note that we always include the scale factor + * to match gas. + */ +const char *const dis_vscale_factor[4] = { ",1)", ",2)", ",4)", ",8)" }; + /* * register decoding for normal references to registers (ie. not addressing) */ @@ -477,9 +510,6 @@ const char *const dis_AVXvgrp7[3][8] = { #endif /* DIS_TEXT */ - - - /* * "decode table" for 64 bit mode MOVSXD instruction (opcode 0x63) */ @@ -505,8 +535,8 @@ const instable_t dis_op0F00[8] = { */ const instable_t dis_op0F01[8] = { -/* [0] */ TNSZ("sgdt",MO,6), TNSZ("sidt",MONITOR_MWAIT,6), TNSZ("lgdt",XGETBV_XSETBV,6), TNSZ("lidt",MO,6), -/* [4] */ TNSZ("smsw",M,2), INVALID, TNSZ("lmsw",M,2), TNS("invlpg",SWAPGS), +/* [0] */ TNSZ("sgdt",VMx,6), TNSZ("sidt",MONITOR_MWAIT,6), TNSZ("lgdt",XGETBV_XSETBV,6), TNSZ("lidt",SVM,6), +/* [4] */ TNSZ("smsw",M,2), INVALID, TNSZ("lmsw",M,2), TNS("invlpg",SWAPGS_RDTSCP), }; /* @@ -515,7 +545,7 @@ const instable_t dis_op0F01[8] = { const instable_t dis_op0F18[8] = { /* [0] */ TNS("prefetchnta",PREF),TNS("prefetcht0",PREF), TNS("prefetcht1",PREF), TNS("prefetcht2",PREF), -/* [4] */ TNSZ("xsave",M,512), TNS("lfence",XMMFENCE), TNS("mfence",XMMFENCE), TNS("sfence",XMMSFNC), +/* [4] */ INVALID, INVALID, INVALID, INVALID, }; /* @@ -523,7 +553,7 @@ const instable_t dis_op0F18[8] = { */ const instable_t dis_op0FAE[8] = { /* [0] */ TNSZ("fxsave",M,512), TNSZ("fxrstor",M,512), TNS("ldmxcsr",M), TNS("stmxcsr",M), -/* [4] */ INVALID, TNS("lfence",XMMFENCE), TNS("mfence",XMMFENCE), TNS("sfence",XMMSFNC), +/* [4] */ TNSZ("xsave",M,512), TNS("lfence",XMMFENCE), TNS("mfence",XMMFENCE), TNS("sfence",XMMSFNC), }; /* @@ -537,15 +567,44 @@ const instable_t dis_op0FBA[8] = { }; /* - * Decode table for 0x0FC7 opcode + * Decode table for 0x0FC7 opcode (group 9) */ const instable_t dis_op0FC7[8] = { /* [0] */ INVALID, TNS("cmpxchg8b",M), INVALID, INVALID, -/* [4] */ INVALID, INVALID, INVALID, INVALID, +/* [4] */ INVALID, INVALID, TNS("vmptrld",MG9), TNS("vmptrst",MG9), +}; + +/* + * Decode table for 0x0FC7 opcode (group 9) mode 3 + */ + +const instable_t dis_op0FC7m3[8] = { + +/* [0] */ INVALID, INVALID, INVALID, INVALID, +/* [4] */ INVALID, INVALID, TNS("rdrand",MG9), TNS("rdseed", MG9), }; +/* + * Decode table for 0x0FC7 opcode with 0x66 prefix + */ + +const instable_t dis_op660FC7[8] = { + +/* [0] */ INVALID, INVALID, INVALID, INVALID, +/* [4] */ INVALID, INVALID, TNS("vmclear",M), INVALID, +}; + +/* + * Decode table for 0x0FC7 opcode with 0xF3 prefix + */ + +const instable_t dis_opF30FC7[8] = { + +/* [0] */ INVALID, INVALID, INVALID, INVALID, +/* [4] */ INVALID, INVALID, TNS("vmxon",M), INVALID, +}; /* * Decode table for 0x0FC8 opcode -- 486 bswap instruction @@ -643,7 +702,7 @@ const instable_t dis_opSIMDdata16[256] = { /* [70] */ TNSZ("pshufd",XMMP,16), INVALID, INVALID, INVALID, /* [74] */ TNSZ("pcmpeqb",XMM,16), TNSZ("pcmpeqw",XMM,16), TNSZ("pcmpeqd",XMM,16), INVALID, /* [78] */ TNSZ("extrq",XMM2I,16), TNSZ("extrq",XMM,16), INVALID, INVALID, -/* [7C] */ INVALID, INVALID, TNSZ("movd",XMM3MXS,4), TNSZ("movdqa",XMMS,16), +/* [7C] */ TNSZ("haddpd",XMM,16), TNSZ("hsubpd",XMM,16), TNSZ("movd",XMM3MXS,4), TNSZ("movdqa",XMMS,16), /* [80] */ INVALID, INVALID, INVALID, INVALID, /* [84] */ INVALID, INVALID, INVALID, INVALID, @@ -670,7 +729,7 @@ const instable_t dis_opSIMDdata16[256] = { /* [C8] */ INVALID, INVALID, INVALID, INVALID, /* [CC] */ INVALID, INVALID, INVALID, INVALID, -/* [D0] */ INVALID, TNSZ("psrlw",XMM,16), TNSZ("psrld",XMM,16), TNSZ("psrlq",XMM,16), +/* [D0] */ TNSZ("addsubpd",XMM,16),TNSZ("psrlw",XMM,16), TNSZ("psrld",XMM,16), TNSZ("psrlq",XMM,16), /* [D4] */ TNSZ("paddq",XMM,16), TNSZ("pmullw",XMM,16), TNSZ("movq",XMMS,8), TNS("pmovmskb",XMMX3), /* [D8] */ TNSZ("psubusb",XMM,16), TNSZ("psubusw",XMM,16), TNSZ("pminub",XMM,16), TNSZ("pand",XMM,16), /* [DC] */ TNSZ("paddusb",XMM,16), TNSZ("paddusw",XMM,16), TNSZ("pmaxub",XMM,16), TNSZ("pandn",XMM,16), @@ -777,7 +836,7 @@ const instable_t dis_opSIMDrepnz[256] = { /* [08] */ INVALID, INVALID, INVALID, INVALID, /* [0C] */ INVALID, INVALID, INVALID, INVALID, -/* [10] */ TNSZ("movsd",XMM,8), TNSZ("movsd",XMMS,8), INVALID, INVALID, +/* [10] */ TNSZ("movsd",XMM,8), TNSZ("movsd",XMMS,8), TNSZ("movddup",XMM,8), INVALID, /* [14] */ INVALID, INVALID, INVALID, INVALID, /* [18] */ INVALID, INVALID, INVALID, INVALID, /* [1C] */ INVALID, INVALID, INVALID, INVALID, @@ -810,7 +869,7 @@ const instable_t dis_opSIMDrepnz[256] = { /* [70] */ TNSZ("pshuflw",XMMP,16),INVALID, INVALID, INVALID, /* [74] */ INVALID, INVALID, INVALID, INVALID, /* [78] */ TNSZ("insertq",XMMX2I,16),TNSZ("insertq",XMM,8),INVALID, INVALID, -/* [7C] */ INVALID, INVALID, INVALID, INVALID, +/* [7C] */ TNSZ("haddps",XMM,16), TNSZ("hsubps",XMM,16), INVALID, INVALID, /* [80] */ INVALID, INVALID, INVALID, INVALID, /* [84] */ INVALID, INVALID, INVALID, INVALID, @@ -837,7 +896,7 @@ const instable_t dis_opSIMDrepnz[256] = { /* [C8] */ INVALID, INVALID, INVALID, INVALID, /* [CC] */ INVALID, INVALID, INVALID, INVALID, -/* [D0] */ INVALID, INVALID, INVALID, INVALID, +/* [D0] */ TNSZ("addsubps",XMM,16),INVALID, INVALID, INVALID, /* [D4] */ INVALID, INVALID, TNS("movdq2q",XMMXM), INVALID, /* [D8] */ INVALID, INVALID, INVALID, INVALID, /* [DC] */ INVALID, INVALID, INVALID, INVALID, @@ -847,7 +906,7 @@ const instable_t dis_opSIMDrepnz[256] = { /* [E8] */ INVALID, INVALID, INVALID, INVALID, /* [EC] */ INVALID, INVALID, INVALID, INVALID, -/* [F0] */ INVALID, INVALID, INVALID, INVALID, +/* [F0] */ TNS("lddqu",XMMM), INVALID, INVALID, INVALID, /* [F4] */ INVALID, INVALID, INVALID, INVALID, /* [F8] */ INVALID, INVALID, INVALID, INVALID, /* [FC] */ INVALID, INVALID, INVALID, INVALID, @@ -935,6 +994,251 @@ const instable_t dis_opAVXF20F[256] = { /* [FC] */ INVALID, INVALID, INVALID, INVALID, }; +const instable_t dis_opAVXF20F3A[256] = { +/* [00] */ INVALID, INVALID, INVALID, INVALID, +/* [04] */ INVALID, INVALID, INVALID, INVALID, +/* [08] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [10] */ INVALID, INVALID, INVALID, INVALID, +/* [14] */ INVALID, INVALID, INVALID, INVALID, +/* [18] */ INVALID, INVALID, INVALID, INVALID, +/* [1C] */ INVALID, INVALID, INVALID, INVALID, + +/* [20] */ INVALID, INVALID, INVALID, INVALID, +/* [24] */ INVALID, INVALID, INVALID, INVALID, +/* [28] */ INVALID, INVALID, INVALID, INVALID, +/* [2C] */ INVALID, INVALID, INVALID, INVALID, + +/* [30] */ INVALID, INVALID, INVALID, INVALID, +/* [34] */ INVALID, INVALID, INVALID, INVALID, +/* [38] */ INVALID, INVALID, INVALID, INVALID, +/* [3C] */ INVALID, INVALID, INVALID, INVALID, + +/* [40] */ INVALID, INVALID, INVALID, INVALID, +/* [44] */ INVALID, INVALID, INVALID, INVALID, +/* [48] */ INVALID, INVALID, INVALID, INVALID, +/* [4C] */ INVALID, INVALID, INVALID, INVALID, + +/* [50] */ INVALID, INVALID, INVALID, INVALID, +/* [54] */ INVALID, INVALID, INVALID, INVALID, +/* [58] */ INVALID, INVALID, INVALID, INVALID, +/* [5C] */ INVALID, INVALID, INVALID, INVALID, + +/* [60] */ INVALID, INVALID, INVALID, INVALID, +/* [64] */ INVALID, INVALID, INVALID, INVALID, +/* [68] */ INVALID, INVALID, INVALID, INVALID, +/* [6C] */ INVALID, INVALID, INVALID, INVALID, + +/* [70] */ INVALID, INVALID, INVALID, INVALID, +/* [74] */ INVALID, INVALID, INVALID, INVALID, +/* [78] */ INVALID, INVALID, INVALID, INVALID, +/* [7C] */ INVALID, INVALID, INVALID, INVALID, + +/* [80] */ INVALID, INVALID, INVALID, INVALID, +/* [84] */ INVALID, INVALID, INVALID, INVALID, +/* [88] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [90] */ INVALID, INVALID, INVALID, INVALID, +/* [94] */ INVALID, INVALID, INVALID, INVALID, +/* [98] */ INVALID, INVALID, INVALID, INVALID, +/* [9C] */ INVALID, INVALID, INVALID, INVALID, + +/* [A0] */ INVALID, INVALID, INVALID, INVALID, +/* [A4] */ INVALID, INVALID, INVALID, INVALID, +/* [A8] */ INVALID, INVALID, INVALID, INVALID, +/* [AC] */ INVALID, INVALID, INVALID, INVALID, + +/* [B0] */ INVALID, INVALID, INVALID, INVALID, +/* [B4] */ INVALID, INVALID, INVALID, INVALID, +/* [B8] */ INVALID, INVALID, INVALID, INVALID, +/* [BC] */ INVALID, INVALID, INVALID, INVALID, + +/* [C0] */ INVALID, INVALID, INVALID, INVALID, +/* [C4] */ INVALID, INVALID, INVALID, INVALID, +/* [C8] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ INVALID, INVALID, INVALID, INVALID, + +/* [D0] */ INVALID, INVALID, INVALID, INVALID, +/* [D4] */ INVALID, INVALID, INVALID, INVALID, +/* [D8] */ INVALID, INVALID, INVALID, INVALID, +/* [DC] */ INVALID, INVALID, INVALID, INVALID, + +/* [E0] */ INVALID, INVALID, INVALID, INVALID, +/* [E4] */ INVALID, INVALID, INVALID, INVALID, +/* [E8] */ INVALID, INVALID, INVALID, INVALID, +/* [EC] */ INVALID, INVALID, INVALID, INVALID, + +/* [F0] */ TNSZvr("rorx",VEX_MXI,6),INVALID, INVALID, INVALID, +/* [F4] */ INVALID, INVALID, INVALID, INVALID, +/* [F8] */ INVALID, INVALID, INVALID, INVALID, +/* [FC] */ INVALID, INVALID, INVALID, INVALID, +}; + +const instable_t dis_opAVXF20F38[256] = { +/* [00] */ INVALID, INVALID, INVALID, INVALID, +/* [04] */ INVALID, INVALID, INVALID, INVALID, +/* [08] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [10] */ INVALID, INVALID, INVALID, INVALID, +/* [14] */ INVALID, INVALID, INVALID, INVALID, +/* [18] */ INVALID, INVALID, INVALID, INVALID, +/* [1C] */ INVALID, INVALID, INVALID, INVALID, + +/* [20] */ INVALID, INVALID, INVALID, INVALID, +/* [24] */ INVALID, INVALID, INVALID, INVALID, +/* [28] */ INVALID, INVALID, INVALID, INVALID, +/* [2C] */ INVALID, INVALID, INVALID, INVALID, + +/* [30] */ INVALID, INVALID, INVALID, INVALID, +/* [34] */ INVALID, INVALID, INVALID, INVALID, +/* [38] */ INVALID, INVALID, INVALID, INVALID, +/* [3C] */ INVALID, INVALID, INVALID, INVALID, + +/* [40] */ INVALID, INVALID, INVALID, INVALID, +/* [44] */ INVALID, INVALID, INVALID, INVALID, +/* [48] */ INVALID, INVALID, INVALID, INVALID, +/* [4C] */ INVALID, INVALID, INVALID, INVALID, + +/* [50] */ INVALID, INVALID, INVALID, INVALID, +/* [54] */ INVALID, INVALID, INVALID, INVALID, +/* [58] */ INVALID, INVALID, INVALID, INVALID, +/* [5C] */ INVALID, INVALID, INVALID, INVALID, + +/* [60] */ INVALID, INVALID, INVALID, INVALID, +/* [64] */ INVALID, INVALID, INVALID, INVALID, +/* [68] */ INVALID, INVALID, INVALID, INVALID, +/* [6C] */ INVALID, INVALID, INVALID, INVALID, + +/* [70] */ INVALID, INVALID, INVALID, INVALID, +/* [74] */ INVALID, INVALID, INVALID, INVALID, +/* [78] */ INVALID, INVALID, INVALID, INVALID, +/* [7C] */ INVALID, INVALID, INVALID, INVALID, + +/* [80] */ INVALID, INVALID, INVALID, INVALID, +/* [84] */ INVALID, INVALID, INVALID, INVALID, +/* [88] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [90] */ INVALID, INVALID, INVALID, INVALID, +/* [94] */ INVALID, INVALID, INVALID, INVALID, +/* [98] */ INVALID, INVALID, INVALID, INVALID, +/* [9C] */ INVALID, INVALID, INVALID, INVALID, + +/* [A0] */ INVALID, INVALID, INVALID, INVALID, +/* [A4] */ INVALID, INVALID, INVALID, INVALID, +/* [A8] */ INVALID, INVALID, INVALID, INVALID, +/* [AC] */ INVALID, INVALID, INVALID, INVALID, + +/* [B0] */ INVALID, INVALID, INVALID, INVALID, +/* [B4] */ INVALID, INVALID, INVALID, INVALID, +/* [B8] */ INVALID, INVALID, INVALID, INVALID, +/* [BC] */ INVALID, INVALID, INVALID, INVALID, + +/* [C0] */ INVALID, INVALID, INVALID, INVALID, +/* [C4] */ INVALID, INVALID, INVALID, INVALID, +/* [C8] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ INVALID, INVALID, INVALID, INVALID, + +/* [D0] */ INVALID, INVALID, INVALID, INVALID, +/* [D4] */ INVALID, INVALID, INVALID, INVALID, +/* [D8] */ INVALID, INVALID, INVALID, INVALID, +/* [DC] */ INVALID, INVALID, INVALID, INVALID, + +/* [E0] */ INVALID, INVALID, INVALID, INVALID, +/* [E4] */ INVALID, INVALID, INVALID, INVALID, +/* [E8] */ INVALID, INVALID, INVALID, INVALID, +/* [EC] */ INVALID, INVALID, INVALID, INVALID, + +/* [F0] */ INVALID, INVALID, INVALID, INVALID, +/* [F4] */ INVALID, TNSZvr("pdep",VEX_RMrX,5),TNSZvr("mulx",VEX_RMrX,5),TNSZvr("shrx",VEX_VRMrX,5), +/* [F8] */ INVALID, INVALID, INVALID, INVALID, +/* [FC] */ INVALID, INVALID, INVALID, INVALID, +}; + +const instable_t dis_opAVXF30F38[256] = { +/* [00] */ INVALID, INVALID, INVALID, INVALID, +/* [04] */ INVALID, INVALID, INVALID, INVALID, +/* [08] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [10] */ INVALID, INVALID, INVALID, INVALID, +/* [14] */ INVALID, INVALID, INVALID, INVALID, +/* [18] */ INVALID, INVALID, INVALID, INVALID, +/* [1C] */ INVALID, INVALID, INVALID, INVALID, + +/* [20] */ INVALID, INVALID, INVALID, INVALID, +/* [24] */ INVALID, INVALID, INVALID, INVALID, +/* [28] */ INVALID, INVALID, INVALID, INVALID, +/* [2C] */ INVALID, INVALID, INVALID, INVALID, + +/* [30] */ INVALID, INVALID, INVALID, INVALID, +/* [34] */ INVALID, INVALID, INVALID, INVALID, +/* [38] */ INVALID, INVALID, INVALID, INVALID, +/* [3C] */ INVALID, INVALID, INVALID, INVALID, + +/* [40] */ INVALID, INVALID, INVALID, INVALID, +/* [44] */ INVALID, INVALID, INVALID, INVALID, +/* [48] */ INVALID, INVALID, INVALID, INVALID, +/* [4C] */ INVALID, INVALID, INVALID, INVALID, + +/* [50] */ INVALID, INVALID, INVALID, INVALID, +/* [54] */ INVALID, INVALID, INVALID, INVALID, +/* [58] */ INVALID, INVALID, INVALID, INVALID, +/* [5C] */ INVALID, INVALID, INVALID, INVALID, + +/* [60] */ INVALID, INVALID, INVALID, INVALID, +/* [64] */ INVALID, INVALID, INVALID, INVALID, +/* [68] */ INVALID, INVALID, INVALID, INVALID, +/* [6C] */ INVALID, INVALID, INVALID, INVALID, + +/* [70] */ INVALID, INVALID, INVALID, INVALID, +/* [74] */ INVALID, INVALID, INVALID, INVALID, +/* [78] */ INVALID, INVALID, INVALID, INVALID, +/* [7C] */ INVALID, INVALID, INVALID, INVALID, + +/* [80] */ INVALID, INVALID, INVALID, INVALID, +/* [84] */ INVALID, INVALID, INVALID, INVALID, +/* [88] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, INVALID, INVALID, INVALID, + +/* [90] */ INVALID, INVALID, INVALID, INVALID, +/* [94] */ INVALID, INVALID, INVALID, INVALID, +/* [98] */ INVALID, INVALID, INVALID, INVALID, +/* [9C] */ INVALID, INVALID, INVALID, INVALID, + +/* [A0] */ INVALID, INVALID, INVALID, INVALID, +/* [A4] */ INVALID, INVALID, INVALID, INVALID, +/* [A8] */ INVALID, INVALID, INVALID, INVALID, +/* [AC] */ INVALID, INVALID, INVALID, INVALID, + +/* [B0] */ INVALID, INVALID, INVALID, INVALID, +/* [B4] */ INVALID, INVALID, INVALID, INVALID, +/* [B8] */ INVALID, INVALID, INVALID, INVALID, +/* [BC] */ INVALID, INVALID, INVALID, INVALID, + +/* [C0] */ INVALID, INVALID, INVALID, INVALID, +/* [C4] */ INVALID, INVALID, INVALID, INVALID, +/* [C8] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ INVALID, INVALID, INVALID, INVALID, + +/* [D0] */ INVALID, INVALID, INVALID, INVALID, +/* [D4] */ INVALID, INVALID, INVALID, INVALID, +/* [D8] */ INVALID, INVALID, INVALID, INVALID, +/* [DC] */ INVALID, INVALID, INVALID, INVALID, + +/* [E0] */ INVALID, INVALID, INVALID, INVALID, +/* [E4] */ INVALID, INVALID, INVALID, INVALID, +/* [E8] */ INVALID, INVALID, INVALID, INVALID, +/* [EC] */ INVALID, INVALID, INVALID, INVALID, + +/* [F0] */ INVALID, INVALID, INVALID, INVALID, +/* [F4] */ INVALID, TNSZvr("pext",VEX_RMrX,5),INVALID, TNSZvr("sarx",VEX_VRMrX,5), +/* [F8] */ INVALID, INVALID, INVALID, INVALID, +/* [FC] */ INVALID, INVALID, INVALID, INVALID, +}; /* * Decode table for SIMD instructions with the repz (0xf3) prefix. */ @@ -944,8 +1248,8 @@ const instable_t dis_opSIMDrepz[256] = { /* [08] */ INVALID, INVALID, INVALID, INVALID, /* [0C] */ INVALID, INVALID, INVALID, INVALID, -/* [10] */ TNSZ("movss",XMM,4), TNSZ("movss",XMMS,4), INVALID, INVALID, -/* [14] */ INVALID, INVALID, INVALID, INVALID, +/* [10] */ TNSZ("movss",XMM,4), TNSZ("movss",XMMS,4), TNSZ("movsldup",XMM,16),INVALID, +/* [14] */ INVALID, INVALID, TNSZ("movshdup",XMM,16),INVALID, /* [18] */ INVALID, INVALID, INVALID, INVALID, /* [1C] */ INVALID, INVALID, INVALID, INVALID, @@ -997,7 +1301,7 @@ const instable_t dis_opSIMDrepz[256] = { /* [B0] */ INVALID, INVALID, INVALID, INVALID, /* [B4] */ INVALID, INVALID, INVALID, INVALID, /* [B8] */ TS("popcnt",MRw), INVALID, INVALID, INVALID, -/* [BC] */ INVALID, TS("lzcnt",MRw), INVALID, INVALID, +/* [BC] */ TNSZ("tzcnt",MRw,5), TS("lzcnt",MRw), INVALID, INVALID, /* [C0] */ INVALID, INVALID, TNSZ("cmpss",XMMP,4), INVALID, /* [C4] */ INVALID, INVALID, INVALID, INVALID, @@ -1101,7 +1405,6 @@ const instable_t dis_opAVXF30F[256] = { /* [F8] */ INVALID, INVALID, INVALID, INVALID, /* [FC] */ INVALID, INVALID, INVALID, INVALID, }; - /* * The following two tables are used to encode crc32 and movbe * since they share the same opcodes. @@ -1116,6 +1419,14 @@ const instable_t dis_op0F38F1[2] = { TS("movbe",MOVBE), }; +/* + * The following table is used to distinguish between adox and adcx which share + * the same opcodes. + */ +const instable_t dis_op0F38F6[2] = { +/* [00] */ TNS("adcx",ADX), + TNS("adox",ADX), +}; const instable_t dis_op0F38[256] = { /* [00] */ TNSZ("pshufb",XMM_66o,16),TNSZ("phaddw",XMM_66o,16),TNSZ("phaddd",XMM_66o,16),TNSZ("phaddsw",XMM_66o,16), @@ -1158,7 +1469,7 @@ const instable_t dis_op0F38[256] = { /* [78] */ INVALID, INVALID, INVALID, INVALID, /* [7C] */ INVALID, INVALID, INVALID, INVALID, -/* [80] */ INVALID, INVALID, INVALID, INVALID, +/* [80] */ TNSy("invept", RM_66r), TNSy("invvpid", RM_66r),TNSy("invpcid", RM_66r),INVALID, /* [84] */ INVALID, INVALID, INVALID, INVALID, /* [88] */ INVALID, INVALID, INVALID, INVALID, /* [8C] */ INVALID, INVALID, INVALID, INVALID, @@ -1180,21 +1491,20 @@ const instable_t dis_op0F38[256] = { /* [C0] */ INVALID, INVALID, INVALID, INVALID, /* [C4] */ INVALID, INVALID, INVALID, INVALID, -/* [C8] */ INVALID, INVALID, INVALID, INVALID, -/* [CC] */ INVALID, INVALID, INVALID, INVALID, +/* [C8] */ TNSZ("sha1nexte",XMM,16),TNSZ("sha1msg1",XMM,16),TNSZ("sha1msg2",XMM,16),TNSZ("sha256rnds2",XMM,16), +/* [CC] */ TNSZ("sha256msg1",XMM,16),TNSZ("sha256msg2",XMM,16),INVALID, INVALID, /* [D0] */ INVALID, INVALID, INVALID, INVALID, /* [D4] */ INVALID, INVALID, INVALID, INVALID, -/* [D8] */ INVALID, INVALID, INVALID, INVALID, -/* [DC] */ INVALID, INVALID, INVALID, INVALID, +/* [D8] */ INVALID, INVALID, INVALID, TNSZ("aesimc",XMM_66r,16), +/* [DC] */ TNSZ("aesenc",XMM_66r,16),TNSZ("aesenclast",XMM_66r,16),TNSZ("aesdec",XMM_66r,16),TNSZ("aesdeclast",XMM_66r,16), /* [E0] */ INVALID, INVALID, INVALID, INVALID, /* [E4] */ INVALID, INVALID, INVALID, INVALID, /* [E8] */ INVALID, INVALID, INVALID, INVALID, /* [EC] */ INVALID, INVALID, INVALID, INVALID, - -/* [F0] */ TNS("crc32b",CRC32), TS("crc32",CRC32), INVALID, INVALID, -/* [F4] */ INVALID, INVALID, INVALID, INVALID, +/* [F0] */ IND(dis_op0F38F0), IND(dis_op0F38F1), INVALID, INVALID, +/* [F4] */ INVALID, INVALID, IND(dis_op0F38F6), INVALID, /* [F8] */ INVALID, INVALID, INVALID, INVALID, /* [FC] */ INVALID, INVALID, INVALID, INVALID, }; @@ -1205,8 +1515,8 @@ const instable_t dis_opAVX660F38[256] = { /* [08] */ TNSZ("vpsignb",VEX_RMrX,16),TNSZ("vpsignw",VEX_RMrX,16),TNSZ("vpsignd",VEX_RMrX,16),TNSZ("vpmulhrsw",VEX_RMrX,16), /* [0C] */ TNSZ("vpermilps",VEX_RMrX,8),TNSZ("vpermilpd",VEX_RMrX,16),TNSZ("vtestps",VEX_RRI,8), TNSZ("vtestpd",VEX_RRI,16), -/* [10] */ INVALID, INVALID, INVALID, INVALID, -/* [14] */ INVALID, INVALID, INVALID, TNSZ("vptest",VEX_RRI,16), +/* [10] */ INVALID, INVALID, INVALID, TNSZ("vcvtph2ps",VEX_MX,16), +/* [14] */ INVALID, INVALID, TNSZ("vpermps",VEX_RMrX,16),TNSZ("vptest",VEX_RRI,16), /* [18] */ TNSZ("vbroadcastss",VEX_MX,4),TNSZ("vbroadcastsd",VEX_MX,8),TNSZ("vbroadcastf128",VEX_MX,16),INVALID, /* [1C] */ TNSZ("vpabsb",VEX_MX,16),TNSZ("vpabsw",VEX_MX,16),TNSZ("vpabsd",VEX_MX,16),INVALID, @@ -1216,12 +1526,12 @@ const instable_t dis_opAVX660F38[256] = { /* [2C] */ TNSZ("vmaskmovps",VEX_RMrX,8),TNSZ("vmaskmovpd",VEX_RMrX,16),TNSZ("vmaskmovps",VEX_RRM,8),TNSZ("vmaskmovpd",VEX_RRM,16), /* [30] */ TNSZ("vpmovzxbw",VEX_MX,16),TNSZ("vpmovzxbd",VEX_MX,16),TNSZ("vpmovzxbq",VEX_MX,16),TNSZ("vpmovzxwd",VEX_MX,16), -/* [34] */ TNSZ("vpmovzxwq",VEX_MX,16),TNSZ("vpmovzxdq",VEX_MX,16),TNSZ("vpermd",VEX_RMrX,16),TNSZ("vpcmpgtq",VEX_RMrX,16), +/* [34] */ TNSZ("vpmovzxwq",VEX_MX,16),TNSZ("vpmovzxdq",VEX_MX,16),TNSZ("vpermd",VEX_RMrX,16),TNSZ("vpcmpgtq",VEX_RMrX,16), /* [38] */ TNSZ("vpminsb",VEX_RMrX,16),TNSZ("vpminsd",VEX_RMrX,16),TNSZ("vpminuw",VEX_RMrX,16),TNSZ("vpminud",VEX_RMrX,16), /* [3C] */ TNSZ("vpmaxsb",VEX_RMrX,16),TNSZ("vpmaxsd",VEX_RMrX,16),TNSZ("vpmaxuw",VEX_RMrX,16),TNSZ("vpmaxud",VEX_RMrX,16), /* [40] */ TNSZ("vpmulld",VEX_RMrX,16),TNSZ("vphminposuw",VEX_MX,16),INVALID, INVALID, -/* [44] */ INVALID, INVALID, INVALID, INVALID, +/* [44] */ INVALID, TSaZ("vpsrlv",VEX_RMrX,16),TNSZ("vpsravd",VEX_RMrX,16),TSaZ("vpsllv",VEX_RMrX,16), /* [48] */ INVALID, INVALID, INVALID, INVALID, /* [4C] */ INVALID, INVALID, INVALID, INVALID, @@ -1242,23 +1552,23 @@ const instable_t dis_opAVX660F38[256] = { /* [80] */ INVALID, INVALID, INVALID, INVALID, /* [84] */ INVALID, INVALID, INVALID, INVALID, -/* [88] */ INVALID, INVALID, INVALID, INVALID, -/* [8C] */ INVALID, INVALID, INVALID, INVALID, - -/* [90] */ INVALID, INVALID, INVALID, INVALID, -/* [94] */ INVALID, INVALID, INVALID, INVALID, -/* [98] */ INVALID, INVALID, INVALID, INVALID, -/* [9C] */ INVALID, INVALID, INVALID, INVALID, +/* [88] */ INVALID, INVALID, INVALID, INVALID, +/* [8C] */ TSaZ("vpmaskmov",VEX_RMrX,16),INVALID, TSaZ("vpmaskmov",VEX_RRM,16),INVALID, + +/* [90] */ TNSZ("vpgatherd",VEX_SbVM,16),TNSZ("vpgatherq",VEX_SbVM,16),TNSZ("vgatherdp",VEX_SbVM,16),TNSZ("vgatherqp",VEX_SbVM,16), +/* [94] */ INVALID, INVALID, TNSZ("vfmaddsub132p",FMA,16),TNSZ("vfmsubadd132p",FMA,16), +/* [98] */ TNSZ("vfmadd132p",FMA,16),TNSZ("vfmadd132s",FMA,16),TNSZ("vfmsub132p",FMA,16),TNSZ("vfmsub132s",FMA,16), +/* [9C] */ TNSZ("vfnmadd132p",FMA,16),TNSZ("vfnmadd132s",FMA,16),TNSZ("vfnmsub132p",FMA,16),TNSZ("vfnmsub132s",FMA,16), /* [A0] */ INVALID, INVALID, INVALID, INVALID, -/* [A4] */ INVALID, INVALID, INVALID, INVALID, -/* [A8] */ INVALID, INVALID, INVALID, INVALID, -/* [AC] */ INVALID, INVALID, INVALID, INVALID, +/* [A4] */ INVALID, INVALID, TNSZ("vfmaddsub213p",FMA,16),TNSZ("vfmsubadd213p",FMA,16), +/* [A8] */ TNSZ("vfmadd213p",FMA,16),TNSZ("vfmadd213s",FMA,16),TNSZ("vfmsub213p",FMA,16),TNSZ("vfmsub213s",FMA,16), +/* [AC] */ TNSZ("vfnmadd213p",FMA,16),TNSZ("vfnmadd213s",FMA,16),TNSZ("vfnmsub213p",FMA,16),TNSZ("vfnmsub213s",FMA,16), /* [B0] */ INVALID, INVALID, INVALID, INVALID, -/* [B4] */ INVALID, INVALID, INVALID, INVALID, -/* [B8] */ INVALID, INVALID, INVALID, INVALID, -/* [BC] */ INVALID, INVALID, INVALID, INVALID, +/* [B4] */ INVALID, INVALID, TNSZ("vfmaddsub231p",FMA,16),TNSZ("vfmsubadd231p",FMA,16), +/* [B8] */ TNSZ("vfmadd231p",FMA,16),TNSZ("vfmadd231s",FMA,16),TNSZ("vfmsub231p",FMA,16),TNSZ("vfmsub231s",FMA,16), +/* [BC] */ TNSZ("vfnmadd231p",FMA,16),TNSZ("vfnmadd231s",FMA,16),TNSZ("vfnmsub231p",FMA,16),TNSZ("vfnmsub231s",FMA,16), /* [C0] */ INVALID, INVALID, INVALID, INVALID, /* [C4] */ INVALID, INVALID, INVALID, INVALID, @@ -1275,7 +1585,7 @@ const instable_t dis_opAVX660F38[256] = { /* [E8] */ INVALID, INVALID, INVALID, INVALID, /* [EC] */ INVALID, INVALID, INVALID, INVALID, /* [F0] */ IND(dis_op0F38F0), IND(dis_op0F38F1), INVALID, INVALID, -/* [F4] */ INVALID, INVALID, INVALID, INVALID, +/* [F4] */ INVALID, INVALID, INVALID, TNSZvr("shlx",VEX_VRMrX,5), /* [F8] */ INVALID, INVALID, INVALID, INVALID, /* [FC] */ INVALID, INVALID, INVALID, INVALID, }; @@ -1302,7 +1612,7 @@ const instable_t dis_op0F3A[256] = { /* [3C] */ INVALID, INVALID, INVALID, INVALID, /* [40] */ TNSZ("dpps",XMMP_66r,16),TNSZ("dppd",XMMP_66r,16),TNSZ("mpsadbw",XMMP_66r,16),INVALID, -/* [44] */ INVALID, INVALID, INVALID, INVALID, +/* [44] */ TNSZ("pclmulqdq",XMMP_66r,16),INVALID, INVALID, INVALID, /* [48] */ INVALID, INVALID, INVALID, INVALID, /* [4C] */ INVALID, INVALID, INVALID, INVALID, @@ -1344,12 +1654,12 @@ const instable_t dis_op0F3A[256] = { /* [C0] */ INVALID, INVALID, INVALID, INVALID, /* [C4] */ INVALID, INVALID, INVALID, INVALID, /* [C8] */ INVALID, INVALID, INVALID, INVALID, -/* [CC] */ INVALID, INVALID, INVALID, INVALID, +/* [CC] */ TNSZ("sha1rnds4",XMMP,16),INVALID, INVALID, INVALID, /* [D0] */ INVALID, INVALID, INVALID, INVALID, /* [D4] */ INVALID, INVALID, INVALID, INVALID, /* [D8] */ INVALID, INVALID, INVALID, INVALID, -/* [DC] */ INVALID, INVALID, INVALID, INVALID, +/* [DC] */ INVALID, INVALID, INVALID, TNSZ("aeskeygenassist",XMMP_66r,16), /* [E0] */ INVALID, INVALID, INVALID, INVALID, /* [E4] */ INVALID, INVALID, INVALID, INVALID, @@ -1371,7 +1681,7 @@ const instable_t dis_opAVX660F3A[256] = { /* [10] */ INVALID, INVALID, INVALID, INVALID, /* [14] */ TNSZ("vpextrb",VEX_RRi,8),TNSZ("vpextrw",VEX_RRi,16),TNSZ("vpextrd",VEX_RRi,16),TNSZ("vextractps",VEX_RM,16), /* [18] */ TNSZ("vinsertf128",VEX_RMRX,16),TNSZ("vextractf128",VEX_RX,16),INVALID, INVALID, -/* [1C] */ INVALID, INVALID, INVALID, INVALID, +/* [1C] */ INVALID, TNSZ("vcvtps2ph",VEX_RX,16), INVALID, INVALID, /* [20] */ TNSZ("vpinsrb",VEX_RMRX,8),TNSZ("vinsertps",VEX_RMRX,16),TNSZ("vpinsrd",VEX_RMRX,16),INVALID, /* [24] */ INVALID, INVALID, INVALID, INVALID, @@ -1384,7 +1694,7 @@ const instable_t dis_opAVX660F3A[256] = { /* [3C] */ INVALID, INVALID, INVALID, INVALID, /* [40] */ TNSZ("vdpps",VEX_RMRX,16),TNSZ("vdppd",VEX_RMRX,16),TNSZ("vmpsadbw",VEX_RMRX,16),INVALID, -/* [44] */ TNSZ("vpclmulqdq",VEX_RMRX,16),INVALID, INVALID, INVALID, +/* [44] */ TNSZ("vpclmulqdq",VEX_RMRX,16),INVALID, TNSZ("vperm2i128",VEX_RMRX,16),INVALID, /* [48] */ INVALID, INVALID, TNSZ("vblendvps",VEX_RMRX,8), TNSZ("vblendvpd",VEX_RMRX,16), /* [4C] */ TNSZ("vpblendvb",VEX_RMRX,16),INVALID, INVALID, INVALID, @@ -1444,6 +1754,15 @@ const instable_t dis_opAVX660F3A[256] = { /* [FC] */ INVALID, INVALID, INVALID, INVALID, }; +/* + * Decode table for 0x0F0D which uses the first byte of the mod_rm to + * indicate a sub-code. + */ +const instable_t dis_op0F0D[8] = { +/* [00] */ INVALID, TNS("prefetchw",PREF), TNS("prefetchwt1",PREF),INVALID, +/* [04] */ INVALID, INVALID, INVALID, INVALID, +}; + /* * Decode table for 0x0F opcodes */ @@ -1453,12 +1772,11 @@ const instable_t dis_op0F[16][16] = { /* [00] */ IND(dis_op0F00), IND(dis_op0F01), TNS("lar",MR), TNS("lsl",MR), /* [04] */ INVALID, TNS("syscall",NORM), TNS("clts",NORM), TNS("sysret",NORM), /* [08] */ TNS("invd",NORM), TNS("wbinvd",NORM), INVALID, TNS("ud2",NORM), -/* [0C] */ INVALID, INVALID, INVALID, INVALID, +/* [0C] */ INVALID, IND(dis_op0F0D), INVALID, INVALID, }, { /* [10] */ TNSZ("movups",XMMO,16), TNSZ("movups",XMMOS,16),TNSZ("movlps",XMMO,8), TNSZ("movlps",XMMOS,8), /* [14] */ TNSZ("unpcklps",XMMO,16),TNSZ("unpckhps",XMMO,16),TNSZ("movhps",XMMOM,8),TNSZ("movhps",XMMOMS,8), /* [18] */ IND(dis_op0F18), INVALID, INVALID, INVALID, -/* APPLE NOTE: Need to handle multi-byte NOP */ /* [1C] */ INVALID, INVALID, INVALID, TS("nop",Mw), }, { /* [20] */ TSy("mov",SREG), TSy("mov",SREG), TSy("mov",SREG), TSy("mov",SREG), @@ -1488,7 +1806,7 @@ const instable_t dis_op0F[16][16] = { }, { /* [70] */ TNSZ("pshufw",MMOPM,8), TNS("psrXXX",MR), TNS("psrXXX",MR), TNS("psrXXX",MR), /* [74] */ TNSZ("pcmpeqb",MMO,8), TNSZ("pcmpeqw",MMO,8), TNSZ("pcmpeqd",MMO,8), TNS("emms",NORM), -/* [78] */ TNS("INVALID",XMMO), TNS("INVALID",XMMO), INVALID, INVALID, +/* [78] */ TNSy("vmread",RM), TNSy("vmwrite",MR), INVALID, INVALID, /* [7C] */ INVALID, INVALID, TNSZ("movd",MMOS,4), TNSZ("movq",MMOS,8), }, { /* [80] */ TNS("jo",D), TNS("jno",D), TNS("jb",D), TNS("jae",D), @@ -1609,8 +1927,8 @@ const instable_t dis_opAVX0F[16][16] = { /* [E8] */ INVALID, INVALID, INVALID, INVALID, /* [EC] */ INVALID, INVALID, INVALID, INVALID, }, { -/* [F0] */ INVALID, INVALID, INVALID, INVALID, -/* [F4] */ INVALID, INVALID, INVALID, INVALID, +/* [F0] */ INVALID, INVALID, TNSZvr("andn",VEX_RMrX,5),TNSZvr("bls",BLS,5), +/* [F4] */ INVALID, TNSZvr("bzhi",VEX_VRMrX,5),INVALID, TNSZvr("bextr",VEX_VRMrX,5), /* [F8] */ INVALID, INVALID, INVALID, INVALID, /* [FC] */ INVALID, INVALID, INVALID, INVALID, } }; @@ -1773,19 +2091,19 @@ const instable_t dis_opFP1n2[8][8] = { /* [2,0] */ TNS("fiaddl",M), TNS("fimull",M), TNS("ficoml",M), TNS("ficompl",M), /* [2,4] */ TNS("fisubl",M), TNS("fisubrl",M), TNS("fidivl",M), TNS("fidivrl",M), }, { -/* [3,0] */ TNS("fildl",M), INVALID, TNS("fistl",M), TNS("fistpl",M), +/* [3,0] */ TNS("fildl",M), TNSZ("tisttpl",M,4), TNS("fistl",M), TNS("fistpl",M), /* [3,4] */ INVALID, TNSZ("fldt",M,10), INVALID, TNSZ("fstpt",M,10), }, { /* [4,0] */ TNSZ("faddl",M,8), TNSZ("fmull",M,8), TNSZ("fcoml",M,8), TNSZ("fcompl",M,8), /* [4,1] */ TNSZ("fsubl",M,8), TNSZ("fsubrl",M,8), TNSZ("fdivl",M,8), TNSZ("fdivrl",M,8), }, { -/* [5,0] */ TNSZ("fldl",M,8), INVALID, TNSZ("fstl",M,8), TNSZ("fstpl",M,8), +/* [5,0] */ TNSZ("fldl",M,8), TNSZ("fisttpll",M,8), TNSZ("fstl",M,8), TNSZ("fstpl",M,8), /* [5,4] */ TNSZ("frstor",M,108), INVALID, TNSZ("fnsave",M,108), TNSZ("fnstsw",M,2), }, { /* [6,0] */ TNSZ("fiadd",M,2), TNSZ("fimul",M,2), TNSZ("ficom",M,2), TNSZ("ficomp",M,2), /* [6,4] */ TNSZ("fisub",M,2), TNSZ("fisubr",M,2), TNSZ("fidiv",M,2), TNSZ("fidivr",M,2), }, { -/* [7,0] */ TNSZ("fild",M,2), INVALID, TNSZ("fist",M,2), TNSZ("fistp",M,2), +/* [7,0] */ TNSZ("fild",M,2), TNSZ("fisttp",M,2), TNSZ("fist",M,2), TNSZ("fistp",M,2), /* [7,4] */ TNSZ("fbld",M,10), TNSZ("fildll",M,8), TNSZ("fbstp",M,10), TNSZ("fistpll",M,8), } }; @@ -1909,7 +2227,7 @@ const instable_t dis_distable[16][16] = { /* [9,0] */ TNS("nop",NORM), TS("xchg",RA), TS("xchg",RA), TS("xchg",RA), /* [9,4] */ TS("xchg",RA), TS("xchg",RA), TS("xchg",RA), TS("xchg",RA), /* [9,8] */ TNS("cXtX",CBW), TNS("cXtX",CWD), TNSx("lcall",SO), TNS("fwait",NORM), -/* [9,C] */ TSZy("pushf",IMPLMEM,4),TSZy("popf",IMPLMEM,4), TNSx("sahf",NORM), TNSx("lahf",NORM), +/* [9,C] */ TSZy("pushf",IMPLMEM,4),TSZy("popf",IMPLMEM,4), TNS("sahf",NORM), TNS("lahf",NORM), }, { /* [A,0] */ TNS("movb",OA), TS("mov",OA), TNS("movb",AO), TS("mov",AO), /* [A,4] */ TNSZ("movsb",SD,1), TS("movs",SD), TNSZ("cmpsb",SD,1), TS("cmps",SD), @@ -2016,6 +2334,80 @@ static int isize64[] = {1, 2, 4, 8}; #define WORD_OPND 8 /* w-bit value indicating word size reg */ #define YMM_OPND 9 /* "value" used to indicate a ymm reg */ +/* + * The AVX2 gather instructions are a bit of a mess. While there's a pattern, + * there's not really a consistent scheme that we can use to know what the mode + * is supposed to be for a given type. Various instructions, like VPGATHERDD, + * always match the value of VEX_L. Other instructions like VPGATHERDQ, have + * some registers match VEX_L, but the VSIB is always XMM. + * + * The simplest way to deal with this is to just define a table based on the + * instruction opcodes, which are 0x90-0x93, so we subtract 0x90 to index into + * them. + * + * We further have to subdivide this based on the value of VEX_W and the value + * of VEX_L. The array is constructed to be indexed as: + * [opcode - 0x90][VEX_W][VEX_L]. + */ +/* w = 0, 0x90 */ +typedef struct dis_gather_regs { + uint_t dgr_arg0; /* src reg */ + uint_t dgr_arg1; /* vsib reg */ + uint_t dgr_arg2; /* dst reg */ + const char *dgr_suffix; /* suffix to append */ +} dis_gather_regs_t; + +static dis_gather_regs_t dis_vgather[4][2][2] = { + { + /* op 0x90, W.0 */ + { + { XMM_OPND, XMM_OPND, XMM_OPND, "d" }, + { YMM_OPND, YMM_OPND, YMM_OPND, "d" } + }, + /* op 0x90, W.1 */ + { + { XMM_OPND, XMM_OPND, XMM_OPND, "q" }, + { YMM_OPND, XMM_OPND, YMM_OPND, "q" } + } + }, + { + /* op 0x91, W.0 */ + { + { XMM_OPND, XMM_OPND, XMM_OPND, "d" }, + { XMM_OPND, YMM_OPND, XMM_OPND, "d" }, + }, + /* op 0x91, W.1 */ + { + { XMM_OPND, XMM_OPND, XMM_OPND, "q" }, + { YMM_OPND, YMM_OPND, YMM_OPND, "q" }, + } + }, + { + /* op 0x92, W.0 */ + { + { XMM_OPND, XMM_OPND, XMM_OPND, "s" }, + { YMM_OPND, YMM_OPND, YMM_OPND, "s" } + }, + /* op 0x92, W.1 */ + { + { XMM_OPND, XMM_OPND, XMM_OPND, "d" }, + { YMM_OPND, XMM_OPND, YMM_OPND, "d" } + } + }, + { + /* op 0x93, W.0 */ + { + { XMM_OPND, XMM_OPND, XMM_OPND, "s" }, + { XMM_OPND, YMM_OPND, XMM_OPND, "s" } + }, + /* op 0x93, W.1 */ + { + { XMM_OPND, XMM_OPND, XMM_OPND, "d" }, + { YMM_OPND, YMM_OPND, YMM_OPND, "d" } + } + } +}; + /* * Get the next byte and separate the op code into the high and low nibbles. */ @@ -2118,6 +2510,7 @@ dtrace_vex_adjust(uint_t vex_byte1, uint_t mode, uint_t *reg, uint_t *r_m) *r_m += 8; } } + /* * Get an immediate operand of the given size, with sign extension. */ @@ -2392,16 +2785,29 @@ dtrace_get_operand(dis86_t *x, uint_t mode, uint_t r_m, int wbit, int opindex) } else { uint_t need_paren = 0; char **regs; + char **bregs; + const char *const *sf; if (x->d86_mode == SIZE32) /* NOTE this is not addr_size! */ regs = (char **)dis_REG32; else regs = (char **)dis_REG64; + if (x->d86_vsib != 0) { + if (wbit == YMM_OPND) /* NOTE this is not addr_size! */ + bregs = (char **)dis_YMMREG; + else + bregs = (char **)dis_XMMREG; + sf = dis_vscale_factor; + } else { + bregs = regs; + sf = dis_scale_factor; + } + /* * print the base (if any) */ if (base == EBP_REGNO && mode == 0) { - if (index != ESP_REGNO) { + if (index != ESP_REGNO || x->d86_vsib != 0) { (void) strlcat(opnd, "(", OPLEN); need_paren = 1; } @@ -2414,10 +2820,10 @@ dtrace_get_operand(dis86_t *x, uint_t mode, uint_t r_m, int wbit, int opindex) /* * print the index (if any) */ - if (index != ESP_REGNO) { + if (index != ESP_REGNO || x->d86_vsib) { (void) strlcat(opnd, ",", OPLEN); - (void) strlcat(opnd, regs[index], OPLEN); - (void) strlcat(opnd, dis_scale_factor[ss], OPLEN); + (void) strlcat(opnd, bregs[index], OPLEN); + (void) strlcat(opnd, sf[ss], OPLEN); } else if (need_paren) (void) strlcat(opnd, ")", OPLEN); @@ -2515,16 +2921,16 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) uint_t w2; /* wbit value for second operand */ uint_t vbit; uint_t mode = 0; /* mode value from ModRM byte */ - uint_t reg; /* reg value from ModRM byte */ - uint_t r_m; /* r_m value from ModRM byte */ - - uint_t opcode1; /* high nibble of 1st byte */ - uint_t opcode2; /* low nibble of 1st byte */ - uint_t opcode3; /* extra opcode bits usually from ModRM byte */ - uint_t opcode4; /* high nibble of 2nd byte */ - uint_t opcode5; /* low nibble of 2nd byte */ - uint_t opcode6; /* high nibble of 3rd byte */ - uint_t opcode7; /* low nibble of 3rd byte */ + uint_t reg = 0; /* reg value from ModRM byte */ + uint_t r_m = 0; /* r_m value from ModRM byte */ + + uint_t opcode1 = 0; /* high nibble of 1st byte */ + uint_t opcode2 = 0; /* low nibble of 1st byte */ + uint_t opcode3 = 0; /* extra opcode bits usually from ModRM byte */ + uint_t opcode4 = 0; /* high nibble of 2nd byte */ + uint_t opcode5 = 0; /* low nibble of 2nd byte */ + uint_t opcode6 = 0; /* high nibble of 3rd byte */ + uint_t opcode7 = 0; /* low nibble of 3rd byte */ uint_t opcode_bytes = 1; /* @@ -2563,7 +2969,13 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) uint_t vex_X = 1; uint_t vex_B = 1; uint_t vex_W = 0; - uint_t vex_L; + uint_t vex_L = 0; + dis_gather_regs_t *vreg; + +#ifdef DIS_TEXT + /* Instruction name for BLS* family of instructions */ + char *blsinstr; +#endif size_t off; @@ -2571,8 +2983,7 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) x->d86_len = 0; x->d86_rmindex = -1; - x->d86_rex_prefix = 0; - x->d86_got_modrm = 0; + x->d86_error = 0; #ifdef DIS_TEXT x->d86_numopnds = 0; x->d86_seg_prefix = NULL; @@ -2585,8 +2996,10 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) x->d86_opnd[i].d86_mode = MODE_NONE; } #endif - x->d86_error = 0; + x->d86_rex_prefix = 0; + x->d86_got_modrm = 0; x->d86_memsize = 0; + x->d86_vsib = 0; if (cpu_mode == SIZE16) { opnd_size = SIZE16; @@ -2610,7 +3023,6 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) x->d86_check_func != NULL && x->d86_check_func(x->d86_data)) { #ifdef DIS_TEXT (void) strncpy(x->d86_mnem, ".byte\t0", OPLEN); - x->d86_mnem[OPLEN - 1] = '\0'; #endif goto done; } @@ -2785,6 +3197,10 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) dp = (instable_t *) &dis_opAVXF30F [(opcode1 << 4) | opcode2]; + } else if (vex_m == VEX_m_0F38) { + dp = (instable_t *) + &dis_opAVXF30F38 + [(opcode1 << 4) | opcode2]; } else { goto error; } @@ -2794,6 +3210,14 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) dp = (instable_t *) &dis_opAVXF20F [(opcode1 << 4) | opcode2]; + } else if (vex_m == VEX_m_0F3A) { + dp = (instable_t *) + &dis_opAVXF20F3A + [(opcode1 << 4) | opcode2]; + } else if (vex_m == VEX_m_0F38) { + dp = (instable_t *) + &dis_opAVXF20F38 + [(opcode1 << 4) | opcode2]; } else { goto error; } @@ -2802,14 +3226,17 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) dp = (instable_t *) &dis_opAVX0F[opcode1][opcode2]; - } - } - + } + } if (vex_prefix) { - if (vex_L) - wbit = YMM_OPND; - else - wbit = XMM_OPND; + if (dp->it_vexwoxmm) { + wbit = LONG_OPND; + } else { + if (vex_L) + wbit = YMM_OPND; + else + wbit = XMM_OPND; + } } /* @@ -2836,7 +3263,6 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) if (addr_size_prefix) addr_size = SIZE32; } - /* * The pause instruction - a repz'd nop. This doesn't fit * with any of the other prefix goop added for SSE, so we'll @@ -2879,6 +3305,8 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) goto error; #endif switch (dp->it_adrmode) { + case XMMP: + break; case XMMP_66r: case XMMPRM_66r: case XMM3PM_66r: @@ -2905,11 +3333,66 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) if (dtrace_get_opcode(x, &opcode6, &opcode7) != 0) goto error; dp = (instable_t *)&dis_op0F38[(opcode6<<4)|opcode7]; + + /* + * Both crc32 and movbe have the same 3rd opcode + * byte of either 0xF0 or 0xF1, so we use another + * indirection to distinguish between the two. + */ + if (dp->it_indirect == (instable_t *)dis_op0F38F0 || + dp->it_indirect == (instable_t *)dis_op0F38F1) { + + dp = dp->it_indirect; + if (rep_prefix != 0xF2) { + /* It is movbe */ + dp++; + } + } + + /* + * The adx family of instructions (adcx and adox) + * continue the classic Intel tradition of abusing + * arbitrary prefixes without actually meaning the + * prefix bit. Therefore, if we find either the + * opnd_size_prefix or rep_prefix we end up zeroing it + * out after making our determination so as to ensure + * that we don't get confused and accidentally print + * repz prefixes and the like on these instructions. + * + * In addition, these instructions are actually much + * closer to AVX instructions in semantics. Importantly, + * they always default to having 32-bit operands. + * However, if the CPU is in 64-bit mode, then and only + * then, does it use REX.w promotes things to 64-bits + * and REX.r allows 64-bit mode to use register r8-r15. + */ + if (dp->it_indirect == (instable_t *)dis_op0F38F6) { + dp = dp->it_indirect; + if (opnd_size_prefix == 0 && + rep_prefix == 0xf3) { + /* It is adox */ + dp++; + } else if (opnd_size_prefix != 0x66 && + rep_prefix != 0) { + /* It isn't adcx */ + goto error; + } + opnd_size_prefix = 0; + rep_prefix = 0; + opnd_size = SIZE32; + if (rex_prefix & REX_W) + opnd_size = SIZE64; + } + #ifdef DIS_TEXT if (LIT_STRNEQL(dp->it_name, "INVALID")) goto error; #endif switch (dp->it_adrmode) { + case ADX: + case XMM: + break; + case RM_66r: case XMM_66r: case XMMM_66r: if (opnd_size_prefix == 0) { @@ -2933,6 +3416,11 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) } rep_prefix = 0; break; + case MOVBE: + if (rep_prefix != 0x0) { + goto error; + } + break; default: goto error; } @@ -2995,9 +3483,12 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) goto error; /* - * deal with MMX/SSE opcodes which are changed by prefixes + * Deal with MMX/SSE opcodes which are changed by prefixes. Note, we do + * need to include UNKNOWN below, as we may have instructions that + * actually have a prefix, but don't exist in any other form. */ switch (dp->it_adrmode) { + case UNKNOWN: case MMO: case MMOIMPL: case MMO3P: @@ -3056,6 +3547,59 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) } break; + case MG9: + /* + * More horribleness: the group 9 (0xF0 0xC7) instructions are + * allowed an optional prefix of 0x66 or 0xF3. This is similar + * to the SIMD business described above, but with a different + * addressing mode (and an indirect table), so we deal with it + * separately (if similarly). + * + * Intel further complicated this with the release of Ivy Bridge + * where they overloaded these instructions based on the ModR/M + * bytes. The VMX instructions have a mode of 0 since they are + * memory instructions but rdrand instructions have a mode of + * 0b11 (REG_ONLY) because they only operate on registers. While + * there are different prefix formats, for now it is sufficient + * to use a single different table. + */ + + /* + * Calculate our offset in dis_op0FC7 (the group 9 table) + */ + if ((uintptr_t)dp - (uintptr_t)dis_op0FC7 > sizeof (dis_op0FC7)) + goto error; + + off = ((uintptr_t)dp - (uintptr_t)dis_op0FC7) / + sizeof (instable_t); + + /* + * If we have a mode of 0b11 then we have to rewrite this. + */ + dtrace_get_modrm(x, &mode, ®, &r_m); + if (mode == REG_ONLY) { + dp = (instable_t *)&dis_op0FC7m3[off]; + break; + } + + /* + * Rewrite if this instruction used one of the magic prefixes. + */ + if (rep_prefix) { + if (rep_prefix == 0xf3) + dp = (instable_t *)&dis_opF30FC7[off]; + else + goto error; + rep_prefix = 0; + } else if (opnd_size_prefix) { + dp = (instable_t *)&dis_op660FC7[off]; + opnd_size_prefix = 0; + if (opnd_size == SIZE16) + opnd_size = SIZE32; + } + break; + + case MMOSH: /* * As with the "normal" SIMD instructions, the MMX @@ -3133,7 +3677,10 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) if (LIT_STRNEQL(dp->it_name, "INVALID")) goto error; (void) strlcat(x->d86_mnem, dp->it_name, OPLEN); - if (dp->it_suffix) { + if (dp->it_avxsuf && dp->it_suffix) { + (void) strlcat(x->d86_mnem, vex_W != 0 ? "q" : "d", + OPLEN); + } else if (dp->it_suffix) { char *types[] = {"", "w", "l", "q"}; if (opcode_bytes == 2 && opcode4 == 4) { /* It's a cmovx.yy. Replace the suffix x */ @@ -3222,6 +3769,27 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) x->d86_opnd_size = opnd_size = SIZE16; dtrace_get_operand(x, mode, r_m, wbit, 0); break; + case MOVBE: + opnd_size = SIZE32; + if (rex_prefix & REX_W) + opnd_size = SIZE64; + x->d86_opnd_size = opnd_size; + + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_rex_adjust(rex_prefix, mode, ®, &r_m); + wbit = WBIT(opcode7); + if (opnd_size_prefix) + x->d86_opnd_size = opnd_size = SIZE16; + if (wbit) { + /* reg -> mem */ + dtrace_get_operand(x, REG_ONLY, reg, LONG_OPND, 0); + dtrace_get_operand(x, mode, r_m, wbit, 1); + } else { + /* mem -> reg */ + dtrace_get_operand(x, REG_ONLY, reg, LONG_OPND, 1); + dtrace_get_operand(x, mode, r_m, wbit, 0); + } + break; /* * imul instruction, with either 8-bit or longer immediate @@ -3235,6 +3803,7 @@ dtrace_disx86(dis86_t *x, uint_t cpu_mode) /* memory or register operand to register, with 'w' bit */ case MRw: + case ADX: wbit = WBIT(opcode2); STANDARD_MODRM(x, mode, reg, r_m, rex_prefix, wbit, 0); break; @@ -3417,15 +3986,22 @@ just_mem: dtrace_get_operand(x, mode, r_m, wbit, 0); break; - case SWAPGS: + case SWAPGS_RDTSCP: if (cpu_mode == SIZE64 && mode == 3 && r_m == 0) { #ifdef DIS_TEXT (void) strncpy(x->d86_mnem, "swapgs", OPLEN); x->d86_mnem[OPLEN - 1] = '\0'; +#endif + NOMEM; + break; + } else if (mode == 3 && r_m == 1) { +#ifdef DIS_TEXT + (void) strncpy(x->d86_mnem, "rdtscp", OPLEN); #endif NOMEM; break; } + /*FALLTHROUGH*/ /* prefetch instruction - memory operand, but no memory acess */ @@ -3435,6 +4011,7 @@ just_mem: /* single memory or register operand */ case M: + case MG9: wbit = LONG_OPND; goto just_mem; @@ -3443,6 +4020,76 @@ just_mem: wbit = BYTE_OPND; goto just_mem; + case VMx: + if (mode == 3) { +#ifdef DIS_TEXT + char *vminstr; + + switch (r_m) { + case 1: + vminstr = "vmcall"; + break; + case 2: + vminstr = "vmlaunch"; + break; + case 3: + vminstr = "vmresume"; + break; + case 4: + vminstr = "vmxoff"; + break; + default: + goto error; + } + + (void) strncpy(x->d86_mnem, vminstr, OPLEN); +#else + if (r_m < 1 || r_m > 4) + goto error; +#endif + + NOMEM; + break; + } + /*FALLTHROUGH*/ + case SVM: + if (mode == 3) { +#if DIS_TEXT + char *vinstr; + + switch (r_m) { + case 0: + vinstr = "vmrun"; + break; + case 1: + vinstr = "vmmcall"; + break; + case 2: + vinstr = "vmload"; + break; + case 3: + vinstr = "vmsave"; + break; + case 4: + vinstr = "stgi"; + break; + case 5: + vinstr = "clgi"; + break; + case 6: + vinstr = "skinit"; + break; + case 7: + vinstr = "invlpga"; + break; + } + + (void) strncpy(x->d86_mnem, vinstr, OPLEN); +#endif + NOMEM; + break; + } + /*FALLTHROUGH*/ case MONITOR_MWAIT: if (mode == 3) { if (r_m == 0) { @@ -3456,6 +4103,18 @@ just_mem: #ifdef DIS_TEXT (void) strncpy(x->d86_mnem, "mwait", OPLEN); x->d86_mnem[OPLEN - 1] = '\0'; +#endif + NOMEM; + break; + } else if (r_m == 2) { +#ifdef DIS_TEXT + (void) strncpy(x->d86_mnem, "clac", OPLEN); +#endif + NOMEM; + break; + } else if (r_m == 3) { +#ifdef DIS_TEXT + (void) strncpy(x->d86_mnem, "stac", OPLEN); #endif NOMEM; break; @@ -3576,14 +4235,14 @@ just_mem: /* memory or register operand to register */ case MR: - if (vex_prefetch) { + if (vex_prefetch) x->d86_got_modrm = 1; - } wbit = LONG_OPND; STANDARD_MODRM(x, mode, reg, r_m, rex_prefix, wbit, 0); break; case RM: + case RM_66r: wbit = LONG_OPND; STANDARD_MODRM(x, mode, reg, r_m, rex_prefix, wbit, 1); break; @@ -3684,13 +4343,13 @@ xmmprm: * movhps and movlhps behave similarly. */ if (mode == REG_ONLY) { - if (LIT_STRNEQL(dp->it_name, "movlps")) + if (LIT_STRNEQL(dp->it_name, "movlps")) { (void) strncpy(x->d86_mnem, "movhlps", OPLEN); x->d86_mnem[OPLEN - 1] = '\0'; - } else if (LIT_STRNEQL(dp->it_name, "movhps")) { + } else if (strcmp(dp->it_name, "movhps") == 0) { (void) strncpy(x->d86_mnem, "movlhps", OPLEN); x->d86_mnem[OPLEN - 1] = '\0'; - } + } } #endif if (dp->it_adrmode == XMMXIMPL) @@ -3705,8 +4364,8 @@ xmmprm: dtrace_get_modrm(x, &mode, ®, &r_m); #ifdef DIS_TEXT if ((LIT_STRNEQL(dp->it_name, "movlps") || - LIT_STRNEQL(dp->it_name, "movhps") || - LIT_STRNEQL(dp->it_name, "movntps")) && + LIT_STRNEQL(dp->it_name, "movhps") || + LIT_STRNEQL(dp->it_name, "movntps")) && mode == REG_ONLY) goto error; #endif @@ -4124,7 +4783,6 @@ xmmprm: } break; - /* float reg */ case F: #ifdef DIS_TEXT @@ -4162,11 +4820,31 @@ xmmprm: dtrace_get_operand(x, mode, r_m, wbit, 0); break; case VEX_RMrX: + case FMA: /* ModR/M.reg := op(VEX.vvvv, ModR/M.r/m) */ x->d86_numopnds = 3; dtrace_get_modrm(x, &mode, ®, &r_m); dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + /* + * In classic Intel fashion, the opcodes for all of the FMA + * instructions all have two possible mnemonics which vary by + * one letter, which is selected based on the value of the wbit. + * When wbit is one, they have the 'd' suffix and when 'wbit' is + * 0, they have the 's' suffix. Otherwise, the FMA instructions + * are all a standard VEX_RMrX. + */ +#ifdef DIS_TEXT + if (dp->it_adrmode == FMA) { + size_t len = strlen(dp->it_name); + (void) strncpy(x->d86_mnem, dp->it_name, OPLEN); + if (len + 1 < OPLEN) { + (void) strncpy(x->d86_mnem + len, + vex_W != 0 ? "d" : "s", OPLEN - len); + } + } +#endif + if (mode != REG_ONLY) { if ((dp == &dis_opAVXF20F[0x10]) || (dp == &dis_opAVXF30F[0x10])) { @@ -4205,6 +4883,53 @@ xmmprm: break; + case VEX_VRMrX: + /* ModR/M.reg := op(MODR/M.r/m, VEX.vvvv) */ + x->d86_numopnds = 3; + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + + dtrace_get_operand(x, REG_ONLY, reg, wbit, 2); + /* + * VEX prefix uses the 1's complement form to encode the + * XMM/YMM regs + */ + dtrace_get_operand(x, REG_ONLY, (0xF - vex_v), wbit, 0); + + dtrace_get_operand(x, mode, r_m, wbit, 1); + break; + + case VEX_SbVM: + /* ModR/M.reg := op(MODR/M.r/m, VSIB, VEX.vvvv) */ + x->d86_numopnds = 3; + x->d86_vsib = 1; + + /* + * All instructions that use VSIB are currently a mess. See the + * comment around the dis_gather_regs_t structure definition. + */ + + vreg = &dis_vgather[opcode2][vex_W][vex_L]; + +#ifdef DIS_TEXT + (void) strncpy(x->d86_mnem, dp->it_name, OPLEN); + (void) strlcat(x->d86_mnem + strlen(dp->it_name), + vreg->dgr_suffix, OPLEN - strlen(dp->it_name)); +#endif + + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + + dtrace_get_operand(x, REG_ONLY, reg, vreg->dgr_arg2, 2); + /* + * VEX prefix uses the 1's complement form to encode the + * XMM/YMM regs + */ + dtrace_get_operand(x, REG_ONLY, (0xF - vex_v), vreg->dgr_arg0, + 0); + dtrace_get_operand(x, mode, r_m, vreg->dgr_arg1, 1); + break; + case VEX_RRX: /* ModR/M.rm := op(VEX.vvvv, ModR/M.reg) */ x->d86_numopnds = 3; @@ -4294,12 +5019,16 @@ L_VEX_MX: dtrace_get_operand(x, mode, r_m, wbit, 0); } else if ((dp == &dis_opAVXF30F[0xE6]) || (dp == &dis_opAVX0F[0x5][0xA]) || + (dp == &dis_opAVX660F38[0x13]) || + (dp == &dis_opAVX660F38[0x18]) || + (dp == &dis_opAVX660F38[0x19]) || (dp == &dis_opAVX660F38[0x58]) || - (dp == &dis_opAVX660F38[0x59]) || (dp == &dis_opAVX660F38[0x78]) || - (dp == &dis_opAVX660F38[0x79])) { + (dp == &dis_opAVX660F38[0x79]) || + (dp == &dis_opAVX660F38[0x59])) { /* vcvtdq2pd <xmm>, <ymm> */ /* or vcvtps2pd <xmm>, <ymm> */ + /* or vcvtph2ps <xmm>, <ymm> */ /* or vbroadcasts* <xmm>, <ymm> */ dtrace_get_operand(x, REG_ONLY, reg, wbit, 1); dtrace_get_operand(x, mode, r_m, XMM_OPND, 0); @@ -4383,7 +5112,9 @@ L_VEX_MX: case VEX_RX: /* ModR/M.rm := op(ModR/M.reg) */ - if (dp == &dis_opAVX660F3A[0x19]) { /* vextractf128 */ + /* vextractf128 || vcvtps2ph */ + if (dp == &dis_opAVX660F3A[0x19] || + dp == &dis_opAVX660F3A[0x1d]) { x->d86_numopnds = 3; dtrace_get_modrm(x, &mode, ®, &r_m); @@ -4445,7 +5176,6 @@ L_VEX_MX: /* one byte immediate number */ dtrace_imm_opnd(x, wbit, 1, 0); break; - case VEX_RIM: /* ModR/M.rm := op(ModR/M.reg, imm) */ x->d86_numopnds = 3; @@ -4513,7 +5243,48 @@ L_VEX_RM: (void) strncpy(x->d86_mnem, "vzeroall", OPLEN); #endif break; + case BLS: { + + /* + * The BLS instructions are VEX instructions that are based on + * VEX.0F38.F3; however, they are considered special group 17 + * and like everything else, they use the bits in 3-5 of the + * MOD R/M to determine the sub instruction. Unlike many others + * like the VMX instructions, these are valid both for memory + * and register forms. + */ + + dtrace_get_modrm(x, &mode, ®, &r_m); + dtrace_vex_adjust(vex_byte1, mode, ®, &r_m); + + switch (reg) { + case 1: +#ifdef DIS_TEXT + blsinstr = "blsr"; +#endif + break; + case 2: +#ifdef DIS_TEXT + blsinstr = "blsmsk"; +#endif + break; + case 3: +#ifdef DIS_TEXT + blsinstr = "blsi"; +#endif + break; + default: + goto error; + } + x->d86_numopnds = 2; +#ifdef DIS_TEXT + (void) strncpy(x->d86_mnem, blsinstr, OPLEN); +#endif + dtrace_get_operand(x, REG_ONLY, (0xF - vex_v), wbit, 1); + dtrace_get_operand(x, mode, r_m, wbit, 0); + break; + } /* an invalid op code */ case AM: case DM: diff --git a/bsd/dev/i386/dtrace_isa.c b/bsd/dev/i386/dtrace_isa.c index dfdeaad82..cb6795580 100644 --- a/bsd/dev/i386/dtrace_isa.c +++ b/bsd/dev/i386/dtrace_isa.c @@ -734,9 +734,9 @@ struct frame { }; uint64_t -dtrace_getarg(int arg, int aframes) +dtrace_getarg(int arg, int aframes, dtrace_mstate_t *mstate, dtrace_vstate_t *vstate) { - uint64_t val; + uint64_t val = 0; struct frame *fp = (struct frame *)__builtin_frame_address(0); uintptr_t *stack; uintptr_t pc; @@ -778,7 +778,7 @@ dtrace_getarg(int arg, int aframes) x86_saved_state64_t *saved_state = saved_state64(tagged_regs); if (arg <= inreg) { - stack = (uintptr_t *)&saved_state->rdi; + stack = (uintptr_t *)(void*)&saved_state->rdi; } else { fp = (struct frame *)(saved_state->isf.rsp); stack = (uintptr_t *)&fp[1]; /* Find marshalled @@ -812,10 +812,11 @@ dtrace_getarg(int arg, int aframes) stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */ load: - DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); - /* dtrace_probe arguments arg0 ... arg4 are 64bits wide */ - val = (uint64_t)(*(((uintptr_t *)stack) + arg)); - DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); + if (dtrace_canload((uint64_t)(stack + arg), sizeof(uint64_t), + mstate, vstate)) { + /* dtrace_probe arguments arg0 ... arg4 are 64bits wide */ + val = dtrace_load64((uint64_t)(stack + arg)); + } return (val); } diff --git a/bsd/dev/i386/fasttrap_isa.c b/bsd/dev/i386/fasttrap_isa.c index 4eeb6d140..a70039322 100644 --- a/bsd/dev/i386/fasttrap_isa.c +++ b/bsd/dev/i386/fasttrap_isa.c @@ -640,6 +640,8 @@ fasttrap_tracepoint_install(proc_t *p, fasttrap_tracepoint_t *tp) if (uwrite(p, &instr, 1, tp->ftt_pc) != 0) return (-1); + tp->ftt_installed = 1; + return (0); } @@ -653,11 +655,13 @@ fasttrap_tracepoint_remove(proc_t *p, fasttrap_tracepoint_t *tp) * instruction. */ if (uread(p, &instr, 1, tp->ftt_pc) != 0) - return (0); + goto end; if (instr != FASTTRAP_INSTR) - return (0); + goto end; if (uwrite(p, &tp->ftt_instr[0], 1, tp->ftt_pc) != 0) return (-1); +end: + tp->ftt_installed = 0; return (0); } @@ -669,6 +673,7 @@ fasttrap_return_common(x86_saved_state_t *regs, user_addr_t pc, pid_t pid, x86_saved_state64_t *regs64; x86_saved_state32_t *regs32; unsigned int p_model; + int retire_tp = 1; dtrace_icookie_t cookie; @@ -708,6 +713,7 @@ fasttrap_return_common(x86_saved_state_t *regs, user_addr_t pc, pid_t pid, } for (id = tp->ftt_retids; id != NULL; id = id->fti_next) { + fasttrap_probe_t *probe = id->fti_probe; /* * If there's a branch that could act as a return site, we * need to trace it, and check here if the program counter is @@ -715,10 +721,23 @@ fasttrap_return_common(x86_saved_state_t *regs, user_addr_t pc, pid_t pid, */ if (tp->ftt_type != FASTTRAP_T_RET && tp->ftt_type != FASTTRAP_T_RET16 && - new_pc - id->fti_probe->ftp_faddr < - id->fti_probe->ftp_fsize) + new_pc - probe->ftp_faddr < probe->ftp_fsize) continue; + if (probe->ftp_prov->ftp_provider_type == DTFTP_PROVIDER_ONESHOT) { + uint8_t already_triggered = atomic_or_8(&probe->ftp_triggered, 1); + if (already_triggered) { + continue; + } + } + /* + * If we have at least one probe associated that + * is not a oneshot probe, don't remove the + * tracepoint + */ + else { + retire_tp = 0; + } /* * Provide a hint to the stack trace functions to add the * following pc to the top of the stack since it's missing @@ -727,14 +746,14 @@ fasttrap_return_common(x86_saved_state_t *regs, user_addr_t pc, pid_t pid, cookie = dtrace_interrupt_disable(); cpu_core[CPU->cpu_id].cpuc_missing_tos = pc; if (ISSET(current_proc()->p_lflag, P_LNOATTACH)) { - dtrace_probe(dtrace_probeid_error, 0 /* state */, id->fti_probe->ftp_id, + dtrace_probe(dtrace_probeid_error, 0 /* state */, probe->ftp_id, 1 /* ndx */, -1 /* offset */, DTRACEFLT_UPRIV); } else if (p_model == DATAMODEL_LP64) { - dtrace_probe(id->fti_probe->ftp_id, + dtrace_probe(probe->ftp_id, pc - id->fti_probe->ftp_faddr, regs64->rax, regs64->rdx, 0, 0); } else { - dtrace_probe(id->fti_probe->ftp_id, + dtrace_probe(probe->ftp_id, pc - id->fti_probe->ftp_faddr, regs32->eax, regs32->edx, 0, 0); } @@ -953,7 +972,7 @@ fasttrap_pid_probe32(x86_saved_state_t *regs) fasttrap_tracepoint_t *tp, tp_local; pid_t pid; dtrace_icookie_t cookie; - uint_t is_enabled = 0; + uint_t is_enabled = 0, retire_tp = 1; uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread()); @@ -1046,45 +1065,59 @@ fasttrap_pid_probe32(x86_saved_state_t *regs) if (ISSET(current_proc()->p_lflag, P_LNOATTACH)) { dtrace_probe(dtrace_probeid_error, 0 /* state */, probe->ftp_id, 1 /* ndx */, -1 /* offset */, DTRACEFLT_UPRIV); - } else if (id->fti_ptype == DTFTP_ENTRY) { - /* - * We note that this was an entry - * probe to help ustack() find the - * first caller. - */ - cookie = dtrace_interrupt_disable(); - DTRACE_CPUFLAG_SET(CPU_DTRACE_ENTRY); - dtrace_probe(probe->ftp_id, s1, s2, - s3, s4, s5); - DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_ENTRY); - dtrace_interrupt_enable(cookie); - } else if (id->fti_ptype == DTFTP_IS_ENABLED) { + } else { + if (probe->ftp_prov->ftp_provider_type == DTFTP_PROVIDER_ONESHOT) { + uint8_t already_triggered = atomic_or_8(&probe->ftp_triggered, 1); + if (already_triggered) { + continue; + } + } /* - * Note that in this case, we don't - * call dtrace_probe() since it's only - * an artificial probe meant to change - * the flow of control so that it - * encounters the true probe. + * If we have at least one probe associated that + * is not a oneshot probe, don't remove the + * tracepoint */ - is_enabled = 1; - } else if (probe->ftp_argmap == NULL) { - dtrace_probe(probe->ftp_id, s0, s1, - s2, s3, s4); - } else { - uint32_t t[5]; - - fasttrap_usdt_args32(probe, regs32, - sizeof (t) / sizeof (t[0]), t); - - dtrace_probe(probe->ftp_id, t[0], t[1], - t[2], t[3], t[4]); - } + else { + retire_tp = 0; + } + if (id->fti_ptype == DTFTP_ENTRY) { + /* + * We note that this was an entry + * probe to help ustack() find the + * first caller. + */ + cookie = dtrace_interrupt_disable(); + DTRACE_CPUFLAG_SET(CPU_DTRACE_ENTRY); + dtrace_probe(probe->ftp_id, s1, s2, + s3, s4, s5); + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_ENTRY); + dtrace_interrupt_enable(cookie); + } else if (id->fti_ptype == DTFTP_IS_ENABLED) { + /* + * Note that in this case, we don't + * call dtrace_probe() since it's only + * an artificial probe meant to change + * the flow of control so that it + * encounters the true probe. + */ + is_enabled = 1; + } else if (probe->ftp_argmap == NULL) { + dtrace_probe(probe->ftp_id, s0, s1, + s2, s3, s4); + } else { + uint32_t t[5]; - /* APPLE NOTE: Oneshot probes get one and only one chance... */ - if (probe->ftp_prov->ftp_provider_type == DTFTP_PROVIDER_ONESHOT) { - fasttrap_tracepoint_remove(p, tp); + fasttrap_usdt_args32(probe, regs32, + sizeof (t) / sizeof (t[0]), t); + + dtrace_probe(probe->ftp_id, t[0], t[1], + t[2], t[3], t[4]); + } } } + if (retire_tp) { + fasttrap_tracepoint_retire(p, tp); + } } /* @@ -1512,6 +1545,7 @@ fasttrap_pid_probe64(x86_saved_state_t *regs) pid_t pid; dtrace_icookie_t cookie; uint_t is_enabled = 0; + int retire_tp = 1; uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread()); @@ -1585,6 +1619,20 @@ fasttrap_pid_probe64(x86_saved_state_t *regs) for (id = tp->ftt_ids; id != NULL; id = id->fti_next) { fasttrap_probe_t *probe = id->fti_probe; + if (probe->ftp_prov->ftp_provider_type == DTFTP_PROVIDER_ONESHOT) { + uint8_t already_triggered = atomic_or_8(&probe->ftp_triggered, 1); + if (already_triggered) { + continue; + } + } + /* + * If we have at least probe associated that + * is not a oneshot probe, don't remove the + * tracepoint + */ + else { + retire_tp = 0; + } if (ISSET(current_proc()->p_lflag, P_LNOATTACH)) { dtrace_probe(dtrace_probeid_error, 0 /* state */, probe->ftp_id, 1 /* ndx */, -1 /* offset */, DTRACEFLT_UPRIV); @@ -1624,10 +1672,9 @@ fasttrap_pid_probe64(x86_saved_state_t *regs) t[2], t[3], t[4]); } - /* APPLE NOTE: Oneshot probes get one and only one chance... */ - if (probe->ftp_prov->ftp_provider_type == DTFTP_PROVIDER_ONESHOT) { - fasttrap_tracepoint_remove(p, tp); - } + } + if (retire_tp) { + fasttrap_tracepoint_retire(p, tp); } } diff --git a/bsd/dev/i386/fbt_x86.c b/bsd/dev/i386/fbt_x86.c index 54f00f290..6553c2412 100644 --- a/bsd/dev/i386/fbt_x86.c +++ b/bsd/dev/i386/fbt_x86.c @@ -102,400 +102,8 @@ extern dtrace_provider_id_t fbt_id; extern fbt_probe_t **fbt_probetab; extern int fbt_probetab_mask; -extern int gIgnoreFBTBlacklist; /* From fbt_init */ - kern_return_t fbt_perfCallback(int, x86_saved_state_t *, uintptr_t *, __unused int); -/* - * Critical routines that must not be probed. PR_5221096, PR_5379018. - * The blacklist must be kept in alphabetic order for purposes of bsearch(). - */ - -static const char * critical_blacklist[] = -{ - "bcopy_phys", - "console_cpu_alloc", - "console_cpu_free", - "cpu_IA32e_disable", - "cpu_IA32e_enable", - "cpu_NMI_interrupt", - "cpu_control", - "cpu_data_alloc", - "cpu_desc_init", - "cpu_desc_init64", - "cpu_desc_load", - "cpu_desc_load64", - "cpu_exit_wait", - "cpu_info", - "cpu_info_count", - "cpu_init", - "cpu_interrupt", - "cpu_machine_init", - "cpu_mode_init", - "cpu_processor_alloc", - "cpu_processor_free", - "cpu_signal_handler", - "cpu_sleep", - "cpu_start", - "cpu_subtype", - "cpu_thread_alloc", - "cpu_thread_halt", - "cpu_thread_init", - "cpu_threadtype", - "cpu_to_processor", - "cpu_topology_sort", - "cpu_topology_start_cpu", - "cpu_type", - "cpuid_cpu_display", - "cpuid_extfeatures", - "handle_pending_TLB_flushes", - "hw_compare_and_store", - "machine_idle_cstate", - "mca_cpu_alloc", - "mca_cpu_init", - "ml_nofault_copy", - "pmap_cpu_alloc", - "pmap_cpu_free", - "pmap_cpu_high_map_vaddr", - "pmap_cpu_high_shared_remap", - "pmap_cpu_init", - "register_cpu_setup_func", - "unregister_cpu_setup_func", - "vstart" -}; -#define CRITICAL_BLACKLIST_COUNT (sizeof(critical_blacklist)/sizeof(critical_blacklist[0])) - -/* - * The transitive closure of entry points that can be reached from probe context. - * (Apart from routines whose names begin with dtrace_). - */ -static const char * probe_ctx_closure[] = -{ - "Debugger", - "IS_64BIT_PROCESS", - "OSCompareAndSwap", - "_disable_preemption", - "_enable_preemption", - "absolutetime_to_microtime", - "act_set_astbsd", - "ast_dtrace_on", - "ast_pending", - "clock_get_calendar_nanotime_nowait", - "copyin", - "copyin_user", - "copyinstr", - "copyout", - "copyoutstr", - "cpu_number", - "current_proc", - "current_processor", - "current_task", - "current_thread", - "debug_enter", - "find_user_regs", - "flush_tlb64", - "get_bsdtask_info", - "get_bsdthread_info", - "hw_atomic_and", - "kauth_cred_get", - "kauth_getgid", - "kauth_getuid", - "kernel_preempt_check", - "mach_absolute_time", - "max_valid_stack_address", - "ml_at_interrupt_context", - "ml_phys_write_byte_64", - "ml_phys_write_half_64", - "ml_phys_write_word_64", - "ml_set_interrupts_enabled", - "panic", - "pmap64_pde", - "pmap64_pdpt", - "pmap_find_phys", - "pmap_get_mapwindow", - "pmap_pde", - "pmap_pte", - "pmap_put_mapwindow", - "pmap_valid_page", - "prf", - "proc_is64bit", - "proc_selfname", - "psignal_lock", - "rtc_nanotime_load", - "rtc_nanotime_read", - "sdt_getargdesc", - "strlcpy", - "sync_iss_to_iks_unconditionally", - "systrace_stub", - "timer_grab" -}; -#define PROBE_CTX_CLOSURE_COUNT (sizeof(probe_ctx_closure)/sizeof(probe_ctx_closure[0])) - - -static int _cmp(const void *a, const void *b) -{ - return strncmp((const char *)a, *(const char **)b, strlen((const char *)a) + 1); -} - -static const void * bsearch( - const void *key, - const void *base0, - size_t nmemb, - size_t size, - int (*compar)(const void *, const void *)) { - - const char *base = base0; - size_t lim; - int cmp; - const void *p; - - for (lim = nmemb; lim != 0; lim >>= 1) { - p = base + (lim >> 1) * size; - cmp = (*compar)(key, p); - if (cmp == 0) - return p; - if (cmp > 0) { /* key > p: move right */ - base = (const char *)p + size; - lim--; - } /* else move left */ - } - return (NULL); -} - -/* - * Module validation - */ -static int -is_module_valid(struct modctl* ctl) -{ - ASSERT(!MOD_FBT_PROBES_PROVIDED(ctl)); - ASSERT(!MOD_FBT_INVALID(ctl)); - - if (0 == ctl->mod_address || 0 == ctl->mod_size) { - return FALSE; - } - - if (0 == ctl->mod_loaded) { - return FALSE; - } - - if (strstr(ctl->mod_modname, "CHUD") != NULL) - return FALSE; - - /* - * If the user sets this, trust they know what they are doing. - */ - if (gIgnoreFBTBlacklist) /* per boot-arg set in fbt_init() */ - return TRUE; - - /* - * These drivers control low level functions that when traced - * cause problems often in the sleep/wake paths as well as - * critical debug and panic paths. - * If somebody really wants to drill in on one of these kexts, then - * they can override blacklisting using the boot-arg above. - */ - - if (strstr(ctl->mod_modname, "AppleACPIEC") != NULL) - return FALSE; - - if (strstr(ctl->mod_modname, "AppleACPIPlatform") != NULL) - return FALSE; - - if (strstr(ctl->mod_modname, "AppleRTC") != NULL) - return FALSE; - - if (strstr(ctl->mod_modname, "IOACPIFamily") != NULL) - return FALSE; - - if (strstr(ctl->mod_modname, "AppleIntelCPUPowerManagement") != NULL) - return FALSE; - - if (strstr(ctl->mod_modname, "AppleProfile") != NULL) - return FALSE; - - if (strstr(ctl->mod_modname, "AppleIntelProfile") != NULL) - return FALSE; - - if (strstr(ctl->mod_modname, "AppleEFI") != NULL) - return FALSE; - - return TRUE; -} - -/* - * FBT probe name validation - */ -static int -is_symbol_valid(const char* name) -{ - /* - * If the user set this, trust they know what they are doing. - */ - if (gIgnoreFBTBlacklist) - return TRUE; - - if (LIT_STRNSTART(name, "dtrace_") && !LIT_STRNSTART(name, "dtrace_safe_")) { - /* - * Anything beginning with "dtrace_" may be called - * from probe context unless it explitly indicates - * that it won't be called from probe context by - * using the prefix "dtrace_safe_". - */ - return FALSE; - } - - if (LIT_STRNSTART(name, "fasttrap_") || - LIT_STRNSTART(name, "fuword") || - LIT_STRNSTART(name, "suword") || - LIT_STRNEQL(name, "sprlock") || - LIT_STRNEQL(name, "sprunlock") || - LIT_STRNEQL(name, "uread") || - LIT_STRNEQL(name, "uwrite")) { - return FALSE; /* Fasttrap inner-workings. */ - } - - if (LIT_STRNSTART(name, "dsmos_")) - return FALSE; /* Don't Steal Mac OS X! */ - - if (LIT_STRNSTART(name, "_dtrace")) - return FALSE; /* Shims in dtrace.c */ - - if (LIT_STRNSTART(name, "chud")) - return FALSE; /* Professional courtesy. */ - - if (LIT_STRNSTART(name, "hibernate_")) - return FALSE; /* Let sleeping dogs lie. */ - - if (LIT_STRNEQL(name, "_ZNK6OSData14getBytesNoCopyEv")) - return FALSE; /* Data::getBytesNoCopy, IOHibernateSystemWake path */ - - if (LIT_STRNEQL(name, "_ZN9IOService14newTemperatureElPS_") || /* IOService::newTemperature */ - LIT_STRNEQL(name, "_ZN9IOService26temperatureCriticalForZoneEPS_")) { /* IOService::temperatureCriticalForZone */ - return FALSE; /* Per the fire code */ - } - - /* - * Place no probes (illegal instructions) in the exception handling path! - */ - if (LIT_STRNEQL(name, "t_invop") || - LIT_STRNEQL(name, "enter_lohandler") || - LIT_STRNEQL(name, "lo_alltraps") || - LIT_STRNEQL(name, "kernel_trap") || - LIT_STRNEQL(name, "interrupt") || - LIT_STRNEQL(name, "i386_astintr")) { - return FALSE; - } - - if (LIT_STRNEQL(name, "current_thread") || - LIT_STRNEQL(name, "ast_pending") || - LIT_STRNEQL(name, "fbt_perfCallback") || - LIT_STRNEQL(name, "machine_thread_get_kern_state") || - LIT_STRNEQL(name, "get_threadtask") || - LIT_STRNEQL(name, "ml_set_interrupts_enabled") || - LIT_STRNEQL(name, "dtrace_invop") || - LIT_STRNEQL(name, "fbt_invop") || - LIT_STRNEQL(name, "sdt_invop") || - LIT_STRNEQL(name, "max_valid_stack_address")) { - return FALSE; - } - - /* - * Voodoo. - */ - if (LIT_STRNSTART(name, "machine_stack_") || - LIT_STRNSTART(name, "mapping_") || - LIT_STRNEQL(name, "tmrCvt") || - - LIT_STRNSTART(name, "tsc_") || - - LIT_STRNSTART(name, "pmCPU") || - LIT_STRNEQL(name, "pmKextRegister") || - LIT_STRNEQL(name, "pmMarkAllCPUsOff") || - LIT_STRNEQL(name, "pmSafeMode") || - LIT_STRNEQL(name, "pmTimerSave") || - LIT_STRNEQL(name, "pmTimerRestore") || - LIT_STRNEQL(name, "pmUnRegister") || - LIT_STRNSTART(name, "pms") || - LIT_STRNEQL(name, "power_management_init") || - LIT_STRNSTART(name, "usimple_") || - LIT_STRNSTART(name, "lck_spin_lock") || - LIT_STRNSTART(name, "lck_spin_unlock") || - - LIT_STRNSTART(name, "rtc_") || - LIT_STRNSTART(name, "_rtc_") || - LIT_STRNSTART(name, "rtclock_") || - LIT_STRNSTART(name, "clock_") || - LIT_STRNSTART(name, "absolutetime_to_") || - LIT_STRNEQL(name, "setPop") || - LIT_STRNEQL(name, "nanoseconds_to_absolutetime") || - LIT_STRNEQL(name, "nanotime_to_absolutetime") || - - LIT_STRNSTART(name, "etimer_") || - - LIT_STRNSTART(name, "commpage_") || - LIT_STRNSTART(name, "pmap_") || - LIT_STRNSTART(name, "ml_") || - LIT_STRNSTART(name, "PE_") || - LIT_STRNEQL(name, "kprintf") || - LIT_STRNSTART(name, "lapic_") || - LIT_STRNSTART(name, "act_machine") || - LIT_STRNSTART(name, "acpi_") || - LIT_STRNSTART(name, "pal_")){ - return FALSE; - } - - /* - * Avoid machine_ routines. PR_5346750. - */ - if (LIT_STRNSTART(name, "machine_")) - return FALSE; - - if (LIT_STRNEQL(name, "handle_pending_TLB_flushes")) - return FALSE; - - /* - * Place no probes on critical routines. PR_5221096 - */ - if (bsearch( name, critical_blacklist, CRITICAL_BLACKLIST_COUNT, sizeof(name), _cmp ) != NULL) - return FALSE; - - /* - * Place no probes that could be hit in probe context. - */ - if (bsearch( name, probe_ctx_closure, PROBE_CTX_CLOSURE_COUNT, sizeof(name), _cmp ) != NULL) { - return FALSE; - } - - /* - * Place no probes that could be hit on the way to the debugger. - */ - if (LIT_STRNSTART(name, "kdp_") || - LIT_STRNSTART(name, "kdb_") || - LIT_STRNSTART(name, "debug_") || - LIT_STRNEQL(name, "Debugger") || - LIT_STRNEQL(name, "Call_DebuggerC") || - LIT_STRNEQL(name, "lock_debugger") || - LIT_STRNEQL(name, "unlock_debugger") || - LIT_STRNEQL(name, "packA") || - LIT_STRNEQL(name, "unpackA") || - LIT_STRNEQL(name, "SysChoked")) { - return FALSE; - } - - - /* - * Place no probes that could be hit on the way to a panic. - */ - if (NULL != strstr(name, "panic_") || - LIT_STRNEQL(name, "panic") || - LIT_STRNEQL(name, "preemption_underflow_panic")) { - return FALSE; - } - - return TRUE; -} - int fbt_invop(uintptr_t addr, uintptr_t *state, uintptr_t rval) { @@ -630,7 +238,9 @@ fbt_perfCallback( retval = KERN_FAILURE; break; } - saved_state->isf.trapno = T_PREEMPT; /* Avoid call to i386_astintr()! */ + + /* Trick trap_from_kernel into not attempting to handle pending AST_URGENT */ + saved_state->isf.trapno = T_PREEMPT; ml_set_interrupts_enabled(oldlevel); } @@ -638,9 +248,8 @@ fbt_perfCallback( return retval; } -/*ARGSUSED*/ -static void -__provide_probe_64(struct modctl *ctl, uintptr_t instrLow, uintptr_t instrHigh, char *modname, char* symbolName, machine_inst_t* symbolStart) +void +fbt_provide_probe(struct modctl *ctl, uintptr_t instrLow, uintptr_t instrHigh, char *modname, char* symbolName, machine_inst_t* symbolStart) { unsigned int j; unsigned int doenable = 0; @@ -918,10 +527,9 @@ again: goto again; } -static void -__kernel_syms_provide_module(void *arg, struct modctl *ctl) +void +fbt_provide_module_kernel_syms(struct modctl *ctl) { -#pragma unused(arg) kernel_mach_header_t *mh; struct load_command *cmd; kernel_segment_command_t *orig_ts = NULL, *orig_le = NULL; @@ -984,78 +592,9 @@ __kernel_syms_provide_module(void *arg, struct modctl *ctl) /* * We're only blacklisting functions in the kernel for now. */ - if (MOD_IS_MACH_KERNEL(ctl) && !is_symbol_valid(name)) + if (MOD_IS_MACH_KERNEL(ctl) && fbt_excluded(name)) continue; - __provide_probe_64(ctl, instrLow, instrHigh, modname, name, (machine_inst_t*)sym[i].n_value); + fbt_provide_probe(ctl, instrLow, instrHigh, modname, name, (machine_inst_t*)sym[i].n_value); } } - -static void -__user_syms_provide_module(void *arg, struct modctl *ctl) -{ -#pragma unused(arg) - char *modname; - unsigned int i; - - modname = ctl->mod_modname; - - dtrace_module_symbols_t* module_symbols = ctl->mod_user_symbols; - if (module_symbols) { - for (i=0; i<module_symbols->dtmodsyms_count; i++) { - - /* - * symbol->dtsym_addr (the symbol address) passed in from - * user space, is already slid for both kexts and kernel. - */ - dtrace_symbol_t* symbol = &module_symbols->dtmodsyms_symbols[i]; - - char* name = symbol->dtsym_name; - - /* Lop off omnipresent leading underscore. */ - if (*name == '_') - name += 1; - - /* - * We're only blacklisting functions in the kernel for now. - */ - if (MOD_IS_MACH_KERNEL(ctl) && !is_symbol_valid(name)) - continue; - - __provide_probe_64(ctl, (uintptr_t)symbol->dtsym_addr, (uintptr_t)(symbol->dtsym_addr + symbol->dtsym_size), modname, name, (machine_inst_t*)(uintptr_t)symbol->dtsym_addr); - } - } -} - -extern int dtrace_kernel_symbol_mode; - -/*ARGSUSED*/ -void -fbt_provide_module(void *arg, struct modctl *ctl) -{ - ASSERT(ctl != NULL); - ASSERT(dtrace_kernel_symbol_mode != DTRACE_KERNEL_SYMBOLS_NEVER); - lck_mtx_assert(&mod_lock, LCK_MTX_ASSERT_OWNED); - - if (MOD_FBT_DONE(ctl)) - return; - - if (!is_module_valid(ctl)) { - ctl->mod_flags |= MODCTL_FBT_INVALID; - return; - } - - if (MOD_HAS_KERNEL_SYMBOLS(ctl)) { - __kernel_syms_provide_module(arg, ctl); - ctl->mod_flags |= MODCTL_FBT_PROBES_PROVIDED; - return; - } - - if (MOD_HAS_USERSPACE_SYMBOLS(ctl)) { - __user_syms_provide_module(arg, ctl); - ctl->mod_flags |= MODCTL_FBT_PROBES_PROVIDED; - if (MOD_FBT_PROVIDE_PRIVATE_PROBES(ctl)) - ctl->mod_flags |= MODCTL_FBT_PRIVATE_PROBES_PROVIDED; - return; - } -} diff --git a/bsd/dev/i386/km.c b/bsd/dev/i386/km.c index d276b6d95..19923cdc2 100644 --- a/bsd/dev/i386/km.c +++ b/bsd/dev/i386/km.c @@ -51,6 +51,7 @@ extern int hz; extern void cnputcusr(char); +extern void cnputsusr(char *, int); extern int cngetc(void); void kminit(void); @@ -359,7 +360,13 @@ kmoutput(struct tty *tp) (void) q_to_b(&tp->t_outq, buf, cc); for (cp = buf; cp < &buf[cc]; cp++) { /* output the buffer one charatcer at a time */ - kmputc(tp->t_dev, *cp & 0x7f); + *cp = *cp & 0x7f; + } + + if (cc > 1) { + cnputsusr((char *)buf, cc); + } else { + kmputc(tp->t_dev, *buf); } } /* diff --git a/bsd/dev/i386/sdt_x86.c b/bsd/dev/i386/sdt_x86.c index aeb7b3410..9bd151891 100644 --- a/bsd/dev/i386/sdt_x86.c +++ b/bsd/dev/i386/sdt_x86.c @@ -118,7 +118,7 @@ sdt_getarg(void *arg, dtrace_id_t id, void *parg, int argno, int aframes) x86_saved_state64_t *saved_state = saved_state64(tagged_regs); if (argno <= inreg) { - stack = (uintptr_t *)&saved_state->rdi; + stack = (uintptr_t *)(void*)&saved_state->rdi; } else { fp = (struct frame *)(saved_state->isf.rsp); stack = (uintptr_t *)&fp[0]; /* Find marshalled diff --git a/bsd/dev/i386/sysctl.c b/bsd/dev/i386/sysctl.c index 1dc26017a..588f53d58 100644 --- a/bsd/dev/i386/sysctl.c +++ b/bsd/dev/i386/sysctl.c @@ -812,6 +812,13 @@ SYSCTL_UINT(_machdep_tsc_nanotime, OID_AUTO, generation, SYSCTL_NODE(_machdep, OID_AUTO, misc, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "Miscellaneous x86 kernel parameters"); +#if (DEVELOPMENT || DEBUG) +extern uint32_t mp_interrupt_watchdog_events; +SYSCTL_UINT(_machdep_misc, OID_AUTO, interrupt_watchdog_events, + CTLFLAG_RW|CTLFLAG_LOCKED, &mp_interrupt_watchdog_events, 0, ""); +#endif + + SYSCTL_PROC(_machdep_misc, OID_AUTO, panic_restart_timeout, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, 0, 0, diff --git a/bsd/dev/i386/systemcalls.c b/bsd/dev/i386/systemcalls.c index 9d7d476f9..3a8d5dffa 100644 --- a/bsd/dev/i386/systemcalls.c +++ b/bsd/dev/i386/systemcalls.c @@ -229,6 +229,12 @@ unix_syscall(x86_saved_state_t *state) uthread->uu_flag &= ~UT_NOTCANCELPT; +#if DEBUG || DEVELOPMENT + kern_allocation_name_t + prior __assert_only = thread_set_allocation_name(NULL); + assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior)); +#endif /* DEBUG || DEVELOPMENT */ + if (__improbable(uthread->uu_lowpri_window)) { /* * task is marked as a low priority I/O type @@ -432,6 +438,12 @@ unix_syscall64(x86_saved_state_t *state) uthread->uu_flag &= ~UT_NOTCANCELPT; +#if DEBUG || DEVELOPMENT + kern_allocation_name_t + prior __assert_only = thread_set_allocation_name(NULL); + assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior)); +#endif /* DEBUG || DEVELOPMENT */ + if (__improbable(uthread->uu_lowpri_window)) { /* * task is marked as a low priority I/O type @@ -565,6 +577,12 @@ unix_syscall_return(int error) uthread->uu_flag &= ~UT_NOTCANCELPT; +#if DEBUG || DEVELOPMENT + kern_allocation_name_t + prior __assert_only = thread_set_allocation_name(NULL); + assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior)); +#endif /* DEBUG || DEVELOPMENT */ + if (uthread->uu_lowpri_window) { /* * task is marked as a low priority I/O type diff --git a/bsd/dev/i386/unix_signal.c b/bsd/dev/i386/unix_signal.c index bb073b026..1c271607f 100644 --- a/bsd/dev/i386/unix_signal.c +++ b/bsd/dev/i386/unix_signal.c @@ -53,6 +53,7 @@ #include <i386/psl.h> #include <i386/machine_routines.h> #include <i386/seg.h> +#include <i386/fpu.h> #include <machine/pal_routines.h> @@ -100,6 +101,28 @@ struct sigframe32 { user32_addr_t uctx; /* struct ucontext32 */ }; +/* + * Declare table of structure flavors and sizes for 64-bit and 32-bit processes + * for the cases of extended states (plain FP, or AVX): + */ +typedef struct { + int flavor; natural_t state_count; size_t mcontext_size; +} xstate_info_t; +static const xstate_info_t thread_state64[] = { + [FP] = { x86_FLOAT_STATE64, x86_FLOAT_STATE64_COUNT, sizeof(struct mcontext64) }, + [AVX] = { x86_AVX_STATE64, x86_AVX_STATE64_COUNT, sizeof(struct mcontext_avx64) }, +#if !defined(RC_HIDE_XNU_J137) + [AVX512] = { x86_AVX512_STATE64, x86_AVX512_STATE64_COUNT, sizeof(struct mcontext_avx512_64) } +#endif +}; +static const xstate_info_t thread_state32[] = { + [FP] = { x86_FLOAT_STATE32, x86_FLOAT_STATE32_COUNT, sizeof(struct mcontext32) }, + [AVX] = { x86_AVX_STATE32, x86_AVX_STATE32_COUNT, sizeof(struct mcontext_avx32) }, +#if !defined(RC_HIDE_XNU_J137) + [AVX512] = { x86_AVX512_STATE32, x86_AVX512_STATE32_COUNT, sizeof(struct mcontext_avx512_32) } +#endif +}; + /* * NOTE: Source and target may *NOT* overlap! * XXX: Unify with bsd/kern/kern_exit.c @@ -139,8 +162,12 @@ void sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused uint32_t code) { union { - struct mcontext_avx32 mctx_avx32; - struct mcontext_avx64 mctx_avx64; + struct mcontext_avx32 mctx_avx32; + struct mcontext_avx64 mctx_avx64; +#if !defined(RC_HIDE_XNU_J137) + struct mcontext_avx512_32 mctx_avx512_32; + struct mcontext_avx512_64 mctx_avx512_64; +#endif } mctx_store, *mctxp = &mctx_store; user_addr_t ua_sp; @@ -162,7 +189,7 @@ sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused uint struct uthread * ut; int stack_size = 0; int infostyle = UC_TRAD; - boolean_t sig_avx; + xstate_t sig_xstate; thread = current_thread(); ut = get_bsdthread_info(thread); @@ -183,7 +210,8 @@ sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused uint sinfo64.si_signo = sig; bzero(mctxp, sizeof(*mctxp)); - sig_avx = ml_fpu_avx_enabled(); + + sig_xstate = current_xstate(); if (proc_is64bit(p)) { x86_thread_state64_t *tstate64; @@ -195,14 +223,8 @@ sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused uint if (thread_getstatus(thread, flavor, (thread_state_t)state, &state_count) != KERN_SUCCESS) goto bad; - if (sig_avx) { - flavor = x86_AVX_STATE64; - state_count = x86_AVX_STATE64_COUNT; - } - else { - flavor = x86_FLOAT_STATE64; - state_count = x86_FLOAT_STATE64_COUNT; - } + flavor = thread_state64[sig_xstate].flavor; + state_count = thread_state64[sig_xstate].state_count; state = (void *)&mctxp->mctx_avx64.fs; if (thread_getstatus(thread, flavor, (thread_state_t)state, &state_count) != KERN_SUCCESS) goto bad; @@ -236,7 +258,7 @@ sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused uint ua_sp -= sizeof (user64_siginfo_t); ua_sip = ua_sp; - ua_sp -= sizeof (struct mcontext_avx64); + ua_sp -= thread_state64[sig_xstate].mcontext_size; ua_mctxp = ua_sp; /* @@ -265,13 +287,13 @@ sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused uint uctx64.uc_stack.ss_flags |= SS_ONSTACK; uctx64.uc_link = 0; - uctx64.uc_mcsize = sig_avx ? sizeof(struct mcontext_avx64) : sizeof(struct mcontext64); + uctx64.uc_mcsize = thread_state64[sig_xstate].mcontext_size; uctx64.uc_mcontext64 = ua_mctxp; if (copyout((caddr_t)&uctx64, ua_uctxp, sizeof (uctx64))) goto bad; - if (copyout((caddr_t)&mctxp->mctx_avx64, ua_mctxp, sizeof (struct mcontext_avx64))) + if (copyout((caddr_t)&mctx_store, ua_mctxp, thread_state64[sig_xstate].mcontext_size)) goto bad; sinfo64.pad[0] = tstate64->rsp; @@ -308,15 +330,8 @@ sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused uint if (thread_getstatus(thread, flavor, (thread_state_t)state, &state_count) != KERN_SUCCESS) goto bad; - if (sig_avx) { - flavor = x86_AVX_STATE32; - state_count = x86_AVX_STATE32_COUNT; - } - else { - flavor = x86_FLOAT_STATE32; - state_count = x86_FLOAT_STATE32_COUNT; - } - + flavor = thread_state32[sig_xstate].flavor; + state_count = thread_state32[sig_xstate].state_count; state = (void *)&mctxp->mctx_avx32.fs; if (thread_getstatus(thread, flavor, (thread_state_t)state, &state_count) != KERN_SUCCESS) goto bad; @@ -347,7 +362,7 @@ sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused uint ua_sp -= sizeof (user32_siginfo_t); ua_sip = ua_sp; - ua_sp -= sizeof (struct mcontext_avx32); + ua_sp -= thread_state32[sig_xstate].mcontext_size; ua_mctxp = ua_sp; ua_sp -= sizeof (struct sigframe32); @@ -393,14 +408,14 @@ sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused uint uctx32.uc_stack.ss_flags |= SS_ONSTACK; uctx32.uc_link = 0; - uctx32.uc_mcsize = sig_avx ? sizeof(struct mcontext_avx32) : sizeof(struct mcontext32); + uctx32.uc_mcsize = thread_state64[sig_xstate].mcontext_size; uctx32.uc_mcontext = CAST_DOWN_EXPLICIT(user32_addr_t, ua_mctxp); if (copyout((caddr_t)&uctx32, ua_uctxp, sizeof (uctx32))) goto bad; - if (copyout((caddr_t)&mctxp->mctx_avx32, ua_mctxp, sizeof (struct mcontext_avx32))) + if (copyout((caddr_t)&mctx_store, ua_mctxp, thread_state32[sig_xstate].mcontext_size)) goto bad; sinfo64.pad[0] = tstate32->esp; @@ -513,7 +528,8 @@ sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused uint * in the siginfo to the handler is supposed to only * contain the status, so we have to shift it out. */ - sinfo64.si_status = WEXITSTATUS(status_and_exitcode); + sinfo64.si_status = (WEXITSTATUS(status_and_exitcode) & 0x00FFFFFF) | (((uint32_t)(p->p_xhighbits) << 24) & 0xFF000000); + p->p_xhighbits = 0; break; } } @@ -648,8 +664,12 @@ int sigreturn(struct proc *p, struct sigreturn_args *uap, __unused int *retval) { union { - struct mcontext_avx32 mctx_avx32; - struct mcontext_avx64 mctx_avx64; + struct mcontext_avx32 mctx_avx32; + struct mcontext_avx64 mctx_avx64; +#if !defined(RC_HIDE_XNU_J137) + struct mcontext_avx512_32 mctx_avx512_32; + struct mcontext_avx512_64 mctx_avx512_64; +#endif } mctx_store, *mctxp = &mctx_store; thread_t thread = current_thread(); @@ -663,8 +683,8 @@ sigreturn(struct proc *p, struct sigreturn_args *uap, __unused int *retval) mach_msg_type_number_t fs_count; unsigned int fs_flavor; void * fs; - int rval = EJUSTRETURN; - boolean_t sig_avx; + int rval = EJUSTRETURN; + xstate_t sig_xstate; ut = (struct uthread *)get_bsdthread_info(thread); @@ -681,7 +701,8 @@ sigreturn(struct proc *p, struct sigreturn_args *uap, __unused int *retval) } bzero(mctxp, sizeof(*mctxp)); - sig_avx = ml_fpu_avx_enabled(); + + sig_xstate = current_xstate(); if (proc_is64bit(p)) { struct user_ucontext64 uctx64; @@ -689,7 +710,7 @@ sigreturn(struct proc *p, struct sigreturn_args *uap, __unused int *retval) if ((error = copyin(uap->uctx, (void *)&uctx64, sizeof (uctx64)))) return(error); - if ((error = copyin(uctx64.uc_mcontext64, (void *)&mctxp->mctx_avx64, sizeof (struct mcontext_avx64)))) + if ((error = copyin(uctx64.uc_mcontext64, (void *)mctxp, thread_state64[sig_xstate].mcontext_size))) return(error); onstack = uctx64.uc_onstack & 01; @@ -699,15 +720,8 @@ sigreturn(struct proc *p, struct sigreturn_args *uap, __unused int *retval) ts_count = x86_THREAD_STATE64_COUNT; ts = (void *)&mctxp->mctx_avx64.ss; - if (sig_avx) { - fs_flavor = x86_AVX_STATE64; - fs_count = x86_AVX_STATE64_COUNT; - } - else { - fs_flavor = x86_FLOAT_STATE64; - fs_count = x86_FLOAT_STATE64_COUNT; - } - + fs_flavor = thread_state64[sig_xstate].flavor; + fs_count = thread_state64[sig_xstate].state_count; fs = (void *)&mctxp->mctx_avx64.fs; } else { @@ -716,7 +730,7 @@ sigreturn(struct proc *p, struct sigreturn_args *uap, __unused int *retval) if ((error = copyin(uap->uctx, (void *)&uctx32, sizeof (uctx32)))) return(error); - if ((error = copyin(CAST_USER_ADDR_T(uctx32.uc_mcontext), (void *)&mctxp->mctx_avx32, sizeof (struct mcontext_avx32)))) + if ((error = copyin(CAST_USER_ADDR_T(uctx32.uc_mcontext), (void *)mctxp, thread_state32[sig_xstate].mcontext_size))) return(error); onstack = uctx32.uc_onstack & 01; @@ -726,15 +740,8 @@ sigreturn(struct proc *p, struct sigreturn_args *uap, __unused int *retval) ts_count = x86_THREAD_STATE32_COUNT; ts = (void *)&mctxp->mctx_avx32.ss; - if (sig_avx) { - fs_flavor = x86_AVX_STATE32; - fs_count = x86_AVX_STATE32_COUNT; - } - else { - fs_flavor = x86_FLOAT_STATE32; - fs_count = x86_FLOAT_STATE32_COUNT; - } - + fs_flavor = thread_state32[sig_xstate].flavor; + fs_count = thread_state32[sig_xstate].state_count; fs = (void *)&mctxp->mctx_avx32.fs; } diff --git a/bsd/dev/monotonic.c b/bsd/dev/monotonic.c new file mode 100644 index 000000000..91ef1f2bd --- /dev/null +++ b/bsd/dev/monotonic.c @@ -0,0 +1,459 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include <kern/monotonic.h> +#include <machine/machine_routines.h> +#include <machine/monotonic.h> +#include <pexpert/pexpert.h> +#include <sys/param.h> /* NULL */ +#include <sys/stat.h> /* dev_t */ +#include <miscfs/devfs/devfs.h> /* must come after sys/stat.h */ +#include <sys/conf.h> /* must come after sys/stat.h */ +#include <sys/sysctl.h> +#include <sys/sysproto.h> +#include <sys/systm.h> +#include <sys/types.h> +#include <sys/monotonic.h> + +static int mt_dev_open(dev_t dev, int flags, int devtype, struct proc *p); +static int mt_dev_close(dev_t dev, int flags, int devtype, struct proc *p); +static int mt_dev_ioctl(dev_t dev, unsigned long cmd, char *uptr, int fflag, + struct proc *p); + +static struct cdevsw mt_cdevsw = { + .d_open = mt_dev_open, + .d_close = mt_dev_close, + .d_read = eno_rdwrt, + .d_write = eno_rdwrt, + .d_ioctl = mt_dev_ioctl, + .d_stop = eno_stop, + .d_reset = eno_reset, + .d_ttys = NULL, + .d_select = eno_select, + .d_mmap = eno_mmap, + .d_strategy = eno_strat, + .d_type = 0 +}; + +/* + * Written at initialization, read-only thereafter. + */ +lck_grp_t *mt_lock_grp = NULL; + +static int mt_dev_major; +decl_lck_mtx_data(static, mt_dev_mtxs[MT_NDEVS]); +static bool mt_dev_owned[MT_NDEVS]; + +static void +mt_dev_lock(dev_t dev) +{ + lck_mtx_lock(&mt_dev_mtxs[minor(dev)]); +} + +static void +mt_dev_unlock(dev_t dev) +{ + lck_mtx_unlock(&mt_dev_mtxs[minor(dev)]); +} + +static void +mt_dev_assert_lock_held(__assert_only dev_t dev) +{ + LCK_MTX_ASSERT(&mt_dev_mtxs[minor(dev)], LCK_MTX_ASSERT_OWNED); +} + +int +mt_dev_init(void) +{ + lck_grp_attr_t *lock_grp_attr = NULL; + int devices = 0; + + lock_grp_attr = lck_grp_attr_alloc_init(); + mt_lock_grp = lck_grp_alloc_init("monotonic", lock_grp_attr); + lck_grp_attr_free(lock_grp_attr); + + mt_dev_major = cdevsw_add(-1 /* allocate a major number */, &mt_cdevsw); + if (mt_dev_major < 0) { + panic("monotonic: cdevsw_add failed: %d", mt_dev_major); + __builtin_trap(); + } + + for (int i = 0; i < MT_NDEVS; i++) { + dev_t dev; + void *dn; + int error; + + error = monotonic_devs[i].mtd_init(); + if (error) { + continue; + } + + dev = makedev(mt_dev_major, i); + dn = devfs_make_node(dev, + DEVFS_CHAR, UID_ROOT, GID_WINDOWSERVER, 0666, + monotonic_devs[i].mtd_name); + if (dn == NULL) { + panic("monotonic: devfs_make_node failed for '%s'", + monotonic_devs[i].mtd_name); + __builtin_trap(); + } + + lck_mtx_init(&mt_dev_mtxs[i], mt_lock_grp, LCK_ATTR_NULL); + + devices++; + } + + return 0; +} + +static int +mt_dev_open(dev_t dev, __unused int flags, __unused int devtype, + __unused struct proc *p) +{ + int error = 0; + + mt_dev_lock(dev); + + if (mt_dev_owned[minor(dev)]) { + error = EBUSY; + goto out; + } + + mt_dev_owned[minor(dev)] = true; + +out: + mt_dev_unlock(dev); + return error; +} + +static int +mt_dev_close(dev_t dev, __unused int flags, __unused int devtype, + __unused struct proc *p) +{ + mt_dev_lock(dev); + + assert(mt_dev_owned[minor(dev)]); + mt_dev_owned[minor(dev)] = false; + + monotonic_devs[minor(dev)].mtd_reset(); + + mt_dev_unlock(dev); + + return 0; +} + +static int +mt_ctl_add(dev_t dev, user_addr_t uptr, __unused int flags, + __unused struct proc *p) +{ + int error; + uint32_t ctr; + union monotonic_ctl_add ctl; + + mt_dev_assert_lock_held(dev); + + error = copyin(uptr, &ctl, sizeof(ctl.in)); + if (error) { + return error; + } + + error = monotonic_devs[minor(dev)].mtd_add(&ctl.in.config, &ctr); + if (error) { + return error; + } + + ctl.out.ctr = ctr; + + error = copyout(&ctl, uptr, sizeof(ctl.out)); + if (error) { + return error; + } + + return 0; +} + +static int +mt_ctl_counts(dev_t dev, user_addr_t uptr, __unused int flags, + __unused struct proc *p) +{ + int error; + uint64_t ctrs; + union monotonic_ctl_counts ctl; + + mt_dev_assert_lock_held(dev); + + error = copyin(uptr, &ctl, sizeof(ctl.in)); + if (error) { + return error; + } + + if (ctl.in.ctr_mask == 0) { + return EINVAL; + } + ctrs = __builtin_popcountll(ctl.in.ctr_mask); + + { + uint64_t counts[ctrs]; + error = monotonic_devs[minor(dev)].mtd_read(ctl.in.ctr_mask, counts); + if (error) { + return error; + } + + error = copyout(&counts, uptr, sizeof(counts)); + if (error) { + return error; + } + } + + return 0; +} + +static int +mt_ctl_enable(dev_t dev, user_addr_t uptr) +{ + int error; + union monotonic_ctl_enable ctl; + + mt_dev_assert_lock_held(dev); + + error = copyin(uptr, &ctl, sizeof(ctl)); + if (error) { + return error; + } + + monotonic_devs[minor(dev)].mtd_enable(ctl.in.enable); + + return 0; +} + +static int +mt_ctl_reset(dev_t dev) +{ + mt_dev_assert_lock_held(dev); + monotonic_devs[minor(dev)].mtd_reset(); + return 0; +} + +static int +mt_dev_ioctl(dev_t dev, unsigned long cmd, char *arg, int flags, + struct proc *p) +{ + int error; + user_addr_t uptr = *(user_addr_t *)(void *)arg; + + mt_dev_lock(dev); + + switch (cmd) { + case MT_IOC_RESET: + error = mt_ctl_reset(dev); + break; + + case MT_IOC_ADD: + error = mt_ctl_add(dev, uptr, flags, p); + break; + + case MT_IOC_ENABLE: + error = mt_ctl_enable(dev, uptr); + break; + + case MT_IOC_COUNTS: + error = mt_ctl_counts(dev, uptr, flags, p); + break; + + default: + error = ENODEV; + break; + } + + mt_dev_unlock(dev); + + return error; +} + +int thread_selfcounts(__unused struct proc *p, + struct thread_selfcounts_args *uap, __unused int *ret_out) +{ + switch (uap->type) { + case 1: { + uint64_t counts[2] = {}; + uint64_t thread_counts[MT_CORE_NFIXED]; + + mt_cur_thread_fixed_counts(thread_counts); + +#ifdef MT_CORE_INSTRS + counts[0] = thread_counts[MT_CORE_INSTRS]; +#endif /* defined(MT_CORE_INSTRS) */ + counts[1] = thread_counts[MT_CORE_CYCLES]; + + return copyout(counts, uap->buf, MIN(sizeof(counts), uap->nbytes)); + } + default: + return EINVAL; + } +} + +enum mt_sysctl { + MT_SUPPORTED, + MT_PMIS, + MT_RETROGRADE, + MT_TASK_THREAD, + MT_DEBUG, + MT_KDBG_TEST, + MT_FIX_CPU_PERF, + MT_FIX_THREAD_PERF, + MT_FIX_TASK_PERF, +}; + +static int +mt_sysctl SYSCTL_HANDLER_ARGS +{ +#pragma unused(oidp, arg2) + uint64_t start[MT_CORE_NFIXED], end[MT_CORE_NFIXED]; + uint64_t counts[2] = {}; + + switch ((enum mt_sysctl)arg1) { + case MT_SUPPORTED: + return sysctl_io_number(req, mt_core_supported, sizeof(mt_core_supported), NULL, NULL); + case MT_PMIS: + return sysctl_io_number(req, mt_pmis, sizeof(mt_pmis), NULL, NULL); + case MT_RETROGRADE: + return sysctl_io_number(req, mt_retrograde, sizeof(mt_retrograde), NULL, NULL); + case MT_TASK_THREAD: + return sysctl_io_number(req, mt_core_supported, sizeof(mt_core_supported), NULL, NULL); + case MT_DEBUG: { + int value = mt_debug; + + int r = sysctl_io_number(req, value, sizeof(value), &value, NULL); + if (r) { + return r; + } + mt_debug = value; + + return 0; + } + case MT_KDBG_TEST: { + if (req->newptr == USER_ADDR_NULL) { + return EINVAL; + } + + int intrs_en = ml_set_interrupts_enabled(FALSE); + MT_KDBG_TMPCPU_START(0x3fff); + MT_KDBG_TMPCPU_END(0x3fff); + + MT_KDBG_TMPTH_START(0x3fff); + MT_KDBG_TMPTH_END(0x3fff); + ml_set_interrupts_enabled(intrs_en); + + return 0; + } + case MT_FIX_CPU_PERF: { + int intrs_en = ml_set_interrupts_enabled(FALSE); + mt_fixed_counts(start); + mt_fixed_counts(end); + ml_set_interrupts_enabled(intrs_en); + + goto copyout_counts; + } + case MT_FIX_THREAD_PERF: { + int intrs_en = ml_set_interrupts_enabled(FALSE); + mt_cur_thread_fixed_counts(start); + mt_cur_thread_fixed_counts(end); + ml_set_interrupts_enabled(intrs_en); + + goto copyout_counts; + } + case MT_FIX_TASK_PERF: { + int intrs_en = ml_set_interrupts_enabled(FALSE); + mt_cur_task_fixed_counts(start); + mt_cur_task_fixed_counts(end); + ml_set_interrupts_enabled(intrs_en); + + goto copyout_counts; + } + default: + return ENOENT; + } + +copyout_counts: + +#ifdef MT_CORE_INSTRS + counts[0] = end[MT_CORE_INSTRS] - start[MT_CORE_INSTRS]; +#endif /* defined(MT_CORE_INSTRS) */ + counts[1] = end[MT_CORE_CYCLES] - start[MT_CORE_CYCLES]; + + return copyout(counts, req->oldptr, MIN(req->oldlen, sizeof(counts))); +} + +SYSCTL_DECL(_kern_monotonic); +SYSCTL_NODE(_kern, OID_AUTO, monotonic, CTLFLAG_RW | CTLFLAG_LOCKED, 0, + "monotonic"); + +SYSCTL_PROC(_kern_monotonic, OID_AUTO, supported, + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED, + (void *)MT_SUPPORTED, sizeof(int), mt_sysctl, "I", + "whether monotonic is supported"); + +SYSCTL_PROC(_kern_monotonic, OID_AUTO, debug, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MASKED, + (void *)MT_DEBUG, sizeof(int), mt_sysctl, "I", + "whether monotonic is printing debug messages"); + +SYSCTL_PROC(_kern_monotonic, OID_AUTO, pmis, + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED, + (void *)MT_PMIS, sizeof(uint64_t), mt_sysctl, "Q", + "how many PMIs have been seen"); + +SYSCTL_PROC(_kern_monotonic, OID_AUTO, retrograde_updates, + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED, + (void *)MT_RETROGRADE, sizeof(uint64_t), mt_sysctl, "Q", + "how many times a counter appeared to go backwards"); + +SYSCTL_PROC(_kern_monotonic, OID_AUTO, task_thread_counting, + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MASKED, + (void *)MT_TASK_THREAD, sizeof(int), mt_sysctl, "I", + "task and thread counting enabled"); + +SYSCTL_PROC(_kern_monotonic, OID_AUTO, kdebug_test, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MASKED | CTLFLAG_LOCKED, + (void *)MT_KDBG_TEST, sizeof(int), mt_sysctl, "O", + "test that kdebug integration works"); + +SYSCTL_PROC(_kern_monotonic, OID_AUTO, fixed_cpu_perf, + CTLFLAG_RW | CTLFLAG_MASKED | CTLFLAG_LOCKED, + (void *)MT_FIX_CPU_PERF, sizeof(uint64_t) * 2, mt_sysctl, "O", + "overhead of accessing the current CPU's counters"); + +SYSCTL_PROC(_kern_monotonic, OID_AUTO, fixed_thread_perf, + CTLFLAG_RW | CTLFLAG_MASKED | CTLFLAG_LOCKED, + (void *)MT_FIX_THREAD_PERF, sizeof(uint64_t) * 2, mt_sysctl, "O", + "overhead of accessing the current thread's counters"); + +SYSCTL_PROC(_kern_monotonic, OID_AUTO, fixed_task_perf, + CTLFLAG_RW | CTLFLAG_MASKED | CTLFLAG_LOCKED, + (void *)MT_FIX_TASK_PERF, sizeof(uint64_t) * 2, mt_sysctl, "O", + "overhead of accessing the current task's counters"); diff --git a/bsd/dev/munge.c b/bsd/dev/munge.c index 227720935..e44a638d6 100644 --- a/bsd/dev/munge.c +++ b/bsd/dev/munge.c @@ -536,6 +536,22 @@ munge_lwww(void *args) out_args[0] = *(volatile uint64_t*)&in_args[0]; } +void +munge_lwwwwwww(void *args) +{ + volatile uint64_t *out_args = (volatile uint64_t*)args; + volatile uint32_t *in_args = (volatile uint32_t*)args; + + out_args[7] = in_args[8]; + out_args[6] = in_args[7]; + out_args[5] = in_args[6]; + out_args[4] = in_args[5]; + out_args[3] = in_args[4]; + out_args[2] = in_args[3]; + out_args[1] = in_args[2]; + out_args[0] = *(volatile uint64_t*)&in_args[0]; +} + void munge_wwlww(void *args) { diff --git a/bsd/dev/unix_startup.c b/bsd/dev/unix_startup.c index 98cbcce99..a63db5b5f 100644 --- a/bsd/dev/unix_startup.c +++ b/bsd/dev/unix_startup.c @@ -85,7 +85,7 @@ int nbuf_headers = 0; #endif SYSCTL_INT (_kern, OID_AUTO, nbuf, CTLFLAG_RD | CTLFLAG_LOCKED, &nbuf_headers, 0, ""); -SYSCTL_INT (_kern, OID_AUTO, maxnbuf, CTLFLAG_RW | CTLFLAG_LOCKED, &max_nbuf_headers, 0, ""); +SYSCTL_INT (_kern, OID_AUTO, maxnbuf, CTLFLAG_RW | CTLFLAG_LOCKED | CTLFLAG_KERN, &max_nbuf_headers, 0, ""); __private_extern__ int customnbuf = 0; int serverperfmode = 0; /* Flag indicates a server boot when set */ @@ -140,7 +140,9 @@ bsd_startupearly(void) &firstaddr, size, FALSE, - VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_FILE), + VM_FLAGS_ANYWHERE, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_FILE, &bufferhdr_map); if (ret != KERN_SUCCESS) @@ -219,7 +221,9 @@ bsd_bufferinit(void) (vm_offset_t *) &mbutl, (vm_size_t) (nmbclusters * MCLBYTES), FALSE, - VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_MBUF), + VM_FLAGS_ANYWHERE, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_MBUF, &mb_map); if (ret != KERN_SUCCESS) @@ -317,6 +321,10 @@ bsd_scale_setup(int scale) if ((scale > 0) && (serverperfmode == 0)) { maxproc *= scale; maxprocperuid = (maxproc * 2) / 3; + if (scale > 2) { + maxfiles *= scale; + maxfilesperproc = maxfiles/2; + } } /* Apply server scaling rules */ if ((scale > 0) && (serverperfmode !=0)) { diff --git a/bsd/i386/_mcontext.h b/bsd/i386/_mcontext.h index 0abb4c87a..e2544d076 100644 --- a/bsd/i386/_mcontext.h +++ b/bsd/i386/_mcontext.h @@ -29,8 +29,9 @@ #ifndef __I386_MCONTEXT_H_ #define __I386_MCONTEXT_H_ +#include <sys/cdefs.h> /* __DARWIN_UNIX03 */ #include <sys/appleapiopts.h> -#include <mach/i386/_structs.h> +#include <mach/machine/_structs.h> #ifndef _STRUCT_MCONTEXT32 #if __DARWIN_UNIX03 @@ -50,6 +51,18 @@ _STRUCT_MCONTEXT_AVX32 _STRUCT_X86_AVX_STATE32 __fs; }; +#if !defined(RC_HIDE_XNU_J137) +#if defined(_STRUCT_X86_AVX512_STATE32) +#define _STRUCT_MCONTEXT_AVX512_32 struct __darwin_mcontext_avx512_32 +_STRUCT_MCONTEXT_AVX512_32 +{ + _STRUCT_X86_EXCEPTION_STATE32 __es; + _STRUCT_X86_THREAD_STATE32 __ss; + _STRUCT_X86_AVX512_STATE32 __fs; +}; +#endif /* _STRUCT_X86_AVX512_STATE32 */ +#endif /* RC_HIDE_XNU_J137 */ + #else /* !__DARWIN_UNIX03 */ #define _STRUCT_MCONTEXT32 struct mcontext32 _STRUCT_MCONTEXT32 @@ -67,6 +80,18 @@ _STRUCT_MCONTEXT_AVX32 _STRUCT_X86_AVX_STATE32 fs; }; +#if !defined(RC_HIDE_XNU_J137) +#if defined(_STRUCT_X86_AVX512_STATE32) +#define _STRUCT_MCONTEXT_AVX512_32 struct mcontext_avx512_32 +_STRUCT_MCONTEXT_AVX512_32 +{ + _STRUCT_X86_EXCEPTION_STATE32 es; + _STRUCT_X86_THREAD_STATE32 ss; + _STRUCT_X86_AVX512_STATE32 fs; +}; +#endif /* _STRUCT_X86_AVX512_STATE32 */ +#endif /* RC_HIDE_XNU_J137 */ + #endif /* __DARWIN_UNIX03 */ #endif /* _STRUCT_MCONTEXT32 */ @@ -88,6 +113,18 @@ _STRUCT_MCONTEXT_AVX64 _STRUCT_X86_AVX_STATE64 __fs; }; +#if !defined(RC_HIDE_XNU_J137) +#if defined(_STRUCT_X86_AVX512_STATE64) +#define _STRUCT_MCONTEXT_AVX512_64 struct __darwin_mcontext_avx512_64 +_STRUCT_MCONTEXT_AVX512_64 +{ + _STRUCT_X86_EXCEPTION_STATE64 __es; + _STRUCT_X86_THREAD_STATE64 __ss; + _STRUCT_X86_AVX512_STATE64 __fs; +}; +#endif /* _STRUCT_X86_AVX512_STATE64 */ +#endif /* RC_HIDE_XNU_J137 */ + #else /* !__DARWIN_UNIX03 */ #define _STRUCT_MCONTEXT64 struct mcontext64 _STRUCT_MCONTEXT64 @@ -105,6 +142,18 @@ _STRUCT_MCONTEXT_AVX64 _STRUCT_X86_AVX_STATE64 fs; }; +#if !defined(RC_HIDE_XNU_J137) +#if defined(_STRUCT_X86_AVX512_STATE64) +#define _STRUCT_MCONTEXT_AVX512_64 struct mcontext_avx512_64 +_STRUCT_MCONTEXT_AVX512_64 +{ + _STRUCT_X86_EXCEPTION_STATE64 es; + _STRUCT_X86_THREAD_STATE64 ss; + _STRUCT_X86_AVX512_STATE64 fs; +}; +#endif /* _STRUCT_X86_AVX512_STATE64 */ +#endif /* RC_HIDE_XNU_J137 */ + #endif /* __DARWIN_UNIX03 */ #endif /* _STRUCT_MCONTEXT64 */ diff --git a/bsd/i386/dis_tables.h b/bsd/i386/dis_tables.h index 5367b5277..6e2ec7f54 100644 --- a/bsd/i386/dis_tables.h +++ b/bsd/i386/dis_tables.h @@ -37,6 +37,13 @@ * for usage information and documentation. */ +/* + * APPLE NOTE: There is a copy of this file for userspace in + * dtrace:sys_dis_tables.h + * + * It needs to be in sync with this file. + */ + #ifdef __cplusplus extern "C" { #endif @@ -54,7 +61,7 @@ extern "C" { #define OPLEN 256 #define PFIXLEN 8 -#define NCPS 12 /* number of chars per symbol */ +#define NCPS 20 /* number of chars per symbol */ /* * data structures that must be provided to dtrace_dis86() @@ -81,6 +88,7 @@ typedef struct dis86 { uint_t d86_opnd_size; uint_t d86_addr_size; uint_t d86_got_modrm; + uint_t d86_vsib; /* Has a VSIB */ struct d86opnd d86_opnd[4]; /* up to 4 operands */ int (*d86_check_func)(void *); int (*d86_get_byte)(void *); diff --git a/bsd/i386/fasttrap_isa.h b/bsd/i386/fasttrap_isa.h index a71101a2d..c2392eb0c 100644 --- a/bsd/i386/fasttrap_isa.h +++ b/bsd/i386/fasttrap_isa.h @@ -57,6 +57,8 @@ typedef struct fasttrap_machtp { uint8_t ftmt_scale; /* branch scale */ uint8_t ftmt_segment; /* segment for memory accesses */ user_addr_t ftmt_dest; /* destination of control flow */ + uint8_t ftmt_installed:1; + uint8_t ftmt_retired:1; } fasttrap_machtp_t; #define ftt_instr ftt_mtp.ftmt_instr @@ -70,6 +72,9 @@ typedef struct fasttrap_machtp { #define ftt_scale ftt_mtp.ftmt_scale #define ftt_segment ftt_mtp.ftmt_segment #define ftt_dest ftt_mtp.ftmt_dest +#define ftt_installed ftt_mtp.ftmt_installed +#define ftt_retired ftt_mtp.ftmt_retired + #define FASTTRAP_T_COMMON 0x00 /* common case -- no emulation */ #define FASTTRAP_T_JCC 0x01 /* near and far conditional jumps */ diff --git a/bsd/kern/ast.h b/bsd/kern/ast.h index 00a8fa199..43d896fd9 100644 --- a/bsd/kern/ast.h +++ b/bsd/kern/ast.h @@ -39,6 +39,12 @@ extern void act_set_astbsd(thread_t); extern void bsd_ast(thread_t); +#define AST_KEVENT_RETURN_TO_KERNEL 0x0001 +#define AST_KEVENT_REDRIVE_THREADREQ 0x0002 + +extern void kevent_ast(thread_t thread, uint16_t bits); +extern void act_set_astkevent(thread_t thread, uint16_t bits); + #if CONFIG_DTRACE extern void ast_dtrace_on(void); #endif diff --git a/bsd/kern/bsd_init.c b/bsd/kern/bsd_init.c index 232e966ee..25395d352 100644 --- a/bsd/kern/bsd_init.c +++ b/bsd/kern/bsd_init.c @@ -119,6 +119,7 @@ #include <mach/exception_types.h> #include <dev/busvar.h> /* for pseudo_inits */ #include <sys/kdebug.h> +#include <sys/monotonic.h> #include <sys/reason.h> #include <mach/mach_types.h> @@ -134,6 +135,7 @@ #include <sys/mcache.h> /* for mcache_init() */ #include <sys/mbuf.h> /* for mbinit() */ #include <sys/event.h> /* for knote_init() */ +#include <sys/eventhandler.h> /* for eventhandler_init() */ #include <sys/kern_memorystatus.h> /* for memorystatus_init() */ #include <sys/aio_kern.h> /* for aio_init() */ #include <sys/semaphore.h> /* for psem_cache_init() */ @@ -167,6 +169,7 @@ #include <net/ntstat.h> /* for nstat_init() */ #include <netinet/tcp_cc.h> /* for tcp_cc_init() */ #include <netinet/mptcp_var.h> /* for mptcp_control_register() */ +#include <net/nwk_wq.h> /* for nwk_wq_init */ #include <kern/assert.h> /* for assert() */ #include <sys/kern_overrides.h> /* for init_system_override() */ @@ -240,7 +243,7 @@ int hostnamelen; char domainname[MAXDOMNAMELEN]; int domainnamelen; -char rootdevice[16]; /* device names have at least 9 chars */ +char rootdevice[DEVMAXNAMESIZE]; #if KMEMSTATS struct kmemstats kmemstats[M_LAST]; @@ -249,6 +252,9 @@ struct kmemstats kmemstats[M_LAST]; struct vnode *rootvp; int boothowto = RB_DEBUG; int minimalboot = 0; +#if CONFIG_EMBEDDED +int darkboot = 0; +#endif #if PROC_REF_DEBUG __private_extern__ int proc_ref_tracking_disabled = 0; /* disable panics on leaked proc refs across syscall boundary */ @@ -283,6 +289,9 @@ __private_extern__ vm_offset_t * execargs_cache = NULL; void bsd_exec_setup(int); +#if __arm64__ +__private_extern__ int bootarg_no64exec = 0; +#endif __private_extern__ int bootarg_vnode_cache_defeat = 0; #if CONFIG_JETSAM && (DEVELOPMENT || DEBUG) @@ -381,6 +390,8 @@ extern int (*mountroot)(void); lck_grp_t * proc_lck_grp; lck_grp_t * proc_slock_grp; lck_grp_t * proc_fdmlock_grp; +lck_grp_t * proc_kqhashlock_grp; +lck_grp_t * proc_knhashlock_grp; lck_grp_t * proc_ucred_mlock_grp; lck_grp_t * proc_mlock_grp; lck_grp_attr_t * proc_lck_grp_attr; @@ -476,13 +487,15 @@ bsd_init(void) proc_lck_grp_attr= lck_grp_attr_alloc_init(); proc_lck_grp = lck_grp_alloc_init("proc", proc_lck_grp_attr); + #if CONFIG_FINE_LOCK_GROUPS proc_slock_grp = lck_grp_alloc_init("proc-slock", proc_lck_grp_attr); - proc_fdmlock_grp = lck_grp_alloc_init("proc-fdmlock", proc_lck_grp_attr); proc_ucred_mlock_grp = lck_grp_alloc_init("proc-ucred-mlock", proc_lck_grp_attr); proc_mlock_grp = lck_grp_alloc_init("proc-mlock", proc_lck_grp_attr); + proc_fdmlock_grp = lck_grp_alloc_init("proc-fdmlock", proc_lck_grp_attr); #endif - + proc_kqhashlock_grp = lck_grp_alloc_init("proc-kqhashlock", proc_lck_grp_attr); + proc_knhashlock_grp = lck_grp_alloc_init("proc-knhashlock", proc_lck_grp_attr); /* Allocate proc lock attribute */ proc_lck_attr = lck_attr_alloc_init(); #if 0 @@ -526,7 +539,6 @@ bsd_init(void) * Initialize the MAC Framework */ mac_policy_initbsd(); - kernproc->p_mac_enforce = 0; #if defined (__i386__) || defined (__x86_64__) /* @@ -653,6 +665,8 @@ bsd_init(void) filedesc0.fd_knlist = NULL; filedesc0.fd_knhash = NULL; filedesc0.fd_knhashmask = 0; + lck_mtx_init(&filedesc0.fd_kqhashlock, proc_kqhashlock_grp, proc_lck_attr); + lck_mtx_init(&filedesc0.fd_knhashlock, proc_knhashlock_grp, proc_lck_attr); /* Create the limits structures. */ kernproc->p_limit = &limit0; @@ -689,7 +703,9 @@ bsd_init(void) &minimum, (vm_size_t)bsd_pageable_map_size, TRUE, - VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_BSD), + VM_FLAGS_ANYWHERE, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_BSD, &bsd_pageable_map); if (ret != KERN_SUCCESS) panic("bsd_init: Failed to allocate bsd pageable map"); @@ -705,12 +721,6 @@ bsd_init(void) bsd_init_kprintf("calling bsd_bufferinit\n"); bsd_bufferinit(); - /* Initialize the execve() semaphore */ - bsd_init_kprintf("calling semaphore_create\n"); - - if (ret != KERN_SUCCESS) - panic("bsd_init: Failed to create execve semaphore"); - /* * Initialize the calendar. */ @@ -753,6 +763,10 @@ bsd_init(void) bsd_init_kprintf("calling knote_init\n"); knote_init(); + /* Initialize event handler */ + bsd_init_kprintf("calling eventhandler_init\n"); + eventhandler_init(); + /* Initialize for async IO */ bsd_init_kprintf("calling aio_init\n"); aio_init(); @@ -797,6 +811,8 @@ bsd_init(void) * until everything is ready. */ #if NETWORKING + bsd_init_kprintf("calling nwk_wq_init\n"); + nwk_wq_init(); bsd_init_kprintf("calling dlil_init\n"); dlil_init(); bsd_init_kprintf("calling proto_kpi_init\n"); @@ -812,7 +828,6 @@ bsd_init(void) flow_divert_init(); #endif /* FLOW_DIVERT */ #endif /* SOCKETS */ - kernproc->p_fd->fd_cdir = NULL; kernproc->p_fd->fd_rdir = NULL; @@ -1026,7 +1041,7 @@ bsd_init(void) mountroot_post_hook(); #if 0 /* not yet */ - consider_zone_gc(); + consider_zone_gc(FALSE); #endif bsd_init_kprintf("done\n"); @@ -1056,6 +1071,10 @@ bsdinit_task(void) mac_cred_label_associate_user(p->p_ucred); #endif + vm_init_before_launchd(); + + + bsd_init_kprintf("bsd_do_post - done"); load_init_program(p); lock_trace = 1; @@ -1167,6 +1186,11 @@ parse_bsd_args(void) minimalboot = 1; } +#if __arm64__ + /* disable 64 bit grading */ + if (PE_parse_boot_argn("-no64exec", namep, sizeof (namep))) + bootarg_no64exec = 1; +#endif /* disable vnode_cache_is_authorized() by setting vnode_cache_defeat */ if (PE_parse_boot_argn("-vnode_cache_defeat", namep, sizeof (namep))) @@ -1204,6 +1228,18 @@ parse_bsd_args(void) #endif /* CONFIG_JETSAM && (DEVELOPMENT || DEBUG) */ +#if CONFIG_EMBEDDED + /* + * The darkboot flag is specified by the bootloader and is stored in + * boot_args->bootFlags. This flag is available starting revision 2. + */ + boot_args *args = (boot_args *) PE_state.bootArgs; + if ((args != NULL) && (args->Revision >= kBootArgsRevision2)) { + darkboot = (args->bootFlags & kBootFlagsDarkBoot) ? 1 : 0; + } else { + darkboot = 0; + } +#endif #if PROC_REF_DEBUG if (PE_parse_boot_argn("-disable_procref_tracking", namep, sizeof(namep))) { diff --git a/bsd/kern/bsd_stubs.c b/bsd/kern/bsd_stubs.c index 85931c3e9..7883a1b75 100644 --- a/bsd/kern/bsd_stubs.c +++ b/bsd/kern/bsd_stubs.c @@ -301,24 +301,23 @@ cdevsw_add_with_bdev(int index, struct cdevsw * csw, int bdev) } int -cdevsw_setkqueueok(int index, struct cdevsw * csw, int use_offset) +cdevsw_setkqueueok(int maj, struct cdevsw * csw, int extra_flags) { struct cdevsw * devsw; uint64_t flags = CDEVSW_SELECT_KQUEUE; - if (index < 0 || index >= nchrdev) - return (-1); + if (maj < 0 || maj >= nchrdev) { + return -1; + } - devsw = &cdevsw[index]; + devsw = &cdevsw[maj]; if ((memcmp((char *)devsw, (char *)csw, sizeof(struct cdevsw)) != 0)) { - return (-1); + return -1; } - if (use_offset) { - flags |= CDEVSW_USE_OFFSET; - } + flags |= extra_flags; - cdevsw_flags[index] = flags; + cdevsw_flags[maj] = flags; return 0; } diff --git a/bsd/kern/decmpfs.c b/bsd/kern/decmpfs.c index bb4b8c2ff..ebca25271 100644 --- a/bsd/kern/decmpfs.c +++ b/bsd/kern/decmpfs.c @@ -68,6 +68,7 @@ UNUSED_SYMBOL(decmpfs_validate_compressed_file) #include <sys/decmpfs.h> #include <sys/uio_internal.h> #include <libkern/OSByteOrder.h> +#include <libkern/section_keywords.h> #pragma mark --- debugging --- @@ -196,7 +197,7 @@ _free(char *ret, __unused int type, const char *file, int line) static lck_grp_t *decmpfs_lockgrp; -static decmpfs_registration * decompressors[CMP_MAX]; /* the registered compressors */ +SECURITY_READ_ONLY_EARLY(static decmpfs_registration *) decompressors[CMP_MAX]; /* the registered compressors */ static lck_rw_t * decompressorsLock; static int decompress_channel; /* channel used by decompress_file to wake up waiters */ static lck_mtx_t *decompress_channel_mtx; @@ -211,10 +212,10 @@ static void * _func_from_offset(uint32_t type, uintptr_t offset) { /* get the function at the given offset in the registration for the given type */ - decmpfs_registration *reg = decompressors[type]; - char *regChar = (char*)reg; - char *func = ®Char[offset]; - void **funcPtr = (void**)func; + const decmpfs_registration *reg = decompressors[type]; + const char *regChar = (const char*)reg; + const char *func = ®Char[offset]; + void * const * funcPtr = (void * const *) func; switch (reg->decmpfs_registration) { case DECMPFS_REGISTRATION_VERSION_V1: @@ -948,13 +949,13 @@ decmpfs_hides_xattr(vfs_context_t ctx, decmpfs_cnode *cp, const char *xattr) #pragma mark --- registration/validation routines --- -static inline int registration_valid(decmpfs_registration *registration) +static inline int registration_valid(const decmpfs_registration *registration) { return registration && ((registration->decmpfs_registration == DECMPFS_REGISTRATION_VERSION_V1) || (registration->decmpfs_registration == DECMPFS_REGISTRATION_VERSION_V3)); } errno_t -register_decmpfs_decompressor(uint32_t compression_type, decmpfs_registration *registration) +register_decmpfs_decompressor(uint32_t compression_type, const decmpfs_registration *registration) { /* called by kexts to register decompressors */ @@ -1375,7 +1376,7 @@ decmpfs_read_compressed(struct vnop_read_args *ap, int *is_compressed, decmpfs_c } /* create the upl */ - kr = ubc_create_upl(vp, curUplPos, curUplSize, &upl, &pli, UPL_SET_LITE); + kr = ubc_create_upl_kernel(vp, curUplPos, curUplSize, &upl, &pli, UPL_SET_LITE, VM_KERN_MEMORY_FILE); if (kr != KERN_SUCCESS) { ErrorLogWithPath("ubc_create_upl error %d\n", (int)kr); err = EINVAL; @@ -1842,7 +1843,7 @@ out: return err; } -static decmpfs_registration Type1Reg = +SECURITY_READ_ONLY_EARLY(static decmpfs_registration) Type1Reg = { .decmpfs_registration = DECMPFS_REGISTRATION_VERSION, .validate = decmpfs_validate_compressed_file_Type1, diff --git a/bsd/kern/imageboot.c b/bsd/kern/imageboot.c index 6ba54a69f..493d740f8 100644 --- a/bsd/kern/imageboot.c +++ b/bsd/kern/imageboot.c @@ -42,6 +42,7 @@ #include <sys/vnode.h> #include <sys/sysproto.h> #include <sys/csr.h> +#include <miscfs/devfs/devfsdefs.h> #include <libkern/crypto/sha2.h> #include <libkern/crypto/rsa.h> #include <libkern/OSKextLibPrivate.h> @@ -54,7 +55,7 @@ extern struct filedesc filedesc0; extern int (*mountroot)(void); -extern char rootdevice[]; +extern char rootdevice[DEVMAXNAMESIZE]; #define DEBUG_IMAGEBOOT 0 @@ -64,7 +65,9 @@ extern char rootdevice[]; #define DBG_TRACE(...) do {} while(0) #endif -extern int di_root_image(const char *path, char devname[], dev_t *dev_p); +extern int di_root_image(const char *path, char *devname, size_t devsz, dev_t *dev_p); +extern int di_root_ramfile_buf(void *buf, size_t bufsz, char *devname, size_t devsz, dev_t *dev_p); + static boolean_t imageboot_setup_new(void); #define kIBFilePrefix "file://" @@ -150,7 +153,7 @@ imageboot_mount_image(const char *root_path, int height) vnode_t newdp; mount_t new_rootfs; - error = di_root_image(root_path, rootdevice, &dev); + error = di_root_image(root_path, rootdevice, DEVMAXNAMESIZE, &dev); if (error) { panic("%s: di_root_image failed: %d\n", __FUNCTION__, error); } @@ -259,7 +262,7 @@ read_file(const char *path, void **bufp, size_t *bufszp) proc_t p = vfs_context_proc(ctx); kauth_cred_t kerncred = vfs_context_ucred(ctx); - NDINIT(&ndp, LOOKUP, OP_OPEN, LOCKLEAF, UIO_SYSSPACE, path, ctx); + NDINIT(&ndp, LOOKUP, OP_OPEN, LOCKLEAF, UIO_SYSSPACE, CAST_USER_ADDR_T(path), ctx); if ((err = namei(&ndp)) != 0) { AUTHPRNT("namei failed (%s)", path); goto out; @@ -493,7 +496,7 @@ validate_root_image(const char *root_path, void *chunklist) /* * Open the DMG */ - NDINIT(&ndp, LOOKUP, OP_OPEN, LOCKLEAF, UIO_SYSSPACE, root_path, ctx); + NDINIT(&ndp, LOOKUP, OP_OPEN, LOCKLEAF, UIO_SYSSPACE, CAST_USER_ADDR_T(root_path), ctx); if ((err = namei(&ndp)) != 0) { AUTHPRNT("namei failed (%s)", root_path); goto out; @@ -831,6 +834,96 @@ auth_imgboot_test(proc_t __unused ap, struct auth_imgboot_test_args *uap, int32_ } #endif +/* + * Attach the image at 'path' as a ramdisk and mount it as our new rootfs. + * All existing mounts are first umounted. + */ +static int +imageboot_mount_ramdisk(const char *path) +{ + int err = 0; + size_t bufsz = 0; + void *buf = NULL; + dev_t dev; + vnode_t newdp; + mount_t new_rootfs; + + /* Read our target image from disk */ + err = read_file(path, &buf, &bufsz); + if (err) { + printf("%s: failed: read_file() = %d\n", __func__, err); + goto out; + } + DBG_TRACE("%s: read '%s' sz = %lu\n", __func__, path, bufsz); + +#if CONFIG_IMGSRC_ACCESS + /* Re-add all root mounts to the mount list in the correct order... */ + mount_list_remove(rootvnode->v_mount); + for (int i = 0; i < MAX_IMAGEBOOT_NESTING; i++) { + struct vnode *vn = imgsrc_rootvnodes[i]; + if (vn) { + vnode_getalways(vn); + imgsrc_rootvnodes[i] = NULLVP; + + mount_t mnt = vn->v_mount; + mount_lock(mnt); + mnt->mnt_flag |= MNT_ROOTFS; + mount_list_add(mnt); + mount_unlock(mnt); + + vnode_rele(vn); + vnode_put(vn); + } + } + mount_list_add(rootvnode->v_mount); +#endif + + /* ... and unmount everything */ + vnode_get_and_drop_always(rootvnode); + filedesc0.fd_cdir = NULL; + rootvnode = NULL; + vfs_unmountall(); + + /* Attach the ramfs image ... */ + err = di_root_ramfile_buf(buf, bufsz, rootdevice, DEVMAXNAMESIZE, &dev); + if (err) { + printf("%s: failed: di_root_ramfile_buf() = %d\n", __func__, err); + goto out; + } + + /* ... and mount it */ + rootdev = dev; + mountroot = NULL; + err = vfs_mountroot(); + if (err) { + printf("%s: failed: vfs_mountroot() = %d\n", __func__, err); + goto out; + } + + /* Switch to new root vnode */ + if (VFS_ROOT(TAILQ_LAST(&mountlist,mntlist), &newdp, vfs_context_kernel())) { + panic("%s: cannot find root vnode", __func__); + } + rootvnode = newdp; + rootvnode->v_flag |= VROOT; + new_rootfs = rootvnode->v_mount; + mount_lock(new_rootfs); + new_rootfs->mnt_flag |= MNT_ROOTFS; + mount_unlock(new_rootfs); + + vnode_ref(newdp); + vnode_put(newdp); + filedesc0.fd_cdir = newdp; + + DBG_TRACE("%s: root switched\n", __func__); + +out: + if (err) { + kfree_safe(buf); + } + return err; +} + static boolean_t imageboot_setup_new() { @@ -839,10 +932,16 @@ imageboot_setup_new() int height = 0; boolean_t done = FALSE; boolean_t auth_root = FALSE; + boolean_t ramdisk_root = FALSE; MALLOC_ZONE(root_path, caddr_t, MAXPATHLEN, M_NAMEI, M_WAITOK); assert(root_path != NULL); + unsigned imgboot_arg; + if (PE_parse_boot_argn("-rootdmg-ramdisk", &imgboot_arg, sizeof(imgboot_arg))) { + ramdisk_root = TRUE; + } + if (PE_parse_boot_argn(IMAGEBOOT_CONTAINER_ARG, root_path, MAXPATHLEN) == TRUE) { printf("%s: container image url is %s\n", __FUNCTION__, root_path); error = imageboot_mount_image(root_path, height); @@ -871,36 +970,41 @@ imageboot_setup_new() } #endif - if (auth_root) { - /* Copy the path to use locally */ - char *path_alloc = kalloc(MAXPATHLEN); - if (path_alloc == NULL) { - panic("imageboot path allocation failed\n"); - } - - char *path = path_alloc; - strlcpy(path, root_path, MAXPATHLEN); + /* Make a copy of the path to URL-decode */ + char *path_alloc = kalloc(MAXPATHLEN); + if (path_alloc == NULL) { + panic("imageboot path allocation failed\n"); + } + char *path = path_alloc; - size_t len = strlen(kIBFilePrefix); - if (strncmp(kIBFilePrefix, path, len) == 0) { - /* its a URL - remove the file:// prefix and percent-decode */ - path += len; - url_decode(path); - } + size_t len = strlen(kIBFilePrefix); + strlcpy(path, root_path, MAXPATHLEN); + if (strncmp(kIBFilePrefix, path, len) == 0) { + /* its a URL - remove the file:// prefix and percent-decode */ + path += len; + url_decode(path); + } + if (auth_root) { AUTHDBG("authenticating root image at %s", path); error = authenticate_root(path); if (error) { panic("root image authentication failed (err = %d)\n", error); } AUTHDBG("successfully authenticated %s", path); + } - kfree_safe(path_alloc); + if (ramdisk_root) { + error = imageboot_mount_ramdisk(path); + } else { + error = imageboot_mount_image(root_path, height); } - error = imageboot_mount_image(root_path, height); - if (error != 0) { - panic("Failed to mount root image."); + kfree_safe(path_alloc); + + if (error) { + panic("Failed to mount root image (err=%d, auth=%d, ramdisk=%d)\n", + error, auth_root, ramdisk_root); } if (auth_root) { diff --git a/bsd/kern/kdebug.c b/bsd/kern/kdebug.c index d1a1b023b..978b02c49 100644 --- a/bsd/kern/kdebug.c +++ b/bsd/kern/kdebug.c @@ -20,8 +20,6 @@ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ -#include <machine/spl.h> - #include <sys/errno.h> #include <sys/param.h> #include <sys/systm.h> @@ -48,6 +46,7 @@ #include <i386/rtclock_protos.h> #include <i386/mp.h> #include <i386/machine_routines.h> +#include <i386/tsc.h> #endif #include <kern/clock.h> @@ -165,46 +164,46 @@ static typefilter_t typefilter_create(void) static void typefilter_deallocate(typefilter_t tf) { - assert(tf); + assert(tf != NULL); assert(tf != kdbg_typefilter); kmem_free(kernel_map, (vm_offset_t)tf, TYPEFILTER_ALLOC_SIZE); } static void typefilter_copy(typefilter_t dst, typefilter_t src) { - assert(src); - assert(dst); + assert(src != NULL); + assert(dst != NULL); memcpy(dst, src, KDBG_TYPEFILTER_BITMAP_SIZE); } static void typefilter_reject_all(typefilter_t tf) { - assert(tf); + assert(tf != NULL); memset(tf, 0, KDBG_TYPEFILTER_BITMAP_SIZE); } static void typefilter_allow_class(typefilter_t tf, uint8_t class) { - assert(tf); + assert(tf != NULL); const uint32_t BYTES_PER_CLASS = 256 / 8; // 256 subclasses, 1 bit each memset(&tf[class * BYTES_PER_CLASS], 0xFF, BYTES_PER_CLASS); } static void typefilter_allow_csc(typefilter_t tf, uint16_t csc) { - assert(tf); + assert(tf != NULL); setbit(tf, csc); } -static boolean_t typefilter_is_debugid_allowed(typefilter_t tf, uint32_t id) +static bool typefilter_is_debugid_allowed(typefilter_t tf, uint32_t id) { - assert(tf); + assert(tf != NULL); return isset(tf, KDBG_EXTRACT_CSC(id)); } static mach_port_t typefilter_create_memory_entry(typefilter_t tf) { - assert(tf); + assert(tf != NULL); mach_port_t memory_entry = MACH_PORT_NULL; memory_object_size_t size = TYPEFILTER_ALLOC_SIZE; @@ -232,7 +231,22 @@ int cpu_number(void); /* XXX <machine/...> include path broken */ void commpage_update_kdebug_state(void); /* XXX sign */ extern int log_leaks; -extern boolean_t kdebug_serial; + +/* + * This flag is for testing purposes only -- it's highly experimental and tools + * have not been updated to support it. + */ +static bool kdbg_continuous_time = false; + +static inline uint64_t +kdbg_timestamp(void) +{ + if (kdbg_continuous_time) { + return mach_continuous_time(); + } else { + return mach_absolute_time(); + } +} #if KDEBUG_MOJO_TRACE #include <sys/kdebugevents.h> @@ -253,7 +267,7 @@ static int kdbg_setpid(kd_regtype *); static void kdbg_thrmap_init(void); static int kdbg_reinit(boolean_t); static int kdbg_bootstrap(boolean_t); -static int kdbg_test(void); +static int kdbg_test(size_t flavor); static int kdbg_write_v1_header(boolean_t write_thread_map, vnode_t vp, vfs_context_t ctx); static int kdbg_write_thread_map(vnode_t vp, vfs_context_t ctx); @@ -271,7 +285,6 @@ static kd_threadmap *kdbg_thrmap_init_internal(unsigned int count, unsigned int *mapcount); static boolean_t kdebug_current_proc_enabled(uint32_t debugid); -boolean_t kdebug_debugid_enabled(uint32_t debugid); static errno_t kdebug_check_trace_string(uint32_t debugid, uint64_t str_id); int kdbg_write_v3_header(user_addr_t, size_t *, int); @@ -297,10 +310,23 @@ extern void IOSleep(int); unsigned int kdebug_enable = 0; /* A static buffer to record events prior to the start of regular logging */ -#define KD_EARLY_BUFFER_MAX 64 -static kd_buf kd_early_buffer[KD_EARLY_BUFFER_MAX]; -static int kd_early_index = 0; -static boolean_t kd_early_overflow = FALSE; + +#define KD_EARLY_BUFFER_SIZE (16 * 1024) +#define KD_EARLY_BUFFER_NBUFS (KD_EARLY_BUFFER_SIZE / sizeof(kd_buf)) +#if CONFIG_EMBEDDED +/* + * On embedded, the space for this is carved out by osfmk/arm/data.s -- clang + * has problems aligning to greater than 4K. + */ +extern kd_buf kd_early_buffer[KD_EARLY_BUFFER_NBUFS]; +#else /* CONFIG_EMBEDDED */ +__attribute__((aligned(KD_EARLY_BUFFER_SIZE))) +static kd_buf kd_early_buffer[KD_EARLY_BUFFER_NBUFS]; +#endif /* !CONFIG_EMBEDDED */ + +static unsigned int kd_early_index = 0; +static bool kd_early_overflow = false; +static bool kd_early_done = false; #define SLOW_NOLOG 0x01 #define SLOW_CHECKS 0x02 @@ -329,8 +355,10 @@ struct kd_storage { kd_buf kds_records[EVENTS_PER_STORAGE_UNIT]; }; -#define MAX_BUFFER_SIZE (1024 * 1024 * 128) -#define N_STORAGE_UNITS_PER_BUFFER (MAX_BUFFER_SIZE / sizeof(struct kd_storage)) +#define MAX_BUFFER_SIZE (1024 * 1024 * 128) +#define N_STORAGE_UNITS_PER_BUFFER (MAX_BUFFER_SIZE / sizeof(struct kd_storage)) +static_assert(N_STORAGE_UNITS_PER_BUFFER <= 0x7ff, + "shoudn't overflow kds_ptr.offset"); struct kd_storage_buffers { struct kd_storage *kdsb_addr; @@ -339,10 +367,10 @@ struct kd_storage_buffers { #define KDS_PTR_NULL 0xffffffff struct kd_storage_buffers *kd_bufs = NULL; -int n_storage_units = 0; -int n_storage_buffers = 0; -int n_storage_threshold = 0; -int kds_waiter = 0; +int n_storage_units = 0; +unsigned int n_storage_buffers = 0; +int n_storage_threshold = 0; +int kds_waiter = 0; #pragma pack(0) struct kd_bufinfo { @@ -460,7 +488,11 @@ static uint32_t kdbg_cpu_count(boolean_t early_trace) { if (early_trace) { +#if CONFIG_EMBEDDED + return ml_get_cpu_count(); +#else return max_ncpus; +#endif } host_basic_info_data_t hinfo; @@ -471,6 +503,41 @@ kdbg_cpu_count(boolean_t early_trace) } #if MACH_ASSERT +#if CONFIG_EMBEDDED +static boolean_t +kdbg_iop_list_is_valid(kd_iop_t* iop) +{ + if (iop) { + /* Is list sorted by cpu_id? */ + kd_iop_t* temp = iop; + do { + assert(!temp->next || temp->next->cpu_id == temp->cpu_id - 1); + assert(temp->next || (temp->cpu_id == kdbg_cpu_count(FALSE) || temp->cpu_id == kdbg_cpu_count(TRUE))); + } while ((temp = temp->next)); + + /* Does each entry have a function and a name? */ + temp = iop; + do { + assert(temp->callback.func); + assert(strlen(temp->callback.iop_name) < sizeof(temp->callback.iop_name)); + } while ((temp = temp->next)); + } + + return TRUE; +} + +static boolean_t +kdbg_iop_list_contains_cpu_id(kd_iop_t* list, uint32_t cpu_id) +{ + while (list) { + if (list->cpu_id == cpu_id) + return TRUE; + list = list->next; + } + + return FALSE; +} +#endif /* CONFIG_EMBEDDED */ #endif /* MACH_ASSERT */ static void @@ -488,6 +555,10 @@ kdbg_set_tracing_enabled(boolean_t enabled, uint32_t trace_type) int s = ml_set_interrupts_enabled(FALSE); lck_spin_lock(kds_spin_lock); if (enabled) { + /* + * The oldest valid time is now; reject old events from IOPs. + */ + kd_ctrl_page.oldest_time = kdbg_timestamp(); kdebug_enable |= trace_type; kd_ctrl_page.kdebug_slowcheck &= ~SLOW_NOLOG; kd_ctrl_page.enabled = 1; @@ -576,10 +647,10 @@ enable_wrap(uint32_t old_slowcheck, boolean_t lostevents) static int create_buffers(boolean_t early_trace) { - int i; - int p_buffer_size; - int f_buffer_size; - int f_buffers; + unsigned int i; + unsigned int p_buffer_size; + unsigned int f_buffer_size; + unsigned int f_buffers; int error = 0; /* @@ -591,6 +662,9 @@ create_buffers(boolean_t early_trace) */ kd_ctrl_page.kdebug_iops = kd_iops; +#if CONFIG_EMBEDDED + assert(kdbg_iop_list_is_valid(kd_ctrl_page.kdebug_iops)); +#endif /* * If the list is valid, it is sorted, newest -> oldest. Each iop entry @@ -675,7 +749,7 @@ create_buffers(boolean_t early_trace) bzero((char *)kdbip, sizeof(struct kd_bufinfo) * kd_ctrl_page.kdebug_cpus); - for (i = 0; i < (int)kd_ctrl_page.kdebug_cpus; i++) { + for (i = 0; i < kd_ctrl_page.kdebug_cpus; i++) { kdbip[i].kd_list_head.raw = KDS_PTR_NULL; kdbip[i].kd_list_tail.raw = KDS_PTR_NULL; kdbip[i].kd_lostevents = FALSE; @@ -696,7 +770,7 @@ out: static void delete_buffers(void) { - int i; + unsigned int i; if (kd_bufs) { for (i = 0; i < n_storage_buffers; i++) { @@ -858,7 +932,7 @@ allocate_storage_unit(int cpu) kd_ctrl_page.oldest_time = oldest_ts; kd_ctrl_page.kdebug_flags |= KDBG_WRAPPED; } - kdsp_actual->kds_timestamp = mach_absolute_time(); + kdsp_actual->kds_timestamp = kdbg_timestamp(); kdsp_actual->kds_next.raw = KDS_PTR_NULL; kdsp_actual->kds_bufcnt = 0; kdsp_actual->kds_readlast = 0; @@ -976,13 +1050,23 @@ kernel_debug_enter( } } - if (kd_ctrl_page.kdebug_flags & KDBG_WRAPPED) { - if (timestamp < kd_ctrl_page.oldest_time) { - goto out1; - } +record_event: + if (timestamp < kd_ctrl_page.oldest_time) { + goto out1; } -record_event: +#if CONFIG_EMBEDDED + /* + * When start_kern_tracing is called by the kernel to trace very + * early kernel events, it saves data to a secondary buffer until + * it is possible to initialize ktrace, and then dumps the events + * into the ktrace buffer using this method. In this case, iops will + * be NULL, and the coreid will be zero. It is not possible to have + * a valid IOP coreid of zero, so pass if both iops is NULL and coreid + * is zero. + */ + assert(kdbg_iop_list_contains_cpu_id(kd_ctrl_page.kdebug_iops, coreid) || (kd_ctrl_page.kdebug_iops == NULL && coreid == 0)); +#endif disable_preemption(); @@ -1004,8 +1088,10 @@ retry_q: if (kds_raw.raw != KDS_PTR_NULL) { kdsp_actual = POINTER_FROM_KDS_PTR(kds_raw); bindx = kdsp_actual->kds_bufindx; - } else + } else { kdsp_actual = NULL; + bindx = EVENTS_PER_STORAGE_UNIT; + } if (kdsp_actual == NULL || bindx >= EVENTS_PER_STORAGE_UNIT) { if (allocate_storage_unit(coreid) == FALSE) { @@ -1138,7 +1224,7 @@ record_event: #if KDEBUG_MOJO_TRACE if (kdebug_enable & KDEBUG_ENABLE_SERIAL) kdebug_serial_print(cpu, debugid, - mach_absolute_time() & KDBG_TIMESTAMP_MASK, + kdbg_timestamp() & KDBG_TIMESTAMP_MASK, arg1, arg2, arg3, arg4, arg5); #endif @@ -1148,9 +1234,11 @@ retry_q: if (kds_raw.raw != KDS_PTR_NULL) { kdsp_actual = POINTER_FROM_KDS_PTR(kds_raw); bindx = kdsp_actual->kds_bufindx; - } else + } else { kdsp_actual = NULL; - + bindx = EVENTS_PER_STORAGE_UNIT; + } + if (kdsp_actual == NULL || bindx >= EVENTS_PER_STORAGE_UNIT) { if (allocate_storage_unit(cpu) == FALSE) { /* @@ -1161,7 +1249,7 @@ retry_q: } goto retry_q; } - now = mach_absolute_time() & KDBG_TIMESTAMP_MASK; + now = kdbg_timestamp() & KDBG_TIMESTAMP_MASK; if ( !OSCompareAndSwap(bindx, bindx + 1, &kdsp_actual->kds_bufindx)) goto retry_q; @@ -1300,17 +1388,17 @@ kernel_debug_early( uintptr_t arg3, uintptr_t arg4) { - /* If tracing is already initialized, use it */ - if (nkdbufs) { + /* If early tracing is over, use the normal path. */ + if (kd_early_done) { KERNEL_DEBUG_CONSTANT(debugid, arg1, arg2, arg3, arg4, 0); return; } - /* Do nothing if the buffer is full or we're not on the boot cpu */ - kd_early_overflow = kd_early_index >= KD_EARLY_BUFFER_MAX; - if (kd_early_overflow || - cpu_number() != master_cpu) + /* Do nothing if the buffer is full or we're not on the boot cpu. */ + kd_early_overflow = kd_early_index >= KD_EARLY_BUFFER_NBUFS; + if (kd_early_overflow || cpu_number() != master_cpu) { return; + } kd_early_buffer[kd_early_index].debugid = debugid; kd_early_buffer[kd_early_index].timestamp = mach_absolute_time(); @@ -1323,31 +1411,33 @@ kernel_debug_early( } /* - * Transfen the contents of the temporary buffer into the trace buffers. + * Transfer the contents of the temporary buffer into the trace buffers. * Precede that by logging the rebase time (offset) - the TSC-based time (in ns) * when mach_absolute_time is set to 0. */ static void kernel_debug_early_end(void) { - int i; - - if (cpu_number() != master_cpu) + if (cpu_number() != master_cpu) { panic("kernel_debug_early_end() not call on boot processor"); + } + /* reset the current oldest time to allow early events */ + kd_ctrl_page.oldest_time = 0; + +#if !CONFIG_EMBEDDED /* Fake sentinel marking the start of kernel time relative to TSC */ - kernel_debug_enter( - 0, - TRACE_TIMESTAMPS, - 0, - (uint32_t)(tsc_rebase_abs_time >> 32), - (uint32_t)tsc_rebase_abs_time, - 0, - 0, - 0); - for (i = 0; i < kd_early_index; i++) { - kernel_debug_enter( + kernel_debug_enter(0, + TRACE_TIMESTAMPS, + 0, + (uint32_t)(tsc_rebase_abs_time >> 32), + (uint32_t)tsc_rebase_abs_time, + tsc_at_boot, 0, + 0); +#endif + for (unsigned int i = 0; i < kd_early_index; i++) { + kernel_debug_enter(0, kd_early_buffer[i].debugid, kd_early_buffer[i].timestamp, kd_early_buffer[i].arg1, @@ -1358,9 +1448,11 @@ kernel_debug_early_end(void) } /* Cut events-lost event on overflow */ - if (kd_early_overflow) - KERNEL_DEBUG_CONSTANT( - TRACE_LOST_EVENTS, 0, 0, 0, 0, 0); + if (kd_early_overflow) { + KDBG_RELEASE(TRACE_LOST_EVENTS, 1); + } + + kd_early_done = true; /* This trace marks the start of kernel tracing */ kernel_debug_string_early("early trace done"); @@ -1444,11 +1536,12 @@ kdebug_typefilter(__unused struct proc* p, vm_map_t user_map = current_map(); ret = mach_to_bsd_errno( - mach_vm_map(user_map, // target map + mach_vm_map_kernel(user_map, // target map &user_addr, // [in, out] target address TYPEFILTER_ALLOC_SIZE, // initial size 0, // mask (alignment?) VM_FLAGS_ANYWHERE, // flags + VM_KERN_MEMORY_NONE, kdbg_typefilter_memory_entry, // port (memory entry!) 0, // offset (in memory entry) FALSE, // should copy @@ -1632,13 +1725,6 @@ kdebug_current_proc_enabled(uint32_t debugid) return TRUE; } -/* - * Returns false if the debugid is disabled by filters, and true if the - * debugid is allowed to be traced. A debugid may not be traced if the - * typefilter disables its class and subclass, it's outside a range - * check, or if it's not an allowed debugid in a value check. Trace - * system events bypass this check. - */ boolean_t kdebug_debugid_enabled(uint32_t debugid) { @@ -1647,13 +1733,17 @@ kdebug_debugid_enabled(uint32_t debugid) return TRUE; } + return kdebug_debugid_explicitly_enabled(debugid); +} + +boolean_t +kdebug_debugid_explicitly_enabled(uint32_t debugid) +{ if (kd_ctrl_page.kdebug_flags & KDBG_TYPEFILTER_CHECK) { return typefilter_is_debugid_allowed(kdbg_typefilter, debugid); } else if (KDBG_EXTRACT_CLASS(debugid) == DBG_TRACE) { return TRUE; - } - - if (kd_ctrl_page.kdebug_flags & KDBG_RANGECHECK) { + } else if (kd_ctrl_page.kdebug_flags & KDBG_RANGECHECK) { if (debugid < kdlog_beg || debugid > kdlog_end) { return FALSE; } @@ -1861,12 +1951,18 @@ kdbg_reinit(boolean_t early_trace) } void -kdbg_trace_data(struct proc *proc, long *arg_pid) +kdbg_trace_data(struct proc *proc, long *arg_pid, long *arg_uniqueid) { - if (!proc) + if (!proc) { *arg_pid = 0; - else + *arg_uniqueid = 0; + } else { *arg_pid = proc->p_pid; + *arg_uniqueid = proc->p_uniqueid; + if ((uint64_t) *arg_uniqueid != proc->p_uniqueid) { + *arg_uniqueid = 0; + } + } } @@ -2007,7 +2103,7 @@ kdbg_cpumap_init_internal(kd_iop_t* iops, uint32_t cpu_count, uint8_t** cpumap, void kdbg_thrmap_init(void) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); if (kd_ctrl_page.kdebug_flags & KDBG_MAPINIT) { return; @@ -2150,7 +2246,7 @@ kdbg_clear(void) kd_ctrl_page.oldest_time = 0; delete_buffers(); - nkdbufs = 0; + nkdbufs = 0; /* Clean up the thread map buffer */ kdbg_clear_thread_map(); @@ -2162,7 +2258,7 @@ kdbg_clear(void) void kdebug_reset(void) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); kdbg_lock_init(); @@ -2173,6 +2269,13 @@ kdebug_reset(void) } } +void +kdebug_free_early_buf(void) +{ + /* Must be done with the buffer, so release it back to the VM. */ + ml_static_mfree((vm_offset_t)&kd_early_buffer, sizeof(kd_early_buffer)); +} + int kdbg_setpid(kd_regtype *kdr) { @@ -2267,7 +2370,7 @@ kdbg_setpidex(kd_regtype *kdr) static int kdbg_initialize_typefilter(typefilter_t tf) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); assert(!kdbg_typefilter); assert(!kdbg_typefilter_memory_entry); typefilter_t deallocate_tf = NULL; @@ -2300,7 +2403,7 @@ kdbg_copyin_typefilter(user_addr_t addr, size_t size) int ret = ENOMEM; typefilter_t tf; - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); if (size != KDBG_TYPEFILTER_BITMAP_SIZE) { return EINVAL; @@ -2938,7 +3041,7 @@ write_error: static void kdbg_clear_thread_map(void) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); if (kd_ctrl_page.kdebug_flags & KDBG_MAPINIT) { assert(kd_mapptr != NULL); @@ -2964,7 +3067,7 @@ kdbg_write_thread_map(vnode_t vp, vfs_context_t ctx) int ret = 0; boolean_t map_initialized; - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); assert(ctx != NULL); map_initialized = (kd_ctrl_page.kdebug_flags & KDBG_MAPINIT); @@ -2995,7 +3098,7 @@ kdbg_copyout_thread_map(user_addr_t buffer, size_t *buffer_size) size_t map_size; int ret = 0; - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); assert(buffer_size != NULL); map_initialized = (kd_ctrl_page.kdebug_flags & KDBG_MAPINIT); @@ -3023,7 +3126,7 @@ kdbg_readthrmap_v3(user_addr_t buffer, size_t buffer_size, int fd) boolean_t map_initialized; size_t map_size; - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); if ((!fd && !buffer) || (fd && buffer)) { return EINVAL; @@ -3079,6 +3182,8 @@ kdbg_wait(uint64_t timeout_ms, boolean_t locked_wait) int wait_result = THREAD_AWAKENED; uint64_t abstime = 0; + ktrace_assert_lock_held(); + if (timeout_ms != 0) { uint64_t ns = timeout_ms * NSEC_PER_MSEC; nanoseconds_to_absolutetime(ns, &abstime); @@ -3093,7 +3198,7 @@ kdbg_wait(uint64_t timeout_ms, boolean_t locked_wait) if (!locked_wait) { /* drop the mutex to allow others to access trace */ - lck_mtx_unlock(ktrace_lock); + ktrace_unlock(); } while (wait_result == THREAD_AWAKENED && @@ -3118,7 +3223,7 @@ kdbg_wait(uint64_t timeout_ms, boolean_t locked_wait) if (!locked_wait) { /* pick the mutex back up again */ - lck_mtx_lock(ktrace_lock); + ktrace_lock(); } /* write out whether we've exceeded the threshold */ @@ -3189,7 +3294,7 @@ kdbg_control(int *name, u_int namelen, user_addr_t where, size_t *sizep) kdbg_lock_init(); assert(kd_ctrl_page.kdebug_flags & KDBG_LOCKINIT); - lck_mtx_lock(ktrace_lock); + ktrace_lock(); /* * Some requests only require "read" access to kdebug trace. Regardless, @@ -3372,12 +3477,12 @@ kdbg_control(int *name, u_int namelen, user_addr_t where, size_t *sizep) if (name[0] == KERN_KDWRITETR || name[0] == KERN_KDWRITETR_V3) { number = nkdbufs * sizeof(kd_buf); - KERNEL_DEBUG_CONSTANT(TRACE_WRITING_EVENTS | DBG_FUNC_START, 0, 0, 0, 0, 0); + KDBG(TRACE_WRITING_EVENTS | DBG_FUNC_START); if (name[0] == KERN_KDWRITETR_V3) ret = kdbg_read(0, &number, vp, &context, RAW_VERSION3); else ret = kdbg_read(0, &number, vp, &context, RAW_VERSION1); - KERNEL_DEBUG_CONSTANT(TRACE_WRITING_EVENTS | DBG_FUNC_END, number, 0, 0, 0, 0); + KDBG(TRACE_WRITING_EVENTS | DBG_FUNC_END, number); *sizep = number; } else { @@ -3439,7 +3544,7 @@ kdbg_control(int *name, u_int namelen, user_addr_t where, size_t *sizep) } case KERN_KDTEST: - ret = kdbg_test(); + ret = kdbg_test(size); break; default: @@ -3447,9 +3552,9 @@ kdbg_control(int *name, u_int namelen, user_addr_t where, size_t *sizep) break; } out: - lck_mtx_unlock(ktrace_lock); + ktrace_unlock(); - return(ret); + return ret; } @@ -3470,6 +3575,7 @@ kdbg_read(user_addr_t buffer, size_t *number, vnode_t vp, vfs_context_t ctx, uin uint32_t rcursor; kd_buf lostevent; union kds_ptr kdsp; + bool traced_retrograde = false; struct kd_storage *kdsp_actual; struct kd_bufinfo *kdbp; struct kd_bufinfo *min_kdbp; @@ -3485,6 +3591,8 @@ kdbg_read(user_addr_t buffer, size_t *number, vnode_t vp, vfs_context_t ctx, uin count = *number/sizeof(kd_buf); *number = 0; + ktrace_assert_lock_held(); + if (count == 0 || !(kd_ctrl_page.kdebug_flags & KDBG_BUFINIT) || kdcopybuf == 0) return EINVAL; @@ -3500,7 +3608,7 @@ kdbg_read(user_addr_t buffer, size_t *number, vnode_t vp, vfs_context_t ctx, uin * disabled, no new events should occur on the AP. */ if (kd_ctrl_page.enabled) { - barrier_max = mach_absolute_time() & KDBG_TIMESTAMP_MASK; + barrier_max = kdbg_timestamp() & KDBG_TIMESTAMP_MASK; } /* @@ -3666,12 +3774,22 @@ next_cpu: */ if (earliest_time < min_kdbp->kd_prev_timebase) { /* - * if so, use the previous timestamp + 1 cycle + * If we haven't already, emit a retrograde events event. */ - min_kdbp->kd_prev_timebase++; + if (traced_retrograde) { + continue; + } + kdbg_set_timestamp_and_cpu(tempbuf, min_kdbp->kd_prev_timebase, kdbg_get_cpu(tempbuf)); - } else + tempbuf->arg1 = tempbuf->debugid; + tempbuf->arg2 = earliest_time; + tempbuf->arg3 = 0; + tempbuf->arg4 = 0; + tempbuf->debugid = TRACE_RETROGRADE_EVENTS; + traced_retrograde = true; + } else { min_kdbp->kd_prev_timebase = earliest_time; + } nextevent: tempbuf_count--; tempbuf_number++; @@ -3734,45 +3852,70 @@ check_error: } static int -kdbg_test(void) +kdbg_test(size_t flavor) { -#define KDEBUG_TEST_CODE(code) BSDDBG_CODE(DBG_BSD_KDEBUG_TEST, (code)) int code = 0; + int dummy_iop = 0; - KDBG(KDEBUG_TEST_CODE(code)); code++; - KDBG(KDEBUG_TEST_CODE(code), 1); code++; - KDBG(KDEBUG_TEST_CODE(code), 1, 2); code++; - KDBG(KDEBUG_TEST_CODE(code), 1, 2, 3); code++; - KDBG(KDEBUG_TEST_CODE(code), 1, 2, 3, 4); code++; - - KDBG_RELEASE(KDEBUG_TEST_CODE(code)); code++; - KDBG_RELEASE(KDEBUG_TEST_CODE(code), 1); code++; - KDBG_RELEASE(KDEBUG_TEST_CODE(code), 1, 2); code++; - KDBG_RELEASE(KDEBUG_TEST_CODE(code), 1, 2, 3); code++; - KDBG_RELEASE(KDEBUG_TEST_CODE(code), 1, 2, 3, 4); code++; - - KDBG_FILTERED(KDEBUG_TEST_CODE(code)); code++; - KDBG_FILTERED(KDEBUG_TEST_CODE(code), 1); code++; - KDBG_FILTERED(KDEBUG_TEST_CODE(code), 1, 2); code++; - KDBG_FILTERED(KDEBUG_TEST_CODE(code), 1, 2, 3); code++; - KDBG_FILTERED(KDEBUG_TEST_CODE(code), 1, 2, 3, 4); code++; - - KDBG_DEBUG(KDEBUG_TEST_CODE(code)); code++; - KDBG_DEBUG(KDEBUG_TEST_CODE(code), 1); code++; - KDBG_DEBUG(KDEBUG_TEST_CODE(code), 1, 2); code++; - KDBG_DEBUG(KDEBUG_TEST_CODE(code), 1, 2, 3); code++; - KDBG_DEBUG(KDEBUG_TEST_CODE(code), 1, 2, 3, 4); code++; +#define KDEBUG_TEST_CODE(code) BSDDBG_CODE(DBG_BSD_KDEBUG_TEST, (code)) + switch (flavor) { + case 1: + /* try each macro */ + KDBG(KDEBUG_TEST_CODE(code)); code++; + KDBG(KDEBUG_TEST_CODE(code), 1); code++; + KDBG(KDEBUG_TEST_CODE(code), 1, 2); code++; + KDBG(KDEBUG_TEST_CODE(code), 1, 2, 3); code++; + KDBG(KDEBUG_TEST_CODE(code), 1, 2, 3, 4); code++; + + KDBG_RELEASE(KDEBUG_TEST_CODE(code)); code++; + KDBG_RELEASE(KDEBUG_TEST_CODE(code), 1); code++; + KDBG_RELEASE(KDEBUG_TEST_CODE(code), 1, 2); code++; + KDBG_RELEASE(KDEBUG_TEST_CODE(code), 1, 2, 3); code++; + KDBG_RELEASE(KDEBUG_TEST_CODE(code), 1, 2, 3, 4); code++; + + KDBG_FILTERED(KDEBUG_TEST_CODE(code)); code++; + KDBG_FILTERED(KDEBUG_TEST_CODE(code), 1); code++; + KDBG_FILTERED(KDEBUG_TEST_CODE(code), 1, 2); code++; + KDBG_FILTERED(KDEBUG_TEST_CODE(code), 1, 2, 3); code++; + KDBG_FILTERED(KDEBUG_TEST_CODE(code), 1, 2, 3, 4); code++; + + KDBG_DEBUG(KDEBUG_TEST_CODE(code)); code++; + KDBG_DEBUG(KDEBUG_TEST_CODE(code), 1); code++; + KDBG_DEBUG(KDEBUG_TEST_CODE(code), 1, 2); code++; + KDBG_DEBUG(KDEBUG_TEST_CODE(code), 1, 2, 3); code++; + KDBG_DEBUG(KDEBUG_TEST_CODE(code), 1, 2, 3, 4); code++; + break; - return 0; + case 2: + if (kd_ctrl_page.kdebug_iops) { + /* avoid the assertion in kernel_debug_enter for a valid IOP */ + dummy_iop = kd_ctrl_page.kdebug_iops[0].cpu_id; + } + + /* ensure old timestamps are not emitted from kernel_debug_enter */ + kernel_debug_enter(dummy_iop, KDEBUG_TEST_CODE(code), + 100 /* very old timestamp */, 0, 0, 0, + 0, (uintptr_t)thread_tid(current_thread())); + code++; + kernel_debug_enter(dummy_iop, KDEBUG_TEST_CODE(code), + kdbg_timestamp(), 0, 0, 0, 0, + (uintptr_t)thread_tid(current_thread())); + code++; + break; + default: + return ENOTSUP; + } #undef KDEBUG_TEST_CODE + + return 0; } void -kdebug_boot_trace(unsigned int n_events, char *filter_desc) +kdebug_init(unsigned int n_events, char *filter_desc) { assert(filter_desc != NULL); -#if (defined(__i386__) || defined(__x86_64__)) +#if defined(__x86_64__) /* only trace MACH events when outputting kdebug to serial */ if (kdebug_serial) { n_events = 1; @@ -3782,7 +3925,7 @@ kdebug_boot_trace(unsigned int n_events, char *filter_desc) filter_desc[2] = '\0'; } } -#endif +#endif /* defined(__x86_64__) */ if (log_leaks && n_events == 0) { n_events = 200000; @@ -3796,7 +3939,7 @@ kdbg_set_typefilter_string(const char *filter_desc) { char *end = NULL; - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); assert(filter_desc != NULL); @@ -3862,15 +4005,16 @@ kdbg_set_typefilter_string(const char *filter_desc) */ void kdebug_trace_start(unsigned int n_events, const char *filter_desc, - boolean_t need_map) + boolean_t at_wake) { uint32_t old1, old2; if (!n_events) { + kd_early_done = true; return; } - lck_mtx_lock(ktrace_lock); + ktrace_start_single_threaded(); kdbg_lock_init(); @@ -3904,19 +4048,20 @@ kdebug_trace_start(unsigned int n_events, const char *filter_desc, */ boolean_t s = ml_set_interrupts_enabled(FALSE); - if (need_map == TRUE) { + if (at_wake) { kdbg_thrmap_init(); } - kdbg_set_tracing_enabled(TRUE, kdebug_serial ? - (KDEBUG_ENABLE_TRACE | KDEBUG_ENABLE_SERIAL) : - KDEBUG_ENABLE_TRACE); + kdbg_set_tracing_enabled(TRUE, KDEBUG_ENABLE_TRACE | (kdebug_serial ? + KDEBUG_ENABLE_SERIAL : 0)); - /* - * Transfer all very early events from the static buffer into the real - * buffers. - */ - kernel_debug_early_end(); + if (!at_wake) { + /* + * Transfer all very early events from the static buffer into the real + * buffers. + */ + kernel_debug_early_end(); + } ml_set_interrupts_enabled(s); @@ -3927,10 +4072,10 @@ kdebug_trace_start(unsigned int n_events, const char *filter_desc, printf("serial output enabled with %lu named events\n", sizeof(kd_events)/sizeof(kd_event_t)); } -#endif +#endif /* KDEBUG_MOJO_TRACE */ out: - lck_mtx_unlock(ktrace_lock); + ktrace_end_single_threaded(); } void @@ -3939,8 +4084,9 @@ kdbg_dump_trace_to_file(const char *filename) vfs_context_t ctx; vnode_t vp; size_t write_size; + int ret; - lck_mtx_lock(ktrace_lock); + ktrace_lock(); if (!(kdebug_enable & KDEBUG_ENABLE_TRACE)) { goto out; @@ -3949,7 +4095,7 @@ kdbg_dump_trace_to_file(const char *filename) if (ktrace_get_owning_pid() != 0) { /* * Another process owns ktrace and is still active, disable tracing to - * capture whatever was being recorded. + * prevent wrapping. */ kdebug_enable = 0; kd_ctrl_page.enabled = 0; @@ -3957,7 +4103,7 @@ kdbg_dump_trace_to_file(const char *filename) goto out; } - KERNEL_DEBUG_CONSTANT(TRACE_PANIC | DBG_FUNC_NONE, 0, 0, 0, 0, 0); + KDBG(TRACE_WRITING_EVENTS | DBG_FUNC_START); kdebug_enable = 0; kd_ctrl_page.enabled = 0; @@ -3972,13 +4118,39 @@ kdbg_dump_trace_to_file(const char *filename) kdbg_write_thread_map(vp, ctx); write_size = nkdbufs * sizeof(kd_buf); - kdbg_read(0, &write_size, vp, ctx, RAW_VERSION1); + ret = kdbg_read(0, &write_size, vp, ctx, RAW_VERSION1); + if (ret) { + goto out_close; + } + /* + * Wait to synchronize the file to capture the I/O in the + * TRACE_WRITING_EVENTS interval. + */ + ret = VNOP_FSYNC(vp, MNT_WAIT, ctx); + + /* + * Balance the starting TRACE_WRITING_EVENTS tracepoint manually. + */ + kd_buf end_event = { + .debugid = TRACE_WRITING_EVENTS | DBG_FUNC_END, + .arg1 = write_size, + .arg2 = ret, + .arg5 = thread_tid(current_thread()), + }; + kdbg_set_timestamp_and_cpu(&end_event, kdbg_timestamp(), + cpu_number()); + + /* this is best effort -- ignore any errors */ + (void)kdbg_write_to_vnode((caddr_t)&end_event, sizeof(kd_buf), vp, ctx, + RAW_file_offset); + +out_close: vnode_close(vp, FWRITE, ctx); sync(current_proc(), (void *)NULL, (int *)NULL); out: - lck_mtx_unlock(ktrace_lock); + ktrace_unlock(); } /* Helper function for filling in the BSD name for an address space @@ -4002,6 +4174,34 @@ void kdbg_get_task_name(char* name_buf, int len, task_t task) snprintf(name_buf, len, "%p [!bsd]", task); } +static int +kdbg_sysctl_continuous SYSCTL_HANDLER_ARGS +{ +#pragma unused(oidp, arg1, arg2) + int value = kdbg_continuous_time; + int ret = sysctl_io_number(req, value, sizeof(value), &value, NULL); + + if (ret || !req->newptr) { + return ret; + } + + kdbg_continuous_time = value; + return 0; +} + +SYSCTL_NODE(_kern, OID_AUTO, kdbg, CTLFLAG_RD | CTLFLAG_LOCKED, 0, + "kdbg"); + +SYSCTL_PROC(_kern_kdbg, OID_AUTO, experimental_continuous, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, 0, + sizeof(int), kdbg_sysctl_continuous, "I", + "Set kdebug to use mach_continuous_time"); + +SYSCTL_QUAD(_kern_kdbg, OID_AUTO, oldest_time, + CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_LOCKED, + &kd_ctrl_page.oldest_time, + "Find the oldest timestamp still in trace"); + #if KDEBUG_MOJO_TRACE static kd_event_t * binary_search(uint32_t id) diff --git a/bsd/kern/kern_acct.c b/bsd/kern/kern_acct.c index aec90c9e0..8bca8fa41 100644 --- a/bsd/kern/kern_acct.c +++ b/bsd/kern/kern_acct.c @@ -93,7 +93,6 @@ #include <sys/ioctl.h> #include <sys/tty.h> #include <sys/sysproto.h> -#include <machine/spl.h> #if CONFIG_MACF #include <security/mac_framework.h> #endif diff --git a/bsd/kern/kern_aio.c b/bsd/kern/kern_aio.c index 3869ad669..08f8d135f 100644 --- a/bsd/kern/kern_aio.c +++ b/bsd/kern/kern_aio.c @@ -1465,6 +1465,8 @@ lio_listio(proc_t p, struct lio_listio_args *uap, int *retval ) struct user_sigevent aiosigev; aio_lio_context *lio_context; boolean_t free_context = FALSE; + uint32_t *paio_offset; + uint32_t *paio_nbytes; KERNEL_DEBUG( (BSDDBG_CODE(DBG_BSD_AIO, AIO_listio)) | DBG_FUNC_START, (int)p, uap->nent, uap->mode, 0, 0 ); @@ -1596,9 +1598,15 @@ lio_listio(proc_t p, struct lio_listio_args *uap, int *retval ) aio_enqueue_work(p, entryp, 1); aio_proc_unlock(p); - KERNEL_DEBUG( (BSDDBG_CODE(DBG_BSD_AIO, AIO_work_queued)) | DBG_FUNC_NONE, - (int)p, (int)entryp->uaiocbp, 0, 0, 0 ); - } + KERNEL_DEBUG_CONSTANT( (BSDDBG_CODE(DBG_BSD_AIO, AIO_work_queued)) | DBG_FUNC_START, + (int)p, (int)entryp->uaiocbp, entryp->flags, entryp->aiocb.aio_fildes, 0 ); + paio_offset = (uint32_t*) &entryp->aiocb.aio_offset; + paio_nbytes = (uint32_t*) &entryp->aiocb.aio_nbytes; + KERNEL_DEBUG_CONSTANT( (BSDDBG_CODE(DBG_BSD_AIO, AIO_work_queued)) | DBG_FUNC_END, + paio_offset[0], (sizeof(entryp->aiocb.aio_offset) == sizeof(uint64_t) ? paio_offset[1] : 0), + paio_nbytes[0], (sizeof(entryp->aiocb.aio_nbytes) == sizeof(uint64_t) ? paio_nbytes[1] : 0), + 0 ); + } switch(uap->mode) { case LIO_WAIT: @@ -1930,9 +1938,11 @@ static int aio_queue_async_request(proc_t procp, user_addr_t aiocbp, int kindOfIO ) { aio_workq_entry *entryp; - int result; - int old_count; - + int result; + int old_count; + uint32_t *paio_offset; + uint32_t *paio_nbytes; + old_count = aio_increment_total_count(); if (old_count >= aio_max_requests) { result = EAGAIN; @@ -1965,10 +1975,16 @@ aio_queue_async_request(proc_t procp, user_addr_t aiocbp, int kindOfIO ) aio_enqueue_work(procp, entryp, 1); aio_proc_unlock(procp); - - KERNEL_DEBUG( (BSDDBG_CODE(DBG_BSD_AIO, AIO_work_queued)) | DBG_FUNC_NONE, - (int)procp, (int)aiocbp, 0, 0, 0 ); - + + paio_offset = (uint32_t*) &entryp->aiocb.aio_offset; + paio_nbytes = (uint32_t*) &entryp->aiocb.aio_nbytes; + KERNEL_DEBUG_CONSTANT( (BSDDBG_CODE(DBG_BSD_AIO, AIO_work_queued)) | DBG_FUNC_START, + (int)procp, (int)aiocbp, entryp->flags, entryp->aiocb.aio_fildes, 0 ); + KERNEL_DEBUG_CONSTANT( (BSDDBG_CODE(DBG_BSD_AIO, AIO_work_queued)) | DBG_FUNC_END, + paio_offset[0], (sizeof(entryp->aiocb.aio_offset) == sizeof(uint64_t) ? paio_offset[1] : 0), + paio_nbytes[0], (sizeof(entryp->aiocb.aio_nbytes) == sizeof(uint64_t) ? paio_nbytes[1] : 0), + 0 ); + return( 0 ); error_exit: diff --git a/bsd/kern/kern_clock.c b/bsd/kern/kern_clock.c index f0e051345..08507cdc5 100644 --- a/bsd/kern/kern_clock.c +++ b/bsd/kern/kern_clock.c @@ -69,8 +69,6 @@ * HISTORY */ -#include <machine/spl.h> - #include <sys/param.h> #include <sys/systm.h> #include <sys/time.h> @@ -167,6 +165,8 @@ timeout_with_leeway( /* * Cancel a timeout. + * Deprecated because it's very inefficient. + * Switch to an allocated thread call instead. */ void untimeout( @@ -201,6 +201,8 @@ bsd_timeout( /* * Cancel a timeout. + * Deprecated because it's very inefficient. + * Switch to an allocated thread call instead. */ void bsd_untimeout( @@ -253,15 +255,14 @@ static int sysctl_clockrate (__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, __unused struct sysctl_req *req) { - struct clockinfo clkinfo; + struct clockinfo clkinfo = { + .hz = hz, + .tick = tick, + .tickadj = 0, + .stathz = hz, + .profhz = hz, + }; - /* - * Construct clockinfo structure. - */ - clkinfo.hz = hz; - clkinfo.tick = tick; - clkinfo.profhz = hz; - clkinfo.stathz = hz; return sysctl_io_opaque(req, &clkinfo, sizeof(clkinfo), NULL); } diff --git a/bsd/kern/kern_control.c b/bsd/kern/kern_control.c index f7ca33348..cda6f1046 100644 --- a/bsd/kern/kern_control.c +++ b/bsd/kern/kern_control.c @@ -839,7 +839,7 @@ ctl_enqueuembuf_list(void *kctlref, u_int32_t unit, struct mbuf *m_list, errno_t error = 0; struct mbuf *m, *nextpkt; int needwakeup = 0; - int len; + int len = 0; u_int32_t kctlflags; /* @@ -1820,9 +1820,9 @@ ctl_unlock(struct socket *so, int refcount, void *lr) } static lck_mtx_t * -ctl_getlock(struct socket *so, int locktype) +ctl_getlock(struct socket *so, int flags) { -#pragma unused(locktype) +#pragma unused(flags) struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb; if (so->so_pcb) { @@ -1903,15 +1903,15 @@ kctl_reg_list SYSCTL_HANDLER_ARGS xkr->xkr_sendbufsize = kctl->sendbufsize; xkr->xkr_lastunit = kctl->lastunit; xkr->xkr_pcbcount = pcbcount; - xkr->xkr_connect = (uint64_t)VM_KERNEL_ADDRPERM(kctl->connect); + xkr->xkr_connect = (uint64_t)VM_KERNEL_UNSLIDE(kctl->connect); xkr->xkr_disconnect = - (uint64_t)VM_KERNEL_ADDRPERM(kctl->disconnect); - xkr->xkr_send = (uint64_t)VM_KERNEL_ADDRPERM(kctl->send); + (uint64_t)VM_KERNEL_UNSLIDE(kctl->disconnect); + xkr->xkr_send = (uint64_t)VM_KERNEL_UNSLIDE(kctl->send); xkr->xkr_send_list = - (uint64_t)VM_KERNEL_ADDRPERM(kctl->send_list); - xkr->xkr_setopt = (uint64_t)VM_KERNEL_ADDRPERM(kctl->setopt); - xkr->xkr_getopt = (uint64_t)VM_KERNEL_ADDRPERM(kctl->getopt); - xkr->xkr_rcvd = (uint64_t)VM_KERNEL_ADDRPERM(kctl->rcvd); + (uint64_t)VM_KERNEL_UNSLIDE(kctl->send_list); + xkr->xkr_setopt = (uint64_t)VM_KERNEL_UNSLIDE(kctl->setopt); + xkr->xkr_getopt = (uint64_t)VM_KERNEL_UNSLIDE(kctl->getopt); + xkr->xkr_rcvd = (uint64_t)VM_KERNEL_UNSLIDE(kctl->rcvd); strlcpy(xkr->xkr_name, kctl->name, sizeof(xkr->xkr_name)); error = SYSCTL_OUT(req, buf, item_size); diff --git a/bsd/kern/kern_core.c b/bsd/kern/kern_core.c index 5cb6e4fa2..73a9a454b 100644 --- a/bsd/kern/kern_core.c +++ b/bsd/kern/kern_core.c @@ -36,7 +36,7 @@ #include <mach/vm_param.h> #include <mach/thread_status.h> - +#include <sys/content_protection.h> #include <sys/param.h> #include <sys/systm.h> #include <sys/signalvar.h> @@ -83,6 +83,21 @@ mythread_state_flavor_t thread_flavor_array [] = { {x86_EXCEPTION_STATE, x86_EXCEPTION_STATE_COUNT}, }; int mynum_flavors=3; +#elif defined (__arm__) +mythread_state_flavor_t thread_flavor_array[]={ + {ARM_THREAD_STATE , ARM_THREAD_STATE_COUNT}, + {ARM_VFP_STATE, ARM_VFP_STATE_COUNT}, + {ARM_EXCEPTION_STATE, ARM_EXCEPTION_STATE_COUNT} + }; +int mynum_flavors=3; + +#elif defined (__arm64__) +mythread_state_flavor_t thread_flavor_array[]={ + {ARM_THREAD_STATE64 , ARM_THREAD_STATE64_COUNT}, + /* ARM64_TODO: VFP */ + {ARM_EXCEPTION_STATE64, ARM_EXCEPTION_STATE64_COUNT} + }; +int mynum_flavors=2; #else #error architecture not supported #endif @@ -124,6 +139,12 @@ process_cpu_type(proc_t core_proc) } else { what_we_think = CPU_TYPE_I386; } +#elif defined (__arm__) || defined(__arm64__) + if (IS_64BIT_PROCESS(core_proc)) { + what_we_think = CPU_TYPE_ARM64; + } else { + what_we_think = CPU_TYPE_ARM; + } #endif return what_we_think; } @@ -138,6 +159,12 @@ process_cpu_subtype(proc_t core_proc) } else { what_we_think = CPU_SUBTYPE_I386_ALL; } +#elif defined (__arm__) || defined(__arm64__) + if (IS_64BIT_PROCESS(core_proc)) { + what_we_think = CPU_SUBTYPE_ARM64_ALL; + } else { + what_we_think = CPU_SUBTYPE_ARM_ALL; + } #endif return what_we_think; } @@ -310,6 +337,9 @@ coredump(proc_t core_proc, uint32_t reserve_mb, int coredump_flags) VATTR_INIT(&va); /* better to do it here than waste more stack in vnode_setsize */ VATTR_SET(&va, va_data_size, 0); + if (core_proc == initproc) { + VATTR_SET(&va, va_dataprotect_class, PROTECTION_CLASS_D); + } vnode_setattr(vp, &va, ctx); core_proc->p_acflag |= ACORE; diff --git a/bsd/kern/kern_credential.c b/bsd/kern/kern_credential.c index 1376ff3c5..bac6af26f 100644 --- a/bsd/kern/kern_credential.c +++ b/bsd/kern/kern_credential.c @@ -74,6 +74,8 @@ #include <security/_label.h> #endif +#include <IOKit/IOBSD.h> + void mach_kauth_cred_uthread_update( void ); #define CRED_DIAGNOSTIC 0 @@ -186,6 +188,22 @@ TAILQ_HEAD(kauth_resolver_done_head, kauth_resolver_work) kauth_resolver_done; #define KAUTH_COMPLAINT_INTERVAL 1000 int kauth_resolver_timeout_cnt = 0; +#if DEVELOPMENT || DEBUG +/* Internal builds get different (less ambiguous) breadcrumbs. */ +#define KAUTH_RESOLVER_FAILED_ERRCODE EOWNERDEAD +#else +/* But non-Internal builds get errors that are allowed by standards. */ +#define KAUTH_RESOLVER_FAILED_ERRCODE EIO +#endif /* DEVELOPMENT || DEBUG */ + +int kauth_resolver_failed_cnt = 0; +#define RESOLVER_FAILED_MESSAGE(fmt, args...) \ +do { \ + if (!(kauth_resolver_failed_cnt++ % 100)) { \ + printf("%s: " fmt "\n", __PRETTY_FUNCTION__, ##args); \ + } \ +} while (0) + static int kauth_resolver_submit(struct kauth_identity_extlookup *lkp, uint64_t extend_data); static int kauth_resolver_complete(user_addr_t message); static int kauth_resolver_getwork(user_addr_t message); @@ -318,7 +336,8 @@ __KERNEL_IS_WAITING_ON_EXTERNAL_CREDENTIAL_RESOLVER__( break; /* woken because the resolver has died? */ if (kauth_resolver_identity == 0) { - error = EIO; + RESOLVER_FAILED_MESSAGE("kauth external resolver died while while waiting for work to complete"); + error = KAUTH_RESOLVER_FAILED_ERRCODE; break; } /* an error? */ @@ -578,6 +597,11 @@ identitysvc(__unused struct proc *p, struct identitysvc_args *uap, __unused int3 int error; pid_t new_id; + if (!IOTaskHasEntitlement(current_task(), IDENTITYSVC_ENTITLEMENT)) { + KAUTH_DEBUG("RESOLVER - pid %d not entitled to call identitysvc", current_proc()->p_pid); + return(EPERM); + } + /* * New server registering itself. */ @@ -755,8 +779,10 @@ kauth_resolver_getwork_continue(int result) * If this is a wakeup from another thread in the resolver * deregistering it, error out the request-for-work thread */ - if (!kauth_resolver_identity) - error = EIO; + if (!kauth_resolver_identity) { + RESOLVER_FAILED_MESSAGE("external resolver died"); + error = KAUTH_RESOLVER_FAILED_ERRCODE; + } KAUTH_RESOLVER_UNLOCK(); return(error); } @@ -897,8 +923,10 @@ kauth_resolver_getwork(user_addr_t message) * If this is a wakeup from another thread in the resolver * deregistering it, error out the request-for-work thread */ - if (!kauth_resolver_identity) - error = EIO; + if (!kauth_resolver_identity) { + printf("external resolver died"); + error = KAUTH_RESOLVER_FAILED_ERRCODE; + } return(error); } return kauth_resolver_getwork2(message); @@ -922,7 +950,7 @@ kauth_resolver_complete(user_addr_t message) struct kauth_identity_extlookup extl; struct kauth_resolver_work *workp; struct kauth_resolver_work *killp; - int error, result, request_flags; + int error, result, want_extend_data; /* * Copy in the mesage, including the extension field, since we are @@ -956,6 +984,7 @@ kauth_resolver_complete(user_addr_t message) case KAUTH_EXTLOOKUP_FATAL: /* fatal error means the resolver is dead */ KAUTH_DEBUG("RESOLVER - resolver %d died, waiting for a new one", kauth_resolver_identity); + RESOLVER_FAILED_MESSAGE("resolver %d died, waiting for a new one", kauth_resolver_identity); /* * Terminate outstanding requests; without an authoritative * resolver, we are now back on our own authority. Tag the @@ -973,7 +1002,7 @@ kauth_resolver_complete(user_addr_t message) /* Cause all waiting-for-work threads to return EIO */ wakeup((caddr_t)&kauth_resolver_unsubmitted); /* and return EIO to the caller */ - error = EIO; + error = KAUTH_RESOLVER_FAILED_ERRCODE; break; case KAUTH_EXTLOOKUP_BADRQ: @@ -983,12 +1012,14 @@ kauth_resolver_complete(user_addr_t message) case KAUTH_EXTLOOKUP_FAILURE: KAUTH_DEBUG("RESOLVER - resolver reported transient failure for request %d", extl.el_seqno); - result = EIO; + RESOLVER_FAILED_MESSAGE("resolver reported transient failure for request %d", extl.el_seqno); + result = KAUTH_RESOLVER_FAILED_ERRCODE; break; default: KAUTH_DEBUG("RESOLVER - resolver returned unexpected status %d", extl.el_result); - result = EIO; + RESOLVER_FAILED_MESSAGE("resolver returned unexpected status %d", extl.el_result); + result = KAUTH_RESOLVER_FAILED_ERRCODE; break; } @@ -1004,9 +1035,9 @@ kauth_resolver_complete(user_addr_t message) /* found it? */ if (workp->kr_seqno == extl.el_seqno) { /* - * Take a snapshot of the original request flags. + * Do we want extend_data? */ - request_flags = workp->kr_work.el_flags; + want_extend_data = (workp->kr_work.el_flags & (KAUTH_EXTLOOKUP_WANT_PWNAM|KAUTH_EXTLOOKUP_WANT_GRNAM)); /* * Get the request of the submitted queue so @@ -1049,11 +1080,13 @@ kauth_resolver_complete(user_addr_t message) * part of a user's address space if they return * flags that mismatch the original request's flags. */ - if ((extl.el_flags & request_flags) & (KAUTH_EXTLOOKUP_VALID_PWNAM|KAUTH_EXTLOOKUP_VALID_GRNAM)) { + if (want_extend_data && (extl.el_flags & (KAUTH_EXTLOOKUP_VALID_PWNAM|KAUTH_EXTLOOKUP_VALID_GRNAM))) { size_t actual; /* notused */ KAUTH_RESOLVER_UNLOCK(); error = copyinstr(extl.el_extend, CAST_DOWN(void *, workp->kr_extend), MAXPATHLEN, &actual); + KAUTH_DEBUG("RESOLVER - resolver got name :%*s: len = %d\n", (int)actual, + actual ? "null" : (char *)extl.el_extend, actual); KAUTH_RESOLVER_LOCK(); } else if (extl.el_flags & (KAUTH_EXTLOOKUP_VALID_PWNAM|KAUTH_EXTLOOKUP_VALID_GRNAM)) { error = EFAULT; @@ -2608,7 +2641,10 @@ kauth_cred_cache_lookup(int from, int to, void *src, void *dst) * atomically. */ if (to == KI_VALID_PWNAM || to == KI_VALID_GRNAM) { + if (dst == NULL) + return (EINVAL); namebuf = dst; + *namebuf = '\0'; } ki.ki_valid = 0; switch(from) { @@ -2632,6 +2668,9 @@ kauth_cred_cache_lookup(int from, int to, void *src, void *dst) default: return(EINVAL); } + /* If we didn't get what we're asking for. Call the resolver */ + if (!error && !(to & ki.ki_valid)) + error = ENOENT; /* lookup failure or error */ if (error != 0) { /* any other error is fatal */ @@ -4574,7 +4613,6 @@ int kauth_proc_label_update(struct proc *p, struct label *label) /* update cred on proc */ PROC_UPDATE_CREDS_ONPROC(p); - mac_proc_set_enforce(p, MAC_ALL_ENFORCE); proc_ucred_unlock(p); } break; @@ -4653,7 +4691,6 @@ kauth_proc_label_update_execve(struct proc *p, vfs_context_t ctx, p->p_ucred = my_new_cred; /* update cred on proc */ PROC_UPDATE_CREDS_ONPROC(p); - mac_proc_set_enforce(p, MAC_ALL_ENFORCE); proc_ucred_unlock(p); } break; diff --git a/bsd/kern/kern_cs.c b/bsd/kern/kern_cs.c index 214b041b7..7a3d22baf 100644 --- a/bsd/kern/kern_cs.c +++ b/bsd/kern/kern_cs.c @@ -84,6 +84,7 @@ const int cs_enforcement_enable = 1; const int cs_library_val_enable = 1; #else /* !SECURE_KERNEL */ int cs_enforcement_panic=0; +int cs_relax_platform_task_ports = 0; #if CONFIG_ENFORCE_SIGNED_CODE #define DEFAULT_CS_ENFORCEMENT_ENABLE 1 @@ -120,6 +121,7 @@ SYSCTL_INT(_vm, OID_AUTO, cs_library_validation, CTLFLAG_RW | CTLFLAG_LOCKED, &c #endif /* !SECURE_KERNEL */ int panic_on_cs_killed = 0; + void cs_init(void) { @@ -140,6 +142,10 @@ cs_init(void) cs_enforcement_panic = (panic != 0); } + PE_parse_boot_argn("cs_relax_platform_task_ports", + &cs_relax_platform_task_ports, + sizeof(cs_relax_platform_task_ports)); + PE_parse_boot_argn("cs_debug", &cs_debug, sizeof (cs_debug)); #if !CONFIG_ENFORCE_LIBRARY_VALIDATION @@ -180,7 +186,9 @@ cs_allow_invalid(struct proc *p) { p->p_csflags |= CS_DEBUGGED; } + proc_unlock(p); + vm_map_switch_protect(get_task_map(p->task), FALSE); #endif return (p->p_csflags & (CS_KILL | CS_HARD)) == 0; @@ -378,6 +386,18 @@ csblob_get_flags(struct cs_blob *blob) return blob->csb_flags; } +/* + * Function: csblob_get_hashtype + * + * Description: This function returns the hash type for a given blob +*/ + +uint8_t +csblob_get_hashtype(struct cs_blob const * const blob) +{ + return blob->csb_hashtype != NULL ? cs_hash_type(blob->csb_hashtype) : 0; +} + /* * Function: csproc_get_blob * @@ -453,6 +473,18 @@ csblob_get_cdhash(struct cs_blob *csblob) return csblob->csb_cdhash; } +/* + * Function: csblob_get_signer_type + * + * Description: This function returns the signer type + * as an integer + */ +unsigned int +csblob_get_signer_type(struct cs_blob *csblob) +{ + return csblob->csb_signer_type; +} + void * csblob_entitlements_dictionary_copy(struct cs_blob *csblob) { @@ -487,6 +519,24 @@ csproc_get_teamid(struct proc *p) return csblob_get_teamid(csblob); } +/* + * Function: csproc_get_signer_type + * + * Description: This function returns the signer type + * of the process p +*/ +unsigned int +csproc_get_signer_type(struct proc *p) +{ + struct cs_blob *csblob; + + csblob = csproc_get_blob(p); + if (csblob == NULL) + return CS_SIGNER_TYPE_UNKNOWN; + + return csblob_get_signer_type(csblob); +} + /* * Function: csvnode_get_teamid * @@ -534,6 +584,26 @@ csproc_get_platform_path(struct proc *p) return (csblob == NULL) ? 0 : csblob->csb_platform_path; } +#if DEVELOPMENT || DEBUG +void +csproc_clear_platform_binary(struct proc *p) +{ + struct cs_blob *csblob = csproc_get_blob(p); + + if (csblob == NULL) { + return; + } + + if (cs_debug) { + printf("clearing platform binary on proc/task: pid = %d\n", p->p_pid); + } + + csblob->csb_platform_binary = 0; + csblob->csb_platform_path = 0; + task_set_platform_binary(proc_task(p), FALSE); +} +#endif + /* * Function: csproc_get_prod_signed * @@ -611,6 +681,46 @@ csfg_get_cdhash(struct fileglob *fg, uint64_t offset, size_t *cdhash_size) return csblob->csb_cdhash; } +/* + * Function: csfg_get_signer_type + * + * Description: This returns the signer type + * for the fileglob fg + */ +unsigned int +csfg_get_signer_type(struct fileglob *fg) +{ + struct ubc_info *uip; + unsigned int signer_type = CS_SIGNER_TYPE_UNKNOWN; + vnode_t vp; + + if (FILEGLOB_DTYPE(fg) != DTYPE_VNODE) + return CS_SIGNER_TYPE_UNKNOWN; + + vp = (struct vnode *)fg->fg_data; + if (vp == NULL) + return CS_SIGNER_TYPE_UNKNOWN; + + vnode_lock(vp); + if (!UBCINFOEXISTS(vp)) + goto out; + + uip = vp->v_ubcinfo; + if (uip == NULL) + goto out; + + if (uip->cs_blobs == NULL) + goto out; + + /* It is OK to extract the signer type from the first blob, + because all blobs of a vnode must have the same signer type. */ + signer_type = uip->cs_blobs->csb_signer_type; +out: + vnode_unlock(vp); + + return signer_type; +} + /* * Function: csfg_get_teamid * @@ -666,11 +776,11 @@ csfg_get_prod_signed(struct fileglob *fg) int prod_signed = 0; if (FILEGLOB_DTYPE(fg) != DTYPE_VNODE) - return NULL; + return 0; vp = (struct vnode *)fg->fg_data; if (vp == NULL) - return NULL; + return 0; vnode_lock(vp); if (!UBCINFOEXISTS(vp)) diff --git a/bsd/kern/kern_csr.c b/bsd/kern/kern_csr.c index bc5e03661..15a5ede70 100644 --- a/bsd/kern/kern_csr.c +++ b/bsd/kern/kern_csr.c @@ -72,7 +72,15 @@ csr_check(csr_config_t mask) return ret; } - ret = (config & mask) ? 0 : EPERM; + // CSR_ALLOW_KERNEL_DEBUGGER needs to be allowed when SIP is disabled + // to allow 3rd-party developers to debug their kexts. Use + // CSR_ALLOW_UNTRUSTED_KEXTS as a proxy for "SIP is disabled" on the + // grounds that you can do the same damage with a kernel debugger as + // you can with an untrusted kext. + if ((config & (CSR_ALLOW_UNTRUSTED_KEXTS|CSR_ALLOW_APPLE_INTERNAL)) != 0) + config |= CSR_ALLOW_KERNEL_DEBUGGER; + + ret = ((config & mask) == mask) ? 0 : EPERM; if (ret == EPERM) { // Override the return value if booted from the BaseSystem and the mask does not contain any flag that should always be enforced. if (csr_allow_all && (mask & CSR_ALWAYS_ENFORCED_FLAGS) == 0) diff --git a/bsd/kern/kern_descrip.c b/bsd/kern/kern_descrip.c index b93ab3bbf..06f9b82a6 100644 --- a/bsd/kern/kern_descrip.c +++ b/bsd/kern/kern_descrip.c @@ -120,6 +120,10 @@ #include <mach/mach_port.h> #include <stdbool.h> +#if CONFIG_MACF +#include <security/mac_framework.h> +#endif + kern_return_t ipc_object_copyin(ipc_space_t, mach_port_name_t, mach_msg_type_name_t, ipc_port_t *); void ipc_port_release_send(ipc_port_t); @@ -418,6 +422,7 @@ _fdrelse(struct proc * p, int fd) while ((nfd = fdp->fd_lastfile) > 0 && fdp->fd_ofiles[nfd] == NULL && !(fdp->fd_ofileflags[nfd] & UF_RESERVED)) + /* JMM - What about files with lingering EV_VANISHED knotes? */ fdp->fd_lastfile--; } @@ -1121,6 +1126,10 @@ fcntl_nocancel(proc_t p, struct fcntl_nocancel_args *uap, int32_t *retval) case F_GETLK: case F_OFD_GETLK: +#if CONFIG_EMBEDDED + case F_GETLKPID: + case F_OFD_GETLKPID: +#endif if (fp->f_type != DTYPE_VNODE) { error = EBADF; goto out; @@ -1966,7 +1975,7 @@ fcntl_nocancel(proc_t p, struct fcntl_nocancel_args *uap, int32_t *retval) error = copyin(argp, &lv32, sizeof (lv32)); lv.lv_file_start = lv32.lv_file_start; - lv.lv_error_message = CAST_USER_ADDR_T(lv32.lv_error_message); + lv.lv_error_message = (void *)(uintptr_t)lv32.lv_error_message; lv.lv_error_message_size = lv32.lv_error_message; } if (error) @@ -1974,7 +1983,7 @@ fcntl_nocancel(proc_t p, struct fcntl_nocancel_args *uap, int32_t *retval) #if CONFIG_MACF error = mac_file_check_library_validation(p, fg, lv.lv_file_start, - lv.lv_error_message, lv.lv_error_message_size); + (user_long_t)lv.lv_error_message, lv.lv_error_message_size); #endif break; @@ -4631,6 +4640,7 @@ fg_free(struct fileglob *fg) } + /* * fdexec * @@ -4648,16 +4658,39 @@ fg_free(struct fileglob *fg) * Returns: void * * Locks: This function internally takes and drops proc_fdlock() + * But assumes tables don't grow/change while unlocked. * */ void -fdexec(proc_t p, short flags) +fdexec(proc_t p, short flags, int self_exec) { struct filedesc *fdp = p->p_fd; int i; boolean_t cloexec_default = (flags & POSIX_SPAWN_CLOEXEC_DEFAULT) != 0; + thread_t self = current_thread(); + struct uthread *ut = get_bsdthread_info(self); + struct kqueue *dealloc_kq = NULL; + + /* + * If the current thread is bound as a workq/workloop + * servicing thread, we need to unbind it first. + */ + if (ut->uu_kqueue_bound && self_exec) { + kevent_qos_internal_unbind(p, 0, self, + ut->uu_kqueue_flags); + } proc_fdlock(p); + + /* + * Deallocate the knotes for this process + * and mark the tables non-existent so + * subsequent kqueue closes go faster. + */ + knotes_dealloc(p); + assert(fdp->fd_knlistsize == -1); + assert(fdp->fd_knhashmask == 0); + for (i = fdp->fd_lastfile; i >= 0; i--) { struct fileproc *fp = fdp->fd_ofiles[i]; @@ -4681,8 +4714,6 @@ fdexec(proc_t p, short flags) || (fp && mac_file_check_inherit(proc_ucred(p), fp->f_fglob)) #endif ) { - if (i < fdp->fd_knlistsize) - knote_fdclose(p, i, TRUE); procfdtbl_clearfd(p, i); if (i == fdp->fd_lastfile && i > 0) fdp->fd_lastfile--; @@ -4704,7 +4735,18 @@ fdexec(proc_t p, short flags) fileproc_free(fp); } } + + /* release the per-process workq kq */ + if (fdp->fd_wqkqueue) { + dealloc_kq = fdp->fd_wqkqueue; + fdp->fd_wqkqueue = NULL; + } + proc_fdunlock(p); + + /* Anything to free? */ + if (dealloc_kq) + kqueue_dealloc(dealloc_kq); } @@ -4892,10 +4934,6 @@ fdcopy(proc_t p, vnode_t uth_cdir) if (*fpp == NULL && i == newfdp->fd_lastfile && i > 0) newfdp->fd_lastfile--; } - newfdp->fd_knlist = NULL; - newfdp->fd_knlistsize = -1; - newfdp->fd_knhash = NULL; - newfdp->fd_knhashmask = 0; } fpp = newfdp->fd_ofiles; flags = newfdp->fd_ofileflags; @@ -4931,6 +4969,20 @@ fdcopy(proc_t p, vnode_t uth_cdir) } proc_fdunlock(p); + + /* + * Initialize knote and kqueue tracking structs + */ + newfdp->fd_knlist = NULL; + newfdp->fd_knlistsize = -1; + newfdp->fd_knhash = NULL; + newfdp->fd_knhashmask = 0; + newfdp->fd_kqhash = NULL; + newfdp->fd_kqhashmask = 0; + newfdp->fd_wqkqueue = NULL; + lck_mtx_init(&newfdp->fd_kqhashlock, proc_kqhashlock_grp, proc_lck_attr); + lck_mtx_init(&newfdp->fd_knhashlock, proc_knhashlock_grp, proc_lck_attr); + return (newfdp); } @@ -4954,6 +5006,7 @@ fdfree(proc_t p) { struct filedesc *fdp; struct fileproc *fp; + struct kqueue *dealloc_kq = NULL; int i; proc_fdlock(p); @@ -4968,13 +5021,17 @@ fdfree(proc_t p) if (&filedesc0 == fdp) panic("filedesc0"); + /* + * deallocate all the knotes up front and claim empty + * tables to make any subsequent kqueue closes faster. + */ + knotes_dealloc(p); + assert(fdp->fd_knlistsize == -1); + assert(fdp->fd_knhashmask == 0); + + /* close file descriptors */ if (fdp->fd_nfiles > 0 && fdp->fd_ofiles) { for (i = fdp->fd_lastfile; i >= 0; i--) { - - /* May still have knotes for fd without open file */ - if (i < fdp->fd_knlistsize) - knote_fdclose(p, i, TRUE); - if ((fp = fdp->fd_ofiles[i]) != NULL) { if (fdp->fd_ofileflags[i] & UF_RESERVED) @@ -4993,10 +5050,18 @@ fdfree(proc_t p) fdp->fd_nfiles = 0; } + if (fdp->fd_wqkqueue) { + dealloc_kq = fdp->fd_wqkqueue; + fdp->fd_wqkqueue = NULL; + } + proc_fdunlock(p); + if (dealloc_kq) + kqueue_dealloc(dealloc_kq); + if (fdp->fd_cdir) - vnode_rele(fdp->fd_cdir); + vnode_rele(fdp->fd_cdir); if (fdp->fd_rdir) vnode_rele(fdp->fd_rdir); @@ -5004,10 +5069,14 @@ fdfree(proc_t p) p->p_fd = NULL; proc_fdunlock(p); - if (fdp->fd_knlist) - FREE(fdp->fd_knlist, M_KQUEUE); - if (fdp->fd_knhash) - FREE(fdp->fd_knhash, M_KQUEUE); + if (fdp->fd_kqhash) { + for (uint32_t j = 0; j <= fdp->fd_kqhashmask; j++) + assert(SLIST_EMPTY(&fdp->fd_kqhash[j])); + FREE(fdp->fd_kqhash, M_KQUEUE); + } + + lck_mtx_destroy(&fdp->fd_kqhashlock, proc_kqhashlock_grp); + lck_mtx_destroy(&fdp->fd_knhashlock, proc_knhashlock_grp); FREE_ZONE(fdp, sizeof(*fdp), M_FILEDESC); } @@ -5437,6 +5506,7 @@ fileport_makefd(proc_t p, struct fileport_makefd_args *uap, int32_t *retval) err = fdalloc(p, 0, &fd); if (err != 0) { proc_fdunlock(p); + fg_drop(fp); goto out; } *fdflags(p, fd) |= UF_EXCLOSE; @@ -5885,9 +5955,10 @@ fo_close(struct fileglob *fg, vfs_context_t ctx) * !0 Filter is active */ int -fo_kqfilter(struct fileproc *fp, struct knote *kn, vfs_context_t ctx) +fo_kqfilter(struct fileproc *fp, struct knote *kn, + struct kevent_internal_s *kev, vfs_context_t ctx) { - return ((*fp->f_ops->fo_kqfilter)(fp, kn, ctx)); + return ((*fp->f_ops->fo_kqfilter)(fp, kn, kev, ctx)); } /* @@ -5904,6 +5975,7 @@ file_issendable(proc_t p, struct fileproc *fp) case DTYPE_SOCKET: case DTYPE_PIPE: case DTYPE_PSXSHM: + case DTYPE_NETPOLICY: return (0 == (fp->f_fglob->fg_lflags & FG_CONFINED)); default: /* DTYPE_KQUEUE, DTYPE_FSEVENTS, DTYPE_PSXSEM */ diff --git a/bsd/kern/kern_event.c b/bsd/kern/kern_event.c index 66cd6e2a5..f64bef436 100644 --- a/bsd/kern/kern_event.c +++ b/bsd/kern/kern_event.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -55,6 +55,7 @@ * @(#)kern_event.c 1.0 (3/31/2000) */ #include <stdint.h> +#include <stdatomic.h> #include <sys/param.h> #include <sys/systm.h> @@ -83,9 +84,13 @@ #include <sys/proc_info.h> #include <sys/codesign.h> #include <sys/pthread_shims.h> +#include <sys/kdebug.h> +#include <sys/reason.h> +#include <os/reason_private.h> #include <kern/locks.h> #include <kern/clock.h> +#include <kern/cpu_data.h> #include <kern/policy_internal.h> #include <kern/thread_call.h> #include <kern/sched_prim.h> @@ -93,16 +98,27 @@ #include <kern/zalloc.h> #include <kern/kalloc.h> #include <kern/assert.h> +#include <kern/ast.h> +#include <kern/thread.h> +#include <kern/kcdata.h> #include <libkern/libkern.h> +#include <libkern/OSAtomic.h> + #include "net/net_str_id.h" #include <mach/task.h> +#include <libkern/section_keywords.h> #if CONFIG_MEMORYSTATUS #include <sys/kern_memorystatus.h> #endif +extern thread_t port_name_to_thread(mach_port_name_t port_name); /* osfmk/kern/ipc_tt.h */ +extern mach_port_name_t ipc_entry_name_mask(mach_port_name_t name); /* osfmk/ipc/ipc_entry.h */ + +#define KEV_EVTID(code) BSDDBG_CODE(DBG_BSD_KEVENT, (code)) + /* * JMM - this typedef needs to be unified with pthread_priority_t * and mach_msg_priority_t. It also needs to be the same type @@ -114,25 +130,25 @@ MALLOC_DEFINE(M_KQUEUE, "kqueue", "memory for kqueue system"); #define KQ_EVENT NO_EVENT64 -static inline void kqlock(struct kqueue *kq); -static inline void kqunlock(struct kqueue *kq); - -static int kqlock2knoteuse(struct kqueue *kq, struct knote *kn); +#define KNUSE_NONE 0x0 +#define KNUSE_STEAL_DROP 0x1 +#define KNUSE_BOOST 0x2 +static int kqlock2knoteuse(struct kqueue *kq, struct knote *kn, int flags); static int kqlock2knotedrop(struct kqueue *kq, struct knote *kn); -static int kqlock2knotedetach(struct kqueue *kq, struct knote *kn); -static int knoteuse2kqlock(struct kqueue *kq, struct knote *kn, int defer_drop); +static int kqlock2knotedetach(struct kqueue *kq, struct knote *kn, int flags); +static int knoteuse2kqlock(struct kqueue *kq, struct knote *kn, int flags); static int kqueue_read(struct fileproc *fp, struct uio *uio, - int flags, vfs_context_t ctx); + int flags, vfs_context_t ctx); static int kqueue_write(struct fileproc *fp, struct uio *uio, - int flags, vfs_context_t ctx); + int flags, vfs_context_t ctx); static int kqueue_ioctl(struct fileproc *fp, u_long com, caddr_t data, - vfs_context_t ctx); + vfs_context_t ctx); static int kqueue_select(struct fileproc *fp, int which, void *wq_link_id, - vfs_context_t ctx); + vfs_context_t ctx); static int kqueue_close(struct fileglob *fg, vfs_context_t ctx); static int kqueue_kqfilter(struct fileproc *fp, struct knote *kn, - vfs_context_t ctx); + struct kevent_internal_s *kev, vfs_context_t ctx); static int kqueue_drain(struct fileproc *fp, vfs_context_t ctx); static const struct fileops kqueueops = { @@ -146,7 +162,9 @@ static const struct fileops kqueueops = { .fo_drain = kqueue_drain, }; -static int kevent_internal(struct proc *p, int fd, +static void kevent_put_kq(struct proc *p, kqueue_id_t id, struct fileproc *fp, struct kqueue *kq); +static int kevent_internal(struct proc *p, + kqueue_id_t id, kqueue_id_t *id_out, user_addr_t changelist, int nchanges, user_addr_t eventlist, int nevents, user_addr_t data_out, uint64_t data_available, @@ -165,10 +183,7 @@ static int kevent_callback(struct kqueue *kq, struct kevent_internal_s *kevp, static void kevent_continue(struct kqueue *kq, void *data, int error); static void kqueue_scan_continue(void *contp, wait_result_t wait_result); static int kqueue_process(struct kqueue *kq, kevent_callback_t callback, void *callback_data, - struct filt_process_s *process_data, kq_index_t servicer_qos_index, - int *countp, struct proc *p); -static int kqueue_begin_processing(struct kqueue *kq, kq_index_t qos_index, unsigned int flags); -static void kqueue_end_processing(struct kqueue *kq, kq_index_t qos_index, unsigned int flags); + struct filt_process_s *process_data, int *countp, struct proc *p); static struct kqtailq *kqueue_get_base_queue(struct kqueue *kq, kq_index_t qos_index); static struct kqtailq *kqueue_get_high_queue(struct kqueue *kq, kq_index_t qos_index); static int kqueue_queue_empty(struct kqueue *kq, kq_index_t qos_index); @@ -176,12 +191,62 @@ static int kqueue_queue_empty(struct kqueue *kq, kq_index_t qos_index); static struct kqtailq *kqueue_get_suppressed_queue(struct kqueue *kq, kq_index_t qos_index); static void kqworkq_request_thread(struct kqworkq *kqwq, kq_index_t qos_index); -static void kqworkq_request_help(struct kqworkq *kqwq, kq_index_t qos_index, uint32_t type); +static void kqworkq_request_help(struct kqworkq *kqwq, kq_index_t qos_index); static void kqworkq_update_override(struct kqworkq *kqwq, kq_index_t qos_index, kq_index_t override_index); -static void kqworkq_bind_thread(struct kqworkq *kqwq, kq_index_t qos_index, thread_t thread, unsigned int flags); +static void kqworkq_bind_thread_impl(struct kqworkq *kqwq, kq_index_t qos_index, thread_t thread, unsigned int flags); static void kqworkq_unbind_thread(struct kqworkq *kqwq, kq_index_t qos_index, thread_t thread, unsigned int flags); static struct kqrequest *kqworkq_get_request(struct kqworkq *kqwq, kq_index_t qos_index); +enum { + KQWL_UO_NONE = 0, + KQWL_UO_OLD_OVERRIDE_IS_SYNC_UI = 0x1, + KQWL_UO_NEW_OVERRIDE_IS_SYNC_UI = 0x2, + KQWL_UO_UPDATE_SUPPRESS_SYNC_COUNTERS = 0x4, + KQWL_UO_UPDATE_OVERRIDE_LAZY = 0x8 +}; + +static void kqworkloop_update_override(struct kqworkloop *kqwl, kq_index_t qos_index, kq_index_t override_index, uint32_t flags); +static void kqworkloop_bind_thread_impl(struct kqworkloop *kqwl, thread_t thread, unsigned int flags); +static void kqworkloop_unbind_thread(struct kqworkloop *kqwl, thread_t thread, unsigned int flags); +static inline kq_index_t kqworkloop_combined_qos(struct kqworkloop *kqwl, boolean_t *); +static void kqworkloop_update_suppress_sync_count(struct kqrequest *kqr, uint32_t flags); +enum { + KQWL_UTQ_NONE, + /* + * The wakeup qos is the qos of QUEUED knotes. + * + * This QoS is accounted for with the events override in the + * kqr_override_index field. It is raised each time a new knote is queued at + * a given QoS. The kqr_wakeup_indexes field is a superset of the non empty + * knote buckets and is recomputed after each event delivery. + */ + KQWL_UTQ_UPDATE_WAKEUP_QOS, + KQWL_UTQ_UPDATE_STAYACTIVE_QOS, + KQWL_UTQ_RECOMPUTE_WAKEUP_QOS, + /* + * The wakeup override is for suppressed knotes that have fired again at + * a higher QoS than the one for which they are suppressed already. + * This override is cleared when the knote suppressed list becomes empty. + */ + KQWL_UTQ_UPDATE_WAKEUP_OVERRIDE, + KQWL_UTQ_RESET_WAKEUP_OVERRIDE, + /* + * The async QoS is the maximum QoS of an event enqueued on this workloop in + * userland. It is copied from the only EVFILT_WORKLOOP knote with + * a NOTE_WL_THREAD_REQUEST bit set allowed on this workloop. If there is no + * such knote, this QoS is 0. + */ + KQWL_UTQ_SET_ASYNC_QOS, + /* + * The sync waiters QoS is the maximum QoS of any thread blocked on an + * EVFILT_WORKLOOP knote marked with the NOTE_WL_SYNC_WAIT bit. + * If there is no such knote, this QoS is 0. + */ + KQWL_UTQ_SET_SYNC_WAITERS_QOS, + KQWL_UTQ_REDRIVE_EVENTS, +}; +static void kqworkloop_update_threads_qos(struct kqworkloop *kqwl, int op, kq_index_t qos); +static void kqworkloop_request_help(struct kqworkloop *kqwl, kq_index_t qos_index); static int knote_process(struct knote *kn, kevent_callback_t callback, void *callback_data, struct filt_process_s *process_data, struct proc *p); @@ -189,9 +254,10 @@ static int knote_process(struct knote *kn, kevent_callback_t callback, void *cal static void knote_put(struct knote *kn); #endif -static int knote_fdadd(struct knote *kn, struct proc *p); -static void knote_fdremove(struct knote *kn, struct proc *p); -static struct knote *knote_fdfind(struct kqueue *kq, struct kevent_internal_s *kev, struct proc *p); +static int kq_add_knote(struct kqueue *kq, struct knote *kn, + struct kevent_internal_s *kev, struct proc *p, int *knoteuse_flags); +static struct knote *kq_find_knote_and_kq_lock(struct kqueue *kq, struct kevent_internal_s *kev, bool is_fd, struct proc *p); +static void kq_remove_knote(struct kqueue *kq, struct knote *kn, struct proc *p, kn_status_t *kn_status, uint16_t *kq_state); static void knote_drop(struct knote *kn, struct proc *p); static struct knote *knote_alloc(void); @@ -212,15 +278,16 @@ static void knote_wakeup(struct knote *kn); static kq_index_t knote_get_queue_index(struct knote *kn); static struct kqtailq *knote_get_queue(struct knote *kn); -static struct kqtailq *knote_get_suppressed_queue(struct knote *kn); static kq_index_t knote_get_req_index(struct knote *kn); static kq_index_t knote_get_qos_index(struct knote *kn); static void knote_set_qos_index(struct knote *kn, kq_index_t qos_index); static kq_index_t knote_get_qos_override_index(struct knote *kn); -static void knote_set_qos_override_index(struct knote *kn, kq_index_t qos_index); +static kq_index_t knote_get_sync_qos_override_index(struct knote *kn); +static void knote_set_qos_override_index(struct knote *kn, kq_index_t qos_index, boolean_t override_is_sync); +static void knote_set_qos_overcommit(struct knote *kn); -static int filt_fileattach(struct knote *kn); -static struct filterops file_filtops = { +static int filt_fileattach(struct knote *kn, struct kevent_internal_s *kev); +SECURITY_READ_ONLY_EARLY(static struct filterops) file_filtops = { .f_isfd = 1, .f_attach = filt_fileattach, }; @@ -229,7 +296,7 @@ static void filt_kqdetach(struct knote *kn); static int filt_kqueue(struct knote *kn, long hint); static int filt_kqtouch(struct knote *kn, struct kevent_internal_s *kev); static int filt_kqprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); -static struct filterops kqread_filtops = { +SECURITY_READ_ONLY_EARLY(static struct filterops) kqread_filtops = { .f_isfd = 1, .f_detach = filt_kqdetach, .f_event = filt_kqueue, @@ -238,17 +305,17 @@ static struct filterops kqread_filtops = { }; /* placeholder for not-yet-implemented filters */ -static int filt_badattach(struct knote *kn); -static struct filterops bad_filtops = { +static int filt_badattach(struct knote *kn, struct kevent_internal_s *kev); +SECURITY_READ_ONLY_EARLY(static struct filterops) bad_filtops = { .f_attach = filt_badattach, }; -static int filt_procattach(struct knote *kn); +static int filt_procattach(struct knote *kn, struct kevent_internal_s *kev); static void filt_procdetach(struct knote *kn); static int filt_proc(struct knote *kn, long hint); static int filt_proctouch(struct knote *kn, struct kevent_internal_s *kev); static int filt_procprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); -static struct filterops proc_filtops = { +SECURITY_READ_ONLY_EARLY(static struct filterops) proc_filtops = { .f_attach = filt_procattach, .f_detach = filt_procdetach, .f_event = filt_proc, @@ -257,60 +324,30 @@ static struct filterops proc_filtops = { }; #if CONFIG_MEMORYSTATUS -extern struct filterops memorystatus_filtops; +extern const struct filterops memorystatus_filtops; #endif /* CONFIG_MEMORYSTATUS */ -extern struct filterops fs_filtops; - -extern struct filterops sig_filtops; - -/* Timer filter */ -static int filt_timerattach(struct knote *kn); -static void filt_timerdetach(struct knote *kn); -static int filt_timer(struct knote *kn, long hint); -static int filt_timertouch(struct knote *kn, struct kevent_internal_s *kev); -static int filt_timerprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); -static struct filterops timer_filtops = { - .f_attach = filt_timerattach, - .f_detach = filt_timerdetach, - .f_event = filt_timer, - .f_touch = filt_timertouch, - .f_process = filt_timerprocess, -}; - -/* Helpers */ -static void filt_timerexpire(void *knx, void *param1); -static int filt_timervalidate(struct knote *kn); -static void filt_timerupdate(struct knote *kn, int num_fired); -static void filt_timercancel(struct knote *kn); +extern const struct filterops fs_filtops; -#define TIMER_RUNNING 0x1 -#define TIMER_CANCELWAIT 0x2 - -static lck_mtx_t _filt_timerlock; -static void filt_timerlock(void); -static void filt_timerunlock(void); +extern const struct filterops sig_filtops; static zone_t knote_zone; static zone_t kqfile_zone; static zone_t kqworkq_zone; +static zone_t kqworkloop_zone; #define KN_HASH(val, mask) (((val) ^ (val >> 8)) & (mask)) -#if 0 -extern struct filterops aio_filtops; -#endif - /* Mach portset filter */ -extern struct filterops machport_filtops; +extern const struct filterops machport_filtops; /* User filter */ -static int filt_userattach(struct knote *kn); +static int filt_userattach(struct knote *kn, struct kevent_internal_s *kev); static void filt_userdetach(struct knote *kn); static int filt_user(struct knote *kn, long hint); static int filt_usertouch(struct knote *kn, struct kevent_internal_s *kev); static int filt_userprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); -static struct filterops user_filtops = { +SECURITY_READ_ONLY_EARLY(static struct filterops) user_filtops = { .f_attach = filt_userattach, .f_detach = filt_userdetach, .f_event = filt_user, @@ -322,20 +359,41 @@ static lck_spin_t _filt_userlock; static void filt_userlock(void); static void filt_userunlock(void); -extern struct filterops pipe_rfiltops; -extern struct filterops pipe_wfiltops; -extern struct filterops ptsd_kqops; -extern struct filterops soread_filtops; -extern struct filterops sowrite_filtops; -extern struct filterops sock_filtops; -extern struct filterops soexcept_filtops; -extern struct filterops spec_filtops; -extern struct filterops bpfread_filtops; -extern struct filterops necp_fd_rfiltops; -extern struct filterops skywalk_channel_rfiltops; -extern struct filterops skywalk_channel_wfiltops; -extern struct filterops fsevent_filtops; -extern struct filterops vnode_filtops; +/* Workloop filter */ +static bool filt_wlneeds_boost(struct kevent_internal_s *kev); +static int filt_wlattach(struct knote *kn, struct kevent_internal_s *kev); +static int filt_wlpost_attach(struct knote *kn, struct kevent_internal_s *kev); +static void filt_wldetach(struct knote *kn); +static int filt_wlevent(struct knote *kn, long hint); +static int filt_wltouch(struct knote *kn, struct kevent_internal_s *kev); +static int filt_wldrop_and_unlock(struct knote *kn, struct kevent_internal_s *kev); +static int filt_wlprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); +SECURITY_READ_ONLY_EARLY(static struct filterops) workloop_filtops = { + .f_needs_boost = filt_wlneeds_boost, + .f_attach = filt_wlattach, + .f_post_attach = filt_wlpost_attach, + .f_detach = filt_wldetach, + .f_event = filt_wlevent, + .f_touch = filt_wltouch, + .f_drop_and_unlock = filt_wldrop_and_unlock, + .f_process = filt_wlprocess, +}; + +extern const struct filterops pipe_rfiltops; +extern const struct filterops pipe_wfiltops; +extern const struct filterops ptsd_kqops; +extern const struct filterops soread_filtops; +extern const struct filterops sowrite_filtops; +extern const struct filterops sock_filtops; +extern const struct filterops soexcept_filtops; +extern const struct filterops spec_filtops; +extern const struct filterops bpfread_filtops; +extern const struct filterops necp_fd_rfiltops; +extern const struct filterops fsevent_filtops; +extern const struct filterops vnode_filtops; +extern const struct filterops tty_filtops; + +const static struct filterops timer_filtops; /* * @@ -353,7 +411,7 @@ extern struct filterops vnode_filtops; * - Add a filterops to the sysfilt_ops. Private filters should be added at the end of * the Private filters section of the array. */ -static struct filterops *sysfilt_ops[EVFILTID_MAX] = { +SECURITY_READ_ONLY_EARLY(static struct filterops *) sysfilt_ops[EVFILTID_MAX] = { /* Public Filters */ [~EVFILT_READ] = &file_filtops, [~EVFILT_WRITE] = &file_filtops, @@ -375,6 +433,8 @@ static struct filterops *sysfilt_ops[EVFILTID_MAX] = { #endif [~EVFILT_EXCEPT] = &file_filtops, + [~EVFILT_WORKLOOP] = &workloop_filtops, + /* Private filters */ [EVFILTID_KQREAD] = &kqread_filtops, [EVFILTID_PIPE_R] = &pipe_rfiltops, @@ -388,7 +448,8 @@ static struct filterops *sysfilt_ops[EVFILTID_MAX] = { [EVFILTID_BPFREAD] = &bpfread_filtops, [EVFILTID_NECP_FD] = &necp_fd_rfiltops, [EVFILTID_FSEVENT] = &fsevent_filtops, - [EVFILTID_VN] = &vnode_filtops + [EVFILTID_VN] = &vnode_filtops, + [EVFILTID_TTY] = &tty_filtops }; /* waitq prepost callback */ @@ -407,27 +468,51 @@ void waitq_set__CALLING_PREPOST_HOOK__(void *kq_hook, void *knote_hook, int qos) #define _PTHREAD_PRIORITY_QOS_CLASS_SHIFT_32 8 #endif +static inline __kdebug_only +uintptr_t +kqr_thread_id(struct kqrequest *kqr) +{ + return (uintptr_t)thread_tid(kqr->kqr_thread); +} + +static inline +boolean_t is_workqueue_thread(thread_t thread) +{ + return (thread_get_tag(thread) & THREAD_TAG_WORKQUEUE); +} + static inline -qos_t canonicalize_kevent_qos(qos_t qos) +void knote_canonicalize_kevent_qos(struct knote *kn) { + struct kqueue *kq = knote_get_kq(kn); unsigned long canonical; + if ((kq->kq_state & (KQ_WORKQ | KQ_WORKLOOP)) == 0) + return; + /* preserve manager and overcommit flags in this case */ - canonical = pthread_priority_canonicalize(qos, FALSE); - return (qos_t)canonical; + canonical = pthread_priority_canonicalize(kn->kn_qos, FALSE); + kn->kn_qos = (qos_t)canonical; } static inline -kq_index_t qos_index_from_qos(qos_t qos, boolean_t propagation) +kq_index_t qos_index_from_qos(struct knote *kn, qos_t qos, boolean_t propagation) { + struct kqueue *kq = knote_get_kq(kn); kq_index_t qos_index; unsigned long flags = 0; + if ((kq->kq_state & (KQ_WORKQ | KQ_WORKLOOP)) == 0) + return QOS_INDEX_KQFILE; + qos_index = (kq_index_t)thread_qos_from_pthread_priority( (unsigned long)qos, &flags); - if (!propagation && (flags & _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG)) - return KQWQ_QOS_MANAGER; + if (kq->kq_state & KQ_WORKQ) { + /* workq kqueues support requesting a manager thread (non-propagation) */ + if (!propagation && (flags & _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG)) + return KQWQ_QOS_MANAGER; + } return qos_index; } @@ -435,38 +520,42 @@ kq_index_t qos_index_from_qos(qos_t qos, boolean_t propagation) static inline qos_t qos_from_qos_index(kq_index_t qos_index) { - if (qos_index == KQWQ_QOS_MANAGER) + /* should only happen for KQ_WORKQ */ + if (qos_index == KQWQ_QOS_MANAGER) return _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG; if (qos_index == 0) - return 0; /* Unspecified */ + return THREAD_QOS_UNSPECIFIED; /* Should have support from pthread kext support */ return (1 << (qos_index - 1 + _PTHREAD_PRIORITY_QOS_CLASS_SHIFT_32)); } +/* kqr lock must be held */ +static inline +unsigned long pthread_priority_for_kqrequest( + struct kqrequest *kqr, + kq_index_t qos_index) +{ + unsigned long priority = qos_from_qos_index(qos_index); + if (kqr->kqr_state & KQR_THOVERCOMMIT) { + priority |= _PTHREAD_PRIORITY_OVERCOMMIT_FLAG; + } + return priority; +} + static inline kq_index_t qos_index_for_servicer(int qos_class, thread_t thread, int flags) { +#pragma unused(thread) kq_index_t qos_index; if (flags & KEVENT_FLAG_WORKQ_MANAGER) return KQWQ_QOS_MANAGER; - /* - * If the caller didn't pass in a class (legacy pthread kext) - * the we use the thread policy QoS of the current thread. - */ - assert(qos_class != -1); - if (qos_class == -1) - qos_index = proc_get_thread_policy(thread, - TASK_POLICY_ATTRIBUTE, - TASK_POLICY_QOS); - else - qos_index = (kq_index_t)qos_class; - - assert(qos_index > 0 && qos_index < KQWQ_NQOS); + qos_index = (kq_index_t)qos_class; + assert(qos_index > 0 && qos_index < KQWQ_QOS_MANAGER); return qos_index; } @@ -499,12 +588,30 @@ kqlock(struct kqueue *kq) lck_spin_lock(&kq->kq_lock); } +static inline void +kqlock_held(__assert_only struct kqueue *kq) +{ + LCK_SPIN_ASSERT(&kq->kq_lock, LCK_ASSERT_OWNED); +} + static inline void kqunlock(struct kqueue *kq) { lck_spin_unlock(&kq->kq_lock); } +static inline void +knhash_lock(proc_t p) +{ + lck_mtx_lock(&p->p_fd->fd_knhashlock); +} + +static inline void +knhash_unlock(proc_t p) +{ + lck_mtx_unlock(&p->p_fd->fd_knhashlock); +} + /* * Convert a kq lock to a knote use referece. @@ -517,17 +624,44 @@ kqunlock(struct kqueue *kq) * - unlock on exit if we get the use reference */ static int -kqlock2knoteuse(struct kqueue *kq, struct knote *kn) +kqlock2knoteuse(struct kqueue *kq, struct knote *kn, int flags) { if (kn->kn_status & (KN_DROPPING | KN_VANISHED)) return (0); assert(kn->kn_status & KN_ATTACHED); kn->kn_inuse++; + if (flags & KNUSE_BOOST) { + set_thread_rwlock_boost(); + } kqunlock(kq); return (1); } +/* + * - kq locked at entry + * - kq unlocked at exit + */ +__disable_tail_calls +static wait_result_t +knoteusewait(struct kqueue *kq, struct knote *kn) +{ + kn->kn_status |= KN_USEWAIT; + waitq_assert_wait64((struct waitq *)&kq->kq_wqs, + CAST_EVENT64_T(&kn->kn_status), + THREAD_UNINT, TIMEOUT_WAIT_FOREVER); + kqunlock(kq); + return thread_block(THREAD_CONTINUE_NULL); +} + +static bool +knoteuse_needs_boost(struct knote *kn, struct kevent_internal_s *kev) +{ + if (knote_fops(kn)->f_needs_boost) { + return knote_fops(kn)->f_needs_boost(kev); + } + return false; +} /* * Convert from a knote use reference back to kq lock. @@ -536,7 +670,7 @@ kqlock2knoteuse(struct kqueue *kq, struct knote *kn) * this is the last one. * * If someone is trying to drop the knote, but the - * caller has events they must deliver, take + * caller has events they must deliver, take * responsibility for the drop later - and wake the * other attempted dropper in a manner that informs * him of the transfer of responsibility. @@ -548,11 +682,16 @@ kqlock2knoteuse(struct kqueue *kq, struct knote *kn) * The kqueue lock is re-taken unconditionally. */ static int -knoteuse2kqlock(struct kqueue *kq, struct knote *kn, int steal_drop) +knoteuse2kqlock(struct kqueue *kq, struct knote *kn, int flags) { int dropped = 0; + int steal_drop = (flags & KNUSE_STEAL_DROP); kqlock(kq); + if (flags & KNUSE_BOOST) { + clear_thread_rwlock_boost(); + } + if (--kn->kn_inuse == 0) { if ((kn->kn_status & KN_ATTACHING) != 0) { @@ -591,12 +730,7 @@ knoteuse2kqlock(struct kqueue *kq, struct knote *kn, int steal_drop) kn->kn_status |= KN_STOLENDROP; /* but we now have to wait to be the last ref */ - kn->kn_status |= KN_USEWAIT; - waitq_assert_wait64((struct waitq *)&kq->kq_wqs, - CAST_EVENT64_T(&kn->kn_status), - THREAD_UNINT, TIMEOUT_WAIT_FOREVER); - kqunlock(kq); - thread_block(THREAD_CONTINUE_NULL); + knoteusewait(kq, kn); kqlock(kq); } else { dropped = 1; @@ -620,25 +754,23 @@ knoteuse2kqlock(struct kqueue *kq, struct knote *kn, int steal_drop) * (caller will start over at lookup). * * - kq locked at entry - * - unlocked on exit + * - unlocked on exit */ static int -kqlock2knotedetach(struct kqueue *kq, struct knote *kn) +kqlock2knotedetach(struct kqueue *kq, struct knote *kn, int flags) { if ((kn->kn_status & KN_DROPPING) || kn->kn_inuse) { /* have to wait for dropper or current uses to go away */ - kn->kn_status |= KN_USEWAIT; - waitq_assert_wait64((struct waitq *)&kq->kq_wqs, - CAST_EVENT64_T(&kn->kn_status), - THREAD_UNINT, TIMEOUT_WAIT_FOREVER); - kqunlock(kq); - thread_block(THREAD_CONTINUE_NULL); + knoteusewait(kq, kn); return (0); } assert((kn->kn_status & KN_VANISHED) == 0); assert(kn->kn_status & KN_ATTACHED); kn->kn_status &= ~KN_ATTACHED; kn->kn_status |= KN_VANISHED; + if (flags & KNUSE_BOOST) { + clear_thread_rwlock_boost(); + } kn->kn_inuse++; kqunlock(kq); return (1); @@ -675,12 +807,7 @@ kqlock2knotedrop(struct kqueue *kq, struct knote *kn) return (oktodrop); } } - kn->kn_status |= KN_USEWAIT; - waitq_assert_wait64((struct waitq *)&kq->kq_wqs, - CAST_EVENT64_T(&kn->kn_status), - THREAD_UNINT, TIMEOUT_WAIT_FOREVER); - kqunlock(kq); - result = thread_block(THREAD_CONTINUE_NULL); + result = knoteusewait(kq, kn); /* THREAD_RESTART == another thread stole the knote drop */ return (result == THREAD_AWAKENED); } @@ -709,9 +836,9 @@ knote_put(struct knote *kn) #endif static int -filt_fileattach(struct knote *kn) +filt_fileattach(struct knote *kn, struct kevent_internal_s *kev) { - return (fo_kqfilter(kn->kn_fp, kn, vfs_context_current())); + return (fo_kqfilter(kn->kn_fp, kn, kev, vfs_context_current())); } #define f_flag f_fglob->fg_flag @@ -781,8 +908,10 @@ filt_kqprocess(struct knote *kn, struct filt_process_s *data, struct kevent_inte return res; } +#pragma mark EVFILT_PROC + static int -filt_procattach(struct knote *kn) +filt_procattach(struct knote *kn, __unused struct kevent_internal_s *kev) { struct proc *p; @@ -1007,32 +1136,95 @@ filt_procprocess(struct knote *kn, struct filt_process_s *data, struct kevent_in return res; } + +#pragma mark EVFILT_TIMER + + /* - * filt_timervalidate - process data from user + * Values stored in the knote at rest (using Mach absolute time units) * - * Converts to either interval or deadline format. + * kn->kn_hook where the thread_call object is stored + * kn->kn_ext[0] next deadline or 0 if immediate expiration + * kn->kn_ext[1] leeway value + * kn->kn_sdata interval timer: the interval + * absolute/deadline timer: 0 + * kn->kn_data fire count + */ + +static lck_mtx_t _filt_timerlock; + +static void filt_timerlock(void) { lck_mtx_lock(&_filt_timerlock); } +static void filt_timerunlock(void) { lck_mtx_unlock(&_filt_timerlock); } + +static inline void filt_timer_assert_locked(void) +{ + LCK_MTX_ASSERT(&_filt_timerlock, LCK_MTX_ASSERT_OWNED); +} + +/* state flags stored in kn_hookid */ +#define TIMER_RUNNING 0x1 +#define TIMER_CANCELWAIT 0x2 + +/* + * filt_timervalidate - process data from user * - * The saved-data field in the knote contains the - * time value. The saved filter-flags indicates - * the unit of measurement. + * Sets up the deadline, interval, and leeway from the provided user data * - * After validation, either the saved-data field - * contains the interval in absolute time, or ext[0] - * contains the expected deadline. If that deadline - * is in the past, ext[0] is 0. + * Input: + * kn_sdata timer deadline or interval time + * kn_sfflags style of timer, unit of measurement * - * Returns EINVAL for unrecognized units of time. + * Output: + * kn_sdata either interval in abstime or 0 if non-repeating timer + * ext[0] fire deadline in abs/cont time + * (or 0 if NOTE_ABSOLUTE and deadline is in past) * - * Timer filter lock is held. + * Returns: + * EINVAL Invalid user data parameters * + * Called with timer filter lock held. */ static int filt_timervalidate(struct knote *kn) { + /* + * There are 4 knobs that need to be chosen for a timer registration: + * + * A) Units of time (what is the time duration of the specified number) + * Absolute and interval take: + * NOTE_SECONDS, NOTE_USECONDS, NOTE_NSECONDS, NOTE_MACHTIME + * Defaults to milliseconds if not specified + * + * B) Clock epoch (what is the zero point of the specified number) + * For interval, there is none + * For absolute, defaults to the gettimeofday/calendar epoch + * With NOTE_MACHTIME, uses mach_absolute_time() + * With NOTE_MACHTIME and NOTE_MACH_CONTINUOUS_TIME, uses mach_continuous_time() + * + * C) The knote's behavior on delivery + * Interval timer causes the knote to arm for the next interval unless one-shot is set + * Absolute is a forced one-shot timer which deletes on delivery + * TODO: Add a way for absolute to be not forced one-shot + * + * D) Whether the time duration is relative to now or absolute + * Interval fires at now + duration when it is set up + * Absolute fires at now + difference between now walltime and passed in walltime + * With NOTE_MACHTIME it fires at an absolute MAT or MCT. + * + * E) Whether the timer continues to tick across sleep + * By default all three do not. + * For interval and absolute, NOTE_MACH_CONTINUOUS_TIME causes them to tick across sleep + * With NOTE_ABSOLUTE | NOTE_MACHTIME | NOTE_MACH_CONTINUOUS_TIME: + * expires when mach_continuous_time() is > the passed in value. + */ + + filt_timer_assert_locked(); + uint64_t multiplier; - uint64_t raw = 0; - switch (kn->kn_sfflags & (NOTE_SECONDS|NOTE_USECONDS|NOTE_NSECONDS)) { + boolean_t use_abstime = FALSE; + + switch (kn->kn_sfflags & (NOTE_SECONDS|NOTE_USECONDS|NOTE_NSECONDS|NOTE_MACHTIME)) { case NOTE_SECONDS: multiplier = NSEC_PER_SEC; break; @@ -1042,6 +1234,10 @@ filt_timervalidate(struct knote *kn) case NOTE_NSECONDS: multiplier = 1; break; + case NOTE_MACHTIME: + multiplier = 0; + use_abstime = TRUE; + break; case 0: /* milliseconds (default) */ multiplier = NSEC_PER_SEC / 1000; break; @@ -1049,89 +1245,123 @@ filt_timervalidate(struct knote *kn) return (EINVAL); } - /* transform the slop delta(leeway) in kn_ext[1] if passed to same time scale */ - if(kn->kn_sfflags & NOTE_LEEWAY){ - nanoseconds_to_absolutetime((uint64_t)kn->kn_ext[1] * multiplier, &raw); - kn->kn_ext[1] = raw; - } + /* transform the leeway in kn_ext[1] to same time scale */ + if (kn->kn_sfflags & NOTE_LEEWAY) { + uint64_t leeway_abs; + + if (use_abstime) { + leeway_abs = (uint64_t)kn->kn_ext[1]; + } else { + uint64_t leeway_ns; + if (os_mul_overflow((uint64_t)kn->kn_ext[1], multiplier, &leeway_ns)) + return (ERANGE); - nanoseconds_to_absolutetime((uint64_t)kn->kn_sdata * multiplier, &raw); + nanoseconds_to_absolutetime(leeway_ns, &leeway_abs); + } - kn->kn_ext[0] = 0; - kn->kn_sdata = 0; + kn->kn_ext[1] = leeway_abs; + } if (kn->kn_sfflags & NOTE_ABSOLUTE) { - clock_sec_t seconds; - clock_nsec_t nanoseconds; - uint64_t now; + uint64_t deadline_abs; + + if (use_abstime) { + deadline_abs = (uint64_t)kn->kn_sdata; + } else { + uint64_t calendar_deadline_ns; + + if (os_mul_overflow((uint64_t)kn->kn_sdata, multiplier, &calendar_deadline_ns)) + return (ERANGE); + + /* calendar_deadline_ns is in nanoseconds since the epoch */ + + clock_sec_t seconds; + clock_nsec_t nanoseconds; + + /* + * Note that the conversion through wall-time is only done once. + * + * If the relationship between MAT and gettimeofday changes, + * the underlying timer does not update. + * + * TODO: build a wall-time denominated timer_call queue + * and a flag to request DTRTing with wall-time timers + */ + clock_get_calendar_nanotime(&seconds, &nanoseconds); + + uint64_t calendar_now_ns = (uint64_t)seconds * NSEC_PER_SEC + nanoseconds; - clock_get_calendar_nanotime(&seconds, &nanoseconds); - nanoseconds_to_absolutetime((uint64_t)seconds * NSEC_PER_SEC + - nanoseconds, &now); + /* if deadline is in the future */ + if (calendar_now_ns < calendar_deadline_ns) { + uint64_t interval_ns = calendar_deadline_ns - calendar_now_ns; + uint64_t interval_abs; - /* if time is in the future */ - if (now < raw) { - raw -= now; + nanoseconds_to_absolutetime(interval_ns, &interval_abs); + + /* + * Note that the NOTE_MACH_CONTINUOUS_TIME flag here only + * causes the timer to keep ticking across sleep, but + * it does not change the calendar timebase. + */ - if (kn->kn_sfflags & NOTE_MACH_CONTINUOUS_TIME) { - clock_continuoustime_interval_to_deadline(raw, - &kn->kn_ext[0]); + if (kn->kn_sfflags & NOTE_MACH_CONTINUOUS_TIME) + clock_continuoustime_interval_to_deadline(interval_abs, + &deadline_abs); + else + clock_absolutetime_interval_to_deadline(interval_abs, + &deadline_abs); } else { - clock_absolutetime_interval_to_deadline(raw, - &kn->kn_ext[0]); + deadline_abs = 0; /* cause immediate expiration */ } } + + kn->kn_ext[0] = deadline_abs; + kn->kn_sdata = 0; /* NOTE_ABSOLUTE is non-repeating */ + } else if (kn->kn_sdata < 0) { + /* + * Negative interval timers fire immediately, once. + * + * Ideally a negative interval would be an error, but certain clients + * pass negative values on accident, and expect an event back. + * + * In the old implementation the timer would repeat with no delay + * N times until mach_absolute_time() + (N * interval) underflowed, + * then it would wait ~forever by accidentally arming a timer for the far future. + * + * We now skip the power-wasting hot spin phase and go straight to the idle phase. + */ + + kn->kn_sdata = 0; /* non-repeating */ + kn->kn_ext[0] = 0; /* expire immediately */ } else { - kn->kn_sdata = raw; + uint64_t interval_abs = 0; + + if (use_abstime) { + interval_abs = (uint64_t)kn->kn_sdata; + } else { + uint64_t interval_ns; + if (os_mul_overflow((uint64_t)kn->kn_sdata, multiplier, &interval_ns)) + return (ERANGE); + + nanoseconds_to_absolutetime(interval_ns, &interval_abs); + } + + uint64_t deadline = 0; + + if (kn->kn_sfflags & NOTE_MACH_CONTINUOUS_TIME) + clock_continuoustime_interval_to_deadline(interval_abs, &deadline); + else + clock_absolutetime_interval_to_deadline(interval_abs, &deadline); + + kn->kn_sdata = interval_abs; /* default to a repeating timer */ + kn->kn_ext[0] = deadline; } return (0); } -/* - * filt_timerupdate - compute the next deadline - * - * Repeating timers store their interval in kn_sdata. Absolute - * timers have already calculated the deadline, stored in ext[0]. - * - * On return, the next deadline (or zero if no deadline is needed) - * is stored in kn_ext[0]. - * - * Timer filter lock is held. - */ -static void -filt_timerupdate(struct knote *kn, int num_fired) -{ - assert(num_fired > 0); - /* if there's no interval, deadline is just in kn_ext[0] */ - if (kn->kn_sdata == 0) - return; - /* if timer hasn't fired before, fire in interval nsecs */ - if (kn->kn_ext[0] == 0) { - assert(num_fired == 1); - if (kn->kn_sfflags & NOTE_MACH_CONTINUOUS_TIME) { - clock_continuoustime_interval_to_deadline(kn->kn_sdata, - &kn->kn_ext[0]); - } else { - clock_absolutetime_interval_to_deadline(kn->kn_sdata, - &kn->kn_ext[0]); - } - } else { - /* - * If timer has fired before, schedule the next pop - * relative to the last intended deadline. - * - * We could check for whether the deadline has expired, - * but the thread call layer can handle that. - * - * Go forward an additional number of periods, in the case the - * timer fired multiple times while the system was asleep. - */ - kn->kn_ext[0] += (kn->kn_sdata * num_fired); - } -} /* * filt_timerexpire - the timer callout routine @@ -1155,6 +1385,7 @@ filt_timerexpire(void *knx, __unused void *spare) /* no "object" for timers, so fake a list */ SLIST_INIT(&timer_list); SLIST_INSERT_HEAD(&timer_list, kn, kn_selnext); + KNOTE(&timer_list, 1); /* if someone is waiting for timer to pop */ @@ -1164,6 +1395,8 @@ filt_timerexpire(void *knx, __unused void *spare) CAST_EVENT64_T(&kn->kn_hook), THREAD_AWAKENED, WAITQ_ALL_PRIORITIES); + + kn->kn_hookid &= ~TIMER_CANCELWAIT; } filt_timerunlock(); @@ -1172,44 +1405,114 @@ filt_timerexpire(void *knx, __unused void *spare) /* * Cancel a running timer (or wait for the pop). * Timer filter lock is held. + * May drop and retake the timer filter lock. */ static void filt_timercancel(struct knote *kn) { - struct kqueue *kq = knote_get_kq(kn); - thread_call_t callout = kn->kn_hook; - boolean_t cancelled; - - if (kn->kn_hookid & TIMER_RUNNING) { - /* cancel the callout if we can */ - cancelled = thread_call_cancel(callout); - if (cancelled) { - kn->kn_hookid &= ~TIMER_RUNNING; - } else { - /* we have to wait for the expire routine. */ - kn->kn_hookid |= TIMER_CANCELWAIT; - waitq_assert_wait64((struct waitq *)&kq->kq_wqs, - CAST_EVENT64_T(&kn->kn_hook), - THREAD_UNINT, TIMEOUT_WAIT_FOREVER); - filt_timerunlock(); - thread_block(THREAD_CONTINUE_NULL); - filt_timerlock(); - assert((kn->kn_hookid & TIMER_RUNNING) == 0); - } + filt_timer_assert_locked(); + + assert((kn->kn_hookid & TIMER_CANCELWAIT) == 0); + + /* if no timer, then we're good */ + if ((kn->kn_hookid & TIMER_RUNNING) == 0) + return; + + thread_call_t callout = (thread_call_t)kn->kn_hook; + + /* cancel the callout if we can */ + if (thread_call_cancel(callout)) { + kn->kn_hookid &= ~TIMER_RUNNING; + return; } + + /* cancel failed, we have to wait for the in-flight expire routine */ + + kn->kn_hookid |= TIMER_CANCELWAIT; + + struct kqueue *kq = knote_get_kq(kn); + + waitq_assert_wait64((struct waitq *)&kq->kq_wqs, + CAST_EVENT64_T(&kn->kn_hook), + THREAD_UNINT, TIMEOUT_WAIT_FOREVER); + + filt_timerunlock(); + thread_block(THREAD_CONTINUE_NULL); + filt_timerlock(); + + assert((kn->kn_hookid & TIMER_CANCELWAIT) == 0); + assert((kn->kn_hookid & TIMER_RUNNING) == 0); +} + +static void +filt_timerarm(struct knote *kn) +{ + filt_timer_assert_locked(); + + assert((kn->kn_hookid & TIMER_RUNNING) == 0); + + thread_call_t callout = (thread_call_t)kn->kn_hook; + + uint64_t deadline = kn->kn_ext[0]; + uint64_t leeway = kn->kn_ext[1]; + + int filter_flags = kn->kn_sfflags; + unsigned int timer_flags = 0; + + if (filter_flags & NOTE_CRITICAL) + timer_flags |= THREAD_CALL_DELAY_USER_CRITICAL; + else if (filter_flags & NOTE_BACKGROUND) + timer_flags |= THREAD_CALL_DELAY_USER_BACKGROUND; + else + timer_flags |= THREAD_CALL_DELAY_USER_NORMAL; + + if (filter_flags & NOTE_LEEWAY) + timer_flags |= THREAD_CALL_DELAY_LEEWAY; + + if (filter_flags & NOTE_MACH_CONTINUOUS_TIME) + timer_flags |= THREAD_CALL_CONTINUOUS; + + thread_call_enter_delayed_with_leeway(callout, NULL, + deadline, leeway, + timer_flags); + + kn->kn_hookid |= TIMER_RUNNING; +} + +/* + * Does this knote need a timer armed for it, or should it be ready immediately? + */ +static boolean_t +filt_timer_is_ready(struct knote *kn) +{ + uint64_t now; + + if (kn->kn_sfflags & NOTE_MACH_CONTINUOUS_TIME) + now = mach_continuous_time(); + else + now = mach_absolute_time(); + + uint64_t deadline = kn->kn_ext[0]; + + if (deadline < now) + return TRUE; + else + return FALSE; } /* * Allocate a thread call for the knote's lifetime, and kick off the timer. */ static int -filt_timerattach(struct knote *kn) +filt_timerattach(struct knote *kn, __unused struct kevent_internal_s *kev) { thread_call_t callout; int error; - int res; - callout = thread_call_allocate(filt_timerexpire, kn); + callout = thread_call_allocate_with_options(filt_timerexpire, + (thread_call_param_t)kn, THREAD_CALL_PRIORITY_HIGH, + THREAD_CALL_OPTIONS_ONCE); + if (NULL == callout) { kn->kn_flags = EV_ERROR; kn->kn_data = ENOMEM; @@ -1217,52 +1520,37 @@ filt_timerattach(struct knote *kn) } filt_timerlock(); - error = filt_timervalidate(kn); - if (error != 0) { - filt_timerunlock(); - thread_call_free(callout); + + if ((error = filt_timervalidate(kn)) != 0) { kn->kn_flags = EV_ERROR; - kn->kn_data = error; + kn->kn_data = error; + filt_timerunlock(); + + __assert_only boolean_t freed = thread_call_free(callout); + assert(freed); return 0; } kn->kn_hook = (void*)callout; kn->kn_hookid = 0; + kn->kn_flags |= EV_CLEAR; - /* absolute=EV_ONESHOT */ + /* NOTE_ABSOLUTE implies EV_ONESHOT */ if (kn->kn_sfflags & NOTE_ABSOLUTE) kn->kn_flags |= EV_ONESHOT; - filt_timerupdate(kn, 1); - if (kn->kn_ext[0]) { - kn->kn_flags |= EV_CLEAR; - unsigned int timer_flags = 0; - if (kn->kn_sfflags & NOTE_CRITICAL) - timer_flags |= THREAD_CALL_DELAY_USER_CRITICAL; - else if (kn->kn_sfflags & NOTE_BACKGROUND) - timer_flags |= THREAD_CALL_DELAY_USER_BACKGROUND; - else - timer_flags |= THREAD_CALL_DELAY_USER_NORMAL; - - if (kn->kn_sfflags & NOTE_LEEWAY) - timer_flags |= THREAD_CALL_DELAY_LEEWAY; - if (kn->kn_sfflags & NOTE_MACH_CONTINUOUS_TIME) - timer_flags |= THREAD_CALL_CONTINUOUS; - - thread_call_enter_delayed_with_leeway(callout, NULL, - kn->kn_ext[0], kn->kn_ext[1], timer_flags); + boolean_t timer_ready = FALSE; - kn->kn_hookid |= TIMER_RUNNING; - } else { - /* fake immediate */ + if ((timer_ready = filt_timer_is_ready(kn))) { + /* cause immediate expiration */ kn->kn_data = 1; + } else { + filt_timerarm(kn); } - res = (kn->kn_data > 0); - filt_timerunlock(); - return res; + return timer_ready; } /* @@ -1280,93 +1568,25 @@ filt_timerdetach(struct knote *kn) filt_timerunlock(); - thread_call_free(callout); -} - - -static int filt_timer_num_fired(struct knote *kn) -{ - /* by default we fire a timer once */ - int num_fired = 1; - - /* - * When the time base is mach_continuous_time, we have to calculate - * the number of times the timer fired while we were asleep. - */ - if ((kn->kn_sfflags & NOTE_MACH_CONTINUOUS_TIME) && - (kn->kn_sdata != 0) && - (kn->kn_ext[0] != 0)) - { - const uint64_t now = mach_continuous_time(); - // time for timer to fire (right now) is kn_ext[0] - // kn_sdata is period for timer to fire - assert(now >= kn->kn_ext[0]); - assert(kn->kn_sdata > 0); - - const uint64_t overrun_ticks = now - kn->kn_ext[0]; - const uint64_t kn_sdata = kn->kn_sdata; - - if (overrun_ticks < kn_sdata) { - num_fired = 1; - } else if (overrun_ticks < (kn_sdata << 1)) { - num_fired = 2; - } else { - num_fired = (overrun_ticks / kn_sdata) + 1; - } - } - - return num_fired; + __assert_only boolean_t freed = thread_call_free(callout); + assert(freed); } /* - * filt_timer - post events to a timer knote + * filt_timerevent - post events to a timer knote * - * Count the timer fire and re-arm as requested. - * This always crosses the threshold of interest, - * so always return an indication that the knote - * should be activated (if not already). + * Called in the context of filt_timerexpire with + * the filt_timerlock held */ static int -filt_timer( - struct knote *kn, - long hint) +filt_timerevent(struct knote *kn, __unused long hint) { -#pragma unused(hint) - - /* real timer pop -- timer lock held by filt_timerexpire */ - int num_fired = filt_timer_num_fired(kn); - kn->kn_data += num_fired; - - if (((kn->kn_hookid & TIMER_CANCELWAIT) == 0) && - ((kn->kn_flags & EV_ONESHOT) == 0)) { - /* evaluate next time to fire */ - filt_timerupdate(kn, num_fired); - - if (kn->kn_ext[0]) { - unsigned int timer_flags = 0; - - /* keep the callout and re-arm */ - if (kn->kn_sfflags & NOTE_CRITICAL) - timer_flags |= THREAD_CALL_DELAY_USER_CRITICAL; - else if (kn->kn_sfflags & NOTE_BACKGROUND) - timer_flags |= THREAD_CALL_DELAY_USER_BACKGROUND; - else - timer_flags |= THREAD_CALL_DELAY_USER_NORMAL; - - if (kn->kn_sfflags & NOTE_LEEWAY) - timer_flags |= THREAD_CALL_DELAY_LEEWAY; + filt_timer_assert_locked(); - thread_call_enter_delayed_with_leeway(kn->kn_hook, NULL, - kn->kn_ext[0], kn->kn_ext[1], timer_flags); - - kn->kn_hookid |= TIMER_RUNNING; - } - } + kn->kn_data = 1; return (1); } - - /* * filt_timertouch - update timer knote with new user input * @@ -1380,13 +1600,18 @@ filt_timertouch( struct kevent_internal_s *kev) { int error; - int res; filt_timerlock(); - /* cancel current call */ + /* + * cancel current call - drops and retakes lock + * TODO: not safe against concurrent touches? + */ filt_timercancel(kn); + /* clear if the timer had previously fired, the user no longer wants to see it */ + kn->kn_data = 0; + /* capture the new values used to compute deadline */ kn->kn_sdata = kev->data; kn->kn_sfflags = kev->fflags; @@ -1400,42 +1625,24 @@ filt_timertouch( error = filt_timervalidate(kn); if (error) { /* no way to report error, so mark it in the knote */ - filt_timerunlock(); kn->kn_flags |= EV_ERROR; kn->kn_data = error; + filt_timerunlock(); return 1; } - /* start timer if necessary */ - filt_timerupdate(kn, 1); - - if (kn->kn_ext[0]) { - unsigned int timer_flags = 0; - if (kn->kn_sfflags & NOTE_CRITICAL) - timer_flags |= THREAD_CALL_DELAY_USER_CRITICAL; - else if (kn->kn_sfflags & NOTE_BACKGROUND) - timer_flags |= THREAD_CALL_DELAY_USER_BACKGROUND; - else - timer_flags |= THREAD_CALL_DELAY_USER_NORMAL; - - if (kn->kn_sfflags & NOTE_LEEWAY) - timer_flags |= THREAD_CALL_DELAY_LEEWAY; - - thread_call_enter_delayed_with_leeway(kn->kn_hook, NULL, - kn->kn_ext[0], kn->kn_ext[1], timer_flags); + boolean_t timer_ready = FALSE; - kn->kn_hookid |= TIMER_RUNNING; - } else { - /* pretend the timer has fired */ + if ((timer_ready = filt_timer_is_ready(kn))) { + /* cause immediate expiration */ kn->kn_data = 1; + } else { + filt_timerarm(kn); } - /* capture if already fired */ - res = (kn->kn_data > 0); - filt_timerunlock(); - return res; + return timer_ready; } /* @@ -1453,43 +1660,114 @@ filt_timerprocess( { filt_timerlock(); - /* user-query */ - if (kn->kn_data == 0) { + if (kn->kn_data == 0 || (kn->kn_hookid & TIMER_CANCELWAIT)) { + /* + * kn_data = 0: + * The timer hasn't yet fired, so there's nothing to deliver + * TIMER_CANCELWAIT: + * touch is in the middle of canceling the timer, + * so don't deliver or re-arm anything + * + * This can happen if a touch resets a timer that had fired + * without being processed + */ filt_timerunlock(); return 0; } + if (kn->kn_sdata != 0 && ((kn->kn_flags & EV_ERROR) == 0)) { + /* + * This is a 'repeating' timer, so we have to emit + * how many intervals expired between the arm + * and the process. + * + * A very strange style of interface, because + * this could easily be done in the client... + */ + + /* The timer better have had expired... */ + assert((kn->kn_hookid & TIMER_RUNNING) == 0); + + uint64_t now; + + if (kn->kn_sfflags & NOTE_MACH_CONTINUOUS_TIME) + now = mach_continuous_time(); + else + now = mach_absolute_time(); + + uint64_t first_deadline = kn->kn_ext[0]; + uint64_t interval_abs = kn->kn_sdata; + uint64_t orig_arm_time = first_deadline - interval_abs; + + assert(now > orig_arm_time); + assert(now > first_deadline); + + uint64_t elapsed = now - orig_arm_time; + + uint64_t num_fired = elapsed / interval_abs; + + /* + * To reach this code, we must have seen the timer pop + * and be in repeating mode, so therefore it must have been + * more than 'interval' time since the attach or last + * successful touch. + * + * An unsuccessful touch would: + * disarm the timer + * clear kn_data + * clear kn_sdata + * set EV_ERROR + * all of which will prevent this code from running. + */ + assert(num_fired > 0); + + /* report how many intervals have elapsed to the user */ + kn->kn_data = (int64_t) num_fired; + + /* We only need to re-arm the timer if it's not about to be destroyed */ + if ((kn->kn_flags & EV_ONESHOT) == 0) { + /* fire at the end of the next interval */ + uint64_t new_deadline = first_deadline + num_fired * interval_abs; + + assert(new_deadline > now); + + kn->kn_ext[0] = new_deadline; + + filt_timerarm(kn); + } + } + /* * Copy out the interesting kevent state, * but don't leak out the raw time calculations. + * + * TODO: potential enhancements - tell the user about: + * - deadline to which this timer thought it was expiring + * - return kn_sfflags in the fflags field so the client can know + * under what flags the timer fired */ *kev = kn->kn_kevent; kev->ext[0] = 0; /* kev->ext[1] = 0; JMM - shouldn't we hide this too? */ - /* - * reset the timer pop count in kn_data - * and (optionally) clear the fflags. - */ + /* we have delivered the event, reset the timer pop count */ kn->kn_data = 0; - if (kn->kn_flags & EV_CLEAR) - kn->kn_fflags = 0; filt_timerunlock(); return 1; } -static void -filt_timerlock(void) -{ - lck_mtx_lock(&_filt_timerlock); -} +SECURITY_READ_ONLY_EARLY(static struct filterops) timer_filtops = { + .f_attach = filt_timerattach, + .f_detach = filt_timerdetach, + .f_event = filt_timerevent, + .f_touch = filt_timertouch, + .f_process = filt_timerprocess, +}; + + +#pragma mark EVFILT_USER -static void -filt_timerunlock(void) -{ - lck_mtx_unlock(&_filt_timerlock); -} static void filt_userlock(void) @@ -1504,12 +1782,12 @@ filt_userunlock(void) } static int -filt_userattach(struct knote *kn) +filt_userattach(struct knote *kn, __unused struct kevent_internal_s *kev) { /* EVFILT_USER knotes are not attached to anything in the kernel */ /* Cant discover this knote until after attach - so no lock needed */ kn->kn_hook = NULL; - if (kn->kn_fflags & NOTE_TRIGGER) { + if (kn->kn_sfflags & NOTE_TRIGGER) { kn->kn_hookid = 1; } else { kn->kn_hookid = 0; @@ -1599,203 +1877,1334 @@ filt_userprocess( return 1; } +#pragma mark EVFILT_WORKLOOP + +#if DEBUG || DEVELOPMENT /* - * JMM - placeholder for not-yet-implemented filters + * see src/queue_internal.h in libdispatch */ -static int -filt_badattach(__unused struct knote *kn) +#define DISPATCH_QUEUE_ENQUEUED 0x1ull +#endif + +static inline void +filt_wllock(struct kqworkloop *kqwl) { - kn->kn_flags |= EV_ERROR; - kn->kn_data = ENOTSUP; - return 0; + lck_mtx_lock(&kqwl->kqwl_statelock); } -struct kqueue * -kqueue_alloc(struct proc *p, unsigned int flags) +static inline void +filt_wlunlock(struct kqworkloop *kqwl) { - struct filedesc *fdp = p->p_fd; - struct kqueue *kq = NULL; - int policy; - void *hook; - uint64_t kq_addr_offset; + lck_mtx_unlock(&kqwl->kqwl_statelock); +} - if (flags & KEVENT_FLAG_WORKQ) { - struct kqworkq *kqwq; - int i; +static inline void +filt_wlheld(__assert_only struct kqworkloop *kqwl) +{ + LCK_MTX_ASSERT(&kqwl->kqwl_statelock, LCK_MTX_ASSERT_OWNED); +} - kqwq = (struct kqworkq *)zalloc(kqworkq_zone); - if (kqwq == NULL) - return NULL; +#define WL_OWNER_SUSPENDED ((thread_t)(~0ull)) /* special owner when suspended */ - kq = &kqwq->kqwq_kqueue; - bzero(kqwq, sizeof (struct kqworkq)); +static inline bool +filt_wlowner_is_valid(thread_t owner) +{ + return owner != THREAD_NULL && owner != WL_OWNER_SUSPENDED; +} - kqwq->kqwq_state = KQ_WORKQ; +static inline bool +filt_wlshould_end_ownership(struct kqworkloop *kqwl, + struct kevent_internal_s *kev, int error) +{ + thread_t owner = kqwl->kqwl_owner; + return (error == 0 || error == ESTALE) && + (kev->fflags & NOTE_WL_END_OWNERSHIP) && + (owner == current_thread() || owner == WL_OWNER_SUSPENDED); +} - for (i = 0; i < KQWQ_NBUCKETS; i++) { - TAILQ_INIT(&kq->kq_queue[i]); - } - for (i = 0; i < KQWQ_NQOS; i++) { - TAILQ_INIT(&kqwq->kqwq_request[i].kqr_suppressed); - } +static inline bool +filt_wlshould_update_ownership(struct kevent_internal_s *kev, int error) +{ + return error == 0 && (kev->fflags & NOTE_WL_DISCOVER_OWNER) && + kev->ext[EV_EXTIDX_WL_ADDR]; +} - lck_spin_init(&kqwq->kqwq_reqlock, kq_lck_grp, kq_lck_attr); - policy = SYNC_POLICY_FIFO; - hook = (void *)kqwq; - - } else { - struct kqfile *kqf; - - kqf = (struct kqfile *)zalloc(kqfile_zone); - if (kqf == NULL) - return NULL; +static inline bool +filt_wlshould_set_async_qos(struct kevent_internal_s *kev, int error, + kq_index_t async_qos) +{ + if (error != 0) { + return false; + } + if (async_qos != THREAD_QOS_UNSPECIFIED) { + return true; + } + if ((kev->fflags & NOTE_WL_THREAD_REQUEST) && (kev->flags & EV_DELETE)) { + /* see filt_wlprocess() */ + return true; + } + return false; +} - kq = &kqf->kqf_kqueue; - bzero(kqf, sizeof (struct kqfile)); - TAILQ_INIT(&kq->kq_queue[0]); - TAILQ_INIT(&kqf->kqf_suppressed); - - policy = SYNC_POLICY_FIFO | SYNC_POLICY_PREPOST; - hook = NULL; +__result_use_check +static int +filt_wlupdateowner(struct kqworkloop *kqwl, struct kevent_internal_s *kev, + int error, kq_index_t async_qos) +{ + struct kqrequest *kqr = &kqwl->kqwl_request; + thread_t cur_owner, new_owner, extra_thread_ref = THREAD_NULL; + kq_index_t cur_override = THREAD_QOS_UNSPECIFIED; + kq_index_t old_owner_override = THREAD_QOS_UNSPECIFIED; + boolean_t ipc_override_is_sync = false; + boolean_t old_owner_override_is_sync = false; + int action = KQWL_UTQ_NONE; - } + filt_wlheld(kqwl); - waitq_set_init(&kq->kq_wqs, policy, NULL, hook); - lck_spin_init(&kq->kq_lock, kq_lck_grp, kq_lck_attr); - kq->kq_p = p; + /* + * The owner is only changed under both the filt_wllock and the + * kqwl_req_lock. Looking at it with either one held is fine. + */ + cur_owner = kqwl->kqwl_owner; + if (filt_wlshould_end_ownership(kqwl, kev, error)) { + new_owner = THREAD_NULL; + } else if (filt_wlshould_update_ownership(kev, error)) { + /* + * Decipher the owner port name, and translate accordingly. + * The low 2 bits were borrowed for other flags, so mask them off. + */ + uint64_t udata = kev->ext[EV_EXTIDX_WL_VALUE]; + mach_port_name_t new_owner_name = (mach_port_name_t)udata & ~0x3; + if (new_owner_name != MACH_PORT_NULL) { + new_owner_name = ipc_entry_name_mask(new_owner_name); + } - if (fdp->fd_knlistsize < 0) { - proc_fdlock(p); - if (fdp->fd_knlistsize < 0) - fdp->fd_knlistsize = 0; /* this process has had a kq */ - proc_fdunlock(p); + if (MACH_PORT_VALID(new_owner_name)) { + new_owner = port_name_to_thread(new_owner_name); + if (new_owner == THREAD_NULL) + return EOWNERDEAD; + extra_thread_ref = new_owner; + } else if (new_owner_name == MACH_PORT_DEAD) { + new_owner = WL_OWNER_SUSPENDED; + } else { + /* + * We never want to learn a new owner that is NULL. + * Ownership should be ended with END_OWNERSHIP. + */ + new_owner = cur_owner; + } + } else { + new_owner = cur_owner; } - kq_addr_offset = ((uintptr_t)kq - (uintptr_t)VM_MIN_KERNEL_AND_KEXT_ADDRESS); - /* Assert that the address can be pointer compacted for use with knote */ - assert(kq_addr_offset < (uint64_t)(1ull << KNOTE_KQ_BITSIZE)); - return (kq); -} - -/* - * kqueue_dealloc - detach all knotes from a kqueue and free it - * - * We walk each list looking for knotes referencing this - * this kqueue. If we find one, we try to drop it. But - * if we fail to get a drop reference, that will wait - * until it is dropped. So, we can just restart again - * safe in the assumption that the list will eventually - * not contain any more references to this kqueue (either - * we dropped them all, or someone else did). - * - * Assumes no new events are being added to the kqueue. - * Nothing locked on entry or exit. - */ -void -kqueue_dealloc(struct kqueue *kq) -{ - struct proc *p; - struct filedesc *fdp; - struct knote *kn; - int i; + if (filt_wlshould_set_async_qos(kev, error, async_qos)) { + action = KQWL_UTQ_SET_ASYNC_QOS; + } + if (cur_owner == new_owner && action == KQWL_UTQ_NONE) { + goto out; + } - if (kq == NULL) - return; + kqwl_req_lock(kqwl); - p = kq->kq_p; - fdp = p->p_fd; + /* If already tracked as servicer, don't track as owner */ + if ((kqr->kqr_state & KQR_BOUND) && new_owner == kqr->kqr_thread) { + kqwl->kqwl_owner = new_owner = THREAD_NULL; + } - proc_fdlock(p); - for (i = 0; i < fdp->fd_knlistsize; i++) { - kn = SLIST_FIRST(&fdp->fd_knlist[i]); - while (kn != NULL) { - if (kq == knote_get_kq(kn)) { - kqlock(kq); - proc_fdunlock(p); - /* drop it ourselves or wait */ - if (kqlock2knotedrop(kq, kn)) { - knote_drop(kn, p); + if (cur_owner != new_owner) { + kqwl->kqwl_owner = new_owner; + if (new_owner == extra_thread_ref) { + /* we just transfered this ref to kqwl_owner */ + extra_thread_ref = THREAD_NULL; + } + cur_override = kqworkloop_combined_qos(kqwl, &ipc_override_is_sync); + old_owner_override = kqr->kqr_dsync_owner_qos; + old_owner_override_is_sync = kqr->kqr_owner_override_is_sync; + + if (filt_wlowner_is_valid(new_owner)) { + /* override it before we drop the old */ + if (cur_override != THREAD_QOS_UNSPECIFIED) { + thread_add_ipc_override(new_owner, cur_override); + } + if (ipc_override_is_sync) { + thread_add_sync_ipc_override(new_owner); + } + /* Update the kqr to indicate that owner has sync ipc override */ + kqr->kqr_dsync_owner_qos = cur_override; + kqr->kqr_owner_override_is_sync = ipc_override_is_sync; + thread_starts_owning_workloop(new_owner); + if ((kqr->kqr_state & (KQR_THREQUESTED | KQR_BOUND)) == KQR_THREQUESTED) { + if (action == KQWL_UTQ_NONE) { + action = KQWL_UTQ_REDRIVE_EVENTS; } - proc_fdlock(p); - /* start over at beginning of list */ - kn = SLIST_FIRST(&fdp->fd_knlist[i]); - continue; } - kn = SLIST_NEXT(kn, kn_link); - } - } - if (fdp->fd_knhashmask != 0) { - for (i = 0; i < (int)fdp->fd_knhashmask + 1; i++) { - kn = SLIST_FIRST(&fdp->fd_knhash[i]); - while (kn != NULL) { - if (kq == knote_get_kq(kn)) { - kqlock(kq); - proc_fdunlock(p); - /* drop it ourselves or wait */ - if (kqlock2knotedrop(kq, kn)) { - knote_drop(kn, p); - } - proc_fdlock(p); - /* start over at beginning of list */ - kn = SLIST_FIRST(&fdp->fd_knhash[i]); - continue; + } else if (new_owner == THREAD_NULL) { + kqr->kqr_dsync_owner_qos = THREAD_QOS_UNSPECIFIED; + kqr->kqr_owner_override_is_sync = false; + if ((kqr->kqr_state & (KQR_THREQUESTED | KQR_WAKEUP)) == KQR_WAKEUP) { + if (action == KQWL_UTQ_NONE) { + action = KQWL_UTQ_REDRIVE_EVENTS; } - kn = SLIST_NEXT(kn, kn_link); } } } - proc_fdunlock(p); - /* - * waitq_set_deinit() remove the KQ's waitq set from - * any select sets to which it may belong. - */ - waitq_set_deinit(&kq->kq_wqs); - lck_spin_destroy(&kq->kq_lock, kq_lck_grp); + if (action != KQWL_UTQ_NONE) { + kqworkloop_update_threads_qos(kqwl, action, async_qos); + } - if (kq->kq_state & KQ_WORKQ) { - struct kqworkq *kqwq = (struct kqworkq *)kq; + kqwl_req_unlock(kqwl); - lck_spin_destroy(&kqwq->kqwq_reqlock, kq_lck_grp); - zfree(kqworkq_zone, kqwq); - } else { - struct kqfile *kqf = (struct kqfile *)kq; + /* Now that we are unlocked, drop the override and ref on old owner */ + if (new_owner != cur_owner && filt_wlowner_is_valid(cur_owner)) { + if (old_owner_override != THREAD_QOS_UNSPECIFIED) { + thread_drop_ipc_override(cur_owner); + } + if (old_owner_override_is_sync) { + thread_drop_sync_ipc_override(cur_owner); + } + thread_ends_owning_workloop(cur_owner); + thread_deallocate(cur_owner); + } - zfree(kqfile_zone, kqf); +out: + if (extra_thread_ref) { + thread_deallocate(extra_thread_ref); } + return error; } -int -kqueue_body(struct proc *p, fp_allocfn_t fp_zalloc, void *cra, int32_t *retval) +static int +filt_wldebounce( + struct kqworkloop *kqwl, + struct kevent_internal_s *kev, + int default_result) { - struct kqueue *kq; - struct fileproc *fp; - int fd, error; + user_addr_t addr = CAST_USER_ADDR_T(kev->ext[EV_EXTIDX_WL_ADDR]); + uint64_t udata; + int error; - error = falloc_withalloc(p, - &fp, &fd, vfs_context_current(), fp_zalloc, cra); - if (error) { - return (error); - } + /* we must have the workloop state mutex held */ + filt_wlheld(kqwl); - kq = kqueue_alloc(p, 0); - if (kq == NULL) { - fp_free(p, fd, fp); - return (ENOMEM); - } + /* Do we have a debounce address to work with? */ + if (addr) { + uint64_t kdata = kev->ext[EV_EXTIDX_WL_VALUE]; + uint64_t mask = kev->ext[EV_EXTIDX_WL_MASK]; - fp->f_flag = FREAD | FWRITE; - fp->f_ops = &kqueueops; - fp->f_data = kq; + error = copyin_word(addr, &udata, sizeof(udata)); + if (error) { + return error; + } - proc_fdlock(p); - *fdflags(p, fd) |= UF_EXCLOSE; - procfdtbl_releasefd(p, fd, NULL); - fp_drop(p, fd, fp, 1); - proc_fdunlock(p); + /* update state as copied in */ + kev->ext[EV_EXTIDX_WL_VALUE] = udata; - *retval = fd; - return (error); + /* If the masked bits don't match, reject it as stale */ + if ((udata & mask) != (kdata & mask)) { + return ESTALE; + } + +#if DEBUG || DEVELOPMENT + if ((kev->fflags & NOTE_WL_THREAD_REQUEST) && !(kev->flags & EV_DELETE)) { + if ((udata & DISPATCH_QUEUE_ENQUEUED) == 0) { + panic("kevent: workloop %#016llx is not enqueued " + "(kev:%p dq_state:%#016llx)", kev->udata, kev, udata); + } + } +#endif + } + + return default_result; +} + +/* + * Remembers the last updated that came in from userspace for debugging reasons. + * - fflags is mirrored from the userspace kevent + * - ext[i, i != VALUE] is mirrored from the userspace kevent + * - ext[VALUE] is set to what the kernel loaded atomically + * - data is set to the error if any + */ +static inline void +filt_wlremember_last_update( + __assert_only struct kqworkloop *kqwl, + struct knote *kn, + struct kevent_internal_s *kev, + int error) +{ + filt_wlheld(kqwl); + kn->kn_fflags = kev->fflags; + kn->kn_data = error; + memcpy(kn->kn_ext, kev->ext, sizeof(kev->ext)); +} + +/* + * Return which operations on EVFILT_WORKLOOP need to be protected against + * knoteusewait() causing priority inversions. + */ +static bool +filt_wlneeds_boost(struct kevent_internal_s *kev) +{ + if (kev == NULL) { + /* + * this is an f_process() usecount, and it can cause a drop to wait + */ + return true; + } + if (kev->fflags & NOTE_WL_THREAD_REQUEST) { + /* + * All operations on thread requests may starve drops or re-attach of + * the same knote, all of them need boosts. None of what we do under + * thread-request usecount holds blocks anyway. + */ + return true; + } + if (kev->fflags & NOTE_WL_SYNC_WAIT) { + /* + * this may call filt_wlwait() and we don't want to hold any boost when + * woken up, this would cause background threads contending on + * dispatch_sync() to wake up at 64 and be preempted immediately when + * this drops. + */ + return false; + } + + /* + * SYNC_WAIT knotes when deleted don't need to be rushed, there's no + * detach/reattach race with these ever. In addition to this, when the + * SYNC_WAIT knote is dropped, the caller is no longer receiving the + * workloop overrides if any, and we'd rather schedule other threads than + * him, he's not possibly stalling anything anymore. + */ + return (kev->flags & EV_DELETE) == 0; +} + +static int +filt_wlattach(struct knote *kn, struct kevent_internal_s *kev) +{ + struct kqueue *kq = knote_get_kq(kn); + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + int error = 0; + kq_index_t qos_index = 0; + + if ((kq->kq_state & KQ_WORKLOOP) == 0) { + error = ENOTSUP; + goto out; + } + +#if DEVELOPMENT || DEBUG + if (kev->ident == 0 && kev->udata == 0 && kev->fflags == 0) { + struct kqrequest *kqr = &kqwl->kqwl_request; + + kqwl_req_lock(kqwl); + kev->fflags = 0; + if (kqr->kqr_dsync_waiters) { + kev->fflags |= NOTE_WL_SYNC_WAIT; + } + if (kqr->kqr_qos_index) { + kev->fflags |= NOTE_WL_THREAD_REQUEST; + } + if (kqwl->kqwl_owner == WL_OWNER_SUSPENDED) { + kev->ext[0] = ~0ull; + } else { + kev->ext[0] = thread_tid(kqwl->kqwl_owner); + } + kev->ext[1] = thread_tid(kqwl->kqwl_request.kqr_thread); + kev->ext[2] = thread_owned_workloops_count(current_thread()); + kev->ext[3] = kn->kn_kevent.ext[3]; + kqwl_req_unlock(kqwl); + error = EBUSY; + goto out; + } +#endif + + /* Some simple validation */ + int command = (kn->kn_sfflags & NOTE_WL_COMMANDS_MASK); + switch (command) { + case NOTE_WL_THREAD_REQUEST: + if (kn->kn_id != kqwl->kqwl_dynamicid) { + error = EINVAL; + goto out; + } + qos_index = qos_index_from_qos(kn, kn->kn_qos, FALSE); + if (qos_index < THREAD_QOS_MAINTENANCE || + qos_index > THREAD_QOS_USER_INTERACTIVE) { + error = ERANGE; + goto out; + } + break; + case NOTE_WL_SYNC_WAIT: + case NOTE_WL_SYNC_WAKE: + if (kq->kq_state & KQ_NO_WQ_THREAD) { + error = ENOTSUP; + goto out; + } + if (kn->kn_id == kqwl->kqwl_dynamicid) { + error = EINVAL; + goto out; + } + if ((kn->kn_flags & EV_DISABLE) == 0) { + error = EINVAL; + goto out; + } + if (kn->kn_sfflags & NOTE_WL_END_OWNERSHIP) { + error = EINVAL; + goto out; + } + break; + default: + error = EINVAL; + goto out; + } + + filt_wllock(kqwl); + kn->kn_hook = NULL; + + if (command == NOTE_WL_THREAD_REQUEST && kqwl->kqwl_request.kqr_qos_index) { + /* + * There already is a thread request, and well, you're only allowed + * one per workloop, so fail the attach. + * + * Note: kqr_qos_index is always set with the wllock held, so we + * don't need to take the kqr lock. + */ + error = EALREADY; + } else { + /* Make sure user and kernel are in agreement on important state */ + error = filt_wldebounce(kqwl, kev, 0); + } + + error = filt_wlupdateowner(kqwl, kev, error, qos_index); + filt_wlunlock(kqwl); +out: + if (error) { + kn->kn_flags |= EV_ERROR; + /* If userland wants ESTALE to be hidden, fail the attach anyway */ + if (error == ESTALE && (kn->kn_sfflags & NOTE_WL_IGNORE_ESTALE)) { + error = 0; + } + kn->kn_data = error; + return 0; + } + + /* Just attaching the thread request successfully will fire it */ + return command == NOTE_WL_THREAD_REQUEST; +} + +__attribute__((noinline,not_tail_called)) +static int +filt_wlwait(struct kqworkloop *kqwl, + struct knote *kn, + struct kevent_internal_s *kev) +{ + filt_wlheld(kqwl); + assert((kn->kn_sfflags & NOTE_WL_SYNC_WAKE) == 0); + + /* + * Hint to the wakeup side that this thread is waiting. Also used by + * stackshot for waitinfo. + */ + kn->kn_hook = current_thread(); + + thread_set_pending_block_hint(current_thread(), kThreadWaitWorkloopSyncWait); + + wait_result_t wr = assert_wait(kn, THREAD_ABORTSAFE); + + if (wr == THREAD_WAITING) { + kq_index_t qos_index = qos_index_from_qos(kn, kev->qos, TRUE); + struct kqrequest *kqr = &kqwl->kqwl_request; + + thread_t thread_to_handoff = THREAD_NULL; /* holds +1 thread ref */ + + thread_t kqwl_owner = kqwl->kqwl_owner; + if (filt_wlowner_is_valid(kqwl_owner)) { + thread_reference(kqwl_owner); + thread_to_handoff = kqwl_owner; + } + + kqwl_req_lock(kqwl); + + if (qos_index) { + assert(kqr->kqr_dsync_waiters < UINT16_MAX); + kqr->kqr_dsync_waiters++; + if (qos_index > kqr->kqr_dsync_waiters_qos) { + kqworkloop_update_threads_qos(kqwl, + KQWL_UTQ_SET_SYNC_WAITERS_QOS, qos_index); + } + } + + if ((kqr->kqr_state & KQR_BOUND) && thread_to_handoff == THREAD_NULL) { + assert(kqr->kqr_thread != THREAD_NULL); + thread_t servicer = kqr->kqr_thread; + + thread_reference(servicer); + thread_to_handoff = servicer; + } + + kqwl_req_unlock(kqwl); + + filt_wlunlock(kqwl); + + /* TODO: use continuation based blocking <rdar://problem/31299584> */ + + /* consume a refcount on thread_to_handoff, then thread_block() */ + wr = thread_handoff(thread_to_handoff); + thread_to_handoff = THREAD_NULL; + + filt_wllock(kqwl); + + /* clear waiting state (only one waiting thread - so no race) */ + assert(kn->kn_hook == current_thread()); + + if (qos_index) { + kqwl_req_lock(kqwl); + assert(kqr->kqr_dsync_waiters > 0); + if (--kqr->kqr_dsync_waiters == 0) { + assert(kqr->kqr_dsync_waiters_qos); + kqworkloop_update_threads_qos(kqwl, + KQWL_UTQ_SET_SYNC_WAITERS_QOS, 0); + } + kqwl_req_unlock(kqwl); + } + } + + kn->kn_hook = NULL; + + switch (wr) { + case THREAD_AWAKENED: + return 0; + case THREAD_INTERRUPTED: + return EINTR; + case THREAD_RESTART: + return ECANCELED; + default: + panic("filt_wlattach: unexpected wait result %d", wr); + return EINVAL; + } +} + +/* called in stackshot context to report the thread responsible for blocking this thread */ +void +kdp_workloop_sync_wait_find_owner(__assert_only thread_t thread, + event64_t event, + thread_waitinfo_t *waitinfo) +{ + struct knote *kn = (struct knote*) event; + assert(kdp_is_in_zone(kn, "knote zone")); + + assert(kn->kn_hook == thread); + + struct kqueue *kq = knote_get_kq(kn); + assert(kdp_is_in_zone(kq, "kqueue workloop zone")); + assert(kq->kq_state & KQ_WORKLOOP); + + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + struct kqrequest *kqr = &kqwl->kqwl_request; + + thread_t kqwl_owner = kqwl->kqwl_owner; + thread_t servicer = kqr->kqr_thread; + + if (kqwl_owner == WL_OWNER_SUSPENDED) { + waitinfo->owner = STACKSHOT_WAITOWNER_SUSPENDED; + } else if (kqwl_owner != THREAD_NULL) { + assert(kdp_is_in_zone(kqwl_owner, "threads")); + + waitinfo->owner = thread_tid(kqwl->kqwl_owner); + } else if (servicer != THREAD_NULL) { + assert(kdp_is_in_zone(servicer, "threads")); + + waitinfo->owner = thread_tid(servicer); + } else if (kqr->kqr_state & KQR_THREQUESTED) { + waitinfo->owner = STACKSHOT_WAITOWNER_THREQUESTED; + } else { + waitinfo->owner = 0; + } + + waitinfo->context = kqwl->kqwl_dynamicid; + + return; +} + +/* + * Takes kqueue locked, returns locked, may drop in the middle and/or block for a while + */ +static int +filt_wlpost_attach(struct knote *kn, struct kevent_internal_s *kev) +{ + struct kqueue *kq = knote_get_kq(kn); + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + int error = 0; + + if (kev->fflags & NOTE_WL_SYNC_WAIT) { + if (kqlock2knoteuse(kq, kn, KNUSE_NONE)) { + filt_wllock(kqwl); + /* if the wake has already preposted, don't wait */ + if ((kn->kn_sfflags & NOTE_WL_SYNC_WAKE) == 0) + error = filt_wlwait(kqwl, kn, kev); + filt_wlunlock(kqwl); + knoteuse2kqlock(kq, kn, KNUSE_NONE); + } + } + return error; +} + +static void +filt_wldetach(__assert_only struct knote *kn) +{ + assert(knote_get_kq(kn)->kq_state & KQ_WORKLOOP); + + /* + * Thread requests have nothing to detach. + * Sync waiters should have been aborted out + * and drop their refs before we could drop/ + * detach their knotes. + */ + assert(kn->kn_hook == NULL); +} + +static int +filt_wlevent( + __unused struct knote *kn, + __unused long hint) +{ + panic("filt_wlevent"); + return 0; +} + +static int +filt_wlvalidate_kev_flags(struct knote *kn, struct kevent_internal_s *kev) +{ + int new_commands = kev->fflags & NOTE_WL_COMMANDS_MASK; + int sav_commands = kn->kn_sfflags & NOTE_WL_COMMANDS_MASK; + int error = 0; + + switch (new_commands) { + case NOTE_WL_THREAD_REQUEST: + /* thread requests can only update themselves */ + if (sav_commands != new_commands) + error = EINVAL; + break; + + case NOTE_WL_SYNC_WAIT: + if (kev->fflags & NOTE_WL_END_OWNERSHIP) + error = EINVAL; + /* FALLTHROUGH */ + case NOTE_WL_SYNC_WAKE: + /* waits and wakes can update themselves or their counterparts */ + if (!(sav_commands & (NOTE_WL_SYNC_WAIT | NOTE_WL_SYNC_WAKE))) + error = EINVAL; + if (kev->fflags & NOTE_WL_UPDATE_QOS) + error = EINVAL; + if ((kev->flags & (EV_ENABLE | EV_DELETE)) == EV_ENABLE) + error = EINVAL; + if (kev->flags & EV_DELETE) { + /* + * Really this is not supported: there is absolutely no reason + * whatsoever to want to fail the drop of a NOTE_WL_SYNC_WAIT knote. + */ + if (kev->ext[EV_EXTIDX_WL_ADDR] && kev->ext[EV_EXTIDX_WL_MASK]) { + error = EINVAL; + } + } + break; + + default: + error = EINVAL; + } + if ((kev->flags & EV_DELETE) && (kev->fflags & NOTE_WL_DISCOVER_OWNER)) { + error = EINVAL; + } + return error; +} + +static int +filt_wltouch( + struct knote *kn, + struct kevent_internal_s *kev) +{ + struct kqueue *kq = knote_get_kq(kn); + int error = 0; + struct kqworkloop *kqwl; + + assert(kq->kq_state & KQ_WORKLOOP); + kqwl = (struct kqworkloop *)kq; + + error = filt_wlvalidate_kev_flags(kn, kev); + if (error) { + goto out; + } + + filt_wllock(kqwl); + + /* Make sure user and kernel are in agreement on important state */ + error = filt_wldebounce(kqwl, kev, 0); + if (error) { + error = filt_wlupdateowner(kqwl, kev, error, 0); + goto out_unlock; + } + + int new_command = kev->fflags & NOTE_WL_COMMANDS_MASK; + switch (new_command) { + case NOTE_WL_THREAD_REQUEST: + assert(kqwl->kqwl_request.kqr_qos_index != THREAD_QOS_UNSPECIFIED); + break; + + case NOTE_WL_SYNC_WAIT: + /* + * we need to allow waiting several times on the same knote because + * of EINTR. If it's already woken though, it won't block. + */ + break; + + case NOTE_WL_SYNC_WAKE: + if (kn->kn_sfflags & NOTE_WL_SYNC_WAKE) { + /* disallow waking the same knote twice */ + error = EALREADY; + goto out_unlock; + } + if (kn->kn_hook) { + thread_wakeup_thread((event_t)kn, (thread_t)kn->kn_hook); + } + break; + + default: + error = EINVAL; + goto out_unlock; + } + + /* + * Save off any additional fflags/data we just accepted + * But only keep the last round of "update" bits we acted on which helps + * debugging a lot. + */ + kn->kn_sfflags &= ~NOTE_WL_UPDATES_MASK; + kn->kn_sfflags |= kev->fflags; + kn->kn_sdata = kev->data; + + kq_index_t qos_index = THREAD_QOS_UNSPECIFIED; + + if (kev->fflags & NOTE_WL_UPDATE_QOS) { + qos_t qos = pthread_priority_canonicalize(kev->qos, FALSE); + + if (kn->kn_qos != qos) { + qos_index = qos_index_from_qos(kn, qos, FALSE); + if (qos_index == THREAD_QOS_UNSPECIFIED) { + error = ERANGE; + goto out_unlock; + } + kqlock(kq); + if (kn->kn_status & KN_QUEUED) { + knote_dequeue(kn); + knote_set_qos_index(kn, qos_index); + knote_enqueue(kn); + knote_wakeup(kn); + } else { + knote_set_qos_index(kn, qos_index); + } + kn->kn_qos = qos; + kqunlock(kq); + } + } + + error = filt_wlupdateowner(kqwl, kev, 0, qos_index); + if (error) { + goto out_unlock; + } + + if (new_command == NOTE_WL_SYNC_WAIT) { + /* if the wake has already preposted, don't wait */ + if ((kn->kn_sfflags & NOTE_WL_SYNC_WAKE) == 0) + error = filt_wlwait(kqwl, kn, kev); + } + +out_unlock: + filt_wlremember_last_update(kqwl, kn, kev, error); + filt_wlunlock(kqwl); +out: + if (error) { + if (error == ESTALE && (kev->fflags & NOTE_WL_IGNORE_ESTALE)) { + /* If userland wants ESTALE to be hidden, do not activate */ + return 0; + } + kev->flags |= EV_ERROR; + kev->data = error; + return 0; + } + /* Just touching the thread request successfully will fire it */ + return new_command == NOTE_WL_THREAD_REQUEST; +} + +static int +filt_wldrop_and_unlock( + struct knote *kn, + struct kevent_internal_s *kev) +{ + struct kqueue *kq = knote_get_kq(kn); + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + int error = 0, knoteuse_flags = KNUSE_NONE; + + kqlock_held(kq); + + assert(kev->flags & EV_DELETE); + assert(kq->kq_state & KQ_WORKLOOP); + + error = filt_wlvalidate_kev_flags(kn, kev); + if (error) { + goto out; + } + + if (kn->kn_sfflags & NOTE_WL_THREAD_REQUEST) { + knoteuse_flags |= KNUSE_BOOST; + } + + /* take a usecount to allow taking the filt_wllock */ + if (!kqlock2knoteuse(kq, kn, knoteuse_flags)) { + /* knote is being dropped already */ + error = EINPROGRESS; + goto out; + } + + filt_wllock(kqwl); + + /* + * Make sure user and kernel are in agreement on important state + * + * Userland will modify bits to cause this to fail for the touch / drop + * race case (when a drop for a thread request quiescing comes in late after + * the workloop has been woken up again). + */ + error = filt_wldebounce(kqwl, kev, 0); + + if (!knoteuse2kqlock(kq, kn, knoteuse_flags)) { + /* knote is no longer alive */ + error = EINPROGRESS; + goto out_unlock; + } + + if (!error && (kn->kn_sfflags & NOTE_WL_THREAD_REQUEST) && kn->kn_inuse) { + /* + * There is a concurrent drop or touch happening, we can't resolve this, + * userland has to redrive. + * + * The race we're worried about here is the following: + * + * f_touch | f_drop_and_unlock + * ------------------------+-------------------------------------------- + * | kqlock() + * | kqlock2knoteuse() + * | filt_wllock() + * | debounces successfully + * kqlock() | + * kqlock2knoteuse | + * filt_wllock() <BLOCKS> | + * | knoteuse2kqlock() + * | filt_wlunlock() + * | kqlock2knotedrop() <BLOCKS, WAKES f_touch> + * debounces successfully | + * filt_wlunlock() | + * caller WAKES f_drop | + * | performs drop, but f_touch should have won + * + * So if the usecount is not 0 here, we need to wait for it to drop and + * redrive the whole logic (including looking up the knote again). + */ + filt_wlunlock(kqwl); + knoteusewait(kq, kn); + return ERESTART; + } + + /* + * If error is 0 this will set kqr_qos_index to THREAD_QOS_UNSPECIFIED + * + * If error is 0 or ESTALE this may drop ownership and cause a thread + * request redrive, however the kqlock is held which prevents f_process() to + * run until we did the drop for real. + */ + error = filt_wlupdateowner(kqwl, kev, error, 0); + if (error) { + goto out_unlock; + } + + if ((kn->kn_sfflags & (NOTE_WL_SYNC_WAIT | NOTE_WL_SYNC_WAKE)) == + NOTE_WL_SYNC_WAIT) { + /* + * When deleting a SYNC_WAIT knote that hasn't been woken up + * explicitly, issue a wake up. + */ + kn->kn_sfflags |= NOTE_WL_SYNC_WAKE; + if (kn->kn_hook) { + thread_wakeup_thread((event_t)kn, (thread_t)kn->kn_hook); + } + } + +out_unlock: + filt_wlremember_last_update(kqwl, kn, kev, error); + filt_wlunlock(kqwl); + +out: + if (error == 0) { + /* If nothing failed, do the regular knote drop. */ + if (kqlock2knotedrop(kq, kn)) { + knote_drop(kn, current_proc()); + } else { + error = EINPROGRESS; + } + } else { + kqunlock(kq); + } + if (error == ESTALE && (kev->fflags & NOTE_WL_IGNORE_ESTALE)) { + error = 0; + } + if (error == EINPROGRESS) { + /* + * filt_wlprocess() makes sure that no event can be delivered for + * NOTE_WL_THREAD_REQUEST knotes once a drop is happening, and + * NOTE_WL_SYNC_* knotes are never fired. + * + * It means that EINPROGRESS is about a state that userland cannot + * observe for this filter (an event being delivered concurrently from + * a drop), so silence the error. + */ + error = 0; + } + return error; +} + +static int +filt_wlprocess( + struct knote *kn, + __unused struct filt_process_s *data, + struct kevent_internal_s *kev) +{ + struct kqueue *kq = knote_get_kq(kn); + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + struct kqrequest *kqr = &kqwl->kqwl_request; + int rc = 0; + + assert(kq->kq_state & KQ_WORKLOOP); + + /* only thread requests should get here */ + assert(kn->kn_sfflags & NOTE_WL_THREAD_REQUEST); + if (kn->kn_sfflags & NOTE_WL_THREAD_REQUEST) { + filt_wllock(kqwl); + assert(kqr->kqr_qos_index != THREAD_QOS_UNSPECIFIED); + if (kqwl->kqwl_owner) { + /* + * <rdar://problem/33584321> userspace sometimes due to events being + * delivered but not triggering a drain session can cause a process + * of the thread request knote. + * + * When that happens, the automatic deactivation due to process + * would swallow the event, so we have to activate the knote again. + */ + kqlock(kq); + knote_activate(kn); + kqunlock(kq); + } else if (kqr->kqr_qos_index) { +#if DEBUG || DEVELOPMENT + user_addr_t addr = CAST_USER_ADDR_T(kn->kn_ext[EV_EXTIDX_WL_ADDR]); + task_t t = current_task(); + uint64_t val; + if (addr && task_is_active(t) && !task_is_halting(t) && + copyin_word(addr, &val, sizeof(val)) == 0 && + val && (val & DISPATCH_QUEUE_ENQUEUED) == 0) { + panic("kevent: workloop %#016llx is not enqueued " + "(kn:%p dq_state:%#016llx kev.dq_state:%#016llx)", + kn->kn_udata, kn, val, + kn->kn_ext[EV_EXTIDX_WL_VALUE]); + } +#endif + *kev = kn->kn_kevent; + kev->fflags = kn->kn_sfflags; + kev->data = kn->kn_sdata; + kev->qos = kn->kn_qos; + rc = 1; + } + filt_wlunlock(kqwl); + } + return rc; +} + +#pragma mark kevent / knotes + +/* + * JMM - placeholder for not-yet-implemented filters + */ +static int +filt_badattach(__unused struct knote *kn, __unused struct kevent_internal_s *kev) +{ + kn->kn_flags |= EV_ERROR; + kn->kn_data = ENOTSUP; + return 0; +} + +struct kqueue * +kqueue_alloc(struct proc *p, unsigned int flags) +{ + struct filedesc *fdp = p->p_fd; + struct kqueue *kq = NULL; + int policy; + void *hook = NULL; + uint64_t kq_addr_offset; + + if (flags & KEVENT_FLAG_WORKQ) { + struct kqworkq *kqwq; + int i; + + kqwq = (struct kqworkq *)zalloc(kqworkq_zone); + if (kqwq == NULL) + return NULL; + + kq = &kqwq->kqwq_kqueue; + bzero(kqwq, sizeof (struct kqworkq)); + + kqwq->kqwq_state = KQ_WORKQ; + + for (i = 0; i < KQWQ_NBUCKETS; i++) { + TAILQ_INIT(&kq->kq_queue[i]); + } + for (i = 0; i < KQWQ_NQOS; i++) { + kqwq->kqwq_request[i].kqr_qos_index = i; + } + + lck_spin_init(&kqwq->kqwq_reqlock, kq_lck_grp, kq_lck_attr); + policy = SYNC_POLICY_FIFO; + hook = (void *)kqwq; + + } else if (flags & KEVENT_FLAG_WORKLOOP) { + struct kqworkloop *kqwl; + int i; + + kqwl = (struct kqworkloop *)zalloc(kqworkloop_zone); + if (kqwl == NULL) + return NULL; + + bzero(kqwl, sizeof (struct kqworkloop)); + + kqwl->kqwl_state = KQ_WORKLOOP | KQ_DYNAMIC; + kqwl->kqwl_retains = 1; /* donate a retain to creator */ + + kq = &kqwl->kqwl_kqueue; + for (i = 0; i < KQWL_NBUCKETS; i++) { + TAILQ_INIT(&kq->kq_queue[i]); + } + TAILQ_INIT(&kqwl->kqwl_request.kqr_suppressed); + + lck_spin_init(&kqwl->kqwl_reqlock, kq_lck_grp, kq_lck_attr); + lck_mtx_init(&kqwl->kqwl_statelock, kq_lck_grp, kq_lck_attr); + + policy = SYNC_POLICY_FIFO; + if (flags & KEVENT_FLAG_WORKLOOP_NO_WQ_THREAD) { + policy |= SYNC_POLICY_PREPOST; + kq->kq_state |= KQ_NO_WQ_THREAD; + } else { + hook = (void *)kqwl; + } + + } else { + struct kqfile *kqf; + + kqf = (struct kqfile *)zalloc(kqfile_zone); + if (kqf == NULL) + return NULL; + + kq = &kqf->kqf_kqueue; + bzero(kqf, sizeof (struct kqfile)); + TAILQ_INIT(&kq->kq_queue[0]); + TAILQ_INIT(&kqf->kqf_suppressed); + + policy = SYNC_POLICY_FIFO | SYNC_POLICY_PREPOST; + } + + waitq_set_init(&kq->kq_wqs, policy, NULL, hook); + lck_spin_init(&kq->kq_lock, kq_lck_grp, kq_lck_attr); + kq->kq_p = p; + + if (fdp->fd_knlistsize < 0) { + proc_fdlock(p); + if (fdp->fd_knlistsize < 0) + fdp->fd_knlistsize = 0; /* this process has had a kq */ + proc_fdunlock(p); + } + + kq_addr_offset = ((uintptr_t)kq - (uintptr_t)VM_MIN_KERNEL_AND_KEXT_ADDRESS); + /* Assert that the address can be pointer compacted for use with knote */ + assert(kq_addr_offset < (uint64_t)(1ull << KNOTE_KQ_BITSIZE)); + return (kq); +} + +/* + * knotes_dealloc - detach all knotes for the process and drop them + * + * Called with proc_fdlock held. + * Returns with it locked. + * May drop it temporarily. + * Process is in such a state that it will not try to allocate + * any more knotes during this process (stopped for exit or exec). + */ +void +knotes_dealloc(proc_t p) +{ + struct filedesc *fdp = p->p_fd; + struct kqueue *kq; + struct knote *kn; + struct klist *kn_hash = NULL; + int i; + + /* Close all the fd-indexed knotes up front */ + if (fdp->fd_knlistsize > 0) { + for (i = 0; i < fdp->fd_knlistsize; i++) { + while ((kn = SLIST_FIRST(&fdp->fd_knlist[i])) != NULL) { + kq = knote_get_kq(kn); + kqlock(kq); + proc_fdunlock(p); + /* drop it ourselves or wait */ + if (kqlock2knotedrop(kq, kn)) { + knote_drop(kn, p); + } + proc_fdlock(p); + } + } + /* free the table */ + FREE(fdp->fd_knlist, M_KQUEUE); + fdp->fd_knlist = NULL; + } + fdp->fd_knlistsize = -1; + + knhash_lock(p); + proc_fdunlock(p); + + /* Clean out all the hashed knotes as well */ + if (fdp->fd_knhashmask != 0) { + for (i = 0; i <= (int)fdp->fd_knhashmask; i++) { + while ((kn = SLIST_FIRST(&fdp->fd_knhash[i])) != NULL) { + kq = knote_get_kq(kn); + kqlock(kq); + knhash_unlock(p); + /* drop it ourselves or wait */ + if (kqlock2knotedrop(kq, kn)) { + knote_drop(kn, p); + } + knhash_lock(p); + } + } + kn_hash = fdp->fd_knhash; + fdp->fd_knhashmask = 0; + fdp->fd_knhash = NULL; + } + + knhash_unlock(p); + + /* free the kn_hash table */ + if (kn_hash) + FREE(kn_hash, M_KQUEUE); + + proc_fdlock(p); +} + + +/* + * kqueue_dealloc - detach all knotes from a kqueue and free it + * + * We walk each list looking for knotes referencing this + * this kqueue. If we find one, we try to drop it. But + * if we fail to get a drop reference, that will wait + * until it is dropped. So, we can just restart again + * safe in the assumption that the list will eventually + * not contain any more references to this kqueue (either + * we dropped them all, or someone else did). + * + * Assumes no new events are being added to the kqueue. + * Nothing locked on entry or exit. + * + * Workloop kqueues cant get here unless all the knotes + * are already gone and all requested threads have come + * and gone (cancelled or arrived). + */ +void +kqueue_dealloc(struct kqueue *kq) +{ + struct proc *p; + struct filedesc *fdp; + struct knote *kn; + int i; + + if (kq == NULL) + return; + + p = kq->kq_p; + fdp = p->p_fd; + + proc_fdlock(p); + for (i = 0; i < fdp->fd_knlistsize; i++) { + kn = SLIST_FIRST(&fdp->fd_knlist[i]); + while (kn != NULL) { + if (kq == knote_get_kq(kn)) { + assert((kq->kq_state & KQ_WORKLOOP) == 0); + kqlock(kq); + proc_fdunlock(p); + /* drop it ourselves or wait */ + if (kqlock2knotedrop(kq, kn)) { + knote_drop(kn, p); + } + proc_fdlock(p); + /* start over at beginning of list */ + kn = SLIST_FIRST(&fdp->fd_knlist[i]); + continue; + } + kn = SLIST_NEXT(kn, kn_link); + } + } + knhash_lock(p); + proc_fdunlock(p); + + if (fdp->fd_knhashmask != 0) { + for (i = 0; i < (int)fdp->fd_knhashmask + 1; i++) { + kn = SLIST_FIRST(&fdp->fd_knhash[i]); + while (kn != NULL) { + if (kq == knote_get_kq(kn)) { + assert((kq->kq_state & KQ_WORKLOOP) == 0); + kqlock(kq); + knhash_unlock(p); + /* drop it ourselves or wait */ + if (kqlock2knotedrop(kq, kn)) { + knote_drop(kn, p); + } + knhash_lock(p); + /* start over at beginning of list */ + kn = SLIST_FIRST(&fdp->fd_knhash[i]); + continue; + } + kn = SLIST_NEXT(kn, kn_link); + } + } + } + knhash_unlock(p); + + if (kq->kq_state & KQ_WORKLOOP) { + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + struct kqrequest *kqr = &kqwl->kqwl_request; + thread_t cur_owner = kqwl->kqwl_owner; + + assert(TAILQ_EMPTY(&kqwl->kqwl_request.kqr_suppressed)); + if (filt_wlowner_is_valid(cur_owner)) { + /* + * If the kqueue had an owner that prevented the thread request to + * go through, then no unbind happened, and we may have lingering + * overrides to drop. + */ + if (kqr->kqr_dsync_owner_qos != THREAD_QOS_UNSPECIFIED) { + thread_drop_ipc_override(cur_owner); + kqr->kqr_dsync_owner_qos = THREAD_QOS_UNSPECIFIED; + } + + if (kqr->kqr_owner_override_is_sync) { + thread_drop_sync_ipc_override(cur_owner); + kqr->kqr_owner_override_is_sync = 0; + } + thread_ends_owning_workloop(cur_owner); + thread_deallocate(cur_owner); + kqwl->kqwl_owner = THREAD_NULL; + } + } + + /* + * waitq_set_deinit() remove the KQ's waitq set from + * any select sets to which it may belong. + */ + waitq_set_deinit(&kq->kq_wqs); + lck_spin_destroy(&kq->kq_lock, kq_lck_grp); + + if (kq->kq_state & KQ_WORKQ) { + struct kqworkq *kqwq = (struct kqworkq *)kq; + + lck_spin_destroy(&kqwq->kqwq_reqlock, kq_lck_grp); + zfree(kqworkq_zone, kqwq); + } else if (kq->kq_state & KQ_WORKLOOP) { + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + + assert(kqwl->kqwl_retains == 0); + lck_spin_destroy(&kqwl->kqwl_reqlock, kq_lck_grp); + lck_mtx_destroy(&kqwl->kqwl_statelock, kq_lck_grp); + zfree(kqworkloop_zone, kqwl); + } else { + struct kqfile *kqf = (struct kqfile *)kq; + + zfree(kqfile_zone, kqf); + } +} + +static inline void +kqueue_retain(struct kqueue *kq) +{ + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + uint32_t previous; + + if ((kq->kq_state & KQ_DYNAMIC) == 0) + return; + + previous = OSIncrementAtomic(&kqwl->kqwl_retains); + if (previous == KQ_WORKLOOP_RETAINS_MAX) + panic("kq(%p) retain overflow", kq); + + if (previous == 0) + panic("kq(%p) resurrection", kq); +} + +#define KQUEUE_CANT_BE_LAST_REF 0 +#define KQUEUE_MIGHT_BE_LAST_REF 1 + +static inline int +kqueue_release(struct kqueue *kq, __assert_only int possibly_last) +{ + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + + if ((kq->kq_state & KQ_DYNAMIC) == 0) { + return 0; + } + + assert(kq->kq_state & KQ_WORKLOOP); /* for now */ + uint32_t refs = OSDecrementAtomic(&kqwl->kqwl_retains); + if (__improbable(refs == 0)) { + panic("kq(%p) over-release", kq); + } + if (refs == 1) { + assert(possibly_last); + } + return refs == 1; +} + +int +kqueue_body(struct proc *p, fp_allocfn_t fp_zalloc, void *cra, int32_t *retval) +{ + struct kqueue *kq; + struct fileproc *fp; + int fd, error; + + error = falloc_withalloc(p, + &fp, &fd, vfs_context_current(), fp_zalloc, cra); + if (error) { + return (error); + } + + kq = kqueue_alloc(p, 0); + if (kq == NULL) { + fp_free(p, fd, fp); + return (ENOMEM); + } + + fp->f_flag = FREAD | FWRITE; + fp->f_ops = &kqueueops; + fp->f_data = kq; + + proc_fdlock(p); + *fdflags(p, fd) |= UF_EXCLOSE; + procfdtbl_releasefd(p, fd, NULL); + fp_drop(p, fd, fp, 1); + proc_fdunlock(p); + + *retval = fd; + return (error); } int @@ -2066,8 +3475,7 @@ kevent_continue(__unused struct kqueue *kq, void *data, int error) fd = cont_args->fd; fp = cont_args->fp; - if (fp != NULL) - fp_drop(p, fd, fp, 0); + kevent_put_kq(p, fd, fp, kq); /* don't abandon other output just because of residual copyout failures */ if (error == 0 && data_available && data_resid != data_size) { @@ -2094,7 +3502,7 @@ kevent(struct proc *p, struct kevent_args *uap, int32_t *retval) unsigned int flags = KEVENT_FLAG_LEGACY32; return kevent_internal(p, - uap->fd, + (kqueue_id_t)uap->fd, NULL, uap->changelist, uap->nchanges, uap->eventlist, uap->nevents, 0ULL, 0ULL, @@ -2114,35 +3522,71 @@ kevent64(struct proc *p, struct kevent64_args *uap, int32_t *retval) flags |= KEVENT_FLAG_LEGACY64; return kevent_internal(p, - uap->fd, - uap->changelist, uap->nchanges, - uap->eventlist, uap->nevents, - 0ULL, 0ULL, - flags, - uap->timeout, - kevent_continue, + (kqueue_id_t)uap->fd, NULL, + uap->changelist, uap->nchanges, + uap->eventlist, uap->nevents, + 0ULL, 0ULL, + flags, + uap->timeout, + kevent_continue, + retval); +} + +int +kevent_qos(struct proc *p, struct kevent_qos_args *uap, int32_t *retval) +{ + /* restrict to user flags */ + uap->flags &= KEVENT_FLAG_USER; + + return kevent_internal(p, + (kqueue_id_t)uap->fd, NULL, + uap->changelist, uap->nchanges, + uap->eventlist, uap->nevents, + uap->data_out, (uint64_t)uap->data_available, + uap->flags, + 0ULL, + kevent_continue, + retval); +} + +int +kevent_qos_internal(struct proc *p, int fd, + user_addr_t changelist, int nchanges, + user_addr_t eventlist, int nevents, + user_addr_t data_out, user_size_t *data_available, + unsigned int flags, + int32_t *retval) +{ + return kevent_internal(p, + (kqueue_id_t)fd, NULL, + changelist, nchanges, + eventlist, nevents, + data_out, (uint64_t)data_available, + (flags | KEVENT_FLAG_KERNEL), + 0ULL, + NULL, retval); } int -kevent_qos(struct proc *p, struct kevent_qos_args *uap, int32_t *retval) +kevent_id(struct proc *p, struct kevent_id_args *uap, int32_t *retval) { /* restrict to user flags */ uap->flags &= KEVENT_FLAG_USER; return kevent_internal(p, - uap->fd, + (kqueue_id_t)uap->id, NULL, uap->changelist, uap->nchanges, uap->eventlist, uap->nevents, uap->data_out, (uint64_t)uap->data_available, - uap->flags, + (uap->flags | KEVENT_FLAG_DYNAMIC_KQUEUE), 0ULL, kevent_continue, retval); } -int -kevent_qos_internal(struct proc *p, int fd, +int +kevent_id_internal(struct proc *p, kqueue_id_t *id, user_addr_t changelist, int nchanges, user_addr_t eventlist, int nevents, user_addr_t data_out, user_size_t *data_available, @@ -2150,11 +3594,11 @@ kevent_qos_internal(struct proc *p, int fd, int32_t *retval) { return kevent_internal(p, - fd, + *id, id, changelist, nchanges, eventlist, nevents, data_out, (uint64_t)data_available, - (flags | KEVENT_FLAG_KERNEL), + (flags | KEVENT_FLAG_KERNEL | KEVENT_FLAG_DYNAMIC_KQUEUE), 0ULL, NULL, retval); @@ -2203,82 +3647,600 @@ kevent_get_timeout(struct proc *p, return 0; } -static int -kevent_set_kq_mode(struct kqueue *kq, unsigned int flags) +static int +kevent_set_kq_mode(struct kqueue *kq, unsigned int flags) +{ + /* each kq should only be used for events of one type */ + kqlock(kq); + if (kq->kq_state & (KQ_KEV32 | KQ_KEV64 | KQ_KEV_QOS)) { + if (flags & KEVENT_FLAG_LEGACY32) { + if ((kq->kq_state & KQ_KEV32) == 0) { + kqunlock(kq); + return EINVAL; + } + } else if (kq->kq_state & KQ_KEV32) { + kqunlock(kq); + return EINVAL; + } + } else if (flags & KEVENT_FLAG_LEGACY32) { + kq->kq_state |= KQ_KEV32; + } else if (flags & KEVENT_FLAG_LEGACY64) { + kq->kq_state |= KQ_KEV64; + } else { + kq->kq_state |= KQ_KEV_QOS; + } + kqunlock(kq); + return 0; +} + +#define KQ_HASH(val, mask) (((val) ^ (val >> 8)) & (mask)) +#define CONFIG_KQ_HASHSIZE CONFIG_KN_HASHSIZE + +static inline void +kqhash_lock(proc_t p) +{ + lck_mtx_lock_spin_always(&p->p_fd->fd_kqhashlock); +} + +static inline void +kqhash_lock_held(__assert_only proc_t p) +{ + LCK_MTX_ASSERT(&p->p_fd->fd_kqhashlock, LCK_MTX_ASSERT_OWNED); +} + +static inline void +kqhash_unlock(proc_t p) +{ + lck_mtx_unlock(&p->p_fd->fd_kqhashlock); +} + +static void +kqueue_hash_init_if_needed(proc_t p) +{ + struct filedesc *fdp = p->p_fd; + + kqhash_lock_held(p); + + if (__improbable(fdp->fd_kqhash == NULL)) { + struct kqlist *alloc_hash; + u_long alloc_mask; + + kqhash_unlock(p); + alloc_hash = hashinit(CONFIG_KQ_HASHSIZE, M_KQUEUE, &alloc_mask); + kqhash_lock(p); + + /* See if we won the race */ + if (fdp->fd_kqhashmask == 0) { + fdp->fd_kqhash = alloc_hash; + fdp->fd_kqhashmask = alloc_mask; + } else { + kqhash_unlock(p); + FREE(alloc_hash, M_KQUEUE); + kqhash_lock(p); + } + } +} + +/* + * Called with the kqhash_lock() held + */ +static void +kqueue_hash_insert( + struct proc *p, + kqueue_id_t id, + struct kqueue *kq) +{ + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + struct filedesc *fdp = p->p_fd; + struct kqlist *list; + + /* should hold the kq hash lock */ + kqhash_lock_held(p); + + if ((kq->kq_state & KQ_DYNAMIC) == 0) { + assert(kq->kq_state & KQ_DYNAMIC); + return; + } + + /* only dynamically allocate workloop kqs for now */ + assert(kq->kq_state & KQ_WORKLOOP); + assert(fdp->fd_kqhash); + + kqwl->kqwl_dynamicid = id; + + list = &fdp->fd_kqhash[KQ_HASH(id, fdp->fd_kqhashmask)]; + SLIST_INSERT_HEAD(list, kqwl, kqwl_hashlink); +} + +/* Called with kqhash_lock held */ +static void +kqueue_hash_remove( + struct proc *p, + struct kqueue *kq) +{ + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + struct filedesc *fdp = p->p_fd; + struct kqlist *list; + + /* should hold the kq hash lock */ + kqhash_lock_held(p); + + if ((kq->kq_state & KQ_DYNAMIC) == 0) { + assert(kq->kq_state & KQ_DYNAMIC); + return; + } + assert(kq->kq_state & KQ_WORKLOOP); /* for now */ + list = &fdp->fd_kqhash[KQ_HASH(kqwl->kqwl_dynamicid, fdp->fd_kqhashmask)]; + SLIST_REMOVE(list, kqwl, kqworkloop, kqwl_hashlink); +} + +/* Called with kqhash_lock held */ +static struct kqueue * +kqueue_hash_lookup(struct proc *p, kqueue_id_t id) +{ + struct filedesc *fdp = p->p_fd; + struct kqlist *list; + struct kqworkloop *kqwl; + + /* should hold the kq hash lock */ + kqhash_lock_held(p); + + if (fdp->fd_kqhashmask == 0) return NULL; + + list = &fdp->fd_kqhash[KQ_HASH(id, fdp->fd_kqhashmask)]; + SLIST_FOREACH(kqwl, list, kqwl_hashlink) { + if (kqwl->kqwl_dynamicid == id) { + struct kqueue *kq = (struct kqueue *)kqwl; + + assert(kq->kq_state & KQ_DYNAMIC); + assert(kq->kq_state & KQ_WORKLOOP); /* for now */ + return kq; + } + } + return NULL; +} + +static inline void +kqueue_release_last(struct proc *p, struct kqueue *kq) +{ + if (kq->kq_state & KQ_DYNAMIC) { + kqhash_lock(p); + if (kqueue_release(kq, KQUEUE_MIGHT_BE_LAST_REF)) { + kqueue_hash_remove(p, kq); + kqhash_unlock(p); + kqueue_dealloc(kq); + } else { + kqhash_unlock(p); + } + } +} + +static struct kqueue * +kevent_get_bound_kq(__assert_only struct proc *p, thread_t thread, + unsigned int kev_flags, unsigned int kq_flags) +{ + struct kqueue *kq; + struct uthread *ut = get_bsdthread_info(thread); + + assert(p == get_bsdthreadtask_info(thread)); + + if (!(ut->uu_kqueue_flags & kev_flags)) + return NULL; + + kq = ut->uu_kqueue_bound; + if (!kq) + return NULL; + + if (!(kq->kq_state & kq_flags)) + return NULL; + + return kq; +} + +static int +kevent_get_kq(struct proc *p, kqueue_id_t id, unsigned int flags, struct fileproc **fpp, int *fdp, struct kqueue **kqp) +{ + struct filedesc *descp = p->p_fd; + struct fileproc *fp = NULL; + struct kqueue *kq; + int fd = 0; + int error = 0; + + /* Was the workloop flag passed? Then it is for sure only a workloop */ + if (flags & KEVENT_FLAG_DYNAMIC_KQUEUE) { + assert(flags & KEVENT_FLAG_WORKLOOP); + if (id == (kqueue_id_t)-1 && + (flags & KEVENT_FLAG_KERNEL) && + (flags & KEVENT_FLAG_WORKLOOP)) { + + assert(is_workqueue_thread(current_thread())); + + /* + * when kevent_id_internal is called from within the + * kernel, and the passed 'id' value is '-1' then we + * look for the currently bound workloop kq. + * + * Until pthread kext avoids calling in to kevent_id_internal + * for threads whose fulfill is canceled, calling in unbound + * can't be fatal. + */ + kq = kevent_get_bound_kq(p, current_thread(), + KEVENT_FLAG_WORKLOOP, KQ_WORKLOOP); + if (kq) { + kqueue_retain(kq); + } else { + struct uthread *ut = get_bsdthread_info(current_thread()); + + /* If thread is unbound due to cancel, just return an error */ + if (ut->uu_kqueue_flags == KEVENT_FLAG_WORKLOOP_CANCELED) { + ut->uu_kqueue_flags = 0; + error = ECANCELED; + } else { + panic("Unbound thread called kevent_internal with id=-1" + " uu_kqueue_flags:0x%x, uu_kqueue_bound:%p", + ut->uu_kqueue_flags, ut->uu_kqueue_bound); + } + } + + *fpp = NULL; + *fdp = 0; + *kqp = kq; + return error; + } + + /* try shortcut on kq lookup for bound threads */ + kq = kevent_get_bound_kq(p, current_thread(), KEVENT_FLAG_WORKLOOP, KQ_WORKLOOP); + if (kq != NULL && ((struct kqworkloop *)kq)->kqwl_dynamicid == id) { + + if (flags & KEVENT_FLAG_DYNAMIC_KQ_MUST_NOT_EXIST) { + error = EEXIST; + kq = NULL; + goto out; + } + + /* retain a reference while working with this kq. */ + assert(kq->kq_state & KQ_DYNAMIC); + kqueue_retain(kq); + error = 0; + goto out; + } + + /* look for the kq on the hash table */ + kqhash_lock(p); + kq = kqueue_hash_lookup(p, id); + if (kq == NULL) { + kqhash_unlock(p); + + if (flags & KEVENT_FLAG_DYNAMIC_KQ_MUST_EXIST) { + error = ENOENT; + goto out; + } + + struct kqueue *alloc_kq; + alloc_kq = kqueue_alloc(p, flags); + if (alloc_kq) { + kqhash_lock(p); + kqueue_hash_init_if_needed(p); + kq = kqueue_hash_lookup(p, id); + if (kq == NULL) { + /* insert our new one */ + kq = alloc_kq; + kqueue_hash_insert(p, id, kq); + kqhash_unlock(p); + } else { + /* lost race, retain existing workloop */ + kqueue_retain(kq); + kqhash_unlock(p); + kqueue_release(alloc_kq, KQUEUE_MIGHT_BE_LAST_REF); + kqueue_dealloc(alloc_kq); + } + } else { + error = ENOMEM; + goto out; + } + } else { + + if (flags & KEVENT_FLAG_DYNAMIC_KQ_MUST_NOT_EXIST) { + kqhash_unlock(p); + kq = NULL; + error = EEXIST; + goto out; + } + + /* retain a reference while working with this kq. */ + assert(kq->kq_state & KQ_DYNAMIC); + kqueue_retain(kq); + kqhash_unlock(p); + } + + } else if (flags & KEVENT_FLAG_WORKQ) { + /* must already exist for bound threads. */ + if (flags & KEVENT_FLAG_KERNEL) { + assert(descp->fd_wqkqueue != NULL); + } + + /* + * use the private kq associated with the proc workq. + * Just being a thread within the process (and not + * being the exit/exec thread) is enough to hold a + * reference on this special kq. + */ + kq = descp->fd_wqkqueue; + if (kq == NULL) { + struct kqueue *alloc_kq = kqueue_alloc(p, KEVENT_FLAG_WORKQ); + if (alloc_kq == NULL) + return ENOMEM; + + knhash_lock(p); + if (descp->fd_wqkqueue == NULL) { + kq = descp->fd_wqkqueue = alloc_kq; + knhash_unlock(p); + } else { + knhash_unlock(p); + kq = descp->fd_wqkqueue; + kqueue_dealloc(alloc_kq); + } + } + } else { + /* get a usecount for the kq itself */ + fd = (int)id; + if ((error = fp_getfkq(p, fd, &fp, &kq)) != 0) + return (error); + } + if ((error = kevent_set_kq_mode(kq, flags)) != 0) { + /* drop the usecount */ + if (fp != NULL) + fp_drop(p, fd, fp, 0); + return error; + } + +out: + *fpp = fp; + *fdp = fd; + *kqp = kq; + + return error; +} + +static void +kevent_put_kq( + struct proc *p, + kqueue_id_t id, + struct fileproc *fp, + struct kqueue *kq) +{ + kqueue_release_last(p, kq); + if (fp != NULL) { + assert((kq->kq_state & KQ_WORKQ) == 0); + fp_drop(p, (int)id, fp, 0); + } +} + +static uint64_t +kevent_workloop_serial_no_copyin(proc_t p, uint64_t workloop_id) +{ + uint64_t serial_no = 0; + user_addr_t addr; + int rc; + + if (workloop_id == 0 || p->p_dispatchqueue_serialno_offset == 0) { + return 0; + } + addr = (user_addr_t)(workloop_id + p->p_dispatchqueue_serialno_offset); + + if (proc_is64bit(p)) { + rc = copyin(addr, (caddr_t)&serial_no, sizeof(serial_no)); + } else { + uint32_t serial_no32 = 0; + rc = copyin(addr, (caddr_t)&serial_no32, sizeof(serial_no32)); + serial_no = serial_no32; + } + return rc == 0 ? serial_no : 0; +} + +int +kevent_exit_on_workloop_ownership_leak(thread_t thread) +{ + proc_t p = current_proc(); + struct filedesc *fdp = p->p_fd; + kqueue_id_t workloop_id = 0; + os_reason_t reason; + mach_vm_address_t addr; + uint32_t reason_size; + + kqhash_lock(p); + if (fdp->fd_kqhashmask > 0) { + for (uint32_t i = 0; i < fdp->fd_kqhashmask + 1; i++) { + struct kqworkloop *kqwl; + + SLIST_FOREACH(kqwl, &fdp->fd_kqhash[i], kqwl_hashlink) { + struct kqueue *kq = &kqwl->kqwl_kqueue; + if ((kq->kq_state & KQ_DYNAMIC) && kqwl->kqwl_owner == thread) { + workloop_id = kqwl->kqwl_dynamicid; + break; + } + } + } + } + kqhash_unlock(p); + assert(workloop_id); + + reason = os_reason_create(OS_REASON_LIBSYSTEM, + OS_REASON_LIBSYSTEM_CODE_WORKLOOP_OWNERSHIP_LEAK); + if (reason == OS_REASON_NULL) { + goto out; + } + + reason->osr_flags |= OS_REASON_FLAG_GENERATE_CRASH_REPORT; + reason_size = 2 * sizeof(uint64_t); + reason_size = kcdata_estimate_required_buffer_size(2, reason_size); + if (os_reason_alloc_buffer(reason, reason_size) != 0) { + goto out; + } + + struct kcdata_descriptor *kcd = &reason->osr_kcd_descriptor; + + if (kcdata_get_memory_addr(kcd, EXIT_REASON_WORKLOOP_ID, + sizeof(workloop_id), &addr) == KERN_SUCCESS) { + kcdata_memcpy(kcd, addr, &workloop_id, sizeof(workloop_id)); + } + + uint64_t serial_no = kevent_workloop_serial_no_copyin(p, workloop_id); + if (serial_no && kcdata_get_memory_addr(kcd, EXIT_REASON_DISPATCH_QUEUE_NO, + sizeof(serial_no), &addr) == KERN_SUCCESS) { + kcdata_memcpy(kcd, addr, &serial_no, sizeof(serial_no)); + } + +out: +#if DEVELOPMENT || DEBUG + psignal_try_thread_with_reason(p, thread, SIGABRT, reason); + return 0; +#else + return exit_with_reason(p, W_EXITCODE(0, SIGKILL), (int *)NULL, + FALSE, FALSE, 0, reason); +#endif +} + + +static int +kevent_servicer_detach_preflight(thread_t thread, unsigned int flags, struct kqueue *kq) +{ + int error = 0; + struct kqworkloop *kqwl; + struct uthread *ut; + struct kqrequest *kqr; + + if (!(flags & KEVENT_FLAG_WORKLOOP) || !(kq->kq_state & KQ_WORKLOOP)) + return EINVAL; + + /* only kq created with KEVENT_FLAG_WORKLOOP_NO_WQ_THREAD from userspace can have attached threads */ + if (!(kq->kq_state & KQ_NO_WQ_THREAD)) + return EINVAL; + + /* allow detach only on not wq threads */ + if (is_workqueue_thread(thread)) + return EINVAL; + + /* check that the current thread is bound to the requested wq */ + ut = get_bsdthread_info(thread); + if (ut->uu_kqueue_bound != kq) + return EINVAL; + + kqwl = (struct kqworkloop *)kq; + kqwl_req_lock(kqwl); + kqr = &kqwl->kqwl_request; + + /* check that the wq is bound to the thread */ + if ((kqr->kqr_state & KQR_BOUND) == 0 || (kqr->kqr_thread != thread)) + error = EINVAL; + + kqwl_req_unlock(kqwl); + + return error; +} + +static void +kevent_servicer_detach_thread(struct proc *p, kqueue_id_t id, thread_t thread, + unsigned int flags, struct kqueue *kq) { - /* each kq should only be used for events of one type */ + struct kqworkloop *kqwl; + struct uthread *ut; + + assert((flags & KEVENT_FLAG_WORKLOOP) && (kq->kq_state & KQ_WORKLOOP)); + + /* allow detach only on not wqthreads threads */ + assert(!is_workqueue_thread(thread)); + + /* only kq created with KEVENT_FLAG_WORKLOOP_NO_WQ_THREAD from userspace can have attached threads */ + assert(kq->kq_state & KQ_NO_WQ_THREAD); + + /* check that the current thread is bound to the requested kq */ + ut = get_bsdthread_info(thread); + assert(ut->uu_kqueue_bound == kq); + + kqwl = (struct kqworkloop *)kq; + kqlock(kq); - if (kq->kq_state & (KQ_KEV32 | KQ_KEV64 | KQ_KEV_QOS)) { - if (flags & KEVENT_FLAG_LEGACY32) { - if ((kq->kq_state & KQ_KEV32) == 0) { - kqunlock(kq); - return EINVAL; - } - } else if (kq->kq_state & KQ_KEV32) { - kqunlock(kq); - return EINVAL; - } - } else if (flags & KEVENT_FLAG_LEGACY32) { - kq->kq_state |= KQ_KEV32; - } else { - /* JMM - set KQ_KEVQOS when we are ready for exclusive */ - kq->kq_state |= KQ_KEV64; - } + + /* unbind the thread. + * unbind itself checks if still processing and ends it. + */ + kqworkloop_unbind_thread(kqwl, thread, flags); + kqunlock(kq); - return 0; + + kevent_put_kq(p, id, NULL, kq); + + return; } static int -kevent_get_kq(struct proc *p, int fd, unsigned int flags, struct fileproc **fpp, struct kqueue **kqp) +kevent_servicer_attach_thread(thread_t thread, unsigned int flags, struct kqueue *kq) { - struct fileproc *fp = NULL; - struct kqueue *kq; - int error; + int error = 0; + struct kqworkloop *kqwl; + struct uthread *ut; + struct kqrequest *kqr; - if (flags & KEVENT_FLAG_WORKQ) { - /* - * use the private kq associated with the proc workq. - * Just being a thread within the process (and not - * being the exit/exec thread) is enough to hold a - * reference on this special kq. - */ - kq = p->p_wqkqueue; - if (kq == NULL) { - struct kqueue *alloc_kq = kqueue_alloc(p, KEVENT_FLAG_WORKQ); - if (alloc_kq == NULL) - return ENOMEM; + if (!(flags & KEVENT_FLAG_WORKLOOP) || !(kq->kq_state & KQ_WORKLOOP)) + return EINVAL; - proc_fdlock(p); - if (p->p_wqkqueue == NULL) { - kq = p->p_wqkqueue = alloc_kq; - proc_fdunlock(p); - } else { - proc_fdunlock(p); - kq = p->p_wqkqueue; - kqueue_dealloc(alloc_kq); - } - } - } else { - /* get a usecount for the kq itself */ - if ((error = fp_getfkq(p, fd, &fp, &kq)) != 0) - return (error); + /* only kq created with KEVENT_FLAG_WORKLOOP_NO_WQ_THREAD from userspace can have attached threads*/ + if (!(kq->kq_state & KQ_NO_WQ_THREAD)) + return EINVAL; + + /* allow attach only on not wqthreads */ + if (is_workqueue_thread(thread)) + return EINVAL; + + /* check that the thread is not already bound */ + ut = get_bsdthread_info(thread); + if (ut->uu_kqueue_bound != NULL) + return EINVAL; + + assert(ut->uu_kqueue_flags == 0); + + kqlock(kq); + kqwl = (struct kqworkloop *)kq; + kqwl_req_lock(kqwl); + kqr = &kqwl->kqwl_request; + + /* check that the kqueue is not already bound */ + if (kqr->kqr_state & (KQR_BOUND | KQR_THREQUESTED | KQR_DRAIN)) { + error = EINVAL; + goto out; } - if ((error = kevent_set_kq_mode(kq, flags)) != 0) { - /* drop the usecount */ - if (fp != NULL) - fp_drop(p, fd, fp, 0); - return error; - } - *fpp = fp; - *kqp = kq; - return 0; + assert(kqr->kqr_thread == NULL); + assert((kqr->kqr_state & KQR_PROCESSING) == 0); + + kqr->kqr_state |= KQR_THREQUESTED; + kqr->kqr_qos_index = THREAD_QOS_UNSPECIFIED; + kqr->kqr_override_index = THREAD_QOS_UNSPECIFIED; + kqr->kqr_dsync_owner_qos = THREAD_QOS_UNSPECIFIED; + kqr->kqr_owner_override_is_sync = 0; + + kqworkloop_bind_thread_impl(kqwl, thread, KEVENT_FLAG_WORKLOOP); + + /* get a ref on the wlkq on behalf of the attached thread */ + kqueue_retain(kq); + +out: + kqwl_req_unlock(kqwl); + kqunlock(kq); + + return error; } +static inline +boolean_t kevent_args_requesting_events(unsigned int flags, int nevents) +{ + return (!(flags & KEVENT_FLAG_ERROR_EVENTS) && nevents > 0); +} static int -kevent_internal(struct proc *p, - int fd, +kevent_internal(struct proc *p, + kqueue_id_t id, kqueue_id_t *id_out, user_addr_t changelist, int nchanges, user_addr_t ueventlist, int nevents, user_addr_t data_out, uint64_t data_available, @@ -2291,17 +4253,40 @@ kevent_internal(struct proc *p, uthread_t ut; struct kqueue *kq; struct fileproc *fp = NULL; + int fd = 0; struct kevent_internal_s kev; int error, noutputs; struct timeval atv; user_size_t data_size; user_size_t data_resid; + thread_t thread = current_thread(); - /* Don't allow user-space threads to process output events from the workq kq */ - if ((flags & (KEVENT_FLAG_WORKQ | KEVENT_FLAG_KERNEL)) == KEVENT_FLAG_WORKQ && - !(flags & KEVENT_FLAG_ERROR_EVENTS) && nevents > 0) + /* Don't allow user-space threads to process output events from the workq kqs */ + if (((flags & (KEVENT_FLAG_WORKQ | KEVENT_FLAG_KERNEL)) == KEVENT_FLAG_WORKQ) && + kevent_args_requesting_events(flags, nevents)) return EINVAL; + /* restrict dynamic kqueue allocation to workloops (for now) */ + if ((flags & (KEVENT_FLAG_DYNAMIC_KQUEUE | KEVENT_FLAG_WORKLOOP)) == KEVENT_FLAG_DYNAMIC_KQUEUE) + return EINVAL; + + if (flags & (KEVENT_FLAG_WORKLOOP_SERVICER_ATTACH | KEVENT_FLAG_WORKLOOP_SERVICER_DETACH | + KEVENT_FLAG_DYNAMIC_KQ_MUST_EXIST | KEVENT_FLAG_DYNAMIC_KQ_MUST_NOT_EXIST | KEVENT_FLAG_WORKLOOP_NO_WQ_THREAD)) { + + /* allowed only on workloops when calling kevent_id from user-space */ + if (!(flags & KEVENT_FLAG_WORKLOOP) || (flags & KEVENT_FLAG_KERNEL) || !(flags & KEVENT_FLAG_DYNAMIC_KQUEUE)) + return EINVAL; + + /* cannot attach and detach simultaneously*/ + if ((flags & KEVENT_FLAG_WORKLOOP_SERVICER_ATTACH) && (flags & KEVENT_FLAG_WORKLOOP_SERVICER_DETACH)) + return EINVAL; + + /* cannot ask for events and detach */ + if ((flags & KEVENT_FLAG_WORKLOOP_SERVICER_DETACH) && kevent_args_requesting_events(flags, nevents)) + return EINVAL; + + } + /* prepare to deal with stack-wise allocation of out events */ if (flags & KEVENT_FLAG_STACK_EVENTS) { int scale = ((flags & KEVENT_FLAG_LEGACY32) ? @@ -2323,10 +4308,42 @@ kevent_internal(struct proc *p, return error; /* get the kq we are going to be working on */ - error = kevent_get_kq(p, fd, flags, &fp, &kq); + error = kevent_get_kq(p, id, flags, &fp, &fd, &kq); if (error) return error; + /* only bound threads can receive events on workloops */ + if ((flags & KEVENT_FLAG_WORKLOOP) && kevent_args_requesting_events(flags, nevents)) { + ut = (uthread_t)get_bsdthread_info(thread); + if (ut->uu_kqueue_bound != kq) { + error = EXDEV; + goto out; + } + + } + + /* attach the current thread if necessary */ + if (flags & KEVENT_FLAG_WORKLOOP_SERVICER_ATTACH) { + error = kevent_servicer_attach_thread(thread, flags, kq); + if (error) + goto out; + } + else { + /* before processing events and committing to the system call, return an error if the thread cannot be detached when requested */ + if (flags & KEVENT_FLAG_WORKLOOP_SERVICER_DETACH) { + error = kevent_servicer_detach_preflight(thread, flags, kq); + if (error) + goto out; + } + } + + if (id_out && kq && (flags & KEVENT_FLAG_WORKLOOP)) { + assert(kq->kq_state & KQ_WORKLOOP); + struct kqworkloop *kqwl; + kqwl = (struct kqworkloop *)kq; + *id_out = kqwl->kqwl_dynamicid; + } + /* register all the change requests the user provided... */ noutputs = 0; while (nchanges > 0 && error == 0) { @@ -2362,9 +4379,8 @@ kevent_internal(struct proc *p, /* process pending events */ if (nevents > 0 && noutputs == 0 && error == 0) { - /* store the continuation/completion data in the uthread */ - ut = (uthread_t)get_bsdthread_info(current_thread()); + ut = (uthread_t)get_bsdthread_info(thread); cont_args = &ut->uu_kevent.ss_kevent; cont_args->fp = fp; cont_args->fd = fd; @@ -2373,7 +4389,7 @@ kevent_internal(struct proc *p, cont_args->eventcount = nevents; cont_args->eventout = noutputs; cont_args->data_available = data_available; - cont_args->process_data.fp_fd = fd; + cont_args->process_data.fp_fd = (int)id; cont_args->process_data.fp_flags = flags; cont_args->process_data.fp_data_out = data_out; cont_args->process_data.fp_data_size = data_size; @@ -2395,6 +4411,15 @@ kevent_internal(struct proc *p, } } + /* detach the current thread if necessary */ + if (flags & KEVENT_FLAG_WORKLOOP_SERVICER_DETACH) { + assert(fp == NULL); + kevent_servicer_detach_thread(p, id, thread, flags, kq); + } + +out: + kevent_put_kq(p, id, fp, kq); + /* don't restart after signals... */ if (error == ERESTART) error = EINTR; @@ -2402,8 +4427,6 @@ kevent_internal(struct proc *p, error = 0; if (error == 0) *retval = noutputs; - if (fp != NULL) - fp_drop(p, fd, fp, 0); return (error); } @@ -2482,10 +4505,12 @@ kevent_register(struct kqueue *kq, struct kevent_internal_s *kev, __unused struct proc *ctxp) { struct proc *p = kq->kq_p; - struct filterops *fops; + const struct filterops *fops; struct knote *kn = NULL; int result = 0; int error = 0; + unsigned short kev_flags = kev->flags; + int knoteuse_flags = KNUSE_NONE; if (kev->filter < 0) { if (kev->filter + EVFILT_SYSCOUNT < 0) { @@ -2511,36 +4536,46 @@ kevent_register(struct kqueue *kq, struct kevent_internal_s *kev, if (kev->flags & EV_DISABLE) kev->flags &= ~EV_ENABLE; -restart: + if (kq->kq_state & KQ_WORKLOOP) { + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWL_REGISTER), + ((struct kqworkloop *)kq)->kqwl_dynamicid, + kev->udata, kev->flags, kev->filter); + } else if (kq->kq_state & KQ_WORKQ) { + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWQ_REGISTER), + 0, kev->udata, kev->flags, kev->filter); + } else { + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQ_REGISTER), + VM_KERNEL_UNSLIDE_OR_PERM(kq), + kev->udata, kev->flags, kev->filter); + } - proc_fdlock(p); +restart: /* find the matching knote from the fd tables/hashes */ - kn = knote_fdfind(kq, kev, p); + kn = kq_find_knote_and_kq_lock(kq, kev, fops->f_isfd, p); if (kn == NULL) { if (kev->flags & EV_ADD) { - struct fileproc *fp = NULL; + struct fileproc *knote_fp = NULL; /* grab a file reference for the new knote */ if (fops->f_isfd) { - if ((error = fp_lookup(p, kev->ident, &fp, 1)) != 0) { - proc_fdunlock(p); + if ((error = fp_lookup(p, kev->ident, &knote_fp, 0)) != 0) { goto out; } } kn = knote_alloc(); if (kn == NULL) { - proc_fdunlock(p); error = ENOMEM; - if (fp != NULL) - fp_drop(p, kev->ident, fp, 0); + if (knote_fp != NULL) + fp_drop(p, kev->ident, knote_fp, 0); goto out; } - kn->kn_fp = fp; - knote_set_kq(kn,kq); + kn->kn_fp = knote_fp; + knote_set_kq(kn, kq); + kqueue_retain(kq); /* retain a kq ref */ kn->kn_filtid = ~kev->filter; kn->kn_inuse = 1; /* for f_attach() */ kn->kn_status = KN_ATTACHING | KN_ATTACHED; @@ -2570,42 +4605,46 @@ restart: kn->kn_data = 0; /* invoke pthread kext to convert kevent qos to thread qos */ - if (kq->kq_state & KQ_WORKQ) { - kn->kn_qos = canonicalize_kevent_qos(kn->kn_qos); - knote_set_qos_index(kn, qos_index_from_qos(kn->kn_qos, FALSE)); - knote_set_qos_override_index(kn, QOS_INDEX_KQFILE); - assert(knote_get_qos_index(kn) < KQWQ_NQOS); - } else { - knote_set_qos_index(kn, QOS_INDEX_KQFILE); - knote_set_qos_override_index(kn, QOS_INDEX_KQFILE); - } + knote_canonicalize_kevent_qos(kn); + knote_set_qos_index(kn, qos_index_from_qos(kn, kn->kn_qos, FALSE)); /* before anyone can find it */ - if (kev->flags & EV_DISABLE) - knote_disable(kn); + if (kev->flags & EV_DISABLE) { + /* + * do this before anyone can find it, + * this can't call knote_disable() because it expects having + * the kqlock held + */ + kn->kn_status |= KN_DISABLED; + } /* Add the knote for lookup thru the fd table */ - error = knote_fdadd(kn, p); - proc_fdunlock(p); - + error = kq_add_knote(kq, kn, kev, p, &knoteuse_flags); if (error) { + (void)kqueue_release(kq, KQUEUE_CANT_BE_LAST_REF); knote_free(kn); - if (fp != NULL) - fp_drop(p, kev->ident, fp, 0); + if (knote_fp != NULL) + fp_drop(p, kev->ident, knote_fp, 0); + + if (error == ERESTART) { + error = 0; + goto restart; + } goto out; } /* fp reference count now applies to knote */ + /* rwlock boost is now held */ /* call filter attach routine */ - result = fops->f_attach(kn); + result = fops->f_attach(kn, kev); /* * Trade knote use count for kq lock. * Cannot be dropped because we held * KN_ATTACHING throughout. */ - knoteuse2kqlock(kq, kn, 1); + knoteuse2kqlock(kq, kn, KNUSE_STEAL_DROP | knoteuse_flags); if (kn->kn_flags & EV_ERROR) { /* @@ -2636,6 +4675,9 @@ restart: goto out; } + /* Mark the thread request overcommit - if appropos */ + knote_set_qos_overcommit(kn); + /* * If the attach routine indicated that an * event is already fired, activate the knote. @@ -2643,28 +4685,37 @@ restart: if (result) knote_activate(kn); + if (knote_fops(kn)->f_post_attach) { + error = knote_fops(kn)->f_post_attach(kn, kev); + if (error) { + kqunlock(kq); + goto out; + } + } + } else { - proc_fdunlock(p); - error = ENOENT; + if ((kev_flags & (EV_ADD | EV_DELETE)) == (EV_ADD | EV_DELETE) && + (kq->kq_state & KQ_WORKLOOP)) { + /* + * For workloops, understand EV_ADD|EV_DELETE as a "soft" delete + * that doesn't care about ENOENT, so just pretend the deletion + * happened. + */ + } else { + error = ENOENT; + } goto out; } } else { - /* existing knote - get kqueue lock */ - kqlock(kq); - proc_fdunlock(p); + /* existing knote: kqueue lock already taken by kq_find_knote_and_kq_lock */ if ((kn->kn_status & (KN_DROPPING | KN_ATTACHING)) != 0) { /* * The knote is not in a stable state, wait for that * transition to complete and then redrive the lookup. */ - kn->kn_status |= KN_USEWAIT; - waitq_assert_wait64((struct waitq *)&kq->kq_wqs, - CAST_EVENT64_T(&kn->kn_status), - THREAD_UNINT, TIMEOUT_WAIT_FOREVER); - kqunlock(kq); - thread_block(THREAD_CONTINUE_NULL); + knoteusewait(kq, kn); goto restart; } @@ -2681,7 +4732,19 @@ restart: kn->kn_status |= KN_DEFERDELETE; kqunlock(kq); error = EINPROGRESS; + } else if (knote_fops(kn)->f_drop_and_unlock) { + /* + * The filter has requested to handle EV_DELETE events + * + * ERESTART means the kevent has to be re-evaluated + */ + error = knote_fops(kn)->f_drop_and_unlock(kn, kev); + if (error == ERESTART) { + error = 0; + goto restart; + } } else if (kqlock2knotedrop(kq, kn)) { + /* standard/default EV_DELETE path */ knote_drop(kn, p); } else { /* @@ -2723,8 +4786,10 @@ restart: * Convert the kqlock to a use reference on the * knote so we can call the filter touch routine. */ - if (kqlock2knoteuse(kq, kn)) { - + if (knoteuse_needs_boost(kn, kev)) { + knoteuse_flags |= KNUSE_BOOST; + } + if (kqlock2knoteuse(kq, kn, knoteuse_flags)) { /* * Call touch routine to notify filter of changes * in filter values (and to re-determine if any @@ -2733,7 +4798,14 @@ restart: result = knote_fops(kn)->f_touch(kn, kev); /* Get the kq lock back (don't defer droppers). */ - if (!knoteuse2kqlock(kq, kn, 0)) { + if (!knoteuse2kqlock(kq, kn, knoteuse_flags)) { + kqunlock(kq); + goto out; + } + + /* Handle errors during touch routine */ + if (kev->flags & EV_ERROR) { + error = kev->data; kqunlock(kq); goto out; } @@ -2752,7 +4824,7 @@ restart: /* still have kqlock held and knote is valid */ kqunlock(kq); - out: +out: /* output local errors through the kevent */ if (error) { kev->flags |= EV_ERROR; @@ -2803,6 +4875,21 @@ knote_process(struct knote *kn, assert(kn->kn_status & (KN_ACTIVE|KN_STAYACTIVE)); assert(!(kn->kn_status & (KN_DISABLED|KN_SUPPRESSED|KN_DROPPING))); + if (kq->kq_state & KQ_WORKLOOP) { + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWL_PROCESS), + ((struct kqworkloop *)kq)->kqwl_dynamicid, + kn->kn_udata, kn->kn_status | (kn->kn_id << 32), + kn->kn_filtid); + } else if (kq->kq_state & KQ_WORKQ) { + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWQ_PROCESS), + 0, kn->kn_udata, kn->kn_status | (kn->kn_id << 32), + kn->kn_filtid); + } else { + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQ_PROCESS), + VM_KERNEL_UNSLIDE_OR_PERM(kq), kn->kn_udata, + kn->kn_status | (kn->kn_id << 32), kn->kn_filtid); + } + /* * For deferred-drop or vanished events, we just create a fake * event to acknowledge end-of-life. Otherwise, we call the @@ -2822,26 +4909,30 @@ knote_process(struct knote *kn, knote_suppress(kn); } else { - + int flags = KNUSE_NONE; /* deactivate - so new activations indicate a wakeup */ knote_deactivate(kn); /* suppress knotes to avoid returning the same event multiple times in a single call. */ knote_suppress(kn); + if (knoteuse_needs_boost(kn, NULL)) { + flags |= KNUSE_BOOST; + } /* convert lock to a knote use reference */ - if (!kqlock2knoteuse(kq, kn)) + if (!kqlock2knoteuse(kq, kn, flags)) panic("dropping knote found on queue\n"); /* call out to the filter to process with just a ref */ result = knote_fops(kn)->f_process(kn, process_data, &kev); + if (result) flags |= KNUSE_STEAL_DROP; /* * convert our reference back to a lock. accept drop * responsibility from others if we've committed to * delivering event data. */ - if (!knoteuse2kqlock(kq, kn, result)) { + if (!knoteuse2kqlock(kq, kn, flags)) { /* knote dropped */ kn = NULL; } @@ -2932,54 +5023,139 @@ kqworkq_begin_processing(struct kqworkq *kqwq, kq_index_t qos_index, int flags) struct kqrequest *kqr; thread_t self = current_thread(); __assert_only struct uthread *ut = get_bsdthread_info(self); - thread_t thread; assert(kqwq->kqwq_state & KQ_WORKQ); assert(qos_index < KQWQ_NQOS); + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWQ_PROCESS_BEGIN) | DBG_FUNC_START, + flags, qos_index); + kqwq_req_lock(kqwq); - kqr = kqworkq_get_request(kqwq, qos_index); - thread = kqr->kqr_thread; + kqr = kqworkq_get_request(kqwq, qos_index); - /* manager skips buckets that haven't ask for its help */ + /* manager skips buckets that haven't asked for its help */ if (flags & KEVENT_FLAG_WORKQ_MANAGER) { /* If nothing for manager to do, just return */ if ((kqr->kqr_state & KQWQ_THMANAGER) == 0) { - assert(kqr->kqr_thread != self); + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWQ_PROCESS_BEGIN) | DBG_FUNC_END, + 0, kqr->kqr_state); kqwq_req_unlock(kqwq); return -1; } - /* bind manager thread from this time on */ - kqworkq_bind_thread(kqwq, qos_index, self, flags); + kqworkq_bind_thread_impl(kqwq, qos_index, self, flags); } else { - /* must have been bound by now */ - assert(thread == self); - assert(ut->uu_kqueue_bound == qos_index); + /* We should already be bound to this kqueue */ + assert(kqr->kqr_state & KQR_BOUND); + assert(kqr->kqr_thread == self); + assert(ut->uu_kqueue_bound == (struct kqueue *)kqwq); + assert(ut->uu_kqueue_qos_index == qos_index); assert((ut->uu_kqueue_flags & flags) == ut->uu_kqueue_flags); } - /* nobody else should still be processing */ - assert(kqr->kqr_state & KQWQ_THREQUESTED); - assert((kqr->kqr_state & KQWQ_PROCESSING) == 0); - - /* anything left to process? */ - if (kqueue_queue_empty(&kqwq->kqwq_kqueue, qos_index)) { - kqwq_req_unlock(kqwq); - return -1; - } + /* + * we should have been requested to be here + * and nobody else should still be processing + */ + assert(kqr->kqr_state & KQR_WAKEUP); + assert(kqr->kqr_state & KQR_THREQUESTED); + assert((kqr->kqr_state & KQR_PROCESSING) == 0); + + /* reset wakeup trigger to catch new events after we start processing */ + kqr->kqr_state &= ~KQR_WAKEUP; /* convert to processing mode */ - /* reset workq triggers and thread requests - maybe processing */ - kqr->kqr_state &= ~(KQWQ_HOOKCALLED | KQWQ_WAKEUP); - kqr->kqr_state |= KQWQ_PROCESSING; + kqr->kqr_state |= KQR_PROCESSING; + + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWQ_PROCESS_BEGIN) | DBG_FUNC_END, + kqr_thread_id(kqr), kqr->kqr_state); + kqwq_req_unlock(kqwq); return 0; } +static inline bool +kqworkloop_is_processing_on_current_thread(struct kqworkloop *kqwl) +{ + struct kqueue *kq = &kqwl->kqwl_kqueue; + + kqlock_held(kq); + + if (kq->kq_state & KQ_PROCESSING) { + /* + * KQ_PROCESSING is unset with the kqlock held, and the kqr thread is + * never modified while KQ_PROCESSING is set, meaning that peeking at + * its value is safe from this context. + */ + return kqwl->kqwl_request.kqr_thread == current_thread(); + } + return false; +} + +static void +kqworkloop_acknowledge_events(struct kqworkloop *kqwl, boolean_t clear_ipc_override) +{ + struct kqrequest *kqr = &kqwl->kqwl_request; + struct knote *kn, *tmp; + + kqlock_held(&kqwl->kqwl_kqueue); + + TAILQ_FOREACH_SAFE(kn, &kqr->kqr_suppressed, kn_tqe, tmp) { + /* + * If a knote that can adjust QoS is disabled because of the automatic + * behavior of EV_DISPATCH, the knotes should stay suppressed so that + * further overrides keep pushing. + */ + if (knote_fops(kn)->f_adjusts_qos && (kn->kn_status & KN_DISABLED) && + (kn->kn_status & (KN_STAYACTIVE | KN_DROPPING)) == 0 && + (kn->kn_flags & (EV_DISPATCH | EV_DISABLE)) == EV_DISPATCH) { + /* + * When called from unbind, clear the sync ipc override on the knote + * for events which are delivered. + */ + if (clear_ipc_override) { + knote_adjust_sync_qos(kn, THREAD_QOS_UNSPECIFIED, FALSE); + } + continue; + } + knote_unsuppress(kn); + } +} + +static int +kqworkloop_begin_processing(struct kqworkloop *kqwl, + __assert_only unsigned int flags) +{ + struct kqrequest *kqr = &kqwl->kqwl_request; + struct kqueue *kq = &kqwl->kqwl_kqueue; + + kqlock_held(kq); + + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWL_PROCESS_BEGIN) | DBG_FUNC_START, + kqwl->kqwl_dynamicid, flags, 0); + + kqwl_req_lock(kqwl); + + /* nobody else should still be processing */ + assert((kqr->kqr_state & KQR_PROCESSING) == 0); + assert((kq->kq_state & KQ_PROCESSING) == 0); + + kqr->kqr_state |= KQR_PROCESSING | KQR_R2K_NOTIF_ARMED; + kq->kq_state |= KQ_PROCESSING; + + kqwl_req_unlock(kqwl); + + kqworkloop_acknowledge_events(kqwl, FALSE); + + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWL_PROCESS_BEGIN) | DBG_FUNC_END, + kqwl->kqwl_dynamicid, flags, 0); + + return 0; +} + /* * Return 0 to indicate that processing should proceed, * -1 if there is nothing to process. @@ -2993,15 +5169,26 @@ kqueue_begin_processing(struct kqueue *kq, kq_index_t qos_index, unsigned int fl { struct kqtailq *suppressq; - if (kq->kq_state & KQ_WORKQ) + kqlock_held(kq); + + if (kq->kq_state & KQ_WORKQ) { return kqworkq_begin_processing((struct kqworkq *)kq, qos_index, flags); + } else if (kq->kq_state & KQ_WORKLOOP) { + return kqworkloop_begin_processing((struct kqworkloop*)kq, flags); + } + + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQ_PROCESS_BEGIN) | DBG_FUNC_START, + VM_KERNEL_UNSLIDE_OR_PERM(kq), flags); assert(qos_index == QOS_INDEX_KQFILE); /* wait to become the exclusive processing thread */ for (;;) { - if (kq->kq_state & KQ_DRAIN) + if (kq->kq_state & KQ_DRAIN) { + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQ_PROCESS_BEGIN) | DBG_FUNC_END, + VM_KERNEL_UNSLIDE_OR_PERM(kq), 2); return -1; + } if ((kq->kq_state & KQ_PROCESSING) == 0) break; @@ -3023,14 +5210,20 @@ kqueue_begin_processing(struct kqueue *kq, kq_index_t qos_index, unsigned int fl /* clear pre-posts and KQ_WAKEUP now, in case we bail early */ waitq_set_clear_preposts(&kq->kq_wqs); kq->kq_state &= ~KQ_WAKEUP; - + /* anything left to process? */ - if (kqueue_queue_empty(kq, qos_index)) + if (kqueue_queue_empty(kq, qos_index)) { + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQ_PROCESS_BEGIN) | DBG_FUNC_END, + VM_KERNEL_UNSLIDE_OR_PERM(kq), 1); return -1; + } /* convert to processing mode */ kq->kq_state |= KQ_PROCESSING; + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQ_PROCESS_BEGIN) | DBG_FUNC_END, + VM_KERNEL_UNSLIDE_OR_PERM(kq)); + return 0; } @@ -3054,52 +5247,63 @@ kqworkq_end_processing(struct kqworkq *kqwq, kq_index_t qos_index, int flags) struct kqtailq *suppressq = kqueue_get_suppressed_queue(kq, qos_index); thread_t self = current_thread(); - __assert_only struct uthread *ut = get_bsdthread_info(self); + struct uthread *ut = get_bsdthread_info(self); struct knote *kn; struct kqrequest *kqr; - int queued_events; - uint16_t pended; thread_t thread; assert(kqwq->kqwq_state & KQ_WORKQ); assert(qos_index < KQWQ_NQOS); - /* leave early if we are not even processing */ - kqwq_req_lock(kqwq); + /* Are we really bound to this kqueue? */ + if (ut->uu_kqueue_bound != kq) { + assert(ut->uu_kqueue_bound == kq); + return; + } + kqr = kqworkq_get_request(kqwq, qos_index); - thread = kqr->kqr_thread; + kqwq_req_lock(kqwq); + + /* Do we claim to be manager? */ if (flags & KEVENT_FLAG_WORKQ_MANAGER) { - assert(ut->uu_kqueue_bound == KQWQ_QOS_MANAGER); - assert(ut->uu_kqueue_flags & KEVENT_FLAG_WORKQ_MANAGER); - /* if this bucket didn't need manager help, bail */ - if ((kqr->kqr_state & KQWQ_THMANAGER) == 0) { - assert(thread != self); + /* bail if not bound that way */ + if (ut->uu_kqueue_qos_index != KQWQ_QOS_MANAGER || + (ut->uu_kqueue_flags & KEVENT_FLAG_WORKQ_MANAGER) == 0) { + assert(ut->uu_kqueue_qos_index == KQWQ_QOS_MANAGER); + assert(ut->uu_kqueue_flags & KEVENT_FLAG_WORKQ_MANAGER); kqwq_req_unlock(kqwq); return; } - assert(kqr->kqr_state & KQWQ_THREQUESTED); - - /* unbound bucket - see if still needs servicing */ - if (thread == THREAD_NULL) { - assert((kqr->kqr_state & KQWQ_PROCESSING) == 0); - assert(TAILQ_EMPTY(suppressq)); - } else { - assert(thread == self); + /* bail if this request wasn't already getting manager help */ + if ((kqr->kqr_state & KQWQ_THMANAGER) == 0 || + (kqr->kqr_state & KQR_PROCESSING) == 0) { + kqwq_req_unlock(kqwq); + return; } - } else { - assert(thread == self); - assert(ut->uu_kqueue_bound == qos_index); - assert((ut->uu_kqueue_flags & KEVENT_FLAG_WORKQ_MANAGER) == 0); + if (ut->uu_kqueue_qos_index != qos_index || + (ut->uu_kqueue_flags & KEVENT_FLAG_WORKQ_MANAGER)) { + assert(ut->uu_kqueue_qos_index == qos_index); + assert((ut->uu_kqueue_flags & KEVENT_FLAG_WORKQ_MANAGER) == 0); + kqwq_req_unlock(kqwq); + return; + } } - kqwq_req_unlock(kqwq); + assert(kqr->kqr_state & KQR_BOUND); + thread = kqr->kqr_thread; + assert(thread == self); - /* Any events queued before we put suppressed ones back? */ - queued_events = !kqueue_queue_empty(kq, qos_index); + assert(kqr->kqr_state & KQR_PROCESSING); + + /* If we didn't drain the whole queue, re-mark a wakeup being needed */ + if (!kqueue_queue_empty(kq, qos_index)) + kqr->kqr_state |= KQR_WAKEUP; + + kqwq_req_unlock(kqwq); /* * Return suppressed knotes to their original state. @@ -3115,51 +5319,95 @@ kqworkq_end_processing(struct kqworkq *kqwq, kq_index_t qos_index, int flags) kqwq_req_lock(kqwq); - /* Determine if wakeup-type events were pended during servicing */ - pended = (kqr->kqr_state & (KQWQ_HOOKCALLED | KQWQ_WAKEUP)); - - /* unbind thread thread */ - kqworkq_unbind_thread(kqwq, qos_index, self, flags); + /* Indicate that we are done processing this request */ + kqr->kqr_state &= ~KQR_PROCESSING; - /* Indicate that we are done processing */ - kqr->kqr_state &= ~(KQWQ_PROCESSING | \ - KQWQ_THREQUESTED | KQWQ_THMANAGER); + /* + * Drop our association with this one request and its + * override on us. + */ + kqworkq_unbind_thread(kqwq, qos_index, thread, flags); /* - * request a new thread if events have happened - * (not just putting stay-active events back). + * request a new thread if we didn't process the whole + * queue or real events have happened (not just putting + * stay-active events back). */ - if ((queued_events || pended) && - !kqueue_queue_empty(kq, qos_index)) { - kqworkq_request_thread(kqwq, qos_index); + if (kqr->kqr_state & KQR_WAKEUP) { + if (kqueue_queue_empty(kq, qos_index)) { + kqr->kqr_state &= ~KQR_WAKEUP; + } else { + kqworkq_request_thread(kqwq, qos_index); + } } - kqwq_req_unlock(kqwq); } +static void +kqworkloop_end_processing(struct kqworkloop *kqwl, int nevents, + unsigned int flags) +{ + struct kqrequest *kqr = &kqwl->kqwl_request; + struct kqueue *kq = &kqwl->kqwl_kqueue; + + kqlock_held(kq); + + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWL_PROCESS_END) | DBG_FUNC_START, + kqwl->kqwl_dynamicid, flags, 0); + + if ((kq->kq_state & KQ_NO_WQ_THREAD) && nevents == 0 && + (flags & KEVENT_FLAG_IMMEDIATE) == 0) { + /* + * <rdar://problem/31634014> We may soon block, but have returned no + * kevents that need to be kept supressed for overriding purposes. + * + * It is hence safe to acknowledge events and unsuppress everything, so + * that if we block we can observe all events firing. + */ + kqworkloop_acknowledge_events(kqwl, TRUE); + } + + kqwl_req_lock(kqwl); + + assert(kqr->kqr_state & KQR_PROCESSING); + assert(kq->kq_state & KQ_PROCESSING); + + kq->kq_state &= ~KQ_PROCESSING; + kqr->kqr_state &= ~KQR_PROCESSING; + kqworkloop_update_threads_qos(kqwl, KQWL_UTQ_RECOMPUTE_WAKEUP_QOS, 0); + + kqwl_req_unlock(kqwl); + + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWL_PROCESS_END) | DBG_FUNC_END, + kqwl->kqwl_dynamicid, flags, 0); +} + /* * Called with kqueue lock held. */ static void -kqueue_end_processing(struct kqueue *kq, kq_index_t qos_index, unsigned int flags) +kqueue_end_processing(struct kqueue *kq, kq_index_t qos_index, + int nevents, unsigned int flags) { struct knote *kn; struct kqtailq *suppressq; int procwait; - if (kq->kq_state & KQ_WORKQ) { - kqworkq_end_processing((struct kqworkq *)kq, qos_index, flags); - return; + kqlock_held(kq); + + assert((kq->kq_state & KQ_WORKQ) == 0); + + if (kq->kq_state & KQ_WORKLOOP) { + return kqworkloop_end_processing((struct kqworkloop *)kq, nevents, flags); } + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQ_PROCESS_END), + VM_KERNEL_UNSLIDE_OR_PERM(kq), flags); + assert(qos_index == QOS_INDEX_KQFILE); /* * Return suppressed knotes to their original state. - * For workq kqueues, suppressed ones that are still - * truly active (not just forced into the queue) will - * set flags we check below to see if anything got - * woken up. */ suppressq = kqueue_get_suppressed_queue(kq, qos_index); while ((kn = TAILQ_FIRST(suppressq)) != NULL) { @@ -3176,8 +5424,226 @@ kqueue_end_processing(struct kqueue *kq, kq_index_t qos_index, unsigned int flag CAST_EVENT64_T(suppressq), THREAD_AWAKENED, WAITQ_ALL_PRIORITIES); - } + } +} + +/* + * kqwq_internal_bind - bind thread to processing workq kqueue + * + * Determines if the provided thread will be responsible for + * servicing the particular QoS class index specified in the + * parameters. Once the binding is done, any overrides that may + * be associated with the cooresponding events can be applied. + * + * This should be called as soon as the thread identity is known, + * preferably while still at high priority during creation. + * + * - caller holds a reference on the process (and workq kq) + * - the thread MUST call kevent_qos_internal after being bound + * or the bucket of events may never be delivered. + * - Nothing locked + * (unless this is a synchronous bind, then the request is locked) + */ +static int +kqworkq_internal_bind( + struct proc *p, + kq_index_t qos_index, + thread_t thread, + unsigned int flags) +{ + struct kqueue *kq; + struct kqworkq *kqwq; + struct kqrequest *kqr; + struct uthread *ut = get_bsdthread_info(thread); + + /* If no process workq, can't be our thread. */ + kq = p->p_fd->fd_wqkqueue; + + if (kq == NULL) + return 0; + + assert(kq->kq_state & KQ_WORKQ); + kqwq = (struct kqworkq *)kq; + + /* + * No need to bind the manager thread to any specific + * bucket, but still claim the thread. + */ + if (qos_index == KQWQ_QOS_MANAGER) { + assert(ut->uu_kqueue_bound == NULL); + assert(flags & KEVENT_FLAG_WORKQ_MANAGER); + ut->uu_kqueue_bound = kq; + ut->uu_kqueue_qos_index = qos_index; + ut->uu_kqueue_flags = flags; + + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWQ_BIND), + thread_tid(thread), flags, qos_index); + + return 1; + } + + /* + * If this is a synchronous bind callback, the request + * lock is already held, so just do the bind. + */ + if (flags & KEVENT_FLAG_SYNCHRONOUS_BIND) { + kqwq_req_held(kqwq); + /* strip out synchronout bind flag */ + flags &= ~KEVENT_FLAG_SYNCHRONOUS_BIND; + kqworkq_bind_thread_impl(kqwq, qos_index, thread, flags); + return 1; + } + + /* + * check the request that corresponds to our qos_index + * to see if there is an outstanding request. + */ + kqr = kqworkq_get_request(kqwq, qos_index); + assert(kqr->kqr_qos_index == qos_index); + kqwq_req_lock(kqwq); + + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWQ_BIND), + thread_tid(thread), flags, qos_index, kqr->kqr_state); + + if ((kqr->kqr_state & KQR_THREQUESTED) && + (kqr->kqr_state & KQR_PROCESSING) == 0) { + + if ((kqr->kqr_state & KQR_BOUND) && + thread == kqr->kqr_thread) { + /* duplicate bind - claim the thread */ + assert(ut->uu_kqueue_bound == kq); + assert(ut->uu_kqueue_qos_index == qos_index); + kqwq_req_unlock(kqwq); + return 1; + } + if ((kqr->kqr_state & (KQR_BOUND | KQWQ_THMANAGER)) == 0) { + /* ours to bind to */ + kqworkq_bind_thread_impl(kqwq, qos_index, thread, flags); + kqwq_req_unlock(kqwq); + return 1; + } + } + kqwq_req_unlock(kqwq); + return 0; +} + +static void +kqworkloop_bind_thread_impl(struct kqworkloop *kqwl, + thread_t thread, + __assert_only unsigned int flags) +{ + assert(flags & KEVENT_FLAG_WORKLOOP); + + /* the request object must be locked */ + kqwl_req_held(kqwl); + + struct kqrequest *kqr = &kqwl->kqwl_request; + struct uthread *ut = get_bsdthread_info(thread); + boolean_t ipc_override_is_sync; + kq_index_t qos_index = kqworkloop_combined_qos(kqwl, &ipc_override_is_sync); + + /* nobody else bound so finally bind (as a workloop) */ + assert(kqr->kqr_state & KQR_THREQUESTED); + assert((kqr->kqr_state & (KQR_BOUND | KQR_PROCESSING)) == 0); + assert(thread != kqwl->kqwl_owner); + + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWL_BIND), + kqwl->kqwl_dynamicid, (uintptr_t)thread_tid(thread), + qos_index, + (uintptr_t)(((uintptr_t)kqr->kqr_override_index << 16) | + (((uintptr_t)kqr->kqr_state) << 8) | + ((uintptr_t)ipc_override_is_sync))); + + kqr->kqr_state |= KQR_BOUND | KQR_R2K_NOTIF_ARMED; + kqr->kqr_thread = thread; + + /* bind the workloop to the uthread */ + ut->uu_kqueue_bound = (struct kqueue *)kqwl; + ut->uu_kqueue_flags = flags; + ut->uu_kqueue_qos_index = qos_index; + assert(ut->uu_kqueue_override_is_sync == 0); + ut->uu_kqueue_override_is_sync = ipc_override_is_sync; + if (qos_index) { + thread_add_ipc_override(thread, qos_index); + } + if (ipc_override_is_sync) { + thread_add_sync_ipc_override(thread); + } +} + +/* + * workloop_fulfill_threadreq - bind thread to processing workloop + * + * The provided thread will be responsible for delivering events + * associated with the given kqrequest. Bind it and get ready for + * the thread to eventually arrive. + * + * If WORKLOOP_FULFILL_THREADREQ_SYNC is specified, the callback + * within the context of the pthread_functions->workq_threadreq + * callout. In this case, the request structure is already locked. + */ +int +workloop_fulfill_threadreq(struct proc *p, + workq_threadreq_t req, + thread_t thread, + int flags) +{ + int sync = (flags & WORKLOOP_FULFILL_THREADREQ_SYNC); + int cancel = (flags & WORKLOOP_FULFILL_THREADREQ_CANCEL); + struct kqrequest *kqr; + struct kqworkloop *kqwl; + + kqwl = (struct kqworkloop *)((uintptr_t)req - + offsetof(struct kqworkloop, kqwl_request) - + offsetof(struct kqrequest, kqr_req)); + kqr = &kqwl->kqwl_request; + + /* validate we're looking at something valid */ + if (kqwl->kqwl_p != p || + (kqwl->kqwl_state & KQ_WORKLOOP) == 0) { + assert(kqwl->kqwl_p == p); + assert(kqwl->kqwl_state & KQ_WORKLOOP); + return EINVAL; + } + + if (!sync) + kqwl_req_lock(kqwl); + + /* Should be a pending request */ + if ((kqr->kqr_state & KQR_BOUND) || + (kqr->kqr_state & KQR_THREQUESTED) == 0) { + + assert((kqr->kqr_state & KQR_BOUND) == 0); + assert(kqr->kqr_state & KQR_THREQUESTED); + if (!sync) + kqwl_req_unlock(kqwl); + return EINPROGRESS; + } + + assert((kqr->kqr_state & KQR_DRAIN) == 0); + + /* + * Is it a cancel indication from pthread. + * If so, we must be exiting/exec'ing. Forget + * our pending request. + */ + if (cancel) { + kqr->kqr_state &= ~KQR_THREQUESTED; + kqr->kqr_state |= KQR_DRAIN; + } else { + /* do the actual bind? */ + kqworkloop_bind_thread_impl(kqwl, thread, KEVENT_FLAG_WORKLOOP); + } + + if (!sync) + kqwl_req_unlock(kqwl); + + if (cancel) + kqueue_release_last(p, &kqwl->kqwl_kqueue); /* may dealloc kq */ + + return 0; } + /* * kevent_qos_internal_bind - bind thread to processing kqueue @@ -3203,90 +5669,96 @@ kevent_qos_internal_bind( thread_t thread, unsigned int flags) { - struct fileproc *fp = NULL; - struct kqueue *kq = NULL; - struct kqworkq *kqwq; - struct kqrequest *kqr; - struct uthread *ut; kq_index_t qos_index; - int res = 0; - assert(thread != THREAD_NULL); assert(flags & KEVENT_FLAG_WORKQ); - if (thread == THREAD_NULL || - (flags & KEVENT_FLAG_WORKQ) == 0) { + if (thread == THREAD_NULL || (flags & KEVENT_FLAG_WORKQ) == 0) { return EINVAL; } - ut = get_bsdthread_info(thread); - - /* find the kqueue */ - res = kevent_get_kq(p, -1, flags, &fp, &kq); - assert(fp == NULL); - if (res) - return res; - /* get the qos index we're going to service */ qos_index = qos_index_for_servicer(qos_class, thread, flags); - - /* No need to bind the manager thread to any bucket */ - if (qos_index == KQWQ_QOS_MANAGER) { - assert(ut->uu_kqueue_bound == 0); - ut->uu_kqueue_bound = qos_index; - ut->uu_kqueue_flags = flags; + + if (kqworkq_internal_bind(p, qos_index, thread, flags)) return 0; - } - kqlock(kq); - assert(kq->kq_state & KQ_WORKQ); - - kqwq = (struct kqworkq *)kq; - kqr = kqworkq_get_request(kqwq, qos_index); + return EINPROGRESS; +} - kqwq_req_lock(kqwq); - /* - * A (non-emergency) request should have been made - * and nobody should already be servicing this bucket. - */ - assert(kqr->kqr_state & KQWQ_THREQUESTED); - assert((kqr->kqr_state & KQWQ_THMANAGER) == 0); - assert((kqr->kqr_state & KQWQ_PROCESSING) == 0); +static void +kqworkloop_internal_unbind( + struct proc *p, + thread_t thread, + unsigned int flags) +{ + struct kqueue *kq; + struct kqworkloop *kqwl; + struct uthread *ut = get_bsdthread_info(thread); - /* Is this is an extraneous bind? */ - if (thread == kqr->kqr_thread) { - assert(ut->uu_kqueue_bound == qos_index); - goto out; - } + assert(ut->uu_kqueue_bound != NULL); + kq = ut->uu_kqueue_bound; + assert(kq->kq_state & KQ_WORKLOOP); + kqwl = (struct kqworkloop *)kq; - /* nobody else bound and we're not bound elsewhere */ - assert(ut->uu_kqueue_bound == 0); - assert(ut->uu_kqueue_flags == 0); - assert(kqr->kqr_thread == THREAD_NULL); + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWL_UNBIND), + kqwl->kqwl_dynamicid, (uintptr_t)thread_tid(thread), + flags, 0); - /* Don't bind if there is a conflict */ - if (kqr->kqr_thread != THREAD_NULL || - (kqr->kqr_state & KQWQ_THMANAGER)) { - res = EINPROGRESS; - goto out; - } + if (!(kq->kq_state & KQ_NO_WQ_THREAD)) { + assert(is_workqueue_thread(thread)); - /* finally bind the thread */ - kqr->kqr_thread = thread; - ut->uu_kqueue_bound = qos_index; - ut->uu_kqueue_flags = flags; + kqlock(kq); + kqworkloop_unbind_thread(kqwl, thread, flags); + kqunlock(kq); - /* add any pending overrides to the thread */ - if (kqr->kqr_override_delta) { - thread_add_ipc_override(thread, qos_index + kqr->kqr_override_delta); + /* If last reference, dealloc the workloop kq */ + kqueue_release_last(p, kq); + } else { + assert(!is_workqueue_thread(thread)); + kevent_servicer_detach_thread(p, kqwl->kqwl_dynamicid, thread, flags, kq); } +} -out: - kqwq_req_unlock(kqwq); - kqunlock(kq); +static void +kqworkq_internal_unbind( + struct proc *p, + kq_index_t qos_index, + thread_t thread, + unsigned int flags) +{ + struct kqueue *kq; + struct kqworkq *kqwq; + struct uthread *ut; + kq_index_t end_index; - return res; + assert(thread == current_thread()); + ut = get_bsdthread_info(thread); + + kq = p->p_fd->fd_wqkqueue; + assert(kq->kq_state & KQ_WORKQ); + assert(ut->uu_kqueue_bound == kq); + + kqwq = (struct kqworkq *)kq; + + /* end servicing any requests we might own */ + end_index = (qos_index == KQWQ_QOS_MANAGER) ? + 0 : qos_index; + kqlock(kq); + + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWQ_UNBIND), + (uintptr_t)thread_tid(thread), flags, qos_index); + + do { + kqworkq_end_processing(kqwq, qos_index, flags); + } while (qos_index-- > end_index); + + ut->uu_kqueue_bound = NULL; + ut->uu_kqueue_qos_index = 0; + ut->uu_kqueue_flags = 0; + + kqunlock(kq); } /* @@ -3306,56 +5778,49 @@ kevent_qos_internal_unbind( thread_t thread, unsigned int flags) { - struct kqueue *kq; - struct uthread *ut; - struct fileproc *fp = NULL; - kq_index_t qos_index; - kq_index_t end_index; - int res; +#pragma unused(qos_class) - assert(flags & KEVENT_FLAG_WORKQ); - assert(thread == current_thread()); + struct uthread *ut; + struct kqueue *kq; + unsigned int bound_flags; + bool check_flags; - if (thread == THREAD_NULL || - (flags & KEVENT_FLAG_WORKQ) == 0) - return EINVAL; - - /* get the kq */ - res = kevent_get_kq(p, -1, flags, &fp, &kq); - assert(fp == NULL); - if (res) - return res; + ut = get_bsdthread_info(thread); + if (ut->uu_kqueue_bound == NULL) { + /* early out if we are already unbound */ + assert(ut->uu_kqueue_flags == 0); + assert(ut->uu_kqueue_qos_index == 0); + assert(ut->uu_kqueue_override_is_sync == 0); + return EALREADY; + } - assert(kq->kq_state & KQ_WORKQ); + assert(flags & (KEVENT_FLAG_WORKQ | KEVENT_FLAG_WORKLOOP)); + assert(thread == current_thread()); - /* get the index we have been servicing */ - qos_index = qos_index_for_servicer(qos_class, thread, flags); + check_flags = flags & KEVENT_FLAG_UNBIND_CHECK_FLAGS; - ut = get_bsdthread_info(thread); + /* Get the kqueue we started with */ + kq = ut->uu_kqueue_bound; + assert(kq != NULL); + assert(kq->kq_state & (KQ_WORKQ | KQ_WORKLOOP)); - /* early out if we were already unbound - or never bound */ - if (ut->uu_kqueue_bound != qos_index) { - __assert_only struct kqworkq *kqwq = (struct kqworkq *)kq; - __assert_only struct kqrequest *kqr = kqworkq_get_request(kqwq, qos_index); + /* get flags and QoS parameters we started with */ + bound_flags = ut->uu_kqueue_flags; - assert(ut->uu_kqueue_bound == 0); - assert(ut->uu_kqueue_flags == 0); - assert(kqr->kqr_thread != thread); - return EALREADY; - } + /* Unbind from the class of workq */ + if (kq->kq_state & KQ_WORKQ) { + if (check_flags && !(flags & KEVENT_FLAG_WORKQ)) { + return EINVAL; + } - /* unbind from all the buckets we might own */ - end_index = (qos_index == KQWQ_QOS_MANAGER) ? - 0 : qos_index; - kqlock(kq); - do { - kqueue_end_processing(kq, qos_index, flags); - } while (qos_index-- > end_index); - kqunlock(kq); + kqworkq_internal_unbind(p, ut->uu_kqueue_qos_index, thread, bound_flags); + } else { + if (check_flags && !(flags & KEVENT_FLAG_WORKLOOP)) { + return EINVAL; + } - /* indicate that we are done processing in the uthread */ - ut->uu_kqueue_bound = 0; - ut->uu_kqueue_flags = 0; + kqworkloop_internal_unbind(p, thread, bound_flags); + } return 0; } @@ -3380,22 +5845,45 @@ kqueue_process(struct kqueue *kq, kevent_callback_t callback, void *callback_data, struct filt_process_s *process_data, - kq_index_t servicer_qos_index, int *countp, struct proc *p) { unsigned int flags = process_data ? process_data->fp_flags : 0; + struct uthread *ut = get_bsdthread_info(current_thread()); kq_index_t start_index, end_index, i; struct knote *kn; int nevents = 0; int error = 0; /* - * Based on the native QoS of the servicer, - * determine the range of QoSes that need checking + * Based on the mode of the kqueue and the bound QoS of the servicer, + * determine the range of thread requests that need checking */ - start_index = servicer_qos_index; - end_index = (start_index == KQWQ_QOS_MANAGER) ? 0 : start_index; + if (kq->kq_state & KQ_WORKQ) { + if (flags & KEVENT_FLAG_WORKQ_MANAGER) { + start_index = KQWQ_QOS_MANAGER; + } else if (ut->uu_kqueue_bound != kq) { + return EJUSTRETURN; + } else { + start_index = ut->uu_kqueue_qos_index; + } + + /* manager services every request in a workq kqueue */ + assert(start_index > 0 && start_index <= KQWQ_QOS_MANAGER); + end_index = (start_index == KQWQ_QOS_MANAGER) ? 0 : start_index; + + } else if (kq->kq_state & KQ_WORKLOOP) { + if (ut->uu_kqueue_bound != kq) + return EJUSTRETURN; + + /* + * Single request servicing + * we want to deliver all events, regardless of the QOS + */ + start_index = end_index = THREAD_QOS_UNSPECIFIED; + } else { + start_index = end_index = QOS_INDEX_KQFILE; + } i = start_index; @@ -3407,41 +5895,39 @@ kqueue_process(struct kqueue *kq, } /* - * loop through the enqueued knotes, processing each one and - * revalidating those that need it. As they are processed, - * they get moved to the inprocess queue (so the loop can end). + * loop through the enqueued knotes associated with this request, + * processing each one. Each request may have several queues + * of knotes to process (depending on the type of kqueue) so we + * have to loop through all the queues as long as we have additional + * space. */ error = 0; struct kqtailq *base_queue = kqueue_get_base_queue(kq, i); struct kqtailq *queue = kqueue_get_high_queue(kq, i); do { - while (error == 0 && - (kn = TAILQ_FIRST(queue)) != NULL) { - /* Process the knote */ + while (error == 0 && (kn = TAILQ_FIRST(queue)) != NULL) { error = knote_process(kn, callback, callback_data, process_data, p); - if (error == EJUSTRETURN) + if (error == EJUSTRETURN) { error = 0; - else + } else { nevents++; - - /* break out if no more space for additional events */ - if (error == EWOULDBLOCK) { - if ((kq->kq_state & KQ_WORKQ) == 0) - kqueue_end_processing(kq, i, flags); - error = 0; - goto out; } + /* error is EWOULDBLOCK when the out event array is full */ } } while (error == 0 && queue-- > base_queue); - /* let somebody else process events if we're not in workq mode */ - if ((kq->kq_state & KQ_WORKQ) == 0) - kqueue_end_processing(kq, i, flags); + if ((kq->kq_state & KQ_WORKQ) == 0) { + kqueue_end_processing(kq, i, nevents, flags); + } + if (error == EWOULDBLOCK) { + /* break out if no more space for additional events */ + error = 0; + break; + } } while (i-- > end_index); -out: *countp = nevents; return (error); } @@ -3463,11 +5949,16 @@ kqueue_scan_continue(void *data, wait_result_t wait_result) kqlock(kq); retry: error = kqueue_process(kq, cont_args->call, cont_args->data, - process_data, cont_args->servicer_qos_index, - &count, current_proc()); + process_data, &count, current_proc()); if (error == 0 && count == 0) { + if (kq->kq_state & KQ_DRAIN) { + kqunlock(kq); + goto drain; + } + if (kq->kq_state & KQ_WAKEUP) goto retry; + waitq_assert_wait64((struct waitq *)&kq->kq_wqs, KQ_EVENT, THREAD_ABORTSAFE, cont_args->deadline); @@ -3485,6 +5976,7 @@ kqueue_scan_continue(void *data, wait_result_t wait_result) error = EINTR; break; case THREAD_RESTART: + drain: error = EBADF; break; default: @@ -3523,7 +6015,6 @@ kqueue_scan(struct kqueue *kq, struct proc *p) { thread_continue_t cont = THREAD_CONTINUE_NULL; - kq_index_t servicer_qos_index; unsigned int flags; uint64_t deadline; int error; @@ -3537,9 +6028,6 @@ kqueue_scan(struct kqueue *kq, */ flags = (process_data) ? process_data->fp_flags : 0; fd = (process_data) ? process_data->fp_fd : -1; - servicer_qos_index = (kq->kq_state & KQ_WORKQ) ? - qos_index_for_servicer(fd, current_thread(), flags) : - QOS_INDEX_KQFILE; first = 1; for (;;) { @@ -3552,8 +6040,7 @@ kqueue_scan(struct kqueue *kq, */ kqlock(kq); error = kqueue_process(kq, callback, callback_data, - process_data, servicer_qos_index, - &count, p); + process_data, &count, p); if (error || count) break; /* lock still held */ @@ -3588,11 +6075,15 @@ kqueue_scan(struct kqueue *kq, cont_args->deadline = deadline; cont_args->data = callback_data; cont_args->process_data = process_data; - cont_args->servicer_qos_index = servicer_qos_index; cont = kqueue_scan_continue; } } + if (kq->kq_state & KQ_DRAIN) { + kqunlock(kq); + return EBADF; + } + /* If awakened during processing, try again */ if (kq->kq_state & KQ_WAKEUP) { kqunlock(kq); @@ -3745,13 +6236,14 @@ kqueue_select(struct fileproc *fp, int which, void *wq_link_id, while ((kn = (struct knote *)TAILQ_FIRST(suppressq)) != NULL) { unsigned peek = 1; - /* If didn't vanish while suppressed - peek at it */ - if (kqlock2knoteuse(kq, kn)) { + assert(!knoteuse_needs_boost(kn, NULL)); + /* If didn't vanish while suppressed - peek at it */ + if (kqlock2knoteuse(kq, kn, KNUSE_NONE)) { peek = knote_fops(kn)->f_peek(kn); /* if it dropped while getting lock - move on */ - if (!knoteuse2kqlock(kq, kn, 0)) + if (!knoteuse2kqlock(kq, kn, KNUSE_NONE)) continue; } @@ -3767,7 +6259,7 @@ kqueue_select(struct fileproc *fp, int which, void *wq_link_id, } out: - kqueue_end_processing(kq, QOS_INDEX_KQFILE, 0); + kqueue_end_processing(kq, QOS_INDEX_KQFILE, retnum, 0); kqunlock(kq); return (retnum); } @@ -3794,7 +6286,8 @@ kqueue_close(struct fileglob *fg, __unused vfs_context_t ctx) * that relationship is torn down. */ static int -kqueue_kqfilter(__unused struct fileproc *fp, struct knote *kn, __unused vfs_context_t ctx) +kqueue_kqfilter(__unused struct fileproc *fp, struct knote *kn, + __unused struct kevent_internal_s *kev, __unused vfs_context_t ctx) { struct kqfile *kqf = (struct kqfile *)kn->kn_fp->f_data; struct kqueue *kq = &kqf->kqf_kqueue; @@ -3877,155 +6370,671 @@ kqueue_stat(struct kqueue *kq, void *ub, int isstat64, proc_t p) if (isstat64 != 0) { struct stat64 *sb64 = (struct stat64 *)ub; - bzero((void *)sb64, sizeof(*sb64)); - sb64->st_size = kq->kq_count; - if (kq->kq_state & KQ_KEV_QOS) - sb64->st_blksize = sizeof(struct kevent_qos_s); - else if (kq->kq_state & KQ_KEV64) - sb64->st_blksize = sizeof(struct kevent64_s); - else if (IS_64BIT_PROCESS(p)) - sb64->st_blksize = sizeof(struct user64_kevent); - else - sb64->st_blksize = sizeof(struct user32_kevent); - sb64->st_mode = S_IFIFO; - } else { - struct stat *sb = (struct stat *)ub; + bzero((void *)sb64, sizeof(*sb64)); + sb64->st_size = kq->kq_count; + if (kq->kq_state & KQ_KEV_QOS) + sb64->st_blksize = sizeof(struct kevent_qos_s); + else if (kq->kq_state & KQ_KEV64) + sb64->st_blksize = sizeof(struct kevent64_s); + else if (IS_64BIT_PROCESS(p)) + sb64->st_blksize = sizeof(struct user64_kevent); + else + sb64->st_blksize = sizeof(struct user32_kevent); + sb64->st_mode = S_IFIFO; + } else { + struct stat *sb = (struct stat *)ub; + + bzero((void *)sb, sizeof(*sb)); + sb->st_size = kq->kq_count; + if (kq->kq_state & KQ_KEV_QOS) + sb->st_blksize = sizeof(struct kevent_qos_s); + else if (kq->kq_state & KQ_KEV64) + sb->st_blksize = sizeof(struct kevent64_s); + else if (IS_64BIT_PROCESS(p)) + sb->st_blksize = sizeof(struct user64_kevent); + else + sb->st_blksize = sizeof(struct user32_kevent); + sb->st_mode = S_IFIFO; + } + kqunlock(kq); + return (0); +} + +/* + * Interact with the pthread kext to request a servicing there. + * Eventually, this will request threads at specific QoS levels. + * For now, it only requests a dispatch-manager-QoS thread, and + * only one-at-a-time. + * + * - Caller holds the workq request lock + * + * - May be called with the kqueue's wait queue set locked, + * so cannot do anything that could recurse on that. + */ +static void +kqworkq_request_thread( + struct kqworkq *kqwq, + kq_index_t qos_index) +{ + struct kqrequest *kqr; + + assert(kqwq->kqwq_state & KQ_WORKQ); + assert(qos_index < KQWQ_NQOS); + + kqr = kqworkq_get_request(kqwq, qos_index); + + assert(kqr->kqr_state & KQR_WAKEUP); + + /* + * If we have already requested a thread, and it hasn't + * started processing yet, there's no use hammering away + * on the pthread kext. + */ + if (kqr->kqr_state & KQR_THREQUESTED) + return; + + assert((kqr->kqr_state & KQR_BOUND) == 0); + + /* request additional workq threads if appropriate */ + if (pthread_functions != NULL && + pthread_functions->workq_reqthreads != NULL) { + unsigned int flags = KEVENT_FLAG_WORKQ; + unsigned long priority; + thread_t wqthread; + + /* Compute the appropriate pthread priority */ + priority = qos_from_qos_index(qos_index); + +#if 0 + /* JMM - for now remain compatible with old invocations */ + /* set the over-commit flag on the request if needed */ + if (kqr->kqr_state & KQR_THOVERCOMMIT) + priority |= _PTHREAD_PRIORITY_OVERCOMMIT_FLAG; +#endif /* 0 */ + + /* Compute a priority based on qos_index. */ + struct workq_reqthreads_req_s request = { + .priority = priority, + .count = 1 + }; + + /* mark that we are making a request */ + kqr->kqr_state |= KQR_THREQUESTED; + if (qos_index == KQWQ_QOS_MANAGER) + kqr->kqr_state |= KQWQ_THMANAGER; + + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWQ_THREQUEST), + 0, qos_index, + (((uintptr_t)kqr->kqr_override_index << 8) | + (uintptr_t)kqr->kqr_state)); + wqthread = (*pthread_functions->workq_reqthreads)(kqwq->kqwq_p, 1, &request); + + /* We've been switched to the emergency/manager thread */ + if (wqthread == (thread_t)-1) { + assert(qos_index != KQWQ_QOS_MANAGER); + kqr->kqr_state |= KQWQ_THMANAGER; + return; + } + + /* + * bind the returned thread identity + * This goes away when we switch to synchronous callback + * binding from the pthread kext. + */ + if (wqthread != NULL) { + kqworkq_bind_thread_impl(kqwq, qos_index, wqthread, flags); + } + } +} + +/* + * If we aren't already busy processing events [for this QoS], + * request workq thread support as appropriate. + * + * TBD - for now, we don't segregate out processing by QoS. + * + * - May be called with the kqueue's wait queue set locked, + * so cannot do anything that could recurse on that. + */ +static void +kqworkq_request_help( + struct kqworkq *kqwq, + kq_index_t qos_index) +{ + struct kqrequest *kqr; + + /* convert to thread qos value */ + assert(qos_index < KQWQ_NQOS); + + kqwq_req_lock(kqwq); + kqr = kqworkq_get_request(kqwq, qos_index); + + if ((kqr->kqr_state & KQR_WAKEUP) == 0) { + /* Indicate that we needed help from this request */ + kqr->kqr_state |= KQR_WAKEUP; + + /* Go assure a thread request has been made */ + kqworkq_request_thread(kqwq, qos_index); + } + kqwq_req_unlock(kqwq); +} + +static void +kqworkloop_threadreq_impl(struct kqworkloop *kqwl, kq_index_t qos_index) +{ + struct kqrequest *kqr = &kqwl->kqwl_request; + unsigned long pri = pthread_priority_for_kqrequest(kqr, qos_index); + int op, ret; + + assert((kqr->kqr_state & (KQR_THREQUESTED | KQR_BOUND)) == KQR_THREQUESTED); + + /* + * New-style thread request supported. Provide + * the pthread kext a pointer to a workq_threadreq_s + * structure for its use until a corresponding + * workloop_fulfill_threqreq callback. + */ + if (current_proc() == kqwl->kqwl_kqueue.kq_p) { + op = WORKQ_THREADREQ_WORKLOOP_NO_THREAD_CALL; + } else { + op = WORKQ_THREADREQ_WORKLOOP; + } +again: + ret = (*pthread_functions->workq_threadreq)(kqwl->kqwl_p, &kqr->kqr_req, + WORKQ_THREADREQ_WORKLOOP, pri, 0); + switch (ret) { + case ENOTSUP: + assert(op == WORKQ_THREADREQ_WORKLOOP_NO_THREAD_CALL); + op = WORKQ_THREADREQ_WORKLOOP; + goto again; + + case ECANCELED: + case EINVAL: + /* + * Process is shutting down or exec'ing. + * All the kqueues are going to be cleaned up + * soon. Forget we even asked for a thread - + * and make sure we don't ask for more. + */ + kqueue_release((struct kqueue *)kqwl, KQUEUE_CANT_BE_LAST_REF); + kqr->kqr_state &= ~KQR_THREQUESTED; + kqr->kqr_state |= KQR_DRAIN; + break; + + case EAGAIN: + assert(op == WORKQ_THREADREQ_WORKLOOP_NO_THREAD_CALL); + act_set_astkevent(current_thread(), AST_KEVENT_REDRIVE_THREADREQ); + break; + + default: + assert(ret == 0); + } +} + +static void +kqworkloop_threadreq_modify(struct kqworkloop *kqwl, kq_index_t qos_index) +{ + struct kqrequest *kqr = &kqwl->kqwl_request; + unsigned long pri = pthread_priority_for_kqrequest(kqr, qos_index); + int ret, op = WORKQ_THREADREQ_CHANGE_PRI_NO_THREAD_CALL; + + assert((kqr->kqr_state & (KQR_THREQUESTED | KQR_BOUND)) == KQR_THREQUESTED); + + if (current_proc() == kqwl->kqwl_kqueue.kq_p) { + op = WORKQ_THREADREQ_CHANGE_PRI_NO_THREAD_CALL; + } else { + op = WORKQ_THREADREQ_CHANGE_PRI; + } +again: + ret = (*pthread_functions->workq_threadreq_modify)(kqwl->kqwl_p, + &kqr->kqr_req, op, pri, 0); + switch (ret) { + case ENOTSUP: + assert(op == WORKQ_THREADREQ_CHANGE_PRI_NO_THREAD_CALL); + op = WORKQ_THREADREQ_CHANGE_PRI; + goto again; + + case EAGAIN: + assert(op == WORKQ_THREADREQ_WORKLOOP_NO_THREAD_CALL); + act_set_astkevent(current_thread(), AST_KEVENT_REDRIVE_THREADREQ); + break; + + case ECANCELED: + case EINVAL: + case 0: + break; + + default: + assert(ret == 0); + } +} + +/* + * Interact with the pthread kext to request a servicing thread. + * This will request a single thread at the highest QoS level + * for which there is work (whether that was the requested QoS + * for an event or an override applied to a lower-QoS request). + * + * - Caller holds the workloop request lock + * + * - May be called with the kqueue's wait queue set locked, + * so cannot do anything that could recurse on that. + */ +static void +kqworkloop_request_thread(struct kqworkloop *kqwl, kq_index_t qos_index) +{ + struct kqrequest *kqr; + + assert(kqwl->kqwl_state & KQ_WORKLOOP); + + kqr = &kqwl->kqwl_request; + + assert(kqwl->kqwl_owner == THREAD_NULL); + assert((kqr->kqr_state & KQR_BOUND) == 0); + assert((kqr->kqr_state & KQR_THREQUESTED) == 0); + assert(!(kqwl->kqwl_kqueue.kq_state & KQ_NO_WQ_THREAD)); + + /* If we're draining thread requests, just bail */ + if (kqr->kqr_state & KQR_DRAIN) + return; + + if (pthread_functions != NULL && + pthread_functions->workq_threadreq != NULL) { + /* + * set request state flags, etc... before calling pthread + * This assures they are set before a possible synchronous + * callback to workloop_fulfill_threadreq(). + */ + kqr->kqr_state |= KQR_THREQUESTED; + + /* Add a thread request reference on the kqueue. */ + kqueue_retain((struct kqueue *)kqwl); + + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWL_THREQUEST), + kqwl->kqwl_dynamicid, + 0, qos_index, kqr->kqr_state); + kqworkloop_threadreq_impl(kqwl, qos_index); + } else { + panic("kqworkloop_request_thread"); + return; + } +} + +static void +kqworkloop_update_sync_override_state(struct kqworkloop *kqwl, boolean_t sync_ipc_override) +{ + struct kqrequest *kqr = &kqwl->kqwl_request; + kqwl_req_lock(kqwl); + kqr->kqr_has_sync_override = sync_ipc_override; + kqwl_req_unlock(kqwl); + +} + +static inline kq_index_t +kqworkloop_combined_qos(struct kqworkloop *kqwl, boolean_t *ipc_override_is_sync) +{ + struct kqrequest *kqr = &kqwl->kqwl_request; + kq_index_t override; + + *ipc_override_is_sync = FALSE; + override = MAX(MAX(kqr->kqr_qos_index, kqr->kqr_override_index), + kqr->kqr_dsync_waiters_qos); + + if (kqr->kqr_sync_suppress_count > 0 || kqr->kqr_has_sync_override) { + *ipc_override_is_sync = TRUE; + override = THREAD_QOS_USER_INTERACTIVE; + } + return override; +} + +static inline void +kqworkloop_request_fire_r2k_notification(struct kqworkloop *kqwl) +{ + struct kqrequest *kqr = &kqwl->kqwl_request; + + kqwl_req_held(kqwl); + + if (kqr->kqr_state & KQR_R2K_NOTIF_ARMED) { + assert(kqr->kqr_state & KQR_BOUND); + assert(kqr->kqr_thread); + + kqr->kqr_state &= ~KQR_R2K_NOTIF_ARMED; + act_set_astkevent(kqr->kqr_thread, AST_KEVENT_RETURN_TO_KERNEL); + } +} + +static void +kqworkloop_update_threads_qos(struct kqworkloop *kqwl, int op, kq_index_t qos) +{ + const uint8_t KQWL_STAYACTIVE_FIRED_BIT = (1 << 0); + + struct kqrequest *kqr = &kqwl->kqwl_request; + boolean_t old_ipc_override_is_sync = FALSE; + kq_index_t old_qos = kqworkloop_combined_qos(kqwl, &old_ipc_override_is_sync); + struct kqueue *kq = &kqwl->kqwl_kqueue; + bool static_thread = (kq->kq_state & KQ_NO_WQ_THREAD); + kq_index_t i; + + /* must hold the kqr lock */ + kqwl_req_held(kqwl); + + switch (op) { + case KQWL_UTQ_UPDATE_WAKEUP_QOS: + if (qos == KQWL_BUCKET_STAYACTIVE) { + /* + * the KQWL_BUCKET_STAYACTIVE is not a QoS bucket, we only remember + * a high watermark (kqr_stayactive_qos) of any stay active knote + * that was ever registered with this workloop. + * + * When waitq_set__CALLING_PREPOST_HOOK__() wakes up any stay active + * knote, we use this high-watermark as a wakeup-index, and also set + * the magic KQWL_BUCKET_STAYACTIVE bit to make sure we remember + * there is at least one stay active knote fired until the next full + * processing of this bucket. + */ + kqr->kqr_wakeup_indexes |= KQWL_STAYACTIVE_FIRED_BIT; + qos = kqr->kqr_stayactive_qos; + assert(qos); + assert(!static_thread); + } + if (kqr->kqr_wakeup_indexes & (1 << qos)) { + assert(kqr->kqr_state & KQR_WAKEUP); + break; + } + + kqr->kqr_wakeup_indexes |= (1 << qos); + kqr->kqr_state |= KQR_WAKEUP; + kqworkloop_request_fire_r2k_notification(kqwl); + goto recompute_async; + + case KQWL_UTQ_UPDATE_STAYACTIVE_QOS: + assert(qos); + if (kqr->kqr_stayactive_qos < qos) { + kqr->kqr_stayactive_qos = qos; + if (kqr->kqr_wakeup_indexes & KQWL_STAYACTIVE_FIRED_BIT) { + assert(kqr->kqr_state & KQR_WAKEUP); + kqr->kqr_wakeup_indexes |= (1 << qos); + goto recompute_async; + } + } + break; + + case KQWL_UTQ_RECOMPUTE_WAKEUP_QOS: + kqlock_held(kq); // to look at kq_queues + kqr->kqr_has_sync_override = FALSE; + i = KQWL_BUCKET_STAYACTIVE; + if (TAILQ_EMPTY(&kqr->kqr_suppressed)) { + kqr->kqr_override_index = THREAD_QOS_UNSPECIFIED; + } + if (!TAILQ_EMPTY(&kq->kq_queue[i]) && + (kqr->kqr_wakeup_indexes & KQWL_STAYACTIVE_FIRED_BIT)) { + /* + * If the KQWL_STAYACTIVE_FIRED_BIT is set, it means a stay active + * knote may have fired, so we need to merge in kqr_stayactive_qos. + * + * Unlike other buckets, this one is never empty but could be idle. + */ + kqr->kqr_wakeup_indexes &= KQWL_STAYACTIVE_FIRED_BIT; + kqr->kqr_wakeup_indexes |= (1 << kqr->kqr_stayactive_qos); + } else { + kqr->kqr_wakeup_indexes = 0; + } + for (i = THREAD_QOS_UNSPECIFIED + 1; i < KQWL_BUCKET_STAYACTIVE; i++) { + if (!TAILQ_EMPTY(&kq->kq_queue[i])) { + kqr->kqr_wakeup_indexes |= (1 << i); + struct knote *kn = TAILQ_FIRST(&kqwl->kqwl_kqueue.kq_queue[i]); + if (i == THREAD_QOS_USER_INTERACTIVE && + kn->kn_qos_override_is_sync) { + kqr->kqr_has_sync_override = TRUE; + } + } + } + if (kqr->kqr_wakeup_indexes) { + kqr->kqr_state |= KQR_WAKEUP; + kqworkloop_request_fire_r2k_notification(kqwl); + } else { + kqr->kqr_state &= ~KQR_WAKEUP; + } + assert(qos == THREAD_QOS_UNSPECIFIED); + goto recompute_async; + + case KQWL_UTQ_RESET_WAKEUP_OVERRIDE: + kqr->kqr_override_index = THREAD_QOS_UNSPECIFIED; + assert(qos == THREAD_QOS_UNSPECIFIED); + goto recompute_async; + + case KQWL_UTQ_UPDATE_WAKEUP_OVERRIDE: + recompute_async: + /* + * When modifying the wakeup QoS or the async override QoS, we always + * need to maintain our invariant that kqr_override_index is at least as + * large as the highest QoS for which an event is fired. + * + * However this override index can be larger when there is an overriden + * suppressed knote pushing on the kqueue. + */ + if (kqr->kqr_wakeup_indexes > (1 << qos)) { + qos = fls(kqr->kqr_wakeup_indexes) - 1; /* fls is 1-based */ + } + if (kqr->kqr_override_index < qos) { + kqr->kqr_override_index = qos; + } + break; + + case KQWL_UTQ_REDRIVE_EVENTS: + break; - bzero((void *)sb, sizeof(*sb)); - sb->st_size = kq->kq_count; - if (kq->kq_state & KQ_KEV_QOS) - sb->st_blksize = sizeof(struct kevent_qos_s); - else if (kq->kq_state & KQ_KEV64) - sb->st_blksize = sizeof(struct kevent64_s); - else if (IS_64BIT_PROCESS(p)) - sb->st_blksize = sizeof(struct user64_kevent); - else - sb->st_blksize = sizeof(struct user32_kevent); - sb->st_mode = S_IFIFO; - } - kqunlock(kq); - return (0); -} + case KQWL_UTQ_SET_ASYNC_QOS: + filt_wlheld(kqwl); + kqr->kqr_qos_index = qos; + break; + case KQWL_UTQ_SET_SYNC_WAITERS_QOS: + filt_wlheld(kqwl); + kqr->kqr_dsync_waiters_qos = qos; + break; -/* - * Interact with the pthread kext to request a servicing there. - * Eventually, this will request threads at specific QoS levels. - * For now, it only requests a dispatch-manager-QoS thread, and - * only one-at-a-time. - * - * - Caller holds the workq request lock - * - * - May be called with the kqueue's wait queue set locked, - * so cannot do anything that could recurse on that. - */ -static void -kqworkq_request_thread( - struct kqworkq *kqwq, - kq_index_t qos_index) -{ - struct kqrequest *kqr; + default: + panic("unknown kqwl thread qos update operation: %d", op); + } - assert(kqwq->kqwq_state & KQ_WORKQ); - assert(qos_index < KQWQ_NQOS); + boolean_t new_ipc_override_is_sync = FALSE; + kq_index_t new_qos = kqworkloop_combined_qos(kqwl, &new_ipc_override_is_sync); + thread_t kqwl_owner = kqwl->kqwl_owner; + thread_t servicer = kqr->kqr_thread; + __assert_only int ret; - kqr = kqworkq_get_request(kqwq, qos_index); + /* + * Apply the diffs to the owner if applicable + */ + if (filt_wlowner_is_valid(kqwl_owner)) { +#if 0 + /* JMM - need new trace hooks for owner overrides */ + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWL_THADJUST), + kqwl->kqwl_dynamicid, + (kqr->kqr_state & KQR_BOUND) ? thread_tid(kqwl_owner) : 0, + (kqr->kqr_qos_index << 8) | new_qos, + (kqr->kqr_override_index << 8) | kqr->kqr_state); +#endif + if (new_qos == kqr->kqr_dsync_owner_qos) { + // nothing to do + } else if (kqr->kqr_dsync_owner_qos == THREAD_QOS_UNSPECIFIED) { + thread_add_ipc_override(kqwl_owner, new_qos); + } else if (new_qos == THREAD_QOS_UNSPECIFIED) { + thread_drop_ipc_override(kqwl_owner); + } else /* kqr->kqr_dsync_owner_qos != new_qos */ { + thread_update_ipc_override(kqwl_owner, new_qos); + } + kqr->kqr_dsync_owner_qos = new_qos; + + if (new_ipc_override_is_sync && + !kqr->kqr_owner_override_is_sync) { + thread_add_sync_ipc_override(kqwl_owner); + } else if (!new_ipc_override_is_sync && + kqr->kqr_owner_override_is_sync) { + thread_drop_sync_ipc_override(kqwl_owner); + } + kqr->kqr_owner_override_is_sync = new_ipc_override_is_sync; + } - /* - * If we have already requested a thread, and it hasn't - * started processing yet, there's no use hammering away - * on the pthread kext. + /* + * apply the diffs to the servicer */ - if (kqr->kqr_state & KQWQ_THREQUESTED) - return; + if (static_thread) { + /* + * Statically bound thread + * + * These threads don't participates in QoS overrides today, just wakeup + * the thread blocked on this kqueue if a new event arrived. + */ - assert(kqr->kqr_thread == THREAD_NULL); + switch (op) { + case KQWL_UTQ_UPDATE_WAKEUP_QOS: + case KQWL_UTQ_UPDATE_STAYACTIVE_QOS: + case KQWL_UTQ_RECOMPUTE_WAKEUP_QOS: + break; - /* request additional workq threads if appropriate */ - if (pthread_functions != NULL && - pthread_functions->workq_reqthreads != NULL) { - unsigned int flags = KEVENT_FLAG_WORKQ; + case KQWL_UTQ_RESET_WAKEUP_OVERRIDE: + case KQWL_UTQ_UPDATE_WAKEUP_OVERRIDE: + case KQWL_UTQ_REDRIVE_EVENTS: + case KQWL_UTQ_SET_ASYNC_QOS: + case KQWL_UTQ_SET_SYNC_WAITERS_QOS: + panic("should never be called"); + break; + } - /* Compute a priority based on qos_index. */ - struct workq_reqthreads_req_s request = { - .priority = qos_from_qos_index(qos_index), - .count = 1 - }; + kqlock_held(kq); - thread_t wqthread; - wqthread = (*pthread_functions->workq_reqthreads)(kqwq->kqwq_p, 1, &request); - kqr->kqr_state |= KQWQ_THREQUESTED; + if ((kqr->kqr_state & KQR_BOUND) && (kqr->kqr_state & KQR_WAKEUP)) { + assert(servicer && !is_workqueue_thread(servicer)); + if (kq->kq_state & (KQ_SLEEP | KQ_SEL)) { + kq->kq_state &= ~(KQ_SLEEP | KQ_SEL); + waitq_wakeup64_all((struct waitq *)&kq->kq_wqs, KQ_EVENT, + THREAD_AWAKENED, WAITQ_ALL_PRIORITIES); + } + } + } else if ((kqr->kqr_state & KQR_THREQUESTED) == 0) { + /* + * No servicer, nor thread-request + * + * Make a new thread request, unless there is an owner (or the workloop + * is suspended in userland) or if there is no asynchronous work in the + * first place. + */ - /* Have we been switched to the emergency/manager thread? */ - if (wqthread == (thread_t)-1) { - flags |= KEVENT_FLAG_WORKQ_MANAGER; - wqthread = THREAD_NULL; - } else if (qos_index == KQWQ_QOS_MANAGER) - flags |= KEVENT_FLAG_WORKQ_MANAGER; + if (kqwl_owner == THREAD_NULL && (kqr->kqr_state & KQR_WAKEUP)) { + kqworkloop_request_thread(kqwl, new_qos); + } + } else if ((kqr->kqr_state & KQR_BOUND) == 0 && + (kqwl_owner || (kqr->kqr_state & KQR_WAKEUP) == 0)) { + /* + * No servicer, thread request in flight we want to cancel + * + * We just got rid of the last knote of the kqueue or noticed an owner + * with a thread request still in flight, take it back. + */ + ret = (*pthread_functions->workq_threadreq_modify)(kqwl->kqwl_p, + &kqr->kqr_req, WORKQ_THREADREQ_CANCEL, 0, 0); + if (ret == 0) { + kqr->kqr_state &= ~KQR_THREQUESTED; + kqueue_release(kq, KQUEUE_CANT_BE_LAST_REF); + } + } else { + boolean_t qos_changed = FALSE; + + /* + * Servicer or request is in flight + * + * Just apply the diff to the servicer or the thread request + */ + if (kqr->kqr_state & KQR_BOUND) { + servicer = kqr->kqr_thread; + struct uthread *ut = get_bsdthread_info(servicer); + if (ut->uu_kqueue_qos_index != new_qos) { + if (ut->uu_kqueue_qos_index == THREAD_QOS_UNSPECIFIED) { + thread_add_ipc_override(servicer, new_qos); + } else if (new_qos == THREAD_QOS_UNSPECIFIED) { + thread_drop_ipc_override(servicer); + } else /* ut->uu_kqueue_qos_index != new_qos */ { + thread_update_ipc_override(servicer, new_qos); + } + ut->uu_kqueue_qos_index = new_qos; + qos_changed = TRUE; + } - /* bind the thread */ - kqworkq_bind_thread(kqwq, qos_index, wqthread, flags); + if (new_ipc_override_is_sync != ut->uu_kqueue_override_is_sync) { + if (new_ipc_override_is_sync && + !ut->uu_kqueue_override_is_sync) { + thread_add_sync_ipc_override(servicer); + } else if (!new_ipc_override_is_sync && + ut->uu_kqueue_override_is_sync) { + thread_drop_sync_ipc_override(servicer); + } + ut->uu_kqueue_override_is_sync = new_ipc_override_is_sync; + qos_changed = TRUE; + } + } else if (old_qos != new_qos) { + assert(new_qos); + kqworkloop_threadreq_modify(kqwl, new_qos); + qos_changed = TRUE; + } + if (qos_changed) { + servicer = kqr->kqr_thread; + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KQWL_THADJUST), + kqwl->kqwl_dynamicid, + (kqr->kqr_state & KQR_BOUND) ? thread_tid(servicer) : 0, + (kqr->kqr_qos_index << 16) | (new_qos << 8) | new_ipc_override_is_sync, + (kqr->kqr_override_index << 8) | kqr->kqr_state); + } } } -/* - * If we aren't already busy processing events [for this QoS], - * request workq thread support as appropriate. - * - * TBD - for now, we don't segregate out processing by QoS. - * - * - May be called with the kqueue's wait queue set locked, - * so cannot do anything that could recurse on that. - */ static void -kqworkq_request_help( - struct kqworkq *kqwq, - kq_index_t qos_index, - uint32_t type) +kqworkloop_request_help(struct kqworkloop *kqwl, kq_index_t qos_index) { - struct kqrequest *kqr; - /* convert to thread qos value */ - assert(qos_index < KQWQ_NQOS); - - kqwq_req_lock(kqwq); - kqr = kqworkq_get_request(kqwq, qos_index); - - /* - * If someone is processing the queue, just mark what type - * of attempt this was (from a kq wakeup or from a waitq hook). - * They'll be noticed at the end of servicing and a new thread - * will be requested at that point. - */ - if (kqr->kqr_state & KQWQ_PROCESSING) { - kqr->kqr_state |= type; - kqwq_req_unlock(kqwq); - return; - } + assert(qos_index < KQWL_NBUCKETS); - kqworkq_request_thread(kqwq, qos_index); - kqwq_req_unlock(kqwq); + kqwl_req_lock(kqwl); + kqworkloop_update_threads_qos(kqwl, KQWL_UTQ_UPDATE_WAKEUP_QOS, qos_index); + kqwl_req_unlock(kqwl); } /* * These arrays described the low and high qindexes for a given qos_index. * The values come from the chart in <sys/eventvar.h> (must stay in sync). */ -static kq_index_t _kq_base_index[KQWQ_NQOS] = {0, 0, 6, 11, 15, 18, 20, 21}; -static kq_index_t _kq_high_index[KQWQ_NQOS] = {0, 5, 10, 14, 17, 19, 20, 21}; +static kq_index_t _kqwq_base_index[KQWQ_NQOS] = {0, 0, 6, 11, 15, 18, 20, 21}; +static kq_index_t _kqwq_high_index[KQWQ_NQOS] = {0, 5, 10, 14, 17, 19, 20, 21}; static struct kqtailq * kqueue_get_base_queue(struct kqueue *kq, kq_index_t qos_index) { - assert(qos_index < KQWQ_NQOS); - return &kq->kq_queue[_kq_base_index[qos_index]]; + if (kq->kq_state & KQ_WORKQ) { + assert(qos_index < KQWQ_NQOS); + return &kq->kq_queue[_kqwq_base_index[qos_index]]; + } else if (kq->kq_state & KQ_WORKLOOP) { + assert(qos_index < KQWL_NBUCKETS); + return &kq->kq_queue[qos_index]; + } else { + assert(qos_index == QOS_INDEX_KQFILE); + return &kq->kq_queue[QOS_INDEX_KQFILE]; + } } static struct kqtailq * kqueue_get_high_queue(struct kqueue *kq, kq_index_t qos_index) { - assert(qos_index < KQWQ_NQOS); - return &kq->kq_queue[_kq_high_index[qos_index]]; + if (kq->kq_state & KQ_WORKQ) { + assert(qos_index < KQWQ_NQOS); + return &kq->kq_queue[_kqwq_high_index[qos_index]]; + } else if (kq->kq_state & KQ_WORKLOOP) { + assert(qos_index < KQWL_NBUCKETS); + return &kq->kq_queue[KQWL_BUCKET_STAYACTIVE]; + } else { + assert(qos_index == QOS_INDEX_KQFILE); + return &kq->kq_queue[QOS_INDEX_KQFILE]; + } } static int @@ -4044,16 +7053,24 @@ kqueue_queue_empty(struct kqueue *kq, kq_index_t qos_index) static struct kqtailq * kqueue_get_suppressed_queue(struct kqueue *kq, kq_index_t qos_index) { + struct kqtailq *res; + struct kqrequest *kqr; + if (kq->kq_state & KQ_WORKQ) { struct kqworkq *kqwq = (struct kqworkq *)kq; - struct kqrequest *kqr; kqr = kqworkq_get_request(kqwq, qos_index); - return &kqr->kqr_suppressed; + res = &kqr->kqr_suppressed; + } else if (kq->kq_state & KQ_WORKLOOP) { + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + + kqr = &kqwl->kqwl_request; + res = &kqr->kqr_suppressed; } else { struct kqfile *kqf = (struct kqfile *)kq; - return &kqf->kqf_suppressed; + res = &kqf->kqf_suppressed; } + return res; } static kq_index_t @@ -4064,15 +7081,19 @@ knote_get_queue_index(struct knote *kn) struct kqueue *kq = knote_get_kq(kn); kq_index_t res; - if ((kq->kq_state & KQ_WORKQ) == 0) { - assert(qos_index == 0); - assert(override_index == 0); + if (kq->kq_state & KQ_WORKQ) { + res = _kqwq_base_index[qos_index]; + if (override_index > qos_index) + res += override_index - qos_index; + assert(res <= _kqwq_high_index[qos_index]); + } else if (kq->kq_state & KQ_WORKLOOP) { + res = MAX(override_index, qos_index); + assert(res < KQWL_NBUCKETS); + } else { + assert(qos_index == QOS_INDEX_KQFILE); + assert(override_index == QOS_INDEX_KQFILE); + res = QOS_INDEX_KQFILE; } - res = _kq_base_index[qos_index]; - if (override_index > qos_index) - res += override_index - qos_index; - - assert(res <= _kq_high_index[qos_index]); return res; } @@ -4084,15 +7105,6 @@ knote_get_queue(struct knote *kn) return &(knote_get_kq(kn))->kq_queue[qindex]; } -static struct kqtailq * -knote_get_suppressed_queue(struct knote *kn) -{ - kq_index_t qos_index = knote_get_qos_index(kn); - struct kqueue *kq = knote_get_kq(kn); - - return kqueue_get_suppressed_queue(kq, qos_index); -} - static kq_index_t knote_get_req_index(struct knote *kn) { @@ -4113,10 +7125,14 @@ knote_set_qos_index(struct knote *kn, kq_index_t qos_index) assert(qos_index < KQWQ_NQOS); assert((kn->kn_status & KN_QUEUED) == 0); - if (kq->kq_state & KQ_WORKQ) - assert(qos_index > QOS_INDEX_KQFILE); - else - assert(qos_index == QOS_INDEX_KQFILE); + if (kq->kq_state & KQ_WORKQ) { + assert(qos_index > THREAD_QOS_UNSPECIFIED); + } else if (kq->kq_state & KQ_WORKLOOP) { + /* XXX this policy decision shouldn't be here */ + if (qos_index == THREAD_QOS_UNSPECIFIED) + qos_index = THREAD_QOS_LEGACY; + } else + qos_index = QOS_INDEX_KQFILE; /* always set requested */ kn->kn_req_index = qos_index; @@ -4126,6 +7142,35 @@ knote_set_qos_index(struct knote *kn, kq_index_t qos_index) kn->kn_qos_index = qos_index; } +static void +knote_set_qos_overcommit(struct knote *kn) +{ + struct kqueue *kq = knote_get_kq(kn); + struct kqrequest *kqr; + + /* turn overcommit on for the appropriate thread request? */ + if (kn->kn_qos & _PTHREAD_PRIORITY_OVERCOMMIT_FLAG) { + if (kq->kq_state & KQ_WORKQ) { + kq_index_t qos_index = knote_get_qos_index(kn); + struct kqworkq *kqwq = (struct kqworkq *)kq; + + kqr = kqworkq_get_request(kqwq, qos_index); + + kqwq_req_lock(kqwq); + kqr->kqr_state |= KQR_THOVERCOMMIT; + kqwq_req_unlock(kqwq); + } else if (kq->kq_state & KQ_WORKLOOP) { + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + + kqr = &kqwl->kqwl_request; + + kqwl_req_lock(kqwl); + kqr->kqr_state |= KQR_THOVERCOMMIT; + kqwl_req_unlock(kqwl); + } + } +} + static kq_index_t knote_get_qos_override_index(struct knote *kn) { @@ -4133,58 +7178,88 @@ knote_get_qos_override_index(struct knote *kn) } static void -knote_set_qos_override_index(struct knote *kn, kq_index_t override_index) +knote_set_qos_override_index(struct knote *kn, kq_index_t override_index, + boolean_t override_is_sync) { struct kqueue *kq = knote_get_kq(kn); kq_index_t qos_index = knote_get_qos_index(kn); + kq_index_t old_override_index = knote_get_qos_override_index(kn); + boolean_t old_override_is_sync = kn->kn_qos_override_is_sync; + uint32_t flags = 0; assert((kn->kn_status & KN_QUEUED) == 0); - if (override_index == KQWQ_QOS_MANAGER) + if (override_index == KQWQ_QOS_MANAGER) { assert(qos_index == KQWQ_QOS_MANAGER); - else + } else { assert(override_index < KQWQ_QOS_MANAGER); + } kn->kn_qos_override = override_index; + kn->kn_qos_override_is_sync = override_is_sync; - /* - * If this is a workq kqueue, apply the override to the - * workq servicing thread. + /* + * If this is a workq/workloop kqueue, apply the override to the + * servicing thread. */ if (kq->kq_state & KQ_WORKQ) { struct kqworkq *kqwq = (struct kqworkq *)kq; - assert(qos_index > QOS_INDEX_KQFILE); + assert(qos_index > THREAD_QOS_UNSPECIFIED); kqworkq_update_override(kqwq, qos_index, override_index); + } else if (kq->kq_state & KQ_WORKLOOP) { + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + + if ((kn->kn_status & KN_SUPPRESSED) == KN_SUPPRESSED) { + flags = flags | KQWL_UO_UPDATE_SUPPRESS_SYNC_COUNTERS; + + if (override_index == THREAD_QOS_USER_INTERACTIVE + && override_is_sync) { + flags = flags | KQWL_UO_NEW_OVERRIDE_IS_SYNC_UI; + } + + if (old_override_index == THREAD_QOS_USER_INTERACTIVE + && old_override_is_sync) { + flags = flags | KQWL_UO_OLD_OVERRIDE_IS_SYNC_UI; + } + } + + assert(qos_index > THREAD_QOS_UNSPECIFIED); + kqworkloop_update_override(kqwl, qos_index, override_index, flags); } } +static kq_index_t +knote_get_sync_qos_override_index(struct knote *kn) +{ + return kn->kn_qos_sync_override; +} + static void kqworkq_update_override(struct kqworkq *kqwq, kq_index_t qos_index, kq_index_t override_index) { struct kqrequest *kqr; - kq_index_t new_delta; - kq_index_t old_delta; + kq_index_t old_override_index; - new_delta = (override_index > qos_index) ? - override_index - qos_index : 0; + if (override_index <= qos_index) { + return; + } kqr = kqworkq_get_request(kqwq, qos_index); kqwq_req_lock(kqwq); - old_delta = kqr->kqr_override_delta; - - if (new_delta > old_delta) { - thread_t wqthread = kqr->kqr_thread; - - /* store the new override delta */ - kqr->kqr_override_delta = new_delta; + old_override_index = kqr->kqr_override_index; + if (override_index > MAX(kqr->kqr_qos_index, old_override_index)) { + kqr->kqr_override_index = override_index; /* apply the override to [incoming?] servicing thread */ - if (wqthread) { + if (kqr->kqr_state & KQR_BOUND) { + thread_t wqthread = kqr->kqr_thread; + /* only apply if non-manager */ + assert(wqthread); if ((kqr->kqr_state & KQWQ_THMANAGER) == 0) { - if (old_delta) + if (old_override_index) thread_update_ipc_override(wqthread, override_index); else thread_add_ipc_override(wqthread, override_index); @@ -4194,70 +7269,212 @@ kqworkq_update_override(struct kqworkq *kqwq, kq_index_t qos_index, kq_index_t o kqwq_req_unlock(kqwq); } -/* called with the kqworkq lock held */ +/* called with the kqworkq lock held */ +static void +kqworkq_bind_thread_impl( + struct kqworkq *kqwq, + kq_index_t qos_index, + thread_t thread, + unsigned int flags) +{ + /* request lock must be held */ + kqwq_req_held(kqwq); + + struct kqrequest *kqr = kqworkq_get_request(kqwq, qos_index); + assert(kqr->kqr_state & KQR_THREQUESTED); + + if (qos_index == KQWQ_QOS_MANAGER) + flags |= KEVENT_FLAG_WORKQ_MANAGER; + + struct uthread *ut = get_bsdthread_info(thread); + + /* + * If this is a manager, and the manager request bit is + * not set, assure no other thread is bound. If the bit + * is set, make sure the old thread is us (or not set). + */ + if (flags & KEVENT_FLAG_WORKQ_MANAGER) { + if ((kqr->kqr_state & KQR_BOUND) == 0) { + kqr->kqr_state |= (KQR_BOUND | KQWQ_THMANAGER); + TAILQ_INIT(&kqr->kqr_suppressed); + kqr->kqr_thread = thread; + ut->uu_kqueue_bound = (struct kqueue *)kqwq; + ut->uu_kqueue_qos_index = KQWQ_QOS_MANAGER; + ut->uu_kqueue_flags = (KEVENT_FLAG_WORKQ | + KEVENT_FLAG_WORKQ_MANAGER); + } else { + assert(kqr->kqr_state & KQR_BOUND); + assert(thread == kqr->kqr_thread); + assert(ut->uu_kqueue_bound == (struct kqueue *)kqwq); + assert(ut->uu_kqueue_qos_index == KQWQ_QOS_MANAGER); + assert(ut->uu_kqueue_flags & KEVENT_FLAG_WORKQ_MANAGER); + } + return; + } + + /* Just a normal one-queue servicing thread */ + assert(kqr->kqr_state & KQR_THREQUESTED); + assert(kqr->kqr_qos_index == qos_index); + + if ((kqr->kqr_state & KQR_BOUND) == 0) { + kqr->kqr_state |= KQR_BOUND; + TAILQ_INIT(&kqr->kqr_suppressed); + kqr->kqr_thread = thread; + + /* apply an ipc QoS override if one is needed */ + if (kqr->kqr_override_index) { + assert(kqr->kqr_qos_index); + assert(kqr->kqr_override_index > kqr->kqr_qos_index); + assert(thread_get_ipc_override(thread) == THREAD_QOS_UNSPECIFIED); + thread_add_ipc_override(thread, kqr->kqr_override_index); + } + + /* indicate that we are processing in the uthread */ + ut->uu_kqueue_bound = (struct kqueue *)kqwq; + ut->uu_kqueue_qos_index = qos_index; + ut->uu_kqueue_flags = flags; + } else { + /* + * probably syncronously bound AND post-request bound + * this logic can go away when we get rid of post-request bind + */ + assert(kqr->kqr_state & KQR_BOUND); + assert(thread == kqr->kqr_thread); + assert(ut->uu_kqueue_bound == (struct kqueue *)kqwq); + assert(ut->uu_kqueue_qos_index == qos_index); + assert((ut->uu_kqueue_flags & flags) == flags); + } +} + +static void +kqworkloop_update_override( + struct kqworkloop *kqwl, + kq_index_t qos_index, + kq_index_t override_index, + uint32_t flags) +{ + struct kqrequest *kqr = &kqwl->kqwl_request; + + kqwl_req_lock(kqwl); + + /* Do not override on attached threads */ + if (kqr->kqr_state & KQR_BOUND) { + assert(kqr->kqr_thread); + + if (kqwl->kqwl_kqueue.kq_state & KQ_NO_WQ_THREAD) { + kqwl_req_unlock(kqwl); + assert(!is_workqueue_thread(kqr->kqr_thread)); + return; + } + } + + /* Update sync ipc counts on kqr for suppressed knotes */ + if (flags & KQWL_UO_UPDATE_SUPPRESS_SYNC_COUNTERS) { + kqworkloop_update_suppress_sync_count(kqr, flags); + } + + if ((flags & KQWL_UO_UPDATE_OVERRIDE_LAZY) == 0) { + kqworkloop_update_threads_qos(kqwl, KQWL_UTQ_UPDATE_WAKEUP_OVERRIDE, + MAX(qos_index, override_index)); + } + kqwl_req_unlock(kqwl); +} + +static void +kqworkloop_update_suppress_sync_count( + struct kqrequest *kqr, + uint32_t flags) +{ + if (flags & KQWL_UO_NEW_OVERRIDE_IS_SYNC_UI) { + kqr->kqr_sync_suppress_count++; + } + + if (flags & KQWL_UO_OLD_OVERRIDE_IS_SYNC_UI) { + assert(kqr->kqr_sync_suppress_count > 0); + kqr->kqr_sync_suppress_count--; + } +} + +/* + * kqworkloop_unbind_thread - Unbind the servicer thread of a workloop kqueue + * + * It will end the processing phase in case it was still processing: + * + * We may have to request a new thread for not KQ_NO_WQ_THREAD workloop. + * This can happen if : + * - there were active events at or above our QoS we never got to (count > 0) + * - we pended waitq hook callouts during processing + * - we pended wakeups while processing (or unsuppressing) + * + * Called with kqueue lock held. + */ + static void -kqworkq_bind_thread( - struct kqworkq *kqwq, - kq_index_t qos_index, +kqworkloop_unbind_thread( + struct kqworkloop *kqwl, thread_t thread, - unsigned int flags) + __unused unsigned int flags) { - struct kqrequest *kqr = kqworkq_get_request(kqwq, qos_index); - thread_t old_thread = kqr->kqr_thread; - struct uthread *ut; + struct kqueue *kq = &kqwl->kqwl_kqueue; + struct kqrequest *kqr = &kqwl->kqwl_request; - assert(kqr->kqr_state & KQWQ_THREQUESTED); + kqlock_held(kq); - /* If no identity yet, just set flags as needed */ - if (thread == THREAD_NULL) { - assert(old_thread == THREAD_NULL); - - /* emergency or unindetified */ - if (flags & KEVENT_FLAG_WORKQ_MANAGER) { - assert((kqr->kqr_state & KQWQ_THMANAGER) == 0); - kqr->kqr_state |= KQWQ_THMANAGER; - } + assert((kq->kq_state & KQ_PROCESSING) == 0); + if (kq->kq_state & KQ_PROCESSING) { return; } - /* Known thread identity */ - ut = get_bsdthread_info(thread); - - /* - * If this is a manager, and the manager request bit is - * not set, assure no other thread is bound. If the bit - * is set, make sure the old thread is us (or not set). + /* + * Forcing the KQ_PROCESSING flag allows for QoS updates because of + * unsuppressing knotes not to be applied until the eventual call to + * kqworkloop_update_threads_qos() below. */ - if (flags & KEVENT_FLAG_WORKQ_MANAGER) { - if ((kqr->kqr_state & KQWQ_THMANAGER) == 0) { - assert(old_thread == THREAD_NULL); - kqr->kqr_state |= KQWQ_THMANAGER; - } else if (old_thread == THREAD_NULL) { - kqr->kqr_thread = thread; - ut->uu_kqueue_bound = KQWQ_QOS_MANAGER; - ut->uu_kqueue_flags = (KEVENT_FLAG_WORKQ | - KEVENT_FLAG_WORKQ_MANAGER); - } else { - assert(thread == old_thread); - assert(ut->uu_kqueue_bound == KQWQ_QOS_MANAGER); - assert(ut->uu_kqueue_flags & KEVENT_FLAG_WORKQ_MANAGER); - } + kq->kq_state |= KQ_PROCESSING; + kqworkloop_acknowledge_events(kqwl, TRUE); + kq->kq_state &= ~KQ_PROCESSING; + + kqwl_req_lock(kqwl); + + /* deal with extraneous unbinds in release kernels */ + assert((kqr->kqr_state & (KQR_BOUND | KQR_PROCESSING)) == KQR_BOUND); + if ((kqr->kqr_state & (KQR_BOUND | KQR_PROCESSING)) != KQR_BOUND) { + kqwl_req_unlock(kqwl); return; } - /* Just a normal one-queue servicing thread */ - assert(old_thread == THREAD_NULL); - assert((kqr->kqr_state & KQWQ_THMANAGER) == 0); + assert(thread == current_thread()); + assert(kqr->kqr_thread == thread); + if (kqr->kqr_thread != thread) { + kqwl_req_unlock(kqwl); + return; + } + + struct uthread *ut = get_bsdthread_info(thread); + kq_index_t old_qos_index = ut->uu_kqueue_qos_index; + boolean_t ipc_override_is_sync = ut->uu_kqueue_override_is_sync; + ut->uu_kqueue_bound = NULL; + ut->uu_kqueue_qos_index = 0; + ut->uu_kqueue_override_is_sync = 0; + ut->uu_kqueue_flags = 0; - kqr->kqr_thread = thread; - - /* apply an ipc QoS override if one is needed */ - if (kqr->kqr_override_delta) - thread_add_ipc_override(thread, qos_index + kqr->kqr_override_delta); + /* unbind the servicer thread, drop overrides */ + kqr->kqr_thread = NULL; + kqr->kqr_state &= ~(KQR_BOUND | KQR_THREQUESTED | KQR_R2K_NOTIF_ARMED); + kqworkloop_update_threads_qos(kqwl, KQWL_UTQ_RECOMPUTE_WAKEUP_QOS, 0); - /* indicate that we are processing in the uthread */ - ut->uu_kqueue_bound = qos_index; - ut->uu_kqueue_flags = flags; + kqwl_req_unlock(kqwl); + + /* + * Drop the override on the current thread last, after the call to + * kqworkloop_update_threads_qos above. + */ + if (old_qos_index) { + thread_drop_ipc_override(thread); + } + if (ipc_override_is_sync) { + thread_drop_sync_ipc_override(thread); + } } /* called with the kqworkq lock held */ @@ -4269,37 +7486,54 @@ kqworkq_unbind_thread( __unused unsigned int flags) { struct kqrequest *kqr = kqworkq_get_request(kqwq, qos_index); - kq_index_t override = 0; + kq_index_t override_index = 0; + + /* request lock must be held */ + kqwq_req_held(kqwq); assert(thread == current_thread()); + if ((kqr->kqr_state & KQR_BOUND) == 0) { + assert(kqr->kqr_state & KQR_BOUND); + return; + } + + assert(kqr->kqr_thread == thread); + assert(TAILQ_EMPTY(&kqr->kqr_suppressed)); + /* * If there is an override, drop it from the current thread * and then we are free to recompute (a potentially lower) * minimum override to apply to the next thread request. */ - if (kqr->kqr_override_delta) { + if (kqr->kqr_override_index) { struct kqtailq *base_queue = kqueue_get_base_queue(&kqwq->kqwq_kqueue, qos_index); struct kqtailq *queue = kqueue_get_high_queue(&kqwq->kqwq_kqueue, qos_index); /* if not bound to a manager thread, drop the current ipc override */ if ((kqr->kqr_state & KQWQ_THMANAGER) == 0) { - assert(thread == kqr->kqr_thread); thread_drop_ipc_override(thread); } /* recompute the new override */ do { if (!TAILQ_EMPTY(queue)) { - override = queue - base_queue; + override_index = queue - base_queue + qos_index; break; } } while (queue-- > base_queue); } - /* unbind the thread and apply the new override */ - kqr->kqr_thread = THREAD_NULL; - kqr->kqr_override_delta = override; + /* Mark it unbound */ + kqr->kqr_thread = NULL; + kqr->kqr_state &= ~(KQR_BOUND | KQR_THREQUESTED | KQWQ_THMANAGER); + + /* apply the new override */ + if (override_index > kqr->kqr_qos_index) { + kqr->kqr_override_index = override_index; + } else { + kqr->kqr_override_index = THREAD_QOS_UNSPECIFIED; + } } struct kqrequest * @@ -4310,53 +7544,124 @@ kqworkq_get_request(struct kqworkq *kqwq, kq_index_t qos_index) } void -knote_adjust_qos(struct knote *kn, qos_t new_qos, qos_t new_override) +knote_adjust_qos(struct knote *kn, qos_t new_qos, qos_t new_override, kq_index_t sync_override_index) { - if (knote_get_kq(kn)->kq_state & KQ_WORKQ) { + struct kqueue *kq = knote_get_kq(kn); + boolean_t override_is_sync = FALSE; + + if (kq->kq_state & (KQ_WORKQ | KQ_WORKLOOP)) { kq_index_t new_qos_index; kq_index_t new_override_index; kq_index_t servicer_qos_index; - new_qos_index = qos_index_from_qos(new_qos, FALSE); - new_override_index = qos_index_from_qos(new_override, TRUE); + new_qos_index = qos_index_from_qos(kn, new_qos, FALSE); + new_override_index = qos_index_from_qos(kn, new_override, TRUE); /* make sure the servicer qos acts as a floor */ - servicer_qos_index = qos_index_from_qos(kn->kn_qos, FALSE); + servicer_qos_index = qos_index_from_qos(kn, kn->kn_qos, FALSE); if (servicer_qos_index > new_qos_index) new_qos_index = servicer_qos_index; if (servicer_qos_index > new_override_index) new_override_index = servicer_qos_index; + if (sync_override_index >= new_override_index) { + new_override_index = sync_override_index; + override_is_sync = TRUE; + } - kqlock(knote_get_kq(kn)); + kqlock(kq); if (new_qos_index != knote_get_req_index(kn) || - new_override_index != knote_get_qos_override_index(kn)) { + new_override_index != knote_get_qos_override_index(kn) || + override_is_sync != kn->kn_qos_override_is_sync) { if (kn->kn_status & KN_QUEUED) { knote_dequeue(kn); knote_set_qos_index(kn, new_qos_index); - knote_set_qos_override_index(kn, new_override_index); + knote_set_qos_override_index(kn, new_override_index, override_is_sync); knote_enqueue(kn); knote_wakeup(kn); } else { knote_set_qos_index(kn, new_qos_index); - knote_set_qos_override_index(kn, new_override_index); + knote_set_qos_override_index(kn, new_override_index, override_is_sync); + } + } + kqunlock(kq); + } +} + +void +knote_adjust_sync_qos(struct knote *kn, kq_index_t sync_qos, boolean_t lock_kq) +{ + struct kqueue *kq = knote_get_kq(kn); + kq_index_t old_sync_override; + kq_index_t qos_index = knote_get_qos_index(kn); + uint32_t flags = 0; + + /* Tracking only happens for UI qos */ + if (sync_qos != THREAD_QOS_USER_INTERACTIVE && + sync_qos != THREAD_QOS_UNSPECIFIED) { + return; + } + + if (lock_kq) + kqlock(kq); + + if (kq->kq_state & KQ_WORKLOOP) { + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + + old_sync_override = knote_get_sync_qos_override_index(kn); + if (old_sync_override != sync_qos) { + kn->kn_qos_sync_override = sync_qos; + + /* update sync ipc counters for suppressed knotes */ + if ((kn->kn_status & KN_SUPPRESSED) == KN_SUPPRESSED) { + flags = flags | KQWL_UO_UPDATE_SUPPRESS_SYNC_COUNTERS; + + /* Do not recalculate kqwl override, it would be done later */ + flags = flags | KQWL_UO_UPDATE_OVERRIDE_LAZY; + + if (sync_qos == THREAD_QOS_USER_INTERACTIVE) { + flags = flags | KQWL_UO_NEW_OVERRIDE_IS_SYNC_UI; + } + + if (old_sync_override == THREAD_QOS_USER_INTERACTIVE) { + flags = flags | KQWL_UO_OLD_OVERRIDE_IS_SYNC_UI; + } + + kqworkloop_update_override(kqwl, qos_index, sync_qos, + flags); } + } - kqunlock(knote_get_kq(kn)); } + if (lock_kq) + kqunlock(kq); } static void knote_wakeup(struct knote *kn) { struct kqueue *kq = knote_get_kq(kn); + kq_index_t qos_index = knote_get_qos_index(kn); + + kqlock_held(kq); if (kq->kq_state & KQ_WORKQ) { /* request a servicing thread */ struct kqworkq *kqwq = (struct kqworkq *)kq; - kq_index_t qos_index = knote_get_qos_index(kn); - kqworkq_request_help(kqwq, qos_index, KQWQ_WAKEUP); + kqworkq_request_help(kqwq, qos_index); + + } else if (kq->kq_state & KQ_WORKLOOP) { + /* request a servicing thread */ + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + if (kqworkloop_is_processing_on_current_thread(kqwl)) { + /* + * kqworkloop_end_processing() will perform the required QoS + * computations when it unsets the processing mode. + */ + return; + } + kqworkloop_request_help(kqwl, qos_index); } else { struct kqfile *kqf = (struct kqfile *)kq; @@ -4377,7 +7682,7 @@ knote_wakeup(struct knote *kn) KNOTE(&kqf->kqf_sel.si_note, 0); } } - + /* * Called with the kqueue locked */ @@ -4426,10 +7731,18 @@ waitq_set__CALLING_PREPOST_HOOK__(void *kq_hook, void *knote_hook, int qos) { #pragma unused(knote_hook, qos) - struct kqworkq *kqwq = (struct kqworkq *)kq_hook; + struct kqueue *kq = (struct kqueue *)kq_hook; - assert(kqwq->kqwq_state & KQ_WORKQ); - kqworkq_request_help(kqwq, KQWQ_QOS_MANAGER, KQWQ_HOOKCALLED); + if (kq->kq_state & KQ_WORKQ) { + struct kqworkq *kqwq = (struct kqworkq *)kq; + + kqworkq_request_help(kqwq, KQWQ_QOS_MANAGER); + + } else if (kq->kq_state & KQ_WORKLOOP) { + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + + kqworkloop_request_help(kqwl, KQWL_BUCKET_STAYACTIVE); + } } void @@ -4461,15 +7774,17 @@ knote(struct klist *list, long hint) kqlock(kq); + assert(!knoteuse_needs_boost(kn, NULL)); + /* If we can get a use reference - deliver event */ - if (kqlock2knoteuse(kq, kn)) { + if (kqlock2knoteuse(kq, kn, KNUSE_NONE)) { int result; /* call the event with only a use count */ result = knote_fops(kn)->f_event(kn, hint); /* if its not going away and triggered */ - if (knoteuse2kqlock(kq, kn, 0) && result) + if (knoteuse2kqlock(kq, kn, KNUSE_NONE) && result) knote_activate(kn); /* kq lock held */ } @@ -4526,19 +7841,21 @@ knote_vanish(struct klist *list) int result; kqlock(kq); - if ((kn->kn_status & KN_DROPPING) == 0) { + assert(!knoteuse_needs_boost(kn, NULL)); + + if ((kn->kn_status & KN_DROPPING) == 0) { /* If EV_VANISH supported - prepare to deliver one */ if (kn->kn_status & KN_REQVANISH) { kn->kn_status |= KN_VANISHED; knote_activate(kn); - } else if (kqlock2knoteuse(kq, kn)) { + } else if (kqlock2knoteuse(kq, kn, KNUSE_NONE)) { /* call the event with only a use count */ result = knote_fops(kn)->f_event(kn, NOTE_REVOKE); - + /* if its not going away and triggered */ - if (knoteuse2kqlock(kq, kn, 0) && result) + if (knoteuse2kqlock(kq, kn, KNUSE_NONE) && result) knote_activate(kn); /* lock held again */ } @@ -4629,16 +7946,17 @@ restart: if ((kn->kn_status & KN_VANISHED) == 0) { proc_fdunlock(p); - /* get detach reference (also marks vanished) */ - if (kqlock2knotedetach(kq, kn)) { + assert(!knoteuse_needs_boost(kn, NULL)); + /* get detach reference (also marks vanished) */ + if (kqlock2knotedetach(kq, kn, KNUSE_NONE)) { /* detach knote and drop fp use reference */ knote_fops(kn)->f_detach(kn); if (knote_fops(kn)->f_isfd) fp_drop(p, kn->kn_id, kn->kn_fp, 0); /* activate it if it's still in existence */ - if (knoteuse2kqlock(kq, kn, 0)) { + if (knoteuse2kqlock(kq, kn, KNUSE_NONE)) { knote_activate(kn); } kqunlock(kq); @@ -4670,7 +7988,66 @@ restart: } /* - * knote_fdadd - Add knote to the fd table for process + * knote_fdfind - lookup a knote in the fd table for process + * + * If the filter is file-based, lookup based on fd index. + * Otherwise use a hash based on the ident. + * + * Matching is based on kq, filter, and ident. Optionally, + * it may also be based on the udata field in the kevent - + * allowing multiple event registration for the file object + * per kqueue. + * + * fd_knhashlock or fdlock held on entry (and exit) + */ +static struct knote * +knote_fdfind(struct kqueue *kq, + struct kevent_internal_s *kev, + bool is_fd, + struct proc *p) +{ + struct filedesc *fdp = p->p_fd; + struct klist *list = NULL; + struct knote *kn = NULL; + + /* + * determine where to look for the knote + */ + if (is_fd) { + /* fd-based knotes are linked off the fd table */ + if (kev->ident < (u_int)fdp->fd_knlistsize) { + list = &fdp->fd_knlist[kev->ident]; + } + } else if (fdp->fd_knhashmask != 0) { + /* hash non-fd knotes here too */ + list = &fdp->fd_knhash[KN_HASH((u_long)kev->ident, fdp->fd_knhashmask)]; + } + + /* + * scan the selected list looking for a match + */ + if (list != NULL) { + SLIST_FOREACH(kn, list, kn_link) { + if (kq == knote_get_kq(kn) && + kev->ident == kn->kn_id && + kev->filter == kn->kn_filter) { + if (kev->flags & EV_UDATA_SPECIFIC) { + if ((kn->kn_status & KN_UDATA_SPECIFIC) && + kev->udata == kn->kn_udata) { + break; /* matching udata-specific knote */ + } + } else if ((kn->kn_status & KN_UDATA_SPECIFIC) == 0) { + break; /* matching non-udata-specific knote */ + } + } + } + } + return kn; +} + +/* + * kq_add_knote- Add knote to the fd table for process + * while checking for duplicates. * * All file-based filters associate a list of knotes by file * descriptor index. All other filters hash the knote by ident. @@ -4678,39 +8055,79 @@ restart: * May have to grow the table of knote lists to cover the * file descriptor index presented. * - * proc_fdlock held on entry (and exit) + * fd_knhashlock and fdlock unheld on entry (and exit). + * + * Takes a rwlock boost if inserting the knote is successful. */ static int -knote_fdadd(struct knote *kn, struct proc *p) +kq_add_knote(struct kqueue *kq, struct knote *kn, + struct kevent_internal_s *kev, + struct proc *p, int *knoteuse_flags) { struct filedesc *fdp = p->p_fd; struct klist *list = NULL; + int ret = 0; + bool is_fd = knote_fops(kn)->f_isfd; + + if (is_fd) + proc_fdlock(p); + else + knhash_lock(p); + + if (knote_fdfind(kq, kev, is_fd, p) != NULL) { + /* found an existing knote: we can't add this one */ + ret = ERESTART; + goto out_locked; + } + + /* knote was not found: add it now */ + if (!is_fd) { + if (fdp->fd_knhashmask == 0) { + u_long size = 0; + + list = hashinit(CONFIG_KN_HASHSIZE, M_KQUEUE, + &size); + if (list == NULL) { + ret = ENOMEM; + goto out_locked; + } + + fdp->fd_knhash = list; + fdp->fd_knhashmask = size; + } - if (! knote_fops(kn)->f_isfd) { - if (fdp->fd_knhashmask == 0) - fdp->fd_knhash = hashinit(CONFIG_KN_HASHSIZE, M_KQUEUE, - &fdp->fd_knhashmask); list = &fdp->fd_knhash[KN_HASH(kn->kn_id, fdp->fd_knhashmask)]; + SLIST_INSERT_HEAD(list, kn, kn_link); + ret = 0; + goto out_locked; + } else { + /* knote is fd based */ + if ((u_int)fdp->fd_knlistsize <= kn->kn_id) { u_int size = 0; if (kn->kn_id >= (uint64_t)p->p_rlimit[RLIMIT_NOFILE].rlim_cur - || kn->kn_id >= (uint64_t)maxfiles) - return (EINVAL); - + || kn->kn_id >= (uint64_t)maxfiles) { + ret = EINVAL; + goto out_locked; + } /* have to grow the fd_knlist */ size = fdp->fd_knlistsize; while (size <= kn->kn_id) size += KQEXTENT; - if (size >= (UINT_MAX/sizeof(struct klist *))) - return (EINVAL); + if (size >= (UINT_MAX/sizeof(struct klist *))) { + ret = EINVAL; + goto out_locked; + } MALLOC(list, struct klist *, size * sizeof(struct klist *), M_KQUEUE, M_WAITOK); - if (list == NULL) - return (ENOMEM); + if (list == NULL) { + ret = ENOMEM; + goto out_locked; + } bcopy((caddr_t)fdp->fd_knlist, (caddr_t)list, fdp->fd_knlistsize * sizeof(struct klist *)); @@ -4721,95 +8138,106 @@ knote_fdadd(struct knote *kn, struct proc *p) fdp->fd_knlist = list; fdp->fd_knlistsize = size; } + list = &fdp->fd_knlist[kn->kn_id]; + SLIST_INSERT_HEAD(list, kn, kn_link); + ret = 0; + goto out_locked; + } - SLIST_INSERT_HEAD(list, kn, kn_link); - return (0); + +out_locked: + if (ret == 0 && knoteuse_needs_boost(kn, kev)) { + set_thread_rwlock_boost(); + *knoteuse_flags = KNUSE_BOOST; + } else { + *knoteuse_flags = KNUSE_NONE; + } + if (is_fd) + proc_fdunlock(p); + else + knhash_unlock(p); + + return ret; } -/* - * knote_fdremove - remove a knote from the fd table for process +/* + * kq_remove_knote - remove a knote from the fd table for process + * and copy kn_status an kq_state while holding kqlock and + * fd table locks. * * If the filter is file-based, remove based on fd index. * Otherwise remove from the hash based on the ident. * - * proc_fdlock held on entry (and exit) + * fd_knhashlock and fdlock unheld on entry (and exit). */ static void -knote_fdremove(struct knote *kn, struct proc *p) +kq_remove_knote(struct kqueue *kq, struct knote *kn, struct proc *p, + kn_status_t *kn_status, uint16_t *kq_state) { struct filedesc *fdp = p->p_fd; struct klist *list = NULL; + bool is_fd; + + is_fd = knote_fops(kn)->f_isfd; - if (knote_fops(kn)->f_isfd) { + if (is_fd) + proc_fdlock(p); + else + knhash_lock(p); + + if (is_fd) { assert ((u_int)fdp->fd_knlistsize > kn->kn_id); list = &fdp->fd_knlist[kn->kn_id]; } else { list = &fdp->fd_knhash[KN_HASH(kn->kn_id, fdp->fd_knhashmask)]; } SLIST_REMOVE(list, kn, knote, kn_link); + + kqlock(kq); + *kn_status = kn->kn_status; + *kq_state = kq->kq_state; + kqunlock(kq); + + if (is_fd) + proc_fdunlock(p); + else + knhash_unlock(p); } -/* - * knote_fdfind - lookup a knote in the fd table for process - * - * If the filter is file-based, lookup based on fd index. - * Otherwise use a hash based on the ident. - * - * Matching is based on kq, filter, and ident. Optionally, - * it may also be based on the udata field in the kevent - - * allowing multiple event registration for the file object - * per kqueue. +/* + * kq_find_knote_and_kq_lock - lookup a knote in the fd table for process + * and, if the knote is found, acquires the kqlock while holding the fd table lock/spinlock. * - * proc_fdlock held on entry (and exit) + * fd_knhashlock or fdlock unheld on entry (and exit) */ -static struct knote * -knote_fdfind(struct kqueue *kq, - struct kevent_internal_s *kev, - struct proc *p) -{ - struct filedesc *fdp = p->p_fd; - struct klist *list = NULL; - struct knote *kn = NULL; - struct filterops *fops; - - fops = sysfilt_ops[~kev->filter]; /* to 0-base index */ - - /* - * determine where to look for the knote - */ - if (fops->f_isfd) { - /* fd-based knotes are linked off the fd table */ - if (kev->ident < (u_int)fdp->fd_knlistsize) { - list = &fdp->fd_knlist[kev->ident]; - } - } else if (fdp->fd_knhashmask != 0) { - /* hash non-fd knotes here too */ - list = &fdp->fd_knhash[KN_HASH((u_long)kev->ident, fdp->fd_knhashmask)]; - } - /* - * scan the selected list looking for a match - */ - if (list != NULL) { - SLIST_FOREACH(kn, list, kn_link) { - if (kq == knote_get_kq(kn) && - kev->ident == kn->kn_id && - kev->filter == kn->kn_filter) { - if (kev->flags & EV_UDATA_SPECIFIC) { - if ((kn->kn_status & KN_UDATA_SPECIFIC) && - kev->udata == kn->kn_udata) { - break; /* matching udata-specific knote */ - } - } else if ((kn->kn_status & KN_UDATA_SPECIFIC) == 0) { - break; /* matching non-udata-specific knote */ - } - } - } +static struct knote * +kq_find_knote_and_kq_lock(struct kqueue *kq, + struct kevent_internal_s *kev, + bool is_fd, + struct proc *p) +{ + struct knote * ret; + + if (is_fd) + proc_fdlock(p); + else + knhash_lock(p); + + ret = knote_fdfind(kq, kev, is_fd, p); + + if (ret) { + kqlock(kq); } - return kn; -} + if (is_fd) + proc_fdunlock(p); + else + knhash_unlock(p); + + return ret; +} /* * knote_drop - disconnect and drop the knote * @@ -4829,31 +8257,38 @@ knote_drop(struct knote *kn, __unused struct proc *ctxp) { struct kqueue *kq = knote_get_kq(kn); struct proc *p = kq->kq_p; - int needswakeup; - - /* We have to have a dropping reference on the knote */ - assert(kn->kn_status & KN_DROPPING); + kn_status_t kn_status; + uint16_t kq_state; /* If we are attached, disconnect from the source first */ if (kn->kn_status & KN_ATTACHED) { knote_fops(kn)->f_detach(kn); } - proc_fdlock(p); - /* Remove the source from the appropriate hash */ - knote_fdremove(kn, p); + kq_remove_knote(kq, kn, p, &kn_status, &kq_state); - /* trade fdlock for kq lock */ - kqlock(kq); - proc_fdunlock(p); + /* + * If a kqueue_dealloc is happening in parallel for the kq + * pointed by the knote the kq could be aready deallocated + * at this point. + * Do not access the kq after the kq_remove_knote if it is + * not a KQ_DYNAMIC. + */ /* determine if anyone needs to know about the drop */ - assert((kn->kn_status & (KN_SUPPRESSED | KN_QUEUED)) == 0); - needswakeup = (kn->kn_status & KN_USEWAIT); - kqunlock(kq); + assert((kn_status & (KN_DROPPING | KN_SUPPRESSED | KN_QUEUED)) == KN_DROPPING); - if (needswakeup) + /* + * If KN_USEWAIT is set, some other thread was trying to drop the kn. + * Or it was in kqueue_dealloc, so the kqueue_dealloc did not happen + * because that thread was waiting on this wake, or it was a drop happening + * because of a kevent_register that takes a reference on the kq, and therefore + * the kq cannot be deallocated in parallel. + * + * It is safe to access kq->kq_wqs if needswakeup is set. + */ + if (kn_status & KN_USEWAIT) waitq_wakeup64_all((struct waitq *)&kq->kq_wqs, CAST_EVENT64_T(&kn->kn_status), THREAD_RESTART, @@ -4863,6 +8298,14 @@ knote_drop(struct knote *kn, __unused struct proc *ctxp) fp_drop(p, kn->kn_id, kn->kn_fp, 0); knote_free(kn); + + /* + * release reference on dynamic kq (and free if last). + * Will only be last if this is from fdfree, etc... + * because otherwise processing thread has reference. + */ + if (kq_state & KQ_DYNAMIC) + kqueue_release_last(p, kq); } /* called with kqueue lock held */ @@ -4872,6 +8315,10 @@ knote_activate(struct knote *kn) if (kn->kn_status & KN_ACTIVE) return; + KDBG_FILTERED(KEV_EVTID(BSD_KEVENT_KNOTE_ACTIVATE), + kn->kn_udata, kn->kn_status | (kn->kn_id << 32), + kn->kn_filtid); + kn->kn_status |= KN_ACTIVE; if (knote_enqueue(kn)) knote_wakeup(kn); @@ -4894,8 +8341,32 @@ knote_enable(struct knote *kn) return; kn->kn_status &= ~KN_DISABLED; - if (knote_enqueue(kn)) + + if (kn->kn_status & KN_SUPPRESSED) { + /* Clear the sync qos on the knote */ + knote_adjust_sync_qos(kn, THREAD_QOS_UNSPECIFIED, FALSE); + + /* + * it is possible for userland to have knotes registered for a given + * workloop `wl_orig` but really handled on another workloop `wl_new`. + * + * In that case, rearming will happen from the servicer thread of + * `wl_new` which if `wl_orig` is no longer being serviced, would cause + * this knote to stay suppressed forever if we only relied on + * kqworkloop_acknowledge_events to be called by `wl_orig`. + * + * However if we see the KQ_PROCESSING bit on `wl_orig` set, we can't + * unsuppress because that would mess with the processing phase of + * `wl_orig`, however it also means kqworkloop_acknowledge_events() + * will be called. + */ + struct kqueue *kq = knote_get_kq(kn); + if ((kq->kq_state & KQ_PROCESSING) == 0) { + knote_unsuppress(kn); + } + } else if (knote_enqueue(kn)) { knote_wakeup(kn); + } } /* called with kqueue lock held */ @@ -4914,14 +8385,27 @@ static void knote_suppress(struct knote *kn) { struct kqtailq *suppressq; + struct kqueue *kq = knote_get_kq(kn); + + kqlock_held(kq); if (kn->kn_status & KN_SUPPRESSED) return; knote_dequeue(kn); kn->kn_status |= KN_SUPPRESSED; - suppressq = knote_get_suppressed_queue(kn); + suppressq = kqueue_get_suppressed_queue(kq, knote_get_qos_index(kn)); TAILQ_INSERT_TAIL(suppressq, kn, kn_tqe); + + if ((kq->kq_state & KQ_WORKLOOP) && + knote_get_qos_override_index(kn) == THREAD_QOS_USER_INTERACTIVE && + kn->kn_qos_override_is_sync) { + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + /* update the sync qos override counter for suppressed knotes */ + kqworkloop_update_override(kqwl, knote_get_qos_index(kn), + knote_get_qos_override_index(kn), + (KQWL_UO_UPDATE_SUPPRESS_SYNC_COUNTERS | KQWL_UO_NEW_OVERRIDE_IS_SYNC_UI)); + } } /* called with kqueue lock held */ @@ -4929,21 +8413,75 @@ static void knote_unsuppress(struct knote *kn) { struct kqtailq *suppressq; + struct kqueue *kq = knote_get_kq(kn); + + kqlock_held(kq); if ((kn->kn_status & KN_SUPPRESSED) == 0) return; + /* Clear the sync qos on the knote */ + knote_adjust_sync_qos(kn, THREAD_QOS_UNSPECIFIED, FALSE); + kn->kn_status &= ~KN_SUPPRESSED; - suppressq = knote_get_suppressed_queue(kn); + suppressq = kqueue_get_suppressed_queue(kq, knote_get_qos_index(kn)); TAILQ_REMOVE(suppressq, kn, kn_tqe); /* udate in-use qos to equal requested qos */ kn->kn_qos_index = kn->kn_req_index; /* don't wakeup if unsuppressing just a stay-active knote */ - if (knote_enqueue(kn) && - (kn->kn_status & KN_ACTIVE)) + if (knote_enqueue(kn) && (kn->kn_status & KN_ACTIVE)) { knote_wakeup(kn); + } + + if ((kq->kq_state & KQ_WORKLOOP) && !(kq->kq_state & KQ_NO_WQ_THREAD) && + knote_get_qos_override_index(kn) == THREAD_QOS_USER_INTERACTIVE && + kn->kn_qos_override_is_sync) { + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + + /* update the sync qos override counter for suppressed knotes */ + kqworkloop_update_override(kqwl, knote_get_qos_index(kn), + knote_get_qos_override_index(kn), + (KQWL_UO_UPDATE_SUPPRESS_SYNC_COUNTERS | KQWL_UO_OLD_OVERRIDE_IS_SYNC_UI)); + } + + if (TAILQ_EMPTY(suppressq) && (kq->kq_state & KQ_WORKLOOP) && + !(kq->kq_state & KQ_NO_WQ_THREAD)) { + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + if (kqworkloop_is_processing_on_current_thread(kqwl)) { + /* + * kqworkloop_end_processing() will perform the required QoS + * computations when it unsets the processing mode. + */ + } else { + kqwl_req_lock(kqwl); + kqworkloop_update_threads_qos(kqwl, KQWL_UTQ_RESET_WAKEUP_OVERRIDE, 0); + kqwl_req_unlock(kqwl); + } + } +} + +/* called with kqueue lock held */ +static void +knote_update_sync_override_state(struct knote *kn) +{ + struct kqtailq *queue = knote_get_queue(kn); + struct kqueue *kq = knote_get_kq(kn); + + if (!(kq->kq_state & KQ_WORKLOOP) || + knote_get_queue_index(kn) != THREAD_QOS_USER_INTERACTIVE) + return; + + /* Update the sync ipc state on workloop */ + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + boolean_t sync_ipc_override = FALSE; + if (!TAILQ_EMPTY(queue)) { + struct knote *kn_head = TAILQ_FIRST(queue); + if (kn_head->kn_qos_override_is_sync) + sync_ipc_override = TRUE; + } + kqworkloop_update_sync_override_state(kqwl, sync_ipc_override); } /* called with kqueue lock held */ @@ -4958,9 +8496,16 @@ knote_enqueue(struct knote *kn) struct kqtailq *queue = knote_get_queue(kn); struct kqueue *kq = knote_get_kq(kn); - TAILQ_INSERT_TAIL(queue, kn, kn_tqe); + kqlock_held(kq); + /* insert at head for sync ipc waiters */ + if (kn->kn_qos_override_is_sync) { + TAILQ_INSERT_HEAD(queue, kn, kn_tqe); + } else { + TAILQ_INSERT_TAIL(queue, kn, kn_tqe); + } kn->kn_status |= KN_QUEUED; kq->kq_count++; + knote_update_sync_override_state(kn); return 1; } return ((kn->kn_status & KN_STAYACTIVE) != 0); @@ -4974,6 +8519,8 @@ knote_dequeue(struct knote *kn) struct kqueue *kq = knote_get_kq(kn); struct kqtailq *queue; + kqlock_held(kq); + if ((kn->kn_status & KN_QUEUED) == 0) return; @@ -4981,6 +8528,7 @@ knote_dequeue(struct knote *kn) TAILQ_REMOVE(queue, kn, kn_tqe); kn->kn_status &= ~KN_QUEUED; kq->kq_count--; + knote_update_sync_override_state(kn); } void @@ -4995,6 +8543,9 @@ knote_init(void) kqworkq_zone = zinit(sizeof(struct kqworkq), 8192*sizeof(struct kqworkq), 8192, "kqueue workq zone"); + kqworkloop_zone = zinit(sizeof(struct kqworkloop), 8192*sizeof(struct kqworkloop), + 8192, "kqueue workloop zone"); + /* allocate kq lock group attribute and group */ kq_lck_grp_attr = lck_grp_attr_alloc_init(); @@ -5016,7 +8567,7 @@ knote_init(void) } SYSINIT(knote, SI_SUB_PSEUDO, SI_ORDER_ANY, knote_init, NULL) -struct filterops * +const struct filterops * knote_fops(struct knote *kn) { return sysfilt_ops[kn->kn_filtid]; @@ -5025,7 +8576,10 @@ knote_fops(struct knote *kn) static struct knote * knote_alloc(void) { - return ((struct knote *)zalloc(knote_zone)); + struct knote *kn; + kn = ((struct knote *)zalloc(knote_zone)); + *kn = (struct knote) { .kn_qos_override = 0, .kn_qos_sync_override = 0, .kn_qos_override_is_sync = 0 }; + return kn; } static void @@ -5105,9 +8659,9 @@ SYSCTL_PROC(_net_systm_kevt, OID_AUTO, pcblist, kevt_pcblist, "S,xkevtpcb", ""); static lck_mtx_t * -event_getlock(struct socket *so, int locktype) +event_getlock(struct socket *so, int flags) { -#pragma unused(locktype) +#pragma unused(flags) struct kern_event_pcb *ev_pcb = (struct kern_event_pcb *)so->so_pcb; if (so->so_pcb != NULL) { @@ -5168,7 +8722,6 @@ event_unlock(struct socket *so, int refcount, void *lr) lr_saved = lr; if (refcount) { - VERIFY(so->so_usecount > 0); so->so_usecount--; } if (so->so_usecount < 0) { @@ -5184,7 +8737,7 @@ event_unlock(struct socket *so, int refcount, void *lr) } mutex_held = (&((struct kern_event_pcb *)so->so_pcb)->evp_mtx); - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); so->unlock_lr[so->next_unlock_lr] = lr_saved; so->next_unlock_lr = (so->next_unlock_lr+1) % SO_LCKDBG_MAX; @@ -5203,7 +8756,7 @@ event_sofreelastref(struct socket *so) { struct kern_event_pcb *ev_pcb = (struct kern_event_pcb *)so->so_pcb; - lck_mtx_assert(&(ev_pcb->evp_mtx), LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&(ev_pcb->evp_mtx), LCK_MTX_ASSERT_OWNED); so->so_pcb = NULL; @@ -5217,7 +8770,7 @@ event_sofreelastref(struct socket *so) so->so_event = sonullevent; lck_mtx_unlock(&(ev_pcb->evp_mtx)); - lck_mtx_assert(&(ev_pcb->evp_mtx), LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&(ev_pcb->evp_mtx), LCK_MTX_ASSERT_NOTOWNED); lck_rw_lock_exclusive(kev_rwlock); LIST_REMOVE(ev_pcb, evp_link); kevtstat.kes_pcbcount--; @@ -5401,7 +8954,7 @@ kev_post_msg(struct kev_msg *event_msg) return (EMSGSIZE); } - m = m_get(M_DONTWAIT, MT_DATA); + m = m_get(M_WAIT, MT_DATA); if (m == 0) { OSIncrementAtomic64((SInt64 *)&kevtstat.kes_nomem); return (ENOMEM); @@ -5459,7 +9012,7 @@ kev_post_msg(struct kev_msg *event_msg) } } - m2 = m_copym(m, 0, m->m_len, M_NOWAIT); + m2 = m_copym(m, 0, m->m_len, M_WAIT); if (m2 == 0) { OSIncrementAtomic64((SInt64 *)&kevtstat.kes_nomem); m_free(m); @@ -5680,27 +9233,84 @@ fill_kqueueinfo(struct kqueue *kq, struct kqueue_info * kinfo) else st->vst_blksize = sizeof(struct kevent); st->vst_mode = S_IFIFO; + st->vst_ino = (kq->kq_state & KQ_DYNAMIC) ? + ((struct kqworkloop *)kq)->kqwl_dynamicid : 0; /* flags exported to libproc as PROC_KQUEUE_* (sys/proc_info.h) */ -#define PROC_KQUEUE_MASK (KQ_SEL|KQ_SLEEP|KQ_KEV32|KQ_KEV64|KQ_KEV_QOS) +#define PROC_KQUEUE_MASK (KQ_SEL|KQ_SLEEP|KQ_KEV32|KQ_KEV64|KQ_KEV_QOS|KQ_WORKQ|KQ_WORKLOOP) kinfo->kq_state = kq->kq_state & PROC_KQUEUE_MASK; return (0); } +static int +fill_kqueue_dyninfo(struct kqueue *kq, struct kqueue_dyninfo *kqdi) +{ + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + struct kqrequest *kqr = &kqwl->kqwl_request; + int err; + + if ((kq->kq_state & KQ_WORKLOOP) == 0) { + return EINVAL; + } + + if ((err = fill_kqueueinfo(kq, &kqdi->kqdi_info))) { + return err; + } + + kqwl_req_lock(kqwl); + + if (kqr->kqr_thread) { + kqdi->kqdi_servicer = thread_tid(kqr->kqr_thread); + } + + if (kqwl->kqwl_owner == WL_OWNER_SUSPENDED) { + kqdi->kqdi_owner = ~0ull; + } else { + kqdi->kqdi_owner = thread_tid(kqwl->kqwl_owner); + } + + kqdi->kqdi_request_state = kqr->kqr_state; + kqdi->kqdi_async_qos = kqr->kqr_qos_index; + kqdi->kqdi_events_qos = kqr->kqr_override_index; + kqdi->kqdi_sync_waiters = kqr->kqr_dsync_waiters; + kqdi->kqdi_sync_waiter_qos = kqr->kqr_dsync_waiters_qos; + + kqwl_req_unlock(kqwl); + + return 0; +} + void knote_markstayactive(struct knote *kn) { - kqlock(knote_get_kq(kn)); + struct kqueue *kq = knote_get_kq(kn); + + kqlock(kq); kn->kn_status |= KN_STAYACTIVE; - /* handle all stayactive knotes on the manager */ - if (knote_get_kq(kn)->kq_state & KQ_WORKQ) + /* + * Making a knote stay active is a property of the knote that must be + * established before it is fully attached. + */ + assert(kn->kn_status & KN_ATTACHING); + + /* handle all stayactive knotes on the (appropriate) manager */ + if (kq->kq_state & KQ_WORKQ) { knote_set_qos_index(kn, KQWQ_QOS_MANAGER); + } else if (kq->kq_state & KQ_WORKLOOP) { + struct kqworkloop *kqwl = (struct kqworkloop *)kq; + kqwl_req_lock(kqwl); + assert(kn->kn_req_index && kn->kn_req_index < THREAD_QOS_LAST); + kqworkloop_update_threads_qos(kqwl, KQWL_UTQ_UPDATE_STAYACTIVE_QOS, + kn->kn_req_index); + kqwl_req_unlock(kqwl); + knote_set_qos_index(kn, KQWL_BUCKET_STAYACTIVE); + } knote_activate(kn); - kqunlock(knote_get_kq(kn)); + kqunlock(kq); } void @@ -5716,27 +9326,27 @@ static unsigned long kevent_extinfo_emit(struct kqueue *kq, struct knote *kn, struct kevent_extinfo *buf, unsigned long buflen, unsigned long nknotes) { - struct kevent_internal_s *kevp; for (; kn; kn = SLIST_NEXT(kn, kn_link)) { if (kq == knote_get_kq(kn)) { if (nknotes < buflen) { struct kevent_extinfo *info = &buf[nknotes]; - struct kevent_qos_s kevqos; + struct kevent_internal_s *kevp = &kn->kn_kevent; kqlock(kq); - kevp = &(kn->kn_kevent); - - bzero(&kevqos, sizeof(kevqos)); - kevqos.ident = kevp->ident; - kevqos.filter = kevp->filter; - kevqos.flags = kevp->flags; - kevqos.fflags = kevp->fflags; - kevqos.data = (int64_t) kevp->data; - kevqos.udata = kevp->udata; - kevqos.ext[0] = kevp->ext[0]; - kevqos.ext[1] = kevp->ext[1]; - - memcpy(&info->kqext_kev, &kevqos, sizeof(info->kqext_kev)); + + info->kqext_kev = (struct kevent_qos_s){ + .ident = kevp->ident, + .filter = kevp->filter, + .flags = kevp->flags, + .fflags = kevp->fflags, + .data = (int64_t)kevp->data, + .udata = kevp->udata, + .ext[0] = kevp->ext[0], + .ext[1] = kevp->ext[1], + .ext[2] = kevp->ext[2], + .ext[3] = kevp->ext[3], + .qos = kn->kn_req_index, + }; info->kqext_sdata = kn->kn_sdata; info->kqext_status = kn->kn_status; info->kqext_sfflags = kn->kn_sfflags; @@ -5752,6 +9362,142 @@ kevent_extinfo_emit(struct kqueue *kq, struct knote *kn, struct kevent_extinfo * return nknotes; } +int +kevent_copyout_proc_dynkqids(void *proc, user_addr_t ubuf, uint32_t ubufsize, + int32_t *nkqueues_out) +{ + proc_t p = (proc_t)proc; + struct filedesc *fdp = p->p_fd; + unsigned int nkqueues = 0; + unsigned long ubuflen = ubufsize / sizeof(kqueue_id_t); + size_t buflen, bufsize; + kqueue_id_t *kq_ids = NULL; + int err = 0; + + assert(p != NULL); + + if (ubuf == USER_ADDR_NULL && ubufsize != 0) { + err = EINVAL; + goto out; + } + + buflen = min(ubuflen, PROC_PIDDYNKQUEUES_MAX); + + if (ubuflen != 0) { + if (os_mul_overflow(sizeof(kqueue_id_t), buflen, &bufsize)) { + err = ERANGE; + goto out; + } + kq_ids = kalloc(bufsize); + assert(kq_ids != NULL); + } + + kqhash_lock(p); + + if (fdp->fd_kqhashmask > 0) { + for (uint32_t i = 0; i < fdp->fd_kqhashmask + 1; i++) { + struct kqworkloop *kqwl; + + SLIST_FOREACH(kqwl, &fdp->fd_kqhash[i], kqwl_hashlink) { + /* report the number of kqueues, even if they don't all fit */ + if (nkqueues < buflen) { + kq_ids[nkqueues] = kqwl->kqwl_dynamicid; + } + nkqueues++; + } + } + } + + kqhash_unlock(p); + + if (kq_ids) { + size_t copysize; + if (os_mul_overflow(sizeof(kqueue_id_t), min(ubuflen, nkqueues), ©size)) { + err = ERANGE; + goto out; + } + + assert(ubufsize >= copysize); + err = copyout(kq_ids, ubuf, copysize); + } + +out: + if (kq_ids) { + kfree(kq_ids, bufsize); + } + + if (!err) { + *nkqueues_out = (int)min(nkqueues, PROC_PIDDYNKQUEUES_MAX); + } + return err; +} + +int +kevent_copyout_dynkqinfo(void *proc, kqueue_id_t kq_id, user_addr_t ubuf, + uint32_t ubufsize, int32_t *size_out) +{ + proc_t p = (proc_t)proc; + struct kqueue *kq; + int err = 0; + struct kqueue_dyninfo kqdi = { }; + + assert(p != NULL); + + if (ubufsize < sizeof(struct kqueue_info)) { + return ENOBUFS; + } + + kqhash_lock(p); + kq = kqueue_hash_lookup(p, kq_id); + if (!kq) { + kqhash_unlock(p); + return ESRCH; + } + kqueue_retain(kq); + kqhash_unlock(p); + + /* + * backward compatibility: allow the argument to this call to only be + * a struct kqueue_info + */ + if (ubufsize >= sizeof(struct kqueue_dyninfo)) { + ubufsize = sizeof(struct kqueue_dyninfo); + err = fill_kqueue_dyninfo(kq, &kqdi); + } else { + ubufsize = sizeof(struct kqueue_info); + err = fill_kqueueinfo(kq, &kqdi.kqdi_info); + } + if (err == 0 && (err = copyout(&kqdi, ubuf, ubufsize)) == 0) { + *size_out = ubufsize; + } + kqueue_release_last(p, kq); + return err; +} + +int +kevent_copyout_dynkqextinfo(void *proc, kqueue_id_t kq_id, user_addr_t ubuf, + uint32_t ubufsize, int32_t *nknotes_out) +{ + proc_t p = (proc_t)proc; + struct kqueue *kq; + int err; + + assert(p != NULL); + + kqhash_lock(p); + kq = kqueue_hash_lookup(p, kq_id); + if (!kq) { + kqhash_unlock(p); + return ESRCH; + } + kqueue_retain(kq); + kqhash_unlock(p); + + err = pid_kqueue_extinfo(p, kq, ubuf, ubufsize, nknotes_out); + kqueue_release_last(p, kq); + return err; +} + int pid_kqueue_extinfo(proc_t p, struct kqueue *kq, user_addr_t ubuf, uint32_t bufsize, int32_t *retval) @@ -5775,21 +9521,21 @@ pid_kqueue_extinfo(proc_t p, struct kqueue *kq, user_addr_t ubuf, bzero(kqext, buflen * sizeof(struct kevent_extinfo)); proc_fdlock(p); - for (i = 0; i < fdp->fd_knlistsize; i++) { kn = SLIST_FIRST(&fdp->fd_knlist[i]); nknotes = kevent_extinfo_emit(kq, kn, kqext, buflen, nknotes); } + proc_fdunlock(p); if (fdp->fd_knhashmask != 0) { for (i = 0; i < (int)fdp->fd_knhashmask + 1; i++) { + kqhash_lock(p); kn = SLIST_FIRST(&fdp->fd_knhash[i]); nknotes = kevent_extinfo_emit(kq, kn, kqext, buflen, nknotes); + kqhash_unlock(p); } } - proc_fdunlock(p); - assert(bufsize >= sizeof(struct kevent_extinfo) * min(buflen, nknotes)); err = copyout(kqext, ubuf, sizeof(struct kevent_extinfo) * min(buflen, nknotes)); @@ -5805,53 +9551,185 @@ pid_kqueue_extinfo(proc_t p, struct kqueue *kq, user_addr_t ubuf, return err; } -static unsigned long -kevent_udatainfo_emit(struct kqueue *kq, struct knote *kn, uint64_t *buf, - unsigned long buflen, unsigned long nknotes) +static unsigned int +klist_copy_udata(struct klist *list, uint64_t *buf, + unsigned int buflen, unsigned int nknotes) { - struct kevent_internal_s *kevp; - for (; kn; kn = SLIST_NEXT(kn, kn_link)) { - if (kq == knote_get_kq(kn)) { - if (nknotes < buflen) { - kqlock(kq); - kevp = &(kn->kn_kevent); - buf[nknotes] = kevp->udata; - kqunlock(kq); - } - - /* we return total number of knotes, which may be more than requested */ - nknotes++; + struct kevent_internal_s *kev; + struct knote *kn; + SLIST_FOREACH(kn, list, kn_link) { + if (nknotes < buflen) { + struct kqueue *kq = knote_get_kq(kn); + kqlock(kq); + kev = &(kn->kn_kevent); + buf[nknotes] = kev->udata; + kqunlock(kq); } + /* we return total number of knotes, which may be more than requested */ + nknotes++; } return nknotes; } +static unsigned int +kqlist_copy_dynamicids(__assert_only proc_t p, struct kqlist *list, + uint64_t *buf, unsigned int buflen, unsigned int nids) +{ + kqhash_lock_held(p); + struct kqworkloop *kqwl; + SLIST_FOREACH(kqwl, list, kqwl_hashlink) { + if (nids < buflen) { + buf[nids] = kqwl->kqwl_dynamicid; + } + nids++; + } + return nids; +} + int -pid_kqueue_udatainfo(proc_t p, struct kqueue *kq, uint64_t *buf, - uint32_t bufsize) +kevent_proc_copy_uptrs(void *proc, uint64_t *buf, int bufsize) { - struct knote *kn; - int i; + proc_t p = (proc_t)proc; struct filedesc *fdp = p->p_fd; - unsigned long nknotes = 0; + unsigned int nuptrs = 0; unsigned long buflen = bufsize / sizeof(uint64_t); + if (buflen > 0) { + assert(buf != NULL); + } + proc_fdlock(p); + for (int i = 0; i < fdp->fd_knlistsize; i++) { + nuptrs = klist_copy_udata(&fdp->fd_knlist[i], buf, buflen, nuptrs); + } + knhash_lock(p); + proc_fdunlock(p); + if (fdp->fd_knhashmask != 0) { + for (int i = 0; i < (int)fdp->fd_knhashmask + 1; i++) { + nuptrs = klist_copy_udata(&fdp->fd_knhash[i], buf, buflen, nuptrs); + } + } + knhash_unlock(p); - for (i = 0; i < fdp->fd_knlistsize; i++) { - kn = SLIST_FIRST(&fdp->fd_knlist[i]); - nknotes = kevent_udatainfo_emit(kq, kn, buf, buflen, nknotes); + kqhash_lock(p); + if (fdp->fd_kqhashmask != 0) { + for (int i = 0; i < (int)fdp->fd_kqhashmask + 1; i++) { + nuptrs = kqlist_copy_dynamicids(p, &fdp->fd_kqhash[i], buf, buflen, + nuptrs); + } } + kqhash_unlock(p); - if (fdp->fd_knhashmask != 0) { - for (i = 0; i < (int)fdp->fd_knhashmask + 1; i++) { - kn = SLIST_FIRST(&fdp->fd_knhash[i]); - nknotes = kevent_udatainfo_emit(kq, kn, buf, buflen, nknotes); + return (int)nuptrs; +} + +static void +kevent_redrive_proc_thread_request(proc_t p) +{ + __assert_only int ret; + ret = (*pthread_functions->workq_threadreq)(p, NULL, WORKQ_THREADREQ_REDRIVE, 0, 0); + assert(ret == 0 || ret == ECANCELED); +} + +static void +kevent_set_return_to_kernel_user_tsd(proc_t p, thread_t thread) +{ + uint64_t ast_addr; + bool proc_is_64bit = !!(p->p_flag & P_LP64); + size_t user_addr_size = proc_is_64bit ? 8 : 4; + uint32_t ast_flags32 = 0; + uint64_t ast_flags64 = 0; + struct uthread *ut = get_bsdthread_info(thread); + + if (ut->uu_kqueue_bound != NULL) { + if (ut->uu_kqueue_flags & KEVENT_FLAG_WORKLOOP) { + ast_flags64 |= R2K_WORKLOOP_PENDING_EVENTS; + } else if (ut->uu_kqueue_flags & KEVENT_FLAG_WORKQ) { + ast_flags64 |= R2K_WORKQ_PENDING_EVENTS; } } - proc_fdunlock(p); - return (int)nknotes; + if (ast_flags64 == 0) { + return; + } + + if (!(p->p_flag & P_LP64)) { + ast_flags32 = (uint32_t)ast_flags64; + assert(ast_flags64 < 0x100000000ull); + } + + ast_addr = thread_rettokern_addr(thread); + if (ast_addr == 0) { + return; + } + + if (copyout((proc_is_64bit ? (void *)&ast_flags64 : (void *)&ast_flags32), + (user_addr_t)ast_addr, + user_addr_size) != 0) { + printf("pid %d (tid:%llu): copyout of return_to_kernel ast flags failed with " + "ast_addr = %llu\n", p->p_pid, thread_tid(current_thread()), ast_addr); + } +} + +void +kevent_ast(thread_t thread, uint16_t bits) +{ + proc_t p = current_proc(); + + if (bits & AST_KEVENT_REDRIVE_THREADREQ) { + kevent_redrive_proc_thread_request(p); + } + if (bits & AST_KEVENT_RETURN_TO_KERNEL) { + kevent_set_return_to_kernel_user_tsd(p, thread); + } +} + +#if DEVELOPMENT || DEBUG + +#define KEVENT_SYSCTL_BOUND_ID 1 + +static int +kevent_sysctl SYSCTL_HANDLER_ARGS +{ +#pragma unused(oidp, arg2) + uintptr_t type = (uintptr_t)arg1; + uint64_t bound_id = 0; + struct uthread *ut; + struct kqueue *kq; + + if (type != KEVENT_SYSCTL_BOUND_ID) { + return EINVAL; + } + + if (req->newptr) { + return EINVAL; + } + + ut = get_bsdthread_info(current_thread()); + if (!ut) { + return EFAULT; + } + + kq = ut->uu_kqueue_bound; + if (kq) { + if (kq->kq_state & KQ_WORKLOOP) { + bound_id = ((struct kqworkloop *)kq)->kqwl_dynamicid; + } else if (kq->kq_state & KQ_WORKQ) { + bound_id = -1; + } + } + + return sysctl_io_number(req, bound_id, sizeof(bound_id), NULL, NULL); } +SYSCTL_NODE(_kern, OID_AUTO, kevent, CTLFLAG_RW | CTLFLAG_LOCKED, 0, + "kevent information"); + +SYSCTL_PROC(_kern_kevent, OID_AUTO, bound_id, + CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_LOCKED | CTLFLAG_MASKED, + (void *)KEVENT_SYSCTL_BOUND_ID, + sizeof(kqueue_id_t), kevent_sysctl, "Q", + "get the ID of the bound kqueue"); + +#endif /* DEVELOPMENT || DEBUG */ diff --git a/bsd/kern/kern_exec.c b/bsd/kern/kern_exec.c index 604da6996..9412a4472 100644 --- a/bsd/kern/kern_exec.c +++ b/bsd/kern/kern_exec.c @@ -138,8 +138,10 @@ #include <kern/policy_internal.h> #include <kern/kalloc.h> +#include <os/log.h> + #if CONFIG_MACF -#include <security/mac.h> +#include <security/mac_framework.h> #include <security/mac_mach_internal.h> #endif @@ -242,6 +244,16 @@ extern const struct fileops vnops; ( ( (user_addr_t)(addr) + (val) - 1) \ & ~((val) - 1) ) + /* Platform Code Exec Logging */ +static int platform_exec_logging = 0; + +SYSCTL_DECL(_security_mac); + +SYSCTL_INT(_security_mac, OID_AUTO, platform_exec_logging, CTLFLAG_RW, &platform_exec_logging, 0, + "log cdhashes for all platform binary executions"); + +static os_log_t peLog = OS_LOG_DEFAULT; + struct image_params; /* Forward */ static int exec_activate_image(struct image_params *imgp); static int exec_copyout_strings(struct image_params *imgp, user_addr_t *stackp); @@ -454,10 +466,6 @@ exec_shell_imgact(struct image_params *imgp) char *ihp; char *line_startp, *line_endp; char *interp; - proc_t p; - struct fileproc *fp; - int fd; - int error; /* * Make sure it's a shell script. If we've already redirected @@ -552,12 +560,18 @@ exec_shell_imgact(struct image_params *imgp) *interp++ = *ihp; *interp = '\0'; +#if !SECURE_KERNEL /* - * If we have a SUID oder SGID script, create a file descriptor + * If we have an SUID or SGID script, create a file descriptor * from the vnode and pass /dev/fd/%d instead of the actual * path name so that the script does not get opened twice */ if (imgp->ip_origvattr->va_mode & (VSUID | VSGID)) { + proc_t p; + struct fileproc *fp; + int fd; + int error; + p = vfs_context_proc(imgp->ip_vfs_context); error = falloc(p, &fp, &fd, imgp->ip_vfs_context); if (error) @@ -575,6 +589,7 @@ exec_shell_imgact(struct image_params *imgp) imgp->ip_interp_sugid_fd = fd; } +#endif return (-3); } @@ -773,6 +788,15 @@ set_proc_name(struct image_params *imgp, proc_t p) p->p_comm[imgp->ip_ndp->ni_cnd.cn_namelen] = '\0'; } +static uint64_t get_va_fsid(struct vnode_attr *vap) +{ + if (VATTR_IS_SUPPORTED(vap, va_fsid64)) { + return *(uint64_t *)&vap->va_fsid64; + } else { + return vap->va_fsid; + } +} + /* * exec_mach_imgact * @@ -984,8 +1008,14 @@ grade: imgp->ip_csflags |= CS_KILL; if (p->p_csflags & CS_EXEC_SET_ENFORCEMENT) imgp->ip_csflags |= CS_ENFORCEMENT; - if (p->p_csflags & CS_EXEC_SET_INSTALLER) - imgp->ip_csflags |= CS_INSTALLER; + if (p->p_csflags & CS_EXEC_INHERIT_SIP) { + if (p->p_csflags & CS_INSTALLER) + imgp->ip_csflags |= CS_INSTALLER; + if (p->p_csflags & CS_DATAVAULT_CONTROLLER) + imgp->ip_csflags |= CS_DATAVAULT_CONTROLLER; + if (p->p_csflags & CS_NVRAM_UNRESTRICTED) + imgp->ip_csflags |= CS_NVRAM_UNRESTRICTED; + } /* * Set up the system reserved areas in the new address space. @@ -995,7 +1025,7 @@ grade: /* * Close file descriptors which specify close-on-exec. */ - fdexec(p, psa != NULL ? psa->psa_flags : 0); + fdexec(p, psa != NULL ? psa->psa_flags : 0, exec); /* * deal with set[ug]id. @@ -1147,17 +1177,10 @@ grade: set_proc_name(imgp, p); #if CONFIG_SECLUDED_MEMORY - if (secluded_for_apps) { + if (secluded_for_apps && + load_result.platform_binary) { if (strncmp(p->p_name, "Camera", - sizeof (p->p_name)) == 0 || -#if 00 - strncmp(p->p_name, - "camerad", - sizeof (p->p_name)) == 0 || -#endif - strncmp(p->p_name, - "testCamera", sizeof (p->p_name)) == 0) { task_set_could_use_secluded_mem(task, TRUE); } else { @@ -1171,10 +1194,34 @@ grade: } #endif /* CONFIG_SECLUDED_MEMORY */ - pal_dbg_set_task_name( task ); + pal_dbg_set_task_name(task); + + /* + * The load result will have already been munged by AMFI to include the + * platform binary flag if boot-args dictated it (AMFI will mark anything + * that doesn't go through the upcall path as a platform binary if its + * enforcement is disabled). + */ + if (load_result.platform_binary) { + if (cs_debug) { + printf("setting platform binary on task: pid = %d\n", p->p_pid); + } + + /* + * We must use 'task' here because the proc's task has not yet been + * switched to the new one. + */ + task_set_platform_binary(task, TRUE); + } else { + if (cs_debug) { + printf("clearing platform binary on task: pid = %d\n", p->p_pid); + } + + task_set_platform_binary(task, FALSE); + } #if DEVELOPMENT || DEBUG - /* + /* * Update the pid an proc name for importance base if any */ task_importance_update_owner_info(task); @@ -1194,8 +1241,18 @@ grade: */ kdbg_trace_string(p, &dbg_arg1, &dbg_arg2, &dbg_arg3, &dbg_arg4); + uintptr_t fsid = 0, fileid = 0; + if (imgp->ip_vattr) { + uint64_t fsid64 = get_va_fsid(imgp->ip_vattr); + fsid = fsid64; + fileid = imgp->ip_vattr->va_fileid; + // check for (unexpected) overflow and trace zero in that case + if (fsid != fsid64 || fileid != imgp->ip_vattr->va_fileid) { + fsid = fileid = 0; + } + } KERNEL_DEBUG_CONSTANT1(TRACE_DATA_EXEC | DBG_FUNC_NONE, - p->p_pid ,0,0,0, (uintptr_t)thread_tid(thread)); + p->p_pid , fsid, fileid, 0, (uintptr_t)thread_tid(thread)); KERNEL_DEBUG_CONSTANT1(TRACE_STRING_EXEC | DBG_FUNC_NONE, dbg_arg1, dbg_arg2, dbg_arg3, dbg_arg4, (uintptr_t)thread_tid(thread)); } @@ -1306,7 +1363,8 @@ struct execsw { * * Description: Iterate through the available image activators, and activate * the image associated with the imgp structure. We start with - * the + * the activator for Mach-o binaries followed by that for Fat binaries + * for Interpreter scripts. * * Parameters: struct image_params * Image parameter block * @@ -1527,9 +1585,11 @@ exec_handle_spawnattr_policy(proc_t p, int psa_apptype, uint64_t psa_qos_clamp, case POSIX_SPAWN_PROC_TYPE_APP_DEFAULT: apptype = TASK_APPTYPE_APP_DEFAULT; break; +#if !CONFIG_EMBEDDED case POSIX_SPAWN_PROC_TYPE_APP_TAL: apptype = TASK_APPTYPE_APP_TAL; break; +#endif /* !CONFIG_EMBEDDED */ default: apptype = TASK_APPTYPE_NONE; /* TODO: Should an invalid value here fail the spawn? */ @@ -2443,9 +2503,25 @@ do_fork1: #if CONFIG_COALITIONS /* set the roles of this task within each given coalition */ if (error == 0) { - kr = coalitions_set_roles(coal, get_threadtask(imgp->ip_new_thread), coal_role); + kr = coalitions_set_roles(coal, new_task, coal_role); if (kr != KERN_SUCCESS) error = EINVAL; + if (kdebug_debugid_enabled(MACHDBG_CODE(DBG_MACH_COALITION, + MACH_COALITION_ADOPT))) { + for (i = 0; i < COALITION_NUM_TYPES; i++) { + if (coal[i] != COALITION_NULL) { + /* + * On 32-bit targets, uniqueid + * will get truncated to 32 bits + */ + KDBG_RELEASE(MACHDBG_CODE( + DBG_MACH_COALITION, + MACH_COALITION_ADOPT), + coalition_id(coal[i]), + get_task_uniqueid(new_task)); + } + } + } } /* drop our references and activations - fork1() now holds them */ @@ -2671,6 +2747,10 @@ do_fork1: OSBitOrAtomic(P_DISABLE_ASLR, &p->p_flag); #endif /* !SECURE_KERNEL */ + /* Randomize high bits of ASLR slide */ + if (px_sa.psa_flags & _POSIX_SPAWN_HIGH_BITS_ASLR) + imgp->ip_flags |= IMGPF_HIGH_BITS_ASLR; + /* * Forcibly disallow execution from data pages for the spawned process * even if it would otherwise be permitted by the architecture default. @@ -2788,8 +2868,10 @@ bad: if (error == 0) { /* reset delay idle sleep status if set */ +#if !CONFIG_EMBEDDED if ((p->p_flag & P_DELAYIDLESLEEP) == P_DELAYIDLESLEEP) OSBitAndAtomic(~((uint32_t)P_DELAYIDLESLEEP), &p->p_flag); +#endif /* !CONFIG_EMBEDDED */ /* upon successful spawn, re/set the proc control state */ if (imgp->ip_px_sa != NULL) { switch (px_sa.psa_pcontrol) { @@ -2816,20 +2898,17 @@ bad: /* * With 2-level high-water-mark support, POSIX_SPAWN_JETSAM_HIWATER_BACKGROUND is no * longer relevant, as background limits are described via the inactive limit slots. - * At the kernel layer, the flag is ignored. * * That said, however, if the POSIX_SPAWN_JETSAM_HIWATER_BACKGROUND is passed in, * we attempt to mimic previous behavior by forcing the BG limit data into the * inactive/non-fatal mode and force the active slots to hold system_wide/fatal mode. - * The kernel layer will flag this mapping. */ if (px_sa.psa_jetsam_flags & POSIX_SPAWN_JETSAM_HIWATER_BACKGROUND) { memorystatus_update(p, px_sa.psa_priority, 0, (px_sa.psa_jetsam_flags & POSIX_SPAWN_JETSAM_USE_EFFECTIVE_PRIORITY), TRUE, -1, TRUE, - px_sa.psa_memlimit_inactive, FALSE, - (px_sa.psa_jetsam_flags & POSIX_SPAWN_JETSAM_HIWATER_BACKGROUND)); + px_sa.psa_memlimit_inactive, FALSE); } else { memorystatus_update(p, px_sa.psa_priority, 0, (px_sa.psa_jetsam_flags & POSIX_SPAWN_JETSAM_USE_EFFECTIVE_PRIORITY), @@ -2837,8 +2916,7 @@ bad: px_sa.psa_memlimit_active, (px_sa.psa_jetsam_flags & POSIX_SPAWN_JETSAM_MEMLIMIT_ACTIVE_FATAL), px_sa.psa_memlimit_inactive, - (px_sa.psa_jetsam_flags & POSIX_SPAWN_JETSAM_MEMLIMIT_INACTIVE_FATAL), - (px_sa.psa_jetsam_flags & POSIX_SPAWN_JETSAM_HIWATER_BACKGROUND)); + (px_sa.psa_jetsam_flags & POSIX_SPAWN_JETSAM_MEMLIMIT_INACTIVE_FATAL)); } } @@ -3163,8 +3241,12 @@ proc_exec_switch_task(proc_t p, task_t old_task, task_t new_task, thread_t new_t * Switch the task pointer of proc to new task. * Before switching the task, wait for proc_refdrain. * After the switch happens, the proc can disappear, - * take a ref before it disappears. + * take a ref before it disappears. Waiting for + * proc_refdrain in exec will block all other threads + * trying to take a proc ref, boost the current thread + * to avoid priority inversion. */ + thread_set_exec_promotion(old_thread); p = proc_refdrain_with_refwait(p, TRUE); /* extra proc ref returned to the caller */ @@ -3194,6 +3276,11 @@ proc_exec_switch_task(proc_t p, task_t old_task, task_t new_task, thread_t new_t */ p->task = new_task; + /* Clear dispatchqueue and workloop ast offset */ + p->p_dispatchqueue_offset = 0; + p->p_dispatchqueue_serialno_offset = 0; + p->p_return_to_kernel_offset = 0; + /* Copy the signal state, dtrace state and set bsd ast on new thread */ act_set_astbsd(new_thread); new_uthread->uu_siglist = old_uthread->uu_siglist; @@ -3228,12 +3315,15 @@ proc_exec_switch_task(proc_t p, task_t old_task, task_t new_task, thread_t new_t task_set_did_exec_flag(old_task); task_clear_exec_copy_flag(new_task); + task_copy_fields_for_exec(new_task, old_task); + proc_transend(p, 1); } } proc_unlock(p); proc_refwake(p); + thread_clear_exec_promotion(old_thread); if (error != 0 || !task_active || !proc_active || !thread_active) { task_terminate_internal(new_task); @@ -4132,6 +4222,12 @@ extern user64_addr_t commpage_text64_location; #define MAIN_STACK_VALUES 4 #define MAIN_STACK_KEY "main_stack=" +#define FSID_KEY "executable_file=" +#define DYLD_FSID_KEY "dyld_file=" +#define CDHASH_KEY "executable_cdhash=" + +#define FSID_MAX_STRING "0x1234567890abcdef,0x1234567890abcdef" + #define HEX_STR_LEN 18 // 64-bit hex value "0x0123456701234567" static int @@ -4251,6 +4347,48 @@ exec_add_apple_strings(struct image_params *imgp, imgp->ip_applec++; } + if (imgp->ip_vattr) { + uint64_t fsid = get_va_fsid(imgp->ip_vattr); + uint64_t fsobjid = imgp->ip_vattr->va_fileid; + + char fsid_string[strlen(FSID_KEY) + strlen(FSID_MAX_STRING) + 1]; + snprintf(fsid_string, sizeof(fsid_string), + FSID_KEY "0x%llx,0x%llx", fsid, fsobjid); + error = exec_add_user_string(imgp, CAST_USER_ADDR_T(fsid_string), UIO_SYSSPACE, FALSE); + if (error) { + goto bad; + } + imgp->ip_applec++; + } + + if (imgp->ip_dyld_fsid || imgp->ip_dyld_fsobjid ) { + char fsid_string[strlen(DYLD_FSID_KEY) + strlen(FSID_MAX_STRING) + 1]; + snprintf(fsid_string, sizeof(fsid_string), + DYLD_FSID_KEY "0x%llx,0x%llx", imgp->ip_dyld_fsid, imgp->ip_dyld_fsobjid); + error = exec_add_user_string(imgp, CAST_USER_ADDR_T(fsid_string), UIO_SYSSPACE, FALSE); + if (error) { + goto bad; + } + imgp->ip_applec++; + } + + uint8_t cdhash[SHA1_RESULTLEN]; + int cdhash_errror = ubc_cs_getcdhash(imgp->ip_vp, imgp->ip_arch_offset, cdhash); + if (cdhash_errror == 0) { + char hash_string[strlen(CDHASH_KEY) + 2*SHA1_RESULTLEN + 1]; + strncpy(hash_string, CDHASH_KEY, sizeof(hash_string)); + char *p = hash_string + sizeof(CDHASH_KEY) - 1; + for (int i = 0; i < SHA1_RESULTLEN; i++) { + snprintf(p, 3, "%02x", (int) cdhash[i]); + p += 2; + } + error = exec_add_user_string(imgp, CAST_USER_ADDR_T(hash_string), UIO_SYSSPACE, FALSE); + if (error) { + goto bad; + } + imgp->ip_applec++; + } + /* Align the tail of the combined applev area */ while (imgp->ip_strspace % img_ptr_size != 0) { *imgp->ip_strendp++ = '\0'; @@ -4298,6 +4436,7 @@ exec_check_permissions(struct image_params *imgp) VATTR_WANTED(vap, va_gid); VATTR_WANTED(vap, va_mode); VATTR_WANTED(vap, va_fsid); + VATTR_WANTED(vap, va_fsid64); VATTR_WANTED(vap, va_fileid); VATTR_WANTED(vap, va_data_size); if ((error = vnode_getattr(vp, vap, imgp->ip_vfs_context)) != 0) @@ -4449,6 +4588,7 @@ exec_handle_sugid(struct image_params *imgp) handle_mac_transition: #endif +#if !SECURE_KERNEL /* * Replace the credential with a copy of itself if euid or * egid change. @@ -4526,6 +4666,7 @@ handle_mac_transition: break; } +#endif /* !SECURE_KERNEL */ #if CONFIG_MACF /* @@ -4785,15 +4926,13 @@ create_unix_stack(vm_map_t map, load_result_t* load_result, return KERN_INVALID_ARGUMENT; } addr = mach_vm_trunc_page(load_result->user_stack - size); - kr = mach_vm_allocate(map, &addr, size, - VM_MAKE_TAG(VM_MEMORY_STACK) | - VM_FLAGS_FIXED); + kr = mach_vm_allocate_kernel(map, &addr, size, + VM_FLAGS_FIXED, VM_MEMORY_STACK); if (kr != KERN_SUCCESS) { // Can't allocate at default location, try anywhere addr = 0; - kr = mach_vm_allocate(map, &addr, size, - VM_MAKE_TAG(VM_MEMORY_STACK) | - VM_FLAGS_ANYWHERE); + kr = mach_vm_allocate_kernel(map, &addr, size, + VM_FLAGS_ANYWHERE, VM_MEMORY_STACK); if (kr != KERN_SUCCESS) { return kr; } @@ -4985,10 +5124,10 @@ load_init_program(proc_t p) mach_vm_offset_t scratch_addr = 0; mach_vm_size_t map_page_size = vm_map_page_size(map); - (void) mach_vm_allocate(map, &scratch_addr, map_page_size, VM_FLAGS_ANYWHERE); -#if CONFIG_MEMORYSTATUS && CONFIG_JETSAM + (void) mach_vm_allocate_kernel(map, &scratch_addr, map_page_size, VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_NONE); +#if CONFIG_MEMORYSTATUS (void) memorystatus_init_at_boot_snapshot(); -#endif /* CONFIG_MEMORYSTATUS && CONFIG_JETSAM */ +#endif /* CONFIG_MEMORYSTATUS */ #if DEBUG || DEVELOPMENT /* Check for boot-arg suffix first */ @@ -4999,6 +5138,7 @@ load_init_program(proc_t p) (strcmp(launchd_suffix, "release") == 0)); if (is_release_suffix) { + printf("load_init_program: attempting to load /sbin/launchd\n"); error = load_init_program_at_path(p, (user_addr_t)scratch_addr, "/sbin/launchd"); if (!error) return; @@ -5008,10 +5148,12 @@ load_init_program(proc_t p) strlcpy(launchd_path, "/usr/local/sbin/launchd.", sizeof(launchd_path)); strlcat(launchd_path, launchd_suffix, sizeof(launchd_path)); - /* All the error data is lost in the loop below, don't - * attempt to save it. */ - if (!load_init_program_at_path(p, (user_addr_t)scratch_addr, launchd_path)) { + printf("load_init_program: attempting to load %s\n", launchd_path); + error = load_init_program_at_path(p, (user_addr_t)scratch_addr, launchd_path); + if (!error) { return; + } else { + printf("load_init_program: failed loading %s: errno %d\n", launchd_path, error); } } } @@ -5019,9 +5161,13 @@ load_init_program(proc_t p) error = ENOENT; for (i = 0; i < sizeof(init_programs)/sizeof(init_programs[0]); i++) { + printf("load_init_program: attempting to load %s\n", init_programs[i]); error = load_init_program_at_path(p, (user_addr_t)scratch_addr, init_programs[i]); - if (!error) + if (!error) { return; + } else { + printf("load_init_program: failed loading %s: errno %d\n", init_programs[i], error); + } } panic("Process 1 exec of %s failed, errno %d", ((i == 0) ? "<null>" : init_programs[i-1]), error); @@ -5136,7 +5282,7 @@ execargs_lock_sleep(void) { static kern_return_t execargs_purgeable_allocate(char **execarg_address) { - kern_return_t kr = vm_allocate(bsd_pageable_map, (vm_offset_t *)execarg_address, BSD_PAGEABLE_SIZE_PER_EXEC, VM_FLAGS_ANYWHERE | VM_FLAGS_PURGABLE); + kern_return_t kr = vm_allocate_kernel(bsd_pageable_map, (vm_offset_t *)execarg_address, BSD_PAGEABLE_SIZE_PER_EXEC, VM_FLAGS_ANYWHERE | VM_FLAGS_PURGABLE, VM_KERN_MEMORY_NONE); assert(kr == KERN_SUCCESS); return kr; } @@ -5302,6 +5448,24 @@ exec_resettextvp(proc_t p, struct image_params *imgp) } +// Includes the 0-byte (therefore "SIZE" instead of "LEN"). +static const size_t CS_CDHASH_STRING_SIZE = CS_CDHASH_LEN * 2 + 1; + +static void cdhash_to_string(char str[CS_CDHASH_STRING_SIZE], uint8_t const * const cdhash) { + static char const nibble[] = "0123456789abcdef"; + + /* Apparently still the safest way to get a hex representation + * of binary data. + * xnu's printf routines have %*D/%20D in theory, but "not really", see: + * <rdar://problem/33328859> confusion around %*D/%nD in printf + */ + for (int i = 0; i < CS_CDHASH_LEN; ++i) { + str[i*2] = nibble[(cdhash[i] & 0xf0) >> 4]; + str[i*2+1] = nibble[cdhash[i] & 0x0f]; + } + str[CS_CDHASH_STRING_SIZE - 1] = 0; +} + /* * If the process is not signed or if it contains entitlements, we * need to communicate through the task_access_port to taskgated. @@ -5320,7 +5484,11 @@ taskgated_required(proc_t p, boolean_t *require_success) if (cs_debug > 2) csvnode_print_debug(p->p_textvp); +#if !CONFIG_EMBEDDED const int can_skip_taskgated = csproc_get_platform_binary(p) && !csproc_get_platform_path(p); +#else + const int can_skip_taskgated = csproc_get_platform_binary(p); +#endif if (can_skip_taskgated) { if (cs_debug) printf("taskgated not required for: %s\n", p->p_name); *require_success = FALSE; @@ -5334,6 +5502,7 @@ taskgated_required(proc_t p, boolean_t *require_success) error = cs_entitlements_blob_get(p, &blob, &length); if (error == 0 && blob != NULL) { +#if !CONFIG_EMBEDDED /* * fatal on the desktop when entitlements are present, * unless we started in single-user mode @@ -5348,6 +5517,7 @@ taskgated_required(proc_t p, boolean_t *require_success) return FALSE; } +#endif if (cs_debug) printf("taskgated required for: %s\n", p->p_name); return TRUE; @@ -5377,7 +5547,7 @@ check_for_signature(proc_t p, struct image_params *imgp) kern_return_t kr = KERN_FAILURE; int error = EACCES; boolean_t unexpected_failure = FALSE; - unsigned char hash[SHA1_RESULTLEN]; + unsigned char hash[CS_CDHASH_LEN]; boolean_t require_success = FALSE; int spawn = (imgp->ip_flags & IMGPF_SPAWN); int vfexec = (imgp->ip_flags & IMGPF_VFORK_EXEC); @@ -5396,7 +5566,7 @@ check_for_signature(proc_t p, struct image_params *imgp) if(p->p_csflags & (CS_HARD|CS_KILL)) { vm_map_switch_protect(get_task_map(p->task), TRUE); } - + /* * image activation may be failed due to policy * which is unexpected but security framework does not @@ -5481,7 +5651,20 @@ check_for_signature(proc_t p, struct image_params *imgp) } done: - if (0 != error) { + if (0 == error) { + /* The process's code signature related properties are + * fully set up, so this is an opportune moment to log + * platform binary execution, if desired. */ + if (platform_exec_logging != 0 && csproc_get_platform_binary(p)) { + uint8_t cdhash[CS_CDHASH_LEN]; + char cdhash_string[CS_CDHASH_STRING_SIZE]; + proc_getcdhash(p, cdhash); + cdhash_to_string(cdhash_string, cdhash); + + os_log(peLog, "CS Platform Exec Logging: Executing platform signed binary " + "'%s' with cdhash %s\n", p->p_name, cdhash_string); + } + } else { if (!unexpected_failure) p->p_csflags |= CS_KILLED; /* make very sure execution fails */ @@ -5525,7 +5708,7 @@ static void exec_prefault_data(proc_t p __unused, struct image_params *imgp, loa vm_map_trunc_page(load_result->entry_point, vm_map_page_mask(current_map())), VM_PROT_READ | VM_PROT_EXECUTE, - FALSE, + FALSE, VM_KERN_MEMORY_NONE, THREAD_UNINT, NULL, 0); if (imgp->ip_flags & IMGPF_IS_64BIT) { @@ -5551,7 +5734,7 @@ static void exec_prefault_data(proc_t p __unused, struct image_params *imgp, loa vm_map_trunc_page(load_result->all_image_info_addr, vm_map_page_mask(current_map())), VM_PROT_READ | VM_PROT_WRITE, - FALSE, + FALSE, VM_KERN_MEMORY_NONE, THREAD_UNINT, NULL, 0); if ((load_result->all_image_info_addr & PAGE_MASK) + expected_all_image_infos_size > PAGE_SIZE) { /* all_image_infos straddles a page */ @@ -5559,14 +5742,14 @@ static void exec_prefault_data(proc_t p __unused, struct image_params *imgp, loa vm_map_trunc_page(load_result->all_image_info_addr + expected_all_image_infos_size - 1, vm_map_page_mask(current_map())), VM_PROT_READ | VM_PROT_WRITE, - FALSE, + FALSE, VM_KERN_MEMORY_NONE, THREAD_UNINT, NULL, 0); } ret = copyin(load_result->all_image_info_addr, &all_image_infos, expected_all_image_infos_size); - if (ret == 0 && all_image_infos.infos32.version >= 9) { + if (ret == 0 && all_image_infos.infos32.version >= DYLD_ALL_IMAGE_INFOS_ADDRESS_MINIMUM_VERSION) { user_addr_t notification_address; user_addr_t dyld_image_address; @@ -5615,25 +5798,25 @@ static void exec_prefault_data(proc_t p __unused, struct image_params *imgp, loa vm_map_trunc_page(notification_address + dyld_slide_amount, vm_map_page_mask(current_map())), VM_PROT_READ | VM_PROT_EXECUTE, - FALSE, + FALSE, VM_KERN_MEMORY_NONE, THREAD_UNINT, NULL, 0); vm_fault(current_map(), vm_map_trunc_page(dyld_image_address + dyld_slide_amount, vm_map_page_mask(current_map())), VM_PROT_READ | VM_PROT_EXECUTE, - FALSE, + FALSE, VM_KERN_MEMORY_NONE, THREAD_UNINT, NULL, 0); vm_fault(current_map(), vm_map_trunc_page(dyld_version_address + dyld_slide_amount, vm_map_page_mask(current_map())), VM_PROT_READ, - FALSE, + FALSE, VM_KERN_MEMORY_NONE, THREAD_UNINT, NULL, 0); vm_fault(current_map(), vm_map_trunc_page(dyld_all_image_infos_address + dyld_slide_amount, vm_map_page_mask(current_map())), VM_PROT_READ | VM_PROT_WRITE, - FALSE, + FALSE, VM_KERN_MEMORY_NONE, THREAD_UNINT, NULL, 0); } } diff --git a/bsd/kern/kern_exit.c b/bsd/kern/kern_exit.c index 4442c7ff4..b8f5def79 100644 --- a/bsd/kern/kern_exit.c +++ b/bsd/kern/kern_exit.c @@ -74,6 +74,7 @@ #include <machine/reg.h> #include <machine/psl.h> +#include <stdatomic.h> #include "compat_43.h" @@ -102,21 +103,20 @@ #include <sys/sysproto.h> #include <sys/signalvar.h> #include <sys/kdebug.h> -#include <sys/filedesc.h> /* fdfree */ -#if SYSV_SHM -#include <sys/shm_internal.h> /* shmexit */ -#endif -#include <sys/acct.h> /* acct_process */ -#if CONFIG_PERSONAS -#include <sys/persona.h> -#endif +#include <sys/filedesc.h> /* fdfree */ +#include <sys/acct.h> /* acct_process */ +#include <sys/codesign.h> +#include <sys/event.h> /* kevent_proc_copy_uptrs */ +#include <sys/sdt.h> #include <security/audit/audit.h> #include <bsm/audit_kevents.h> #include <mach/mach_types.h> -#include <kern/exc_resource.h> +#include <mach/task.h> +#include <mach/thread_act.h> +#include <kern/exc_resource.h> #include <kern/kern_types.h> #include <kern/kalloc.h> #include <kern/task.h> @@ -126,53 +126,53 @@ #include <kern/sched_prim.h> #include <kern/assert.h> #include <kern/policy_internal.h> +#include <kern/exc_guard.h> -#include <sys/codesign.h> +#include <vm/vm_protos.h> + +#include <pexpert/pexpert.h> +#if SYSV_SHM +#include <sys/shm_internal.h> /* shmexit */ +#endif /* SYSV_SHM */ +#if CONFIG_PERSONAS +#include <sys/persona.h> +#endif /* CONFIG_PERSONAS */ #if CONFIG_MEMORYSTATUS #include <sys/kern_memorystatus.h> -#endif - +#endif /* CONFIG_MEMORYSTATUS */ #if CONFIG_DTRACE /* Do not include dtrace.h, it redefines kmem_[alloc/free] */ void dtrace_proc_exit(proc_t p); - #include <sys/dtrace_ptss.h> -#endif - +#endif /* CONFIG_DTRACE */ #if CONFIG_MACF -#include <security/mac.h> +#include <security/mac_framework.h> #include <security/mac_mach_internal.h> #include <sys/syscall.h> -#endif - -#include <mach/mach_types.h> -#include <mach/task.h> -#include <mach/thread_act.h> - -#include <vm/vm_protos.h> - -#include <sys/sdt.h> +#endif /* CONFIG_MACF */ void proc_prepareexit(proc_t p, int rv, boolean_t perf_notify); -void gather_populate_corpse_crashinfo(proc_t p, void *crash_info_ptr, mach_exception_data_type_t code, mach_exception_data_type_t subcode, uint64_t *udata_buffer, int num_udata); +void gather_populate_corpse_crashinfo(proc_t p, task_t corpse_task, + mach_exception_data_type_t code, mach_exception_data_type_t subcode, + uint64_t *udata_buffer, int num_udata, void *reason); mach_exception_data_type_t proc_encode_exit_exception_code(proc_t p); void vfork_exit(proc_t p, int rv); -void vproc_exit(proc_t p); __private_extern__ void munge_user64_rusage(struct rusage *a_rusage_p, struct user64_rusage *a_user_rusage_p); __private_extern__ void munge_user32_rusage(struct rusage *a_rusage_p, struct user32_rusage *a_user_rusage_p); static int reap_child_locked(proc_t parent, proc_t child, int deadparent, int reparentedtoinit, int locked, int droplock); -static void populate_corpse_crashinfo(proc_t p, void *crash_info_ptr, struct rusage_superset *rup, mach_exception_data_type_t code, mach_exception_data_type_t subcode, uint64_t *udata_buffer, int num_udata); +static void populate_corpse_crashinfo(proc_t p, task_t corpse_task, + struct rusage_superset *rup, mach_exception_data_type_t code, + mach_exception_data_type_t subcode, uint64_t *udata_buffer, + int num_udata, os_reason_t reason); static void proc_update_corpse_exception_codes(proc_t p, mach_exception_data_type_t *code, mach_exception_data_type_t *subcode); extern int proc_pidpathinfo_internal(proc_t p, uint64_t arg, char *buffer, uint32_t buffersize, int32_t *retval); -static void abort_with_payload_internal(proc_t p, uint32_t reason_namespace, uint64_t reason_code, user_addr_t payload, - uint32_t payload_size, user_addr_t reason_string, uint64_t reason_flags); - static __attribute__((noinline)) void launchd_crashed_panic(proc_t p, int rv); extern void proc_piduniqidentifierinfo(proc_t p, struct proc_uniqidentifierinfo *p_uniqidinfo); extern void task_coalition_ids(task_t task, uint64_t ids[COALITION_NUM_TYPES]); extern uint64_t get_task_phys_footprint_limit(task_t); int proc_list_uptrs(void *p, uint64_t *udata_buffer, int size); +extern uint64_t task_corpse_get_crashed_thread_id(task_t corpse_task); /* @@ -184,6 +184,7 @@ int waitidcontinue(int result); kern_return_t sys_perf_notify(thread_t thread, int pid); kern_return_t task_exception_notify(exception_type_t exception, mach_exception_data_type_t code, mach_exception_data_type_t subcode); +kern_return_t task_violated_guard(mach_exception_code_t, mach_exception_subcode_t, void *); void delay(int); void gather_rusage_info(proc_t p, rusage_info_current *ru, int flavor); @@ -226,7 +227,7 @@ copyoutsiginfo(user_siginfo_t *native, boolean_t is64, user_addr_t uaddr) { if (is64) { user64_siginfo_t sinfo64; - + bzero(&sinfo64, sizeof (sinfo64)); siginfo_user_to_user64(native, &sinfo64); return (copyout(&sinfo64, uaddr, sizeof (sinfo64))); @@ -239,13 +240,16 @@ copyoutsiginfo(user_siginfo_t *native, boolean_t is64, user_addr_t uaddr) } } -void gather_populate_corpse_crashinfo(proc_t p, void *crash_info_ptr, mach_exception_data_type_t code, mach_exception_data_type_t subcode, uint64_t *udata_buffer, int num_udata) +void gather_populate_corpse_crashinfo(proc_t p, task_t corpse_task, + mach_exception_data_type_t code, mach_exception_data_type_t subcode, + uint64_t *udata_buffer, int num_udata, void *reason) { struct rusage_superset rup; gather_rusage_info(p, &rup.ri, RUSAGE_INFO_CURRENT); rup.ri.ri_phys_footprint = 0; - populate_corpse_crashinfo(p, crash_info_ptr, &rup, code, subcode, udata_buffer, num_udata); + populate_corpse_crashinfo(p, corpse_task, &rup, code, subcode, + udata_buffer, num_udata, reason); } static void proc_update_corpse_exception_codes(proc_t p, mach_exception_data_type_t *code, mach_exception_data_type_t *subcode) @@ -291,7 +295,10 @@ mach_exception_data_type_t proc_encode_exit_exception_code(proc_t p) return (mach_exception_data_type_t)subcode; } -static void populate_corpse_crashinfo(proc_t p, void *crash_info_ptr, struct rusage_superset *rup, mach_exception_data_type_t code, mach_exception_data_type_t subcode, uint64_t *udata_buffer, int num_udata) +static void +populate_corpse_crashinfo(proc_t p, task_t corpse_task, struct rusage_superset *rup, + mach_exception_data_type_t code, mach_exception_data_type_t subcode, + uint64_t *udata_buffer, int num_udata, os_reason_t reason) { mach_vm_address_t uaddr = 0; mach_exception_data_type_t exc_codes[EXCEPTION_CODE_MAX]; @@ -301,10 +308,11 @@ static void populate_corpse_crashinfo(proc_t p, void *crash_info_ptr, struct rus struct proc_uniqidentifierinfo p_uniqidinfo; struct proc_workqueueinfo pwqinfo; int retval = 0; - uint64_t crashed_threadid = thread_tid(current_thread()); + uint64_t crashed_threadid = task_corpse_get_crashed_thread_id(corpse_task); unsigned int pflags = 0; uint64_t max_footprint_mb; uint64_t max_footprint; + void *crash_info_ptr = task_get_corpseinfo(corpse_task); #if CONFIG_MEMORYSTATUS int memstat_dirty_flags = 0; @@ -322,8 +330,11 @@ static void populate_corpse_crashinfo(proc_t p, void *crash_info_ptr, struct rus kcdata_memcpy(crash_info_ptr, uaddr, &p->p_ppid, sizeof(p->p_ppid)); } - if (KERN_SUCCESS == kcdata_get_memory_addr(crash_info_ptr, TASK_CRASHINFO_CRASHED_THREADID, sizeof(uint64_t), &uaddr)) { - kcdata_memcpy(crash_info_ptr, uaddr, &crashed_threadid, sizeof(uint64_t)); + /* Don't include the crashed thread ID if there's an exit reason that indicates it's irrelevant */ + if ((p->p_exit_reason == OS_REASON_NULL) || !(p->p_exit_reason->osr_flags & OS_REASON_FLAG_NO_CRASHED_TID)) { + if (KERN_SUCCESS == kcdata_get_memory_addr(crash_info_ptr, TASK_CRASHINFO_CRASHED_THREADID, sizeof(uint64_t), &uaddr)) { + kcdata_memcpy(crash_info_ptr, uaddr, &crashed_threadid, sizeof(uint64_t)); + } } if (KERN_SUCCESS == @@ -427,23 +438,26 @@ static void populate_corpse_crashinfo(proc_t p, void *crash_info_ptr, struct rus } #endif - if (p->p_exit_reason != OS_REASON_NULL) { + if (p->p_exit_reason != OS_REASON_NULL && reason == OS_REASON_NULL) { + reason = p->p_exit_reason; + } + if (reason != OS_REASON_NULL) { if (KERN_SUCCESS == kcdata_get_memory_addr(crash_info_ptr, EXIT_REASON_SNAPSHOT, sizeof(struct exit_reason_snapshot), &uaddr)) { struct exit_reason_snapshot ers = { - .ers_namespace = p->p_exit_reason->osr_namespace, - .ers_code = p->p_exit_reason->osr_code, - .ers_flags = p->p_exit_reason->osr_flags + .ers_namespace = reason->osr_namespace, + .ers_code = reason->osr_code, + .ers_flags = reason->osr_flags }; kcdata_memcpy(crash_info_ptr, uaddr, &ers, sizeof(ers)); } - if (p->p_exit_reason->osr_kcd_buf != 0) { - uint32_t reason_buf_size = kcdata_memory_get_used_bytes(&p->p_exit_reason->osr_kcd_descriptor); + if (reason->osr_kcd_buf != 0) { + uint32_t reason_buf_size = kcdata_memory_get_used_bytes(&reason->osr_kcd_descriptor); assert(reason_buf_size != 0); if (KERN_SUCCESS == kcdata_get_memory_addr(crash_info_ptr, KCDATA_TYPE_NESTED_KCDATA, reason_buf_size, &uaddr)) { - kcdata_memcpy(crash_info_ptr, uaddr, p->p_exit_reason->osr_kcd_buf, reason_buf_size); + kcdata_memcpy(crash_info_ptr, uaddr, reason->osr_kcd_buf, reason_buf_size); } } } @@ -510,6 +524,14 @@ launchd_crashed_panic(proc_t p, int rv) launchd_exit_reason_desc : "none"); } + const char *launchd_crashed_prefix_str; + + if (strnstr(p->p_name, "preinit", sizeof(p->p_name))) { + launchd_crashed_prefix_str = "LTE preinit process exited"; + } else { + launchd_crashed_prefix_str = "initproc exited"; + } + #if (DEVELOPMENT || DEBUG) && CONFIG_COREDUMP /* * For debugging purposes, generate a core file of initproc before @@ -525,6 +547,7 @@ launchd_crashed_panic(proc_t p, int rv) clock_usec_t tv_usec; uint32_t tv_msec; + err = coredump(p, 300, COREDUMP_IGNORE_ULIMIT | COREDUMP_FULLFSYNC); coredump_end = mach_absolute_time(); @@ -545,48 +568,107 @@ launchd_crashed_panic(proc_t p, int rv) sync(p, (void *)NULL, (int *)NULL); if (p->p_exit_reason == OS_REASON_NULL) { - panic_plain(LAUNCHD_CRASHED_PREFIX " -- no exit reason available -- (signal %d, exit status %d %s)", - WTERMSIG(rv), WEXITSTATUS(rv), ((p->p_csflags & CS_KILLED) ? "CS_KILLED" : "")); + panic_with_options(0, NULL, DEBUGGER_OPTION_INITPROC_PANIC, "%s -- no exit reason available -- (signal %d, exit status %d %s)", + launchd_crashed_prefix_str, WTERMSIG(rv), WEXITSTATUS(rv), ((p->p_csflags & CS_KILLED) ? "CS_KILLED" : "")); } else { - panic_plain(LAUNCHD_CRASHED_PREFIX " %s -- exit reason namespace %d subcode 0x%llx description: %." LAUNCHD_PANIC_REASON_STRING_MAXLEN "s", + panic_with_options(0, NULL, DEBUGGER_OPTION_INITPROC_PANIC, "%s %s -- exit reason namespace %d subcode 0x%llx description: %." LAUNCHD_PANIC_REASON_STRING_MAXLEN "s", ((p->p_csflags & CS_KILLED) ? "CS_KILLED" : ""), - p->p_exit_reason->osr_namespace, p->p_exit_reason->osr_code, + launchd_crashed_prefix_str, p->p_exit_reason->osr_namespace, p->p_exit_reason->osr_code, launchd_exit_reason_desc ? launchd_exit_reason_desc : "none"); } } -static void -abort_with_payload_internal(proc_t p, uint32_t reason_namespace, uint64_t reason_code, user_addr_t payload, uint32_t payload_size, - user_addr_t reason_string, uint64_t reason_flags) +#define OS_REASON_IFLAG_USER_FAULT 0x1 + +#define OS_REASON_TOTAL_USER_FAULTS_PER_PROC 5 + +static int +abort_with_payload_internal(proc_t p, + uint32_t reason_namespace, uint64_t reason_code, + user_addr_t payload, uint32_t payload_size, + user_addr_t reason_string, uint64_t reason_flags, + uint32_t internal_flags) { os_reason_t exit_reason = OS_REASON_NULL; + kern_return_t kr = KERN_SUCCESS; + + if (internal_flags & OS_REASON_IFLAG_USER_FAULT) { + uint32_t old_value = atomic_load_explicit(&p->p_user_faults, + memory_order_relaxed); + for (;;) { + if (old_value >= OS_REASON_TOTAL_USER_FAULTS_PER_PROC) { + return EQFULL; + } + // this reloads the value in old_value + if (atomic_compare_exchange_strong_explicit(&p->p_user_faults, + &old_value, old_value + 1, memory_order_relaxed, + memory_order_relaxed)) { + break; + } + } + } KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_PROC, BSD_PROC_EXITREASON_CREATE) | DBG_FUNC_NONE, p->p_pid, reason_namespace, reason_code, 0, 0); - exit_reason = build_userspace_exit_reason(reason_namespace, reason_code, payload, payload_size, reason_string, - reason_flags); + exit_reason = build_userspace_exit_reason(reason_namespace, reason_code, + payload, payload_size, reason_string, reason_flags); - /* - * We use SIGABRT (rather than calling exit directly from here) so that - * the debugger can catch abort_with_{reason,payload} calls. - */ - psignal_try_thread_with_reason(p, current_thread(), SIGABRT, exit_reason); + if (internal_flags & OS_REASON_IFLAG_USER_FAULT) { + mach_exception_code_t code = 0; - return; + EXC_GUARD_ENCODE_TYPE(code, GUARD_TYPE_USER); /* simulated EXC_GUARD */ + EXC_GUARD_ENCODE_FLAVOR(code, 0); + EXC_GUARD_ENCODE_TARGET(code, reason_namespace); + + if (exit_reason == OS_REASON_NULL) { + kr = KERN_RESOURCE_SHORTAGE; + } else { + kr = task_violated_guard(code, reason_code, exit_reason); + } + os_reason_free(exit_reason); + } else { + /* + * We use SIGABRT (rather than calling exit directly from here) so that + * the debugger can catch abort_with_{reason,payload} calls. + */ + psignal_try_thread_with_reason(p, current_thread(), SIGABRT, exit_reason); + } + + switch (kr) { + case KERN_SUCCESS: + return 0; + case KERN_NOT_SUPPORTED: + return ENOTSUP; + case KERN_INVALID_ARGUMENT: + return EINVAL; + case KERN_RESOURCE_SHORTAGE: + default: + return EBUSY; + } } int abort_with_payload(struct proc *cur_proc, struct abort_with_payload_args *args, __unused void *retval) { - abort_with_payload_internal(cur_proc, args->reason_namespace, args->reason_code, args->payload, args->payload_size, - args->reason_string, args->reason_flags); + abort_with_payload_internal(cur_proc, args->reason_namespace, + args->reason_code, args->payload, args->payload_size, + args->reason_string, args->reason_flags, 0); return 0; } +int +os_fault_with_payload(struct proc *cur_proc, + struct os_fault_with_payload_args *args, __unused int *retval) +{ + return abort_with_payload_internal(cur_proc, args->reason_namespace, + args->reason_code, args->payload, args->payload_size, + args->reason_string, args->reason_flags, OS_REASON_IFLAG_USER_FAULT); +} + /* * exit -- @@ -596,6 +678,7 @@ __attribute__((noreturn)) void exit(proc_t p, struct exit_args *uap, int *retval) { + p->p_xhighbits = ((uint32_t)(uap->rval) & 0xFF000000) >> 24; exit1(p, W_EXITCODE(uap->rval, 0), retval); thread_exception_return(); @@ -858,11 +941,11 @@ skipcheck: int buf_size = 0; /* Get all the udata pointers from kqueue */ - est_knotes = proc_list_uptrs(p, NULL, 0); + est_knotes = kevent_proc_copy_uptrs(p, NULL, 0); if (est_knotes > 0) { buf_size = (est_knotes + 32) * sizeof(uint64_t); buffer = (uint64_t *) kalloc(buf_size); - num_knotes = proc_list_uptrs(p, buffer, buf_size); + num_knotes = kevent_proc_copy_uptrs(p, buffer, buf_size); if (num_knotes > est_knotes + 32) { num_knotes = est_knotes + 32; } @@ -870,7 +953,8 @@ skipcheck: /* Update the code, subcode based on exit reason */ proc_update_corpse_exception_codes(p, &code, &subcode); - populate_corpse_crashinfo(p, task_get_corpseinfo(p->task), rup, code, subcode, buffer, num_knotes); + populate_corpse_crashinfo(p, p->task, rup, + code, subcode, buffer, num_knotes, NULL); if (buffer != NULL) { kfree(buffer, buf_size); } @@ -981,9 +1065,6 @@ proc_exit(proc_t p) task_clear_cpuusage(p->task, TRUE); workqueue_mark_exiting(p); - workqueue_exit(p); - kqueue_dealloc(p->p_wqkqueue); - p->p_wqkqueue = NULL; _aio_exit( p ); @@ -993,6 +1074,12 @@ proc_exit(proc_t p) */ fdfree(p); + /* + * Once all the knotes, kqueues & workloops are destroyed, get rid of the + * workqueue. + */ + workqueue_exit(p); + if (uth->uu_lowpri_window) { /* * task is marked as a low priority I/O type @@ -1213,6 +1300,15 @@ proc_exit(proc_t p) proc_childdrainend(p); proc_list_unlock(); +#if CONFIG_MACF + /* + * Notify MAC policies that proc is dead. + * This should be replaced with proper label management + * (rdar://problem/32126399). + */ + mac_proc_notify_exit(p); +#endif + /* * Release reference to text vnode */ @@ -1318,6 +1414,8 @@ proc_exit(proc_t p) if (pp != initproc) { proc_lock(pp); pp->si_pid = p->p_pid; + pp->p_xhighbits = p->p_xhighbits; + p->p_xhighbits = 0; pp->si_status = p->p_xstat; pp->si_code = CLD_EXITED; /* @@ -1576,16 +1674,16 @@ reap_child_locked(proc_t parent, proc_t child, int deadparent, int reparentedtoi #if CONFIG_FINE_LOCK_GROUPS lck_mtx_destroy(&child->p_mlock, proc_mlock_grp); - lck_mtx_destroy(&child->p_fdmlock, proc_fdmlock_grp); lck_mtx_destroy(&child->p_ucred_mlock, proc_ucred_mlock_grp); + lck_mtx_destroy(&child->p_fdmlock, proc_fdmlock_grp); #if CONFIG_DTRACE lck_mtx_destroy(&child->p_dtrace_sprlock, proc_lck_grp); #endif lck_spin_destroy(&child->p_slock, proc_slock_grp); #else /* CONFIG_FINE_LOCK_GROUPS */ lck_mtx_destroy(&child->p_mlock, proc_lck_grp); - lck_mtx_destroy(&child->p_fdmlock, proc_lck_grp); lck_mtx_destroy(&child->p_ucred_mlock, proc_lck_grp); + lck_mtx_destroy(&child->p_fdmlock, proc_lck_grp); #if CONFIG_DTRACE lck_mtx_destroy(&child->p_dtrace_sprlock, proc_lck_grp); #endif @@ -1937,7 +2035,8 @@ loop1: #endif siginfo.si_signo = SIGCHLD; siginfo.si_pid = p->p_pid; - siginfo.si_status = WEXITSTATUS(p->p_xstat); + siginfo.si_status = (WEXITSTATUS(p->p_xstat) & 0x00FFFFFF) | (((uint32_t)(p->p_xhighbits) << 24) & 0xFF000000); + p->p_xhighbits = 0; if (WIFSIGNALED(p->p_xstat)) { siginfo.si_code = WCOREDUMP(p->p_xstat) ? CLD_DUMPED : CLD_KILLED; @@ -2201,23 +2300,15 @@ vfork_exit_internal(proc_t p, int rv, int forceexit) p->p_sigignore = ~0; proc_unlock(p); - proc_free_realitimer(p); - ut->uu_siglist = 0; - vproc_exit(p); -} + /* begin vproc_exit */ -void -vproc_exit(proc_t p) -{ proc_t q; proc_t pp; - + vnode_t tvp; -#ifdef FIXME - struct task *task = p->task; -#endif + struct pgrp * pg; struct session *sessp; struct rusage_superset *rup; @@ -2488,6 +2579,8 @@ vproc_exit(proc_t p) pth_proc_hashdelete(p); #endif /* PSYNCH */ + proc_free_realitimer(p); + /* * Other substructures are freed from wait(). */ @@ -2515,6 +2608,8 @@ vproc_exit(proc_t p) if (pp != initproc) { proc_lock(pp); pp->si_pid = p->p_pid; + pp->p_xhighbits = p->p_xhighbits; + p->p_xhighbits = 0; pp->si_status = p->p_xstat; pp->si_code = CLD_EXITED; /* diff --git a/bsd/kern/kern_fork.c b/bsd/kern/kern_fork.c index a42891ae7..952b6f8fb 100644 --- a/bsd/kern/kern_fork.c +++ b/bsd/kern/kern_fork.c @@ -124,10 +124,12 @@ static void (*dtrace_proc_waitfor_hook)(proc_t) = NULL; #include <kern/thread_call.h> #include <kern/zalloc.h> -#include <machine/spl.h> +#include <os/log.h> + +#include <os/log.h> #if CONFIG_MACF -#include <security/mac.h> +#include <security/mac_framework.h> #include <security/mac_mach_internal.h> #endif @@ -385,6 +387,14 @@ fork1(proc_t parent_proc, thread_t *child_threadp, int kind, coalition_t *coalit uid = kauth_getruid(); proc_list_lock(); if ((nprocs >= maxproc - 1 && uid != 0) || nprocs >= maxproc) { +#if (DEVELOPMENT || DEBUG) && CONFIG_EMBEDDED + /* + * On the development kernel, panic so that the fact that we hit + * the process limit is obvious, as this may very well wedge the + * system. + */ + panic("The process table is full; parent pid=%d", parent_proc->p_pid); +#endif proc_list_unlock(); tablefull("proc"); return (EAGAIN); @@ -400,6 +410,15 @@ fork1(proc_t parent_proc, thread_t *child_threadp, int kind, coalition_t *coalit count = chgproccnt(uid, 1); if (uid != 0 && (rlim_t)count > parent_proc->p_rlimit[RLIMIT_NPROC].rlim_cur) { +#if (DEVELOPMENT || DEBUG) && CONFIG_EMBEDDED + /* + * On the development kernel, panic so that the fact that we hit + * the per user process limit is obvious. This may be less dire + * than hitting the global process limit, but we cannot rely on + * that. + */ + panic("The per-user process limit has been hit; parent pid=%d, uid=%d", parent_proc->p_pid, uid); +#endif err = EAGAIN; goto bad; } @@ -1210,6 +1229,13 @@ retry: LIST_INSERT_HEAD(PIDHASH(child_proc->p_pid), child_proc, p_hash); proc_list_unlock(); + if (child_proc->p_uniqueid == startup_serial_num_procs) { + /* + * Turn off startup serial logging now that we have reached + * the defined number of startup processes. + */ + startup_serial_logging_active = false; + } /* * We've identified the PID we are going to use; initialize the new @@ -1232,7 +1258,11 @@ retry: * Increase reference counts on shared objects. * The p_stats and p_sigacts substructs are set in vm_fork. */ +#if !CONFIG_EMBEDDED child_proc->p_flag = (parent_proc->p_flag & (P_LP64 | P_DISABLE_ASLR | P_DELAYIDLESLEEP | P_SUGID)); +#else /* !CONFIG_EMBEDDED */ + child_proc->p_flag = (parent_proc->p_flag & (P_LP64 | P_DISABLE_ASLR | P_SUGID)); +#endif /* !CONFIG_EMBEDDED */ if (parent_proc->p_flag & P_PROFIL) startprofclock(child_proc); @@ -1354,9 +1384,11 @@ retry: if ((parent_proc->p_lflag & P_LREGISTER) != 0) { child_proc->p_lflag |= P_LREGISTER; } - child_proc->p_wqkqueue = NULL; child_proc->p_dispatchqueue_offset = parent_proc->p_dispatchqueue_offset; child_proc->p_dispatchqueue_serialno_offset = parent_proc->p_dispatchqueue_serialno_offset; + child_proc->p_return_to_kernel_offset = parent_proc->p_return_to_kernel_offset; + child_proc->p_mach_thread_self_offset = parent_proc->p_mach_thread_self_offset; + child_proc->p_pth_tsd_offset = parent_proc->p_pth_tsd_offset; #if PSYNCH pth_proc_hashinit(child_proc); #endif /* PSYNCH */ @@ -1397,7 +1429,7 @@ bad: void proc_lock(proc_t p) { - lck_mtx_assert(proc_list_mlock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(proc_list_mlock, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(&p->p_mlock); } @@ -1615,12 +1647,11 @@ uthread_cleanup(task_t task, void *uthread, void * bsd_info) assert(uth->uu_ar == NULL); if (uth->uu_kqueue_bound) { - kevent_qos_internal_unbind(p, - uth->uu_kqueue_bound, + kevent_qos_internal_unbind(p, + 0, /* didn't save qos_class */ uth->uu_thread, uth->uu_kqueue_flags); - uth->uu_kqueue_flags = 0; - uth->uu_kqueue_bound = 0; + assert(uth->uu_kqueue_override_is_sync == 0); } sel = &uth->uu_select; diff --git a/bsd/kern/kern_guarded.c b/bsd/kern/kern_guarded.c index a24987fc0..ea583e9cf 100644 --- a/bsd/kern/kern_guarded.c +++ b/bsd/kern/kern_guarded.c @@ -31,6 +31,7 @@ #include <sys/filedesc.h> #include <sys/kernel.h> #include <sys/file_internal.h> +#include <kern/exc_guard.h> #include <sys/guarded.h> #include <kern/kalloc.h> #include <sys/sysproto.h> @@ -45,6 +46,14 @@ #include <sys/kdebug.h> #include <stdbool.h> #include <vm/vm_protos.h> +#include <libkern/section_keywords.h> +#if CONFIG_MACF && CONFIG_VNGUARD +#include <security/mac.h> +#include <security/mac_framework.h> +#include <security/mac_policy.h> +#include <pexpert/pexpert.h> +#include <sys/sysctl.h> +#endif #define f_flag f_fglob->fg_flag @@ -60,6 +69,7 @@ extern int wr_uio(struct proc *p, struct fileproc *fp, uio_t uio, user_ssize_t * kern_return_t task_exception_notify(exception_type_t exception, mach_exception_data_type_t code, mach_exception_data_type_t subcode); +kern_return_t task_violated_guard(mach_exception_code_t, mach_exception_subcode_t, void *); /* * Most fd's have an underlying fileproc struct; but some may be @@ -78,10 +88,7 @@ struct guarded_fileproc { struct fileproc gf_fileproc; u_int gf_magic; u_int gf_attrs; - thread_t gf_thread; guardid_t gf_guard; - int gf_exc_fd; - u_int gf_exc_code; }; const size_t sizeof_guarded_fileproc = sizeof (struct guarded_fileproc); @@ -180,48 +187,23 @@ fp_isguarded(struct fileproc *fp, u_int attrs) extern char *proc_name_address(void *p); int -fp_guard_exception(proc_t p, int fd, struct fileproc *fp, u_int code) +fp_guard_exception(proc_t p, int fd, struct fileproc *fp, u_int flavor) { if (FILEPROC_TYPE(fp) != FTYPE_GUARDED) panic("%s corrupt fp %p flags %x", __func__, fp, fp->f_flags); struct guarded_fileproc *gfp = FP_TO_GFP(fp); - /* all gfd fields protected via proc_fdlock() */ proc_fdlock_assert(p, LCK_MTX_ASSERT_OWNED); - if (NULL == gfp->gf_thread) { - thread_t t = current_thread(); - gfp->gf_thread = t; - gfp->gf_exc_fd = fd; - gfp->gf_exc_code = code; - - /* - * This thread was the first to attempt the - * operation that violated the guard on this fd; - * generate an exception. - */ - printf("%s: guarded fd exception: " - "fd %d code 0x%x guard 0x%llx\n", - proc_name_address(p), gfp->gf_exc_fd, - gfp->gf_exc_code, gfp->gf_guard); - - thread_guard_violation(t, GUARD_TYPE_FD); - } else { - /* - * We already recorded a violation on this fd for a - * different thread, so posting an exception is - * already in progress. We could pause for a bit - * and check again, or we could panic (though that seems - * heavy handed), or we could just press on with the - * error return alone. For now, resort to printf. - */ - printf("%s: guarded fd exception+: " - "fd %d code 0x%x guard 0x%llx\n", - proc_name_address(p), gfp->gf_exc_fd, - gfp->gf_exc_code, gfp->gf_guard); - } + mach_exception_code_t code = 0; + EXC_GUARD_ENCODE_TYPE(code, GUARD_TYPE_FD); + EXC_GUARD_ENCODE_FLAVOR(code, flavor); + EXC_GUARD_ENCODE_TARGET(code, fd); + mach_exception_subcode_t subcode = gfp->gf_guard; + thread_t t = current_thread(); + thread_guard_violation(t, code, subcode); return (EPERM); } @@ -229,73 +211,14 @@ fp_guard_exception(proc_t p, int fd, struct fileproc *fp, u_int code) * (Invoked before returning to userland from the syscall handler.) */ void -fd_guard_ast(thread_t t) +fd_guard_ast( + thread_t __unused t, + mach_exception_code_t code, + mach_exception_subcode_t subcode) { + task_exception_notify(EXC_GUARD, code, subcode); proc_t p = current_proc(); - struct filedesc *fdp = p->p_fd; - int i; - - proc_fdlock(p); - for (i = fdp->fd_lastfile; i >= 0; i--) { - struct fileproc *fp = fdp->fd_ofiles[i]; - - if (fp == NULL || - FILEPROC_TYPE(fp) != FTYPE_GUARDED) - continue; - - struct guarded_fileproc *gfp = FP_TO_GFP(fp); - - if (GUARDED_FILEPROC_MAGIC != gfp->gf_magic) - panic("%s: corrupt gfp %p flags %x", - __func__, gfp, fp->f_flags); - - if (gfp->gf_thread == t) { - mach_exception_data_type_t code, subcode; - - gfp->gf_thread = NULL; - - /* - * EXC_GUARD exception code namespace. - * - * code: - * +-------------------------------------------------+ - * | [63:61] guard type | [60:0] guard-specific data | - * +-------------------------------------------------+ - * - * subcode: - * +-------------------------------------------------+ - * | [63:0] guard-specific data | - * +-------------------------------------------------+ - * - * At the moment, we have just one guard type: file - * descriptor guards. - * - * File descriptor guards use the exception codes like - * so: - * - * code: - * +--------------------------------------------------+ - * |[63:61] GUARD_TYPE_FD | [60:32] flavor | [31:0] fd| - * +--------------------------------------------------+ - * - * subcode: - * +--------------------------------------------------+ - * | [63:0] guard value | - * +--------------------------------------------------+ - */ - code = (((uint64_t)GUARD_TYPE_FD) << 61) | - (((uint64_t)gfp->gf_exc_code) << 32) | - ((uint64_t)gfp->gf_exc_fd); - subcode = gfp->gf_guard; - proc_fdunlock(p); - - (void) task_exception_notify(EXC_GUARD, code, subcode); - psignal(p, SIGKILL); - - return; - } - } - proc_fdunlock(p); + psignal(p, SIGKILL); } /* @@ -665,6 +588,7 @@ restart: case DTYPE_PIPE: case DTYPE_SOCKET: case DTYPE_KQUEUE: + case DTYPE_NETPOLICY: break; default: error = ENOTSUP; @@ -981,3 +905,627 @@ falloc_guarded(struct proc *p, struct fileproc **fp, int *fd, return (falloc_withalloc(p, fp, fd, ctx, guarded_fileproc_alloc_init, &crarg)); } + +#if CONFIG_MACF && CONFIG_VNGUARD + +/* + * Guarded vnodes + * + * Uses MAC hooks to guard operations on vnodes in the system. Given an fd, + * add data to the label on the fileglob and the vnode it points at. + * The data contains a pointer to the fileglob, the set of attributes to + * guard, a guard value for uniquification, and the pid of the process + * who set the guard up in the first place. + * + * The fd must have been opened read/write, and the underlying + * fileglob is FG_CONFINED so that there's no ambiguity about the + * owning process. + * + * When there's a callback for a vnode operation of interest (rename, unlink, + * etc.) check to see if the guard permits that operation, and if not + * take an action e.g. log a message or generate a crash report. + * + * The label is removed from the vnode and the fileglob when the fileglob + * is closed. + * + * The initial action to be taken can be specified by a boot arg (vnguard=0x42) + * and change via the "kern.vnguard.flags" sysctl. + */ + +struct vng_owner; + +struct vng_info { /* lives on the vnode label */ + guardid_t vgi_guard; + unsigned vgi_attrs; + TAILQ_HEAD(, vng_owner) vgi_owners; +}; + +struct vng_owner { /* lives on the fileglob label */ + proc_t vgo_p; + struct fileglob *vgo_fg; + struct vng_info *vgo_vgi; + TAILQ_ENTRY(vng_owner) vgo_link; +}; + +static struct vng_info * +new_vgi(unsigned attrs, guardid_t guard) +{ + struct vng_info *vgi = kalloc(sizeof (*vgi)); + vgi->vgi_guard = guard; + vgi->vgi_attrs = attrs; + TAILQ_INIT(&vgi->vgi_owners); + return vgi; +} + +static struct vng_owner * +new_vgo(proc_t p, struct fileglob *fg) +{ + struct vng_owner *vgo = kalloc(sizeof (*vgo)); + memset(vgo, 0, sizeof (*vgo)); + vgo->vgo_p = p; + vgo->vgo_fg = fg; + return vgo; +} + +static void +vgi_add_vgo(struct vng_info *vgi, struct vng_owner *vgo) +{ + vgo->vgo_vgi = vgi; + TAILQ_INSERT_HEAD(&vgi->vgi_owners, vgo, vgo_link); +} + +static boolean_t +vgi_remove_vgo(struct vng_info *vgi, struct vng_owner *vgo) +{ + TAILQ_REMOVE(&vgi->vgi_owners, vgo, vgo_link); + vgo->vgo_vgi = NULL; + return TAILQ_EMPTY(&vgi->vgi_owners); +} + +static void +free_vgi(struct vng_info *vgi) +{ + assert(TAILQ_EMPTY(&vgi->vgi_owners)); +#if DEVELOP || DEBUG + memset(vgi, 0xbeadfade, sizeof (*vgi)); +#endif + kfree(vgi, sizeof (*vgi)); +} + +static void +free_vgo(struct vng_owner *vgo) +{ +#if DEVELOP || DEBUG + memset(vgo, 0x2bedf1d0, sizeof (*vgo)); +#endif + kfree(vgo, sizeof (*vgo)); +} + +static int label_slot; +static lck_rw_t llock; +static lck_grp_t *llock_grp; + +static __inline void * +vng_lbl_get(struct label *label) +{ + lck_rw_assert(&llock, LCK_RW_ASSERT_HELD); + void *data; + if (NULL == label) + data = NULL; + else + data = (void *)mac_label_get(label, label_slot); + return data; +} + +static __inline struct vng_info * +vng_lbl_get_withattr(struct label *label, unsigned attrmask) +{ + struct vng_info *vgi = vng_lbl_get(label); + assert(NULL == vgi || (vgi->vgi_attrs & ~VNG_ALL) == 0); + if (NULL != vgi && 0 == (vgi->vgi_attrs & attrmask)) + vgi = NULL; + return vgi; +} + +static __inline void +vng_lbl_set(struct label *label, void *data) +{ + assert(NULL != label); + lck_rw_assert(&llock, LCK_RW_ASSERT_EXCLUSIVE); + mac_label_set(label, label_slot, (intptr_t)data); +} + +static int +vnguard_sysc_setguard(proc_t p, const struct vnguard_set *vns) +{ + const int fd = vns->vns_fd; + + if ((vns->vns_attrs & ~VNG_ALL) != 0 || + 0 == vns->vns_attrs || 0 == vns->vns_guard) + return EINVAL; + + int error; + struct fileproc *fp; + if (0 != (error = fp_lookup(p, fd, &fp, 0))) + return error; + do { + /* + * To avoid trivial DoS, insist that the caller + * has read/write access to the file. + */ + if ((FREAD|FWRITE) != (fp->f_flag & (FREAD|FWRITE))) { + error = EBADF; + break; + } + struct fileglob *fg = fp->f_fglob; + if (FILEGLOB_DTYPE(fg) != DTYPE_VNODE) { + error = EBADF; + break; + } + /* + * Confinement means there's only one fd pointing at + * this fileglob, and will always be associated with + * this pid. + */ + if (0 == (FG_CONFINED & fg->fg_lflags)) { + error = EBADF; + break; + } + struct vnode *vp = fg->fg_data; + if (!vnode_isreg(vp) || NULL == vp->v_mount) { + error = EBADF; + break; + } + error = vnode_getwithref(vp); + if (0 != error) { + fp_drop(p, fd, fp, 0); + break; + } + /* Ensure the target vnode -has- a label */ + struct vfs_context *ctx = vfs_context_current(); + mac_vnode_label_update(ctx, vp, NULL); + + struct vng_info *nvgi = new_vgi(vns->vns_attrs, vns->vns_guard); + struct vng_owner *nvgo = new_vgo(p, fg); + + lck_rw_lock_exclusive(&llock); + + do { + /* + * A vnode guard is associated with one or more + * fileglobs in one or more processes. + */ + struct vng_info *vgi = vng_lbl_get(vp->v_label); + struct vng_owner *vgo = vng_lbl_get(fg->fg_label); + + if (NULL == vgi) { + /* vnode unguarded, add the first guard */ + if (NULL != vgo) + panic("vnguard label on fileglob " + "but not vnode"); + /* add a kusecount so we can unlabel later */ + error = vnode_ref_ext(vp, O_EVTONLY, 0); + if (0 == error) { + /* add the guard */ + vgi_add_vgo(nvgi, nvgo); + vng_lbl_set(vp->v_label, nvgi); + vng_lbl_set(fg->fg_label, nvgo); + } else { + free_vgo(nvgo); + free_vgi(nvgi); + } + } else { + /* vnode already guarded */ + free_vgi(nvgi); + if (vgi->vgi_guard != vns->vns_guard) + error = EPERM; /* guard mismatch */ + else if (vgi->vgi_attrs != vns->vns_attrs) + error = EACCES; /* attr mismatch */ + if (0 != error || NULL != vgo) { + free_vgo(nvgo); + break; + } + /* record shared ownership */ + vgi_add_vgo(vgi, nvgo); + vng_lbl_set(fg->fg_label, nvgo); + } + } while (0); + + lck_rw_unlock_exclusive(&llock); + vnode_put(vp); + } while (0); + + fp_drop(p, fd, fp, 0); + return error; +} + +static int +vng_policy_syscall(proc_t p, int cmd, user_addr_t arg) +{ + int error = EINVAL; + + switch (cmd) { + case VNG_SYSC_PING: + if (0 == arg) + error = 0; + break; + case VNG_SYSC_SET_GUARD: { + struct vnguard_set vns; + error = copyin(arg, (void *)&vns, sizeof (vns)); + if (error) + break; + error = vnguard_sysc_setguard(p, &vns); + break; + } + default: + break; + } + return (error); +} + +/* + * This is called just before the fileglob disappears in fg_free(). + * Take the exclusive lock: no other thread can add or remove + * a vng_info to any vnode in the system. + */ +static void +vng_file_label_destroy(struct label *label) +{ + lck_rw_lock_exclusive(&llock); + struct vng_owner *lvgo = vng_lbl_get(label); + if (lvgo) { + vng_lbl_set(label, 0); + struct vng_info *vgi = lvgo->vgo_vgi; + assert(vgi); + if (vgi_remove_vgo(vgi, lvgo)) { + /* that was the last reference */ + vgi->vgi_attrs = 0; + struct fileglob *fg = lvgo->vgo_fg; + assert(fg); + if (DTYPE_VNODE == FILEGLOB_DTYPE(fg)) { + struct vnode *vp = fg->fg_data; + int error = vnode_getwithref(vp); + if (0 == error) { + vng_lbl_set(vp->v_label, 0); + lck_rw_unlock_exclusive(&llock); + /* may trigger VNOP_INACTIVE */ + vnode_rele_ext(vp, O_EVTONLY, 0); + vnode_put(vp); + free_vgi(vgi); + free_vgo(lvgo); + return; + } + } + } + free_vgo(lvgo); + } + lck_rw_unlock_exclusive(&llock); +} + +static int vng_policy_flags; + +static int +vng_guard_violation(const struct vng_info *vgi, + unsigned opval, const char *nm) +{ + int retval = 0; + + if (vng_policy_flags & kVNG_POLICY_EPERM) { + /* deny the operation */ + retval = EPERM; + } + + if (vng_policy_flags & kVNG_POLICY_LOGMSG) { + /* log a message */ + const char *op; + switch (opval) { + case VNG_RENAME_FROM: + op = "rename-from"; + break; + case VNG_RENAME_TO: + op = "rename-to"; + break; + case VNG_UNLINK: + op = "unlink"; + break; + case VNG_LINK: + op = "link"; + break; + case VNG_EXCHDATA: + op = "exchdata"; + break; + case VNG_WRITE_OTHER: + op = "write"; + break; + case VNG_TRUNC_OTHER: + op = "truncate"; + break; + default: + op = "(unknown)"; + break; + } + proc_t p = current_proc(); + const struct vng_owner *vgo; + TAILQ_FOREACH(vgo, &vgi->vgi_owners, vgo_link) { + printf("%s[%d]: %s%s: '%s' guarded by %s[%d] (0x%llx)\n", + proc_name_address(p), proc_pid(p), op, + 0 != retval ? " denied" : "", + NULL != nm ? nm : "(unknown)", + proc_name_address(vgo->vgo_p), proc_pid(vgo->vgo_p), + vgi->vgi_guard); + } + } + + if (vng_policy_flags & (kVNG_POLICY_EXC|kVNG_POLICY_EXC_CORPSE)) { + /* EXC_GUARD exception */ + const struct vng_owner *vgo = TAILQ_FIRST(&vgi->vgi_owners); + pid_t pid = vgo ? proc_pid(vgo->vgo_p) : 0; + mach_exception_code_t code; + mach_exception_subcode_t subcode; + + code = 0; + EXC_GUARD_ENCODE_TYPE(code, GUARD_TYPE_VN); + EXC_GUARD_ENCODE_FLAVOR(code, opval); + EXC_GUARD_ENCODE_TARGET(code, pid); + subcode = vgi->vgi_guard; + + if (vng_policy_flags & kVNG_POLICY_EXC_CORPSE) { + task_violated_guard(code, subcode, NULL); + /* not fatal */ + } else { + thread_t t = current_thread(); + thread_guard_violation(t, code, subcode); + } + } else if (vng_policy_flags & kVNG_POLICY_SIGKILL) { + proc_t p = current_proc(); + psignal(p, SIGKILL); + } + + return retval; +} + +/* + * A vnode guard was tripped on this thread. + * + * (Invoked before returning to userland from the syscall handler.) + */ +void +vn_guard_ast(thread_t __unused t, + mach_exception_data_type_t code, mach_exception_data_type_t subcode) +{ + task_exception_notify(EXC_GUARD, code, subcode); + proc_t p = current_proc(); + psignal(p, SIGKILL); +} + +/* + * vnode callbacks + */ + +static int +vng_vnode_check_rename(kauth_cred_t __unused cred, + struct vnode *__unused dvp, struct label *__unused dlabel, + struct vnode *__unused vp, struct label *label, + struct componentname *cnp, + struct vnode *__unused tdvp, struct label *__unused tdlabel, + struct vnode *__unused tvp, struct label *tlabel, + struct componentname *tcnp) +{ + int error = 0; + if (NULL != label || NULL != tlabel) { + lck_rw_lock_shared(&llock); + const struct vng_info *vgi = + vng_lbl_get_withattr(label, VNG_RENAME_FROM); + if (NULL != vgi) + error = vng_guard_violation(vgi, + VNG_RENAME_FROM, cnp->cn_nameptr); + if (0 == error) { + vgi = vng_lbl_get_withattr(tlabel, VNG_RENAME_TO); + if (NULL != vgi) + error = vng_guard_violation(vgi, + VNG_RENAME_TO, tcnp->cn_nameptr); + } + lck_rw_unlock_shared(&llock); + } + return error; +} + +static int +vng_vnode_check_link(kauth_cred_t __unused cred, + struct vnode *__unused dvp, struct label *__unused dlabel, + struct vnode *vp, struct label *label, struct componentname *__unused cnp) +{ + int error = 0; + if (NULL != label) { + lck_rw_lock_shared(&llock); + const struct vng_info *vgi = + vng_lbl_get_withattr(label, VNG_LINK); + if (vgi) { + const char *nm = vnode_getname(vp); + error = vng_guard_violation(vgi, VNG_LINK, nm); + if (nm) + vnode_putname(nm); + } + lck_rw_unlock_shared(&llock); + } + return error; +} + +static int +vng_vnode_check_unlink(kauth_cred_t __unused cred, + struct vnode *__unused dvp, struct label *__unused dlabel, + struct vnode *__unused vp, struct label *label, struct componentname *cnp) +{ + int error = 0; + if (NULL != label) { + lck_rw_lock_shared(&llock); + const struct vng_info *vgi = + vng_lbl_get_withattr(label, VNG_UNLINK); + if (vgi) + error = vng_guard_violation(vgi, VNG_UNLINK, + cnp->cn_nameptr); + lck_rw_unlock_shared(&llock); + } + return error; +} + +/* + * Only check violations for writes performed by "other processes" + */ +static int +vng_vnode_check_write(kauth_cred_t __unused actv_cred, + kauth_cred_t __unused file_cred, struct vnode *vp, struct label *label) +{ + int error = 0; + if (NULL != label) { + lck_rw_lock_shared(&llock); + const struct vng_info *vgi = + vng_lbl_get_withattr(label, VNG_WRITE_OTHER); + if (vgi) { + proc_t p = current_proc(); + const struct vng_owner *vgo; + TAILQ_FOREACH(vgo, &vgi->vgi_owners, vgo_link) { + if (vgo->vgo_p == p) + goto done; + } + const char *nm = vnode_getname(vp); + error = vng_guard_violation(vgi, + VNG_WRITE_OTHER, nm); + if (nm) + vnode_putname(nm); + } + done: + lck_rw_unlock_shared(&llock); + } + return error; +} + +/* + * Only check violations for truncates performed by "other processes" + */ +static int +vng_vnode_check_truncate(kauth_cred_t __unused actv_cred, + kauth_cred_t __unused file_cred, struct vnode *vp, + struct label *label) +{ + int error = 0; + if (NULL != label) { + lck_rw_lock_shared(&llock); + const struct vng_info *vgi = + vng_lbl_get_withattr(label, VNG_TRUNC_OTHER); + if (vgi) { + proc_t p = current_proc(); + const struct vng_owner *vgo; + TAILQ_FOREACH(vgo, &vgi->vgi_owners, vgo_link) { + if (vgo->vgo_p == p) + goto done; + } + const char *nm = vnode_getname(vp); + error = vng_guard_violation(vgi, + VNG_TRUNC_OTHER, nm); + if (nm) + vnode_putname(nm); + } + done: + lck_rw_unlock_shared(&llock); + } + return error; +} + +static int +vng_vnode_check_exchangedata(kauth_cred_t __unused cred, + struct vnode *fvp, struct label *flabel, + struct vnode *svp, struct label *slabel) +{ + int error = 0; + if (NULL != flabel || NULL != slabel) { + lck_rw_lock_shared(&llock); + const struct vng_info *vgi = + vng_lbl_get_withattr(flabel, VNG_EXCHDATA); + if (NULL != vgi) { + const char *nm = vnode_getname(fvp); + error = vng_guard_violation(vgi, + VNG_EXCHDATA, nm); + if (nm) + vnode_putname(nm); + } + if (0 == error) { + vgi = vng_lbl_get_withattr(slabel, VNG_EXCHDATA); + if (NULL != vgi) { + const char *nm = vnode_getname(svp); + error = vng_guard_violation(vgi, + VNG_EXCHDATA, nm); + if (nm) + vnode_putname(nm); + } + } + lck_rw_unlock_shared(&llock); + } + return error; +} + +/* + * Configuration gorp + */ + +static void +vng_init(struct mac_policy_conf *mpc) +{ + llock_grp = lck_grp_alloc_init(mpc->mpc_name, LCK_GRP_ATTR_NULL); + lck_rw_init(&llock, llock_grp, LCK_ATTR_NULL); +} + +SECURITY_READ_ONLY_EARLY(static struct mac_policy_ops) vng_policy_ops = { + .mpo_file_label_destroy = vng_file_label_destroy, + + .mpo_vnode_check_link = vng_vnode_check_link, + .mpo_vnode_check_unlink = vng_vnode_check_unlink, + .mpo_vnode_check_rename = vng_vnode_check_rename, + .mpo_vnode_check_write = vng_vnode_check_write, + .mpo_vnode_check_truncate = vng_vnode_check_truncate, + .mpo_vnode_check_exchangedata = vng_vnode_check_exchangedata, + + .mpo_policy_syscall = vng_policy_syscall, + .mpo_policy_init = vng_init, +}; + +static const char *vng_labelnames[] = { + "vnguard", +}; + +#define ACOUNT(arr) ((unsigned)(sizeof (arr) / sizeof (arr[0]))) + +SECURITY_READ_ONLY_LATE(static struct mac_policy_conf) vng_policy_conf = { + .mpc_name = VNG_POLICY_NAME, + .mpc_fullname = "Guarded vnode policy", + .mpc_field_off = &label_slot, + .mpc_labelnames = vng_labelnames, + .mpc_labelname_count = ACOUNT(vng_labelnames), + .mpc_ops = &vng_policy_ops, + .mpc_loadtime_flags = 0, + .mpc_runtime_flags = 0 +}; + +static mac_policy_handle_t vng_policy_handle; + +void +vnguard_policy_init(void) +{ + if (0 == PE_i_can_has_debugger(NULL)) + return; + vng_policy_flags = kVNG_POLICY_LOGMSG | kVNG_POLICY_EXC_CORPSE; + PE_parse_boot_argn("vnguard", &vng_policy_flags, sizeof (vng_policy_flags)); + if (vng_policy_flags) + mac_policy_register(&vng_policy_conf, &vng_policy_handle, NULL); +} + +#if DEBUG || DEVELOPMENT +#include <sys/sysctl.h> + +SYSCTL_DECL(_kern_vnguard); +SYSCTL_NODE(_kern, OID_AUTO, vnguard, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "vnguard"); +SYSCTL_INT(_kern_vnguard, OID_AUTO, flags, CTLFLAG_RW | CTLFLAG_LOCKED, + &vng_policy_flags, 0, "vnguard policy flags"); +#endif + +#endif /* CONFIG_MACF && CONFIG_VNGUARD */ diff --git a/bsd/kern/kern_kpc.c b/bsd/kern/kern_kpc.c index 38bc2abbd..098b7349f 100644 --- a/bsd/kern/kern_kpc.c +++ b/bsd/kern/kern_kpc.c @@ -60,25 +60,39 @@ typedef int (*getint_t)(void); typedef int (*setint_t)(int); -/* safety */ static int kpc_initted = 0; -/* locking and buffer for large data requests */ -#define SYSCTL_BUFFER_SIZE (33 * sizeof(uint64_t)) -static lck_grp_attr_t *sysctl_buffer_lckgrp_attr = NULL; -static lck_grp_t *sysctl_buffer_lckgrp = NULL; -static lck_mtx_t sysctl_buffer_lock; -static void *sysctl_buffer = NULL; +static lck_grp_attr_t *sysctl_lckgrp_attr = NULL; +static lck_grp_t *sysctl_lckgrp = NULL; +static lck_mtx_t sysctl_lock; + +#if defined(__x86_64__) +/* 18 cores, 7 counters each */ +#define KPC_MAX_COUNTERS_COPIED (18 * 7) +#elif defined(__arm64__) +#include <pexpert/arm64/board_config.h> +#if defined(CPU_COUNT) +#define KPC_MAX_COUNTERS_COPIED (CPU_COUNT * 10) +#else /* defined(CPU_COUNT) */ +#define KPC_MAX_COUNTERS_COPIED (2 * 10) +#endif /* !defined(CPU_COUNT) */ +#elif defined(__arm__) +#define KPC_MAX_COUNTERS_COPIED (16) +#else /* !defined(__arm__) && !defined(__arm64__) && !defined(__x86_64__) */ +#error "unknown architecture for kpc buffer sizes" +#endif /* !defined(__arm__) && !defined(__arm64__) && !defined(__x86_64__) */ + +static_assert((KPC_MAX_COUNTERS_COPIED * sizeof(uint64_t)) < 1024, + "kpc's stack could grow too large"); typedef int (*setget_func_t)(int); void kpc_init(void) { - sysctl_buffer_lckgrp_attr = lck_grp_attr_alloc_init(); - sysctl_buffer_lckgrp = lck_grp_alloc_init("kpc", - sysctl_buffer_lckgrp_attr); - lck_mtx_init(&sysctl_buffer_lock, sysctl_buffer_lckgrp, LCK_ATTR_NULL); + sysctl_lckgrp_attr = lck_grp_attr_alloc_init(); + sysctl_lckgrp = lck_grp_alloc_init("kpc", sysctl_lckgrp_attr); + lck_mtx_init(&sysctl_lock, sysctl_lckgrp, LCK_ATTR_NULL); kpc_arch_init(); kpc_common_init(); @@ -156,26 +170,6 @@ sysctl_setget_int( struct sysctl_req *req, } static int -kpc_sysctl_acquire_buffer(void) -{ - if( sysctl_buffer == NULL ) - { - sysctl_buffer = kalloc(SYSCTL_BUFFER_SIZE); - if( sysctl_buffer ) - { - bzero( sysctl_buffer, SYSCTL_BUFFER_SIZE ); - } - } - - if( !sysctl_buffer ) - { - return ENOMEM; - } - - return 0; -} - -static int sysctl_kpc_get_counters(uint32_t counters, uint32_t *size, void *buf) { @@ -218,7 +212,7 @@ sysctl_kpc_get_shadow_counters(uint32_t counters, return 0; } -static int +static int sysctl_kpc_get_thread_counters(uint32_t tid, uint32_t *size, void *buf) { @@ -233,7 +227,7 @@ sysctl_kpc_get_thread_counters(uint32_t tid, *size = count * sizeof(uint64_t); return r; -} +} static int sysctl_kpc_get_config(uint32_t classes, void* buf) @@ -279,35 +273,23 @@ sysctl_kpc_set_actionid(uint32_t classes, void* buf) static int -sysctl_get_bigarray( struct sysctl_req *req, - int (*get_fn)(uint32_t, uint32_t*, void*) ) +sysctl_get_bigarray(struct sysctl_req *req, + int (*get_fn)(uint32_t, uint32_t*, void*)) { - int error = 0; - uint32_t bufsize = SYSCTL_BUFFER_SIZE; + uint64_t buf[KPC_MAX_COUNTERS_COPIED] = {}; + uint32_t bufsize = sizeof(buf); uint32_t arg = 0; /* get the argument */ - error = SYSCTL_IN( req, &arg, sizeof(arg) ); - if(error) - { - printf( "kpc: no arg?\n" ); + int error = SYSCTL_IN(req, &arg, sizeof(arg)); + if (error) { return error; } - /* get the wired buffer */ - error = kpc_sysctl_acquire_buffer(); - if (error) - return error; - - /* atomically get the array into the wired buffer. We have a double - * copy, but this is better than page faulting / interrupting during - * a copy. - */ - error = get_fn( arg, &bufsize, sysctl_buffer ); - - /* do the copy out */ - if( !error ) - error = SYSCTL_OUT( req, sysctl_buffer, bufsize ); + error = get_fn(arg, &bufsize, &buf); + if (!error) { + error = SYSCTL_OUT(req, &buf, bufsize); + } return error; } @@ -332,79 +314,53 @@ sysctl_actionid_size( uint32_t classes ) } static int -sysctl_getset_bigarray( struct sysctl_req *req, - int (*size_fn)(uint32_t arg), - int (*get_fn)(uint32_t, void*), - int (*set_fn)(uint32_t, void*) ) +sysctl_getset_bigarray(struct sysctl_req *req, int (*size_fn)(uint32_t arg), + int (*get_fn)(uint32_t, void*), int (*set_fn)(uint32_t, void*)) { int error = 0; - uint32_t bufsize = SYSCTL_BUFFER_SIZE; - uint32_t regsize = 0; + uint64_t buf[KPC_MAX_COUNTERS_COPIED] = {}; + uint32_t bufsize = sizeof(buf); uint64_t arg; /* get the config word */ - error = SYSCTL_IN( req, &arg, sizeof(arg) ); - if(error) - { - printf( "kpc: no arg?\n" ); + error = SYSCTL_IN(req, &arg, sizeof(arg)); + if (error) { return error; } - /* Work out size of registers */ - regsize = size_fn((uint32_t)arg); - - /* Ignore NULL requests */ - if(regsize == 0) - return EINVAL; - - /* ensure not too big */ - if( regsize > bufsize ) + /* Determine the size of registers to modify. */ + uint32_t regsize = size_fn((uint32_t)arg); + if (regsize == 0 || regsize > bufsize) { return EINVAL; + } - /* get the wired buffer */ - error = kpc_sysctl_acquire_buffer(); - if (error) - return error; - - // if writing... - if(req->newptr) - { - // copy in the rest in -- sysctl remembers we did one already - error = SYSCTL_IN( req, sysctl_buffer, - regsize ); - - // if SYSCTL_IN fails it means we are only doing a read - if(!error) { - // set it - error = set_fn( (uint32_t)arg, sysctl_buffer ); - if( error ) - goto fail; + /* if writing */ + if (req->newptr) { + /* copy the rest -- SYSCTL_IN knows the copyin should be shifted */ + error = SYSCTL_IN(req, &buf, regsize); + + /* SYSCTL_IN failure means only need to read */ + if (!error) { + error = set_fn((uint32_t)arg, &buf); + if (error) { + return error; + } } } - // if reading - if(req->oldptr) - { - // read it - error = get_fn( (uint32_t)arg, sysctl_buffer ); - if( error ) - goto fail; + /* if reading */ + if (req->oldptr) { + error = get_fn((uint32_t)arg, &buf); + if (error) { + return error; + } - // copy out the full set - error = SYSCTL_OUT( req, sysctl_buffer, regsize ); + error = SYSCTL_OUT(req, &buf, regsize); } - -fail: + return error; } - - -/* - * #define SYSCTL_HANDLER_ARGS (struct sysctl_oid *oidp, \ - * void *arg1, int arg2, \ - * struct sysctl_req *req ) - */ static int kpc_sysctl SYSCTL_HANDLER_ARGS { @@ -412,11 +368,11 @@ kpc_sysctl SYSCTL_HANDLER_ARGS // __unused struct sysctl_oid *unused_oidp = oidp; (void)arg2; - + if( !kpc_initted ) panic("kpc_init not called"); - lck_mtx_lock(ktrace_lock); + ktrace_lock(); // Most sysctls require an access check, but a few are public. switch( (uintptr_t) arg1 ) { @@ -430,15 +386,15 @@ kpc_sysctl SYSCTL_HANDLER_ARGS // Require kperf access to read or write anything else. // This is either root or the blessed pid. if ((ret = ktrace_read_check())) { - lck_mtx_unlock(ktrace_lock); + ktrace_unlock(); return ret; } break; } - lck_mtx_unlock(ktrace_lock); + ktrace_unlock(); - lck_mtx_lock(&sysctl_buffer_lock); + lck_mtx_lock(&sysctl_lock); /* which request */ switch( (uintptr_t) arg1 ) @@ -505,7 +461,7 @@ kpc_sysctl SYSCTL_HANDLER_ARGS case REQ_SW_INC: ret = sysctl_set_int( req, (setget_func_t)kpc_set_sw_inc ); - break; + break; case REQ_PMU_VERSION: ret = sysctl_get_int(oidp, req, kpc_get_pmu_version()); @@ -516,8 +472,8 @@ kpc_sysctl SYSCTL_HANDLER_ARGS break; } - lck_mtx_unlock(&sysctl_buffer_lock); - + lck_mtx_unlock(&sysctl_lock); + return ret; } diff --git a/bsd/kern/kern_ktrace.c b/bsd/kern/kern_ktrace.c index af4573ef8..628a19a55 100644 --- a/bsd/kern/kern_ktrace.c +++ b/bsd/kern/kern_ktrace.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015 Apple Inc. All rights reserved. + * Copyright (c) 2015-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -70,7 +70,7 @@ char *proc_name_address(void *p); kern_return_t ktrace_background_available_notify_user(void); -lck_mtx_t *ktrace_lock; +static lck_mtx_t *ktrace_mtx; /* * The overall state of ktrace, whether it is unconfigured, in foreground mode, @@ -105,7 +105,7 @@ static uint32_t ktrace_active_mask = 0; * * Background tools must be RunAtLoad daemons. */ -static boolean_t should_notify_on_init = TRUE; +static bool should_notify_on_init = true; /* Set the owning process of ktrace. */ static void ktrace_set_owning_proc(proc_t p); @@ -124,7 +124,10 @@ static void ktrace_promote_background(void); * This is managed by the user space-oriented function ktrace_set_owning_pid * and ktrace_unset_owning_pid. */ -boolean_t ktrace_keep_ownership_on_reset = FALSE; +bool ktrace_keep_ownership_on_reset = false; + +/* Whether the kernel is the owner of ktrace. */ +bool ktrace_owner_kernel = false; /* Allow user space to unset the owning pid and potentially reset ktrace. */ static void ktrace_set_invalid_owning_pid(void); @@ -135,6 +138,48 @@ static void ktrace_set_invalid_owning_pid(void); */ int ktrace_root_set_owner_allowed = 0; +/* + * If ktrace is guaranteed that it's the only thread running on the system + * (e.g., during boot or wake) this flag disables locking requirements. + */ +static bool ktrace_single_threaded = false; + +void +ktrace_lock(void) +{ + if (!ktrace_single_threaded) { + lck_mtx_lock(ktrace_mtx); + } +} + +void +ktrace_unlock(void) +{ + if (!ktrace_single_threaded) { + lck_mtx_unlock(ktrace_mtx); + } +} + +void +ktrace_assert_lock_held(void) +{ + if (!ktrace_single_threaded) { + lck_mtx_assert(ktrace_mtx, LCK_MTX_ASSERT_OWNED); + } +} + +void +ktrace_start_single_threaded(void) +{ + ktrace_single_threaded = true; +} + +void +ktrace_end_single_threaded(void) +{ + ktrace_single_threaded = false; +} + static void ktrace_reset_internal(uint32_t reset_mask) { @@ -155,7 +200,7 @@ ktrace_reset_internal(uint32_t reset_mask) ktrace_promote_background(); } else if (ktrace_state == KTRACE_STATE_BG) { /* background tool is resetting ktrace */ - should_notify_on_init = TRUE; + should_notify_on_init = true; ktrace_release_ownership(); ktrace_state = KTRACE_STATE_OFF; } @@ -165,7 +210,7 @@ ktrace_reset_internal(uint32_t reset_mask) void ktrace_reset(uint32_t reset_mask) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); if (ktrace_active_mask == 0) { if (!ktrace_keep_ownership_on_reset) { @@ -180,7 +225,6 @@ ktrace_reset(uint32_t reset_mask) static void ktrace_promote_background(void) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); assert(ktrace_state != KTRACE_STATE_BG); /* @@ -189,9 +233,9 @@ ktrace_promote_background(void) * for the host special port). */ if (ktrace_background_available_notify_user() == KERN_FAILURE) { - should_notify_on_init = TRUE; + should_notify_on_init = true; } else { - should_notify_on_init = FALSE; + should_notify_on_init = false; } ktrace_release_ownership(); @@ -201,14 +245,13 @@ ktrace_promote_background(void) bool ktrace_background_active(void) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); return (ktrace_state == KTRACE_STATE_BG); } int ktrace_read_check(void) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); if (proc_uniqueid(current_proc()) == ktrace_owning_unique_id) { @@ -222,7 +265,7 @@ ktrace_read_check(void) static void ktrace_ownership_maintenance(void) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); /* do nothing if ktrace is not owned */ if (ktrace_owning_unique_id == 0) { @@ -248,7 +291,7 @@ ktrace_ownership_maintenance(void) int ktrace_configure(uint32_t config_mask) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); assert(config_mask != 0); proc_t p = current_proc(); @@ -274,6 +317,7 @@ ktrace_configure(uint32_t config_mask) return EPERM; } + ktrace_owner_kernel = false; ktrace_set_owning_proc(p); ktrace_active_mask |= config_mask; return 0; @@ -295,7 +339,7 @@ ktrace_disable(enum ktrace_state state_to_match) int ktrace_get_owning_pid(void) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); ktrace_ownership_maintenance(); return ktrace_owning_pid; @@ -304,18 +348,24 @@ ktrace_get_owning_pid(void) void ktrace_kernel_configure(uint32_t config_mask) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + assert(ktrace_single_threaded == true); + + if (ktrace_owner_kernel) { + ktrace_active_mask |= config_mask; + return; + } if (ktrace_state != KTRACE_STATE_OFF) { - if (ktrace_active_mask & KTRACE_KPERF) { + if (ktrace_active_mask & config_mask & KTRACE_KPERF) { kperf_reset(); } - if (ktrace_active_mask & KTRACE_KDEBUG) { + if (ktrace_active_mask & config_mask & KTRACE_KDEBUG) { kdebug_reset(); } } - ktrace_active_mask = config_mask; + ktrace_owner_kernel = true; + ktrace_active_mask |= config_mask; ktrace_state = KTRACE_STATE_FG; ktrace_release_ownership(); @@ -328,7 +378,7 @@ ktrace_init_background(void) { int err = 0; - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); if ((err = priv_check_cred(kauth_cred_get(), PRIV_KTRACE_BACKGROUND, 0))) { return err; @@ -350,7 +400,7 @@ ktrace_init_background(void) return EINVAL; } } - should_notify_on_init = FALSE; + should_notify_on_init = false; } proc_t p = current_proc(); @@ -369,7 +419,7 @@ void ktrace_set_invalid_owning_pid(void) { if (ktrace_keep_ownership_on_reset) { - ktrace_keep_ownership_on_reset = FALSE; + ktrace_keep_ownership_on_reset = false; ktrace_reset_internal(ktrace_active_mask); } } @@ -377,7 +427,7 @@ ktrace_set_invalid_owning_pid(void) int ktrace_set_owning_pid(int pid) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); + ktrace_assert_lock_held(); /* allow user space to successfully unset owning pid */ if (pid == -1) { @@ -397,7 +447,7 @@ ktrace_set_owning_pid(int pid) return ESRCH; } - ktrace_keep_ownership_on_reset = TRUE; + ktrace_keep_ownership_on_reset = true; ktrace_set_owning_proc(p); proc_rele(p); @@ -407,8 +457,8 @@ ktrace_set_owning_pid(int pid) static void ktrace_set_owning_proc(proc_t p) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); - assert(p); + ktrace_assert_lock_held(); + assert(p != NULL); if (ktrace_state != KTRACE_STATE_FG) { if (proc_uniqueid(p) == ktrace_bg_unique_id) { @@ -425,10 +475,11 @@ ktrace_set_owning_proc(proc_t p) ktrace_active_mask = 0; } ktrace_state = KTRACE_STATE_FG; - should_notify_on_init = FALSE; + should_notify_on_init = false; } } + ktrace_owner_kernel = false; ktrace_owning_unique_id = proc_uniqueid(p); ktrace_owning_pid = proc_pid(p); strlcpy(ktrace_last_owner_execname, proc_name_address(p), @@ -438,8 +489,6 @@ ktrace_set_owning_proc(proc_t p) static void ktrace_release_ownership(void) { - lck_mtx_assert(ktrace_lock, LCK_MTX_ASSERT_OWNED); - ktrace_owning_unique_id = 0; ktrace_owning_pid = 0; } @@ -477,7 +526,7 @@ ktrace_sysctl SYSCTL_HANDLER_ARGS int ret = 0; uintptr_t type = (uintptr_t)arg1; - lck_mtx_lock(ktrace_lock); + ktrace_lock(); if (!kauth_cred_issuser(kauth_cred_get())) { ret = EPERM; @@ -498,7 +547,7 @@ ktrace_sysctl SYSCTL_HANDLER_ARGS } out: - lck_mtx_unlock(ktrace_lock); + ktrace_unlock(); return ret; } @@ -508,7 +557,7 @@ ktrace_init(void) { static lck_grp_attr_t *lock_grp_attr = NULL; static lck_grp_t *lock_grp = NULL; - static boolean_t initialized = FALSE; + static bool initialized = false; if (initialized) { return; @@ -518,7 +567,7 @@ ktrace_init(void) lock_grp = lck_grp_alloc_init("ktrace", lock_grp_attr); lck_grp_attr_free(lock_grp_attr); - ktrace_lock = lck_mtx_alloc_init(lock_grp, LCK_ATTR_NULL); - assert(ktrace_lock); - initialized = TRUE; + ktrace_mtx = lck_mtx_alloc_init(lock_grp, LCK_ATTR_NULL); + assert(ktrace_mtx != NULL);; + initialized = true; } diff --git a/bsd/kern/kern_lockf.c b/bsd/kern/kern_lockf.c index f0bfa129f..5284f060c 100644 --- a/bsd/kern/kern_lockf.c +++ b/bsd/kern/kern_lockf.c @@ -330,6 +330,12 @@ lf_advlock(struct vnop_advlock_args *ap) FREE(lock, M_LOCKF); break; +#if CONFIG_EMBEDDED + case F_GETLKPID: + error = lf_getlock(lock, fl, fl->l_pid); + FREE(lock, M_LOCKF); + break; +#endif default: FREE(lock, M_LOCKF); diff --git a/bsd/kern/kern_malloc.c b/bsd/kern/kern_malloc.c index d33382ec3..e5ae62f3f 100644 --- a/bsd/kern/kern_malloc.c +++ b/bsd/kern/kern_malloc.c @@ -300,6 +300,9 @@ const char *memname[] = { "fdvnodedata" /* 122 M_FD_VN_DATA */ "fddirbuf", /* 123 M_FD_DIRBUF */ "netagent", /* 124 M_NETAGENT */ + "Event Handler",/* 125 M_EVENTHANDLER */ + "Link Layer Table", /* 126 M_LLTABLE */ + "Network Work Queue", /* 127 M_NWKWQ */ "" }; @@ -485,6 +488,9 @@ struct kmzones { { 0, KMZ_MALLOC, FALSE }, /* 122 M_FD_VN_DATA */ { 0, KMZ_MALLOC, FALSE }, /* 123 M_FD_DIRBUF */ { 0, KMZ_MALLOC, FALSE }, /* 124 M_NETAGENT */ + { 0, KMZ_MALLOC, FALSE }, /* 125 M_EVENTHANDLER */ + { 0, KMZ_MALLOC, FALSE }, /* 126 M_LLTABLE */ + { 0, KMZ_MALLOC, FALSE }, /* 127 M_NWKWQ */ #undef SOS #undef SOX }; @@ -557,7 +563,7 @@ _MALLOC_external( int type, int flags) { - static vm_allocation_site_t site = { VM_KERN_MEMORY_KALLOC, VM_TAG_BT }; + static vm_allocation_site_t site = { .tag = VM_KERN_MEMORY_KALLOC, .flags = VM_TAG_BT }; return (__MALLOC(size, type, flags, &site)); } @@ -650,9 +656,9 @@ __REALLOC( if (kalloc_bucket_size(size) == alloc) { if (flags & M_ZERO) { if (alloc < size) - bzero((uintptr_t)addr + alloc, (size - alloc)); + bzero(addr + alloc, (size - alloc)); else - bzero((uintptr_t)addr + size, (alloc - size)); + bzero(addr + size, (alloc - size)); } return addr; } @@ -751,6 +757,51 @@ _FREE_ZONE( kfree(elem, size); } +#if DEBUG || DEVELOPMENT + +extern unsigned int zone_map_jetsam_limit; + +static int +sysctl_zone_map_jetsam_limit SYSCTL_HANDLER_ARGS +{ +#pragma unused(oidp, arg1, arg2) + int oldval = 0, val = 0, error = 0; + + oldval = zone_map_jetsam_limit; + error = sysctl_io_number(req, oldval, sizeof(int), &val, NULL); + if (error || !req->newptr) { + return (error); + } + + if (val <= 0 || val > 100) { + printf("sysctl_zone_map_jetsam_limit: new jetsam limit value is invalid.\n"); + return EINVAL; + } + + zone_map_jetsam_limit = val; + return (0); +} + +SYSCTL_PROC(_kern, OID_AUTO, zone_map_jetsam_limit, CTLTYPE_INT|CTLFLAG_RW, 0, 0, + sysctl_zone_map_jetsam_limit, "I", "Zone map jetsam limit"); + +extern boolean_t run_zone_test(void); + +static int +sysctl_run_zone_test SYSCTL_HANDLER_ARGS +{ +#pragma unused(oidp, arg1, arg2) + int ret_val = run_zone_test(); + + return SYSCTL_OUT(req, &ret_val, sizeof(ret_val)); +} + +SYSCTL_PROC(_kern, OID_AUTO, run_zone_test, + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED, + 0, 0, &sysctl_run_zone_test, "I", "Test zone allocator KPI"); + +#endif /* DEBUG || DEVELOPMENT */ + #if CONFIG_ZLEAKS SYSCTL_DECL(_kern_zleak); @@ -863,3 +914,18 @@ SYSCTL_PROC(_kern_zleak, OID_AUTO, zone_threshold, sysctl_zleak_threshold, "Q", "zleak per-zone threshold"); #endif /* CONFIG_ZLEAKS */ + +extern uint64_t get_zones_collectable_bytes(void); + +static int +sysctl_zones_collectable_bytes SYSCTL_HANDLER_ARGS +{ +#pragma unused(oidp, arg1, arg2) + uint64_t zones_free_mem = get_zones_collectable_bytes(); + + return SYSCTL_OUT(req, &zones_free_mem, sizeof(zones_free_mem)); +} + +SYSCTL_PROC(_kern, OID_AUTO, zones_collectable_bytes, + CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED, + 0, 0, &sysctl_zones_collectable_bytes, "Q", "Collectable memory in zones"); diff --git a/bsd/kern/kern_memorystatus.c b/bsd/kern/kern_memorystatus.c index 46b037aba..74a80dc55 100644 --- a/bsd/kern/kern_memorystatus.c +++ b/bsd/kern/kern_memorystatus.c @@ -36,6 +36,7 @@ #include <kern/thread.h> #include <kern/host.h> #include <kern/policy_internal.h> +#include <kern/thread_group.h> #include <IOKit/IOBSD.h> @@ -45,6 +46,7 @@ #include <mach/task.h> #include <mach/host_priv.h> #include <mach/mach_host.h> +#include <os/log.h> #include <pexpert/pexpert.h> #include <sys/coalition.h> #include <sys/kern_event.h> @@ -68,9 +70,10 @@ #include <sys/kern_memorystatus.h> #include <mach/machine/sdt.h> +#include <libkern/section_keywords.h> /* For logging clarity */ -static const char *jetsam_kill_cause_name[] = { +static const char *memorystatus_kill_cause_name[] = { "" , "jettisoned" , /* kMemorystatusKilled */ "highwater" , /* kMemorystatusKilledHiwat */ @@ -81,12 +84,35 @@ static const char *jetsam_kill_cause_name[] = { "per-process-limit" , /* kMemorystatusKilledPerProcessLimit */ "diagnostic" , /* kMemorystatusKilledDiagnostic */ "idle-exit" , /* kMemorystatusKilledIdleExit */ + "zone-map-exhaustion" , /* kMemorystatusKilledZoneMapExhaustion */ }; -#if CONFIG_JETSAM +static const char * +memorystatus_priority_band_name(int32_t priority) +{ + switch (priority) { + case JETSAM_PRIORITY_FOREGROUND: + return "FOREGROUND"; + case JETSAM_PRIORITY_AUDIO_AND_ACCESSORY: + return "AUDIO_AND_ACCESSORY"; + case JETSAM_PRIORITY_CONDUCTOR: + return "CONDUCTOR"; + case JETSAM_PRIORITY_HOME: + return "HOME"; + case JETSAM_PRIORITY_EXECUTIVE: + return "EXECUTIVE"; + case JETSAM_PRIORITY_IMPORTANT: + return "IMPORTANT"; + case JETSAM_PRIORITY_CRITICAL: + return "CRITICAL"; + } + + return ("?"); +} + /* Does cause indicate vm or fc thrashing? */ static boolean_t -is_thrashing(unsigned cause) +is_reason_thrashing(unsigned cause) { switch (cause) { case kMemorystatusKilledVMThrashing: @@ -97,9 +123,26 @@ is_thrashing(unsigned cause) } } -/* Callback into vm_compressor.c to signal that thrashing has been mitigated. */ -extern void vm_thrashing_jetsam_done(void); -#endif /* CONFIG_JETSAM */ +/* Is the zone map almost full? */ +static boolean_t +is_reason_zone_map_exhaustion(unsigned cause) +{ + if (cause == kMemorystatusKilledZoneMapExhaustion) + return TRUE; + return FALSE; +} + +/* + * Returns the current zone map size and capacity to include in the jetsam snapshot. + * Defined in zalloc.c + */ +extern void get_zone_map_size(uint64_t *current_size, uint64_t *capacity); + +/* + * Returns the name of the largest zone and its size to include in the jetsam snapshot. + * Defined in zalloc.c + */ +extern void get_largest_zone_info(char *zone_name, size_t zone_name_len, uint64_t *zone_size); /* These are very verbose printfs(), enable with * MEMORYSTATUS_DEBUG_LOG @@ -193,13 +236,13 @@ static uint64_t memorystatus_apps_idle_delay_time = 0; * Memorystatus kevents */ -static int filt_memorystatusattach(struct knote *kn); +static int filt_memorystatusattach(struct knote *kn, struct kevent_internal_s *kev); static void filt_memorystatusdetach(struct knote *kn); static int filt_memorystatus(struct knote *kn, long hint); static int filt_memorystatustouch(struct knote *kn, struct kevent_internal_s *kev); static int filt_memorystatusprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); -struct filterops memorystatus_filtops = { +SECURITY_READ_ONLY_EARLY(struct filterops) memorystatus_filtops = { .f_attach = filt_memorystatusattach, .f_detach = filt_memorystatusdetach, .f_event = filt_memorystatus, @@ -229,6 +272,8 @@ static void memorystatus_reschedule_idle_demotion_locked(void); static void memorystatus_update_priority_locked(proc_t p, int priority, boolean_t head_insert, boolean_t skip_demotion_check); +int memorystatus_update_priority_for_appnap(proc_t p, boolean_t is_appnap); + vm_pressure_level_t convert_internal_pressure_level_to_dispatch_level(vm_pressure_level_t); boolean_t is_knote_registered_modify_task_pressure_bits(struct knote*, int, task_t, vm_pressure_level_t, vm_pressure_level_t); @@ -250,6 +295,8 @@ typedef struct memstat_bucket { memstat_bucket_t memstat_bucket[MEMSTAT_BUCKET_COUNT]; +int memorystatus_get_proccnt_upto_priority(int32_t max_bucket_index); + uint64_t memstat_idle_demotion_deadline = 0; int system_procs_aging_band = JETSAM_PRIORITY_AGING_BAND1; @@ -268,8 +315,9 @@ int applications_aging_band = JETSAM_PRIORITY_IDLE; unsigned int jetsam_aging_policy = kJetsamAgingPolicyLegacy; extern int corpse_for_fatal_memkill; -extern unsigned long total_corpses_count; +extern unsigned long total_corpses_count(void) __attribute__((pure)); extern void task_purge_all_corpses(void); +boolean_t memorystatus_allowed_vm_map_fork(__unused task_t); #if 0 @@ -459,6 +507,11 @@ static unsigned int memorystatus_dirty_count = 0; SYSCTL_INT(_kern, OID_AUTO, max_task_pmem, CTLFLAG_RD|CTLFLAG_LOCKED|CTLFLAG_MASKED, &max_task_footprint_mb, 0, ""); +#if CONFIG_EMBEDDED + +SYSCTL_INT(_kern, OID_AUTO, memorystatus_level, CTLFLAG_RD|CTLFLAG_LOCKED, &memorystatus_level, 0, ""); + +#endif /* CONFIG_EMBEDDED */ int memorystatus_get_level(__unused struct proc *p, struct memorystatus_get_level_args *args, __unused int *ret) @@ -488,12 +541,6 @@ static boolean_t memorystatus_kill_specific_process(pid_t victim_pid, uint32_t c static boolean_t memorystatus_kill_process_sync(pid_t victim_pid, uint32_t cause, os_reason_t jetsam_reason); -/* Jetsam */ - -#if CONFIG_JETSAM - -static int memorystatus_cmd_set_jetsam_memory_limit(pid_t pid, int32_t high_water_mark, __unused int32_t *retval, boolean_t is_fatal_limit); - static int memorystatus_cmd_set_memlimit_properties(pid_t pid, user_addr_t buffer, size_t buffer_size, __unused int32_t *retval); static int memorystatus_set_memlimit_properties(pid_t pid, memorystatus_memlimit_properties_t *entry); @@ -508,12 +555,8 @@ static boolean_t memorystatus_idle_snapshot = 0; unsigned int memorystatus_delta = 0; -static unsigned int memorystatus_available_pages_critical_base = 0; -//static unsigned int memorystatus_last_foreground_pressure_pages = (unsigned int)-1; -static unsigned int memorystatus_available_pages_critical_idle_offset = 0; - /* Jetsam Loop Detection */ -static boolean_t memorystatus_jld_enabled = TRUE; /* Enables jetsam loop detection on all devices */ +static boolean_t memorystatus_jld_enabled = FALSE; /* Enable jetsam loop detection */ static uint32_t memorystatus_jld_eval_period_msecs = 0; /* Init pass sets this based on device memory size */ static int memorystatus_jld_eval_aggressive_count = 3; /* Raise the priority max after 'n' aggressive loops */ static int memorystatus_jld_eval_aggressive_priority_band_max = 15; /* Kill aggressively up through this band */ @@ -545,14 +588,6 @@ SYSCTL_UINT(_kern, OID_AUTO, memorystatus_jld_eval_aggressive_count, CTLFLAG_RW| SYSCTL_UINT(_kern, OID_AUTO, memorystatus_jld_eval_aggressive_priority_band_max, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_jld_eval_aggressive_priority_band_max, 0, ""); #endif /* DEVELOPMENT || DEBUG */ -#if DEVELOPMENT || DEBUG -static unsigned int memorystatus_jetsam_panic_debug = 0; -static unsigned int memorystatus_jetsam_policy_offset_pages_diagnostic = 0; -#endif - -static unsigned int memorystatus_jetsam_policy = kPolicyDefault; -static unsigned int memorystatus_thread_wasted_wakeup = 0; - static uint32_t kill_under_pressure_cause = 0; /* @@ -586,11 +621,10 @@ static void memorystatus_get_task_phys_footprint_page_counts(task_t task, static void memorystatus_get_task_memory_region_count(task_t task, uint64_t *count); static uint32_t memorystatus_build_state(proc_t p); -static void memorystatus_update_levels_locked(boolean_t critical_only); //static boolean_t memorystatus_issue_pressure_kevent(boolean_t pressured); static boolean_t memorystatus_kill_top_process(boolean_t any, boolean_t sort_flag, uint32_t cause, os_reason_t jetsam_reason, int32_t *priority, uint32_t *errors); -static boolean_t memorystatus_kill_top_process_aggressive(boolean_t any, uint32_t cause, os_reason_t jetsam_reason, int aggr_count, int32_t priority_max, uint32_t *errors); +static boolean_t memorystatus_kill_top_process_aggressive(uint32_t cause, int aggr_count, int32_t priority_max, uint32_t *errors); static boolean_t memorystatus_kill_elevated_process(uint32_t cause, os_reason_t jetsam_reason, int aggr_count, uint32_t *errors); static boolean_t memorystatus_kill_hiwat_proc(uint32_t *errors); @@ -607,8 +641,6 @@ typedef int (*cmpfunc_t)(const void *a, const void *b); extern void qsort(void *a, size_t n, size_t es, cmpfunc_t cmp); static int memstat_asc_cmp(const void *a, const void *b); -#endif /* CONFIG_JETSAM */ - /* VM pressure */ extern unsigned int vm_page_free_count; @@ -621,28 +653,48 @@ extern unsigned int vm_page_wire_count; extern unsigned int vm_page_secluded_count; #endif /* CONFIG_SECLUDED_MEMORY */ -#if VM_PRESSURE_EVENTS - -boolean_t memorystatus_warn_process(pid_t pid, __unused boolean_t is_active, __unused boolean_t is_fatal, boolean_t exceeded); - -vm_pressure_level_t memorystatus_vm_pressure_level = kVMPressureNormal; - -#if CONFIG_MEMORYSTATUS +#if CONFIG_JETSAM unsigned int memorystatus_available_pages = (unsigned int)-1; unsigned int memorystatus_available_pages_pressure = 0; unsigned int memorystatus_available_pages_critical = 0; -unsigned int memorystatus_frozen_count = 0; -unsigned int memorystatus_suspended_count = 0; -unsigned int memorystatus_policy_more_free_offset_pages = 0; +static unsigned int memorystatus_available_pages_critical_base = 0; +static unsigned int memorystatus_available_pages_critical_idle_offset = 0; -#if CONFIG_JETSAM #if DEVELOPMENT || DEBUG SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_available_pages, 0, ""); #else -SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages, CTLFLAG_RD| CTLFLAG_MASKED | CTLFLAG_LOCKED, &memorystatus_available_pages, 0, ""); +SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages, CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED, &memorystatus_available_pages, 0, ""); #endif /* DEVELOPMENT || DEBUG */ + +static unsigned int memorystatus_jetsam_policy = kPolicyDefault; +unsigned int memorystatus_policy_more_free_offset_pages = 0; +static void memorystatus_update_levels_locked(boolean_t critical_only); +static unsigned int memorystatus_thread_wasted_wakeup = 0; + +/* Callback into vm_compressor.c to signal that thrashing has been mitigated. */ +extern void vm_thrashing_jetsam_done(void); +static int memorystatus_cmd_set_jetsam_memory_limit(pid_t pid, int32_t high_water_mark, __unused int32_t *retval, boolean_t is_fatal_limit); + +int32_t max_kill_priority = JETSAM_PRIORITY_MAX; + +#else /* CONFIG_JETSAM */ + +uint64_t memorystatus_available_pages = (uint64_t)-1; +uint64_t memorystatus_available_pages_pressure = (uint64_t)-1; +uint64_t memorystatus_available_pages_critical = (uint64_t)-1; + +int32_t max_kill_priority = JETSAM_PRIORITY_IDLE; #endif /* CONFIG_JETSAM */ +unsigned int memorystatus_frozen_count = 0; +unsigned int memorystatus_suspended_count = 0; + +#if VM_PRESSURE_EVENTS + +boolean_t memorystatus_warn_process(pid_t pid, __unused boolean_t is_active, __unused boolean_t is_fatal, boolean_t exceeded); + +vm_pressure_level_t memorystatus_vm_pressure_level = kVMPressureNormal; + /* * We use this flag to signal if we have any HWM offenders * on the system. This way we can reduce the number of wakeups @@ -658,7 +710,6 @@ SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages, CTLFLAG_RD| CTLFLAG_M boolean_t memorystatus_hwm_candidates = 0; static int memorystatus_send_note(int event_code, void *data, size_t data_length); -#endif /* CONFIG_MEMORYSTATUS */ #endif /* VM_PRESSURE_EVENTS */ @@ -669,7 +720,8 @@ lck_grp_attr_t *disconnect_page_mappings_lck_grp_attr; lck_grp_t *disconnect_page_mappings_lck_grp; static lck_mtx_t disconnect_page_mappings_mutex; -#endif +extern boolean_t kill_on_no_paging_space; +#endif /* DEVELOPMENT || DEBUG */ /* Freeze */ @@ -714,7 +766,7 @@ static unsigned int memorystatus_suspended_footprint_total = 0; /* pages */ extern uint64_t vm_swap_get_free_space(void); -static boolean_t memorystatus_freeze_update_throttle(); +static boolean_t memorystatus_freeze_update_throttle(void); #endif /* CONFIG_FREEZE */ @@ -851,10 +903,6 @@ sysctl_memorystatus_highwater_enable SYSCTL_HANDLER_ARGS use_active = proc_jetsam_state_is_active_locked(p); if (enable) { - /* - * No need to consider P_MEMSTAT_MEMLIMIT_BACKGROUND anymore. - * Background limits are described via the inactive limit slots. - */ if (use_active == TRUE) { CACHE_ACTIVE_LIMITS_LOCKED(p, is_fatal); @@ -937,12 +985,14 @@ sysctl_memorystatus_vm_pressure_send SYSCTL_HANDLER_ARGS /* * See event.h ... fflags for EVFILT_MEMORYSTATUS */ - if (!((fflags == NOTE_MEMORYSTATUS_PRESSURE_NORMAL)|| - (fflags == NOTE_MEMORYSTATUS_PRESSURE_WARN) || - (fflags == NOTE_MEMORYSTATUS_PRESSURE_CRITICAL) || - (fflags == NOTE_MEMORYSTATUS_LOW_SWAP) || - (fflags == NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) || - (fflags == NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL))) { + if (!((fflags == NOTE_MEMORYSTATUS_PRESSURE_NORMAL)|| + (fflags == NOTE_MEMORYSTATUS_PRESSURE_WARN) || + (fflags == NOTE_MEMORYSTATUS_PRESSURE_CRITICAL) || + (fflags == NOTE_MEMORYSTATUS_LOW_SWAP) || + (fflags == NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) || + (fflags == NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) || + (((fflags & NOTE_MEMORYSTATUS_MSL_STATUS) != 0 && + ((fflags & ~NOTE_MEMORYSTATUS_MSL_STATUS) == 0))))) { printf("memorystatus_vm_pressure_send: notification [0x%x] not supported \n", fflags); error = 1; @@ -987,15 +1037,17 @@ SYSCTL_PROC(_kern, OID_AUTO, memorystatus_vm_pressure_send, CTLTYPE_QUAD|CTLFLAG #endif /* VM_PRESSURE_EVENTS */ -#if CONFIG_JETSAM - SYSCTL_INT(_kern, OID_AUTO, memorystatus_idle_snapshot, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_idle_snapshot, 0, ""); +#if CONFIG_JETSAM SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages_critical, CTLFLAG_RD|CTLFLAG_LOCKED, &memorystatus_available_pages_critical, 0, ""); SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages_critical_base, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_available_pages_critical_base, 0, ""); SYSCTL_UINT(_kern, OID_AUTO, memorystatus_available_pages_critical_idle_offset, CTLFLAG_RW|CTLFLAG_LOCKED, &memorystatus_available_pages_critical_idle_offset, 0, ""); SYSCTL_UINT(_kern, OID_AUTO, memorystatus_policy_more_free_offset_pages, CTLFLAG_RW, &memorystatus_policy_more_free_offset_pages, 0, ""); +static unsigned int memorystatus_jetsam_panic_debug = 0; +static unsigned int memorystatus_jetsam_policy_offset_pages_diagnostic = 0; + /* Diagnostic code */ enum { @@ -1250,8 +1302,6 @@ SYSCTL_PROC(_kern, OID_AUTO, memorystatus_disconnect_page_mappings, CTLTYPE_INT| #endif /* DEVELOPMENT || DEBUG */ - -#if CONFIG_JETSAM /* * Picks the sorting routine for a given jetsam priority band. * @@ -1292,6 +1342,12 @@ static int memorystatus_sort_bucket(unsigned int bucket_index, int sort_order) #endif proc_list_lock(); + + if (memstat_bucket[bucket_index].count == 0) { + proc_list_unlock(); + return (0); + } + switch (bucket_index) { case JETSAM_PRIORITY_FOREGROUND: if (memorystatus_sort_by_largest_coalition_locked(bucket_index, coal_sort_order) == 0) { @@ -1362,8 +1418,6 @@ static void memorystatus_sort_by_largest_process_locked(unsigned int bucket_inde } } -#endif /* CONFIG_JETSAM */ - static proc_t memorystatus_get_first_proc_locked(unsigned int *bucket_index, boolean_t search) { memstat_bucket_t *current_bucket; proc_t next_p; @@ -1418,19 +1472,24 @@ memorystatus_init(void) disconnect_page_mappings_lck_grp = lck_grp_alloc_init("disconnect_page_mappings", disconnect_page_mappings_lck_grp_attr); lck_mtx_init(&disconnect_page_mappings_mutex, disconnect_page_mappings_lck_grp, NULL); + + if (kill_on_no_paging_space == TRUE) { + max_kill_priority = JETSAM_PRIORITY_MAX; + } #endif - nanoseconds_to_absolutetime((uint64_t)DEFERRED_IDLE_EXIT_TIME_SECS * NSEC_PER_SEC, &memorystatus_sysprocs_idle_delay_time); - nanoseconds_to_absolutetime((uint64_t)DEFERRED_IDLE_EXIT_TIME_SECS * NSEC_PER_SEC, &memorystatus_apps_idle_delay_time); /* Init buckets */ for (i = 0; i < MEMSTAT_BUCKET_COUNT; i++) { TAILQ_INIT(&memstat_bucket[i].list); memstat_bucket[i].count = 0; } - memorystatus_idle_demotion_call = thread_call_allocate((thread_call_func_t)memorystatus_perform_idle_demotion, NULL); +#if CONFIG_JETSAM + nanoseconds_to_absolutetime((uint64_t)DEFERRED_IDLE_EXIT_TIME_SECS * NSEC_PER_SEC, &memorystatus_sysprocs_idle_delay_time); + nanoseconds_to_absolutetime((uint64_t)DEFERRED_IDLE_EXIT_TIME_SECS * NSEC_PER_SEC, &memorystatus_apps_idle_delay_time); + /* Apply overrides */ PE_get_default("kern.jetsam_delta", &delta_percentage, sizeof(delta_percentage)); if (delta_percentage == 0) { @@ -1500,7 +1559,6 @@ memorystatus_init(void) assert(JETSAM_PRIORITY_ELEVATED_INACTIVE > system_procs_aging_band); assert(JETSAM_PRIORITY_ELEVATED_INACTIVE > applications_aging_band); -#if CONFIG_JETSAM /* Take snapshots for idle-exit kills by default? First check the boot-arg... */ if (!PE_parse_boot_argn("jetsam_idle_snapshot", &memorystatus_idle_snapshot, sizeof (memorystatus_idle_snapshot))) { /* ...no boot-arg, so check the device tree */ @@ -1512,6 +1570,22 @@ memorystatus_init(void) memorystatus_available_pages_critical_base = (critical_threshold_percentage / delta_percentage) * memorystatus_delta; memorystatus_policy_more_free_offset_pages = (policy_more_free_offset_percentage / delta_percentage) * memorystatus_delta; + /* Jetsam Loop Detection */ + if (max_mem <= (512 * 1024 * 1024)) { + /* 512 MB devices */ + memorystatus_jld_eval_period_msecs = 8000; /* 8000 msecs == 8 second window */ + } else { + /* 1GB and larger devices */ + memorystatus_jld_eval_period_msecs = 6000; /* 6000 msecs == 6 second window */ + } + + memorystatus_jld_enabled = TRUE; + + /* No contention at this point */ + memorystatus_update_levels_locked(FALSE); + +#endif /* CONFIG_JETSAM */ + memorystatus_jetsam_snapshot_max = maxproc; memorystatus_jetsam_snapshot = (memorystatus_jetsam_snapshot_t*)kalloc(sizeof(memorystatus_jetsam_snapshot_t) + @@ -1524,19 +1598,6 @@ memorystatus_init(void) memset(&memorystatus_at_boot_snapshot, 0, sizeof(memorystatus_jetsam_snapshot_t)); - /* No contention at this point */ - memorystatus_update_levels_locked(FALSE); - - /* Jetsam Loop Detection */ - if (max_mem <= (512 * 1024 * 1024)) { - /* 512 MB devices */ - memorystatus_jld_eval_period_msecs = 8000; /* 8000 msecs == 8 second window */ - } else { - /* 1GB and larger devices */ - memorystatus_jld_eval_period_msecs = 6000; /* 6000 msecs == 6 second window */ - } -#endif - #if CONFIG_FREEZE memorystatus_freeze_threshold = (freeze_threshold_percentage / delta_percentage) * memorystatus_delta; #endif @@ -1585,6 +1646,14 @@ memorystatus_do_kill(proc_t p, uint32_t cause, os_reason_t jetsam_reason) { #else #pragma unused(cause) #endif + + if (p->p_memstat_effectivepriority >= JETSAM_PRIORITY_FOREGROUND) { + printf("memorystatus: killing process %d [%s] in high band %s (%d) - memorystatus_available_pages: %llu\n", p->p_pid, + (*p->p_name ? p->p_name : "unknown"), + memorystatus_priority_band_name(p->p_memstat_effectivepriority), p->p_memstat_effectivepriority, + (uint64_t)memorystatus_available_pages); + } + int jetsam_flags = P_LTERM_JETSAM; switch (cause) { case kMemorystatusKilledHiwat: jetsam_flags |= P_JETSAM_HIWAT; break; @@ -1614,7 +1683,12 @@ memorystatus_check_levels_locked(void) { #if CONFIG_JETSAM /* Update levels */ memorystatus_update_levels_locked(TRUE); -#endif +#else /* CONFIG_JETSAM */ + /* + * Nothing to do here currently since we update + * memorystatus_available_pages in vm_pressure_response. + */ +#endif /* CONFIG_JETSAM */ } /* @@ -2101,9 +2175,6 @@ memorystatus_update_priority_locked(proc_t p, int priority, boolean_t head_inser boolean_t ledger_update_needed = TRUE; /* - * No need to consider P_MEMSTAT_MEMLIMIT_BACKGROUND anymore. - * Background limits are described via the inactive limit slots. - * * Here, we must update the cached memory limit if the task * is transitioning between: * active <--> inactive @@ -2231,11 +2302,6 @@ memorystatus_update_priority_locked(proc_t p, int priority, boolean_t head_inser * memlimit_inactive_is_fatal When a process is inactive and exceeds its memory footprint, * this describes whether or not it should be immediatly fatal. * - * memlimit_background This process has a high-water-mark while in the background. - * No longer meaningful. Background limits are described via - * the inactive slots. Flag is ignored. - * - * * Returns: 0 Success * non-0 Failure */ @@ -2243,8 +2309,7 @@ memorystatus_update_priority_locked(proc_t p, int priority, boolean_t head_inser int memorystatus_update(proc_t p, int priority, uint64_t user_data, boolean_t effective, boolean_t update_memlimit, int32_t memlimit_active, boolean_t memlimit_active_is_fatal, - int32_t memlimit_inactive, boolean_t memlimit_inactive_is_fatal, - __unused boolean_t memlimit_background) + int32_t memlimit_inactive, boolean_t memlimit_inactive_is_fatal) { int ret; boolean_t head_insert = false; @@ -2308,23 +2373,6 @@ memorystatus_update(proc_t p, int priority, uint64_t user_data, boolean_t effect memlimit_active, (memlimit_active_is_fatal ? "F " : "NF"), memlimit_inactive, (memlimit_inactive_is_fatal ? "F " : "NF")); - if (memlimit_background) { - - /* - * With 2-level HWM support, we no longer honor P_MEMSTAT_MEMLIMIT_BACKGROUND. - * Background limits are described via the inactive limit slots. - */ - - // p->p_memstat_state |= P_MEMSTAT_MEMLIMIT_BACKGROUND; - -#if DEVELOPMENT || DEBUG - printf("memorystatus_update: WARNING %s[%d] set unused flag P_MEMSTAT_MEMLIMIT_BACKGROUND [A==%dMB %s] [IA==%dMB %s]\n", - (*p->p_name ? p->p_name : "unknown"), p->p_pid, - memlimit_active, (memlimit_active_is_fatal ? "F " : "NF"), - memlimit_inactive, (memlimit_inactive_is_fatal ? "F " : "NF")); -#endif /* DEVELOPMENT || DEBUG */ - } - if (memlimit_active <= 0) { /* * This process will have a system_wide task limit when active. @@ -2572,6 +2620,19 @@ memorystatus_update_idle_priority_locked(proc_t p) { * explicitly because it won't be going through the demotion paths * that take care to apply the limits appropriately. */ + + if (p->p_memstat_state & P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND) { + + /* + * This process has the 'elevated inactive jetsam band' attribute. + * So, there will be no trip to IDLE after all. + * Instead, we pin the process in the elevated band, + * where its ACTIVE limits will apply. + */ + + priority = JETSAM_PRIORITY_ELEVATED_INACTIVE; + } + memorystatus_update_priority_locked(p, priority, false, true); } else { @@ -2908,6 +2969,8 @@ memorystatus_dirty_set(proc_t p, boolean_t self, uint32_t pcontrol) { if (proc_jetsam_state_is_active_locked(p) == TRUE) { /* + * process is pinned in elevated band + * or * process is dirty */ CACHE_ACTIVE_LIMITS_LOCKED(p, is_fatal); @@ -3178,8 +3241,6 @@ memorystatus_build_state(proc_t p) { return snapshot_state; } -#if !CONFIG_JETSAM - static boolean_t kill_idle_exit_proc(void) { @@ -3220,7 +3281,7 @@ kill_idle_exit_proc(void) proc_list_unlock(); if (victim_p) { - printf("memorystatus_thread: idle exiting pid %d [%s]\n", victim_p->p_pid, (*victim_p->p_name ? victim_p->p_name : "(unknown)")); + printf("memorystatus: killing_idle_process pid %d [%s]\n", victim_p->p_pid, (*victim_p->p_name ? victim_p->p_name : "unknown")); killed = memorystatus_do_kill(victim_p, kMemorystatusKilledIdleExit, jetsam_reason); proc_rele(victim_p); } else { @@ -3229,14 +3290,11 @@ kill_idle_exit_proc(void) return killed; } -#endif -#if CONFIG_JETSAM static void memorystatus_thread_wake(void) { thread_wakeup((event_t)&memorystatus_wakeup); } -#endif /* CONFIG_JETSAM */ extern void vm_pressure_response(void); @@ -3252,32 +3310,305 @@ memorystatus_thread_block(uint32_t interval_ms, thread_continue_t continuation) return thread_block(continuation); } +static boolean_t +memorystatus_avail_pages_below_pressure(void) +{ +#if CONFIG_EMBEDDED +/* + * Instead of CONFIG_EMBEDDED for these *avail_pages* routines, we should + * key off of the system having dynamic swap support. With full swap support, + * the system shouldn't really need to worry about various page thresholds. + */ + return (memorystatus_available_pages <= memorystatus_available_pages_pressure); +#else /* CONFIG_EMBEDDED */ + return FALSE; +#endif /* CONFIG_EMBEDDED */ +} + +static boolean_t +memorystatus_avail_pages_below_critical(void) +{ +#if CONFIG_EMBEDDED + return (memorystatus_available_pages <= memorystatus_available_pages_critical); +#else /* CONFIG_EMBEDDED */ + return FALSE; +#endif /* CONFIG_EMBEDDED */ +} + +static boolean_t +memorystatus_post_snapshot(int32_t priority, uint32_t cause) +{ +#if CONFIG_EMBEDDED +#pragma unused(cause) + /* + * Don't generate logs for steady-state idle-exit kills, + * unless it is overridden for debug or by the device + * tree. + */ + + return ((priority != JETSAM_PRIORITY_IDLE) || memorystatus_idle_snapshot); + +#else /* CONFIG_EMBEDDED */ + /* + * Don't generate logs for steady-state idle-exit kills, + * unless + * - it is overridden for debug or by the device + * tree. + * OR + * - the kill causes are important i.e. not kMemorystatusKilledIdleExit + */ + + boolean_t snapshot_eligible_kill_cause = (is_reason_thrashing(cause) || is_reason_zone_map_exhaustion(cause)); + return ((priority != JETSAM_PRIORITY_IDLE) || memorystatus_idle_snapshot || snapshot_eligible_kill_cause); +#endif /* CONFIG_EMBEDDED */ +} + +static boolean_t +memorystatus_action_needed(void) +{ +#if CONFIG_EMBEDDED + return (is_reason_thrashing(kill_under_pressure_cause) || + is_reason_zone_map_exhaustion(kill_under_pressure_cause) || + memorystatus_available_pages <= memorystatus_available_pages_pressure); +#else /* CONFIG_EMBEDDED */ + return (is_reason_thrashing(kill_under_pressure_cause) || + is_reason_zone_map_exhaustion(kill_under_pressure_cause)); +#endif /* CONFIG_EMBEDDED */ +} + +static boolean_t +memorystatus_act_on_hiwat_processes(uint32_t *errors, uint32_t *hwm_kill, boolean_t *post_snapshot, __unused boolean_t *is_critical) +{ + boolean_t killed = memorystatus_kill_hiwat_proc(errors); + + if (killed) { + *hwm_kill = *hwm_kill + 1; + *post_snapshot = TRUE; + return TRUE; + } else { + memorystatus_hwm_candidates = FALSE; + } + +#if CONFIG_JETSAM + /* No highwater processes to kill. Continue or stop for now? */ + if (!is_reason_thrashing(kill_under_pressure_cause) && + !is_reason_zone_map_exhaustion(kill_under_pressure_cause) && + (memorystatus_available_pages > memorystatus_available_pages_critical)) { + /* + * We are _not_ out of pressure but we are above the critical threshold and there's: + * - no compressor thrashing + * - enough zone memory + * - no more HWM processes left. + * For now, don't kill any other processes. + */ + + if (*hwm_kill == 0) { + memorystatus_thread_wasted_wakeup++; + } + + *is_critical = FALSE; + + return TRUE; + } +#endif /* CONFIG_JETSAM */ + + return FALSE; +} + +static boolean_t +memorystatus_act_aggressive(uint32_t cause, os_reason_t jetsam_reason, int *jld_idle_kills, boolean_t *corpse_list_purged, boolean_t *post_snapshot) +{ + if (memorystatus_jld_enabled == TRUE) { + + boolean_t killed; + uint32_t errors = 0; + + /* Jetsam Loop Detection - locals */ + memstat_bucket_t *bucket; + int jld_bucket_count = 0; + struct timeval jld_now_tstamp = {0,0}; + uint64_t jld_now_msecs = 0; + int elevated_bucket_count = 0; + + /* Jetsam Loop Detection - statics */ + static uint64_t jld_timestamp_msecs = 0; + static int jld_idle_kill_candidates = 0; /* Number of available processes in band 0,1 at start */ + static int jld_eval_aggressive_count = 0; /* Bumps the max priority in aggressive loop */ + static int32_t jld_priority_band_max = JETSAM_PRIORITY_UI_SUPPORT; + /* + * Jetsam Loop Detection: attempt to detect + * rapid daemon relaunches in the lower bands. + */ + + microuptime(&jld_now_tstamp); + + /* + * Ignore usecs in this calculation. + * msecs granularity is close enough. + */ + jld_now_msecs = (jld_now_tstamp.tv_sec * 1000); + + proc_list_lock(); + switch (jetsam_aging_policy) { + case kJetsamAgingPolicyLegacy: + bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE]; + jld_bucket_count = bucket->count; + bucket = &memstat_bucket[JETSAM_PRIORITY_AGING_BAND1]; + jld_bucket_count += bucket->count; + break; + case kJetsamAgingPolicySysProcsReclaimedFirst: + case kJetsamAgingPolicyAppsReclaimedFirst: + bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE]; + jld_bucket_count = bucket->count; + bucket = &memstat_bucket[system_procs_aging_band]; + jld_bucket_count += bucket->count; + bucket = &memstat_bucket[applications_aging_band]; + jld_bucket_count += bucket->count; + break; + case kJetsamAgingPolicyNone: + default: + bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE]; + jld_bucket_count = bucket->count; + break; + } + + bucket = &memstat_bucket[JETSAM_PRIORITY_ELEVATED_INACTIVE]; + elevated_bucket_count = bucket->count; + + proc_list_unlock(); + + /* + * memorystatus_jld_eval_period_msecs is a tunable + * memorystatus_jld_eval_aggressive_count is a tunable + * memorystatus_jld_eval_aggressive_priority_band_max is a tunable + */ + if ( (jld_bucket_count == 0) || + (jld_now_msecs > (jld_timestamp_msecs + memorystatus_jld_eval_period_msecs))) { + + /* + * Refresh evaluation parameters + */ + jld_timestamp_msecs = jld_now_msecs; + jld_idle_kill_candidates = jld_bucket_count; + *jld_idle_kills = 0; + jld_eval_aggressive_count = 0; + jld_priority_band_max = JETSAM_PRIORITY_UI_SUPPORT; + } + + if (*jld_idle_kills > jld_idle_kill_candidates) { + jld_eval_aggressive_count++; + +#if DEVELOPMENT || DEBUG + printf("memorystatus: aggressive%d: beginning of window: %lld ms, : timestamp now: %lld ms\n", + jld_eval_aggressive_count, + jld_timestamp_msecs, + jld_now_msecs); + printf("memorystatus: aggressive%d: idle candidates: %d, idle kills: %d\n", + jld_eval_aggressive_count, + jld_idle_kill_candidates, + *jld_idle_kills); +#endif /* DEVELOPMENT || DEBUG */ + + if ((jld_eval_aggressive_count == memorystatus_jld_eval_aggressive_count) && + (total_corpses_count() > 0) && (*corpse_list_purged == FALSE)) { + /* + * If we reach this aggressive cycle, corpses might be causing memory pressure. + * So, in an effort to avoid jetsams in the FG band, we will attempt to purge + * corpse memory prior to this final march through JETSAM_PRIORITY_UI_SUPPORT. + */ + task_purge_all_corpses(); + *corpse_list_purged = TRUE; + } + else if (jld_eval_aggressive_count > memorystatus_jld_eval_aggressive_count) { + /* + * Bump up the jetsam priority limit (eg: the bucket index) + * Enforce bucket index sanity. + */ + if ((memorystatus_jld_eval_aggressive_priority_band_max < 0) || + (memorystatus_jld_eval_aggressive_priority_band_max >= MEMSTAT_BUCKET_COUNT)) { + /* + * Do nothing. Stick with the default level. + */ + } else { + jld_priority_band_max = memorystatus_jld_eval_aggressive_priority_band_max; + } + } + + /* Visit elevated processes first */ + while (elevated_bucket_count) { + + elevated_bucket_count--; + + /* + * memorystatus_kill_elevated_process() drops a reference, + * so take another one so we can continue to use this exit reason + * even after it returns. + */ + + os_reason_ref(jetsam_reason); + killed = memorystatus_kill_elevated_process( + cause, + jetsam_reason, + jld_eval_aggressive_count, + &errors); + + if (killed) { + *post_snapshot = TRUE; + if (memorystatus_avail_pages_below_pressure()) { + /* + * Still under pressure. + * Find another pinned processes. + */ + continue; + } else { + return TRUE; + } + } else { + /* + * No pinned processes left to kill. + * Abandon elevated band. + */ + break; + } + } + + /* + * memorystatus_kill_top_process_aggressive() allocates its own + * jetsam_reason so the kMemorystatusKilledVMThrashing cause + * is consistent throughout the aggressive march. + */ + killed = memorystatus_kill_top_process_aggressive( + kMemorystatusKilledVMThrashing, + jld_eval_aggressive_count, + jld_priority_band_max, + &errors); + + if (killed) { + /* Always generate logs after aggressive kill */ + *post_snapshot = TRUE; + *jld_idle_kills = 0; + return TRUE; + } + } + + return FALSE; + } + + return FALSE; +} + + static void memorystatus_thread(void *param __unused, wait_result_t wr __unused) { static boolean_t is_vm_privileged = FALSE; -#if CONFIG_JETSAM boolean_t post_snapshot = FALSE; uint32_t errors = 0; uint32_t hwm_kill = 0; boolean_t sort_flag = TRUE; boolean_t corpse_list_purged = FALSE; - - /* Jetsam Loop Detection - locals */ - memstat_bucket_t *bucket; - int jld_bucket_count = 0; - struct timeval jld_now_tstamp = {0,0}; - uint64_t jld_now_msecs = 0; - int elevated_bucket_count = 0; - - /* Jetsam Loop Detection - statics */ - static uint64_t jld_timestamp_msecs = 0; - static int jld_idle_kill_candidates = 0; /* Number of available processes in band 0,1 at start */ - static int jld_idle_kills = 0; /* Number of procs killed during eval period */ - static int jld_eval_aggressive_count = 0; /* Bumps the max priority in aggressive loop */ - static int32_t jld_priority_band_max = JETSAM_PRIORITY_UI_SUPPORT; -#endif + int jld_idle_kills = 0; if (is_vm_privileged == FALSE) { /* @@ -3289,12 +3620,10 @@ memorystatus_thread(void *param __unused, wait_result_t wr __unused) if (vm_restricted_to_single_processor == TRUE) thread_vm_bind_group_add(); - + thread_set_thread_name(current_thread(), "VM_memorystatus"); memorystatus_thread_block(0, memorystatus_thread); } -#if CONFIG_JETSAM - KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_SCAN) | DBG_FUNC_START, memorystatus_available_pages, memorystatus_jld_enabled, memorystatus_jld_eval_period_msecs, memorystatus_jld_eval_aggressive_count,0); @@ -3310,8 +3639,7 @@ memorystatus_thread(void *param __unused, wait_result_t wr __unused) * If we run out of HWM processes and our available pages drops below the critical threshold, then, * we target the least recently used process in order of increasing jetsam priority (exception: the FG band). */ - while (is_thrashing(kill_under_pressure_cause) || - memorystatus_available_pages <= memorystatus_available_pages_pressure) { + while (memorystatus_action_needed()) { boolean_t killed; int32_t priority; uint32_t cause; @@ -3326,6 +3654,9 @@ memorystatus_thread(void *param __unused, wait_result_t wr __unused) case kMemorystatusKilledVMThrashing: jetsam_reason_code = JETSAM_REASON_MEMORY_VMTHRASHING; break; + case kMemorystatusKilledZoneMapExhaustion: + jetsam_reason_code = JETSAM_REASON_ZONE_MAP_EXHAUSTION; + break; case kMemorystatusKilledVMPageShortage: /* falls through */ default: @@ -3335,30 +3666,16 @@ memorystatus_thread(void *param __unused, wait_result_t wr __unused) } /* Highwater */ - killed = memorystatus_kill_hiwat_proc(&errors); - if (killed) { - hwm_kill++; - post_snapshot = TRUE; - goto done; - } else { - memorystatus_hwm_candidates = FALSE; - } - - /* No highwater processes to kill. Continue or stop for now? */ - if (!is_thrashing(kill_under_pressure_cause) && - (memorystatus_available_pages > memorystatus_available_pages_critical)) { - /* - * We are _not_ out of pressure but we are above the critical threshold and there's: - * - no compressor thrashing - * - no more HWM processes left. - * For now, don't kill any other processes. - */ - - if (hwm_kill == 0) { - memorystatus_thread_wasted_wakeup++; + boolean_t is_critical = TRUE; + if (memorystatus_act_on_hiwat_processes(&errors, &hwm_kill, &post_snapshot, &is_critical)) { + if (is_critical == FALSE) { + /* + * For now, don't kill any other processes. + */ + break; + } else { + goto done; } - - break; } jetsam_reason = os_reason_create(OS_REASON_JETSAM, jetsam_reason_code); @@ -3366,166 +3683,8 @@ memorystatus_thread(void *param __unused, wait_result_t wr __unused) printf("memorystatus_thread: failed to allocate jetsam reason\n"); } - if (memorystatus_jld_enabled == TRUE) { - - /* - * Jetsam Loop Detection: attempt to detect - * rapid daemon relaunches in the lower bands. - */ - - microuptime(&jld_now_tstamp); - - /* - * Ignore usecs in this calculation. - * msecs granularity is close enough. - */ - jld_now_msecs = (jld_now_tstamp.tv_sec * 1000); - - proc_list_lock(); - switch (jetsam_aging_policy) { - case kJetsamAgingPolicyLegacy: - bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE]; - jld_bucket_count = bucket->count; - bucket = &memstat_bucket[JETSAM_PRIORITY_AGING_BAND1]; - jld_bucket_count += bucket->count; - break; - case kJetsamAgingPolicySysProcsReclaimedFirst: - case kJetsamAgingPolicyAppsReclaimedFirst: - bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE]; - jld_bucket_count = bucket->count; - bucket = &memstat_bucket[system_procs_aging_band]; - jld_bucket_count += bucket->count; - bucket = &memstat_bucket[applications_aging_band]; - jld_bucket_count += bucket->count; - break; - case kJetsamAgingPolicyNone: - default: - bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE]; - jld_bucket_count = bucket->count; - break; - } - - bucket = &memstat_bucket[JETSAM_PRIORITY_ELEVATED_INACTIVE]; - elevated_bucket_count = bucket->count; - - proc_list_unlock(); - - /* - * memorystatus_jld_eval_period_msecs is a tunable - * memorystatus_jld_eval_aggressive_count is a tunable - * memorystatus_jld_eval_aggressive_priority_band_max is a tunable - */ - if ( (jld_bucket_count == 0) || - (jld_now_msecs > (jld_timestamp_msecs + memorystatus_jld_eval_period_msecs))) { - - /* - * Refresh evaluation parameters - */ - jld_timestamp_msecs = jld_now_msecs; - jld_idle_kill_candidates = jld_bucket_count; - jld_idle_kills = 0; - jld_eval_aggressive_count = 0; - jld_priority_band_max = JETSAM_PRIORITY_UI_SUPPORT; - } - - if (jld_idle_kills > jld_idle_kill_candidates) { - jld_eval_aggressive_count++; - -#if DEVELOPMENT || DEBUG - printf("memorystatus: aggressive%d: beginning of window: %lld ms, : timestamp now: %lld ms\n", - jld_eval_aggressive_count, - jld_timestamp_msecs, - jld_now_msecs); - printf("memorystatus: aggressive%d: idle candidates: %d, idle kills: %d\n", - jld_eval_aggressive_count, - jld_idle_kill_candidates, - jld_idle_kills); -#endif /* DEVELOPMENT || DEBUG */ - - if ((jld_eval_aggressive_count == memorystatus_jld_eval_aggressive_count) && - (total_corpses_count > 0) && (corpse_list_purged == FALSE)) { - /* - * If we reach this aggressive cycle, corpses might be causing memory pressure. - * So, in an effort to avoid jetsams in the FG band, we will attempt to purge - * corpse memory prior to this final march through JETSAM_PRIORITY_UI_SUPPORT. - */ - task_purge_all_corpses(); - corpse_list_purged = TRUE; - } - else if (jld_eval_aggressive_count > memorystatus_jld_eval_aggressive_count) { - /* - * Bump up the jetsam priority limit (eg: the bucket index) - * Enforce bucket index sanity. - */ - if ((memorystatus_jld_eval_aggressive_priority_band_max < 0) || - (memorystatus_jld_eval_aggressive_priority_band_max >= MEMSTAT_BUCKET_COUNT)) { - /* - * Do nothing. Stick with the default level. - */ - } else { - jld_priority_band_max = memorystatus_jld_eval_aggressive_priority_band_max; - } - } - - /* Visit elevated processes first */ - while (elevated_bucket_count) { - - elevated_bucket_count--; - - /* - * memorystatus_kill_elevated_process() drops a reference, - * so take another one so we can continue to use this exit reason - * even after it returns. - */ - - os_reason_ref(jetsam_reason); - killed = memorystatus_kill_elevated_process( - kMemorystatusKilledVMThrashing, - jetsam_reason, - jld_eval_aggressive_count, - &errors); - - if (killed) { - post_snapshot = TRUE; - if (memorystatus_available_pages <= memorystatus_available_pages_pressure) { - /* - * Still under pressure. - * Find another pinned processes. - */ - continue; - } else { - goto done; - } - } else { - /* - * No pinned processes left to kill. - * Abandon elevated band. - */ - break; - } - } - - /* - * memorystatus_kill_top_process_aggressive() drops a reference, - * so take another one so we can continue to use this exit reason - * even after it returns - */ - os_reason_ref(jetsam_reason); - killed = memorystatus_kill_top_process_aggressive( - TRUE, - kMemorystatusKilledVMThrashing, - jetsam_reason, - jld_eval_aggressive_count, - jld_priority_band_max, - &errors); - - if (killed) { - /* Always generate logs after aggressive kill */ - post_snapshot = TRUE; - jld_idle_kills = 0; - goto done; - } - } + if (memorystatus_act_aggressive(cause, jetsam_reason, &jld_idle_kills, &corpse_list_purged, &post_snapshot)) { + goto done; } /* @@ -3540,12 +3699,8 @@ memorystatus_thread(void *param __unused, wait_result_t wr __unused) sort_flag = FALSE; if (killed) { - /* - * Don't generate logs for steady-state idle-exit kills, - * unless it is overridden for debug or by the device - * tree. - */ - if ((priority != JETSAM_PRIORITY_IDLE) || memorystatus_idle_snapshot) { + if (memorystatus_post_snapshot(priority, cause) == TRUE) { + post_snapshot = TRUE; } @@ -3561,7 +3716,7 @@ memorystatus_thread(void *param __unused, wait_result_t wr __unused) } } - if ((priority >= JETSAM_PRIORITY_UI_SUPPORT) && (total_corpses_count > 0) && (corpse_list_purged == FALSE)) { + if ((priority >= JETSAM_PRIORITY_UI_SUPPORT) && (total_corpses_count() > 0) && (corpse_list_purged == FALSE)) { /* * If we have jetsammed a process in or above JETSAM_PRIORITY_UI_SUPPORT * then we attempt to relieve pressure by purging corpse memory. @@ -3572,20 +3727,20 @@ memorystatus_thread(void *param __unused, wait_result_t wr __unused) goto done; } - if (memorystatus_available_pages <= memorystatus_available_pages_critical) { + if (memorystatus_avail_pages_below_critical()) { /* * Still under pressure and unable to kill a process - purge corpse memory */ - if (total_corpses_count > 0) { + if (total_corpses_count() > 0) { task_purge_all_corpses(); corpse_list_purged = TRUE; } - if (memorystatus_available_pages <= memorystatus_available_pages_critical) { + if (memorystatus_avail_pages_below_critical()) { /* * Still under pressure and unable to kill a process - panic */ - panic("memorystatus_jetsam_thread: no victim! available pages:%d\n", memorystatus_available_pages); + panic("memorystatus_jetsam_thread: no victim! available pages:%llu\n", (uint64_t)memorystatus_available_pages); } } @@ -3596,9 +3751,13 @@ done: * To avoid that, we reset the flag here and notify the * compressor. */ - if (is_thrashing(kill_under_pressure_cause)) { + if (is_reason_thrashing(kill_under_pressure_cause)) { kill_under_pressure_cause = 0; +#if CONFIG_JETSAM vm_thrashing_jetsam_done(); +#endif /* CONFIG_JETSAM */ + } else if (is_reason_zone_map_exhaustion(kill_under_pressure_cause)) { + kill_under_pressure_cause = 0; } os_reason_free(jetsam_reason); @@ -3610,17 +3769,6 @@ done: memorystatus_clear_errors(); } -#if VM_PRESSURE_EVENTS - /* - * LD: We used to target the foreground process first and foremost here. - * Now, we target all processes, starting from the non-suspended, background - * processes first. We will target foreground too. - * - * memorystatus_update_vm_pressure(TRUE); - */ - //vm_pressure_response(); -#endif - if (post_snapshot) { proc_list_lock(); size_t snapshot_size = sizeof(memorystatus_jetsam_snapshot_t) + @@ -3645,18 +3793,9 @@ done: KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_SCAN) | DBG_FUNC_END, memorystatus_available_pages, 0, 0, 0, 0); -#else /* CONFIG_JETSAM */ - - /* - * Jetsam not enabled - */ - -#endif /* CONFIG_JETSAM */ - memorystatus_thread_block(0, memorystatus_thread); } -#if !CONFIG_JETSAM /* * Returns TRUE: * when an idle-exitable proc was killed @@ -3665,9 +3804,20 @@ done: * when the attempt to kill an idle-exitable proc failed */ boolean_t memorystatus_idle_exit_from_VM(void) { + + /* + * This routine should no longer be needed since we are + * now using jetsam bands on all platforms and so will deal + * with IDLE processes within the memorystatus thread itself. + * + * But we still use it because we observed that macos systems + * started heavy compression/swapping with a bunch of + * idle-exitable processes alive and doing nothing. We decided + * to rather kill those processes than start swapping earlier. + */ + return(kill_idle_exit_proc()); } -#endif /* !CONFIG_JETSAM */ /* * Callback invoked when allowable physical memory footprint exceeded @@ -3691,7 +3841,7 @@ memorystatus_on_ledger_footprint_exceeded(boolean_t warning, boolean_t memlimit_ */ if (memorystatus_warn_process(p->p_pid, memlimit_is_active, memlimit_is_fatal, FALSE /* not exceeded */) != TRUE) { /* Print warning, since it's possible that task has not registered for pressure notifications */ - printf("task_exceeded_footprint: failed to warn the current task (%d exiting, or no handler registered?).\n", p->p_pid); + os_log(OS_LOG_DEFAULT, "memorystatus_on_ledger_footprint_exceeded: failed to warn the current task (%d exiting, or no handler registered?).\n", p->p_pid); } return; } @@ -3743,9 +3893,10 @@ memorystatus_log_exception(const int max_footprint_mb, boolean_t memlimit_is_act * Hard memory limit is a fatal custom-task-limit or system-wide per-task memory limit. */ - printf("process %d (%s) exceeded physical memory footprint, the %s%sMemoryLimit of %d MB\n", - p->p_pid, (*p->p_name ? p->p_name : "unknown"), (memlimit_is_active ? "Active" : "Inactive"), - (memlimit_is_fatal ? "Hard" : "Soft"), max_footprint_mb); + os_log_with_startup_serial(OS_LOG_DEFAULT, "EXC_RESOURCE -> %s[%d] exceeded mem limit: %s%s %d MB (%s)\n", + (*p->p_name ? p->p_name : "unknown"), p->p_pid, (memlimit_is_active ? "Active" : "Inactive"), + (memlimit_is_fatal ? "Hard" : "Soft"), max_footprint_mb, + (memlimit_is_fatal ? "fatal" : "non-fatal")); return; } @@ -3753,7 +3904,13 @@ memorystatus_log_exception(const int max_footprint_mb, boolean_t memlimit_is_act /* * Description: - * Evaluates active vs. inactive process state. + * Evaluates process state to determine which limit + * should be applied (active vs. inactive limit). + * + * Processes that have the 'elevated inactive jetsam band' attribute + * are first evaluated based on their current priority band. + * presently elevated ==> active + * * Processes that opt into dirty tracking are evaluated * based on clean vs dirty state. * dirty ==> active @@ -3771,7 +3928,15 @@ memorystatus_log_exception(const int max_footprint_mb, boolean_t memlimit_is_act static boolean_t proc_jetsam_state_is_active_locked(proc_t p) { - if (p->p_memstat_dirty & P_DIRTY_TRACK) { + if ((p->p_memstat_state & P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND) && + (p->p_memstat_effectivepriority == JETSAM_PRIORITY_ELEVATED_INACTIVE)) { + /* + * process has the 'elevated inactive jetsam band' attribute + * and process is present in the elevated band + * implies active state + */ + return TRUE; + } else if (p->p_memstat_dirty & P_DIRTY_TRACK) { /* * process has opted into dirty tracking * active state is based on dirty vs. clean @@ -3808,7 +3973,6 @@ static boolean_t memorystatus_kill_process_sync(pid_t victim_pid, uint32_t cause, os_reason_t jetsam_reason) { boolean_t res; -#if CONFIG_JETSAM uint32_t errors = 0; if (victim_pid == -1) { @@ -3842,12 +4006,7 @@ memorystatus_kill_process_sync(pid_t victim_pid, uint32_t cause, os_reason_t jet proc_list_unlock(); } } -#else /* !CONFIG_JETSAM */ - - res = memorystatus_kill_specific_process(victim_pid, cause, jetsam_reason); -#endif /* CONFIG_JETSAM */ - return res; } @@ -3873,7 +4032,6 @@ memorystatus_kill_specific_process(pid_t victim_pid, uint32_t cause, os_reason_t proc_list_lock(); -#if CONFIG_JETSAM if (memorystatus_jetsam_snapshot_count == 0) { memorystatus_init_jetsam_snapshot_locked(NULL,0); } @@ -3886,19 +4044,9 @@ memorystatus_kill_specific_process(pid_t victim_pid, uint32_t cause, os_reason_t proc_list_unlock(); - printf("%lu.%02d memorystatus: specifically killing pid %d [%s] (%s %d) - memorystatus_available_pages: %d\n", - (unsigned long)tv_sec, tv_msec, victim_pid, (*p->p_name ? p->p_name : "(unknown)"), - jetsam_kill_cause_name[cause], p->p_memstat_effectivepriority, memorystatus_available_pages); -#else /* !CONFIG_JETSAM */ - proc_list_unlock(); - - killtime = mach_absolute_time(); - absolutetime_to_microtime(killtime, &tv_sec, &tv_usec); - tv_msec = tv_usec / 1000; - printf("%lu.%02d memorystatus: specifically killing pid %d [%s] (%s %d)\n", - (unsigned long)tv_sec, tv_msec, victim_pid, (*p->p_name ? p->p_name : "(unknown)"), - jetsam_kill_cause_name[cause], p->p_memstat_effectivepriority); -#endif /* CONFIG_JETSAM */ + os_log_with_startup_serial(OS_LOG_DEFAULT, "%lu.%03d memorystatus: killing_specific_process pid %d [%s] (%s %d) - memorystatus_available_pages: %llu\n", + (unsigned long)tv_sec, tv_msec, victim_pid, (*p->p_name ? p->p_name : "unknown"), + memorystatus_kill_cause_name[cause], p->p_memstat_effectivepriority, (uint64_t)memorystatus_available_pages); killed = memorystatus_do_kill(p, cause, jetsam_reason); proc_rele(p); @@ -3964,6 +4112,8 @@ jetsam_on_ledger_cpulimit_exceeded(void) } } +#endif /* CONFIG_JETSAM */ + static void memorystatus_get_task_memory_region_count(task_t task, uint64_t *count) { @@ -3973,6 +4123,69 @@ memorystatus_get_task_memory_region_count(task_t task, uint64_t *count) *count = get_task_memory_region_count(task); } +/* + * Called during EXC_RESOURCE handling when a process exceeds a soft + * memory limit. This is the corpse fork path and here we decide if + * vm_map_fork will be allowed when creating the corpse. + * The current task is suspended. + * + * By default, a vm_map_fork is allowed to proceed. + * + * A few simple policy assumptions: + * Desktop platform is not considered in this path. + * The vm_map_fork is always allowed. + * + * If the device has a zero system-wide task limit, + * then the vm_map_fork is allowed. + * + * And if a process's memory footprint calculates less + * than or equal to half of the system-wide task limit, + * then the vm_map_fork is allowed. This calculation + * is based on the assumption that a process can + * munch memory up to the system-wide task limit. + */ +boolean_t +memorystatus_allowed_vm_map_fork(__unused task_t task) +{ + boolean_t is_allowed = TRUE; /* default */ + +#if CONFIG_EMBEDDED + + uint64_t footprint_in_bytes = 0; + uint64_t purgeable_in_bytes = 0; + uint64_t max_allowed_bytes = 0; + + if (max_task_footprint_mb == 0) { + return (is_allowed); + } + + purgeable_in_bytes = get_task_purgeable_size(task); + footprint_in_bytes = get_task_phys_footprint(task); + + /* + * Maximum is half the system-wide task limit. + */ + max_allowed_bytes = ((((uint64_t)max_task_footprint_mb) * 1024ULL * 1024ULL) >> 1); + + if (footprint_in_bytes > purgeable_in_bytes) { + footprint_in_bytes -= purgeable_in_bytes; + } + + if (footprint_in_bytes <= max_allowed_bytes) { + return (is_allowed); + } else { + printf("memorystatus disallowed vm_map_fork %lld %lld\n", footprint_in_bytes, max_allowed_bytes); + return (!is_allowed); + } + +#else /* CONFIG_EMBEDDED */ + + return (is_allowed); + +#endif /* CONFIG_EMBEDDED */ + +} + static void memorystatus_get_task_page_counts(task_t task, uint32_t *footprint, uint32_t *max_footprint, uint32_t *max_footprint_lifetime, uint32_t *purgeable_pages) { @@ -3986,7 +4199,7 @@ memorystatus_get_task_page_counts(task_t task, uint32_t *footprint, uint32_t *ma *footprint = (uint32_t)pages; if (max_footprint) { - pages = (get_task_phys_footprint_max(task) / PAGE_SIZE_64); + pages = (get_task_phys_footprint_recent_max(task) / PAGE_SIZE_64); assert(((uint32_t)pages) == pages); *max_footprint = (uint32_t)pages; } @@ -4213,6 +4426,7 @@ exit: return; } +#if CONFIG_JETSAM void memorystatus_pages_update(unsigned int pages_avail) { memorystatus_available_pages = pages_avail; @@ -4256,6 +4470,7 @@ void memorystatus_pages_update(unsigned int pages_avail) } #endif /* VM_PRESSURE_EVENTS */ } +#endif /* CONFIG_JETSAM */ static boolean_t memorystatus_init_jetsam_snapshot_entry_locked(proc_t p, memorystatus_jetsam_snapshot_entry_t *entry, uint64_t gencount) @@ -4355,6 +4570,10 @@ memorystatus_init_snapshot_vmstats(memorystatus_jetsam_snapshot_t *snapshot) snapshot->stats.compressor_pages = vm_stat.compressor_page_count; snapshot->stats.total_uncompressed_pages_in_compressor = vm_stat.total_uncompressed_pages_in_compressor; } + + get_zone_map_size(&snapshot->stats.zone_map_size, &snapshot->stats.zone_map_capacity); + get_largest_zone_info(snapshot->stats.largest_zone_name, sizeof(snapshot->stats.largest_zone_name), + &snapshot->stats.largest_zone_size); } /* @@ -4433,6 +4652,7 @@ memorystatus_init_jetsam_snapshot_locked(memorystatus_jetsam_snapshot_t *od_snap #if DEVELOPMENT || DEBUG +#if CONFIG_JETSAM static int memorystatus_cmd_set_panic_bits(user_addr_t buffer, uint32_t buffer_size) { int ret; @@ -4456,6 +4676,7 @@ memorystatus_cmd_set_panic_bits(user_addr_t buffer, uint32_t buffer_size) { return ret; } +#endif /* CONFIG_JETSAM */ /* * Triggers a sort_order on a specified jetsam priority band. @@ -4492,7 +4713,7 @@ memorystatus_kill_top_process(boolean_t any, boolean_t sort_flag, uint32_t cause { pid_t aPid; proc_t p = PROC_NULL, next_p = PROC_NULL; - boolean_t new_snapshot = FALSE, killed = FALSE; + boolean_t new_snapshot = FALSE, force_new_snapshot = FALSE, killed = FALSE; int kill_count = 0; unsigned int i = 0; uint32_t aPid_ep; @@ -4500,6 +4721,7 @@ memorystatus_kill_top_process(boolean_t any, boolean_t sort_flag, uint32_t cause clock_sec_t tv_sec; clock_usec_t tv_usec; uint32_t tv_msec; + int32_t local_max_kill_prio = JETSAM_PRIORITY_IDLE; #ifndef CONFIG_FREEZE #pragma unused(any) @@ -4509,14 +4731,53 @@ memorystatus_kill_top_process(boolean_t any, boolean_t sort_flag, uint32_t cause memorystatus_available_pages, 0, 0, 0, 0); +#if CONFIG_JETSAM + if (sort_flag == TRUE) { + (void)memorystatus_sort_bucket(JETSAM_PRIORITY_FOREGROUND, JETSAM_SORT_DEFAULT); + } + + local_max_kill_prio = max_kill_priority; + + force_new_snapshot = FALSE; + +#else /* CONFIG_JETSAM */ + if (sort_flag == TRUE) { - (void)memorystatus_sort_bucket(JETSAM_PRIORITY_FOREGROUND, JETSAM_SORT_DEFAULT); + (void)memorystatus_sort_bucket(JETSAM_PRIORITY_IDLE, JETSAM_SORT_DEFAULT); + } + + /* + * On macos, we currently only have 2 reasons to be here: + * + * kMemorystatusKilledZoneMapExhaustion + * AND + * kMemorystatusKilledVMThrashing + * + * If we are here because of kMemorystatusKilledZoneMapExhaustion, we will consider + * any and all processes as eligible kill candidates since we need to avoid a panic. + * + * Since this function can be called async. it is harder to toggle the max_kill_priority + * value before and after a call. And so we use this local variable to set the upper band + * on the eligible kill bands. + */ + if (cause == kMemorystatusKilledZoneMapExhaustion) { + local_max_kill_prio = JETSAM_PRIORITY_MAX; + } else { + local_max_kill_prio = max_kill_priority; } + /* + * And, because we are here under extreme circumstances, we force a snapshot even for + * IDLE kills. + */ + force_new_snapshot = TRUE; + +#endif /* CONFIG_JETSAM */ + proc_list_lock(); next_p = memorystatus_get_first_proc_locked(&i, TRUE); - while (next_p) { + while (next_p && (next_p->p_memstat_effectivepriority <= local_max_kill_prio)) { #if DEVELOPMENT || DEBUG int activeProcess; int procSuspendedForDiagnosis; @@ -4537,12 +4798,12 @@ memorystatus_kill_top_process(boolean_t any, boolean_t sort_flag, uint32_t cause continue; /* with lock held */ } -#if DEVELOPMENT || DEBUG +#if CONFIG_JETSAM && (DEVELOPMENT || DEBUG) if ((memorystatus_jetsam_policy & kPolicyDiagnoseActive) && procSuspendedForDiagnosis) { printf("jetsam: continuing after ignoring proc suspended already for diagnosis - %d\n", aPid); continue; } -#endif /* DEVELOPMENT || DEBUG */ +#endif /* CONFIG_JETSAM && (DEVELOPMENT || DEBUG) */ if (cause == kMemorystatusKilledVnodes) { @@ -4550,8 +4811,8 @@ memorystatus_kill_top_process(boolean_t any, boolean_t sort_flag, uint32_t cause * If the system runs out of vnodes, we systematically jetsam * processes in hopes of stumbling onto a vnode gain that helps * the system recover. The process that happens to trigger - * this path has no known relationship to the vnode consumption. - * We attempt to safeguard that process e.g: do not jetsam it. + * this path has no known relationship to the vnode shortage. + * Deadlock avoidance: attempt to safeguard the caller. */ if (p == current_proc()) { @@ -4576,11 +4837,14 @@ memorystatus_kill_top_process(boolean_t any, boolean_t sort_flag, uint32_t cause { /* * Capture a snapshot if none exists and: + * - we are forcing a new snapshot creation, either because: + * - on a particular platform we need these snapshots every time, OR + * - a boot-arg/embedded device tree property has been set. * - priority was not requested (this is something other than an ambient kill) * - the priority was requested *and* the targeted process is not at idle priority */ if ((memorystatus_jetsam_snapshot_count == 0) && - (memorystatus_idle_snapshot || ((!priority) || (priority && (aPid_ep != JETSAM_PRIORITY_IDLE))))) { + (force_new_snapshot || memorystatus_idle_snapshot || ((!priority) || (priority && (aPid_ep != JETSAM_PRIORITY_IDLE))))) { memorystatus_init_jetsam_snapshot_locked(NULL,0); new_snapshot = TRUE; } @@ -4597,7 +4861,7 @@ memorystatus_kill_top_process(boolean_t any, boolean_t sort_flag, uint32_t cause absolutetime_to_microtime(killtime, &tv_sec, &tv_usec); tv_msec = tv_usec / 1000; -#if DEVELOPMENT || DEBUG +#if CONFIG_JETSAM && (DEVELOPMENT || DEBUG) if ((memorystatus_jetsam_policy & kPolicyDiagnoseActive) && activeProcess) { MEMORYSTATUS_DEBUG(1, "jetsam: suspending pid %d [%s] (active) for diagnosis - memory_status_level: %d\n", aPid, (*p->p_name ? p->p_name: "(unknown)"), memorystatus_level); @@ -4621,18 +4885,18 @@ memorystatus_kill_top_process(boolean_t any, boolean_t sort_flag, uint32_t cause goto exit; } else -#endif /* DEVELOPMENT || DEBUG */ +#endif /* CONFIG_JETSAM && (DEVELOPMENT || DEBUG) */ { /* Shift queue, update stats */ memorystatus_update_jetsam_snapshot_entry_locked(p, cause, killtime); if (proc_ref_locked(p) == p) { proc_list_unlock(); - printf("%lu.%02d memorystatus: %s %d [%s] (%s %d) - memorystatus_available_pages: %d\n", + os_log_with_startup_serial(OS_LOG_DEFAULT, "%lu.%03d memorystatus: %s pid %d [%s] (%s %d) - memorystatus_available_pages: %llu\n", (unsigned long)tv_sec, tv_msec, - ((aPid_ep == JETSAM_PRIORITY_IDLE) ? "idle exiting pid" : "jetsam killing top process pid"), - aPid, (*p->p_name ? p->p_name : "(unknown)"), - jetsam_kill_cause_name[cause], aPid_ep, memorystatus_available_pages); + ((aPid_ep == JETSAM_PRIORITY_IDLE) ? "killing_idle_process" : "killing_top_process"), + aPid, (*p->p_name ? p->p_name : "unknown"), + memorystatus_kill_cause_name[cause], aPid_ep, (uint64_t)memorystatus_available_pages); /* * memorystatus_do_kill() drops a reference, so take another one so we can @@ -4704,7 +4968,7 @@ exit: * Jetsam aggressively */ static boolean_t -memorystatus_kill_top_process_aggressive(boolean_t any, uint32_t cause, os_reason_t jetsam_reason, int aggr_count, +memorystatus_kill_top_process_aggressive(uint32_t cause, int aggr_count, int32_t priority_max, uint32_t *errors) { pid_t aPid; @@ -4718,14 +4982,18 @@ memorystatus_kill_top_process_aggressive(boolean_t any, uint32_t cause, os_reaso clock_sec_t tv_sec; clock_usec_t tv_usec; uint32_t tv_msec; - -#pragma unused(any) + os_reason_t jetsam_reason = OS_REASON_NULL; KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_JETSAM) | DBG_FUNC_START, memorystatus_available_pages, priority_max, 0, 0, 0); memorystatus_sort_bucket(JETSAM_PRIORITY_FOREGROUND, JETSAM_SORT_DEFAULT); + jetsam_reason = os_reason_create(OS_REASON_JETSAM, cause); + if (jetsam_reason == OS_REASON_NULL) { + printf("memorystatus_kill_top_process_aggressive: failed to allocate exit reason\n"); + } + proc_list_lock(); next_p = memorystatus_get_first_proc_locked(&i, TRUE); @@ -4735,20 +5003,22 @@ memorystatus_kill_top_process_aggressive(boolean_t any, uint32_t cause, os_reaso int procSuspendedForDiagnosis; #endif /* DEVELOPMENT || DEBUG */ - if ((unsigned int)(next_p->p_memstat_effectivepriority) != i) { + if (((next_p->p_listflag & P_LIST_EXITED) != 0) || + ((unsigned int)(next_p->p_memstat_effectivepriority) != i)) { /* - * We have raced with next_p running on another core, as it has - * moved to a different jetsam priority band. This means we have - * lost our place in line while traversing the jetsam list. We + * We have raced with next_p running on another core. + * It may be exiting or it may have moved to a different + * jetsam priority band. This means we have lost our + * place in line while traversing the jetsam list. We * attempt to recover by rewinding to the beginning of the band * we were already traversing. By doing this, we do not guarantee * that no process escapes this aggressive march, but we can make * skipping an entire range of processes less likely. (PR-21069019) */ - MEMORYSTATUS_DEBUG(1, "memorystatus: aggressive%d: rewinding %s moved from band %d --> %d\n", - aggr_count, (*next_p->p_name ? next_p->p_name : "unknown"), i, next_p->p_memstat_effectivepriority); + MEMORYSTATUS_DEBUG(1, "memorystatus: aggressive%d: rewinding band %d, %s(%d) moved or exiting.\n", + aggr_count, i, (*next_p->p_name ? next_p->p_name : "unknown"), next_p->p_pid); next_p = memorystatus_get_first_proc_locked(&i, TRUE); continue; @@ -4780,12 +5050,12 @@ memorystatus_kill_top_process_aggressive(boolean_t any, uint32_t cause, os_reaso continue; } -#if DEVELOPMENT || DEBUG +#if CONFIG_JETSAM && (DEVELOPMENT || DEBUG) if ((memorystatus_jetsam_policy & kPolicyDiagnoseActive) && procSuspendedForDiagnosis) { printf("jetsam: continuing after ignoring proc suspended already for diagnosis - %d\n", aPid); continue; } -#endif /* DEVELOPMENT || DEBUG */ +#endif /* CONFIG_JETSAM && (DEVELOPMENT || DEBUG) */ /* * Capture a snapshot if none exists. @@ -4836,11 +5106,11 @@ memorystatus_kill_top_process_aggressive(boolean_t any, uint32_t cause, os_reaso } proc_list_unlock(); - printf("%lu.%01d memorystatus: aggressive%d: %s %d [%s] (%s %d) - memorystatus_available_pages: %d\n", - (unsigned long)tv_sec, tv_msec, aggr_count, - ((aPid_ep == JETSAM_PRIORITY_IDLE) ? "idle exiting pid" : "jetsam killing pid"), - aPid, (*p->p_name ? p->p_name : "(unknown)"), - jetsam_kill_cause_name[cause], aPid_ep, memorystatus_available_pages); + printf("%lu.%03d memorystatus: %s%d pid %d [%s] (%s %d) - memorystatus_available_pages: %llu\n", + (unsigned long)tv_sec, tv_msec, + ((aPid_ep == JETSAM_PRIORITY_IDLE) ? "killing_idle_process_aggressive" : "killing_top_process_aggressive"), + aggr_count, aPid, (*p->p_name ? p->p_name : "unknown"), + memorystatus_kill_cause_name[cause], aPid_ep, (uint64_t)memorystatus_available_pages); memorystatus_level_snapshot = memorystatus_level; @@ -4977,30 +5247,17 @@ memorystatus_kill_hiwat_proc(uint32_t *errors) continue; } -#if 0 - /* - * No need to consider P_MEMSTAT_MEMLIMIT_BACKGROUND anymore. - * Background limits are described via the inactive limit slots. - * Their fatal/non-fatal setting will drive whether or not to be - * considered in this kill path. - */ - - /* skip if a currently inapplicable limit is encountered */ - if ((p->p_memstat_state & P_MEMSTAT_MEMLIMIT_BACKGROUND) && (p->p_memstat_effectivepriority >= JETSAM_PRIORITY_FOREGROUND)) { - continue; - } -#endif footprint_in_bytes = get_task_phys_footprint(p->task); memlimit_in_bytes = (((uint64_t)p->p_memstat_memlimit) * 1024ULL * 1024ULL); /* convert MB to bytes */ skip = (footprint_in_bytes <= memlimit_in_bytes); -#if DEVELOPMENT || DEBUG +#if CONFIG_JETSAM && (DEVELOPMENT || DEBUG) if (!skip && (memorystatus_jetsam_policy & kPolicyDiagnoseActive)) { if (p->p_memstat_state & P_MEMSTAT_DIAG_SUSPENDED) { continue; } } -#endif /* DEVELOPMENT || DEBUG */ +#endif /* CONFIG_JETSAM && (DEVELOPMENT || DEBUG) */ #if CONFIG_FREEZE if (!skip) { @@ -5015,13 +5272,13 @@ memorystatus_kill_hiwat_proc(uint32_t *errors) if (skip) { continue; } else { -#if DEVELOPMENT || DEBUG +#if CONFIG_JETSAM && (DEVELOPMENT || DEBUG) MEMORYSTATUS_DEBUG(1, "jetsam: %s pid %d [%s] - %lld Mb > 1 (%d Mb)\n", (memorystatus_jetsam_policy & kPolicyDiagnoseActive) ? "suspending": "killing", aPid, (*p->p_name ? p->p_name : "unknown"), (footprint_in_bytes / (1024ULL * 1024ULL)), /* converted bytes to MB */ p->p_memstat_memlimit); -#endif /* DEVELOPMENT || DEBUG */ +#endif /* CONFIG_JETSAM && (DEVELOPMENT || DEBUG) */ if (memorystatus_jetsam_snapshot_count == 0) { memorystatus_init_jetsam_snapshot_locked(NULL,0); @@ -5034,7 +5291,7 @@ memorystatus_kill_hiwat_proc(uint32_t *errors) absolutetime_to_microtime(killtime, &tv_sec, &tv_usec); tv_msec = tv_usec / 1000; -#if DEVELOPMENT || DEBUG +#if CONFIG_JETSAM && (DEVELOPMENT || DEBUG) if (memorystatus_jetsam_policy & kPolicyDiagnoseActive) { MEMORYSTATUS_DEBUG(1, "jetsam: pid %d suspended for diagnosis - memorystatus_available_pages: %d\n", aPid, memorystatus_available_pages); memorystatus_update_jetsam_snapshot_entry_locked(p, kMemorystatusKilledDiagnostic, killtime); @@ -5050,15 +5307,15 @@ memorystatus_kill_hiwat_proc(uint32_t *errors) goto exit; } else -#endif /* DEVELOPMENT || DEBUG */ +#endif /* CONFIG_JETSAM && (DEVELOPMENT || DEBUG) */ { memorystatus_update_jetsam_snapshot_entry_locked(p, kMemorystatusKilledHiwat, killtime); if (proc_ref_locked(p) == p) { proc_list_unlock(); - printf("%lu.%02d memorystatus: jetsam killing pid %d [%s] (highwater %d) - memorystatus_available_pages: %d\n", - (unsigned long)tv_sec, tv_msec, aPid, (*p->p_name ? p->p_name : "(unknown)"), aPid_ep, memorystatus_available_pages); + os_log_with_startup_serial(OS_LOG_DEFAULT, "%lu.%03d memorystatus: killing_highwater_process pid %d [%s] (highwater %d) - memorystatus_available_pages: %llu\n", + (unsigned long)tv_sec, tv_msec, aPid, (*p->p_name ? p->p_name : "unknown"), aPid_ep, (uint64_t)memorystatus_available_pages); /* * memorystatus_do_kill drops a reference, so take another one so we can @@ -5199,11 +5456,11 @@ memorystatus_kill_elevated_process(uint32_t cause, os_reason_t jetsam_reason, in proc_list_unlock(); - printf("%lu.%01d memorystatus: elevated%d: jetsam killing pid %d [%s] (%s %d) - memorystatus_available_pages: %d\n", + os_log_with_startup_serial(OS_LOG_DEFAULT, "%lu.%03d memorystatus: killing_top_process_elevated%d pid %d [%s] (%s %d) - memorystatus_available_pages: %llu\n", (unsigned long)tv_sec, tv_msec, aggr_count, - aPid, (*p->p_name ? p->p_name : "(unknown)"), - jetsam_kill_cause_name[cause], aPid_ep, memorystatus_available_pages); + aPid, (*p->p_name ? p->p_name : "unknown"), + memorystatus_kill_cause_name[cause], aPid_ep, (uint64_t)memorystatus_available_pages); /* * memorystatus_do_kill drops a reference, so take another one so we can @@ -5273,7 +5530,7 @@ memorystatus_kill_process_async(pid_t victim_pid, uint32_t cause) { * add the appropriate exit reason code mapping. */ if ((victim_pid != -1) || (cause != kMemorystatusKilledVMPageShortage && cause != kMemorystatusKilledVMThrashing && - cause != kMemorystatusKilledFCThrashing)) { + cause != kMemorystatusKilledFCThrashing && cause != kMemorystatusKilledZoneMapExhaustion)) { return FALSE; } @@ -5282,31 +5539,32 @@ memorystatus_kill_process_async(pid_t victim_pid, uint32_t cause) { return TRUE; } -boolean_t -memorystatus_kill_on_VM_page_shortage(boolean_t async) { +boolean_t +memorystatus_kill_on_VM_thrashing(boolean_t async) { if (async) { - return memorystatus_kill_process_async(-1, kMemorystatusKilledVMPageShortage); + return memorystatus_kill_process_async(-1, kMemorystatusKilledVMThrashing); } else { - os_reason_t jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_MEMORY_VMPAGESHORTAGE); + os_reason_t jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_MEMORY_VMTHRASHING); if (jetsam_reason == OS_REASON_NULL) { - printf("memorystatus_kill_on_VM_page_shortage -- sync: failed to allocate jetsam reason\n"); + printf("memorystatus_kill_on_VM_thrashing -- sync: failed to allocate jetsam reason\n"); } - return memorystatus_kill_process_sync(-1, kMemorystatusKilledVMPageShortage, jetsam_reason); + return memorystatus_kill_process_sync(-1, kMemorystatusKilledVMThrashing, jetsam_reason); } } -boolean_t -memorystatus_kill_on_VM_thrashing(boolean_t async) { +#if CONFIG_JETSAM +boolean_t +memorystatus_kill_on_VM_page_shortage(boolean_t async) { if (async) { - return memorystatus_kill_process_async(-1, kMemorystatusKilledVMThrashing); + return memorystatus_kill_process_async(-1, kMemorystatusKilledVMPageShortage); } else { - os_reason_t jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_MEMORY_VMTHRASHING); + os_reason_t jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_MEMORY_VMPAGESHORTAGE); if (jetsam_reason == OS_REASON_NULL) { - printf("memorystatus_kill_on_VM_thrashing -- sync: failed to allocate jetsam reason\n"); + printf("memorystatus_kill_on_VM_page_shortage -- sync: failed to allocate jetsam reason\n"); } - return memorystatus_kill_process_sync(-1, kMemorystatusKilledVMThrashing, jetsam_reason); + return memorystatus_kill_process_sync(-1, kMemorystatusKilledVMPageShortage, jetsam_reason); } } @@ -5338,6 +5596,22 @@ memorystatus_kill_on_vnode_limit(void) { #endif /* CONFIG_JETSAM */ +boolean_t +memorystatus_kill_on_zone_map_exhaustion(pid_t pid) { + boolean_t res = FALSE; + if (pid == -1) { + res = memorystatus_kill_process_async(-1, kMemorystatusKilledZoneMapExhaustion); + } else { + os_reason_t jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_ZONE_MAP_EXHAUSTION); + if (jetsam_reason == OS_REASON_NULL) { + printf("memorystatus_kill_on_zone_map_exhaustion: failed to allocate jetsam reason\n"); + } + + res = memorystatus_kill_process_sync(pid, kMemorystatusKilledZoneMapExhaustion, jetsam_reason); + } + return res; +} + #if CONFIG_FREEZE __private_extern__ void @@ -5873,6 +6147,38 @@ memorystatus_warn_process(pid_t pid, __unused boolean_t is_active, __unused bool * filt_memorystatus(). */ +#if CONFIG_EMBEDDED + if (!limit_exceeded) { + /* + * Intentionally set either the unambiguous limit warning, + * the system-wide critical or the system-wide warning + * notification bit. + */ + + if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) { + kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_WARN; + found_knote = TRUE; + send_knote_count++; + } else if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PRESSURE_CRITICAL) { + kn->kn_fflags = NOTE_MEMORYSTATUS_PRESSURE_CRITICAL; + found_knote = TRUE; + send_knote_count++; + } else if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PRESSURE_WARN) { + kn->kn_fflags = NOTE_MEMORYSTATUS_PRESSURE_WARN; + found_knote = TRUE; + send_knote_count++; + } + } else { + /* + * Send this notification when a process has exceeded a soft limit. + */ + if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) { + kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL; + found_knote = TRUE; + send_knote_count++; + } + } +#else /* CONFIG_EMBEDDED */ if (!limit_exceeded) { /* @@ -5969,6 +6275,7 @@ memorystatus_warn_process(pid_t pid, __unused boolean_t is_active, __unused bool } } } +#endif /* CONFIG_EMBEDDED */ } } @@ -6091,7 +6398,11 @@ vm_pressure_select_optimal_candidate_to_notify(struct klist *, int, boolean_t); /* * This value is the threshold that a process must meet to be considered for scavenging. */ +#if CONFIG_EMBEDDED +#define VM_PRESSURE_MINIMUM_RSIZE 1 /* MB */ +#else /* CONFIG_EMBEDDED */ #define VM_PRESSURE_MINIMUM_RSIZE 10 /* MB */ +#endif /* CONFIG_EMBEDDED */ #define VM_PRESSURE_NOTIFY_WAIT_PERIOD 10000 /* milliseconds */ @@ -6213,7 +6524,7 @@ vm_pressure_select_optimal_candidate_to_notify(struct klist *candidate_list, int pressure_increase = TRUE; } else { - if (level >= pressure_snapshot) { + if (level && (level >= pressure_snapshot)) { pressure_increase = TRUE; } else { pressure_increase = FALSE; @@ -6286,7 +6597,11 @@ vm_pressure_select_optimal_candidate_to_notify(struct klist *candidate_list, int } #endif /* CONFIG_MEMORYSTATUS */ +#if CONFIG_EMBEDDED + curr_task_importance = p->p_memstat_effectivepriority; +#else /* CONFIG_EMBEDDED */ curr_task_importance = task_importance_estimate(t); +#endif /* CONFIG_EMBEDDED */ /* * Privileged listeners are only considered in the multi-level pressure scheme @@ -6462,15 +6777,24 @@ memorystatus_update_vm_pressure(boolean_t target_foreground_process) if (level_snapshot == kVMPressureWarning || level_snapshot == kVMPressureUrgent) { - if (curr_ts < next_warning_notification_sent_at_ts) { - delay(INTER_NOTIFICATION_DELAY * 4 /* 1 sec */); - return KERN_SUCCESS; + if (next_warning_notification_sent_at_ts) { + if (curr_ts < next_warning_notification_sent_at_ts) { + delay(INTER_NOTIFICATION_DELAY * 4 /* 1 sec */); + return KERN_SUCCESS; + } + + next_warning_notification_sent_at_ts = 0; + memorystatus_klist_reset_all_for_level(kVMPressureWarning); } } else if (level_snapshot == kVMPressureCritical) { - if (curr_ts < next_critical_notification_sent_at_ts) { - delay(INTER_NOTIFICATION_DELAY * 4 /* 1 sec */); - return KERN_SUCCESS; + if (next_critical_notification_sent_at_ts) { + if (curr_ts < next_critical_notification_sent_at_ts) { + delay(INTER_NOTIFICATION_DELAY * 4 /* 1 sec */); + return KERN_SUCCESS; + } + next_critical_notification_sent_at_ts = 0; + memorystatus_klist_reset_all_for_level(kVMPressureCritical); } } } @@ -6523,16 +6847,16 @@ memorystatus_update_vm_pressure(boolean_t target_foreground_process) if (level_snapshot != kVMPressureNormal) { if (level_snapshot == kVMPressureWarning || level_snapshot == kVMPressureUrgent) { nanoseconds_to_absolutetime(WARNING_NOTIFICATION_RESTING_PERIOD * NSEC_PER_SEC, &curr_ts); - next_warning_notification_sent_at_ts = mach_absolute_time() + curr_ts; - memorystatus_klist_reset_all_for_level(kVMPressureWarning); + /* Next warning notification (if nothing changes) won't be sent before...*/ + next_warning_notification_sent_at_ts = mach_absolute_time() + curr_ts; } if (level_snapshot == kVMPressureCritical) { nanoseconds_to_absolutetime(CRITICAL_NOTIFICATION_RESTING_PERIOD * NSEC_PER_SEC, &curr_ts); - next_critical_notification_sent_at_ts = mach_absolute_time() + curr_ts; - memorystatus_klist_reset_all_for_level(kVMPressureCritical); + /* Next critical notification (if nothing changes) won't be sent before...*/ + next_critical_notification_sent_at_ts = mach_absolute_time() + curr_ts; } } return KERN_FAILURE; @@ -6686,6 +7010,14 @@ static int sysctl_memorystatus_vm_pressure_level SYSCTL_HANDLER_ARGS { #pragma unused(arg1, arg2, oidp) +#if CONFIG_EMBEDDED + int error = 0; + + error = priv_check_cred(kauth_cred_get(), PRIV_VM_PRESSURE, 0); + if (error) + return (error); + +#endif /* CONFIG_EMBEDDED */ vm_pressure_level_t dispatch_level = convert_internal_pressure_level_to_dispatch_level(memorystatus_vm_pressure_level); return SYSCTL_OUT(req, &dispatch_level, sizeof(dispatch_level)); @@ -6860,12 +7192,6 @@ memorystatus_get_priority_list(memorystatus_priority_entry_t **list_ptr, size_t list_entry->priority = p->p_memstat_effectivepriority; list_entry->user_data = p->p_memstat_userdata; - /* - * No need to consider P_MEMSTAT_MEMLIMIT_BACKGROUND anymore. - * Background limits are described via the inactive limit slots. - * So, here, the cached limit should always be valid. - */ - if (p->p_memstat_memlimit <= 0) { task_get_phys_footprint_limit(p->task, &list_entry->limit); } else { @@ -6888,37 +7214,78 @@ memorystatus_get_priority_list(memorystatus_priority_entry_t **list_ptr, size_t } static int -memorystatus_cmd_get_priority_list(user_addr_t buffer, size_t buffer_size, int32_t *retval) { - int error = EINVAL; +memorystatus_get_priority_pid(pid_t pid, user_addr_t buffer, size_t buffer_size) { + int error = 0; + memorystatus_priority_entry_t mp_entry; + + /* Validate inputs */ + if ((pid == 0) || (buffer == USER_ADDR_NULL) || (buffer_size != sizeof(memorystatus_priority_entry_t))) { + return EINVAL; + } + + proc_t p = proc_find(pid); + if (!p) { + return ESRCH; + } + + memset (&mp_entry, 0, sizeof(memorystatus_priority_entry_t)); + + mp_entry.pid = p->p_pid; + mp_entry.priority = p->p_memstat_effectivepriority; + mp_entry.user_data = p->p_memstat_userdata; + if (p->p_memstat_memlimit <= 0) { + task_get_phys_footprint_limit(p->task, &mp_entry.limit); + } else { + mp_entry.limit = p->p_memstat_memlimit; + } + mp_entry.state = memorystatus_build_state(p); + + proc_rele(p); + + error = copyout(&mp_entry, buffer, buffer_size); + + return (error); +} + +static int +memorystatus_cmd_get_priority_list(pid_t pid, user_addr_t buffer, size_t buffer_size, int32_t *retval) { + int error = 0; boolean_t size_only; - memorystatus_priority_entry_t *list = NULL; size_t list_size; + + /* + * When a non-zero pid is provided, the 'list' has only one entry. + */ size_only = ((buffer == USER_ADDR_NULL) ? TRUE: FALSE); - - error = memorystatus_get_priority_list(&list, &buffer_size, &list_size, size_only); - if (error) { - goto out; - } - if (!size_only) { - error = copyout(list, buffer, list_size); + if (pid != 0) { + list_size = sizeof(memorystatus_priority_entry_t) * 1; + if (!size_only) { + error = memorystatus_get_priority_pid(pid, buffer, buffer_size); + } + } else { + memorystatus_priority_entry_t *list = NULL; + error = memorystatus_get_priority_list(&list, &buffer_size, &list_size, size_only); + + if (error == 0) { + if (!size_only) { + error = copyout(list, buffer, list_size); + } + } + + if (list) { + kfree(list, buffer_size); + } } - + if (error == 0) { *retval = list_size; } -out: - - if (list) { - kfree(list, buffer_size); - } - return error; + return (error); } -#if CONFIG_JETSAM - static void memorystatus_clear_errors(void) { @@ -6942,6 +7309,7 @@ memorystatus_clear_errors(void) KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_CLEAR_ERRORS) | DBG_FUNC_END, 0, 0, 0, 0, 0); } +#if CONFIG_JETSAM static void memorystatus_update_levels_locked(boolean_t critical_only) { @@ -6974,7 +7342,7 @@ memorystatus_update_levels_locked(boolean_t critical_only) { memorystatus_available_pages_critical = memorystatus_available_pages_pressure; } } -#endif +#endif /* DEBUG || DEVELOPMENT */ if (memorystatus_jetsam_policy & kPolicyMoreFree) { memorystatus_available_pages_critical += memorystatus_policy_more_free_offset_pages; @@ -6994,6 +7362,7 @@ memorystatus_update_levels_locked(boolean_t critical_only) { #endif } + static int sysctl_kern_memorystatus_policy_more_free SYSCTL_HANDLER_ARGS { @@ -7038,6 +7407,8 @@ sysctl_kern_memorystatus_policy_more_free SYSCTL_HANDLER_ARGS SYSCTL_PROC(_kern, OID_AUTO, memorystatus_policy_more_free, CTLTYPE_INT|CTLFLAG_WR|CTLFLAG_LOCKED|CTLFLAG_MASKED, 0, 0, &sysctl_kern_memorystatus_policy_more_free, "I", ""); +#endif /* CONFIG_JETSAM */ + /* * Get the at_boot snapshot */ @@ -7466,7 +7837,7 @@ memorystatus_cmd_set_priority_properties(pid_t pid, user_addr_t buffer, size_t b return EPERM; } - error = memorystatus_update(p, mpp_entry.priority, mpp_entry.user_data, FALSE, FALSE, 0, 0, FALSE, FALSE, FALSE); + error = memorystatus_update(p, mpp_entry.priority, mpp_entry.user_data, FALSE, FALSE, 0, 0, FALSE, FALSE); proc_rele(p); } @@ -7643,6 +8014,7 @@ memorystatus_get_pressure_status_kdp() { * - so mapping is (active/fatal, inactive/fatal) */ +#if CONFIG_JETSAM static int memorystatus_cmd_set_jetsam_memory_limit(pid_t pid, int32_t high_water_mark, __unused int32_t *retval, boolean_t is_fatal_limit) { int error = 0; @@ -7661,6 +8033,7 @@ memorystatus_cmd_set_jetsam_memory_limit(pid_t pid, int32_t high_water_mark, __u error = memorystatus_set_memlimit_properties(pid, &entry); return (error); } +#endif /* CONFIG_JETSAM */ static int memorystatus_set_memlimit_properties(pid_t pid, memorystatus_memlimit_properties_t *entry) { @@ -7754,10 +8127,6 @@ memorystatus_set_memlimit_properties(pid_t pid, memorystatus_memlimit_properties if (memorystatus_highwater_enabled) { boolean_t is_fatal; boolean_t use_active; - /* - * No need to consider P_MEMSTAT_MEMLIMIT_BACKGROUND anymore. - * Background limits are described via the inactive limit slots. - */ if (proc_jetsam_state_is_active_locked(p) == TRUE) { CACHE_ACTIVE_LIMITS_LOCKED(p, is_fatal); @@ -7800,8 +8169,6 @@ proc_get_memstat_priority(proc_t p, boolean_t effective_priority) return 0; } -#endif /* CONFIG_JETSAM */ - int memorystatus_control(struct proc *p __unused, struct memorystatus_control_args *args, int *ret) { int error = EINVAL; @@ -7831,9 +8198,8 @@ memorystatus_control(struct proc *p __unused, struct memorystatus_control_args * switch (args->command) { case MEMORYSTATUS_CMD_GET_PRIORITY_LIST: - error = memorystatus_cmd_get_priority_list(args->buffer, args->buffersize, ret); + error = memorystatus_cmd_get_priority_list(args->pid, args->buffer, args->buffersize, ret); break; -#if CONFIG_JETSAM case MEMORYSTATUS_CMD_SET_PRIORITY_PROPERTIES: error = memorystatus_cmd_set_priority_properties(args->pid, args->buffer, args->buffersize, ret); break; @@ -7855,6 +8221,7 @@ memorystatus_control(struct proc *p __unused, struct memorystatus_control_args * case MEMORYSTATUS_CMD_GET_PRESSURE_STATUS: error = memorystatus_cmd_get_pressure_status(ret); break; +#if CONFIG_JETSAM case MEMORYSTATUS_CMD_SET_JETSAM_HIGH_WATER_MARK: /* * This call does not distinguish between active and inactive limits. @@ -7871,6 +8238,7 @@ memorystatus_control(struct proc *p __unused, struct memorystatus_control_args * */ error = memorystatus_cmd_set_jetsam_memory_limit(args->pid, (int32_t)args->flags, ret, TRUE); break; +#endif /* CONFIG_JETSAM */ /* Test commands */ #if DEVELOPMENT || DEBUG case MEMORYSTATUS_CMD_TEST_JETSAM: @@ -7884,9 +8252,11 @@ memorystatus_control(struct proc *p __unused, struct memorystatus_control_args * case MEMORYSTATUS_CMD_TEST_JETSAM_SORT: error = memorystatus_cmd_test_jetsam_sort(args->pid, (int32_t)args->flags); break; +#if CONFIG_JETSAM case MEMORYSTATUS_CMD_SET_JETSAM_PANIC_BITS: error = memorystatus_cmd_set_panic_bits(args->buffer, args->buffersize); break; +#endif /* CONFIG_JETSAM */ #else /* DEVELOPMENT || DEBUG */ #pragma unused(jetsam_reason) #endif /* DEVELOPMENT || DEBUG */ @@ -7909,18 +8279,15 @@ memorystatus_control(struct proc *p __unused, struct memorystatus_control_args * memorystatus_aggressive_jetsam_lenient = FALSE; error = 0; break; -#endif /* CONFIG_JETSAM */ case MEMORYSTATUS_CMD_PRIVILEGED_LISTENER_ENABLE: case MEMORYSTATUS_CMD_PRIVILEGED_LISTENER_DISABLE: error = memorystatus_low_mem_privileged_listener(args->command); break; -#if CONFIG_JETSAM case MEMORYSTATUS_CMD_ELEVATED_INACTIVEJETSAMPRIORITY_ENABLE: case MEMORYSTATUS_CMD_ELEVATED_INACTIVEJETSAMPRIORITY_DISABLE: error = memorystatus_update_inactive_jetsam_priority_band(args->pid, args->command, args->flags ? TRUE : FALSE); break; -#endif /* CONFIG_JETSAM */ default: break; @@ -7932,7 +8299,7 @@ out: static int -filt_memorystatusattach(struct knote *kn) +filt_memorystatusattach(struct knote *kn, __unused struct kevent_internal_s *kev) { int error; @@ -8025,6 +8392,7 @@ filt_memorystatustouch(struct knote *kn, struct kevent_internal_s *kev) prev_kn_sfflags = kn->kn_sfflags; kn->kn_sfflags = (kev->fflags & EVFILT_MEMORYSTATUS_ALL_MASK); +#if !CONFIG_EMBEDDED /* * Only on desktop do we restrict notifications to * one per active/inactive state (soft limits only). @@ -8078,6 +8446,7 @@ filt_memorystatustouch(struct knote *kn, struct kevent_internal_s *kev) kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_INACTIVE; } } +#endif /* !CONFIG_EMBEDDED */ if ((kn->kn_status & KN_UDATA_SPECIFIC) == 0) kn->kn_udata = kev->udata; @@ -8140,8 +8509,9 @@ memorystatus_knote_register(struct knote *kn) { /* * Support only userspace visible flags. */ - if ((kn->kn_sfflags & EVFILT_MEMORYSTATUS_ALL_MASK) == kn->kn_sfflags) { + if ((kn->kn_sfflags & EVFILT_MEMORYSTATUS_ALL_MASK) == (unsigned int) kn->kn_sfflags) { +#if !CONFIG_EMBEDDED if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) { kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_ACTIVE; kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_INACTIVE; @@ -8151,6 +8521,7 @@ memorystatus_knote_register(struct knote *kn) { kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_ACTIVE; kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_INACTIVE; } +#endif /* !CONFIG_EMBEDDED */ KNOTE_ATTACH(&memorystatus_klist, kn); @@ -8183,7 +8554,6 @@ memorystatus_issue_pressure_kevent(boolean_t pressured) { #endif /* CONFIG_JETSAM && VM_PRESSURE_EVENTS */ #endif /* 0 */ -#if CONFIG_JETSAM /* Coalition support */ /* sorting info for a particular priority bucket */ @@ -8445,4 +8815,131 @@ memorystatus_move_list_locked(unsigned int bucket_index, pid_t *pid_list, int li } return(found_pids); } -#endif /* CONFIG_JETSAM */ + +int +memorystatus_get_proccnt_upto_priority(int32_t max_bucket_index) +{ + int32_t i = JETSAM_PRIORITY_IDLE; + int count = 0; + + if (max_bucket_index >= MEMSTAT_BUCKET_COUNT) { + return(-1); + } + + while(i <= max_bucket_index) { + count += memstat_bucket[i++].count; + } + + return count; +} + +int +memorystatus_update_priority_for_appnap(proc_t p, boolean_t is_appnap) +{ +#if !CONFIG_JETSAM + if (!p || (!isApp(p)) || (p->p_memstat_state & P_MEMSTAT_INTERNAL)) { + /* + * Ineligible processes OR system processes e.g. launchd. + */ + return -1; + } + + /* + * For macOS only: + * We would like to use memorystatus_update() here to move the processes + * within the bands. Unfortunately memorystatus_update() calls + * memorystatus_update_priority_locked() which uses any band transitions + * as an indication to modify ledgers. For that it needs the task lock + * and since we came into this function with the task lock held, we'll deadlock. + * + * Unfortunately we can't completely disable ledger updates because we still + * need the ledger updates for a subset of processes i.e. daemons. + * When all processes on all platforms support memory limits, we can simply call + * memorystatus_update(). + + * It also has some logic to deal with 'aging' which, currently, is only applicable + * on CONFIG_JETSAM configs. So, till every platform has CONFIG_JETSAM we'll need + * to do this explicit band transition. + */ + + memstat_bucket_t *current_bucket, *new_bucket; + int32_t priority = 0; + + proc_list_lock(); + + if (((p->p_listflag & P_LIST_EXITED) != 0) || + (p->p_memstat_state & (P_MEMSTAT_ERROR | P_MEMSTAT_TERMINATED))) { + /* + * If the process is on its way out OR + * jetsam has alread tried and failed to kill this process, + * let's skip the whole jetsam band transition. + */ + proc_list_unlock(); + return(0); + } + + if (is_appnap) { + current_bucket = &memstat_bucket[p->p_memstat_effectivepriority]; + new_bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE]; + priority = JETSAM_PRIORITY_IDLE; + } else { + if (p->p_memstat_effectivepriority != JETSAM_PRIORITY_IDLE) { + /* + * It is possible that someone pulled this process + * out of the IDLE band without updating its app-nap + * parameters. + */ + proc_list_unlock(); + return (0); + } + + current_bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE]; + new_bucket = &memstat_bucket[p->p_memstat_requestedpriority]; + priority = p->p_memstat_requestedpriority; + } + + TAILQ_REMOVE(¤t_bucket->list, p, p_memstat_list); + current_bucket->count--; + + TAILQ_INSERT_TAIL(&new_bucket->list, p, p_memstat_list); + new_bucket->count++; + + /* + * Record idle start or idle delta. + */ + if (p->p_memstat_effectivepriority == priority) { + /* + * This process is not transitioning between + * jetsam priority buckets. Do nothing. + */ + } else if (p->p_memstat_effectivepriority == JETSAM_PRIORITY_IDLE) { + uint64_t now; + /* + * Transitioning out of the idle priority bucket. + * Record idle delta. + */ + assert(p->p_memstat_idle_start != 0); + now = mach_absolute_time(); + if (now > p->p_memstat_idle_start) { + p->p_memstat_idle_delta = now - p->p_memstat_idle_start; + } + } else if (priority == JETSAM_PRIORITY_IDLE) { + /* + * Transitioning into the idle priority bucket. + * Record idle start. + */ + p->p_memstat_idle_start = mach_absolute_time(); + } + + p->p_memstat_effectivepriority = priority; + + proc_list_unlock(); + + return (0); + +#else /* !CONFIG_JETSAM */ + #pragma unused(p) + #pragma unused(is_appnap) + return -1; +#endif /* !CONFIG_JETSAM */ +} diff --git a/bsd/kern/kern_mib.c b/bsd/kern/kern_mib.c index 9ef3a2479..dd57cd722 100644 --- a/bsd/kern/kern_mib.c +++ b/bsd/kern/kern_mib.c @@ -119,6 +119,9 @@ extern vm_map_t bsd_pageable_map; #include <i386/cpuid.h> /* for cpuid_info() */ #endif +#if defined(__arm__) || defined(__arm64__) +#include <arm/cpuid.h> /* for cpuid_info() & cache_info() */ +#endif #ifndef MAX @@ -337,7 +340,11 @@ sysctl_pagesize32 (__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req) { long long l; +#if __arm64__ + l = (long long) (1 << page_shift_user32); +#else /* __arm64__ */ l = (long long) PAGE_SIZE; +#endif /* __arm64__ */ return sysctl_io_number(req, l, sizeof(l), NULL, NULL); } @@ -367,17 +374,28 @@ SYSCTL_OPAQUE (_hw, OID_AUTO, cacheconfig, CTLFLAG_RD | CTLFLAG_LOCKED, &cachec SYSCTL_OPAQUE (_hw, OID_AUTO, cachesize, CTLFLAG_RD | CTLFLAG_LOCKED, &cachesize, sizeof(cachesize), "Q", ""); SYSCTL_PROC (_hw, OID_AUTO, pagesize, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, 0, 0, sysctl_pagesize, "Q", ""); SYSCTL_PROC (_hw, OID_AUTO, pagesize32, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, 0, 0, sysctl_pagesize32, "Q", ""); +#if DEBUG || DEVELOPMENT || (!defined(__arm__) && !defined(__arm64__)) SYSCTL_QUAD (_hw, OID_AUTO, busfrequency, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.bus_frequency_hz, ""); SYSCTL_QUAD (_hw, OID_AUTO, busfrequency_min, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.bus_frequency_min_hz, ""); SYSCTL_QUAD (_hw, OID_AUTO, busfrequency_max, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.bus_frequency_max_hz, ""); SYSCTL_QUAD (_hw, OID_AUTO, cpufrequency, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.cpu_frequency_hz, ""); SYSCTL_QUAD (_hw, OID_AUTO, cpufrequency_min, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.cpu_frequency_min_hz, ""); SYSCTL_QUAD (_hw, OID_AUTO, cpufrequency_max, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.cpu_frequency_max_hz, ""); +#endif SYSCTL_PROC (_hw, OID_AUTO, cachelinesize, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, 0, HW_CACHELINE | CTLHW_RETQUAD, sysctl_hw_generic, "Q", ""); SYSCTL_PROC (_hw, OID_AUTO, l1icachesize, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, 0, HW_L1ICACHESIZE | CTLHW_RETQUAD, sysctl_hw_generic, "Q", ""); SYSCTL_PROC (_hw, OID_AUTO, l1dcachesize, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, 0, HW_L1DCACHESIZE | CTLHW_RETQUAD, sysctl_hw_generic, "Q", ""); SYSCTL_PROC (_hw, OID_AUTO, l2cachesize, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, 0, HW_L2CACHESIZE | CTLHW_RETQUAD, sysctl_hw_generic, "Q", ""); SYSCTL_PROC (_hw, OID_AUTO, l3cachesize, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, 0, HW_L3CACHESIZE | CTLHW_RETQUAD, sysctl_hw_generic, "Q", ""); +#if (defined(__arm__) || defined(__arm64__)) && (DEBUG || DEVELOPMENT) +SYSCTL_QUAD (_hw, OID_AUTO, memfrequency, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.mem_frequency_hz, ""); +SYSCTL_QUAD (_hw, OID_AUTO, memfrequency_min, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.mem_frequency_min_hz, ""); +SYSCTL_QUAD (_hw, OID_AUTO, memfrequency_max, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.mem_frequency_max_hz, ""); +SYSCTL_QUAD (_hw, OID_AUTO, prffrequency, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.prf_frequency_hz, ""); +SYSCTL_QUAD (_hw, OID_AUTO, prffrequency_min, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.prf_frequency_min_hz, ""); +SYSCTL_QUAD (_hw, OID_AUTO, prffrequency_max, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.prf_frequency_max_hz, ""); +SYSCTL_QUAD (_hw, OID_AUTO, fixfrequency, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.fix_frequency_hz, ""); +#endif /* __arm__ || __arm64__ */ SYSCTL_PROC(_hw, OID_AUTO, tbfrequency, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, 0, 0, sysctl_tbfrequency, "Q", ""); SYSCTL_QUAD (_hw, HW_MEMSIZE, memsize, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &max_mem, ""); SYSCTL_INT (_hw, OID_AUTO, packages, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &packages, 0, ""); @@ -412,8 +430,10 @@ SYSCTL_NODE(_hw, OID_AUTO, features, CTLFLAG_RD | CTLFLAG_LOCKED, NULL, "hardwar * The *_compat nodes are *NOT* visible within the kernel. */ SYSCTL_PROC(_hw, HW_PAGESIZE, pagesize_compat, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED, 0, HW_PAGESIZE, sysctl_hw_generic, "I", ""); +#if DEBUG || DEVELOPMENT || (!defined(__arm__) && !defined(__arm64__)) SYSCTL_COMPAT_INT (_hw, HW_BUS_FREQ, busfrequency_compat, CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.bus_clock_rate_hz, 0, ""); SYSCTL_COMPAT_INT (_hw, HW_CPU_FREQ, cpufrequency_compat, CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED, &gPEClockFrequencyInfo.cpu_clock_rate_hz, 0, ""); +#endif SYSCTL_PROC(_hw, HW_CACHELINE, cachelinesize_compat, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED, 0, HW_CACHELINE, sysctl_hw_generic, "I", ""); SYSCTL_PROC(_hw, HW_L1ICACHESIZE, l1icachesize_compat, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED, 0, HW_L1ICACHESIZE, sysctl_hw_generic, "I", ""); SYSCTL_PROC(_hw, HW_L1DCACHESIZE, l1dcachesize_compat, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MASKED | CTLFLAG_LOCKED, 0, HW_L1DCACHESIZE, sysctl_hw_generic, "I", ""); @@ -466,6 +486,48 @@ SYSCTL_PROC(_hw_optional, OID_AUTO, hle, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_KERN SYSCTL_PROC(_hw_optional, OID_AUTO, adx, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, (void *) kHasADX, 0, sysctl_cpu_capability, "I", ""); SYSCTL_PROC(_hw_optional, OID_AUTO, mpx, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, (void *) kHasMPX, 0, sysctl_cpu_capability, "I", ""); SYSCTL_PROC(_hw_optional, OID_AUTO, sgx, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, (void *) kHasSGX, 0, sysctl_cpu_capability, "I", ""); +#if !defined(RC_HIDE_XNU_J137) +SYSCTL_PROC(_hw_optional, OID_AUTO, avx512f, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, (void *) kHasAVX512F, 0, sysctl_cpu_capability, "I", ""); +SYSCTL_PROC(_hw_optional, OID_AUTO, avx512cd, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, (void *) kHasAVX512CD, 0, sysctl_cpu_capability, "I", ""); +SYSCTL_PROC(_hw_optional, OID_AUTO, avx512dq, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, (void *) kHasAVX512DQ, 0, sysctl_cpu_capability, "I", ""); +SYSCTL_PROC(_hw_optional, OID_AUTO, avx512bw, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, (void *) kHasAVX512BW, 0, sysctl_cpu_capability, "I", ""); +SYSCTL_PROC(_hw_optional, OID_AUTO, avx512vl, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, (void *) kHasAVX512VL, 0, sysctl_cpu_capability, "I", ""); +SYSCTL_PROC(_hw_optional, OID_AUTO, avx512ifma, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, (void *) kHasAVX512IFMA, 0, sysctl_cpu_capability, "I", ""); +SYSCTL_PROC(_hw_optional, OID_AUTO, avx512vbmi, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, (void *) kHasAVX512VBMI, 0, sysctl_cpu_capability, "I", ""); +#endif /* not RC_HIDE_XNU_J137 */ +#elif defined (__arm__) || defined (__arm64__) +int watchpoint_flag = -1; +int breakpoint_flag = -1; +int gNeon = -1; +int gNeonHpfp = -1; +int gARMv81Atomics = 0; + +#if defined (__arm__) +int arm64_flag = 0; +#elif defined (__arm64__) /* end __arm__*/ +int arm64_flag = 1; +#else /* end __arm64__*/ +int arm64_flag = -1; +#endif + +SYSCTL_INT(_hw_optional, OID_AUTO, watchpoint, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &watchpoint_flag, 0, ""); +SYSCTL_INT(_hw_optional, OID_AUTO, breakpoint, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &breakpoint_flag, 0, ""); +SYSCTL_INT(_hw_optional, OID_AUTO, neon, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gNeon, 0, ""); +SYSCTL_INT(_hw_optional, OID_AUTO, neon_hpfp, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gNeonHpfp, 0, ""); +SYSCTL_INT(_hw_optional, OID_AUTO, armv8_1_atomics, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &gARMv81Atomics, 0, ""); + +/* + * Without this little ifdef dance, the preprocessor replaces "arm64" with "1", + * leaving us with a less-than-helpful sysctl.hwoptional.1. + */ +#ifdef arm64 +#undef arm64 +SYSCTL_INT(_hw_optional, OID_AUTO, arm64, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &arm64_flag, 0, ""); +#define arm64 1 +#else +SYSCTL_INT(_hw_optional, OID_AUTO, arm64, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &arm64_flag, 0, ""); +#endif + #else #error Unsupported arch #endif /* !__i386__ && !__x86_64 && !__arm__ && ! __arm64__ */ @@ -485,6 +547,8 @@ sysctl_mib_init(void) cputhreadtype = cpu_threadtype(); #if defined(__i386__) || defined (__x86_64__) cpu64bit = (_get_cpu_capabilities() & k64Bit) == k64Bit; +#elif defined(__arm__) || defined (__arm64__) + cpu64bit = (cputype & CPU_ARCH_ABI64) == CPU_ARCH_ABI64; #else #error Unsupported arch #endif @@ -523,6 +587,33 @@ sysctl_mib_init(void) packages = roundup(ml_cpu_cache_sharing(0), cpuid_info()->thread_count) / cpuid_info()->thread_count; +#elif defined(__arm__) || defined(__arm64__) /* end __i386 */ + + cpufamily = cpuid_get_cpufamily(); + + watchpoint_flag = arm_debug_info()->num_watchpoint_pairs; + breakpoint_flag = arm_debug_info()->num_breakpoint_pairs; + + arm_mvfp_info_t *mvfp_info; + mvfp_info = arm_mvfp_info(); + gNeon = mvfp_info->neon; + gNeonHpfp = mvfp_info->neon_hpfp; + + cacheconfig[0] = ml_get_max_cpus(); + cacheconfig[1] = 1; + cacheconfig[2] = cache_info()->c_l2size ? 1:0; + cacheconfig[3] = 0; + cacheconfig[4] = 0; + cacheconfig[5] = 0; + cacheconfig[6] = 0; + + cachesize[0] = ml_get_machine_mem(); + cachesize[1] = cache_info()->c_dsize; /* Using the DCache */ + cachesize[2] = cache_info()->c_l2size; + cachesize[3] = 0; + cachesize[4] = 0; + + packages = 1; #else #error unknown architecture #endif /* !__i386__ && !__x86_64 && !__arm__ && !__arm64__ */ diff --git a/bsd/kern/kern_mman.c b/bsd/kern/kern_mman.c index 318400dc9..adb144567 100644 --- a/bsd/kern/kern_mman.c +++ b/bsd/kern/kern_mman.c @@ -128,6 +128,10 @@ #include <vm/vm_pager.h> #include <vm/vm_protos.h> +#if CONFIG_MACF +#include <security/mac_framework.h> +#endif + /* * XXX Internally, we use VM_PROT_* somewhat interchangeably, but the correct * XXX usage is PROT_* from an interface perspective. Thus the values of @@ -150,7 +154,9 @@ mmap(proc_t p, struct mmap_args *uap, user_addr_t *retval) vm_map_size_t user_size; vm_object_offset_t pageoff; vm_object_offset_t file_pos; - int alloc_flags=0; + int alloc_flags = 0; + vm_tag_t tag = VM_KERN_MEMORY_NONE; + vm_map_kernel_flags_t vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; boolean_t docow; vm_prot_t maxprot; void *handle; @@ -293,12 +299,15 @@ mmap(proc_t p, struct mmap_args *uap, user_addr_t *retval) * Use "fd" to pass (some) Mach VM allocation flags, * (see the VM_FLAGS_* definitions). */ - alloc_flags = fd & (VM_FLAGS_ALIAS_MASK | VM_FLAGS_SUPERPAGE_MASK | + alloc_flags = fd & (VM_FLAGS_ALIAS_MASK | + VM_FLAGS_SUPERPAGE_MASK | VM_FLAGS_PURGABLE); if (alloc_flags != fd) { /* reject if there are any extra flags */ return EINVAL; } + VM_GET_FLAGS_ALIAS(alloc_flags, tag); + alloc_flags &= ~VM_FLAGS_ALIAS_MASK; } handle = NULL; @@ -487,7 +496,7 @@ mmap(proc_t p, struct mmap_args *uap, user_addr_t *retval) alloc_flags |= VM_FLAGS_NO_CACHE; if (flags & MAP_JIT) { - alloc_flags |= VM_FLAGS_MAP_JIT; + vmk_flags.vmkf_map_jit = TRUE; } if (flags & MAP_RESILIENT_CODESIGN) { @@ -518,7 +527,8 @@ mmap(proc_t p, struct mmap_args *uap, user_addr_t *retval) map_anon_retry: result = vm_map_enter_mem_object(user_map, &user_addr, user_size, - 0, alloc_flags, + 0, alloc_flags, vmk_flags, + tag, IPC_PORT_NULL, 0, FALSE, prot, maxprot, (flags & MAP_SHARED) ? @@ -599,7 +609,8 @@ map_file_retry: } result = vm_map_enter_mem_object_control(user_map, &user_addr, user_size, - 0, alloc_flags, + 0, alloc_flags, vmk_flags, + tag, control, file_pos, docow, prot, maxprot, (flags & MAP_SHARED) ? @@ -650,8 +661,10 @@ bad: fp_drop(p, fd, fp, 0); KERNEL_DEBUG_CONSTANT((BSDDBG_CODE(DBG_BSD_SC_EXTENDED_INFO, SYS_mmap) | DBG_FUNC_NONE), fd, (uint32_t)(*retval), (uint32_t)user_size, error, 0); +#ifndef CONFIG_EMBEDDED KERNEL_DEBUG_CONSTANT((BSDDBG_CODE(DBG_BSD_SC_EXTENDED_INFO2, SYS_mmap) | DBG_FUNC_NONE), (uint32_t)(*retval >> 32), (uint32_t)(user_size >> 32), (uint32_t)(file_pos >> 32), (uint32_t)file_pos, 0); +#endif return(error); } @@ -675,7 +688,9 @@ msync_nocancel(__unused proc_t p, struct msync_nocancel_args *uap, __unused int3 user_map = current_map(); addr = (mach_vm_offset_t) uap->addr; size = (mach_vm_size_t)uap->len; +#ifndef CONFIG_EMBEDDED KERNEL_DEBUG_CONSTANT((BSDDBG_CODE(DBG_BSD_SC_EXTENDED_INFO, SYS_msync) | DBG_FUNC_NONE), (uint32_t)(addr >> 32), (uint32_t)(size >> 32), 0, 0, 0); +#endif if (addr & vm_map_page_mask(user_map)) { /* UNIX SPEC: user address is not page-aligned, return EINVAL */ return EINVAL; @@ -802,6 +817,10 @@ mprotect(__unused proc_t p, struct mprotect_args *uap, __unused int32_t *retval) prot |= VM_PROT_READ; #endif /* 3936456 */ +#if defined(__arm64__) + if (prot & VM_PROT_STRIP_READ) + prot &= ~(VM_PROT_READ | VM_PROT_STRIP_READ); +#endif #if CONFIG_MACF /* @@ -952,6 +971,24 @@ madvise(__unused proc_t p, struct madvise_args *uap, __unused int32_t *retval) start = (mach_vm_offset_t) uap->addr; size = (mach_vm_size_t) uap->len; +#if __arm64__ + if (start == 0 && + size != 0 && + (uap->behav == MADV_FREE || + uap->behav == MADV_FREE_REUSABLE)) { + printf("** FOURK_COMPAT: %d[%s] " + "failing madvise(0x%llx,0x%llx,%s)\n", + p->p_pid, p->p_comm, start, size, + ((uap->behav == MADV_FREE_REUSABLE) + ? "MADV_FREE_REUSABLE" + : "MADV_FREE")); + DTRACE_VM3(fourk_compat_madvise, + uint64_t, start, + uint64_t, size, + int, uap->behav); + return EINVAL; + } +#endif /* __arm64__ */ user_map = current_map(); @@ -971,17 +1008,19 @@ madvise(__unused proc_t p, struct madvise_args *uap, __unused int32_t *retval) int mincore(__unused proc_t p, struct mincore_args *uap, __unused int32_t *retval) { - mach_vm_offset_t addr, first_addr, end; - vm_map_t map; - user_addr_t vec; - int error; - int vecindex, lastvecindex; + mach_vm_offset_t addr = 0, first_addr = 0, end = 0, cur_end = 0; + vm_map_t map = VM_MAP_NULL; + user_addr_t vec = 0; + int error = 0; + int vecindex = 0, lastvecindex = 0; int mincoreinfo=0; - int pqueryinfo; - kern_return_t ret; - int numref; - - char c; + int pqueryinfo = 0; + unsigned int pqueryinfo_vec_size = 0; + vm_page_info_basic_t info = NULL; + mach_msg_type_number_t count = 0; + char *kernel_vec = NULL; + int req_vec_size_pages = 0, cur_vec_size_pages = 0; + kern_return_t kr = KERN_SUCCESS; map = current_map(); @@ -991,82 +1030,117 @@ mincore(__unused proc_t p, struct mincore_args *uap, __unused int32_t *retval) */ first_addr = addr = vm_map_trunc_page(uap->addr, vm_map_page_mask(map)); - end = addr + vm_map_round_page(uap->len, + end = vm_map_round_page(uap->addr + uap->len, vm_map_page_mask(map)); if (end < addr) return (EINVAL); + if (end == addr) + return (0); + /* - * Address of byte vector + * We are going to loop through the whole 'req_vec_size' pages + * range in chunks of 'cur_vec_size'. */ - vec = uap->vec; - map = current_map(); + req_vec_size_pages = (end - addr) >> PAGE_SHIFT; + cur_vec_size_pages = MIN(req_vec_size_pages, (int)(MAX_PAGE_RANGE_QUERY >> PAGE_SHIFT)); + + kernel_vec = (void*) _MALLOC(cur_vec_size_pages * sizeof(char), M_TEMP, M_WAITOK); + + if (kernel_vec == NULL) { + return (ENOMEM); + } /* - * Do this on a map entry basis so that if the pages are not - * in the current processes address space, we can easily look - * up the pages elsewhere. + * Address of byte vector */ - lastvecindex = -1; - for( ; addr < end; addr += PAGE_SIZE ) { - pqueryinfo = 0; - ret = mach_vm_page_query(map, addr, &pqueryinfo, &numref); - if (ret != KERN_SUCCESS) - pqueryinfo = 0; - mincoreinfo = 0; - if (pqueryinfo & VM_PAGE_QUERY_PAGE_PRESENT) - mincoreinfo |= MINCORE_INCORE; - if (pqueryinfo & VM_PAGE_QUERY_PAGE_REF) - mincoreinfo |= MINCORE_REFERENCED; - if (pqueryinfo & VM_PAGE_QUERY_PAGE_DIRTY) - mincoreinfo |= MINCORE_MODIFIED; - - - /* - * calculate index into user supplied byte vector - */ - vecindex = (addr - first_addr)>> PAGE_SHIFT; + vec = uap->vec; + + pqueryinfo_vec_size = cur_vec_size_pages * sizeof(struct vm_page_info_basic); + info = (void*) _MALLOC(pqueryinfo_vec_size, M_TEMP, M_WAITOK); + + if (info == NULL) { + FREE(kernel_vec, M_TEMP); + return (ENOMEM); + } + + while (addr < end) { + + cur_end = addr + (cur_vec_size_pages * PAGE_SIZE_64); + + count = VM_PAGE_INFO_BASIC_COUNT; + kr = vm_map_page_range_info_internal(map, + addr, + cur_end, + VM_PAGE_INFO_BASIC, + (vm_page_info_t) info, + &count); + + assert(kr == KERN_SUCCESS); /* - * If we have skipped map entries, we need to make sure that - * the byte vector is zeroed for those skipped entries. + * Do this on a map entry basis so that if the pages are not + * in the current processes address space, we can easily look + * up the pages elsewhere. */ - while((lastvecindex + 1) < vecindex) { - c = 0; - error = copyout(&c, vec + lastvecindex, 1); - if (error) { - return (EFAULT); - } - ++lastvecindex; + lastvecindex = -1; + for( ; addr < cur_end; addr += PAGE_SIZE ) { + + pqueryinfo = info[lastvecindex + 1].disposition; + + mincoreinfo = 0; + + if (pqueryinfo & VM_PAGE_QUERY_PAGE_PRESENT) + mincoreinfo |= MINCORE_INCORE; + if (pqueryinfo & VM_PAGE_QUERY_PAGE_REF) + mincoreinfo |= MINCORE_REFERENCED; + if (pqueryinfo & VM_PAGE_QUERY_PAGE_DIRTY) + mincoreinfo |= MINCORE_MODIFIED; + if (pqueryinfo & VM_PAGE_QUERY_PAGE_PAGED_OUT) + mincoreinfo |= MINCORE_PAGED_OUT; + if (pqueryinfo & VM_PAGE_QUERY_PAGE_COPIED) + mincoreinfo |= MINCORE_COPIED; + if ((pqueryinfo & VM_PAGE_QUERY_PAGE_EXTERNAL) == 0) + mincoreinfo |= MINCORE_ANONYMOUS; + /* + * calculate index into user supplied byte vector + */ + vecindex = (addr - first_addr)>> PAGE_SHIFT; + kernel_vec[vecindex] = (char)mincoreinfo; + lastvecindex = vecindex; } - /* - * Pass the page information to the user - */ - c = (char)mincoreinfo; - error = copyout(&c, vec + vecindex, 1); + + assert(vecindex == (cur_vec_size_pages - 1)); + + error = copyout(kernel_vec, vec, cur_vec_size_pages * sizeof(char) /* a char per page */); + if (error) { - return (EFAULT); + break; } - lastvecindex = vecindex; + + /* + * For the next chunk, we'll need: + * - bump the location in the user buffer for our next disposition. + * - new length + * - starting address + */ + vec += cur_vec_size_pages * sizeof(char); + req_vec_size_pages = (end - addr) >> PAGE_SHIFT; + cur_vec_size_pages = MIN(req_vec_size_pages, (int)(MAX_PAGE_RANGE_QUERY >> PAGE_SHIFT)); + + first_addr = addr; } + FREE(kernel_vec, M_TEMP); + FREE(info, M_TEMP); - /* - * Zero the last entries in the byte vector. - */ - vecindex = (end - first_addr) >> PAGE_SHIFT; - while((lastvecindex + 1) < vecindex) { - c = 0; - error = copyout(&c, vec + lastvecindex, 1); - if (error) { - return (EFAULT); - } - ++lastvecindex; + if (error) { + return (EFAULT); } - + return (0); } @@ -1097,7 +1171,7 @@ mlock(__unused proc_t p, struct mlock_args *uap, __unused int32_t *retvalval) size = vm_map_round_page(size+pageoff, vm_map_page_mask(user_map)); /* have to call vm_map_wire directly to pass "I don't know" protections */ - result = vm_map_wire(user_map, addr, addr+size, VM_PROT_NONE | VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_MLOCK), TRUE); + result = vm_map_wire_kernel(user_map, addr, addr+size, VM_PROT_NONE, VM_KERN_MEMORY_MLOCK, TRUE); if (result == KERN_RESOURCE_SHORTAGE) return EAGAIN; @@ -1125,7 +1199,7 @@ munlock(__unused proc_t p, struct munlock_args *uap, __unused int32_t *retval) user_map = current_map(); /* JMM - need to remove all wirings by spec - this just removes one */ - result = mach_vm_wire(host_priv_self(), user_map, addr, size, VM_PROT_NONE); + result = mach_vm_wire_kernel(host_priv_self(), user_map, addr, size, VM_PROT_NONE, VM_KERN_MEMORY_MLOCK); return (result == KERN_SUCCESS ? 0 : ENOMEM); } diff --git a/bsd/kern/kern_newsysctl.c b/bsd/kern/kern_newsysctl.c index 6895674f9..4103009fe 100644 --- a/bsd/kern/kern_newsysctl.c +++ b/bsd/kern/kern_newsysctl.c @@ -80,6 +80,10 @@ #include <security/audit/audit.h> #include <pexpert/pexpert.h> +#if CONFIG_MACF +#include <security/mac_framework.h> +#endif + lck_grp_t * sysctl_lock_group = NULL; lck_rw_t * sysctl_geometry_lock = NULL; lck_mtx_t * sysctl_unlocked_node_lock = NULL; diff --git a/bsd/kern/kern_ntptime.c b/bsd/kern/kern_ntptime.c new file mode 100644 index 000000000..a922c3676 --- /dev/null +++ b/bsd/kern/kern_ntptime.c @@ -0,0 +1,782 @@ +/*- + *********************************************************************** + * * + * Copyright (c) David L. Mills 1993-2001 * + * * + * Permission to use, copy, modify, and distribute this software and * + * its documentation for any purpose and without fee is hereby * + * granted, provided that the above copyright notice appears in all * + * copies and that both the copyright notice and this permission * + * notice appear in supporting documentation, and that the name * + * University of Delaware not be used in advertising or publicity * + * pertaining to distribution of the software without specific, * + * written prior permission. The University of Delaware makes no * + * representations about the suitability this software for any * + * purpose. It is provided "as is" without express or implied * + * warranty. * + * * + **********************************************************************/ + + +/* + * Adapted from the original sources for FreeBSD and timecounters by: + * Poul-Henning Kamp <phk@FreeBSD.org>. + * + * The 32bit version of the "LP" macros seems a bit past its "sell by" + * date so I have retained only the 64bit version and included it directly + * in this file. + * + * Only minor changes done to interface with the timecounters over in + * sys/kern/kern_clock.c. Some of the comments below may be (even more) + * confusing and/or plain wrong in that context. + */ + +/* + * Copyright (c) 2017 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include <sys/cdefs.h> +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/eventhandler.h> +#include <sys/kernel.h> +#include <sys/priv.h> +#include <sys/proc.h> +#include <sys/lock.h> +#include <sys/time.h> +#include <sys/timex.h> +#include <kern/clock.h> +#include <sys/sysctl.h> +#include <sys/sysproto.h> +#include <sys/kauth.h> +#include <kern/thread_call.h> +#include <kern/timer_call.h> +#include <machine/machine_routines.h> +#if CONFIG_MACF +#include <security/mac_framework.h> +#endif +#include <IOKit/IOBSD.h> + +typedef int64_t l_fp; +#define L_ADD(v, u) ((v) += (u)) +#define L_SUB(v, u) ((v) -= (u)) +#define L_ADDHI(v, a) ((v) += (int64_t)(a) << 32) +#define L_NEG(v) ((v) = -(v)) +#define L_RSHIFT(v, n) \ + do { \ + if ((v) < 0) \ + (v) = -(-(v) >> (n)); \ + else \ + (v) = (v) >> (n); \ + } while (0) +#define L_MPY(v, a) ((v) *= (a)) +#define L_CLR(v) ((v) = 0) +#define L_ISNEG(v) ((v) < 0) +#define L_LINT(v, a) \ + do { \ + if ((a) > 0) \ + ((v) = (int64_t)(a) << 32); \ + else \ + ((v) = -((int64_t)(-(a)) << 32)); \ + } while (0) +#define L_GINT(v) ((v) < 0 ? -(-(v) >> 32) : (v) >> 32) + +/* + * Generic NTP kernel interface + * + * These routines constitute the Network Time Protocol (NTP) interfaces + * for user and daemon application programs. The ntp_gettime() routine + * provides the time, maximum error (synch distance) and estimated error + * (dispersion) to client user application programs. The ntp_adjtime() + * routine is used by the NTP daemon to adjust the calendar clock to an + * externally derived time. The time offset and related variables set by + * this routine are used by other routines in this module to adjust the + * phase and frequency of the clock discipline loop which controls the + * system clock. + * + * When the kernel time is reckoned directly in nanoseconds (NTP_NANO + * defined), the time at each tick interrupt is derived directly from + * the kernel time variable. When the kernel time is reckoned in + * microseconds, (NTP_NANO undefined), the time is derived from the + * kernel time variable together with a variable representing the + * leftover nanoseconds at the last tick interrupt. In either case, the + * current nanosecond time is reckoned from these values plus an + * interpolated value derived by the clock routines in another + * architecture-specific module. The interpolation can use either a + * dedicated counter or a processor cycle counter (PCC) implemented in + * some architectures. + * + */ +/* + * Phase/frequency-lock loop (PLL/FLL) definitions + * + * The nanosecond clock discipline uses two variable types, time + * variables and frequency variables. Both types are represented as 64- + * bit fixed-point quantities with the decimal point between two 32-bit + * halves. On a 32-bit machine, each half is represented as a single + * word and mathematical operations are done using multiple-precision + * arithmetic. On a 64-bit machine, ordinary computer arithmetic is + * used. + * + * A time variable is a signed 64-bit fixed-point number in ns and + * fraction. It represents the remaining time offset to be amortized + * over succeeding tick interrupts. The maximum time offset is about + * 0.5 s and the resolution is about 2.3e-10 ns. + * + * 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 3 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |s s s| ns | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | fraction | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * + * A frequency variable is a signed 64-bit fixed-point number in ns/s + * and fraction. It represents the ns and fraction to be added to the + * kernel time variable at each second. The maximum frequency offset is + * about +-500000 ns/s and the resolution is about 2.3e-10 ns/s. + * + * 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 3 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |s s s s s s s s s s s s s| ns/s | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | fraction | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + */ + +#define SHIFT_PLL 4 +#define SHIFT_FLL 2 + +static int time_state = TIME_OK; +int time_status = STA_UNSYNC; +static long time_tai; +static long time_constant; +static long time_precision = 1; +static long time_maxerror = MAXPHASE / 1000; +static unsigned long last_time_maxerror_update; +long time_esterror = MAXPHASE / 1000; +static long time_reftime; +static l_fp time_offset; +static l_fp time_freq; +static int64_t time_adjtime; +static int updated; + +static lck_spin_t * ntp_lock; +static lck_grp_t * ntp_lock_grp; +static lck_attr_t * ntp_lock_attr; +static lck_grp_attr_t *ntp_lock_grp_attr; + +#define NTP_LOCK(enable) \ + enable = ml_set_interrupts_enabled(FALSE); \ + lck_spin_lock(ntp_lock); + +#define NTP_UNLOCK(enable) \ + lck_spin_unlock(ntp_lock);\ + ml_set_interrupts_enabled(enable); + +#define NTP_ASSERT_LOCKED() LCK_SPIN_ASSERT(ntp_lock, LCK_ASSERT_OWNED) + +static timer_call_data_t ntp_loop_update; +static uint64_t ntp_loop_deadline; +static uint32_t ntp_loop_active; +static uint32_t ntp_loop_period; +#define NTP_LOOP_PERIOD_INTERVAL (NSEC_PER_SEC) /*1 second interval*/ + +void ntp_init(void); +static void hardupdate(long offset); +static void ntp_gettime1(struct ntptimeval *ntvp); +static bool ntp_is_time_error(int tsl); + +static void ntp_loop_update_call(void); +static void refresh_ntp_loop(void); +static void start_ntp_loop(void); + +static bool +ntp_is_time_error(int tsl) +{ + + if (tsl & (STA_UNSYNC | STA_CLOCKERR)) + return (true); + + return (false); +} + +static void +ntp_gettime1(struct ntptimeval *ntvp) +{ + struct timespec atv; + + NTP_ASSERT_LOCKED(); + + nanotime(&atv); + ntvp->time.tv_sec = atv.tv_sec; + ntvp->time.tv_nsec = atv.tv_nsec; + if ((unsigned long)atv.tv_sec > last_time_maxerror_update) { + time_maxerror += (MAXFREQ / 1000)*(atv.tv_sec-last_time_maxerror_update); + last_time_maxerror_update = atv.tv_sec; + } + ntvp->maxerror = time_maxerror; + ntvp->esterror = time_esterror; + ntvp->tai = time_tai; + ntvp->time_state = time_state; + + if (ntp_is_time_error(time_status)) + ntvp->time_state = TIME_ERROR; +} + +int +ntp_gettime(struct proc *p, struct ntp_gettime_args *uap, __unused int32_t *retval) +{ + struct ntptimeval ntv; + int error; + boolean_t enable; + + NTP_LOCK(enable); + ntp_gettime1(&ntv); + NTP_UNLOCK(enable); + + if (IS_64BIT_PROCESS(p)) { + struct user64_ntptimeval user_ntv; + user_ntv.time.tv_sec = ntv.time.tv_sec; + user_ntv.time.tv_nsec = ntv.time.tv_nsec; + user_ntv.maxerror = ntv.maxerror; + user_ntv.esterror = ntv.esterror; + user_ntv.tai = ntv.tai; + user_ntv.time_state = ntv.time_state; + error = copyout(&user_ntv, uap->ntvp, sizeof(user_ntv)); + } else { + struct user32_ntptimeval user_ntv; + user_ntv.time.tv_sec = ntv.time.tv_sec; + user_ntv.time.tv_nsec = ntv.time.tv_nsec; + user_ntv.maxerror = ntv.maxerror; + user_ntv.esterror = ntv.esterror; + user_ntv.tai = ntv.tai; + user_ntv.time_state = ntv.time_state; + error = copyout(&user_ntv, uap->ntvp, sizeof(user_ntv)); + } + + if (error) + return error; + + return ntv.time_state; +} + +int +ntp_adjtime(struct proc *p, struct ntp_adjtime_args *uap, __unused int32_t *retval) +{ + struct timex ntv; + long freq; + int modes; + int error, ret = 0; + clock_sec_t sec; + clock_usec_t microsecs; + boolean_t enable; + + if (IS_64BIT_PROCESS(p)) { + struct user64_timex user_ntv; + error = copyin(uap->tp, &user_ntv, sizeof(user_ntv)); + ntv.modes = user_ntv.modes; + ntv.offset = user_ntv.offset; + ntv.freq = user_ntv.freq; + ntv.maxerror = user_ntv.maxerror; + ntv.esterror = user_ntv.esterror; + ntv.status = user_ntv.status; + ntv.constant = user_ntv.constant; + ntv.precision = user_ntv.precision; + ntv.tolerance = user_ntv.tolerance; + + } else { + struct user32_timex user_ntv; + error = copyin(uap->tp, &user_ntv, sizeof(user_ntv)); + ntv.modes = user_ntv.modes; + ntv.offset = user_ntv.offset; + ntv.freq = user_ntv.freq; + ntv.maxerror = user_ntv.maxerror; + ntv.esterror = user_ntv.esterror; + ntv.status = user_ntv.status; + ntv.constant = user_ntv.constant; + ntv.precision = user_ntv.precision; + ntv.tolerance = user_ntv.tolerance; + } + if (error) + return (error); + + /* + * Update selected clock variables - only the superuser can + * change anything. Note that there is no error checking here on + * the assumption the superuser should know what it is doing. + * Note that either the time constant or TAI offset are loaded + * from the ntv.constant member, depending on the mode bits. If + * the STA_PLL bit in the status word is cleared, the state and + * status words are reset to the initial values at boot. + */ + modes = ntv.modes; + if (modes) { + /* Check that this task is entitled to set the time or it is root */ + if (!IOTaskHasEntitlement(current_task(), SETTIME_ENTITLEMENT)) { +#if CONFIG_MACF + error = mac_system_check_settime(kauth_cred_get()); + if (error) + return (error); +#endif + if ((error = priv_check_cred(kauth_cred_get(), PRIV_ADJTIME, 0))) + return (error); + + } + } + + NTP_LOCK(enable); + + if (modes & MOD_MAXERROR) { + clock_gettimeofday(&sec, µsecs); + time_maxerror = ntv.maxerror; + last_time_maxerror_update = sec; + } + if (modes & MOD_ESTERROR) + time_esterror = ntv.esterror; + if (modes & MOD_STATUS) { + if (time_status & STA_PLL && !(ntv.status & STA_PLL)) { + time_state = TIME_OK; + time_status = STA_UNSYNC; + } + time_status &= STA_RONLY; + time_status |= ntv.status & ~STA_RONLY; + /* + * Nor PPS or leaps seconds are supported. + * Filter out unsupported bits. + */ + time_status &= STA_SUPPORTED; + } + if (modes & MOD_TIMECONST) { + if (ntv.constant < 0) + time_constant = 0; + else if (ntv.constant > MAXTC) + time_constant = MAXTC; + else + time_constant = ntv.constant; + } + if (modes & MOD_TAI) { + if (ntv.constant > 0) + time_tai = ntv.constant; + } + if (modes & MOD_NANO) + time_status |= STA_NANO; + if (modes & MOD_MICRO) + time_status &= ~STA_NANO; + if (modes & MOD_CLKB) + time_status |= STA_CLK; + if (modes & MOD_CLKA) + time_status &= ~STA_CLK; + if (modes & MOD_FREQUENCY) { + freq = (ntv.freq * 1000LL) >> 16; + if (freq > MAXFREQ) + L_LINT(time_freq, MAXFREQ); + else if (freq < -MAXFREQ) + L_LINT(time_freq, -MAXFREQ); + else { + /* + * ntv.freq is [PPM * 2^16] = [us/s * 2^16] + * time_freq is [ns/s * 2^32] + */ + time_freq = ntv.freq * 1000LL * 65536LL; + } + } + if (modes & MOD_OFFSET) { + if (time_status & STA_NANO) + hardupdate(ntv.offset); + else + hardupdate(ntv.offset * 1000); + } + + ret = ntp_is_time_error(time_status) ? TIME_ERROR : time_state; + + /* + * Retrieve all clock variables. Note that the TAI offset is + * returned only by ntp_gettime(); + */ + if (IS_64BIT_PROCESS(p)) { + struct user64_timex user_ntv; + + if (time_status & STA_NANO) + user_ntv.offset = L_GINT(time_offset); + else + user_ntv.offset = L_GINT(time_offset) / 1000; + user_ntv.freq = L_GINT((time_freq / 1000LL) << 16); + user_ntv.maxerror = time_maxerror; + user_ntv.esterror = time_esterror; + user_ntv.status = time_status; + user_ntv.constant = time_constant; + if (time_status & STA_NANO) + user_ntv.precision = time_precision; + else + user_ntv.precision = time_precision / 1000; + user_ntv.tolerance = MAXFREQ * SCALE_PPM; + + /* unlock before copyout */ + NTP_UNLOCK(enable); + + error = copyout(&user_ntv, uap->tp, sizeof(user_ntv)); + + } + else{ + struct user32_timex user_ntv; + + if (time_status & STA_NANO) + user_ntv.offset = L_GINT(time_offset); + else + user_ntv.offset = L_GINT(time_offset) / 1000; + user_ntv.freq = L_GINT((time_freq / 1000LL) << 16); + user_ntv.maxerror = time_maxerror; + user_ntv.esterror = time_esterror; + user_ntv.status = time_status; + user_ntv.constant = time_constant; + if (time_status & STA_NANO) + user_ntv.precision = time_precision; + else + user_ntv.precision = time_precision / 1000; + user_ntv.tolerance = MAXFREQ * SCALE_PPM; + + /* unlock before copyout */ + NTP_UNLOCK(enable); + + error = copyout(&user_ntv, uap->tp, sizeof(user_ntv)); + } + + if (modes) + start_ntp_loop(); + + if (error == 0) + *retval = ret; + + return (error); +} + +int64_t +ntp_get_freq(void){ + return time_freq; +} + +/* + * Compute the adjustment to add to the next second. + */ +void +ntp_update_second(int64_t *adjustment, clock_sec_t secs) +{ + int tickrate; + l_fp time_adj; + l_fp ftemp, old_time_adjtime, old_offset; + + NTP_ASSERT_LOCKED(); + + if (secs > last_time_maxerror_update) { + time_maxerror += (MAXFREQ / 1000)*(secs-last_time_maxerror_update); + last_time_maxerror_update = secs; + } + + old_offset = time_offset; + old_time_adjtime = time_adjtime; + + ftemp = time_offset; + L_RSHIFT(ftemp, SHIFT_PLL + time_constant); + time_adj = ftemp; + L_SUB(time_offset, ftemp); + L_ADD(time_adj, time_freq); + + /* + * Apply any correction from adjtime. If more than one second + * off we slew at a rate of 5ms/s (5000 PPM) else 500us/s (500PPM) + * until the last second is slewed the final < 500 usecs. + */ + if (time_adjtime != 0) { + if (time_adjtime > 1000000) + tickrate = 5000; + else if (time_adjtime < -1000000) + tickrate = -5000; + else if (time_adjtime > 500) + tickrate = 500; + else if (time_adjtime < -500) + tickrate = -500; + else + tickrate = time_adjtime; + time_adjtime -= tickrate; + L_LINT(ftemp, tickrate * 1000); + L_ADD(time_adj, ftemp); + } + + if (old_time_adjtime || ((time_offset || old_offset) && (time_offset != old_offset))) { + updated = 1; + } + else{ + updated = 0; + } + + *adjustment = time_adj; +} + +/* + * hardupdate() - local clock update + * + * This routine is called by ntp_adjtime() when an offset is provided + * to update the local clock phase and frequency. + * The implementation is of an adaptive-parameter, hybrid + * phase/frequency-lock loop (PLL/FLL). The routine computes new + * time and frequency offset estimates for each call. + * Presumably, calls to ntp_adjtime() occur only when the caller + * believes the local clock is valid within some bound (+-128 ms with + * NTP). + * + * For uncompensated quartz crystal oscillators and nominal update + * intervals less than 256 s, operation should be in phase-lock mode, + * where the loop is disciplined to phase. For update intervals greater + * than 1024 s, operation should be in frequency-lock mode, where the + * loop is disciplined to frequency. Between 256 s and 1024 s, the mode + * is selected by the STA_MODE status bit. + */ +static void +hardupdate(offset) + long offset; +{ + long mtemp = 0; + long time_monitor; + clock_sec_t time_uptime; + l_fp ftemp; + + NTP_ASSERT_LOCKED(); + + if (!(time_status & STA_PLL)) + return; + + if (offset > MAXPHASE) + time_monitor = MAXPHASE; + else if (offset < -MAXPHASE) + time_monitor = -MAXPHASE; + else + time_monitor = offset; + L_LINT(time_offset, time_monitor); + + clock_get_calendar_uptime(&time_uptime); + + if (time_status & STA_FREQHOLD || time_reftime == 0) { + time_reftime = time_uptime; + } + + mtemp = time_uptime - time_reftime; + L_LINT(ftemp, time_monitor); + L_RSHIFT(ftemp, (SHIFT_PLL + 2 + time_constant) << 1); + L_MPY(ftemp, mtemp); + L_ADD(time_freq, ftemp); + time_status &= ~STA_MODE; + if (mtemp >= MINSEC && (time_status & STA_FLL || mtemp > + MAXSEC)) { + L_LINT(ftemp, (time_monitor << 4) / mtemp); + L_RSHIFT(ftemp, SHIFT_FLL + 4); + L_ADD(time_freq, ftemp); + time_status |= STA_MODE; + } + time_reftime = time_uptime; + + if (L_GINT(time_freq) > MAXFREQ) + L_LINT(time_freq, MAXFREQ); + else if (L_GINT(time_freq) < -MAXFREQ) + L_LINT(time_freq, -MAXFREQ); +} + + +static int +kern_adjtime(struct timeval *delta) +{ + struct timeval atv; + int64_t ltr, ltw; + boolean_t enable; + + if (delta == NULL) + return (EINVAL); + + ltw = (int64_t)delta->tv_sec * (int64_t)USEC_PER_SEC + delta->tv_usec; + + NTP_LOCK(enable); + ltr = time_adjtime; + time_adjtime = ltw; + NTP_UNLOCK(enable); + + atv.tv_sec = ltr / (int64_t)USEC_PER_SEC; + atv.tv_usec = ltr % (int64_t)USEC_PER_SEC; + if (atv.tv_usec < 0) { + atv.tv_usec += (suseconds_t)USEC_PER_SEC; + atv.tv_sec--; + } + + *delta = atv; + + start_ntp_loop(); + + return (0); +} + +int +adjtime(struct proc *p, struct adjtime_args *uap, __unused int32_t *retval) +{ + + struct timeval atv; + int error; + + /* Check that this task is entitled to set the time or it is root */ + if (!IOTaskHasEntitlement(current_task(), SETTIME_ENTITLEMENT)) { + +#if CONFIG_MACF + error = mac_system_check_settime(kauth_cred_get()); + if (error) + return (error); +#endif + if ((error = priv_check_cred(kauth_cred_get(), PRIV_ADJTIME, 0))) + return (error); + } + + if (IS_64BIT_PROCESS(p)) { + struct user64_timeval user_atv; + error = copyin(uap->delta, &user_atv, sizeof(user_atv)); + atv.tv_sec = user_atv.tv_sec; + atv.tv_usec = user_atv.tv_usec; + } else { + struct user32_timeval user_atv; + error = copyin(uap->delta, &user_atv, sizeof(user_atv)); + atv.tv_sec = user_atv.tv_sec; + atv.tv_usec = user_atv.tv_usec; + } + if (error) + return (error); + + kern_adjtime(&atv); + + if (uap->olddelta) { + if (IS_64BIT_PROCESS(p)) { + struct user64_timeval user_atv; + user_atv.tv_sec = atv.tv_sec; + user_atv.tv_usec = atv.tv_usec; + error = copyout(&user_atv, uap->olddelta, sizeof(user_atv)); + } else { + struct user32_timeval user_atv; + user_atv.tv_sec = atv.tv_sec; + user_atv.tv_usec = atv.tv_usec; + error = copyout(&user_atv, uap->olddelta, sizeof(user_atv)); + } + } + + return (error); + +} + +static void +ntp_loop_update_call(void) +{ + boolean_t enable; + + NTP_LOCK(enable); + + /* + * Update the scale factor used by clock_calend. + * NOTE: clock_update_calendar will call ntp_update_second to compute the next adjustment. + */ + clock_update_calendar(); + + refresh_ntp_loop(); + + NTP_UNLOCK(enable); +} + +static void +refresh_ntp_loop(void) +{ + + NTP_ASSERT_LOCKED(); + if (--ntp_loop_active == 0) { + /* + * Activate the timer only if the next second adjustment might change. + * ntp_update_second checks it and sets updated accordingly. + */ + if (updated) { + clock_deadline_for_periodic_event(ntp_loop_period, mach_absolute_time(), &ntp_loop_deadline); + + if (!timer_call_enter(&ntp_loop_update, ntp_loop_deadline, TIMER_CALL_SYS_CRITICAL)) + ntp_loop_active++; + } + } + +} + +/* + * This function triggers a timer that each second will calculate the adjustment to + * provide to clock_calendar to scale the time (used by gettimeofday-family syscalls). + * The periodic timer will stop when the adjustment will reach a stable value. + */ +static void +start_ntp_loop(void) +{ + boolean_t enable; + + NTP_LOCK(enable); + + ntp_loop_deadline = mach_absolute_time() + ntp_loop_period; + + if (!timer_call_enter(&ntp_loop_update, ntp_loop_deadline, TIMER_CALL_SYS_CRITICAL)) { + ntp_loop_active++; + } + + NTP_UNLOCK(enable); +} + + +static void +init_ntp_loop(void) +{ + uint64_t abstime; + + ntp_loop_active = 0; + nanoseconds_to_absolutetime(NTP_LOOP_PERIOD_INTERVAL, &abstime); + ntp_loop_period = (uint32_t)abstime; + timer_call_setup(&ntp_loop_update, (timer_call_func_t)ntp_loop_update_call, NULL); +} + +void +ntp_init(void) +{ + + L_CLR(time_offset); + L_CLR(time_freq); + + ntp_lock_grp_attr = lck_grp_attr_alloc_init(); + ntp_lock_grp = lck_grp_alloc_init("ntp_lock", ntp_lock_grp_attr); + ntp_lock_attr = lck_attr_alloc_init(); + ntp_lock = lck_spin_alloc_init(ntp_lock_grp, ntp_lock_attr); + + updated = 0; + + init_ntp_loop(); +} + +SYSINIT(ntpclocks, SI_SUB_CLOCKS, SI_ORDER_MIDDLE, ntp_init, NULL); diff --git a/bsd/kern/kern_overrides.c b/bsd/kern/kern_overrides.c index 7b2a4622c..8e70d80f0 100644 --- a/bsd/kern/kern_overrides.c +++ b/bsd/kern/kern_overrides.c @@ -39,7 +39,6 @@ #include <sys/kauth.h> #include <sys/unistd.h> #include <sys/priv.h> -#include <security/audit/audit.h> #include <mach/mach_types.h> #include <mach/vm_param.h> diff --git a/bsd/kern/kern_priv.c b/bsd/kern/kern_priv.c index ee17dd2a8..462c3fbd2 100644 --- a/bsd/kern/kern_priv.c +++ b/bsd/kern/kern_priv.c @@ -75,7 +75,7 @@ int proc_check_footprint_priv(void); * only a few to grant it. */ int -priv_check_cred(kauth_cred_t cred, int priv, __unused int flags) +priv_check_cred(kauth_cred_t cred, int priv, int flags) { #if !CONFIG_MACF #pragma unused(priv) @@ -92,15 +92,18 @@ priv_check_cred(kauth_cred_t cred, int priv, __unused int flags) goto out; #endif - /* - * Having determined if privilege is restricted by various policies, - * now determine if privilege is granted. At this point, any policy - * may grant privilege. For now, we allow short-circuit boolean - * evaluation, so may not call all policies. Perhaps we should. - */ - if (kauth_cred_getuid(cred) == 0) { - error = 0; - goto out; + /* Only grant all privileges to root if DEFAULT_UNPRIVELEGED flag is NOT set. */ + if (!(flags & PRIVCHECK_DEFAULT_UNPRIVILEGED_FLAG)) { + /* + * Having determined if privilege is restricted by various policies, + * now determine if privilege is granted. At this point, any policy + * may grant privilege. For now, we allow short-circuit boolean + * evaluation, so may not call all policies. Perhaps we should. + */ + if (kauth_cred_getuid(cred) == 0) { + error = 0; + goto out; + } } /* diff --git a/bsd/kern/kern_proc.c b/bsd/kern/kern_proc.c index 8628e301f..c599a4bc7 100644 --- a/bsd/kern/kern_proc.c +++ b/bsd/kern/kern_proc.c @@ -112,6 +112,10 @@ #include <sys/bsdtask_info.h> #include <sys/persona.h> +#if CONFIG_CSR +#include <sys/csr.h> +#endif + #if CONFIG_MEMORYSTATUS #include <sys/kern_memorystatus.h> #endif @@ -150,13 +154,21 @@ extern struct tty cons; extern int cs_debug; +#if DEVELOPMENT || DEBUG +extern int cs_enforcement_enable; +#endif + #if DEBUG #define __PROC_INTERNAL_DEBUG 1 #endif #if CONFIG_COREDUMP /* Name to give to core files */ +#if CONFIG_EMBEDDED +__XNU_PRIVATE_EXTERN char corefilename[MAXPATHLEN+1] = {"/private/var/cores/%N.core"}; +#else __XNU_PRIVATE_EXTERN char corefilename[MAXPATHLEN+1] = {"/cores/core.%P"}; #endif +#endif #if PROC_REF_DEBUG #include <kern/backtrace.h> @@ -488,11 +500,16 @@ proc_t proc_ref_locked(proc_t p) { proc_t p1 = p; + int pid = proc_pid(p); - /* if process still in creation return failure */ - if ((p == PROC_NULL) || ((p->p_listflag & P_LIST_INCREATE) != 0)) - return (PROC_NULL); retry: + /* + * if process still in creation or proc got recycled + * during msleep then return failure. + */ + if ((p == PROC_NULL) || (p1 != p) || ((p->p_listflag & P_LIST_INCREATE) != 0)) + return (PROC_NULL); + /* * Do not return process marked for termination * or proc_refdrain called without ref wait. @@ -508,6 +525,11 @@ retry: ((p->p_listflag & P_LIST_REFWAIT) != 0))) { if ((p->p_listflag & P_LIST_REFWAIT) != 0 && uthread_needs_to_wait_in_proc_refwait()) { msleep(&p->p_listflag, proc_list_mlock, 0, "proc_refwait", 0) ; + /* + * the proc might have been recycled since we dropped + * the proc list lock, get the proc again. + */ + p = pfind_locked(pid); goto retry; } p->p_refcount++; @@ -1148,6 +1170,7 @@ bsd_set_dependency_capable(task_t task) } +#ifndef __arm__ int IS_64BIT_PROCESS(proc_t p) { @@ -1156,6 +1179,7 @@ IS_64BIT_PROCESS(proc_t p) else return(0); } +#endif /* * Locate a process by number @@ -1297,6 +1321,7 @@ pinsertchild(proc_t parent, proc_t child) child->p_pptr = parent; child->p_ppid = parent->p_pid; child->p_puniqueid = parent->p_uniqueid; + child->p_xhighbits = 0; pg = proc_pgrp(parent); pgrp_add(pg, parent, child); @@ -1949,6 +1974,7 @@ csops_internal(pid_t pid, int ops, user_addr_t uaddr, user_size_t usersize, user case CS_OPS_MARKRESTRICT: case CS_OPS_SET_STATUS: case CS_OPS_CLEARINSTALLER: + case CS_OPS_CLEARPLATFORM: if ((error = mac_proc_check_set_cs_info(current_proc(), pt, ops))) goto out; break; @@ -2149,7 +2175,7 @@ csops_internal(pid_t pid, int ops, user_addr_t uaddr, user_size_t usersize, user error = ENOENT; break; } - + length = strlen(identity) + 1; /* include NUL */ idlen = htonl(length + sizeof(fakeheader)); memcpy(&fakeheader[4], &idlen, sizeof(idlen)); @@ -2168,10 +2194,34 @@ csops_internal(pid_t pid, int ops, user_addr_t uaddr, user_size_t usersize, user case CS_OPS_CLEARINSTALLER: proc_lock(pt); - pt->p_csflags &= ~(CS_INSTALLER | CS_EXEC_SET_INSTALLER); + pt->p_csflags &= ~(CS_INSTALLER | CS_DATAVAULT_CONTROLLER | CS_EXEC_INHERIT_SIP); proc_unlock(pt); break; + case CS_OPS_CLEARPLATFORM: +#if DEVELOPMENT || DEBUG + if (cs_enforcement_enable) { + error = ENOTSUP; + break; + } + +#if CONFIG_CSR + if (csr_check(CSR_ALLOW_APPLE_INTERNAL) != 0) { + error = ENOTSUP; + break; + } +#endif + + proc_lock(pt); + pt->p_csflags &= ~(CS_PLATFORM_BINARY|CS_PLATFORM_PATH); + csproc_clear_platform_binary(pt); + proc_unlock(pt); + break; +#else + error = ENOTSUP; + break; +#endif /* !DEVELOPMENT || DEBUG */ + default: error = EINVAL; break; @@ -2201,7 +2251,7 @@ proc_iterate( for (;;) { proc_list_lock(); - pid_count_available = nprocs; + pid_count_available = nprocs + 1; //kernel_task is not counted in nprocs assert(pid_count_available > 0); pid_list_size_needed = pid_count_available * sizeof(pid_t); @@ -3170,6 +3220,10 @@ extern boolean_t kill_on_no_paging_space; #endif /* DEVELOPMENT || DEBUG */ #define MB_SIZE (1024 * 1024ULL) +boolean_t memorystatus_kill_on_VM_thrashing(boolean_t); + +extern int32_t max_kill_priority; +extern int memorystatus_get_proccnt_upto_priority(int32_t max_bucket_index); int no_paging_space_action() @@ -3235,6 +3289,22 @@ no_paging_space_action() } } + /* + * We have some processes within our jetsam bands of consideration and hence can be killed. + * So we will invoke the memorystatus thread to go ahead and kill something. + */ + if (memorystatus_get_proccnt_upto_priority(max_kill_priority) > 0) { + + last_no_space_action = now; + memorystatus_kill_on_VM_thrashing(TRUE /* async */); + return (1); + } + + /* + * No eligible processes to kill. So let's suspend/kill the largest + * process depending on its policy control specifications. + */ + if (nps.pcs_max_size > 0) { if ((p = proc_find(nps.pcs_pid)) != PROC_NULL) { @@ -3246,22 +3316,6 @@ no_paging_space_action() */ last_no_space_action = now; -#if DEVELOPMENT || DEBUG - if (kill_on_no_paging_space == TRUE) { - /* - * We found the largest process that has a process policy i.e. one of - * PC_KILL, PC_SUSP, PC_THROTTLE. - * But we are in a mode where we will kill it regardless of its policy. - */ - printf("low swap: killing largest process with pid %d (%s) and size %llu MB\n", p->p_pid, p->p_comm, (nps.pcs_max_size/MB_SIZE)); - psignal(p, SIGKILL); - - proc_rele(p); - - return 1; - } -#endif /* DEVELOPMENT || DEBUG */ - proc_dopcontrol(p); proc_rele(p); diff --git a/bsd/kern/kern_resource.c b/bsd/kern/kern_resource.c index 625916715..7bc8466bc 100644 --- a/bsd/kern/kern_resource.c +++ b/bsd/kern/kern_resource.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2008 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -81,8 +81,6 @@ #include <sys/malloc.h> #include <sys/proc_internal.h> #include <sys/kauth.h> -#include <machine/spl.h> - #include <sys/mount_internal.h> #include <sys/sysproto.h> @@ -104,6 +102,9 @@ #include <kern/clock.h> /* for absolutetime_to_microtime() */ #include <netinet/in.h> /* for TRAFFIC_MGT_SO_* */ #include <sys/socketvar.h> /* for struct socket */ +#if NECP +#include <net/necp.h> +#endif /* NECP */ #include <vm/vm_map.h> @@ -112,6 +113,10 @@ #include <sys/priv.h> #include <IOKit/IOBSD.h> +#if CONFIG_MACF +#include <security/mac_framework.h> +#endif + int donice(struct proc *curp, struct proc *chgp, int n); int dosetrlimit(struct proc *p, u_int which, struct rlimit *limp); int uthread_get_background_state(uthread_t); @@ -128,7 +133,8 @@ int fill_task_rusage(task_t task, rusage_info_current *ri); void fill_task_billed_usage(task_t task, rusage_info_current *ri); int fill_task_io_rusage(task_t task, rusage_info_current *ri); int fill_task_qos_rusage(task_t task, rusage_info_current *ri); -static void rusage_info_conversion(rusage_info_t ri_info, rusage_info_current *ri_current, int flavor); +uint64_t get_task_logical_writes(task_t task); +void fill_task_monotonic_rusage(task_t task, rusage_info_current *ri); int proc_get_rusage(proc_t p, int flavor, user_addr_t buffer, __unused int is_zombie); @@ -583,7 +589,7 @@ static int proc_set_darwin_role(proc_t curp, proc_t targetp, int priority) { int error = 0; - uint32_t flagsp; + uint32_t flagsp = 0; kauth_cred_t ucred, target_cred; @@ -762,16 +768,20 @@ do_background_socket(struct proc *p, thread_t thread) fdp = p->p_fd; for (i = 0; i < fdp->fd_nfiles; i++) { - struct socket *sockp; - fp = fdp->fd_ofiles[i]; - if (fp == NULL || (fdp->fd_ofileflags[i] & UF_RESERVED) != 0 || - FILEGLOB_DTYPE(fp->f_fglob) != DTYPE_SOCKET) { + if (fp == NULL || (fdp->fd_ofileflags[i] & UF_RESERVED) != 0) { continue; } - sockp = (struct socket *)fp->f_fglob->fg_data; - socket_set_traffic_mgt_flags(sockp, TRAFFIC_MGT_SO_BACKGROUND); - sockp->so_background_thread = NULL; + if (FILEGLOB_DTYPE(fp->f_fglob) == DTYPE_SOCKET) { + struct socket *sockp = (struct socket *)fp->f_fglob->fg_data; + socket_set_traffic_mgt_flags(sockp, TRAFFIC_MGT_SO_BACKGROUND); + sockp->so_background_thread = NULL; + } +#if NECP + else if (FILEGLOB_DTYPE(fp->f_fglob) == DTYPE_NETPOLICY) { + necp_set_client_as_background(p, fp, background); + } +#endif /* NECP */ } } } else { @@ -785,17 +795,23 @@ do_background_socket(struct proc *p, thread_t thread) struct socket *sockp; fp = fdp->fd_ofiles[ i ]; - if ( fp == NULL || (fdp->fd_ofileflags[ i ] & UF_RESERVED) != 0 || - FILEGLOB_DTYPE(fp->f_fglob) != DTYPE_SOCKET ) { + if (fp == NULL || (fdp->fd_ofileflags[ i ] & UF_RESERVED) != 0) { continue; } - sockp = (struct socket *)fp->f_fglob->fg_data; - /* skip if only clearing this thread's sockets */ - if ((thread) && (sockp->so_background_thread != thread)) { - continue; + if (FILEGLOB_DTYPE(fp->f_fglob) == DTYPE_SOCKET) { + sockp = (struct socket *)fp->f_fglob->fg_data; + /* skip if only clearing this thread's sockets */ + if ((thread) && (sockp->so_background_thread != thread)) { + continue; + } + socket_clear_traffic_mgt_flags(sockp, TRAFFIC_MGT_SO_BACKGROUND); + sockp->so_background_thread = NULL; } - socket_clear_traffic_mgt_flags(sockp, TRAFFIC_MGT_SO_BACKGROUND); - sockp->so_background_thread = NULL; +#if NECP + else if (FILEGLOB_DTYPE(fp->f_fglob) == DTYPE_NETPOLICY) { + necp_set_client_as_background(p, fp, background); + } +#endif /* NECP */ } } @@ -1509,9 +1525,15 @@ iopolicysys_disk(struct proc *p __unused, int cmd, int scope, int policy, struct break; case IOPOL_SCOPE_DARWIN_BG: +#if CONFIG_EMBEDDED + /* Embedded doesn't want this as BG is always IOPOL_THROTTLE */ + error = ENOTSUP; + goto out; +#else /* CONFIG_EMBEDDED */ thread = THREAD_NULL; policy_flavor = TASK_POLICY_DARWIN_BG_IOPOL; break; +#endif /* CONFIG_EMBEDDED */ default: error = EINVAL; @@ -1670,7 +1692,13 @@ gather_rusage_info(proc_t p, rusage_info_current *ru, int flavor) struct rusage_info_child *ri_child; assert(p->p_stats != NULL); + memset(ru, 0, sizeof(*ru)); switch(flavor) { + case RUSAGE_INFO_V4: + ru->ri_logical_writes = get_task_logical_writes(p->task); + ru->ri_lifetime_max_phys_footprint = get_task_phys_footprint_lifetime_max(p->task); + fill_task_monotonic_rusage(p->task, ru); + /* fall through */ case RUSAGE_INFO_V3: fill_task_qos_rusage(p->task, ru); @@ -1705,131 +1733,57 @@ gather_rusage_info(proc_t p, rusage_info_current *ru, int flavor) } } -static void -rusage_info_conversion(rusage_info_t ri_info, rusage_info_current *ri_current, int flavor) -{ - struct rusage_info_v0 *ri_v0; - struct rusage_info_v1 *ri_v1; - struct rusage_info_v2 *ri_v2; - - switch (flavor) { - - case RUSAGE_INFO_V2: - ri_v2 = (struct rusage_info_v2 *)ri_info; - ri_v2->ri_diskio_bytesread = ri_current->ri_diskio_bytesread; - ri_v2->ri_diskio_byteswritten = ri_current->ri_diskio_byteswritten; - /* fall through */ - - case RUSAGE_INFO_V1: - ri_v1 = (struct rusage_info_v1 *)ri_info; - ri_v1->ri_child_user_time = ri_current->ri_child_user_time; - ri_v1->ri_child_system_time = ri_current->ri_child_system_time; - ri_v1->ri_child_pkg_idle_wkups = ri_current->ri_child_pkg_idle_wkups; - ri_v1->ri_child_interrupt_wkups = ri_current->ri_child_interrupt_wkups; - ri_v1->ri_child_pageins = ri_current->ri_child_pageins; - ri_v1->ri_child_elapsed_abstime = ri_current->ri_child_elapsed_abstime; - /* fall through */ - - case RUSAGE_INFO_V0: - ri_v0 = (struct rusage_info_v0 *)ri_info; - memcpy(&ri_v0->ri_uuid[0], &ri_current->ri_uuid[0], sizeof(ri_v0->ri_uuid)); - ri_v0->ri_user_time = ri_current->ri_user_time; - ri_v0->ri_system_time = ri_current->ri_system_time; - ri_v0->ri_pkg_idle_wkups = ri_current->ri_pkg_idle_wkups; - ri_v0->ri_interrupt_wkups = ri_current->ri_interrupt_wkups; - ri_v0->ri_pageins = ri_current->ri_pageins; - ri_v0->ri_wired_size = ri_current->ri_wired_size; - ri_v0->ri_resident_size = ri_current->ri_resident_size; - ri_v0->ri_phys_footprint = ri_current->ri_phys_footprint; - ri_v0->ri_proc_start_abstime = ri_current->ri_proc_start_abstime; - ri_v0->ri_proc_exit_abstime = ri_current->ri_proc_exit_abstime; - - break; - - default: - break; - } -} - - int proc_get_rusage(proc_t p, int flavor, user_addr_t buffer, __unused int is_zombie) { - struct rusage_info_v0 ri_v0; - struct rusage_info_v1 ri_v1; - struct rusage_info_v2 ri_v2; - struct rusage_info_v3 ri_v3; - rusage_info_current ri_current; int error = 0; + size_t size = 0; switch (flavor) { case RUSAGE_INFO_V0: - /* - * If task is still alive, collect info from the live task itself. - * Otherwise, look to the cached info in the zombie proc. - */ - if (p->p_ru == NULL) { - gather_rusage_info(p, &ri_current, flavor); - ri_current.ri_proc_exit_abstime = 0; - rusage_info_conversion(&ri_v0, &ri_current, flavor); - } else { - rusage_info_conversion(&ri_v0, &p->p_ru->ri, flavor); - } - error = copyout(&ri_v0, buffer, sizeof (ri_v0)); + size = sizeof(struct rusage_info_v0); break; case RUSAGE_INFO_V1: - /* - * If task is still alive, collect info from the live task itself. - * Otherwise, look to the cached info in the zombie proc. - */ - if (p->p_ru == NULL) { - gather_rusage_info(p, &ri_current, flavor); - ri_current.ri_proc_exit_abstime = 0; - rusage_info_conversion(&ri_v1, &ri_current, flavor); - } else { - rusage_info_conversion(&ri_v1, &p->p_ru->ri, flavor); - } - error = copyout(&ri_v1, buffer, sizeof (ri_v1)); + size = sizeof(struct rusage_info_v1); break; case RUSAGE_INFO_V2: - /* - * If task is still alive, collect info from the live task itself. - * Otherwise, look to the cached info in the zombie proc. - */ - if (p->p_ru == NULL) { - gather_rusage_info(p, &ri_current, flavor); - ri_current.ri_proc_exit_abstime = 0; - rusage_info_conversion(&ri_v2, &ri_current, flavor); - } else { - rusage_info_conversion(&ri_v2, &p->p_ru->ri, flavor); - } - error = copyout(&ri_v2, buffer, sizeof (ri_v2)); + size = sizeof(struct rusage_info_v2); break; case RUSAGE_INFO_V3: - /* - * If task is still alive, collect info from the live task itself. - * Otherwise, look to the cached info in the zombie proc. - */ - if (p->p_ru == NULL) { - gather_rusage_info(p, &ri_v3, flavor); - ri_v3.ri_proc_exit_abstime = 0; - } else { - ri_v3 = p->p_ru->ri; - } - error = copyout(&ri_v3, buffer, sizeof (ri_v3)); + size = sizeof(struct rusage_info_v3); break; - default: - error = EINVAL; + case RUSAGE_INFO_V4: + size = sizeof(struct rusage_info_v4); break; + + default: + return EINVAL; + } + + if(size == 0) { + return EINVAL; } - return (error); + /* + * If task is still alive, collect info from the live task itself. + * Otherwise, look to the cached info in the zombie proc. + */ + if (p->p_ru == NULL) { + gather_rusage_info(p, &ri_current, flavor); + ri_current.ri_proc_exit_abstime = 0; + error = copyout(&ri_current, buffer, size); + } else { + ri_current = p->p_ru->ri; + error = copyout(&p->p_ru->ri, buffer, size); + } + + return (error); } static int @@ -1955,3 +1909,10 @@ int thread_selfusage(struct proc *p __unused, struct thread_selfusage_args *uap return (0); } + +#if !MONOTONIC +int thread_selfcounts(__unused struct proc *p, __unused struct thread_selfcounts_args *uap, __unused int *ret_out) +{ + return ENOTSUP; +} +#endif /* !MONOTONIC */ diff --git a/bsd/kern/kern_shutdown.c b/bsd/kern/kern_shutdown.c index 216a32baf..2be1a1310 100644 --- a/bsd/kern/kern_shutdown.c +++ b/bsd/kern/kern_shutdown.c @@ -84,6 +84,8 @@ static void sd_log(vfs_context_t, const char *, ...); static void proc_shutdown(void); static void kernel_hwm_panic_info(void); extern void IOSystemShutdownNotification(void); +extern void halt_log_enter(const char * what, const void * pc, uint64_t time); + #if DEVELOPMENT || DEBUG extern boolean_t kdp_has_polled_corefile(void); #endif /* DEVELOPMENT || DEBUG */ @@ -112,31 +114,29 @@ static int sd_callback2(proc_t p, void * arg); static int sd_callback3(proc_t p, void * arg); extern boolean_t panic_include_zprint; -extern vm_offset_t panic_kext_memory_info; -extern vm_size_t panic_kext_memory_size; +extern mach_memory_info_t *panic_kext_memory_info; +extern vm_size_t panic_kext_memory_size; static void kernel_hwm_panic_info(void) { - mach_memory_info_t *memory_info; - unsigned int num_sites; - kern_return_t kr; + unsigned int num_sites; + kern_return_t kr; panic_include_zprint = TRUE; - panic_kext_memory_info = 0; + panic_kext_memory_info = NULL; panic_kext_memory_size = 0; - num_sites = VM_KERN_MEMORY_COUNT + VM_KERN_COUNTER_COUNT; - panic_kext_memory_size = round_page(num_sites * sizeof(mach_zone_info_t)); - - kr = kmem_alloc(kernel_map, (vm_offset_t *) &panic_kext_memory_info, panic_kext_memory_size, VM_KERN_MEMORY_OSFMK); + num_sites = vm_page_diagnose_estimate(); + panic_kext_memory_size = num_sites * sizeof(panic_kext_memory_info[0]); + + kr = kmem_alloc(kernel_map, (vm_offset_t *)&panic_kext_memory_info, round_page(panic_kext_memory_size), VM_KERN_MEMORY_OSFMK); if (kr != KERN_SUCCESS) { - panic_kext_memory_info = 0; + panic_kext_memory_info = NULL; return; } - memory_info = (mach_memory_info_t *)panic_kext_memory_info; - vm_page_diagnose(memory_info, num_sites, 0); - return; + + vm_page_diagnose(panic_kext_memory_info, num_sites, 0); } int @@ -149,6 +149,7 @@ int reboot_kernel(int howto, char *message) { int hostboot_option=0; + uint64_t startTime; if (!OSCompareAndSwap(0, 1, &system_inshutdown)) { if ( (howto&RB_QUICK) == RB_QUICK) @@ -177,19 +178,28 @@ reboot_kernel(int howto, char *message) */ /* handle live procs (deallocate their root and current directories), suspend initproc */ + + startTime = mach_absolute_time(); proc_shutdown(); + halt_log_enter("proc_shutdown", 0, mach_absolute_time() - startTime); #if CONFIG_AUDIT + startTime = mach_absolute_time(); audit_shutdown(); + halt_log_enter("audit_shutdown", 0, mach_absolute_time() - startTime); #endif if (unmountroot_pre_hook != NULL) unmountroot_pre_hook(); + startTime = mach_absolute_time(); sync((proc_t)NULL, (void *)NULL, (int *)NULL); - if (kdebug_enable) + if (kdebug_enable) { + startTime = mach_absolute_time(); kdbg_dump_trace_to_file("/var/log/shutdown/shutdown.trace"); + halt_log_enter("shutdown.trace", 0, mach_absolute_time() - startTime); + } /* * Unmount filesystems @@ -199,10 +209,13 @@ reboot_kernel(int howto, char *message) if (!(howto & RB_PANIC) || !kdp_has_polled_corefile()) #endif /* DEVELOPMENT || DEBUG */ { + startTime = mach_absolute_time(); vfs_unmountall(); + halt_log_enter("vfs_unmountall", 0, mach_absolute_time() - startTime); } /* Wait for the buffer cache to clean remaining dirty buffers */ + startTime = mach_absolute_time(); for (iter = 0; iter < 100; iter++) { nbusy = count_busy_buffers(); if (nbusy == 0) @@ -214,6 +227,7 @@ reboot_kernel(int howto, char *message) printf("giving up\n"); else printf("done\n"); + halt_log_enter("bufferclean", 0, mach_absolute_time() - startTime); } #if NETWORKING /* @@ -221,7 +235,9 @@ reboot_kernel(int howto, char *message) * because that will lock out softints which the disk * drivers depend on to finish DMAs. */ + startTime = mach_absolute_time(); if_down_all(); + halt_log_enter("if_down_all", 0, mach_absolute_time() - startTime); #endif /* NETWORKING */ force_reboot: @@ -455,7 +471,7 @@ sd_callback3(proc_t p, void * args) * * POSIX modifications: * - * For POSIX fcntl() file locking call vno_lockrelease() on + * For POSIX fcntl() file locking call vno_lockrelease() on * the file to release all of its record locks, if any. */ @@ -474,10 +490,10 @@ proc_shutdown(void) * Kill as many procs as we can. (Except ourself...) */ self = (struct proc *)current_proc(); - + /* * Signal the init with SIGTERM so that he does not launch - * new processes + * new processes */ p = proc_find(1); if (p && p != self) { @@ -506,11 +522,11 @@ sigterm_loop: proc_list_lock(); if (proc_shutdown_exitcount != 0) { /* - * now wait for up to 30 seconds to allow those procs catching SIGTERM - * to digest it - * as soon as these procs have exited, we'll continue on to the next step - */ - ts.tv_sec = 30; + * now wait for up to 3 seconds to allow those procs catching SIGTERM + * to digest it + * as soon as these procs have exited, we'll continue on to the next step + */ + ts.tv_sec = 3; ts.tv_nsec = 0; error = msleep(&proc_shutdown_exitcount, proc_list_mlock, PWAIT, "shutdownwait", &ts); if (error != 0) { @@ -523,7 +539,6 @@ sigterm_loop: p->p_listflag &= ~P_LIST_EXITCOUNT; } } - } proc_list_unlock(); } @@ -532,7 +547,6 @@ sigterm_loop: * log the names of the unresponsive tasks */ - proc_list_lock(); for (p = allproc.lh_first; p; p = p->p_list.le_next) { @@ -543,8 +557,6 @@ sigterm_loop: } proc_list_unlock(); - - delay_for_interval(1000 * 5, 1000 * 1000); } /* @@ -560,16 +572,18 @@ sigterm_loop: /* post a SIGKILL to all that catch SIGTERM and not marked for delay */ proc_rebootscan(sd_callback2, (void *)&sdargs, sd_filt2, (void *)&sfargs); + error = 0; + if (sdargs.activecount != 0 && proc_shutdown_exitcount!= 0) { proc_list_lock(); if (proc_shutdown_exitcount != 0) { /* - * wait for up to 60 seconds to allow these procs to exit normally - * - * History: The delay interval was changed from 100 to 200 - * for NFS requests in particular. - */ - ts.tv_sec = 60; + * wait for up to 60 seconds to allow these procs to exit normally + * + * History: The delay interval was changed from 100 to 200 + * for NFS requests in particular. + */ + ts.tv_sec = 10; ts.tv_nsec = 0; error = msleep(&proc_shutdown_exitcount, proc_list_mlock, PWAIT, "shutdownwait", &ts); if (error != 0) { @@ -586,6 +600,23 @@ sigterm_loop: proc_list_unlock(); } + if (error == ETIMEDOUT) { + /* + * log the names of the unresponsive tasks + */ + + proc_list_lock(); + + for (p = allproc.lh_first; p; p = p->p_list.le_next) { + if (p->p_shutdownstate == 2) { + printf("%s[%d]: didn't act on SIGKILL\n", p->p_comm, p->p_pid); + sd_log(ctx, "%s[%d]: didn't act on SIGKILL\n", p->p_comm, p->p_pid); + } + } + + proc_list_unlock(); + } + /* * if we still have procs that haven't exited, then brute force 'em */ @@ -596,6 +627,8 @@ sigterm_loop: sdargs.countproc = 0; sdargs.activecount = 0; + + /* post a SIGTERM to all that catch SIGTERM and not marked for delay */ proc_rebootscan(sd_callback3, (void *)&sdargs, sd_filt2, (void *)&sfargs); printf("\n"); diff --git a/bsd/kern/kern_sig.c b/bsd/kern/kern_sig.c index 5d38f292e..ca7b6c584 100644 --- a/bsd/kern/kern_sig.c +++ b/bsd/kern/kern_sig.c @@ -95,8 +95,6 @@ #include <security/audit/audit.h> -#include <machine/spl.h> - #include <kern/cpu_number.h> #include <sys/vm.h> @@ -116,6 +114,11 @@ #include <sys/sdt.h> #include <sys/codesign.h> +#include <libkern/section_keywords.h> + +#if CONFIG_MACF +#include <security/mac_framework.h> +#endif /* * Missing prototypes that Mach should export @@ -149,13 +152,13 @@ kern_return_t semaphore_timedwait_trap_internal(mach_port_name_t, unsigned int, kern_return_t semaphore_wait_signal_trap_internal(mach_port_name_t, mach_port_name_t, void (*)(kern_return_t)); kern_return_t semaphore_wait_trap_internal(mach_port_name_t, void (*)(kern_return_t)); -static int filt_sigattach(struct knote *kn); +static int filt_sigattach(struct knote *kn, struct kevent_internal_s *kev); static void filt_sigdetach(struct knote *kn); static int filt_signal(struct knote *kn, long hint); static int filt_signaltouch(struct knote *kn, struct kevent_internal_s *kev); static int filt_signalprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); -struct filterops sig_filtops = { +SECURITY_READ_ONLY_EARLY(struct filterops) sig_filtops = { .f_attach = filt_sigattach, .f_detach = filt_sigdetach, .f_event = filt_signal, @@ -481,11 +484,11 @@ sigaction(proc_t p, struct sigaction_args *uap, __unused int32_t *retval) sa->sa_flags |= SA_NOCLDWAIT; if (IS_64BIT_PROCESS(p)) { - struct user64_sigaction vec64; + struct user64_sigaction vec64 = {}; sigaction_kern_to_user64(sa, &vec64); error = copyout(&vec64, uap->osa, sizeof(vec64)); } else { - struct user32_sigaction vec32; + struct user32_sigaction vec32 = {}; sigaction_kern_to_user32(sa, &vec32); error = copyout(&vec32, uap->osa, sizeof(vec32)); } @@ -1405,11 +1408,11 @@ sigaltstack(__unused proc_t p, struct sigaltstack_args *uap, __unused int32_t *r onstack = pstk->ss_flags & SA_ONSTACK; if (uap->oss) { if (IS_64BIT_PROCESS(p)) { - struct user64_sigaltstack ss64; + struct user64_sigaltstack ss64 = {}; sigaltstack_kern_to_user64(pstk, &ss64); error = copyout(&ss64, uap->oss, sizeof(ss64)); } else { - struct user32_sigaltstack ss32; + struct user32_sigaltstack ss32 = {}; sigaltstack_kern_to_user32(pstk, &ss32); error = copyout(&ss32, uap->oss, sizeof(ss32)); } @@ -1668,7 +1671,7 @@ terminate_with_payload_internal(struct proc *cur_proc, int target_pid, uint32_t reason_code, 0, 0); signal_reason = build_userspace_exit_reason(reason_namespace, reason_code, payload, payload_size, - reason_string, reason_flags); + reason_string, (reason_flags | OS_REASON_FLAG_NO_CRASHED_TID)); if (target_pid == cur_proc->p_pid) { /* @@ -2507,6 +2510,7 @@ psignal_internal(proc_t p, task_t task, thread_t thread, int flavor, int signum, assert(signal_reason == NULL); OSBitOrAtomic(P_CONTINUED, &sig_proc->p_flag); sig_proc->p_contproc = sig_proc->p_pid; + sig_proc->p_xstat = signum; (void) task_resume_internal(sig_task); @@ -2760,6 +2764,8 @@ issignal_locked(proc_t p) proc_lock(pp); pp->si_pid = p->p_pid; + pp->p_xhighbits = p->p_xhighbits; + p->p_xhighbits = 0; pp->si_status = p->p_xstat; pp->si_code = CLD_TRAPPED; pp->si_uid = r_uid; @@ -3211,7 +3217,7 @@ postsig_locked(int signum) */ static int -filt_sigattach(struct knote *kn) +filt_sigattach(struct knote *kn, __unused struct kevent_internal_s *kev) { proc_t p = current_proc(); /* can attach only to oneself */ diff --git a/bsd/kern/kern_symfile.c b/bsd/kern/kern_symfile.c index f1586ef1f..46018b2de 100644 --- a/bsd/kern/kern_symfile.c +++ b/bsd/kern/kern_symfile.c @@ -148,7 +148,7 @@ kern_ioctl_file_extents(struct kern_direct_file_io_ref_t * ref, u_long theIoctl, if (filechunk > (size_t)(end - offset)) filechunk = (size_t)(end - offset); error = VNOP_BLOCKMAP(ref->vp, offset, filechunk, &blkno, - &filechunk, NULL, VNODE_WRITE, NULL); + &filechunk, NULL, VNODE_WRITE | VNODE_BLOCKMAP_NO_TRACK, NULL); if (error) break; if (-1LL == blkno) continue; fileblk = blkno * ref->blksize; @@ -382,7 +382,7 @@ kern_open_file_for_direct_io(const char * name, daddr64_t blkno; error = VNOP_BLOCKMAP(ref->vp, f_offset, filechunk, &blkno, - &filechunk, NULL, VNODE_WRITE, NULL); + &filechunk, NULL, VNODE_WRITE | VNODE_BLOCKMAP_NO_TRACK, NULL); if (error) goto out; if (-1LL == blkno) continue; fileblk = blkno * ref->blksize; diff --git a/bsd/kern/kern_synch.c b/bsd/kern/kern_synch.c index 5f3b8546b..841cdeba9 100644 --- a/bsd/kern/kern_synch.c +++ b/bsd/kern/kern_synch.c @@ -40,8 +40,6 @@ #include <sys/vnode.h> #include <sys/kernel.h> -#include <machine/spl.h> - #include <kern/queue.h> #include <sys/lock.h> #include <kern/thread.h> diff --git a/bsd/kern/kern_sysctl.c b/bsd/kern/kern_sysctl.c index 29b714684..af90ce9ff 100644 --- a/bsd/kern/kern_sysctl.c +++ b/bsd/kern/kern_sysctl.c @@ -104,18 +104,24 @@ #include <sys/reboot.h> #include <sys/memory_maintenance.h> #include <sys/priv.h> +#include <stdatomic.h> #include <security/audit/audit.h> #include <kern/kalloc.h> +#include <machine/smp.h> #include <mach/machine.h> #include <mach/mach_host.h> #include <mach/mach_types.h> +#include <mach/processor_info.h> #include <mach/vm_param.h> +#include <kern/debug.h> #include <kern/mach_param.h> #include <kern/task.h> #include <kern/thread.h> +#include <kern/thread_group.h> #include <kern/processor.h> +#include <kern/cpu_number.h> #include <kern/debug.h> #include <kern/sched_prim.h> #include <vm/vm_kern.h> @@ -166,7 +172,6 @@ extern int lowpri_IO_window_msecs; extern int lowpri_IO_delay_msecs; extern int nx_enabled; extern int speculative_reads_disabled; -extern int ignore_is_ssd; extern unsigned int speculative_prefetch_max; extern unsigned int speculative_prefetch_max_iosize; extern unsigned int preheat_max_bytes; @@ -227,8 +232,6 @@ netboot_root(void); int pcsamples_ops(int *name, u_int namelen, user_addr_t where, size_t *sizep, proc_t p); -__private_extern__ kern_return_t -reset_vmobjectcache(unsigned int val1, unsigned int val2); int sysctl_procargs(int *name, u_int namelen, user_addr_t where, size_t *sizep, proc_t cur_proc); @@ -257,7 +260,9 @@ STATIC int sysctl_kdebug_ops SYSCTL_HANDLER_ARGS; #if COUNT_SYSCALLS STATIC int sysctl_docountsyscalls SYSCTL_HANDLER_ARGS; #endif /* COUNT_SYSCALLS */ +#if !CONFIG_EMBEDDED STATIC int sysctl_doprocargs SYSCTL_HANDLER_ARGS; +#endif /* !CONFIG_EMBEDDED */ STATIC int sysctl_doprocargs2 SYSCTL_HANDLER_ARGS; STATIC int sysctl_prochandle SYSCTL_HANDLER_ARGS; STATIC int sysctl_aiomax(struct sysctl_oid *oidp, void *arg1, int arg2, struct sysctl_req *req); @@ -752,6 +757,7 @@ sysctl_prochandle SYSCTL_HANDLER_ARGS int uidcheck = 0; int ruidcheck = 0; int ttycheck = 0; + int success = 0; if (namelen != 1 && !(namelen == 0 && cmd == KERN_PROC_ALL)) return (EINVAL); @@ -808,8 +814,16 @@ sysctl_prochandle SYSCTL_HANDLER_ARGS if (namelen) args.uidval = name[0]; - proc_iterate((PROC_ALLPROCLIST | PROC_ZOMBPROCLIST), - sysdoproc_callback, &args, filterfn, name); + success = proc_iterate((PROC_ALLPROCLIST | PROC_ZOMBPROCLIST), + sysdoproc_callback, &args, filterfn, name); + + /* + * rdar://problem/28433391: if we can't iterate over the processes, + * make sure to return an error. + */ + + if (success != 0) + return (ENOMEM); if (error) return (error); @@ -1180,6 +1194,7 @@ SYSCTL_PROC(_kern, KERN_KDEBUG, kdebug, CTLTYPE_NODE|CTLFLAG_RD | CTLFLAG_LOCKED ""); +#if !CONFIG_EMBEDDED /* * Return the top *sizep bytes of the user stack, or the entire area of the * user stack down through the saved exec_path, whichever is smaller. @@ -1210,6 +1225,7 @@ SYSCTL_PROC(_kern, KERN_PROCARGS, procargs, CTLTYPE_NODE|CTLFLAG_RD | CTLFLAG_LO sysctl_doprocargs, /* Handler function */ NULL, /* Data pointer */ ""); +#endif /* !CONFIG_EMBEDDED */ STATIC int sysctl_doprocargs2 SYSCTL_HANDLER_ARGS @@ -1258,6 +1274,7 @@ sysctl_procargsx(int *name, u_int namelen, user_addr_t where, int pid; kauth_cred_t my_cred; uid_t uid; + int argc = -1; if ( namelen < 1 ) return(EINVAL); @@ -1304,24 +1321,23 @@ sysctl_procargsx(int *name, u_int namelen, user_addr_t where, proc_rele(p); return(EFAULT); } - - size = p->p_argslen; + + size = p->p_argslen; proc_rele(p); - if (argc_yes) { - size += sizeof(int); - } - else { + if (argc_yes) { + size += sizeof(int); + } else { /* * old PROCARGS will return the executable's path and plus some * extra space for work alignment and data tags */ - size += PATH_MAX + (6 * sizeof(int)); - } + size += PATH_MAX + (6 * sizeof(int)); + } size += (size & (sizeof(int) - 1)) ? (sizeof(int) - (size & (sizeof(int) - 1))) : 0; *sizep = size; return (0); } - + my_cred = kauth_cred_proc_ref(p); uid = kauth_cred_getuid(my_cred); kauth_cred_unref(&my_cred); @@ -1337,7 +1353,6 @@ sysctl_procargsx(int *name, u_int namelen, user_addr_t where, arg_addr = p->user_stack - arg_size; - /* * Before we can block (any VM code), make another * reference to the map to keep it alive. We do @@ -1348,7 +1363,10 @@ sysctl_procargsx(int *name, u_int namelen, user_addr_t where, proc_rele(p); return(EINVAL); } - + + /* save off argc before releasing the proc */ + argc = p->p_argc; + argslen = p->p_argslen; /* * Once we have a task reference we can convert that into a @@ -1421,7 +1439,7 @@ sysctl_procargsx(int *name, u_int namelen, user_addr_t where, if (argc_yes) { /* Put processes argc as the first word in the copyout buffer */ - suword(where, p->p_argc); + suword(where, argc); error = copyout(data, (where + sizeof(int)), size); size += sizeof(int); } else { @@ -1668,15 +1686,43 @@ SYSCTL_PROC(_kern, KERN_OSVERSION, osversion, osversion, 256 /* OSVERSIZE*/, sysctl_osversion, "A", ""); +static uint64_t osvariant_status = 0; + +STATIC int +sysctl_osvariant_status(__unused struct sysctl_oid *oidp, void *arg1, int arg2, struct sysctl_req *req) +{ + if (req->newptr != 0) { + /* + * Can only ever be set by launchd, and only once at boot. + */ + if (req->p->p_pid != 1 || osvariant_status != 0) { + return EPERM; + } + } + + return sysctl_handle_quad(oidp, arg1, arg2, req); +} + +SYSCTL_PROC(_kern, OID_AUTO, osvariant_status, + CTLFLAG_RW | CTLTYPE_QUAD | CTLFLAG_LOCKED | CTLFLAG_MASKED, + &osvariant_status, sizeof(osvariant_status), + sysctl_osvariant_status, "Q", "Opaque flags used to cache OS variant information"); + STATIC int sysctl_sysctl_bootargs (__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req) { int error; - char buf[256]; + /* BOOT_LINE_LENGTH */ +#if CONFIG_EMBEDDED + size_t boot_args_len = 256; +#else + size_t boot_args_len = 1024; +#endif + char buf[boot_args_len]; - strlcpy(buf, PE_boot_args(), 256); - error = sysctl_io_string(req, buf, 256, 0, NULL); + strlcpy(buf, PE_boot_args(), boot_args_len); + error = sysctl_io_string(req, buf, boot_args_len, 0, NULL); return(error); } @@ -1732,7 +1778,6 @@ sysctl_maxvnodes (__unused struct sysctl_oid *oidp, __unused void *arg1, __unuse int error = sysctl_io_number(req, desiredvnodes, sizeof(int), &desiredvnodes, NULL); if (oldval != desiredvnodes) { - reset_vmobjectcache(oldval, desiredvnodes); resize_namecache(desiredvnodes); } @@ -1768,7 +1813,71 @@ extern int sched_smt_balance; SYSCTL_INT(_kern, OID_AUTO, sched_smt_balance, CTLFLAG_KERN| CTLFLAG_RW| CTLFLAG_LOCKED, &sched_smt_balance, 0, ""); -#endif +#if __arm__ || __arm64__ +extern uint32_t perfcontrol_requested_recommended_cores; +SYSCTL_UINT(_kern, OID_AUTO, sched_recommended_cores, + CTLFLAG_KERN | CTLFLAG_RD | CTLFLAG_LOCKED, + &perfcontrol_requested_recommended_cores, 0, ""); + +/* Scheduler perfcontrol callouts sysctls */ +SYSCTL_DECL(_kern_perfcontrol_callout); +SYSCTL_NODE(_kern, OID_AUTO, perfcontrol_callout, CTLFLAG_RW | CTLFLAG_LOCKED, 0, + "scheduler perfcontrol callouts"); + +extern int perfcontrol_callout_stats_enabled; +SYSCTL_INT(_kern_perfcontrol_callout, OID_AUTO, stats_enabled, + CTLFLAG_KERN| CTLFLAG_RW| CTLFLAG_LOCKED, + &perfcontrol_callout_stats_enabled, 0, ""); + +extern uint64_t perfcontrol_callout_stat_avg(perfcontrol_callout_type_t type, + perfcontrol_callout_stat_t stat); + +/* On-Core Callout */ +STATIC int +sysctl_perfcontrol_callout_stat +(__unused struct sysctl_oid *oidp, void *arg1, int arg2, struct sysctl_req *req) +{ + perfcontrol_callout_stat_t stat = (perfcontrol_callout_stat_t)arg1; + perfcontrol_callout_type_t type = (perfcontrol_callout_type_t)arg2; + return sysctl_io_number(req, (int)perfcontrol_callout_stat_avg(type, stat), + sizeof(int), NULL, NULL); +} + +SYSCTL_PROC(_kern_perfcontrol_callout, OID_AUTO, oncore_instr, + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED, + (void *)PERFCONTROL_STAT_INSTRS, PERFCONTROL_CALLOUT_ON_CORE, + sysctl_perfcontrol_callout_stat, "I", ""); +SYSCTL_PROC(_kern_perfcontrol_callout, OID_AUTO, oncore_cycles, + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED, + (void *)PERFCONTROL_STAT_CYCLES, PERFCONTROL_CALLOUT_ON_CORE, + sysctl_perfcontrol_callout_stat, "I", ""); +SYSCTL_PROC(_kern_perfcontrol_callout, OID_AUTO, offcore_instr, + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED, + (void *)PERFCONTROL_STAT_INSTRS, PERFCONTROL_CALLOUT_OFF_CORE, + sysctl_perfcontrol_callout_stat, "I", ""); +SYSCTL_PROC(_kern_perfcontrol_callout, OID_AUTO, offcore_cycles, + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED, + (void *)PERFCONTROL_STAT_CYCLES, PERFCONTROL_CALLOUT_OFF_CORE, + sysctl_perfcontrol_callout_stat, "I", ""); +SYSCTL_PROC(_kern_perfcontrol_callout, OID_AUTO, context_instr, + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED, + (void *)PERFCONTROL_STAT_INSTRS, PERFCONTROL_CALLOUT_CONTEXT, + sysctl_perfcontrol_callout_stat, "I", ""); +SYSCTL_PROC(_kern_perfcontrol_callout, OID_AUTO, context_cycles, + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED, + (void *)PERFCONTROL_STAT_CYCLES, PERFCONTROL_CALLOUT_CONTEXT, + sysctl_perfcontrol_callout_stat, "I", ""); +SYSCTL_PROC(_kern_perfcontrol_callout, OID_AUTO, update_instr, + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED, + (void *)PERFCONTROL_STAT_INSTRS, PERFCONTROL_CALLOUT_STATE_UPDATE, + sysctl_perfcontrol_callout_stat, "I", ""); +SYSCTL_PROC(_kern_perfcontrol_callout, OID_AUTO, update_cycles, + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED, + (void *)PERFCONTROL_STAT_CYCLES, PERFCONTROL_CALLOUT_STATE_UPDATE, + sysctl_perfcontrol_callout_stat, "I", ""); + +#endif /* __arm__ || __arm64__ */ +#endif /* (DEVELOPMENT || DEBUG) */ STATIC int sysctl_securelvl @@ -1847,10 +1956,6 @@ SYSCTL_INT(_kern, KERN_SPECULATIVE_READS, speculative_reads_disabled, CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_LOCKED, &speculative_reads_disabled, 0, ""); -SYSCTL_INT(_kern, OID_AUTO, ignore_is_ssd, - CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_LOCKED, - &ignore_is_ssd, 0, ""); - SYSCTL_UINT(_kern, OID_AUTO, preheat_max_bytes, CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_LOCKED, &preheat_max_bytes, 0, ""); @@ -1908,12 +2013,12 @@ sysctl_boottime struct proc *p = req->p; if (proc_is64bit(p)) { - struct user64_timeval t; + struct user64_timeval t = {}; t.tv_sec = tv.tv_sec; t.tv_usec = tv.tv_usec; return sysctl_io_opaque(req, &t, sizeof(t), NULL); } else { - struct user32_timeval t; + struct user32_timeval t = {}; t.tv_sec = tv.tv_sec; t.tv_usec = tv.tv_usec; return sysctl_io_opaque(req, &t, sizeof(t), NULL); @@ -2001,7 +2106,7 @@ sysctl_imgsrcinfo (__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req) { int error; - struct imgsrc_info info[MAX_IMAGEBOOT_NESTING]; /* 2 for now, no problem */ + struct imgsrc_info info[MAX_IMAGEBOOT_NESTING] = {}; /* 2 for now, no problem */ uint32_t i; vnode_t rvp, devvp; @@ -2083,7 +2188,7 @@ SYSCTL_NODE(_kern_timer, OID_AUTO, longterm, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "lo enum { THRESHOLD, QCOUNT, ENQUEUES, DEQUEUES, ESCALATES, SCANS, PREEMPTS, - LATENCY, LATENCY_MIN, LATENCY_MAX + LATENCY, LATENCY_MIN, LATENCY_MAX, SCAN_LIMIT, PAUSES }; extern uint64_t timer_sysctl_get(int); extern int timer_sysctl_set(int, uint64_t); @@ -2108,10 +2213,13 @@ sysctl_timer SYSCTL_PROC(_kern_timer_longterm, OID_AUTO, threshold, CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED, (void *) THRESHOLD, 0, sysctl_timer, "Q", ""); +SYSCTL_PROC(_kern_timer_longterm, OID_AUTO, scan_limit, + CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED, + (void *) SCAN_LIMIT, 0, sysctl_timer, "Q", ""); SYSCTL_PROC(_kern_timer_longterm, OID_AUTO, qlen, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_LOCKED, (void *) QCOUNT, 0, sysctl_timer, "Q", ""); -#if DEBUG +#if DEBUG SYSCTL_PROC(_kern_timer_longterm, OID_AUTO, enqueues, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_LOCKED, (void *) ENQUEUES, 0, sysctl_timer, "Q", ""); @@ -2136,6 +2244,9 @@ SYSCTL_PROC(_kern_timer_longterm, OID_AUTO, latency_min, SYSCTL_PROC(_kern_timer_longterm, OID_AUTO, latency_max, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_LOCKED, (void *) LATENCY_MAX, 0, sysctl_timer, "Q", ""); +SYSCTL_PROC(_kern_timer_longterm, OID_AUTO, scan_pauses, + CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_LOCKED, + (void *) PAUSES, 0, sysctl_timer, "Q", ""); #endif /* DEBUG */ STATIC int @@ -2404,11 +2515,11 @@ sysctl_loadavg (__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req) { if (proc_is64bit(req->p)) { - struct user64_loadavg loadinfo64; + struct user64_loadavg loadinfo64 = {}; fill_loadavg64(&averunnable, &loadinfo64); return sysctl_io_opaque(req, &loadinfo64, sizeof(loadinfo64), NULL); } else { - struct user32_loadavg loadinfo32; + struct user32_loadavg loadinfo32 = {}; fill_loadavg32(&averunnable, &loadinfo32); return sysctl_io_opaque(req, &loadinfo32, sizeof(loadinfo32), NULL); } @@ -2448,7 +2559,7 @@ sysctl_swapusage uint64_t swap_avail; vm_size_t swap_pagesize; boolean_t swap_encrypted; - struct xsw_usage xsu; + struct xsw_usage xsu = {}; error = macx_swapinfo(&swap_total, &swap_avail, @@ -2657,9 +2768,15 @@ vm_map_size_t vm_user_wire_limit; /* * There needs to be a more automatic/elegant way to do this */ +#if defined(__ARM__) +SYSCTL_INT(_vm, OID_AUTO, global_no_user_wire_amount, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_global_no_user_wire_amount, 0, ""); +SYSCTL_INT(_vm, OID_AUTO, global_user_wire_limit, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_global_user_wire_limit, 0, ""); +SYSCTL_INT(_vm, OID_AUTO, user_wire_limit, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_user_wire_limit, 0, ""); +#else SYSCTL_QUAD(_vm, OID_AUTO, global_no_user_wire_amount, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_global_no_user_wire_amount, ""); SYSCTL_QUAD(_vm, OID_AUTO, global_user_wire_limit, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_global_user_wire_limit, ""); SYSCTL_QUAD(_vm, OID_AUTO, user_wire_limit, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_user_wire_limit, ""); +#endif extern int vm_map_copy_overwrite_aligned_src_not_internal; extern int vm_map_copy_overwrite_aligned_src_not_symmetric; @@ -2693,7 +2810,9 @@ extern uint32_t vm_compressor_majorcompact_threshold_divisor; extern uint32_t vm_compressor_unthrottle_threshold_divisor; extern uint32_t vm_compressor_catchup_threshold_divisor; extern uint32_t vm_compressor_time_thread; -extern uint64_t vm_compressor_thread_runtime; +#if DEVELOPMENT || DEBUG +extern vmct_stats_t vmct_stats; +#endif SYSCTL_QUAD(_vm, OID_AUTO, compressor_input_bytes, CTLFLAG_RD | CTLFLAG_LOCKED, &c_segment_input_bytes, ""); SYSCTL_QUAD(_vm, OID_AUTO, compressor_compressed_bytes, CTLFLAG_RD | CTLFLAG_LOCKED, &c_segment_compressed_bytes, ""); @@ -2719,7 +2838,26 @@ SYSCTL_INT(_vm, OID_AUTO, compressor_catchup_threshold_divisor, CTLFLAG_RW | CTL SYSCTL_STRING(_vm, OID_AUTO, swapfileprefix, CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_LOCKED, swapfilename, sizeof(swapfilename) - SWAPFILENAME_INDEX_LEN, ""); SYSCTL_INT(_vm, OID_AUTO, compressor_timing_enabled, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_compressor_time_thread, 0, ""); -SYSCTL_QUAD(_vm, OID_AUTO, compressor_thread_runtime, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_compressor_thread_runtime, ""); + +#if DEVELOPMENT || DEBUG +SYSCTL_QUAD(_vm, OID_AUTO, compressor_thread_runtime0, CTLFLAG_RD | CTLFLAG_LOCKED, &vmct_stats.vmct_runtimes[0], ""); +SYSCTL_QUAD(_vm, OID_AUTO, compressor_thread_runtime1, CTLFLAG_RD | CTLFLAG_LOCKED, &vmct_stats.vmct_runtimes[1], ""); + +SYSCTL_QUAD(_vm, OID_AUTO, compressor_threads_total, CTLFLAG_RD | CTLFLAG_LOCKED, &vmct_stats.vmct_cthreads_total, ""); + +SYSCTL_QUAD(_vm, OID_AUTO, compressor_thread_pages0, CTLFLAG_RD | CTLFLAG_LOCKED, &vmct_stats.vmct_pages[0], ""); +SYSCTL_QUAD(_vm, OID_AUTO, compressor_thread_pages1, CTLFLAG_RD | CTLFLAG_LOCKED, &vmct_stats.vmct_pages[1], ""); + +SYSCTL_QUAD(_vm, OID_AUTO, compressor_thread_iterations0, CTLFLAG_RD | CTLFLAG_LOCKED, &vmct_stats.vmct_iterations[0], ""); +SYSCTL_QUAD(_vm, OID_AUTO, compressor_thread_iterations1, CTLFLAG_RD | CTLFLAG_LOCKED, &vmct_stats.vmct_iterations[1], ""); + +SYSCTL_INT(_vm, OID_AUTO, compressor_thread_minpages0, CTLFLAG_RD | CTLFLAG_LOCKED, &vmct_stats.vmct_minpages[0], 0, ""); +SYSCTL_INT(_vm, OID_AUTO, compressor_thread_minpages1, CTLFLAG_RD | CTLFLAG_LOCKED, &vmct_stats.vmct_minpages[1], 0, ""); + +SYSCTL_INT(_vm, OID_AUTO, compressor_thread_maxpages0, CTLFLAG_RD | CTLFLAG_LOCKED, &vmct_stats.vmct_maxpages[0], 0, ""); +SYSCTL_INT(_vm, OID_AUTO, compressor_thread_maxpages1, CTLFLAG_RD | CTLFLAG_LOCKED, &vmct_stats.vmct_maxpages[1], 0, ""); + +#endif SYSCTL_QUAD(_vm, OID_AUTO, lz4_compressions, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.lz4_compressions, ""); SYSCTL_QUAD(_vm, OID_AUTO, lz4_compression_failures, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.lz4_compression_failures, ""); @@ -2733,6 +2871,15 @@ SYSCTL_QUAD(_vm, OID_AUTO, lz4_decompressed_bytes, CTLFLAG_RD | CTLFLAG_LOCKED, SYSCTL_QUAD(_vm, OID_AUTO, uc_decompressions, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.uc_decompressions, ""); SYSCTL_QUAD(_vm, OID_AUTO, wk_compressions, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wk_compressions, ""); + +SYSCTL_QUAD(_vm, OID_AUTO, wk_catime, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wk_cabstime, ""); + +SYSCTL_QUAD(_vm, OID_AUTO, wkh_catime, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wkh_cabstime, ""); +SYSCTL_QUAD(_vm, OID_AUTO, wkh_compressions, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wkh_compressions, ""); + +SYSCTL_QUAD(_vm, OID_AUTO, wks_catime, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wks_cabstime, ""); +SYSCTL_QUAD(_vm, OID_AUTO, wks_compressions, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wks_compressions, ""); + SYSCTL_QUAD(_vm, OID_AUTO, wk_compressions_exclusive, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wk_compressions_exclusive, ""); SYSCTL_QUAD(_vm, OID_AUTO, wk_sv_compressions, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wk_sv_compressions, ""); SYSCTL_QUAD(_vm, OID_AUTO, wk_mzv_compressions, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wk_mzv_compressions, ""); @@ -2740,7 +2887,21 @@ SYSCTL_QUAD(_vm, OID_AUTO, wk_compression_failures, CTLFLAG_RD | CTLFLAG_LOCKED, SYSCTL_QUAD(_vm, OID_AUTO, wk_compressed_bytes_exclusive, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wk_compressed_bytes_exclusive, ""); SYSCTL_QUAD(_vm, OID_AUTO, wk_compressed_bytes_total, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wk_compressed_bytes_total, ""); +SYSCTL_QUAD(_vm, OID_AUTO, wks_compressed_bytes, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wks_compressed_bytes, ""); +SYSCTL_QUAD(_vm, OID_AUTO, wks_compression_failures, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wks_compression_failures, ""); +SYSCTL_QUAD(_vm, OID_AUTO, wks_sv_compressions, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wks_sv_compressions, ""); + + SYSCTL_QUAD(_vm, OID_AUTO, wk_decompressions, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wk_decompressions, ""); + +SYSCTL_QUAD(_vm, OID_AUTO, wk_datime, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wk_dabstime, ""); + +SYSCTL_QUAD(_vm, OID_AUTO, wkh_datime, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wkh_dabstime, ""); +SYSCTL_QUAD(_vm, OID_AUTO, wkh_decompressions, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wkh_decompressions, ""); + +SYSCTL_QUAD(_vm, OID_AUTO, wks_datime, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wks_dabstime, ""); +SYSCTL_QUAD(_vm, OID_AUTO, wks_decompressions, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wks_decompressions, ""); + SYSCTL_QUAD(_vm, OID_AUTO, wk_decompressed_bytes, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wk_decompressed_bytes, ""); SYSCTL_QUAD(_vm, OID_AUTO, wk_sv_decompressions, CTLFLAG_RD | CTLFLAG_LOCKED, &compressor_stats.wk_sv_decompressions, ""); @@ -2752,6 +2913,21 @@ SYSCTL_INT(_vm, OID_AUTO, lz4_max_preselects, CTLFLAG_RW | CTLFLAG_LOCKED, &vmct SYSCTL_INT(_vm, OID_AUTO, lz4_run_preselection_threshold, CTLFLAG_RW | CTLFLAG_LOCKED, &vmctune.lz4_run_preselection_threshold, 0, ""); SYSCTL_INT(_vm, OID_AUTO, lz4_run_continue_bytes, CTLFLAG_RW | CTLFLAG_LOCKED, &vmctune.lz4_run_continue_bytes, 0, ""); SYSCTL_INT(_vm, OID_AUTO, lz4_profitable_bytes, CTLFLAG_RW | CTLFLAG_LOCKED, &vmctune.lz4_profitable_bytes, 0, ""); +#if DEVELOPMENT || DEBUG +extern int vm_compressor_current_codec; +extern int vm_compressor_test_seg_wp; +extern boolean_t vm_compressor_force_sw_wkdm; +SYSCTL_INT(_vm, OID_AUTO, compressor_codec, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_compressor_current_codec, 0, ""); +SYSCTL_INT(_vm, OID_AUTO, compressor_test_wp, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_compressor_test_seg_wp, 0, ""); + +SYSCTL_INT(_vm, OID_AUTO, wksw_force, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_compressor_force_sw_wkdm, 0, ""); +extern int precompy, wkswhw; + +SYSCTL_INT(_vm, OID_AUTO, precompy, CTLFLAG_RW | CTLFLAG_LOCKED, &precompy, 0, ""); +SYSCTL_INT(_vm, OID_AUTO, wkswhw, CTLFLAG_RW | CTLFLAG_LOCKED, &wkswhw, 0, ""); +extern unsigned int vm_ktrace_enabled; +SYSCTL_INT(_vm, OID_AUTO, vm_ktrace, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_ktrace_enabled, 0, ""); +#endif #if CONFIG_PHANTOM_CACHE extern uint32_t phantom_cache_thrashing_threshold; @@ -2767,7 +2943,6 @@ SYSCTL_INT(_vm, OID_AUTO, phantom_cache_thrashing_threshold_ssd, CTLFLAG_RW | CT #if CONFIG_BACKGROUND_QUEUE extern uint32_t vm_page_background_count; -extern uint32_t vm_page_background_limit; extern uint32_t vm_page_background_target; extern uint32_t vm_page_background_internal_count; extern uint32_t vm_page_background_external_count; @@ -2781,7 +2956,6 @@ extern uint64_t vm_pageout_rejected_bq_external; SYSCTL_INT(_vm, OID_AUTO, vm_page_background_mode, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_page_background_mode, 0, ""); SYSCTL_INT(_vm, OID_AUTO, vm_page_background_exclude_external, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_page_background_exclude_external, 0, ""); -SYSCTL_INT(_vm, OID_AUTO, vm_page_background_limit, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_page_background_limit, 0, ""); SYSCTL_INT(_vm, OID_AUTO, vm_page_background_target, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_page_background_target, 0, ""); SYSCTL_INT(_vm, OID_AUTO, vm_page_background_count, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_background_count, 0, ""); SYSCTL_INT(_vm, OID_AUTO, vm_page_background_internal_count, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_background_internal_count, 0, ""); @@ -2865,6 +3039,10 @@ SYSCTL_INT (_kern, OID_AUTO, stack_size, SYSCTL_INT (_kern, OID_AUTO, stack_depth_max, CTLFLAG_RD | CTLFLAG_LOCKED, (int *) &kernel_stack_depth_max, 0, "Max kernel stack depth at interrupt or context switch"); +extern unsigned int kern_feature_overrides; +SYSCTL_INT (_kern, OID_AUTO, kern_feature_overrides, + CTLFLAG_RD | CTLFLAG_LOCKED, &kern_feature_overrides, 0, "Kernel feature override mask"); + /* * enable back trace for port allocations */ @@ -2887,6 +3065,14 @@ SYSCTL_STRING(_kern, OID_AUTO, sched, * Only support runtime modification on embedded platforms * with development config enabled */ +#if CONFIG_EMBEDDED +#if !SECURE_KERNEL +extern int precise_user_kernel_time; +SYSCTL_INT(_kern, OID_AUTO, precise_user_kernel_time, + CTLFLAG_RW | CTLFLAG_LOCKED, + &precise_user_kernel_time, 0, "Precise accounting of kernel vs. user time"); +#endif +#endif /* Parameters related to timer coalescing tuning, to be replaced @@ -3069,6 +3255,77 @@ SYSCTL_INT(_kern, OID_AUTO, hv_support, &hv_support_available, 0, ""); #endif +#if CONFIG_EMBEDDED +STATIC int +sysctl_darkboot SYSCTL_HANDLER_ARGS +{ + int err = 0, value = 0; +#pragma unused(oidp, arg1, arg2, err, value, req) + + /* + * Handle the sysctl request. + * + * If this is a read, the function will set the value to the current darkboot value. Otherwise, + * we'll get the request identifier into "value" and then we can honor it. + */ + if ((err = sysctl_io_number(req, darkboot, sizeof(int), &value, NULL)) != 0) { + goto exit; + } + + /* writing requested, let's process the request */ + if (req->newptr) { + /* writing is protected by an entitlement */ + if (priv_check_cred(kauth_cred_get(), PRIV_DARKBOOT, 0) != 0) { + err = EPERM; + goto exit; + } + + switch (value) { + case MEMORY_MAINTENANCE_DARK_BOOT_UNSET: + /* + * If the darkboot sysctl is unset, the NVRAM variable + * must be unset too. If that's not the case, it means + * someone is doing something crazy and not supported. + */ + if (darkboot != 0) { + int ret = PERemoveNVRAMProperty(MEMORY_MAINTENANCE_DARK_BOOT_NVRAM_NAME); + if (ret) { + darkboot = 0; + } else { + err = EINVAL; + } + } + break; + case MEMORY_MAINTENANCE_DARK_BOOT_SET: + darkboot = 1; + break; + case MEMORY_MAINTENANCE_DARK_BOOT_SET_PERSISTENT: { + /* + * Set the NVRAM and update 'darkboot' in case + * of success. Otherwise, do not update + * 'darkboot' and report the failure. + */ + if (PEWriteNVRAMBooleanProperty(MEMORY_MAINTENANCE_DARK_BOOT_NVRAM_NAME, TRUE)) { + darkboot = 1; + } else { + err = EINVAL; + } + + break; + } + default: + err = EINVAL; + } + } + +exit: + return err; +} + +SYSCTL_PROC(_kern, OID_AUTO, darkboot, + CTLFLAG_KERN | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED | CTLFLAG_ANYBODY, + 0, 0, sysctl_darkboot, "I", ""); +#endif /* * This is set by core audio to tell tailspin (ie background tracing) how long @@ -3132,3 +3389,184 @@ SYSCTL_COMPAT_INT(_kern, OID_AUTO, development, CTLFLAG_RD | CTLFLAG_MASKED, NUL #else SYSCTL_COMPAT_INT(_kern, OID_AUTO, development, CTLFLAG_RD | CTLFLAG_MASKED, NULL, 0, ""); #endif + + +#if DEVELOPMENT || DEBUG + +static int +sysctl_panic_test SYSCTL_HANDLER_ARGS +{ +#pragma unused(arg1, arg2) + int rval = 0; + char str[32] = "entry prelog postlog postcore"; + + rval = sysctl_handle_string(oidp, str, sizeof(str), req); + + if (rval == 0 && req->newptr) { + if (strncmp("entry", str, strlen("entry")) == 0) { + panic_with_options(0, NULL, DEBUGGER_OPTION_RECURPANIC_ENTRY, "test recursive panic at entry"); + } else if (strncmp("prelog", str, strlen("prelog")) == 0) { + panic_with_options(0, NULL, DEBUGGER_OPTION_RECURPANIC_PRELOG, "test recursive panic prior to writing a paniclog"); + } else if (strncmp("postlog", str, strlen("postlog")) == 0) { + panic_with_options(0, NULL, DEBUGGER_OPTION_RECURPANIC_POSTLOG, "test recursive panic subsequent to paniclog"); + } else if (strncmp("postcore", str, strlen("postcore")) == 0) { + panic_with_options(0, NULL, DEBUGGER_OPTION_RECURPANIC_POSTCORE, "test recursive panic subsequent to on-device core"); + } + } + + return rval; +} + +static int +sysctl_debugger_test SYSCTL_HANDLER_ARGS +{ +#pragma unused(arg1, arg2) + int rval = 0; + char str[32] = "entry prelog postlog postcore"; + + rval = sysctl_handle_string(oidp, str, sizeof(str), req); + + if (rval == 0 && req->newptr) { + if (strncmp("entry", str, strlen("entry")) == 0) { + DebuggerWithContext(0, NULL, "test recursive panic via debugger at entry", DEBUGGER_OPTION_RECURPANIC_ENTRY); + } else if (strncmp("prelog", str, strlen("prelog")) == 0) { + DebuggerWithContext(0, NULL, "test recursive panic via debugger prior to writing a paniclog", DEBUGGER_OPTION_RECURPANIC_PRELOG); + } else if (strncmp("postlog", str, strlen("postlog")) == 0) { + DebuggerWithContext(0, NULL, "test recursive panic via debugger subsequent to paniclog", DEBUGGER_OPTION_RECURPANIC_POSTLOG); + } else if (strncmp("postcore", str, strlen("postcore")) == 0) { + DebuggerWithContext(0, NULL, "test recursive panic via debugger subsequent to on-device core", DEBUGGER_OPTION_RECURPANIC_POSTCORE); + } + } + + return rval; +} + +decl_lck_spin_data(, spinlock_panic_test_lock) + +__attribute__((noreturn)) +static void +spinlock_panic_test_acquire_spinlock(void * arg __unused, wait_result_t wres __unused) +{ + lck_spin_lock(&spinlock_panic_test_lock); + while (1) { ; } +} + +static int +sysctl_spinlock_panic_test SYSCTL_HANDLER_ARGS +{ +#pragma unused(oidp, arg1, arg2) + if (req->newlen == 0) + return EINVAL; + + thread_t panic_spinlock_thread; + /* Initialize panic spinlock */ + lck_grp_t * panic_spinlock_grp; + lck_grp_attr_t * panic_spinlock_grp_attr; + lck_attr_t * panic_spinlock_attr; + + panic_spinlock_grp_attr = lck_grp_attr_alloc_init(); + panic_spinlock_grp = lck_grp_alloc_init("panic_spinlock", panic_spinlock_grp_attr); + panic_spinlock_attr = lck_attr_alloc_init(); + + lck_spin_init(&spinlock_panic_test_lock, panic_spinlock_grp, panic_spinlock_attr); + + + /* Create thread to acquire spinlock */ + if (kernel_thread_start(spinlock_panic_test_acquire_spinlock, NULL, &panic_spinlock_thread) != KERN_SUCCESS) { + return EBUSY; + } + + /* Try to acquire spinlock -- should panic eventually */ + lck_spin_lock(&spinlock_panic_test_lock); + while(1) { ; } +} + +__attribute__((noreturn)) +static void +simultaneous_panic_worker +(void * arg, wait_result_t wres __unused) +{ + atomic_int *start_panic = (atomic_int *)arg; + + while (!atomic_load(start_panic)) { ; } + panic("SIMULTANEOUS PANIC TEST: INITIATING PANIC FROM CPU %d", cpu_number()); + __builtin_unreachable(); +} + +static int +sysctl_simultaneous_panic_test SYSCTL_HANDLER_ARGS +{ +#pragma unused(oidp, arg1, arg2) + if (req->newlen == 0) + return EINVAL; + + int i = 0, threads_to_create = 2 * processor_count; + atomic_int start_panic = 0; + unsigned int threads_created = 0; + thread_t new_panic_thread; + + for (i = threads_to_create; i > 0; i--) { + if (kernel_thread_start(simultaneous_panic_worker, (void *) &start_panic, &new_panic_thread) == KERN_SUCCESS) { + threads_created++; + } + } + + /* FAIL if we couldn't create at least processor_count threads */ + if (threads_created < processor_count) { + panic("SIMULTANEOUS PANIC TEST: FAILED TO CREATE ENOUGH THREADS, ONLY CREATED %d (of %d)", + threads_created, threads_to_create); + } + + atomic_exchange(&start_panic, 1); + while (1) { ; } +} + +SYSCTL_PROC(_debug, OID_AUTO, panic_test, CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_MASKED, 0, 0, sysctl_panic_test, "A", "panic test"); +SYSCTL_PROC(_debug, OID_AUTO, debugger_test, CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_MASKED, 0, 0, sysctl_debugger_test, "A", "debugger test"); +SYSCTL_PROC(_debug, OID_AUTO, spinlock_panic_test, CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_MASKED, 0, 0, sysctl_spinlock_panic_test, "A", "spinlock panic test"); +SYSCTL_PROC(_debug, OID_AUTO, simultaneous_panic_test, CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_MASKED, 0, 0, sysctl_simultaneous_panic_test, "A", "simultaneous panic test"); + + +#endif /* DEVELOPMENT || DEBUG */ + +const uint32_t thread_groups_supported = 0; + +STATIC int +sysctl_thread_groups_supported (__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req) +{ + int value = thread_groups_supported; + return sysctl_io_number(req, value, sizeof(value), NULL, NULL); +} + +SYSCTL_PROC(_kern, OID_AUTO, thread_groups_supported, CTLFLAG_RD | CTLFLAG_LOCKED | CTLFLAG_KERN, + 0, 0, &sysctl_thread_groups_supported, "I", "thread groups supported"); + +static int +sysctl_grade_cputype SYSCTL_HANDLER_ARGS +{ +#pragma unused(arg1, arg2, oidp) + int error = 0; + int type_tuple[2] = {}; + int return_value = 0; + + error = SYSCTL_IN(req, &type_tuple, sizeof(type_tuple)); + + if (error) { + return error; + } + + return_value = grade_binary(type_tuple[0], type_tuple[1]); + + error = SYSCTL_OUT(req, &return_value, sizeof(return_value)); + + if (error) { + return error; + } + + return error; +} + +SYSCTL_PROC(_kern, OID_AUTO, grade_cputype, + CTLFLAG_RW|CTLFLAG_ANYBODY|CTLFLAG_MASKED|CTLFLAG_LOCKED|CTLTYPE_OPAQUE, + 0, 0, &sysctl_grade_cputype, "S", + "grade value of cpu_type_t+cpu_sub_type_t"); diff --git a/bsd/kern/kern_time.c b/bsd/kern/kern_time.c index 7f94f9b50..92da86a67 100644 --- a/bsd/kern/kern_time.c +++ b/bsd/kern/kern_time.c @@ -88,6 +88,8 @@ #if CONFIG_MACF #include <security/mac_framework.h> #endif +#include <IOKit/IOBSD.h> +#include <sys/time.h> #define HZ 100 /* XXX */ @@ -174,13 +176,20 @@ settimeofday(__unused struct proc *p, struct settimeofday_args *uap, __unused i bzero(&atv, sizeof(atv)); + /* Check that this task is entitled to set the time or it is root */ + if (!IOTaskHasEntitlement(current_task(), SETTIME_ENTITLEMENT)) { + #if CONFIG_MACF - error = mac_system_check_settime(kauth_cred_get()); - if (error) - return (error); + error = mac_system_check_settime(kauth_cred_get()); + if (error) + return (error); #endif - if ((error = suser(kauth_cred_get(), &p->p_acflag))) - return (error); +#ifndef CONFIG_EMBEDDED + if ((error = suser(kauth_cred_get(), &p->p_acflag))) + return (error); +#endif + } + /* Verify all parameters before changing time */ if (uap->tv) { if (IS_64BIT_PROCESS(p)) { @@ -220,59 +229,6 @@ setthetime( clock_set_calendar_microtime(tv->tv_sec, tv->tv_usec); } -/* - * XXX Y2038 bug because of clock_adjtime() first argument - */ -/* ARGSUSED */ -int -adjtime(struct proc *p, struct adjtime_args *uap, __unused int32_t *retval) -{ - struct timeval atv; - int error; - -#if CONFIG_MACF - error = mac_system_check_settime(kauth_cred_get()); - if (error) - return (error); -#endif - if ((error = priv_check_cred(kauth_cred_get(), PRIV_ADJTIME, 0))) - return (error); - if (IS_64BIT_PROCESS(p)) { - struct user64_timeval user_atv; - error = copyin(uap->delta, &user_atv, sizeof(user_atv)); - atv.tv_sec = user_atv.tv_sec; - atv.tv_usec = user_atv.tv_usec; - } else { - struct user32_timeval user_atv; - error = copyin(uap->delta, &user_atv, sizeof(user_atv)); - atv.tv_sec = user_atv.tv_sec; - atv.tv_usec = user_atv.tv_usec; - } - if (error) - return (error); - - /* - * Compute the total correction and the rate at which to apply it. - */ - clock_adjtime(&atv.tv_sec, &atv.tv_usec); - - if (uap->olddelta) { - if (IS_64BIT_PROCESS(p)) { - struct user64_timeval user_atv; - user_atv.tv_sec = atv.tv_sec; - user_atv.tv_usec = atv.tv_usec; - error = copyout(&user_atv, uap->olddelta, sizeof(user_atv)); - } else { - struct user32_timeval user_atv; - user_atv.tv_sec = atv.tv_sec; - user_atv.tv_usec = atv.tv_usec; - error = copyout(&user_atv, uap->olddelta, sizeof(user_atv)); - } - } - - return (0); -} - /* * Verify the calendar value. If negative, * reset to zero (the epoch). diff --git a/bsd/kern/kern_xxx.c b/bsd/kern/kern_xxx.c index a4e96162e..c4674fc33 100644 --- a/bsd/kern/kern_xxx.c +++ b/bsd/kern/kern_xxx.c @@ -102,27 +102,38 @@ reboot(struct proc *p, struct reboot_args *uap, __unused int32_t *retval) message[0] = '\0'; - if ((error = suser(kauth_cred_get(), &p->p_acflag))) - return(error); - + if ((error = suser(kauth_cred_get(), &p->p_acflag))) { +#if (DEVELOPMENT || DEBUG) + /* allow non-root user to call panic on dev/debug kernels */ + if (!(uap->opt & RB_PANIC)) + return error; +#else + return error; +#endif + } + if (uap->opt & RB_COMMAND) return ENOSYS; if (uap->opt & RB_PANIC) { -#if !(DEVELOPMENT || DEBUG) - if (p != initproc) { - return EPERM; - } -#endif error = copyinstr(uap->command, (void *)message, sizeof(message), (size_t *)&dummy); } #if CONFIG_MACF +#if (DEVELOPMENT || DEBUG) + if (uap->opt & RB_PANIC) { + /* on dev/debug kernels: allow anyone to call panic */ + goto skip_cred_check; + } +#endif if (error) return (error); my_cred = kauth_cred_proc_ref(p); error = mac_system_check_reboot(my_cred, uap->opt); kauth_cred_unref(&my_cred); +#if (DEVELOPMENT || DEBUG) +skip_cred_check: +#endif #endif if (!error) { OSBitOrAtomic(P_REBOOT, &p->p_flag); /* No more signals for this proc */ diff --git a/bsd/kern/kpi_mbuf.c b/bsd/kern/kpi_mbuf.c index 587394e49..6cb79b10e 100644 --- a/bsd/kern/kpi_mbuf.c +++ b/bsd/kern/kpi_mbuf.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2015 Apple Inc. All rights reserved. + * Copyright (c) 2004-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -1572,15 +1572,16 @@ mbuf_last_pkt(const mbuf_t m, u_int32_t *retval) errno_t mbuf_get_timestamp(mbuf_t m, u_int64_t *ts, boolean_t *valid) { - if (m == NULL || !(m->m_flags & M_PKTHDR) || ts == NULL || - valid == NULL) + if (m == NULL || !(m->m_flags & M_PKTHDR) || ts == NULL) return (EINVAL); - if ((m->m_pkthdr.pkt_flags & PKTF_DRV_TS_VALID) == 0) { - *valid = FALSE; + if ((m->m_pkthdr.pkt_flags & PKTF_TS_VALID) == 0) { + if (valid != NULL) + *valid = FALSE; *ts = 0; } else { - *valid = TRUE; + if (valid != NULL) + *valid = TRUE; *ts = m->m_pkthdr.pkt_timestamp; } return (0); @@ -1593,10 +1594,10 @@ mbuf_set_timestamp(mbuf_t m, u_int64_t ts, boolean_t valid) return (EINVAL); if (valid == FALSE) { - m->m_pkthdr.pkt_flags &= ~PKTF_DRV_TS_VALID; + m->m_pkthdr.pkt_flags &= ~PKTF_TS_VALID; m->m_pkthdr.pkt_timestamp = 0; } else { - m->m_pkthdr.pkt_flags |= PKTF_DRV_TS_VALID; + m->m_pkthdr.pkt_flags |= PKTF_TS_VALID; m->m_pkthdr.pkt_timestamp = ts; } return (0); @@ -1846,7 +1847,7 @@ m_do_tx_compl_callback(struct mbuf *m, struct ifnet *ifp) #if (DEBUG || DEVELOPMENT) if (mbuf_tx_compl_debug != 0 && ifp != NULL && (ifp->if_xflags & IFXF_TIMESTAMP_ENABLED) != 0 && - (m->m_pkthdr.pkt_flags & PKTF_DRV_TS_VALID) == 0) { + (m->m_pkthdr.pkt_flags & PKTF_TS_VALID) == 0) { struct timespec now; nanouptime(&now); @@ -1866,7 +1867,9 @@ m_do_tx_compl_callback(struct mbuf *m, struct ifnet *ifp) if (callback != NULL) { callback(m->m_pkthdr.pkt_compl_context, - ifp, m->m_pkthdr.pkt_timestamp, + ifp, + (m->m_pkthdr.pkt_flags & PKTF_TS_VALID) ? + m->m_pkthdr.pkt_timestamp: 0, m->m_pkthdr.drv_tx_compl_arg, m->m_pkthdr.drv_tx_compl_data, m->m_pkthdr.drv_tx_status); diff --git a/bsd/kern/kpi_socket.c b/bsd/kern/kpi_socket.c index 2f1b1d96a..a7b17264d 100644 --- a/bsd/kern/kpi_socket.c +++ b/bsd/kern/kpi_socket.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003-2016 Apple Inc. All rights reserved. + * Copyright (c) 2003-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -44,17 +44,30 @@ #include <sys/filio.h> #include <sys/uio_internal.h> #include <kern/locks.h> +#include <net/net_api_stats.h> #include <netinet/in.h> #include <libkern/OSAtomic.h> +#include <stdbool.h> static errno_t sock_send_internal(socket_t, const struct msghdr *, mbuf_t, int, size_t *); -static void sock_setupcalls_common(socket_t, sock_upcall, void *, - sock_upcall, void *); + +#undef sock_accept +#undef sock_socket +errno_t sock_accept(socket_t so, struct sockaddr *from, int fromlen, + int flags, sock_upcall callback, void *cookie, socket_t *new_so); +errno_t sock_socket(int domain, int type, int protocol, sock_upcall callback, + void *context, socket_t *new_so); + +static errno_t sock_accept_common(socket_t sock, struct sockaddr *from, + int fromlen, int flags, sock_upcall callback, void *cookie, + socket_t *new_sock, bool is_internal); +static errno_t sock_socket_common(int domain, int type, int protocol, + sock_upcall callback, void *context, socket_t *new_so, bool is_internal); errno_t -sock_accept(socket_t sock, struct sockaddr *from, int fromlen, int flags, - sock_upcall callback, void *cookie, socket_t *new_sock) +sock_accept_common(socket_t sock, struct sockaddr *from, int fromlen, int flags, + sock_upcall callback, void *cookie, socket_t *new_sock, bool is_internal) { struct sockaddr *sa; struct socket *new_so; @@ -82,7 +95,7 @@ check_again: } if (sock->so_proto->pr_getlock != NULL) { - mutex_held = (*sock->so_proto->pr_getlock)(sock, 0); + mutex_held = (*sock->so_proto->pr_getlock)(sock, PR_F_WILLUNLOCK); dosocklock = 1; } else { mutex_held = sock->so_proto->pr_domain->dom_mtx; @@ -121,6 +134,15 @@ check_again: so_release_accept_list(sock); + /* + * Count the accepted socket as an in-kernel socket + */ + new_so->so_flags1 |= SOF1_IN_KERNEL_SOCKET; + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_in_kernel_total); + if (is_internal) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_in_kernel_os_total); + } + /* * Pass the pre-accepted socket to any interested socket filter(s). * Upon failure, the socket would have been closed by the callee. @@ -142,7 +164,7 @@ check_again: } if (dosocklock) { - lck_mtx_assert(new_so->so_proto->pr_getlock(new_so, 0), + LCK_MTX_ASSERT(new_so->so_proto->pr_getlock(new_so, 0), LCK_MTX_ASSERT_NOTOWNED); socket_lock(new_so, 1); } @@ -153,7 +175,11 @@ check_again: /* see comments in sock_setupcall() */ if (callback != NULL) { - sock_setupcalls_common(new_so, callback, cookie, NULL, NULL); +#if CONFIG_EMBEDDED + sock_setupcalls_locked(new_so, callback, cookie, callback, cookie, 0); +#else + sock_setupcalls_locked(new_so, callback, cookie, NULL, NULL, 0); +#endif /* !CONFIG_EMBEDDED */ } if (sa != NULL && from != NULL) { @@ -178,6 +204,22 @@ check_again: return (error); } +errno_t +sock_accept(socket_t sock, struct sockaddr *from, int fromlen, int flags, + sock_upcall callback, void *cookie, socket_t *new_sock) +{ + return (sock_accept_common(sock, from, fromlen, flags, + callback, cookie, new_sock, false)); +} + +errno_t +sock_accept_internal(socket_t sock, struct sockaddr *from, int fromlen, int flags, + sock_upcall callback, void *cookie, socket_t *new_sock) +{ + return (sock_accept_common(sock, from, fromlen, flags, + callback, cookie, new_sock, true)); +} + errno_t sock_bind(socket_t sock, const struct sockaddr *to) { @@ -247,7 +289,7 @@ sock_connect(socket_t sock, const struct sockaddr *to, int flags) } if (sock->so_proto->pr_getlock != NULL) - mutex_held = (*sock->so_proto->pr_getlock)(sock, 0); + mutex_held = (*sock->so_proto->pr_getlock)(sock, PR_F_WILLUNLOCK); else mutex_held = sock->so_proto->pr_domain->dom_mtx; @@ -313,7 +355,7 @@ sock_connectwait(socket_t sock, const struct timeval *tv) } if (sock->so_proto->pr_getlock != NULL) - mutex_held = (*sock->so_proto->pr_getlock)(sock, 0); + mutex_held = (*sock->so_proto->pr_getlock)(sock, PR_F_WILLUNLOCK); else mutex_held = sock->so_proto->pr_domain->dom_mtx; @@ -892,10 +934,9 @@ sock_shutdown(socket_t sock, int how) return (soshutdown(sock, how)); } - errno_t -sock_socket(int domain, int type, int protocol, sock_upcall callback, - void *context, socket_t *new_so) +sock_socket_common(int domain, int type, int protocol, sock_upcall callback, + void *context, socket_t *new_so, bool is_internal) { int error = 0; @@ -905,10 +946,18 @@ sock_socket(int domain, int type, int protocol, sock_upcall callback, /* socreate will create an initial so_count */ error = socreate(domain, new_so, type, protocol); if (error == 0) { + /* + * This is an in-kernel socket + */ + (*new_so)->so_flags1 |= SOF1_IN_KERNEL_SOCKET; + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_in_kernel_total); + if (is_internal) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_in_kernel_os_total); + } + /* see comments in sock_setupcall() */ if (callback != NULL) { - sock_setupcalls_common(*new_so, callback, context, - NULL, NULL); + sock_setupcall(*new_so, callback, context); } /* * last_pid and last_upid should be zero for sockets @@ -920,6 +969,22 @@ sock_socket(int domain, int type, int protocol, sock_upcall callback, return (error); } +errno_t +sock_socket_internal(int domain, int type, int protocol, sock_upcall callback, + void *context, socket_t *new_so) +{ + return (sock_socket_common(domain, type, protocol, callback, + context, new_so, true)); +} + +errno_t +sock_socket(int domain, int type, int protocol, sock_upcall callback, + void *context, socket_t *new_so) +{ + return (sock_socket_common(domain, type, protocol, callback, + context, new_so, false)); +} + void sock_close(socket_t sock) { @@ -1135,26 +1200,30 @@ socket_defunct(struct proc *p, socket_t so, int level) return (retval); } -static void -sock_setupcalls_common(socket_t sock, sock_upcall rcallback, void *rcontext, - sock_upcall wcallback, void *wcontext) +void +sock_setupcalls_locked(socket_t sock, sock_upcall rcallback, void *rcontext, + sock_upcall wcallback, void *wcontext, int locked) { if (rcallback != NULL) { sock->so_rcv.sb_flags |= SB_UPCALL; + if (locked) + sock->so_rcv.sb_flags |= SB_UPCALL_LOCK; sock->so_rcv.sb_upcall = rcallback; sock->so_rcv.sb_upcallarg = rcontext; } else { - sock->so_rcv.sb_flags &= ~SB_UPCALL; + sock->so_rcv.sb_flags &= ~(SB_UPCALL | SB_UPCALL_LOCK); sock->so_rcv.sb_upcall = NULL; sock->so_rcv.sb_upcallarg = NULL; } if (wcallback != NULL) { sock->so_snd.sb_flags |= SB_UPCALL; + if (locked) + sock->so_snd.sb_flags |= SB_UPCALL_LOCK; sock->so_snd.sb_upcall = wcallback; sock->so_snd.sb_upcallarg = wcontext; } else { - sock->so_snd.sb_flags &= ~SB_UPCALL; + sock->so_snd.sb_flags &= ~(SB_UPCALL | SB_UPCALL_LOCK); sock->so_snd.sb_upcall = NULL; sock->so_snd.sb_upcallarg = NULL; } @@ -1176,7 +1245,11 @@ sock_setupcall(socket_t sock, sock_upcall callback, void *context) * the read and write callbacks and their respective parameters. */ socket_lock(sock, 1); - sock_setupcalls_common(sock, callback, context, NULL, NULL); +#if CONFIG_EMBEDDED + sock_setupcalls_locked(sock, callback, context, callback, context, 0); +#else + sock_setupcalls_locked(sock, callback, context, NULL, NULL, 0); +#endif /* !CONFIG_EMBEDDED */ socket_unlock(sock, 1); return (0); @@ -1193,23 +1266,21 @@ sock_setupcalls(socket_t sock, sock_upcall rcallback, void *rcontext, * Note that we don't wait for any in progress upcall to complete. */ socket_lock(sock, 1); - sock_setupcalls_common(sock, rcallback, rcontext, wcallback, wcontext); + sock_setupcalls_locked(sock, rcallback, rcontext, wcallback, wcontext, 0); socket_unlock(sock, 1); return (0); } -errno_t -sock_catchevents(socket_t sock, sock_evupcall ecallback, void *econtext, +void +sock_catchevents_locked(socket_t sock, sock_evupcall ecallback, void *econtext, u_int32_t emask) { - if (sock == NULL) - return (EINVAL); + socket_lock_assert_owned(sock); /* * Note that we don't wait for any in progress upcall to complete. */ - socket_lock(sock, 1); if (ecallback != NULL) { sock->so_event = ecallback; sock->so_eventarg = econtext; @@ -1219,6 +1290,17 @@ sock_catchevents(socket_t sock, sock_evupcall ecallback, void *econtext, sock->so_eventarg = NULL; sock->so_eventmask = 0; } +} + +errno_t +sock_catchevents(socket_t sock, sock_evupcall ecallback, void *econtext, + u_int32_t emask) +{ + if (sock == NULL) + return (EINVAL); + + socket_lock(sock, 1); + sock_catchevents_locked(sock, ecallback, econtext, emask); socket_unlock(sock, 1); return (0); diff --git a/bsd/kern/kpi_socketfilter.c b/bsd/kern/kpi_socketfilter.c index e8e4115f0..138c6b299 100644 --- a/bsd/kern/kpi_socketfilter.c +++ b/bsd/kern/kpi_socketfilter.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003-2016 Apple Inc. All rights reserved. + * Copyright (c) 2003-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -40,6 +40,7 @@ #include <kern/debug.h> #include <net/kext_net.h> #include <net/if.h> +#include <net/net_api_stats.h> #include <netinet/in_var.h> #include <netinet/ip.h> #include <netinet/ip_var.h> @@ -51,6 +52,7 @@ #include <libkern/libkern.h> #include <libkern/OSAtomic.h> +#include <stdbool.h> #include <string.h> #define SFEF_ATTACHED 0x1 /* SFE is on socket list */ @@ -91,6 +93,13 @@ static thread_t sock_filter_cleanup_thread = NULL; static void sflt_cleanup_thread(void *, wait_result_t); static void sflt_detach_locked(struct socket_filter_entry *entry); +#undef sflt_register +static errno_t sflt_register_common(const struct sflt_filter *filter, int domain, + int type, int protocol, bool is_internal); +errno_t sflt_register(const struct sflt_filter *filter, int domain, + int type, int protocol); + + #pragma mark -- Internal State Management -- __private_extern__ int @@ -1249,9 +1258,9 @@ struct solist { struct socket *so; }; -errno_t -sflt_register(const struct sflt_filter *filter, int domain, int type, - int protocol) +static errno_t +sflt_register_common(const struct sflt_filter *filter, int domain, int type, + int protocol, bool is_internal) { struct socket_filter *sock_filt = NULL; struct socket_filter *match = NULL; @@ -1317,6 +1326,12 @@ sflt_register(const struct sflt_filter *filter, int domain, int type, sock_filt->sf_proto = pr; } sflt_retain_locked(sock_filt); + + OSIncrementAtomic64(&net_api_stats.nas_sfltr_register_count); + INC_ATOMIC_INT64_LIM(net_api_stats.nas_sfltr_register_total); + if (is_internal) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_sfltr_register_os_total); + } } lck_rw_unlock_exclusive(sock_filter_lock); @@ -1414,6 +1429,20 @@ sflt_register(const struct sflt_filter *filter, int domain, int type, return (error); } +errno_t +sflt_register_internal(const struct sflt_filter *filter, int domain, int type, + int protocol) +{ + return (sflt_register_common(filter, domain, type, protocol, true)); +} + +errno_t +sflt_register(const struct sflt_filter *filter, int domain, int type, + int protocol) +{ + return (sflt_register_common(filter, domain, type, protocol, false)); +} + errno_t sflt_unregister(sflt_handle handle) { @@ -1427,6 +1456,8 @@ sflt_unregister(sflt_handle handle) } if (filter) { + VERIFY(OSDecrementAtomic64(&net_api_stats.nas_sfltr_register_count) > 0); + /* Remove it from the global list */ TAILQ_REMOVE(&sock_filter_head, filter, sf_global_next); diff --git a/bsd/kern/mach_loader.c b/bsd/kern/mach_loader.c index ca2fb4338..43f1a78b0 100644 --- a/bsd/kern/mach_loader.c +++ b/bsd/kern/mach_loader.c @@ -248,7 +248,7 @@ struct macho_data; static load_return_t get_macho_vnode( - char *path, + const char *path, integer_t archbits, struct mach_header *mach_header, off_t *file_offset, @@ -285,6 +285,7 @@ note_all_image_info_section(const struct segment_command_64 *scp, } *sectionp; unsigned int i; + if (strncmp(scp->segname, "__DATA", sizeof(scp->segname)) != 0) return; for (i = 0; i < scp->nsects; ++i) { @@ -302,6 +303,15 @@ note_all_image_info_section(const struct segment_command_64 *scp, } } +#if __arm64__ +/* + * Allow bypassing some security rules (hard pagezero, no write+execute) + * in exchange for better binary compatibility for legacy apps built + * before 16KB-alignment was enforced. + */ +int fourk_binary_compatibility_unsafe = TRUE; +int fourk_binary_compatibility_allow_wx = FALSE; +#endif /* __arm64__ */ load_return_t load_machfile( @@ -324,8 +334,10 @@ load_machfile( int in_exec = (imgp->ip_flags & IMGPF_EXEC); task_t task = current_task(); proc_t p = current_proc(); - mach_vm_offset_t aslr_offset = 0; - mach_vm_offset_t dyld_aslr_offset = 0; + int64_t aslr_page_offset = 0; + int64_t dyld_aslr_page_offset = 0; + int64_t aslr_section_size = 0; + int64_t aslr_section_offset = 0; kern_return_t kret; if (macho_size > file_size) { @@ -348,7 +360,14 @@ load_machfile( vm_compute_max_offset(result->is64bit), TRUE); -#if (__ARM_ARCH_7K__ >= 2) && defined(PLATFORM_WatchOS) +#if defined(__arm64__) + if (result->is64bit) { + /* enforce 16KB alignment of VM map entries */ + vm_map_set_page_shift(map, SIXTEENK_PAGE_SHIFT); + } else { + vm_map_set_page_shift(map, page_shift_user32); + } +#elif (__ARM_ARCH_7K__ >= 2) && defined(PLATFORM_WatchOS) /* enforce 16KB alignment for watch targets with new ABI */ vm_map_set_page_shift(map, SIXTEENK_PAGE_SHIFT); #endif /* __arm64__ */ @@ -367,24 +386,25 @@ load_machfile( * normally permits it. */ if ((header->flags & MH_NO_HEAP_EXECUTION) && !(imgp->ip_flags & IMGPF_ALLOW_DATA_EXEC)) vm_map_disallow_data_exec(map); - + /* * Compute a random offset for ASLR, and an independent random offset for dyld. */ if (!(imgp->ip_flags & IMGPF_DISABLE_ASLR)) { - uint64_t max_slide_pages; + vm_map_get_max_aslr_slide_section(map, &aslr_section_offset, &aslr_section_size); + aslr_section_offset = (random() % aslr_section_offset) * aslr_section_size; - max_slide_pages = vm_map_get_max_aslr_slide_pages(map); + aslr_page_offset = random(); + aslr_page_offset %= vm_map_get_max_aslr_slide_pages(map); + aslr_page_offset <<= vm_map_page_shift(map); - aslr_offset = random(); - aslr_offset %= max_slide_pages; - aslr_offset <<= vm_map_page_shift(map); + dyld_aslr_page_offset = random(); + dyld_aslr_page_offset %= vm_map_get_max_loader_aslr_slide_pages(map); + dyld_aslr_page_offset <<= vm_map_page_shift(map); - dyld_aslr_offset = random(); - dyld_aslr_offset %= max_slide_pages; - dyld_aslr_offset <<= vm_map_page_shift(map); + aslr_page_offset += aslr_section_offset; } - + if (!result) result = &myresult; @@ -396,7 +416,7 @@ load_machfile( result->is64bit = ((imgp->ip_flags & IMGPF_IS_64BIT) == IMGPF_IS_64BIT); lret = parse_machfile(vp, map, thread, header, file_offset, macho_size, - 0, (int64_t)aslr_offset, (int64_t)dyld_aslr_offset, result, + 0, aslr_page_offset, dyld_aslr_page_offset, result, NULL, imgp); if (lret != LOAD_SUCCESS) { @@ -411,12 +431,44 @@ load_machfile( if (!result->is64bit) { enforce_hard_pagezero = FALSE; } -#endif + + /* + * For processes with IMGPF_HIGH_BITS_ASLR, add a few random high bits + * to the start address for "anywhere" memory allocations. + */ +#define VM_MAP_HIGH_START_BITS_COUNT 8 +#define VM_MAP_HIGH_START_BITS_SHIFT 27 + if (result->is64bit && + (imgp->ip_flags & IMGPF_HIGH_BITS_ASLR)) { + int random_bits; + vm_map_offset_t high_start; + + random_bits = random(); + random_bits &= (1 << VM_MAP_HIGH_START_BITS_COUNT)-1; + high_start = (((vm_map_offset_t)random_bits) + << VM_MAP_HIGH_START_BITS_SHIFT); + vm_map_set_high_start(map, high_start); + } +#endif /* __x86_64__ */ + /* * Check to see if the page zero is enforced by the map->min_offset. */ if (enforce_hard_pagezero && (vm_map_has_hard_pagezero(map, 0x1000) == FALSE)) { +#if __arm64__ + if (!result->is64bit && /* not 64-bit */ + !(header->flags & MH_PIE) && /* not PIE */ + (vm_map_page_shift(map) != FOURK_PAGE_SHIFT || + PAGE_SHIFT != FOURK_PAGE_SHIFT) && /* page size != 4KB */ + result->has_pagezero && /* has a "soft" page zero */ + fourk_binary_compatibility_unsafe) { + /* + * For backwards compatibility of "4K" apps on + * a 16K system, do not enforce a hard page zero... + */ + } else +#endif /* __arm64__ */ { vm_map_deallocate(map); /* will lose pmap reference too */ return (LOAD_BADMACHO); @@ -459,8 +511,7 @@ load_machfile( workqueue_mark_exiting(p); task_complete_halt(task); workqueue_exit(p); - kqueue_dealloc(p->p_wqkqueue); - p->p_wqkqueue = NULL; + /* * Roll up accounting info to new task. The roll up is done after * task_complete_halt to make sure the thread accounting info is @@ -514,9 +565,8 @@ parse_machfile( integer_t dlarchbits = 0; void * control; load_return_t ret = LOAD_SUCCESS; - caddr_t addr; - void * kl_addr; - vm_size_t size,kl_size; + void * addr; + vm_size_t alloc_size, cmds_size; size_t offset; size_t oldoffset; /* for overflow check */ int pass; @@ -532,6 +582,14 @@ parse_machfile( boolean_t dyld_no_load_addr = FALSE; boolean_t is_dyld = FALSE; vm_map_offset_t effective_page_mask = MAX(PAGE_MASK, vm_map_page_mask(map)); +#if __arm64__ + uint32_t pagezero_end = 0; + uint32_t executable_end = 0; + uint32_t writable_start = 0; + vm_map_size_t effective_page_size; + + effective_page_size = MAX(PAGE_SIZE, vm_map_page_size(map)); +#endif /* __arm64__ */ if (header->magic == MH_MAGIC_64 || header->magic == MH_CIGAM_64) { @@ -563,6 +621,20 @@ parse_machfile( if (depth != 1) { return (LOAD_FAILURE); } +#if CONFIG_EMBEDDED + if (header->flags & MH_DYLDLINK) { + /* Check properties of dynamic executables */ + if (!(header->flags & MH_PIE) && pie_required(header->cputype, header->cpusubtype & ~CPU_SUBTYPE_MASK)) { + return (LOAD_FAILURE); + } + result->needs_dynlinker = TRUE; + } else { + /* Check properties of static executables (disallowed except for development) */ +#if !(DEVELOPMENT || DEBUG) + return (LOAD_FAILURE); +#endif + } +#endif /* CONFIG_EMBEDDED */ break; case MH_DYLINKER: @@ -581,43 +653,32 @@ parse_machfile( */ control = ubc_getobject(vp, UBC_FLAGS_NONE); - /* - * Map portion that must be accessible directly into - * kernel's map. - */ - if ((off_t)(mach_header_sz + header->sizeofcmds) > macho_size) - return(LOAD_BADMACHO); - - /* - * Round size of Mach-O commands up to page boundry. - */ - size = round_page(mach_header_sz + header->sizeofcmds); - if (size <= 0) - return(LOAD_BADMACHO); + /* ensure header + sizeofcmds falls within the file */ + if (os_add_overflow(mach_header_sz, header->sizeofcmds, &cmds_size) || + (off_t)cmds_size > macho_size || + round_page_overflow(cmds_size, &alloc_size)) { + return LOAD_BADMACHO; + } /* * Map the load commands into kernel memory. */ - addr = 0; - kl_size = size; - kl_addr = kalloc(size); - addr = (caddr_t)kl_addr; - if (addr == NULL) - return(LOAD_NOSPACE); + addr = kalloc(alloc_size); + if (addr == NULL) { + return LOAD_NOSPACE; + } - error = vn_rdwr(UIO_READ, vp, addr, size, file_offset, + error = vn_rdwr(UIO_READ, vp, addr, alloc_size, file_offset, UIO_SYSSPACE, 0, kauth_cred_get(), &resid, p); if (error) { - if (kl_addr) - kfree(kl_addr, kl_size); - return(LOAD_IOERROR); + kfree(addr, alloc_size); + return LOAD_IOERROR; } if (resid) { /* We must be able to read in as much as the mach_header indicated */ - if (kl_addr) - kfree(kl_addr, kl_size); - return(LOAD_BADMACHO); + kfree(addr, alloc_size); + return LOAD_BADMACHO; } /* @@ -637,6 +698,11 @@ parse_machfile( */ boolean_t slide_realign = FALSE; +#if __arm64__ + if (!abi64) { + slide_realign = TRUE; + } +#endif for (pass = 0; pass <= 3; pass++) { @@ -645,6 +711,60 @@ parse_machfile( * address, pass 0 can be skipped */ continue; } else if (pass == 1) { +#if __arm64__ + boolean_t is_pie; + int64_t adjust; + + is_pie = ((header->flags & MH_PIE) != 0); + if (pagezero_end != 0 && + pagezero_end < effective_page_size) { + /* need at least 1 page for PAGEZERO */ + adjust = effective_page_size; + MACHO_PRINTF(("pagezero boundary at " + "0x%llx; adjust slide from " + "0x%llx to 0x%llx%s\n", + (uint64_t) pagezero_end, + slide, + slide + adjust, + (is_pie + ? "" + : " BUT NO PIE ****** :-("))); + if (is_pie) { + slide += adjust; + pagezero_end += adjust; + executable_end += adjust; + writable_start += adjust; + } + } + if (pagezero_end != 0) { + result->has_pagezero = TRUE; + } + if (executable_end == writable_start && + (executable_end & effective_page_mask) != 0 && + (executable_end & FOURK_PAGE_MASK) == 0) { + + /* + * The TEXT/DATA boundary is 4K-aligned but + * not page-aligned. Adjust the slide to make + * it page-aligned and avoid having a page + * with both write and execute permissions. + */ + adjust = + (effective_page_size - + (executable_end & effective_page_mask)); + MACHO_PRINTF(("page-unaligned X-W boundary at " + "0x%llx; adjust slide from " + "0x%llx to 0x%llx%s\n", + (uint64_t) executable_end, + slide, + slide + adjust, + (is_pie + ? "" + : " BUT NO PIE ****** :-("))); + if (is_pie) + slide += adjust; + } +#endif /* __arm64__ */ if (dyld_no_load_addr && binresult) { /* @@ -684,12 +804,18 @@ parse_machfile( ncmds = header->ncmds; while (ncmds--) { + + /* ensure enough space for a minimal load command */ + if (offset + sizeof(struct load_command) > cmds_size) { + ret = LOAD_BADMACHO; + break; + } + /* * Get a pointer to the command. */ lcp = (struct load_command *)(addr + offset); oldoffset = offset; - offset += lcp->cmdsize; /* * Perform prevalidation of the struct load_command @@ -699,9 +825,9 @@ parse_machfile( * straddle or exist past the reserved section at the * start of the image. */ - if (oldoffset > offset || - lcp->cmdsize < sizeof(struct load_command) || - offset > header->sizeofcmds + mach_header_sz) { + if (os_add_overflow(offset, lcp->cmdsize, &offset) || + lcp->cmdsize < sizeof(struct load_command) || + offset > cmds_size) { ret = LOAD_BADMACHO; break; } @@ -723,6 +849,31 @@ parse_machfile( } } +#if __arm64__ + assert(!abi64); + + if (scp->initprot == 0 && scp->maxprot == 0 && scp->vmaddr == 0) { + /* PAGEZERO */ + if (os_add3_overflow(scp->vmaddr, scp->vmsize, slide, &pagezero_end)) { + ret = LOAD_BADMACHO; + break; + } + } + if (scp->initprot & VM_PROT_EXECUTE) { + /* TEXT */ + if (os_add3_overflow(scp->vmaddr, scp->vmsize, slide, &executable_end)) { + ret = LOAD_BADMACHO; + break; + } + } + if (scp->initprot & VM_PROT_WRITE) { + /* DATA */ + if (os_add_overflow(scp->vmaddr, slide, &writable_start)) { + ret = LOAD_BADMACHO; + break; + } + } +#endif /* __arm64__ */ break; } @@ -850,7 +1001,7 @@ parse_machfile( case LC_UUID: if (pass == 1 && depth == 1) { ret = load_uuid((struct uuid_command *) lcp, - (char *)addr + mach_header_sz + header->sizeofcmds, + (char *)addr + cmds_size, result); } break; @@ -893,7 +1044,7 @@ parse_machfile( if (cs_debug > 10) printf("validating initial pages of %s\n", vp->v_name); - while (off < size && ret == LOAD_SUCCESS) { + while (off < alloc_size && ret == LOAD_SUCCESS) { tainted = CS_VALIDATE_TAINTED; valid = cs_validate_range(vp, @@ -975,6 +1126,7 @@ parse_machfile( if (cs_enforcement(NULL)) { ret = LOAD_FAILURE; } else { +#if !CONFIG_EMBEDDED /* * No embedded signatures: look for detached by taskgated, * this is only done on OSX, on embedded platforms we expect everything @@ -995,6 +1147,7 @@ parse_machfile( /* get flags to be applied to the process */ result->csflags |= cs_flag_data; } +#endif } } @@ -1016,6 +1169,11 @@ parse_machfile( if (result->thread_count == 0) { ret = LOAD_FAILURE; } +#if CONFIG_EMBEDDED + if (result->needs_dynlinker && !(result->csflags & CS_DYLD_PLATFORM)) { + ret = LOAD_FAILURE; + } +#endif } } @@ -1023,11 +1181,9 @@ parse_machfile( ret = LOAD_BADMACHO_UPX; } - if (kl_addr) { - kfree(kl_addr, kl_size); - } + kfree(addr, alloc_size); - return(ret); + return ret; } #if CONFIG_CODE_DECRYPTION @@ -1155,10 +1311,10 @@ map_segment( vm_prot_t initprot, vm_prot_t maxprot) { - int extra_vm_flags, cur_extra_vm_flags; vm_map_offset_t cur_offset, cur_start, cur_end; kern_return_t ret; vm_map_offset_t effective_page_mask; + vm_map_kernel_flags_t vmk_flags, cur_vmk_flags; if (vm_end < vm_start || file_end < file_start) { @@ -1172,24 +1328,71 @@ map_segment( effective_page_mask = MAX(PAGE_MASK, vm_map_page_mask(map)); - extra_vm_flags = 0; + vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; if (vm_map_page_aligned(vm_start, effective_page_mask) && vm_map_page_aligned(vm_end, effective_page_mask) && vm_map_page_aligned(file_start, effective_page_mask) && vm_map_page_aligned(file_end, effective_page_mask)) { /* all page-aligned and map-aligned: proceed */ } else { +#if __arm64__ + /* use an intermediate "4K" pager */ + vmk_flags.vmkf_fourk = TRUE; +#else /* __arm64__ */ panic("map_segment: unexpected mis-alignment " "vm[0x%llx:0x%llx] file[0x%llx:0x%llx]\n", (uint64_t) vm_start, (uint64_t) vm_end, (uint64_t) file_start, (uint64_t) file_end); +#endif /* __arm64__ */ } cur_offset = 0; cur_start = vm_start; cur_end = vm_start; +#if __arm64__ + if (!vm_map_page_aligned(vm_start, effective_page_mask)) { + /* one 4K pager for the 1st page */ + cur_end = vm_map_round_page(cur_start, effective_page_mask); + if (cur_end > vm_end) { + cur_end = vm_start + (file_end - file_start); + } + if (control != MEMORY_OBJECT_CONTROL_NULL) { + ret = vm_map_enter_mem_object_control( + map, + &cur_start, + cur_end - cur_start, + (mach_vm_offset_t)0, + VM_FLAGS_FIXED, + vmk_flags, + VM_KERN_MEMORY_NONE, + control, + file_start + cur_offset, + TRUE, /* copy */ + initprot, maxprot, + VM_INHERIT_DEFAULT); + } else { + ret = vm_map_enter_mem_object( + map, + &cur_start, + cur_end - cur_start, + (mach_vm_offset_t)0, + VM_FLAGS_FIXED, + vmk_flags, + VM_KERN_MEMORY_NONE, + IPC_PORT_NULL, + 0, /* offset */ + TRUE, /* copy */ + initprot, maxprot, + VM_INHERIT_DEFAULT); + } + if (ret != KERN_SUCCESS) { + return (LOAD_NOSPACE); + } + cur_offset += cur_end - cur_start; + } +#endif /* __arm64__ */ if (cur_end >= vm_start + (file_end - file_start)) { /* all mapped: done */ goto done; @@ -1203,10 +1406,10 @@ map_segment( if ((vm_start & effective_page_mask) != (file_start & effective_page_mask)) { /* one 4K pager for the middle */ - cur_extra_vm_flags = extra_vm_flags; + cur_vmk_flags = vmk_flags; } else { /* regular mapping for the middle */ - cur_extra_vm_flags = 0; + cur_vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; } cur_end = vm_map_trunc_page(vm_start + (file_end - file_start), @@ -1217,7 +1420,9 @@ map_segment( &cur_start, cur_end - cur_start, (mach_vm_offset_t)0, - VM_FLAGS_FIXED | cur_extra_vm_flags, + VM_FLAGS_FIXED, + cur_vmk_flags, + VM_KERN_MEMORY_NONE, control, file_start + cur_offset, TRUE, /* copy */ @@ -1229,7 +1434,9 @@ map_segment( &cur_start, cur_end - cur_start, (mach_vm_offset_t)0, - VM_FLAGS_FIXED | cur_extra_vm_flags, + VM_FLAGS_FIXED, + cur_vmk_flags, + VM_KERN_MEMORY_NONE, IPC_PORT_NULL, 0, /* offset */ TRUE, /* copy */ @@ -1246,6 +1453,46 @@ map_segment( goto done; } cur_start = cur_end; +#if __arm64__ + if (!vm_map_page_aligned(vm_start + (file_end - file_start), + effective_page_mask)) { + /* one 4K pager for the last page */ + cur_end = vm_start + (file_end - file_start); + if (control != MEMORY_OBJECT_CONTROL_NULL) { + ret = vm_map_enter_mem_object_control( + map, + &cur_start, + cur_end - cur_start, + (mach_vm_offset_t)0, + VM_FLAGS_FIXED, + vmk_flags, + VM_KERN_MEMORY_NONE, + control, + file_start + cur_offset, + TRUE, /* copy */ + initprot, maxprot, + VM_INHERIT_DEFAULT); + } else { + ret = vm_map_enter_mem_object( + map, + &cur_start, + cur_end - cur_start, + (mach_vm_offset_t)0, + VM_FLAGS_FIXED, + vmk_flags, + VM_KERN_MEMORY_NONE, + IPC_PORT_NULL, + 0, /* offset */ + TRUE, /* copy */ + initprot, maxprot, + VM_INHERIT_DEFAULT); + } + if (ret != KERN_SUCCESS) { + return (LOAD_NOSPACE); + } + cur_offset += cur_end - cur_start; + } +#endif /* __arm64__ */ done: assert(cur_end >= vm_start + (file_end - file_start)); return LOAD_SUCCESS; @@ -1279,6 +1526,10 @@ load_segment( boolean_t verbose; vm_map_size_t effective_page_size; vm_map_offset_t effective_page_mask; +#if __arm64__ + vm_map_kernel_flags_t vmk_flags; + boolean_t fourk_align; +#endif /* __arm64__ */ effective_page_size = MAX(PAGE_SIZE, vm_map_page_size(map)); effective_page_mask = MAX(PAGE_MASK, vm_map_page_mask(map)); @@ -1287,9 +1538,24 @@ load_segment( if (LC_SEGMENT_64 == lcp->cmd) { segment_command_size = sizeof(struct segment_command_64); single_section_size = sizeof(struct section_64); +#if __arm64__ + /* 64-bit binary: should already be 16K-aligned */ + fourk_align = FALSE; +#endif /* __arm64__ */ } else { segment_command_size = sizeof(struct segment_command); single_section_size = sizeof(struct section); +#if __arm64__ + /* 32-bit binary: might need 4K-alignment */ + if (effective_page_size != FOURK_PAGE_SIZE) { + /* not using 4K page size: need fourk_pager */ + fourk_align = TRUE; + verbose = TRUE; + } else { + /* using 4K page size: no need for re-alignment */ + fourk_align = FALSE; + } +#endif /* __arm64__ */ } if (lcp->cmdsize < segment_command_size) return (LOAD_BADMACHO); @@ -1336,6 +1602,17 @@ load_segment( */ file_offset = pager_offset + scp->fileoff; /* limited to 32 bits */ file_size = scp->filesize; +#if __arm64__ + if (fourk_align) { + if ((file_offset & FOURK_PAGE_MASK) != 0) { + /* + * we can't mmap() it if it's not at least 4KB-aligned + * in the file + */ + return LOAD_BADMACHO; + } + } else +#endif /* __arm64__ */ if ((file_offset & PAGE_MASK_64) != 0 || /* we can't mmap() it if it's not page-aligned in the file */ (file_offset & vm_map_page_mask(map)) != 0) { @@ -1393,6 +1670,13 @@ load_segment( "page_zero up to 0x%llx\n", (uint64_t) vm_end)); } +#if __arm64__ + if (fourk_align) { + /* raise min_offset as much as page-alignment allows */ + vm_end_aligned = vm_map_trunc_page(vm_end, + effective_page_mask); + } else +#endif /* __arm64__ */ { vm_end = vm_map_round_page(vm_end, PAGE_MASK_64); @@ -1400,14 +1684,67 @@ load_segment( } ret = vm_map_raise_min_offset(map, vm_end_aligned); +#if __arm64__ + if (ret == 0 && + vm_end > vm_end_aligned) { + /* use fourk_pager to map the rest of pagezero */ + assert(fourk_align); + vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; + vmk_flags.vmkf_fourk = TRUE; + ret = vm_map_enter_mem_object( + map, + &vm_end_aligned, + vm_end - vm_end_aligned, + (mach_vm_offset_t) 0, /* mask */ + VM_FLAGS_FIXED, + vmk_flags, + VM_KERN_MEMORY_NONE, + IPC_PORT_NULL, + 0, + FALSE, /* copy */ + (scp->initprot & VM_PROT_ALL), + (scp->maxprot & VM_PROT_ALL), + VM_INHERIT_DEFAULT); + } +#endif /* __arm64__ */ if (ret != KERN_SUCCESS) { return (LOAD_FAILURE); } return (LOAD_SUCCESS); } else { +#if CONFIG_EMBEDDED + /* not PAGEZERO: should not be mapped at address 0 */ + if (filetype != MH_DYLINKER && scp->vmaddr == 0) { + return LOAD_BADMACHO; + } +#endif /* CONFIG_EMBEDDED */ } +#if __arm64__ + if (fourk_align) { + /* 4K-align */ + file_start = vm_map_trunc_page(file_offset, + FOURK_PAGE_MASK); + file_end = vm_map_round_page(file_offset + file_size, + FOURK_PAGE_MASK); + vm_start = vm_map_trunc_page(vm_offset, + FOURK_PAGE_MASK); + vm_end = vm_map_round_page(vm_offset + vm_size, + FOURK_PAGE_MASK); + if (!strncmp(scp->segname, "__LINKEDIT", 11) && + page_aligned(file_start) && + vm_map_page_aligned(file_start, vm_map_page_mask(map)) && + page_aligned(vm_start) && + vm_map_page_aligned(vm_start, vm_map_page_mask(map))) { + /* XXX last segment: ignore mis-aligned tail */ + file_end = vm_map_round_page(file_end, + effective_page_mask); + vm_end = vm_map_round_page(vm_end, + effective_page_mask); + } + } else +#endif /* __arm64__ */ { file_start = vm_map_trunc_page(file_offset, effective_page_mask); @@ -1463,7 +1800,7 @@ load_segment( if (delta_size > 0) { mach_vm_offset_t tmp; - ret = mach_vm_allocate(kernel_map, &tmp, delta_size, VM_FLAGS_ANYWHERE| VM_MAKE_TAG(VM_KERN_MEMORY_BSD)); + ret = mach_vm_allocate_kernel(kernel_map, &tmp, delta_size, VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_BSD); if (ret != KERN_SUCCESS) { return(LOAD_RESOURCE); } @@ -1879,6 +2216,15 @@ extern char dyld_alt_path[]; extern int use_alt_dyld; #endif +static uint64_t get_va_fsid(struct vnode_attr *vap) +{ + if (VATTR_IS_SUPPORTED(vap, va_fsid64)) { + return *(uint64_t *)&vap->va_fsid64; + } else { + return vap->va_fsid; + } +} + static load_return_t load_dylinker( struct dylinker_command *lcp, @@ -1891,8 +2237,7 @@ load_dylinker( struct image_params *imgp ) { - char *name; - char *p; + const char *name; struct vnode *vp = NULLVP; /* set by get_macho_vnode() */ struct mach_header *header; off_t file_offset = 0; /* set by get_macho_vnode() */ @@ -1906,19 +2251,17 @@ load_dylinker( struct macho_data __macho_data; } *dyld_data; - if (lcp->cmdsize < sizeof(*lcp)) - return (LOAD_BADMACHO); + if (lcp->cmdsize < sizeof(*lcp) || lcp->name.offset >= lcp->cmdsize) + return LOAD_BADMACHO; - name = (char *)lcp + lcp->name.offset; + name = (const char *)lcp + lcp->name.offset; - /* - * Check for a proper null terminated string. - */ - p = name; - do { - if (p >= (char *)lcp + lcp->cmdsize) - return(LOAD_BADMACHO); - } while (*p++); + /* Check for a proper null terminated string. */ + size_t maxsz = lcp->cmdsize - lcp->name.offset; + size_t namelen = strnlen(name, maxsz); + if (namelen >= maxsz) { + return LOAD_BADMACHO; + } #if (DEVELOPMENT || DEBUG) @@ -1985,6 +2328,17 @@ load_dylinker( } } + struct vnode_attr va; + VATTR_INIT(&va); + VATTR_WANTED(&va, va_fsid64); + VATTR_WANTED(&va, va_fsid); + VATTR_WANTED(&va, va_fileid); + int error = vnode_getattr(vp, &va, imgp->ip_vfs_context); + if (error == 0) { + imgp->ip_dyld_fsid = get_va_fsid(&va); + imgp->ip_dyld_fsobjid = va.va_fileid; + } + vnode_put(vp); novp_out: FREE(dyld_data, M_TEMP); @@ -2088,7 +2442,7 @@ load_code_signature( out: if (ret == LOAD_SUCCESS) { if (blob == NULL) - panic("sucess, but no blob!"); + panic("success, but no blob!"); result->csflags |= blob->csb_flags; result->platform_binary = blob->csb_platform_binary; @@ -2259,7 +2613,7 @@ remap_now: static load_return_t get_macho_vnode( - char *path, + const char *path, integer_t archbits, struct mach_header *mach_header, off_t *file_offset, diff --git a/bsd/kern/mach_loader.h b/bsd/kern/mach_loader.h index 760ea45d9..d58ae5a4a 100644 --- a/bsd/kern/mach_loader.h +++ b/bsd/kern/mach_loader.h @@ -63,6 +63,7 @@ typedef struct _load_result { mach_vm_address_t all_image_info_addr; mach_vm_size_t all_image_info_size; + int thread_count; unsigned int /* boolean_t */ unixproc :1, diff --git a/bsd/kern/mach_process.c b/bsd/kern/mach_process.c index 184f568a4..72d262def 100644 --- a/bsd/kern/mach_process.c +++ b/bsd/kern/mach_process.c @@ -95,6 +95,10 @@ #include <pexpert/pexpert.h> +#if CONFIG_MACF +#include <security/mac_framework.h> +#endif + /* XXX ken/bsd_kern.c - prototype should be in common header */ int get_task_userstop(task_t); @@ -128,6 +132,10 @@ ptrace(struct proc *p, struct ptrace_args *uap, int32_t *retval) AUDIT_ARG(value32, uap->data); if (uap->req == PT_DENY_ATTACH) { +#if (DEVELOPMENT || DEBUG) && CONFIG_EMBEDDED + if (PE_i_can_has_debugger(NULL)) + return(0); +#endif proc_lock(p); if (ISSET(p->p_lflag, P_LTRACED)) { proc_unlock(p); @@ -227,6 +235,12 @@ retry_trace_me:; #pragma clang diagnostic pop int err; +#if CONFIG_EMBEDDED + if (tr_sigexc == 0) { + error = ENOTSUP; + goto out; + } +#endif if ( kauth_authorize_process(proc_ucred(p), KAUTH_PROCESS_CANTRACE, t, (uintptr_t)&err, 0, 0) == 0 ) { diff --git a/bsd/kern/makesyscalls.sh b/bsd/kern/makesyscalls.sh index 29dd74b74..411d5ce61 100755 --- a/bsd/kern/makesyscalls.sh +++ b/bsd/kern/makesyscalls.sh @@ -35,6 +35,7 @@ output_syshdrfile=0 output_syscalltablefile=0 output_auditevfile=0 output_tracecodes=0 +output_systrace=0 use_stdout=0 @@ -50,7 +51,7 @@ syscallprefix="SYS_" switchname="sysent" namesname="syscallnames" tracecodename="syscall.codes" - +systraceargsfile="systrace_args.c" # tmp files: syslegal="sysent.syslegal.$$" sysent="sysent.switch.$$" @@ -61,10 +62,13 @@ syscallnamestempfile="syscallnamesfile.$$" syshdrtempfile="syshdrtempfile.$$" audittempfile="audittempfile.$$" tracecodetempfile="tracecodetempfile.$$" +systraceargstempfile="systraceargstempfile.$$" +systraceargdesctempfile="systraceargdesctempfile.$$" +systracerettempfile="systracerettempfile.$$" -trap "rm $syslegal $sysent $sysinc $sysarg $sysprotoend $syscallnamestempfile $syshdrtempfile $audittempfile $tracecodetempfile" 0 +trap "rm $syslegal $sysent $sysinc $sysarg $sysprotoend $syscallnamestempfile $syshdrtempfile $audittempfile $tracecodetempfile $systraceargstempfile $systraceargdesctempfile $systracerettempfile" 0 -touch $syslegal $sysent $sysinc $sysarg $sysprotoend $syscallnamestempfile $syshdrtempfile $audittempfile $tracecodetempfile +touch $syslegal $sysent $sysinc $sysarg $sysprotoend $syscallnamestempfile $syshdrtempfile $audittempfile $tracecodetempfile $systraceargstempfile $systraceargdesctempfile $systracerettempfile case $# in 0) @@ -93,6 +97,9 @@ if [ -n "$1" ]; then audit) output_auditevfile=1 ;; + systrace) + output_systrace=1 + ;; trace) output_tracecodes=1 use_stdout=1 @@ -141,6 +148,9 @@ s/\$//g syscallnamestempfile = \"$syscallnamestempfile\" syshdrfile = \"$syshdrfile\" syshdrtempfile = \"$syshdrtempfile\" + systraceargstempfile = \"$systraceargstempfile\" + systraceargdesctempfile = \"$systraceargdesctempfile\" + systracerettempfile = \"$systracerettempfile\" audittempfile = \"$audittempfile\" tracecodetempfile = \"$tracecodetempfile\" syscallprefix = \"$syscallprefix\" @@ -233,6 +243,19 @@ s/\$//g printf "#include <bsm/audit_kevents.h>\n\n" > audittempfile printf "#if CONFIG_AUDIT\n\n" > audittempfile printf "au_event_t sys_au_event[] = {\n" > audittempfile + + printf "/*\n * System call argument to DTrace register array conversion.\n */\n" > systraceargstempfile + printf "#include <sys/systrace_args.h>\n" > systraceargstempfile + printf "void\nsystrace_args(int sysnum, void *params, uint64_t *uarg)\n{\n" > systraceargstempfile + printf "\tint64_t *iarg = (int64_t *) uarg;\n" > systraceargstempfile + printf "\tswitch (sysnum) {\n" > systraceargstempfile + + printf "void\nsystrace_entry_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)\n{\n\tconst char *p = NULL;\n" > systraceargdesctempfile + printf "\tswitch (sysnum) {\n" > systraceargdesctempfile + + printf "void\nsystrace_return_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)\n{\n\tconst char *p = NULL;\n" > systracerettempfile + printf "\tswitch (sysnum) {\n" > systracerettempfile + next } NF == 0 || $1 ~ /^;/ { @@ -248,6 +271,9 @@ s/\$//g print > syscallnamestempfile print > sysprotoend print > audittempfile + print > systraceargstempfile + print > systraceargdesctempfile + print > systracerettempfile savesyscall = syscall_num skip_for_header = 0 next @@ -258,6 +284,9 @@ s/\$//g print > syscallnamestempfile print > sysprotoend print > audittempfile + print > systraceargstempfile + print > systraceargdesctempfile + print > systracerettempfile syscall_num = savesyscall skip_for_header = 1 next @@ -268,6 +297,9 @@ s/\$//g print > syscallnamestempfile print > sysprotoend print > audittempfile + print > systraceargstempfile + print > systraceargdesctempfile + print > systracerettempfile skip_for_header = 0 next } @@ -467,6 +499,40 @@ s/\$//g size32 = 0 if ((funcname != "nosys" && funcname != "enosys") || (syscall_num == 0 && funcname == "nosys")) { + printf("\t/* %s */\n\tcase %d: {\n", funcname, syscall_num) > systraceargstempfile + printf("\t/* %s */\n\tcase %d:\n", funcname, syscall_num) > systraceargdesctempfile + printf("\t/* %s */\n\tcase %d:\n", funcname, syscall_num) > systracerettempfile + if (argc > 0) { + printf("\t\tswitch(ndx) {\n") > systraceargdesctempfile + printf("\t\tstruct %s *p = params;\n", argalias) > systraceargstempfile + for (i = 1; i <= argc; i++) { + arg = argtype[i] + sub("__restrict$", "", arg) + if (index(arg, "*") > 0) + printf("\t\tcase %d:\n\t\t\tp = \"userland %s\";\n\t\t\tbreak;\n", i - 1, arg) > systraceargdesctempfile + else + printf("\t\tcase %d:\n\t\t\tp = \"%s\";\n\t\t\tbreak;\n", i - 1, arg) > systraceargdesctempfile + if (index(arg, "*") > 0 || arg == "caddr_t") + printf("\t\tuarg[%d] = (intptr_t) p->%s; /* %s */\n", \ + i - 1, \ + argname[i], arg) > systraceargstempfile + else if (substr(arg, 1, 1) == "u" || arg == "size_t") + printf("\t\tuarg[%d] = p->%s; /* %s */\n", \ + i - 1, \ + argname[i], arg) > systraceargstempfile + else + printf("\t\tiarg[%d] = p->%s; /* %s */\n", \ + i - 1, \ + argname[i], arg) > systraceargstempfile + } + printf("\t\tdefault:\n\t\t\tbreak;\n\t\t};\n") > systraceargdesctempfile + + } + printf("\t\tbreak;\n\t}\n", argc) > systraceargstempfile + printf("\t\tif (ndx == 0 || ndx == 1)\n") > systracerettempfile + printf("\t\t\tp = \"%s\";\n", returntype) > systracerettempfile + printf("\t\tbreak;\n") > systracerettempfile + printf("\t\tbreak;\n") > systraceargdesctempfile if (argc != 0) { if (add_sysproto_entry == 1) { printf("struct %s {\n", argalias) > sysarg @@ -707,6 +773,10 @@ s/\$//g printf("#endif /* !%s */\n", syscall_h) > syshdrtempfile printf("};\n\n") > audittempfile printf("#endif /* AUDIT */\n") > audittempfile + + printf "\tdefault:\n\t\tbreak;\n\t};\n}\n" > systraceargstempfile + printf "\tdefault:\n\t\tbreak;\n\t};\n\tif (p != NULL)\n\t\tstrlcpy(desc, p, descsz);\n}\n" > systraceargdesctempfile + printf "\tdefault:\n\t\tbreak;\n\t};\n\tif (p != NULL)\n\t\tstrlcpy(desc, p, descsz);\n}\n" > systracerettempfile } ' # define value in syscall table file to permit redifintion because of the way @@ -733,6 +803,12 @@ if [ $output_auditevfile -eq 1 ]; then cat $syslegal $audittempfile > $auditevfile fi +if [ $output_systrace -eq 1 ]; then + cat $systraceargstempfile > $systraceargsfile + cat $systraceargdesctempfile >> $systraceargsfile + cat $systracerettempfile >> $systraceargsfile +fi + if [ $output_tracecodes -eq 1 ]; then if [ $use_stdout -eq 1 ]; then cat $tracecodetempfile @@ -740,3 +816,4 @@ if [ $output_tracecodes -eq 1 ]; then cat $tracecodetempfile > $tracecodename fi fi + diff --git a/bsd/kern/mcache.c b/bsd/kern/mcache.c index 823fbf95e..0794dc1db 100644 --- a/bsd/kern/mcache.c +++ b/bsd/kern/mcache.c @@ -69,7 +69,7 @@ #include <sys/mcache.h> #define MCACHE_SIZE(n) \ - ((size_t)(&((mcache_t *)0)->mc_cpu[n])) + __builtin_offsetof(mcache_t, mc_cpu[n]) /* Allocate extra in case we need to manually align the pointer */ #define MCACHE_ALLOC_SIZE \ @@ -154,6 +154,7 @@ static void mcache_bkt_purge(mcache_t *); static void mcache_bkt_destroy(mcache_t *, mcache_bkttype_t *, mcache_bkt_t *, int); static void mcache_bkt_ws_update(mcache_t *); +static void mcache_bkt_ws_zero(mcache_t *); static void mcache_bkt_ws_reap(mcache_t *); static void mcache_dispatch(void (*)(void *), void *); static void mcache_cache_reap(mcache_t *); @@ -307,11 +308,7 @@ mcache_create_common(const char *name, size_t bufsize, size_t align, return (NULL); } - if (!(wait & MCR_NOSLEEP)) - buf = zalloc(mcache_zone); - else - buf = zalloc_noblock(mcache_zone); - + buf = zalloc(mcache_zone); if (buf == NULL) goto fail; @@ -333,10 +330,14 @@ mcache_create_common(const char *name, size_t bufsize, size_t align, * Guaranteed alignment is valid only when we use the internal * slab allocator (currently set to use the zone allocator). */ - if (!need_zone) + if (!need_zone) { align = 1; - else if (align == 0) - align = MCACHE_ALIGN; + } else { + /* Enforce 64-bit minimum alignment for zone-based buffers */ + if (align == 0) + align = MCACHE_ALIGN; + align = P2ROUNDUP(align, MCACHE_ALIGN); + } if ((align & (align - 1)) != 0) panic("mcache_create: bad alignment %lu", align); @@ -368,9 +369,8 @@ mcache_create_common(const char *name, size_t bufsize, size_t align, */ chunksize = MAX(bufsize, sizeof (u_int64_t)); if (need_zone) { - /* Enforce 64-bit minimum alignment for zone-based buffers */ - align = MAX(align, sizeof (u_int64_t)); - chunksize += sizeof (void *) + align; + VERIFY(align != 0 && (align % MCACHE_ALIGN) == 0); + chunksize += sizeof (uint64_t) + align; chunksize = P2ROUNDUP(chunksize, align); if ((cp->mc_slab_zone = zinit(chunksize, 64 * 1024 * ncpu, PAGE_SIZE, cp->mc_name)) == NULL) @@ -898,11 +898,12 @@ mcache_destroy(mcache_t *cp) * implementation uses the zone allocator for simplicity reasons. */ static unsigned int -mcache_slab_alloc(void *arg, mcache_obj_t ***plist, unsigned int num, int wait) +mcache_slab_alloc(void *arg, mcache_obj_t ***plist, unsigned int num, + int wait) { +#pragma unused(wait) mcache_t *cp = arg; unsigned int need = num; - size_t offset = 0; size_t rsize = P2ROUNDUP(cp->mc_bufsize, sizeof (u_int64_t)); u_int32_t flags = cp->mc_flags; void *buf, *base, **pbuf; @@ -910,26 +911,14 @@ mcache_slab_alloc(void *arg, mcache_obj_t ***plist, unsigned int num, int wait) *list = NULL; - /* - * The address of the object returned to the caller is an - * offset from the 64-bit aligned base address only if the - * cache's alignment requirement is neither 1 nor 8 bytes. - */ - if (cp->mc_align != 1 && cp->mc_align != sizeof (u_int64_t)) - offset = cp->mc_align; - for (;;) { - if (!(wait & MCR_NOSLEEP)) - buf = zalloc(cp->mc_slab_zone); - else - buf = zalloc_noblock(cp->mc_slab_zone); - + buf = zalloc(cp->mc_slab_zone); if (buf == NULL) break; - /* Get the 64-bit aligned base address for this object */ + /* Get the aligned base address for this object */ base = (void *)P2ROUNDUP((intptr_t)buf + sizeof (u_int64_t), - sizeof (u_int64_t)); + cp->mc_align); /* * Wind back a pointer size from the aligned base and @@ -938,6 +927,9 @@ mcache_slab_alloc(void *arg, mcache_obj_t ***plist, unsigned int num, int wait) pbuf = (void **)((intptr_t)base - sizeof (void *)); *pbuf = buf; + VERIFY (((intptr_t)base + cp->mc_bufsize) <= + ((intptr_t)buf + cp->mc_chunksize)); + /* * If auditing is enabled, patternize the contents of * the buffer starting from the 64-bit aligned base to @@ -951,14 +943,8 @@ mcache_slab_alloc(void *arg, mcache_obj_t ***plist, unsigned int num, int wait) mcache_set_pattern(MCACHE_FREE_PATTERN, base, rsize); } - /* - * Fix up the object's address to fulfill the cache's - * alignment requirement (if needed) and return this - * to the caller. - */ - VERIFY(((intptr_t)base + offset + cp->mc_bufsize) <= - ((intptr_t)buf + cp->mc_chunksize)); - *list = (mcache_obj_t *)((intptr_t)base + offset); + VERIFY(IS_P2ALIGNED(base, cp->mc_align)); + *list = (mcache_obj_t *)base; (*list)->obj_next = NULL; list = *plist = &(*list)->obj_next; @@ -979,40 +965,31 @@ mcache_slab_free(void *arg, mcache_obj_t *list, __unused boolean_t purged) { mcache_t *cp = arg; mcache_obj_t *nlist; - size_t offset = 0; size_t rsize = P2ROUNDUP(cp->mc_bufsize, sizeof (u_int64_t)); u_int32_t flags = cp->mc_flags; void *base; void **pbuf; - /* - * The address of the object is an offset from a 64-bit - * aligned base address only if the cache's alignment - * requirement is neither 1 nor 8 bytes. - */ - if (cp->mc_align != 1 && cp->mc_align != sizeof (u_int64_t)) - offset = cp->mc_align; - for (;;) { nlist = list->obj_next; list->obj_next = NULL; - /* Get the 64-bit aligned base address of this object */ - base = (void *)((intptr_t)list - offset); - VERIFY(IS_P2ALIGNED(base, sizeof (u_int64_t))); + base = list; + VERIFY(IS_P2ALIGNED(base, cp->mc_align)); /* Get the original address since we're about to free it */ pbuf = (void **)((intptr_t)base - sizeof (void *)); + VERIFY(((intptr_t)base + cp->mc_bufsize) <= + ((intptr_t)*pbuf + cp->mc_chunksize)); + if (flags & MCF_DEBUG) { VERIFY(((intptr_t)base + rsize) <= ((intptr_t)*pbuf + cp->mc_chunksize)); - mcache_audit_free_verify(NULL, base, offset, rsize); + mcache_audit_free_verify(NULL, base, 0, rsize); } /* Free it to zone */ - VERIFY(((intptr_t)base + offset + cp->mc_bufsize) <= - ((intptr_t)*pbuf + cp->mc_chunksize)); zfree(cp->mc_slab_zone, *pbuf); /* No more objects to free; return to mcache */ @@ -1028,24 +1005,14 @@ static void mcache_slab_audit(void *arg, mcache_obj_t *list, boolean_t alloc) { mcache_t *cp = arg; - size_t offset = 0; size_t rsize = P2ROUNDUP(cp->mc_bufsize, sizeof (u_int64_t)); void *base, **pbuf; - /* - * The address of the object returned to the caller is an - * offset from the 64-bit aligned base address only if the - * cache's alignment requirement is neither 1 nor 8 bytes. - */ - if (cp->mc_align != 1 && cp->mc_align != sizeof (u_int64_t)) - offset = cp->mc_align; - while (list != NULL) { mcache_obj_t *next = list->obj_next; - /* Get the 64-bit aligned base address of this object */ - base = (void *)((intptr_t)list - offset); - VERIFY(IS_P2ALIGNED(base, sizeof (u_int64_t))); + base = list; + VERIFY(IS_P2ALIGNED(base, cp->mc_align)); /* Get the original address */ pbuf = (void **)((intptr_t)base - sizeof (void *)); @@ -1056,7 +1023,7 @@ mcache_slab_audit(void *arg, mcache_obj_t *list, boolean_t alloc) if (!alloc) mcache_set_pattern(MCACHE_FREE_PATTERN, base, rsize); else - mcache_audit_free_verify_set(NULL, base, offset, rsize); + mcache_audit_free_verify_set(NULL, base, 0, rsize); list = list->obj_next = next; } @@ -1181,13 +1148,7 @@ mcache_bkt_purge(mcache_t *cp) mcache_bkt_destroy(cp, btp, pbp, pobjs); } - /* - * Updating the working set back to back essentially sets - * the working set size to zero, so everything is reapable. - */ - mcache_bkt_ws_update(cp); - mcache_bkt_ws_update(cp); - + mcache_bkt_ws_zero(cp); mcache_bkt_ws_reap(cp); } @@ -1246,6 +1207,22 @@ mcache_bkt_ws_update(mcache_t *cp) MCACHE_UNLOCK(&cp->mc_bkt_lock); } +/* + * Mark everything as eligible for reaping (working set is zero). + */ +static void +mcache_bkt_ws_zero(mcache_t *cp) +{ + MCACHE_LOCK(&cp->mc_bkt_lock); + + cp->mc_full.bl_reaplimit = cp->mc_full.bl_total; + cp->mc_full.bl_min = cp->mc_full.bl_total; + cp->mc_empty.bl_reaplimit = cp->mc_empty.bl_total; + cp->mc_empty.bl_min = cp->mc_empty.bl_total; + + MCACHE_UNLOCK(&cp->mc_bkt_lock); +} + /* * Reap all buckets that are beyond the working set. */ @@ -1314,6 +1291,18 @@ mcache_reap(void) mcache_dispatch(mcache_reap_start, flag); } +__private_extern__ void +mcache_reap_now(mcache_t *cp, boolean_t purge) +{ + if (purge) { + mcache_bkt_purge(cp); + mcache_cache_bkt_enable(cp); + } else { + mcache_bkt_ws_zero(cp); + mcache_bkt_ws_reap(cp); + } +} + static void mcache_cache_reap(mcache_t *cp) { diff --git a/bsd/kern/policy_check.c b/bsd/kern/policy_check.c index cbaf30ca4..85a92f5fe 100644 --- a/bsd/kern/policy_check.c +++ b/bsd/kern/policy_check.c @@ -13,6 +13,7 @@ #include <security/mac.h> #include <security/mac_policy.h> +#include <libkern/section_keywords.h> #include <libkern/OSDebug.h> /* OSBPrintBacktrace */ @@ -118,7 +119,7 @@ common_hook(void) return rv; } -#if (MAC_POLICY_OPS_VERSION != 47) +#if (MAC_POLICY_OPS_VERSION != 52) # error "struct mac_policy_ops doesn't match definition in mac_policy.h" #endif /* @@ -127,7 +128,7 @@ common_hook(void) * Please note that this struct initialization should be kept in sync with * security/mac_policy.h (mac_policy_ops struct definition). */ -static struct mac_policy_ops policy_ops = { +const static struct mac_policy_ops policy_ops = { CHECK_SET_HOOK(audit_check_postselect) CHECK_SET_HOOK(audit_check_preselect) @@ -275,7 +276,7 @@ static struct mac_policy_ops policy_ops = { CHECK_SET_HOOK(proc_check_set_host_exception_port) CHECK_SET_HOOK(exc_action_check_exception_send) CHECK_SET_HOOK(exc_action_label_associate) - CHECK_SET_HOOK(exc_action_label_copy) + CHECK_SET_HOOK(exc_action_label_populate) CHECK_SET_HOOK(exc_action_label_destroy) CHECK_SET_HOOK(exc_action_label_init) CHECK_SET_HOOK(exc_action_label_update) @@ -284,8 +285,8 @@ static struct mac_policy_ops policy_ops = { .mpo_reserved2 = (mpo_reserved_hook_t *)common_hook, .mpo_reserved3 = (mpo_reserved_hook_t *)common_hook, .mpo_reserved4 = (mpo_reserved_hook_t *)common_hook, - .mpo_reserved5 = (mpo_reserved_hook_t *)common_hook, - .mpo_reserved6 = (mpo_reserved_hook_t *)common_hook, + CHECK_SET_HOOK(skywalk_flow_check_connect) + CHECK_SET_HOOK(skywalk_flow_check_listen) CHECK_SET_HOOK(posixsem_check_create) CHECK_SET_HOOK(posixsem_check_open) @@ -363,7 +364,7 @@ static struct mac_policy_ops policy_ops = { CHECK_SET_HOOK(system_check_settime) CHECK_SET_HOOK(system_check_swapoff) CHECK_SET_HOOK(system_check_swapon) - .mpo_reserved7 = (mpo_reserved_hook_t *)common_hook, + CHECK_SET_HOOK(socket_check_ioctl) CHECK_SET_HOOK(sysvmsg_label_associate) CHECK_SET_HOOK(sysvmsg_label_destroy) @@ -396,7 +397,7 @@ static struct mac_policy_ops policy_ops = { CHECK_SET_HOOK(sysvshm_label_init) CHECK_SET_HOOK(sysvshm_label_recycle) - .mpo_reserved8 = (mpo_reserved_hook_t *)common_hook, + CHECK_SET_HOOK(proc_notify_exit) CHECK_SET_HOOK(mount_check_snapshot_revert) CHECK_SET_HOOK(vnode_check_getattr) CHECK_SET_HOOK(mount_check_snapshot_create) @@ -493,7 +494,7 @@ static struct mac_policy_ops policy_ops = { CHECK_SET_HOOK(system_check_kas_info) - CHECK_SET_HOOK(proc_check_cpumon) + CHECK_SET_HOOK(vnode_check_lookup_preflight) CHECK_SET_HOOK(vnode_notify_open) @@ -519,7 +520,7 @@ static struct mac_policy_ops policy_ops = { /* * Policy definition */ -static struct mac_policy_conf policy_conf = { +static SECURITY_READ_ONLY_LATE(struct mac_policy_conf) policy_conf = { .mpc_name = "CHECK", .mpc_fullname = "Check Assumptions Policy", .mpc_field_off = NULL, /* no label slot */ @@ -530,7 +531,7 @@ static struct mac_policy_conf policy_conf = { .mpc_runtime_flags = 0, }; -static mac_policy_handle_t policy_handle; +static SECURITY_READ_ONLY_LATE(mac_policy_handle_t) policy_handle; /* * Init routine; for a loadable policy, this would be called during the KEXT diff --git a/bsd/kern/posix_sem.c b/bsd/kern/posix_sem.c index ca7ee4c6a..9dc882363 100644 --- a/bsd/kern/posix_sem.c +++ b/bsd/kern/posix_sem.c @@ -168,20 +168,21 @@ struct psemstats psemstats; /* cache effectiveness statistics */ static int psem_access(struct pseminfo *pinfo, int mode, kauth_cred_t cred); static int psem_cache_search(struct pseminfo **, - struct psemname *, struct psemcache **); + struct psemname *, struct psemcache **); static int psem_delete(struct pseminfo * pinfo); static int psem_read (struct fileproc *fp, struct uio *uio, - int flags, vfs_context_t ctx); + int flags, vfs_context_t ctx); static int psem_write (struct fileproc *fp, struct uio *uio, - int flags, vfs_context_t ctx); + int flags, vfs_context_t ctx); static int psem_ioctl (struct fileproc *fp, u_long com, - caddr_t data, vfs_context_t ctx); + caddr_t data, vfs_context_t ctx); static int psem_select (struct fileproc *fp, int which, void *wql, vfs_context_t ctx); static int psem_closefile (struct fileglob *fp, vfs_context_t ctx); static int psem_unlink_internal(struct pseminfo *pinfo, struct psemcache *pcache); -static int psem_kqfilter (struct fileproc *fp, struct knote *kn, vfs_context_t ctx); +static int psem_kqfilter (struct fileproc *fp, struct knote *kn, + struct kevent_internal_s *kev, vfs_context_t ctx); static const struct fileops psemops = { .fo_type = DTYPE_PSXSEM, @@ -1093,36 +1094,36 @@ psem_delete(struct pseminfo * pinfo) } static int -psem_read(__unused struct fileproc *fp, __unused struct uio *uio, - __unused int flags, __unused vfs_context_t ctx) +psem_read(__unused struct fileproc *fp, __unused struct uio *uio, + __unused int flags, __unused vfs_context_t ctx) { return(ENOTSUP); } static int -psem_write(__unused struct fileproc *fp, __unused struct uio *uio, - __unused int flags, __unused vfs_context_t ctx) +psem_write(__unused struct fileproc *fp, __unused struct uio *uio, + __unused int flags, __unused vfs_context_t ctx) { return(ENOTSUP); } static int -psem_ioctl(__unused struct fileproc *fp, __unused u_long com, - __unused caddr_t data, __unused vfs_context_t ctx) +psem_ioctl(__unused struct fileproc *fp, __unused u_long com, + __unused caddr_t data, __unused vfs_context_t ctx) { return(ENOTSUP); } static int -psem_select(__unused struct fileproc *fp, __unused int which, - __unused void *wql, __unused vfs_context_t ctx) +psem_select(__unused struct fileproc *fp, __unused int which, + __unused void *wql, __unused vfs_context_t ctx) { return(ENOTSUP); } static int -psem_kqfilter(__unused struct fileproc *fp, struct knote *kn, - __unused vfs_context_t ctx) +psem_kqfilter(__unused struct fileproc *fp, struct knote *kn, + __unused struct kevent_internal_s *kev, __unused vfs_context_t ctx) { kn->kn_flags = EV_ERROR; kn->kn_data = ENOTSUP; diff --git a/bsd/kern/posix_shm.c b/bsd/kern/posix_shm.c index 2ddd346bc..169eba5c5 100644 --- a/bsd/kern/posix_shm.c +++ b/bsd/kern/posix_shm.c @@ -176,16 +176,17 @@ long pshmnument; /* number of cache entries allocated */ struct pshmstats pshmstats; /* cache effectiveness statistics */ static int pshm_read (struct fileproc *fp, struct uio *uio, - int flags, vfs_context_t ctx); + int flags, vfs_context_t ctx); static int pshm_write (struct fileproc *fp, struct uio *uio, - int flags, vfs_context_t ctx); + int flags, vfs_context_t ctx); static int pshm_ioctl (struct fileproc *fp, u_long com, - caddr_t data, vfs_context_t ctx); + caddr_t data, vfs_context_t ctx); static int pshm_select (struct fileproc *fp, int which, void *wql, vfs_context_t ctx); static int pshm_close(struct pshminfo *pinfo, int dropref); static int pshm_closefile (struct fileglob *fg, vfs_context_t ctx); -static int pshm_kqfilter(struct fileproc *fp, struct knote *kn, vfs_context_t ctx); +static int pshm_kqfilter(struct fileproc *fp, struct knote *kn, + struct kevent_internal_s *kev, vfs_context_t ctx); int pshm_access(struct pshminfo *pinfo, int mode, kauth_cred_t cred, proc_t p); int pshm_cache_purge_all(proc_t p); @@ -869,6 +870,7 @@ pshm_mmap(__unused proc_t p, struct mmap_args *uap, user_addr_t *retval, struct vm_object_offset_t map_pos; vm_map_t user_map; int alloc_flags; + vm_map_kernel_flags_t vmk_flags; boolean_t docow; kern_return_t kret; struct pshminfo * pinfo; @@ -945,13 +947,15 @@ pshm_mmap(__unused proc_t p, struct mmap_args *uap, user_addr_t *retval, struct docow = FALSE; mapped_size = 0; - - /* reserver the entire space first... */ + vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; + /* reserve the entire space first... */ kret = vm_map_enter_mem_object(user_map, &user_addr, user_size, 0, alloc_flags, + vmk_flags, + VM_KERN_MEMORY_NONE, IPC_PORT_NULL, 0, FALSE, @@ -978,12 +982,15 @@ pshm_mmap(__unused proc_t p, struct mmap_args *uap, user_addr_t *retval, struct if (map_size > user_size) { map_size = user_size; } + vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; kret = vm_map_enter_mem_object( user_map, &user_addr, map_size, 0, VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE, + vmk_flags, + VM_KERN_MEMORY_NONE, pshmobj->pshmo_memobject, file_pos - map_pos, docow, @@ -1255,36 +1262,36 @@ pshm_closefile(struct fileglob *fg, __unused vfs_context_t ctx) } static int -pshm_read(__unused struct fileproc *fp, __unused struct uio *uio, - __unused int flags, __unused vfs_context_t ctx) +pshm_read(__unused struct fileproc *fp, __unused struct uio *uio, + __unused int flags, __unused vfs_context_t ctx) { return(ENOTSUP); } static int -pshm_write(__unused struct fileproc *fp, __unused struct uio *uio, - __unused int flags, __unused vfs_context_t ctx) +pshm_write(__unused struct fileproc *fp, __unused struct uio *uio, + __unused int flags, __unused vfs_context_t ctx) { return(ENOTSUP); } static int -pshm_ioctl(__unused struct fileproc *fp, __unused u_long com, - __unused caddr_t data, __unused vfs_context_t ctx) +pshm_ioctl(__unused struct fileproc *fp, __unused u_long com, + __unused caddr_t data, __unused vfs_context_t ctx) { return(ENOTSUP); } static int -pshm_select(__unused struct fileproc *fp, __unused int which, __unused void *wql, - __unused vfs_context_t ctx) +pshm_select(__unused struct fileproc *fp, __unused int which, __unused void *wql, + __unused vfs_context_t ctx) { return(ENOTSUP); } static int pshm_kqfilter(__unused struct fileproc *fp, struct knote *kn, - __unused vfs_context_t ctx) + __unused struct kevent_internal_s *kev, __unused vfs_context_t ctx) { kn->kn_flags = EV_ERROR; kn->kn_data = ENOTSUP; diff --git a/bsd/kern/proc_info.c b/bsd/kern/proc_info.c index e0707e35a..9efb0b1ca 100644 --- a/bsd/kern/proc_info.c +++ b/bsd/kern/proc_info.c @@ -84,7 +84,7 @@ #include <vm/vm_protos.h> -/* Needed by proc_pidnoteexit() */ +/* Needed by proc_pidnoteexit(), proc_pidlistuptrs() */ #include <sys/event.h> #include <sys/codesign.h> @@ -93,6 +93,10 @@ #include <sys/coalition.h> #endif +#if CONFIG_MACF +#include <security/mac_framework.h> +#endif + struct pshmnode; struct psemnode; struct pipe; @@ -101,6 +105,7 @@ struct atalk; uint64_t get_dispatchqueue_offset_from_proc(void *); uint64_t get_dispatchqueue_serialno_offset_from_proc(void *); +uint64_t get_return_to_kernel_offset_from_proc(void *p); int proc_info_internal(int callnum, int pid, int flavor, uint64_t arg, user_addr_t buffer, uint32_t buffersize, int32_t * retval); /* @@ -166,7 +171,8 @@ void __attribute__ ((noinline)) proc_pidcoalitioninfo(proc_t p, struct proc_pidc int __attribute__ ((noinline)) proc_pidnoteexit(proc_t p, uint64_t arg, uint32_t *data); int __attribute__ ((noinline)) proc_pidexitreasoninfo(proc_t p, struct proc_exitreasoninfo *peri, struct proc_exitreasonbasicinfo *pberi); int __attribute__ ((noinline)) proc_pidoriginatorpid_uuid(uuid_t uuid, uint32_t buffersize, pid_t *pid); - +int __attribute__ ((noinline)) proc_pidlistuptrs(proc_t p, user_addr_t buffer, uint32_t buffersize, int32_t *retval); +int __attribute__ ((noinline)) proc_piddynkqueueinfo(pid_t pid, int flavor, kqueue_id_t id, user_addr_t buffer, uint32_t buffersize, int32_t *retval); /* protos for proc_pidfdinfo calls */ int __attribute__ ((noinline)) pid_vnodeinfo(vnode_t vp, uint32_t vid, struct fileproc * fp,proc_t proc, int fd, user_addr_t buffer, uint32_t buffersize, int32_t * retval); @@ -187,9 +193,6 @@ int proc_security_policy(proc_t targetp, int callnum, int flavor, boolean_t chec static void munge_vinfo_stat(struct stat64 *sbp, struct vinfo_stat *vsbp); static int proc_piduuidinfo(pid_t pid, uuid_t uuid_buf, uint32_t buffersize); int proc_pidpathinfo_internal(proc_t p, __unused uint64_t arg, char *buf, uint32_t buffersize, __unused int32_t *retval); -int proc_listfd_kqueue(proc_t p, int32_t *fdlist, int len); -int proc_kqueue_udata_info(proc_t p, int32_t fd, uint64_t *buffer, int bufsize); -int proc_list_uptrs(proc_t p, uint64_t *udata_buffer, int size); extern int cansignal(struct proc *, kauth_cred_t, struct proc *, int, int); extern int proc_get_rusage(proc_t proc, int flavor, user_addr_t buffer, int is_zombie); @@ -217,6 +220,16 @@ uint64_t get_dispatchqueue_serialno_offset_from_proc(void *p) } } +uint64_t get_return_to_kernel_offset_from_proc(void *p) +{ + if (p != NULL) { + proc_t pself = (proc_t)p; + return (pself->p_return_to_kernel_offset); + } else { + return (uint64_t)0; + } +} + /***************************** proc_info ********************/ int @@ -257,8 +270,10 @@ proc_info_internal(int callnum, int pid, int flavor, uint64_t arg, user_addr_t b buffersize, retval); case PROC_INFO_CALL_CANUSEFGHW: return proc_can_use_foreground_hw(pid, buffer, buffersize, retval); + case PROC_INFO_CALL_PIDDYNKQUEUEINFO: + return proc_piddynkqueueinfo(pid, flavor, (kqueue_id_t)arg, buffer, buffersize, retval); default: - return(EINVAL); + return EINVAL; } return(EINVAL); @@ -268,10 +283,12 @@ proc_info_internal(int callnum, int pid, int flavor, uint64_t arg, user_addr_t b int proc_listpids(uint32_t type, uint32_t typeinfo, user_addr_t buffer, uint32_t buffersize, int32_t * retval) { - int numprocs, wantpids; + uint32_t numprocs = 0; + uint32_t wantpids; char * kbuf; int * ptr; - int n, skip; + uint32_t n; + int skip; struct proc * p; struct tty * tp; int error = 0; @@ -283,7 +300,7 @@ proc_listpids(uint32_t type, uint32_t typeinfo, user_addr_t buffer, uint32_t bu /* if the buffer is null, return num of procs */ if (buffer == (user_addr_t)0) { - *retval = ((nprocs+20) * sizeof(int)); + *retval = ((nprocs + 20) * sizeof(int)); return(0); } @@ -291,13 +308,17 @@ proc_listpids(uint32_t type, uint32_t typeinfo, user_addr_t buffer, uint32_t bu return(ENOMEM); } wantpids = buffersize/sizeof(int); - numprocs = nprocs+20; - if (numprocs > wantpids) + if ((nprocs + 20) > 0) { + numprocs = (uint32_t)(nprocs + 20); + } + if (numprocs > wantpids) { numprocs = wantpids; + } kbuf = (char *)kalloc((vm_size_t)(numprocs * sizeof(int))); - if (kbuf == NULL) + if (kbuf == NULL) { return(ENOMEM); + } bzero(kbuf, sizeof(int)); proc_list_lock(); @@ -358,6 +379,11 @@ proc_loop: skip = 1; } break; + case PROC_KDBG_ONLY: + if (p->p_kdebug == 0) { + skip = 1; + } + break; default: skip = 1; break; @@ -393,7 +419,8 @@ proc_loop: int proc_pidfdlist(proc_t p, user_addr_t buffer, uint32_t buffersize, int32_t *retval) { - int numfds, needfds; + uint32_t numfds = 0; + uint32_t needfds; char * kbuf; struct proc_fdinfo * pfd; struct fileproc * fp; @@ -401,7 +428,9 @@ proc_pidfdlist(proc_t p, user_addr_t buffer, uint32_t buffersize, int32_t *retv int count = 0; int error = 0; - numfds = p->p_fd->fd_nfiles; + if (p->p_fd->fd_nfiles > 0) { + numfds = (uint32_t)p->p_fd->fd_nfiles; + } if (buffer == (user_addr_t) 0) { numfds += 20; @@ -412,8 +441,9 @@ proc_pidfdlist(proc_t p, user_addr_t buffer, uint32_t buffersize, int32_t *retv /* buffersize is big enough atleast for one struct */ needfds = buffersize/sizeof(struct proc_fdinfo); - if (numfds > needfds) + if (numfds > needfds) { numfds = needfds; + } kbuf = (char *)kalloc((vm_size_t)(numfds * sizeof(struct proc_fdinfo))); if (kbuf == NULL) @@ -424,7 +454,7 @@ proc_pidfdlist(proc_t p, user_addr_t buffer, uint32_t buffersize, int32_t *retv pfd = (struct proc_fdinfo *)kbuf; - for (n = 0; ((n < numfds) && (n < p->p_fd->fd_nfiles)); n++) { + for (n = 0; ((n < (int)numfds) && (n < p->p_fd->fd_nfiles)); n++) { if (((fp = p->p_fd->fd_ofiles[n]) != 0) && ((p->p_fd->fd_ofileflags[n] & UF_RESERVED) == 0)) { file_type_t fdtype = FILEGLOB_DTYPE(fp->f_fglob); @@ -621,8 +651,10 @@ proc_pidbsdinfo(proc_t p, struct proc_bsdinfo * pbsd, int zombie) pbsd->pbi_flags |= PROC_FLAG_CTTY; } +#if !CONFIG_EMBEDDED if ((p->p_flag & P_DELAYIDLESLEEP) == P_DELAYIDLESLEEP) pbsd->pbi_flags |= PROC_FLAG_DELAYIDLESLEEP; +#endif /* !CONFIG_EMBEDDED */ switch(PROC_CONTROL_STATE(p)) { case P_PCTHROTTLE: @@ -700,8 +732,10 @@ proc_pidshortbsdinfo(proc_t p, struct proc_bsdshortinfo * pbsd_shortp, int zombi pbsd_shortp->pbsi_flags |= PROC_FLAG_PSUGID; if ((p->p_flag & P_EXEC) == P_EXEC) pbsd_shortp->pbsi_flags |= PROC_FLAG_EXEC; +#if !CONFIG_EMBEDDED if ((p->p_flag & P_DELAYIDLESLEEP) == P_DELAYIDLESLEEP) pbsd_shortp->pbsi_flags |= PROC_FLAG_DELAYIDLESLEEP; +#endif /* !CONFIG_EMBEDDED */ switch(PROC_CONTROL_STATE(p)) { case P_PCTHROTTLE: @@ -886,20 +920,22 @@ proc_pidthreadpathinfo(proc_t p, uint64_t arg, struct proc_threadwithpathinfo * int proc_pidlistthreads(proc_t p, user_addr_t buffer, uint32_t buffersize, int32_t *retval) { - int count = 0; + uint32_t count = 0; int ret = 0; int error = 0; void * kbuf; - int numthreads; + uint32_t numthreads = 0; - - count = buffersize/(sizeof(uint64_t)); - numthreads = get_numthreads(p->task); + int num = get_numthreads(p->task) + 10; + if (num > 0) { + numthreads = (uint32_t)num; + } - numthreads += 10; + count = buffersize/(sizeof(uint64_t)); - if (numthreads > count) + if (numthreads > count) { numthreads = count; + } kbuf = (void *)kalloc(numthreads * sizeof(uint64_t)); if (kbuf == NULL) @@ -1647,7 +1683,7 @@ proc_pidinfo(int pid, int flavor, uint64_t arg, user_addr_t buffer, uint32_t bu switch (flavor) { case PROC_PIDLISTFDS: size = PROC_PIDLISTFD_SIZE; - if (buffer == (user_addr_t)0) + if (buffer == USER_ADDR_NULL) size = 0; break; case PROC_PIDTBSDINFO: @@ -1732,6 +1768,18 @@ proc_pidinfo(int pid, int flavor, uint64_t arg, user_addr_t buffer, uint32_t bu case PROC_PIDREGIONPATHINFO3: size = PROC_PIDREGIONPATHINFO3_SIZE; break; + case PROC_PIDLISTUPTRS: + size = PROC_PIDLISTUPTRS_SIZE; + if (buffer == USER_ADDR_NULL) { + size = 0; + } + break; + case PROC_PIDLISTDYNKQUEUES: + size = PROC_PIDLISTDYNKQUEUES_SIZE; + if (buffer == USER_ADDR_NULL) { + size = 0; + } + break; default: return(EINVAL); } @@ -2007,8 +2055,17 @@ proc_pidinfo(int pid, int flavor, uint64_t arg, user_addr_t buffer, uint32_t bu } break; + case PROC_PIDLISTUPTRS: + error = proc_pidlistuptrs(p, buffer, buffersize, retval); + break; + + case PROC_PIDLISTDYNKQUEUES: + error = kevent_copyout_proc_dynkqids(p, buffer, buffersize, retval); + break; + default: error = ENOTSUP; + break; } out: @@ -2356,7 +2413,7 @@ proc_pidfdinfo(int pid, int flavor, int fd, user_addr_t buffer, uint32_t buffer struct kqueue * kq; if (fd == -1) { - if ((kq = p->p_wqkqueue) == NULL) { + if ((kq = p->p_fd->fd_wqkqueue) == NULL) { /* wqkqueue is initialized on-demand */ error = 0; break; @@ -2374,7 +2431,7 @@ proc_pidfdinfo(int pid, int flavor, int fd, user_addr_t buffer, uint32_t buffer struct kqueue * kq; if (fd == -1) { - if ((kq = p->p_wqkqueue) == NULL) { + if ((kq = p->p_fd->fd_wqkqueue) == NULL) { /* wqkqueue is initialized on-demand */ error = 0; break; @@ -2401,119 +2458,53 @@ out: return(error); } -int -proc_listfd_kqueue(proc_t p, int32_t *fdlist, int len) -{ - int numfds; - struct fileproc * fp; - int n; - int count = 0; - - numfds = p->p_fd->fd_nfiles; - if (len < numfds) { - return -1; - } - - proc_fdlock(p); - for (n = 0; ((n < numfds) && (n < p->p_fd->fd_nfiles)); n++) { - if (((fp = p->p_fd->fd_ofiles[n]) != 0) - && ((p->p_fd->fd_ofileflags[n] & UF_RESERVED) == 0) - && (FILEGLOB_DTYPE(fp->f_fglob) == PROX_FDTYPE_KQUEUE)) { - fdlist[count++] = n; - } - } - proc_fdunlock(p); - return count; -} +#define MAX_UPTRS 16392 int -proc_kqueue_udata_info(proc_t p, int32_t fd, uint64_t *buffer, int bufsize) +proc_pidlistuptrs(proc_t p, user_addr_t buffer, uint32_t buffersize, int32_t *retval) { - struct kqueue *kq; - struct fileproc * fp = NULL; - int retval; + uint32_t count = 0; + int error = 0; + void *kbuf = NULL; + int32_t nuptrs = 0; - if (fd == -1) { - /* wqkqueue is initialized on-demand */ - if ((kq = p->p_wqkqueue) == NULL) { - return 0; + if (buffer != USER_ADDR_NULL) { + count = buffersize / sizeof(uint64_t); + if (count > MAX_UPTRS) { + count = MAX_UPTRS; + buffersize = count * sizeof(uint64_t); } - } else { - int error = fp_getfkq(p, fd, &fp, &kq); - if (error != 0) { - return 0; + if (count > 0) { + kbuf = kalloc(buffersize); + assert(kbuf != NULL); } - } - - retval = pid_kqueue_udatainfo(p, kq, buffer, bufsize); - if (fp) { - fp_drop(p, fd, fp , 0); - } - - return retval; -} - -int -proc_list_uptrs(proc_t p, uint64_t *udata_buffer, int size) -{ - int32_t *fdlist = NULL; - int nfds; - int i; - int count = 0; - int ret; - int knote_max = 4096; - uint64_t *buffer; - int bufsize = knote_max * sizeof(uint64_t); - - fdlist = (int32_t *)kalloc((OPEN_MAX + 1) * sizeof(int32_t)); - if (!fdlist) { - return -1; - } - - nfds = proc_listfd_kqueue(p, &fdlist[1], OPEN_MAX); - if (nfds < 0 || nfds > OPEN_MAX) { - kfree(fdlist, (OPEN_MAX + 1) * sizeof(int32_t)); - return 0; - } - - /* Add FD -1, the implicit workq kqueue */ - fdlist[0] = -1; - nfds++; - - if (size == 0) { - bufsize = 0; - buffer = NULL; } else { - bufsize = knote_max * sizeof(uint64_t); - buffer = (uint64_t *)kalloc(bufsize); + buffersize = 0; } - for (i = 0; i < nfds; i++) { -again: - ret = proc_kqueue_udata_info(p, fdlist[i], buffer, bufsize); - if (bufsize != 0 && ret > knote_max) { - kfree(buffer, bufsize); - knote_max = ret + 32; - bufsize = knote_max * sizeof(uint64_t); - buffer = kalloc(bufsize); - goto again; - } - - if (ret == 0) - continue; + nuptrs = kevent_proc_copy_uptrs(p, kbuf, buffersize); - /* Copy the udata ptrs */ - if (size >= (int)((count + ret) * sizeof(uint64_t))) { - memcpy(&udata_buffer[count], buffer, ret * sizeof(uint64_t)); + if (kbuf) { + size_t copysize; + if (os_mul_overflow(nuptrs, sizeof(uint64_t), ©size)) { + error = ERANGE; + goto out; + } + if (copysize > buffersize) { + copysize = buffersize; } - count = count + ret; + error = copyout(kbuf, buffer, copysize); } - kfree(fdlist, (OPEN_MAX + 1) * sizeof(int32_t)); - if (buffer) { - kfree(buffer, bufsize); +out: + *retval = nuptrs; + + if (kbuf) { + kfree(kbuf, buffersize); + kbuf = NULL; } - return count; + + return error; } /* @@ -3174,3 +3165,41 @@ proc_pidnoteexit(proc_t p, uint64_t flags, uint32_t *data) return (0); } +int +proc_piddynkqueueinfo(int pid, int flavor, kqueue_id_t kq_id, + user_addr_t ubuf, uint32_t bufsize, int32_t *retval) +{ + proc_t p; + int err; + + if (ubuf == USER_ADDR_NULL) { + return EFAULT; + } + + p = proc_find(pid); + if (p == PROC_NULL) { + return ESRCH; + } + + err = proc_security_policy(p, PROC_INFO_CALL_PIDDYNKQUEUEINFO, 0, CHECK_SAME_USER); + if (err) { + goto out; + } + + switch (flavor) { + case PROC_PIDDYNKQUEUE_INFO: + err = kevent_copyout_dynkqinfo(p, kq_id, ubuf, bufsize, retval); + break; + case PROC_PIDDYNKQUEUE_EXTINFO: + err = kevent_copyout_dynkqextinfo(p, kq_id, ubuf, bufsize, retval); + break; + default: + err = ENOTSUP; + break; + } + +out: + proc_rele(p); + + return err; +} diff --git a/bsd/kern/process_policy.c b/bsd/kern/process_policy.c index bcf69d7c8..0a0b43e66 100644 --- a/bsd/kern/process_policy.c +++ b/bsd/kern/process_policy.c @@ -34,6 +34,7 @@ #include <sys/systm.h> #include <sys/kernel.h> #include <sys/malloc.h> +#include <sys/priv.h> #include <sys/proc_internal.h> #include <sys/proc.h> #include <sys/kauth.h> @@ -43,8 +44,6 @@ #include <sys/vm.h> #include <sys/user.h> -#include <security/audit/audit.h> - #include <mach/machine.h> #include <mach/mach_types.h> #include <mach/vm_param.h> @@ -72,6 +71,15 @@ #include <kern/ipc_misc.h> #include <vm/vm_protos.h> +#if CONFIG_EMBEDDED +#include <sys/kern_memorystatus.h> +#endif /* CONFIG_EMBEDDED */ + +#if CONFIG_MACF +#include <security/mac.h> +#include <security/mac_framework.h> +#endif /* CONFIG_MACF */ + static int handle_lowresource(int scope, int action, int policy, int policy_subtype, user_addr_t attrp, proc_t proc, uint64_t target_threadid); static int handle_cpuuse(int action, user_addr_t attrp, proc_t proc, uint64_t target_threadid); static int handle_apptype(int scope, int action, int policy, int policy_subtype, user_addr_t attrp, proc_t proc, uint64_t target_threadid); @@ -80,6 +88,9 @@ static int handle_boost(int scope, int action, int policy, int policy_subtype, u extern kern_return_t task_suspend(task_t); extern kern_return_t task_resume(task_t); +#if CONFIG_EMBEDDED +static int handle_applifecycle(int scope, int action, int policy, int policy_subtype, user_addr_t attrp, proc_t proc, uint64_t target_threadid); +#endif /* CONFIG_EMBEDDED */ /***************************** process_policy ********************/ @@ -104,8 +115,13 @@ process_policy(__unused struct proc *p, struct process_policy_args * uap, __unus pid_t target_pid = uap->target_pid; uint64_t target_threadid = uap->target_threadid; proc_t target_proc = PROC_NULL; +#if CONFIG_MACF || !CONFIG_EMBEDDED proc_t curp = current_proc(); +#endif kauth_cred_t my_cred; +#if CONFIG_EMBEDDED + kauth_cred_t target_cred; +#endif if ((scope != PROC_POLICY_SCOPE_PROCESS) && (scope != PROC_POLICY_SCOPE_THREAD)) { return(EINVAL); @@ -121,6 +137,13 @@ process_policy(__unused struct proc *p, struct process_policy_args * uap, __unus my_cred = kauth_cred_get(); +#if CONFIG_EMBEDDED + target_cred = kauth_cred_proc_ref(target_proc); + + if (!kauth_cred_issuser(my_cred) && kauth_cred_getruid(my_cred) && + kauth_cred_getuid(my_cred) != kauth_cred_getuid(target_cred) && + kauth_cred_getruid(my_cred) != kauth_cred_getuid(target_cred)) +#else /* * Resoure starvation control can be used by unpriv resource owner but priv at the time of ownership claim. This is * checked in low resource handle routine. So bypass the checks here. @@ -128,6 +151,7 @@ process_policy(__unused struct proc *p, struct process_policy_args * uap, __unus if ((policy != PROC_POLICY_RESOURCE_STARVATION) && (policy != PROC_POLICY_APPTYPE) && (!kauth_cred_issuser(my_cred) && curp != p)) +#endif { error = EPERM; goto out; @@ -137,6 +161,10 @@ process_policy(__unused struct proc *p, struct process_policy_args * uap, __unus switch (policy) { case PROC_POLICY_BOOST: case PROC_POLICY_RESOURCE_USAGE: +#if CONFIG_EMBEDDED + case PROC_POLICY_APPTYPE: + case PROC_POLICY_APP_LIFECYCLE: +#endif /* These policies do their own appropriate mac checks */ break; default: @@ -175,6 +203,11 @@ process_policy(__unused struct proc *p, struct process_policy_args * uap, __unus error = handle_cpuuse(action, attrp, target_proc, target_threadid); break; +#if CONFIG_EMBEDDED + case PROC_POLICY_APP_LIFECYCLE: + error = handle_applifecycle(scope, action, policy, policy_subtype, attrp, target_proc, target_threadid); + break; +#endif /* CONFIG_EMBEDDED */ case PROC_POLICY_APPTYPE: error = handle_apptype(scope, action, policy, policy_subtype, attrp, target_proc, target_threadid); break; @@ -188,6 +221,9 @@ process_policy(__unused struct proc *p, struct process_policy_args * uap, __unus out: proc_rele(target_proc); +#if CONFIG_EMBEDDED + kauth_cred_unref(&target_cred); +#endif return(error); } @@ -217,40 +253,42 @@ static int handle_cpuuse(int action, user_addr_t attrp, proc_t proc, __unused uint64_t target_threadid) { proc_policy_cpuusage_attr_t cpuattr; -#if CONFIG_MACF +#if CONFIG_MACF || !CONFIG_EMBEDDED proc_t curp = current_proc(); #endif - int entitled = FALSE; + Boolean privileged = FALSE; Boolean canEnable = FALSE; uint64_t interval = -1ULL; int error = 0; uint8_t percentage; +#if !CONFIG_EMBEDDED + /* On macOS, tasks can only set and clear their own CPU limits. */ + if ((action == PROC_POLICY_ACTION_APPLY || action == PROC_POLICY_ACTION_RESTORE) + && curp != proc) { + return (EPERM); + } + /* No privilege required on macOS. */ + privileged = TRUE; +#endif + #if CONFIG_MACF - /* - * iOS only allows processes to override their own CPU usage monitor - * parameters if they have com.apple.private.kernel.override-cpumon. - * - * Until rdar://24799462 improves our scheme, we are also using the - * same entitlement to indicate which processes can resume monitoring - * when they otherwise wouldn't be able to. - */ - entitled = (mac_proc_check_cpumon(curp) == 0) ? TRUE : FALSE; - canEnable = (entitled && action == PROC_POLICY_ACTION_ENABLE); + /* Is caller privileged to set less-restrictive scheduling parameters? */ + if (!privileged) { + privileged = (priv_check_cred(kauth_cred_get(), PRIV_PROC_CPUMON_OVERRIDE, 0) == 0); + } + canEnable = (privileged && action == PROC_POLICY_ACTION_ENABLE); if (!canEnable && curp != proc) { - /* can the current process change scheduling parameters? */ + /* + * Can the current process change scheduling parameters for + * the target process? + */ error = mac_proc_check_sched(curp, proc); if (error) return error; } #endif - // on macOS tasks can only set and clear their own CPU limits - if ((action == PROC_POLICY_ACTION_APPLY || action == PROC_POLICY_ACTION_RESTORE) - && proc != current_proc()) { - return (EPERM); - } - switch (action) { case PROC_POLICY_ACTION_GET: error = proc_get_task_ruse_cpu(proc->task, &cpuattr.ppattr_cpu_attr, @@ -286,12 +324,12 @@ handle_cpuuse(int action, user_addr_t attrp, proc_t proc, __unused uint64_t targ cpuattr.ppattr_cpu_percentage, interval, cpuattr.ppattr_cpu_attr_deadline, - entitled); + privileged); break; /* restore process to prior state */ case PROC_POLICY_ACTION_RESTORE: - error = proc_clear_task_ruse_cpu(proc->task, entitled); + error = proc_clear_task_ruse_cpu(proc->task, privileged); break; /* re-enable suspended monitor */ @@ -310,6 +348,78 @@ handle_cpuuse(int action, user_addr_t attrp, proc_t proc, __unused uint64_t targ return(error); } +#if CONFIG_EMBEDDED +static int +handle_applifecycle(__unused int scope, + int action, + __unused int policy, + int policy_subtype, + user_addr_t attrp, + proc_t proc, + uint64_t target_threadid) +{ + int error = 0; + int state = 0; + + switch(policy_subtype) { + case PROC_POLICY_APPLIFE_NONE: + error = 0; + break; + + case PROC_POLICY_APPLIFE_STATE: + /* appstate is no longer supported */ + error = ENOTSUP; + break; + + case PROC_POLICY_APPLIFE_DEVSTATUS: +#if CONFIG_MACF + /* ToDo - this should be a generic check, since we could potentially hang other behaviours here. */ + error = mac_proc_check_suspend_resume(current_proc(), MAC_PROC_CHECK_HIBERNATE); + if (error) { + error = EPERM; + goto out; + } +#endif +#if CONFIG_MEMORYSTATUS + if (action == PROC_POLICY_ACTION_APPLY) { + /* Used as a freeze hint */ + memorystatus_on_inactivity(proc); + + /* in future use devicestatus for pid_socketshutdown() */ + error = 0; + } else +#endif + { + error = EINVAL; + } + break; + + case PROC_POLICY_APPLIFE_PIDBIND: +#if CONFIG_MACF + error = mac_proc_check_suspend_resume(current_proc(), MAC_PROC_CHECK_PIDBIND); + if (error) { + error = EPERM; + goto out; + } +#endif + error = copyin((user_addr_t)attrp, (int *)&state, sizeof(int)); + if (error != 0) + goto out; + if (action == PROC_POLICY_ACTION_APPLY) { + /* bind the thread in target_thread in current process to target_proc */ + error = proc_lf_pidbind(current_task(), target_threadid, proc->task, state); + } else + error = EINVAL; + break; + default: + error = EINVAL; + break; + } + +out: + return(error); +} +#endif /* CONFIG_EMBEDDED */ static int handle_apptype( int scope, @@ -499,7 +609,7 @@ proc_get_originatorbgstate(uint32_t *is_backgrounded) { uint32_t bgstate; proc_t p = current_proc(); - uint32_t flagsp; + uint32_t flagsp = 0; kern_return_t kr; pid_t pid; int ret; diff --git a/bsd/kern/pthread_shims.c b/bsd/kern/pthread_shims.c index 76e76c957..c8e42fc8c 100644 --- a/bsd/kern/pthread_shims.c +++ b/bsd/kern/pthread_shims.c @@ -28,6 +28,7 @@ #define PTHREAD_INTERNAL 1 +#include <stdatomic.h> #include <kern/debug.h> #include <kern/mach_param.h> #include <kern/sched_prim.h> @@ -41,7 +42,9 @@ #include <mach/task.h> #include <mach/thread_act.h> #include <sys/param.h> +#include <sys/eventvar.h> #include <sys/pthread_shims.h> +#include <sys/proc_info.h> #include <sys/proc_internal.h> #include <sys/sysproto.h> #include <sys/systm.h> @@ -53,7 +56,11 @@ #define PTHREAD_SHIMS_VERSION 1 /* on arm, the callbacks function has two #ifdef arm ponters */ +#if defined(__arm__) +#define PTHREAD_CALLBACK_MEMBER map_is_1gb +#else #define PTHREAD_CALLBACK_MEMBER ml_get_max_cpus +#endif /* compile time asserts to check the length of structures in pthread_shims.h */ static_assert((sizeof(struct pthread_functions_s) - offsetof(struct pthread_functions_s, psynch_rw_yieldwrlock) - sizeof(void*)) == (sizeof(void*) * 100)); @@ -63,10 +70,6 @@ static_assert((sizeof(struct pthread_callbacks_s) - offsetof(struct pthread_call extern kern_return_t mach_port_deallocate(ipc_space_t, mach_port_name_t); extern kern_return_t semaphore_signal_internal_trap(mach_port_name_t); -/* Used for stackshot introspection */ -extern void kdp_pthread_find_owner(thread_t thread, struct stackshot_thread_waitinfo *waitinfo); -extern void* kdp_pthread_get_thread_kwq(thread_t thread); - #define PTHREAD_STRUCT_ACCESSOR(get, set, rettype, structtype, member) \ static rettype \ get(structtype x) { \ @@ -84,7 +87,10 @@ PTHREAD_STRUCT_ACCESSOR(proc_get_stack_addr_hint, proc_set_stack_addr_hint, user PTHREAD_STRUCT_ACCESSOR(proc_get_dispatchqueue_offset, proc_set_dispatchqueue_offset, uint64_t, struct proc*, p_dispatchqueue_offset); PTHREAD_STRUCT_ACCESSOR(proc_get_dispatchqueue_serialno_offset, proc_set_dispatchqueue_serialno_offset, uint64_t, struct proc*, p_dispatchqueue_serialno_offset); PTHREAD_STRUCT_ACCESSOR(proc_get_pthread_tsd_offset, proc_set_pthread_tsd_offset, uint32_t, struct proc *, p_pth_tsd_offset); +PTHREAD_STRUCT_ACCESSOR(proc_get_mach_thread_self_tsd_offset, proc_set_mach_thread_self_tsd_offset, uint64_t, struct proc *, p_mach_thread_self_offset); PTHREAD_STRUCT_ACCESSOR(proc_get_pthhash, proc_set_pthhash, void*, struct proc*, p_pthhash); +PTHREAD_STRUCT_ACCESSOR(proc_get_return_to_kernel_offset, proc_set_return_to_kernel_offset, uint64_t, struct proc*, p_return_to_kernel_offset); +PTHREAD_STRUCT_ACCESSOR(proc_get_user_stack, proc_set_user_stack, user_addr_t, struct proc*, user_stack); PTHREAD_STRUCT_ACCESSOR(uthread_get_threadlist, uthread_set_threadlist, void*, struct uthread*, uu_threadlist); PTHREAD_STRUCT_ACCESSOR(uthread_get_sigmask, uthread_set_sigmask, sigset_t, struct uthread*, uu_sigmask); @@ -184,6 +190,14 @@ qos_main_thread_active(void) return TRUE; } +#if defined(__arm__) +/* On iOS, the stack placement depends on the address space size */ +static uint32_t +map_is_1gb(vm_map_t map) +{ + return ((!vm_map_is_64bit(map)) && (get_map_max(map) == ml_get_max_offset(FALSE, MACHINE_MAX_OFFSET_MIN))); +} +#endif static int proc_usynch_get_requested_thread_qos(struct uthread *uth) { @@ -501,10 +515,17 @@ thread_qos_from_pthread_priority(unsigned long priority, unsigned long *flags) unsigned long pthread_priority_canonicalize(unsigned long priority, boolean_t propagation) { - if (pthread_functions->pthread_priority_canonicalize2) { - return pthread_functions->pthread_priority_canonicalize2(priority, propagation); + return pthread_functions->pthread_priority_canonicalize2(priority, propagation); +} + +boolean_t +workq_thread_has_been_unbound(thread_t th, int qos_class) +{ + if (pthread_functions->workq_thread_has_been_unbound) { + return pthread_functions->workq_thread_has_been_unbound(th, qos_class); } else { - return pthread_functions->pthread_priority_canonicalize(priority); + panic("pthread kext does not support workq_thread_has_been_unbound"); + return false; } } @@ -524,6 +545,28 @@ kdp_pthread_get_thread_kwq(thread_t thread) return NULL; } +static void +thread_will_park_or_terminate(thread_t thread) +{ + if (thread_owned_workloops_count(thread)) { + (void)kevent_exit_on_workloop_ownership_leak(thread); + } +} + +#if defined(__arm64__) +static unsigned __int128 +atomic_fetch_add_128_relaxed(_Atomic unsigned __int128 *ptr, unsigned __int128 value) +{ + return atomic_fetch_add_explicit(ptr, value, memory_order_relaxed); +} + +static unsigned __int128 +atomic_load_128_relaxed(_Atomic unsigned __int128 *ptr) +{ + return atomic_load_explicit(ptr, memory_order_relaxed); +} +#endif + /* * The callbacks structure (defined in pthread_shims.h) contains a collection * of kernel functions that were not deemed sensible to expose as a KPI to all @@ -559,7 +602,9 @@ static const struct pthread_callbacks_s pthread_callbacks = { .vm_map_page_info = vm_map_page_info, .vm_map_switch = vm_map_switch, .thread_set_wq_state32 = thread_set_wq_state32, +#if !defined(__arm__) .thread_set_wq_state64 = thread_set_wq_state64, +#endif .uthread_get_threadlist = uthread_get_threadlist, .uthread_set_threadlist = uthread_set_threadlist, @@ -593,6 +638,8 @@ static const struct pthread_callbacks_s pthread_callbacks = { .zfree = zfree, .zinit = zinit, + .workloop_fulfill_threadreq = workloop_fulfill_threadreq, + .__pthread_testcancel = __pthread_testcancel, .mach_port_deallocate = mach_port_deallocate, @@ -604,6 +651,13 @@ static const struct pthread_callbacks_s pthread_callbacks = { .convert_thread_to_port = convert_thread_to_port, .ml_get_max_cpus = (void*)ml_get_max_cpus, +#if defined(__arm__) + .map_is_1gb = map_is_1gb, +#endif +#if defined(__arm64__) + .atomic_fetch_add_128_relaxed = atomic_fetch_add_128_relaxed, + .atomic_load_128_relaxed = atomic_load_128_relaxed, +#endif .proc_get_dispatchqueue_serialno_offset = proc_get_dispatchqueue_serialno_offset, .proc_set_dispatchqueue_serialno_offset = proc_set_dispatchqueue_serialno_offset, @@ -612,6 +666,8 @@ static const struct pthread_callbacks_s pthread_callbacks = { .proc_set_stack_addr_hint = proc_set_stack_addr_hint, .proc_get_pthread_tsd_offset = proc_get_pthread_tsd_offset, .proc_set_pthread_tsd_offset = proc_set_pthread_tsd_offset, + .proc_get_mach_thread_self_tsd_offset = proc_get_mach_thread_self_tsd_offset, + .proc_set_mach_thread_self_tsd_offset = proc_set_mach_thread_self_tsd_offset, .thread_set_tsd_base = thread_set_tsd_base, @@ -632,6 +688,15 @@ static const struct pthread_callbacks_s pthread_callbacks = { .proc_usynch_thread_qos_squash_override_for_resource = proc_usynch_thread_qos_squash_override_for_resource, .task_get_default_manager_qos = task_get_default_manager_qos, .thread_create_workq_waiting = thread_create_workq_waiting, + + .proc_get_return_to_kernel_offset = proc_get_return_to_kernel_offset, + .proc_set_return_to_kernel_offset = proc_set_return_to_kernel_offset, + .thread_will_park_or_terminate = thread_will_park_or_terminate, + + .qos_max_parallelism = qos_max_parallelism, + + .proc_get_user_stack = proc_get_user_stack, + .proc_set_user_stack = proc_set_user_stack, }; pthread_callbacks_t pthread_kern = &pthread_callbacks; diff --git a/bsd/kern/stackshot.c b/bsd/kern/stackshot.c index 5910e059f..cb3918bae 100644 --- a/bsd/kern/stackshot.c +++ b/bsd/kern/stackshot.c @@ -178,7 +178,8 @@ kern_stack_snapshot_with_reason(__unused char *reason) config.sc_pid = -1; config.sc_flags = (STACKSHOT_SAVE_LOADINFO | STACKSHOT_GET_GLOBAL_MEM_STATS | STACKSHOT_SAVE_IN_KERNEL_BUFFER | - STACKSHOT_KCDATA_FORMAT | STACKSHOT_ENABLE_UUID_FAULTING); + STACKSHOT_KCDATA_FORMAT | STACKSHOT_ENABLE_UUID_FAULTING | STACKSHOT_THREAD_WAITINFO | + STACKSHOT_NO_IO_STATS); config.sc_delta_timestamp = 0; config.sc_out_buffer_addr = 0; config.sc_out_size_addr = 0; diff --git a/bsd/kern/subr_eventhandler.c b/bsd/kern/subr_eventhandler.c new file mode 100644 index 000000000..65c5975e3 --- /dev/null +++ b/bsd/kern/subr_eventhandler.c @@ -0,0 +1,359 @@ +/* + * Copyright (c) 2016-2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/*- + * Copyright (c) 1999 Michael Smith <msmith@freebsd.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +#include <sys/param.h> +#include <sys/kernel.h> +#include <kern/queue.h> +#include <kern/locks.h> +#include <sys/malloc.h> +#include <sys/proc.h> +#include <sys/systm.h> +#include <sys/mcache.h> +#include <sys/eventhandler.h> +#include <sys/sysctl.h> + +int evh_debug = 0; + +MALLOC_DEFINE(M_EVENTHANDLER, "eventhandler", "Event handler records"); + +SYSCTL_NODE(_kern, OID_AUTO, eventhandler, CTLFLAG_RW | CTLFLAG_LOCKED, + 0, "Eventhandler"); +SYSCTL_INT(_kern_eventhandler, OID_AUTO, debug, CTLFLAG_RW | CTLFLAG_LOCKED, + &evh_debug, 0, "Eventhandler debug mode"); + +struct eventhandler_entry_arg eventhandler_entry_dummy_arg = {{0}}; + +/* List of 'slow' lists */ +static struct eventhandler_lists_ctxt evthdlr_lists_ctxt_glb; +static lck_grp_attr_t *eventhandler_mutex_grp_attr; +static lck_grp_t *eventhandler_mutex_grp; +static lck_attr_t *eventhandler_mutex_attr; + +static lck_grp_attr_t *el_lock_grp_attr; +lck_grp_t *el_lock_grp; +lck_attr_t *el_lock_attr; + +struct eventhandler_entry_generic +{ + struct eventhandler_entry ee; + void (* func)(void); +}; + +static struct eventhandler_list *_eventhandler_find_list( + struct eventhandler_lists_ctxt *evthdlr_lists_ctxt, const char *name); + +void +eventhandler_lists_ctxt_init(struct eventhandler_lists_ctxt *evthdlr_lists_ctxt) +{ + VERIFY(evthdlr_lists_ctxt != NULL); + + TAILQ_INIT(&evthdlr_lists_ctxt->eventhandler_lists); + evthdlr_lists_ctxt->eventhandler_lists_initted = 1; + lck_mtx_init(&evthdlr_lists_ctxt->eventhandler_mutex, + eventhandler_mutex_grp, eventhandler_mutex_attr); +} + +/* + * Initialize the eventhandler mutex and list. + */ +void +eventhandler_init(void) +{ + eventhandler_mutex_grp_attr = lck_grp_attr_alloc_init(); + eventhandler_mutex_grp = lck_grp_alloc_init("eventhandler", + eventhandler_mutex_grp_attr); + eventhandler_mutex_attr = lck_attr_alloc_init(); + + el_lock_grp_attr = lck_grp_attr_alloc_init(); + el_lock_grp = lck_grp_alloc_init("eventhandler list", + el_lock_grp_attr); + el_lock_attr = lck_attr_alloc_init(); + + eventhandler_lists_ctxt_init(&evthdlr_lists_ctxt_glb); +} + +/* + * Insertion is O(n) due to the priority scan, but optimises to O(1) + * if all priorities are identical. + */ +static eventhandler_tag +eventhandler_register_internal( + struct eventhandler_lists_ctxt *evthdlr_lists_ctxt, + struct eventhandler_list *list, + const char *name, eventhandler_tag epn) +{ + struct eventhandler_list *new_list; + struct eventhandler_entry *ep; + + if (evthdlr_lists_ctxt == NULL) + evthdlr_lists_ctxt = &evthdlr_lists_ctxt_glb; + + VERIFY(evthdlr_lists_ctxt->eventhandler_lists_initted); /* eventhandler registered too early */ + VERIFY(epn != NULL); /* cannot register NULL event */ + + /* lock the eventhandler lists */ + lck_mtx_lock(&evthdlr_lists_ctxt->eventhandler_mutex); + + /* Do we need to find/create the (slow) list? */ + if (list == NULL) { + /* look for a matching, existing list */ + list = _eventhandler_find_list(evthdlr_lists_ctxt, name); + + /* Do we need to create the list? */ + if (list == NULL) { + lck_mtx_unlock(&evthdlr_lists_ctxt->eventhandler_mutex); + + MALLOC(new_list, struct eventhandler_list *, + sizeof(struct eventhandler_list) + strlen(name) + 1, + M_EVENTHANDLER, M_WAITOK); + + /* If someone else created it already, then use that one. */ + lck_mtx_lock(&evthdlr_lists_ctxt->eventhandler_mutex); + list = _eventhandler_find_list(evthdlr_lists_ctxt, name); + if (list != NULL) { + FREE(new_list, M_EVENTHANDLER); + } else { + evhlog((LOG_DEBUG, "%s: creating list \"%s\"", __func__, name)); + list = new_list; + list->el_flags = 0; + list->el_runcount = 0; + bzero(&list->el_lock, sizeof(list->el_lock)); + list->el_name = (char *)list + sizeof(struct eventhandler_list); + strlcpy(list->el_name, name, strlen(name) + 1); + TAILQ_INSERT_HEAD(&evthdlr_lists_ctxt->eventhandler_lists, list, el_link); + } + } + } + if (!(list->el_flags & EHL_INITTED)) { + TAILQ_INIT(&list->el_entries); + EHL_LOCK_INIT(list); + list->el_flags |= EHL_INITTED; + } + lck_mtx_unlock(&evthdlr_lists_ctxt->eventhandler_mutex); + + KASSERT(epn->ee_priority != EHE_DEAD_PRIORITY, + ("%s: handler for %s registered with dead priority", __func__, name)); + + /* sort it into the list */ + evhlog((LOG_DEBUG, "%s: adding item %p (function %p to \"%s\"", __func__, VM_KERNEL_ADDRPERM(epn), + VM_KERNEL_UNSLIDE(((struct eventhandler_entry_generic *)epn)->func), name)); + EHL_LOCK(list); + TAILQ_FOREACH(ep, &list->el_entries, ee_link) { + if (ep->ee_priority != EHE_DEAD_PRIORITY && + epn->ee_priority < ep->ee_priority) { + TAILQ_INSERT_BEFORE(ep, epn, ee_link); + break; + } + } + if (ep == NULL) + TAILQ_INSERT_TAIL(&list->el_entries, epn, ee_link); + EHL_UNLOCK(list); + return(epn); +} + +eventhandler_tag +eventhandler_register(struct eventhandler_lists_ctxt *evthdlr_lists_ctxt, + struct eventhandler_list *list, const char *name, + void *func, struct eventhandler_entry_arg arg, int priority) +{ + struct eventhandler_entry_generic *eg; + + /* allocate an entry for this handler, populate it */ + MALLOC(eg, struct eventhandler_entry_generic *, + sizeof(struct eventhandler_entry_generic), + M_EVENTHANDLER, M_WAITOK | M_ZERO); + + eg->func = func; + eg->ee.ee_arg = arg; + eg->ee.ee_priority = priority; + + return (eventhandler_register_internal(evthdlr_lists_ctxt, list, name, &eg->ee)); +} + +void +eventhandler_deregister(struct eventhandler_list *list, eventhandler_tag tag) +{ + struct eventhandler_entry *ep = tag; + + EHL_LOCK_ASSERT(list, LCK_MTX_ASSERT_OWNED); + if (ep != NULL) { + /* remove just this entry */ + if (list->el_runcount == 0) { + evhlog((LOG_DEBUG, "%s: removing item %p from \"%s\"", __func__, VM_KERNEL_ADDRPERM(ep), + list->el_name)); + /* + * We may have purged the list because of certain events. + * Make sure that is not the case when a specific entry + * is being removed. + */ + if (!TAILQ_EMPTY(&list->el_entries)) + TAILQ_REMOVE(&list->el_entries, ep, ee_link); + FREE(ep, M_EVENTHANDLER); + } else { + evhlog((LOG_DEBUG, "%s: marking item %p from \"%s\" as dead", __func__, + VM_KERNEL_ADDRPERM(ep), list->el_name)); + ep->ee_priority = EHE_DEAD_PRIORITY; + } + } else { + /* remove entire list */ + if (list->el_runcount == 0) { + evhlog((LOG_DEBUG, "%s: removing all items from \"%s\"", __func__, + list->el_name)); + while (!TAILQ_EMPTY(&list->el_entries)) { + ep = TAILQ_FIRST(&list->el_entries); + TAILQ_REMOVE(&list->el_entries, ep, ee_link); + FREE(ep, M_EVENTHANDLER); + } + } else { + evhlog((LOG_DEBUG, "%s: marking all items from \"%s\" as dead", + __func__, list->el_name)); + TAILQ_FOREACH(ep, &list->el_entries, ee_link) + ep->ee_priority = EHE_DEAD_PRIORITY; + } + } + while (list->el_runcount > 0) + msleep((caddr_t)list, &list->el_lock, 0, "evhrm", 0); + EHL_UNLOCK(list); +} + +/* + * Internal version for use when eventhandler list is already locked. + */ +static struct eventhandler_list * +_eventhandler_find_list(struct eventhandler_lists_ctxt *evthdlr_lists_ctxt, + const char *name) +{ + struct eventhandler_list *list; + + VERIFY(evthdlr_lists_ctxt != NULL); + + LCK_MTX_ASSERT(&evthdlr_lists_ctxt->eventhandler_mutex, LCK_MTX_ASSERT_OWNED); + TAILQ_FOREACH(list, &evthdlr_lists_ctxt->eventhandler_lists, el_link) { + if (!strcmp(name, list->el_name)) + break; + } + return (list); +} + +/* + * Lookup a "slow" list by name. Returns with the list locked. + */ +struct eventhandler_list * +eventhandler_find_list(struct eventhandler_lists_ctxt *evthdlr_lists_ctxt, + const char *name) +{ + struct eventhandler_list *list; + + if (evthdlr_lists_ctxt == NULL) + evthdlr_lists_ctxt = &evthdlr_lists_ctxt_glb; + + if (!evthdlr_lists_ctxt->eventhandler_lists_initted) + return(NULL); + + /* scan looking for the requested list */ + lck_mtx_lock(&evthdlr_lists_ctxt->eventhandler_mutex); + list = _eventhandler_find_list(evthdlr_lists_ctxt, name); + if (list != NULL) + EHL_LOCK(list); + lck_mtx_unlock(&evthdlr_lists_ctxt->eventhandler_mutex); + + return(list); +} + +/* + * Prune "dead" entries from an eventhandler list. + */ +void +eventhandler_prune_list(struct eventhandler_list *list) +{ + struct eventhandler_entry *ep, *en; + int pruned = 0; + + evhlog((LOG_DEBUG, "%s: pruning list \"%s\"", __func__, list->el_name)); + EHL_LOCK_ASSERT(list, LCK_MTX_ASSERT_OWNED); + TAILQ_FOREACH_SAFE(ep, &list->el_entries, ee_link, en) { + if (ep->ee_priority == EHE_DEAD_PRIORITY) { + TAILQ_REMOVE(&list->el_entries, ep, ee_link); + FREE(ep, M_EVENTHANDLER); + pruned++; + } + } + if (pruned > 0) + wakeup(list); +} + +/* + * This should be called when last reference to an object + * is being released. + * The individual event type lists must be purged when the object + * becomes defunct. + */ +void +eventhandler_lists_ctxt_destroy(struct eventhandler_lists_ctxt *evthdlr_lists_ctxt) +{ + struct eventhandler_list *list = NULL; + struct eventhandler_list *list_next = NULL; + + lck_mtx_lock(&evthdlr_lists_ctxt->eventhandler_mutex); + TAILQ_FOREACH_SAFE(list, &evthdlr_lists_ctxt->eventhandler_lists, + el_link, list_next) { + VERIFY(TAILQ_EMPTY(&list->el_entries)); + EHL_LOCK_DESTROY(list); + FREE(list, M_EVENTHANDLER); + } + lck_mtx_unlock(&evthdlr_lists_ctxt->eventhandler_mutex); + lck_mtx_destroy(&evthdlr_lists_ctxt->eventhandler_mutex, + eventhandler_mutex_grp); + return; +} diff --git a/bsd/kern/subr_log.c b/bsd/kern/subr_log.c index d0a78427b..5a748b5b5 100644 --- a/bsd/kern/subr_log.c +++ b/bsd/kern/subr_log.c @@ -690,7 +690,7 @@ oslogwakeup(void) static void oslog_streamwakeup_locked(void) { - lck_spin_assert(&oslog_stream_lock, LCK_ASSERT_OWNED); + LCK_SPIN_ASSERT(&oslog_stream_lock, LCK_ASSERT_OWNED); if (!oslog_stream_open) { return; } @@ -777,7 +777,7 @@ oslogioctl(__unused dev_t dev, u_long com, caddr_t data, __unused int flag, __un /* return number of characters immediately available */ case LOGBUFFERMAP: - kernel_firehose_buffer = kernel_firehose_addr; + kernel_firehose_buffer = (firehose_buffer_t)kernel_firehose_addr; ret = mach_make_memory_entry_64(kernel_map, &buffer_size, @@ -786,11 +786,12 @@ oslogioctl(__unused dev_t dev, u_long com, caddr_t data, __unused int flag, __un &mem_entry_ptr, MACH_PORT_NULL); if (ret == KERN_SUCCESS) { - ret = mach_vm_map(get_task_map(current_task()), + ret = mach_vm_map_kernel(get_task_map(current_task()), &user_addr, buffer_size, 0, /* mask */ VM_FLAGS_ANYWHERE, + VM_KERN_MEMORY_NONE, mem_entry_ptr, 0, /* offset */ FALSE, /* copy */ @@ -868,9 +869,9 @@ oslog_init(void) panic("Failed to allocate memory for firehose logging buffer"); } kernel_firehose_addr += PAGE_SIZE; - bzero(kernel_firehose_addr, size); + bzero((void *)kernel_firehose_addr, size); /* register buffer with firehose */ - kernel_firehose_addr = __firehose_buffer_create((size_t *) &size); + kernel_firehose_addr = (vm_offset_t)__firehose_buffer_create((size_t *) &size); kprintf("oslog_init completed\n"); } @@ -907,7 +908,7 @@ oslog_stream_find_free_buf_entry_locked(void) struct msgbuf *mbp; oslog_stream_buf_entry_t buf_entry = NULL; - lck_spin_assert(&oslog_stream_lock, LCK_ASSERT_OWNED); + LCK_SPIN_ASSERT(&oslog_stream_lock, LCK_ASSERT_OWNED); mbp = oslog_streambufp; @@ -946,7 +947,7 @@ oslog_stream_find_free_buf_entry_locked(void) void oslog_streamwrite_metadata_locked(oslog_stream_buf_entry_t m_entry) { - lck_spin_assert(&oslog_stream_lock, LCK_ASSERT_OWNED); + LCK_SPIN_ASSERT(&oslog_stream_lock, LCK_ASSERT_OWNED); STAILQ_INSERT_TAIL(&oslog_stream_buf_head, m_entry, buf_entries); return; @@ -956,7 +957,7 @@ static void oslog_streamwrite_append_bytes(const char *buffer, int buflen) { struct msgbuf *mbp; - lck_spin_assert(&oslog_stream_lock, LCK_ASSERT_OWNED); + LCK_SPIN_ASSERT(&oslog_stream_lock, LCK_ASSERT_OWNED); mbp = oslog_streambufp; // Check if we have enough space in the stream buffer to write the data @@ -995,7 +996,7 @@ oslog_streamwrite_locked(firehose_tracepoint_id_u ftid, uint16_t ft_size = offsetof(struct firehose_tracepoint_s, ft_data); int ft_length = ft_size + publen; - lck_spin_assert(&oslog_stream_lock, LCK_ASSERT_OWNED); + LCK_SPIN_ASSERT(&oslog_stream_lock, LCK_ASSERT_OWNED); mbp = oslog_streambufp; if (ft_length > mbp->msg_size) { diff --git a/bsd/kern/subr_prf.c b/bsd/kern/subr_prf.c index ebca1b666..f254816fa 100644 --- a/bsd/kern/subr_prf.c +++ b/bsd/kern/subr_prf.c @@ -98,7 +98,6 @@ #include <sys/subr_prf.h> #include <kern/cpu_number.h> /* for cpu_number() */ -#include <machine/spl.h> #include <libkern/libkern.h> #include <os/log_private.h> @@ -117,10 +116,10 @@ struct snprintf_arg { /* * In case console is off, - * panicstr contains argument to last + * debugger_panic_str contains argument to last * call to panic. */ -extern const char *panicstr; +extern const char *debugger_panic_str; extern void cnputc(char); /* standard console putc */ void (*v_putc)(char) = cnputc; /* routine to putc on virtual console */ @@ -417,7 +416,7 @@ putchar(int c, void *arg) struct putchar_args *pca = arg; char **sp = (char**) pca->tty; - if (panicstr) + if (debugger_panic_str) constty = 0; if ((pca->flags & TOCONS) && pca->tty == NULL && constty) { pca->tty = constty; @@ -449,6 +448,7 @@ vprintf_log_locked(const char *fmt, va_list ap) return 0; } +#if !CONFIG_EMBEDDED /* * Scaled down version of vsprintf(3). @@ -471,6 +471,7 @@ vsprintf(char *buf, const char *cfmt, va_list ap) } return 0; } +#endif /* !CONFIG_EMBEDDED */ /* * Scaled down version of snprintf(3). diff --git a/bsd/kern/subr_prof.c b/bsd/kern/subr_prof.c index 80b6edc27..20a0f5be3 100644 --- a/bsd/kern/subr_prof.c +++ b/bsd/kern/subr_prof.c @@ -70,7 +70,6 @@ #include <sys/kernel.h> #include <sys/proc_internal.h> #include <sys/user.h> -#include <machine/spl.h> #include <machine/machine_routines.h> #include <sys/mount_internal.h> diff --git a/bsd/kern/sys_coalition.c b/bsd/kern/sys_coalition.c index e35a8a878..85100962e 100644 --- a/bsd/kern/sys_coalition.c +++ b/bsd/kern/sys_coalition.c @@ -1,4 +1,5 @@ #include <kern/kern_types.h> +#include <kern/thread_group.h> #include <mach/mach_types.h> #include <mach/boolean.h> @@ -31,6 +32,7 @@ coalition_create_syscall(user_addr_t cidp, uint32_t flags) uint64_t cid; coalition_t coal; int type = COALITION_CREATE_FLAGS_GET_TYPE(flags); + int role = COALITION_CREATE_FLAGS_GET_ROLE(flags); boolean_t privileged = !!(flags & COALITION_CREATE_FLAGS_PRIVILEGED); if ((flags & (~COALITION_CREATE_FLAGS_MASK)) != 0) @@ -38,7 +40,7 @@ coalition_create_syscall(user_addr_t cidp, uint32_t flags) if (type < 0 || type > COALITION_TYPE_MAX) return EINVAL; - kr = coalition_create_internal(type, privileged, &coal); + kr = coalition_create_internal(type, role, privileged, &coal); if (kr != KERN_SUCCESS) { /* for now, the only kr is KERN_RESOURCE_SHORTAGE */ error = ENOMEM; @@ -118,7 +120,7 @@ coalition_request_terminate_syscall(user_addr_t cidp, uint32_t flags) * Request the kernel to deallocate the coalition identified by ID, which * must be both terminated and empty. This balances the reference taken * in coalition_create. - * The memory containig the coalition object may not be freed just yet, if + * The memory containing the coalition object may not be freed just yet, if * other kernel operations still hold references to it. * * Returns: @@ -232,6 +234,26 @@ coalition_info_resource_usage(coalition_t coal, user_addr_t buffer, user_size_t return copyout(&cru, buffer, MIN(bufsize, sizeof(cru))); } +#define coalition_info_set_name_internal(...) 0 + +static int +coalition_info_efficiency(coalition_t coal, user_addr_t buffer, user_size_t bufsize) +{ + int error = 0; + if (coalition_type(coal) != COALITION_TYPE_JETSAM) + return EINVAL; + uint64_t flags = 0; + error = copyin(buffer, &flags, MIN(bufsize, sizeof(flags))); + if (error) + return error; + if ((flags & COALITION_EFFICIENCY_VALID_FLAGS) == 0) + return EINVAL; + if (flags & COALITION_FLAGS_EFFICIENT) { + coalition_set_efficient(coal); + } + return error; +} + int coalition_info(proc_t p, struct coalition_info_args *uap, __unused int32_t *retval) { user_addr_t cidp = uap->cid; @@ -271,6 +293,12 @@ int coalition_info(proc_t p, struct coalition_info_args *uap, __unused int32_t * case COALITION_INFO_RESOURCE_USAGE: error = coalition_info_resource_usage(coal, buffer, bufsize); break; + case COALITION_INFO_SET_NAME: + error = coalition_info_set_name_internal(coal, buffer, bufsize); + break; + case COALITION_INFO_SET_EFFICIENCY: + error = coalition_info_efficiency(coal, buffer, bufsize); + break; default: error = EINVAL; } @@ -280,7 +308,7 @@ bad: return error; } -#if defined(DEVELOPMENT) || defined(DEBUG) +#if DEVELOPMENT || DEBUG static int sysctl_coalition_get_ids SYSCTL_HANDLER_ARGS { #pragma unused(oidp, arg1, arg2) diff --git a/bsd/kern/sys_generic.c b/bsd/kern/sys_generic.c index 2d1431763..9cfbe7e91 100644 --- a/bsd/kern/sys_generic.c +++ b/bsd/kern/sys_generic.c @@ -97,6 +97,7 @@ #include <sys/proc.h> #include <sys/kauth.h> +#include <machine/smp.h> #include <mach/mach_types.h> #include <kern/kern_types.h> #include <kern/assert.h> @@ -140,6 +141,10 @@ #include <kern/kalloc.h> #include <sys/vnode_internal.h> +#if CONFIG_MACF +#include <security/mac_framework.h> +#endif + /* XXX should be in a header file somewhere */ void evsofree(struct socket *); void evpipefree(struct pipe *); @@ -159,7 +164,6 @@ __private_extern__ int dofilewrite(vfs_context_t ctx, struct fileproc *fp, __private_extern__ int preparefileread(struct proc *p, struct fileproc **fp_ret, int fd, int check_for_vnode); __private_extern__ void donefileread(struct proc *p, struct fileproc *fp_ret, int fd); - /* Conflict wait queue for when selects collide (opaque type) */ struct waitq select_conflict_queue; @@ -770,7 +774,7 @@ ioctl(struct proc *p, struct ioctl_args *uap, __unused int32_t *retval) boolean_t is64bit = FALSE; int tmp = 0; #define STK_PARAMS 128 - char stkbuf[STK_PARAMS]; + char stkbuf[STK_PARAMS] = {}; int fd = uap->fd; u_long com = uap->com; struct vfs_context context = *vfs_context_current(); @@ -3111,6 +3115,14 @@ gethostuuid(struct proc *p, struct gethostuuid_args *uap, __unused int32_t *retv __darwin_uuid_t uuid_kern; /* for IOKit call */ if (!uap->spi) { +#if CONFIG_EMBEDDED +#if CONFIG_MACF + if ((error = mac_system_check_info(kauth_cred_get(), "hw.uuid")) != 0) { + /* EPERM invokes userspace upcall if present */ + return (error); + } +#endif +#endif } /* Convert the 32/64 bit timespec into a mach_timespec_t */ @@ -3289,7 +3301,7 @@ telemetry(__unused struct proc *p, struct telemetry_args *args, __unused int32_t return (error); } -#if defined(DEVELOPMENT) || defined(DEBUG) +#if DEVELOPMENT || DEBUG #if CONFIG_WAITQ_DEBUG static uint64_t g_wqset_num = 0; struct g_wqset { @@ -3669,4 +3681,6 @@ SYSCTL_PROC(_kern, OID_AUTO, wqset_clear_preposts, CTLTYPE_QUAD | CTLFLAG_RW | C 0, 0, sysctl_wqset_clear_preposts, "Q", "clear preposts on given waitq set"); #endif /* CONFIG_WAITQ_DEBUG */ -#endif /* defined(DEVELOPMENT) || defined(DEBUG) */ +#endif /* DEVELOPMENT || DEBUG */ + + diff --git a/bsd/kern/sys_pipe.c b/bsd/kern/sys_pipe.c index f6adf702a..9e8b346e9 100644 --- a/bsd/kern/sys_pipe.c +++ b/bsd/kern/sys_pipe.c @@ -146,6 +146,11 @@ #include <kern/kalloc.h> #include <vm/vm_kern.h> #include <libkern/OSAtomic.h> +#include <libkern/section_keywords.h> + +#if CONFIG_MACF +#include <security/mac_framework.h> +#endif #define f_flag f_fglob->fg_flag #define f_msgcount f_fglob->fg_msgcount @@ -158,14 +163,14 @@ * interfaces to the outside world exported through file operations */ static int pipe_read(struct fileproc *fp, struct uio *uio, - int flags, vfs_context_t ctx); + int flags, vfs_context_t ctx); static int pipe_write(struct fileproc *fp, struct uio *uio, - int flags, vfs_context_t ctx); + int flags, vfs_context_t ctx); static int pipe_close(struct fileglob *fg, vfs_context_t ctx); static int pipe_select(struct fileproc *fp, int which, void * wql, vfs_context_t ctx); static int pipe_kqfilter(struct fileproc *fp, struct knote *kn, - vfs_context_t ctx); + struct kevent_internal_s *kev, vfs_context_t ctx); static int pipe_ioctl(struct fileproc *fp, u_long cmd, caddr_t data, vfs_context_t ctx); static int pipe_drain(struct fileproc *fp,vfs_context_t ctx); @@ -191,7 +196,7 @@ static int filt_pipewrite(struct knote *kn, long hint); static int filt_pipewritetouch(struct knote *kn, struct kevent_internal_s *kev); static int filt_pipewriteprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); -struct filterops pipe_rfiltops = { +SECURITY_READ_ONLY_EARLY(struct filterops) pipe_rfiltops = { .f_isfd = 1, .f_detach = filt_pipedetach, .f_event = filt_piperead, @@ -199,7 +204,7 @@ struct filterops pipe_rfiltops = { .f_process = filt_pipereadprocess, }; -struct filterops pipe_wfiltops = { +SECURITY_READ_ONLY_EARLY(struct filterops) pipe_wfiltops = { .f_isfd = 1, .f_detach = filt_pipedetach, .f_event = filt_pipewrite, @@ -299,6 +304,7 @@ pipeinit(void) } +#ifndef CONFIG_EMBEDDED /* Bitmap for things to touch in pipe_touch() */ #define PIPE_ATIME 0x00000001 /* time of last access */ #define PIPE_MTIME 0x00000002 /* time of last modification */ @@ -307,25 +313,26 @@ pipeinit(void) static void pipe_touch(struct pipe *tpipe, int touch) { - struct timeval now; + struct timespec now; - microtime(&now); + nanotime(&now); if (touch & PIPE_ATIME) { tpipe->st_atimespec.tv_sec = now.tv_sec; - tpipe->st_atimespec.tv_nsec = now.tv_usec * 1000; + tpipe->st_atimespec.tv_nsec = now.tv_nsec; } if (touch & PIPE_MTIME) { tpipe->st_mtimespec.tv_sec = now.tv_sec; - tpipe->st_mtimespec.tv_nsec = now.tv_usec * 1000; + tpipe->st_mtimespec.tv_nsec = now.tv_nsec; } if (touch & PIPE_CTIME) { tpipe->st_ctimespec.tv_sec = now.tv_sec; - tpipe->st_ctimespec.tv_nsec = now.tv_usec * 1000; + tpipe->st_ctimespec.tv_nsec = now.tv_nsec; } } +#endif static const unsigned int pipesize_blocks[] = {512,1024,2048,4096, 4096 * 2, PIPE_SIZE , PIPE_SIZE * 4 }; @@ -658,8 +665,10 @@ pipe_create(struct pipe **cpipep) */ bzero(cpipe, sizeof *cpipe); +#ifndef CONFIG_EMBEDDED /* Initial times are all the time of creation of the pipe */ pipe_touch(cpipe, PIPE_ATIME | PIPE_MTIME | PIPE_CTIME); +#endif return (0); } @@ -860,8 +869,10 @@ unlocked_error: if ((rpipe->pipe_buffer.size - rpipe->pipe_buffer.cnt) > 0) pipeselwakeup(rpipe, rpipe->pipe_peer); +#ifndef CONFIG_EMBEDDED /* update last read time */ pipe_touch(rpipe, PIPE_ATIME); +#endif PIPE_UNLOCK(rpipe); @@ -1117,9 +1128,11 @@ pipe_write(struct fileproc *fp, struct uio *uio, __unused int flags, pipeselwakeup(wpipe, wpipe); } +#ifndef CONFIG_EMBEDDED /* Update modification, status change (# of bytes in pipe) times */ pipe_touch(rpipe, PIPE_MTIME | PIPE_CTIME); pipe_touch(wpipe, PIPE_MTIME | PIPE_CTIME); +#endif PIPE_UNLOCK(rpipe); return (error); @@ -1536,7 +1549,8 @@ filt_pipewriteprocess(struct knote *kn, struct filt_process_s *data, struct keve /*ARGSUSED*/ static int -pipe_kqfilter(__unused struct fileproc *fp, struct knote *kn, __unused vfs_context_t ctx) +pipe_kqfilter(__unused struct fileproc *fp, struct knote *kn, + __unused struct kevent_internal_s *kev, __unused vfs_context_t ctx) { struct pipe *cpipe = (struct pipe *)kn->kn_fp->f_data; int res; @@ -1623,7 +1637,7 @@ fill_pipeinfo(struct pipe * cpipe, struct pipe_info * pinfo) #if CONFIG_MACF int error; #endif - struct timeval now; + struct timespec now; struct vinfo_stat * ub; int pipe_size = 0; int pipe_count; @@ -1676,15 +1690,15 @@ fill_pipeinfo(struct pipe * cpipe, struct pipe_info * pinfo) ub->vst_uid = kauth_getuid(); ub->vst_gid = kauth_getgid(); - microtime(&now); + nanotime(&now); ub->vst_atime = now.tv_sec; - ub->vst_atimensec = now.tv_usec * 1000; + ub->vst_atimensec = now.tv_nsec; ub->vst_mtime = now.tv_sec; - ub->vst_mtimensec = now.tv_usec * 1000; + ub->vst_mtimensec = now.tv_nsec; ub->vst_ctime = now.tv_sec; - ub->vst_ctimensec = now.tv_usec * 1000; + ub->vst_ctimensec = now.tv_nsec; /* * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen, st_uid, st_gid. diff --git a/bsd/kern/sys_socket.c b/bsd/kern/sys_socket.c index 7b9e78b6f..cc4d778bd 100644 --- a/bsd/kern/sys_socket.c +++ b/bsd/kern/sys_socket.c @@ -189,6 +189,12 @@ soioctl(struct socket *so, u_long cmd, caddr_t data, struct proc *p) int error = 0; int int_arg; +#if CONFIG_MACF_SOCKET_SUBSET + error = mac_socket_check_ioctl(kauth_cred_get(), so, cmd); + if (error) + return (error); +#endif + socket_lock(so, 1); /* call the socket filter's ioctl handler anything but ours */ @@ -374,7 +380,7 @@ soo_stat(struct socket *so, void *ub, int isstat64) /* warning avoidance ; protected by isstat64 */ struct stat64 *sb64 = (struct stat64 *)0; -#if CONFIG_MACF_SOCKET +#if CONFIG_MACF_SOCKET_SUBSET ret = mac_socket_check_stat(kauth_cred_get(), so); if (ret) return (ret); diff --git a/bsd/kern/sys_ulock.c b/bsd/kern/sys_ulock.c index 4da93446c..c8bf0da86 100644 --- a/bsd/kern/sys_ulock.c +++ b/bsd/kern/sys_ulock.c @@ -131,7 +131,6 @@ typedef struct ull { static const bool ull_debug = false; extern void ulock_initialize(void); -extern void kdp_ulock_find_owner(struct waitq * waitq, event64_t event, thread_waitinfo_t *waitinfo); #define ULL_MUST_EXIST 0x0001 static ull_t *ull_get(ulk_t *, uint32_t); @@ -141,7 +140,6 @@ static thread_t ull_promote_owner_locked(ull_t* ull, thread_t thread); #if DEVELOPMENT || DEBUG static int ull_simulate_copyin_fault = 0; -static int ull_panic_on_corruption = 0; static void ull_dump(ull_t *ull) @@ -210,13 +208,6 @@ ulock_initialize(void) 0, "ulocks"); zone_change(ull_zone, Z_NOENCRYPT, TRUE); - -#if DEVELOPMENT || DEBUG - if (!PE_parse_boot_argn("ulock_panic_on_corruption", - &ull_panic_on_corruption, sizeof(ull_panic_on_corruption))) { - ull_panic_on_corruption = 0; - } -#endif } #if DEVELOPMENT || DEBUG @@ -282,7 +273,7 @@ ull_free(ull_t *ull) { assert(ull->ull_owner == THREAD_NULL); - lck_mtx_assert(&ull->ull_lock, LCK_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&ull->ull_lock, LCK_ASSERT_NOTOWNED); lck_mtx_destroy(&ull->ull_lock, ull_lck_grp); @@ -501,17 +492,6 @@ ulock_wait(struct proc *p, struct ulock_wait_args *args, int32_t *retval) /* HACK: don't bail on MACH_PORT_DEAD, to avoid blowing up the no-tsd pthread lock */ if (owner_name != MACH_PORT_DEAD && owner_thread == THREAD_NULL) { -#if DEBUG || DEVELOPMENT - if (ull_panic_on_corruption) { - if (flags & ULF_NO_ERRNO) { - // ULF_NO_ERRNO is used by libplatform ulocks, but not libdispatch ones. - // Don't panic on libdispatch ulock corruptions; the userspace likely - // mismanaged a dispatch queue. - panic("ulock_wait: ulock is corrupted; value=0x%x, ull=%p", - (uint32_t)(args->value), ull); - } - } -#endif /* * Translation failed - even though the lock value is up to date, * whatever was stored in the lock wasn't actually a thread port. @@ -733,7 +713,7 @@ ulock_wake(struct proc *p, struct ulock_wake_args *args, __unused int32_t *retva } else { /* * TODO: WAITQ_SELECT_MAX_PRI forces a linear scan of the (hashed) global waitq. - * Move to a ulock-private, priority sorted waitq to avoid that. + * Move to a ulock-private, priority sorted waitq (i.e. SYNC_POLICY_FIXED_PRIORITY) to avoid that. * * TODO: 'owner is not current_thread (or null)' likely means we can avoid this wakeup * <rdar://problem/25487001> diff --git a/bsd/kern/sys_work_interval.c b/bsd/kern/sys_work_interval.c index 53d4a2930..561a1a0bb 100644 --- a/bsd/kern/sys_work_interval.c +++ b/bsd/kern/sys_work_interval.c @@ -33,44 +33,70 @@ #include <sys/work_interval.h> #include <kern/sched_prim.h> #include <kern/thread.h> -#include <kern/policy_internal.h> +#include <kern/work_interval.h> #include <libkern/libkern.h> int -work_interval_ctl(__unused proc_t p, struct work_interval_ctl_args *uap, __unused int32_t *retval) +work_interval_ctl(__unused proc_t p, struct work_interval_ctl_args *uap, + __unused int32_t *retval) { - uint32_t operation = uap->operation; - int error = 0; - kern_return_t kret = KERN_SUCCESS; - uint64_t work_interval_id; - struct work_interval_notification notification; + uint32_t operation = uap->operation; + int error = 0; + kern_return_t kret = KERN_SUCCESS; + struct work_interval_notification notification; + + /* Two different structs, because headers are complicated */ + struct work_interval_create_params create_params; + struct kern_work_interval_create_args create_args; switch (operation) { case WORK_INTERVAL_OPERATION_CREATE: - if (uap->arg == USER_ADDR_NULL || uap->work_interval_id != 0) { + return ENOTSUP; + case WORK_INTERVAL_OPERATION_CREATE2: + if (uap->arg == USER_ADDR_NULL || uap->work_interval_id != 0) + return EINVAL; + if (uap->len < sizeof(create_params)) return EINVAL; - } - if (uap->len < sizeof(work_interval_id)) { - return ERANGE; - } /* * Privilege check performed up-front, and then the work * ID is allocated for use by the thread */ - error = priv_check_cred(kauth_cred_get(), PRIV_WORK_INTERVAL, 0); - if (error) { - return (error); - } + if ((error = priv_check_cred(kauth_cred_get(), PRIV_WORK_INTERVAL, 0))) + return error; - kret = thread_policy_create_work_interval(current_thread(), - &work_interval_id); - if (kret == KERN_SUCCESS) { - error = copyout(&work_interval_id, uap->arg, sizeof(work_interval_id)); - } else { - error = EINVAL; - } + if ((error = copyin(uap->arg, &create_params, sizeof(create_params)))) + return error; + + create_args = (struct kern_work_interval_create_args) { + .wica_id = create_params.wicp_id, + .wica_port = create_params.wicp_port, + .wica_create_flags = create_params.wicp_create_flags, + }; + + kret = kern_work_interval_create(current_thread(), &create_args); + + /* thread already has a work interval */ + if (kret == KERN_FAILURE) + return EALREADY; + + /* port copyout failed */ + if (kret == KERN_RESOURCE_SHORTAGE) + return ENOMEM; + + /* some other failure */ + if (kret != KERN_SUCCESS) + return EINVAL; + + create_params = (struct work_interval_create_params) { + .wicp_id = create_args.wica_id, + .wicp_port = create_args.wica_port, + .wicp_create_flags = create_args.wica_create_flags, + }; + + if ((error = copyout(&create_params, uap->arg, sizeof(create_params)))) + return error; break; case WORK_INTERVAL_OPERATION_DESTROY: @@ -83,48 +109,61 @@ work_interval_ctl(__unused proc_t p, struct work_interval_ctl_args *uap, __unuse * operation would have allocated a work interval ID for the current * thread, which the scheduler will validate. */ - kret = thread_policy_destroy_work_interval(current_thread(), - uap->work_interval_id); - if (kret != KERN_SUCCESS) { - error = EINVAL; - } + kret = kern_work_interval_destroy(current_thread(), uap->work_interval_id); + if (kret != KERN_SUCCESS) + return EINVAL; break; case WORK_INTERVAL_OPERATION_NOTIFY: - if (uap->arg == USER_ADDR_NULL || uap->work_interval_id == 0) { + if (uap->arg == USER_ADDR_NULL || uap->work_interval_id == 0) return EINVAL; - } - if (uap->len < sizeof(notification)) { + + if (uap->len < sizeof(notification)) return EINVAL; - } /* * No privilege check, we assume a previous WORK_INTERVAL_OPERATION_CREATE * operation would have allocated a work interval ID for the current * thread, which the scheduler will validate. */ - error = copyin(uap->arg, ¬ification, sizeof(notification)); - if (error) { - break; - } + if ((error = copyin(uap->arg, ¬ification, sizeof(notification)))) + return error; - kret = sched_work_interval_notify(current_thread(), - uap->work_interval_id, - notification.start, - notification.finish, - notification.deadline, - notification.next_start, - notification.flags); - if (kret != KERN_SUCCESS) { - error = EINVAL; - break; + struct kern_work_interval_args kwi_args = { + .work_interval_id = uap->work_interval_id, + .start = notification.start, + .finish = notification.finish, + .deadline = notification.deadline, + .next_start = notification.next_start, + .notify_flags = notification.notify_flags, + .create_flags = notification.create_flags, + }; + + kret = kern_work_interval_notify(current_thread(), &kwi_args); + if (kret != KERN_SUCCESS) + return EINVAL; + + break; + case WORK_INTERVAL_OPERATION_JOIN: + if (uap->arg != USER_ADDR_NULL) { + return EINVAL; } + /* + * No privilege check, because the work interval port + * is a capability. + */ + kret = kern_work_interval_join(current_thread(), + (mach_port_name_t)uap->work_interval_id); + if (kret != KERN_SUCCESS) + return EINVAL; + break; + default: - error = ENOTSUP; - break; + return ENOTSUP; } return (error); } + diff --git a/bsd/kern/syscalls.master b/bsd/kern/syscalls.master index 8b3b9b169..f18cc9121 100644 --- a/bsd/kern/syscalls.master +++ b/bsd/kern/syscalls.master @@ -262,16 +262,16 @@ 174 AUE_NULL ALL { int nosys(void); } { old getdents } 175 AUE_NULL ALL { int nosys(void); } { old gc_control } 176 AUE_NULL ALL { int nosys(void); } { old add_profil } -177 AUE_KDEBUGTRACE ALL { int kdebug_typefilter(void** addr, size_t* size) NO_SYSCALL_STUB; } -178 AUE_KDEBUGTRACE ALL { uint64_t kdebug_trace_string(uint32_t debugid, uint64_t str_id, const char *str) NO_SYSCALL_STUB; } -179 AUE_KDEBUGTRACE ALL { int kdebug_trace64(uint32_t code, uint64_t arg1, uint64_t arg2, uint64_t arg3, uint64_t arg4) NO_SYSCALL_STUB; } -180 AUE_KDEBUGTRACE ALL { int kdebug_trace(uint32_t code, u_long arg1, u_long arg2, u_long arg3, u_long arg4) NO_SYSCALL_STUB; } +177 AUE_NULL ALL { int kdebug_typefilter(void** addr, size_t* size) NO_SYSCALL_STUB; } +178 AUE_NULL ALL { uint64_t kdebug_trace_string(uint32_t debugid, uint64_t str_id, const char *str) NO_SYSCALL_STUB; } +179 AUE_NULL ALL { int kdebug_trace64(uint32_t code, uint64_t arg1, uint64_t arg2, uint64_t arg3, uint64_t arg4) NO_SYSCALL_STUB; } +180 AUE_NULL ALL { int kdebug_trace(uint32_t code, u_long arg1, u_long arg2, u_long arg3, u_long arg4) NO_SYSCALL_STUB; } 181 AUE_SETGID ALL { int setgid(gid_t gid); } 182 AUE_SETEGID ALL { int setegid(gid_t egid); } 183 AUE_SETEUID ALL { int seteuid(uid_t euid); } 184 AUE_SIGRETURN ALL { int sigreturn(struct ucontext *uctx, int infostyle) NO_SYSCALL_STUB; } 185 AUE_NULL ALL { int enosys(void); } { old chud } -186 AUE_NULL ALL { int nosys(void); } +186 AUE_NULL ALL { int thread_selfcounts(int type, user_addr_t buf, user_size_t nbytes); } 187 AUE_FDATASYNC ALL { int fdatasync(int fd); } 188 AUE_STAT ALL { int stat(user_addr_t path, user_addr_t ub); } 189 AUE_FSTAT ALL { int fstat(int fd, user_addr_t ub); } @@ -565,7 +565,7 @@ 372 AUE_NULL ALL { uint64_t thread_selfid (void) NO_SYSCALL_STUB; } 373 AUE_LEDGER ALL { int ledger(int cmd, caddr_t arg1, caddr_t arg2, caddr_t arg3); } 374 AUE_NULL ALL { int kevent_qos(int fd, const struct kevent_qos_s *changelist, int nchanges, struct kevent_qos_s *eventlist, int nevents, void *data_out, size_t *data_available, unsigned int flags); } -375 AUE_NULL ALL { int nosys(void); } +375 AUE_NULL ALL { int kevent_id(uint64_t id, const struct kevent_qos_s *changelist, int nchanges, struct kevent_qos_s *eventlist, int nevents, void *data_out, size_t *data_available, unsigned int flags); } 376 AUE_NULL ALL { int nosys(void); } 377 AUE_NULL ALL { int nosys(void); } 378 AUE_NULL ALL { int nosys(void); } @@ -666,8 +666,16 @@ 432 AUE_NULL ALL { int audit_session_port(au_asid_t asid, user_addr_t portnamep); } 433 AUE_NULL ALL { int pid_suspend(int pid); } 434 AUE_NULL ALL { int pid_resume(int pid); } +#if CONFIG_EMBEDDED +435 AUE_NULL ALL { int pid_hibernate(int pid); } +#else 435 AUE_NULL ALL { int nosys(void); } +#endif +#if SOCKETS +436 AUE_NULL ALL { int pid_shutdown_sockets(int pid, int level); } +#else 436 AUE_NULL ALL { int nosys(void); } +#endif 437 AUE_NULL ALL { int nosys(void); } { old shared_region_slide_np } 438 AUE_NULL ALL { int shared_region_map_and_slide_np(int fd, uint32_t count, const struct shared_file_mapping_np *mappings, uint32_t slide, uint64_t* slide_start, uint32_t slide_size) NO_SYSCALL_STUB; } 439 AUE_NULL ALL { int kas_info(int selector, void *value, size_t *size); } @@ -676,7 +684,7 @@ #else 440 AUE_NULL ALL { int nosys(void); } #endif -441 AUE_OPEN_RWTC ALL { int guarded_open_np(const char *path, const guardid_t *guard, u_int guardflags, int flags, int mode) NO_SYSCALL_STUB; } +441 AUE_OPEN_RWTC ALL { int guarded_open_np(user_addr_t path, const guardid_t *guard, u_int guardflags, int flags, int mode) NO_SYSCALL_STUB; } 442 AUE_CLOSE ALL { int guarded_close_np(int fd, const guardid_t *guard); } 443 AUE_KQUEUE ALL { int guarded_kqueue_np(const guardid_t *guard, u_int guardflags); } 444 AUE_NULL ALL { int change_fdguard_np(int fd, const guardid_t *guard, u_int guardflags, const guardid_t *nguard, u_int nguardflags, int *fdflagsp); } @@ -752,7 +760,7 @@ #else 483 AUE_NULL ALL { int enosys(void); } #endif /* CSR */ -484 AUE_NULL ALL { int guarded_open_dprotected_np(const char *path, const guardid_t *guard, u_int guardflags, int flags, int dpclass, int dpflags, int mode) NO_SYSCALL_STUB; } +484 AUE_NULL ALL { int guarded_open_dprotected_np(user_addr_t path, const guardid_t *guard, u_int guardflags, int flags, int dpclass, int dpflags, int mode) NO_SYSCALL_STUB; } 485 AUE_NULL ALL { user_ssize_t guarded_write_np(int fd, const guardid_t *guard, user_addr_t cbuf, user_size_t nbyte); } 486 AUE_PWRITE ALL { user_ssize_t guarded_pwrite_np(int fd, const guardid_t *guard, user_addr_t buf, user_size_t nbyte, off_t offset); } 487 AUE_WRITEV ALL { user_ssize_t guarded_writev_np(int fd, const guardid_t *guard, struct iovec *iovp, int iovcnt); } @@ -815,3 +823,16 @@ 519 AUE_NULL ALL { int enosys(void); } 520 AUE_KILL ALL { int terminate_with_payload(int pid, uint32_t reason_namespace, uint64_t reason_code, void *payload, uint32_t payload_size, const char *reason_string, uint64_t reason_flags) NO_SYSCALL_STUB; } 521 AUE_EXIT ALL { void abort_with_payload(uint32_t reason_namespace, uint64_t reason_code, void *payload, uint32_t payload_size, const char *reason_string, uint64_t reason_flags) NO_SYSCALL_STUB; } +#if NECP +522 AUE_NECP ALL { int necp_session_open(int flags); } } +523 AUE_NECP ALL { int necp_session_action(int necp_fd, uint32_t action, uint8_t *in_buffer, size_t in_buffer_length, uint8_t *out_buffer, size_t out_buffer_length); } +#else /* NECP */ +522 AUE_NULL ALL { int enosys(void); } +523 AUE_NULL ALL { int enosys(void); } +#endif /* NECP */ +524 AUE_SETATTRLISTAT ALL { int setattrlistat(int fd, const char *path, struct attrlist *alist, void *attributeBuffer, size_t bufferSize, uint32_t options); } +525 AUE_NET ALL { int net_qos_guideline(struct net_qos_param *param, uint32_t param_len); } +526 AUE_FMOUNT ALL { int fmount(const char *type, int fd, int flags, void *data); } +527 AUE_NULL ALL { int ntp_adjtime(struct timex *tp); } +528 AUE_NULL ALL { int ntp_gettime(struct ntptimeval *ntvp); } +529 AUE_NULL ALL { int os_fault_with_payload(uint32_t reason_namespace, uint64_t reason_code, void *payload, uint32_t payload_size, const char *reason_string, uint64_t reason_flags); } diff --git a/bsd/kern/sysv_msg.c b/bsd/kern/sysv_msg.c index 56c670424..fad2cfbf7 100644 --- a/bsd/kern/sysv_msg.c +++ b/bsd/kern/sysv_msg.c @@ -67,6 +67,10 @@ #include <sys/sysproto.h> #include <sys/ipcs.h> +#if CONFIG_MACF +#include <security/mac_framework.h> +#endif + #if SYSV_MSG static int msginit(void *); @@ -268,7 +272,7 @@ msginit(__unused void *dummy) } MALLOC(msqids, struct msqid_kernel *, - sizeof(struct user_msqid_ds) * msginfo.msgmni, + sizeof(struct msqid_kernel) * msginfo.msgmni, M_SHM, M_WAITOK); if (msqids == NULL) { printf("msginit: can't allocate msqids"); diff --git a/bsd/kern/sysv_shm.c b/bsd/kern/sysv_shm.c index 4a47b7ac2..a962e9ab1 100644 --- a/bsd/kern/sysv_shm.c +++ b/bsd/kern/sysv_shm.c @@ -483,11 +483,12 @@ shmat(struct proc *p, struct shmat_args *uap, user_addr_t *retval) mapped_size = 0; /* first reserve enough space... */ - rv = mach_vm_map(current_map(), + rv = mach_vm_map_kernel(current_map(), &attach_va, map_size, 0, vm_flags, + VM_KERN_MEMORY_NONE, IPC_PORT_NULL, 0, FALSE, @@ -511,6 +512,8 @@ shmat(struct proc *p, struct shmat_args *uap, user_addr_t *retval) shm_handle->shm_handle_size, /* segment size */ (mach_vm_offset_t)0, /* alignment mask */ VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_NONE, shm_handle->shm_object, (mach_vm_offset_t)0, FALSE, @@ -622,7 +625,7 @@ shmctl(__unused struct proc *p, struct shmctl_args *uap, int32_t *retval) error = copyout(&shmid_ds, uap->buf, sizeof(shmid_ds)); } else { - struct user32_shmid_ds shmid_ds32; + struct user32_shmid_ds shmid_ds32 = {}; shmid_ds_64to32(&shmseg->u, &shmid_ds32); /* Clear kernel reserved pointer before copying to user space */ @@ -1166,7 +1169,7 @@ IPCS_shm_sysctl(__unused struct sysctl_oid *oidp, __unused void *arg1, struct user32_IPCS_command u32; struct user_IPCS_command u64; } ipcs; - struct user32_shmid_ds shmid_ds32; /* post conversion, 32 bit version */ + struct user32_shmid_ds shmid_ds32 = {}; /* post conversion, 32 bit version */ struct user_shmid_ds shmid_ds; /* 64 bit version */ void *shmid_dsp; size_t ipcs_sz = sizeof(struct user_IPCS_command); diff --git a/bsd/kern/trace_codes b/bsd/kern/trace_codes index 1b7187a01..981ab7f57 100644 --- a/bsd/kern/trace_codes +++ b/bsd/kern/trace_codes @@ -107,7 +107,7 @@ 0x10c00bc MSC_kern_invalid_#47 0x10c00c0 MSC_macx_swapon 0x10c00c4 MSC_macx_swapoff -0x10c00c8 MSC_kern_invalid_#50 +0x10c00c8 MSC_thread_get_special_reply_port 0x10c00cc MSC_macx_triggers 0x10c00d0 MSC_macx_backing_store_suspend 0x10c00d4 MSC_macx_backing_store_recovery @@ -228,11 +228,19 @@ 0x1300420 MACH_vm_pageout_cache_evict 0x1300424 MACH_vm_pageout_thread_block 0x1300428 MACH_vm_pageout_jetsam +0x130042c MACH_vm_info1 +0x1300430 MACH_vm_info2 +0x1300434 MACH_vm_info3 +0x1300438 MACH_vm_info4 +0x130043c MACH_vm_info5 +0x1300440 MACH_vm_info6 +0x1300444 MACH_vm_info7 0x1300480 MACH_vm_upl_page_wait 0x1300484 MACH_vm_iopl_page_wait 0x1300488 MACH_vm_page_wait_block 0x130048C MACH_vm_page_sleep 0x1300490 MACH_vm_page_expedite +0x1300494 MACH_vm_page_expedite_no_memory 0x13004c0 MACH_vm_pressure_event 0x1300500 MACH_vm_data_write 0x1320000 vm_disconnect_all_page_mappings @@ -253,6 +261,8 @@ 0x1400024 MACH_IDLE 0x1400028 MACH_STACK_DEPTH 0x140002c MACH_MOVED +0x1400030 MACH_PSET_LOAD_AVERAGE +0x1400034 MACH_AMP_DEBUG 0x1400038 MACH_FAILSAFE 0x140003C MACH_BLOCK 0x1400040 MACH_WAIT @@ -284,6 +294,10 @@ 0x14000B4 MACH_SCHED_LOAD 0x14000B8 MACH_REC_CORES_FAILSAFE 0x14000BC MACH_SCHED_QUANTUM_EXPIRED +0x14000C0 MACH_EXEC_PROMOTE +0x14000C4 MACH_EXEC_DEMOTE +0x14000C8 MACH_AMP_SIGNAL_SPILL +0x14000CC MACH_AMP_STEAL 0x1500000 MACH_MSGID_INVALID 0x1600000 MTX_SLEEP 0x1600004 MTX_SLEEP_DEADLINE @@ -329,7 +343,11 @@ 0x1700034 PMAP_flush_kernel_TLBS 0x1700038 PMAP_flush_delayed_TLBS 0x170003c PMAP_flush_TLBS_TO +0x1700040 PMAP_flush_EPT +0x1700044 PMAP_fast_fault 0x1800000 MACH_CLOCK_EPOCH_CHANGE +0x1800004 MACH_CLOCK_BRIDGE_RCV_TS +0x1800008 MACH_CLOCK_BRIDGE_REMOTE_TIME 0x1900000 MP_TLB_FLUSH 0x1900004 MP_CPUS_CALL 0x1900008 MP_CPUS_CALL_LOCAL @@ -355,6 +373,17 @@ 0x1a30008 ENERGY_PERF_GPU_TIME 0x1a40000 SYSDIAGNOSE_notify_user 0x1a50000 ZALLOC_ZCRAM +0x1a60000 THREAD_GROUP_NEW +0x1a60004 THREAD_GROUP_FREE +0x1a60008 THREAD_GROUP_SET +0x1a6000c THREAD_GROUP_NAME +0x1a60010 THREAD_GROUP_NAME_FREE +0x1a60014 THREAD_GROUP_FLAGS +0x1a70000 COALITION_NEW +0x1a70004 COALITION_FREE +0x1a70008 COALITION_ADOPT +0x1a7000c COALITION_REMOVE +0x1a70010 COALITION_THREAD_GROUP_SET 0x2010000 L_IP_In_Beg 0x2010004 L_IP_Out_Beg 0x2010008 L_IP_In_End @@ -856,12 +885,34 @@ 0x402002C MEMSTAT_dirty_clear 0x4020030 MEMSTAT_grp_set_properties 0x4020034 MEMSTAT_do_kill +0x4030004 KEVENT_kq_processing_begin +0x4030008 KEVENT_kq_processing_end +0x403000c KEVENT_kqwq_processing_begin +0x4030010 KEVENT_kqwq_processing_end +0x4030014 KEVENT_kqwq_bind +0x4030018 KEVENT_kqwq_unbind +0x403001c KEVENT_kqwq_thread_request +0x4030020 KEVENT_kqwl_processing_begin +0x4030024 KEVENT_kqwl_processing_end +0x4030028 KEVENT_kqwl_thread_request +0x403002c KEVENT_kqwl_thread_adjust +0x4030030 KEVENT_kq_register +0x4030034 KEVENT_kqwq_register +0x4030038 KEVENT_kqwl_register +0x403003c KEVENT_knote_activate +0x4030040 KEVENT_kq_process +0x4030044 KEVENT_kqwq_process +0x4030048 KEVENT_kqwl_process +0x403004c KEVENT_kqwl_bind +0x4030050 KEVENT_kqwl_unbind +0x4030054 KEVENT_knote_enable 0x40e0104 BSC_msync_extended_info 0x40e0264 BSC_pread_extended_info 0x40e0268 BSC_pwrite_extended_info 0x40e0314 BSC_mmap_extended_info 0x40f0314 BSC_mmap_extended_info2 0x5000004 INTC_Handler +0x5000008 INTC_Spurious 0x5010004 WL_CheckForWork 0x5010008 WL_RunEventSources 0x5020004 IES_client @@ -938,6 +989,8 @@ 0x50700ec PM_DriverResponseDelay 0x50700f0 PM_PCIDevChangeStart 0x50700f4 PM_PCIDevChangeDone +0x50700f8 PM_SleepWakeMessage +0x50700fc PM_DriverPSChangeDelay 0x5080004 IOSERVICE_BUSY 0x5080008 IOSERVICE_NONBUSY 0x508000c IOSERVICE_MODULESTALL @@ -1135,6 +1188,11 @@ 0x5330024 HIBERNATE_aes_decrypt_cbc 0x5330028 HIBERNATE_flush_compressor 0x533002c HIBERNATE_fastwake_warmup +0x5330030 HIBERNATE_teardown +0x5330034 HIBERNATE_rebuild +0x5330038 HIBERNATE_stats +0x533003c HIBERNATE_idle_kernel +0x5350000 BOOTER_timestamps 0x7000004 TRACE_DATA_NEWTHREAD 0x7000008 TRACE_DATA_EXEC 0x700000c TRACE_DATA_THREAD_TERMINATE @@ -1149,6 +1207,7 @@ 0x7020008 TRACE_LOST_EVENTS 0x702000c TRACE_WRITING_EVENTS 0x7020010 TRACE_INFO_STRING +0x7020014 TRACE_RETROGRADE_EVENTS 0x8000000 USER_TEST 0x8000004 USER_run 0x8000008 USER_join @@ -1206,130 +1265,10 @@ 0xa008000 P_CS_SYNC_DISK 0xa008004 P_CS_WaitForBuffer 0xa008008 P_CS_NoBuffer -0xb000000 AFP_asp_tcp_usr_send -0xb000004 AFP_asp_tcp_usr_send_after_Request -0xb000008 AFP_asp_tcp_usr_send_after_FindDSIReq -0xb00000c AFP_asp_tcp_usr_send_after_Reply -0xb000010 AFP_asp_tcp_slowtimo -0xb000014 AFP_asp_tcp_usr_control -0xb000018 AFP_asp_tcp_fasttimo -0xb000020 AFP_Send -0xb000024 AFP_Send_before_sosend -0xb000028 AFP_Send_after_sosend -0xb00002c AFP_Send_before_write -0xb000030 AFP_Send_after_write -0xb000040 AFP_Reply -0xb000044 AFP_Reply_rcvdAlready -0xb000048 AFP_Reply_before_RcvLock -0xb00004c AFP_Reply_fail_RcvLock -0xb000050 AFP_Reply_before_ReadDSIHdr -0xb000054 AFP_Reply_after_ReadDSIHdr -0xb000058 AFP_Reply_fail_ReadDSIHdr -0xb00005c AFP_Reply_after_FindDSIReqInfo -0xb000060 AFP_Reply_SetAFPCmd -0xb000064 AFP_Reply_before_ReadDSIPacket -0xb000068 AFP_Reply_setRcvdReplyLen -0xb000070 AFP_SendReply -0xb000080 AFP_CreateDSIHeader -0xb000084 AFP_CreateDSIHeader_after_GetReqID -0xb000090 AFP_Request -0xb0000a0 AFP_ReceiveLock -0xb0000b0 AFP_ReceiveWakeUp -0xb0000c0 AFP_ReceiveUnLock -0xb0000e0 AFP_SendLock -0xb0000e4 AFP_SendUnLock -0xb0000f0 AFP_SendQueueLock -0xb000100 AFP_SendQueueUnLock -0xb000110 AFP_ReadDSIHeader -0xb000120 AFP_Receive -0xb000124 AFP_Receive_before_sorcv -0xb000128 AFP_Receive_after_sorcv -0xb000130 AFP_ReadDSIPacket -0xb000140 AFP_DoCopyOut -0xb000150 AFP_DoCopyIn -0xb000160 AFP_CheckRcvTickle -0xb000164 AFP_CheckRcvTickleTO -0xb000170 AFP_CheckSendTickle -0xb000180 AFP_CheckIncomingPkts -0xb000190 AFP_ProcessOptions -0xb000200 AFP_FindDSIReqInfo -0xb000204 AFP_FindDSIReqInfo_foundReqInfo -0xb000208 AFP_FindDSIReqInfo_flags -0xb00020c AFP_FindDSIReqLeave -0xb000210 AFP_UsrDisconnect -0xc000000 AFPVFS_UserReply -0xc000004 AFPVFS_UserReplyGetMbuf -0xc000008 AFPVFS_UserReplysosend -0xc000010 AFPVFS_UserCommand -0xc000018 AFPVFS_UserCommandsosend -0xc000020 AFPVFS_ReadFork -0xc000024 AFPVFS_ReadForkFillQPB -0xc000028 AFPVFS_ReadForkNbrRequests -0xc00002c AFPVFS_ReadForkSendQPB -0xc000030 AFPVFS_ReadForkSendErr -0xc000040 AFPVFS_ReadForkGetReply -0xc000044 AFPVFS_ReadForkGetReplyResult -0xc000050 AFPVFS_WriteFork -0xc000054 AFPVFS_WriteForkFillQPB -0xc000058 AFPVFS_WriteForkNbrRequests -0xc00005c AFPVFS_WriteForkSendQPB -0xc000060 AFPVFS_WriteForkSendErr -0xc000064 AFPVFS_WriteForkGetReply -0xc000068 AFPVFS_WriteForkGetReplyResult -0xc000070 AFPVFS_GetAttr -0xc000080 AFPVFS_SetAttr -0xc000090 AFPVFS_GetAttrList -0xc0000a0 AFPVFS_SetAttrList -0xc0000b0 AFPVFS_FSCTL -0xc0000c0 AFPVFS_LookUp -0xc0000d0 AFPVFS_CacheLookUp -0xc0000e0 AFPVFS_Write -0xc0000e4 AFPVFS_WriteNoCluster -0xc0000e8 AFPVFS_WriteDone -0xc0000f0 AFPVFS_DoWrite -0xc000100 AFPVFS_Lock -0xc000110 AFPVFS_Statfs -0xc000120 AFPVFS_Sync -0xc000130 AFPVFS_VGet -0xc000140 AFPVFS_FlushFiles -0xc000150 AFPVFS_Create -0xc000160 AFPVFS_Mknod -0xc000170 AFPVFS_Open -0xc000180 AFPVFS_Close -0xc000190 AFPVFS_Access -0xc000194 AFPVFS_AccessUID -0xc000198 AFPVFS_AccessGID -0xc00019c AFPVFS_AccessWID -0xc0001a0 AFPVFS_Writeperm -0xc0001b0 AFPVFS_Chmod -0xc0001c0 AFPVFS_Chflags -0xc0001d0 AFPVFS_Exchange -0xc0001e0 AFPVFS_Chid -0xc0001f0 AFPVFS_Fsync -0xc000200 AFPVFS_Remove -0xc000210 AFPVFS_Rename -0xc000220 AFPVFS_Copyfile -0xc000230 AFPVFS_Mkdir -0xc000240 AFPVFS_Symlink -0xc000250 AFPVFS_Readdir -0xc000260 AFPVFS_Readdirattr -0xc000264 AFPVFS_Readdirattr1 -0xc000268 AFPVFS_Readdirattr2 -0xc00026c AFPVFS_Readdirattr3 -0xc000270 AFPVFS_Readlink -0xc000280 AFPVFS_Abortop -0xc000290 AFPVFS_Inactive -0xc0002a0 AFPVFS_Reclaim -0xc0002b0 AFPVFS_Unlock -0xc0002c0 AFPVFS_Islocked -0xc0002d0 AFPVFS_Pathconf -0xc0002e0 AFPVFS_Update -0xc0002f0 AFPVFS_Makenode -0xc000300 AFPVFS_Allocate -0xc000310 AFPVFS_Search -0xc000320 AFPVFS_Reconnect -0xc0003e0 AFPVFS_Rmdir -0xc0003f0 AFPVFS_Vinit +0xc010000 MT_InstrsCycles +0xc010004 MT_InsCyc_CPU_CSwitch +0xcfe0000 MT_TmpThread +0xcff0000 MT_TmpCPU 0x11000000 DNC_PURGE1 0x11000004 DNC_PURGE2 0x11000008 DNC_FOUND @@ -1451,43 +1390,6 @@ 0x20000004 RTC_sync_TBR 0x21010000 SCROLL_BEGIN 0x21020000 BOOT_BEGIN -0x21030200 LOGIN_BEGIN -0x21030204 LOGINWINDOW_LAUNCHED -0x21030208 LOGINWINDOW_LAUNCHES_SA -0x2103020c LOGINWINDOW_GUI_APPEARS -0x21030210 LOGINWINDOW_LOGIN_CLICKED -0x21030214 LOGINWINDOW_ASKS_AUTH -0x21030218 LOGINWINDOW_AUTH_SUCCEEDED -0x2103021c LOGINWINDOW_LAUNCHES_DOCK -0x21030220 LOGINWINDOW_LAUNCHES_SUIS -0x21030224 LOGINWINDOW_LAUNCHES_FINDER -0x21030228 LOGINWINDOW_DOCK_LAUNCHED -0x2103022c LOGINWINDOW_SUIS_LAUNCHED -0x21030230 LOGINWINDOW_FINDER_LAUNCHED -0x21030234 LOGINWINDOW_LOGOUT_CLICKED -0x21030238 LOGINWINDOW_QUIT_FGAPPS -0x2103023c LOGINWINDOW_FGAPPS_QUIT -0x21030240 LOGINWINDOW_QUIT_SUIS -0x21030244 LOGINWINDOW_SUIS_DIES -0x21030248 LOGINWINDOW_QUIT_FINDER -0x2103024c LOGINWINDOW_FINDER_DIES -0x21030250 LOGINWINDOW_QUIT_DOCK -0x21030254 LOGINWINDOW_DOCK_DIES -0x21030258 LOGINWINDOW_EXIT -0x2103025c LOGINWINDOW_FUS_SELUSERNAME -0x21030260 LOGINWINDOW_FUS_SELLOGINWIND -0x21030270 LOGIN_APPLICATION_EXECUTING -0x21030274 LOGIN_APPLICATION_USABLE -0x21030300 LOGIN_END -0x21030500 LOGINWINDOW_APP_TERMINATION_REQUEST -0x21030504 LOGINWINDOW_LOGOUT_START -0x21030508 LOGINWINDOW_DESKTOP_UP -0x2103050c LOGINWINDOW_DESKTOP_UP_NOTIFICATION -0x21030510 LOGINWINDOW_DISPLAYWAKE -0x21030514 LOGINWINDOW_SYSTEMWAKE -0x21030518 LOGINWINDOW_UNLOCKUI_TRIGGERED -0x2103051c LOGINWINDOW_UNLOCKUI_SHOWN -0x21030520 LOGINWINDOW_NO_LOCKUI_SHOWN 0x21040000 APP_AudioOverload 0x21050000 TOOL_PRIVATE_1 0x21050004 TOOL_PRIVATE_2 @@ -1561,6 +1463,10 @@ 0x25010038 PERF_THD_Disp_Data_32 0x2501003c PERF_THD_Sched_Data1_32 0x25010040 PERF_THD_Sched_Data2_32 +0x25010044 PERF_THD_Inscyc_Data +0x25010048 PERF_THD_Inscyc_Data_32 +0x2501004c PERF_THD_Sched_Data_2 +0x25010050 PERF_THD_Sched_Data2_32_2 0x25020000 PERF_STK_KSample 0x25020004 PERF_STK_USched 0x25020008 PERF_STK_USample @@ -1571,7 +1477,7 @@ 0x2502001c PERF_STK_Error 0x25020020 PERF_STK_Backtrace 0x25020024 PERF_STK_Log -0x25030000 PERF_TMR_AllSched +0x25030000 PERF_TMR_Fire 0x25030004 PERF_TMR_Schedule 0x25030008 PERF_TMR_Handler 0x25040000 PERF_ATS_Thread @@ -1606,6 +1512,7 @@ 0x25080010 PERF_TK_Snap_Data2_32 0x250a0000 PERF_MI_Sample 0x250a0004 PERF_MI_Data +0x250a0008 PERF_MI_SysMem_Data 0x26100008 imp_assertion_hold 0x2610000c imp_assertion_hold_ext 0x26100020 imp_assertion_externalize @@ -1631,6 +1538,10 @@ 0x261a0004 imp_usynch_remove_override 0x261b0000 imp_donor_update_live_donor 0x261b0004 imp_donor_init_donor_state +0x261d0000 imp_sync_ipc_qos_applied +0x261d0004 imp_sync_ipc_qos_removed +0x261d0008 imp_sync_ipc_qos_overflow +0x261d000c imp_sync_ipc_qos_underflow 0x26210010 imp_task_int_bg 0x26210014 imp_task_ext_bg 0x26210020 imp_thread_int_bg @@ -1660,6 +1571,11 @@ 0x26310018 imp_task_watchers_bg 0x26320028 imp_thread_pidbind_bg 0x26330028 imp_thread_workq_bg +0x26350028 imp_thread_qos +0x26360028 imp_thread_qos_override +0x26380028 imp_thread_qos_and_relprio +0x263c0028 imp_thread_qos_promote +0x263d0028 imp_thread_qos_ipc_override 0x27000000 PERF_PCEVENT 0x27001000 PERF_CPU_IDLE 0x27001100 PERF_CPU_IDLE_TIMER @@ -1686,6 +1602,7 @@ 0x2700E030 PERF_SRAMEMA_DOM3 0x28100004 BANK_SETTLE_CPU_TIME 0x28100008 BANK_SECURE_ORIGINATOR_CHANGED +0x2810000c BANK_SETTLE_ENERGY 0x2a100004 ATM_MIN_CALLED 0x2a100008 ATM_LINK_LIST_TRIM 0x2a200004 ATM_VALUE_REPLACED @@ -1706,7 +1623,6 @@ 0xff000334 MSG_host_get_io_master 0xff000338 MSG_host_get_clock_service 0xff00033c MSG_kmod_get_info -0xff000340 MSG_host_zone_info 0xff000344 MSG_host_virtual_physical_table_info 0xff000348 MSG_host_ipc_hash_info 0xff00034c MSG_enable_bluebox diff --git a/bsd/kern/tty.c b/bsd/kern/tty.c index 580852a0c..61d59ee62 100644 --- a/bsd/kern/tty.c +++ b/bsd/kern/tty.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997-2007 Apple Inc. All rights reserved. + * Copyright (c) 1997-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -113,18 +113,8 @@ #include <dev/kmreg_com.h> #include <machine/cons.h> #include <sys/resource.h> /* averunnable */ - -/* - * Debugging assertions for tty locks - */ -#define TTY_DEBUG 1 -#if TTY_DEBUG -#define TTY_LOCK_OWNED(tp) do {lck_mtx_assert(&tp->t_lock, LCK_MTX_ASSERT_OWNED); } while (0) -#define TTY_LOCK_NOTOWNED(tp) do {lck_mtx_assert(&tp->t_lock, LCK_MTX_ASSERT_NOTOWNED); } while (0) -#else -#define TTY_LOCK_OWNED(tp) -#define TTY_LOCK_NOTOWNED(tp) -#endif +#include <kern/waitq.h> +#include <libkern/section_keywords.h> static lck_grp_t *tty_lck_grp; static lck_grp_attr_t *tty_lck_grp_attr; @@ -142,7 +132,7 @@ static void ttyunblock(struct tty *tp); static int ttywflush(struct tty *tp); static int proc_compare(proc_t p1, proc_t p2); -static void ttyhold(struct tty *tp); +void ttyhold(struct tty *tp); static void ttydeallocate(struct tty *tp); static int isctty(proc_t p, struct tty *tp); @@ -330,7 +320,6 @@ tty_unlock(struct tty *tp) lck_mtx_unlock(&tp->t_lock); } - /* * ttyopen (LDISC) * @@ -372,6 +361,7 @@ ttyclose(struct tty *tp) { struct pgrp * oldpg; struct session * oldsessp; + struct knote *kn; TTY_LOCK_OWNED(tp); /* debug assert */ @@ -408,8 +398,15 @@ ttyclose(struct tty *tp) pg_rele(oldpg); tty_lock(tp); tp->t_state = 0; + SLIST_FOREACH(kn, &tp->t_wsel.si_note, kn_selnext) { + KNOTE_DETACH(&tp->t_wsel.si_note, kn); + } selthreadclear(&tp->t_wsel); + SLIST_FOREACH(kn, &tp->t_rsel.si_note, kn_selnext) { + KNOTE_DETACH(&tp->t_rsel.si_note, kn); + } selthreadclear(&tp->t_rsel); + return (0); } @@ -1517,18 +1514,27 @@ int ttyselect(struct tty *tp, int rw, void *wql, proc_t p) { int retval = 0; + /* + * Attaching knotes to TTYs needs to call selrecord in order to hook + * up the waitq to the selinfo, regardless of data being ready. See + * filt_ttyattach. + */ + bool needs_selrecord = rw & FMARK; + rw &= ~FMARK; - if (tp == NULL) - return (ENXIO); + if (tp == NULL) { + return ENXIO; + } - TTY_LOCK_OWNED(tp); /* debug assert */ + TTY_LOCK_OWNED(tp); + + if (tp->t_state & TS_ZOMBIE) { + retval = 1; + goto out; + } switch (rw) { case FREAD: - if (ISSET(tp->t_state, TS_ZOMBIE)) { - return(1); - } - retval = ttnread(tp); if (retval > 0) { break; @@ -1537,12 +1543,8 @@ ttyselect(struct tty *tp, int rw, void *wql, proc_t p) selrecord(p, &tp->t_rsel, wql); break; case FWRITE: - if (ISSET(tp->t_state, TS_ZOMBIE)) { - return(1); - } - if ((tp->t_outq.c_cc <= tp->t_lowat) && - ISSET(tp->t_state, TS_CONNECTED)) { + (tp->t_state & TS_CONNECTED)) { retval = tp->t_hiwat - tp->t_outq.c_cc; break; } @@ -1550,6 +1552,19 @@ ttyselect(struct tty *tp, int rw, void *wql, proc_t p) selrecord(p, &tp->t_wsel, wql); break; } + +out: + if (retval > 0 && needs_selrecord) { + switch (rw) { + case FREAD: + selrecord(p, &tp->t_rsel, wql); + break; + case FWRITE: + selrecord(p, &tp->t_wsel, wql); + break; + } + } + return retval; } @@ -3066,7 +3081,7 @@ ttymalloc(void) /* * Increment the reference count on a tty. */ -static void +void ttyhold(struct tty *tp) { TTY_LOCK_OWNED(tp); @@ -3146,3 +3161,387 @@ isctty_sp(proc_t p, struct tty *tp, struct session *sessp) return(sessp == tp->t_session && p->p_flag & P_CONTROLT); } + + +static int filt_ttyattach(struct knote *kn, struct kevent_internal_s *kev); +static void filt_ttydetach(struct knote *kn); +static int filt_ttyevent(struct knote *kn, long hint); +static int filt_ttytouch(struct knote *kn, struct kevent_internal_s *kev); +static int filt_ttyprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); + +SECURITY_READ_ONLY_EARLY(struct filterops) tty_filtops = { + .f_isfd = 1, + .f_attach = filt_ttyattach, + .f_detach = filt_ttydetach, + .f_event = filt_ttyevent, + .f_touch = filt_ttytouch, + .f_process = filt_ttyprocess +}; + +/* + * Called with struct tty locked. Returns non-zero if there is data to be read + * or written. + */ +static int +filt_tty_common(struct knote *kn, struct tty *tp) +{ + int retval = 0; + + TTY_LOCK_OWNED(tp); /* debug assert */ + + if (tp->t_state & TS_ZOMBIE) { + kn->kn_flags |= EV_EOF; + return 1; + } + + switch (knote_get_seltype(kn)) { + case FREAD: + retval = ttnread(tp); + break; + case FWRITE: + if ((tp->t_outq.c_cc <= tp->t_lowat) && + (tp->t_state & TS_CONNECTED)) { + retval = tp->t_hiwat - tp->t_outq.c_cc; + } + break; + } + + kn->kn_data = retval; + + /* + * TODO(mwidmann, jandrus): For native knote low watermark support, + * check the kn_sfflags for NOTE_LOWAT and check against kn_sdata. + * + * res = ((kn->kn_sfflags & NOTE_LOWAT) != 0) ? + * (kn->kn_data >= kn->kn_sdata) : kn->kn_data; + */ + + return retval; +} + +/* + * Find the struct tty from a waitq, which is a member of one of the two struct + * selinfos inside the struct tty. Use the seltype to determine which selinfo. + */ +static struct tty * +tty_from_waitq(struct waitq *wq, int seltype) +{ + struct selinfo *si; + struct tty *tp = NULL; + + /* + * The waitq is part of the selinfo structure managed by the driver. For + * certain drivers, we want to hook the knote into the selinfo + * structure's si_note field so selwakeup can call KNOTE. + * + * While 'wq' is not really a queue element, this macro only uses the + * pointer to calculate the offset into a structure given an element + * name. + */ + si = qe_element(wq, struct selinfo, si_waitq); + + /* + * For TTY drivers, the selinfo structure is somewhere in the struct + * tty. There are two different selinfo structures, and the one used + * corresponds to the type of filter requested. + * + * While 'si' is not really a queue element, this macro only uses the + * pointer to calculate the offset into a structure given an element + * name. + */ + switch (seltype) { + case FREAD: + tp = qe_element(si, struct tty, t_rsel); + break; + case FWRITE: + tp = qe_element(si, struct tty, t_wsel); + break; + } + + return tp; +} + +static struct tty * +tty_from_knote(struct knote *kn) +{ + return (struct tty *)kn->kn_hook; +} + +/* + * Try to lock the TTY structure associated with a knote. + * + * On success, this function returns a locked TTY structure. Otherwise, NULL is + * returned. + */ +__attribute__((warn_unused_result)) +static struct tty * +tty_lock_from_knote(struct knote *kn) +{ + struct tty *tp = tty_from_knote(kn); + if (tp) { + tty_lock(tp); + } + + return tp; +} + +/* + * Set the knote's struct tty to the kn_hook field. + * + * The idea is to fake a call to select with our own waitq set. If the driver + * calls selrecord, we'll get a link to their waitq and access to the tty + * structure. + * + * Returns -1 on failure, with the error set in the knote, or selres on success. + */ +static int +tty_set_knote_hook(struct knote *kn) +{ + uthread_t uth; + vfs_context_t ctx; + vnode_t vp; + kern_return_t kr; + struct waitq *wq = NULL; + struct waitq_set *old_wqs; + struct waitq_set tmp_wqs; + uint64_t rsvd, rsvd_arg; + uint64_t *rlptr = NULL; + int selres = -1; + struct tty *tp; + + uth = get_bsdthread_info(current_thread()); + + ctx = vfs_context_current(); + vp = (vnode_t)kn->kn_fp->f_fglob->fg_data; + + /* + * Reserve a link element to avoid potential allocation under + * a spinlock. + */ + rsvd = rsvd_arg = waitq_link_reserve(NULL); + rlptr = (void *)&rsvd_arg; + + /* + * Trick selrecord into hooking a known waitq set into the device's selinfo + * waitq. Once the link is in place, we can get back into the selinfo from + * the waitq and subsequently the tty (see tty_from_waitq). + * + * We can't use a real waitq set (such as the kqueue's) because wakeups + * might happen before we can unlink it. + */ + kr = waitq_set_init(&tmp_wqs, SYNC_POLICY_FIFO | SYNC_POLICY_PREPOST, NULL, + NULL); + assert(kr == KERN_SUCCESS); + + old_wqs = uth->uu_wqset; + uth->uu_wqset = &tmp_wqs; + /* + * FMARK forces selects to always call selrecord, even if data is + * available. See ttselect, ptsselect, ptcselect. + * + * selres also contains the data currently available in the tty. + */ + selres = VNOP_SELECT(vp, knote_get_seltype(kn) | FMARK, 0, rlptr, ctx); + uth->uu_wqset = old_wqs; + + /* + * Make sure to cleanup the reserved link - this guards against + * drivers that may not actually call selrecord(). + */ + waitq_link_release(rsvd); + if (rsvd == rsvd_arg) { + /* + * The driver didn't call selrecord -- there's no tty hooked up so we + * can't attach. + */ + knote_set_error(kn, ENOTTY); + selres = -1; + goto out; + } + + /* rlptr may not point to a properly aligned pointer */ + memcpy(&wq, rlptr, sizeof(void *)); + + tp = tty_from_waitq(wq, knote_get_seltype(kn)); + assert(tp != NULL); + + /* + * Take a reference and stash the tty in the knote. + */ + tty_lock(tp); + ttyhold(tp); + kn->kn_hook = tp; + tty_unlock(tp); + +out: + /* + * Cleaning up the wqset will unlink its waitq and clean up any preposts + * that occurred as a result of data coming in while the tty was attached. + */ + waitq_set_deinit(&tmp_wqs); + + return selres; +} + +static int +filt_ttyattach(struct knote *kn, __unused struct kevent_internal_s *kev) +{ + int selres = 0; + struct tty *tp; + + /* + * This function should be called from filt_specattach (spec_vnops.c), + * so most of the knote data structure should already be initialized. + */ + + /* don't support offsets in ttys or drivers that don't use struct tty */ + if (kn->kn_vnode_use_ofst || !kn->kn_vnode_kqok) { + knote_set_error(kn, ENOTSUP); + return 0; + } + + /* + * Connect the struct tty to the knote through the selinfo structure + * referenced by the waitq within the selinfo. + */ + selres = tty_set_knote_hook(kn); + if (selres < 0) { + return 0; + } + + /* + * Attach the knote to selinfo's klist. + */ + tp = tty_lock_from_knote(kn); + if (!tp) { + knote_set_error(kn, ENOENT); + return 0; + } + + switch (knote_get_seltype(kn)) { + case FREAD: + KNOTE_ATTACH(&tp->t_rsel.si_note, kn); + break; + case FWRITE: + KNOTE_ATTACH(&tp->t_wsel.si_note, kn); + break; + } + + tty_unlock(tp); + + return selres; +} + +static void +filt_ttydetach(struct knote *kn) +{ + struct tty *tp; + + tp = tty_lock_from_knote(kn); + if (!tp) { + knote_set_error(kn, ENOENT); + return; + } + + struct selinfo *si = NULL; + switch (knote_get_seltype(kn)) { + case FREAD: + si = &tp->t_rsel; + break; + case FWRITE: + si = &tp->t_wsel; + break; + /* knote_get_seltype will panic on default */ + } + + KNOTE_DETACH(&si->si_note, kn); + kn->kn_hook = NULL; + + tty_unlock(tp); + ttyfree(tp); +} + +static int +filt_ttyevent(struct knote *kn, long hint) +{ + int ret; + struct tty *tp; + bool revoked = hint & NOTE_REVOKE; + hint &= ~NOTE_REVOKE; + + tp = tty_from_knote(kn); + if (!tp) { + knote_set_error(kn, ENOENT); + return 0; + } + + if (!hint) { + tty_lock(tp); + } + + if (revoked) { + kn->kn_flags |= EV_EOF | EV_ONESHOT; + ret = 1; + } else { + ret = filt_tty_common(kn, tp); + } + + if (!hint) { + tty_unlock(tp); + } + + return ret; +} + +static int +filt_ttytouch(struct knote *kn, struct kevent_internal_s *kev) +{ + struct tty *tp; + int res = 0; + + tp = tty_lock_from_knote(kn); + if (!tp) { + knote_set_error(kn, ENOENT); + return 0; + } + + kn->kn_sdata = kev->data; + kn->kn_sfflags = kev->fflags; + if ((kn->kn_status & KN_UDATA_SPECIFIC) == 0) + kn->kn_udata = kev->udata; + + if (kn->kn_vnode_kqok) { + res = filt_tty_common(kn, tp); + } + + tty_unlock(tp); + + return res; +} + +static int +filt_ttyprocess(struct knote *kn, __unused struct filt_process_s *data, struct kevent_internal_s *kev) +{ + struct tty *tp; + int res; + + tp = tty_lock_from_knote(kn); + if (!tp) { + knote_set_error(kn, ENOENT); + return 0; + } + + res = filt_tty_common(kn, tp); + + if (res) { + *kev = kn->kn_kevent; + if (kn->kn_flags & EV_CLEAR) { + kn->kn_fflags = 0; + kn->kn_data = 0; + } + } + + tty_unlock(tp); + + return res; +} diff --git a/bsd/kern/tty_ptmx.c b/bsd/kern/tty_ptmx.c index 43db005d0..cf89b93eb 100644 --- a/bsd/kern/tty_ptmx.c +++ b/bsd/kern/tty_ptmx.c @@ -2,7 +2,7 @@ * Copyright (c) 1997-2013 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* @@ -82,6 +82,7 @@ #include <sys/sysctl.h> #include <miscfs/devfs/devfs.h> #include <miscfs/devfs/devfsdefs.h> /* DEVFS_LOCK()/DEVFS_UNLOCK() */ +#include <libkern/section_keywords.h> #if CONFIG_MACF #include <security/mac_framework.h> @@ -169,39 +170,6 @@ SYSCTL_PROC(_kern_tty, OID_AUTO, ptmx_max, static int ptmx_clone(dev_t dev, int minor); -/* - * Set of locks to keep the interaction between kevents and revoke - * from causing havoc. - */ - -#define LOG2_PTSD_KE_NLCK 2 -#define PTSD_KE_NLCK (1l << LOG2_PTSD_KE_NLCK) -#define PTSD_KE_LOCK_INDEX(x) ((x) & (PTSD_KE_NLCK - 1)) - -static lck_mtx_t ptsd_kevent_lock[PTSD_KE_NLCK]; - -static void -ptsd_kevent_lock_init(void) -{ - int i; - lck_grp_t *lgrp = lck_grp_alloc_init("ptsd kevent", LCK_GRP_ATTR_NULL); - - for (i = 0; i < PTSD_KE_NLCK; i++) - lck_mtx_init(&ptsd_kevent_lock[i], lgrp, LCK_ATTR_NULL); -} - -static void -ptsd_kevent_mtx_lock(int minor) -{ - lck_mtx_lock(&ptsd_kevent_lock[PTSD_KE_LOCK_INDEX(minor)]); -} - -static void -ptsd_kevent_mtx_unlock(int minor) -{ - lck_mtx_unlock(&ptsd_kevent_lock[PTSD_KE_LOCK_INDEX(minor)]); -} - static struct tty_dev_t _ptmx_driver; int @@ -213,12 +181,12 @@ ptmx_init( __unused int config_count) */ /* Get a major number for /dev/ptmx */ - if((ptmx_major = cdevsw_add(-15, &ptmx_cdev)) == -1) { + if ((ptmx_major = cdevsw_add(-15, &ptmx_cdev)) == -1) { printf("ptmx_init: failed to obtain /dev/ptmx major number\n"); return (ENOENT); } - if (cdevsw_setkqueueok(ptmx_major, &ptmx_cdev, 0) == -1) { + if (cdevsw_setkqueueok(ptmx_major, &ptmx_cdev, CDEVSW_IS_PTC) == -1) { panic("Failed to set flags on ptmx cdevsw entry."); } @@ -228,16 +196,11 @@ ptmx_init( __unused int config_count) printf("ptmx_init: failed to obtain /dev/ptmx major number\n"); return (ENOENT); } - - if (cdevsw_setkqueueok(ptsd_major, &ptsd_cdev, 0) == -1) { + + if (cdevsw_setkqueueok(ptsd_major, &ptsd_cdev, CDEVSW_IS_PTS) == -1) { panic("Failed to set flags on ptmx cdevsw entry."); } - /* - * Locks to guard against races between revoke and kevents - */ - ptsd_kevent_lock_init(); - /* Create the /dev/ptmx device {<major>,0} */ (void)devfs_make_node_clone(makedev(ptmx_major, 0), DEVFS_CHAR, UID_ROOT, GID_TTY, 0666, @@ -341,8 +304,8 @@ ptmx_get_ioctl(int minor, int open_flag) _state.pis_total += PTMX_GROW_VECTOR; if (old_pis_ioctl_list) FREE(old_pis_ioctl_list, M_TTYS); - } - + } + /* is minor in range now? */ if (minor < 0 || minor >= _state.pis_total) { ttyfree(new_ptmx_ioctl->pt_tty); @@ -350,14 +313,14 @@ ptmx_get_ioctl(int minor, int open_flag) FREE(new_ptmx_ioctl, M_TTYS); return (NULL); } - + if (_state.pis_ioctl_list[minor] != NULL) { ttyfree(new_ptmx_ioctl->pt_tty); DEVFS_UNLOCK(); FREE(new_ptmx_ioctl, M_TTYS); /* Special error value so we know to redrive the open, we've been raced */ - return (struct ptmx_ioctl*)-1; + return (struct ptmx_ioctl*)-1; } @@ -381,11 +344,11 @@ ptmx_get_ioctl(int minor, int open_flag) printf("devfs_make_node() call failed for ptmx_get_ioctl()!!!!\n"); } } - + if (minor < 0 || minor >= _state.pis_total) { return (NULL); } - + return (_state.pis_ioctl_list[minor]); } @@ -398,7 +361,7 @@ ptmx_free_ioctl(int minor, int open_flag) struct ptmx_ioctl *old_ptmx_ioctl = NULL; DEVFS_LOCK(); - + if (minor < 0 || minor >= _state.pis_total) { DEVFS_UNLOCK(); return (-1); @@ -498,22 +461,20 @@ ptmx_clone(__unused dev_t dev, int action) /* * kqueue support. */ -int ptsd_kqfilter(dev_t, struct knote *); +int ptsd_kqfilter(dev_t dev, struct knote *kn); static void ptsd_kqops_detach(struct knote *); static int ptsd_kqops_event(struct knote *, long); static int ptsd_kqops_touch(struct knote *kn, struct kevent_internal_s *kev); static int ptsd_kqops_process(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); -struct filterops ptsd_kqops = { +SECURITY_READ_ONLY_EARLY(struct filterops) ptsd_kqops = { .f_isfd = 1, + /* attach is handled by ptsd_kqfilter -- the dev node must be passed in */ .f_detach = ptsd_kqops_detach, .f_event = ptsd_kqops_event, .f_touch = ptsd_kqops_touch, .f_process = ptsd_kqops_process, -}; - -#define PTSD_KNOTE_VALID NULL -#define PTSD_KNOTE_REVOKED ((void *)-911l) +}; /* * In the normal case, by the time the driver_close() routine is called @@ -527,243 +488,210 @@ struct filterops ptsd_kqops = { static void ptsd_kqops_detach(struct knote *kn) { - struct ptmx_ioctl *pti; struct tty *tp; - dev_t dev, lockdev = (dev_t)kn->kn_hookid; - - ptsd_kevent_mtx_lock(minor(lockdev)); - - if ((dev = (dev_t)kn->kn_hookid) != 0) { - pti = ptmx_get_ioctl(minor(dev), 0); - if (pti != NULL && (tp = pti->pt_tty) != NULL) { - tty_lock(tp); - if (kn->kn_filter == EVFILT_READ) - KNOTE_DETACH(&tp->t_rsel.si_note, kn); - else - KNOTE_DETACH(&tp->t_wsel.si_note, kn); - tty_unlock(tp); - kn->kn_hookid = 0; + + tp = kn->kn_hook; + assert(tp != NULL); + + tty_lock(tp); + + /* + * Only detach knotes from open ttys -- ttyclose detaches all knotes + * under the lock and unsets TS_ISOPEN. + */ + if (tp->t_state & TS_ISOPEN) { + switch (kn->kn_filter) { + case EVFILT_READ: + KNOTE_DETACH(&tp->t_rsel.si_note, kn); + break; + + case EVFILT_WRITE: + KNOTE_DETACH(&tp->t_wsel.si_note, kn); + break; + + default: + panic("invalid knote %p detach, filter: %d", kn, kn->kn_filter); + break; } } - ptsd_kevent_mtx_unlock(minor(lockdev)); + kn->kn_hook = NULL; + tty_unlock(tp); + + ttyfree(tp); } static int -ptsd_kqops_common(struct knote *kn, dev_t dev, long hint) +ptsd_kqops_common(struct knote *kn, struct tty *tp) { - struct ptmx_ioctl *pti; - struct tty *tp; int retval = 0; - do { - if (kn->kn_hook != PTSD_KNOTE_VALID ) { - /* We were revoked */ - kn->kn_data = 0; - kn->kn_flags |= EV_EOF; - retval = 1; - break; - } + TTY_LOCK_OWNED(tp); - pti = ptmx_get_ioctl(minor(dev), 0); - if (pti == NULL || (tp = pti->pt_tty) == NULL) { - kn->kn_data = ENXIO; - kn->kn_flags |= EV_ERROR; + switch (kn->kn_filter) { + case EVFILT_READ: + kn->kn_data = ttnread(tp); + if (kn->kn_data > 0) { retval = 1; - break; } + break; - if (hint == 0) - tty_lock(tp); - - if (kn->kn_filter == EVFILT_READ) { - kn->kn_data = ttnread(tp); - if (kn->kn_data > 0) - retval = 1; - if (ISSET(tp->t_state, TS_ZOMBIE)) { - kn->kn_flags |= EV_EOF; - retval = 1; - } - } else { /* EVFILT_WRITE */ - if ((tp->t_outq.c_cc <= tp->t_lowat) && - ISSET(tp->t_state, TS_CONNECTED)) { - kn->kn_data = tp->t_outq.c_cn - tp->t_outq.c_cc; - retval = 1; - } - if (ISSET(tp->t_state, TS_ZOMBIE)) { - kn->kn_flags |= EV_EOF; - retval = 1; - } + case EVFILT_WRITE: + if ((tp->t_outq.c_cc <= tp->t_lowat) && + (tp->t_state & TS_CONNECTED)) { + kn->kn_data = tp->t_outq.c_cn - tp->t_outq.c_cc; + retval = 1; } + break; - if (hint == 0) - tty_unlock(tp); + default: + panic("ptsd kevent: unexpected filter: %d, kn = %p, tty = %p", + kn->kn_filter, kn, tp); + break; + } - } while (0); + if (tp->t_state & TS_ZOMBIE) { + kn->kn_flags |= EV_EOF; + retval = 1; + } - return (retval); -} + return retval; +} static int ptsd_kqops_event(struct knote *kn, long hint) { - dev_t dev = (dev_t)kn->kn_hookid; - int res; + struct tty *tp = kn->kn_hook; + int ret; + bool revoked = hint & NOTE_REVOKE; + hint &= ~NOTE_REVOKE; - ptsd_kevent_mtx_lock(minor(dev)); - res = ptsd_kqops_common(kn, dev, hint); - ptsd_kevent_mtx_unlock(minor(dev)); - return res; + if (!hint) { + tty_lock(tp); + } + + if (revoked) { + kn->kn_flags |= EV_EOF | EV_ONESHOT; + ret = 1; + } else { + ret = ptsd_kqops_common(kn, tp); + } + + if (!hint) { + tty_unlock(tp); + } + + return ret; } - static int ptsd_kqops_touch(struct knote *kn, struct kevent_internal_s *kev) { - dev_t dev = (dev_t)kn->kn_hookid; - int res; + struct tty *tp; + int ret; + + tp = kn->kn_hook; - ptsd_kevent_mtx_lock(minor(dev)); + tty_lock(tp); /* accept new kevent state */ kn->kn_sfflags = kev->fflags; kn->kn_sdata = kev->data; - if ((kn->kn_status & KN_UDATA_SPECIFIC) == 0) + if ((kn->kn_status & KN_UDATA_SPECIFIC) == 0) { kn->kn_udata = kev->udata; + } /* recapture fired state of knote */ - res = ptsd_kqops_common(kn, dev, 0); + ret = ptsd_kqops_common(kn, tp); - ptsd_kevent_mtx_unlock(minor(dev)); + tty_unlock(tp); - return res; + return ret; } static int -ptsd_kqops_process(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev) +ptsd_kqops_process(struct knote *kn, __unused struct filt_process_s *data, + struct kevent_internal_s *kev) { -#pragma unused(data) - dev_t dev = (dev_t)kn->kn_hookid; - int res; + struct tty *tp = kn->kn_hook; + int ret; - ptsd_kevent_mtx_lock(minor(dev)); - res = ptsd_kqops_common(kn, dev, 0); - if (res) { + tty_lock(tp); + ret = ptsd_kqops_common(kn, tp); + if (ret) { *kev = kn->kn_kevent; if (kn->kn_flags & EV_CLEAR) { kn->kn_fflags = 0; kn->kn_data = 0; } } - ptsd_kevent_mtx_unlock(minor(dev)); - return res; + tty_unlock(tp); + + return ret; } int ptsd_kqfilter(dev_t dev, struct knote *kn) { - struct tty *tp = NULL; + struct tty *tp = NULL; struct ptmx_ioctl *pti = NULL; - int retval = 0; + int ret; /* make sure we're talking about the right device type */ if (cdevsw[major(dev)].d_open != ptsopen) { - kn->kn_flags = EV_ERROR; - kn->kn_data = EINVAL; + knote_set_error(kn, ENODEV); return 0; } if ((pti = ptmx_get_ioctl(minor(dev), 0)) == NULL) { - kn->kn_flags = EV_ERROR; - kn->kn_data = ENXIO; - return 0; + knote_set_error(kn, ENXIO); + return 0; } tp = pti->pt_tty; tty_lock(tp); - kn->kn_hookid = dev; - kn->kn_hook = PTSD_KNOTE_VALID; - kn->kn_filtid = EVFILTID_PTSD; - - switch (kn->kn_filter) { - case EVFILT_READ: - KNOTE_ATTACH(&tp->t_rsel.si_note, kn); - break; - case EVFILT_WRITE: - KNOTE_ATTACH(&tp->t_wsel.si_note, kn); - break; - default: - kn->kn_flags = EV_ERROR; - kn->kn_data = EINVAL; - break; - } - - tty_unlock(tp); + assert(tp->t_state & TS_ISOPEN); - ptsd_kevent_mtx_lock(minor(dev)); + kn->kn_filtid = EVFILTID_PTSD; + /* the tty will be freed when detaching the knote */ + ttyhold(tp); + kn->kn_hook = tp; + + switch (kn->kn_filter) { + case EVFILT_READ: + KNOTE_ATTACH(&tp->t_rsel.si_note, kn); + break; + case EVFILT_WRITE: + KNOTE_ATTACH(&tp->t_wsel.si_note, kn); + break; + default: + panic("ptsd kevent: unexpected filter: %d, kn = %p, tty = %p", + kn->kn_filter, kn, tp); + break; + } /* capture current event state */ - retval = ptsd_kqops_common(kn, dev, 0); + ret = ptsd_kqops_common(kn, tp); - ptsd_kevent_mtx_unlock(minor(dev)); + tty_unlock(tp); - return (retval); + return ret; } /* * Support for revoke(2). - * - * Mark all the kn_hook fields so that future invocations of the - * f_event op will just say "EOF" *without* looking at the - * ptmx_ioctl structure (which may disappear or be recycled at - * the end of ptsd_close). Issue wakeups to post that EOF to - * anyone listening. And finally remove the knotes from the - * tty's klists to keep ttyclose() happy, and set the hookid to - * zero to make the final detach passively successful. */ static void -ptsd_revoke_knotes(int minor, struct tty *tp) +ptsd_revoke_knotes(__unused int minor, struct tty *tp) { - struct klist *list; - struct knote *kn, *tkn; - - /* (Hold and drop the right locks in the right order.) */ - - ptsd_kevent_mtx_lock(minor); tty_lock(tp); - list = &tp->t_rsel.si_note; - SLIST_FOREACH(kn, list, kn_selnext) - kn->kn_hook = PTSD_KNOTE_REVOKED; - - list = &tp->t_wsel.si_note; - SLIST_FOREACH(kn, list, kn_selnext) - kn->kn_hook = PTSD_KNOTE_REVOKED; - - tty_unlock(tp); - ptsd_kevent_mtx_unlock(minor); - - tty_lock(tp); ttwakeup(tp); - ttwwakeup(tp); - tty_unlock(tp); + KNOTE(&tp->t_rsel.si_note, NOTE_REVOKE | 1 /* the lock is already held */); - ptsd_kevent_mtx_lock(minor); - tty_lock(tp); - - list = &tp->t_rsel.si_note; - SLIST_FOREACH_SAFE(kn, list, kn_selnext, tkn) { - (void) KNOTE_DETACH(list, kn); - kn->kn_hookid = 0; - } - - list = &tp->t_wsel.si_note; - SLIST_FOREACH_SAFE(kn, list, kn_selnext, tkn) { - (void) KNOTE_DETACH(list, kn); - kn->kn_hookid = 0; - } + ttwwakeup(tp); + KNOTE(&tp->t_wsel.si_note, NOTE_REVOKE | 1); tty_unlock(tp); - ptsd_kevent_mtx_unlock(minor); } diff --git a/bsd/kern/tty_pty.c b/bsd/kern/tty_pty.c index 7c4f14e6a..c5f899a46 100644 --- a/bsd/kern/tty_pty.c +++ b/bsd/kern/tty_pty.c @@ -92,12 +92,9 @@ #warning You have only one pty defined, redefining to 32. #endif -#define PTY_MAJOR 5 -#define TTY_MAJOR 4 - /* - * pts == /dev/tty[pqrsPQRS][0123456789abcdefghijklmnopqrstuv] - * ptc == /dev/pty[pqrsPQRS][0123456789abcdefghijklmnopqrstuv] + * pts == /dev/tty[pqrsPQRS][0-9a-v] + * ptc == /dev/pty[pqrsPQRS][0-9a-v] */ static struct ptmx_ioctl pt_ioctl[NPTY]; @@ -154,23 +151,32 @@ pty_init(int n_ptys) for (j = 0; j < 10; j++) { for (i = 0; i < HEX_BASE; i++) { int m = j * HEX_BASE + i; - if (m >= n_ptys) + if (m >= n_ptys) { goto done; - pt_ioctl[m].pt_devhandle = devfs_make_node(makedev(TTY_MAJOR, m), - DEVFS_CHAR, UID_ROOT, GID_WHEEL, 0666, - "tty%c%x", j + START_CHAR, i); - (void)devfs_make_node(makedev(PTY_MAJOR, m), - DEVFS_CHAR, UID_ROOT, GID_WHEEL, 0666, - "pty%c%x", j + START_CHAR, i); + } + pt_ioctl[m].pt_devhandle = devfs_make_node(makedev(PTS_MAJOR, m), + DEVFS_CHAR, UID_ROOT, GID_WHEEL, 0666, + "tty%c%x", j + START_CHAR, i); + (void)devfs_make_node(makedev(PTC_MAJOR, m), + DEVFS_CHAR, UID_ROOT, GID_WHEEL, 0666, + "pty%c%x", j + START_CHAR, i); } } + done: - _pty_driver.master = PTY_MAJOR; - _pty_driver.slave = TTY_MAJOR; + _pty_driver.master = PTC_MAJOR; + _pty_driver.slave = PTS_MAJOR; _pty_driver.open_reset = 1; _pty_driver.open = &pty_get_ioctl; _pty_driver.name = &pty_get_name; tty_dev_register(&_pty_driver); - return (0); + + if (cdevsw_setkqueueok(PTC_MAJOR, &cdevsw[PTC_MAJOR], CDEVSW_IS_PTC) == -1) { + panic("Can't mark ptc as kqueue ok"); + } + if (cdevsw_setkqueueok(PTS_MAJOR, &cdevsw[PTS_MAJOR], CDEVSW_IS_PTS) == -1) { + panic("Can't mark pts as kqueue ok"); + } + return 0; } #endif // DEVFS diff --git a/bsd/kern/ubc_subr.c b/bsd/kern/ubc_subr.c index 8e6f56929..e05d13c3e 100644 --- a/bsd/kern/ubc_subr.c +++ b/bsd/kern/ubc_subr.c @@ -65,6 +65,7 @@ #include <kern/kalloc.h> #include <kern/zalloc.h> #include <kern/thread.h> +#include <vm/pmap.h> #include <vm/vm_kern.h> #include <vm/vm_protos.h> /* last */ @@ -158,6 +159,12 @@ struct cs_hash { cs_md_final cs_final; }; +uint8_t cs_hash_type( + struct cs_hash const * const cs_hash) +{ + return cs_hash->cs_type; +} + static const struct cs_hash cs_hash_sha1 = { .cs_type = CS_HASHTYPE_SHA1, .cs_size = CS_SHA1_LEN, @@ -495,6 +502,9 @@ cs_validate_csblob( uint32_t n, count; const CS_CodeDirectory *best_cd = NULL; unsigned int best_rank = 0; +#if PLATFORM_WatchOS + const CS_CodeDirectory *sha1_cd = NULL; +#endif if (length < sizeof(CS_SuperBlob)) return EBADEXEC; @@ -543,6 +553,15 @@ cs_validate_csblob( printf("multiple hash=%d CodeDirectories in signature; rejecting\n", best_cd->hashType); return EBADEXEC; } +#if PLATFORM_WatchOS + if (candidate->hashType == CS_HASHTYPE_SHA1) { + if (sha1_cd != NULL) { + printf("multiple sha1 CodeDirectories in signature; rejecting\n"); + return EBADEXEC; + } + sha1_cd = candidate; + } +#endif } else if (type == CSSLOT_ENTITLEMENTS) { if (ntohl(subBlob->magic) != CSMAGIC_EMBEDDED_ENTITLEMENTS) { return EBADEXEC; @@ -555,6 +574,37 @@ cs_validate_csblob( } } +#if PLATFORM_WatchOS + /* To keep watchOS fast enough, we have to resort to sha1 for + * some code. + * + * At the time of writing this comment, known sha1 attacks are + * collision attacks (not preimage or second preimage + * attacks), which do not apply to platform binaries since + * they have a fixed hash in the trust cache. Given this + * property, we only prefer sha1 code directories for adhoc + * signatures, which always have to be in a trust cache to be + * valid (can-load-cdhash does not exist for watchOS). Those + * are, incidentally, also the platform binaries, for which we + * care about the performance hit that sha256 would bring us. + * + * Platform binaries may still contain a (not chosen) sha256 + * code directory, which keeps software updates that switch to + * sha256-only small. + */ + + if (*rcd != NULL && sha1_cd != NULL && (ntohl(sha1_cd->flags) & CS_ADHOC)) { + if (sha1_cd->flags != (*rcd)->flags) { + printf("mismatched flags between hash %d (flags: %#x) and sha1 (flags: %#x) cd.\n", + (int)(*rcd)->hashType, (*rcd)->flags, sha1_cd->flags); + *rcd = NULL; + return EBADEXEC; + } + + *rcd = sha1_cd; + } +#endif + } else if (ntohl(blob->magic) == CSMAGIC_CODEDIRECTORY) { if ((error = cs_validate_codedirectory((const CS_CodeDirectory *)(const void *)addr, length)) != 0) @@ -1080,7 +1130,7 @@ errno_t ubc_setsize_ex(struct vnode *vp, off_t nsize, ubc_setsize_opts_t opts) * zero the tail of this page if it's currently * present in the cache */ - kret = ubc_create_upl(vp, lastpg, PAGE_SIZE, &upl, &pl, UPL_SET_LITE); + kret = ubc_create_upl_kernel(vp, lastpg, PAGE_SIZE, &upl, &pl, UPL_SET_LITE, VM_KERN_MEMORY_FILE); if (kret != KERN_SUCCESS) panic("ubc_setsize: ubc_create_upl (error = %d)\n", kret); @@ -2285,13 +2335,26 @@ ubc_range_op( * ubc_upl_abort(), or ubc_upl_abort_range(). */ kern_return_t -ubc_create_upl( +ubc_create_upl_external( struct vnode *vp, off_t f_offset, int bufsize, upl_t *uplp, upl_page_info_t **plp, int uplflags) +{ + return (ubc_create_upl_kernel(vp, f_offset, bufsize, uplp, plp, uplflags, vm_tag_bt())); +} + +kern_return_t +ubc_create_upl_kernel( + struct vnode *vp, + off_t f_offset, + int bufsize, + upl_t *uplp, + upl_page_info_t **plp, + int uplflags, + vm_tag_t tag) { memory_object_control_t control; kern_return_t kr; @@ -2351,7 +2414,7 @@ ubc_create_upl( if (control == MEMORY_OBJECT_CONTROL_NULL) return KERN_INVALID_ARGUMENT; - kr = memory_object_upl_request(control, f_offset, bufsize, uplp, NULL, NULL, uplflags); + kr = memory_object_upl_request(control, f_offset, bufsize, uplp, NULL, NULL, uplflags, tag); if (kr == KERN_SUCCESS && plp != NULL) *plp = UPL_GET_INTERNAL_PAGE_LIST(*uplp); return kr; @@ -3080,6 +3143,7 @@ ubc_cs_blob_add( blob->csb_mem_offset = 0; blob->csb_mem_kaddr = *addr; blob->csb_flags = 0; + blob->csb_signer_type = CS_SIGNER_TYPE_UNKNOWN; blob->csb_platform_binary = 0; blob->csb_platform_path = 0; blob->csb_teamid = NULL; @@ -3127,20 +3191,20 @@ ubc_cs_blob_add( kr); } } else { - memcpy(new_blob_addr, blob->csb_mem_kaddr, size); + memcpy((void *)new_blob_addr, (void *)blob->csb_mem_kaddr, size); if (cd == NULL) { new_cd = NULL; } else { - new_cd = ((uintptr_t)cd + new_cd = (void *)(((uintptr_t)cd - (uintptr_t)blob->csb_mem_kaddr - + (uintptr_t)new_blob_addr); + + (uintptr_t)new_blob_addr)); } if (entitlements == NULL) { new_entitlements = NULL; } else { - new_entitlements = ((uintptr_t)entitlements + new_entitlements = (void *)(((uintptr_t)entitlements - (uintptr_t)blob->csb_mem_kaddr - + (uintptr_t)new_blob_addr); + + (uintptr_t)new_blob_addr)); } // printf("CODE SIGNING: %s:%d kaddr 0x%llx cd %p ents %p -> blob 0x%llx cd %p ents %p\n", __FUNCTION__, __LINE__, (uint64_t)blob->csb_mem_kaddr, cd, entitlements, (uint64_t)new_blob_addr, new_cd, new_entitlements); ubc_cs_blob_deallocate(blob->csb_mem_kaddr, @@ -3187,8 +3251,10 @@ ubc_cs_blob_add( */ #if CONFIG_MACF unsigned int cs_flags = blob->csb_flags; - error = mac_vnode_check_signature(vp, blob, imgp, &cs_flags, flags); + unsigned int signer_type = blob->csb_signer_type; + error = mac_vnode_check_signature(vp, blob, imgp, &cs_flags, &signer_type, flags); blob->csb_flags = cs_flags; + blob->csb_signer_type = signer_type; if (error) { if (cs_debug) @@ -3201,8 +3267,8 @@ ubc_cs_blob_add( error = EPERM; goto out; } -#endif - +#endif + if (blob->csb_flags & CS_PLATFORM_BINARY) { if (cs_debug > 1) printf("check_signature[pid: %d]: platform binary\n", current_proc()->p_pid); @@ -3252,8 +3318,11 @@ ubc_cs_blob_add( oblob = oblob->csb_next) { off_t oblob_start_offset, oblob_end_offset; - /* check for conflicting teamid */ - if (blob->csb_platform_binary) { //platform binary needs to be the same for app slices + if (blob->csb_signer_type != oblob->csb_signer_type) { // signer type needs to be the same for slices + vnode_unlock(vp); + error = EALREADY; + goto out; + } else if (blob->csb_platform_binary) { //platform binary needs to be the same for app slices if (!oblob->csb_platform_binary) { vnode_unlock(vp); error = EALREADY; @@ -3575,20 +3644,22 @@ ubc_cs_blob_revalidate( assert(size == blob->csb_mem_size); unsigned int cs_flags = (ntohl(cd->flags) & CS_ALLOWED_MACHO) | CS_VALID; - + unsigned int signer_type = CS_SIGNER_TYPE_UNKNOWN; /* callout to mac_vnode_check_signature */ #if CONFIG_MACF - error = mac_vnode_check_signature(vp, blob, imgp, &cs_flags, flags); + error = mac_vnode_check_signature(vp, blob, imgp, &cs_flags, &signer_type, flags); if (cs_debug && error) { printf("revalidate: check_signature[pid: %d], error = %d\n", current_proc()->p_pid, error); } #else (void)flags; + (void)signer_type; #endif /* update generation number if success */ vnode_lock_spin(vp); blob->csb_flags = cs_flags; + blob->csb_signer_type = signer_type; if (UBCINFOEXISTS(vp)) { if (error == 0) vp->v_ubcinfo->cs_add_gen = cs_blob_generation_count; @@ -3957,7 +4028,7 @@ ubc_cs_is_range_codesigned( } #if CHECK_CS_VALIDATION_BITMAP -#define stob(s) ((atop_64((s)) + 07) >> 3) +#define stob(s) (((atop_64(round_page_64(s))) + 07) >> 3) extern boolean_t root_fs_upgrade_try; /* diff --git a/bsd/kern/uipc_domain.c b/bsd/kern/uipc_domain.c index 7fde6ee3e..e03a08e6d 100644 --- a/bsd/kern/uipc_domain.c +++ b/bsd/kern/uipc_domain.c @@ -76,10 +76,14 @@ #include <sys/queue.h> #include <net/dlil.h> +#include <net/nwk_wq.h> #include <mach/boolean.h> #include <pexpert/pexpert.h> +/* Eventhandler context for protocol events */ +struct eventhandler_lists_ctxt protoctl_evhdlr_ctxt; + static void pr_init_old(struct protosw *, struct domain *); static void init_proto(struct protosw *, struct domain *); static void attach_proto(struct protosw *, struct domain *); @@ -102,7 +106,7 @@ static lck_grp_attr_t *domain_proto_mtx_grp_attr; decl_lck_mtx_data(static, domain_proto_mtx); decl_lck_mtx_data(static, domain_timeout_mtx); -static u_int64_t _net_uptime; +u_int64_t _net_uptime; #if (DEVELOPMENT || DEBUG) @@ -637,7 +641,7 @@ done: static void domain_sched_timeout(void) { - lck_mtx_assert(&domain_timeout_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&domain_timeout_mtx, LCK_MTX_ASSERT_OWNED); if (!domain_timeout_run && domain_draining) { domain_timeout_run = TRUE; @@ -705,6 +709,7 @@ domaininit(void) struct domain *dp; domain_guard_t guard; + eventhandler_lists_ctxt_init(&protoctl_evhdlr_ctxt); /* * allocate lock group attribute and group for domain mutexes */ @@ -940,31 +945,32 @@ pfctlinput2(int cmd, struct sockaddr *sa, void *ctlparam) TAILQ_FOREACH(dp, &domains, dom_entry) { TAILQ_FOREACH(pp, &dp->dom_protosw, pr_entry) { if (pp->pr_ctlinput != NULL) - (*pp->pr_ctlinput)(cmd, sa, ctlparam); + (*pp->pr_ctlinput)(cmd, sa, ctlparam, NULL); } } domain_guard_release(guard); } void -net_update_uptime(void) +net_update_uptime_with_time(const struct timeval *tvp) { - struct timeval tv; - - microuptime(&tv); - _net_uptime = tv.tv_sec; + _net_uptime = tvp->tv_sec; /* * Round up the timer to the nearest integer value because otherwise * we might setup networking timers that are off by almost 1 second. */ - if (tv.tv_usec > 500000) + if (tvp->tv_usec > 500000) _net_uptime++; } void -net_update_uptime_secs(uint64_t secs) +net_update_uptime(void) { - _net_uptime = secs; + struct timeval tv; + + microuptime(&tv); + + net_update_uptime_with_time(&tv); } /* @@ -997,13 +1003,13 @@ net_uptime(void) void domain_proto_mtx_lock_assert_held(void) { - lck_mtx_assert(&domain_proto_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&domain_proto_mtx, LCK_MTX_ASSERT_OWNED); } void domain_proto_mtx_lock_assert_notheld(void) { - lck_mtx_assert(&domain_proto_mtx, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&domain_proto_mtx, LCK_MTX_ASSERT_NOTOWNED); } domain_guard_t @@ -1013,11 +1019,11 @@ domain_guard_deploy(void) marks = net_thread_marks_push(NET_THREAD_HELD_DOMAIN); if (marks != net_thread_marks_none) { - lck_mtx_assert(&domain_proto_mtx, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&domain_proto_mtx, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(&domain_proto_mtx); } else - lck_mtx_assert(&domain_proto_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&domain_proto_mtx, LCK_MTX_ASSERT_OWNED); return ((domain_guard_t)(const void*)marks); } @@ -1028,12 +1034,12 @@ domain_guard_release(domain_guard_t guard) net_thread_marks_t marks = (net_thread_marks_t)(const void*)guard; if (marks != net_thread_marks_none) { - lck_mtx_assert(&domain_proto_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&domain_proto_mtx, LCK_MTX_ASSERT_OWNED); lck_mtx_unlock(&domain_proto_mtx); net_thread_marks_pop(marks); } else - lck_mtx_assert(&domain_proto_mtx, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&domain_proto_mtx, LCK_MTX_ASSERT_NOTOWNED); } domain_unguard_t @@ -1043,11 +1049,11 @@ domain_unguard_deploy(void) marks = net_thread_unmarks_push(NET_THREAD_HELD_DOMAIN); if (marks != net_thread_marks_none) { - lck_mtx_assert(&domain_proto_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&domain_proto_mtx, LCK_MTX_ASSERT_OWNED); lck_mtx_unlock(&domain_proto_mtx); } else - lck_mtx_assert(&domain_proto_mtx, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&domain_proto_mtx, LCK_MTX_ASSERT_NOTOWNED); return ((domain_unguard_t)(const void*)marks); } @@ -1058,14 +1064,15 @@ domain_unguard_release(domain_unguard_t unguard) net_thread_marks_t marks = (net_thread_marks_t)(const void*)unguard; if (marks != net_thread_marks_none) { - lck_mtx_assert(&domain_proto_mtx, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&domain_proto_mtx, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(&domain_proto_mtx); net_thread_unmarks_pop(marks); } else - lck_mtx_assert(&domain_proto_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&domain_proto_mtx, LCK_MTX_ASSERT_OWNED); } + #if (DEVELOPMENT || DEBUG) static int @@ -1085,4 +1092,3 @@ sysctl_do_drain_domains SYSCTL_HANDLER_ARGS } #endif /* DEVELOPMENT || DEBUG */ - \ No newline at end of file diff --git a/bsd/kern/uipc_mbuf.c b/bsd/kern/uipc_mbuf.c index cee3242d1..04df5a515 100644 --- a/bsd/kern/uipc_mbuf.c +++ b/bsd/kern/uipc_mbuf.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998-2016 Apple Inc. All rights reserved. + * Copyright (c) 1998-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -317,9 +317,6 @@ static lck_grp_attr_t *mbuf_mlock_grp_attr; /* Back-end (common) layer */ static boolean_t mbuf_worker_needs_wakeup; /* wait channel for mbuf worker */ static int mbuf_worker_ready; /* worker thread is runnable */ -static int mbuf_expand_mcl; /* number of cluster creation requets */ -static int mbuf_expand_big; /* number of big cluster creation requests */ -static int mbuf_expand_16k; /* number of 16KB cluster creation requests */ static int ncpu; /* number of CPUs */ static ppnum_t *mcl_paddr; /* Array of cluster physical addresses */ static ppnum_t mcl_pages; /* Size of array (# physical pages) */ @@ -496,7 +493,7 @@ static struct mleak_table mleak_table; static mleak_stat_t *mleak_stat; #define MLEAK_STAT_SIZE(n) \ - ((size_t)(&((mleak_stat_t *)0)->ml_trace[n])) + __builtin_offsetof(mleak_stat_t, ml_trace[n]) struct mallocation { mcache_obj_t *element; /* the alloc'ed element, NULL if unused */ @@ -588,6 +585,7 @@ typedef struct { int mtbl_maxlimit; /* maximum allowed */ u_int32_t mtbl_wantpurge; /* purge during next reclaim */ uint32_t mtbl_avgtotal; /* average total on iOS */ + u_int32_t mtbl_expand; /* worker should expand the class */ } mbuf_table_t; #define m_class(c) mbuf_table[c].mtbl_class @@ -613,6 +611,7 @@ typedef struct { #define m_ctotal(c) mbuf_table[c].mtbl_stats->mbcl_ctotal #define m_peak(c) mbuf_table[c].mtbl_stats->mbcl_peak_reported #define m_release_cnt(c) mbuf_table[c].mtbl_stats->mbcl_release_cnt +#define m_region_expand(c) mbuf_table[c].mtbl_expand static mbuf_table_t mbuf_table[] = { /* @@ -621,13 +620,13 @@ static mbuf_table_t mbuf_table[] = { * usage patterns on iOS. */ { MC_MBUF, NULL, TAILQ_HEAD_INITIALIZER(m_slablist(MC_MBUF)), - NULL, NULL, 0, 0, 0, 0, 3000 }, + NULL, NULL, 0, 0, 0, 0, 3000, 0 }, { MC_CL, NULL, TAILQ_HEAD_INITIALIZER(m_slablist(MC_CL)), - NULL, NULL, 0, 0, 0, 0, 2000 }, + NULL, NULL, 0, 0, 0, 0, 2000, 0 }, { MC_BIGCL, NULL, TAILQ_HEAD_INITIALIZER(m_slablist(MC_BIGCL)), - NULL, NULL, 0, 0, 0, 0, 1000 }, + NULL, NULL, 0, 0, 0, 0, 1000, 0 }, { MC_16KCL, NULL, TAILQ_HEAD_INITIALIZER(m_slablist(MC_16KCL)), - NULL, NULL, 0, 0, 0, 0, 1000 }, + NULL, NULL, 0, 0, 0, 0, 200, 0 }, /* * The following are special caches; they serve as intermediate * caches backed by the above rudimentary caches. Each object @@ -636,9 +635,9 @@ static mbuf_table_t mbuf_table[] = { * deal with the slab structures; instead, the constructed * cached elements are simply stored in the freelists. */ - { MC_MBUF_CL, NULL, { NULL, NULL }, NULL, NULL, 0, 0, 0, 0, 2000 }, - { MC_MBUF_BIGCL, NULL, { NULL, NULL }, NULL, NULL, 0, 0, 0, 0, 1000 }, - { MC_MBUF_16KCL, NULL, { NULL, NULL }, NULL, NULL, 0, 0, 0, 0, 1000 }, + { MC_MBUF_CL, NULL, { NULL, NULL }, NULL, NULL, 0, 0, 0, 0, 2000, 0 }, + { MC_MBUF_BIGCL, NULL, { NULL, NULL }, NULL, NULL, 0, 0, 0, 0, 1000, 0 }, + { MC_MBUF_16KCL, NULL, { NULL, NULL }, NULL, NULL, 0, 0, 0, 0, 200, 0 }, }; #define NELEM(a) (sizeof (a) / sizeof ((a)[0])) @@ -665,8 +664,13 @@ static char *mbuf_dump_buf; * mb_drain_maxint controls the amount of time to wait (in seconds) before * consecutive calls to m_drain(). */ +#if CONFIG_EMBEDDED +static unsigned int mb_watchdog = 1; +static unsigned int mb_drain_maxint = 60; +#else static unsigned int mb_watchdog = 0; static unsigned int mb_drain_maxint = 0; +#endif /* CONFIG_EMBEDDED */ uintptr_t mb_obscure_extfree __attribute__((visibility("hidden"))); uintptr_t mb_obscure_extref __attribute__((visibility("hidden"))); @@ -932,7 +936,7 @@ struct mb_stat *mb_stat; struct omb_stat *omb_stat; /* For backwards compatibility */ #define MB_STAT_SIZE(n) \ - ((size_t)(&((mb_stat_t *)0)->mbs_class[n])) + __builtin_offsetof(mb_stat_t, mbs_class[n]) #define OMB_STAT_SIZE(n) \ ((size_t)(&((struct omb_stat *)0)->mbs_class[n])) @@ -999,7 +1003,7 @@ mbuf_mtypes_sync(boolean_t locked) mtypes_cpu_t mtc; if (locked) - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); bzero(&mtc, sizeof (mtc)); for (m = 0; m < ncpu; m++) { @@ -1037,7 +1041,7 @@ mbuf_stat_sync(void) mcache_t *cp; int k, m, bktsize; - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); for (k = 0; k < NELEM(mbuf_table); k++) { cp = m_cache(k); @@ -1455,6 +1459,7 @@ mbinit(void) _CASSERT(MBUF_TSO_IPV6 == CSUM_TSO_IPV6); _CASSERT(MBUF_CSUM_REQ_SUM16 == CSUM_PARTIAL); _CASSERT(MBUF_CSUM_TCP_SUM16 == MBUF_CSUM_REQ_SUM16); + _CASSERT(MBUF_CSUM_REQ_ZERO_INVERT == CSUM_ZERO_INVERT); _CASSERT(MBUF_CSUM_REQ_IP == CSUM_IP); _CASSERT(MBUF_CSUM_REQ_TCP == CSUM_TCP); _CASSERT(MBUF_CSUM_REQ_UDP == CSUM_UDP); @@ -1571,6 +1576,21 @@ mbinit(void) mleak_activate(); + /* + * Allocate structure for per-CPU statistics that's aligned + * on the CPU cache boundary; this code assumes that we never + * uninitialize this framework, since the original address + * before alignment is not saved. + */ + ncpu = ml_get_max_cpus(); + MALLOC(buf, void *, MBUF_MTYPES_SIZE(ncpu) + CPU_CACHE_LINE_SIZE, + M_TEMP, M_WAITOK); + VERIFY(buf != NULL); + + mbuf_mtypes = (mbuf_mtypes_t *)P2ROUNDUP((intptr_t)buf, + CPU_CACHE_LINE_SIZE); + bzero(mbuf_mtypes, MBUF_MTYPES_SIZE(ncpu)); + /* Calculate the number of pages assigned to the cluster pool */ mcl_pages = (nmbclusters << MCLSHIFT) / PAGE_SIZE; MALLOC(mcl_paddr, ppnum_t *, mcl_pages * sizeof (ppnum_t), @@ -1661,21 +1681,6 @@ mbinit(void) (void *)(uintptr_t)m, flags, MCR_SLEEP); } - /* - * Allocate structure for per-CPU statistics that's aligned - * on the CPU cache boundary; this code assumes that we never - * uninitialize this framework, since the original address - * before alignment is not saved. - */ - ncpu = ml_get_max_cpus(); - MALLOC(buf, void *, MBUF_MTYPES_SIZE(ncpu) + CPU_CACHE_LINE_SIZE, - M_TEMP, M_WAITOK); - VERIFY(buf != NULL); - - mbuf_mtypes = (mbuf_mtypes_t *)P2ROUNDUP((intptr_t)buf, - CPU_CACHE_LINE_SIZE); - bzero(mbuf_mtypes, MBUF_MTYPES_SIZE(ncpu)); - /* * Set the max limit on sb_max to be 1/16 th of the size of * memory allocated for mbuf clusters. @@ -1742,7 +1747,7 @@ slab_alloc(mbuf_class_t class, int wait) mcl_slab_t *sp; mcache_obj_t *buf; - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); /* This should always be NULL for us */ VERIFY(m_cobjlist(class) == NULL); @@ -1876,7 +1881,7 @@ slab_free(mbuf_class_t class, mcache_obj_t *buf) boolean_t reinit_supercl = false; mbuf_class_t super_class; - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); VERIFY(class != MC_16KCL || njcl > 0); VERIFY(buf->obj_next == NULL); @@ -2147,7 +2152,7 @@ mbuf_slab_alloc(void *arg, mcache_obj_t ***plist, unsigned int num, int wait) mbuf_sleep(class, need, wait)) break; - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); } } @@ -2280,7 +2285,7 @@ cslab_alloc(mbuf_class_t class, mcache_obj_t ***plist, unsigned int num) VERIFY(need > 0); VERIFY(class != MC_MBUF_16KCL || njcl > 0); - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); /* Get what we can from the freelist */ while ((*list = m_cobjlist(class)) != NULL) { @@ -2344,7 +2349,7 @@ cslab_free(mbuf_class_t class, mcache_obj_t *list, int purged) ASSERT(MBUF_CLASS_VALID(class) && MBUF_CLASS_COMPOSITE(class)); VERIFY(class != MC_MBUF_16KCL || njcl > 0); - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); if (class == MC_MBUF_CL) { cl_class = MC_CL; @@ -2805,7 +2810,7 @@ m_clalloc(const u_int32_t num, const int wait, const u_int32_t bufsize) else class = MC_16KCL; - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); /* * Multiple threads may attempt to populate the cluster map one @@ -2819,7 +2824,7 @@ m_clalloc(const u_int32_t num, const int wait, const u_int32_t bufsize) mb_clalloc_waiters++; (void) msleep(mb_clalloc_waitchan, mbuf_mlock, (PZERO-1), "m_clalloc", NULL); - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); } /* We are busy now; tell everyone else to go away */ @@ -2830,7 +2835,7 @@ m_clalloc(const u_int32_t num, const int wait, const u_int32_t bufsize) * to grow the pool asynchronously using the mbuf worker thread. */ i = m_howmany(num, bufsize); - if (i == 0 || (wait & M_DONTWAIT)) + if (i <= 0 || (wait & M_DONTWAIT)) goto out; lck_mtx_unlock(mbuf_mlock); @@ -3000,7 +3005,7 @@ m_clalloc(const u_int32_t num, const int wait, const u_int32_t bufsize) return (count); out: - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); /* We're done; let others enter */ mb_clalloc_busy = FALSE; @@ -3024,8 +3029,8 @@ out: * at this time. */ i += m_total(MC_BIGCL); - if (i > mbuf_expand_big) { - mbuf_expand_big = i; + if (i > m_region_expand(MC_BIGCL)) { + m_region_expand(MC_BIGCL) = i; } } if (m_infree(MC_BIGCL) >= num) @@ -3037,8 +3042,8 @@ out: * at this time. */ i += m_total(MC_16KCL); - if (i > mbuf_expand_16k) { - mbuf_expand_16k = i; + if (i > m_region_expand(MC_16KCL)) { + m_region_expand(MC_16KCL) = i; } } if (m_infree(MC_16KCL) >= num) @@ -3060,7 +3065,7 @@ freelist_populate(mbuf_class_t class, unsigned int num, int wait) VERIFY(class == MC_MBUF || class == MC_CL || class == MC_BIGCL || class == MC_16KCL); - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); VERIFY(PAGE_SIZE == m_maxsize(MC_BIGCL) || PAGE_SIZE == m_maxsize(MC_16KCL)); @@ -3083,12 +3088,6 @@ freelist_populate(mbuf_class_t class, unsigned int num, int wait) i = m_clalloc(numpages, wait, m_maxsize(super_class)); - /* Respect the minimum limit of super class */ - if (m_total(super_class) == m_maxlimit(super_class) && - m_infree(super_class) <= m_minlimit(super_class)) - if (wait & MCR_COMP) - return (0); - /* how many objects will we cut the page into? */ int numobj = PAGE_SIZE / m_maxsize(class); @@ -3138,6 +3137,7 @@ freelist_populate(mbuf_class_t class, unsigned int num, int wait) mbstat.m_bigclusters = m_total(MC_BIGCL); m_total(class) += numobj; + VERIFY(m_total(class) <= m_maxlimit(class)); m_infree(class) += numobj; if (!mb_peak_newreport && mbuf_report_usage(class)) @@ -3204,7 +3204,7 @@ freelist_populate(mbuf_class_t class, unsigned int num, int wait) static void freelist_init(mbuf_class_t class) { - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); VERIFY(class == MC_CL || class == MC_BIGCL); VERIFY(m_total(class) == 0); @@ -3269,7 +3269,7 @@ mbuf_steal(mbuf_class_t class, unsigned int num) mcache_obj_t **list = ⊤ unsigned int tot = 0; - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); switch (class) { case MC_MBUF: @@ -3305,7 +3305,7 @@ m_reclaim(mbuf_class_t class, unsigned int num, boolean_t comp) { int m, bmap = 0; - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); VERIFY(m_total(MC_CL) <= m_maxlimit(MC_CL)); VERIFY(m_total(MC_BIGCL) <= m_maxlimit(MC_BIGCL)); @@ -3962,10 +3962,11 @@ m_classifier_init(struct mbuf *m, uint32_t pktf_mask) (void) m_set_service_class(m, MBUF_SC_BE); if (!(m->m_pkthdr.pkt_flags & PKTF_IFAINFO)) m->m_pkthdr.pkt_ifainfo = 0; -#if MEASURE_BW - m->m_pkthdr.pkt_bwseq = 0; -#endif /* MEASURE_BW */ - m->m_pkthdr.pkt_timestamp = 0; + /* + * Preserve timestamp if requested + */ + if (!(m->m_pkthdr.pkt_flags & PKTF_TS_VALID)) + m->m_pkthdr.pkt_timestamp = 0; } void @@ -3980,9 +3981,6 @@ m_copy_classifier(struct mbuf *to, struct mbuf *from) to->m_pkthdr.pkt_flags = from->m_pkthdr.pkt_flags; (void) m_set_service_class(to, from->m_pkthdr.pkt_svc); to->m_pkthdr.pkt_ifainfo = from->m_pkthdr.pkt_ifainfo; -#if MEASURE_BW - to->m_pkthdr.pkt_bwseq = from->m_pkthdr.pkt_bwseq; -#endif /* MEASURE_BW */ } /* @@ -5039,23 +5037,33 @@ nospace: void m_copydata(struct mbuf *m, int off, int len, void *vp) { + int off0 = off, len0 = len; + struct mbuf *m0 = m; unsigned count; char *cp = vp; - if (off < 0 || len < 0) - panic("m_copydata: invalid offset %d or len %d", off, len); + if (__improbable(off < 0 || len < 0)) { + panic("%s: invalid offset %d or len %d", __func__, off, len); + /* NOTREACHED */ + } while (off > 0) { - if (m == NULL) - panic("m_copydata: invalid mbuf chain"); + if (__improbable(m == NULL)) { + panic("%s: invalid mbuf chain %p [off %d, len %d]", + __func__, m0, off0, len0); + /* NOTREACHED */ + } if (off < m->m_len) break; off -= m->m_len; m = m->m_next; } while (len > 0) { - if (m == NULL) - panic("m_copydata: invalid mbuf chain"); + if (__improbable(m == NULL)) { + panic("%s: invalid mbuf chain %p [off %d, len %d]", + __func__, m0, off0, len0); + /* NOTREACHED */ + } count = MIN(m->m_len - off, len); bcopy(MTOD(m, caddr_t) + off, cp, count); len -= count; @@ -5445,7 +5453,7 @@ m_howmany(int num, size_t bufsize) VERIFY(bufsize == m_maxsize(MC_BIGCL) || bufsize == m_maxsize(MC_16KCL)); - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); /* Numbers in 2K cluster units */ m_mbclusters = m_total(MC_MBUF) >> NMBPCLSHIFT; @@ -5535,9 +5543,8 @@ m_howmany(int num, size_t bufsize) i = MAX(i, j); /* Check to ensure we don't go over limit */ - if (i + m_16kclusters >= m_maxlimit(MC_16KCL)) - i = m_maxlimit(MC_16KCL) - m_16kclusters; - VERIFY((m_total(MC_16KCL) + i) <= m_maxlimit(MC_16KCL)); + if ((i + m_total(MC_16KCL)) >= m_maxlimit(MC_16KCL)) + i = m_maxlimit(MC_16KCL) - m_total(MC_16KCL); } return (i); } @@ -6196,6 +6203,8 @@ m_defrag_offset(struct mbuf *m0, u_int32_t off, int how) if (length > MCLBYTES) length = MCLBYTES; length -= ((m_new == m_final) ? off : 0); + if (length < 0) + goto nospace; if (m_new == NULL) { if (length > MLEN) @@ -6357,7 +6366,7 @@ mbuf_sleep(mbuf_class_t class, unsigned int num, int wait) { boolean_t mcache_retry = FALSE; - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); /* Check if there's anything at the cache layer */ if (mbuf_cached_above(class, wait)) { @@ -6399,6 +6408,14 @@ mbuf_sleep(mbuf_class_t class, unsigned int num, int wait) mbuf_watchdog(); mb_waiters++; + m_region_expand(class) += num; + /* wake up the worker thread */ + if (class > MC_MBUF && mbuf_worker_ready && + mbuf_worker_needs_wakeup) { + wakeup((caddr_t)&mbuf_worker_needs_wakeup); + mbuf_worker_needs_wakeup = FALSE; + } + (void) msleep(mb_waitchan, mbuf_mlock, (PZERO-1), m_cname(class), NULL); /* We are now up; stop getting notified until next round */ @@ -6425,41 +6442,41 @@ mbuf_worker_thread(void) while (1) { lck_mtx_lock(mbuf_mlock); mbuf_expand = 0; - if (mbuf_expand_mcl) { + if (m_region_expand(MC_CL) > 0) { int n; /* Adjust to current number of cluster in use */ - n = mbuf_expand_mcl - + n = m_region_expand(MC_CL) - (m_total(MC_CL) - m_infree(MC_CL)); if ((n + m_total(MC_CL)) > m_maxlimit(MC_CL)) n = m_maxlimit(MC_CL) - m_total(MC_CL); - mbuf_expand_mcl = 0; + m_region_expand(MC_CL) = 0; if (n > 0 && freelist_populate(MC_CL, n, M_WAIT) > 0) mbuf_expand++; } - if (mbuf_expand_big) { + if (m_region_expand(MC_BIGCL) > 0) { int n; /* Adjust to current number of 4 KB cluster in use */ - n = mbuf_expand_big - + n = m_region_expand(MC_BIGCL) - (m_total(MC_BIGCL) - m_infree(MC_BIGCL)); if ((n + m_total(MC_BIGCL)) > m_maxlimit(MC_BIGCL)) n = m_maxlimit(MC_BIGCL) - m_total(MC_BIGCL); - mbuf_expand_big = 0; + m_region_expand(MC_BIGCL) = 0; if (n > 0 && freelist_populate(MC_BIGCL, n, M_WAIT) > 0) mbuf_expand++; } - if (mbuf_expand_16k) { + if (m_region_expand(MC_16KCL) > 0) { int n; /* Adjust to current number of 16 KB cluster in use */ - n = mbuf_expand_16k - + n = m_region_expand(MC_16KCL) - (m_total(MC_16KCL) - m_infree(MC_16KCL)); if ((n + m_total(MC_16KCL)) > m_maxlimit(MC_16KCL)) n = m_maxlimit(MC_16KCL) - m_total(MC_16KCL); - mbuf_expand_16k = 0; + m_region_expand(MC_16KCL) = 0; if (n > 0) (void) freelist_populate(MC_16KCL, n, M_WAIT); @@ -6501,7 +6518,7 @@ slab_get(void *buf) mcl_slabg_t *slg; unsigned int ix, k; - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); VERIFY(MBUF_IN_MAP(buf)); ix = ((unsigned char *)buf - mbutl) >> MBSHIFT; @@ -6686,7 +6703,7 @@ mcl_audit_init(void *buf, mcache_audit_t **mca_list, VERIFY(ix < maxclaudit); /* Make sure we haven't been here before */ - for (i = 0; i < NMBPG; i++) + for (i = 0; i < num; i++) VERIFY(mclaudit[ix].cl_audit[i] == NULL); mca = mca_tail = *mca_list; @@ -7616,8 +7633,8 @@ m_set_ext(struct mbuf *m, struct ext_ref *rfa, m_ext_free_func_t ext_free, m->m_ext.ext_free = (m_ext_free_func_t) (((uintptr_t)ext_free) ^ rfa->ext_token); if (ext_arg != NULL) { - m->m_ext.ext_arg = (((uintptr_t)ext_arg) ^ - rfa->ext_token); + m->m_ext.ext_arg = + (caddr_t)(((uintptr_t)ext_arg) ^ rfa->ext_token); } else { m->m_ext.ext_arg = NULL; } @@ -7633,10 +7650,12 @@ m_set_ext(struct mbuf *m, struct ext_ref *rfa, m_ext_free_func_t ext_free, * to obscure the ext_free and ext_arg pointers. */ if (ext_free != NULL) { - m->m_ext.ext_free = ((uintptr_t)ext_free ^ + m->m_ext.ext_free = + (m_ext_free_func_t)((uintptr_t)ext_free ^ mb_obscure_extfree); if (ext_arg != NULL) { - m->m_ext.ext_arg = ((uintptr_t)ext_arg ^ + m->m_ext.ext_arg = + (caddr_t)((uintptr_t)ext_arg ^ mb_obscure_extfree); } else { m->m_ext.ext_arg = NULL; @@ -7667,7 +7686,7 @@ m_get_ext_free(struct mbuf *m) rfa = m_get_rfa(m); if (rfa == NULL) - return ((uintptr_t)m->m_ext.ext_free ^ mb_obscure_extfree); + return ((m_ext_free_func_t)((uintptr_t)m->m_ext.ext_free ^ mb_obscure_extfree)); else return ((m_ext_free_func_t)(((uintptr_t)m->m_ext.ext_free) ^ rfa->ext_token)); @@ -7682,7 +7701,7 @@ m_get_ext_arg(struct mbuf *m) rfa = m_get_rfa(m); if (rfa == NULL) { - return ((uintptr_t)m->m_ext.ext_arg ^ mb_obscure_extfree); + return ((caddr_t)((uintptr_t)m->m_ext.ext_arg ^ mb_obscure_extfree)); } else { return ((caddr_t)(((uintptr_t)m->m_ext.ext_arg) ^ rfa->ext_token)); @@ -7919,8 +7938,14 @@ m_drain(void) 0); nsp->sl_flags = 0; } - if (mclaudit != NULL) - mcl_audit_free(sp->sl_base, 1); + if (mclaudit != NULL) { + if (sp->sl_len == PAGE_SIZE) { + mcl_audit_free(sp->sl_base, + NMBPG); + } else { + mcl_audit_free(sp->sl_base, 1); + } + } break; default: /* diff --git a/bsd/kern/uipc_mbuf2.c b/bsd/kern/uipc_mbuf2.c index 74ac53ba7..fc22ee904 100644 --- a/bsd/kern/uipc_mbuf2.c +++ b/bsd/kern/uipc_mbuf2.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2013 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -664,9 +664,8 @@ m_tag_init(struct mbuf *m, int all) * (e.g. m_dup_pkthdr), don't zero them out. */ if (all) { - bzero(m_pftag(m), sizeof (struct pf_mtag)); - bzero(&m->m_pkthdr.proto_mtag, sizeof (m->m_pkthdr.proto_mtag)); - bzero(&m->m_pkthdr.necp_mtag, sizeof (m->m_pkthdr.necp_mtag)); + bzero(&m->m_pkthdr.builtin_mtag._net_mtag, + sizeof (m->m_pkthdr.builtin_mtag._net_mtag)); } } @@ -825,27 +824,75 @@ m_service_class_from_val(u_int32_t v) } uint16_t -m_adj_sum16(struct mbuf *m, uint32_t start, uint32_t ulpoff, uint32_t sum) +m_adj_sum16(struct mbuf *m, uint32_t start, uint32_t dataoff, + uint32_t datalen, uint32_t sum) { - int len = (ulpoff - start); + uint32_t total_sub = 0; /* total to subtract */ + uint32_t mlen = m_pktlen(m); /* frame length */ + uint32_t bytes = (dataoff + datalen); /* bytes covered by sum */ + int len; + ASSERT(bytes <= mlen); + + /* + * Take care of excluding (len > 0) or including (len < 0) + * extraneous octets at the beginning of the packet, taking + * into account the start offset. + */ + len = (dataoff - start); + if (len > 0) + total_sub = m_sum16(m, start, len); + else if (len < 0) + sum += m_sum16(m, dataoff, -len); + + /* + * Take care of excluding any postpended extraneous octets. + */ + len = (mlen - bytes); if (len > 0) { - uint32_t adj = m_sum16(m, start, len); - if (adj >= sum) - sum = ~(adj - sum) & 0xffff; + struct mbuf *m0 = m; + uint32_t extra = m_sum16(m, bytes, len); + uint32_t off = bytes, off0 = off; + + while (off > 0) { + if (__improbable(m == NULL)) { + panic("%s: invalid mbuf chain %p [off %u, " + "len %u]", __func__, m0, off0, len); + /* NOTREACHED */ + } + if (off < m->m_len) + break; + off -= m->m_len; + m = m->m_next; + } + + /* if we started on odd-alignment, swap the value */ + if ((uintptr_t)(mtod(m, uint8_t *) + off) & 1) + total_sub += ((extra << 8) & 0xffff) | (extra >> 8); + else + total_sub += extra; + + total_sub = (total_sub >> 16) + (total_sub & 0xffff); + } + + /* + * 1's complement subtract any extraneous octets. + */ + if (total_sub != 0) { + if (total_sub >= sum) + sum = ~(total_sub - sum) & 0xffff; else - sum -= adj; - } else if (len < 0) { - sum += m_sum16(m, ulpoff, -len); + sum -= total_sub; } - ADDCARRY(sum); + /* fold 32-bit to 16-bit */ + sum = (sum >> 16) + (sum & 0xffff); /* 17-bit */ + sum = (sum >> 16) + (sum & 0xffff); /* 16-bit + carry */ + sum = (sum >> 16) + (sum & 0xffff); /* final carry */ - return (sum); + return (sum & 0xffff); } -extern int cpu_in_cksum(struct mbuf *m, int len, int off, uint32_t initial_sum); - uint16_t m_sum16(struct mbuf *m, uint32_t off, uint32_t len) { @@ -859,9 +906,10 @@ m_sum16(struct mbuf *m, uint32_t off, uint32_t len) * a M_PKTHDR one. */ if ((mlen = m_length2(m, NULL)) < (off + len)) { - panic("%s: mbuf len (%d) < off+len (%d+%d)\n", __func__, - mlen, off, len); + panic("%s: mbuf %p len (%d) < off+len (%d+%d)\n", __func__, + m, mlen, off, len); + /* NOTREACHED */ } - return (~cpu_in_cksum(m, len, off, 0) & 0xffff); + return (os_cpu_in_cksum_mbuf(m, len, off, 0)); } diff --git a/bsd/kern/uipc_socket.c b/bsd/kern/uipc_socket.c index aa41e355a..1c5af8cf3 100644 --- a/bsd/kern/uipc_socket.c +++ b/bsd/kern/uipc_socket.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998-2016 Apple Inc. All rights reserved. + * Copyright (c) 1998-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -98,6 +98,7 @@ #include <sys/kern_event.h> #include <net/route.h> #include <net/init.h> +#include <net/net_api_stats.h> #include <net/ntstat.h> #include <net/content_filter.h> #include <netinet/in.h> @@ -119,9 +120,9 @@ #include <sys/kpi_mbuf.h> #include <sys/mcache.h> #include <sys/unpcb.h> +#include <libkern/section_keywords.h> #if CONFIG_MACF -#include <security/mac.h> #include <security/mac_framework.h> #endif /* MAC */ @@ -159,19 +160,19 @@ static lck_mtx_t *so_cache_mtx; #include <machine/limits.h> -static int filt_sorattach(struct knote *kn); +static int filt_sorattach(struct knote *kn, struct kevent_internal_s *kev); static void filt_sordetach(struct knote *kn); static int filt_soread(struct knote *kn, long hint); static int filt_sortouch(struct knote *kn, struct kevent_internal_s *kev); static int filt_sorprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); -static int filt_sowattach(struct knote *kn); +static int filt_sowattach(struct knote *kn, struct kevent_internal_s *kev); static void filt_sowdetach(struct knote *kn); static int filt_sowrite(struct knote *kn, long hint); static int filt_sowtouch(struct knote *kn, struct kevent_internal_s *kev); static int filt_sowprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); -static int filt_sockattach(struct knote *kn); +static int filt_sockattach(struct knote *kn, struct kevent_internal_s *kev); static void filt_sockdetach(struct knote *kn); static int filt_sockev(struct knote *kn, long hint); static int filt_socktouch(struct knote *kn, struct kevent_internal_s *kev); @@ -180,7 +181,7 @@ static int filt_sockprocess(struct knote *kn, struct filt_process_s *data, struc static int sooptcopyin_timeval(struct sockopt *, struct timeval *); static int sooptcopyout_timeval(struct sockopt *, const struct timeval *); -struct filterops soread_filtops = { +SECURITY_READ_ONLY_EARLY(struct filterops) soread_filtops = { .f_isfd = 1, .f_attach = filt_sorattach, .f_detach = filt_sordetach, @@ -189,7 +190,7 @@ struct filterops soread_filtops = { .f_process = filt_sorprocess, }; -struct filterops sowrite_filtops = { +SECURITY_READ_ONLY_EARLY(struct filterops) sowrite_filtops = { .f_isfd = 1, .f_attach = filt_sowattach, .f_detach = filt_sowdetach, @@ -198,7 +199,7 @@ struct filterops sowrite_filtops = { .f_process = filt_sowprocess, }; -struct filterops sock_filtops = { +SECURITY_READ_ONLY_EARLY(struct filterops) sock_filtops = { .f_isfd = 1, .f_attach = filt_sockattach, .f_detach = filt_sockdetach, @@ -207,7 +208,7 @@ struct filterops sock_filtops = { .f_process = filt_sockprocess, }; -struct filterops soexcept_filtops = { +SECURITY_READ_ONLY_EARLY(struct filterops) soexcept_filtops = { .f_isfd = 1, .f_attach = filt_sorattach, .f_detach = filt_sordetach, @@ -368,6 +369,7 @@ SYSCTL_STRUCT(_kern_ipc, OID_AUTO, extbkidlestat, CTLFLAG_RD | CTLFLAG_LOCKED, int so_set_extended_bk_idle(struct socket *, int); + /* * SOTCDB_NO_DSCP is set by default, to prevent the networking stack from * setting the DSCP code on the packet based on the service class; see @@ -630,6 +632,12 @@ soalloc(int waitok, int dom, int type) if (so != NULL) { so->so_gencnt = OSIncrementAtomic64((SInt64 *)&so_gencnt); so->so_zone = socket_zone; + + /* + * Increment the socket allocation statistics + */ + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_alloc_total); + #if CONFIG_MACF_SOCKET /* Convert waitok to M_WAITOK/M_NOWAIT for MAC Framework. */ if (mac_socket_label_init(so, !waitok) != 0) { @@ -677,19 +685,48 @@ socreate_internal(int dom, struct socket **aso, int type, int proto, if (so == NULL) return (ENOBUFS); + switch (dom) { + case PF_LOCAL: + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_domain_local_total); + break; + case PF_INET: + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_domain_inet_total); + if (type == SOCK_STREAM) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_inet_stream_total); + } else { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_inet_dgram_total); + } + break; + case PF_ROUTE: + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_domain_route_total); + break; + case PF_NDRV: + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_domain_ndrv_total); + break; + case PF_KEY: + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_domain_key_total); + break; + case PF_INET6: + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_domain_inet6_total); + if (type == SOCK_STREAM) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_inet6_stream_total); + } else { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_inet6_dgram_total); + } + break; + case PF_SYSTEM: + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_domain_system_total); + break; + case PF_MULTIPATH: + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_domain_multipath_total); + break; + default: + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_domain_other_total); + break; + } + if (flags & SOCF_ASYNC) so->so_state |= SS_NBIO; -#if MULTIPATH - if (flags & SOCF_MP_SUBFLOW) { - /* - * A multipath subflow socket is used internally in the kernel, - * therefore it does not have a file desciptor associated by - * default. - */ - so->so_state |= SS_NOFDREF; - so->so_flags |= SOF_MP_SUBFLOW; - } -#endif /* MULTIPATH */ TAILQ_INIT(&so->so_incomp); TAILQ_INIT(&so->so_comp); @@ -864,7 +901,6 @@ sobindlock(struct socket *so, struct sockaddr *nam, int dolock) if (dolock) socket_lock(so, 1); - VERIFY(so->so_usecount > 1); so_update_last_owner_locked(so, p); so_update_policy(so); @@ -1067,8 +1103,8 @@ so_acquire_accept_list(struct socket *head, struct socket *so) if (head->so_proto->pr_getlock == NULL) { return; } - mutex_held = (*head->so_proto->pr_getlock)(head, 0); - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + mutex_held = (*head->so_proto->pr_getlock)(head, PR_F_WILLUNLOCK); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); if (!(head->so_flags1 & SOF1_ACCEPT_LIST_HELD)) { head->so_flags1 |= SOF1_ACCEPT_LIST_HELD; @@ -1097,8 +1133,8 @@ so_release_accept_list(struct socket *head) lck_mtx_t *mutex_held; mutex_held = (*head->so_proto->pr_getlock)(head, 0); - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); - + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); + head->so_flags1 &= ~SOF1_ACCEPT_LIST_HELD; wakeup((caddr_t)&head->so_incomp); } @@ -1159,8 +1195,8 @@ sofreelastref(struct socket *so, int dealloc) } else { if (head->so_proto->pr_getlock != NULL) { so_release_accept_list(head); - socket_unlock(head, 1); - } + socket_unlock(head, 1); + } printf("sofree: not queued\n"); } } @@ -1188,10 +1224,10 @@ soclose_wait_locked(struct socket *so) lck_mtx_t *mutex_held; if (so->so_proto->pr_getlock != NULL) - mutex_held = (*so->so_proto->pr_getlock)(so, 0); + mutex_held = (*so->so_proto->pr_getlock)(so, PR_F_WILLUNLOCK); else mutex_held = so->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); /* * Double check here and return if there's no outstanding upcall; @@ -1202,9 +1238,10 @@ soclose_wait_locked(struct socket *so) so->so_rcv.sb_flags &= ~SB_UPCALL; so->so_snd.sb_flags &= ~SB_UPCALL; so->so_flags |= SOF_CLOSEWAIT; + (void) msleep((caddr_t)&so->so_upcallusecount, mutex_held, (PZERO - 1), "soclose_wait_locked", NULL); - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); so->so_flags &= ~SOF_CLOSEWAIT; } @@ -1365,7 +1402,7 @@ again: (so->so_state & SS_NBIO)) goto drop; if (so->so_proto->pr_getlock != NULL) - mutex_held = (*so->so_proto->pr_getlock)(so, 0); + mutex_held = (*so->so_proto->pr_getlock)(so, PR_F_WILLUNLOCK); else mutex_held = so->so_proto->pr_domain->dom_mtx; while (so->so_state & SS_ISCONNECTED) { @@ -1408,9 +1445,6 @@ discard: } so->so_state |= SS_NOFDREF; - if (so->so_flags & SOF_MP_SUBFLOW) - so->so_flags &= ~SOF_MP_SUBFLOW; - if ((so->so_flags & SOF_KNOTE) != 0) KNOTE(&so->so_klist, SO_FILT_HINT_LOCKED); @@ -1461,7 +1495,7 @@ soabort(struct socket *so) mutex_held = (*so->so_proto->pr_getlock)(so, 0); else mutex_held = so->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); #endif if ((so->so_flags & SOF_ABORTED) == 0) { @@ -1547,7 +1581,7 @@ soacceptfilter(struct socket *so, struct socket *head) * the following is done while holding the lock since * the socket has been exposed to the filter(s) earlier. */ - so->so_state &= ~SS_COMP; + so->so_state &= ~SS_NOFDREF; socket_unlock(so, 1); soclose(so); /* Propagate socket filter's error code to the caller */ @@ -1802,12 +1836,6 @@ sodisconnectx(struct socket *so, sae_associd_t aid, sae_connid_t cid) return (error); } -int -sopeelofflocked(struct socket *so, sae_associd_t aid, struct socket **psop) -{ - return ((*so->so_proto->pr_usrreqs->pru_peeloff)(so, aid, psop)); -} - #define SBLOCKWAIT(f) (((f) & MSG_DONTWAIT) ? 0 : SBL_WAIT) /* @@ -1893,18 +1921,9 @@ defunct: if ((so->so_proto->pr_flags & PR_CONNREQUIRED) != 0) { if (((so->so_state & SS_ISCONFIRMING) == 0) && (resid != 0 || clen == 0) && - !(so->so_flags1 & SOF1_PRECONNECT_DATA)) { -#if MPTCP - /* - * MPTCP Fast Join sends data before the - * socket is truly connected. - */ - if ((so->so_flags & (SOF_MP_SUBFLOW | - SOF_MPTCP_FASTJOIN)) != - (SOF_MP_SUBFLOW | SOF_MPTCP_FASTJOIN)) -#endif /* MPTCP */ + !(so->so_flags1 & SOF1_PRECONNECT_DATA)) return (ENOTCONN); - } + } else if (addr == 0 && !(flags&MSG_HOLD)) { return ((so->so_proto->pr_flags & PR_CONNREQUIRED) ? ENOTCONN : EDESTADDRREQ); @@ -2072,8 +2091,7 @@ sosend(struct socket *so, struct sockaddr *addr, struct uio *uio, if (so->so_type != SOCK_STREAM && (flags & MSG_OOB) != 0) { error = EOPNOTSUPP; - socket_unlock(so, 1); - goto out; + goto out_locked; } /* @@ -2092,8 +2110,7 @@ sosend(struct socket *so, struct sockaddr *addr, struct uio *uio, if (resid < 0 || resid > INT_MAX || (so->so_type == SOCK_STREAM && !(so->so_flags & SOF_ENABLE_MSGS) && (flags & MSG_EOR))) { error = EINVAL; - socket_unlock(so, 1); - goto out; + goto out_locked; } dontroute = (flags & MSG_DONTROUTE) && @@ -2111,7 +2128,7 @@ sosend(struct socket *so, struct sockaddr *addr, struct uio *uio, error = sosendcheck(so, addr, resid, clen, atomic, flags, &sblocked, control); if (error) - goto release; + goto out_locked; mp = ⊤ if (so->so_flags & SOF_ENABLE_MSGS) @@ -2296,7 +2313,7 @@ sosend(struct socket *so, struct sockaddr *addr, struct uio *uio, if (freelist == NULL) { error = ENOBUFS; socket_lock(so, 0); - goto release; + goto out_locked; } /* * For datagram protocols, @@ -2352,7 +2369,7 @@ sosend(struct socket *so, struct sockaddr *addr, struct uio *uio, socket_lock(so, 0); if (error) - goto release; + goto out_locked; } if (flags & (MSG_HOLD|MSG_SEND)) { @@ -2372,7 +2389,7 @@ sosend(struct socket *so, struct sockaddr *addr, struct uio *uio, so->so_tail = mb1; if (flags & MSG_HOLD) { top = NULL; - goto release; + goto out_locked; } top = so->so_temp; } @@ -2407,7 +2424,7 @@ sosend(struct socket *so, struct sockaddr *addr, struct uio *uio, control = NULL; top = NULL; } - goto release; + goto out_locked; } #if CONTENT_FILTER /* @@ -2423,7 +2440,7 @@ sosend(struct socket *so, struct sockaddr *addr, struct uio *uio, control = NULL; top = NULL; } - goto release; + goto out_locked; } #endif /* CONTENT_FILTER */ } @@ -2450,16 +2467,15 @@ sosend(struct socket *so, struct sockaddr *addr, struct uio *uio, top = NULL; mp = ⊤ if (error) - goto release; + goto out_locked; } while (resid && space > 0); } while (resid); -release: +out_locked: if (sblocked) sbunlock(&so->so_snd, FALSE); /* will unlock socket */ else socket_unlock(so, 1); -out: if (top != NULL) m_freem(top); if (control != NULL) @@ -2469,12 +2485,7 @@ out: if (control_copy != NULL) m_freem(control_copy); - /* - * One write has been done. This was enough. Get back to "normal" - * behavior. - */ - if (so->so_flags1 & SOF1_PRECONNECT_DATA) - so->so_flags1 &= ~SOF1_PRECONNECT_DATA; + soclearfastopen(so); if (en_tracing) { /* resid passed here is the bytes left in uio */ @@ -4485,7 +4496,7 @@ sorflush(struct socket *so) else mutex_held = so->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); #endif /* notyet */ sflt_notify(so, sock_evt_flush_read, NULL); @@ -4664,19 +4675,25 @@ sooptcopyin_timeval(struct sockopt *sopt, struct timeval *tv_p) return (0); } -static int -soopt_cred_check(struct socket *so, int priv) +int +soopt_cred_check(struct socket *so, int priv, boolean_t allow_root) { kauth_cred_t cred = NULL; proc_t ep = PROC_NULL; - int error; + uid_t uid; + int error = 0; if (so->so_flags & SOF_DELEGATED) { ep = proc_find(so->e_pid); if (ep) cred = kauth_cred_proc_ref(ep); } - error = priv_check_cred(cred ? cred : so->so_cred, priv, 0); + + uid = kauth_cred_getuid(cred ? cred : so->so_cred); + + /* uid is 0 for root */ + if (uid != 0 || !allow_root) + error = priv_check_cred(cred ? cred : so->so_cred, priv, 0); if (cred) kauth_cred_unref(&cred); if (ep != PROC_NULL) @@ -4966,7 +4983,7 @@ sosetoptlock(struct socket *so, struct sockopt *sopt, int dolock) goto out; if (optval != 0) { error = soopt_cred_check(so, - PRIV_NET_RESTRICTED_AWDL); + PRIV_NET_RESTRICTED_AWDL, false); if (error == 0) inp_set_awdl_unrestricted( sotoinpcb(so)); @@ -4985,7 +5002,7 @@ sosetoptlock(struct socket *so, struct sockopt *sopt, int dolock) if (optval != 0 && inp_get_intcoproc_allowed(sotoinpcb(so)) == FALSE) { error = soopt_cred_check(so, - PRIV_NET_RESTRICTED_INTCOPROC); + PRIV_NET_RESTRICTED_INTCOPROC, false); if (error == 0) inp_set_intcoproc_allowed( sotoinpcb(so)); @@ -5232,27 +5249,50 @@ sosetoptlock(struct socket *so, struct sockopt *sopt, int dolock) case SO_NECP_ATTRIBUTES: error = necp_set_socket_attributes(so, sopt); break; -#endif /* NECP */ -#if MPTCP - case SO_MPTCP_FASTJOIN: - if (!((so->so_flags & SOF_MP_SUBFLOW) || - ((SOCK_CHECK_DOM(so, PF_MULTIPATH)) && - (SOCK_CHECK_PROTO(so, IPPROTO_TCP))))) { - error = ENOPROTOOPT; + case SO_NECP_CLIENTUUID: + if (SOCK_DOM(so) == PF_MULTIPATH) { + /* Handled by MPTCP itself */ break; } - error = sooptcopyin(sopt, &optval, sizeof (optval), - sizeof (optval)); - if (error != 0) + if (SOCK_DOM(so) != PF_INET && SOCK_DOM(so) != PF_INET6) { + error = EINVAL; goto out; - if (optval == 0) - so->so_flags &= ~SOF_MPTCP_FASTJOIN; - else - so->so_flags |= SOF_MPTCP_FASTJOIN; + } + + struct inpcb *inp = sotoinpcb(so); + if (!uuid_is_null(inp->necp_client_uuid)) { + // Clear out the old client UUID if present + necp_inpcb_remove_cb(inp); + } + + error = sooptcopyin(sopt, &inp->necp_client_uuid, + sizeof(uuid_t), sizeof(uuid_t)); + if (error != 0) { + goto out; + } + + if (uuid_is_null(inp->necp_client_uuid)) { + error = EINVAL; + goto out; + } + + error = necp_client_register_socket_flow(so->last_pid, + inp->necp_client_uuid, inp); + if (error != 0) { + uuid_clear(inp->necp_client_uuid); + goto out; + } + + if (inp->inp_lport != 0) { + // There is bound local port, so this is not + // a fresh socket. Assign to the client. + necp_client_assign_from_socket(so->last_pid, inp->necp_client_uuid, inp); + } + break; -#endif /* MPTCP */ +#endif /* NECP */ case SO_EXTENDED_BK_IDLE: error = sooptcopyin(sopt, &optval, sizeof (optval), @@ -5349,8 +5389,8 @@ sooptcopyout_timeval(struct sockopt *sopt, const struct timeval *tv_p) { int error; size_t len; - struct user64_timeval tv64; - struct user32_timeval tv32; + struct user64_timeval tv64 = {}; + struct user32_timeval tv32 = {}; const void * val; size_t valsize; @@ -5694,6 +5734,23 @@ integer: case SO_NECP_ATTRIBUTES: error = necp_get_socket_attributes(so, sopt); break; + + case SO_NECP_CLIENTUUID: + { + uuid_t *ncu; + + if (SOCK_DOM(so) == PF_MULTIPATH) { + ncu = &mpsotomppcb(so)->necp_client_uuid; + } else if (SOCK_DOM(so) == PF_INET || SOCK_DOM(so) == PF_INET6) { + ncu = &sotoinpcb(so)->necp_client_uuid; + } else { + error = EINVAL; + goto out; + } + + error = sooptcopyout(sopt, ncu, sizeof(uuid_t)); + break; + } #endif /* NECP */ #if CONTENT_FILTER @@ -5708,19 +5765,6 @@ integer: } #endif /* CONTENT_FILTER */ -#if MPTCP - case SO_MPTCP_FASTJOIN: - if (!((so->so_flags & SOF_MP_SUBFLOW) || - ((SOCK_CHECK_DOM(so, PF_MULTIPATH)) && - (SOCK_CHECK_PROTO(so, IPPROTO_TCP))))) { - error = ENOPROTOOPT; - break; - } - optval = (so->so_flags & SOF_MPTCP_FASTJOIN); - /* Fixed along with rdar://19391339 */ - goto integer; -#endif /* MPTCP */ - case SO_EXTENDED_BK_IDLE: optval = (so->so_flags1 & SOF1_EXTEND_BK_IDLE_WANTED); goto integer; @@ -5940,7 +5984,8 @@ sopoll(struct socket *so, int events, kauth_cred_t cred, void * wql) } int -soo_kqfilter(struct fileproc *fp, struct knote *kn, vfs_context_t ctx) +soo_kqfilter(struct fileproc *fp, struct knote *kn, + struct kevent_internal_s *kev, vfs_context_t ctx) { #pragma unused(fp) #if !CONFIG_MACF_SOCKET @@ -5987,7 +6032,7 @@ soo_kqfilter(struct fileproc *fp, struct knote *kn, vfs_context_t ctx) * call the appropriate sub-filter attach * with the socket still locked */ - result = knote_fops(kn)->f_attach(kn); + result = knote_fops(kn)->f_attach(kn, kev); socket_unlock(so, 1); @@ -6067,7 +6112,7 @@ filt_soread_common(struct knote *kn, struct socket *so) } static int -filt_sorattach(struct knote *kn) +filt_sorattach(struct knote *kn, __unused struct kevent_internal_s *kev) { struct socket *so = (struct socket *)kn->kn_fp->f_fglob->fg_data; @@ -6234,7 +6279,7 @@ filt_sowrite_common(struct knote *kn, struct socket *so) } static int -filt_sowattach(struct knote *kn) +filt_sowattach(struct knote *kn, __unused struct kevent_internal_s *kev) { struct socket *so = (struct socket *)kn->kn_fp->f_fglob->fg_data; @@ -6426,7 +6471,7 @@ filt_sockev_common(struct knote *kn, struct socket *so, long ev_hint) } static int -filt_sockattach(struct knote *kn) +filt_sockattach(struct knote *kn, __unused struct kevent_internal_s *kev) { struct socket *so = (struct socket *)kn->kn_fp->f_fglob->fg_data; @@ -6610,19 +6655,18 @@ solockhistory_nr(struct socket *so) return (lock_history_str); } -int +void socket_lock(struct socket *so, int refcount) { - int error = 0; void *lr_saved; lr_saved = __builtin_return_address(0); if (so->so_proto->pr_lock) { - error = (*so->so_proto->pr_lock)(so, refcount, lr_saved); + (*so->so_proto->pr_lock)(so, refcount, lr_saved); } else { #ifdef MORE_LOCKING_DEBUG - lck_mtx_assert(so->so_proto->pr_domain->dom_mtx, + LCK_MTX_ASSERT(so->so_proto->pr_domain->dom_mtx, LCK_MTX_ASSERT_NOTOWNED); #endif lck_mtx_lock(so->so_proto->pr_domain->dom_mtx); @@ -6631,14 +6675,37 @@ socket_lock(struct socket *so, int refcount) so->lock_lr[so->next_lock_lr] = lr_saved; so->next_lock_lr = (so->next_lock_lr+1) % SO_LCKDBG_MAX; } +} - return (error); +void +socket_lock_assert_owned(struct socket *so) +{ + lck_mtx_t *mutex_held; + + if (so->so_proto->pr_getlock != NULL) + mutex_held = (*so->so_proto->pr_getlock)(so, 0); + else + mutex_held = so->so_proto->pr_domain->dom_mtx; + + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); } int +socket_try_lock(struct socket *so) +{ + lck_mtx_t *mtx; + + if (so->so_proto->pr_getlock != NULL) + mtx = (*so->so_proto->pr_getlock)(so, 0); + else + mtx = so->so_proto->pr_domain->dom_mtx; + + return (lck_mtx_try_lock(mtx)); +} + +void socket_unlock(struct socket *so, int refcount) { - int error = 0; void *lr_saved; lck_mtx_t *mutex_held; @@ -6650,11 +6717,11 @@ socket_unlock(struct socket *so, int refcount) } if (so && so->so_proto->pr_unlock) { - error = (*so->so_proto->pr_unlock)(so, refcount, lr_saved); + (*so->so_proto->pr_unlock)(so, refcount, lr_saved); } else { mutex_held = so->so_proto->pr_domain->dom_mtx; #ifdef MORE_LOCKING_DEBUG - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); #endif so->unlock_lr[so->next_unlock_lr] = lr_saved; so->next_unlock_lr = (so->next_unlock_lr+1) % SO_LCKDBG_MAX; @@ -6674,8 +6741,6 @@ socket_unlock(struct socket *so, int refcount) } lck_mtx_unlock(mutex_held); } - - return (error); } /* Called with socket locked, will unlock socket */ @@ -6688,7 +6753,7 @@ sofree(struct socket *so) mutex_held = (*so->so_proto->pr_getlock)(so, 0); else mutex_held = so->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); sofreelastref(so, 0); } @@ -6919,7 +6984,7 @@ sodefunct(struct proc *p, struct socket *so, int level) * Explicitly handle connectionless-protocol disconnection * and release any remaining data in the socket buffers. */ - if (!(so->so_flags & SS_ISDISCONNECTED)) + if (!(so->so_state & SS_ISDISCONNECTED)) (void) soisdisconnected(so); if (so->so_error == 0) @@ -6993,6 +7058,12 @@ so_set_extended_bk_idle(struct socket *so, int optval) struct filedesc *fdp; int count = 0; + /* + * Unlock socket to avoid lock ordering issue with + * the proc fd table lock + */ + socket_unlock(so, 0); + proc_fdlock(p); fdp = p->p_fd; @@ -7012,6 +7083,10 @@ so_set_extended_bk_idle(struct socket *so, int optval) if (count >= soextbkidlestat.so_xbkidle_maxperproc) break; } + proc_fdunlock(p); + + socket_lock(so, 0); + if (count >= soextbkidlestat.so_xbkidle_maxperproc) { OSIncrementAtomic(&soextbkidlestat.so_xbkidle_toomany); error = EBUSY; @@ -7029,8 +7104,6 @@ so_set_extended_bk_idle(struct socket *so, int optval) SOCK_DOM(so), SOCK_TYPE(so), (so->so_flags1 & SOF1_EXTEND_BK_IDLE_WANTED) ? "is" : "not"); - - proc_fdunlock(p); } return (error); @@ -7145,6 +7218,7 @@ so_set_recv_anyif(struct socket *so, int optval) sotoinpcb(so)->inp_flags &= ~INP_RECV_ANYIF; } + return (ret); } @@ -7212,6 +7286,9 @@ so_set_restrictions(struct socket *so, uint32_t vals) } } + if (SOCK_DOM(so) == PF_MULTIPATH) + mptcp_set_restrictions(so); + return (0); } diff --git a/bsd/kern/uipc_socket2.c b/bsd/kern/uipc_socket2.c index 45c151848..08ec21e97 100644 --- a/bsd/kern/uipc_socket2.c +++ b/bsd/kern/uipc_socket2.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998-2017 Apple Inc. All rights reserved. + * Copyright (c) 1998-2015 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -89,6 +89,7 @@ #include <net/content_filter.h> #include <netinet/in.h> #include <netinet/in_pcb.h> +#include <netinet/tcp_var.h> #include <sys/kdebug.h> #include <libkern/OSAtomic.h> @@ -142,10 +143,6 @@ int32_t total_sbmb_cnt_floor __attribute__((aligned(8))) = 0; int32_t total_sbmb_cnt_peak __attribute__((aligned(8))) = 0; int64_t sbmb_limreached __attribute__((aligned(8))) = 0; -/* Control whether to throttle sockets eligible to be throttled */ -__private_extern__ u_int32_t net_io_policy_throttled = 0; -static int sysctl_io_policy_throttled SYSCTL_HANDLER_ARGS; - u_int32_t net_io_policy_log = 0; /* log socket policy changes */ #if CONFIG_PROC_UUID_POLICY u_int32_t net_io_policy_uuid = 1; /* enable UUID socket policy */ @@ -202,7 +199,7 @@ soisconnected(struct socket *so) if (so->so_head != NULL && (so->so_state & SS_INCOMP)) { struct socket *head = so->so_head; int locked = 0; - + /* * Enforce lock order when the protocol has per socket locks */ @@ -233,7 +230,7 @@ soisconnected(struct socket *so) if (locked != 0) { socket_unlock(head, 1); socket_lock(so, 0); - } + } } else if (locked != 0) { so_release_accept_list(head); socket_unlock(head, 1); @@ -327,7 +324,7 @@ sonewconn_internal(struct socket *head, int connstatus) mutex_held = (*head->so_proto->pr_getlock)(head, 0); else mutex_held = head->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); if (!soqlencomp) { /* @@ -551,11 +548,11 @@ sbwait(struct sockbuf *sb) } if (so->so_proto->pr_getlock != NULL) - mutex_held = (*so->so_proto->pr_getlock)(so, 0); + mutex_held = (*so->so_proto->pr_getlock)(so, PR_F_WILLUNLOCK); else mutex_held = so->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); ts.tv_sec = sb->sb_timeo.tv_sec; ts.tv_nsec = sb->sb_timeo.tv_usec * 1000; @@ -630,15 +627,18 @@ sowakeup(struct socket *so, struct sockbuf *sb) if (sb->sb_flags & SB_UPCALL) { void (*sb_upcall)(struct socket *, void *, int); caddr_t sb_upcallarg; + int lock = !(sb->sb_flags & SB_UPCALL_LOCK); sb_upcall = sb->sb_upcall; sb_upcallarg = sb->sb_upcallarg; /* Let close know that we're about to do an upcall */ so->so_upcallusecount++; - socket_unlock(so, 0); + if (lock) + socket_unlock(so, 0); (*sb_upcall)(so, sb_upcallarg, M_DONTWAIT); - socket_lock(so, 0); + if (lock) + socket_lock(so, 0); so->so_upcallusecount--; /* Tell close that it's safe to proceed */ @@ -897,7 +897,7 @@ sbcheck(struct sockbuf *sb) else mutex_held = sb->sb_so->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); if (sbchecking == 0) return; @@ -1468,9 +1468,6 @@ sbappendmptcpstream_rcv(struct sockbuf *sb, struct mbuf *m) SBLASTMBUFCHK(sb, __func__); - if (mptcp_adj_rmap(so, m) != 0) - return (0); - /* No filter support (SB_RECV) on mptcp subflow sockets */ sbcompress(sb, m, sb->sb_mbtail); @@ -1862,14 +1859,18 @@ sbdrop(struct sockbuf *sb, int len) next = (m = sb->sb_mb) ? m->m_nextpkt : 0; #if MPTCP - if ((m != NULL) && (len > 0) && - (!(sb->sb_flags & SB_RECV)) && + if (m != NULL && len > 0 && !(sb->sb_flags & SB_RECV) && ((sb->sb_so->so_flags & SOF_MP_SUBFLOW) || - ((SOCK_CHECK_DOM(sb->sb_so, PF_MULTIPATH)) && - (SOCK_CHECK_PROTO(sb->sb_so, IPPROTO_TCP)))) && - (!(sb->sb_so->so_flags1 & SOF1_POST_FALLBACK_SYNC))) { + (SOCK_CHECK_DOM(sb->sb_so, PF_MULTIPATH) && + SOCK_CHECK_PROTO(sb->sb_so, IPPROTO_TCP))) && + !(sb->sb_so->so_flags1 & SOF1_POST_FALLBACK_SYNC)) { mptcp_preproc_sbdrop(sb->sb_so, m, (unsigned int)len); } + if (m != NULL && len > 0 && !(sb->sb_flags & SB_RECV) && + (sb->sb_so->so_flags & SOF_MP_SUBFLOW) && + (sb->sb_so->so_flags1 & SOF1_POST_FALLBACK_SYNC)) { + mptcp_fallback_sbdrop(sb->sb_so, m, len); + } #endif /* MPTCP */ KERNEL_DEBUG((DBG_FNC_SBDROP | DBG_FUNC_START), sb, len, 0, 0, 0); @@ -2137,13 +2138,6 @@ pru_listen_notsupp(struct socket *so, struct proc *p) return (EOPNOTSUPP); } -int -pru_peeloff_notsupp(struct socket *so, sae_associd_t aid, struct socket **psop) -{ -#pragma unused(so, aid, psop) - return (EOPNOTSUPP); -} - int pru_peeraddr_notsupp(struct socket *so, struct sockaddr **nam) { @@ -2290,7 +2284,6 @@ pru_sanitize(struct pr_usrreqs *pru) DEFAULT(pru->pru_disconnect, pru_disconnect_notsupp); DEFAULT(pru->pru_disconnectx, pru_disconnectx_notsupp); DEFAULT(pru->pru_listen, pru_listen_notsupp); - DEFAULT(pru->pru_peeloff, pru_peeloff_notsupp); DEFAULT(pru->pru_peeraddr, pru_peeraddr_notsupp); DEFAULT(pru->pru_rcvd, pru_rcvd_notsupp); DEFAULT(pru->pru_rcvoob, pru_rcvoob_notsupp); @@ -2363,7 +2356,7 @@ int msgq_sbspace(struct socket *so, struct mbuf *control) { int space = 0, error; - u_int32_t msgpri; + u_int32_t msgpri = 0; VERIFY(so->so_type == SOCK_STREAM && SOCK_PROTO(so) == IPPROTO_TCP); if (control != NULL) { @@ -2566,11 +2559,11 @@ sblock(struct sockbuf *sb, uint32_t flags) * us the lock. This will be fixed in future. */ if (so->so_proto->pr_getlock != NULL) - mutex_held = (*so->so_proto->pr_getlock)(so, 0); + mutex_held = (*so->so_proto->pr_getlock)(so, PR_F_WILLUNLOCK); else mutex_held = so->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); sb->sb_wantlock++; VERIFY(sb->sb_wantlock != 0); @@ -2659,11 +2652,11 @@ sbunlock(struct sockbuf *sb, boolean_t keeplocked) lck_mtx_t *mutex_held; if (so->so_proto->pr_getlock != NULL) - mutex_held = (*so->so_proto->pr_getlock)(so, 0); + mutex_held = (*so->so_proto->pr_getlock)(so, PR_F_WILLUNLOCK); else mutex_held = so->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); VERIFY(so->so_usecount > 0); so->so_usecount--; @@ -2711,18 +2704,10 @@ soevupcall(struct socket *so, u_int32_t hint) { if (so->so_event != NULL) { caddr_t so_eventarg = so->so_eventarg; - int locked = hint & SO_FILT_HINT_LOCKED; hint &= so->so_eventmask; - if (hint != 0) { - if (locked) - socket_unlock(so, 0); - + if (hint != 0) so->so_event(so, so_eventarg, hint); - - if (locked) - socket_lock(so, 0); - } } } @@ -2836,6 +2821,7 @@ sotoxsocket(struct socket *so, struct xsocket *xso) } +#if !CONFIG_EMBEDDED void sotoxsocket64(struct socket *so, struct xsocket64 *xso) @@ -2865,6 +2851,7 @@ sotoxsocket64(struct socket *so, struct xsocket64 *xso) xso->so_uid = kauth_cred_getuid(so->so_cred); } +#endif /* !CONFIG_EMBEDDED */ /* * This does the same for sockbufs. Note that the xsockbuf structure, @@ -2894,12 +2881,7 @@ sbtoxsockbuf(struct sockbuf *sb, struct xsockbuf *xsb) inline int soisthrottled(struct socket *so) { - /* - * On non-embedded, we rely on implicit throttling by the - * application, as we're missing the system wide "decision maker" - */ - return ( - (so->so_flags1 & SOF1_TRAFFIC_MGT_SO_BACKGROUND)); + return (so->so_flags1 & SOF1_TRAFFIC_MGT_SO_BACKGROUND); } inline int @@ -2930,6 +2912,16 @@ soissrcbesteffort(struct socket *so) so->so_traffic_class == SO_TC_OAM); } +void +soclearfastopen(struct socket *so) +{ + if (so->so_flags1 & SOF1_PRECONNECT_DATA) + so->so_flags1 &= ~SOF1_PRECONNECT_DATA; + + if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) + so->so_flags1 &= ~SOF1_DATA_IDEMPOTENT; +} + void sonullevent(struct socket *so, void *arg, uint32_t hint) { @@ -2963,27 +2955,6 @@ sysctl_sb_max SYSCTL_HANDLER_ARGS return (error); } -static int -sysctl_io_policy_throttled SYSCTL_HANDLER_ARGS -{ -#pragma unused(arg1, arg2) - int i, err; - - i = net_io_policy_throttled; - - err = sysctl_handle_int(oidp, &i, 0, req); - if (err != 0 || req->newptr == USER_ADDR_NULL) - return (err); - - if (i != net_io_policy_throttled) - SOTHROTTLELOG("throttle: network IO policy throttling is " - "now %s\n", i ? "ON" : "OFF"); - - net_io_policy_throttled = i; - - return (err); -} - SYSCTL_PROC(_kern_ipc, KIPC_MAXSOCKBUF, maxsockbuf, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &sb_max, 0, &sysctl_sb_max, "IU", "Maximum socket buffer size"); @@ -3004,8 +2975,39 @@ SYSCTL_INT(_kern_ipc, KIPC_SOQLIMITCOMPAT, soqlimitcompat, CTLFLAG_RW | CTLFLAG_LOCKED, &soqlimitcompat, 1, "Enable socket queue limit compatibility"); -SYSCTL_INT(_kern_ipc, OID_AUTO, soqlencomp, CTLFLAG_RW | CTLFLAG_LOCKED, - &soqlencomp, 0, "Listen backlog represents only complete queue"); +/* + * Hack alert -- rdar://33572856 + * A loopback test we cannot change was failing because it sets + * SO_SENDTIMEO to 5 seconds and that's also the value + * of the minimum persist timer. Because of the persist timer, + * the connection was not idle for 5 seconds and SO_SNDTIMEO + * was not triggering at 5 seconds causing the test failure. + * As a workaround we check the sysctl soqlencomp the test is already + * setting to set disable auto tuning of the receive buffer. + */ + +extern u_int32_t tcp_do_autorcvbuf; + +static int +sysctl_soqlencomp SYSCTL_HANDLER_ARGS +{ +#pragma unused(oidp, arg1, arg2) + u_int32_t new_value; + int changed = 0; + int error = sysctl_io_number(req, soqlencomp, sizeof (u_int32_t), + &new_value, &changed); + if (!error && changed) { + soqlencomp = new_value; + if (new_value != 0) { + tcp_do_autorcvbuf = 0; + tcptv_persmin_val = 6 * TCP_RETRANSHZ; + } + } + return (error); +} +SYSCTL_PROC(_kern_ipc, OID_AUTO, soqlencomp, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, + &soqlencomp, 0, &sysctl_soqlencomp, "IU", ""); SYSCTL_INT(_kern_ipc, OID_AUTO, sbmb_cnt, CTLFLAG_RD | CTLFLAG_LOCKED, &total_sbmb_cnt, 0, ""); @@ -3019,10 +3021,6 @@ SYSCTL_QUAD(_kern_ipc, OID_AUTO, sbmb_limreached, CTLFLAG_RD | CTLFLAG_LOCKED, SYSCTL_NODE(_kern_ipc, OID_AUTO, io_policy, CTLFLAG_RW, 0, "network IO policy"); -SYSCTL_PROC(_kern_ipc_io_policy, OID_AUTO, throttled, - CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &net_io_policy_throttled, 0, - sysctl_io_policy_throttled, "I", ""); - SYSCTL_INT(_kern_ipc_io_policy, OID_AUTO, log, CTLFLAG_RW | CTLFLAG_LOCKED, &net_io_policy_log, 0, ""); diff --git a/bsd/kern/uipc_syscalls.c b/bsd/kern/uipc_syscalls.c index 0aaba750a..53043d70b 100644 --- a/bsd/kern/uipc_syscalls.c +++ b/bsd/kern/uipc_syscalls.c @@ -156,7 +156,6 @@ static int connectx_nocancel(struct proc *, struct connectx_args *, int *); static int connectitx(struct socket *, struct sockaddr *, struct sockaddr *, struct proc *, uint32_t, sae_associd_t, sae_connid_t *, uio_t, unsigned int, user_ssize_t *); -static int peeloff_nocancel(struct proc *, struct peeloff_args *, int *); static int disconnectx_nocancel(struct proc *, struct disconnectx_args *, int *); static int socket_common(struct proc *, int, int, int, pid_t, int32_t *, int); @@ -439,7 +438,7 @@ accept_nocancel(struct proc *p, struct accept_nocancel_args *uap, socket_lock(head, 1); if (head->so_proto->pr_getlock != NULL) { - mutex_held = (*head->so_proto->pr_getlock)(head, 0); + mutex_held = (*head->so_proto->pr_getlock)(head, PR_F_WILLUNLOCK); dosocklock = 1; } else { mutex_held = head->so_proto->pr_domain->dom_mtx; @@ -931,7 +930,7 @@ connectit(struct socket *so, struct sockaddr *sa) lck_mtx_t *mutex_held; if (so->so_proto->pr_getlock != NULL) - mutex_held = (*so->so_proto->pr_getlock)(so, 0); + mutex_held = (*so->so_proto->pr_getlock)(so, PR_F_WILLUNLOCK); else mutex_held = so->so_proto->pr_domain->dom_mtx; error = msleep((caddr_t)&so->so_timeo, mutex_held, @@ -979,7 +978,7 @@ connectitx(struct socket *so, struct sockaddr *src, so->so_flags1 |= SOF1_DATA_IDEMPOTENT; if (flags & CONNECT_DATA_AUTHENTICATED) - so->so_flags |= SOF1_DATA_AUTHENTICATED; + so->so_flags1 |= SOF1_DATA_AUTHENTICATED; } /* @@ -1029,7 +1028,7 @@ connectitx(struct socket *so, struct sockaddr *src, lck_mtx_t *mutex_held; if (so->so_proto->pr_getlock != NULL) - mutex_held = (*so->so_proto->pr_getlock)(so, 0); + mutex_held = (*so->so_proto->pr_getlock)(so, PR_F_WILLUNLOCK); else mutex_held = so->so_proto->pr_domain->dom_mtx; error = msleep((caddr_t)&so->so_timeo, mutex_held, @@ -1052,80 +1051,13 @@ out: int peeloff(struct proc *p, struct peeloff_args *uap, int *retval) { +#pragma unused(p, uap, retval) /* * Due to similiarity with a POSIX interface, define as * an unofficial cancellation point. */ __pthread_testcancel(1); - return (peeloff_nocancel(p, uap, retval)); -} - -static int -peeloff_nocancel(struct proc *p, struct peeloff_args *uap, int *retval) -{ - struct fileproc *fp; - struct socket *mp_so, *so = NULL; - int newfd, fd = uap->s; - short fflag; /* type must match fp->f_flag */ - int error; - - *retval = -1; - - error = fp_getfsock(p, fd, &fp, &mp_so); - if (error != 0) { - if (error == EOPNOTSUPP) - error = ENOTSOCK; - goto out_nofile; - } - if (mp_so == NULL) { - error = EBADF; - goto out; - } - - socket_lock(mp_so, 1); - error = sopeelofflocked(mp_so, uap->aid, &so); - if (error != 0) { - socket_unlock(mp_so, 1); - goto out; - } - VERIFY(so != NULL); - socket_unlock(mp_so, 0); /* keep ref on mp_so for us */ - - fflag = fp->f_flag; - error = falloc(p, &fp, &newfd, vfs_context_current()); - if (error != 0) { - /* drop this socket (probably ran out of file descriptors) */ - soclose(so); - sodereference(mp_so); /* our mp_so ref */ - goto out; - } - - fp->f_flag = fflag; - fp->f_ops = &socketops; - fp->f_data = (caddr_t)so; - - /* - * If the socket has been marked as inactive by sosetdefunct(), - * disallow further operations on it. - */ - if (so->so_flags & SOF_DEFUNCT) { - sodefunct(current_proc(), so, - SHUTDOWN_SOCKET_LEVEL_DISCONNECT_INTERNAL); - } - - proc_fdlock(p); - procfdtbl_releasefd(p, newfd, NULL); - fp_drop(p, newfd, fp, 1); - proc_fdunlock(p); - - sodereference(mp_so); /* our mp_so ref */ - *retval = newfd; - -out: - file_drop(fd); - -out_nofile: - return (error); + return (0); } int diff --git a/bsd/kern/uipc_usrreq.c b/bsd/kern/uipc_usrreq.c index 4f31897bd..b8b429c08 100644 --- a/bsd/kern/uipc_usrreq.c +++ b/bsd/kern/uipc_usrreq.c @@ -305,7 +305,7 @@ uipc_detach(struct socket *so) if (unp == 0) return (EINVAL); - lck_mtx_assert(&unp->unp_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&unp->unp_mtx, LCK_MTX_ASSERT_OWNED); unp_detach(unp); return (0); } @@ -987,8 +987,18 @@ unp_bind( return (EAFNOSUPPORT); } + /* + * Check if the socket is already bound to an address + */ if (unp->unp_vnode != NULL) return (EINVAL); + /* + * Check if the socket may have been shut down + */ + if ((so->so_state & (SS_CANTRCVMORE | SS_CANTSENDMORE)) == + (SS_CANTRCVMORE | SS_CANTSENDMORE)) + return (EINVAL); + namelen = soun->sun_len - offsetof(struct sockaddr_un, sun_path); if (namelen <= 0) return (EINVAL); @@ -1311,7 +1321,7 @@ decref_out: } out: - lck_mtx_assert(&unp->unp_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&unp->unp_mtx, LCK_MTX_ASSERT_OWNED); vnode_put(vp); return (error); } @@ -1332,8 +1342,8 @@ unp_connect2(struct socket *so, struct socket *so2) unp2 = sotounpcb(so2); - lck_mtx_assert(&unp->unp_mtx, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&unp2->unp_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&unp->unp_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&unp2->unp_mtx, LCK_MTX_ASSERT_OWNED); /* Verify both sockets are still opened */ if (unp == 0 || unp2 == 0) @@ -1397,8 +1407,8 @@ unp_connect2(struct socket *so, struct socket *so2) default: panic("unknown socket type %d in unp_connect2", so->so_type); } - lck_mtx_assert(&unp->unp_mtx, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&unp2->unp_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&unp->unp_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&unp2->unp_mtx, LCK_MTX_ASSERT_OWNED); return (0); } @@ -1460,8 +1470,8 @@ try_again: } so_locked = 1; - lck_mtx_assert(&unp->unp_mtx, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&unp2->unp_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&unp->unp_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&unp2->unp_mtx, LCK_MTX_ASSERT_OWNED); /* Check for the UNP_DONTDISCONNECT flag, if it * is set, release both sockets and go to sleep @@ -1500,7 +1510,7 @@ try_again: case SOCK_STREAM: unp2->unp_conn = NULL; - VERIFY(so2->so_usecount > 0); + VERIFY(so->so_usecount > 0); so->so_usecount--; /* Set the socket state correctly but do a wakeup later when @@ -1535,7 +1545,7 @@ out: socket_lock(so,0); soisdisconnected(so); } - lck_mtx_assert(&unp->unp_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&unp->unp_mtx, LCK_MTX_ASSERT_OWNED); return; } @@ -1707,6 +1717,7 @@ SYSCTL_PROC(_net_local_stream, OID_AUTO, pcblist, (caddr_t)(long)SOCK_STREAM, 0, unp_pcblist, "S,xunpcb", "List of active local stream sockets"); +#if !CONFIG_EMBEDDED static int unp_pcblist64 SYSCTL_HANDLER_ARGS @@ -1855,6 +1866,7 @@ SYSCTL_PROC(_net_local_stream, OID_AUTO, pcblist64, (caddr_t)(long)SOCK_STREAM, 0, unp_pcblist64, "S,xunpcb64", "List of active local stream sockets 64 bit"); +#endif /* !CONFIG_EMBEDDED */ static void unp_shutdown(struct unpcb *unp) @@ -2461,7 +2473,7 @@ unp_unlock(struct socket *so, int refcount, void * lr) } else { mutex_held = &((struct unpcb *)so->so_pcb)->unp_mtx; } - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); so->unlock_lr[so->next_unlock_lr] = lr_saved; so->next_unlock_lr = (so->next_unlock_lr+1) % SO_LCKDBG_MAX; @@ -2485,7 +2497,7 @@ unp_unlock(struct socket *so, int refcount, void * lr) } lck_mtx_t * -unp_getlock(struct socket *so, __unused int locktype) +unp_getlock(struct socket *so, __unused int flags) { struct unpcb *unp = (struct unpcb *)so->so_pcb; diff --git a/bsd/libkern/libkern.h b/bsd/libkern/libkern.h index bfef07c6a..e2a47fdef 100644 --- a/bsd/libkern/libkern.h +++ b/bsd/libkern/libkern.h @@ -78,6 +78,9 @@ #include <sys/types.h> #include <mach/vm_param.h> +#if defined(__arm__) || defined(__arm64__) +#include <arm/arch.h> /* for _ARM_ARCH_* */ +#endif #ifdef __APPLE_API_OBSOLETE /* BCD conversions. */ @@ -140,6 +143,9 @@ ulmin(u_int32_t a, u_int32_t b) /* Prototypes for non-quad routines. */ extern int ffs(int); +extern int ffsll(unsigned long long); +extern int fls(int); +extern int flsll(unsigned long long); extern u_int32_t random(void); extern int scanc(u_int, u_char *, const u_char *, int); extern int skpc(int, int, char *); @@ -154,7 +160,7 @@ extern void url_decode(char *str); int snprintf(char *, size_t, const char *, ...) __printflike(3,4); /* sprintf() is being deprecated. Please use snprintf() instead. */ -int sprintf(char *bufp, const char *, ...) __deprecated; +int sprintf(char *bufp, const char *, ...) __deprecated __printflike(2,3); int sscanf(const char *, char const *, ...) __scanflike(2,3); int printf(const char *, ...) __printflike(1,2); @@ -184,19 +190,23 @@ extern int copyin_word(const user_addr_t user_addr, uint64_t *kernel_addr, vm_si int vsscanf(const char *, char const *, va_list); -extern int vprintf(const char *, va_list); -extern int vsnprintf(char *, size_t, const char *, va_list); +extern int vprintf(const char *, va_list) __printflike(1,0); +extern int vsnprintf(char *, size_t, const char *, va_list) __printflike(3,0); #if XNU_KERNEL_PRIVATE -extern int vprintf_log_locked(const char *, va_list); +extern int vprintf_log_locked(const char *, va_list) __printflike(1,0); extern void osobject_retain(void * object); extern void osobject_release(void * object); #endif /* vsprintf() is being deprecated. Please use vsnprintf() instead. */ -extern int vsprintf(char *bufp, const char *, va_list) __deprecated; +extern int vsprintf(char *bufp, const char *, va_list) __deprecated __printflike(2,0); #ifdef KERNEL_PRIVATE +#ifdef __arm__ +void flush_inner_dcaches(void); +void clean_inner_dcaches(void); +#endif extern void invalidate_icache(vm_offset_t, unsigned, int); extern void flush_dcache(vm_offset_t, unsigned, int); #else @@ -210,8 +220,13 @@ extern void flush_dcache64(addr64_t, unsigned, int); static inline int clz(unsigned int num) { +#if (__arm__ || __arm64__) + // On ARM, clz(0) is defined to return number of bits in the input type + return __builtin_clz(num); +#else // On Intel, clz(0) is undefined return num ? __builtin_clz(num) : sizeof(num) * CHAR_BIT; +#endif } __END_DECLS diff --git a/bsd/libkern/url_encode.c b/bsd/libkern/url_encode.c index 28534ada1..0e3cf1562 100644 --- a/bsd/libkern/url_encode.c +++ b/bsd/libkern/url_encode.c @@ -74,10 +74,7 @@ url_decode(char *str) * string down a few characters */ *esc++ = c; str = memmove(esc, str, strlen(str)+1); - } else { - str++; } - } else { str++; } diff --git a/bsd/machine/Makefile b/bsd/machine/Makefile index 5b190e442..bffe6a814 100644 --- a/bsd/machine/Makefile +++ b/bsd/machine/Makefile @@ -20,7 +20,7 @@ KERNELFILES = \ disklabel.h \ byte_order.h endian.h \ limits.h param.h profile.h \ - signal.h spl.h types.h \ + signal.h types.h \ vmparam.h _types.h _limits.h _param.h \ _mcontext.h diff --git a/bsd/machine/_limits.h b/bsd/machine/_limits.h index c1d8abd07..736a5886b 100644 --- a/bsd/machine/_limits.h +++ b/bsd/machine/_limits.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/_limits.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/_limits.h" #else #error architecture not supported #endif diff --git a/bsd/machine/_mcontext.h b/bsd/machine/_mcontext.h index ee9b1d943..e22043742 100644 --- a/bsd/machine/_mcontext.h +++ b/bsd/machine/_mcontext.h @@ -27,6 +27,8 @@ */ #if defined (__i386__) || defined (__x86_64__) #include "i386/_mcontext.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/_mcontext.h" #else #error architecture not supported #endif diff --git a/bsd/machine/_param.h b/bsd/machine/_param.h index beb2cb939..96b0c2fef 100644 --- a/bsd/machine/_param.h +++ b/bsd/machine/_param.h @@ -27,6 +27,8 @@ */ #if defined (__i386__) || defined (__x86_64__) #include "i386/_param.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/_param.h" #else #error architecture not supported #endif diff --git a/bsd/machine/_types.h b/bsd/machine/_types.h index 92c65bf6c..be86a2368 100644 --- a/bsd/machine/_types.h +++ b/bsd/machine/_types.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/_types.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/_types.h" #else #error architecture not supported #endif diff --git a/bsd/machine/disklabel.h b/bsd/machine/disklabel.h index 490bbda8a..a29df81d3 100644 --- a/bsd/machine/disklabel.h +++ b/bsd/machine/disklabel.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/disklabel.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/disklabel.h" #else #error architecture not supported #endif diff --git a/bsd/machine/endian.h b/bsd/machine/endian.h index 871af6483..9cefbf79a 100644 --- a/bsd/machine/endian.h +++ b/bsd/machine/endian.h @@ -33,6 +33,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/endian.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/endian.h" #else #error architecture not supported #endif diff --git a/bsd/machine/exec.h b/bsd/machine/exec.h index a5712128a..d4bc6a86a 100644 --- a/bsd/machine/exec.h +++ b/bsd/machine/exec.h @@ -46,6 +46,8 @@ boolean_t pie_required(cpu_type_t, cpu_subtype_t); #if defined (__i386__) || defined(__x86_64__) #include "i386/exec.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/exec.h" #else #error architecture not supported #endif diff --git a/bsd/machine/fasttrap_isa.h b/bsd/machine/fasttrap_isa.h index cfe9e297a..7f31b4eec 100644 --- a/bsd/machine/fasttrap_isa.h +++ b/bsd/machine/fasttrap_isa.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/fasttrap_isa.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/fasttrap_isa.h" #else #error architecture not supported #endif diff --git a/bsd/machine/limits.h b/bsd/machine/limits.h index e96709f89..39b348a7b 100644 --- a/bsd/machine/limits.h +++ b/bsd/machine/limits.h @@ -4,6 +4,8 @@ This file is public domain. */ #if defined (__i386__) || defined(__x86_64__) #include <i386/limits.h> +#elif defined (__arm__) || defined (__arm64__) +#include <arm/limits.h> #else #error architecture not supported #endif diff --git a/bsd/machine/param.h b/bsd/machine/param.h index 2724da7e1..74b280059 100644 --- a/bsd/machine/param.h +++ b/bsd/machine/param.h @@ -33,6 +33,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/param.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/param.h" #else #error architecture not supported #endif diff --git a/bsd/machine/profile.h b/bsd/machine/profile.h index cc8a5eac0..14f2977ee 100644 --- a/bsd/machine/profile.h +++ b/bsd/machine/profile.h @@ -35,6 +35,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/profile.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/profile.h" #else #error architecture not supported #endif diff --git a/bsd/machine/psl.h b/bsd/machine/psl.h index 01c6e0a25..6c260a01b 100644 --- a/bsd/machine/psl.h +++ b/bsd/machine/psl.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/psl.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/psl.h" #else #error architecture not supported #endif diff --git a/bsd/machine/ptrace.h b/bsd/machine/ptrace.h index 3320c2226..cb5ecd990 100644 --- a/bsd/machine/ptrace.h +++ b/bsd/machine/ptrace.h @@ -33,6 +33,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/ptrace.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/ptrace.h" #else #error architecture not supported #endif diff --git a/bsd/machine/reboot.h b/bsd/machine/reboot.h index 864f1970c..0a00f2ec2 100644 --- a/bsd/machine/reboot.h +++ b/bsd/machine/reboot.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/reboot.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/reboot.h" #else #error architecture not supported #endif diff --git a/bsd/machine/reg.h b/bsd/machine/reg.h index 30e5dc524..8f4128740 100644 --- a/bsd/machine/reg.h +++ b/bsd/machine/reg.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/reg.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/reg.h" #else #error architecture not supported #endif diff --git a/bsd/machine/signal.h b/bsd/machine/signal.h index 4b7f69c19..46b23f231 100644 --- a/bsd/machine/signal.h +++ b/bsd/machine/signal.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/signal.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/signal.h" #else #error architecture not supported #endif diff --git a/bsd/machine/smp.h b/bsd/machine/smp.h new file mode 100644 index 000000000..f97a38fe8 --- /dev/null +++ b/bsd/machine/smp.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _BSD_MACHINE_SMP_H_ +#define _BSD_MACHINE_SMP_H_ + +#if defined (__i386__) || defined(__x86_64__) +#include "i386/smp.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/smp.h" +#else +#error architecture not supported +#endif + +#endif /* _BSD_MACHINE_SMP_H_ */ diff --git a/bsd/machine/types.h b/bsd/machine/types.h index 5d6d4db44..c14795279 100644 --- a/bsd/machine/types.h +++ b/bsd/machine/types.h @@ -33,6 +33,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/types.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/types.h" #else #error architecture not supported #endif diff --git a/bsd/machine/vmparam.h b/bsd/machine/vmparam.h index 54b212382..3817b5a67 100644 --- a/bsd/machine/vmparam.h +++ b/bsd/machine/vmparam.h @@ -30,6 +30,8 @@ #if defined (__i386__) || defined(__x86_64__) #include "i386/vmparam.h" +#elif defined (__arm__) || defined (__arm64__) +#include "arm/vmparam.h" #else #error architecture not supported #endif diff --git a/bsd/man/man2/Makefile b/bsd/man/man2/Makefile index 53d932f6d..331ca334e 100644 --- a/bsd/man/man2/Makefile +++ b/bsd/man/man2/Makefile @@ -62,13 +62,19 @@ DATAFILES = \ fpathconf.2 \ fremovexattr.2 \ fsetxattr.2 \ + fsgetpath.2 \ fstat.2 \ fstat64.2 \ fstatat.2 \ fstatfs.2 \ fstatfs64.2 \ fsync.2 \ + fs_snapshot_create.2 \ + fs_snapshot_list.2 \ + fs_snapshot_delete.2 \ + fs_snapshot_rename.2 \ ftruncate.2 \ + futimens.2 \ futimes.2 \ getattrlist.2 \ getattrlistat.2 \ @@ -143,7 +149,6 @@ DATAFILES = \ poll.2 \ posix_madvise.2 \ pread.2 \ - profil.2 \ pselect.2 \ pthread_setugid_np.2 \ ptrace.2 \ @@ -179,6 +184,7 @@ DATAFILES = \ sendmsg.2 \ sendto.2 \ setattrlist.2 \ + setattrlistat.2 \ setaudit.2 \ setaudit_addr.2 \ setauid.2 \ @@ -227,6 +233,7 @@ DATAFILES = \ unlink.2 \ unlinkat.2 \ unmount.2 \ + utimensat.2 \ utimes.2 \ vfork.2 \ wait.2 \ diff --git a/bsd/man/man2/clonefile.2 b/bsd/man/man2/clonefile.2 index c9fd137e6..114321e0d 100644 --- a/bsd/man/man2/clonefile.2 +++ b/bsd/man/man2/clonefile.2 @@ -61,7 +61,9 @@ or .Xr mkdirat 2 or .Xr symlinkat 2 -if the current user does not have privileges to change ownership. +if the current user does not have privileges to change ownership. If the optional +flag CLONE_NOOWNERCOPY is passed, the ownership information is the same as if the +the current user does not have privileges to change ownership . .It @@ -133,6 +135,17 @@ names a symbolic link. . .El .Pp +.Bl -tag -width CLONE_NOOWNERCOPY +. +.It CLONE_NOOWNERCOPY +Don't copy ownership information from the source when run called with superuser privileges. +The symbolic link is itself cloned if +.Fa src +names a symbolic link. +. +.El +.Pp +The The .Fn clonefile , .Fn clonefileat diff --git a/bsd/man/man2/connectx.2 b/bsd/man/man2/connectx.2 index 7fbca576e..6ee2ba8b2 100644 --- a/bsd/man/man2/connectx.2 +++ b/bsd/man/man2/connectx.2 @@ -154,15 +154,6 @@ specifies the length of that buffer. .\" .Fn connectx .\" calls. .\" .Pp -.\" If the initial connection is established without any protocol-level -.\" multipath association, the error -.\" .Er EPROTO -.\" will be returned, and the connection can be extracted to a new socket with -.\" the same properties of -.\" .Fa socket , -.\" by calling -.\" .Xr peeloff 2 . -.\" .Pp .\" An association representing one or more connections, or a single connection .\" may be dissolved by calling .\" .Xr disconnectx 2 . @@ -363,11 +354,6 @@ Because .Fa socket is listening, no connection is allowed. .\" ========== -.\".It Bq Er EPROTO -.\"The connection was successfully established without any protocol-level -.\"association. The connection can be extracted to a new socket using -.\".Xr peeloff 2 . -.\" ========== .\".It Bq Er EPROTOTYPE .\".Fa address .\"has a different type than the socket @@ -385,7 +371,6 @@ Connection establishment timed out without establishing a connection. .Xr disconnectx 2 , .Xr getsockopt 2 , .\".Xr kqueue 2 , -.\".Xr peeloff 2 , .\".Xr shutdown 2 , .Xr select 2 , .Xr socket 2 , diff --git a/bsd/man/man2/exchangedata.2 b/bsd/man/man2/exchangedata.2 index 83dc23c1b..10a22ae34 100644 --- a/bsd/man/man2/exchangedata.2 +++ b/bsd/man/man2/exchangedata.2 @@ -182,10 +182,12 @@ An I/O error occurred while reading from or writing to the file system. . .Sh SEE ALSO . -.Xr getattrlist 2 +.Xr getattrlist 2 , +.Xr rename 2 . .Sh HISTORY A .Fn exchangedata function call appeared in Darwin 1.3.1 (Mac OS X version 10.0). . +It was deprecated in macOS 10.13. diff --git a/bsd/man/man2/fcntl.2 b/bsd/man/man2/fcntl.2 index dc3a20805..b55972bd9 100644 --- a/bsd/man/man2/fcntl.2 +++ b/bsd/man/man2/fcntl.2 @@ -188,11 +188,11 @@ A value of zero in turns data caching on. .It Dv F_LOG2PHYS Get disk device information. -Currently this only includes the +Currently this only returns the disk device address that corresponds -to the current file offset. Note that if the -file offset is not backed by physical blocks -we can return -1 as the offset. This is subject +to the current file offset. Note that the system +may return -1 as the disk device address if the file is not +backed by physical blocks. This is subject to change. .It Dv F_LOG2PHYS_EXT Variant of F_LOG2PHYS that uses the passed in diff --git a/bsd/man/man2/fs_snapshot_create.2 b/bsd/man/man2/fs_snapshot_create.2 new file mode 100644 index 000000000..57e3b3d34 --- /dev/null +++ b/bsd/man/man2/fs_snapshot_create.2 @@ -0,0 +1,201 @@ +.\" Copyright (c) 2017 Apple Computer, Inc. All rights reserved. +.\" +.\" The contents of this file constitute Original Code as defined in and +.\" are subject to the Apple Public Source License Version 1.1 (the +.\" "License"). You may not use this file except in compliance with the +.\" License. Please obtain a copy of the License at +.\" http://www.apple.com/publicsource and read it before using this file. +.\" +.\" This Original Code and all software distributed under the License are +.\" distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER +.\" EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, +.\" INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, +.\" FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the +.\" License for the specific language governing rights and limitations +.\" under the License. +.\" +.\" @(#)fs_snapshot_create.2 +. +.Dd July 4th, 2017 +.Dt FS_SNAPSHOT_CREATE 2 +.Os Darwin +.Sh NAME +.Nm fs_snasphot_create +.Nd create read only snapshot of a mounted filesystem +.Sh SYNOPSIS +.Fd #include <sys/attr.h> +.Fd #include <sys/snapshot.h> +.Pp +.Ft int +.Fn fs_snapshot_create "int dirfd" "const char * name" "uint32_t flags" +. +.Ft int +.Fn fs_snapshot_delete "int dirfd" "const char * name" "uint32_t flags" +. +.Ft int +.Fn fs_snapshot_list "int dirfd" "struct attrlist * name" "void * attrbuf" "size_t bufsize" "uint32_t flags" +. +.Ft int +.Fn fs_snapshot_rename "int dirfd" "const char * old" "const char * new" "uint32_t flags" +. +.Ft int +.Fn fs_snapshot_mount "int dirfd" "const char * dir" "const char * snapshot" "uint32_t flags" +. +.Ft int +.Fn fs_snapshot_revert "int dirfd" "const char * name" "uint32_t flags" +. +.Sh DESCRIPTION +The +.Fn fs_snapshot_create +function, for supported Filesystems, causes a snapshot of the Filesystem to be created. A snapshot is a read only copy +of the filesystem frozen at a point in time. The Filesystem is identified by the the +.Fa dirfd +parameter which should be a file descriptor associated with the root directory of the filesystem for the snapshot is to be created. +.Fa name +can be any valid name for a component name (except . and ..). +. +The +.Fn fs_snapshot_delete +function causes the named snapshot +.Fa name +to be deleted and the +.Fn fs_snapshot_rename +function causes the named snapshot +.Fa old +to be renamed to the name +.Fa new . +Available snapshots along with their attributes can be listed by calling +.Fn fs_snapshot_list +which is to be used in exactly the same way as +.Xr getattrlistbulk 2 . +. +The +.Fa flags +parameter specifies the options that can be passed. No options are currently defined. +.Pp +Snapshots may be useful for backing up the Filesystem and to restore the Filesystem to a previous state. +Snapshots are expected to consume no additional storage on creation but might consume additional storage as the active +Filesystem is modified. Similarly deletion of files on the active filesystem may not result in the storage being available +if the snapshot contains the file. Additionally, the underlying Filesystem may impose a limit on the number +of snapshots that can be taken. For supporting Filesystems, a snapshot may be used as a source for a mount. This can be done +by the +.Fn fs_snapshot_mount +function. The snapshot will be mounted read only. When a snapshot is mounted, it cannot be deleted but it can be renamed. +To revert the filesystem to a previous snapshot, the +.Fn fs_snapshot_revert +can be used. It should be noted that reverting a filesystem to a snapshot is a destructive operation and causes all +changes made to the filesystem (including snapshots created after the snapshot being reverted to) to be lost. +. +.Pp +All snapshot functions require superuser privileges and also require an additional entitlement. +. +.Sh RETURN VALUES +Upon successful completion, +.Fn fs_snapshot_create +, +.Fn fs_snapshot_delete +, +.Fn fs_snapshot_rename +and +.Fn fs_snapshot_list +returns 0. Otherwise, a value of -1 is returned and errno is set to indicate the error. +.Pp +.Sh COMPATIBILITY +Not all volumes support snapshots. A volume can be tested for snapshot support +by using +.Xr getattrlist 2 +to get the volume capabilities attribute ATTR_VOL_CAPABILITIES, and then testing the VOL_CAP_INT_SNAPSHOT flag. +.Pp +.Sh ERRORS +The +.Fn fs_snapshot_create +, +.Fn fs_snapshot_delete +, +.Fn fs_snapshot_rename +and +.Fn fs_snapshot_list +function will fail if: +.Bl -tag -width Er +. +.It Bq Er EACCES +Read permissions are denied for the caller on the filesystem +. +.It Bq Er ENOTSUP +The underlying filesystem does not support this call. +. +.It Bq Er EINVAL +The value of the +.Fa flags +parameter is invalid. +. +.It Bq Er ENOSPC +There is no free space remaining on the file system containing the file. +. +.It Bq Er ENOSPC +The limit for the maximum number of snapshots for a filesystem has been reached. +. +.It Bq Er EIO +An I/O error occurred while reading from or writing to the file system. +. +.It Bq Er EPERM +The calling process does not have appropriate privileges. +. +.It Bq Er EROFS +The requested operation requires modifications in a read-only file system. +. +.It Bq Er ENAMETOOLONG +The length of a component of a pathname is longer than {NAME_MAX}. +. +.It Bq Er EBADF +dirfd is not a valid file descriptor. +. +.It Bq Er ENOTDIR +dirfd is a file descriptor associated with a non-directory file. +.El +.Pp +In addition, the +.Fn fs_snapshot_create +or +.Fn fs_snapshot_rename +functions may fail with the following errors +.Bl -tag -width Er +.It Bq Er EEXIST +The The named snapshot to be created already exists or the new name already +exists for the snapshot being renamed. +. +.El +.Pp +.Fn fs_snapshot_create +or +.Fn fs_snapshot_rename +functions may fail with the following errors +.Bl -tag -width Er +.It Bq Er ENOENT +The named snapshot does not exist. +.El +. +.Pp +.Fn fs_snapshot_delete +function may fail with +.Bl -tag -width Er +.It Bq Er EBUSY +The named snapshot is currently mounted. +.El +. +.Sh SEE ALSO +. +.Xr getattrlist 2 , +.Xr getattrlistbulk 2 +. +.Sh HISTORY +The +.Fn fs_snapshot_create +, +.Fn fs_snapshot_delete +, +.Fn fs_snapshot_delete +and +.Fn fs_snapshot_list +function calls appeared in macOS version 10.13 +. diff --git a/bsd/man/man2/fs_snapshot_delete.2 b/bsd/man/man2/fs_snapshot_delete.2 new file mode 100644 index 000000000..20620315a --- /dev/null +++ b/bsd/man/man2/fs_snapshot_delete.2 @@ -0,0 +1 @@ +.so man2/fs_snapshot_create.2 diff --git a/bsd/man/man2/fs_snapshot_list.2 b/bsd/man/man2/fs_snapshot_list.2 new file mode 100644 index 000000000..20620315a --- /dev/null +++ b/bsd/man/man2/fs_snapshot_list.2 @@ -0,0 +1 @@ +.so man2/fs_snapshot_create.2 diff --git a/bsd/man/man2/fs_snapshot_rename.2 b/bsd/man/man2/fs_snapshot_rename.2 new file mode 100644 index 000000000..20620315a --- /dev/null +++ b/bsd/man/man2/fs_snapshot_rename.2 @@ -0,0 +1 @@ +.so man2/fs_snapshot_create.2 diff --git a/bsd/man/man2/fsgetpath.2 b/bsd/man/man2/fsgetpath.2 new file mode 100644 index 000000000..317c45cb7 --- /dev/null +++ b/bsd/man/man2/fsgetpath.2 @@ -0,0 +1,126 @@ +.\" Copyright (c) 2017 Apple Computer, Inc. All rights reserved. +.\" +.\" The contents of this file constitute Original Code as defined in and +.\" are subject to the Apple Public Source License Version 1.1 (the +.\" "License"). You may not use this file except in compliance with the +.\" License. Please obtain a copy of the License at +.\" http://www.apple.com/publicsource and read it before using this file. +.\" +.\" This Original Code and all software distributed under the License are +.\" distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER +.\" EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, +.\" INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, +.\" FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the +.\" License for the specific language governing rights and limitations +.\" under the License. +.\" +.\" @(#)fsgetpath.2 +. +.Dd July 27, 2017 +.Dt FSGETPATH 2 +.Os Darwin +.Sh NAME +.Nm fsgetpath +.Nd get the path associated with filesystem node identifier (inode number/link id/object id) +.Sh SYNOPSIS +.Fd #include <sys/attr.h> +.Fd #include <sys/fsgetpath.h> +.Pp +.Ft ssize_t +.Fn fsgetpath "char * restrict_buf" "size_t buflen" "fsid_t * fsid" "uint64_t obj_id" +. +.Sh DESCRIPTION +The +.Fn fsgetpath +function returns the path in a caller provided buffer +.Fa restrict_buf +of length indicated by +.Fa buflen +associated with a filesystem object identified by +.Fa fsid +and +.Fa obj_id. +.Fa fsid +is a pointer to a structure which identifies a filesystem to which the object belongs. +It is obtained by the value returned for ATTR_CMN_FSID in a previous call to +.Xr getattrlist 2 +or the +.Fa f_fsid +field of the +.Vt statfs +structure returned by +.Xr statfs 2 . +.Fa obj_id +can be any one of of a object identifier i.e. ATTR_CMN_FILEID returned by +.Xr getattrlist 2 +or +.Fa st_ino +field of the +.Vt stat +structure returned by +.Xr stat 2 +or a link id returned in ATTR_CMNEXT_LINKID by a previous call to +.Xr getattrlist 2 . +Using a linkid will result in a more accurate path in case the filesystem object is a +hard link. If a inode number is passed and the object is a hard link, any one of the +multiple paths to that filesystem object may be returned. +.Sh RETURN VALUES +Upon successful completion, +.Fn fsgetpath +returns the path length. Otherwise, a value of -1 is returned and errno is set to indicate the error. +.Pp +.Sh COMPATIBILITY +Not all volumes support +.Fn fsgetpath . +A volume can be tested for +.Fn fsgetpath +support by using +.Xr getattrlist 2 +to get the volume capabilities attribute ATTR_VOL_CAPABILITIES, and then testing the VOL_CAP_FMT_PATH_FROM_ID flag. +.Pp +.Sh ERRORS +The +.Fn fsgetpath +function will fail if: +.Bl -tag -width Er +. +.It Bq Er EACCES +Read permissions are denied on any component of the pathname. +. +.It Bq Er ENOTSUP +The underlying filesystem does not support this call. +. +.It Bq Er EINVAL +.Fa buflen +is larger than PAGE_SIZE +. +.It Bq Er EIO +An I/O error occurred while reading from the file system. +. +.It Bq Er EPERM +The calling process does not have appropriate privileges. +. +.It Bq Er ENOENT +The Filesystem object does not exist. +. +.It Bq Er EFAULT +restrict_buf points to memory not valid in the callers address space. +. +.It Bq Er ENOSPC +restrict_buf is not large enough to hold the path. +. +.El +. +.Pp +. +.Sh SEE ALSO +. +.Xr getattrlist 2 +.Xr statfs 2 +.Xr stat 2 +. +.Sh HISTORY +The +.Fn fsgetpath +function call appeared in macOS version 10.13 +. diff --git a/bsd/man/man2/futimens.2 b/bsd/man/man2/futimens.2 new file mode 100644 index 000000000..a365c7b53 --- /dev/null +++ b/bsd/man/man2/futimens.2 @@ -0,0 +1 @@ +.so man2/utimensat.2 diff --git a/bsd/man/man2/getattrlist.2 b/bsd/man/man2/getattrlist.2 index d0c23207c..a92c08dcd 100644 --- a/bsd/man/man2/getattrlist.2 +++ b/bsd/man/man2/getattrlist.2 @@ -435,14 +435,19 @@ An structure that uniquely identifies the file system object within a mounted volume for the duration of it's mount; this identifier is not guaranteed to be persistent for the volume and may change every time the volume is mounted. -If the VOL_CAP_FMT_64BIT_OBJECT_IDS capability is set, this is instead a 64-bit -object identifier. .Pp On HFS+ volumes, the ATTR_CMN_OBJID of a file system object is distinct from the ATTR_CMN_OBJID of any hard link to that file system object. Although the ATTR_CMN_OBJID of a file system object may appear similar (in whole or in part) to it's ATTR_CMN_FILEID (see description of ATTR_CMN_FILEID below), \fBno relation between the two attributes should ever be implied.\fP +.Pp +ATTR_CMN_OBJID is deprecated sarting with macOS 10.13, iOS 11.0, watchOS 4.0 and +tvOS 11.0 and ATTR_CMNEXT_LINKID should be used in its place. +ATTR_CMN_OBJID can only be used on older operating systems only if the file +system doesn't 64 bit IDs. See the +.Fn getLinkIDInfo +function in the EXAMPLES section. . .It ATTR_CMN_OBJPERMANENTID An @@ -450,8 +455,6 @@ An structure that uniquely and persistently identifies the file system object within its volume; persistence implies that this attribute is unaffected by mount/unmount operations on the volume. -If the VOL_CAP_FMT_64BIT_OBJECT_IDS capability is set, this is instead a 64-bit -object identifier. .Pp Some file systems can not return this attribute when the volume is mounted read-only and will fail the request with error @@ -467,8 +470,6 @@ structure that uniquely identifies the parent directory of the file system object within a mounted volume, for the duration of the volume mount; this identifier is not guaranteed to be persistent for the volume and may change every time the volume is mounted. -If the VOL_CAP_FMT_64BIT_OBJECT_IDS capability is set, this is instead a 64-bit -object identifier. .Pp . If a file system object is hard linked from multiple directories, the parent @@ -677,7 +678,7 @@ Analoguous to .It ATTR_CMN_FILEID A .Vt u_int64_t -that uniquely identifies the file system object within it's mounted volume. +that uniquely identifies the file system object within its mounted volume. Equivalent to .Fa st_ino field of the @@ -1168,6 +1169,18 @@ An containing the number of bytes that are \fBnot\fP trapped inside a clone or snapshot, and which would be freed immediately if the file were deleted. . +.It ATTR_CMNEXT_LINKID +A +.Vt u_int64_t +that uniquely identifies the file system object within a mounted volume for the +duration of its mount. +.Pp +On HFS+ and APFS volumes, the ATTR_CMNEXT_LINKID of a file system +object is distinct from the ATTR_CMNEXT_LINKID of any hard link to that file +system object. Although the ATTR_CMNEXT_LINKID of a file system object may appear +similar (in whole or in part) to its ATTR_CMN_FILEID (see description of +ATTR_CMN_FILEID above), \fBno relation between the two attributes should ever be implied.\fP +. .El . .Sh VOLUME CAPABILITIES @@ -1528,6 +1541,12 @@ See .Xr clonefileat 2 for more details. . +.It VOL_CAP_INT_SNAPSHOT +If this bit is set, the file system supports snapshots. +See +.Xr fs_snapshot_create 2 +for more details. +. .It VOL_CAP_INT_NAMEDSTREAMS If this bit is set, the volume format implementation supports native named streams. @@ -2050,6 +2069,56 @@ main(int argc, char **argv) return 0; } .Ed +.Pp + The getLinkIDInfo() function determines if ATTR_CMNEXT_LINKID and ATTR_CMN_OBJID + are valid to use on the file system specified by path. +. +.Bd -literal +int getLinkIDInfo(const char *path, bool *cmnExtLinkIDValid, bool *cmnObjIDValid) +{ + int result; + struct statfs statfsBuf; + struct attrlist attrList; + struct volAttrsBuf { + u_int32_t length; + vol_capabilities_attr_t capabilities; + vol_attributes_attr_t attributes; + } __attribute__((aligned(4), packed)); + struct volAttrsBuf volAttrs; +.Pp + memset(&attrList, 0, sizeof(attrList)); + attrList.bitmapcount = ATTR_BIT_MAP_COUNT; + attrList.volattr = ATTR_VOL_INFO | ATTR_VOL_CAPABILITIES | ATTR_VOL_ATTRIBUTES; + // get the file system's mount point path for the input path + result = statfs(path, &statfsBuf); + if ( result == 0 ) { + // get the supported capabilities and attributes + result = getattrlist(statfsBuf.f_mntonname, &attrList, &volAttrs, sizeof(volAttrs), FSOPT_ATTR_CMN_EXTENDED); + if ( result == 0 ) { + if ( volAttrs.attributes.validattr.forkattr & ATTR_CMNEXT_LINKID ) { + // ATTR_CMNEXT_LINKID is available; do not use ATTR_CMN_OBJID + *cmnExtLinkIDValid = true; + *cmnObjIDValid = false; + } + else { + // ATTR_CMNEXT_LINKID is not available + cmnExtLinkIDValid = false; + // ATTR_CMN_OBJID can only be used if the file system does not use 64-bit object IDs + if ( (volAttrs.capabilities.capabilities[VOL_CAPABILITIES_FORMAT] & VOL_CAP_FMT_64BIT_OBJECT_IDS) && (volAttrs.capabilities.valid[VOL_CAPABILITIES_FORMAT] & VOL_CAP_FMT_64BIT_OBJECT_IDS) ) { + *cmnObjIDValid = false; + } + else { + *cmnObjIDValid = true; + } + } + } + } + if ( result != 0 ) { + *cmnExtLinkIDValid = *cmnObjIDValid = false; + } + return result; +} +.Ed .Pp . .Sh SEE ALSO diff --git a/bsd/man/man2/kqueue.2 b/bsd/man/man2/kqueue.2 index 6f1ce36ef..d1a535542 100644 --- a/bsd/man/man2/kqueue.2 +++ b/bsd/man/man2/kqueue.2 @@ -595,41 +595,65 @@ area provided to .Fn kevent_qos if there is enough space remaining there. .It EVFILT_TIMER -Establishes an interval timer with the data -timer identified by -.Va ident . -When adding a timer, +Establishes an interval timer identified by +.Va ident +where .Va data -specifies the timeout period and +specifies the timeout period (in milliseconds). +.Pp .Va fflags -can be set to one of the following: -.Bl -tag -width NOTE_ABSOLUTE +can include one of the following flags to specify a different unit: +.Bl -tag -width NOTE_NSECONDS .It NOTE_SECONDS -data is in seconds +.Va data +is in seconds .It NOTE_USECONDS -data is in microseconds +.Va data +is in microseconds .It NOTE_NSECONDS -data is in nanoseconds -.It NOTE_ABSOLUTE -data is an absolute timeout +.Va data +is in nanoseconds +.It NOTE_MACHTIME +.Va data +is in Mach absolute time units +.El +.Pp +.Va fflags +can also include +.Dv NOTE_ABSOLUTE, +which establishes an +.Dv EV_ONESHOT +timer with an absolute deadline instead of an interval. +The absolute deadline is expressed in terms of +.Xr gettimeofday 2 . +With +.Dv NOTE_MACHTIME, +the deadline is expressed in terms of +.Fn mach_absolute_time . +.Pp +The timer can be coalesced with other timers to save power. The following flags can be set in +.Va fflags +to modify this behavior: +.Bl -tag -width NOTE_BACKGROUND .It NOTE_CRITICAL -system makes a best effort to fire this timer as scheduled. +override default power-saving techniques to more strictly respect the leeway value .It NOTE_BACKGROUND -system has extra leeway to coalesce this timer. +apply more power-saving techniques to coalesce this timer with other timers .It NOTE_LEEWAY -ext[1] holds user-supplied slop in deadline for timer coalescing. +.Va ext[1] +holds user-supplied slop in deadline for timer coalescing. .El .Pp -If fflags is not set, the default is milliseconds. The timer will be periodic unless EV_ONESHOT is specified. +The timer will be periodic unless +.Dv EV_ONESHOT +is specified. On return, .Va data -contains the number of times the timeout has expired since the last call to -.Fn kevent , -.Fn kevent64 -or -.Fn kevent_qos . - -This filter automatically sets the EV_CLEAR flag internally. +contains the number of times the timeout has expired since the last arming or last delivery of the timer event. +.Pp +This filter automatically sets the +.Dv EV_CLEAR +flag. .El .Pp ---- diff --git a/bsd/man/man2/mount.2 b/bsd/man/man2/mount.2 index e3245d9f3..586707fa6 100644 --- a/bsd/man/man2/mount.2 +++ b/bsd/man/man2/mount.2 @@ -39,6 +39,7 @@ .Os BSD 4 .Sh NAME .Nm mount , +.Nm fmount, .Nm unmount .Nd mount or dismount a filesystem .Sh SYNOPSIS @@ -47,6 +48,8 @@ .Ft int .Fn mount "const char *type" "const char *dir" "int flags" "void *data" .Ft int +.Fn fmount "const char *type" "int fd" "int flags" "void *data" +.Ft int .Fn unmount "const char *dir" "int flags" .Sh DESCRIPTION The @@ -122,6 +125,15 @@ The format for these argument structures is described in the manual page for each filesystem. .Pp The +.Fn fmount +function call is equivalent to the +.Fn mount +function call, except in the use of the second argument. +It takes an open file descriptor representing mount point +instead of the string literal containing full path to the mount +point in the filesystem hierarchy. +.Pp +The .Fn unmount function call disassociates the filesystem from the specified mount point @@ -139,7 +151,9 @@ even if the filesystem is later remounted. .Sh RETURN VALUES The .Fn mount -returns the value 0 if the mount was successful, otherwise -1 is returned +and +.Fn fmount +return the value 0 if the mount was successful, otherwise -1 is returned and the variable .Va errno is set to indicate the error. @@ -151,6 +165,8 @@ and the variable is set to indicate the error. .Sh ERRORS .Fn mount +and +.Fn fmount will fail when one of the following occurs: .Bl -tag -width [ENAMETOOLONG] .It Bq Er EPERM @@ -217,7 +233,8 @@ points outside the process's allocated address space. .El .Sh SEE ALSO .Xr mount 8 , -.Xr unmount 8 +.Xr unmount 8 , +.Xr open 2 .Sh BUGS Some of the error codes need translation to more obvious messages. .Sh HISTORY @@ -226,3 +243,5 @@ and .Fn unmount function calls appeared in .At v6 . +.Fn fmount +function call first appeared in macOS version 10.13. diff --git a/bsd/man/man2/peeloff.2 b/bsd/man/man2/peeloff.2 deleted file mode 100644 index 3ba0acb18..000000000 --- a/bsd/man/man2/peeloff.2 +++ /dev/null @@ -1,99 +0,0 @@ -.\" -.\" Copyright (c) 2012 Apple Inc. All rights reserved. -.\" -.\" @APPLE_OSREFERENCE_LICENSE_HEADER_START@ -.\" -.\" This file contains Original Code and/or Modifications of Original Code -.\" as defined in and that are subject to the Apple Public Source License -.\" Version 2.0 (the 'License'). You may not use this file except in -.\" compliance with the License. The rights granted to you under the License -.\" may not be used to create, or enable the creation or redistribution of, -.\" unlawful or unlicensed copies of an Apple operating system, or to -.\" circumvent, violate, or enable the circumvention or violation of, any -.\" terms of an Apple operating system software license agreement. -.\" -.\" Please obtain a copy of the License at -.\" http://www.opensource.apple.com/apsl/ and read it before using this file. -.\" -.\" The Original Code and all software distributed under the License are -.\" distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER -.\" EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, -.\" INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, -.\" FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. -.\" Please see the License for the specific language governing rights and -.\" limitations under the License. -.\" -.\" @APPLE_OSREFERENCE_LICENSE_HEADER_END@ -.\" -.Dd November 14, 2012 -.Dt PEELOFF 2 -.Os Darwin -.Sh NAME -.Nm peeloff -.Nd extracts an association from a socket -.Sh SYNOPSIS -.Fd #include <sys/socket.h> -.Ft int -.Fo peeloff -.Fa "int socket" -.Fa "sae_associd_t associd" -.Fc -.Sh DESCRIPTION -The parameter -.Fa socket -is a socket. The communication domain of the socket determines the -availability and behavior of -.Fn peeloff . -In general, -.Fn peeloff -attempts to extract the association specified by -.Fa associd -into its own separate socket. -.Pp -The parameter -.Fa associd -specifies the association identifier. It may be set to -.Dv SAE_ASSOCID_ANY -when there is only one association present; or one of the identifiers -returned from -.Xr getassocids 3 . -.Sh RETURN VALUES -The -.Fn peeloff -function returns -1 on error and the global variable -.Va errno -is set to indicate the error. If it succeeds, it returns a non-negative -integer that is a descriptor for the extracted association. -.Sh ERRORS -The -.Fn peeloff -system call succeeds unless: -.Bl -tag -width Er -.\" =========== -.It Bq Er EBADF -.Fa Socket -is not a valid descriptor. -.\" =========== -.It Bq Er EINVAL -The -.Fa associd -argument is invalid; cannot be extracted; or the underlying protocol -is no longer attached to -.Fa socket . -.\" =========== -.It Bq Er ENOTSOCK -.Fa Socket -is a file, not a socket. -.El -.Sh SEE ALSO -.Xr connectx 2 , -.Xr disconnectx 2 , -.Xr socket 2 , -.Xr getassocids 3 , -.Xr getconnids 3 , -.Xr getconninfo 3 , -.Xr compat 5 -.Sh HISTORY -The -.Fn peeloff -function call appeared in Darwin 13.0.0 diff --git a/bsd/man/man2/posix_spawn.2 b/bsd/man/man2/posix_spawn.2 index aa339f665..83dea9536 100644 --- a/bsd/man/man2/posix_spawn.2 +++ b/bsd/man/man2/posix_spawn.2 @@ -324,6 +324,9 @@ A component of the path prefix is not a directory. .It Bq Er ETXTBSY The new process file is a pure procedure (shared text) file that is currently open for writing or reading by some process. +.\" ========== +.It Bq Er EBADARCH +The new process file has no architectures appropriate for the current system. .El .Pp Additionally, they may fail for any of the reasons listed in diff --git a/bsd/man/man2/profil.2 b/bsd/man/man2/profil.2 deleted file mode 100644 index 40ee9a463..000000000 --- a/bsd/man/man2/profil.2 +++ /dev/null @@ -1,144 +0,0 @@ -.\" $NetBSD: profil.2,v 1.3 1995/11/22 23:07:23 cgd Exp $ -.\" -.\" Copyright (c) 1993 -.\" The Regents of the University of California. All rights reserved. -.\" -.\" This code is derived from software contributed to Berkeley by -.\" Donn Seeley of BSDI. -.\" -.\" Redistribution and use in source and binary forms, with or without -.\" modification, are permitted provided that the following conditions -.\" are met: -.\" 1. Redistributions of source code must retain the above copyright -.\" notice, this list of conditions and the following disclaimer. -.\" 2. Redistributions in binary form must reproduce the above copyright -.\" notice, this list of conditions and the following disclaimer in the -.\" documentation and/or other materials provided with the distribution. -.\" 3. All advertising materials mentioning features or use of this software -.\" must display the following acknowledgement: -.\" This product includes software developed by the University of -.\" California, Berkeley and its contributors. -.\" 4. Neither the name of the University nor the names of its contributors -.\" may be used to endorse or promote products derived from this software -.\" without specific prior written permission. -.\" -.\" THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND -.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -.\" ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE -.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -.\" SUCH DAMAGE. -.\" -.\" @(#)profil.2 8.1 (Berkeley) 6/4/93 -.\" -.Dd September 26, 2008 -.Dt PROFIL 2 -.Os -.Sh NAME -.Nm profil -.Nd control process profiling -.Sh LIBRARY -.Lb libc -.Sh SYNOPSIS -.In unistd.h -.Ft int -.Fn profil "char *samples" "size_t size" "u_long offset" "u_int scale" -.Sh DESCRIPTION -.Pp -.Fd -- This function is now deprecated. It will always return EINVAL. -- -.Pp -The intended replacements are the user-level developer tools, like CHUD and dtrace. -.Pp ------- -.Pp -The -.Fn profil -function enables or disables -program counter profiling of the current process. -If profiling is enabled, -then at every clock tick, -the kernel updates an appropriate count in the -.Fa samples -buffer. -.Pp -The buffer -.Fa samples -contains -.Fa size -bytes and is divided into -a series of 16-bit bins. -Each bin counts the number of times the program counter -was in a particular address range in the process -when a clock tick occurred while profiling was enabled. -For a given program counter address, -the number of the corresponding bin is given -by the relation: -.Bd -literal -offset indent -[(pc - offset) / 2] * scale / 65536 -.Ed -.Pp -The -.Fa offset -parameter is the lowest address at which -the kernel takes program counter samples. -The -.Fa scale -parameter ranges from 1 to 65536 and -can be used to change the span of the bins. -A scale of 65536 maps each bin to 2 bytes of address range; -a scale of 32768 gives 4 bytes, 16384 gives 8 bytes and so on. -Intermediate values provide approximate intermediate ranges. -A -.Fa scale -value of 0 disables profiling. -.Sh RETURN VALUES -If the -.Fa scale -value is nonzero and the buffer -.Fa samples -contains an illegal address, -.Fn profil -returns \-1, -profiling is terminated and -.Va errno -is set appropriately. -Otherwise -.Fn profil -returns 0. -.Sh FILES -.Bl -tag -width /usr/lib/gcrt0.o -compact -.It Pa /usr/lib/gcrt0.o -profiling C run-time startup file -.It Pa gmon.out -conventional name for profiling output file -.El -.Sh ERRORS -The following error may be reported: -.Bl -tag -width Er -.It Bq Er EFAULT -The buffer -.Fa samples -contains an invalid address. -.El -.Sh SEE ALSO -.Xr gprof 1 -.Sh HISTORY -The -.Fn profil -function appeared in -.At v7 . -.Sh BUGS -This routine should be named -.Fn profile . -.Pp -The -.Fa samples -argument should really be a vector of type -.Fa "unsigned short" . -.Pp -The format of the gmon.out file is undocumented. diff --git a/bsd/man/man2/readlink.2 b/bsd/man/man2/readlink.2 index 375eca319..2940e9ced 100644 --- a/bsd/man/man2/readlink.2 +++ b/bsd/man/man2/readlink.2 @@ -85,6 +85,7 @@ in the .Fa fd parameter, the current working directory is used and the behavior is identical to a call to +.Fn readlink . .Sh RETURN VALUES The call returns the count of characters placed in the buffer if it succeeds, or a -1 if an error occurs, placing the error diff --git a/bsd/man/man2/setattrlist.2 b/bsd/man/man2/setattrlist.2 index 68898eb3c..4dcc7340d 100644 --- a/bsd/man/man2/setattrlist.2 +++ b/bsd/man/man2/setattrlist.2 @@ -21,15 +21,18 @@ .Os Darwin .Sh NAME .Nm setattrlist , -.Nm fsetattrlist +.Nm fsetattrlist , +.Nm setattrlistat .Nd set file system attributes .Sh SYNOPSIS .Fd #include <sys/attr.h> .Fd #include <unistd.h> .Ft int -.Fn setattrlist "const char* path" "struct attrlist * attrList" "void * attrBuf" "size_t attrBufSize" "unsigned long options" +.Fn setattrlist "const char * path" "struct attrlist * attrList" "void * attrBuf" "size_t attrBufSize" "unsigned long options" .Ft int .Fn fsetattrlist "int fd" "struct attrlist * attrList" "void * attrBuf" "size_t attrBufSize" "unsigned long options" +.Ft int +.Fn setattrlistat "int dir_fd" "const char * path" "struct attrlist * attrList" "void * attrBuf" "size_t attrBufSize" "uint32_t options" . .Sh DESCRIPTION The @@ -59,6 +62,27 @@ The .Fa options parameter lets you control specific aspects of the function's behaviour. .Pp +The +.Fn setattrlistat +system call is equivalent to +.Fn setattrlist +except in the case where +.Fa path +specifies a relative path. +In this case the attributes are set for the file system object named by +path relative to the directory associated with the file descriptor +.Fa fd +instead of the current working directory. +If +.Fn setattrlistat +is passed the special value +.Dv AT_FDCWD +in the +.Fa fd +parameter, the current working directory is used and the behavior is +identical to a call to +.Fn setattrlist . +.Pp . The functions are only supported by certain volume format implementations. @@ -122,6 +146,8 @@ ATTR_CMN_FLAGS ATTR_CMN_EXTENDED_SECURITY .It ATTR_CMN_GRPUUID +.It +ATTR_CMN_ADDEDTIME .Pp .It ATTR_VOL_NAME @@ -151,6 +177,8 @@ ATTR_CMN_CRTIME ATTR_CMN_MODTIME .It ATTR_CMN_ACCTIME +.It +ATTR_CMN_ADDEDTIME .Pp ATTR_CMN_CHGTIME .Fa cannot be set programmatically. Any attempt to set change time is ignored. @@ -336,6 +364,31 @@ is too small to hold all the attributes that you are trying to set. An I/O error occurred while reading from or writing to the file system. .El .Pp +.Pp +In addition to the errors returned by the +.Fn setattrlist , +the +.Fn setattrlistat +function may fail if: +.Bl -tag -width Er +.It Bq Er EBADF +The +.Fa path +argument does not specify an absolute path and the +.Fa fd +argument is neither +.Dv AT_FDCWD +nor a valid file descriptor open for searching. +.It Bq Er ENOTDIR +The +.Fa path +argument is not an absolute path and +.Fa fd +is neither +.Dv AT_FDCWD +nor a file descriptor associated with a directory. +.El +.Pp . .Sh CAVEATS . @@ -443,5 +496,6 @@ static int FInfoDemo( .Sh HISTORY A .Fn setattrlist -function call appeared in Darwin 1.3.1 (Mac OS X version 10.0). +function call appeared in Darwin 1.3.1 (Mac OS X version 10.0). The setatrlistat function call first +appeared in macOS version 10.13. . diff --git a/bsd/man/man2/setattrlistat.2 b/bsd/man/man2/setattrlistat.2 new file mode 100644 index 000000000..f823a2cac --- /dev/null +++ b/bsd/man/man2/setattrlistat.2 @@ -0,0 +1 @@ +.so man2/setattrlist.2 diff --git a/bsd/man/man2/utimensat.2 b/bsd/man/man2/utimensat.2 new file mode 100644 index 000000000..0b5e8f8e8 --- /dev/null +++ b/bsd/man/man2/utimensat.2 @@ -0,0 +1,256 @@ +.\" $NetBSD: utimes.2,v 1.13 1999/03/22 19:45:11 garbled Exp $ +.\" +.\" Copyright (c) 1990, 1993 +.\" The Regents of the University of California. All rights reserved. +.\" Copyright (c) 2012, Jilles Tjoelker +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" 3. Neither the name of the University nor the names of its contributors +.\" may be used to endorse or promote products derived from this software +.\" without specific prior written permission. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND +.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE +.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. +.\" +.\" @(#)utimes.2 8.1 (Berkeley) 6/4/93 +.\" $FreeBSD$ +.\" +.Dd January 17, 2016 +.Dt UTIMENSAT 2 +.Os +.Sh NAME +.Nm futimens , +.Nm utimensat +.Nd set file access and modification times +.Sh SYNOPSIS +.In sys/stat.h +.Ft int +.Fn futimens "int fd" "const struct timespec times[2]" +.Ft int +.Fo utimensat +.Fa "int fd" +.Fa "const char *path" +.Fa "const struct timespec times[2]" +.Fa "int flag" +.Fc +.Sh DESCRIPTION +The access and modification times of the file named by +.Fa path +or referenced by +.Fa fd +are changed as specified by the argument +.Fa times . +The inode-change-time of the file is set to the current time. +.Pp +If +.Fa path +specifies a relative path, +it is relative to the current working directory if +.Fa fd +is +.Dv AT_FDCWD +and otherwise relative to the directory associated with the file descriptor +.Fa fd . +.Pp +The +.Va tv_nsec +field of a +.Vt timespec +structure +can be set to the special value +.Dv UTIME_NOW +to set the current time, or to +.Dv UTIME_OMIT +to leave the time unchanged. +In either case, the +.Va tv_sec +field is ignored. +.Pp +If +.Fa times +is +.No non- Ns Dv NULL , +it is assumed to point to an array of two timespec structures. +The access time is set to the value of the first element, and the +modification time is set to the value of the second element. +If +.Fa times +is +.Dv NULL , +this is equivalent to passing +a pointer to an array of two timespec structures +with both +.Va tv_nsec +fields set to +.Dv UTIME_NOW . +.Pp +If both +.Va tv_nsec +fields are +.Dv UTIME_OMIT , +the timestamps remain unchanged and +no permissions are needed for the file itself, +although search permissions may be required for the path prefix. +The call may or may not succeed if the named file does not exist. +.Pp +If both +.Va tv_nsec +fields are +.Dv UTIME_NOW , +the caller must be the owner of the file, have permission to +write the file, or be the super-user. +.Pp +For all other values of the timestamps, +the caller must be the owner of the file or be the super-user. +.Pp +The values for the +.Fa flag +argument of the +.Fn utimensat +system call +are constructed by a bitwise-inclusive OR of flags from the following list, +defined in +.In fcntl.h : +.Bl -tag -width indent +.It Dv AT_SYMLINK_NOFOLLOW +If +.Fa path +names a symbolic link, the symbolic link's times are changed. +By default, +.Fn utimensat +changes the times of the file referenced by the symbolic link. +.El +.Sh RETURN VALUES +.Rv -std +.Sh ERRORS +These system calls will fail if: +.Bl -tag -width Er +.It Bq Er EACCES +The +.Fa times +argument is +.Dv NULL , +or both +.Va tv_nsec +values are +.Dv UTIME_NOW , +and the effective user ID of the process does not +match the owner of the file, and is not the super-user, and write +access is denied. +.It Bq Er EINVAL +The +.Va tv_nsec +component of at least one of the values specified by the +.Fa times +argument has a value less than 0 or greater than 999999999 and is not equal to +.Dv UTIME_NOW +or +.Dv UTIME_OMIT . +.It Bq Er EIO +An I/O error occurred while reading or writing the affected inode. +.It Bq Er EPERM +The +.Fa times +argument is not +.Dv NULL +nor are both +.Va tv_nsec +values +.Dv UTIME_NOW , +nor are both +.Va tv_nsec +values +.Dv UTIME_OMIT +and the calling process's effective user ID +does not match the owner of the file and is not the super-user. +.It Bq Er EPERM +The named file has its immutable or append-only flag set, see the +.Xr chflags 2 +manual page for more information. +.It Bq Er EROFS +The file system containing the file is mounted read-only. +.El +.Pp +The +.Fn futimens +system call +will fail if: +.Bl -tag -width Er +.It Bq Er EBADF +The +.Fa fd +argument +does not refer to a valid descriptor. +.El +.Pp +The +.Fn utimensat +system call +will fail if: +.Bl -tag -width Er +.It Bq Er EACCES +Search permission is denied for a component of the path prefix. +.It Bq Er EBADF +The +.Fa path +argument does not specify an absolute path and the +.Fa fd +argument is neither +.Dv AT_FDCWD +nor a valid file descriptor. +.It Bq Er EFAULT +The +.Fa path +argument +points outside the process's allocated address space. +.It Bq Er ELOOP +Too many symbolic links were encountered in translating the pathname. +.It Bq Er ENAMETOOLONG +A component of a pathname exceeded +.Dv NAME_MAX +characters, or an entire path name exceeded +.Dv PATH_MAX +characters. +.It Bq Er ENOENT +The named file does not exist. +.It Bq Er ENOTDIR +A component of the path prefix is not a directory. +.It Bq Er ENOTDIR +The +.Fa path +argument is not an absolute path and +.Fa fd +is neither +.Dv AT_FDCWD +nor a file descriptor associated with a directory. +.El +.Sh SEE ALSO +.Xr chflags 2 , +.Xr stat 2 , +.Xr symlink 2 , +.Xr utimes 2 , +.Xr utime 3 , +.Xr symlink 7 +.Sh STANDARDS +The +.Fn futimens +and +.Fn utimensat +system calls are expected to conform to +.St -p1003.1-2008 . diff --git a/bsd/man/man9/monotonic.9 b/bsd/man/man9/monotonic.9 new file mode 100644 index 000000000..38dbe2aa6 --- /dev/null +++ b/bsd/man/man9/monotonic.9 @@ -0,0 +1,81 @@ +.\" Copyright (c) 2017, Apple Inc. All rights reserved. +.\" +.Dd March 1, 2017 +.Dt MONOTONIC 9 +.Os Darwin +.\" +.Sh NAME +.Nm monotonic +.Nd performance counter access system +.\" +.Sh DESCRIPTION +.Nm +allows kernel and user space clients to configure and read hardware performance +counters. The hardware counters can be virtualized to count per-thread and +per-process. +.Nm +is split into three major layers: +.Bl -dash +.It +The machine-dependent implementations manipulate hardware registers to configure +and access the counters. This layer provides a machine-independent interface +that can be used by the next layer. +.It +A set of hooks and kernel routines manage the counters and provide higher-level +abstractions, like 64-bit counters and counting only events that occurred on a +thread or in a process. +.It +A user space interface that is presented as device nodes under +.Pa /dev/monotonic . +See +.Xr monotonic 4 . +Mach thread and task ports are used for the per-thread and per-process counts, +with special inspection routines. Some counter values are also reflected into +.Fn getrusage , +for use after a process has exited. See +.Xr getrusage 2 . +.El +.Pp +.\".Sh DIAGNOSTICS +.\" +.Sh SEE ALSO +.Xr count 1 , +.Xr mperf 1 , +.Xr easyperf 1 , +.Xr getrusage 2 , +.Xr monotonic 4 , +.Xr perf 1 +.\" +.Sh HISTORY +.Nm +replaces the kernel performance counter system, kpc. For the time being, +.Nm +backs portions of the existing kpc +.Fn sysctl +interface. Prior to kpc, the AppleProfileFamily kernel extensions provided +performance counter interfaces. The kernel extensions themselves expanded upon +functionality provided for PowerPC by CHUD. +.\".Sh CAVEATS +.\" +.Sh SECURITY CONSIDERATIONS +.Pp +.Pp +.Bl -dash +.It +Hardware performance counters are an ideal tool for side-channel attacks. By +observing how the counters are affected by an otherwise opaque process, an +attacker can obtain sensitive data or key material. +.Pp +For this reason, the hardware performance counters cannot be queried directly +from user space. Instead, all processes, including those owned by root, can +only query the thread and process counters if they have the corresponding Mach +thread or task port. +.It +When used in sampling mode, hardware performance counters can induce interrupt +storms that translate to denial-of-service attacks on a system. Even a careless +user can stumble over this issue, since reasonable periods for some events are +far too aggressive for others. +.Pp +If a hardware performance counter takes too many interrupts in a short amount of +time, it will be disabled. +.El diff --git a/bsd/miscfs/devfs/devfs.h b/bsd/miscfs/devfs/devfs.h index 6615b97b5..c2cc577c5 100644 --- a/bsd/miscfs/devfs/devfs.h +++ b/bsd/miscfs/devfs/devfs.h @@ -57,6 +57,7 @@ #define _MISCFS_DEVFS_DEVFS_H_ #include <sys/appleapiopts.h> +#include <sys/cdefs.h> #define DEVFS_CHAR 0 #define DEVFS_BLOCK 1 diff --git a/bsd/miscfs/devfs/devfs_fdesc_support.c b/bsd/miscfs/devfs/devfs_fdesc_support.c index bf4e3bb06..5d9355efc 100644 --- a/bsd/miscfs/devfs/devfs_fdesc_support.c +++ b/bsd/miscfs/devfs/devfs_fdesc_support.c @@ -568,9 +568,15 @@ devfs_devfd_readdir(struct vnop_readdir_args *ap) if (ap->a_flags & (VNODE_READDIR_EXTENDED | VNODE_READDIR_REQSEEKOFF)) return (EINVAL); + /* + * There needs to be space for at least one entry. + */ + if (uio_resid(uio) < UIO_MX) + return (EINVAL); + i = uio->uio_offset / UIO_MX; error = 0; - while (uio_resid(uio) > 0) { + while (uio_resid(uio) >= UIO_MX) { if (i >= p->p_fd->fd_nfiles) break; diff --git a/bsd/miscfs/devfs/devfs_tree.c b/bsd/miscfs/devfs/devfs_tree.c index 21912549a..adbc2e78f 100644 --- a/bsd/miscfs/devfs/devfs_tree.c +++ b/bsd/miscfs/devfs/devfs_tree.c @@ -328,7 +328,7 @@ dev_finddir(const char * path, while (*scan != '/' && *scan) scan++; - strlcpy(component, start, scan - start); + strlcpy(component, start, (scan - start) + 1); if (*scan == '/') scan++; diff --git a/bsd/miscfs/devfs/devfs_vnops.c b/bsd/miscfs/devfs/devfs_vnops.c index fbd3246f7..41029dd7c 100644 --- a/bsd/miscfs/devfs/devfs_vnops.c +++ b/bsd/miscfs/devfs/devfs_vnops.c @@ -115,6 +115,7 @@ static int devfs_update(struct vnode *vp, struct timeval *access, void devfs_rele_node(devnode_t *); static void devfs_consider_time_update(devnode_t *dnp, uint32_t just_changed_flags); static boolean_t devfs_update_needed(long now_s, long last_s); +static boolean_t devfs_is_name_protected(struct vnode *dvp, const char *name); void dn_times_locked(devnode_t * dnp, struct timeval *t1, struct timeval *t2, struct timeval *t3, uint32_t just_changed_flags); void dn_times_now(devnode_t *dnp, uint32_t just_changed_flags); void dn_mark_for_delayed_times_update(devnode_t *dnp, uint32_t just_changed_flags); @@ -184,6 +185,33 @@ dn_times_now(devnode_t * dnp, uint32_t just_changed_flags) DEVFS_ATTR_UNLOCK(); } +/* + * Critical devfs devices cannot be renamed or removed. + * However, links to them may be moved/unlinked. So we block + * remove/rename on a per-name basis, rather than per-node. + */ +static boolean_t +devfs_is_name_protected(struct vnode *dvp, const char *name) +{ + /* + * Only names in root are protected. E.g. /dev/null is protected, + * but /dev/foo/null isn't. + */ + if (!vnode_isvroot(dvp)) + return FALSE; + + if ((strcmp("console", name) == 0) || + (strcmp("tty", name) == 0) || + (strcmp("null", name) == 0) || + (strcmp("zero", name) == 0) || + (strcmp("klog", name) == 0)) { + + return TRUE; + } + + return FALSE; +} + /* * Convert a component of a pathname into a pointer to a locked node. @@ -795,6 +823,7 @@ devfs_vnop_remove(struct vnop_remove_args *ap) * are the end of the path. Get pointers to all our * devfs structures. */ + DEVFS_LOCK(); tp = VTODN(vp); @@ -808,6 +837,14 @@ devfs_vnop_remove(struct vnop_remove_args *ap) goto abort; } + /* + * Don't allow removing critical devfs devices + */ + if (devfs_is_name_protected(dvp, cnp->cn_nameptr)) { + error = EINVAL; + goto abort; +} + /* * Make sure that we don't try do something stupid */ @@ -1006,6 +1043,15 @@ devfs_rename(struct vnop_rename_args *ap) doingdirectory++; } + /* + * Don't allow renaming critical devfs devices + */ + if (devfs_is_name_protected(fdvp, fcnp->cn_nameptr) || + devfs_is_name_protected(tdvp, tcnp->cn_nameptr)) { + error = EINVAL; + goto out; + } + /* * If ".." must be changed (ie the directory gets a new * parent) then the source directory must not be in the @@ -1570,7 +1616,7 @@ static struct vnodeopv_entry_desc devfs_vnodeop_entries[] = { #if CONFIG_MACF { &vnop_setlabel_desc, (VOPFUNC)devfs_setlabel }, /* setlabel */ #endif - { (struct vnodeop_desc*)NULL, (int(*)())NULL } + { (struct vnodeop_desc*)NULL, (int(*)(void *))NULL } }; struct vnodeopv_desc devfs_vnodeop_opv_desc = { &devfs_vnodeop_p, devfs_vnodeop_entries }; @@ -1616,7 +1662,7 @@ static struct vnodeopv_entry_desc devfs_spec_vnodeop_entries[] = { #if CONFIG_MACF { &vnop_setlabel_desc, (VOPFUNC)devfs_setlabel }, /* setlabel */ #endif - { (struct vnodeop_desc*)NULL, (int(*)())NULL } + { (struct vnodeop_desc*)NULL, (int(*)(void *))NULL } }; struct vnodeopv_desc devfs_spec_vnodeop_opv_desc = { &devfs_spec_vnodeop_p, devfs_spec_vnodeop_entries }; @@ -1640,7 +1686,7 @@ static struct vnodeopv_entry_desc devfs_devfd_vnodeop_entries[] = { #if CONFIG_MACF { &vnop_setlabel_desc, (VOPFUNC)devfs_setlabel }, /* setlabel */ #endif - { (struct vnodeop_desc*)NULL, (int(*)())NULL } + { (struct vnodeop_desc*)NULL, (int(*)(void *))NULL } }; struct vnodeopv_desc devfs_devfd_vnodeop_opv_desc = { &devfs_devfd_vnodeop_p, devfs_devfd_vnodeop_entries}; diff --git a/bsd/miscfs/devfs/devfsdefs.h b/bsd/miscfs/devfs/devfsdefs.h index 79e99f512..6fdac8849 100644 --- a/bsd/miscfs/devfs/devfsdefs.h +++ b/bsd/miscfs/devfs/devfsdefs.h @@ -72,8 +72,6 @@ #include <sys/appleapiopts.h> -#include <security/mac.h> - __BEGIN_DECLS #ifdef __APPLE_API_PRIVATE #define DEVMAXNAMESIZE 32 /* XXX */ diff --git a/bsd/miscfs/fifofs/fifo_vnops.c b/bsd/miscfs/fifofs/fifo_vnops.c index 0b7bc1d3d..6146bcbfa 100644 --- a/bsd/miscfs/fifofs/fifo_vnops.c +++ b/bsd/miscfs/fifofs/fifo_vnops.c @@ -117,7 +117,7 @@ struct vnodeopv_entry_desc fifo_vnodeop_entries[] = { { &vnop_blktooff_desc, (VOPFUNC)err_blktooff }, /* blktooff */ { &vnop_offtoblk_desc, (VOPFUNC)err_offtoblk }, /* offtoblk */ { &vnop_blockmap_desc, (VOPFUNC)err_blockmap }, /* blockmap */ - { (struct vnodeop_desc*)NULL, (int(*)())NULL } + { (struct vnodeop_desc*)NULL, (int(*)(void *))NULL } }; struct vnodeopv_desc fifo_vnodeop_opv_desc = { &fifo_vnodeop_p, fifo_vnodeop_entries }; diff --git a/bsd/miscfs/nullfs/nullfs.h b/bsd/miscfs/nullfs/nullfs.h index 5d55e2c8b..e29b9e696 100644 --- a/bsd/miscfs/nullfs/nullfs.h +++ b/bsd/miscfs/nullfs/nullfs.h @@ -163,4 +163,4 @@ __END_DECLS #endif /* KERNEL */ -#endif \ No newline at end of file +#endif diff --git a/bsd/miscfs/routefs/routefs_ops.c b/bsd/miscfs/routefs/routefs_ops.c index 194325406..db6db101d 100644 --- a/bsd/miscfs/routefs/routefs_ops.c +++ b/bsd/miscfs/routefs/routefs_ops.c @@ -553,7 +553,7 @@ static struct vnodeopv_entry_desc routefs_vnodeop_entries[] = { #if CONFIG_MACF { &vnop_setlabel_desc, (VOPFUNC)routefserr_setlabel }, /* setlabel */ #endif - { (struct vnodeop_desc*)NULL, (int(*)())NULL } + { (struct vnodeop_desc*)NULL, (int(*)(void *))NULL } }; struct vnodeopv_desc routefs_vnodeop_opv_desc = { &routefs_vnodeop_p, routefs_vnodeop_entries }; diff --git a/bsd/miscfs/specfs/spec_vnops.c b/bsd/miscfs/specfs/spec_vnops.c index f698e5d68..625e9736f 100644 --- a/bsd/miscfs/specfs/spec_vnops.c +++ b/bsd/miscfs/specfs/spec_vnops.c @@ -84,6 +84,7 @@ #include <machine/machine_routines.h> #include <miscfs/specfs/specdev.h> #include <vfs/vfs_support.h> +#include <vfs/vfs_disk_conditioner.h> #include <kern/assert.h> #include <kern/task.h> @@ -91,18 +92,18 @@ #include <kern/thread.h> #include <kern/policy_internal.h> #include <kern/timer_call.h> +#include <kern/waitq.h> #include <pexpert/pexpert.h> #include <sys/kdebug.h> +#include <libkern/section_keywords.h> /* XXX following three prototypes should be in a header file somewhere */ extern dev_t chrtoblk(dev_t dev); extern boolean_t iskmemdev(dev_t dev); extern int bpfkqfilter(dev_t dev, struct knote *kn); -extern int ptsd_kqfilter(dev_t dev, struct knote *kn); - -extern int ignore_is_ssd; +extern int ptsd_kqfilter(dev_t, struct knote *); struct vnode *speclisth[SPECHSZ]; @@ -155,7 +156,7 @@ struct vnodeopv_entry_desc spec_vnodeop_entries[] = { { &vnop_blktooff_desc, (VOPFUNC)spec_blktooff }, /* blktooff */ { &vnop_offtoblk_desc, (VOPFUNC)spec_offtoblk }, /* offtoblk */ { &vnop_blockmap_desc, (VOPFUNC)spec_blockmap }, /* blockmap */ - { (struct vnodeop_desc*)NULL, (int(*)())NULL } + { (struct vnodeop_desc*)NULL, (int(*)(void *))NULL } }; struct vnodeopv_desc spec_vnodeop_opv_desc = { &spec_vnodeop_p, spec_vnodeop_entries }; @@ -239,6 +240,7 @@ static void throttle_info_end_io_internal(struct _throttle_io_info_t *info, int static int throttle_info_update_internal(struct _throttle_io_info_t *info, uthread_t ut, int flags, boolean_t isssd, boolean_t inflight, struct bufattr *bap); static int throttle_get_thread_throttle_level(uthread_t ut); static int throttle_get_thread_throttle_level_internal(uthread_t ut, int io_tier); +void throttle_info_mount_reset_period(mount_t mp, int isssd); /* * Trivial lookup routine that always fails. @@ -725,10 +727,10 @@ spec_select(struct vnop_select_args *ap) } } -static int filt_specattach(struct knote *kn); +static int filt_specattach(struct knote *kn, struct kevent_internal_s *kev); int -spec_kqfilter(vnode_t vp, struct knote *kn) +spec_kqfilter(vnode_t vp, struct knote *kn, struct kevent_internal_s *kev) { dev_t dev; @@ -741,7 +743,7 @@ spec_kqfilter(vnode_t vp, struct knote *kn) * Try a bpf device, as defined in bsd/net/bpf.c * If it doesn't error out the attach, then it * claimed it. Otherwise, fall through and try - * a regular spec attach. + * other attaches. */ int32_t tmp_flags = kn->kn_flags; int64_t tmp_data = kn->kn_data; @@ -755,8 +757,31 @@ spec_kqfilter(vnode_t vp, struct knote *kn) kn->kn_data = tmp_data; #endif + if (major(dev) > nchrdev) { + knote_set_error(kn, ENXIO); + return 0; + } + + kn->kn_vnode_kqok = !!(cdevsw_flags[major(dev)] & CDEVSW_SELECT_KQUEUE); + kn->kn_vnode_use_ofst = !!(cdevsw_flags[major(dev)] & CDEVSW_USE_OFFSET); + + if (cdevsw_flags[major(dev)] & CDEVSW_IS_PTS) { + kn->kn_filtid = EVFILTID_PTSD; + return ptsd_kqfilter(dev, kn); + } else if (cdevsw[major(dev)].d_type == D_TTY && + !(cdevsw_flags[major(dev)] & CDEVSW_IS_PTC) && + kn->kn_vnode_kqok) { + /* + * TTYs from drivers that use struct ttys use their own filter + * routines. The PTC driver doesn't use the tty for character + * counts, so it must go through the select fallback. + */ + kn->kn_filtid = EVFILTID_TTY; + return knote_fops(kn)->f_attach(kn, kev); + } + /* Try to attach to other char special devices */ - return filt_specattach(kn); + return filt_specattach(kn, kev); } /* @@ -1503,6 +1528,27 @@ throttle_info_mount_rel(mount_t mp) mp->mnt_throttle_info = NULL; } +/* + * Reset throttling periods for the given mount point + * + * private interface used by disk conditioner to reset + * throttling periods when 'is_ssd' status changes + */ +void +throttle_info_mount_reset_period(mount_t mp, int isssd) +{ + struct _throttle_io_info_t *info; + + if (mp == NULL) + info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1]; + else if (mp->mnt_throttle_info == NULL) + info = &_throttle_io_info[mp->mnt_devbsdunit]; + else + info = mp->mnt_throttle_info; + + throttle_init_throttle_period(info, isssd); +} + void throttle_info_get_last_io_time(mount_t mp, struct timeval *tv) { @@ -1535,7 +1581,6 @@ update_last_io_time(mount_t mp) mp->mnt_last_write_completed_timestamp = info->throttle_last_write_timestamp; } - int throttle_get_io_policy(uthread_t *ut) { @@ -2031,7 +2076,7 @@ void *throttle_info_update_by_mount(mount_t mp) ut = get_bsdthread_info(current_thread()); if (mp != NULL) { - if ((mp->mnt_kern_flag & MNTK_SSD) && !ignore_is_ssd) + if (disk_conditioner_mount_is_ssd(mp)) isssd = TRUE; info = &_throttle_io_info[mp->mnt_devbsdunit]; } else @@ -2158,6 +2203,11 @@ int throttle_lowpri_window(void) return ut->uu_lowpri_window; } + +#if CONFIG_IOSCHED +int upl_get_cached_tier(void *); +#endif + int spec_strategy(struct vnop_strategy_args *ap) { @@ -2176,14 +2226,35 @@ spec_strategy(struct vnop_strategy_args *ap) boolean_t upgrade = FALSE; int code = 0; +#if !CONFIG_EMBEDDED proc_t curproc = current_proc(); +#endif /* !CONFIG_EMBEDDED */ bp = ap->a_bp; bdev = buf_device(bp); mp = buf_vnode(bp)->v_mount; bap = &bp->b_attr; +#if CONFIG_IOSCHED + if (bp->b_flags & B_CLUSTER) { + + io_tier = upl_get_cached_tier(bp->b_upl); + + if (io_tier == -1) + io_tier = throttle_get_io_policy(&ut); +#if DEVELOPMENT || DEBUG + else { + int my_io_tier = throttle_get_io_policy(&ut); + + if (io_tier != my_io_tier) + KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_THROTTLE, IO_TIER_UPL_MISMATCH)) | DBG_FUNC_NONE, buf_kernel_addrperm_addr(bp), my_io_tier, io_tier, 0, 0); + } +#endif + } else + io_tier = throttle_get_io_policy(&ut); +#else io_tier = throttle_get_io_policy(&ut); +#endif passive = throttle_get_passive_io_policy(&ut); /* @@ -2233,8 +2304,10 @@ spec_strategy(struct vnop_strategy_args *ap) bap->ba_flags |= BA_PASSIVE; } +#if !CONFIG_EMBEDDED if ((curproc != NULL) && ((curproc->p_flag & P_DELAYIDLESLEEP) == P_DELAYIDLESLEEP)) bap->ba_flags |= BA_DELAYIDLESLEEP; +#endif /* !CONFIG_EMBEDDED */ bflags = bp->b_flags; @@ -2275,7 +2348,7 @@ spec_strategy(struct vnop_strategy_args *ap) thread_update_io_stats(current_thread(), buf_count(bp), code); if (mp != NULL) { - if ((mp->mnt_kern_flag & MNTK_SSD) && !ignore_is_ssd) + if (disk_conditioner_mount_is_ssd(mp)) isssd = TRUE; /* * Partially initialized mounts don't have a final devbsdunit and should not be tracked. @@ -2327,6 +2400,11 @@ spec_strategy(struct vnop_strategy_args *ap) typedef int strategy_fcn_ret_t(struct buf *bp); strategy_ret = (*(strategy_fcn_ret_t*)bdevsw[major(bdev)].d_strategy)(bp); + + // disk conditioner needs to track when this I/O actually starts + // which means track it after `strategy` which may include delays + // from inflight I/Os + microuptime(&bp->b_timestamp_tv); if (IO_SATISFIED_BY_CACHE == strategy_ret) { /* @@ -2562,37 +2640,136 @@ spec_offtoblk(struct vnop_offtoblk_args *ap) } static void filt_specdetach(struct knote *kn); -static int filt_spec(struct knote *kn, long hint); +static int filt_specevent(struct knote *kn, long hint); static int filt_spectouch(struct knote *kn, struct kevent_internal_s *kev); static int filt_specprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); static unsigned filt_specpeek(struct knote *kn); -struct filterops spec_filtops = { - .f_isfd = 1, - .f_attach = filt_specattach, - .f_detach = filt_specdetach, - .f_event = filt_spec, - .f_touch = filt_spectouch, - .f_process = filt_specprocess, - .f_peek = filt_specpeek +SECURITY_READ_ONLY_EARLY(struct filterops) spec_filtops = { + .f_isfd = 1, + .f_attach = filt_specattach, + .f_detach = filt_specdetach, + .f_event = filt_specevent, + .f_touch = filt_spectouch, + .f_process = filt_specprocess, + .f_peek = filt_specpeek }; + +/* + * Given a waitq that is assumed to be embedded within a selinfo structure, + * return the containing selinfo structure. While 'wq' is not really a queue + * element, this macro simply does the offset_of calculation to get back to a + * containing struct given the struct type and member name. + */ +#define selinfo_from_waitq(wq) \ + qe_element((wq), struct selinfo, si_waitq) + static int -filter_to_seltype(int16_t filter) +spec_knote_select_and_link(struct knote *kn) { - switch (filter) { - case EVFILT_READ: - return FREAD; - case EVFILT_WRITE: - return FWRITE; - default: - panic("filt_to_seltype(): invalid filter %d\n", filter); + uthread_t uth; + vfs_context_t ctx; + vnode_t vp; + struct waitq_set *old_wqs; + uint64_t rsvd, rsvd_arg; + uint64_t *rlptr = NULL; + struct selinfo *si = NULL; + int selres = 0; + + uth = get_bsdthread_info(current_thread()); + + ctx = vfs_context_current(); + vp = (vnode_t)kn->kn_fp->f_fglob->fg_data; + + int error = vnode_getwithvid(vp, kn->kn_hookid); + if (error != 0) { + knote_set_error(kn, ENOENT); return 0; } + + /* + * This function may be called many times to link or re-link the + * underlying vnode to the kqueue. If we've already linked the two, + * we will have a valid kn_hook_data which ties us to the underlying + * device's waitq via a the waitq's prepost table object. However, + * devices can abort any select action by calling selthreadclear(). + * This is OK because the table object will be invalidated by the + * driver (through a call to selthreadclear), so any attempt to access + * the associated waitq will fail because the table object is invalid. + * + * Even if we've already registered, we need to pass a pointer + * to a reserved link structure. Otherwise, selrecord() will + * infer that we're in the second pass of select() and won't + * actually do anything! + */ + rsvd = rsvd_arg = waitq_link_reserve(NULL); + rlptr = (void *)&rsvd_arg; + + /* + * Trick selrecord() into hooking kqueue's wait queue set + * set into device's selinfo wait queue + */ + old_wqs = uth->uu_wqset; + uth->uu_wqset = &(knote_get_kq(kn)->kq_wqs); + selres = VNOP_SELECT(vp, knote_get_seltype(kn), 0, rlptr, ctx); + uth->uu_wqset = old_wqs; + + /* + * make sure to cleanup the reserved link - this guards against + * drivers that may not actually call selrecord(). + */ + waitq_link_release(rsvd); + if (rsvd != rsvd_arg) { + /* the driver / handler called selrecord() */ + struct waitq *wq; + memcpy(&wq, rlptr, sizeof(void *)); + + /* + * The waitq is part of the selinfo structure managed by the + * driver. For certain drivers, we want to hook the knote into + * the selinfo structure's si_note field so selwakeup can call + * KNOTE. + */ + si = selinfo_from_waitq(wq); + + /* + * The waitq_get_prepost_id() function will (potentially) + * allocate a prepost table object for the waitq and return + * the table object's ID to us. It will also set the + * waitq_prepost_id field within the waitq structure. + * + * We can just overwrite kn_hook_data because it's simply a + * table ID used to grab a reference when needed. + * + * We have a reference on the vnode, so we know that the + * device won't go away while we get this ID. + */ + kn->kn_hook_data = waitq_get_prepost_id(wq); + } else { + assert(selres != 0); + } + + vnode_put(vp); + + return selres; } -static int -filt_specattach(struct knote *kn) +static void filt_spec_common(struct knote *kn, int selres) +{ + if (kn->kn_vnode_use_ofst) { + if (kn->kn_fp->f_fglob->fg_offset >= (uint32_t)selres) { + kn->kn_data = 0; + } else { + kn->kn_data = ((uint32_t)selres) - kn->kn_fp->f_fglob->fg_offset; + } + } else { + kn->kn_data = selres; + } +} + +static int +filt_specattach(struct knote *kn, __unused struct kevent_internal_s *kev) { vnode_t vp; dev_t dev; @@ -2603,12 +2780,6 @@ filt_specattach(struct knote *kn) dev = vnode_specrdev(vp); - if (major(dev) > nchrdev) { - kn->kn_flags |= EV_ERROR; - kn->kn_data = ENXIO; - return 0; - } - /* * For a few special kinds of devices, we can attach knotes with * no restrictions because their "select" vectors return the amount @@ -2616,25 +2787,32 @@ filt_specattach(struct knote *kn) * data of 1, indicating that the caller doesn't care about actual * data counts, just an indication that the device has data. */ - - if ((cdevsw_flags[major(dev)] & CDEVSW_SELECT_KQUEUE) == 0 && + if (!kn->kn_vnode_kqok && ((kn->kn_sfflags & NOTE_LOWAT) == 0 || kn->kn_sdata != 1)) { - kn->kn_flags |= EV_ERROR; - kn->kn_data = EINVAL; + knote_set_error(kn, EINVAL); return 0; } - kn->kn_hook_data = 0; + /* + * This forces the select fallback to call through VNOP_SELECT and hook + * up selinfo on every filter routine. + * + * Pseudo-terminal controllers are opted out of native kevent support -- + * remove this when they get their own EVFILTID. + */ + if (cdevsw_flags[major(dev)] & CDEVSW_IS_PTC) { + kn->kn_vnode_kqok = 0; + } kn->kn_filtid = EVFILTID_SPEC; + kn->kn_hook_data = 0; kn->kn_hookid = vnode_vid(vp); knote_markstayactive(kn); - - return 0; + return spec_knote_select_and_link(kn); } -static void +static void filt_specdetach(struct knote *kn) { knote_clearstayactive(kn); @@ -2657,15 +2835,16 @@ filt_specdetach(struct knote *kn) } } -static int -filt_spec(__unused struct knote *kn, __unused long hint) +static int +filt_specevent(struct knote *kn, __unused long hint) { - panic("filt_spec()"); + /* + * Nothing should call knote or knote_vanish on this knote. + */ + panic("filt_specevent(%p)", kn); return 0; } - - static int filt_spectouch(struct knote *kn, struct kevent_internal_s *kev) { @@ -2674,7 +2853,10 @@ filt_spectouch(struct knote *kn, struct kevent_internal_s *kev) if ((kn->kn_status & KN_UDATA_SPECIFIC) == 0) kn->kn_udata = kev->udata; - /* stayqueued knotes don't need hints from touch */ + if (kev->flags & EV_ENABLE) { + return spec_knote_select_and_link(kn); + } + return 0; } @@ -2684,96 +2866,26 @@ filt_specprocess(struct knote *kn, struct filt_process_s *data, struct kevent_in #pragma unused(data) vnode_t vp; uthread_t uth; - struct waitq_set *old_wqs; vfs_context_t ctx; int res; int selres; int error; - int use_offset; - dev_t dev; - uint64_t flags; - uint64_t rsvd, rsvd_arg; - uint64_t *rlptr = NULL; uth = get_bsdthread_info(current_thread()); ctx = vfs_context_current(); vp = (vnode_t)kn->kn_fp->f_fglob->fg_data; - /* JMM - locking against touches? */ + /* FIXME JMM - locking against touches? */ error = vnode_getwithvid(vp, kn->kn_hookid); if (error != 0) { kn->kn_flags |= (EV_EOF | EV_ONESHOT); - *kev = kn->kn_kevent; + *kev = kn->kn_kevent; return 1; } - - dev = vnode_specrdev(vp); - flags = cdevsw_flags[major(dev)]; - use_offset = ((flags & CDEVSW_USE_OFFSET) != 0); - /* - * This function may be called many times to link or re-link the - * underlying vnode to the kqueue. If we've already linked the two, - * we will have a valid kn_hook_data which ties us to the underlying - * device's waitq via a the waitq's prepost table object. However, - * devices can abort any select action by calling selthreadclear(). - * This is OK because the table object will be invalidated by the - * driver (through a call to selthreadclear), so any attempt to access - * the associated waitq will fail because the table object is invalid. - * - * Even if we've already registered, we need to pass a pointer - * to a reserved link structure. Otherwise, selrecord() will - * infer that we're in the second pass of select() and won't - * actually do anything! - */ - rsvd = rsvd_arg = waitq_link_reserve(NULL); - rlptr = (void *)&rsvd_arg; - - /* - * Trick selrecord() into hooking kqueue's wait queue set - * set into device's selinfo wait queue - */ - old_wqs = uth->uu_wqset; - uth->uu_wqset = &(knote_get_kq(kn)->kq_wqs); - selres = VNOP_SELECT(vp, filter_to_seltype(kn->kn_filter), - 0, rlptr, ctx); - uth->uu_wqset = old_wqs; - - /* - * make sure to cleanup the reserved link - this guards against - * drivers that may not actually call selrecord(). - */ - waitq_link_release(rsvd); - if (rsvd != rsvd_arg) { - /* the driver / handler called selrecord() */ - struct waitq *wq; - memcpy(&wq, rlptr, sizeof(void *)); - - /* - * The waitq_get_prepost_id() function will (potentially) - * allocate a prepost table object for the waitq and return - * the table object's ID to us. It will also set the - * waitq_prepost_id field within the waitq structure. - * - * We can just overwrite kn_hook_data because it's simply a - * table ID used to grab a reference when needed. - * - * We have a reference on the vnode, so we know that the - * device won't go away while we get this ID. - */ - kn->kn_hook_data = waitq_get_prepost_id(wq); - } - - if (use_offset) { - if (kn->kn_fp->f_fglob->fg_offset >= (uint32_t)selres) { - kn->kn_data = 0; - } else { - kn->kn_data = ((uint32_t)selres) - kn->kn_fp->f_fglob->fg_offset; - } - } else { - kn->kn_data = selres; - } + selres = spec_knote_select_and_link(kn); + filt_spec_common(kn, selres); vnode_put(vp); @@ -2794,64 +2906,11 @@ filt_specprocess(struct knote *kn, struct filt_process_s *data, struct kevent_in static unsigned filt_specpeek(struct knote *kn) { - vnode_t vp; - uthread_t uth; - struct waitq_set *old_wqs; - vfs_context_t ctx; - int error, selres; - uint64_t rsvd, rsvd_arg; - uint64_t *rlptr = NULL; - - uth = get_bsdthread_info(current_thread()); - ctx = vfs_context_current(); - vp = (vnode_t)kn->kn_fp->f_fglob->fg_data; - - error = vnode_getwithvid(vp, kn->kn_hookid); - if (error != 0) { - return 1; /* Just like VNOP_SELECT() on recycled vnode */ - } - - /* - * Even if we've already registered, we need to pass a pointer - * to a reserved link structure. Otherwise, selrecord() will - * infer that we're in the second pass of select() and won't - * actually do anything! - */ - rsvd = rsvd_arg = waitq_link_reserve(NULL); - rlptr = (void *)&rsvd_arg; + int selres = 0; - old_wqs = uth->uu_wqset; - uth->uu_wqset = &(knote_get_kq(kn)->kq_wqs); - selres = VNOP_SELECT(vp, filter_to_seltype(kn->kn_filter), - 0, (void *)rlptr, ctx); - uth->uu_wqset = old_wqs; + selres = spec_knote_select_and_link(kn); + filt_spec_common(kn, selres); - /* - * make sure to cleanup the reserved link - this guards against - * drivers that may not actually call selrecord() - */ - waitq_link_release(rsvd); - if (rsvd != rsvd_arg) { - /* the driver / handler called selrecord() */ - struct waitq *wq; - memcpy(&wq, rlptr, sizeof(void *)); - - /* - * The waitq_get_prepost_id() function will (potentially) - * allocate a prepost table object for the waitq and return - * the table object's ID to us. It will also set the - * waitq_prepost_id field within the waitq structure. - * - * We can just overwrite kn_hook_data because it's simply a - * table ID used to grab a reference when needed. - * - * We have a reference on the vnode, so we know that the - * device won't go away while we get this ID. - */ - kn->kn_hook_data = waitq_get_prepost_id(wq); - } - - vnode_put(vp); - return selres; + return kn->kn_data; } diff --git a/bsd/miscfs/specfs/specdev.h b/bsd/miscfs/specfs/specdev.h index 2b14d796b..3d6d0258f 100644 --- a/bsd/miscfs/specfs/specdev.h +++ b/bsd/miscfs/specfs/specdev.h @@ -135,7 +135,7 @@ int spec_blktooff (struct vnop_blktooff_args *); int spec_offtoblk (struct vnop_offtoblk_args *); int spec_fsync_internal (vnode_t, int, vfs_context_t); int spec_blockmap (struct vnop_blockmap_args *); -int spec_kqfilter (vnode_t vp, struct knote *kn); +int spec_kqfilter (vnode_t vp, struct knote *kn, struct kevent_internal_s *kev); #endif /* BSD_KERNEL_PRIVATE */ int spec_ebadf(void *); diff --git a/bsd/net/Makefile b/bsd/net/Makefile index 234c2a15b..98db2444f 100644 --- a/bsd/net/Makefile +++ b/bsd/net/Makefile @@ -50,10 +50,12 @@ PRIVATE_DATAFILES = \ if_utun.h \ if_var.h \ if_vlan_var.h \ + if_fake_var.h \ iptap.h \ lacp.h \ ndrv_var.h \ necp.h \ + net_api_stats.h \ netsrc.h \ network_agent.h \ ntstat.h \ diff --git a/bsd/net/altq/altq.h b/bsd/net/altq/altq.h index 590c6810b..a3b18841f 100644 --- a/bsd/net/altq/altq.h +++ b/bsd/net/altq/altq.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 Apple Inc. All rights reserved. + * Copyright (c) 2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -65,11 +65,8 @@ #define ALTQT_CBQ PKTSCHEDT_CBQ /* cbq */ #define ALTQT_HFSC PKTSCHEDT_HFSC /* hfsc */ #define ALTQT_PRIQ PKTSCHEDT_PRIQ /* priority queue */ -#define ALTQT_FAIRQ PKTSCHEDT_FAIRQ /* fairq */ -#define ALTQT_QFQ PKTSCHEDT_QFQ /* quick fair queueing */ +#define ALTQT_FAIRQ PKTSCHEDT_FAIRQ /* fairq */ +#define ALTQT_QFQ PKTSCHEDT_QFQ /* quick fair queueing */ #define ALTQT_MAX PKTSCHEDT_MAX /* should be max disc type + 1 */ #endif /* PRIVATE */ -#ifdef BSD_KERNEL_PRIVATE -#include <net/altq/altq_var.h> -#endif /* BSD_KERNEL_PRIVATE */ #endif /* _ALTQ_ALTQ_H_ */ diff --git a/bsd/net/altq/altq_cbq.c b/bsd/net/altq/altq_cbq.c deleted file mode 100644 index 8ced30eeb..000000000 --- a/bsd/net/altq/altq_cbq.c +++ /dev/null @@ -1,272 +0,0 @@ -/* - * Copyright (c) 2007-2013 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* $OpenBSD: altq_cbq.c,v 1.23 2007/09/13 20:40:02 chl Exp $ */ -/* $KAME: altq_cbq.c,v 1.9 2000/12/14 08:12:45 thorpej Exp $ */ - -/* - * Copyright (c) Sun Microsystems, Inc. 1993-1998 All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the SMCC Technology - * Development Group at Sun Microsystems, Inc. - * - * 4. The name of the Sun Microsystems, Inc nor may not be used to endorse or - * promote products derived from this software without specific prior - * written permission. - * - * SUN MICROSYSTEMS DOES NOT CLAIM MERCHANTABILITY OF THIS SOFTWARE OR THE - * SUITABILITY OF THIS SOFTWARE FOR ANY PARTICULAR PURPOSE. The software is - * provided "as is" without express or implied warranty of any kind. - * - * These notices must be retained in any copies of any part of this software. - */ - -#if PF_ALTQ && PKTSCHED_CBQ - -#include <sys/cdefs.h> -#include <sys/param.h> -#include <sys/malloc.h> -#include <sys/mbuf.h> -#include <sys/systm.h> -#include <sys/errno.h> -#include <sys/kernel.h> - -#include <net/if.h> -#include <net/pfvar.h> -#include <net/net_osdep.h> -#include <net/altq/altq.h> -#include <net/altq/altq_cbq.h> -#include <netinet/in.h> - -/* - * Forward Declarations. - */ -static int altq_cbq_request(struct ifaltq *, enum altrq, void *); -static int altq_cbq_enqueue(struct ifaltq *, struct mbuf *); -static struct mbuf *altq_cbq_dequeue(struct ifaltq *, enum altdq_op); - -int -altq_cbq_pfattach(struct pf_altq *a) -{ - struct ifnet *ifp; - int error; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((ifp = ifunit(a->ifname)) == NULL || a->altq_disc == NULL) - return (EINVAL); - - IFCQ_LOCK(&ifp->if_snd); - error = altq_attach(IFCQ_ALTQ(&ifp->if_snd), ALTQT_CBQ, a->altq_disc, - altq_cbq_enqueue, altq_cbq_dequeue, NULL, altq_cbq_request); - IFCQ_UNLOCK(&ifp->if_snd); - - return (error); -} - -int -altq_cbq_add(struct pf_altq *a) -{ - cbq_state_t *cbqp; - struct ifnet *ifp; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((ifp = ifunit(a->ifname)) == NULL) - return (EINVAL); - if (!ALTQ_IS_READY(IFCQ_ALTQ(&ifp->if_snd))) - return (ENODEV); - - cbqp = cbq_alloc(ifp, M_WAITOK, TRUE); - if (cbqp == NULL) - return (ENOMEM); - - /* keep the state in pf_altq */ - a->altq_disc = cbqp; - - return (0); -} - -int -altq_cbq_remove(struct pf_altq *a) -{ - cbq_state_t *cbqp; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((cbqp = a->altq_disc) == NULL) - return (EINVAL); - a->altq_disc = NULL; - - return (cbq_destroy(cbqp)); -} - -int -altq_cbq_add_queue(struct pf_altq *a) -{ - struct cbq_opts *opts = &a->pq_u.cbq_opts; - cbq_state_t *cbqp; - int err; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((cbqp = a->altq_disc) == NULL) - return (EINVAL); - - IFCQ_LOCK(cbqp->ifnp.ifq_); - err = cbq_add_queue(cbqp, a->qlimit, a->priority, - opts->minburst, opts->maxburst, opts->pktsize, opts->maxpktsize, - opts->ns_per_byte, opts->maxidle, opts->minidle, opts->offtime, - opts->flags, a->parent_qid, a->qid, NULL); - IFCQ_UNLOCK(cbqp->ifnp.ifq_); - - return (err); -} - -int -altq_cbq_remove_queue(struct pf_altq *a) -{ - cbq_state_t *cbqp; - int err; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((cbqp = a->altq_disc) == NULL) - return (EINVAL); - - IFCQ_LOCK(cbqp->ifnp.ifq_); - err = cbq_remove_queue(cbqp, a->qid); - IFCQ_UNLOCK(cbqp->ifnp.ifq_); - - return (err); -} - -int -altq_cbq_getqstats(struct pf_altq *a, void *ubuf, int *nbytes) -{ - struct ifclassq *ifq = NULL; - cbq_state_t *cbqp; - class_stats_t stats; - int error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((unsigned)*nbytes < sizeof (stats)) - return (EINVAL); - - if ((cbqp = altq_lookup(a->ifname, ALTQT_CBQ)) == NULL) - return (EBADF); - - ifq = cbqp->ifnp.ifq_; - IFCQ_LOCK_ASSERT_HELD(ifq); /* lock held by altq_lookup */ - error = cbq_get_class_stats(cbqp, a->qid, &stats); - IFCQ_UNLOCK(ifq); - if (error != 0) - return (error); - - if ((error = copyout((caddr_t)&stats, (user_addr_t)(uintptr_t)ubuf, - sizeof (stats))) != 0) - return (error); - - *nbytes = sizeof (stats); - - return (0); -} - -static int -altq_cbq_request(struct ifaltq *altq, enum altrq req, void *arg) -{ - cbq_state_t *cbqp = (cbq_state_t *)altq->altq_disc; - - switch (req) { - case ALTRQ_PURGE: - cbq_purge(cbqp); - break; - - case ALTRQ_PURGE_SC: - /* not supported for ALTQ instance */ - break; - - case ALTRQ_EVENT: - cbq_event(cbqp, (cqev_t)arg); - break; - - case ALTRQ_THROTTLE: - default: - break; - } - return (0); -} - -/* - * altq_cbq_enqueue is an enqueue function to be registered to - * (*altq_enqueue) in struct ifaltq. - */ -static int -altq_cbq_enqueue(struct ifaltq *altq, struct mbuf *m) -{ - /* grab class set by classifier */ - if (!(m->m_flags & M_PKTHDR)) { - /* should not happen */ - printf("%s: packet for %s does not have pkthdr\n", __func__, - if_name(altq->altq_ifcq->ifcq_ifp)); - m_freem(m); - return (ENOBUFS); - } - - return (cbq_enqueue(altq->altq_disc, NULL, m, m_pftag(m))); -} - -/* - * altq_cbq_dequeue is a dequeue function to be registered to - * (*altq_dequeue) in struct ifaltq. - * - * note: ALTDQ_POLL returns the next packet without removing the packet - * from the queue. ALTDQ_REMOVE is a normal dequeue operation. - * ALTDQ_REMOVE must return the same packet if called immediately - * after ALTDQ_POLL. - */ -static struct mbuf * -altq_cbq_dequeue(struct ifaltq *altq, enum altdq_op op) -{ - return (cbq_dequeue(altq->altq_disc, (cqdq_op_t)op)); -} -#endif /* PF_ALTQ && PKTSCHED_CBQ */ diff --git a/bsd/net/altq/altq_cbq.h b/bsd/net/altq/altq_cbq.h index fba7310c9..3a47e8df0 100644 --- a/bsd/net/altq/altq_cbq.h +++ b/bsd/net/altq/altq_cbq.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 Apple Inc. All rights reserved. + * Copyright (c) 2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -66,21 +66,4 @@ #include <net/altq/altq.h> #include <net/pktsched/pktsched_cbq.h> -#ifdef BSD_KERNEL_PRIVATE -#ifdef __cplusplus -extern "C" { -#endif - -extern int altq_cbq_pfattach(struct pf_altq *); -extern int altq_cbq_add(struct pf_altq *); -extern int altq_cbq_remove(struct pf_altq *); -extern int altq_cbq_add_queue(struct pf_altq *); -extern int altq_cbq_remove_queue(struct pf_altq *); -extern int altq_cbq_getqstats(struct pf_altq *, void *, int *); - -#ifdef __cplusplus -} -#endif - -#endif /* BSD_KERNEL_PRIVATE */ #endif /* !_NET_ALTQ_ALTQ_CBQ_H_ */ diff --git a/bsd/net/altq/altq_fairq.c b/bsd/net/altq/altq_fairq.c deleted file mode 100644 index 69dcf2209..000000000 --- a/bsd/net/altq/altq_fairq.c +++ /dev/null @@ -1,304 +0,0 @@ -/* - * Copyright (c) 2011-2013 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* - * Copyright (c) 2008 The DragonFly Project. All rights reserved. - * - * This code is derived from software contributed to The DragonFly Project - * by Matthew Dillon <dillon@backplane.com> - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * 3. Neither the name of The DragonFly Project nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific, prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * $DragonFly: src/sys/net/altq/altq_fairq.c,v 1.2 2008/05/14 11:59:23 sephe Exp $ - */ -/* - * Matt: I gutted altq_priq.c and used it as a skeleton on which to build - * fairq. The fairq algorithm is completely different then priq, of course, - * but because I used priq's skeleton I believe I should include priq's - * copyright. - * - * Copyright (C) 2000-2003 - * Sony Computer Science Laboratories Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY SONY CSL AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL SONY CSL OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#if PF_ALTQ && PKTSCHED_FAIRQ - -#include <sys/cdefs.h> -#include <sys/param.h> -#include <sys/malloc.h> -#include <sys/mbuf.h> -#include <sys/systm.h> -#include <sys/errno.h> -#include <sys/kernel.h> - -#include <net/if.h> -#include <net/pfvar.h> -#include <net/net_osdep.h> -#include <net/altq/altq.h> -#include <net/altq/altq_fairq.h> -#include <netinet/in.h> - -/* - * function prototypes - */ -static int altq_fairq_enqueue(struct ifaltq *, struct mbuf *); -static struct mbuf *altq_fairq_dequeue(struct ifaltq *, enum altdq_op); -static int altq_fairq_request(struct ifaltq *, enum altrq, void *); - -int -altq_fairq_pfattach(struct pf_altq *a) -{ - struct ifnet *ifp; - int error; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((ifp = ifunit(a->ifname)) == NULL || a->altq_disc == NULL) - return (EINVAL); - - IFCQ_LOCK(&ifp->if_snd); - error = altq_attach(IFCQ_ALTQ(&ifp->if_snd), ALTQT_FAIRQ, a->altq_disc, - altq_fairq_enqueue, altq_fairq_dequeue, NULL, altq_fairq_request); - IFCQ_UNLOCK(&ifp->if_snd); - - return (error); -} - -int -altq_fairq_add(struct pf_altq *a) -{ - struct fairq_if *fif; - struct ifnet *ifp; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((ifp = ifunit(a->ifname)) == NULL) - return (EINVAL); - if (!ALTQ_IS_READY(IFCQ_ALTQ(&ifp->if_snd))) - return (ENODEV); - - fif = fairq_alloc(ifp, M_WAITOK, TRUE); - if (fif == NULL) - return (ENOMEM); - - /* keep the state in pf_altq */ - a->altq_disc = fif; - - return (0); -} - -int -altq_fairq_remove(struct pf_altq *a) -{ - struct fairq_if *fif; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((fif = a->altq_disc) == NULL) - return (EINVAL); - a->altq_disc = NULL; - - return (fairq_destroy(fif)); -} - -int -altq_fairq_add_queue(struct pf_altq *a) -{ - struct fairq_if *fif; - struct fairq_opts *opts = &a->pq_u.fairq_opts; - int err; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((fif = a->altq_disc) == NULL) - return (EINVAL); - - IFCQ_LOCK(fif->fif_ifq); - err = fairq_add_queue(fif, a->priority, a->qlimit, a->bandwidth, - opts->nbuckets, opts->flags, opts->hogs_m1, opts->lssc_m1, - opts->lssc_d, opts->lssc_m2, a->qid, NULL); - IFCQ_UNLOCK(fif->fif_ifq); - - return (err); -} - -int -altq_fairq_remove_queue(struct pf_altq *a) -{ - struct fairq_if *fif; - int err; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((fif = a->altq_disc) == NULL) - return (EINVAL); - - IFCQ_LOCK(fif->fif_ifq); - err = fairq_remove_queue(fif, a->qid); - IFCQ_UNLOCK(fif->fif_ifq); - - return (err); -} - -int -altq_fairq_getqstats(struct pf_altq *a, void *ubuf, int *nbytes) -{ - struct ifclassq *ifq = NULL; - struct fairq_if *fif; - struct fairq_classstats stats; - int error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((unsigned)*nbytes < sizeof (stats)) - return (EINVAL); - - if ((fif = altq_lookup(a->ifname, ALTQT_FAIRQ)) == NULL) - return (EBADF); - - ifq = fif->fif_ifq; - IFCQ_LOCK_ASSERT_HELD(ifq); /* lock held by altq_lookup */ - error = fairq_get_class_stats(fif, a->qid, &stats); - IFCQ_UNLOCK(ifq); - if (error != 0) - return (error); - - if ((error = copyout((caddr_t)&stats, (user_addr_t)(uintptr_t)ubuf, - sizeof (stats))) != 0) - return (error); - - *nbytes = sizeof (stats); - - return (0); -} - -static int -altq_fairq_request(struct ifaltq *altq, enum altrq req, void *arg) -{ - struct fairq_if *fif = (struct fairq_if *)altq->altq_disc; - - switch (req) { - case ALTRQ_PURGE: - fairq_purge(fif); - break; - - case ALTRQ_PURGE_SC: - /* not supported for ALTQ instance */ - break; - - case ALTRQ_EVENT: - fairq_event(fif, (cqev_t)arg); - break; - - case ALTRQ_THROTTLE: - default: - break; - } - return (0); -} - -/* - * altq_fairq_enqueue is an enqueue function to be registered to - * (*altq_enqueue) in struct ifaltq. - */ -static int -altq_fairq_enqueue(struct ifaltq *altq, struct mbuf *m) -{ - /* grab class set by classifier */ - if (!(m->m_flags & M_PKTHDR)) { - /* should not happen */ - printf("%s: packet for %s does not have pkthdr\n", __func__, - if_name(altq->altq_ifcq->ifcq_ifp)); - m_freem(m); - return (ENOBUFS); - } - - return (fairq_enqueue(altq->altq_disc, NULL, m, m_pftag(m))); -} - -/* - * altq_fairq_dequeue is a dequeue function to be registered to - * (*altq_dequeue) in struct ifaltq. - * - * note: ALTDQ_POLL returns the next packet without removing the packet - * from the queue. ALTDQ_REMOVE is a normal dequeue operation. - * ALTDQ_REMOVE must return the same packet if called immediately - * after ALTDQ_POLL. - */ -static struct mbuf * -altq_fairq_dequeue(struct ifaltq *altq, enum altdq_op op) -{ - return (fairq_dequeue(altq->altq_disc, (cqdq_op_t)op)); -} -#endif /* PF_ALTQ && PKTSCHED_FAIRQ */ diff --git a/bsd/net/altq/altq_fairq.h b/bsd/net/altq/altq_fairq.h index d9d536ca8..f9c20940a 100644 --- a/bsd/net/altq/altq_fairq.h +++ b/bsd/net/altq/altq_fairq.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 Apple Inc. All rights reserved. + * Copyright (c) 2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -69,20 +69,4 @@ #include <net/altq/altq.h> #include <net/pktsched/pktsched_fairq.h> -#ifdef BSD_KERNEL_PRIVATE -#ifdef __cplusplus -extern "C" { -#endif - -extern int altq_fairq_pfattach(struct pf_altq *); -extern int altq_fairq_add(struct pf_altq *); -extern int altq_fairq_remove(struct pf_altq *); -extern int altq_fairq_add_queue(struct pf_altq *); -extern int altq_fairq_remove_queue(struct pf_altq *); -extern int altq_fairq_getqstats(struct pf_altq *, void *, int *); - -#ifdef __cplusplus -} -#endif -#endif /* BSD_KERNEL_PRIVATE */ #endif /* _NET_ALTQ_ALTQ_FAIRQ_H_ */ diff --git a/bsd/net/altq/altq_hfsc.c b/bsd/net/altq/altq_hfsc.c deleted file mode 100644 index adaf0d35c..000000000 --- a/bsd/net/altq/altq_hfsc.c +++ /dev/null @@ -1,290 +0,0 @@ -/* - * Copyright (c) 2007-2013 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* $OpenBSD: altq_hfsc.c,v 1.25 2007/09/13 20:40:02 chl Exp $ */ -/* $KAME: altq_hfsc.c,v 1.17 2002/11/29 07:48:33 kjc Exp $ */ - -/* - * Copyright (c) 1997-1999 Carnegie Mellon University. All Rights Reserved. - * - * Permission to use, copy, modify, and distribute this software and - * its documentation is hereby granted (including for commercial or - * for-profit use), provided that both the copyright notice and this - * permission notice appear in all copies of the software, derivative - * works, or modified versions, and any portions thereof. - * - * THIS SOFTWARE IS EXPERIMENTAL AND IS KNOWN TO HAVE BUGS, SOME OF - * WHICH MAY HAVE SERIOUS CONSEQUENCES. CARNEGIE MELLON PROVIDES THIS - * SOFTWARE IN ITS ``AS IS'' CONDITION, AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT - * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR - * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE - * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH - * DAMAGE. - * - * Carnegie Mellon encourages (but does not require) users of this - * software to return any improvements or extensions that they make, - * and to grant Carnegie Mellon the rights to redistribute these - * changes without encumbrance. - */ - -#include <sys/cdefs.h> - -#if PF_ALTQ && PKTSCHED_HFSC - -#include <sys/cdefs.h> -#include <sys/param.h> -#include <sys/malloc.h> -#include <sys/mbuf.h> -#include <sys/systm.h> -#include <sys/errno.h> -#include <sys/kernel.h> - -#include <net/if.h> -#include <net/pfvar.h> -#include <net/net_osdep.h> -#include <net/altq/altq.h> -#include <net/altq/altq_hfsc.h> -#include <netinet/in.h> - -/* - * function prototypes - */ -static int altq_hfsc_request(struct ifaltq *, enum altrq, void *); -static int altq_hfsc_enqueue(struct ifaltq *, struct mbuf *); -static struct mbuf *altq_hfsc_dequeue(struct ifaltq *, enum altdq_op); - -int -altq_hfsc_pfattach(struct pf_altq *a) -{ - struct ifnet *ifp; - int error; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((ifp = ifunit(a->ifname)) == NULL || a->altq_disc == NULL) - return (EINVAL); - - IFCQ_LOCK(&ifp->if_snd); - error = altq_attach(IFCQ_ALTQ(&ifp->if_snd), ALTQT_HFSC, a->altq_disc, - altq_hfsc_enqueue, altq_hfsc_dequeue, NULL, altq_hfsc_request); - IFCQ_UNLOCK(&ifp->if_snd); - - return (error); -} - -int -altq_hfsc_add(struct pf_altq *a) -{ - struct hfsc_if *hif; - struct ifnet *ifp; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((ifp = ifunit(a->ifname)) == NULL) - return (EINVAL); - if (!ALTQ_IS_READY(IFCQ_ALTQ(&ifp->if_snd))) - return (ENODEV); - - hif = hfsc_alloc(ifp, M_WAITOK, TRUE); - if (hif == NULL) - return (ENOMEM); - - /* keep the state in pf_altq */ - a->altq_disc = hif; - - return (0); -} - -int -altq_hfsc_remove(struct pf_altq *a) -{ - struct hfsc_if *hif; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((hif = a->altq_disc) == NULL) - return (EINVAL); - a->altq_disc = NULL; - - return (hfsc_destroy(hif)); -} - -int -altq_hfsc_add_queue(struct pf_altq *a) -{ - struct hfsc_if *hif; - struct hfsc_opts *opts = &a->pq_u.hfsc_opts; - struct service_curve rtsc, lssc, ulsc; - int err; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((hif = a->altq_disc) == NULL) - return (EINVAL); - - bzero(&rtsc, sizeof (rtsc)); - bzero(&lssc, sizeof (lssc)); - bzero(&ulsc, sizeof (ulsc)); - - rtsc.m1 = opts->rtsc_m1; - rtsc.d = opts->rtsc_d; - rtsc.m2 = opts->rtsc_m2; - rtsc.fl = opts->rtsc_fl; - lssc.m1 = opts->lssc_m1; - lssc.d = opts->lssc_d; - lssc.m2 = opts->lssc_m2; - lssc.fl = opts->lssc_fl; - ulsc.m1 = opts->ulsc_m1; - ulsc.d = opts->ulsc_d; - ulsc.m2 = opts->ulsc_m2; - ulsc.fl = opts->ulsc_fl; - - IFCQ_LOCK(hif->hif_ifq); - err = hfsc_add_queue(hif, &rtsc, &lssc, &ulsc, a->qlimit, - opts->flags, a->parent_qid, a->qid, NULL); - IFCQ_UNLOCK(hif->hif_ifq); - - return (err); -} - -int -altq_hfsc_remove_queue(struct pf_altq *a) -{ - struct hfsc_if *hif; - int err; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((hif = a->altq_disc) == NULL) - return (EINVAL); - - IFCQ_LOCK(hif->hif_ifq); - err = hfsc_remove_queue(hif, a->qid); - IFCQ_UNLOCK(hif->hif_ifq); - - return (err); -} - -int -altq_hfsc_getqstats(struct pf_altq *a, void *ubuf, int *nbytes) -{ - struct ifclassq *ifq = NULL; - struct hfsc_if *hif; - struct hfsc_classstats stats; - int error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((unsigned)*nbytes < sizeof (stats)) - return (EINVAL); - - if ((hif = altq_lookup(a->ifname, ALTQT_HFSC)) == NULL) - return (EBADF); - - ifq = hif->hif_ifq; - IFCQ_LOCK_ASSERT_HELD(ifq); /* lock held by altq_lookup */ - error = hfsc_get_class_stats(hif, a->qid, &stats); - IFCQ_UNLOCK(ifq); - if (error != 0) - return (error); - - if ((error = copyout((caddr_t)&stats, (user_addr_t)(uintptr_t)ubuf, - sizeof (stats))) != 0) - return (error); - - *nbytes = sizeof (stats); - - return (0); -} - -static int -altq_hfsc_request(struct ifaltq *altq, enum altrq req, void *arg) -{ - struct hfsc_if *hif = (struct hfsc_if *)altq->altq_disc; - - switch (req) { - case ALTRQ_PURGE: - hfsc_purge(hif); - break; - - case ALTRQ_PURGE_SC: - /* not supported for ALTQ instance */ - break; - - case ALTRQ_EVENT: - hfsc_event(hif, (cqev_t)arg); - break; - - case ALTRQ_THROTTLE: - default: - break; - } - return (0); -} - -/* - * altq_hfsc_enqueue is an enqueue function to be registered to - * (*altq_enqueue) in struct ifaltq. - */ -static int -altq_hfsc_enqueue(struct ifaltq *altq, struct mbuf *m) -{ - /* grab class set by classifier */ - if (!(m->m_flags & M_PKTHDR)) { - /* should not happen */ - printf("%s: packet for %s does not have pkthdr\n", __func__, - if_name(altq->altq_ifcq->ifcq_ifp)); - m_freem(m); - return (ENOBUFS); - } - - return (hfsc_enqueue(altq->altq_disc, NULL, m, m_pftag(m))); -} - -/* - * altq_hfsc_dequeue is a dequeue function to be registered to - * (*altq_dequeue) in struct ifaltq. - * - * note: ALTDQ_POLL returns the next packet without removing the packet - * from the queue. ALTDQ_REMOVE is a normal dequeue operation. - * ALTDQ_REMOVE must return the same packet if called immediately - * after ALTDQ_POLL. - */ -static struct mbuf * -altq_hfsc_dequeue(struct ifaltq *altq, enum altdq_op op) -{ - return (hfsc_dequeue(altq->altq_disc, (cqdq_op_t)op)); -} -#endif /* PF_ALTQ && PKTSCHED_HFSC */ diff --git a/bsd/net/altq/altq_hfsc.h b/bsd/net/altq/altq_hfsc.h index 6b46293e7..0addc4fea 100644 --- a/bsd/net/altq/altq_hfsc.h +++ b/bsd/net/altq/altq_hfsc.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 Apple Inc. All rights reserved. + * Copyright (c) 2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -65,20 +65,4 @@ #include <net/altq/altq.h> #include <net/pktsched/pktsched_hfsc.h> -#ifdef BSD_KERNEL_PRIVATE -#ifdef __cplusplus -extern "C" { -#endif - -extern int altq_hfsc_pfattach(struct pf_altq *); -extern int altq_hfsc_add(struct pf_altq *); -extern int altq_hfsc_remove(struct pf_altq *); -extern int altq_hfsc_add_queue(struct pf_altq *); -extern int altq_hfsc_remove_queue(struct pf_altq *); -extern int altq_hfsc_getqstats(struct pf_altq *, void *, int *); - -#ifdef __cplusplus -} -#endif -#endif /* BSD_KERNEL_PRIVATE */ #endif /* _NET_ALTQ_ALTQ_HFSC_H_ */ diff --git a/bsd/net/altq/altq_priq.c b/bsd/net/altq/altq_priq.c deleted file mode 100644 index a86a48383..000000000 --- a/bsd/net/altq/altq_priq.c +++ /dev/null @@ -1,266 +0,0 @@ -/* - * Copyright (c) 2007-2012 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* $OpenBSD: altq_priq.c,v 1.21 2007/09/13 20:40:02 chl Exp $ */ -/* $KAME: altq_priq.c,v 1.1 2000/10/18 09:15:23 kjc Exp $ */ - -/* - * Copyright (C) 2000-2003 - * Sony Computer Science Laboratories Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY SONY CSL AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL SONY CSL OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -/* - * priority queue - */ - -#if PF_ALTQ && PKTSCHED_PRIQ - -#include <sys/cdefs.h> -#include <sys/param.h> -#include <sys/malloc.h> -#include <sys/mbuf.h> -#include <sys/systm.h> -#include <sys/errno.h> -#include <sys/kernel.h> - -#include <net/if.h> -#include <net/pfvar.h> -#include <net/net_osdep.h> -#include <net/altq/altq.h> -#include <net/altq/altq_priq.h> -#include <netinet/in.h> - -/* - * function prototypes - */ -static int altq_priq_enqueue(struct ifaltq *, struct mbuf *); -static struct mbuf *altq_priq_dequeue(struct ifaltq *, enum altdq_op); -static int altq_priq_request(struct ifaltq *, enum altrq, void *); - -int -altq_priq_pfattach(struct pf_altq *a) -{ - struct ifnet *ifp; - int error; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((ifp = ifunit(a->ifname)) == NULL || a->altq_disc == NULL) - return (EINVAL); - - IFCQ_LOCK(&ifp->if_snd); - error = altq_attach(IFCQ_ALTQ(&ifp->if_snd), ALTQT_PRIQ, a->altq_disc, - altq_priq_enqueue, altq_priq_dequeue, NULL, altq_priq_request); - IFCQ_UNLOCK(&ifp->if_snd); - - return (error); -} - -int -altq_priq_add(struct pf_altq *a) -{ - struct priq_if *pif; - struct ifnet *ifp; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((ifp = ifunit(a->ifname)) == NULL) - return (EINVAL); - if (!ALTQ_IS_READY(IFCQ_ALTQ(&ifp->if_snd))) - return (ENODEV); - - pif = priq_alloc(ifp, M_WAITOK, TRUE); - if (pif == NULL) - return (ENOMEM); - - /* keep the state in pf_altq */ - a->altq_disc = pif; - - return (0); -} - -int -altq_priq_remove(struct pf_altq *a) -{ - struct priq_if *pif; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((pif = a->altq_disc) == NULL) - return (EINVAL); - a->altq_disc = NULL; - - return (priq_destroy(pif)); -} - -int -altq_priq_add_queue(struct pf_altq *a) -{ - struct priq_if *pif; - int err; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((pif = a->altq_disc) == NULL) - return (EINVAL); - - IFCQ_LOCK(pif->pif_ifq); - err = priq_add_queue(pif, a->priority, a->qlimit, - a->pq_u.priq_opts.flags, a->qid, NULL); - IFCQ_UNLOCK(pif->pif_ifq); - - return (err); -} - -int -altq_priq_remove_queue(struct pf_altq *a) -{ - struct priq_if *pif; - int err; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((pif = a->altq_disc) == NULL) - return (EINVAL); - - IFCQ_LOCK(pif->pif_ifq); - err = priq_remove_queue(pif, a->qid); - IFCQ_UNLOCK(pif->pif_ifq); - - return (err); -} - -int -altq_priq_getqstats(struct pf_altq *a, void *ubuf, int *nbytes) -{ - struct ifclassq *ifq = NULL; - struct priq_if *pif; - struct priq_classstats stats; - int error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((unsigned)*nbytes < sizeof (stats)) - return (EINVAL); - - if ((pif = altq_lookup(a->ifname, ALTQT_PRIQ)) == NULL) - return (EBADF); - - ifq = pif->pif_ifq; - IFCQ_LOCK_ASSERT_HELD(ifq); /* lock held by altq_lookup */ - error = priq_get_class_stats(pif, a->qid, &stats); - IFCQ_UNLOCK(ifq); - if (error != 0) - return (error); - - if ((error = copyout((caddr_t)&stats, (user_addr_t)(uintptr_t)ubuf, - sizeof (stats))) != 0) - return (error); - - *nbytes = sizeof (stats); - - return (0); -} - -static int -altq_priq_request(struct ifaltq *altq, enum altrq req, void *arg) -{ - struct priq_if *pif = (struct priq_if *)altq->altq_disc; - - switch (req) { - case ALTRQ_PURGE: - priq_purge(pif); - break; - - case ALTRQ_PURGE_SC: - case ALTRQ_THROTTLE: - /* not supported for ALTQ instance */ - break; - - case ALTRQ_EVENT: - priq_event(pif, (cqev_t)arg); - break; - } - return (0); -} - -/* - * altq_priq_enqueue is an enqueue function to be registered to - * (*altq_enqueue) in struct ifaltq. - */ -static int -altq_priq_enqueue(struct ifaltq *altq, struct mbuf *m) -{ - /* grab class set by classifier */ - if (!(m->m_flags & M_PKTHDR)) { - /* should not happen */ - printf("%s: packet for %s does not have pkthdr\n", __func__, - if_name(altq->altq_ifcq->ifcq_ifp)); - m_freem(m); - return (ENOBUFS); - } - - return (priq_enqueue(altq->altq_disc, NULL, m, m_pftag(m))); -} - -/* - * altq_priq_dequeue is a dequeue function to be registered to - * (*altq_dequeue) in struct ifaltq. - * - * note: ALTDQ_POLL returns the next packet without removing the packet - * from the queue. ALTDQ_REMOVE is a normal dequeue operation. - * ALTDQ_REMOVE must return the same packet if called immediately - * after ALTDQ_POLL. - */ -static struct mbuf * -altq_priq_dequeue(struct ifaltq *altq, enum altdq_op op) -{ - return (priq_dequeue(altq->altq_disc, (cqdq_op_t)op)); -} -#endif /* PF_ALTQ && PKTSCHED_PRIQ */ diff --git a/bsd/net/altq/altq_priq.h b/bsd/net/altq/altq_priq.h index f6b6372e9..f1f92e939 100644 --- a/bsd/net/altq/altq_priq.h +++ b/bsd/net/altq/altq_priq.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 Apple Inc. All rights reserved. + * Copyright (c) 2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -61,20 +61,4 @@ #include <net/altq/altq.h> #include <net/pktsched/pktsched_priq.h> -#ifdef BSD_KERNEL_PRIVATE -#ifdef __cplusplus -extern "C" { -#endif - -extern int altq_priq_pfattach(struct pf_altq *); -extern int altq_priq_add(struct pf_altq *); -extern int altq_priq_remove(struct pf_altq *); -extern int altq_priq_add_queue(struct pf_altq *); -extern int altq_priq_remove_queue(struct pf_altq *); -extern int altq_priq_getqstats(struct pf_altq *, void *, int *); - -#ifdef __cplusplus -} -#endif -#endif /* BSD_KERNEL_PRIVATE */ #endif /* _NET_ALTQ_ALTQ_PRIQ_H_ */ diff --git a/bsd/net/altq/altq_qfq.c b/bsd/net/altq/altq_qfq.c deleted file mode 100644 index 0f7c52983..000000000 --- a/bsd/net/altq/altq_qfq.c +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Copyright (c) 2011-2013 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* - * quick fair queueing - */ - -#if PF_ALTQ - -#include <sys/cdefs.h> -#include <sys/param.h> -#include <sys/malloc.h> -#include <sys/mbuf.h> -#include <sys/systm.h> -#include <sys/errno.h> -#include <sys/kernel.h> - -#include <net/if.h> -#include <net/pfvar.h> -#include <net/net_osdep.h> -#include <net/altq/altq.h> -#include <net/altq/altq_qfq.h> -#include <netinet/in.h> - -/* - * function prototypes - */ -static int altq_qfq_enqueue(struct ifaltq *, struct mbuf *); -static struct mbuf *altq_qfq_dequeue(struct ifaltq *, enum altdq_op); -static int altq_qfq_request(struct ifaltq *, enum altrq, void *); - -int -altq_qfq_pfattach(struct pf_altq *a) -{ - struct ifnet *ifp; - int error; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((ifp = ifunit(a->ifname)) == NULL || a->altq_disc == NULL) - return (EINVAL); - - IFCQ_LOCK(&ifp->if_snd); - error = altq_attach(IFCQ_ALTQ(&ifp->if_snd), ALTQT_QFQ, a->altq_disc, - altq_qfq_enqueue, altq_qfq_dequeue, NULL, altq_qfq_request); - IFCQ_UNLOCK(&ifp->if_snd); - - return (error); -} - -int -altq_qfq_add(struct pf_altq *a) -{ - struct qfq_if *qif; - struct ifnet *ifp; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((ifp = ifunit(a->ifname)) == NULL) - return (EINVAL); - if (!ALTQ_IS_READY(IFCQ_ALTQ(&ifp->if_snd))) - return (ENODEV); - - qif = qfq_alloc(ifp, M_WAITOK, TRUE); - if (qif == NULL) - return (ENOMEM); - - /* keep the state in pf_altq */ - a->altq_disc = qif; - - return (0); -} - -int -altq_qfq_remove(struct pf_altq *a) -{ - struct qfq_if *qif; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((qif = a->altq_disc) == NULL) - return (EINVAL); - a->altq_disc = NULL; - - return (qfq_destroy(qif)); -} - -int -altq_qfq_add_queue(struct pf_altq *a) -{ - struct qfq_if *qif; - int err; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((qif = a->altq_disc) == NULL) - return (EINVAL); - - IFCQ_LOCK(qif->qif_ifq); - err = qfq_add_queue(qif, a->qlimit, a->weight, a->pq_u.qfq_opts.lmax, - a->pq_u.qfq_opts.flags, a->qid, NULL); - IFCQ_UNLOCK(qif->qif_ifq); - - return (err); -} - -int -altq_qfq_remove_queue(struct pf_altq *a) -{ - struct qfq_if *qif; - int err; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((qif = a->altq_disc) == NULL) - return (EINVAL); - - IFCQ_LOCK(qif->qif_ifq); - err = qfq_remove_queue(qif, a->qid); - IFCQ_UNLOCK(qif->qif_ifq); - - return (err); -} - -int -altq_qfq_getqstats(struct pf_altq *a, void *ubuf, int *nbytes) -{ - struct ifclassq *ifq = NULL; - struct qfq_if *qif; - struct qfq_classstats stats; - int error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((unsigned)*nbytes < sizeof (stats)) - return (EINVAL); - - if ((qif = altq_lookup(a->ifname, ALTQT_QFQ)) == NULL) - return (EBADF); - - ifq = qif->qif_ifq; - IFCQ_LOCK_ASSERT_HELD(ifq); /* lock held by altq_lookup */ - error = qfq_get_class_stats(qif, a->qid, &stats); - IFCQ_UNLOCK(ifq); - if (error != 0) - return (error); - - if ((error = copyout((caddr_t)&stats, (user_addr_t)(uintptr_t)ubuf, - sizeof (stats))) != 0) - return (error); - - *nbytes = sizeof (stats); - - return (0); -} - -static int -altq_qfq_request(struct ifaltq *altq, enum altrq req, void *arg) -{ - struct qfq_if *qif = (struct qfq_if *)altq->altq_disc; - - switch (req) { - case ALTRQ_PURGE: - qfq_purge(qif); - break; - - case ALTRQ_PURGE_SC: - /* not supported for ALTQ instance */ - break; - - case ALTRQ_EVENT: - qfq_event(qif, (cqev_t)arg); - break; - - case ALTRQ_THROTTLE: - default: - break; - } - return (0); -} - -/* - * altq_qfq_enqueue is an enqueue function to be registered to - * (*altq_enqueue) in struct ifaltq. - */ -static int -altq_qfq_enqueue(struct ifaltq *altq, struct mbuf *m) -{ - /* grab class set by classifier */ - if (!(m->m_flags & M_PKTHDR)) { - /* should not happen */ - printf("%s: packet for %s does not have pkthdr\n", __func__, - if_name(altq->altq_ifcq->ifcq_ifp)); - m_freem(m); - return (ENOBUFS); - } - - return (qfq_enqueue(altq->altq_disc, NULL, m, m_pftag(m))); -} - -/* - * altq_qfq_dequeue is a dequeue function to be registered to - * (*altq_dequeue) in struct ifaltq. - * - * note: ALTDQ_POLL returns the next packet without removing the packet - * from the queue. ALTDQ_REMOVE is a normal dequeue operation. - * ALTDQ_REMOVE must return the same packet if called immediately - * after ALTDQ_POLL. - */ -static struct mbuf * -altq_qfq_dequeue(struct ifaltq *altq, enum altdq_op op) -{ - return (qfq_dequeue(altq->altq_disc, (cqdq_op_t)op)); -} -#endif /* PF_ALTQ */ diff --git a/bsd/net/altq/altq_qfq.h b/bsd/net/altq/altq_qfq.h index 790742229..942fca369 100644 --- a/bsd/net/altq/altq_qfq.h +++ b/bsd/net/altq/altq_qfq.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 Apple Inc. All rights reserved. + * Copyright (c) 2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -33,20 +33,4 @@ #include <net/altq/altq.h> #include <net/pktsched/pktsched_qfq.h> -#ifdef BSD_KERNEL_PRIVATE -#ifdef __cplusplus -extern "C" { -#endif - -extern int altq_qfq_pfattach(struct pf_altq *); -extern int altq_qfq_add(struct pf_altq *); -extern int altq_qfq_remove(struct pf_altq *); -extern int altq_qfq_add_queue(struct pf_altq *); -extern int altq_qfq_remove_queue(struct pf_altq *); -extern int altq_qfq_getqstats(struct pf_altq *, void *, int *); - -#ifdef __cplusplus -} -#endif -#endif /* BSD_KERNEL_PRIVATE */ #endif /* _NET_ALTQ_ALTQ_QFQ_H_ */ diff --git a/bsd/net/altq/altq_subr.c b/bsd/net/altq/altq_subr.c deleted file mode 100644 index 5b00e6f5b..000000000 --- a/bsd/net/altq/altq_subr.c +++ /dev/null @@ -1,487 +0,0 @@ -/* - * Copyright (c) 2007-2011 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* $OpenBSD: altq_subr.c,v 1.24 2007/12/11 00:30:14 mikeb Exp $ */ -/* $KAME: altq_subr.c,v 1.11 2002/01/11 08:11:49 kjc Exp $ */ - -/* - * Copyright (C) 1997-2003 - * Sony Computer Science Laboratories Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY SONY CSL AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL SONY CSL OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include <sys/cdefs.h> - -#include <sys/param.h> -#include <sys/malloc.h> -#include <sys/mbuf.h> -#include <sys/systm.h> -#include <sys/proc.h> -#include <sys/socket.h> -#include <sys/socketvar.h> -#include <sys/kernel.h> -#include <sys/errno.h> -#include <sys/syslog.h> -#include <sys/sysctl.h> -#include <sys/queue.h> -#include <sys/mcache.h> - -#include <net/if.h> -#include <net/if_var.h> -#include <net/if_dl.h> -#include <net/if_types.h> -#include <net/pfvar.h> -#include <net/altq/altq.h> -#include <net/pktsched/pktsched.h> - -#include <pexpert/pexpert.h> - -SYSCTL_NODE(_net, OID_AUTO, altq, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "ALTQ"); - -static u_int32_t altq_debug; -SYSCTL_UINT(_net_altq, OID_AUTO, debug, CTLFLAG_RW, &altq_debug, 0, - "Enable ALTQ debugging"); - -/* - * look up the queue state by the interface name and the queueing type; - * upon success, returns with the interface send queue lock held, and - * the caller is responsible for releasing it. - */ -void * -altq_lookup(char *name, u_int32_t type) -{ - struct ifnet *ifp; - void *state = NULL; - - if ((ifp = ifunit(name)) != NULL) { - IFCQ_LOCK(&ifp->if_snd); - if (type != ALTQT_NONE && - IFCQ_ALTQ(&ifp->if_snd)->altq_type == type) - state = IFCQ_ALTQ(&ifp->if_snd)->altq_disc; - if (state == NULL) - IFCQ_UNLOCK(&ifp->if_snd); - } - - if (state != NULL) - IFCQ_LOCK_ASSERT_HELD(&ifp->if_snd); - - return (state); -} - -int -altq_attach(struct ifaltq *altq, u_int32_t type, void *discipline, - altq_enq_func enqueue, altq_deq_func dequeue, - altq_deq_sc_func dequeue_sc, altq_req_func request) -{ - IFCQ_LOCK_ASSERT_HELD(altq->altq_ifcq); - - if (!ALTQ_IS_READY(altq)) - return (ENXIO); - - VERIFY(enqueue != NULL); - VERIFY(!(dequeue != NULL && dequeue_sc != NULL)); - VERIFY(request != NULL); - - altq->altq_type = type; - altq->altq_disc = discipline; - altq->altq_enqueue = enqueue; - altq->altq_dequeue = dequeue; - altq->altq_dequeue_sc = dequeue_sc; - altq->altq_request = request; - altq->altq_flags &= (ALTQF_CANTCHANGE|ALTQF_ENABLED); - - return (0); -} - -int -altq_detach(struct ifaltq *altq) -{ - IFCQ_LOCK_ASSERT_HELD(altq->altq_ifcq); - - if (!ALTQ_IS_READY(altq)) - return (ENXIO); - if (ALTQ_IS_ENABLED(altq)) - return (EBUSY); - if (!ALTQ_IS_ATTACHED(altq)) - return (0); - - altq->altq_type = ALTQT_NONE; - altq->altq_disc = NULL; - altq->altq_enqueue = NULL; - altq->altq_dequeue = NULL; - altq->altq_dequeue_sc = NULL; - altq->altq_request = NULL; - altq->altq_flags &= ALTQF_CANTCHANGE; - - return (0); -} - -int -altq_enable(struct ifaltq *altq) -{ - struct ifclassq *ifq = altq->altq_ifcq; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - if (!ALTQ_IS_READY(altq)) - return (ENXIO); - if (ALTQ_IS_ENABLED(altq)) - return (0); - - altq->altq_flags |= ALTQF_ENABLED; - - return (0); -} - -int -altq_disable(struct ifaltq *altq) -{ - struct ifclassq *ifq = altq->altq_ifcq; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - if (!ALTQ_IS_ENABLED(altq)) - return (0); - - if_qflush(ifq->ifcq_ifp, 1); - - altq->altq_flags &= ~ALTQF_ENABLED; - - return (0); -} - -/* - * add a discipline or a queue - */ -int -altq_add(struct pf_altq *a) -{ - int error = 0; - - VERIFY(machclk_freq != 0); - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if (a->qname[0] != 0) - return (altq_add_queue(a)); - - switch (a->scheduler) { -#if PKTSCHED_CBQ - case ALTQT_CBQ: - error = altq_cbq_add(a); - break; -#endif /* PKTSCHED_CBQ */ -#if PKTSCHED_PRIQ - case ALTQT_PRIQ: - error = altq_priq_add(a); - break; -#endif /* PKTSCHED_PRIQ */ -#if PKTSCHED_HFSC - case ALTQT_HFSC: - error = altq_hfsc_add(a); - break; -#endif /* PKTSCHED_HFSC */ -#if PKTSCHED_FAIRQ - case ALTQT_FAIRQ: - error = altq_fairq_add(a); - break; -#endif /* PKTSCHED_FAIRQ */ - case ALTQT_QFQ: - error = altq_qfq_add(a); - break; - default: - error = ENXIO; - } - - return (error); -} - -/* - * remove a discipline or a queue - */ -int -altq_remove(struct pf_altq *a) -{ - int error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if (a->qname[0] != 0) - return (altq_remove_queue(a)); - - switch (a->scheduler) { -#if PKTSCHED_CBQ - case ALTQT_CBQ: - error = altq_cbq_remove(a); - break; -#endif /* PKTSCHED_CBQ */ -#if PKTSCHED_PRIQ - case ALTQT_PRIQ: - error = altq_priq_remove(a); - break; -#endif /* PKTSCHED_PRIQ */ -#if PKTSCHED_HFSC - case ALTQT_HFSC: - error = altq_hfsc_remove(a); - break; -#endif /* PKTSCHED_HFSC */ -#if PKTSCHED_FAIRQ - case ALTQT_FAIRQ: - error = altq_fairq_remove(a); - break; -#endif /* PKTSCHED_FAIRQ */ - case ALTQT_QFQ: - error = altq_qfq_remove(a); - break; - default: - error = ENXIO; - } - - return (error); -} - -/* - * add a queue to the discipline - */ -int -altq_add_queue(struct pf_altq *a) -{ - int error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - switch (a->scheduler) { -#if PKTSCHED_CBQ - case ALTQT_CBQ: - error = altq_cbq_add_queue(a); - break; -#endif /* PKTSCHED_CBQ */ -#if PKTSCHED_PRIQ - case ALTQT_PRIQ: - error = altq_priq_add_queue(a); - break; -#endif /* PKTSCHED_PRIQ */ -#if PKTSCHED_HFSC - case ALTQT_HFSC: - error = altq_hfsc_add_queue(a); - break; -#endif /* PKTSCHED_HFSC */ -#if PKTSCHED_FAIRQ - case ALTQT_FAIRQ: - error = altq_fairq_add_queue(a); - break; -#endif /* PKTSCHED_FAIRQ */ - case ALTQT_QFQ: - error = altq_qfq_add_queue(a); - break; - default: - error = ENXIO; - } - - return (error); -} - -/* - * remove a queue from the discipline - */ -int -altq_remove_queue(struct pf_altq *a) -{ - int error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - switch (a->scheduler) { -#if PKTSCHED_CBQ - case ALTQT_CBQ: - error = altq_cbq_remove_queue(a); - break; -#endif /* PKTSCHED_CBQ */ -#if PKTSCHED_PRIQ - case ALTQT_PRIQ: - error = altq_priq_remove_queue(a); - break; -#endif /* PKTSCHED_PRIQ */ -#if PKTSCHED_HFSC - case ALTQT_HFSC: - error = altq_hfsc_remove_queue(a); - break; -#endif /* PKTSCHED_HFSC */ -#if PKTSCHED_FAIRQ - case ALTQT_FAIRQ: - error = altq_fairq_remove_queue(a); - break; -#endif /* PKTSCHED_FAIRQ */ - case ALTQT_QFQ: - error = altq_qfq_remove_queue(a); - break; - default: - error = ENXIO; - } - - return (error); -} - -/* - * get queue statistics - */ -int -altq_getqstats(struct pf_altq *a, void *ubuf, int *nbytes) -{ - int error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - switch (a->scheduler) { -#if PKTSCHED_CBQ - case ALTQT_CBQ: - error = altq_cbq_getqstats(a, ubuf, nbytes); - break; -#endif /* PKTSCHED_CBQ */ -#if PKTSCHED_PRIQ - case ALTQT_PRIQ: - error = altq_priq_getqstats(a, ubuf, nbytes); - break; -#endif /* PKTSCHED_PRIQ */ -#if PKTSCHED_HFSC - case ALTQT_HFSC: - error = altq_hfsc_getqstats(a, ubuf, nbytes); - break; -#endif /* PKTSCHED_HFSC */ -#if PKTSCHED_FAIRQ - case ALTQT_FAIRQ: - error = altq_fairq_getqstats(a, ubuf, nbytes); - break; -#endif /* PKTSCHED_FAIRQ */ - case ALTQT_QFQ: - error = altq_qfq_getqstats(a, ubuf, nbytes); - break; - default: - error = ENXIO; - } - - return (error); -} - -/* - * attach a discipline to the interface. if one already exists, it is - * overridden. - */ -int -altq_pfattach(struct pf_altq *a) -{ - int error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - switch (a->scheduler) { - case ALTQT_NONE: - break; -#if PKTSCHED_CBQ - case ALTQT_CBQ: - error = altq_cbq_pfattach(a); - break; -#endif /* PKTSCHED_CBQ */ -#if PKTSCHED_PRIQ - case ALTQT_PRIQ: - error = altq_priq_pfattach(a); - break; -#endif /* PKTSCHED_PRIQ */ -#if PKTSCHED_HFSC - case ALTQT_HFSC: - error = altq_hfsc_pfattach(a); - break; -#endif /* PKTSCHED_HFSC */ -#if PKTSCHED_FAIRQ - case ALTQT_FAIRQ: - error = altq_fairq_pfattach(a); - break; -#endif /* PKTSCHED_FAIRQ */ - case ALTQT_QFQ: - error = altq_qfq_pfattach(a); - break; - default: - error = ENXIO; - } - - return (error); -} - -/* - * detach a discipline from the interface. - * it is possible that the discipline was already overridden by another - * discipline. - */ -int -altq_pfdetach(struct pf_altq *a) -{ - struct ifnet *ifp; - int error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((ifp = ifunit(a->ifname)) == NULL) - return (EINVAL); - - /* if this discipline is no longer referenced, just return */ - IFCQ_LOCK(&ifp->if_snd); - if (a->altq_disc == NULL || - a->altq_disc != IFCQ_ALTQ(&ifp->if_snd)->altq_disc) { - IFCQ_UNLOCK(&ifp->if_snd); - return (0); - } - - if (ALTQ_IS_ENABLED(IFCQ_ALTQ(&ifp->if_snd))) - error = altq_disable(IFCQ_ALTQ(&ifp->if_snd)); - if (error == 0) - error = altq_detach(IFCQ_ALTQ(&ifp->if_snd)); - IFCQ_UNLOCK(&ifp->if_snd); - return (error); -} - - diff --git a/bsd/net/altq/altq_var.h b/bsd/net/altq/altq_var.h deleted file mode 100644 index e866a4dc4..000000000 --- a/bsd/net/altq/altq_var.h +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright (c) 2011 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* $NetBSD: altq_var.h,v 1.10 2006/10/15 13:17:13 peter Exp $ */ -/* $KAME: altq_var.h,v 1.18 2005/04/13 03:44:25 suz Exp $ */ - -/* - * Copyright (C) 1998-2003 - * Sony Computer Science Laboratories Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY SONY CSL AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL SONY CSL OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ -#ifndef _NET_ALTQ_ALTQ_VAR_H_ -#define _NET_ALTQ_ALTQ_VAR_H_ - -#ifdef BSD_KERNEL_PRIVATE -#if PF_ALTQ -#include <sys/param.h> -#include <sys/kernel.h> -#include <sys/queue.h> -#include <net/pktsched/pktsched.h> -#include <net/classq/classq.h> -#include <net/altq/if_altq.h> -#if PKTSCHED_HFSC -#include <net/altq/altq_hfsc.h> -#endif /* PKTSCHED_HFSC */ -#if PKTSCHED_FAIRQ -#include <net/altq/altq_fairq.h> -#endif /* PKTSCHED_FAIRQ */ -#if PKTSCHED_CBQ -#include <net/altq/altq_cbq.h> -#endif /* PKTSCHED_CBQ */ -#if PKTSCHED_PRIQ -#include <net/altq/altq_priq.h> -#endif /* PKTSCHED_PRIQ */ -#include <net/altq/altq_qfq.h> - -struct pf_altq; - -extern void *altq_lookup(char *, u_int32_t); -extern int altq_pfattach(struct pf_altq *); -extern int altq_pfdetach(struct pf_altq *); -extern int altq_add(struct pf_altq *); -extern int altq_remove(struct pf_altq *); -extern int altq_add_queue(struct pf_altq *); -extern int altq_remove_queue(struct pf_altq *); -extern int altq_getqstats(struct pf_altq *, void *, int *); - -#endif /* PF_ALTQ */ -#endif /* BSD_KERNEL_PRIVATE */ -#endif /* _NET_ALTQ_ALTQ_VAR_H_ */ diff --git a/bsd/net/altq/if_altq.h b/bsd/net/altq/if_altq.h deleted file mode 100644 index 6d634cf5d..000000000 --- a/bsd/net/altq/if_altq.h +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright (c) 2008-2012 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ -/* $OpenBSD: if_altq.h,v 1.11 2007/11/18 12:51:48 mpf Exp $ */ -/* $KAME: if_altq.h,v 1.6 2001/01/29 19:59:09 itojun Exp $ */ - -/* - * Copyright (C) 1997-2003 - * Sony Computer Science Laboratories Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY SONY CSL AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL SONY CSL OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ -#ifndef _NET_ALTQ_IF_ALTQ_H_ -#define _NET_ALTQ_IF_ALTQ_H_ - -#ifdef BSD_KERNEL_PRIVATE -#if PF_ALTQ -#include <net/classq/if_classq.h> - -/* altq request types */ -typedef enum altrq { - ALTRQ_PURGE = CLASSQRQ_PURGE, /* purge all packets */ - ALTRQ_PURGE_SC = CLASSQRQ_PURGE_SC, /* purge SC flow */ - ALTRQ_EVENT = CLASSQRQ_EVENT, /* interface events */ - ALTRQ_THROTTLE = CLASSQRQ_THROTTLE, /* throttle packets */ -} altrq_t; - -struct ifaltq; -enum altdq_op; - -typedef int (*altq_enq_func)(struct ifaltq *, struct mbuf *); -typedef struct mbuf *(*altq_deq_func)(struct ifaltq *, enum altdq_op); -typedef struct mbuf *(*altq_deq_sc_func)(struct ifaltq *, - mbuf_svc_class_t, enum altdq_op); -typedef int (*altq_req_func)(struct ifaltq *, enum altrq, void *); - -/* - * Structure defining a queue for a network interface. - */ -struct ifaltq { - struct ifclassq *altq_ifcq; /* back pointer to interface queue */ - - /* alternate queueing related fields */ - u_int32_t altq_type; /* discipline type */ - u_int32_t altq_flags; /* flags (e.g. ready, in-use) */ - void *altq_disc; /* for discipline-specific use */ - - altq_enq_func altq_enqueue; - altq_deq_func altq_dequeue; - altq_deq_sc_func altq_dequeue_sc; - altq_req_func altq_request; -}; - -/* altq_flags */ -#define ALTQF_READY 0x01 /* driver supports alternate queueing */ -#define ALTQF_ENABLED 0x02 /* altq is in use */ -#define ALTQF_DRIVER1 0x40 /* driver specific */ - -/* altq_flags set internally only: */ -#define ALTQF_CANTCHANGE (ALTQF_READY) - -/* altq_dequeue op arg */ -typedef enum altdq_op { - ALTDQ_REMOVE = CLASSQDQ_REMOVE, /* dequeue mbuf from the queue */ - ALTDQ_POLL = CLASSQDQ_POLL, /* don't dequeue mbuf from the queue */ -} altdq_op_t; - -#define ALTQ_IS_READY(_altq) ((_altq)->altq_flags & ALTQF_READY) -#define ALTQ_IS_ENABLED(_altq) ((_altq)->altq_flags & ALTQF_ENABLED) -#define ALTQ_IS_ATTACHED(_altq) ((_altq)->altq_disc != NULL) - -#define ALTQ_ENQUEUE(_altq, _m, _err) do { \ - (_err) = (*(_altq)->altq_enqueue)(_altq, _m); \ -} while (0) - -#define ALTQ_DEQUEUE(_altq, _m) do { \ - (_m) = (*(_altq)->altq_dequeue)(_altq, ALTDQ_REMOVE); \ -} while (0) - -#define ALTQ_DEQUEUE_SC(_altq, _sc, _m) do { \ - (_m) = (*(_altq)->altq_dequeue_sc)(_altq, _sc, ALTDQ_REMOVE); \ -} while (0) - -#define ALTQ_POLL(_altq, _m) do { \ - (_m) = (*(_altq)->altq_dequeue)(_altq, ALTDQ_POLL); \ -} while (0) - -#define ALTQ_POLL_SC(_altq, _sc, _m) do { \ - (_m) = (*(_altq)->altq_dequeue_sc)(_altq, _sc, ALTDQ_POLL); \ -} while (0) - -#define ALTQ_PURGE(_altq) do { \ - (void) (*(_altq)->altq_request)(_altq, ALTRQ_PURGE, NULL); \ -} while (0) - -#define ALTQ_PURGE_SC(_altq, _sc, _flow, _packets, _bytes) do { \ - cqrq_purge_sc_t _req = { _sc, _flow, 0, 0 }; \ - (void) (*(_altq)->altq_request)(_altq, ALTRQ_PURGE_SC, &_req); \ - (_packets) = _req.packets; \ - (_bytes) = _req.bytes; \ -} while (0) - -#define ALTQ_UPDATE(_altq, _ev) do { \ - (void) (*(_altq)->altq_request)(_altq, ALTRQ_EVENT, \ - (void *)(_ev)); \ -} while (0) - -#define ALTQ_SET_READY(_altq) do { \ - IFCQ_LOCK_ASSERT_HELD((_altq)->altq_ifcq); \ - (_altq)->altq_flags |= ALTQF_READY; \ -} while (0) - -#define ALTQ_CLEAR_READY(_altq) do { \ - IFCQ_LOCK_ASSERT_HELD((_altq)->altq_ifcq); \ - (_altq)->altq_flags &= ~ALTQF_READY; \ -} while (0) - -extern int altq_attach(struct ifaltq *, u_int32_t, void *, - altq_enq_func, altq_deq_func, altq_deq_sc_func, altq_req_func); -extern int altq_detach(struct ifaltq *); -extern int altq_enable(struct ifaltq *); -extern int altq_disable(struct ifaltq *); -#endif /* PF_ALTQ */ -#endif /* BSD_KERNEL_PRIVATE */ -#endif /* _NET_ALTQ_IF_ALTQ_H_ */ diff --git a/bsd/net/bpf.c b/bsd/net/bpf.c index 66e0c3b54..3bf795149 100644 --- a/bsd/net/bpf.c +++ b/bsd/net/bpf.c @@ -126,6 +126,7 @@ #include <kern/locks.h> #include <kern/thread_call.h> +#include <libkern/section_keywords.h> #if CONFIG_MACF_NET #include <security/mac_framework.h> @@ -139,6 +140,8 @@ extern int tvtohz(struct timeval *); #define PRINET 26 /* interruptible */ +typedef void (*pktcopyfunc_t)(const void *, void *, size_t); + /* * The default read buffer size is patchable. */ @@ -156,7 +159,11 @@ SYSCTL_UINT(_debug, OID_AUTO, bpf_maxdevices, CTLFLAG_RW | CTLFLAG_LOCKED, * For OS X is off by default so process need to use the ioctl BPF_WANT_PKTAP * explicitly to be able to use DLT_PKTAP. */ +#if CONFIG_EMBEDDED +static unsigned int bpf_wantpktap = 1; +#else static unsigned int bpf_wantpktap = 0; +#endif SYSCTL_UINT(_debug, OID_AUTO, bpf_wantpktap, CTLFLAG_RW | CTLFLAG_LOCKED, &bpf_wantpktap, 0, ""); @@ -190,21 +197,18 @@ static lck_grp_t *bpf_mlock_grp; static lck_grp_attr_t *bpf_mlock_grp_attr; static lck_attr_t *bpf_mlock_attr; -static mbuf_tag_id_t bpf_mtag_id; #endif /* __APPLE__ */ static int bpf_allocbufs(struct bpf_d *); static errno_t bpf_attachd(struct bpf_d *d, struct bpf_if *bp); static int bpf_detachd(struct bpf_d *d, int); static void bpf_freed(struct bpf_d *); -static void bpf_mcopy(const void *, void *, size_t); static int bpf_movein(struct uio *, int, struct mbuf **, struct sockaddr *, int *); -static int bpf_setif(struct bpf_d *, ifnet_t ifp, u_int32_t dlt); +static int bpf_setif(struct bpf_d *, ifnet_t ifp); static void bpf_timed_out(void *, void *); static void bpf_wakeup(struct bpf_d *); -static void catchpacket(struct bpf_d *, u_char *, struct mbuf *, u_int, - u_int, int, void (*)(const void *, void *, size_t)); +static void catchpacket(struct bpf_d *, struct bpf_packet *, u_int, int); static void reset_d(struct bpf_d *); static int bpf_setf(struct bpf_d *, u_int, user_addr_t, u_long); static int bpf_getdltlist(struct bpf_d *, caddr_t, struct proc *); @@ -503,26 +507,18 @@ bpf_attachd(struct bpf_d *d, struct bpf_if *bp) struct bpf_if *tmp, *primary = NULL; for (tmp = bpf_iflist; tmp; tmp = tmp->bif_next) { - if (tmp->bif_ifp != bp->bif_ifp) - continue; - primary = tmp; - /* - * Make DLT_PKTAP only if process knows how - * to deal with it, otherwise find another one - */ - if (tmp->bif_dlt == DLT_PKTAP && - !(d->bd_flags & BPF_WANT_PKTAP)) - continue; - break; + if (tmp->bif_ifp == bp->bif_ifp) { + primary = tmp; + break; + } } bp->bif_ifp->if_bpf = primary; } - /* Only call dlil_set_bpf_tap for primary dlt */ if (bp->bif_ifp->if_bpf == bp) - dlil_set_bpf_tap(bp->bif_ifp, BPF_TAP_INPUT_OUTPUT, bpf_tap_callback); - - if (bp->bif_tap) + dlil_set_bpf_tap(bp->bif_ifp, BPF_TAP_INPUT_OUTPUT, bpf_tap_callback); + + if (bp->bif_tap != NULL) error = bp->bif_tap(bp->bif_ifp, bp->bif_dlt, BPF_TAP_INPUT_OUTPUT); } @@ -531,12 +527,11 @@ bpf_attachd(struct bpf_d *d, struct bpf_if *bp) */ d->bd_flags &= ~(BPF_DETACHING | BPF_DETACHED); - if (bp->bif_ifp->if_bpf != NULL && - bp->bif_ifp->if_bpf->bif_dlt == DLT_PKTAP) + if (bp->bif_dlt == DLT_PKTAP) { d->bd_flags |= BPF_FINALIZE_PKTAP; - else + } else { d->bd_flags &= ~BPF_FINALIZE_PKTAP; - + } return error; } @@ -699,7 +694,7 @@ bpf_acquire_d(struct bpf_d *d) { void *lr_saved = __builtin_return_address(0); - lck_mtx_assert(bpf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(bpf_mlock, LCK_MTX_ASSERT_OWNED); d->bd_refcnt += 1; @@ -712,7 +707,7 @@ bpf_release_d(struct bpf_d *d) { void *lr_saved = __builtin_return_address(0); - lck_mtx_assert(bpf_mlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(bpf_mlock, LCK_MTX_ASSERT_OWNED); if (d->bd_refcnt <= 0) panic("%s: %p refcnt <= 0", __func__, d); @@ -1056,7 +1051,7 @@ bpfread(dev_t dev, struct uio *uio, int ioflag) } if (error == EINTR || error == ERESTART) { - if (d->bd_hbuf) { + if (d->bd_hbuf != NULL) { /* * Because we msleep, the hold buffer might * be filled when we wake up. Avoid rotating @@ -1064,7 +1059,7 @@ bpfread(dev_t dev, struct uio *uio, int ioflag) */ break; } - if (d->bd_slen) { + if (d->bd_slen != 0) { /* * Sometimes we may be interrupted often and * the sleep above will not timeout. @@ -1077,6 +1072,11 @@ bpfread(dev_t dev, struct uio *uio, int ioflag) } bpf_release_d(d); lck_mtx_unlock(bpf_mlock); + if (error == ERESTART) { + printf("%s: %llx ERESTART to EINTR\n", + __func__, (uint64_t)VM_KERNEL_ADDRPERM(d)); + error = EINTR; + } return (error); } if (error == EWOULDBLOCK) { @@ -1147,6 +1147,7 @@ bpfread(dev_t dev, struct uio *uio, int ioflag) } ehp->bh_flowid = 0; } + if (flags & BPF_FINALIZE_PKTAP) { struct pktap_header *pktaphdr; @@ -1617,6 +1618,11 @@ bpfioctl(dev_t dev, u_long cmd, caddr_t addr, __unused int flags, u_int dlt; bcopy(addr, &dlt, sizeof (dlt)); + + if (dlt == DLT_PKTAP && !(d->bd_flags & BPF_WANT_PKTAP)) { + printf("BIOCSDLT downgrade DLT_PKTAP to DLT_RAW\n"); + dlt = DLT_RAW; + } error = bpf_setdlt(d, dlt); } break; @@ -1647,7 +1653,7 @@ bpfioctl(dev_t dev, u_long cmd, caddr_t addr, __unused int flags, if (ifp == NULL) error = ENXIO; else - error = bpf_setif(d, ifp, 0); + error = bpf_setif(d, ifp); break; } @@ -1949,7 +1955,7 @@ bpf_setf(struct bpf_d *d, u_int bf_len, user_addr_t bf_insns, * Return an errno or 0. */ static int -bpf_setif(struct bpf_d *d, ifnet_t theywant, u_int32_t dlt) +bpf_setif(struct bpf_d *d, ifnet_t theywant) { struct bpf_if *bp; int error; @@ -1966,14 +1972,12 @@ bpf_setif(struct bpf_d *d, ifnet_t theywant, u_int32_t dlt) for (bp = bpf_iflist; bp != 0; bp = bp->bif_next) { struct ifnet *ifp = bp->bif_ifp; - if (ifp == 0 || ifp != theywant || (dlt != 0 && dlt != bp->bif_dlt)) + if (ifp == 0 || ifp != theywant) continue; /* - * If the process knows how to deal with DLT_PKTAP, use it - * by default + * Do not use DLT_PKTAP, unless requested explicitly */ - if (dlt == 0 && bp->bif_dlt == DLT_PKTAP && - !(d->bd_flags & BPF_WANT_PKTAP)) + if (bp->bif_dlt == DLT_PKTAP && !(d->bd_flags & BPF_WANT_PKTAP)) continue; /* * We found the requested interface. @@ -2030,7 +2034,7 @@ bpf_getdltlist(struct bpf_d *d, caddr_t addr, struct proc *p) if (bp->bif_ifp != ifp) continue; /* - * Return DLT_PKTAP only to processes that know how to handle it + * Do not use DLT_PKTAP, unless requested explicitly */ if (bp->bif_dlt == DLT_PKTAP && !(d->bd_flags & BPF_WANT_PKTAP)) continue; @@ -2073,8 +2077,15 @@ bpf_setdlt(struct bpf_d *d, uint32_t dlt) ifp = d->bd_bif->bif_ifp; for (bp = bpf_iflist; bp; bp = bp->bif_next) { - if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) + if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) { + /* + * Do not use DLT_PKTAP, unless requested explicitly + */ + if (bp->bif_dlt == DLT_PKTAP && !(d->bd_flags & BPF_WANT_PKTAP)) { + continue; + } break; + } } if (bp != NULL) { opromisc = d->bd_promisc; @@ -2202,7 +2213,7 @@ static int filt_bpfread(struct knote *, long); static int filt_bpftouch(struct knote *kn, struct kevent_internal_s *kev); static int filt_bpfprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); -struct filterops bpfread_filtops = { +SECURITY_READ_ONLY_EARLY(struct filterops) bpfread_filtops = { .f_isfd = 1, .f_detach = filt_bpfdetach, .f_event = filt_bpfread, @@ -2373,12 +2384,11 @@ filt_bpfprocess(struct knote *kn, struct filt_process_s *data, struct kevent_int /* * Copy data from an mbuf chain into a buffer. This code is derived - * from m_copydata in sys/uipc_mbuf.c. + * from m_copydata in kern/uipc_mbuf.c. */ static void -bpf_mcopy(const void *src_arg, void *dst_arg, size_t len) +bpf_mcopy(struct mbuf * m, void *dst_arg, size_t len) { - struct mbuf *m = (struct mbuf *)(uintptr_t)(src_arg); u_int count; u_char *dst; @@ -2398,19 +2408,18 @@ static inline void bpf_tap_imp( ifnet_t ifp, u_int32_t dlt, - mbuf_t m, - void* hdr, - size_t hlen, + struct bpf_packet *bpf_pkt, int outbound) { + struct bpf_d *d; + u_int slen; struct bpf_if *bp; - struct mbuf *savedm = m; /* * It's possible that we get here after the bpf descriptor has been * detached from the interface; in such a case we simply return. * Lock ordering is important since we can be called asynchronously - * (from the IOKit) to process an inbound packet; when that happens + * (from IOKit) to process an inbound packet; when that happens * we would have been holding its "gateLock" and will be acquiring * "bpf_mlock" upon entering this routine. Due to that, we release * "bpf_mlock" prior to calling ifnet_set_promiscuous (which will @@ -2423,62 +2432,70 @@ bpf_tap_imp( lck_mtx_unlock(bpf_mlock); return; } - bp = ifp->if_bpf; - for (bp = ifp->if_bpf; bp && bp->bif_ifp == ifp && - (dlt != 0 && bp->bif_dlt != dlt); bp = bp->bif_next) - ; - if (bp && bp->bif_ifp == ifp && bp->bif_dlist != NULL) { - struct bpf_d *d; - struct m_hdr hack_hdr; - u_int pktlen = 0; - u_int slen = 0; - struct mbuf *m0; - - if (hdr) { - /* - * This is gross. We mock up an mbuf that points to the - * header buffer. This means we don't have to copy the - * header. A number of interfaces prepended headers just - * for bpf by allocating an mbuf on the stack. We want to - * give developers an easy way to prepend a header for bpf. - * Since a developer allocating an mbuf on the stack is bad, - * we do even worse here, allocating only a header to point - * to a buffer the developer supplied. This makes assumptions - * that bpf_filter and catchpacket will not look at anything - * in the mbuf other than the header. This was true at the - * time this code was written. - */ - hack_hdr.mh_next = m; - hack_hdr.mh_nextpkt = NULL; - hack_hdr.mh_len = hlen; - hack_hdr.mh_data = hdr; - hack_hdr.mh_type = m->m_type; - hack_hdr.mh_flags = 0; - - __IGNORE_WCASTALIGN(m = (mbuf_t)&hack_hdr); + for (bp = ifp->if_bpf; bp != NULL; bp = bp->bif_next) { + if (bp->bif_ifp != ifp) { + /* wrong interface */ + bp = NULL; + break; } - - for (m0 = m; m0 != 0; m0 = m0->m_next) - pktlen += m0->m_len; - - for (d = bp->bif_dlist; d; d = d->bd_next) { - if (outbound && !d->bd_seesent) - continue; - ++d->bd_rcount; - slen = bpf_filter(d->bd_filter, (u_char *)m, pktlen, 0); - if (slen != 0) { + if (dlt == 0 || bp->bif_dlt == dlt) { + /* tapping default DLT or DLT matches */ + break; + } + } + if (bp == NULL) { + goto done; + } + for (d = bp->bif_dlist; d; d = d->bd_next) { + if (outbound && !d->bd_seesent) + continue; + ++d->bd_rcount; + slen = bpf_filter(d->bd_filter, (u_char *)bpf_pkt, + bpf_pkt->bpfp_total_length, 0); + if (slen != 0) { #if CONFIG_MACF_NET - if (mac_bpfdesc_check_receive(d, bp->bif_ifp) != 0) - continue; + if (mac_bpfdesc_check_receive(d, bp->bif_ifp) != 0) + continue; #endif - catchpacket(d, (u_char *)m, savedm, pktlen, - slen, outbound, bpf_mcopy); - } + catchpacket(d, bpf_pkt, slen, outbound); } } + + done: lck_mtx_unlock(bpf_mlock); } +static inline void +bpf_tap_mbuf( + ifnet_t ifp, + u_int32_t dlt, + mbuf_t m, + void* hdr, + size_t hlen, + int outbound) +{ + struct bpf_packet bpf_pkt; + struct mbuf *m0; + + if (ifp->if_bpf == NULL) { + /* quickly check without taking lock */ + return; + } + bpf_pkt.bpfp_type = BPF_PACKET_TYPE_MBUF; + bpf_pkt.bpfp_mbuf = m; + bpf_pkt.bpfp_total_length = 0; + for (m0 = m; m0 != NULL; m0 = m0->m_next) + bpf_pkt.bpfp_total_length += m0->m_len; + bpf_pkt.bpfp_header = hdr; + if (hdr != NULL) { + bpf_pkt.bpfp_total_length += hlen; + bpf_pkt.bpfp_header_length = hlen; + } else { + bpf_pkt.bpfp_header_length = 0; + } + bpf_tap_imp(ifp, dlt, &bpf_pkt, outbound); +} + void bpf_tap_out( ifnet_t ifp, @@ -2487,7 +2504,7 @@ bpf_tap_out( void* hdr, size_t hlen) { - bpf_tap_imp(ifp, dlt, m, hdr, hlen, 1); + bpf_tap_mbuf(ifp, dlt, m, hdr, hlen, 1); } void @@ -2498,29 +2515,50 @@ bpf_tap_in( void* hdr, size_t hlen) { - bpf_tap_imp(ifp, dlt, m, hdr, hlen, 0); + bpf_tap_mbuf(ifp, dlt, m, hdr, hlen, 0); } /* Callback registered with Ethernet driver. */ static int bpf_tap_callback(struct ifnet *ifp, struct mbuf *m) { - bpf_tap_imp(ifp, 0, m, NULL, 0, mbuf_pkthdr_rcvif(m) == NULL); + bpf_tap_mbuf(ifp, 0, m, NULL, 0, mbuf_pkthdr_rcvif(m) == NULL); return 0; } + +static void +copy_bpf_packet(struct bpf_packet * pkt, void * dst, size_t len) +{ + /* copy the optional header */ + if (pkt->bpfp_header_length != 0) { + size_t count = min(len, pkt->bpfp_header_length); + bcopy(pkt->bpfp_header, dst, count); + len -= count; + dst += count; + } + if (len == 0) { + /* nothing past the header */ + return; + } + /* copy the packet */ + switch (pkt->bpfp_type) { + case BPF_PACKET_TYPE_MBUF: + bpf_mcopy(pkt->bpfp_mbuf, dst, len); + break; + default: + break; + } +} + /* * Move the packet data from interface memory (pkt) into the * store buffer. Return 1 if it's time to wakeup a listener (buffer full), - * otherwise 0. "copy" is the routine called to do the actual data - * transfer. bcopy is passed in to copy contiguous chunks, while - * bpf_mcopy is passed in to copy mbuf chains. In the latter case, - * pkt is really an mbuf. + * otherwise 0. */ static void -catchpacket(struct bpf_d *d, u_char *pkt, struct mbuf *m, u_int pktlen, - u_int snaplen, int outbound, - void (*cpfn)(const void *, void *, size_t)) +catchpacket(struct bpf_d *d, struct bpf_packet * pkt, + u_int snaplen, int outbound) { struct bpf_hdr *hp; struct bpf_hdr_ext *ehp; @@ -2529,8 +2567,6 @@ catchpacket(struct bpf_d *d, u_char *pkt, struct mbuf *m, u_int pktlen, int do_wakeup = 0; u_char *payload; struct timeval tv; - struct m_tag *mt = NULL; - struct bpf_mtag *bt = NULL; hdrlen = (d->bd_flags & BPF_EXTENDED_HDR) ? d->bd_bif->bif_exthdrlen : d->bd_bif->bif_hdrlen; @@ -2540,7 +2576,7 @@ catchpacket(struct bpf_d *d, u_char *pkt, struct mbuf *m, u_int pktlen, * much. Otherwise, transfer the whole packet (unless * we hit the buffer size limit). */ - totlen = hdrlen + min(snaplen, pktlen); + totlen = hdrlen + min(snaplen, pkt->bpfp_total_length); if (totlen > d->bd_bufsize) totlen = d->bd_bufsize; @@ -2596,26 +2632,27 @@ catchpacket(struct bpf_d *d, u_char *pkt, struct mbuf *m, u_int pktlen, */ microtime(&tv); if (d->bd_flags & BPF_EXTENDED_HDR) { + struct mbuf *m; + + m = (pkt->bpfp_type == BPF_PACKET_TYPE_MBUF) + ? pkt->bpfp_mbuf : NULL; ehp = (struct bpf_hdr_ext *)(void *)(d->bd_sbuf + curlen); memset(ehp, 0, sizeof(*ehp)); ehp->bh_tstamp.tv_sec = tv.tv_sec; ehp->bh_tstamp.tv_usec = tv.tv_usec; - ehp->bh_datalen = pktlen; + + ehp->bh_datalen = pkt->bpfp_total_length; ehp->bh_hdrlen = hdrlen; - ehp->bh_caplen = totlen - hdrlen; - mt = m_tag_locate(m, bpf_mtag_id, 0, NULL); - if (mt && mt->m_tag_len >= sizeof(*bt)) { - bt = (struct bpf_mtag *)(mt + 1); - ehp->bh_pid = bt->bt_pid; - strlcpy(ehp->bh_comm, bt->bt_comm, - sizeof(ehp->bh_comm)); - ehp->bh_svc = so_svc2tc(bt->bt_svc); - if (bt->bt_direction == BPF_MTAG_DIR_OUT) + caplen = ehp->bh_caplen = totlen - hdrlen; + if (m == NULL) { + if (outbound) { ehp->bh_flags |= BPF_HDR_EXT_FLAGS_DIR_OUT; - else + } else { ehp->bh_flags |= BPF_HDR_EXT_FLAGS_DIR_IN; - m_tag_delete(m, mt); + } } else if (outbound) { + ehp->bh_flags |= BPF_HDR_EXT_FLAGS_DIR_OUT; + /* only do lookups on non-raw INPCB */ if ((m->m_pkthdr.pkt_flags & (PKTF_FLOW_ID| PKTF_FLOW_LOCALSRC|PKTF_FLOW_RAWSOCK)) == @@ -2625,7 +2662,6 @@ catchpacket(struct bpf_d *d, u_char *pkt, struct mbuf *m, u_int pktlen, ehp->bh_proto = m->m_pkthdr.pkt_proto; } ehp->bh_svc = so_svc2tc(m->m_pkthdr.pkt_svc); - ehp->bh_flags |= BPF_HDR_EXT_FLAGS_DIR_OUT; if (m->m_pkthdr.pkt_flags & PKTF_TCP_REXMT) ehp->bh_pktflags |= BPF_PKTFLAGS_TCP_REXMT; if (m->m_pkthdr.pkt_flags & PKTF_START_SEQ) @@ -2641,21 +2677,19 @@ catchpacket(struct bpf_d *d, u_char *pkt, struct mbuf *m, u_int pktlen, } else ehp->bh_flags |= BPF_HDR_EXT_FLAGS_DIR_IN; payload = (u_char *)ehp + hdrlen; - caplen = ehp->bh_caplen; } else { hp = (struct bpf_hdr *)(void *)(d->bd_sbuf + curlen); hp->bh_tstamp.tv_sec = tv.tv_sec; hp->bh_tstamp.tv_usec = tv.tv_usec; - hp->bh_datalen = pktlen; + hp->bh_datalen = pkt->bpfp_total_length; hp->bh_hdrlen = hdrlen; - hp->bh_caplen = totlen - hdrlen; + caplen = hp->bh_caplen = totlen - hdrlen; payload = (u_char *)hp + hdrlen; - caplen = hp->bh_caplen; } /* * Copy the packet data into the store buffer and update its length. */ - (*cpfn)(pkt, payload, caplen); + copy_bpf_packet(pkt, payload, caplen); d->bd_slen = curlen + totlen; d->bd_scnt += 1; @@ -2744,10 +2778,13 @@ bpf_attach( bpf_send_func send, bpf_tap_func tap) { + struct bpf_if *bp; struct bpf_if *bp_new; - struct bpf_if *bp_temp; + struct bpf_if *bp_before_first = NULL; struct bpf_if *bp_first = NULL; - + struct bpf_if *bp_last = NULL; + boolean_t found; + bp_new = (struct bpf_if *) _MALLOC(sizeof(*bp_new), M_DEVBUF, M_WAIT | M_ZERO); if (bp_new == 0) @@ -2756,20 +2793,34 @@ bpf_attach( lck_mtx_lock(bpf_mlock); /* - * Check if this interface/dlt is already attached, record first - * attachment for this interface. + * Check if this interface/dlt is already attached. Remember the + * first and last attachment for this interface, as well as the + * element before the first attachment. */ - for (bp_temp = bpf_iflist; bp_temp && (bp_temp->bif_ifp != ifp || - bp_temp->bif_dlt != dlt); bp_temp = bp_temp->bif_next) { - if (bp_temp->bif_ifp == ifp && bp_first == NULL) - bp_first = bp_temp; + found = FALSE; + for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) { + if (bp->bif_ifp != ifp) { + if (bp_first != NULL) { + /* no more elements for this interface */ + break; + } + bp_before_first = bp; + } else { + if (bp->bif_dlt == dlt) { + found = TRUE; + break; + } + if (bp_first == NULL) { + bp_first = bp; + } + bp_last = bp; + } } - - if (bp_temp != NULL) { + if (found) { + lck_mtx_unlock(bpf_mlock); printf("bpfattach - %s with dlt %d is already attached\n", if_name(ifp), dlt); FREE(bp_new, M_DEVBUF); - lck_mtx_unlock(bpf_mlock); return EEXIST; } @@ -2784,9 +2835,21 @@ bpf_attach( bpf_iflist = bp_new; } else { - /* Add this after the first entry for this interface */ - bp_new->bif_next = bp_first->bif_next; - bp_first->bif_next = bp_new; + if (ifnet_type(ifp) == IFT_ETHER && dlt == DLT_EN10MB) { + /* Make this the first entry for this interface */ + if (bp_before_first != NULL) { + /* point the previous to us */ + bp_before_first->bif_next = bp_new; + } else { + /* we're the new head */ + bpf_iflist = bp_new; + } + bp_new->bif_next = bp_first; + } else { + /* Add this after the last entry for this interface */ + bp_new->bif_next = bp_last->bif_next; + bp_last->bif_next = bp_new; + } } /* @@ -2825,8 +2888,7 @@ bpfdetach(struct ifnet *ifp) struct bpf_d *d; if (bpf_debug != 0) - printf("%s: %s\n", - __func__, if_name(ifp)); + printf("%s: %s\n", __func__, if_name(ifp)); lck_mtx_lock(bpf_mlock); @@ -2900,8 +2962,6 @@ bpf_init(__unused void *unused) for (i = 0 ; i < NBPFILTER; i++) bpf_make_dev_t(maj); - - VERIFY(mbuf_tag_id_find(BPF_CONTROL_NAME, &bpf_mtag_id) == 0); } #else cdevsw_add(&bpf_cdevsw); diff --git a/bsd/net/bpf.h b/bsd/net/bpf.h index edd79c7f7..ff4eb1bff 100644 --- a/bsd/net/bpf.h +++ b/bsd/net/bpf.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -1203,8 +1203,12 @@ struct bpf_mtag { */ #define DLT_NFC_LLCP 245 +/* + * USB packets, beginning with a Darwin (macOS, etc.) USB header. + */ +#define DLT_USB_DARWIN 266 -#define DLT_MATCHING_MAX 245 /* highest value in the "matching" range */ +#define DLT_MATCHING_MAX 266 /* highest value in the "matching" range */ /* * The instruction encodings. @@ -1299,6 +1303,21 @@ struct bpf_dltlist { struct ifnet; struct mbuf; +#define BPF_PACKET_TYPE_MBUF 0 + +struct bpf_packet { + int bpfp_type; + void * bpfp_header; /* optional */ + size_t bpfp_header_length; + union { + struct mbuf *bpfpu_mbuf; + void * bpfpu_ptr; + } bpfp_u; +#define bpfp_mbuf bpfp_u.bpfpu_mbuf +#define bpfp_ptr bpfp_u.bpfpu_ptr + size_t bpfp_total_length; /* length including optional header */ +}; + extern int bpf_validate(const struct bpf_insn *, int); extern void bpfdetach(struct ifnet *); extern void bpfilterattach(int); @@ -1341,7 +1360,7 @@ typedef u_int32_t bpf_tap_mode; @param packet The packet to be sent. */ typedef errno_t (*bpf_send_func)(ifnet_t interface, u_int32_t data_link_type, - mbuf_t packet); + mbuf_t packet); /*! @typedef bpf_tap_func @@ -1359,7 +1378,7 @@ typedef errno_t (*bpf_send_func)(ifnet_t interface, u_int32_t data_link_type, @param direction The direction of the tap. */ typedef errno_t (*bpf_tap_func)(ifnet_t interface, u_int32_t data_link_type, - bpf_tap_mode direction); + bpf_tap_mode direction); /*! @function bpfattach @@ -1412,7 +1431,7 @@ extern void bpf_tap_in(ifnet_t interface, u_int32_t dlt, mbuf_t packet, /*! @function bpf_tap_out - @discussion Call this function when your interface trasmits a + @discussion Call this function when your interface transmits a packet. This function will check if any bpf devices need a a copy of the packet. @param interface The interface the packet was or will be transmitted on. diff --git a/bsd/net/bpf_filter.c b/bsd/net/bpf_filter.c index 362472a95..80e31cd06 100644 --- a/bsd/net/bpf_filter.c +++ b/bsd/net/bpf_filter.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2011 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -100,106 +100,189 @@ #endif #include <net/bpf.h> #ifdef KERNEL -#define MINDEX(m, k) \ -{ \ - unsigned int len = m->m_len; \ - \ - while (k >= len) { \ - k -= len; \ - m = m->m_next; \ - if (m == 0) \ - return 0; \ - len = m->m_len; \ - } \ -} extern unsigned int bpf_maxbufsize; -static u_int16_t m_xhalf(struct mbuf *m, bpf_u_int32 k, int *err); -static u_int32_t m_xword(struct mbuf *m, bpf_u_int32 k, int *err); +static inline u_int32_t +get_word_from_buffers(u_char * cp, u_char * np, int num_from_cp) +{ + u_int32_t val; -static u_int32_t -m_xword(struct mbuf *m, bpf_u_int32 k, int *err) + switch (num_from_cp) { + case 1: + val = ((u_int32_t)cp[0] << 24) | + ((u_int32_t)np[0] << 16) | + ((u_int32_t)np[1] << 8) | + (u_int32_t)np[2]; + break; + + case 2: + val = ((u_int32_t)cp[0] << 24) | + ((u_int32_t)cp[1] << 16) | + ((u_int32_t)np[0] << 8) | + (u_int32_t)np[1]; + break; + default: + val = ((u_int32_t)cp[0] << 24) | + ((u_int32_t)cp[1] << 16) | + ((u_int32_t)cp[2] << 8) | + (u_int32_t)np[0]; + break; + } + return (val); +} + +static u_char * +m_hdr_offset(struct mbuf **m_p, void * hdr, size_t hdrlen, bpf_u_int32 * k_p, + size_t * len_p) { + u_char *cp; + bpf_u_int32 k = *k_p; size_t len; - u_char *cp, *np; - struct mbuf *m0; - - len = m->m_len; - while (k >= len) { - k -= len; - m = m->m_next; - if (m == 0) - goto bad; + + if (k >= hdrlen) { + struct mbuf *m = *m_p; + + /* there's no header or the offset we want is past the header */ + k -= hdrlen; len = m->m_len; + while (k >= len) { + k -= len; + m = m->m_next; + if (m == NULL) + return (NULL); + len = m->m_len; + } + cp = mtod(m, u_char *) + k; + + /* return next mbuf, in case it's needed */ + *m_p = m->m_next; + + /* update the offset */ + *k_p = k; + } else { + len = hdrlen; + cp = (u_char *)hdr + k; } - cp = mtod(m, u_char *) + k; + *len_p = len; + return (cp); +} + +static u_int32_t +m_xword(struct mbuf *m, void * hdr, size_t hdrlen, bpf_u_int32 k, int *err) +{ + size_t len; + u_char *cp, *np; + + cp = m_hdr_offset(&m, hdr, hdrlen, &k, &len); + if (cp == NULL) + goto bad; if (len - k >= 4) { *err = 0; return EXTRACT_LONG(cp); } - m0 = m->m_next; - if (m0 == 0 || m0->m_len + len - k < 4) + if (m == 0 || m->m_len + len - k < 4) goto bad; *err = 0; - np = mtod(m0, u_char *); - switch (len - k) { - - case 1: - return - ((u_int32_t)cp[0] << 24) | - ((u_int32_t)np[0] << 16) | - ((u_int32_t)np[1] << 8) | - (u_int32_t)np[2]; - - case 2: - return - ((u_int32_t)cp[0] << 24) | - ((u_int32_t)cp[1] << 16) | - ((u_int32_t)np[0] << 8) | - (u_int32_t)np[1]; + np = mtod(m, u_char *); + return get_word_from_buffers(cp, np, len - k); - default: - return - ((u_int32_t)cp[0] << 24) | - ((u_int32_t)cp[1] << 16) | - ((u_int32_t)cp[2] << 8) | - (u_int32_t)np[0]; - } bad: *err = 1; return 0; } static u_int16_t -m_xhalf(struct mbuf *m, bpf_u_int32 k, int *err) +m_xhalf(struct mbuf *m, void * hdr, size_t hdrlen, bpf_u_int32 k, int *err) { size_t len; u_char *cp; - struct mbuf *m0; - - len = m->m_len; - while (k >= len) { - k -= len; - m = m->m_next; - if (m == 0) - goto bad; - len = m->m_len; - } - cp = mtod(m, u_char *) + k; + + cp = m_hdr_offset(&m, hdr, hdrlen, &k, &len); + if (cp == NULL) + goto bad; if (len - k >= 2) { *err = 0; return EXTRACT_SHORT(cp); } - m0 = m->m_next; - if (m0 == 0) + if (m == 0) goto bad; *err = 0; - return (cp[0] << 8) | mtod(m0, u_char *)[0]; + return (cp[0] << 8) | mtod(m, u_char *)[0]; bad: *err = 1; return 0; } + +static u_int8_t +m_xbyte(struct mbuf *m, void * hdr, size_t hdrlen, bpf_u_int32 k, int *err) +{ + size_t len; + u_char *cp; + + cp = m_hdr_offset(&m, hdr, hdrlen, &k, &len); + if (cp == NULL) + goto bad; + *err = 0; + return (*cp); + bad: + *err = 1; + return 0; + +} + + +static u_int32_t +bp_xword(struct bpf_packet *bp, bpf_u_int32 k, int *err) +{ + void * hdr = bp->bpfp_header; + size_t hdrlen = bp->bpfp_header_length; + + switch (bp->bpfp_type) { + case BPF_PACKET_TYPE_MBUF: + return m_xword(bp->bpfp_mbuf, hdr, hdrlen, k, err); + default: + break; + } + *err = 1; + return 0; + +} + +static u_int16_t +bp_xhalf(struct bpf_packet *bp, bpf_u_int32 k, int *err) +{ + void * hdr = bp->bpfp_header; + size_t hdrlen = bp->bpfp_header_length; + + switch (bp->bpfp_type) { + case BPF_PACKET_TYPE_MBUF: + return m_xhalf(bp->bpfp_mbuf, hdr, hdrlen, k, err); + default: + break; + } + *err = 1; + return 0; + +} + +static u_int8_t +bp_xbyte(struct bpf_packet *bp, bpf_u_int32 k, int *err) +{ + void * hdr = bp->bpfp_header; + size_t hdrlen = bp->bpfp_header_length; + + switch (bp->bpfp_type) { + case BPF_PACKET_TYPE_MBUF: + return m_xbyte(bp->bpfp_mbuf, hdr, hdrlen, k, err); + default: + break; + } + *err = 1; + return 0; + +} + #endif /* @@ -213,6 +296,10 @@ bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen) u_int32_t A = 0, X = 0; bpf_u_int32 k; int32_t mem[BPF_MEMWORDS]; +#ifdef KERNEL + int merr; + struct bpf_packet * bp = (struct bpf_packet *)(void *)p; +#endif /* KERNEL */ bzero(mem, sizeof(mem)); @@ -230,9 +317,9 @@ bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen) default: #ifdef KERNEL return 0; -#else +#else /* KERNEL */ abort(); -#endif +#endif /* KERNEL */ case BPF_RET|BPF_K: return (u_int)pc->k; @@ -243,23 +330,21 @@ bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen) k = pc->k; if (k > buflen || sizeof(int32_t) > buflen - k) { #ifdef KERNEL - int merr; - if (buflen != 0) return 0; - A = m_xword((struct mbuf *)(void *)p, k, &merr); + A = bp_xword(bp, k, &merr); if (merr != 0) return 0; continue; -#else +#else /* KERNEL */ return 0; -#endif +#endif /* KERNEL */ } #if BPF_ALIGN if (((intptr_t)(p + k) & 3) != 0) A = EXTRACT_LONG(&p[k]); else -#endif +#endif /* BPF_ALIGN */ A = ntohl(*(int32_t *)(void *)(p + k)); continue; @@ -267,15 +352,15 @@ bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen) k = pc->k; if (k > buflen || sizeof(int16_t) > buflen - k) { #ifdef KERNEL - int merr; - if (buflen != 0) return 0; - A = m_xhalf((struct mbuf *)(void *)p, k, &merr); + A = bp_xhalf(bp, k, &merr); + if (merr != 0) + return 0; continue; -#else +#else /* KERNEL */ return 0; -#endif +#endif /* KERNEL */ } A = EXTRACT_SHORT(&p[k]); continue; @@ -284,17 +369,15 @@ bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen) k = pc->k; if (k >= buflen) { #ifdef KERNEL - struct mbuf *m; - if (buflen != 0) return 0; - m = (struct mbuf *)(void *)p; - MINDEX(m, k); - A = mtod(m, u_char *)[k]; + A = bp_xbyte(bp, k, &merr); + if (merr != 0) + return 0; continue; -#else +#else /* KERNEL */ return 0; -#endif +#endif /* KERNEL */ } A = p[k]; continue; @@ -312,23 +395,21 @@ bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen) if (pc->k > buflen || X > buflen - pc->k || sizeof(int32_t) > buflen - k) { #ifdef KERNEL - int merr; - if (buflen != 0) return 0; - A = m_xword((struct mbuf *)(void *)p, k, &merr); + A = bp_xword(bp, k, &merr); if (merr != 0) return 0; continue; -#else +#else /* KERNEL */ return 0; -#endif +#endif /* KERNEL */ } #if BPF_ALIGN if (((intptr_t)(p + k) & 3) != 0) A = EXTRACT_LONG(&p[k]); else -#endif +#endif /* BPF_ALIGN */ A = ntohl(*(int32_t *)(void *)(p + k)); continue; @@ -337,17 +418,15 @@ bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen) if (X > buflen || pc->k > buflen - X || sizeof(int16_t) > buflen - k) { #ifdef KERNEL - int merr; - if (buflen != 0) return 0; - A = m_xhalf((struct mbuf *)(void *)p, k, &merr); + A = bp_xhalf(bp, k, &merr); if (merr != 0) return 0; continue; -#else +#else /* KERNEL */ return 0; -#endif +#endif /* KERNEL */ } A = EXTRACT_SHORT(&p[k]); continue; @@ -356,17 +435,15 @@ bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen) k = X + pc->k; if (pc->k >= buflen || X >= buflen - pc->k) { #ifdef KERNEL - struct mbuf *m; - if (buflen != 0) return 0; - m = (struct mbuf *)(void *)p; - MINDEX(m, k); - A = mtod(m, u_char *)[k]; + A = bp_xbyte(bp, k, &merr); + if (merr != 0) + return 0; continue; -#else +#else /* KERNEL */ return 0; -#endif +#endif /* KERNEL */ } A = p[k]; continue; @@ -375,13 +452,12 @@ bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen) k = pc->k; if (k >= buflen) { #ifdef KERNEL - struct mbuf *m; - if (buflen != 0) return 0; - m = (struct mbuf *)(void *)p; - MINDEX(m, k); - X = (mtod(m, u_char *)[k] & 0xf) << 2; + X = bp_xbyte(bp, k, &merr); + if (merr != 0) + return 0; + X = (X & 0xf) << 2; continue; #else return 0; diff --git a/bsd/net/bridgestp.c b/bsd/net/bridgestp.c index 10d86fa4a..972a1ae14 100644 --- a/bsd/net/bridgestp.c +++ b/bsd/net/bridgestp.c @@ -121,7 +121,7 @@ static void bstp_task_drain(struct bstp_task *); #define BSTP_LOCK_DESTROY(_bs) lck_mtx_free((_bs)->bs_mtx, bstp_lock_grp) #define BSTP_LOCK(_bs) lck_mtx_lock((_bs)->bs_mtx) #define BSTP_UNLOCK(_bs) lck_mtx_unlock((_bs)->bs_mtx) -#define BSTP_LOCK_ASSERT(_bs) lck_mtx_assert((_bs)->bs_mtx, LCK_MTX_ASSERT_OWNED) +#define BSTP_LOCK_ASSERT(_bs) LCK_MTX_ASSERT((_bs)->bs_mtx, LCK_MTX_ASSERT_OWNED) #ifdef BRIDGESTP_DEBUG diff --git a/bsd/net/classq/classq.c b/bsd/net/classq/classq.c index 67c1f44ea..35d86188f 100644 --- a/bsd/net/classq/classq.c +++ b/bsd/net/classq/classq.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2007-2016 Apple Inc. All rights reserved. + * Copyright (c) 2007-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -73,6 +73,7 @@ #include <libkern/libkern.h> + u_int32_t classq_verbose = 0; /* more noise if greater than 1 */ SYSCTL_NODE(_net, OID_AUTO, classq, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "classq"); @@ -81,44 +82,100 @@ SYSCTL_UINT(_net_classq, OID_AUTO, verbose, CTLFLAG_RW|CTLFLAG_LOCKED, &classq_verbose, 0, "Class queue verbosity level"); void -_qinit(class_queue_t *q, int type, int lim) +_qinit(class_queue_t *q, int type, int lim, classq_pkt_type_t ptype) { - MBUFQ_INIT(&q->mbufq); + switch (ptype) { + case QP_MBUF: + MBUFQ_INIT(&qmbufq(q)); + break; + + + default: + VERIFY(0); + /* NOTREACHED */ + } + qlimit(q) = lim; qlen(q) = 0; qsize(q) = 0; qtype(q) = type; + qptype(q) = ptype; qstate(q) = QS_RUNNING; } /* add a packet at the tail of the queue */ void -_addq(class_queue_t *q, struct mbuf *m) +_addq(class_queue_t *q, void *pkt) { - MBUFQ_ENQUEUE(&q->mbufq, m); + uint32_t size = 0; + + switch (qptype(q)) { + case QP_MBUF: { + struct mbuf *m = pkt; + MBUFQ_ENQUEUE(&qmbufq(q), m); + size = m_length(m); + break; + } + + + default: + VERIFY(0); + /* NOTREACHED */ + } + qlen(q)++; VERIFY(qlen(q) != 0); - qsize(q) += m_length(m); + qsize(q) += size; } /* add one or more packets at the tail of the queue */ void -_addq_multi(class_queue_t *q, struct mbuf *m_head, struct mbuf *m_tail, +_addq_multi(class_queue_t *q, void *pkt_head, void *pkt_tail, u_int32_t cnt, u_int32_t size) { - MBUFQ_ENQUEUE_MULTI(&q->mbufq, m_head, m_tail); + switch (qptype(q)) { + case QP_MBUF: { + struct mbuf *m_head = pkt_head; + struct mbuf *m_tail = pkt_tail; + MBUFQ_ENQUEUE_MULTI(&qmbufq(q), m_head, m_tail); + break; + } + + + default: + VERIFY(0); + /* NOTREACHED */ + } + qlen(q) += cnt; qsize(q) += size; } /* get a packet at the head of the queue */ -struct mbuf * +void * _getq(class_queue_t *q) { - struct mbuf *m; + void *pkt = NULL; + uint32_t pkt_len; + + switch (qptype(q)) { + case QP_MBUF: { + struct mbuf *m; + MBUFQ_DEQUEUE(&qmbufq(q), m); + if (m != NULL) { + pkt_len = m_length(m); + pkt = m; + } + break; + } + + + default: + VERIFY(0); + /* NOTREACHED */ + } - MBUFQ_DEQUEUE(&q->mbufq, m); - if (m == NULL) { + if (pkt == NULL) { VERIFY(qlen(q) == 0); if (qsize(q) > 0) qsize(q) = 0; @@ -128,73 +185,97 @@ _getq(class_queue_t *q) qlen(q)--; /* qsize is an approximation, so adjust if necessary */ - if (((int)qsize(q) - m_length(m)) > 0) - qsize(q) -= m_length(m); + if (((int)qsize(q) - pkt_len) > 0) + qsize(q) -= pkt_len; else if (qsize(q) != 0) qsize(q) = 0; - return (m); + return (pkt); } -static struct mbuf * +static void * _getq_flow_or_scidx(class_queue_t *q, u_int32_t val, boolean_t isflowid) { - struct mbuf *m, *m_tmp; - - MBUFQ_FOREACH_SAFE(m, &q->mbufq, m_tmp) { - if ((isflowid && (val == 0 || ((m->m_flags & M_PKTHDR) && - m->m_pkthdr.pkt_flowid == val))) || - (!isflowid && - MBUF_SCIDX(mbuf_get_service_class(m)) < val)) { - /* remove it from the class queue */ - MBUFQ_REMOVE(&q->mbufq, m); - MBUFQ_NEXT(m) = NULL; - break; + void *pkt = NULL; + uint32_t pkt_len; + + switch (qptype(q)) { + case QP_MBUF: { + struct mbuf *m, *m_tmp; + + MBUFQ_FOREACH_SAFE(m, &qmbufq(q), m_tmp) { + if ((isflowid && (val == 0 || + ((m->m_flags & M_PKTHDR) && + m->m_pkthdr.pkt_flowid == val))) || + (!isflowid && + MBUF_SCIDX(mbuf_get_service_class(m)) < val)) { + /* remove it from the class queue */ + MBUFQ_REMOVE(&qmbufq(q), m); + MBUFQ_NEXT(m) = NULL; + break; + } + } + if (m != NULL) { + pkt = m; + pkt_len = m_length(m); } + break; } - if (m != NULL) { - u_int32_t l = m_length(m); + default: + VERIFY(0); + /* NOTREACHED */ + } + + if (pkt != NULL) { VERIFY(qlen(q) > 0); qlen(q)--; /* qsize is an approximation, so adjust if necessary */ - if (((int)qsize(q) - l) > 0) - qsize(q) -= l; + if (((int)qsize(q) - pkt_len) > 0) + qsize(q) -= pkt_len; else if (qsize(q) != 0) qsize(q) = 0; } - return (m); - + return (pkt); } /* get a packet of a specific flow beginning from the head of the queue */ -struct mbuf * +void * _getq_flow(class_queue_t *q, u_int32_t flow) { return (_getq_flow_or_scidx(q, flow, TRUE)); } /* Get a packet whose MBUF_SCIDX() < scidx from head of queue */ -struct mbuf * +void * _getq_scidx_lt(class_queue_t *q, u_int32_t scidx) { return (_getq_flow_or_scidx(q, scidx, FALSE)); } -/* get all packets starting from the head of the queue */ -struct mbuf * -_getq_all(class_queue_t *q, struct mbuf **last, u_int32_t *qlenp, +/* get all packets (chained) starting from the head of the queue */ +void * +_getq_all(class_queue_t *q, void **last, u_int32_t *qlenp, u_int64_t *qsizep) { - struct mbuf *m; + void *pkt = NULL; + + switch (qptype(q)) { + case QP_MBUF: + pkt = MBUFQ_FIRST(&qmbufq(q)); + if (last != NULL) + *last = MBUFQ_LAST(&qmbufq(q)); + MBUFQ_INIT(&qmbufq(q)); + break; - m = MBUFQ_FIRST(&q->mbufq); - if (last != NULL) - *last = MBUFQ_LAST(&q->mbufq); - MBUFQ_INIT(&q->mbufq); + + default: + VERIFY(0); + /* NOTREACHED */ + } if (qlenp != NULL) *qlenp = qlen(q); @@ -204,14 +285,13 @@ _getq_all(class_queue_t *q, struct mbuf **last, u_int32_t *qlenp, qlen(q) = 0; qsize(q) = 0; - return (m); + return (pkt); } -/* drop a packet at the tail of the queue */ -struct mbuf * -_getq_tail(class_queue_t *q) +static inline struct mbuf * +_getq_tail_mbuf(class_queue_t *q) { - struct mq_head *head = &q->mbufq; + struct mq_head *head = &qmbufq(q); struct mbuf *m = MBUFQ_LAST(head); if (m != NULL) { @@ -247,15 +327,36 @@ _getq_tail(class_queue_t *q) return (m); } -/* randomly select a packet in the queue */ -struct mbuf * -_getq_random(class_queue_t *q) +/* drop a packet at the tail of the queue */ +void * +_getq_tail(class_queue_t *q) +{ + void *t = NULL; + + switch (qptype(q)) { + case QP_MBUF: + t = _getq_tail_mbuf(q); + break; + + default: + VERIFY(0); + /* NOTREACHED */ + } + + return (t); +} + +static inline struct mbuf * +_getq_random_mbuf(class_queue_t *q) { - struct mq_head *head = &q->mbufq; + struct mq_head *head = &qmbufq(q); struct mbuf *m = NULL; unsigned int n; u_int32_t rnd; + /* XXX: Add support for Kernel packet when needed */ + VERIFY((qptype(q) == QP_MBUF)); + n = qlen(q); if (n == 0) { VERIFY(MBUFQ_EMPTY(head)); @@ -265,7 +366,7 @@ _getq_random(class_queue_t *q) } m = MBUFQ_FIRST(head); - read_random(&rnd, sizeof (rnd)); + read_frandom(&rnd, sizeof (rnd)); n = (rnd % n) + 1; if (n == 1) { @@ -301,11 +402,29 @@ _getq_random(class_queue_t *q) return (m); } -/* remove a packet from the queue */ -void -_removeq(class_queue_t *q, struct mbuf *m) +/* randomly select a packet in the queue */ +void * +_getq_random(class_queue_t *q) +{ + void *r = NULL; + + switch (qptype(q)) { + case QP_MBUF: + r = _getq_random_mbuf(q); + break; + + default: + VERIFY(0); + /* NOTREACHED */ + } + + return (r); +} + +static inline void +_removeq_mbuf(class_queue_t *q, struct mbuf *m) { - struct mq_head *head = &q->mbufq; + struct mq_head *head = &qmbufq(q); struct mbuf *m0, **mtail; m0 = MBUFQ_FIRST(head); @@ -339,14 +458,30 @@ _removeq(class_queue_t *q, struct mbuf *m) MBUFQ_NEXT(m) = NULL; } +/* remove a packet from the queue */ +void +_removeq(class_queue_t *q, void *pkt) +{ + switch (qptype(q)) { + case QP_MBUF: + _removeq_mbuf(q, pkt); + break; + + default: + VERIFY(0); + /* NOTREACHED */ + } +} + void _flushq(class_queue_t *q) { (void) _flushq_flow(q, 0, NULL, NULL); } -void -_flushq_flow(class_queue_t *q, u_int32_t flow, u_int32_t *cnt, u_int32_t *len) +static inline void +_flushq_flow_mbuf(class_queue_t *q, u_int32_t flow, u_int32_t *cnt, + u_int32_t *len) { MBUFQ_HEAD(mq_freeq) freeq; struct mbuf *m, *m_tmp; @@ -354,11 +489,11 @@ _flushq_flow(class_queue_t *q, u_int32_t flow, u_int32_t *cnt, u_int32_t *len) MBUFQ_INIT(&freeq); - MBUFQ_FOREACH_SAFE(m, &q->mbufq, m_tmp) { + MBUFQ_FOREACH_SAFE(m, &qmbufq(q), m_tmp) { if (flow == 0 || ((m->m_flags & M_PKTHDR) && m->m_pkthdr.pkt_flowid == flow)) { /* remove it from the class queue */ - MBUFQ_REMOVE(&q->mbufq, m); + MBUFQ_REMOVE(&qmbufq(q), m); MBUFQ_NEXT(m) = NULL; /* and add it to the free queue */ @@ -389,3 +524,17 @@ _flushq_flow(class_queue_t *q, u_int32_t flow, u_int32_t *cnt, u_int32_t *len) if (len != NULL) *len = l; } + +void +_flushq_flow(class_queue_t *q, u_int32_t flow, u_int32_t *cnt, u_int32_t *len) +{ + switch (qptype(q)) { + case QP_MBUF: + _flushq_flow_mbuf(q, flow, cnt, len); + break; + + default: + VERIFY(0); + /* NOTREACHED */ + } +} diff --git a/bsd/net/classq/classq.h b/bsd/net/classq/classq.h index 750ce5452..f36f9d727 100644 --- a/bsd/net/classq/classq.h +++ b/bsd/net/classq/classq.h @@ -72,15 +72,20 @@ extern "C" { #endif +/* + * Packet types + */ +typedef enum classq_pkt_type { + QP_INVALID = 0, + QP_MBUF, /* mbuf packet */ +} classq_pkt_type_t; + /* * Packet Queue types */ typedef enum classq_type { Q_DROPHEAD, Q_DROPTAIL, - Q_RED, - Q_RIO, - Q_BLUE, Q_SFB } classq_type_t; @@ -114,21 +119,26 @@ struct pktcntr { * Packet Queue structures and macros to manipulate them. */ typedef struct _class_queue_ { - MBUFQ_HEAD(mq_head) mbufq; /* Packet queue */ + union { + MBUFQ_HEAD(mq_head) __mbufq; /* mbuf packet queue */ + } __pktq_u; u_int32_t qlen; /* Queue length (in number of packets) */ u_int32_t qlim; /* Queue limit (in number of packets*) */ u_int64_t qsize; /* Approx. queue size (in number of bytes) */ classq_type_t qtype; /* Queue type */ classq_state_t qstate; /* Queue state */ + classq_pkt_type_t qptype; /* Packet type */ } class_queue_t; +#define qmbufq(q) (q)->__pktq_u.__mbufq /* Get mbuf packet queue */ +#define qptype(q) (q)->qptype /* Get queue packet type */ #define qtype(q) (q)->qtype /* Get queue type */ #define qstate(q) (q)->qstate /* Get queue state */ #define qlimit(q) (q)->qlim /* Max packets to be queued */ #define qlen(q) (q)->qlen /* Current queue length. */ #define qsize(q) (q)->qsize /* Approx. bytes in queue */ -/* #define qtail(q) MBUFQ_LAST(&(q)->mbufq) */ -#define qhead(q) MBUFQ_FIRST(&(q)->mbufq) + +#define qhead(q) MBUFQ_FIRST(&qmbufq(q)) #define qempty(q) (qlen(q) == 0) /* Is the queue empty?? */ #define q_is_red(q) (qtype(q) == Q_RED) /* Is the queue a RED queue */ @@ -157,18 +167,16 @@ extern u_int32_t classq_verbose; SYSCTL_DECL(_net_classq); -extern void _qinit(class_queue_t *, int, int); -extern void _addq(class_queue_t *, struct mbuf *); -extern void _addq_multi(class_queue_t *, struct mbuf *, struct mbuf *, - u_int32_t, u_int32_t); -extern struct mbuf *_getq(class_queue_t *); -extern struct mbuf *_getq_all(class_queue_t *, struct mbuf **, - u_int32_t *, u_int64_t *); -extern struct mbuf *_getq_tail(class_queue_t *); -extern struct mbuf *_getq_random(class_queue_t *); -extern struct mbuf *_getq_flow(class_queue_t *, u_int32_t); -extern struct mbuf *_getq_scidx_lt(class_queue_t *, u_int32_t); -extern void _removeq(class_queue_t *, struct mbuf *); +extern void _qinit(class_queue_t *, int, int, classq_pkt_type_t); +extern void _addq(class_queue_t *, void *); +extern void _addq_multi(class_queue_t *, void *, void *, u_int32_t, u_int32_t); +extern void *_getq(class_queue_t *); +extern void *_getq_all(class_queue_t *, void **, u_int32_t *, u_int64_t *); +extern void *_getq_tail(class_queue_t *); +extern void *_getq_random(class_queue_t *); +extern void *_getq_flow(class_queue_t *, u_int32_t); +extern void *_getq_scidx_lt(class_queue_t *, u_int32_t); +extern void _removeq(class_queue_t *, void *); extern void _flushq(class_queue_t *); extern void _flushq_flow(class_queue_t *, u_int32_t, u_int32_t *, u_int32_t *); diff --git a/bsd/net/classq/classq_blue.c b/bsd/net/classq/classq_blue.c deleted file mode 100644 index fdf21b069..000000000 --- a/bsd/net/classq/classq_blue.c +++ /dev/null @@ -1,385 +0,0 @@ -/* - * Copyright (c) 2007-2012 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* $NetBSD: altq_blue.c,v 1.21 2006/11/16 01:32:37 christos Exp $ */ -/* $KAME: altq_blue.c,v 1.15 2005/04/13 03:44:24 suz Exp $ */ - -/* - * Copyright (C) 1997-2002 - * Sony Computer Science Laboratories Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY SONY CSL AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL SONY CSL OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - */ -/* - * Copyright (c) 1990-1994 Regents of the University of California. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the Computer Systems - * Engineering Group at Lawrence Berkeley Laboratory. - * 4. Neither the name of the University nor of the Laboratory may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include <sys/cdefs.h> - -#if CLASSQ_BLUE - -#include <sys/param.h> -#include <sys/malloc.h> -#include <sys/mbuf.h> -#include <sys/socket.h> -#include <sys/sockio.h> -#include <sys/systm.h> -#include <sys/syslog.h> -#include <sys/proc.h> -#include <sys/errno.h> -#include <sys/kernel.h> -#include <sys/kauth.h> - -#include <kern/zalloc.h> - -#include <net/if.h> -#include <net/if_var.h> -#include <net/if_types.h> - -#include <netinet/in.h> -#include <netinet/in_systm.h> -#include <netinet/ip.h> -#if INET6 -#include <netinet/ip6.h> -#endif - -#include <net/classq/classq_blue.h> -#include <net/net_osdep.h> -#include <dev/random/randomdev.h> - -/* - * Blue is proposed and implemented by Wu-chang Feng <wuchang@eecs.umich.edu>. - * more information on Blue is available from - * http://www.eecs.umich.edu/~wuchang/blue/ - */ - -#define BLUE_LIMIT 200 /* default max queue lenght */ - -#define BLUE_ZONE_MAX 32 /* maximum elements in zone */ -#define BLUE_ZONE_NAME "classq_blue" /* zone name */ - -static unsigned int blue_size; /* size of zone element */ -static struct zone *blue_zone; /* zone for blue */ - -/* internal function prototypes */ -static struct mbuf *blue_getq_flow(struct blue *, class_queue_t *, - u_int32_t, boolean_t); -static int blue_drop_early(struct blue *); - -void -blue_init(void) -{ - _CASSERT(BLUEF_ECN4 == CLASSQF_ECN4); - _CASSERT(BLUEF_ECN6 == CLASSQF_ECN6); - - blue_size = sizeof (struct blue); - blue_zone = zinit(blue_size, BLUE_ZONE_MAX * blue_size, - 0, BLUE_ZONE_NAME); - if (blue_zone == NULL) { - panic("%s: failed allocating %s", __func__, BLUE_ZONE_NAME); - /* NOTREACHED */ - } - zone_change(blue_zone, Z_EXPAND, TRUE); - zone_change(blue_zone, Z_CALLERACCT, TRUE); -} - -/* - * blue support routines - */ -struct blue * -blue_alloc(struct ifnet *ifp, u_int32_t max_pmark, u_int32_t hold_time, - u_int32_t flags) -{ - struct blue *bp; - - VERIFY(ifp != NULL); - - bp = zalloc(blue_zone); - if (bp == NULL) - return (NULL); - - bzero(bp, blue_size); - bp->blue_idle = 1; - bp->blue_ifp = ifp; - bp->blue_flags = (flags & BLUEF_USERFLAGS); -#if !PF_ECN - if (bp->blue_flags & BLUEF_ECN) { - bp->blue_flags &= ~BLUEF_ECN; - log(LOG_ERR, "%s: BLUE ECN not available; ignoring " - "BLUEF_ECN flag!\n", if_name(ifp)); - } -#endif /* !PF_ECN */ - - - if (max_pmark == 0) - bp->blue_max_pmark = 1000; - else - bp->blue_max_pmark = max_pmark; - - if (hold_time == 0) - bp->blue_hold_time = 50000; - else - bp->blue_hold_time = hold_time; - - microuptime(&bp->blue_last); - - return (bp); -} - -void -blue_destroy(struct blue *bp) -{ - zfree(blue_zone, bp); -} - -void -blue_getstats(struct blue *bp, struct blue_stats *sp) -{ - sp->q_pmark = bp->blue_pmark; - sp->drop_forced = bp->blue_stats.drop_forced; - sp->drop_unforced = bp->blue_stats.drop_unforced; - sp->marked_packets = bp->blue_stats.marked_packets; -} - -#define DTYPE_NODROP 0 /* no drop */ -#define DTYPE_FORCED 1 /* a "forced" drop */ -#define DTYPE_EARLY 2 /* an "unforced" (early) drop */ - -int -blue_addq(struct blue *bp, class_queue_t *q, struct mbuf *m, - struct pf_mtag *tag) -{ -#if !PF_ECN -#pragma unused(tag) -#endif /* !PF_ECN */ - int droptype; - - /* - * if we were idle, this is an enqueue onto an empty queue - * and we should decrement marking probability - */ - if (bp->blue_idle) { - struct timeval now; - u_int32_t t; - - bp->blue_idle = 0; - microuptime(&now); - t = (now.tv_sec - bp->blue_last.tv_sec); - if (t > 1) { - bp->blue_pmark = 1; - microuptime(&bp->blue_last); - } else { - t = t * 1000000 + (now.tv_usec - bp->blue_last.tv_usec); - if (t > bp->blue_hold_time) { - bp->blue_pmark--; - if (bp->blue_pmark < 0) - bp->blue_pmark = 0; - microuptime(&bp->blue_last); - } - } - } - - /* see if we drop early */ - droptype = DTYPE_NODROP; - if (blue_drop_early(bp) && qlen(q) > 1) { - /* mark or drop by blue */ -#if PF_ECN - if ((bp->blue_flags & BLUEF_ECN) && - (tag->pftag_proto == IPPROTO_TCP) && /* only for TCP */ - mark_ecn(m, tag, bp->blue_flags)) { - /* successfully marked. do not drop. */ - bp->blue_stats.marked_packets++; - } else -#endif /* PF_ECN */ - { - /* unforced drop by blue */ - droptype = DTYPE_EARLY; - } - } - - /* if the queue length hits the hard limit, it's a forced drop */ - if (droptype == DTYPE_NODROP && qlen(q) >= qlimit(q)) - droptype = DTYPE_FORCED; - - /* if successful or forced drop, enqueue this packet. */ - if (droptype != DTYPE_EARLY) - _addq(q, m); - - if (droptype != DTYPE_NODROP) { - if (droptype == DTYPE_EARLY) { - /* drop the incoming packet */ - bp->blue_stats.drop_unforced++; - } else { - struct timeval now; - u_int32_t t; - /* forced drop, select a victim packet in the queue. */ - m = _getq_random(q); - microuptime(&now); - t = (now.tv_sec - bp->blue_last.tv_sec); - t = t * 1000000 + (now.tv_usec - bp->blue_last.tv_usec); - if (t > bp->blue_hold_time) { - bp->blue_pmark += bp->blue_max_pmark >> 3; - if (bp->blue_pmark > bp->blue_max_pmark) - bp->blue_pmark = bp->blue_max_pmark; - microuptime(&bp->blue_last); - } - bp->blue_stats.drop_forced++; - } - IFCQ_CONVERT_LOCK(&bp->blue_ifp->if_snd); - m_freem(m); - return (CLASSQEQ_DROPPED); - } - /* successfully queued */ - return (CLASSQEQ_SUCCESS); -} - -static struct mbuf * -blue_getq_flow(struct blue *bp, class_queue_t *q, u_int32_t flow, - boolean_t purge) -{ -#pragma unused(purge) - struct mbuf *m; - - /* flow of 0 means head of queue */ - if ((m = ((flow == 0) ? _getq(q) : _getq_flow(q, flow))) == NULL) { - if (bp->blue_idle == 0) { - bp->blue_idle = 1; - microuptime(&bp->blue_last); - } - return (NULL); - } - - bp->blue_idle = 0; - return (m); -} - -struct mbuf * -blue_getq(struct blue *bp, class_queue_t *q) -{ - return (blue_getq_flow(bp, q, 0, FALSE)); -} - -void -blue_purgeq(struct blue *bp, class_queue_t *q, u_int32_t flow, - u_int32_t *packets, u_int32_t *bytes) -{ - u_int32_t cnt = 0, len = 0; - struct mbuf *m; - - IFCQ_CONVERT_LOCK(&bp->blue_ifp->if_snd); - - while ((m = blue_getq_flow(bp, q, flow, TRUE)) != NULL) { - cnt++; - len += m_pktlen(m); - m_freem(m); - } - - if (packets != NULL) - *packets = cnt; - if (bytes != NULL) - *bytes = len; -} - -/* - * early-drop probability is kept in blue_pmark - */ -static int -blue_drop_early(struct blue *bp) -{ - if ((RandomULong() % (unsigned)bp->blue_max_pmark) < - (unsigned)bp->blue_pmark) { - /* drop or mark */ - return (1); - } - /* no drop/mark */ - return (0); -} - -void -blue_updateq(struct blue *bp, cqev_t ev) -{ -#pragma unused(bp, ev) - /* nothing for now */ -} - -int -blue_suspendq(struct blue *bp, class_queue_t *q, boolean_t on) -{ -#pragma unused(bp, q, on) - return (ENOTSUP); -} -#endif /* CLASSQ_BLUE */ diff --git a/bsd/net/classq/classq_blue.h b/bsd/net/classq/classq_blue.h index e6c546e23..777122b94 100644 --- a/bsd/net/classq/classq_blue.h +++ b/bsd/net/classq/classq_blue.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2012 Apple Inc. All rights reserved. + * Copyright (c) 2011-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -75,51 +75,6 @@ struct blue_stats { u_int64_t marked_packets; }; -#ifdef BSD_KERNEL_PRIVATE -/* blue flags */ -#define BLUEF_ECN4 0x01 /* use packet marking for IPv4 packets */ -#define BLUEF_ECN6 0x02 /* use packet marking for IPv6 packets */ -#define BLUEF_ECN (BLUEF_ECN4 | BLUEF_ECN6) - -#define BLUEF_USERFLAGS \ - (BLUEF_ECN4 | BLUEF_ECN6) - -typedef struct blue { - u_int32_t blue_flags; /* blue flags */ - - /* blue parameters */ - int32_t blue_pmark; /* 0-1000 (mark probability*10000) */ - int32_t blue_max_pmark; /* sets precision of marking probability */ - u_int32_t blue_hold_time; /* hold time in usec */ - struct ifnet *blue_ifp; /* back pointer to ifnet */ - - /* variables for internal use */ - u_int32_t blue_idle; /* queue was empty */ - struct timeval blue_last; /* timestamp when the queue becomes idle */ - - /* statistics */ - struct { - struct pktcntr xmit_cnt; - struct pktcntr drop_cnt; - u_int64_t drop_forced; - u_int64_t drop_unforced; - u_int64_t marked_packets; - } blue_stats; -} blue_t; - -extern void blue_init(void); -extern struct blue *blue_alloc(struct ifnet *, u_int32_t, u_int32_t, u_int32_t); -extern void blue_destroy(struct blue *); -extern int blue_addq(struct blue *, class_queue_t *, struct mbuf *, - struct pf_mtag *); -extern struct mbuf *blue_getq(struct blue *, class_queue_t *); -extern void blue_purgeq(struct blue *, class_queue_t *, u_int32_t, - u_int32_t *, u_int32_t *); -extern void blue_getstats(struct blue *, struct blue_stats *); -extern void blue_updateq(struct blue *, cqev_t); -extern int blue_suspendq(struct blue *, class_queue_t *, boolean_t); -#endif /* BSD_KERNEL_PRIVATE */ - #ifdef __cplusplus } #endif diff --git a/bsd/net/classq/classq_fq_codel.c b/bsd/net/classq/classq_fq_codel.c index f78da89d6..3710ad112 100644 --- a/bsd/net/classq/classq_fq_codel.c +++ b/bsd/net/classq/classq_fq_codel.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 Apple Inc. All rights reserved. + * Copyright (c) 2016-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -41,15 +41,19 @@ #include <kern/zalloc.h> #include <netinet/in.h> +#include <net/classq/classq.h> +#include <net/classq/if_classq.h> +#include <net/pktsched/pktsched.h> #include <net/pktsched/pktsched_fq_codel.h> #include <net/classq/classq_fq_codel.h> static struct zone *flowq_zone = NULL; -static size_t flowq_size; #define FQ_ZONE_MAX (32 * 1024) /* across all interfaces */ -#define FQ_SEQ_LT(a,b) ((int)((a)-(b)) < 0) -#define FQ_SEQ_GT(a,b) ((int)((a)-(b)) > 0) + +#define DTYPE_NODROP 0 /* no drop */ +#define DTYPE_FORCED 1 /* a "forced" drop */ +#define DTYPE_EARLY 2 /* an "unforced" (early) drop */ void fq_codel_init(void) @@ -57,9 +61,8 @@ fq_codel_init(void) if (flowq_zone != NULL) return; - flowq_size = sizeof (fq_t); - flowq_zone = zinit(flowq_size, FQ_ZONE_MAX * flowq_size, - 0, "flowq_zone"); + flowq_zone = zinit(sizeof (struct flowq), + FQ_ZONE_MAX * sizeof (struct flowq), 0, "flowq_zone"); if (flowq_zone == NULL) { panic("%s: failed to allocate flowq_zone", __func__); /* NOTREACHED */ @@ -69,27 +72,29 @@ fq_codel_init(void) } fq_t * -fq_alloc(int how) +fq_alloc(classq_pkt_type_t ptype) { fq_t *fq = NULL; - fq = (how == M_WAITOK) ? zalloc(flowq_zone) : - zalloc_noblock(flowq_zone); + fq = zalloc(flowq_zone); if (fq == NULL) { log(LOG_ERR, "%s: unable to allocate from flowq_zone\n"); return (NULL); } - bzero(fq, flowq_size); - MBUFQ_INIT(&fq->fq_mbufq); + bzero(fq, sizeof (*fq)); + fq->fq_ptype = ptype; + if (ptype == QP_MBUF) { + MBUFQ_INIT(&fq->fq_mbufq); + } return (fq); } void fq_destroy(fq_t *fq) { - VERIFY(MBUFQ_EMPTY(&fq->fq_mbufq)); + VERIFY(fq_empty(fq)); VERIFY(!(fq->fq_flags & (FQF_NEW_FLOW | FQF_OLD_FLOW))); - bzero(fq, flowq_size); + VERIFY(fq->fq_bytes == 0); zfree(flowq_zone, fq); } @@ -99,7 +104,7 @@ fq_detect_dequeue_stall(fq_if_t *fqs, fq_t *flowq, fq_if_classq_t *fq_cl, { u_int64_t maxgetqtime; if (FQ_IS_DELAYHIGH(flowq) || flowq->fq_getqtime == 0 || - MBUFQ_EMPTY(&flowq->fq_mbufq) || + fq_empty(flowq) || flowq->fq_bytes < FQ_MIN_FC_THRESHOLD_BYTES) return; maxgetqtime = flowq->fq_getqtime + fqs->fqs_update_interval; @@ -116,61 +121,78 @@ fq_detect_dequeue_stall(fq_if_t *fqs, fq_t *flowq, fq_if_classq_t *fq_cl, void fq_head_drop(fq_if_t *fqs, fq_t *fq) { - struct mbuf *m = NULL; + pktsched_pkt_t pkt; + uint32_t *pkt_flags; + uint64_t *pkt_timestamp; struct ifclassq *ifq = fqs->fqs_ifq; - m = fq_getq_flow(fqs, fq); - if (m == NULL) + _PKTSCHED_PKT_INIT(&pkt); + if (fq_getq_flow_internal(fqs, fq, &pkt) == NULL) return; - IFCQ_DROP_ADD(ifq, 1, m_length(m)); + pktsched_get_pkt_vars(&pkt, &pkt_flags, &pkt_timestamp, NULL, NULL, + NULL, NULL); + + *pkt_timestamp = 0; + if (pkt.pktsched_ptype == QP_MBUF) + *pkt_flags &= ~PKTF_PRIV_GUARDED; + + IFCQ_DROP_ADD(ifq, 1, pktsched_get_pkt_len(&pkt)); IFCQ_CONVERT_LOCK(ifq); - m_freem(m); + pktsched_free_pkt(&pkt); } int -fq_addq(fq_if_t *fqs, struct mbuf *m, fq_if_classq_t *fq_cl) +fq_addq(fq_if_t *fqs, pktsched_pkt_t *pkt, fq_if_classq_t *fq_cl) { - struct pkthdr *pkt = &m->m_pkthdr; int droptype = DTYPE_NODROP, fc_adv = 0, ret = CLASSQEQ_SUCCESS; u_int64_t now; fq_t *fq = NULL; + uint64_t *pkt_timestamp; + uint32_t *pkt_flags; + uint32_t pkt_flowid, pkt_tx_start_seq; + uint8_t pkt_proto, pkt_flowsrc; + + pktsched_get_pkt_vars(pkt, &pkt_flags, &pkt_timestamp, &pkt_flowid, + &pkt_flowsrc, &pkt_proto, &pkt_tx_start_seq); + + if (pkt->pktsched_ptype == QP_MBUF) { + /* See comments in <rdar://problem/14040693> */ + VERIFY(!(*pkt_flags & PKTF_PRIV_GUARDED)); + *pkt_flags |= PKTF_PRIV_GUARDED; + } - VERIFY(!(pkt->pkt_flags & PKTF_PRIV_GUARDED)); - pkt->pkt_flags |= PKTF_PRIV_GUARDED; - - if (pkt->pkt_timestamp > 0) { - now = pkt->pkt_timestamp; + if (*pkt_timestamp > 0) { + now = *pkt_timestamp; } else { - now = mach_absolute_time(); - pkt->pkt_timestamp = now; + struct timespec now_ts; + nanouptime(&now_ts); + now = (now_ts.tv_sec * NSEC_PER_SEC) + now_ts.tv_nsec; + *pkt_timestamp = now; } /* find the flowq for this packet */ - fq = fq_if_hash_pkt(fqs, pkt->pkt_flowid, m_get_service_class(m), - now, TRUE); + fq = fq_if_hash_pkt(fqs, pkt_flowid, pktsched_get_pkt_svc(pkt), + now, TRUE, pkt->pktsched_ptype); if (fq == NULL) { /* drop the packet if we could not allocate a flow queue */ fq_cl->fcl_stat.fcl_drop_memfailure++; IFCQ_CONVERT_LOCK(fqs->fqs_ifq); - m_freem(m); - return (CLASSQEQ_DROPPED); + return (CLASSQEQ_DROP); } - - VERIFY(fq_cl->fcl_service_class == - (u_int32_t)mbuf_get_service_class(m)); + VERIFY(fq->fq_ptype == pkt->pktsched_ptype); fq_detect_dequeue_stall(fqs, fq, fq_cl, &now); if (FQ_IS_DELAYHIGH(fq)) { if ((fq->fq_flags & FQF_FLOWCTL_CAPABLE) && - (pkt->pkt_flags & PKTF_FLOW_ADV)) { + (*pkt_flags & PKTF_FLOW_ADV)) { fc_adv = 1; /* * If the flow is suspended or it is not * TCP, drop the packet */ - if (pkt->pkt_proto != IPPROTO_TCP) { + if (pkt_proto != IPPROTO_TCP) { droptype = DTYPE_EARLY; fq_cl->fcl_stat.fcl_drop_early++; } @@ -179,7 +201,7 @@ fq_addq(fq_if_t *fqs, struct mbuf *m, fq_if_classq_t *fq_cl) * Need to drop a packet, instead of dropping this * one, try to drop from the head of the queue */ - if (!MBUFQ_EMPTY(&fq->fq_mbufq)) { + if (!fq_empty(fq)) { fq_head_drop(fqs, fq); droptype = DTYPE_NODROP; } else { @@ -190,28 +212,17 @@ fq_addq(fq_if_t *fqs, struct mbuf *m, fq_if_classq_t *fq_cl) } - /* - * check if this packet is a retransmission of another pkt already - * in the queue - */ - if ((pkt->pkt_flags & (PKTF_TCP_REXMT|PKTF_START_SEQ)) == - (PKTF_TCP_REXMT|PKTF_START_SEQ) && fq->fq_dequeue_seq != 0) { - if (FQ_SEQ_GT(pkt->tx_start_seq, fq->fq_dequeue_seq)) { - fq_cl->fcl_stat.fcl_dup_rexmts++; - droptype = DTYPE_FORCED; - } - } - /* Set the return code correctly */ if (fc_adv == 1 && droptype != DTYPE_FORCED) { - if (fq_if_add_fcentry(fqs, pkt, fq_cl)) { + if (fq_if_add_fcentry(fqs, pkt, pkt_flowid, pkt_flowsrc, + fq_cl)) { fq->fq_flags |= FQF_FLOWCTL_ON; /* deliver flow control advisory error */ if (droptype == DTYPE_NODROP) { ret = CLASSQEQ_SUCCESS_FC; } else { /* dropped due to flow control */ - ret = CLASSQEQ_DROPPED_FC; + ret = CLASSQEQ_DROP_FC; } } else { /* @@ -219,7 +230,7 @@ fq_addq(fq_if_t *fqs, struct mbuf *m, fq_if_classq_t *fq_cl) * better to drop */ droptype = DTYPE_FORCED; - ret = CLASSQEQ_DROPPED_FC; + ret = CLASSQEQ_DROP_FC; fq_cl->fcl_stat.fcl_flow_control_fail++; } } @@ -231,13 +242,38 @@ fq_addq(fq_if_t *fqs, struct mbuf *m, fq_if_classq_t *fq_cl) * tail drop. */ if (droptype == DTYPE_NODROP && fq_if_at_drop_limit(fqs)) { - fq_if_drop_packet(fqs); + if (fqs->fqs_large_flow == fq) { + /* + * Drop from the head of the current fq. Since a + * new packet will be added to the tail, it is ok + * to leave fq in place. + */ + fq_head_drop(fqs, fq); + } else { + if (fqs->fqs_large_flow == NULL) { + droptype = DTYPE_FORCED; + fq_cl->fcl_stat.fcl_drop_overflow++; + + /* + * if this fq was freshly created and there + * is nothing to enqueue, free it + */ + if (fq_empty(fq) && !(fq->fq_flags & + (FQF_NEW_FLOW | FQF_OLD_FLOW))) { + fq_if_destroy_flow(fqs, fq_cl, fq); + fq = NULL; + } + } else { + fq_if_drop_packet(fqs); + } + } } if (droptype == DTYPE_NODROP) { - MBUFQ_ENQUEUE(&fq->fq_mbufq, m); - fq->fq_bytes += m_length(m); - fq_cl->fcl_stat.fcl_byte_cnt += m_length(m); + uint32_t pkt_len = pktsched_get_pkt_len(pkt); + fq_enqueue(fq, pkt->pktsched_pkt); + fq->fq_bytes += pkt_len; + fq_cl->fcl_stat.fcl_byte_cnt += pkt_len; fq_cl->fcl_stat.fcl_pkt_cnt++; /* @@ -247,8 +283,7 @@ fq_addq(fq_if_t *fqs, struct mbuf *m, fq_if_classq_t *fq_cl) fq_if_is_flow_heavy(fqs, fq); } else { IFCQ_CONVERT_LOCK(fqs->fqs_ifq); - m_freem(m); - return ((ret != CLASSQEQ_SUCCESS) ? ret : CLASSQEQ_DROPPED); + return ((ret != CLASSQEQ_SUCCESS) ? ret : CLASSQEQ_DROP); } /* @@ -267,79 +302,97 @@ fq_addq(fq_if_t *fqs, struct mbuf *m, fq_if_classq_t *fq_cl) return (ret); } -struct mbuf * -fq_getq_flow(fq_if_t *fqs, fq_t *fq) +void * +fq_getq_flow_internal(fq_if_t *fqs, fq_t *fq, pktsched_pkt_t *pkt) { - struct mbuf *m = NULL; - struct ifclassq *ifq = fqs->fqs_ifq; + void *p; + uint32_t plen; fq_if_classq_t *fq_cl; - u_int64_t now; - int64_t qdelay; - struct pkthdr *pkt; - u_int32_t mlen; + struct ifclassq *ifq = fqs->fqs_ifq; - MBUFQ_DEQUEUE(&fq->fq_mbufq, m); - if (m == NULL) + fq_dequeue(fq, p); + if (p == NULL) return (NULL); - mlen = m_length(m); + pktsched_pkt_encap(pkt, fq->fq_ptype, p); + plen = pktsched_get_pkt_len(pkt); - VERIFY(fq->fq_bytes >= mlen); - fq->fq_bytes -= mlen; + VERIFY(fq->fq_bytes >= plen); + fq->fq_bytes -= plen; fq_cl = &fqs->fqs_classq[fq->fq_sc_index]; - fq_cl->fcl_stat.fcl_byte_cnt -= mlen; + fq_cl->fcl_stat.fcl_byte_cnt -= plen; fq_cl->fcl_stat.fcl_pkt_cnt--; IFCQ_DEC_LEN(ifq); - IFCQ_DEC_BYTES(ifq, mlen); + IFCQ_DEC_BYTES(ifq, plen); - pkt = &m->m_pkthdr; - now = mach_absolute_time(); + /* Reset getqtime so that we don't count idle times */ + if (fq_empty(fq)) + fq->fq_getqtime = 0; + + return (p); +} + +void * +fq_getq_flow(fq_if_t *fqs, fq_t *fq, pktsched_pkt_t *pkt) +{ + void *p; + fq_if_classq_t *fq_cl; + u_int64_t now; + int64_t qdelay = 0; + struct timespec now_ts; + uint32_t *pkt_flags, pkt_tx_start_seq; + uint64_t *pkt_timestamp; + + p = fq_getq_flow_internal(fqs, fq, pkt); + if (p == NULL) + return (NULL); + + pktsched_get_pkt_vars(pkt, &pkt_flags, &pkt_timestamp, NULL, NULL, + NULL, &pkt_tx_start_seq); + + nanouptime(&now_ts); + now = (now_ts.tv_sec * NSEC_PER_SEC) + now_ts.tv_nsec; /* this will compute qdelay in nanoseconds */ - qdelay = now - pkt->pkt_timestamp; + if (now > *pkt_timestamp) + qdelay = now - *pkt_timestamp; + fq_cl = &fqs->fqs_classq[fq->fq_sc_index]; if (fq->fq_min_qdelay == 0 || (qdelay > 0 && (u_int64_t)qdelay < fq->fq_min_qdelay)) fq->fq_min_qdelay = qdelay; - if (now >= fq->fq_updatetime || MBUFQ_EMPTY(&fq->fq_mbufq)) { - if (fq->fq_min_qdelay >= fqs->fqs_target_qdelay) { + if (now >= fq->fq_updatetime) { + if (fq->fq_min_qdelay > fqs->fqs_target_qdelay) { if (!FQ_IS_DELAYHIGH(fq)) FQ_SET_DELAY_HIGH(fq); - } - - if (!FQ_IS_DELAYHIGH(fq) || MBUFQ_EMPTY(&fq->fq_mbufq)) { + } else { FQ_CLEAR_DELAY_HIGH(fq); - if (fq->fq_flags & FQF_FLOWCTL_ON) { - fq_if_flow_feedback(fqs, fq, fq_cl); - } } + /* Reset measured queue delay and update time */ fq->fq_updatetime = now + fqs->fqs_update_interval; fq->fq_min_qdelay = 0; } + if (!FQ_IS_DELAYHIGH(fq) || fq_empty(fq)) { + FQ_CLEAR_DELAY_HIGH(fq); + if (fq->fq_flags & FQF_FLOWCTL_ON) { + fq_if_flow_feedback(fqs, fq, fq_cl); + } + } - if ((pkt->pkt_flags & PKTF_START_SEQ) && (fq->fq_dequeue_seq == 0 || - (FQ_SEQ_LT(fq->fq_dequeue_seq, pkt->tx_start_seq)))) - fq->fq_dequeue_seq = pkt->tx_start_seq; - - pkt->pkt_timestamp = 0; - pkt->pkt_flags &= ~PKTF_PRIV_GUARDED; - - if (MBUFQ_EMPTY(&fq->fq_mbufq)) { - /* - * Remove from large_flow field, if this happened to be - * the one that is tagged. - */ - if (fqs->fqs_large_flow == fq) - fqs->fqs_large_flow = NULL; - + if (fq_empty(fq)) { /* Reset getqtime so that we don't count idle times */ fq->fq_getqtime = 0; } else { fq->fq_getqtime = now; } + fq_if_is_flow_heavy(fqs, fq); + + *pkt_timestamp = 0; + if (pkt->pktsched_ptype == QP_MBUF) + *pkt_flags &= ~PKTF_PRIV_GUARDED; - return (m); + return (p); } diff --git a/bsd/net/classq/classq_fq_codel.h b/bsd/net/classq/classq_fq_codel.h index 35f8341b2..6683256c1 100644 --- a/bsd/net/classq/classq_fq_codel.h +++ b/bsd/net/classq/classq_fq_codel.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 Apple Inc. All rights reserved. + * Copyright (c) 2016-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -40,7 +40,7 @@ extern "C" { #endif #define FQ_MIN_FC_THRESHOLD_BYTES 7500 -#define FQ_IS_DELAYHIGH(_fq_) ((_fq_)->fq_flags & FQF_DELAY_HIGH) +#define FQ_IS_DELAYHIGH(_fq_) ((_fq_)->fq_flags & FQF_DELAY_HIGH) #define FQ_SET_DELAY_HIGH(_fq_) do { \ (_fq_)->fq_flags |= FQF_DELAY_HIGH; \ } while (0) @@ -49,7 +49,9 @@ extern "C" { } while (0) typedef struct flowq { - MBUFQ_HEAD(pktq_head) fq_mbufq; /* Packet queue */ + union { + MBUFQ_HEAD(mbufq_head) __mbufq; /* mbuf packet queue */ + } __fq_pktq_u; #define FQF_FLOWCTL_CAPABLE 0x01 /* Use flow control instead of drop */ #define FQF_DELAY_HIGH 0x02 /* Min delay is greater than target */ #define FQF_NEW_FLOW 0x04 /* Currently on new flows queue */ @@ -65,19 +67,34 @@ typedef struct flowq { SLIST_ENTRY(flowq) fq_hashlink; /* for flow queue hash table */ STAILQ_ENTRY(flowq) fq_actlink; /* for new/old flow queues */ u_int32_t fq_flowhash; /* Flow hash */ - u_int32_t fq_dequeue_seq; /* Last dequeue seq */ + classq_pkt_type_t fq_ptype; /* Packet type */ } fq_t; +#define fq_mbufq __fq_pktq_u.__mbufq + +#define fq_empty(_q) MBUFQ_EMPTY(&(_q)->fq_mbufq) + +#define fq_enqueue(_q, _p) MBUFQ_ENQUEUE(&(_q)->fq_mbufq, (mbuf_t)_p) + +#define fq_dequeue(_q, _p) do { \ + mbuf_t _m; \ + MBUFQ_DEQUEUE(&(_q)->fq_mbufq, _m); \ + (_p) = _m; \ +} while (0) + struct fq_codel_sched_data; struct fq_if_classq; /* Function definitions */ extern void fq_codel_init(void); -extern fq_t *fq_alloc(int); +extern fq_t *fq_alloc(classq_pkt_type_t); extern void fq_destroy(fq_t *); -extern int fq_addq(struct fq_codel_sched_data *, struct mbuf *, +extern int fq_addq(struct fq_codel_sched_data *, pktsched_pkt_t *, struct fq_if_classq *); -extern struct mbuf *fq_getq_flow(struct fq_codel_sched_data *, fq_t *); +extern void *fq_getq_flow(struct fq_codel_sched_data *, fq_t *, + pktsched_pkt_t *); +extern void *fq_getq_flow_internal(struct fq_codel_sched_data *, + fq_t *, pktsched_pkt_t *); extern void fq_head_drop(struct fq_codel_sched_data *, fq_t *); #ifdef __cplusplus diff --git a/bsd/net/classq/classq_red.c b/bsd/net/classq/classq_red.c deleted file mode 100644 index 63cbd8ede..000000000 --- a/bsd/net/classq/classq_red.c +++ /dev/null @@ -1,630 +0,0 @@ -/* - * Copyright (c) 2007-2012 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* $OpenBSD: altq_red.c,v 1.14 2007/09/13 20:40:02 chl Exp $ */ -/* $KAME: altq_red.c,v 1.10 2002/04/03 05:38:51 kjc Exp $ */ - -/* - * Copyright (C) 1997-2003 - * Sony Computer Science Laboratories Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY SONY CSL AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL SONY CSL OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - */ -/* - * Copyright (c) 1990-1994 Regents of the University of California. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the Computer Systems - * Engineering Group at Lawrence Berkeley Laboratory. - * 4. Neither the name of the University nor of the Laboratory may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include <sys/cdefs.h> - -#if CLASSQ_RED - -#include <sys/param.h> -#include <sys/malloc.h> -#include <sys/mbuf.h> -#include <sys/socket.h> -#include <sys/systm.h> -#include <sys/syslog.h> -#include <sys/errno.h> -#include <sys/kauth.h> -#include <dev/random/randomdev.h> -#include <kern/zalloc.h> - -#include <net/if.h> - -#include <netinet/in.h> -#include <netinet/in_systm.h> -#include <netinet/ip.h> -#if INET6 -#include <netinet/ip6.h> -#endif - -#include <net/classq/classq_red.h> -#include <net/net_osdep.h> - -/* - * ALTQ/RED (Random Early Detection) implementation using 32-bit - * fixed-point calculation. - * - * written by kjc using the ns code as a reference. - * you can learn more about red and ns from Sally's home page at - * http://www-nrg.ee.lbl.gov/floyd/ - * - * most of the red parameter values are fixed in this implementation - * to prevent fixed-point overflow/underflow. - * if you change the parameters, watch out for overflow/underflow! - * - * the parameters used are recommended values by Sally. - * the corresponding ns config looks: - * q_weight=0.00195 - * minthresh=5 maxthresh=15 queue-size=60 - * linterm=30 - * dropmech=drop-tail - * bytes=false (can't be handled by 32-bit fixed-point) - * doubleq=false dqthresh=false - * wait=true - */ -/* - * alternative red parameters for a slow link. - * - * assume the queue length becomes from zero to L and keeps L, it takes - * N packets for q_avg to reach 63% of L. - * when q_weight is 0.002, N is about 500 packets. - * for a slow link like dial-up, 500 packets takes more than 1 minute! - * when q_weight is 0.008, N is about 127 packets. - * when q_weight is 0.016, N is about 63 packets. - * bursts of 50 packets are allowed for 0.002, bursts of 25 packets - * are allowed for 0.016. - * see Sally's paper for more details. - */ -/* normal red parameters */ -#define W_WEIGHT 512 /* inverse of weight of EWMA (511/512) */ - /* q_weight = 0.00195 */ - -/* red parameters for a slow link */ -#define W_WEIGHT_1 128 /* inverse of weight of EWMA (127/128) */ - /* q_weight = 0.0078125 */ - -/* red parameters for a very slow link (e.g., dialup) */ -#define W_WEIGHT_2 64 /* inverse of weight of EWMA (63/64) */ - /* q_weight = 0.015625 */ - -/* fixed-point uses 12-bit decimal places */ -#define FP_SHIFT 12 /* fixed-point shift */ - -/* red parameters for drop probability */ -#define INV_P_MAX 10 /* inverse of max drop probability */ -#define TH_MIN 5 /* min threshold */ -#define TH_MAX 15 /* max threshold */ - -#define RED_LIMIT 60 /* default max queue lenght */ - -#define RED_ZONE_MAX 32 /* maximum elements in zone */ -#define RED_ZONE_NAME "classq_red" /* zone name */ - -static unsigned int red_size; /* size of zone element */ -static struct zone *red_zone; /* zone for red */ - -/* - * our default policy for forced-drop is drop-tail. - * (in altq-1.1.2 or earlier, the default was random-drop. - * but it makes more sense to punish the cause of the surge.) - * to switch to the random-drop policy, define "RED_RANDOM_DROP". - */ - -/* default red parameter values */ -static int default_th_min = TH_MIN; -static int default_th_max = TH_MAX; -static int default_inv_pmax = INV_P_MAX; - -static struct mbuf *red_getq_flow(struct red *, class_queue_t *, - u_int32_t, boolean_t); - -void -red_init(void) -{ - _CASSERT(REDF_ECN4 == CLASSQF_ECN4); - _CASSERT(REDF_ECN6 == CLASSQF_ECN6); - - red_size = sizeof (red_t); - red_zone = zinit(red_size, RED_ZONE_MAX * red_size, - 0, RED_ZONE_NAME); - if (red_zone == NULL) { - panic("%s: failed allocating %s", __func__, RED_ZONE_NAME); - /* NOTREACHED */ - } - zone_change(red_zone, Z_EXPAND, TRUE); - zone_change(red_zone, Z_CALLERACCT, TRUE); -} - -/* - * red support routines - */ -red_t * -red_alloc(struct ifnet *ifp, int weight, int inv_pmax, int th_min, - int th_max, int flags, int pkttime) -{ - red_t *rp; - int w, i; - int npkts_per_sec; - - VERIFY(ifp != NULL); - - rp = zalloc(red_zone); - if (rp == NULL) - return (NULL); - - bzero(rp, red_size); - rp->red_avg = 0; - rp->red_idle = 1; - - if (weight == 0) - rp->red_weight = W_WEIGHT; - else - rp->red_weight = weight; - if (inv_pmax == 0) - rp->red_inv_pmax = default_inv_pmax; - else - rp->red_inv_pmax = inv_pmax; - if (th_min == 0) - rp->red_thmin = default_th_min; - else - rp->red_thmin = th_min; - if (th_max == 0) - rp->red_thmax = default_th_max; - else - rp->red_thmax = th_max; - - rp->red_ifp = ifp; - rp->red_flags = (flags & REDF_USERFLAGS); -#if !PF_ECN - if (rp->red_flags & REDF_ECN) { - rp->red_flags &= ~REDF_ECN; - log(LOG_ERR, "%s: RED ECN not available; ignoring " - "REDF_ECN flag!\n", if_name(ifp)); - } -#endif /* !PF_ECN */ - - if (pkttime == 0) - /* default packet time: 1000 bytes / 10Mbps * 8 * 1000000 */ - rp->red_pkttime = 800; - else - rp->red_pkttime = pkttime; - - if (weight == 0) { - /* when the link is very slow, adjust red parameters */ - npkts_per_sec = 1000000 / rp->red_pkttime; - if (npkts_per_sec < 50) { - /* up to about 400Kbps */ - rp->red_weight = W_WEIGHT_2; - } else if (npkts_per_sec < 300) { - /* up to about 2.4Mbps */ - rp->red_weight = W_WEIGHT_1; - } - } - - /* calculate wshift. weight must be power of 2 */ - w = rp->red_weight; - for (i = 0; w > 1; i++) - w = w >> 1; - rp->red_wshift = i; - w = 1 << rp->red_wshift; - if (w != rp->red_weight) { - printf("invalid weight value %d for red! use %d\n", - rp->red_weight, w); - rp->red_weight = w; - } - - /* - * thmin_s and thmax_s are scaled versions of th_min and th_max - * to be compared with avg. - */ - rp->red_thmin_s = rp->red_thmin << (rp->red_wshift + FP_SHIFT); - rp->red_thmax_s = rp->red_thmax << (rp->red_wshift + FP_SHIFT); - - /* - * precompute probability denominator - * probd = (2 * (TH_MAX-TH_MIN) / pmax) in fixed-point - */ - rp->red_probd = (2 * (rp->red_thmax - rp->red_thmin) * - rp->red_inv_pmax) << FP_SHIFT; - - /* allocate weight table */ - rp->red_wtab = wtab_alloc(rp->red_weight); - if (rp->red_wtab == NULL) { - red_destroy(rp); - return (NULL); - } - - microuptime(&rp->red_last); - return (rp); -} - -void -red_destroy(red_t *rp) -{ - if (rp->red_wtab != NULL) { - wtab_destroy(rp->red_wtab); - rp->red_wtab = NULL; - } - zfree(red_zone, rp); -} - -void -red_getstats(red_t *rp, struct red_stats *sp) -{ - sp->q_avg = rp->red_avg >> rp->red_wshift; - sp->drop_forced = rp->red_stats.drop_forced; - sp->drop_unforced = rp->red_stats.drop_unforced; - sp->marked_packets = rp->red_stats.marked_packets; -} - -int -red_addq(red_t *rp, class_queue_t *q, struct mbuf *m, struct pf_mtag *tag) -{ -#if !PF_ECN -#pragma unused(tag) -#endif /* !PF_ECN */ - int avg, droptype; - int n; - - avg = rp->red_avg; - - /* - * if we were idle, we pretend that n packets arrived during - * the idle period. - */ - if (rp->red_idle) { - struct timeval now; - int t; - - rp->red_idle = 0; - microuptime(&now); - t = (now.tv_sec - rp->red_last.tv_sec); - if (t > 60) { - /* - * being idle for more than 1 minute, set avg to zero. - * this prevents t from overflow. - */ - avg = 0; - } else { - t = t * 1000000 + (now.tv_usec - rp->red_last.tv_usec); - n = t / rp->red_pkttime - 1; - - /* the following line does (avg = (1 - Wq)^n * avg) */ - if (n > 0) - avg = (avg >> FP_SHIFT) * - pow_w(rp->red_wtab, n); - } - } - - /* run estimator. (note: avg is scaled by WEIGHT in fixed-point) */ - avg += (qlen(q) << FP_SHIFT) - (avg >> rp->red_wshift); - rp->red_avg = avg; /* save the new value */ - - /* - * red_count keeps a tally of arriving traffic that has not - * been dropped. - */ - rp->red_count++; - - /* see if we drop early */ - droptype = DTYPE_NODROP; - if (avg >= rp->red_thmin_s && qlen(q) > 1) { - if (avg >= rp->red_thmax_s) { - /* avg >= th_max: forced drop */ - droptype = DTYPE_FORCED; - } else if (rp->red_old == 0) { - /* first exceeds th_min */ - rp->red_count = 1; - rp->red_old = 1; - } else if (drop_early((avg - rp->red_thmin_s) >> rp->red_wshift, - rp->red_probd, rp->red_count)) { - /* mark or drop by red */ -#if PF_ECN - if ((rp->red_flags & REDF_ECN) && - (tag->pftag_proto == IPPROTO_TCP) && /* only TCP */ - mark_ecn(m, tag, rp->red_flags)) { - /* successfully marked. do not drop. */ - rp->red_count = 0; - rp->red_stats.marked_packets++; - } else -#endif /* PF_ECN */ - { - /* unforced drop by red */ - droptype = DTYPE_EARLY; - } - } - } else { - /* avg < th_min */ - rp->red_old = 0; - } - - /* - * if the queue length hits the hard limit, it's a forced drop. - */ - if (droptype == DTYPE_NODROP && qlen(q) >= qlimit(q)) - droptype = DTYPE_FORCED; - -#ifdef RED_RANDOM_DROP - /* if successful or forced drop, enqueue this packet. */ - if (droptype != DTYPE_EARLY) - _addq(q, m); -#else - /* if successful, enqueue this packet. */ - if (droptype == DTYPE_NODROP) - _addq(q, m); -#endif - if (droptype != DTYPE_NODROP) { - if (droptype == DTYPE_EARLY) { - /* drop the incoming packet */ - rp->red_stats.drop_unforced++; - } else { - /* forced drop, select a victim packet in the queue. */ -#ifdef RED_RANDOM_DROP - m = _getq_random(q); -#endif - rp->red_stats.drop_forced++; - } - rp->red_count = 0; - IFCQ_CONVERT_LOCK(&rp->red_ifp->if_snd); - m_freem(m); - return (CLASSQEQ_DROPPED); - } - /* successfully queued */ - return (CLASSQEQ_SUCCESS); -} - -/* - * early-drop probability is calculated as follows: - * prob = p_max * (avg - th_min) / (th_max - th_min) - * prob_a = prob / (2 - count*prob) - * = (avg-th_min) / (2*(th_max-th_min)*inv_p_max - count*(avg-th_min)) - * here prob_a increases as successive undrop count increases. - * (prob_a starts from prob/2, becomes prob when (count == (1 / prob)), - * becomes 1 when (count >= (2 / prob))). - */ -int -drop_early(int fp_len, int fp_probd, int count) -{ - int d; /* denominator of drop-probability */ - - d = fp_probd - count * fp_len; - if (d <= 0) - /* count exceeds the hard limit: drop or mark */ - return (1); - - /* - * now the range of d is [1..600] in fixed-point. (when - * th_max-th_min=10 and p_max=1/30) - * drop probability = (avg - TH_MIN) / d - */ - - if ((RandomULong() % d) < (unsigned)fp_len) { - /* drop or mark */ - return (1); - } - /* no drop/mark */ - return (0); -} - -static struct mbuf * -red_getq_flow(struct red *rp, class_queue_t *q, u_int32_t flow, boolean_t purge) -{ -#pragma unused(purge) - struct mbuf *m; - - /* flow of 0 means head of queue */ - if ((m = ((flow == 0) ? _getq(q) : _getq_flow(q, flow))) == NULL) { - if (rp->red_idle == 0) { - rp->red_idle = 1; - microuptime(&rp->red_last); - } - return (NULL); - } - - rp->red_idle = 0; - return (m); -} - -struct mbuf * -red_getq(red_t *rp, class_queue_t *q) -{ - return (red_getq_flow(rp, q, 0, FALSE)); -} - -void -red_purgeq(struct red *rp, class_queue_t *q, u_int32_t flow, u_int32_t *packets, - u_int32_t *bytes) -{ - u_int32_t cnt = 0, len = 0; - struct mbuf *m; - - IFCQ_CONVERT_LOCK(&rp->red_ifp->if_snd); - - while ((m = red_getq_flow(rp, q, flow, TRUE)) != NULL) { - cnt++; - len += m_pktlen(m); - m_freem(m); - } - - if (packets != NULL) - *packets = cnt; - if (bytes != NULL) - *bytes = len; -} - -void -red_updateq(red_t *rp, cqev_t ev) -{ -#pragma unused(rp, ev) - /* nothing for now */ -} - -int -red_suspendq(red_t *rp, class_queue_t *q, boolean_t on) -{ -#pragma unused(rp, q, on) - return (ENOTSUP); -} - -/* - * helper routine to calibrate avg during idle. - * pow_w(wtab, n) returns (1 - Wq)^n in fixed-point - * here Wq = 1/weight and the code assumes Wq is close to zero. - * - * w_tab[n] holds ((1 - Wq)^(2^n)) in fixed-point. - */ -static struct wtab *wtab_list = NULL; /* pointer to wtab list */ - -struct wtab * -wtab_alloc(int weight) -{ - struct wtab *w; - int i; - - for (w = wtab_list; w != NULL; w = w->w_next) - if (w->w_weight == weight) { - w->w_refcount++; - return (w); - } - - w = _MALLOC(sizeof (struct wtab), M_DEVBUF, M_WAITOK|M_ZERO); - if (w == NULL) - return (NULL); - - w->w_weight = weight; - w->w_refcount = 1; - w->w_next = wtab_list; - wtab_list = w; - - /* initialize the weight table */ - w->w_tab[0] = ((weight - 1) << FP_SHIFT) / weight; - for (i = 1; i < 32; i++) { - w->w_tab[i] = (w->w_tab[i-1] * w->w_tab[i-1]) >> FP_SHIFT; - if (w->w_tab[i] == 0 && w->w_param_max == 0) - w->w_param_max = 1 << i; - } - - return (w); -} - -void -wtab_destroy(struct wtab *w) -{ - struct wtab *prev; - - if (--w->w_refcount > 0) - return; - - if (wtab_list == w) - wtab_list = w->w_next; - else for (prev = wtab_list; prev->w_next != NULL; prev = prev->w_next) - if (prev->w_next == w) { - prev->w_next = w->w_next; - break; - } - - _FREE(w, M_DEVBUF); -} - -int32_t -pow_w(struct wtab *w, int n) -{ - int i, bit; - int32_t val; - - if (n >= w->w_param_max) - return (0); - - val = 1 << FP_SHIFT; - if (n <= 0) - return (val); - - bit = 1; - i = 0; - while (n) { - if (n & bit) { - val = (val * w->w_tab[i]) >> FP_SHIFT; - n &= ~bit; - } - i++; - bit <<= 1; - } - return (val); -} - -#endif /* CLASSQ_RED */ diff --git a/bsd/net/classq/classq_red.h b/bsd/net/classq/classq_red.h index 58956b504..ceae286dc 100644 --- a/bsd/net/classq/classq_red.h +++ b/bsd/net/classq/classq_red.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2012 Apple Inc. All rights reserved. + * Copyright (c) 2011-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -85,82 +85,6 @@ struct red_stats { u_int32_t marked_packets; }; -#ifdef BSD_KERNEL_PRIVATE -/* weight table structure for idle time calibration */ -struct wtab { - struct wtab *w_next; - int w_weight; - int w_param_max; - int w_refcount; - int32_t w_tab[32]; -}; - -/* red flags */ -#define REDF_ECN4 0x01 /* use packet marking for IPv4 packets */ -#define REDF_ECN6 0x02 /* use packet marking for IPv6 packets */ -#define REDF_ECN (REDF_ECN4 | REDF_ECN6) -#define REDF_FLOWVALVE 0x04 /* use flowvalve (aka penalty-box) */ - -#define REDF_USERFLAGS \ - (REDF_ECN4 | REDF_ECN6 | REDF_FLOWVALVE) - -typedef struct red { - int red_pkttime; /* average packet time in micro sec */ - /* used for idle calibration */ - int red_flags; /* red flags */ - struct ifnet *red_ifp; /* back pointer to ifnet */ - - /* red parameters */ - int red_weight; /* weight for EWMA */ - int red_inv_pmax; /* inverse of max drop probability */ - int red_thmin; /* red min threshold */ - int red_thmax; /* red max threshold */ - - /* variables for internal use */ - int red_wshift; /* log(red_weight) */ - int red_thmin_s; /* th_min scaled by avgshift */ - int red_thmax_s; /* th_max scaled by avgshift */ - int red_probd; /* drop probability denominator */ - - int red_avg; /* queue len avg scaled by avgshift */ - int red_count; /* packet count since last dropped/ */ - /* marked packet */ - int red_idle; /* queue was empty */ - int red_old; /* avg is above th_min */ - struct wtab *red_wtab; /* weight table */ - struct timeval red_last; /* time when the queue becomes idle */ - - struct { - struct pktcntr xmit_cnt; - struct pktcntr drop_cnt; - u_int32_t drop_forced; - u_int32_t drop_unforced; - u_int32_t marked_packets; - } red_stats; -} red_t; - -/* red drop types */ -#define DTYPE_NODROP 0 /* no drop */ -#define DTYPE_FORCED 1 /* a "forced" drop */ -#define DTYPE_EARLY 2 /* an "unforced" (early) drop */ - -extern void red_init(void); -extern red_t *red_alloc(struct ifnet *, int, int, int, int, int, int); -extern void red_destroy(red_t *); -extern void red_getstats(red_t *, struct red_stats *); -extern int red_addq(red_t *, class_queue_t *, struct mbuf *, struct pf_mtag *); -extern struct mbuf *red_getq(red_t *, class_queue_t *); -extern void red_purgeq(struct red *, class_queue_t *, u_int32_t, - u_int32_t *, u_int32_t *); -extern void red_updateq(red_t *, cqev_t); -extern int red_suspendq(red_t *, class_queue_t *, boolean_t); - -extern int drop_early(int, int, int); -extern struct wtab *wtab_alloc(int); -extern void wtab_destroy(struct wtab *); -extern int32_t pow_w(struct wtab *, int); -#endif /* BSD_KERNEL_PRIVATE */ - #ifdef __cplusplus } #endif diff --git a/bsd/net/classq/classq_rio.c b/bsd/net/classq/classq_rio.c deleted file mode 100644 index 91f7da00a..000000000 --- a/bsd/net/classq/classq_rio.c +++ /dev/null @@ -1,554 +0,0 @@ -/* - * Copyright (c) 2007-2013 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* $OpenBSD: altq_rio.c,v 1.11 2007/09/13 20:40:02 chl Exp $ */ -/* $KAME: altq_rio.c,v 1.8 2000/12/14 08:12:46 thorpej Exp $ */ - -/* - * Copyright (C) 1998-2003 - * Sony Computer Science Laboratories Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY SONY CSL AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL SONY CSL OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ -/* - * Copyright (c) 1990-1994 Regents of the University of California. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the Computer Systems - * Engineering Group at Lawrence Berkeley Laboratory. - * 4. Neither the name of the University nor of the Laboratory may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include <sys/cdefs.h> - -#if CLASSQ_RIO - -#include <sys/param.h> -#include <sys/malloc.h> -#include <sys/mbuf.h> -#include <sys/socket.h> -#include <sys/systm.h> -#include <sys/syslog.h> -#include <sys/errno.h> -#include <sys/kauth.h> -#include <sys/kauth.h> - -#include <kern/zalloc.h> - -#include <net/if.h> - -#include <netinet/in.h> -#include <netinet/in_systm.h> -#include <netinet/ip.h> -#if INET6 -#include <netinet/ip6.h> -#endif - -#include <net/classq/classq_red.h> -#include <net/classq/classq_rio.h> -#include <net/net_osdep.h> - -/* - * RIO: RED with IN/OUT bit - * described in - * "Explicit Allocation of Best Effort Packet Delivery Service" - * David D. Clark and Wenjia Fang, MIT Lab for Computer Science - * http://diffserv.lcs.mit.edu/Papers/exp-alloc-ddc-wf.{ps,pdf} - * - * this implementation is extended to support more than 2 drop precedence - * values as described in RFC2597 (Assured Forwarding PHB Group). - * - */ -/* - * AF DS (differentiated service) codepoints. - * (classes can be mapped to CBQ or H-FSC classes.) - * - * 0 1 2 3 4 5 6 7 - * +---+---+---+---+---+---+---+---+ - * | CLASS |DropPre| 0 | CU | - * +---+---+---+---+---+---+---+---+ - * - * class 1: 001 - * class 2: 010 - * class 3: 011 - * class 4: 100 - * - * low drop prec: 01 - * medium drop prec: 10 - * high drop prec: 11 - */ - -/* normal red parameters */ -#define W_WEIGHT 512 /* inverse of weight of EWMA (511/512) */ - /* q_weight = 0.00195 */ - -/* red parameters for a slow link */ -#define W_WEIGHT_1 128 /* inverse of weight of EWMA (127/128) */ - /* q_weight = 0.0078125 */ - -/* red parameters for a very slow link (e.g., dialup) */ -#define W_WEIGHT_2 64 /* inverse of weight of EWMA (63/64) */ - /* q_weight = 0.015625 */ - -/* fixed-point uses 12-bit decimal places */ -#define FP_SHIFT 12 /* fixed-point shift */ - -/* red parameters for drop probability */ -#define INV_P_MAX 10 /* inverse of max drop probability */ -#define TH_MIN 5 /* min threshold */ -#define TH_MAX 15 /* max threshold */ - -#define RIO_LIMIT 60 /* default max queue lenght */ - -/* default rio parameter values */ -static struct redparams default_rio_params[RIO_NDROPPREC] = { - /* th_min, th_max, inv_pmax */ - { TH_MAX * 2 + TH_MIN, TH_MAX * 3, INV_P_MAX }, /* low drop precedence */ - { TH_MAX + TH_MIN, TH_MAX * 2, INV_P_MAX }, /* medium drop precedence */ - { TH_MIN, TH_MAX, INV_P_MAX } /* high drop precedence */ -}; - -#define RIO_ZONE_MAX 32 /* maximum elements in zone */ -#define RIO_ZONE_NAME "classq_rio" /* zone name */ - -static unsigned int rio_size; /* size of zone element */ -static struct zone *rio_zone; /* zone for rio */ - -/* internal function prototypes */ -static struct mbuf *rio_getq_flow(struct rio *, class_queue_t *, - u_int32_t, boolean_t); -static int dscp2index(u_int8_t); - -void -rio_init(void) -{ - _CASSERT(RIOF_ECN4 == CLASSQF_ECN4); - _CASSERT(RIOF_ECN6 == CLASSQF_ECN6); - - rio_size = sizeof (rio_t); - rio_zone = zinit(rio_size, RIO_ZONE_MAX * rio_size, - 0, RIO_ZONE_NAME); - if (rio_zone == NULL) { - panic("%s: failed allocating %s", __func__, RIO_ZONE_NAME); - /* NOTREACHED */ - } - zone_change(rio_zone, Z_EXPAND, TRUE); - zone_change(rio_zone, Z_CALLERACCT, TRUE); -} - -rio_t * -rio_alloc(struct ifnet *ifp, int weight, struct redparams *params, - int flags, int pkttime) -{ - rio_t *rp; - int w, i; - int npkts_per_sec; - - VERIFY(ifp != NULL); - - rp = zalloc(rio_zone); - if (rp == NULL) - return (NULL); - - bzero(rp, rio_size); - rp->rio_ifp = ifp; - rp->rio_flags = (flags & RIOF_USERFLAGS); -#if !PF_ECN - if (rp->rio_flags & RIOF_ECN) { - rp->rio_flags &= ~RIOF_ECN; - log(LOG_ERR, "%s: RIO ECN not available; ignoring " - "RIOF_ECN flag!\n", if_name(ifp)); - } - if (rp->rio_flags & RIOF_CLEARDSCP) { - rp->rio_flags &= ~RIOF_CLEARDSCP; - log(LOG_ERR, "%s: RIO ECN not available; ignoring " - "RIOF_CLEARDSCP flag!\n", if_name(ifp)); - } -#endif /* !PF_ECN */ - - if (pkttime == 0) - /* default packet time: 1000 bytes / 10Mbps * 8 * 1000000 */ - rp->rio_pkttime = 800; - else - rp->rio_pkttime = pkttime; - - if (weight != 0) - rp->rio_weight = weight; - else { - /* use default */ - rp->rio_weight = W_WEIGHT; - - /* when the link is very slow, adjust red parameters */ - npkts_per_sec = 1000000 / rp->rio_pkttime; - if (npkts_per_sec < 50) { - /* up to about 400Kbps */ - rp->rio_weight = W_WEIGHT_2; - } else if (npkts_per_sec < 300) { - /* up to about 2.4Mbps */ - rp->rio_weight = W_WEIGHT_1; - } - } - - /* calculate wshift. weight must be power of 2 */ - w = rp->rio_weight; - for (i = 0; w > 1; i++) - w = w >> 1; - rp->rio_wshift = i; - w = 1 << rp->rio_wshift; - if (w != rp->rio_weight) { - printf("invalid weight value %d for red! use %d\n", - rp->rio_weight, w); - rp->rio_weight = w; - } - - /* allocate weight table */ - rp->rio_wtab = wtab_alloc(rp->rio_weight); - if (rp->rio_wtab == NULL) { - rio_destroy(rp); - return (NULL); - } - - for (i = 0; i < RIO_NDROPPREC; i++) { - struct dropprec_state *prec = &rp->rio_precstate[i]; - - prec->avg = 0; - prec->idle = 1; - - if (params == NULL || params[i].inv_pmax == 0) - prec->inv_pmax = default_rio_params[i].inv_pmax; - else - prec->inv_pmax = params[i].inv_pmax; - if (params == NULL || params[i].th_min == 0) - prec->th_min = default_rio_params[i].th_min; - else - prec->th_min = params[i].th_min; - if (params == NULL || params[i].th_max == 0) - prec->th_max = default_rio_params[i].th_max; - else - prec->th_max = params[i].th_max; - - /* - * th_min_s and th_max_s are scaled versions of th_min - * and th_max to be compared with avg. - */ - prec->th_min_s = prec->th_min << (rp->rio_wshift + FP_SHIFT); - prec->th_max_s = prec->th_max << (rp->rio_wshift + FP_SHIFT); - - /* - * precompute probability denominator - * probd = (2 * (TH_MAX-TH_MIN) / pmax) in fixed-point - */ - prec->probd = (2 * (prec->th_max - prec->th_min) * - prec->inv_pmax) << FP_SHIFT; - - microuptime(&prec->last); - } - - return (rp); -} - -void -rio_destroy(rio_t *rp) -{ - if (rp->rio_wtab != NULL) { - wtab_destroy(rp->rio_wtab); - rp->rio_wtab = NULL; - } - zfree(rio_zone, rp); -} - -void -rio_getstats(rio_t *rp, struct red_stats *sp) -{ - int i; - - for (i = 0; i < RIO_NDROPPREC; i++) { - bcopy(&rp->q_stats[i], sp, sizeof (struct red_stats)); - sp->q_avg = rp->rio_precstate[i].avg >> rp->rio_wshift; - sp++; - } -} - -#if (RIO_NDROPPREC == 3) -/* - * internally, a drop precedence value is converted to an index - * starting from 0. - */ -static int -dscp2index(u_int8_t dscp) -{ -#define AF_DROPPRECMASK 0x18 - - int dpindex = dscp & AF_DROPPRECMASK; - - if (dpindex == 0) - return (0); - return ((dpindex >> 3) - 1); -} -#endif - -/* Store RIO precindex in the module private scratch space */ -#define pkt_precidx pkt_mpriv.__mpriv_u.__mpriv32[0].__mpriv32_u.__val32 - -#define RIOM_SET_PRECINDEX(pkt, idx) do { \ - (pkt)->pkt_precidx = (idx); \ -} while (0) - -#define RIOM_GET_PRECINDEX(pkt) \ - ({ u_int32_t idx; idx = (pkt)->pkt_precidx; \ - RIOM_SET_PRECINDEX(pkt, 0); idx; }) - -int -rio_addq(rio_t *rp, class_queue_t *q, struct mbuf *m, struct pf_mtag *tag) -{ -#if !PF_ECN -#pragma unused(tag) -#endif /* !PF_ECN */ -#define DSCP_MASK 0xfc - int avg, droptype; - u_int8_t dsfield, odsfield; - int dpindex, i, n, t; - struct timeval now; - struct dropprec_state *prec; - -#if PF_ECN - dsfield = odsfield = read_dsfield(m, tag); -#else - dsfield = odsfield = 0; -#endif /* !PF_ECN */ - dpindex = dscp2index(dsfield); - - /* - * update avg of the precedence states whose drop precedence - * is larger than or equal to the drop precedence of the packet - */ - now.tv_sec = 0; - for (i = dpindex; i < RIO_NDROPPREC; i++) { - prec = &rp->rio_precstate[i]; - avg = prec->avg; - if (prec->idle) { - prec->idle = 0; - if (now.tv_sec == 0) - microuptime(&now); - t = (now.tv_sec - prec->last.tv_sec); - if (t > 60) - avg = 0; - else { - t = t * 1000000 + - (now.tv_usec - prec->last.tv_usec); - n = t / rp->rio_pkttime; - /* calculate (avg = (1 - Wq)^n * avg) */ - if (n > 0) { - avg = (avg >> FP_SHIFT) * - pow_w(rp->rio_wtab, n); - } - } - } - - /* run estimator. (avg is scaled by WEIGHT in fixed-point) */ - avg += (prec->qlen << FP_SHIFT) - (avg >> rp->rio_wshift); - prec->avg = avg; /* save the new value */ - /* - * count keeps a tally of arriving traffic that has not - * been dropped. - */ - prec->count++; - } - - prec = &rp->rio_precstate[dpindex]; - avg = prec->avg; - - /* see if we drop early */ - droptype = DTYPE_NODROP; - if (avg >= prec->th_min_s && prec->qlen > 1) { - if (avg >= prec->th_max_s) { - /* avg >= th_max: forced drop */ - droptype = DTYPE_FORCED; - } else if (prec->old == 0) { - /* first exceeds th_min */ - prec->count = 1; - prec->old = 1; - } else if (drop_early((avg - prec->th_min_s) >> rp->rio_wshift, - prec->probd, prec->count)) { - /* unforced drop by red */ - droptype = DTYPE_EARLY; - } - } else { - /* avg < th_min */ - prec->old = 0; - } - - /* - * if the queue length hits the hard limit, it's a forced drop. - */ - if (droptype == DTYPE_NODROP && qlen(q) >= qlimit(q)) - droptype = DTYPE_FORCED; - - if (droptype != DTYPE_NODROP) { - /* always drop incoming packet (as opposed to randomdrop) */ - for (i = dpindex; i < RIO_NDROPPREC; i++) - rp->rio_precstate[i].count = 0; - - if (droptype == DTYPE_EARLY) - rp->q_stats[dpindex].drop_unforced++; - else - rp->q_stats[dpindex].drop_forced++; - - IFCQ_CONVERT_LOCK(&rp->rio_ifp->if_snd); - m_freem(m); - return (CLASSQEQ_DROPPED); - } - - for (i = dpindex; i < RIO_NDROPPREC; i++) - rp->rio_precstate[i].qlen++; - - /* save drop precedence index in mbuf hdr */ - RIOM_SET_PRECINDEX(&m->m_pkthdr, dpindex); - - if (rp->rio_flags & RIOF_CLEARDSCP) - dsfield &= ~DSCP_MASK; - -#if PF_ECN - if (dsfield != odsfield) - write_dsfield(m, tag, dsfield); -#endif /* PF_ECN */ - - _addq(q, m); - - return (CLASSQEQ_SUCCESS); -} - -static struct mbuf * -rio_getq_flow(struct rio *rp, class_queue_t *q, u_int32_t flow, boolean_t purge) -{ -#pragma unused(purge) - struct mbuf *m; - int dpindex, i; - - /* flow of 0 means head of queue */ - if ((m = ((flow == 0) ? _getq(q) : _getq_flow(q, flow))) == NULL) - return (NULL); - - VERIFY(m->m_flags & M_PKTHDR); - - dpindex = RIOM_GET_PRECINDEX(&m->m_pkthdr); - for (i = dpindex; i < RIO_NDROPPREC; i++) { - if (--rp->rio_precstate[i].qlen == 0) { - if (rp->rio_precstate[i].idle == 0) { - rp->rio_precstate[i].idle = 1; - microuptime(&rp->rio_precstate[i].last); - } - } - } - return (m); -} - -struct mbuf * -rio_getq(rio_t *rp, class_queue_t *q) -{ - return (rio_getq_flow(rp, q, 0, FALSE)); -} - -void -rio_purgeq(struct rio *rp, class_queue_t *q, u_int32_t flow, u_int32_t *packets, - u_int32_t *bytes) -{ - u_int32_t cnt = 0, len = 0; - struct mbuf *m; - - IFCQ_CONVERT_LOCK(&rp->rio_ifp->if_snd); - - while ((m = rio_getq_flow(rp, q, flow, TRUE)) != NULL) { - cnt++; - len += m_pktlen(m); - m_freem(m); - } - - if (packets != NULL) - *packets = cnt; - if (bytes != NULL) - *bytes = len; -} - -void -rio_updateq(rio_t *rp, cqev_t ev) -{ -#pragma unused(rp, ev) - /* nothing for now */ -} - -int -rio_suspendq(rio_t *rp, class_queue_t *q, boolean_t on) -{ -#pragma unused(rp, q, on) - return (ENOTSUP); -} -#endif /* CLASSQ_RIO */ diff --git a/bsd/net/classq/classq_rio.h b/bsd/net/classq/classq_rio.h index fb3c24199..8f0a12a07 100644 --- a/bsd/net/classq/classq_rio.h +++ b/bsd/net/classq/classq_rio.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2012 Apple Inc. All rights reserved. + * Copyright (c) 2011-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -73,65 +73,6 @@ extern "C" { */ #define RIO_NDROPPREC 3 /* number of drop precedence values */ -#ifdef BSD_KERNEL_PRIVATE -/* rio flags */ -#define RIOF_ECN4 0x01 /* use packet marking for IPv4 packets */ -#define RIOF_ECN6 0x02 /* use packet marking for IPv6 packets */ -#define RIOF_ECN (RIOF_ECN4 | RIOF_ECN6) -#define RIOF_CLEARDSCP 0x200 /* clear diffserv codepoint */ - -#define RIOF_USERFLAGS \ - (RIOF_ECN4 | RIOF_ECN6 | RIOF_CLEARDSCP) - -typedef struct rio { - /* per drop precedence structure */ - struct dropprec_state { - /* red parameters */ - int inv_pmax; /* inverse of max drop probability */ - int th_min; /* red min threshold */ - int th_max; /* red max threshold */ - - /* variables for internal use */ - int th_min_s; /* th_min scaled by avgshift */ - int th_max_s; /* th_max scaled by avgshift */ - int probd; /* drop probability denominator */ - - int qlen; /* queue length */ - int avg; /* (scaled) queue length average */ - int count; /* packet count since the last */ - /* dropped/marked packet */ - int idle; /* queue was empty */ - int old; /* avg is above th_min */ - struct timeval last; /* timestamp when queue becomes idle */ - } rio_precstate[RIO_NDROPPREC]; - - int rio_wshift; /* log(red_weight) */ - int rio_weight; /* weight for EWMA */ - struct wtab *rio_wtab; /* weight table */ - - int rio_pkttime; /* average packet time in micro sec */ - /* used for idle calibration */ - int rio_flags; /* rio flags */ - struct ifnet *rio_ifp; /* back pointer to ifnet */ - - u_int8_t rio_codepoint; /* codepoint value to tag packets */ - u_int8_t rio_codepointmask; /* codepoint mask bits */ - - struct red_stats q_stats[RIO_NDROPPREC]; /* statistics */ -} rio_t; - -extern void rio_init(void); -extern rio_t *rio_alloc(struct ifnet *, int, struct redparams *, int, int); -extern void rio_destroy(rio_t *); -extern void rio_getstats(rio_t *, struct red_stats *); -extern int rio_addq(rio_t *, class_queue_t *, struct mbuf *, struct pf_mtag *); -extern struct mbuf *rio_getq(rio_t *, class_queue_t *); -extern void rio_purgeq(struct rio *, class_queue_t *, u_int32_t, - u_int32_t *, u_int32_t *); -extern void rio_updateq(rio_t *, cqev_t); -extern int rio_suspendq(rio_t *, class_queue_t *, boolean_t); -#endif /* BSD_KERNEL_PRIVATE */ - #ifdef __cplusplus } #endif diff --git a/bsd/net/classq/classq_sfb.c b/bsd/net/classq/classq_sfb.c index c679ca43f..438abf2c3 100644 --- a/bsd/net/classq/classq_sfb.c +++ b/bsd/net/classq/classq_sfb.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2013 Apple Inc. All rights reserved. + * Copyright (c) 2011-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -220,12 +220,6 @@ /* Place the flow control entries in current bin on level 0 */ #define SFB_FC_LEVEL 0 -/* Store SFB hash and flags in the module private scratch space */ -#define pkt_sfb_hash8 pkt_mpriv.__mpriv_u.__mpriv32[0].__mpriv32_u.__val8 -#define pkt_sfb_hash16 pkt_mpriv.__mpriv_u.__mpriv32[0].__mpriv32_u.__val16 -#define pkt_sfb_hash32 pkt_mpriv.__mpriv_u.__mpriv32[0].__mpriv32_u.__val32 -#define pkt_sfb_flags pkt_mpriv.__mpriv_u.__mpriv32[1].__mpriv32_u.__val32 - static unsigned int sfb_size; /* size of zone element */ static struct zone *sfb_zone; /* zone for sfb */ @@ -237,28 +231,29 @@ static struct zone *sfb_fcl_zone; /* zone for sfb_fc_lists */ /* internal function prototypes */ static u_int32_t sfb_random(struct sfb *); -static struct mbuf *sfb_getq_flow(struct sfb *, class_queue_t *, u_int32_t, - boolean_t); +static void *sfb_getq_flow(struct sfb *, class_queue_t *, u_int32_t, boolean_t, + pktsched_pkt_t *); static void sfb_resetq(struct sfb *, cqev_t); static void sfb_calc_holdtime(struct sfb *, u_int64_t); static void sfb_calc_pboxtime(struct sfb *, u_int64_t); static void sfb_calc_hinterval(struct sfb *, u_int64_t *); static void sfb_calc_update_interval(struct sfb *, u_int64_t); static void sfb_swap_bins(struct sfb *, u_int32_t); -static inline int sfb_pcheck(struct sfb *, struct pkthdr *); -static int sfb_penalize(struct sfb *, struct pkthdr *, struct timespec *); +static inline int sfb_pcheck(struct sfb *, uint32_t); +static int sfb_penalize(struct sfb *, uint32_t, uint32_t *, struct timespec *); static void sfb_adjust_bin(struct sfb *, struct sfbbinstats *, struct timespec *, struct timespec *, boolean_t); static void sfb_decrement_bin(struct sfb *, struct sfbbinstats *, struct timespec *, struct timespec *); static void sfb_increment_bin(struct sfb *, struct sfbbinstats *, struct timespec *, struct timespec *); -static inline void sfb_dq_update_bins(struct sfb *, struct pkthdr *, +static inline void sfb_dq_update_bins(struct sfb *, uint32_t, uint32_t, struct timespec *, u_int32_t qsize); -static inline void sfb_eq_update_bins(struct sfb *, struct pkthdr *); -static int sfb_drop_early(struct sfb *, struct pkthdr *, u_int16_t *, +static inline void sfb_eq_update_bins(struct sfb *, uint32_t, uint32_t); +static int sfb_drop_early(struct sfb *, uint32_t, u_int16_t *, struct timespec *); -static boolean_t sfb_bin_addfcentry(struct sfb *, struct pkthdr *); +static boolean_t sfb_bin_addfcentry(struct sfb *, pktsched_pkt_t *, + uint32_t, uint8_t, uint32_t); static void sfb_fclist_append(struct sfb *, struct sfb_fcl *); static void sfb_fclists_clean(struct sfb *sp); static int sfb_bin_mark_or_drop(struct sfb *sp, struct sfbbinstats *bin); @@ -409,7 +404,7 @@ sfb_calc_pboxtime(struct sfb *sp, u_int64_t outbw) static void sfb_calc_hinterval(struct sfb *sp, u_int64_t *t) { - u_int64_t hinterval; + u_int64_t hinterval = 0; struct timespec now; if (t != NULL) { @@ -487,7 +482,7 @@ sfb_alloc(struct ifnet *ifp, u_int32_t qid, u_int32_t qlim, u_int32_t flags) } #endif /* !PF_ECN */ - sfb_resetq(sp, -1); + sfb_resetq(sp, CLASSQ_EV_INIT); return (sp); } @@ -496,7 +491,6 @@ static void sfb_fclist_append(struct sfb *sp, struct sfb_fcl *fcl) { IFCQ_CONVERT_LOCK(&sp->sfb_ifp->if_snd); - VERIFY(STAILQ_EMPTY(&fcl->fclist) || fcl->cnt > 0); sp->sfb_stats.flow_feedback += fcl->cnt; fcl->cnt = 0; @@ -670,11 +664,12 @@ sfb_swap_bins(struct sfb *sp, u_int32_t len) } static inline int -sfb_pcheck(struct sfb *sp, struct pkthdr *pkt) +sfb_pcheck(struct sfb *sp, uint32_t pkt_sfb_hash) { #if SFB_LEVELS != 2 int i, n; #endif /* SFB_LEVELS != 2 */ + uint8_t *pkt_sfb_hash8 = (uint8_t *)&pkt_sfb_hash; int s; s = sp->sfb_current; @@ -689,17 +684,17 @@ sfb_pcheck(struct sfb *sp, struct pkthdr *pkt) * Level 0: bin index at [0] for set 0; [2] for set 1 * Level 1: bin index at [1] for set 0; [3] for set 1 */ - if (SFB_BINST(sp, 0, SFB_BINMASK(pkt->pkt_sfb_hash8[(s << 1)]), + if (SFB_BINST(sp, 0, SFB_BINMASK(pkt_sfb_hash8[(s << 1)]), s)->pmark < SFB_PMARK_TH || - SFB_BINST(sp, 1, SFB_BINMASK(pkt->pkt_sfb_hash8[(s << 1) + 1]), + SFB_BINST(sp, 1, SFB_BINMASK(pkt_sfb_hash8[(s << 1) + 1]), s)->pmark < SFB_PMARK_TH) return (0); #else /* SFB_LEVELS != 2 */ for (i = 0; i < SFB_LEVELS; i++) { if (s == 0) /* set 0, bin index [0,1] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i]); + n = SFB_BINMASK(pkt_sfb_hash8[i]); else /* set 1, bin index [2,3] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i + 2]); + n = SFB_BINMASK(pkt_sfb_hash8[i + 2]); if (SFB_BINST(sp, i, n, s)->pmark < SFB_PMARK_TH) return (0); @@ -709,12 +704,14 @@ sfb_pcheck(struct sfb *sp, struct pkthdr *pkt) } static int -sfb_penalize(struct sfb *sp, struct pkthdr *pkt, struct timespec *now) +sfb_penalize(struct sfb *sp, uint32_t pkt_sfb_hash, uint32_t *pkt_sfb_flags, + struct timespec *now) { struct timespec delta = { 0, 0 }; + uint8_t *pkt_sfb_hash8 = (uint8_t *)&pkt_sfb_hash; /* If minimum pmark of current bins is < SFB_PMARK_TH, we're done */ - if (!sfb_ratelimit || !sfb_pcheck(sp, pkt)) + if (!sfb_ratelimit || !sfb_pcheck(sp, pkt_sfb_hash)) return (0); net_timersub(now, &sp->sfb_pboxfreeze, &delta); @@ -733,22 +730,22 @@ sfb_penalize(struct sfb *sp, struct pkthdr *pkt, struct timespec *now) */ #if SFB_LEVELS == 2 /* Level 0: bin index at [0] for set 0; [2] for set 1 */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[(w << 1)]); + n = SFB_BINMASK(pkt_sfb_hash8[(w << 1)]); bin = SFB_BINST(sp, 0, n, w); if (bin->pkts >= sp->sfb_allocation) sfb_increment_bin(sp, bin, SFB_BINFT(sp, 0, n, w), now); /* Level 0: bin index at [1] for set 0; [3] for set 1 */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[(w << 1) + 1]); + n = SFB_BINMASK(pkt_sfb_hash8[(w << 1) + 1]); bin = SFB_BINST(sp, 1, n, w); if (bin->pkts >= sp->sfb_allocation) sfb_increment_bin(sp, bin, SFB_BINFT(sp, 1, n, w), now); #else /* SFB_LEVELS != 2 */ for (i = 0; i < SFB_LEVELS; i++) { if (w == 0) /* set 0, bin index [0,1] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i]); + n = SFB_BINMASK(pkt_sfb_hash8[i]); else /* set 1, bin index [2,3] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i + 2]); + n = SFB_BINMASK(pkt_sfb_hash8[i + 2]); bin = SFB_BINST(sp, i, n, w); if (bin->pkts >= sp->sfb_allocation) { @@ -761,7 +758,7 @@ sfb_penalize(struct sfb *sp, struct pkthdr *pkt, struct timespec *now) } /* non-conformant or else misclassified flow; queue it anyway */ - pkt->pkt_sfb_flags |= SFB_PKT_PBOX; + *pkt_sfb_flags |= SFB_PKT_PBOX; *(&sp->sfb_pboxfreeze) = *now; return (0); @@ -807,7 +804,7 @@ sfb_increment_bin(struct sfb *sp, struct sfbbinstats *bin, struct timespec *ft, } static inline void -sfb_dq_update_bins(struct sfb *sp, struct pkthdr *pkt, +sfb_dq_update_bins(struct sfb *sp, uint32_t pkt_sfb_hash, uint32_t pkt_len, struct timespec *now, u_int32_t qsize) { #if SFB_LEVELS != 2 || SFB_FC_LEVEL != 0 @@ -816,6 +813,7 @@ sfb_dq_update_bins(struct sfb *sp, struct pkthdr *pkt, struct sfbbinstats *bin; int s, n; struct sfb_fcl *fcl = NULL; + uint8_t *pkt_sfb_hash8 = (uint8_t *)&pkt_sfb_hash; s = sp->sfb_current; VERIFY((s + (s ^ 1)) == 1); @@ -825,12 +823,12 @@ sfb_dq_update_bins(struct sfb *sp, struct pkthdr *pkt, */ #if SFB_LEVELS == 2 && SFB_FC_LEVEL == 0 /* Level 0: bin index at [0] for set 0; [2] for set 1 */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[(s << 1)]); + n = SFB_BINMASK(pkt_sfb_hash8[(s << 1)]); bin = SFB_BINST(sp, 0, n, s); - VERIFY(bin->pkts > 0 && bin->bytes >= (u_int32_t)pkt->len); + VERIFY(bin->pkts > 0 && bin->bytes >= pkt_len); bin->pkts--; - bin->bytes -= pkt->len; + bin->bytes -= pkt_len; if (bin->pkts == 0) sfb_decrement_bin(sp, bin, SFB_BINFT(sp, 0, n, s), now); @@ -850,26 +848,26 @@ sfb_dq_update_bins(struct sfb *sp, struct pkthdr *pkt, fcl = NULL; /* Level 1: bin index at [1] for set 0; [3] for set 1 */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[(s << 1) + 1]); + n = SFB_BINMASK(pkt_sfb_hash8[(s << 1) + 1]); bin = SFB_BINST(sp, 1, n, s); - VERIFY(bin->pkts > 0 && bin->bytes >= (u_int64_t)pkt->len); + VERIFY(bin->pkts > 0 && bin->bytes >= (u_int64_t)pkt_len); bin->pkts--; - bin->bytes -= pkt->len; + bin->bytes -= pkt_len; if (bin->pkts == 0) sfb_decrement_bin(sp, bin, SFB_BINFT(sp, 1, n, s), now); #else /* SFB_LEVELS != 2 || SFB_FC_LEVEL != 0 */ for (i = 0; i < SFB_LEVELS; i++) { if (s == 0) /* set 0, bin index [0,1] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i]); + n = SFB_BINMASK(pkt_sfb_hash8[i]); else /* set 1, bin index [2,3] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i + 2]); + n = SFB_BINMASK(pkt_sfb_hash8[i + 2]); bin = SFB_BINST(sp, i, n, s); - VERIFY(bin->pkts > 0 && bin->bytes >= pkt->len); + VERIFY(bin->pkts > 0 && bin->bytes >= pkt_len); bin->pkts--; - bin->bytes -= pkt->len; + bin->bytes -= pkt_len; if (bin->pkts == 0) sfb_decrement_bin(sp, bin, SFB_BINFT(sp, i, n, s), now); @@ -890,13 +888,14 @@ sfb_dq_update_bins(struct sfb *sp, struct pkthdr *pkt, } static inline void -sfb_eq_update_bins(struct sfb *sp, struct pkthdr *pkt) +sfb_eq_update_bins(struct sfb *sp, uint32_t pkt_sfb_hash, uint32_t pkt_len) { #if SFB_LEVELS != 2 int i, n; #endif /* SFB_LEVELS != 2 */ int s; struct sfbbinstats *bin; + uint8_t *pkt_sfb_hash8 = (uint8_t *)&pkt_sfb_hash; s = sp->sfb_current; VERIFY((s + (s ^ 1)) == 1); @@ -906,44 +905,42 @@ sfb_eq_update_bins(struct sfb *sp, struct pkthdr *pkt) #if SFB_LEVELS == 2 /* Level 0: bin index at [0] for set 0; [2] for set 1 */ bin = SFB_BINST(sp, 0, - SFB_BINMASK(pkt->pkt_sfb_hash8[(s << 1)]), s); + SFB_BINMASK(pkt_sfb_hash8[(s << 1)]), s); bin->pkts++; - bin->bytes += pkt->len; + bin->bytes += pkt_len; /* Level 1: bin index at [1] for set 0; [3] for set 1 */ bin = SFB_BINST(sp, 1, - SFB_BINMASK(pkt->pkt_sfb_hash8[(s << 1) + 1]), s); + SFB_BINMASK(pkt_sfb_hash8[(s << 1) + 1]), s); bin->pkts++; - bin->bytes += pkt->len; + bin->bytes += pkt_len; #else /* SFB_LEVELS != 2 */ for (i = 0; i < SFB_LEVELS; i++) { if (s == 0) /* set 0, bin index [0,1] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i]); + n = SFB_BINMASK(pkt_sfb_hash8[i]); else /* set 1, bin index [2,3] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i + 2]); + n = SFB_BINMASK(pkt_sfb_hash8[i + 2]); bin = SFB_BINST(sp, i, n, s); bin->pkts++; - bin->bytes += pkt->len; + bin->bytes += pkt_len; } #endif /* SFB_LEVELS != 2 */ } static boolean_t -sfb_bin_addfcentry(struct sfb *sp, struct pkthdr *pkt) +sfb_bin_addfcentry(struct sfb *sp, pktsched_pkt_t *pkt, uint32_t pkt_sfb_hash, + uint8_t flowsrc, uint32_t flowid) { struct flowadv_fcentry *fce; - u_int32_t flowsrc, flowid; struct sfb_fcl *fcl; int s; + uint8_t *pkt_sfb_hash8 = (uint8_t *)&pkt_sfb_hash; s = sp->sfb_current; VERIFY((s + (s ^ 1)) == 1); - flowsrc = pkt->pkt_flowsrc; - flowid = pkt->pkt_flowid; - if (flowid == 0) { sp->sfb_stats.null_flowid++; return (FALSE); @@ -953,9 +950,9 @@ sfb_bin_addfcentry(struct sfb *sp, struct pkthdr *pkt) * Use value at index 0 for set 0 and * value at index 2 for set 1 */ - fcl = SFB_FC_LIST(sp, SFB_BINMASK(pkt->pkt_sfb_hash8[(s << 1)])); + fcl = SFB_FC_LIST(sp, SFB_BINMASK(pkt_sfb_hash8[(s << 1)])); STAILQ_FOREACH(fce, &fcl->fclist, fce_link) { - if (fce->fce_flowsrc == flowsrc && + if ((uint8_t)fce->fce_flowsrc_type == flowsrc && fce->fce_flowid == flowid) { /* Already on flow control list; just return */ return (TRUE); @@ -963,10 +960,8 @@ sfb_bin_addfcentry(struct sfb *sp, struct pkthdr *pkt) } IFCQ_CONVERT_LOCK(&sp->sfb_ifp->if_snd); - fce = flowadv_alloc_entry(M_WAITOK); + fce = pktsched_alloc_fcentry(pkt, sp->sfb_ifp, M_WAITOK); if (fce != NULL) { - fce->fce_flowsrc = flowsrc; - fce->fce_flowid = flowid; STAILQ_INSERT_TAIL(&fcl->fclist, fce, fce_link); fcl->cnt++; sp->sfb_stats.flow_controlled++; @@ -1003,7 +998,7 @@ sfb_bin_mark_or_drop(struct sfb *sp, struct sfbbinstats *bin) * early-drop probability is kept in pmark of each bin of the flow */ static int -sfb_drop_early(struct sfb *sp, struct pkthdr *pkt, u_int16_t *pmin, +sfb_drop_early(struct sfb *sp, uint32_t pkt_sfb_hash, u_int16_t *pmin, struct timespec *now) { #if SFB_LEVELS != 2 @@ -1011,6 +1006,7 @@ sfb_drop_early(struct sfb *sp, struct pkthdr *pkt, u_int16_t *pmin, #endif /* SFB_LEVELS != 2 */ struct sfbbinstats *bin; int s, n, ret = 0; + uint8_t *pkt_sfb_hash8 = (uint8_t *)&pkt_sfb_hash; s = sp->sfb_current; VERIFY((s + (s ^ 1)) == 1); @@ -1022,7 +1018,7 @@ sfb_drop_early(struct sfb *sp, struct pkthdr *pkt, u_int16_t *pmin, */ #if SFB_LEVELS == 2 /* Level 0: bin index at [0] for set 0; [2] for set 1 */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[(s << 1)]); + n = SFB_BINMASK(pkt_sfb_hash8[(s << 1)]); bin = SFB_BINST(sp, 0, n, s); if (*pmin > (u_int16_t)bin->pmark) *pmin = (u_int16_t)bin->pmark; @@ -1035,7 +1031,7 @@ sfb_drop_early(struct sfb *sp, struct pkthdr *pkt, u_int16_t *pmin, ret = sfb_bin_mark_or_drop(sp, bin); /* Level 1: bin index at [1] for set 0; [3] for set 1 */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[(s << 1) + 1]); + n = SFB_BINMASK(pkt_sfb_hash8[(s << 1) + 1]); bin = SFB_BINST(sp, 1, n, s); if (*pmin > (u_int16_t)bin->pmark) *pmin = (u_int16_t)bin->pmark; @@ -1045,9 +1041,9 @@ sfb_drop_early(struct sfb *sp, struct pkthdr *pkt, u_int16_t *pmin, #else /* SFB_LEVELS != 2 */ for (i = 0; i < SFB_LEVELS; i++) { if (s == 0) /* set 0, bin index [0,1] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i]); + n = SFB_BINMASK(pkt_sfb_hash8[i]); else /* set 1, bin index [2,3] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i + 2]); + n = SFB_BINMASK(pkt_sfb_hash8[i + 2]); bin = SFB_BINST(sp, i, n, s); if (*pmin > (u_int16_t)bin->pmark) @@ -1095,31 +1091,45 @@ sfb_detect_dequeue_stall(struct sfb *sp, class_queue_t *q, #define DTYPE_EARLY 2 /* an "unforced" (early) drop */ int -sfb_addq(struct sfb *sp, class_queue_t *q, struct mbuf *m, struct pf_mtag *t) +sfb_addq(struct sfb *sp, class_queue_t *q, pktsched_pkt_t *pkt, + struct pf_mtag *t) { #if !PF_ECN #pragma unused(t) #endif /* !PF_ECN */ - struct pkthdr *pkt = &m->m_pkthdr; struct timespec now; int droptype, s; - u_int16_t pmin; + uint16_t pmin; int fc_adv = 0; int ret = CLASSQEQ_SUCCESS; - u_int32_t maxqsize = 0; + uint32_t maxqsize = 0; + uint64_t *pkt_timestamp; + uint32_t *pkt_sfb_hash; + uint16_t *pkt_sfb_hash16; + uint32_t *pkt_sfb_flags; + uint32_t pkt_flowid; + uint32_t *pkt_flags; + uint8_t pkt_proto, pkt_flowsrc; s = sp->sfb_current; VERIFY((s + (s ^ 1)) == 1); - /* See comments in <rdar://problem/14040693> */ - VERIFY(!(pkt->pkt_flags & PKTF_PRIV_GUARDED)); - pkt->pkt_flags |= PKTF_PRIV_GUARDED; + pktsched_get_pkt_vars(pkt, &pkt_flags, &pkt_timestamp, &pkt_flowid, + &pkt_flowsrc, &pkt_proto, NULL); + pkt_sfb_hash = pktsched_get_pkt_sfb_vars(pkt, &pkt_sfb_flags); + pkt_sfb_hash16 = (uint16_t *)pkt_sfb_hash; - if (pkt->pkt_timestamp > 0) { - net_nsectimer(&pkt->pkt_timestamp, &now); + if (pkt->pktsched_ptype == QP_MBUF) { + /* See comments in <rdar://problem/14040693> */ + VERIFY(!(*pkt_flags & PKTF_PRIV_GUARDED)); + *pkt_flags |= PKTF_PRIV_GUARDED; + } + + if (*pkt_timestamp > 0) { + net_nsectimer(pkt_timestamp, &now); } else { nanouptime(&now); - net_timernsec(&now, &pkt->pkt_timestamp); + net_timernsec(&now, pkt_timestamp); } /* time to swap the bins? */ @@ -1142,12 +1152,12 @@ sfb_addq(struct sfb *sp, class_queue_t *q, struct mbuf *m, struct pf_mtag *t) if (qsize(q) == 0 && !net_timerisset(&sp->sfb_getqtime)) *(&sp->sfb_getqtime) = *(&now); - pkt->pkt_sfb_flags = 0; - pkt->pkt_sfb_hash16[s] = - (SFB_HASH(&pkt->pkt_flowid, sizeof (pkt->pkt_flowid), + *pkt_sfb_flags = 0; + pkt_sfb_hash16[s] = + (SFB_HASH(&pkt_flowid, sizeof (pkt_flowid), (*sp->sfb_bins)[s].fudge) & SFB_HASHMASK); - pkt->pkt_sfb_hash16[s ^ 1] = - (SFB_HASH(&pkt->pkt_flowid, sizeof (pkt->pkt_flowid), + pkt_sfb_hash16[s ^ 1] = + (SFB_HASH(&pkt_flowid, sizeof (pkt_flowid), (*sp->sfb_bins)[s ^ 1].fudge) & SFB_HASHMASK); /* check if the queue has been stalled */ @@ -1155,21 +1165,23 @@ sfb_addq(struct sfb *sp, class_queue_t *q, struct mbuf *m, struct pf_mtag *t) /* see if we drop early */ droptype = DTYPE_NODROP; - if (sfb_drop_early(sp, pkt, &pmin, &now)) { + if (sfb_drop_early(sp, *pkt_sfb_hash, &pmin, &now)) { /* flow control, mark or drop by sfb */ if ((sp->sfb_flags & SFBF_FLOWCTL) && - (pkt->pkt_flags & PKTF_FLOW_ADV)) { + (*pkt_flags & PKTF_FLOW_ADV)) { fc_adv = 1; /* drop all during suspension or for non-TCP */ if ((sp->sfb_flags & SFBF_SUSPENDED) || - pkt->pkt_proto != IPPROTO_TCP) { + pkt_proto != IPPROTO_TCP) { droptype = DTYPE_EARLY; sp->sfb_stats.drop_early++; } } #if PF_ECN + /* XXX: only supported for mbuf */ else if ((sp->sfb_flags & SFBF_ECN) && - (pkt->pkt_proto == IPPROTO_TCP) && /* only for TCP */ + (pkt->pktsched_ptype == QP_MBUF) && + (pkt_proto == IPPROTO_TCP) && /* only for TCP */ ((sfb_random(sp) & SFB_MAX_PMARK) <= pmin) && mark_ecn(m, t, sp->sfb_flags) && !(sp->sfb_flags & SFBF_SUSPENDED)) { @@ -1185,7 +1197,8 @@ sfb_addq(struct sfb *sp, class_queue_t *q, struct mbuf *m, struct pf_mtag *t) } /* non-responsive flow penalty? */ - if (droptype == DTYPE_NODROP && sfb_penalize(sp, pkt, &now)) { + if (droptype == DTYPE_NODROP && sfb_penalize(sp, *pkt_sfb_hash, + pkt_sfb_flags, &now)) { droptype = DTYPE_FORCED; sp->sfb_stats.drop_pbox++; } @@ -1200,9 +1213,9 @@ sfb_addq(struct sfb *sp, class_queue_t *q, struct mbuf *m, struct pf_mtag *t) * drop */ if (droptype == DTYPE_NODROP && qlen(q) >= maxqsize) { - if (pkt->pkt_proto == IPPROTO_TCP && + if (pkt_proto == IPPROTO_TCP && qlen(q) < (maxqsize + (maxqsize >> 1)) && - ((pkt->pkt_flags & PKTF_TCP_REXMT) || + ((*pkt_flags & PKTF_TCP_REXMT) || (sp->sfb_flags & SFBF_LAST_PKT_DROPPED))) { /* * At some level, dropping packets will make the @@ -1221,30 +1234,32 @@ sfb_addq(struct sfb *sp, class_queue_t *q, struct mbuf *m, struct pf_mtag *t) } if (fc_adv == 1 && droptype != DTYPE_FORCED && - sfb_bin_addfcentry(sp, pkt)) { + sfb_bin_addfcentry(sp, pkt, *pkt_sfb_hash, pkt_flowsrc, + pkt_flowid)) { /* deliver flow control advisory error */ if (droptype == DTYPE_NODROP) { ret = CLASSQEQ_SUCCESS_FC; VERIFY(!(sp->sfb_flags & SFBF_SUSPENDED)); } else if (sp->sfb_flags & SFBF_SUSPENDED) { - /* dropped due to suspension */ - ret = CLASSQEQ_DROPPED_SP; + /* drop due to suspension */ + ret = CLASSQEQ_DROP_SP; } else { - /* dropped due to flow-control */ - ret = CLASSQEQ_DROPPED_FC; + /* drop due to flow-control */ + ret = CLASSQEQ_DROP_FC; } } /* if successful enqueue this packet, else drop it */ if (droptype == DTYPE_NODROP) { - _addq(q, m); + VERIFY(pkt->pktsched_ptype == qptype(q)); + _addq(q, pkt->pktsched_pkt); } else { IFCQ_CONVERT_LOCK(&sp->sfb_ifp->if_snd); - m_freem(m); - return ((ret != CLASSQEQ_SUCCESS) ? ret : CLASSQEQ_DROPPED); + return ((ret != CLASSQEQ_SUCCESS) ? ret : CLASSQEQ_DROP); } - if (!(pkt->pkt_sfb_flags & SFB_PKT_PBOX)) - sfb_eq_update_bins(sp, pkt); + if (!(*pkt_sfb_flags & SFB_PKT_PBOX)) + sfb_eq_update_bins(sp, *pkt_sfb_hash, + pktsched_get_pkt_len(pkt)); else sp->sfb_stats.pbox_packets++; @@ -1252,12 +1267,17 @@ sfb_addq(struct sfb *sp, class_queue_t *q, struct mbuf *m, struct pf_mtag *t) return (ret); } -static struct mbuf * -sfb_getq_flow(struct sfb *sp, class_queue_t *q, u_int32_t flow, boolean_t purge) +static void * +sfb_getq_flow(struct sfb *sp, class_queue_t *q, u_int32_t flow, boolean_t purge, + pktsched_pkt_t *pkt) { struct timespec now; - struct mbuf *m; - struct pkthdr *pkt; + classq_pkt_type_t ptype; + uint64_t *pkt_timestamp; + uint32_t *pkt_flags; + uint32_t *pkt_sfb_flags; + uint32_t *pkt_sfb_hash; + void *p; if (!purge && (sp->sfb_flags & SFBF_SUSPENDED)) return (NULL); @@ -1265,16 +1285,21 @@ sfb_getq_flow(struct sfb *sp, class_queue_t *q, u_int32_t flow, boolean_t purge) nanouptime(&now); /* flow of 0 means head of queue */ - if ((m = ((flow == 0) ? _getq(q) : _getq_flow(q, flow))) == NULL) { + if ((p = ((flow == 0) ? _getq(q) : _getq_flow(q, flow))) == NULL) { if (!purge) net_timerclear(&sp->sfb_getqtime); return (NULL); } - VERIFY(m->m_flags & M_PKTHDR); + ptype = qptype(q); + pktsched_pkt_encap(pkt, ptype, p); + pktsched_get_pkt_vars(pkt, &pkt_flags, &pkt_timestamp, NULL, + NULL, NULL, NULL); + pkt_sfb_hash = pktsched_get_pkt_sfb_vars(pkt, &pkt_sfb_flags); - pkt = &m->m_pkthdr; - VERIFY(pkt->pkt_flags & PKTF_PRIV_GUARDED); + /* See comments in <rdar://problem/14040693> */ + if (ptype == QP_MBUF) + VERIFY(*pkt_flags & PKTF_PRIV_GUARDED); if (!purge) { /* calculate EWMA of dequeues */ @@ -1306,8 +1331,8 @@ sfb_getq_flow(struct sfb *sp, class_queue_t *q, u_int32_t flow, boolean_t purge) if (!purge && SFB_QUEUE_DELAYBASED(sp)) { u_int64_t dequeue_ns, queue_delay = 0; net_timernsec(&now, &dequeue_ns); - if (dequeue_ns > pkt->pkt_timestamp) - queue_delay = dequeue_ns - pkt->pkt_timestamp; + if (dequeue_ns > *pkt_timestamp) + queue_delay = dequeue_ns - *pkt_timestamp; if (sp->sfb_min_qdelay == 0 || (queue_delay > 0 && queue_delay < sp->sfb_min_qdelay)) @@ -1326,7 +1351,7 @@ sfb_getq_flow(struct sfb *sp, class_queue_t *q, u_int32_t flow, boolean_t purge) sp->sfb_min_qdelay = 0; } } - pkt->pkt_timestamp = 0; + *pkt_timestamp = 0; /* * Clearpkts are the ones which were in the queue when the hash @@ -1337,18 +1362,20 @@ sfb_getq_flow(struct sfb *sp, class_queue_t *q, u_int32_t flow, boolean_t purge) * this reason. A rule of thumb is to set it to K*D, where D is * the time taken to drain queue. */ - if (pkt->pkt_sfb_flags & SFB_PKT_PBOX) { - pkt->pkt_sfb_flags &= ~SFB_PKT_PBOX; + if (*pkt_sfb_flags & SFB_PKT_PBOX) { + *pkt_sfb_flags &= ~SFB_PKT_PBOX; if (sp->sfb_clearpkts > 0) sp->sfb_clearpkts--; } else if (sp->sfb_clearpkts > 0) { sp->sfb_clearpkts--; } else { - sfb_dq_update_bins(sp, pkt, &now, qsize(q)); + sfb_dq_update_bins(sp, *pkt_sfb_hash, pktsched_get_pkt_len(pkt), + &now, qsize(q)); } /* See comments in <rdar://problem/14040693> */ - pkt->pkt_flags &= ~PKTF_PRIV_GUARDED; + if (ptype == QP_MBUF) + *pkt_flags &= ~PKTF_PRIV_GUARDED; /* * If the queue becomes empty before the update interval, reset @@ -1361,14 +1388,13 @@ sfb_getq_flow(struct sfb *sp, class_queue_t *q, u_int32_t flow, boolean_t purge) net_timerclear(&sp->sfb_update_time); net_timerclear(&sp->sfb_getqtime); } - - return (m); + return (p); } -struct mbuf * -sfb_getq(struct sfb *sp, class_queue_t *q) +void +sfb_getq(struct sfb *sp, class_queue_t *q, pktsched_pkt_t *pkt) { - return (sfb_getq_flow(sp, q, 0, FALSE)); + sfb_getq_flow(sp, q, 0, FALSE, pkt); } void @@ -1376,14 +1402,13 @@ sfb_purgeq(struct sfb *sp, class_queue_t *q, u_int32_t flow, u_int32_t *packets, u_int32_t *bytes) { u_int32_t cnt = 0, len = 0; - struct mbuf *m; + pktsched_pkt_t pkt; IFCQ_CONVERT_LOCK(&sp->sfb_ifp->if_snd); - - while ((m = sfb_getq_flow(sp, q, flow, TRUE)) != NULL) { + while (sfb_getq_flow(sp, q, flow, TRUE, &pkt) != NULL) { cnt++; - len += m_pktlen(m); - m_freem(m); + len += pktsched_get_pkt_len(&pkt); + pktsched_free_pkt(&pkt); } if (packets != NULL) diff --git a/bsd/net/classq/classq_sfb.h b/bsd/net/classq/classq_sfb.h index 2a28a7192..480ee5d73 100644 --- a/bsd/net/classq/classq_sfb.h +++ b/bsd/net/classq/classq_sfb.h @@ -104,7 +104,7 @@ struct sfb_fcl { #define SFBF_FLOWCTL 0x04 /* enable flow control advisories */ #define SFBF_DELAYBASED 0x08 /* queueing is delay based */ #define SFBF_DELAYHIGH 0x10 /* Estimated delay is greater than target */ -#define SFBF_LAST_PKT_DROPPED 0x20 /* Last packet dropped */ +#define SFBF_LAST_PKT_DROPPED 0x20 /* Last packet dropped */ #define SFBF_SUSPENDED 0x1000 /* queue is suspended */ #define SFBF_USERFLAGS \ @@ -126,7 +126,7 @@ typedef struct sfb { /* target queue delay and interval for queue sizing */ u_int64_t sfb_target_qdelay; struct timespec sfb_update_interval; - u_int64_t sfb_fc_threshold; /* for flow control feedback */ + u_int64_t sfb_fc_threshold; /* for flow control feedback */ /* variables for computing estimated delay of the queue */ u_int64_t sfb_min_qdelay; @@ -154,9 +154,9 @@ typedef struct sfb { extern void sfb_init(void); extern struct sfb *sfb_alloc(struct ifnet *, u_int32_t, u_int32_t, u_int32_t); extern void sfb_destroy(struct sfb *); -extern int sfb_addq(struct sfb *, class_queue_t *, struct mbuf *, +extern int sfb_addq(struct sfb *, class_queue_t *, pktsched_pkt_t *, struct pf_mtag *); -extern struct mbuf *sfb_getq(struct sfb *, class_queue_t *); +extern void sfb_getq(struct sfb *, class_queue_t *, pktsched_pkt_t *); extern void sfb_purgeq(struct sfb *, class_queue_t *, u_int32_t, u_int32_t *, u_int32_t *); extern void sfb_getstats(struct sfb *, struct sfb_stats *); diff --git a/bsd/net/classq/classq_subr.c b/bsd/net/classq/classq_subr.c index 55f42daf6..669e45ad6 100644 --- a/bsd/net/classq/classq_subr.c +++ b/bsd/net/classq/classq_subr.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2015 Apple Inc. All rights reserved. + * Copyright (c) 2011-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -40,32 +40,18 @@ #include <net/net_osdep.h> #include <net/classq/classq.h> #include <pexpert/pexpert.h> -#if CLASSQ_RED -#include <net/classq/classq_red.h> -#endif /* CLASSQ_RED */ -#if CLASSQ_RIO -#include <net/classq/classq_rio.h> -#endif /* CLASSQ_RIO */ -#if CLASSQ_BLUE -#include <net/classq/classq_blue.h> -#endif /* CLASSQ_BLUE */ #include <net/classq/classq_sfb.h> #include <net/pktsched/pktsched.h> #include <net/pktsched/pktsched_fq_codel.h> #include <libkern/libkern.h> -#if PF_ALTQ -#include <net/altq/altq.h> -#endif /* PF_ALTQ */ static errno_t ifclassq_dequeue_common(struct ifclassq *, mbuf_svc_class_t, - u_int32_t, u_int32_t, struct mbuf **, struct mbuf **, u_int32_t *, - u_int32_t *, boolean_t); -static struct mbuf *ifclassq_poll_common(struct ifclassq *, - mbuf_svc_class_t, boolean_t); -static struct mbuf *ifclassq_tbr_dequeue_common(struct ifclassq *, int, - mbuf_svc_class_t, boolean_t); + u_int32_t, u_int32_t, void **, void **, u_int32_t *, u_int32_t *, + boolean_t, classq_pkt_type_t *); +static void *ifclassq_tbr_dequeue_common(struct ifclassq *, mbuf_svc_class_t, + boolean_t, classq_pkt_type_t *); static u_int64_t ifclassq_target_qdelay = 0; SYSCTL_QUAD(_net_classq, OID_AUTO, target_qdelay, CTLFLAG_RW|CTLFLAG_LOCKED, @@ -85,21 +71,12 @@ classq_init(void) _CASSERT(MBUF_SC_BE == 0); _CASSERT(IFCQ_SC_MAX == MBUF_SC_MAX_CLASSES); -#if CLASSQ_RED - red_init(); -#endif /* CLASSQ_RED */ -#if CLASSQ_RIO - rio_init(); -#endif /* CLASSQ_RIO */ -#if CLASSQ_BLUE - blue_init(); -#endif /* CLASSQ_BLUE */ sfb_init(); fq_codel_scheduler_init(); if (!PE_parse_boot_argn("fq_codel", &ifclassq_sched_fq_codel, sizeof (ifclassq_sched_fq_codel))) - ifclassq_sched_fq_codel = 0; + ifclassq_sched_fq_codel = 1; } int @@ -147,26 +124,7 @@ ifclassq_setup(struct ifnet *ifp, u_int32_t sflags, boolean_t reuse) if (err == 0) ifq->ifcq_flags = (IFCQF_READY | IFCQF_ENABLED); } - -#if PF_ALTQ - ifq->ifcq_drain = 0; - IFCQ_ALTQ(ifq)->altq_ifcq = ifq; - VERIFY(IFCQ_ALTQ(ifq)->altq_type == ALTQT_NONE); - VERIFY(IFCQ_ALTQ(ifq)->altq_flags == 0); - VERIFY(IFCQ_ALTQ(ifq)->altq_disc == NULL); - VERIFY(IFCQ_ALTQ(ifq)->altq_enqueue == NULL); - VERIFY(IFCQ_ALTQ(ifq)->altq_dequeue == NULL); - VERIFY(IFCQ_ALTQ(ifq)->altq_dequeue_sc == NULL); - VERIFY(IFCQ_ALTQ(ifq)->altq_request == NULL); - - if ((ifp->if_eflags & IFEF_TXSTART) && - ifp->if_output_sched_model != IFNET_SCHED_MODEL_DRIVER_MANAGED) - ALTQ_SET_READY(IFCQ_ALTQ(ifq)); - else - ALTQ_CLEAR_READY(IFCQ_ALTQ(ifq)); -#endif /* PF_ALTQ */ IFCQ_UNLOCK(ifq); - return (err); } @@ -176,24 +134,6 @@ ifclassq_teardown(struct ifnet *ifp) struct ifclassq *ifq = &ifp->if_snd; IFCQ_LOCK(ifq); -#if PF_ALTQ - if (ALTQ_IS_READY(IFCQ_ALTQ(ifq))) { - if (ALTQ_IS_ENABLED(IFCQ_ALTQ(ifq))) - altq_disable(IFCQ_ALTQ(ifq)); - if (ALTQ_IS_ATTACHED(IFCQ_ALTQ(ifq))) - altq_detach(IFCQ_ALTQ(ifq)); - IFCQ_ALTQ(ifq)->altq_flags = 0; - } - ifq->ifcq_drain = 0; - IFCQ_ALTQ(ifq)->altq_ifcq = NULL; - VERIFY(IFCQ_ALTQ(ifq)->altq_type == ALTQT_NONE); - VERIFY(IFCQ_ALTQ(ifq)->altq_flags == 0); - VERIFY(IFCQ_ALTQ(ifq)->altq_disc == NULL); - VERIFY(IFCQ_ALTQ(ifq)->altq_enqueue == NULL); - VERIFY(IFCQ_ALTQ(ifq)->altq_dequeue == NULL); - VERIFY(IFCQ_ALTQ(ifq)->altq_dequeue_sc == NULL); - VERIFY(IFCQ_ALTQ(ifq)->altq_request == NULL); -#endif /* PF_ALTQ */ if (IFCQ_IS_READY(ifq)) { if (IFCQ_TBR_IS_ENABLED(ifq)) { @@ -228,6 +168,7 @@ int ifclassq_pktsched_setup(struct ifclassq *ifq) { struct ifnet *ifp = ifq->ifcq_ifp; + classq_pkt_type_t ptype = QP_MBUF; int err = 0; IFCQ_LOCK_ASSERT_HELD(ifq); @@ -235,21 +176,27 @@ ifclassq_pktsched_setup(struct ifclassq *ifq) switch (ifp->if_output_sched_model) { case IFNET_SCHED_MODEL_DRIVER_MANAGED: - err = pktsched_setup(ifq, PKTSCHEDT_TCQ, ifq->ifcq_sflags); + if (ifclassq_sched_fq_codel != 0) { + err = pktsched_setup(ifq, PKTSCHEDT_FQ_CODEL, + ifq->ifcq_sflags, ptype); + } else { + err = pktsched_setup(ifq, PKTSCHEDT_TCQ, + ifq->ifcq_sflags, ptype); + } break; case IFNET_SCHED_MODEL_NORMAL: if (ifclassq_sched_fq_codel != 0) { err = pktsched_setup(ifq, PKTSCHEDT_FQ_CODEL, - ifq->ifcq_sflags); + ifq->ifcq_sflags, ptype); } else { err = pktsched_setup(ifq, PKTSCHEDT_QFQ, - ifq->ifcq_sflags); + ifq->ifcq_sflags, ptype); } break; case IFNET_SCHED_MODEL_FQ_CODEL: err = pktsched_setup(ifq, PKTSCHEDT_FQ_CODEL, - ifq->ifcq_sflags); + ifq->ifcq_sflags, ptype); break; default: VERIFY(0); @@ -295,74 +242,114 @@ ifclassq_get_len(struct ifclassq *ifq, mbuf_svc_class_t sc, u_int32_t *packets, return (err); } +inline void +ifclassq_set_packet_metadata(struct ifclassq *ifq, struct ifnet *ifp, + void *p, classq_pkt_type_t ptype) +{ + if (!IFNET_IS_CELLULAR(ifp)) + return; + + switch (ptype) { + case QP_MBUF: { + struct mbuf *m = p; + m->m_pkthdr.pkt_flags |= PKTF_VALID_UNSENT_DATA; + m->m_pkthdr.bufstatus_if = IFCQ_BYTES(ifq); + m->m_pkthdr.bufstatus_sndbuf = ifp->if_sndbyte_unsent; + break; + } + + + default: + VERIFY(0); + /* NOTREACHED */ + } +} + errno_t -ifclassq_enqueue(struct ifclassq *ifq, struct mbuf *m) +ifclassq_enqueue(struct ifclassq *ifq, void *p, classq_pkt_type_t ptype, + boolean_t *pdrop) { errno_t err; - IFCQ_LOCK_SPIN(ifq); + switch (ptype) { + case QP_MBUF: + IFCQ_LOCK_SPIN(ifq); + break; -#if PF_ALTQ - if (ALTQ_IS_ENABLED(IFCQ_ALTQ(ifq))) { - ALTQ_ENQUEUE(IFCQ_ALTQ(ifq), m, err); - } else { - u_int32_t qlen = IFCQ_LEN(ifq); - IFCQ_ENQUEUE(ifq, m, err); - if (IFCQ_LEN(ifq) > qlen) - ifq->ifcq_drain += (IFCQ_LEN(ifq) - qlen); + default: + IFCQ_LOCK(ifq); + break; } -#else /* !PF_ALTQ */ - IFCQ_ENQUEUE(ifq, m, err); -#endif /* PF_ALTQ */ + IFCQ_ENQUEUE(ifq, p, ptype, err, pdrop); IFCQ_UNLOCK(ifq); - return (err); } errno_t ifclassq_dequeue(struct ifclassq *ifq, u_int32_t pkt_limit, - u_int32_t byte_limit, struct mbuf **head, - struct mbuf **tail, u_int32_t *cnt, u_int32_t *len) + u_int32_t byte_limit, void **head, void **tail, + u_int32_t *cnt, u_int32_t *len, classq_pkt_type_t *ptype) { return (ifclassq_dequeue_common(ifq, MBUF_SC_UNSPEC, pkt_limit, - byte_limit, head, tail, cnt, len, FALSE)); + byte_limit, head, tail, cnt, len, FALSE, ptype)); } errno_t ifclassq_dequeue_sc(struct ifclassq *ifq, mbuf_svc_class_t sc, - u_int32_t pkt_limit, struct mbuf **head, struct mbuf **tail, - u_int32_t *cnt, u_int32_t *len) + u_int32_t pkt_limit, u_int32_t byte_limit, void **head, void **tail, + u_int32_t *cnt, u_int32_t *len, classq_pkt_type_t *ptype) { - return (ifclassq_dequeue_common(ifq, sc, pkt_limit, - CLASSQ_DEQUEUE_MAX_BYTE_LIMIT, head, tail, cnt, len, TRUE)); + return (ifclassq_dequeue_common(ifq, sc, pkt_limit, byte_limit, + head, tail, cnt, len, TRUE, ptype)); } static errno_t ifclassq_dequeue_common(struct ifclassq *ifq, mbuf_svc_class_t sc, - u_int32_t pkt_limit, u_int32_t byte_limit, struct mbuf **head, - struct mbuf **tail, u_int32_t *cnt, u_int32_t *len, boolean_t drvmgt) + u_int32_t pkt_limit, u_int32_t byte_limit, void **head, + void **tail, u_int32_t *cnt, u_int32_t *len, boolean_t drvmgt, + classq_pkt_type_t *ptype) { struct ifnet *ifp = ifq->ifcq_ifp; - u_int32_t i = 0, l = 0; - struct mbuf **first, *last; -#if PF_ALTQ - struct ifaltq *altq = IFCQ_ALTQ(ifq); - boolean_t draining; -#endif /* PF_ALTQ */ + u_int32_t i = 0, l = 0, lock_spin = 1 ; + void **first, *last; VERIFY(!drvmgt || MBUF_VALID_SC(sc)); + *ptype = 0; + + + if (IFCQ_TBR_IS_ENABLED(ifq)) + goto dequeue_loop; + /* * If the scheduler support dequeueing multiple packets at the * same time, call that one instead. */ + if (drvmgt && ifq->ifcq_dequeue_sc_multi != NULL) { + int err; - if (ifq->ifcq_dequeue_multi != NULL) { + if (lock_spin) + IFCQ_LOCK_SPIN(ifq); + else + IFCQ_LOCK(ifq); + err = ifq->ifcq_dequeue_sc_multi(ifq, sc, pkt_limit, + byte_limit, head, tail, cnt, len, ptype); + IFCQ_UNLOCK(ifq); + + if (err == 0 && (*head) == NULL) + err = EAGAIN; + return (err); + } else if (ifq->ifcq_dequeue_multi != NULL) { int err; - IFCQ_LOCK_SPIN(ifq); - err = ifq->ifcq_dequeue_multi(ifq, CLASSQDQ_REMOVE, - pkt_limit, byte_limit, head, tail, cnt, len); + + if (lock_spin) + IFCQ_LOCK_SPIN(ifq); + else + IFCQ_LOCK(ifq); + + err = ifq->ifcq_dequeue_multi(ifq, pkt_limit, byte_limit, + head, tail, cnt, len, ptype); IFCQ_UNLOCK(ifq); if (err == 0 && (*head) == NULL) @@ -370,75 +357,50 @@ ifclassq_dequeue_common(struct ifclassq *ifq, mbuf_svc_class_t sc, return (err); } +dequeue_loop: *head = NULL; first = &(*head); last = NULL; - IFCQ_LOCK_SPIN(ifq); + if (lock_spin) + IFCQ_LOCK_SPIN(ifq); + else + IFCQ_LOCK(ifq); while (i < pkt_limit && l < byte_limit) { -#if PF_ALTQ - u_int32_t qlen; - - qlen = IFCQ_LEN(ifq); - draining = IFCQ_IS_DRAINING(ifq); - - if (drvmgt) { - if (IFCQ_TBR_IS_ENABLED(ifq)) - IFCQ_TBR_DEQUEUE_SC(ifq, sc, *head); - else if (draining) - IFCQ_DEQUEUE_SC(ifq, sc, *head); - else if (ALTQ_IS_ENABLED(altq)) - ALTQ_DEQUEUE_SC(altq, sc, *head); - else - *head = NULL; - } else { - if (IFCQ_TBR_IS_ENABLED(ifq)) - IFCQ_TBR_DEQUEUE(ifq, *head); - else if (draining) - IFCQ_DEQUEUE(ifq, *head); - else if (ALTQ_IS_ENABLED(altq)) - ALTQ_DEQUEUE(altq, *head); - else - *head = NULL; - } - - if (draining && *head != NULL) { - VERIFY(ifq->ifcq_drain >= (qlen - IFCQ_LEN(ifq))); - ifq->ifcq_drain -= (qlen - IFCQ_LEN(ifq)); - } -#else /* ! PF_ALTQ */ + classq_pkt_type_t tmp_ptype; if (drvmgt) { if (IFCQ_TBR_IS_ENABLED(ifq)) - IFCQ_TBR_DEQUEUE_SC(ifq, sc, *head); + IFCQ_TBR_DEQUEUE_SC(ifq, sc, *head, &tmp_ptype); else - IFCQ_DEQUEUE_SC(ifq, sc, *head); + IFCQ_DEQUEUE_SC(ifq, sc, *head, &tmp_ptype); } else { if (IFCQ_TBR_IS_ENABLED(ifq)) - IFCQ_TBR_DEQUEUE(ifq, *head); + IFCQ_TBR_DEQUEUE(ifq, *head, &tmp_ptype); else - IFCQ_DEQUEUE(ifq, *head); + IFCQ_DEQUEUE(ifq, *head, &tmp_ptype); } -#endif /* !PF_ALTQ */ if (*head == NULL) break; - (*head)->m_nextpkt = NULL; - last = *head; + switch (tmp_ptype) { + case QP_MBUF: + (*((mbuf_t *)head))->m_nextpkt = NULL; + last = *head; + l += (*((mbuf_t *)head))->m_pkthdr.len; + ifclassq_set_packet_metadata(ifq, ifp, (*head), + QP_MBUF); + head = (void **)&(*((mbuf_t *)head))->m_nextpkt; + break; - l += (*head)->m_pkthdr.len; -#if MEASURE_BW - (*head)->m_pkthdr.pkt_bwseq = - atomic_add_64_ov(&(ifp->if_bw.cur_seq), m_pktlen(*head)); -#endif /* MEASURE_BW */ - if (IFNET_IS_CELLULAR(ifp)) { - (*head)->m_pkthdr.pkt_flags |= PKTF_VALID_UNSENT_DATA; - (*head)->m_pkthdr.bufstatus_if = IFCQ_BYTES(ifq); - (*head)->m_pkthdr.bufstatus_sndbuf = ifp->if_sndbyte_unsent; + default: + VERIFY(0); + /* NOTREACHED */ } - head = &(*head)->m_nextpkt; + + *ptype = tmp_ptype; i++; } @@ -454,76 +416,11 @@ ifclassq_dequeue_common(struct ifclassq *ifq, mbuf_svc_class_t sc, return ((*first != NULL) ? 0 : EAGAIN); } -struct mbuf * -ifclassq_poll(struct ifclassq *ifq) -{ - return (ifclassq_poll_common(ifq, MBUF_SC_UNSPEC, FALSE)); -} - -struct mbuf * -ifclassq_poll_sc(struct ifclassq *ifq, mbuf_svc_class_t sc) -{ - return (ifclassq_poll_common(ifq, sc, TRUE)); -} - -static struct mbuf * -ifclassq_poll_common(struct ifclassq *ifq, mbuf_svc_class_t sc, - boolean_t drvmgt) -{ -#if PF_ALTQ - struct ifaltq *altq = IFCQ_ALTQ(ifq); -#endif /* PF_ALTQ */ - struct mbuf *m; - - VERIFY(!drvmgt || MBUF_VALID_SC(sc)); - -#if PF_ALTQ - if (drvmgt) { - if (IFCQ_TBR_IS_ENABLED(ifq)) - IFCQ_TBR_POLL_SC(ifq, sc, m); - else if (IFCQ_IS_DRAINING(ifq)) - IFCQ_POLL_SC(ifq, sc, m); - else if (ALTQ_IS_ENABLED(altq)) - ALTQ_POLL_SC(altq, sc, m); - else - m = NULL; - } else { - if (IFCQ_TBR_IS_ENABLED(ifq)) - IFCQ_TBR_POLL(ifq, m); - else if (IFCQ_IS_DRAINING(ifq)) - IFCQ_POLL(ifq, m); - else if (ALTQ_IS_ENABLED(altq)) - ALTQ_POLL(altq, m); - else - m = NULL; - } -#else /* ! PF_ALTQ */ - if (drvmgt) { - if (IFCQ_TBR_IS_ENABLED(ifq)) - IFCQ_TBR_POLL_SC(ifq, sc, m); - else - IFCQ_POLL_SC(ifq, sc, m); - } else { - if (IFCQ_TBR_IS_ENABLED(ifq)) - IFCQ_TBR_POLL(ifq, m); - else - IFCQ_POLL(ifq, m); - } -#endif /* !PF_ALTQ */ - - return (m); -} - void ifclassq_update(struct ifclassq *ifq, cqev_t ev) { IFCQ_LOCK_ASSERT_HELD(ifq); VERIFY(IFCQ_IS_READY(ifq)); - -#if PF_ALTQ - if (ALTQ_IS_ENABLED(IFCQ_ALTQ(ifq))) - ALTQ_UPDATE(IFCQ_ALTQ(ifq), ev); -#endif /* PF_ALTQ */ IFCQ_UPDATE(ifq, ev); } @@ -531,13 +428,12 @@ int ifclassq_attach(struct ifclassq *ifq, u_int32_t type, void *discipline, ifclassq_enq_func enqueue, ifclassq_deq_func dequeue, ifclassq_deq_sc_func dequeue_sc, ifclassq_deq_multi_func dequeue_multi, - ifclassq_req_func request) + ifclassq_deq_sc_multi_func dequeue_sc_multi, ifclassq_req_func request) { IFCQ_LOCK_ASSERT_HELD(ifq); VERIFY(ifq->ifcq_disc == NULL); VERIFY(enqueue != NULL); - VERIFY(!(dequeue != NULL && dequeue_sc != NULL)); VERIFY(request != NULL); ifq->ifcq_type = type; @@ -546,6 +442,7 @@ ifclassq_attach(struct ifclassq *ifq, u_int32_t type, void *discipline, ifq->ifcq_dequeue = dequeue; ifq->ifcq_dequeue_sc = dequeue_sc; ifq->ifcq_dequeue_multi = dequeue_multi; + ifq->ifcq_dequeue_sc_multi = dequeue_sc_multi; ifq->ifcq_request = request; return (0); @@ -652,24 +549,25 @@ ifclassq_ev2str(cqev_t ev) #define TBR_SCALE(x) ((int64_t)(x) << TBR_SHIFT) #define TBR_UNSCALE(x) ((x) >> TBR_SHIFT) -struct mbuf * -ifclassq_tbr_dequeue(struct ifclassq *ifq, int op) +void * +ifclassq_tbr_dequeue(struct ifclassq *ifq, classq_pkt_type_t *ptype) { - return (ifclassq_tbr_dequeue_common(ifq, op, MBUF_SC_UNSPEC, FALSE)); + return (ifclassq_tbr_dequeue_common(ifq, MBUF_SC_UNSPEC, FALSE, ptype)); } -struct mbuf * -ifclassq_tbr_dequeue_sc(struct ifclassq *ifq, int op, mbuf_svc_class_t sc) +void * +ifclassq_tbr_dequeue_sc(struct ifclassq *ifq, mbuf_svc_class_t sc, + classq_pkt_type_t *ptype) { - return (ifclassq_tbr_dequeue_common(ifq, op, sc, TRUE)); + return (ifclassq_tbr_dequeue_common(ifq, sc, TRUE, ptype)); } -static struct mbuf * -ifclassq_tbr_dequeue_common(struct ifclassq *ifq, int op, - mbuf_svc_class_t sc, boolean_t drvmgt) +static void * +ifclassq_tbr_dequeue_common(struct ifclassq *ifq, mbuf_svc_class_t sc, + boolean_t drvmgt, classq_pkt_type_t *ptype) { struct tb_regulator *tbr; - struct mbuf *m; + void *p; int64_t interval; u_int64_t now; @@ -679,64 +577,46 @@ ifclassq_tbr_dequeue_common(struct ifclassq *ifq, int op, VERIFY(IFCQ_TBR_IS_ENABLED(ifq)); tbr = &ifq->ifcq_tbr; - if (op == CLASSQDQ_REMOVE && tbr->tbr_lastop == CLASSQDQ_POLL) { - /* if this is a remove after poll, bypass tbr check */ - } else { - /* update token only when it is negative */ - if (tbr->tbr_token <= 0) { - now = read_machclk(); - interval = now - tbr->tbr_last; - if (interval >= tbr->tbr_filluptime) { + /* update token only when it is negative */ + if (tbr->tbr_token <= 0) { + now = read_machclk(); + interval = now - tbr->tbr_last; + if (interval >= tbr->tbr_filluptime) { + tbr->tbr_token = tbr->tbr_depth; + } else { + tbr->tbr_token += interval * tbr->tbr_rate; + if (tbr->tbr_token > tbr->tbr_depth) tbr->tbr_token = tbr->tbr_depth; - } else { - tbr->tbr_token += interval * tbr->tbr_rate; - if (tbr->tbr_token > tbr->tbr_depth) - tbr->tbr_token = tbr->tbr_depth; - } - tbr->tbr_last = now; } - /* if token is still negative, don't allow dequeue */ - if (tbr->tbr_token <= 0) - return (NULL); + tbr->tbr_last = now; } + /* if token is still negative, don't allow dequeue */ + if (tbr->tbr_token <= 0) + return (NULL); /* * ifclassq takes precedence over ALTQ queue; * ifcq_drain count is adjusted by the caller. */ -#if PF_ALTQ - if (IFCQ_IS_DRAINING(ifq)) { -#endif /* PF_ALTQ */ - if (op == CLASSQDQ_POLL) { - if (drvmgt) - IFCQ_POLL_SC(ifq, sc, m); - else - IFCQ_POLL(ifq, m); - } else { - if (drvmgt) - IFCQ_DEQUEUE_SC(ifq, sc, m); - else - IFCQ_DEQUEUE(ifq, m); - } -#if PF_ALTQ - } else { - struct ifaltq *altq = IFCQ_ALTQ(ifq); - if (ALTQ_IS_ENABLED(altq)) { - if (drvmgt) - m = (*altq->altq_dequeue_sc)(altq, sc, op); - else - m = (*altq->altq_dequeue)(altq, op); - } else { - m = NULL; + if (drvmgt) + IFCQ_DEQUEUE_SC(ifq, sc, p, ptype); + else + IFCQ_DEQUEUE(ifq, p, ptype); + + if (p != NULL) { + switch (*ptype) { + case QP_MBUF: + tbr->tbr_token -= TBR_SCALE(m_pktlen((mbuf_t)p)); + break; + + + default: + VERIFY(0); + /* NOTREACHED */ } } -#endif /* PF_ALTQ */ - - if (m != NULL && op == CLASSQDQ_REMOVE) - tbr->tbr_token -= TBR_SCALE(m_pktlen(m)); - tbr->tbr_lastop = op; - return (m); + return (p); } /* @@ -840,7 +720,6 @@ ifclassq_tbr_set(struct ifclassq *ifq, struct tb_profile *profile, } tbr->tbr_token = tbr->tbr_depth; tbr->tbr_last = read_machclk(); - tbr->tbr_lastop = CLASSQDQ_REMOVE; if (tbr->tbr_rate > 0 && (ifp->if_flags & IFF_UP)) { struct timespec ts = @@ -874,18 +753,18 @@ ifclassq_tbr_set(struct ifclassq *ifq, struct tb_profile *profile, void ifclassq_calc_target_qdelay(struct ifnet *ifp, u_int64_t *if_target_qdelay) { - u_int64_t target_qdelay = 0; - target_qdelay = IFCQ_TARGET_QDELAY(&ifp->if_snd); + u_int64_t qdelay = 0; + qdelay = IFCQ_TARGET_QDELAY(&ifp->if_snd); if (ifclassq_target_qdelay != 0) - target_qdelay = ifclassq_target_qdelay; + qdelay = ifclassq_target_qdelay; /* * If we do not know the effective bandwidth, use the default * target queue delay. */ - if (target_qdelay == 0) - target_qdelay = IFQ_TARGET_DELAY; + if (qdelay == 0) + qdelay = IFQ_TARGET_DELAY; /* * If a delay has been added to ifnet start callback for @@ -894,9 +773,9 @@ ifclassq_calc_target_qdelay(struct ifnet *ifp, u_int64_t *if_target_qdelay) */ if ((ifp->if_eflags & IFEF_ENQUEUE_MULTI) && ifp->if_start_delay_timeout > 0) - target_qdelay += ifp->if_start_delay_timeout; + qdelay += ifp->if_start_delay_timeout; - *(if_target_qdelay) = target_qdelay; + *(if_target_qdelay) = qdelay; } void diff --git a/bsd/net/classq/if_classq.h b/bsd/net/classq/if_classq.h index de8ddc60c..148426903 100644 --- a/bsd/net/classq/if_classq.h +++ b/bsd/net/classq/if_classq.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2013 Apple Inc. All rights reserved. + * Copyright (c) 2011-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -34,11 +34,9 @@ #ifdef BSD_KERNEL_PRIVATE #include <net/classq/classq.h> -/* classq dequeue op arg */ -typedef enum cqdq_op { - CLASSQDQ_REMOVE = 1, /* dequeue mbuf from the queue */ - CLASSQDQ_POLL = 2, /* don't dequeue mbuf from the queue */ -} cqdq_op_t; + +/* maximum number of packets stored across all queues */ +#define IFCQ_DEFAULT_PKT_DROP_LIMIT 2048 /* classq request types */ typedef enum cqrq { @@ -70,10 +68,6 @@ typedef struct cqrq_stat_sc { u_int32_t bytes; /* (out) bytes enqueued */ } cqrq_stat_sc_t; -#if PF_ALTQ -#include <net/altq/if_altq.h> -#endif /* PF_ALTQ */ - /* * A token-bucket regulator limits the rate that a network driver can * dequeue packets from the output queue. Modern cards are able to buffer @@ -92,7 +86,6 @@ struct tb_regulator { int64_t tbr_filluptime; /* (scaled) time to fill up bucket */ u_int64_t tbr_last; /* last time token was updated */ - int tbr_lastop; /* last dequeue operation type */ /* needed for poll-and-dequeue */ }; @@ -107,13 +100,16 @@ struct ifclassq; enum cqdq_op; enum cqrq; -typedef int (*ifclassq_enq_func)(struct ifclassq *, struct mbuf *); -typedef struct mbuf *(*ifclassq_deq_func)(struct ifclassq *, enum cqdq_op); -typedef struct mbuf *(*ifclassq_deq_sc_func)(struct ifclassq *, - mbuf_svc_class_t, enum cqdq_op); -typedef int (*ifclassq_deq_multi_func)(struct ifclassq *, enum cqdq_op, - u_int32_t, u_int32_t, struct mbuf **, struct mbuf **, u_int32_t *, - u_int32_t *); +typedef int (*ifclassq_enq_func)(struct ifclassq *, void *, classq_pkt_type_t, + boolean_t *); +typedef void *(*ifclassq_deq_func)(struct ifclassq *, classq_pkt_type_t *); +typedef void *(*ifclassq_deq_sc_func)(struct ifclassq *, + mbuf_svc_class_t, classq_pkt_type_t *); +typedef int (*ifclassq_deq_multi_func)(struct ifclassq *, u_int32_t, + u_int32_t, void **, void **, u_int32_t *, u_int32_t *, classq_pkt_type_t *); +typedef int (*ifclassq_deq_sc_multi_func)(struct ifclassq *, + mbuf_svc_class_t, u_int32_t, u_int32_t, void **, void **, + u_int32_t *, u_int32_t *, classq_pkt_type_t *); typedef int (*ifclassq_req_func)(struct ifclassq *, enum cqrq, void *); /* @@ -133,6 +129,7 @@ struct ifclassq { u_int32_t ifcq_sflags; /* scheduler flags */ u_int32_t ifcq_target_qdelay; /* target queue delay */ u_int32_t ifcq_bytes; /* bytes count */ + u_int32_t ifcq_pkt_drop_limit; void *ifcq_disc; /* for scheduler-specific use */ /* * ifcq_disc_slots[] represents the leaf classes configured for the @@ -162,22 +159,13 @@ struct ifclassq { ifclassq_deq_func ifcq_dequeue; ifclassq_deq_sc_func ifcq_dequeue_sc; ifclassq_deq_multi_func ifcq_dequeue_multi; + ifclassq_deq_sc_multi_func ifcq_dequeue_sc_multi; ifclassq_req_func ifcq_request; /* token bucket regulator */ struct tb_regulator ifcq_tbr; /* TBR */ - -#if PF_ALTQ - u_int32_t ifcq_drain; - struct ifaltq ifcq_altq; -#endif /* PF_ALTQ */ }; -#if PF_ALTQ -#define IFCQ_ALTQ(_ifcq) (&(_ifcq)->ifcq_altq) -#define IFCQ_IS_DRAINING(_ifcq) ((_ifcq)->ifcq_drain > 0) -#endif /* PF_ALTQ */ - /* ifcq_flags */ #define IFCQF_READY 0x01 /* ifclassq supports discipline */ #define IFCQF_ENABLED 0x02 /* ifclassq is in use */ @@ -188,17 +176,20 @@ struct ifclassq { #define IFCQ_TBR_IS_ENABLED(_ifcq) ((_ifcq)->ifcq_flags & IFCQF_TBR) /* classq enqueue return value */ -#define CLASSQEQ_DROPPED (-1) /* packet dropped (freed) */ -#define CLASSQEQ_SUCCESS 0 /* success, packet enqueued */ -#define CLASSQEQ_SUCCESS_FC 1 /* packet enqueued; */ - /* give flow control feedback */ -#define CLASSQEQ_DROPPED_FC 2 /* packet dropped; */ - /* give flow control feedback */ -#define CLASSQEQ_DROPPED_SP 3 /* packet dropped due to suspension; */ - /* give flow control feedback */ +/* packet has to be dropped */ +#define CLASSQEQ_DROP (-1) +/* packet successfully enqueued */ +#define CLASSQEQ_SUCCESS 0 +/* packet enqueued; give flow control feedback */ +#define CLASSQEQ_SUCCESS_FC 1 +/* packet needs to be dropped due to flowcontrol; give flow control feedback */ +#define CLASSQEQ_DROP_FC 2 +/* packet needs to be dropped due to suspension; give flow control feedback */ +#define CLASSQEQ_DROP_SP 3 /* interface event argument for CLASSQRQ_EVENT */ typedef enum cqev { + CLASSQ_EV_INIT = 0, CLASSQ_EV_LINK_BANDWIDTH = 1, /* link bandwidth has changed */ CLASSQ_EV_LINK_LATENCY = 2, /* link latency has changed */ CLASSQ_EV_LINK_MTU = 3, /* link MTU has changed */ @@ -207,11 +198,7 @@ typedef enum cqev { } cqev_t; #endif /* BSD_KERNEL_PRIVATE */ -#include <net/pktsched/pktsched_priq.h> -#include <net/pktsched/pktsched_fairq.h> #include <net/pktsched/pktsched_tcq.h> -#include <net/pktsched/pktsched_cbq.h> -#include <net/pktsched/pktsched_hfsc.h> #include <net/pktsched/pktsched_qfq.h> #include <net/pktsched/pktsched_fq_codel.h> @@ -225,11 +212,7 @@ struct if_ifclassq_stats { struct pktcntr ifqs_dropcnt; u_int32_t ifqs_scheduler; union { - struct priq_classstats ifqs_priq_stats; - struct fairq_classstats ifqs_fairq_stats; struct tcq_classstats ifqs_tcq_stats; - struct cbq_classstats ifqs_cbq_stats; - struct hfsc_classstats ifqs_hfsc_stats; struct qfq_classstats ifqs_qfq_stats; struct fq_codel_classstats ifqs_fq_codel_stats; }; @@ -244,10 +227,10 @@ struct if_ifclassq_stats { * For ifclassq lock */ #define IFCQ_LOCK_ASSERT_HELD(_ifcq) \ - lck_mtx_assert(&(_ifcq)->ifcq_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_ifcq)->ifcq_lock, LCK_MTX_ASSERT_OWNED) #define IFCQ_LOCK_ASSERT_NOTHELD(_ifcq) \ - lck_mtx_assert(&(_ifcq)->ifcq_lock, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_ifcq)->ifcq_lock, LCK_MTX_ASSERT_NOTOWNED) #define IFCQ_LOCK(_ifcq) \ lck_mtx_lock(&(_ifcq)->ifcq_lock) @@ -266,40 +249,24 @@ struct if_ifclassq_stats { /* * For ifclassq operations */ -#define IFCQ_ENQUEUE(_ifq, _m, _err) do { \ - (_err) = (*(_ifq)->ifcq_enqueue)(_ifq, _m); \ +#define IFCQ_ENQUEUE(_ifq, _p, _t, _err, _drop) do { \ + (_err) = (*(_ifq)->ifcq_enqueue)(_ifq, _p, _t, _drop); \ } while (0) -#define IFCQ_DEQUEUE(_ifq, _m) do { \ - (_m) = (*(_ifq)->ifcq_dequeue)(_ifq, CLASSQDQ_REMOVE); \ +#define IFCQ_DEQUEUE(_ifq, _p, _t) do { \ + (_p) = (*(_ifq)->ifcq_dequeue)(_ifq, _t); \ } while (0) -#define IFCQ_DEQUEUE_SC(_ifq, _sc, _m) do { \ - (_m) = (*(_ifq)->ifcq_dequeue_sc)(_ifq, _sc, CLASSQDQ_REMOVE); \ +#define IFCQ_DEQUEUE_SC(_ifq, _sc, _p, _t) do { \ + (_p) = (*(_ifq)->ifcq_dequeue_sc)(_ifq, _sc, _t); \ } while (0) -#define IFCQ_TBR_DEQUEUE(_ifcq, _m) do { \ - (_m) = ifclassq_tbr_dequeue(_ifcq, CLASSQDQ_REMOVE); \ +#define IFCQ_TBR_DEQUEUE(_ifcq, _p, _t) do { \ + (_p) = ifclassq_tbr_dequeue(_ifcq, _t); \ } while (0) -#define IFCQ_TBR_DEQUEUE_SC(_ifcq, _sc, _m) do { \ - (_m) = ifclassq_tbr_dequeue_sc(_ifcq, CLASSQDQ_REMOVE, _sc); \ -} while (0) - -#define IFCQ_POLL(_ifq, _m) do { \ - (_m) = (*(_ifq)->ifcq_dequeue)(_ifq, CLASSQDQ_POLL); \ -} while (0) - -#define IFCQ_POLL_SC(_ifq, _sc, _m) do { \ - (_m) = (*(_ifq)->ifcq_dequeue_sc)(_ifq, _sc, CLASSQDQ_POLL); \ -} while (0) - -#define IFCQ_TBR_POLL(_ifcq, _m) do { \ - (_m) = ifclassq_tbr_dequeue(_ifcq, CLASSQDQ_POLL); \ -} while (0) - -#define IFCQ_TBR_POLL_SC(_ifcq, _sc, _m) do { \ - (_m) = ifclassq_tbr_dequeue_sc(_ifcq, CLASSQDQ_POLL, _sc); \ +#define IFCQ_TBR_DEQUEUE_SC(_ifcq, _sc, _p, _t) do { \ + (_p) = ifclassq_tbr_dequeue_sc(_ifcq, _sc, _t); \ } while (0) #define IFCQ_PURGE(_ifq) do { \ @@ -349,9 +316,9 @@ struct if_ifclassq_stats { #define IFCQ_SET_MAXLEN(_ifcq, _len) ((_ifcq)->ifcq_maxlen = (_len)) #define IFCQ_TARGET_QDELAY(_ifcq) ((_ifcq)->ifcq_target_qdelay) #define IFCQ_BYTES(_ifcq) ((_ifcq)->ifcq_bytes) -#define IFCQ_INC_BYTES(_ifcq, _len) \ +#define IFCQ_INC_BYTES(_ifcq, _len) \ ((_ifcq)->ifcq_bytes = (_ifcq)->ifcq_bytes + (_len)) -#define IFCQ_DEC_BYTES(_ifcq, _len) \ +#define IFCQ_DEC_BYTES(_ifcq, _len) \ ((_ifcq)->ifcq_bytes = (_ifcq)->ifcq_bytes - (_len)) #define IFCQ_XMIT_ADD(_ifcq, _pkt, _len) do { \ @@ -362,6 +329,8 @@ struct if_ifclassq_stats { PKTCNTR_ADD(&(_ifcq)->ifcq_dropcnt, _pkt, _len); \ } while (0) +#define IFCQ_PKT_DROP_LIMIT(_ifcq) ((_ifcq)->ifcq_pkt_drop_limit) + extern int ifclassq_setup(struct ifnet *, u_int32_t, boolean_t); extern void ifclassq_teardown(struct ifnet *); extern int ifclassq_pktsched_setup(struct ifclassq *); @@ -369,28 +338,33 @@ extern void ifclassq_set_maxlen(struct ifclassq *, u_int32_t); extern u_int32_t ifclassq_get_maxlen(struct ifclassq *); extern int ifclassq_get_len(struct ifclassq *, mbuf_svc_class_t, u_int32_t *, u_int32_t *); -extern errno_t ifclassq_enqueue(struct ifclassq *, struct mbuf *); +extern errno_t ifclassq_enqueue(struct ifclassq *, void *, classq_pkt_type_t, + boolean_t *); extern errno_t ifclassq_dequeue(struct ifclassq *, u_int32_t, u_int32_t, - struct mbuf **, struct mbuf **, u_int32_t *, u_int32_t *); + void **, void **, u_int32_t *, u_int32_t *, classq_pkt_type_t *); extern errno_t ifclassq_dequeue_sc(struct ifclassq *, mbuf_svc_class_t, - u_int32_t, struct mbuf **, struct mbuf **, u_int32_t *, u_int32_t *); -extern struct mbuf *ifclassq_poll(struct ifclassq *); -extern struct mbuf *ifclassq_poll_sc(struct ifclassq *, mbuf_svc_class_t); + u_int32_t, u_int32_t, void **, void **, u_int32_t *, u_int32_t *, + classq_pkt_type_t *); +extern void *ifclassq_poll(struct ifclassq *, classq_pkt_type_t *); +extern void *ifclassq_poll_sc(struct ifclassq *, mbuf_svc_class_t, + classq_pkt_type_t *); extern void ifclassq_update(struct ifclassq *, cqev_t); extern int ifclassq_attach(struct ifclassq *, u_int32_t, void *, ifclassq_enq_func, ifclassq_deq_func, ifclassq_deq_sc_func, - ifclassq_deq_multi_func, ifclassq_req_func); + ifclassq_deq_multi_func, ifclassq_deq_sc_multi_func, ifclassq_req_func); extern int ifclassq_detach(struct ifclassq *); extern int ifclassq_getqstats(struct ifclassq *, u_int32_t, void *, u_int32_t *); extern const char *ifclassq_ev2str(cqev_t); extern int ifclassq_tbr_set(struct ifclassq *, struct tb_profile *, boolean_t); -extern struct mbuf *ifclassq_tbr_dequeue(struct ifclassq *, int); -extern struct mbuf *ifclassq_tbr_dequeue_sc(struct ifclassq *, int, - mbuf_svc_class_t); +extern void *ifclassq_tbr_dequeue(struct ifclassq *, classq_pkt_type_t *); +extern void *ifclassq_tbr_dequeue_sc(struct ifclassq *, mbuf_svc_class_t, + classq_pkt_type_t *); extern void ifclassq_calc_target_qdelay(struct ifnet *ifp, u_int64_t *if_target_qdelay); extern void ifclassq_calc_update_interval(u_int64_t *update_interval); +extern void ifclassq_set_packet_metadata(struct ifclassq *ifq, + struct ifnet *ifp, void *p, classq_pkt_type_t ptype); #endif /* BSD_KERNEL_PRIVATE */ #endif /* PRIVATE */ diff --git a/bsd/net/content_filter.c b/bsd/net/content_filter.c index 68cdae1e8..4a685a80a 100644 --- a/bsd/net/content_filter.c +++ b/bsd/net/content_filter.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2014 Apple Inc. All rights reserved. + * Copyright (c) 2013-2017 Apple Inc. All rights reserved. * * @APPLE_LICENSE_HEADER_START@ * @@ -32,7 +32,7 @@ * also kept in kernel buffer until the user space agents makes a pass or drop * decision. This unidirectional flow of content avoids unnecessary data copies * back to the kernel. - * * + * * A user space filter agent opens a kernel control socket with the name * CONTENT_FILTER_CONTROL_NAME to attach to the socket content filter subsystem. * When connected, a "struct content_filter" is created and set as the @@ -54,7 +54,7 @@ * NECP FILTER CONTROL UNIT * * A user space filter agent uses the Network Extension Control Policy (NECP) - * database specify which TCP/IP sockets needs to be filtered. The NECP + * database to specify which TCP/IP sockets need to be filtered. The NECP * criteria may be based on a variety of properties like user ID or proc UUID. * * The NECP "filter control unit" is used by the socket content filter subsystem @@ -77,7 +77,7 @@ * 4) The NECP filter control unit is then used to find the corresponding * kernel control socket instance. * - * Note: NECP currently supports a ingle filter control unit per TCP/IP socket + * Note: NECP currently supports a single filter control unit per TCP/IP socket * but this restriction may be soon lifted. * * @@ -117,16 +117,19 @@ * * After a CFM_OP_SOCKET_ATTACHED is delivered, CFM_OP_DATA_OUT and * CFM_OP_DATA_OUT events are not delivered until a CFM_OP_DATA_UPDATE - * action message is send by the user space filter agent. + * action message is sent by the user space filter agent. * * Note: absolute 64 bits offsets should be large enough for the foreseeable - * future. A 64-bits counter will wrap after 468 years are 10 Gbit/sec: + * future. A 64-bits counter will wrap after 468 years at 10 Gbit/sec: * 2E64 / ((10E9 / 8) * 60 * 60 * 24 * 365.25) = 467.63 * - * They are two kinds of content filter actions: + * They are two kinds of primary content filter actions: * - CFM_OP_DATA_UPDATE: to update pass or peek offsets for each direction. * - CFM_OP_DROP: to shutdown socket and disallow further data flow * + * There is also an action to mark a given client flow as already filtered + * at a higher level, CFM_OP_BLESS_CLIENT. + * * * ACTION MESSAGES * @@ -196,7 +199,7 @@ * CONTENT FILTER QUEUES * * Data that is being filtered is steered away from the TCP/IP socket buffer - * and instead will sit in one of three content filter queue until the data + * and instead will sit in one of three content filter queues until the data * can be re-injected into the TCP/IP socket buffer. * * A content filter queue is represented by "struct cfil_queue" that contains @@ -209,7 +212,7 @@ * b) The "cfe_pending_q" of "struct cfil_entry" * c) The "cfi_inject_q" of "struct cfil_info" * - * Note: The seqyence (a),(b) may be repeated several times if there are more + * Note: The sequence (a),(b) may be repeated several times if there is more * than one content filter attached to the TCP/IP socket. * * The "cfe_ctl_q" queue holds data than cannot be delivered to the @@ -417,6 +420,16 @@ struct cfil_entry { #define CFEF_SENT_SOCK_CLOSED 0x0040 /* closed event was sent */ #define CFEF_CFIL_DETACHED 0x0080 /* filter was detached */ + +#define CFI_ADD_TIME_LOG(cfil, t1, t0, op) \ + struct timeval _tdiff; \ + if ((cfil)->cfi_op_list_ctr < CFI_MAX_TIME_LOG_ENTRY) { \ + timersub(t1, t0, &_tdiff); \ + (cfil)->cfi_op_time[(cfil)->cfi_op_list_ctr] = (uint32_t)(_tdiff.tv_sec * 1000 + _tdiff.tv_usec / 1000);\ + (cfil)->cfi_op_list[(cfil)->cfi_op_list_ctr] = (unsigned char)op; \ + (cfil)->cfi_op_list_ctr ++; \ + } + /* * struct cfil_info * @@ -427,6 +440,10 @@ struct cfil_info { struct socket *cfi_so; uint64_t cfi_flags; uint64_t cfi_sock_id; + struct timeval64 cfi_first_event; + uint32_t cfi_op_list_ctr; + uint32_t cfi_op_time[CFI_MAX_TIME_LOG_ENTRY]; /* time interval in microseconds since first event */ + unsigned char cfi_op_list[CFI_MAX_TIME_LOG_ENTRY]; struct cfi_buf { /* @@ -451,7 +468,7 @@ struct cfil_info { } cfi_snd, cfi_rcv; struct cfil_entry cfi_entries[MAX_CONTENT_FILTER]; -}; +} __attribute__((aligned(8))); #define CFIF_DROP 0x0001 /* drop action applied */ #define CFIF_CLOSE_WAIT 0x0002 /* waiting for filter to close */ @@ -527,6 +544,7 @@ SYSCTL_STRUCT(_net_cfil, OID_AUTO, stats, CTLFLAG_RD|CTLFLAG_LOCKED, static int cfil_action_data_pass(struct socket *, uint32_t, int, uint64_t, uint64_t); static int cfil_action_drop(struct socket *, uint32_t); +static int cfil_action_bless_client(uint32_t, struct cfil_msg_hdr *); static int cfil_dispatch_closed_event(struct socket *, int); static int cfil_data_common(struct socket *, int, struct sockaddr *, struct mbuf *, struct mbuf *, uint32_t); @@ -541,6 +559,7 @@ static void cfil_info_free(struct socket *, struct cfil_info *); static struct cfil_info * cfil_info_alloc(struct socket *); static int cfil_info_attach_unit(struct socket *, uint32_t); static struct socket * cfil_socket_from_sock_id(cfil_sock_id_t); +static struct socket *cfil_socket_from_client_uuid(uuid_t, bool *); static int cfil_service_pending_queue(struct socket *, uint32_t, int); static int cfil_data_service_ctl_q(struct socket *, uint32_t, int); static void cfil_info_verify(struct cfil_info *); @@ -647,23 +666,13 @@ cfil_rw_lock_exclusive_to_shared(lck_rw_t *lck) static void cfil_rw_lock_assert_held(lck_rw_t *lck, int exclusive) { - lck_rw_assert(lck, +#if !MACH_ASSERT +#pragma unused(lck, exclusive) +#endif + LCK_RW_ASSERT(lck, exclusive ? LCK_RW_ASSERT_EXCLUSIVE : LCK_RW_ASSERT_HELD); } -static void -socket_lock_assert_owned(struct socket *so) -{ - lck_mtx_t *mutex_held; - - if (so->so_proto->pr_getlock != NULL) - mutex_held = (*so->so_proto->pr_getlock)(so, 0); - else - mutex_held = so->so_proto->pr_domain->dom_mtx; - - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); -} - /* * Return the number of bytes in the mbuf chain using the same * method as m_length() or sballoc() @@ -1131,11 +1140,11 @@ cfil_acquire_sockbuf(struct socket *so, int outgoing) while ((sb->sb_flags & SB_LOCK) || (sb->sb_cfil_thread != NULL && sb->sb_cfil_thread != tp)) { if (so->so_proto->pr_getlock != NULL) - mutex_held = (*so->so_proto->pr_getlock)(so, 0); + mutex_held = (*so->so_proto->pr_getlock)(so, PR_F_WILLUNLOCK); else mutex_held = so->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); sb->sb_wantlock++; VERIFY(sb->sb_wantlock != 0); @@ -1244,6 +1253,28 @@ cfil_socket_from_sock_id(cfil_sock_id_t cfil_sock_id) return (so); } +static struct socket * +cfil_socket_from_client_uuid(uuid_t necp_client_uuid, bool *cfil_attached) +{ + struct socket *so = NULL; + struct inpcb *inp = NULL; + struct inpcbinfo *pcbinfo = &tcbinfo; + + lck_rw_lock_shared(pcbinfo->ipi_lock); + LIST_FOREACH(inp, pcbinfo->ipi_listhead, inp_list) { + if (inp->inp_state != INPCB_STATE_DEAD && + inp->inp_socket != NULL && + uuid_compare(inp->necp_client_uuid, necp_client_uuid) == 0) { + *cfil_attached = (inp->inp_socket->so_cfil != NULL); + so = inp->inp_socket; + break; + } + } + lck_rw_done(pcbinfo->ipi_lock); + + return (so); +} + static errno_t cfil_ctl_send(kern_ctl_ref kctlref, u_int32_t kcunit, void *unitinfo, mbuf_t m, int flags) @@ -1295,6 +1326,17 @@ cfil_ctl_send(kern_ctl_ref kctlref, u_int32_t kcunit, void *unitinfo, mbuf_t m, case CFM_OP_DROP: OSIncrementAtomic(&cfil_stats.cfs_ctl_action_drop); break; + case CFM_OP_BLESS_CLIENT: + if (msghdr->cfm_len != sizeof(struct cfil_msg_bless_client)) { + OSIncrementAtomic(&cfil_stats.cfs_ctl_action_bad_len); + error = EINVAL; + CFIL_LOG(LOG_ERR, "bad len: %u for op %u", + msghdr->cfm_len, + msghdr->cfm_op); + goto done; + } + error = cfil_action_bless_client(kcunit, msghdr); + goto done; default: OSIncrementAtomic(&cfil_stats.cfs_ctl_action_bad_op); CFIL_LOG(LOG_ERR, "bad op %u", msghdr->cfm_op); @@ -1360,6 +1402,7 @@ cfil_ctl_send(kern_ctl_ref kctlref, u_int32_t kcunit, void *unitinfo, mbuf_t m, } microuptime(&entry->cfe_last_action); + CFI_ADD_TIME_LOG(so->so_cfil, &entry->cfe_last_action, &so->so_cfil->cfi_first_event, msghdr->cfm_op); action_msg = (struct cfil_msg_action *)msghdr; @@ -1440,9 +1483,88 @@ cfil_ctl_getopt(kern_ctl_ref kctlref, u_int32_t kcunit, void *unitinfo, error = EINVAL; goto done; } - if (data != NULL) + if (data != NULL) { *(uint32_t *)data = cfc->cf_necp_control_unit; + } break; + case CFIL_OPT_GET_SOCKET_INFO: + if (*len != sizeof(struct cfil_opt_sock_info)) { + CFIL_LOG(LOG_ERR, "len does not match %lu", *len); + error = EINVAL; + goto done; + } + if (data == NULL) { + CFIL_LOG(LOG_ERR, "data not passed"); + error = EINVAL; + goto done; + } + + struct cfil_opt_sock_info *sock_info = + (struct cfil_opt_sock_info *) data; + struct socket *sock = + cfil_socket_from_sock_id(sock_info->cfs_sock_id); + if (sock == NULL) { + CFIL_LOG(LOG_NOTICE, "bad sock_id %llx", + sock_info->cfs_sock_id); + error = ENOENT; + goto done; + } + + // Unlock here so that we never hold both cfil_lck_rw and the + // socket_lock at the same time. Otherwise, this can deadlock + // because soclose() takes the socket_lock and then exclusive + // cfil_lck_rw and we require the opposite order. + + // WARNING: Be sure to never use anything protected + // by cfil_lck_rw beyond this point. + // WARNING: Be sure to avoid fallthrough and + // goto return_already_unlocked from this branch. + cfil_rw_unlock_shared(&cfil_lck_rw); + + socket_lock(sock, 1); + + if (sock->so_cfil == NULL) { + CFIL_LOG(LOG_NOTICE, "so %llx not attached, cannot fetch info", + (uint64_t)VM_KERNEL_ADDRPERM(sock)); + error = EINVAL; + socket_unlock(sock, 1); + goto return_already_unlocked; + } + + // Fill out family, type, and protocol + sock_info->cfs_sock_family = sock->so_proto->pr_domain->dom_family; + sock_info->cfs_sock_type = sock->so_proto->pr_type; + sock_info->cfs_sock_protocol = sock->so_proto->pr_protocol; + + // Source and destination addresses + struct inpcb *inp = sotoinpcb(sock); + if (inp->inp_vflag & INP_IPV6) { + fill_ip6_sockaddr_4_6(&sock_info->cfs_local, + &inp->in6p_laddr, inp->inp_lport); + fill_ip6_sockaddr_4_6(&sock_info->cfs_remote, + &inp->in6p_faddr, inp->inp_fport); + } else if (inp->inp_vflag & INP_IPV4) { + fill_ip_sockaddr_4_6(&sock_info->cfs_local, + inp->inp_laddr, inp->inp_lport); + fill_ip_sockaddr_4_6(&sock_info->cfs_remote, + inp->inp_faddr, inp->inp_fport); + } + + // Set the pid info + sock_info->cfs_pid = sock->last_pid; + memcpy(sock_info->cfs_uuid, sock->last_uuid, sizeof(uuid_t)); + + if (sock->so_flags & SOF_DELEGATED) { + sock_info->cfs_e_pid = sock->e_pid; + memcpy(sock_info->cfs_e_uuid, sock->e_uuid, sizeof(uuid_t)); + } else { + sock_info->cfs_e_pid = sock->last_pid; + memcpy(sock_info->cfs_e_uuid, sock->last_uuid, sizeof(uuid_t)); + } + + socket_unlock(sock, 1); + + goto return_already_unlocked; default: error = ENOPROTOOPT; break; @@ -1450,6 +1572,10 @@ cfil_ctl_getopt(kern_ctl_ref kctlref, u_int32_t kcunit, void *unitinfo, done: cfil_rw_unlock_shared(&cfil_lck_rw); + return (error); + +return_already_unlocked: + return (error); } @@ -1547,7 +1673,7 @@ cfil_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t kcunit, void *unitinfo, int flags) cfc->cf_flags &= ~CFF_FLOW_CONTROLLED; cfil_rw_lock_exclusive_to_shared(&cfil_lck_rw); - lck_rw_assert(&cfil_lck_rw, LCK_RW_ASSERT_SHARED); + LCK_RW_ASSERT(&cfil_lck_rw, LCK_RW_ASSERT_SHARED); } /* * Flow control will be raised again as soon as an entry cannot enqueue @@ -1915,7 +2041,8 @@ cfil_sock_attach(struct socket *so) so->so_proto->pr_domain->dom_family != PF_INET6) || so->so_proto->pr_type != SOCK_STREAM || so->so_proto->pr_protocol != IPPROTO_TCP || - (so->so_flags & SOF_MP_SUBFLOW) != 0) + (so->so_flags & SOF_MP_SUBFLOW) != 0 || + (so->so_flags1 & SOF1_CONTENT_FILTER_SKIP) != 0) goto done; filter_control_unit = necp_socket_get_content_filter_control_unit(so); @@ -1990,7 +2117,7 @@ cfil_dispatch_attach_event(struct socket *so, uint32_t filter_control_unit) struct cfil_entry *entry = NULL; struct cfil_msg_sock_attached msg_attached; uint32_t kcunit; - struct content_filter *cfc; + struct content_filter *cfc = NULL; socket_lock_assert_owned(so); @@ -2063,6 +2190,9 @@ cfil_dispatch_attach_event(struct socket *so, uint32_t filter_control_unit) goto done; } microuptime(&entry->cfe_last_event); + so->so_cfil->cfi_first_event.tv_sec = entry->cfe_last_event.tv_sec; + so->so_cfil->cfi_first_event.tv_usec = entry->cfe_last_event.tv_usec; + entry->cfe_flags |= CFEF_SENT_SOCK_ATTACHED; OSIncrementAtomic(&cfil_stats.cfs_attach_event_ok); done: @@ -2158,6 +2288,7 @@ cfil_dispatch_disconnect_event(struct socket *so, uint32_t kcunit, int outgoing) goto done; } microuptime(&entry->cfe_last_event); + CFI_ADD_TIME_LOG(so->so_cfil, &entry->cfe_last_event, &so->so_cfil->cfi_first_event, msg_disconnected.cfm_op); /* Remember we have sent the disconnection message */ if (outgoing) { @@ -2193,7 +2324,7 @@ int cfil_dispatch_closed_event(struct socket *so, int kcunit) { struct cfil_entry *entry; - struct cfil_msg_hdr msg_closed; + struct cfil_msg_sock_closed msg_closed; errno_t error = 0; struct content_filter *cfc; @@ -2222,23 +2353,42 @@ cfil_dispatch_closed_event(struct socket *so, int kcunit) if ((entry->cfe_flags & CFEF_SENT_SOCK_ATTACHED) == 0) goto done; - bzero(&msg_closed, sizeof(struct cfil_msg_hdr)); - msg_closed.cfm_len = sizeof(struct cfil_msg_hdr); - msg_closed.cfm_version = CFM_VERSION_CURRENT; - msg_closed.cfm_type = CFM_TYPE_EVENT; - msg_closed.cfm_op = CFM_OP_SOCKET_CLOSED; - msg_closed.cfm_sock_id = entry->cfe_cfil_info->cfi_sock_id; + microuptime(&entry->cfe_last_event); + CFI_ADD_TIME_LOG(so->so_cfil, &entry->cfe_last_event, &so->so_cfil->cfi_first_event, CFM_OP_SOCKET_CLOSED); + + bzero(&msg_closed, sizeof(struct cfil_msg_sock_closed)); + msg_closed.cfc_msghdr.cfm_len = sizeof(struct cfil_msg_sock_closed); + msg_closed.cfc_msghdr.cfm_version = CFM_VERSION_CURRENT; + msg_closed.cfc_msghdr.cfm_type = CFM_TYPE_EVENT; + msg_closed.cfc_msghdr.cfm_op = CFM_OP_SOCKET_CLOSED; + msg_closed.cfc_msghdr.cfm_sock_id = entry->cfe_cfil_info->cfi_sock_id; + msg_closed.cfc_first_event.tv_sec = so->so_cfil->cfi_first_event.tv_sec; + msg_closed.cfc_first_event.tv_usec = so->so_cfil->cfi_first_event.tv_usec; + memcpy(msg_closed.cfc_op_time, so->so_cfil->cfi_op_time, sizeof(uint32_t)*CFI_MAX_TIME_LOG_ENTRY); + memcpy(msg_closed.cfc_op_list, so->so_cfil->cfi_op_list, sizeof(unsigned char)*CFI_MAX_TIME_LOG_ENTRY); + msg_closed.cfc_op_list_ctr = so->so_cfil->cfi_op_list_ctr; + + CFIL_LOG(LOG_INFO, "sock id %llu, op ctr %d, start time %llu.%llu", msg_closed.cfc_msghdr.cfm_sock_id, so->so_cfil->cfi_op_list_ctr, so->so_cfil->cfi_first_event.tv_sec, so->so_cfil->cfi_first_event.tv_usec); + /* for debugging + if (msg_closed.cfc_op_list_ctr > CFI_MAX_TIME_LOG_ENTRY) { + msg_closed.cfc_op_list_ctr = CFI_MAX_TIME_LOG_ENTRY; // just in case + } + for (unsigned int i = 0; i < msg_closed.cfc_op_list_ctr ; i++) { + CFIL_LOG(LOG_ERR, "MD: socket %llu event %2u, time + %u msec", msg_closed.cfc_msghdr.cfm_sock_id, (unsigned short)msg_closed.cfc_op_list[i], msg_closed.cfc_op_time[i]); + } + */ + error = ctl_enqueuedata(entry->cfe_filter->cf_kcref, entry->cfe_filter->cf_kcunit, &msg_closed, - sizeof(struct cfil_msg_hdr), + sizeof(struct cfil_msg_sock_closed), CTL_DATA_EOR); if (error != 0) { CFIL_LOG(LOG_ERR, "ctl_enqueuedata() failed: %d", error); goto done; } - microuptime(&entry->cfe_last_event); + entry->cfe_flags |= CFEF_SENT_SOCK_CLOSED; OSIncrementAtomic(&cfil_stats.cfs_closed_event_ok); done: @@ -2305,6 +2455,7 @@ cfil_dispatch_data_event(struct socket *so, uint32_t kcunit, int outgoing, struct cfil_entry *entry; struct cfe_buf *entrybuf; struct content_filter *cfc; + struct timeval tv; cfil_rw_lock_shared(&cfil_lck_rw); @@ -2396,6 +2547,9 @@ cfil_dispatch_data_event(struct socket *so, uint32_t kcunit, int outgoing, } } + microuptime(&tv); + CFI_ADD_TIME_LOG(so->so_cfil, &tv, &so->so_cfil->cfi_first_event, data_req->cfd_msghdr.cfm_op); + /* Pass the message to the content filter */ error = ctl_enqueuembuf(entry->cfe_filter->cf_kcref, entry->cfe_filter->cf_kcunit, @@ -3230,6 +3384,35 @@ done: return (error); } +int +cfil_action_bless_client(uint32_t kcunit, struct cfil_msg_hdr *msghdr) +{ + errno_t error = 0; + + cfil_rw_lock_exclusive(&cfil_lck_rw); + + bool cfil_attached = false; + struct cfil_msg_bless_client *blessmsg = (struct cfil_msg_bless_client *)msghdr; + struct socket *so = cfil_socket_from_client_uuid(blessmsg->cfb_client_uuid, &cfil_attached); + if (so == NULL) { + error = ENOENT; + } else { + // The client gets a pass automatically + socket_lock(so, 1); + if (cfil_attached) { + (void)cfil_action_data_pass(so, kcunit, 1, CFM_MAX_OFFSET, CFM_MAX_OFFSET); + (void)cfil_action_data_pass(so, kcunit, 0, CFM_MAX_OFFSET, CFM_MAX_OFFSET); + } else { + so->so_flags1 |= SOF1_CONTENT_FILTER_SKIP; + } + socket_unlock(so, 1); + } + + cfil_rw_unlock_exclusive(&cfil_lck_rw); + + return (error); +} + static int cfil_update_entry_offsets(struct socket *so, int outgoing, unsigned int datalen) { @@ -3608,10 +3791,10 @@ cfil_sock_close_wait(struct socket *so) CFIL_LOG(LOG_INFO, "so %llx", (uint64_t)VM_KERNEL_ADDRPERM(so)); if (so->so_proto->pr_getlock != NULL) - mutex_held = (*so->so_proto->pr_getlock)(so, 0); + mutex_held = (*so->so_proto->pr_getlock)(so, PR_F_WILLUNLOCK); else mutex_held = so->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); while (cfil_filters_attached(so)) { /* diff --git a/bsd/net/content_filter.h b/bsd/net/content_filter.h index 7291b2fb4..e4d1ce5d4 100644 --- a/bsd/net/content_filter.h +++ b/bsd/net/content_filter.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2014 Apple Inc. All rights reserved. + * Copyright (c) 2013-2017 Apple Inc. All rights reserved. * * @APPLE_LICENSE_HEADER_START@ * @@ -49,6 +49,14 @@ __BEGIN_DECLS */ #define CONTENT_FILTER_CONTROL_NAME "com.apple.content-filter" +/* + * Opaque socket identifier + */ +typedef uint64_t cfil_sock_id_t; + +#define CFIL_SOCK_ID_NONE UINT64_MAX + + /* * CFIL_OPT_NECP_CONTROL_UNIT * To set or get the NECP filter control unit for the kernel control socket @@ -56,6 +64,30 @@ __BEGIN_DECLS */ #define CFIL_OPT_NECP_CONTROL_UNIT 1 /* uint32_t */ +/* + * CFIL_OPT_GET_SOCKET_INFO + * To get information about a given socket that is being filtered. + */ +#define CFIL_OPT_GET_SOCKET_INFO 2 /* uint32_t */ + +/* + * struct cfil_opt_sock_info + * + * Contains information about a socket that is being filtered. + */ +struct cfil_opt_sock_info { + cfil_sock_id_t cfs_sock_id; + int cfs_sock_family; /* e.g. PF_INET */ + int cfs_sock_type; /* e.g. SOCK_STREAM */ + int cfs_sock_protocol; /* e.g. IPPROTO_TCP */ + union sockaddr_in_4_6 cfs_local; + union sockaddr_in_4_6 cfs_remote; + pid_t cfs_pid; + pid_t cfs_e_pid; + uuid_t cfs_uuid; + uuid_t cfs_e_uuid; +}; + /* * How many filter may be active simultaneously */ @@ -87,13 +119,7 @@ __BEGIN_DECLS */ #define CFM_OP_DATA_UPDATE 16 /* update pass or peek offsets */ #define CFM_OP_DROP 17 /* shutdown socket, no more data */ - -/* - * Opaque socket identifier - */ -typedef uint64_t cfil_sock_id_t; - -#define CFIL_SOCK_ID_NONE UINT64_MAX +#define CFM_OP_BLESS_CLIENT 18 /* mark a client flow as already filtered, passes a uuid */ /* * struct cfil_msg_hdr @@ -158,6 +184,27 @@ struct cfil_msg_data_event { /* Actual content data immediatly follows */ }; +#define CFI_MAX_TIME_LOG_ENTRY 6 +/* + * struct cfil_msg_sock_closed + * + * Information about a socket being closed to the content filter + * + * Action: No reply is expected as this does not block the closing of the + * TCP/IP. + * + * Valid Types: CFM_TYPE_EVENT + * + * Valid Op: CFM_OP_SOCKET_CLOSED + */ +struct cfil_msg_sock_closed { + struct cfil_msg_hdr cfc_msghdr; + struct timeval64 cfc_first_event; + uint32_t cfc_op_list_ctr; + uint32_t cfc_op_time[CFI_MAX_TIME_LOG_ENTRY]; /* time interval in microseconds since first event */ + unsigned char cfc_op_list[CFI_MAX_TIME_LOG_ENTRY]; +} __attribute__((aligned(8))); + /* * struct cfil_msg_action * @@ -183,6 +230,20 @@ struct cfil_msg_action { uint64_t cfa_out_peek_offset; }; +/* + * struct cfil_msg_bless_client + * + * Marks a client UUID as already filtered at a higher level. + * + * Valid Type: CFM_TYPE_ACTION + * + * Valid Ops: CFM_OP_BLESS_CLIENT + */ +struct cfil_msg_bless_client { + struct cfil_msg_hdr cfb_msghdr; + uuid_t cfb_client_uuid; +}; + #define CFM_MAX_OFFSET UINT64_MAX /* @@ -361,8 +422,8 @@ extern void cfil_sock_buf_update(struct sockbuf *sb); extern cfil_sock_id_t cfil_sock_id_from_socket(struct socket *so); -__END_DECLS - #endif /* BSD_KERNEL_PRIVATE */ +__END_DECLS + #endif /* __CONTENT_FILTER_H__ */ diff --git a/bsd/net/dlil.c b/bsd/net/dlil.c index b38d4e597..50e85b274 100644 --- a/bsd/net/dlil.c +++ b/bsd/net/dlil.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999-2016 Apple Inc. All rights reserved. + * Copyright (c) 1999-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -73,6 +73,8 @@ #include <net/classq/classq_sfb.h> #include <net/flowhash.h> #include <net/ntstat.h> +#include <net/if_llatbl.h> +#include <net/net_api_stats.h> #if INET #include <netinet/in_var.h> @@ -113,15 +115,13 @@ #if PF #include <net/pfvar.h> #endif /* PF */ -#if PF_ALTQ -#include <net/altq/altq.h> -#endif /* PF_ALTQ */ #include <net/pktsched/pktsched.h> #if NECP #include <net/necp.h> #endif /* NECP */ + #define DBG_LAYER_BEG DLILDBG_CODE(DBG_DLIL_STATIC, 0) #define DBG_LAYER_END DLILDBG_CODE(DBG_DLIL_STATIC, 2) #define DBG_FNC_DLIL_INPUT DLILDBG_CODE(DBG_DLIL_STATIC, (1 << 8)) @@ -186,7 +186,6 @@ struct if_proto { SLIST_HEAD(proto_hash_entry, if_proto); -#define DLIL_SDLMAXLEN 64 #define DLIL_SDLDATALEN \ (DLIL_SDLMAXLEN - offsetof(struct sockaddr_dl, sdl_data[0])) @@ -353,8 +352,6 @@ static errno_t ifproto_media_send_arp(struct ifnet *, u_short, const struct sockaddr_dl *, const struct sockaddr *, const struct sockaddr_dl *, const struct sockaddr *); -static errno_t ifp_if_output(struct ifnet *, struct mbuf *); -static void ifp_if_start(struct ifnet *); static errno_t ifp_if_input(struct ifnet *ifp, struct mbuf *m_head, struct mbuf *m_tail, const struct ifnet_stat_increment_param *s, boolean_t poll, struct thread *tp); @@ -367,8 +364,14 @@ static errno_t ifp_if_add_proto(struct ifnet *, protocol_family_t, const struct ifnet_demux_desc *, u_int32_t); static errno_t ifp_if_del_proto(struct ifnet *, protocol_family_t); static errno_t ifp_if_check_multi(struct ifnet *, const struct sockaddr *); +#if CONFIG_EMBEDDED +static errno_t ifp_if_framer(struct ifnet *, struct mbuf **, + const struct sockaddr *, const char *, const char *, + u_int32_t *, u_int32_t *); +#else static errno_t ifp_if_framer(struct ifnet *, struct mbuf **, const struct sockaddr *, const char *, const char *); +#endif /* CONFIG_EMBEDDED */ static errno_t ifp_if_framer_extended(struct ifnet *, struct mbuf **, const struct sockaddr *, const char *, const char *, u_int32_t *, u_int32_t *); @@ -391,9 +394,9 @@ static void dlil_input_packet_list_common(struct ifnet *, struct mbuf *, static errno_t ifnet_input_common(struct ifnet *, struct mbuf *, struct mbuf *, const struct ifnet_stat_increment_param *, boolean_t, boolean_t); -#if DEBUG +#if DEBUG || DEVELOPMENT static void dlil_verify_sum16(void); -#endif /* DEBUG */ +#endif /* DEBUG || DEVELOPMENT */ static void dlil_output_cksum_dbg(struct ifnet *, struct mbuf *, uint32_t, protocol_family_t); static void dlil_input_cksum_dbg(struct ifnet *, struct mbuf *, char *, @@ -408,6 +411,8 @@ static struct ifnet *ifnet_detaching_dequeue(void); static void ifnet_start_thread_fn(void *, wait_result_t); static void ifnet_poll_thread_fn(void *, wait_result_t); static void ifnet_poll(struct ifnet *); +static errno_t ifnet_enqueue_common(struct ifnet *, void *, + classq_pkt_type_t, boolean_t, boolean_t *); static void ifp_src_route_copyout(struct ifnet *, struct route *); static void ifp_src_route_copyin(struct ifnet *, struct route *); @@ -432,6 +437,10 @@ static int sysctl_get_ports_used SYSCTL_HANDLER_ARGS; struct chain_len_stats tx_chain_len_stats; static int sysctl_tx_chain_len_stats SYSCTL_HANDLER_ARGS; +#if TEST_INPUT_THREAD_TERMINATION +static int sysctl_input_thread_termination_spin SYSCTL_HANDLER_ARGS; +#endif /* TEST_INPUT_THREAD_TERMINATION */ + /* The following are protected by dlil_ifnet_lock */ static TAILQ_HEAD(, ifnet) ifnet_detaching_head; static u_int32_t ifnet_detaching_cnt; @@ -489,8 +498,12 @@ extern uint32_t tcp_count_opportunistic(unsigned int ifindex, __private_extern__ void link_rtrequest(int, struct rtentry *, struct sockaddr *); #if CONFIG_MACF +#ifdef CONFIG_EMBEDDED +int dlil_lladdr_ckreq = 1; +#else int dlil_lladdr_ckreq = 0; #endif +#endif #if DEBUG int dlil_verbose = 1; @@ -588,13 +601,14 @@ SYSCTL_PROC(_net_link_generic_system, OID_AUTO, rxpoll, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &if_rxpoll, 0, sysctl_rxpoll, "I", "enable opportunistic input polling"); -u_int32_t if_bw_smoothing_val = 3; -SYSCTL_UINT(_net_link_generic_system, OID_AUTO, if_bw_smoothing_val, - CTLFLAG_RW | CTLFLAG_LOCKED, &if_bw_smoothing_val, 0, ""); - -u_int32_t if_bw_measure_size = 10; -SYSCTL_INT(_net_link_generic_system, OID_AUTO, if_bw_measure_size, - CTLFLAG_RW | CTLFLAG_LOCKED, &if_bw_measure_size, 0, ""); +#if TEST_INPUT_THREAD_TERMINATION +static u_int32_t if_input_thread_termination_spin = 0; +SYSCTL_PROC(_net_link_generic_system, OID_AUTO, input_thread_termination_spin, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, + &if_input_thread_termination_spin, 0, + sysctl_input_thread_termination_spin, + "I", "input thread termination spin limit"); +#endif /* TEST_INPUT_THREAD_TERMINATION */ static u_int32_t cur_dlil_input_threads = 0; SYSCTL_UINT(_net_link_generic_system, OID_AUTO, dlil_input_threads, @@ -723,12 +737,25 @@ SYSCTL_UINT(_net_link_generic_system, OID_AUTO, tx_chain_len_count, SYSCTL_NODE(_net_link_generic_system, OID_AUTO, get_ports_used, CTLFLAG_RD | CTLFLAG_LOCKED, sysctl_get_ports_used, ""); +static uint32_t threshold_notify = 1; /* enable/disable */ +SYSCTL_UINT(_net_link_generic_system, OID_AUTO, threshold_notify, + CTLFLAG_RW | CTLFLAG_LOCKED, &threshold_notify, 0, ""); + +static uint32_t threshold_interval = 2; /* in seconds */ +SYSCTL_UINT(_net_link_generic_system, OID_AUTO, threshold_interval, + CTLFLAG_RW | CTLFLAG_LOCKED, &threshold_interval, 0, ""); + #if (DEVELOPMENT || DEBUG) static int sysctl_get_kao_frames SYSCTL_HANDLER_ARGS; SYSCTL_NODE(_net_link_generic_system, OID_AUTO, get_kao_frames, CTLFLAG_RD | CTLFLAG_LOCKED, sysctl_get_kao_frames, ""); #endif /* DEVELOPMENT || DEBUG */ +struct net_api_stats net_api_stats; +SYSCTL_STRUCT(_net, OID_AUTO, api_stats, CTLFLAG_RD|CTLFLAG_LOCKED, + &net_api_stats, net_api_stats, ""); + + unsigned int net_rxpoll = 1; unsigned int net_affinity = 1; static kern_return_t dlil_affinity_set(struct thread *, u_int32_t); @@ -738,6 +765,24 @@ extern u_int32_t inject_buckets; static lck_grp_attr_t *dlil_grp_attributes = NULL; static lck_attr_t *dlil_lck_attributes = NULL; +/* DLIL data threshold thread call */ +static void dlil_dt_tcall_fn(thread_call_param_t, thread_call_param_t); + +static void dlil_mit_tcall_fn(thread_call_param_t, thread_call_param_t); + +uint32_t dlil_rcv_mit_pkts_min = 5; +uint32_t dlil_rcv_mit_pkts_max = 64; +uint32_t dlil_rcv_mit_interval = (500 * 1000); + +#if (DEVELOPMENT || DEBUG) +SYSCTL_UINT(_net_link_generic_system, OID_AUTO, rcv_mit_pkts_min, + CTLFLAG_RW | CTLFLAG_LOCKED, &dlil_rcv_mit_pkts_min, 0, ""); +SYSCTL_UINT(_net_link_generic_system, OID_AUTO, rcv_mit_pkts_max, + CTLFLAG_RW | CTLFLAG_LOCKED, &dlil_rcv_mit_pkts_max, 0, ""); +SYSCTL_UINT(_net_link_generic_system, OID_AUTO, rcv_mit_interval, + CTLFLAG_RW | CTLFLAG_LOCKED, &dlil_rcv_mit_interval, 0, ""); +#endif /* DEVELOPMENT || DEBUG */ + #define DLIL_INPUT_CHECK(m, ifp) { \ struct ifnet *_rcvif = mbuf_pkthdr_rcvif(m); \ @@ -878,6 +923,9 @@ if_proto_free(struct if_proto *proto) __private_extern__ void ifnet_lock_assert(struct ifnet *ifp, ifnet_lock_assert_t what) { +#if !MACH_ASSERT +#pragma unused(ifp) +#endif unsigned int type = 0; int ass = 1; @@ -904,7 +952,7 @@ ifnet_lock_assert(struct ifnet *ifp, ifnet_lock_assert_t what) /* NOTREACHED */ } if (ass) - lck_rw_assert(&ifp->if_lock, type); + LCK_RW_ASSERT(&ifp->if_lock, type); } __private_extern__ void @@ -986,7 +1034,7 @@ ifnet_head_done(void) __private_extern__ void ifnet_head_assert_exclusive(void) { - lck_rw_assert(&ifnet_head_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&ifnet_head_lock, LCK_RW_ASSERT_EXCLUSIVE); } /* @@ -1208,11 +1256,11 @@ dlil_create_input_thread(ifnet_t ifp, struct dlil_threading_info *inp) limit = (u_int32_t)-1; } - _qinit(&inp->rcvq_pkts, Q_DROPTAIL, limit); + _qinit(&inp->rcvq_pkts, Q_DROPTAIL, limit, QP_MBUF); if (inp == dlil_main_input_thread) { struct dlil_main_threading_info *inpm = (struct dlil_main_threading_info *)inp; - _qinit(&inpm->lo_rcvq_pkts, Q_DROPTAIL, limit); + _qinit(&inpm->lo_rcvq_pkts, Q_DROPTAIL, limit, QP_MBUF); } error = kernel_thread_start(func, inp, &inp->input_thr); @@ -1231,7 +1279,7 @@ dlil_create_input_thread(ifnet_t ifp, struct dlil_threading_info *inp) * Randomize to reduce the probability * of affinity tag namespace collision. */ - read_random(&tag, sizeof (tag)); + read_frandom(&tag, sizeof (tag)); if (dlil_affinity_set(tp, tag) == KERN_SUCCESS) { thread_reference(tp); inp->tag = tag; @@ -1251,23 +1299,37 @@ dlil_create_input_thread(ifnet_t ifp, struct dlil_threading_info *inp) return (error); } -static void -dlil_terminate_input_thread(struct dlil_threading_info *inp) +#if TEST_INPUT_THREAD_TERMINATION +static int +sysctl_input_thread_termination_spin SYSCTL_HANDLER_ARGS { - struct ifnet *ifp; +#pragma unused(arg1, arg2) + uint32_t i; + int err; - VERIFY(current_thread() == inp->input_thr); - VERIFY(inp != dlil_main_input_thread); + i = if_input_thread_termination_spin; - OSAddAtomic(-1, &cur_dlil_input_threads); + err = sysctl_handle_int(oidp, &i, 0, req); + if (err != 0 || req->newptr == USER_ADDR_NULL) + return (err); + + if (net_rxpoll == 0) + return (ENXIO); + if_input_thread_termination_spin = i; + return (err); +} +#endif /* TEST_INPUT_THREAD_TERMINATION */ + +static void +dlil_clean_threading_info(struct dlil_threading_info *inp) +{ lck_mtx_destroy(&inp->input_lck, inp->lck_grp); lck_grp_free(inp->lck_grp); inp->input_waiting = 0; inp->wtot = 0; bzero(inp->input_name, sizeof (inp->input_name)); - ifp = inp->ifp; inp->ifp = NULL; VERIFY(qhead(&inp->rcvq_pkts) == NULL && qempty(&inp->rcvq_pkts)); qlimit(&inp->rcvq_pkts) = 0; @@ -1293,15 +1355,44 @@ dlil_terminate_input_thread(struct dlil_threading_info *inp) #if IFNET_INPUT_SANITY_CHK inp->input_mbuf_cnt = 0; #endif /* IFNET_INPUT_SANITY_CHK */ +} - if (dlil_verbose) { - printf("%s: input thread terminated\n", - if_name(ifp)); +static void +dlil_terminate_input_thread(struct dlil_threading_info *inp) +{ + struct ifnet *ifp = inp->ifp; + + VERIFY(current_thread() == inp->input_thr); + VERIFY(inp != dlil_main_input_thread); + + OSAddAtomic(-1, &cur_dlil_input_threads); + +#if TEST_INPUT_THREAD_TERMINATION + { /* do something useless that won't get optimized away */ + uint32_t v = 1; + for (uint32_t i = 0; + i < if_input_thread_termination_spin; + i++) { + v = (i + 1) * v; + } + printf("the value is %d\n", v); } +#endif /* TEST_INPUT_THREAD_TERMINATION */ + + lck_mtx_lock_spin(&inp->input_lck); + VERIFY((inp->input_waiting & DLIL_INPUT_TERMINATE) != 0); + inp->input_waiting |= DLIL_INPUT_TERMINATE_COMPLETE; + wakeup_one((caddr_t)&inp->input_waiting); + lck_mtx_unlock(&inp->input_lck); /* for the extra refcnt from kernel_thread_start() */ thread_deallocate(current_thread()); + if (dlil_verbose) { + printf("%s: input thread terminated\n", + if_name(ifp)); + } + /* this is the end */ thread_terminate(current_thread()); /* NOTREACHED */ @@ -1370,6 +1461,7 @@ dlil_init(void) _CASSERT(IF_HWASSIST_CSUM_UDPIPV6 == IFNET_CSUM_UDPIPV6); _CASSERT(IF_HWASSIST_CSUM_FRAGMENT_IPV6 == IFNET_IPV6_FRAGMENT); _CASSERT(IF_HWASSIST_CSUM_PARTIAL == IFNET_CSUM_PARTIAL); + _CASSERT(IF_HWASSIST_CSUM_ZERO_INVERT == IFNET_CSUM_ZERO_INVERT); _CASSERT(IF_HWASSIST_VLAN_TAGGING == IFNET_VLAN_TAGGING); _CASSERT(IF_HWASSIST_VLAN_MTU == IFNET_VLAN_MTU); _CASSERT(IF_HWASSIST_TSO_V4 == IFNET_TSO_IPV4); @@ -1387,6 +1479,7 @@ dlil_init(void) _CASSERT(CSUM_UDPIPV6 == IF_HWASSIST_CSUM_UDPIPV6); _CASSERT(CSUM_FRAGMENT_IPV6 == IF_HWASSIST_CSUM_FRAGMENT_IPV6); _CASSERT(CSUM_PARTIAL == IF_HWASSIST_CSUM_PARTIAL); + _CASSERT(CSUM_ZERO_INVERT == IF_HWASSIST_CSUM_ZERO_INVERT); _CASSERT(CSUM_VLAN_TAG_VALID == IF_HWASSIST_VLAN_TAGGING); /* @@ -1524,6 +1617,7 @@ dlil_init(void) zone_change(dlif_udpstat_zone, Z_CALLERACCT, FALSE); ifnet_llreach_init(); + eventhandler_lists_ctxt_init(&ifnet_evhdlr_ctxt); TAILQ_INIT(&dlil_ifnet_head); TAILQ_INIT(&ifnet_head); @@ -1590,10 +1684,13 @@ dlil_init(void) /* Initialize the service class to dscp map */ net_qos_map_init(); -#if DEBUG +#if DEBUG || DEVELOPMENT /* Run self-tests */ dlil_verify_sum16(); -#endif /* DEBUG */ +#endif /* DEBUG || DEVELOPMENT */ + + /* Initialize link layer table */ + lltable_glbl_init(); /* * Create and start up the main DLIL input thread and the interface @@ -1607,12 +1704,13 @@ dlil_init(void) /* NOTREACHED */ } thread_deallocate(thread); + } static void if_flt_monitor_busy(struct ifnet *ifp) { - lck_mtx_assert(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED); ++ifp->if_flt_busy; VERIFY(ifp->if_flt_busy != 0); @@ -1627,7 +1725,7 @@ if_flt_monitor_unbusy(struct ifnet *ifp) static void if_flt_monitor_enter(struct ifnet *ifp) { - lck_mtx_assert(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED); while (ifp->if_flt_busy) { ++ifp->if_flt_waiters; @@ -1640,7 +1738,7 @@ if_flt_monitor_enter(struct ifnet *ifp) static void if_flt_monitor_leave(struct ifnet *ifp) { - lck_mtx_assert(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED); VERIFY(ifp->if_flt_busy != 0); --ifp->if_flt_busy; @@ -1692,7 +1790,7 @@ dlil_attach_filter(struct ifnet *ifp, const struct iff_filter *if_filter, lck_mtx_lock(&ifp->if_flt_lock); if_flt_monitor_enter(ifp); - lck_mtx_assert(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED); TAILQ_INSERT_TAIL(&ifp->if_flt_head, filter, filt_next); if_flt_monitor_leave(ifp); @@ -1708,6 +1806,11 @@ dlil_attach_filter(struct ifnet *ifp, const struct iff_filter *if_filter, OSAddAtomic(1, &dlil_filter_disable_tso_count); routegenid_update(); } + OSIncrementAtomic64(&net_api_stats.nas_iflt_attach_count); + INC_ATOMIC_INT64_LIM(net_api_stats.nas_iflt_attach_total); + if ((filter->filt_flags & DLIL_IFF_INTERNAL)) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_iflt_attach_os_total); + } if (dlil_verbose) { printf("%s: %s filter attached\n", if_name(ifp), if_filter->iff_name); @@ -1753,7 +1856,7 @@ dlil_detach_filter_internal(interface_filter_t filter, int detached) lck_mtx_lock(&ifp->if_flt_lock); if_flt_monitor_enter(ifp); - lck_mtx_assert(&ifp->if_flt_lock, + LCK_MTX_ASSERT(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED); /* Remove the filter from the list */ @@ -1795,6 +1898,8 @@ destroy: routegenid_update(); } + VERIFY(OSDecrementAtomic64(&net_api_stats.nas_iflt_attach_count) > 0); + /* Free the filter */ zfree(dlif_filt_zone, filter); filter = NULL; @@ -1958,7 +2063,7 @@ dlil_input_thread_func(void *v, wait_result_t w) inp->wtot = 0; - dlil_input_stats_sync(ifp, inp); + dlil_input_stats_sync(ifp, inp); lck_mtx_unlock(&inp->input_lck); @@ -2033,7 +2138,12 @@ dlil_rxpoll_input_thread_func(void *v, wait_result_t w) if (inp->input_waiting & DLIL_INPUT_TERMINATE) { /* Free up pending packets */ + lck_mtx_convert_spin(&inp->input_lck); _flushq(&inp->rcvq_pkts); + if (inp->input_mit_tcall != NULL) { + if (thread_call_isactive(inp->input_mit_tcall)) + thread_call_cancel(inp->input_mit_tcall); + } lck_mtx_unlock(&inp->input_lck); dlil_terminate_input_thread(inp); @@ -2257,7 +2367,7 @@ dlil_rxpoll_set_params(struct ifnet *ifp, struct ifnet_poll_params *p, if (!locked) lck_mtx_lock(&inp->input_lck); - lck_mtx_assert(&inp->input_lck, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&inp->input_lck, LCK_MTX_ASSERT_OWNED); /* * Normally, we'd reset the parameters to the auto-tuned values @@ -2380,7 +2490,7 @@ static errno_t ifnet_input_common(struct ifnet *ifp, struct mbuf *m_head, struct mbuf *m_tail, const struct ifnet_stat_increment_param *s, boolean_t ext, boolean_t poll) { - ifnet_input_handler_func handler_func; + dlil_input_func input_func; struct ifnet_stat_increment_param _s; u_int32_t m_cnt = 0, m_size = 0; struct mbuf *last; @@ -2407,8 +2517,8 @@ ifnet_input_common(struct ifnet *ifp, struct mbuf *m_head, struct mbuf *m_tail, return (EINVAL); } - handler_func = ifp->if_input_handler; - VERIFY(handler_func != NULL); + input_func = ifp->if_input_dlil; + VERIFY(input_func != NULL); if (m_tail == NULL) { last = m_head; @@ -2476,7 +2586,7 @@ ifnet_input_common(struct ifnet *ifp, struct mbuf *m_head, struct mbuf *m_tail, _s.packets_in = m_cnt; _s.bytes_in = m_size; - err = (*handler_func)(ifp, m_head, m_tail, s, poll, current_thread()); + err = (*input_func)(ifp, m_head, m_tail, s, poll, current_thread()); if (ifp != lo_ifp) { /* Release the IO refcnt */ @@ -2486,31 +2596,6 @@ ifnet_input_common(struct ifnet *ifp, struct mbuf *m_head, struct mbuf *m_tail, return (err); } -errno_t -ifnet_set_input_handler(struct ifnet *ifp, ifnet_input_handler_func fn) -{ - return (atomic_test_set_ptr(&ifp->if_input_handler, - dlil_input_handler, fn) ? 0 : EBUSY); -} - -void -ifnet_reset_input_handler(struct ifnet *ifp) -{ - atomic_set_ptr(&ifp->if_input_handler, dlil_input_handler); -} - -errno_t -ifnet_set_output_handler(struct ifnet *ifp, ifnet_output_handler_func fn) -{ - return (atomic_test_set_ptr(&ifp->if_output_handler, - dlil_output_handler, fn) ? 0 : EBUSY); -} - -void -ifnet_reset_output_handler(struct ifnet *ifp) -{ - atomic_set_ptr(&ifp->if_output_handler, dlil_output_handler); -} errno_t dlil_output_handler(struct ifnet *ifp, struct mbuf *m) @@ -2555,7 +2640,7 @@ dlil_input_handler(struct ifnet *ifp, struct mbuf *m_head, /* * Take a reference on the current thread; during detach, - * we will need to refer to it in order ot tear down its + * we will need to refer to it in order to tear down its * affinity. */ thread_reference(tp); @@ -2612,16 +2697,31 @@ dlil_input_handler(struct ifnet *ifp, struct mbuf *m_head, if (inp == dlil_main_input_thread) dlil_input_stats_sync(ifp, inp); - inp->input_waiting |= DLIL_INPUT_WAITING; - if (!(inp->input_waiting & DLIL_INPUT_RUNNING)) { - inp->wtot++; - wakeup_one((caddr_t)&inp->input_waiting); + if (qlen(&inp->rcvq_pkts) >= dlil_rcv_mit_pkts_min && + qlen(&inp->rcvq_pkts) < dlil_rcv_mit_pkts_max && + (ifp->if_family == IFNET_FAMILY_ETHERNET || + ifp->if_type == IFT_CELLULAR) + ) { + if (!thread_call_isactive(inp->input_mit_tcall)) { + uint64_t deadline; + clock_interval_to_deadline(dlil_rcv_mit_interval, + 1, &deadline); + (void) thread_call_enter_delayed( + inp->input_mit_tcall, deadline); + } + } else { + inp->input_waiting |= DLIL_INPUT_WAITING; + if (!(inp->input_waiting & DLIL_INPUT_RUNNING)) { + inp->wtot++; + wakeup_one((caddr_t)&inp->input_waiting); + } } lck_mtx_unlock(&inp->input_lck); return (0); } + static void ifnet_start_common(struct ifnet *ifp, int resetfc) { @@ -2645,7 +2745,8 @@ ifnet_start_common(struct ifnet *ifp, int resetfc) (resetfc || !(ifp->if_eflags & IFEF_ENQUEUE_MULTI) || IFCQ_LEN(&ifp->if_snd) >= ifp->if_start_delay_qlen || ifp->if_start_delayed == 0)) { - wakeup_one((caddr_t)&ifp->if_start_thread); + (void) thread_wakeup_thread((caddr_t)&ifp->if_start_thread, + ifp->if_start_thread); } lck_mtx_unlock(&ifp->if_start_lock); } @@ -2669,7 +2770,8 @@ ifnet_start_thread_fn(void *v, wait_result_t w) /* Construct the name for this thread, and then apply it. */ bzero(thread_name, sizeof(thread_name)); - snprintf(thread_name, sizeof(thread_name), "ifnet_start_%s", ifp->if_xname); + (void) snprintf(thread_name, sizeof (thread_name), + "ifnet_start_%s", ifp->if_xname); thread_set_thread_name(ifp->if_start_thread, thread_name); /* @@ -2699,17 +2801,16 @@ ifnet_start_thread_fn(void *v, wait_result_t w) } } - snprintf(ifname, sizeof (ifname), "%s_starter", - if_name(ifp)); + (void) snprintf(ifname, sizeof (ifname), "%s_starter", if_name(ifp)); lck_mtx_lock_spin(&ifp->if_start_lock); for (;;) { - if (ifp->if_start_thread != NULL) + if (ifp->if_start_thread != NULL) { (void) msleep(&ifp->if_start_thread, &ifp->if_start_lock, (PZERO - 1) | PSPIN, ifname, ts); - + } /* interface is detached? */ if (ifp->if_start_thread == THREAD_NULL) { ifnet_set_start_cycle(ifp, NULL); @@ -3131,34 +3232,86 @@ ifnet_get_rcvq_maxlen(struct ifnet *ifp, u_int32_t *maxqlen) return (0); } -errno_t -ifnet_enqueue(struct ifnet *ifp, struct mbuf *m) +void +ifnet_enqueue_multi_setup(struct ifnet *ifp, uint16_t delay_qlen, + uint16_t delay_timeout) +{ + if (delay_qlen > 0 && delay_timeout > 0) { + ifp->if_eflags |= IFEF_ENQUEUE_MULTI; + ifp->if_start_delay_qlen = min(100, delay_qlen); + ifp->if_start_delay_timeout = min(20000, delay_timeout); + /* convert timeout to nanoseconds */ + ifp->if_start_delay_timeout *= 1000; + kprintf("%s: forced IFEF_ENQUEUE_MULTI qlen %u timeout %u\n", + ifp->if_xname, (uint32_t)delay_qlen, + (uint32_t)delay_timeout); + } else { + ifp->if_eflags &= ~IFEF_ENQUEUE_MULTI; + } +} + +static inline errno_t +ifnet_enqueue_common(struct ifnet *ifp, void *p, classq_pkt_type_t ptype, + boolean_t flush, boolean_t *pdrop) { - int error; + volatile uint64_t *fg_ts = NULL; + volatile uint64_t *rt_ts = NULL; + struct mbuf *m = p; struct timespec now; - u_int64_t now_nsec; + u_int64_t now_nsec = 0; + int error = 0; - if (ifp == NULL || m == NULL || !(m->m_flags & M_PKTHDR) || - m->m_nextpkt != NULL) { - if (m != NULL) - m_freem_list(m); - return (EINVAL); - } else if (!(ifp->if_eflags & IFEF_TXSTART) || - !(ifp->if_refflags & IFRF_ATTACHED)) { - /* flag tested without lock for performance */ - m_freem(m); - return (ENXIO); - } else if (!(ifp->if_flags & IFF_UP)) { - m_freem(m); - return (ENETDOWN); - } + ASSERT(ifp->if_eflags & IFEF_TXSTART); + + /* + * If packet already carries a timestamp, either from dlil_output() + * or from flowswitch, use it here. Otherwise, record timestamp. + * PKTF_TS_VALID is always cleared prior to entering classq, i.e. + * the timestamp value is used internally there. + */ + switch (ptype) { + case QP_MBUF: + ASSERT(m->m_flags & M_PKTHDR); + ASSERT(m->m_nextpkt == NULL); + + if (!(m->m_pkthdr.pkt_flags & PKTF_TS_VALID) || + m->m_pkthdr.pkt_timestamp == 0) { + nanouptime(&now); + net_timernsec(&now, &now_nsec); + m->m_pkthdr.pkt_timestamp = now_nsec; + } + m->m_pkthdr.pkt_flags &= ~PKTF_TS_VALID; + /* + * If the packet service class is not background, + * update the timestamp to indicate recent activity + * on a foreground socket. + */ + if ((m->m_pkthdr.pkt_flags & PKTF_FLOW_ID) && + m->m_pkthdr.pkt_flowsrc == FLOWSRC_INPCB) { + if (!(m->m_pkthdr.pkt_flags & PKTF_SO_BACKGROUND)) { + ifp->if_fg_sendts = _net_uptime; + if (fg_ts != NULL) + *fg_ts = _net_uptime; + } + if (m->m_pkthdr.pkt_flags & PKTF_SO_REALTIME) { + ifp->if_rt_sendts = _net_uptime; + if (rt_ts != NULL) + *rt_ts = _net_uptime; + } + } + break; - nanouptime(&now); - net_timernsec(&now, &now_nsec); - m->m_pkthdr.pkt_timestamp = now_nsec; - m->m_pkthdr.pkt_flags &= ~PKTF_DRV_TS_VALID; + + default: + VERIFY(0); + /* NOTREACHED */ + } if (ifp->if_eflags & IFEF_ENQUEUE_MULTI) { + if (now_nsec == 0) { + nanouptime(&now); + net_timernsec(&now, &now_nsec); + } /* * If the driver chose to delay start callback for * coalescing multiple packets, Then use the following @@ -3213,8 +3366,17 @@ ifnet_enqueue(struct ifnet *ifp, struct mbuf *m) ifp->if_eflags &= ~(IFEF_DELAY_START); } - /* enqueue the packet */ - error = ifclassq_enqueue(&ifp->if_snd, m); + switch (ptype) { + case QP_MBUF: + /* enqueue the packet (caller consumes object) */ + error = ifclassq_enqueue(&ifp->if_snd, m, QP_MBUF, pdrop); + m = NULL; + break; + + + default: + break; + } /* * Tell the driver to start dequeueing; do this even when the queue @@ -3222,16 +3384,51 @@ ifnet_enqueue(struct ifnet *ifp, struct mbuf *m) * be dequeueing from other unsuspended queues. */ if (!(ifp->if_eflags & IFEF_ENQUEUE_MULTI) && - (error == 0 || error == EQFULL || error == EQSUSPENDED)) + ((error == 0 && flush) || error == EQFULL || error == EQSUSPENDED)) ifnet_start(ifp); return (error); } +errno_t +ifnet_enqueue(struct ifnet *ifp, struct mbuf *m) +{ + boolean_t pdrop; + return (ifnet_enqueue_mbuf(ifp, m, TRUE, &pdrop)); +} + +errno_t +ifnet_enqueue_mbuf(struct ifnet *ifp, struct mbuf *m, boolean_t flush, + boolean_t *pdrop) +{ + if (ifp == NULL || m == NULL || !(m->m_flags & M_PKTHDR) || + m->m_nextpkt != NULL) { + if (m != NULL) { + m_freem_list(m); + *pdrop = TRUE; + } + return (EINVAL); + } else if (!(ifp->if_eflags & IFEF_TXSTART) || + !IF_FULLY_ATTACHED(ifp)) { + /* flag tested without lock for performance */ + m_freem(m); + *pdrop = TRUE; + return (ENXIO); + } else if (!(ifp->if_flags & IFF_UP)) { + m_freem(m); + *pdrop = TRUE; + return (ENETDOWN); + } + + return (ifnet_enqueue_common(ifp, m, QP_MBUF, flush, pdrop)); +} + + errno_t ifnet_dequeue(struct ifnet *ifp, struct mbuf **mp) { errno_t rc; + classq_pkt_type_t ptype; if (ifp == NULL || mp == NULL) return (EINVAL); else if (!(ifp->if_eflags & IFEF_TXSTART) || @@ -3239,8 +3436,10 @@ ifnet_dequeue(struct ifnet *ifp, struct mbuf **mp) return (ENXIO); if (!ifnet_is_attached(ifp, 1)) return (ENXIO); + rc = ifclassq_dequeue(&ifp->if_snd, 1, CLASSQ_DEQUEUE_MAX_BYTE_LIMIT, - mp, NULL, NULL, NULL); + (void **)mp, NULL, NULL, NULL, &ptype); + VERIFY((*mp == NULL) || (ptype == QP_MBUF)); ifnet_decr_iorefcnt(ifp); return (rc); @@ -3251,6 +3450,7 @@ ifnet_dequeue_service_class(struct ifnet *ifp, mbuf_svc_class_t sc, struct mbuf **mp) { errno_t rc; + classq_pkt_type_t ptype; if (ifp == NULL || mp == NULL || !MBUF_VALID_SC(sc)) return (EINVAL); else if (!(ifp->if_eflags & IFEF_TXSTART) || @@ -3259,7 +3459,10 @@ ifnet_dequeue_service_class(struct ifnet *ifp, mbuf_svc_class_t sc, if (!ifnet_is_attached(ifp, 1)) return (ENXIO); - rc = ifclassq_dequeue_sc(&ifp->if_snd, sc, 1, mp, NULL, NULL, NULL); + rc = ifclassq_dequeue_sc(&ifp->if_snd, sc, 1, + CLASSQ_DEQUEUE_MAX_BYTE_LIMIT, (void **)mp, NULL, NULL, + NULL, &ptype); + VERIFY((*mp == NULL) || (ptype == QP_MBUF)); ifnet_decr_iorefcnt(ifp); return (rc); } @@ -3269,6 +3472,7 @@ ifnet_dequeue_multi(struct ifnet *ifp, u_int32_t pkt_limit, struct mbuf **head, struct mbuf **tail, u_int32_t *cnt, u_int32_t *len) { errno_t rc; + classq_pkt_type_t ptype; if (ifp == NULL || head == NULL || pkt_limit < 1) return (EINVAL); else if (!(ifp->if_eflags & IFEF_TXSTART) || @@ -3278,7 +3482,9 @@ ifnet_dequeue_multi(struct ifnet *ifp, u_int32_t pkt_limit, return (ENXIO); rc = ifclassq_dequeue(&ifp->if_snd, pkt_limit, - CLASSQ_DEQUEUE_MAX_BYTE_LIMIT, head, tail, cnt, len); + CLASSQ_DEQUEUE_MAX_BYTE_LIMIT, (void **)head, (void **)tail, cnt, + len, &ptype); + VERIFY((*head == NULL) || (ptype == QP_MBUF)); ifnet_decr_iorefcnt(ifp); return (rc); } @@ -3288,6 +3494,7 @@ ifnet_dequeue_multi_bytes(struct ifnet *ifp, u_int32_t byte_limit, struct mbuf **head, struct mbuf **tail, u_int32_t *cnt, u_int32_t *len) { errno_t rc; + classq_pkt_type_t ptype; if (ifp == NULL || head == NULL || byte_limit < 1) return (EINVAL); else if (!(ifp->if_eflags & IFEF_TXSTART) || @@ -3297,7 +3504,8 @@ ifnet_dequeue_multi_bytes(struct ifnet *ifp, u_int32_t byte_limit, return (ENXIO); rc = ifclassq_dequeue(&ifp->if_snd, CLASSQ_DEQUEUE_MAX_PKT_LIMIT, - byte_limit, head, tail, cnt, len); + byte_limit, (void **)head, (void **)tail, cnt, len, &ptype); + VERIFY((*head == NULL) || (ptype == QP_MBUF)); ifnet_decr_iorefcnt(ifp); return (rc); } @@ -3308,6 +3516,7 @@ ifnet_dequeue_service_class_multi(struct ifnet *ifp, mbuf_svc_class_t sc, u_int32_t *len) { errno_t rc; + classq_pkt_type_t ptype; if (ifp == NULL || head == NULL || pkt_limit < 1 || !MBUF_VALID_SC(sc)) return (EINVAL); @@ -3316,12 +3525,16 @@ ifnet_dequeue_service_class_multi(struct ifnet *ifp, mbuf_svc_class_t sc, return (ENXIO); if (!ifnet_is_attached(ifp, 1)) return (ENXIO); - rc = ifclassq_dequeue_sc(&ifp->if_snd, sc, pkt_limit, head, - tail, cnt, len); + + rc = ifclassq_dequeue_sc(&ifp->if_snd, sc, pkt_limit, + CLASSQ_DEQUEUE_MAX_BYTE_LIMIT, (void **)head, + (void **)tail, cnt, len, &ptype); + VERIFY((*head == NULL) || (ptype == QP_MBUF)); ifnet_decr_iorefcnt(ifp); return (rc); } +#if !CONFIG_EMBEDDED errno_t ifnet_framer_stub(struct ifnet *ifp, struct mbuf **m, const struct sockaddr *dest, const char *dest_linkaddr, @@ -3334,6 +3547,7 @@ ifnet_framer_stub(struct ifnet *ifp, struct mbuf **m, return (ifp->if_framer_legacy(ifp, m, dest, dest_linkaddr, frame_type)); } +#endif /* !CONFIG_EMBEDDED */ static int dlil_interface_filters_input(struct ifnet *ifp, struct mbuf **m_p, @@ -3522,17 +3736,12 @@ dlil_input_stats_sync(struct ifnet *ifp, struct dlil_threading_info *inp) atomic_add_64(&ifp->if_data.ifi_iqdrops, s->dropped); s->dropped = 0; } - /* - * If we went over the threshold, notify NetworkStatistics. - */ - if (ifp->if_data_threshold && - (ifp->if_ibytes + ifp->if_obytes) - ifp->if_dt_bytes > - ifp->if_data_threshold) { - ifp->if_dt_bytes = ifp->if_ibytes + ifp->if_obytes; + if (ifp->if_data_threshold != 0) { lck_mtx_convert_spin(&inp->input_lck); - nstat_ifnet_threshold_reached(ifp->if_index); + ifnet_notify_data_threshold(ifp); } + /* * No need for atomic operations as they are modified here * only from within the DLIL input thread context. @@ -3612,7 +3821,10 @@ dlil_input_packet_list_common(struct ifnet *ifp_param, struct mbuf *m, goto next; } iorefcnt = 1; - pktf_mask = 0; + /* + * Preserve the time stamp if it was set. + */ + pktf_mask = PKTF_TS_VALID; } else { /* * If this arrived on lo0, preserve interface addr @@ -3954,81 +4166,6 @@ dlil_get_socket_type(struct mbuf **mp, int family, int raw) } #endif -/* - * This is mostly called from the context of the DLIL input thread; - * because of that there is no need for atomic operations. - */ -static __inline void -ifp_inc_traffic_class_in(struct ifnet *ifp, struct mbuf *m) -{ - if (!(m->m_flags & M_PKTHDR)) - return; - - switch (m_get_traffic_class(m)) { - case MBUF_TC_BE: - ifp->if_tc.ifi_ibepackets++; - ifp->if_tc.ifi_ibebytes += m->m_pkthdr.len; - break; - case MBUF_TC_BK: - ifp->if_tc.ifi_ibkpackets++; - ifp->if_tc.ifi_ibkbytes += m->m_pkthdr.len; - break; - case MBUF_TC_VI: - ifp->if_tc.ifi_ivipackets++; - ifp->if_tc.ifi_ivibytes += m->m_pkthdr.len; - break; - case MBUF_TC_VO: - ifp->if_tc.ifi_ivopackets++; - ifp->if_tc.ifi_ivobytes += m->m_pkthdr.len; - break; - default: - break; - } - - if (mbuf_is_traffic_class_privileged(m)) { - ifp->if_tc.ifi_ipvpackets++; - ifp->if_tc.ifi_ipvbytes += m->m_pkthdr.len; - } -} - -/* - * This is called from DLIL output, hence multiple threads could end - * up modifying the statistics. We trade off acccuracy for performance - * by not using atomic operations here. - */ -static __inline void -ifp_inc_traffic_class_out(struct ifnet *ifp, struct mbuf *m) -{ - if (!(m->m_flags & M_PKTHDR)) - return; - - switch (m_get_traffic_class(m)) { - case MBUF_TC_BE: - ifp->if_tc.ifi_obepackets++; - ifp->if_tc.ifi_obebytes += m->m_pkthdr.len; - break; - case MBUF_TC_BK: - ifp->if_tc.ifi_obkpackets++; - ifp->if_tc.ifi_obkbytes += m->m_pkthdr.len; - break; - case MBUF_TC_VI: - ifp->if_tc.ifi_ovipackets++; - ifp->if_tc.ifi_ovibytes += m->m_pkthdr.len; - break; - case MBUF_TC_VO: - ifp->if_tc.ifi_ovopackets++; - ifp->if_tc.ifi_ovobytes += m->m_pkthdr.len; - break; - default: - break; - } - - if (mbuf_is_traffic_class_privileged(m)) { - ifp->if_tc.ifi_opvpackets++; - ifp->if_tc.ifi_opvbytes += m->m_pkthdr.len; - } -} - static void dlil_count_chain_len(mbuf_t m, struct chain_len_stats *cls) { @@ -4082,7 +4219,6 @@ errno_t dlil_output(ifnet_t ifp, protocol_family_t proto_family, mbuf_t packetlist, void *route, const struct sockaddr *dest, int raw, struct flowadv *adv) { - ifnet_output_handler_func handler_func; char *frame_type = NULL; char *dst_linkaddr = NULL; int retval = 0; @@ -4096,6 +4232,8 @@ dlil_output(ifnet_t ifp, protocol_family_t proto_family, mbuf_t packetlist, u_int32_t pre = 0, post = 0; u_int32_t fpkts = 0, fbytes = 0; int32_t flen = 0; + struct timespec now; + u_int64_t now_nsec; KERNEL_DEBUG(DBG_FNC_DLIL_OUTPUT | DBG_FUNC_START, 0, 0, 0, 0, 0); @@ -4109,8 +4247,7 @@ dlil_output(ifnet_t ifp, protocol_family_t proto_family, mbuf_t packetlist, } iorefcnt = 1; - handler_func = ifp->if_output_handler; - VERIFY(handler_func != NULL); + VERIFY(ifp->if_output_dlil != NULL); /* update the driver's multicast filter, if needed */ if (ifp->if_updatemcasts > 0 && if_mcasts_update(ifp) == 0) @@ -4276,20 +4413,6 @@ preout_again: goto cleanup; } - /* - * If the packet service class is not background, - * update the timestamp to indicate recent activity - * on a foreground socket. - */ - if ((m->m_pkthdr.pkt_flags & PKTF_FLOW_ID) && - m->m_pkthdr.pkt_flowsrc == FLOWSRC_INPCB) { - if (!(m->m_pkthdr.pkt_flags & PKTF_SO_BACKGROUND)) - ifp->if_fg_sendts = net_uptime(); - - if (m->m_pkthdr.pkt_flags & PKTF_SO_REALTIME) - ifp->if_rt_sendts = net_uptime(); - } - ifp_inc_traffic_class_out(ifp, m); pktap_output(ifp, proto_family, m, pre, post); @@ -4300,6 +4423,29 @@ preout_again: dlil_count_chain_len(m, &tx_chain_len_stats); } + /* + * Record timestamp; ifnet_enqueue() will use this info + * rather than redoing the work. An optimization could + * involve doing this just once at the top, if there are + * no interface filters attached, but that's probably + * not a big deal. + */ + nanouptime(&now); + net_timernsec(&now, &now_nsec); + (void) mbuf_set_timestamp(m, now_nsec, TRUE); + + /* + * Discard partial sum information if this packet originated + * from another interface; the packet would already have the + * final checksum and we shouldn't recompute it. + */ + if ((m->m_pkthdr.pkt_flags & PKTF_FORWARDED) && + (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID|CSUM_PARTIAL)) == + (CSUM_DATA_VALID|CSUM_PARTIAL)) { + m->m_pkthdr.csum_flags &= ~CSUM_TX_FLAGS; + m->m_pkthdr.csum_data = 0; + } + /* * Finally, call the driver. */ @@ -4319,7 +4465,7 @@ preout_again: } KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_START, 0, 0, 0, 0, 0); - retval = (*handler_func)(ifp, m); + retval = (*ifp->if_output_dlil)(ifp, m); if (retval == EQFULL || retval == EQSUSPENDED) { if (adv != NULL && adv->code == FADV_SUCCESS) { adv->code = (retval == EQFULL ? @@ -4354,7 +4500,7 @@ next: KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_START, 0, 0, 0, 0, 0); if (ifp->if_eflags & IFEF_SENDLIST) { - retval = (*handler_func)(ifp, send_head); + retval = (*ifp->if_output_dlil)(ifp, send_head); if (retval == EQFULL || retval == EQSUSPENDED) { if (adv != NULL) { adv->code = (retval == EQFULL ? @@ -4379,7 +4525,7 @@ next: send_m = send_head; send_head = send_m->m_nextpkt; send_m->m_nextpkt = NULL; - retval = (*handler_func)(ifp, send_m); + retval = (*ifp->if_output_dlil)(ifp, send_m); if (retval == EQFULL || retval == EQSUSPENDED) { if (adv != NULL) { adv->code = (retval == EQFULL ? @@ -4853,7 +4999,7 @@ ifnet_lookup(struct ifnet *ifp) { struct ifnet *_ifp; - lck_rw_assert(&ifnet_head_lock, LCK_RW_ASSERT_HELD); + LCK_RW_ASSERT(&ifnet_head_lock, LCK_RW_ASSERT_HELD); TAILQ_FOREACH(_ifp, &ifnet_head, if_link) { if (_ifp == ifp) break; @@ -4872,8 +5018,7 @@ ifnet_is_attached(struct ifnet *ifp, int refio) int ret; lck_mtx_lock_spin(&ifp->if_ref_lock); - if ((ret = ((ifp->if_refflags & (IFRF_ATTACHED | IFRF_DETACHING)) == - IFRF_ATTACHED))) { + if ((ret = IF_FULLY_ATTACHED(ifp))) { if (refio > 0) ifp->if_refio++; } @@ -4891,8 +5036,7 @@ void ifnet_incr_iorefcnt(struct ifnet *ifp) { lck_mtx_lock_spin(&ifp->if_ref_lock); - VERIFY((ifp->if_refflags & (IFRF_ATTACHED | IFRF_DETACHING)) == - IFRF_ATTACHED); + VERIFY(IF_FULLY_ATTACHED(ifp)); VERIFY(ifp->if_refio > 0); ifp->if_refio++; lck_mtx_unlock(&ifp->if_ref_lock); @@ -4903,17 +5047,16 @@ ifnet_decr_iorefcnt(struct ifnet *ifp) { lck_mtx_lock_spin(&ifp->if_ref_lock); VERIFY(ifp->if_refio > 0); - VERIFY((ifp->if_refflags & (IFRF_ATTACHED | IFRF_DETACHING)) != 0); + VERIFY(ifp->if_refflags & (IFRF_ATTACHED | IFRF_DETACHING)); ifp->if_refio--; /* * if there are no more outstanding io references, wakeup the * ifnet_detach thread if detaching flag is set. */ - if (ifp->if_refio == 0 && - (ifp->if_refflags & IFRF_DETACHING) != 0) { + if (ifp->if_refio == 0 && (ifp->if_refflags & IFRF_DETACHING)) wakeup(&(ifp->if_refio)); - } + lck_mtx_unlock(&ifp->if_ref_lock); } @@ -4967,26 +5110,39 @@ errno_t dlil_if_free(struct ifnet *ifp) { struct dlil_ifnet *dl_if = (struct dlil_ifnet *)ifp; + bool need_release = FALSE; if (dl_if == NULL) return (EINVAL); lck_mtx_lock_spin(&dl_if->dl_if_lock); - if (dl_if->dl_if_refcnt == 0) { + switch (dl_if->dl_if_refcnt) { + case 0: panic("%s: negative refcnt for ifp=%p", __func__, ifp); /* NOTREACHED */ + break; + case 1: + if ((ifp->if_refflags & IFRF_EMBRYONIC) != 0) { + need_release = TRUE; + } + break; + default: + break; } --dl_if->dl_if_refcnt; if (dl_if->dl_if_trace != NULL) (*dl_if->dl_if_trace)(dl_if, FALSE); lck_mtx_unlock(&dl_if->dl_if_lock); - + if (need_release) { + dlil_if_release(ifp); + } return (0); } static errno_t dlil_attach_protocol_internal(struct if_proto *proto, - const struct ifnet_demux_desc *demux_list, u_int32_t demux_count) + const struct ifnet_demux_desc *demux_list, u_int32_t demux_count, + uint32_t * proto_count) { struct kev_dl_proto_data ev_pr_data; struct ifnet *ifp = proto->ifp; @@ -5041,6 +5197,9 @@ dlil_attach_protocol_internal(struct if_proto *proto, dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_PROTO_ATTACHED, (struct net_event_data *)&ev_pr_data, sizeof (struct kev_dl_proto_data)); + if (proto_count != NULL) { + *proto_count = ev_pr_data.proto_remaining_count; + } return (retval); } @@ -5050,6 +5209,7 @@ ifnet_attach_protocol(ifnet_t ifp, protocol_family_t protocol, { int retval = 0; struct if_proto *ifproto = NULL; + uint32_t proto_count = 0; ifnet_head_lock_shared(); if (ifp == NULL || protocol == 0 || proto_details == NULL) { @@ -5082,21 +5242,25 @@ ifnet_attach_protocol(ifnet_t ifp, protocol_family_t protocol, ifproto->kpi.v1.send_arp = proto_details->send_arp; retval = dlil_attach_protocol_internal(ifproto, - proto_details->demux_list, proto_details->demux_count); - - if (dlil_verbose) { - printf("%s: attached v1 protocol %d\n", if_name(ifp), - protocol); - } + proto_details->demux_list, proto_details->demux_count, + &proto_count); end: if (retval != 0 && retval != EEXIST && ifp != NULL) { DLIL_PRINTF("%s: failed to attach v1 protocol %d (err=%d)\n", if_name(ifp), protocol, retval); + } else { + if (dlil_verbose) { + printf("%s: attached v1 protocol %d (count = %d)\n", + if_name(ifp), + protocol, proto_count); + } } ifnet_head_done(); - if (retval != 0 && ifproto != NULL) + if (retval == 0) { + } else if (ifproto != NULL) { zfree(dlif_proto_zone, ifproto); + } return (retval); } @@ -5106,6 +5270,7 @@ ifnet_attach_protocol_v2(ifnet_t ifp, protocol_family_t protocol, { int retval = 0; struct if_proto *ifproto = NULL; + uint32_t proto_count = 0; ifnet_head_lock_shared(); if (ifp == NULL || protocol == 0 || proto_details == NULL) { @@ -5138,21 +5303,25 @@ ifnet_attach_protocol_v2(ifnet_t ifp, protocol_family_t protocol, ifproto->kpi.v2.send_arp = proto_details->send_arp; retval = dlil_attach_protocol_internal(ifproto, - proto_details->demux_list, proto_details->demux_count); - - if (dlil_verbose) { - printf("%s: attached v2 protocol %d\n", if_name(ifp), - protocol); - } + proto_details->demux_list, proto_details->demux_count, + &proto_count); end: if (retval != 0 && retval != EEXIST && ifp != NULL) { DLIL_PRINTF("%s: failed to attach v2 protocol %d (err=%d)\n", if_name(ifp), protocol, retval); + } else { + if (dlil_verbose) { + printf("%s: attached v2 protocol %d (count = %d)\n", + if_name(ifp), + protocol, proto_count); + } } ifnet_head_done(); - if (retval != 0 && ifproto != NULL) + if (retval == 0) { + } else if (ifproto != NULL) { zfree(dlif_proto_zone, ifproto); + } return (retval); } @@ -5316,8 +5485,8 @@ ifnet_attach(ifnet_t ifp, const struct sockaddr_dl *ll_addr) } lck_mtx_lock_spin(&ifp->if_ref_lock); - if (ifp->if_refflags & IFRF_ATTACHED) { - panic_plain("%s: flags mismatch (attached set) ifp=%p", + if (!(ifp->if_refflags & IFRF_EMBRYONIC)) { + panic_plain("%s: flags mismatch (embryonic not set) ifp=%p", __func__, ifp); /* NOTREACHED */ } @@ -5396,7 +5565,6 @@ ifnet_attach(ifnet_t ifp, const struct sockaddr_dl *ll_addr) VERIFY(ifindex2ifnet[ifp->if_index] == NULL); /* allocate (if needed) and initialize a link address */ - VERIFY(!(dl_if->dl_if_flags & DLIF_REUSE) || ifp->if_lladdr != NULL); ifa = dlil_alloc_lladdr(ifp, ll_addr); if (ifa == NULL) { ifnet_lock_done(ifp); @@ -5453,6 +5621,10 @@ ifnet_attach(ifnet_t ifp, const struct sockaddr_dl *ll_addr) if (if_delaybased_queue) sflags |= PKTSCHEDF_QALG_DELAYBASED; + if (ifp->if_output_sched_model == + IFNET_SCHED_MODEL_DRIVER_MANAGED) + sflags |= PKTSCHEDF_QALG_DRIVER_MANAGED; + /* Initialize transmit queue(s) */ err = ifclassq_setup(ifp, sflags, (dl_if->dl_if_flags & DLIF_REUSE)); if (err != 0) { @@ -5500,6 +5672,12 @@ ifnet_attach(ifnet_t ifp, const struct sockaddr_dl *ll_addr) } } + if (ifp->if_inp != NULL && ifp->if_inp->input_mit_tcall == NULL) { + ifp->if_inp->input_mit_tcall = + thread_call_allocate_with_priority(dlil_mit_tcall_fn, + ifp, THREAD_CALL_PRIORITY_KERNEL); + } + /* * If the driver supports the new transmit model, calculate flow hash * and create a workloop starter thread to invoke the if_start callback @@ -5508,19 +5686,19 @@ ifnet_attach(ifnet_t ifp, const struct sockaddr_dl *ll_addr) if (ifp->if_eflags & IFEF_TXSTART) { ifp->if_flowhash = ifnet_calc_flowhash(ifp); VERIFY(ifp->if_flowhash != 0); - - VERIFY(ifp->if_start != NULL); VERIFY(ifp->if_start_thread == THREAD_NULL); ifnet_set_start_cycle(ifp, NULL); ifp->if_start_active = 0; ifp->if_start_req = 0; ifp->if_start_flags = 0; - if ((err = kernel_thread_start(ifnet_start_thread_fn, ifp, - &ifp->if_start_thread)) != KERN_SUCCESS) { - panic_plain("%s: ifp=%p couldn't get a start thread; " + VERIFY(ifp->if_start != NULL); + if ((err = kernel_thread_start(ifnet_start_thread_fn, + ifp, &ifp->if_start_thread)) != KERN_SUCCESS) { + panic_plain("%s: " + "ifp=%p couldn't get a start thread; " "err=%d", __func__, ifp, err); - /* NOTREACHED */ + /* NOTREACHED */ } ml_thread_policy(ifp->if_start_thread, MACHINE_GROUP, (MACHINE_NETWORK_GROUP|MACHINE_NETWORK_WORKLOOP)); @@ -5577,7 +5755,10 @@ ifnet_attach(ifnet_t ifp, const struct sockaddr_dl *ll_addr) /* Clear logging parameters */ bzero(&ifp->if_log, sizeof (ifp->if_log)); + + /* Clear foreground/realtime activity timestamps */ ifp->if_fg_sendts = 0; + ifp->if_rt_sendts = 0; VERIFY(ifp->if_delegated.ifp == NULL); VERIFY(ifp->if_delegated.type == 0); @@ -5636,6 +5817,7 @@ ifnet_attach(ifnet_t ifp, const struct sockaddr_dl *ll_addr) ifnet_lock_done(ifp); ifnet_head_done(); + lck_mtx_lock(&ifp->if_cached_route_lock); /* Enable forwarding cached route */ ifp->if_fwd_cacheok = 1; @@ -5679,6 +5861,7 @@ ifnet_attach(ifnet_t ifp, const struct sockaddr_dl *ll_addr) #endif /* INET6 */ VERIFY(ifp->if_data_threshold == 0); + VERIFY(ifp->if_dt_tcall != NULL); /* * Finally, mark this ifnet as attached. @@ -5686,7 +5869,7 @@ ifnet_attach(ifnet_t ifp, const struct sockaddr_dl *ll_addr) lck_mtx_lock(rnh_lock); ifnet_lock_exclusive(ifp); lck_mtx_lock_spin(&ifp->if_ref_lock); - ifp->if_refflags = IFRF_ATTACHED; + ifp->if_refflags = IFRF_ATTACHED; /* clears embryonic */ lck_mtx_unlock(&ifp->if_ref_lock); if (net_rtref) { /* boot-args override; enable idle notification */ @@ -5887,6 +6070,7 @@ ifnet_detach(ifnet_t ifp) lck_mtx_unlock(rnh_lock); return (ENXIO); } + VERIFY(!(ifp->if_refflags & IFRF_EMBRYONIC)); /* Indicate this interface is being detached */ ifp->if_refflags &= ~IFRF_ATTACHED; ifp->if_refflags |= IFRF_DETACHING; @@ -5933,6 +6117,7 @@ ifnet_detach(ifnet_t ifp) ifnet_head_done(); lck_mtx_unlock(rnh_lock); + /* Release reference held on the delegated interface */ if (delegated_ifp != NULL) ifnet_release(delegated_ifp); @@ -5982,7 +6167,11 @@ ifnet_detach(ifnet_t ifp) ifp->if_fwd_cacheok = 0; lck_mtx_unlock(&ifp->if_cached_route_lock); + /* Disable data threshold and wait for any pending event posting */ ifp->if_data_threshold = 0; + VERIFY(ifp->if_dt_tcall != NULL); + (void) thread_call_cancel_wait(ifp->if_dt_tcall); + /* * Drain any deferred IGMPv3/MLDv2 query responses, but keep the * references to the info structures and leave them attached to @@ -6114,7 +6303,7 @@ ifnet_detach_final(struct ifnet *ifp) lck_mtx_lock(&ifp->if_flt_lock); if_flt_monitor_enter(ifp); - lck_mtx_assert(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED); fhead = ifp->if_flt_head; TAILQ_INIT(&ifp->if_flt_head); @@ -6242,22 +6431,36 @@ ifnet_detach_final(struct ifnet *ifp) /* disassociate ifp DLIL input thread */ ifp->if_inp = NULL; + /* tell the input thread to terminate */ lck_mtx_lock_spin(&inp->input_lck); inp->input_waiting |= DLIL_INPUT_TERMINATE; if (!(inp->input_waiting & DLIL_INPUT_RUNNING)) { wakeup_one((caddr_t)&inp->input_waiting); } lck_mtx_unlock(&inp->input_lck); + + /* wait for the input thread to terminate */ + lck_mtx_lock_spin(&inp->input_lck); + while ((inp->input_waiting & DLIL_INPUT_TERMINATE_COMPLETE) + == 0) { + (void) msleep(&inp->input_waiting, &inp->input_lck, + (PZERO - 1) | PSPIN, inp->input_name, NULL); + } + lck_mtx_unlock(&inp->input_lck); + + /* clean-up input thread state */ + dlil_clean_threading_info(inp); + } /* The driver might unload, so point these to ourselves */ if_free = ifp->if_free; - ifp->if_output_handler = ifp_if_output; + ifp->if_output_dlil = ifp_if_output; ifp->if_output = ifp_if_output; ifp->if_pre_enqueue = ifp_if_output; ifp->if_start = ifp_if_start; ifp->if_output_ctl = ifp_if_ctl; - ifp->if_input_handler = ifp_if_input; + ifp->if_input_dlil = ifp_if_input; ifp->if_input_poll = ifp_if_input_poll; ifp->if_input_ctl = ifp_if_ctl; ifp->if_ioctl = ifp_if_ioctl; @@ -6288,6 +6491,7 @@ ifnet_detach_final(struct ifnet *ifp) ifp->if_eflags &= ~IFEF_QOSMARKING_ENABLED; if_set_qosmarking_mode(ifp, IFRTYPE_QOSMARKING_MODE_NONE); + ifnet_lock_done(ifp); #if PF @@ -6319,6 +6523,8 @@ ifnet_detach_final(struct ifnet *ifp) lck_mtx_unlock(&ifp->if_cached_route_lock); VERIFY(ifp->if_data_threshold == 0); + VERIFY(ifp->if_dt_tcall != NULL); + VERIFY(!thread_call_isactive(ifp->if_dt_tcall)); ifnet_llreach_ifdetach(ifp); @@ -6345,7 +6551,7 @@ ifnet_detach_final(struct ifnet *ifp) ifnet_release(ifp); } -static errno_t +errno_t ifp_if_output(struct ifnet *ifp, struct mbuf *m) { #pragma unused(ifp) @@ -6353,7 +6559,7 @@ ifp_if_output(struct ifnet *ifp, struct mbuf *m) return (0); } -static void +void ifp_if_start(struct ifnet *ifp) { ifnet_purge(ifp); @@ -6421,12 +6627,23 @@ ifp_if_check_multi(struct ifnet *ifp, const struct sockaddr *sa) return (EOPNOTSUPP); } +#if CONFIG_EMBEDDED +static errno_t +ifp_if_framer(struct ifnet *ifp, struct mbuf **m, + const struct sockaddr *sa, const char *ll, const char *t, + u_int32_t *pre, u_int32_t *post) +#else static errno_t ifp_if_framer(struct ifnet *ifp, struct mbuf **m, const struct sockaddr *sa, const char *ll, const char *t) +#endif /* !CONFIG_EMBEDDED */ { #pragma unused(ifp, m, sa, ll, t) +#if CONFIG_EMBEDDED + return (ifp_if_framer_extended(ifp, m, sa, ll, t, pre, post)); +#else return (ifp_if_framer_extended(ifp, m, sa, ll, t, NULL, NULL)); +#endif /* !CONFIG_EMBEDDED */ } static errno_t @@ -6535,7 +6752,7 @@ int dlil_if_acquire(u_int32_t family, const void *uniqueid, MALLOC(dlifp1->dl_if_uniqueid, void *, uniqueid_len, M_NKE, M_WAITOK); if (dlifp1->dl_if_uniqueid == NULL) { - zfree(dlif_zone, dlifp1); + zfree(dlif_zone, buf); ret = ENOMEM; goto end; } @@ -6557,6 +6774,7 @@ int dlil_if_acquire(u_int32_t family, const void *uniqueid, ifp1->if_desc.ifd_len = 0; ifp1->if_desc.ifd_desc = dlifp1->dl_if_descstorage; + #if CONFIG_MACF_NET mac_ifnet_label_init(ifp1); #endif @@ -6601,6 +6819,14 @@ int dlil_if_acquire(u_int32_t family, const void *uniqueid, lck_mtx_init(&ifp1->if_poll_lock, ifnet_rcv_lock_group, ifnet_lock_attr); + /* thread call allocation is done with sleeping zalloc */ + ifp1->if_dt_tcall = thread_call_allocate_with_options(dlil_dt_tcall_fn, + ifp1, THREAD_CALL_PRIORITY_KERNEL, THREAD_CALL_OPTIONS_ONCE); + if (ifp1->if_dt_tcall == NULL) { + panic_plain("%s: couldn't create if_dt_tcall", __func__); + /* NOTREACHED */ + } + TAILQ_INSERT_TAIL(&dlil_ifnet_head, dlifp1, dl_if_link); *ifp = ifp1; @@ -6619,6 +6845,11 @@ dlil_if_release(ifnet_t ifp) { struct dlil_ifnet *dlifp = (struct dlil_ifnet *)ifp; + VERIFY(OSDecrementAtomic64(&net_api_stats.nas_ifnet_alloc_count) > 0); + if (!(ifp->if_xflags & IFXF_ALLOC_KPI)) { + VERIFY(OSDecrementAtomic64(&net_api_stats.nas_ifnet_alloc_os_count) > 0); + } + ifnet_lock_exclusive(ifp); lck_mtx_lock(&dlifp->dl_if_lock); dlifp->dl_if_flags &= ~DLIF_INUSE; @@ -6656,7 +6887,7 @@ dlil_if_unlock(void) __private_extern__ void dlil_if_lock_assert(void) { - lck_mtx_assert(&dlil_ifnet_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&dlil_ifnet_lock, LCK_MTX_ASSERT_OWNED); } __private_extern__ void @@ -6750,17 +6981,16 @@ ifnet_cached_rtlookup_inet(struct ifnet *ifp, struct in_addr src_ip) } dst->sin_addr = src_ip; - if (src_rt.ro_rt == NULL) { - src_rt.ro_rt = rtalloc1_scoped((struct sockaddr *)dst, - 0, 0, ifp->if_index); + VERIFY(src_rt.ro_rt == NULL); + src_rt.ro_rt = rtalloc1_scoped((struct sockaddr *)dst, + 0, 0, ifp->if_index); - if (src_rt.ro_rt != NULL) { - /* retain a ref, copyin consumes one */ - struct rtentry *rte = src_rt.ro_rt; - RT_ADDREF(rte); - ifp_src_route_copyin(ifp, &src_rt); - src_rt.ro_rt = rte; - } + if (src_rt.ro_rt != NULL) { + /* retain a ref, copyin consumes one */ + struct rtentry *rte = src_rt.ro_rt; + RT_ADDREF(rte); + ifp_src_route_copyin(ifp, &src_rt); + src_rt.ro_rt = rte; } } @@ -6814,12 +7044,21 @@ if_lqm_update(struct ifnet *ifp, int lqm, int locked) VERIFY(lqm >= IFNET_LQM_MIN && lqm <= IFNET_LQM_MAX); /* Normalize to edge */ - if (lqm >= 0 && lqm <= IFNET_LQM_THRESH_BAD) - lqm = IFNET_LQM_THRESH_BAD; - else if (lqm > IFNET_LQM_THRESH_BAD && lqm <= IFNET_LQM_THRESH_POOR) + if (lqm >= 0 && lqm <= IFNET_LQM_THRESH_ABORT) { + lqm = IFNET_LQM_THRESH_ABORT; + atomic_bitset_32(&tcbinfo.ipi_flags, + INPCBINFO_HANDLE_LQM_ABORT); + inpcb_timer_sched(&tcbinfo, INPCB_TIMER_FAST); + } else if (lqm > IFNET_LQM_THRESH_ABORT && + lqm <= IFNET_LQM_THRESH_MINIMALLY_VIABLE) { + lqm = IFNET_LQM_THRESH_MINIMALLY_VIABLE; + } else if (lqm > IFNET_LQM_THRESH_MINIMALLY_VIABLE && + lqm <= IFNET_LQM_THRESH_POOR) { lqm = IFNET_LQM_THRESH_POOR; - else if (lqm > IFNET_LQM_THRESH_POOR && lqm <= IFNET_LQM_THRESH_GOOD) + } else if (lqm > IFNET_LQM_THRESH_POOR && + lqm <= IFNET_LQM_THRESH_GOOD) { lqm = IFNET_LQM_THRESH_GOOD; + } /* * Take the lock if needed @@ -7000,6 +7239,10 @@ if_probe_connectivity(struct ifnet *ifp, u_int32_t conn_probe) ifp->if_eflags |= IFEF_PROBE_CONNECTIVITY; ifnet_lock_done(ifp); +#if NECP + necp_update_all_clients(); +#endif /* NECP */ + tcp_probe_connectivity(ifp, conn_probe); return (0); } @@ -7265,19 +7508,7 @@ dlil_ifaddr_bytes(const struct sockaddr_dl *sdl, size_t *sizep, [0] = 2 }; - switch (sdl->sdl_type) { - case IFT_ETHER: - VERIFY(size == ETHER_ADDR_LEN); - bytes = unspec; - break; - case IFT_IEEE1394: - VERIFY(size == FIREWIRE_EUI64_LEN); - bytes = unspec; - break; - default: - VERIFY(FALSE); - break; - }; + bytes = unspec; } } #else @@ -7402,11 +7633,6 @@ ifnet_set_throttle(struct ifnet *ifp, u_int32_t level) switch (level) { case IFNET_THROTTLE_OFF: case IFNET_THROTTLE_OPPORTUNISTIC: -#if PF_ALTQ - /* Throttling works only for IFCQ, not ALTQ instances */ - if (ALTQ_IS_ENABLED(IFCQ_ALTQ(ifq))) - return (ENXIO); -#endif /* PF_ALTQ */ break; default: return (EINVAL); @@ -7569,7 +7795,7 @@ ifnet_flowid(struct ifnet *ifp, uint32_t *flowid) if (ifp == NULL || flowid == NULL) { return (EINVAL); } else if (!(ifp->if_eflags & IFEF_TXSTART) || - !(ifp->if_refflags & IFRF_ATTACHED)) { + !IF_FULLY_ATTACHED(ifp)) { return (ENXIO); } @@ -7586,7 +7812,7 @@ ifnet_disable_output(struct ifnet *ifp) if (ifp == NULL) { return (EINVAL); } else if (!(ifp->if_eflags & IFEF_TXSTART) || - !(ifp->if_refflags & IFRF_ATTACHED)) { + !IF_FULLY_ATTACHED(ifp)) { return (ENXIO); } @@ -7604,7 +7830,7 @@ ifnet_enable_output(struct ifnet *ifp) if (ifp == NULL) { return (EINVAL); } else if (!(ifp->if_eflags & IFEF_TXSTART) || - !(ifp->if_refflags & IFRF_ATTACHED)) { + !IF_FULLY_ATTACHED(ifp)) { return (ENXIO); } @@ -7838,7 +8064,7 @@ ifnet_get_netsignature(struct ifnet *ifp, uint8_t family, uint8_t *len, { int error = 0; - if (ifp == NULL || len == NULL || flags == NULL || data == NULL) + if (ifp == NULL || len == NULL || data == NULL) return (EINVAL); switch (family) { @@ -7883,12 +8109,104 @@ ifnet_get_netsignature(struct ifnet *ifp, uint8_t family, uint8_t *len, break; } - if (error == 0) + if (error == 0 && flags != NULL) *flags = 0; return (error); } +#if INET6 +int +ifnet_set_nat64prefix(struct ifnet *ifp, struct ipv6_prefix *prefixes) +{ + int i, error = 0, one_set = 0; + + if_inet6data_lock_exclusive(ifp); + + if (IN6_IFEXTRA(ifp) == NULL) { + error = ENOMEM; + goto out; + } + + for (i = 0; i < NAT64_MAX_NUM_PREFIXES; i++) { + uint32_t prefix_len = + prefixes[i].prefix_len; + struct in6_addr *prefix = + &prefixes[i].ipv6_prefix; + + if (prefix_len == 0) { + /* Allow clearing the signature */ + IN6_IFEXTRA(ifp)->nat64_prefixes[i].prefix_len = 0; + bzero(&IN6_IFEXTRA(ifp)->nat64_prefixes[i].ipv6_prefix, + sizeof(struct in6_addr)); + + continue; + } else if (prefix_len != NAT64_PREFIX_LEN_32 && + prefix_len != NAT64_PREFIX_LEN_40 && + prefix_len != NAT64_PREFIX_LEN_48 && + prefix_len != NAT64_PREFIX_LEN_56 && + prefix_len != NAT64_PREFIX_LEN_64 && + prefix_len != NAT64_PREFIX_LEN_96) { + error = EINVAL; + goto out; + } + + if (IN6_IS_SCOPE_EMBED(prefix)) { + error = EINVAL; + goto out; + } + + IN6_IFEXTRA(ifp)->nat64_prefixes[i].prefix_len = prefix_len; + bcopy(prefix, &IN6_IFEXTRA(ifp)->nat64_prefixes[i].ipv6_prefix, + sizeof(struct in6_addr)); + one_set = 1; + } + +out: + if_inet6data_lock_done(ifp); + + if (error == 0 && one_set != 0) + necp_update_all_clients(); + + return (error); +} + +int +ifnet_get_nat64prefix(struct ifnet *ifp, struct ipv6_prefix *prefixes) +{ + int i, found_one = 0, error = 0; + + if (ifp == NULL) + return (EINVAL); + + if_inet6data_lock_shared(ifp); + + if (IN6_IFEXTRA(ifp) == NULL) { + error = ENOMEM; + goto out; + } + + for (i = 0; i < NAT64_MAX_NUM_PREFIXES; i++) { + if (IN6_IFEXTRA(ifp)->nat64_prefixes[i].prefix_len != 0) + found_one = 1; + } + + if (found_one == 0) { + error = ENOENT; + goto out; + } + + if (prefixes) + bcopy(IN6_IFEXTRA(ifp)->nat64_prefixes, prefixes, + sizeof(IN6_IFEXTRA(ifp)->nat64_prefixes)); + +out: + if_inet6data_lock_done(ifp); + + return (error); +} +#endif + static void dlil_output_cksum_dbg(struct ifnet *ifp, struct mbuf *m, uint32_t hoff, protocol_family_t pf) @@ -7932,7 +8250,7 @@ static void dlil_input_cksum_dbg(struct ifnet *ifp, struct mbuf *m, char *frame_header, protocol_family_t pf) { - uint16_t sum; + uint16_t sum = 0; uint32_t hlen; if (frame_header == NULL || @@ -8042,7 +8360,8 @@ dlil_input_cksum_dbg(struct ifnet *ifp, struct mbuf *m, char *frame_header, if (aoff == rxoff || aoff > (uint32_t)m->m_pkthdr.len) return; - sum = m_adj_sum16(m, rxoff, aoff, sum); + sum = m_adj_sum16(m, rxoff, aoff, + m_pktlen(m) - aoff, sum); m->m_pkthdr.csum_rx_val = sum; m->m_pkthdr.csum_rx_start = (aoff + hlen); @@ -8137,7 +8456,7 @@ sysctl_tx_chain_len_stats SYSCTL_HANDLER_ARGS } -#if DEBUG +#if DEBUG || DEVELOPMENT /* Blob for sum16 verification */ static uint8_t sumdata[] = { 0x1f, 0x8b, 0x08, 0x08, 0x4c, 0xe5, 0x9a, 0x4f, 0x00, 0x03, @@ -8177,19 +8496,26 @@ static uint8_t sumdata[] = { /* Precomputed 16-bit 1's complement sums for various spans of the above data */ static struct { - int len; - uint16_t sum; + boolean_t init; + uint16_t len; + uint16_t sumr; /* reference */ + uint16_t sumrp; /* reference, precomputed */ } sumtbl[] = { - { 11, 0xcb6d }, - { 20, 0x20dd }, - { 27, 0xbabd }, - { 32, 0xf3e8 }, - { 37, 0x197d }, - { 43, 0x9eae }, - { 64, 0x4678 }, - { 127, 0x9399 }, - { 256, 0xd147 }, - { 325, 0x0358 } + { FALSE, 0, 0, 0x0000 }, + { FALSE, 1, 0, 0x001f }, + { FALSE, 2, 0, 0x8b1f }, + { FALSE, 3, 0, 0x8b27 }, + { FALSE, 7, 0, 0x790e }, + { FALSE, 11, 0, 0xcb6d }, + { FALSE, 20, 0, 0x20dd }, + { FALSE, 27, 0, 0xbabd }, + { FALSE, 32, 0, 0xf3e8 }, + { FALSE, 37, 0, 0x197d }, + { FALSE, 43, 0, 0x9eae }, + { FALSE, 64, 0, 0x4678 }, + { FALSE, 127, 0, 0x9399 }, + { FALSE, 256, 0, 0xd147 }, + { FALSE, 325, 0, 0x0358 }, }; #define SUMTBL_MAX ((int)sizeof (sumtbl) / (int)sizeof (sumtbl[0])) @@ -8203,6 +8529,8 @@ dlil_verify_sum16(void) /* Make sure test data plus extra room for alignment fits in cluster */ _CASSERT((sizeof (sumdata) + (sizeof (uint64_t) * 2)) <= MCLBYTES); + kprintf("DLIL: running SUM16 self-tests ... "); + m = m_getcl(M_WAITOK, MT_DATA, M_PKTHDR); MH_ALIGN(m, sizeof (uint32_t)); /* 32-bit starting alignment */ buf = mtod(m, uint8_t *); /* base address */ @@ -8213,7 +8541,7 @@ dlil_verify_sum16(void) /* Verify for all possible alignments */ for (i = 0; i < (int)sizeof (uint64_t); i++) { - uint16_t sum; + uint16_t sum, sumr; uint8_t *c; /* Copy over test data to mbuf */ @@ -8226,11 +8554,25 @@ dlil_verify_sum16(void) m->m_len = len; sum = m_sum16(m, 0, len); + if (!sumtbl[n].init) { + sumr = in_cksum_mbuf_ref(m, len, 0, 0); + sumtbl[n].sumr = sumr; + sumtbl[n].init = TRUE; + } else { + sumr = sumtbl[n].sumr; + } + /* Something is horribly broken; stop now */ - if (sum != sumtbl[n].sum) { - panic("%s: broken m_sum16 for len=%d align=%d " - "sum=0x%04x [expected=0x%04x]\n", __func__, - len, i, sum, sumtbl[n].sum); + if (sumr != sumtbl[n].sumrp) { + panic_plain("\n%s: broken in_cksum_mbuf_ref() " + "for len=%d align=%d sum=0x%04x " + "[expected=0x%04x]\n", __func__, + len, i, sum, sumr); + /* NOTREACHED */ + } else if (sum != sumr) { + panic_plain("\n%s: broken m_sum16() for len=%d " + "align=%d sum=0x%04x [expected=0x%04x]\n", + __func__, len, i, sum, sumr); /* NOTREACHED */ } @@ -8240,10 +8582,10 @@ dlil_verify_sum16(void) sum = m_sum16(m, i, len); /* Something is horribly broken; stop now */ - if (sum != sumtbl[n].sum) { - panic("%s: broken m_sum16 for len=%d offset=%d " - "sum=0x%04x [expected=0x%04x]\n", __func__, - len, i, sum, sumtbl[n].sum); + if (sum != sumr) { + panic_plain("\n%s: broken m_sum16() for len=%d " + "offset=%d sum=0x%04x [expected=0x%04x]\n", + __func__, len, i, sum, sumr); /* NOTREACHED */ } #if INET @@ -8251,10 +8593,10 @@ dlil_verify_sum16(void) sum = b_sum16(c, len); /* Something is horribly broken; stop now */ - if (sum != sumtbl[n].sum) { - panic("%s: broken b_sum16 for len=%d align=%d " - "sum=0x%04x [expected=0x%04x]\n", __func__, - len, i, sum, sumtbl[n].sum); + if (sum != sumr) { + panic_plain("\n%s: broken b_sum16() for len=%d " + "align=%d sum=0x%04x [expected=0x%04x]\n", + __func__, len, i, sum, sumr); /* NOTREACHED */ } #endif /* INET */ @@ -8262,9 +8604,9 @@ dlil_verify_sum16(void) } m_freem(m); - printf("DLIL: SUM16 self-tests PASSED\n"); + kprintf("PASSED\n"); } -#endif /* DEBUG */ +#endif /* DEBUG || DEVELOPMENT */ #define CASE_STRINGIFY(x) case x: return #x @@ -8353,7 +8695,7 @@ sysctl_get_ports_used SYSCTL_HANDLER_ARGS ifp = ifindex2ifnet[idx]; ifnet_head_done(); - bitfield = _MALLOC(bitstr_size(65536), M_TEMP, M_WAITOK); + bitfield = _MALLOC(bitstr_size(65536), M_TEMP, M_WAITOK | M_ZERO); if (bitfield == NULL) { error = ENOMEM; goto done; @@ -8371,6 +8713,48 @@ done: return (error); } +static void +dlil_dt_tcall_fn(thread_call_param_t arg0, thread_call_param_t arg1) +{ +#pragma unused(arg1) + struct ifnet *ifp = arg0; + + if (ifnet_is_attached(ifp, 1)) { + nstat_ifnet_threshold_reached(ifp->if_index); + ifnet_decr_iorefcnt(ifp); + } +} + +void +ifnet_notify_data_threshold(struct ifnet *ifp) +{ + uint64_t bytes = (ifp->if_ibytes + ifp->if_obytes); + uint64_t oldbytes = ifp->if_dt_bytes; + + ASSERT(ifp->if_dt_tcall != NULL); + + /* + * If we went over the threshold, notify NetworkStatistics. + * We rate-limit it based on the threshold interval value. + */ + if (threshold_notify && (bytes - oldbytes) > ifp->if_data_threshold && + OSCompareAndSwap64(oldbytes, bytes, &ifp->if_dt_bytes) && + !thread_call_isactive(ifp->if_dt_tcall)) { + uint64_t tival = (threshold_interval * NSEC_PER_SEC); + uint64_t now = mach_absolute_time(), deadline = now; + uint64_t ival; + + if (tival != 0) { + nanoseconds_to_absolutetime(tival, &ival); + clock_deadline_for_periodic_event(ival, now, &deadline); + (void) thread_call_enter_delayed(ifp->if_dt_tcall, + deadline); + } else { + (void) thread_call_enter(ifp->if_dt_tcall); + } + } +} + #if (DEVELOPMENT || DEBUG) /* * The sysctl variable name contains the input parameters of @@ -8468,3 +8852,34 @@ done: return (error); } #endif /* DEVELOPMENT || DEBUG */ + +void +ifnet_update_stats_per_flow(struct ifnet_stats_per_flow *ifs, + struct ifnet *ifp) +{ + tcp_update_stats_per_flow(ifs, ifp); +} + +static void +dlil_mit_tcall_fn(thread_call_param_t arg0, thread_call_param_t arg1) +{ +#pragma unused(arg1) + struct ifnet *ifp = (struct ifnet *)arg0; + struct dlil_threading_info *inp = ifp->if_inp; + + ifnet_lock_shared(ifp); + if (!IF_FULLY_ATTACHED(ifp) || inp == NULL) { + ifnet_lock_done(ifp); + return; + } + + lck_mtx_lock_spin(&inp->input_lck); + inp->input_waiting |= DLIL_INPUT_WAITING; + if (!(inp->input_waiting & DLIL_INPUT_RUNNING) || + !qempty(&inp->rcvq_pkts)) { + inp->wtot++; + wakeup_one((caddr_t)&inp->input_waiting); + } + lck_mtx_unlock(&inp->input_lck); + ifnet_lock_done(ifp); +} diff --git a/bsd/net/dlil.h b/bsd/net/dlil.h index a412d25fc..5a6c669e4 100644 --- a/bsd/net/dlil.h +++ b/bsd/net/dlil.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999-2016 Apple Inc. All rights reserved. + * Copyright (c) 1999-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -202,6 +202,7 @@ struct dlil_threading_info { */ u_int64_t input_mbuf_cnt; /* total # of packets processed */ #endif + thread_call_t input_mit_tcall; /* coalescing input processing */ }; /* @@ -221,11 +222,13 @@ struct dlil_main_threading_info { #define DLIL_PROTO_REGISTER 0x20000000 #define DLIL_PROTO_WAITING 0x10000000 #define DLIL_INPUT_TERMINATE 0x08000000 +#define DLIL_INPUT_TERMINATE_COMPLETE 0x04000000 /* * Flags for dlil_attach_filter() */ #define DLIL_IFF_TSO 0x01 /* Interface filter supports TSO */ +#define DLIL_IFF_INTERNAL 0x02 /* Apple internal -- do not count towards stats */ extern int dlil_verbose; extern uint32_t hwcksum_dbg; @@ -236,6 +239,8 @@ extern struct dlil_threading_info *dlil_main_input_thread; extern void dlil_init(void); extern errno_t ifp_if_ioctl(struct ifnet *, unsigned long, void *); +extern errno_t ifp_if_output(struct ifnet *, struct mbuf *); +extern void ifp_if_start(struct ifnet *); extern errno_t dlil_set_bpf_tap(ifnet_t, bpf_tap_mode, bpf_packet_func); @@ -325,6 +330,7 @@ extern int dlil_post_complete_msg(struct ifnet *, struct kev_msg *); extern int dlil_alloc_local_stats(struct ifnet *); + /* * dlil_if_acquire is obsolete. Use ifnet_allocate. */ @@ -364,6 +370,83 @@ extern errno_t dlil_input_handler(struct ifnet *, struct mbuf *, struct mbuf *, const struct ifnet_stat_increment_param *, boolean_t, struct thread *); + +/* + * This is mostly called from the context of the DLIL input thread; + * because of that there is no need for atomic operations. + */ +__attribute__((always_inline)) +static inline void +ifp_inc_traffic_class_in(struct ifnet *ifp, struct mbuf *m) +{ + if (!(m->m_flags & M_PKTHDR)) + return; + + switch (m_get_traffic_class(m)) { + case MBUF_TC_BE: + ifp->if_tc.ifi_ibepackets++; + ifp->if_tc.ifi_ibebytes += m->m_pkthdr.len; + break; + case MBUF_TC_BK: + ifp->if_tc.ifi_ibkpackets++; + ifp->if_tc.ifi_ibkbytes += m->m_pkthdr.len; + break; + case MBUF_TC_VI: + ifp->if_tc.ifi_ivipackets++; + ifp->if_tc.ifi_ivibytes += m->m_pkthdr.len; + break; + case MBUF_TC_VO: + ifp->if_tc.ifi_ivopackets++; + ifp->if_tc.ifi_ivobytes += m->m_pkthdr.len; + break; + default: + break; + } + + if (mbuf_is_traffic_class_privileged(m)) { + ifp->if_tc.ifi_ipvpackets++; + ifp->if_tc.ifi_ipvbytes += m->m_pkthdr.len; + } +} + +/* + * This is called from DLIL output, hence multiple threads could end + * up modifying the statistics. We trade off acccuracy for performance + * by not using atomic operations here. + */ +__attribute__((always_inline)) +static inline void +ifp_inc_traffic_class_out(struct ifnet *ifp, struct mbuf *m) +{ + if (!(m->m_flags & M_PKTHDR)) + return; + + switch (m_get_traffic_class(m)) { + case MBUF_TC_BE: + ifp->if_tc.ifi_obepackets++; + ifp->if_tc.ifi_obebytes += m->m_pkthdr.len; + break; + case MBUF_TC_BK: + ifp->if_tc.ifi_obkpackets++; + ifp->if_tc.ifi_obkbytes += m->m_pkthdr.len; + break; + case MBUF_TC_VI: + ifp->if_tc.ifi_ovipackets++; + ifp->if_tc.ifi_ovibytes += m->m_pkthdr.len; + break; + case MBUF_TC_VO: + ifp->if_tc.ifi_ovopackets++; + ifp->if_tc.ifi_ovobytes += m->m_pkthdr.len; + break; + default: + break; + } + + if (mbuf_is_traffic_class_privileged(m)) { + ifp->if_tc.ifi_opvpackets++; + ifp->if_tc.ifi_opvbytes += m->m_pkthdr.len; + } +} #endif /* BSD_KERNEL_PRIVATE */ #endif /* KERNEL_PRIVATE */ #endif /* KERNEL */ diff --git a/bsd/net/ether_if_module.c b/bsd/net/ether_if_module.c index 8fd6074f1..3335bb6c4 100644 --- a/bsd/net/ether_if_module.c +++ b/bsd/net/ether_if_module.c @@ -102,11 +102,12 @@ #if IF_BRIDGE #include <net/if_bridgevar.h> #endif /* IF_BRIDGE */ +#if IF_FAKE +#include <net/if_fake_var.h> +#endif /* IF_FAKE */ #include <net/dlil.h> -#define memcpy(x,y,z) bcopy(y, x, z) - SYSCTL_DECL(_net_link); SYSCTL_NODE(_net_link, IFT_ETHER, ether, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "Ethernet"); @@ -118,7 +119,11 @@ struct en_desc { }; /* descriptors are allocated in blocks of ETHER_DESC_BLK_SIZE */ +#if CONFIG_EMBEDDED +#define ETHER_DESC_BLK_SIZE (2) /* IP, ARP */ +#else #define ETHER_DESC_BLK_SIZE (10) +#endif /* * Header for the demux list, hangs off of IFP at if_family_cookie @@ -636,6 +641,10 @@ ether_family_init(void) #if IF_BRIDGE bridgeattach(0); #endif /* IF_BRIDGE */ +#if IF_FAKE + if_fake_init(); +#endif /* IF_FAKE */ + done: return (error); diff --git a/bsd/net/ethernet.h b/bsd/net/ethernet.h index 5f29b6e71..3f61bc94f 100644 --- a/bsd/net/ethernet.h +++ b/bsd/net/ethernet.h @@ -130,6 +130,17 @@ struct ether_addr *ether_aton(const char *); #ifdef BSD_KERNEL_PRIVATE extern u_char etherbroadcastaddr[ETHER_ADDR_LEN]; +#if defined (__arm__) + +#include <string.h> + +static __inline__ int +_ether_cmp(const void * a, const void * b) +{ + return (memcmp(a, b, ETHER_ADDR_LEN)); +} + +#else /* __arm__ */ static __inline__ int _ether_cmp(const void * a, const void * b) @@ -145,6 +156,7 @@ _ether_cmp(const void * a, const void * b) return (0); } +#endif /* __arm__ */ #endif /* BSD_KERNEL_PRIVATE */ #define ETHER_IS_MULTICAST(addr) (*(addr) & 0x01) /* is address mcast/bcast? */ diff --git a/bsd/net/flowadv.c b/bsd/net/flowadv.c index 14ad67758..214c5c715 100644 --- a/bsd/net/flowadv.c +++ b/bsd/net/flowadv.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -196,7 +196,7 @@ flowadv_thread_cont(int err) { #pragma unused(err) for (;;) { - lck_mtx_assert(&fadv_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&fadv_lock, LCK_MTX_ASSERT_OWNED); while (STAILQ_EMPTY(&fadv_list)) { VERIFY(!fadv_active); (void) msleep0(&fadv_list, &fadv_lock, (PSOCK | PSPIN), @@ -215,7 +215,7 @@ flowadv_thread_cont(int err) STAILQ_NEXT(fce, fce_link) = NULL; lck_mtx_unlock(&fadv_lock); - switch (fce->fce_flowsrc) { + switch (fce->fce_flowsrc_type) { case FLOWSRC_INPCB: inp_flowadv(fce->fce_flowid); break; @@ -224,6 +224,7 @@ flowadv_thread_cont(int err) ifnet_flowadv(fce->fce_flowid); break; + case FLOWSRC_PF: default: break; diff --git a/bsd/net/flowadv.h b/bsd/net/flowadv.h index f56eb4b8b..76ae8a498 100644 --- a/bsd/net/flowadv.h +++ b/bsd/net/flowadv.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -33,6 +33,7 @@ #include <sys/types.h> #include <sys/queue.h> + #define FADV_SUCCESS 0 /* success */ #define FADV_FLOW_CONTROLLED 1 /* regular flow control */ #define FADV_SUSPENDED 2 /* flow control due to suspension */ @@ -44,7 +45,7 @@ struct flowadv { #ifdef BSD_KERNEL_PRIVATE struct flowadv_fcentry { STAILQ_ENTRY(flowadv_fcentry) fce_link; - u_int32_t fce_flowsrc; /* FLOWSRC values */ + u_int32_t fce_flowsrc_type; /* FLOWSRC values */ u_int32_t fce_flowid; }; diff --git a/bsd/net/flowhash.c b/bsd/net/flowhash.c index a45796023..85761bd68 100644 --- a/bsd/net/flowhash.c +++ b/bsd/net/flowhash.c @@ -64,17 +64,16 @@ static inline u_int64_t mh3_fmix64(u_int64_t); /* * The following hash algorithms are selected based on performance: * - * Intel 32-bit: MurmurHash3_x86_32 - * Intel 64-bit: MurmurHash3_x64_128 - * ARM, et al: JHash + * 64-bit: MurmurHash3_x64_128 + * 32-bit: JHash */ -#if defined(__x86_64__) +#if defined(__LP64__) net_flowhash_fn_t *net_flowhash = net_flowhash_mh3_x64_128; -#else /* !__i386__ && !__x86_64__ */ +#else /* !__LP64__ */ net_flowhash_fn_t *net_flowhash = net_flowhash_jhash; -#endif /* !__i386__ && !__x86_64__ */ +#endif /* !__LP64__ */ -#if defined(__i386__) || defined(__x86_64__) +#if defined(__i386__) || defined(__x86_64__) || defined(__arm64__) static inline u_int32_t getblock32(const u_int32_t *p, int i) { @@ -86,7 +85,7 @@ getblock64(const u_int64_t *p, int i) { return (p[i]); } -#else /* !__i386__ && !__x86_64 */ +#else /* !__i386__ && !__x86_64__ && !__arm64__*/ static inline u_int32_t getblock32(const u_int32_t *p, int i) { @@ -146,7 +145,7 @@ getblock64(const u_int64_t *p, int i) } return (value); } -#endif /* !__i386__ && !__x86_64 */ +#endif /* !__i386__ && !__x86_64 && !__arm64__ */ static inline u_int32_t mh3_fmix32(u_int32_t h) @@ -255,20 +254,44 @@ net_flowhash_mh3_x64_128(const void *key, u_int32_t len, const u_int32_t seed) k2 = getblock64(blocks, i * 2 + 1); k1 *= MH3_X64_128_C1; +#if defined(__x86_64__) + __asm__ ( "rol $31, %[k1]\n\t" :[k1] "+r" (k1) : :); +#elif defined(__arm64__) + __asm__ ( "ror %[k1], %[k1], #(64-31)\n\t" :[k1] "+r" (k1) : :); +#else /* !__x86_64__ && !__arm64__ */ k1 = ROTL64(k1, 31); +#endif /* !__x86_64__ && !__arm64__ */ k1 *= MH3_X64_128_C2; h1 ^= k1; - h1 = ROTL64(h1, 27); +#if defined(__x86_64__) + __asm__ ( "rol $27, %[h1]\n\t" :[h1] "+r" (h1) : :); +#elif defined(__arm64__) + __asm__ ( "ror %[h1], %[h1], #(64-27)\n\t" :[h1] "+r" (h1) : :); +#else /* !__x86_64__ && !__arm64__ */ + h1 = ROTL64(h1, 27); +#endif /* !__x86_64__ && !__arm64__ */ h1 += h2; h1 = h1 * 5 + 0x52dce729; k2 *= MH3_X64_128_C2; - k2 = ROTL64(k2, 33); +#if defined(__x86_64__) + __asm__ ( "rol $33, %[k2]\n\t" :[k2] "+r" (k2) : :); +#elif defined(__arm64__) + __asm__ ( "ror %[k2], %[k2], #(64-33)\n\t" :[k2] "+r" (k2) : :); +#else /* !__x86_64__ && !__arm64__ */ + k2 = ROTL64(k2, 33); +#endif /* !__x86_64__ && !__arm64__ */ k2 *= MH3_X64_128_C1; h2 ^= k2; - h2 = ROTL64(h2, 31); +#if defined(__x86_64__) + __asm__ ( "rol $31, %[h2]\n\t" :[h2] "+r" (h2) : :); +#elif defined(__arm64__) + __asm__ ( "ror %[h2], %[h2], #(64-31)\n\t" :[h2] "+r" (h2) : :); +#else /* !__x86_64__ && !__arm64__ */ + h2 = ROTL64(h2, 31); +#endif /* !__x86_64__ && !__arm64__ */ h2 += h1; h2 = h2 * 5+ 0x38495ab5; } @@ -300,7 +323,13 @@ net_flowhash_mh3_x64_128(const void *key, u_int32_t len, const u_int32_t seed) case 9: k2 ^= ((u_int64_t)tail[8]) << 0; k2 *= MH3_X64_128_C2; - k2 = ROTL64(k2, 33); +#if defined(__x86_64__) + __asm__ ( "rol $33, %[k2]\n\t" :[k2] "+r" (k2) : :); +#elif defined(__arm64__) + __asm__ ( "ror %[k2], %[k2], #(64-33)\n\t" :[k2] "+r" (k2) : :); +#else /* !__x86_64__ && !__arm64__ */ + k2 = ROTL64(k2, 33); +#endif /* !__x86_64__ && !__arm64__ */ k2 *= MH3_X64_128_C1; h2 ^= k2; /* FALLTHRU */ @@ -328,7 +357,13 @@ net_flowhash_mh3_x64_128(const void *key, u_int32_t len, const u_int32_t seed) case 1: k1 ^= ((u_int64_t)tail[0]) << 0; k1 *= MH3_X64_128_C1; - k1 = ROTL64(k1, 31); +#if defined(__x86_64__) + __asm__ ( "rol $31, %[k1]\n\t" :[k1] "+r" (k1) : :); +#elif defined(__arm64__) + __asm__ ( "ror %[k1], %[k1], #(64-31)\n\t" :[k1] "+r" (k1) : :); +#else /* !__x86_64__ && !__arm64__ */ + k1 = ROTL64(k1, 31); +#endif /* !__x86_64__ && !__arm64__ */ k1 *= MH3_X64_128_C2; h1 ^= k1; }; diff --git a/bsd/net/if.c b/bsd/net/if.c index 1bb5a6e1f..244132134 100644 --- a/bsd/net/if.c +++ b/bsd/net/if.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -99,10 +99,11 @@ #include <net/if_ppp.h> #include <net/ethernet.h> #include <net/network_agent.h> - #include <net/radix.h> #include <net/route.h> #include <net/dlil.h> +#include <net/nwk_wq.h> + #include <sys/domain.h> #include <libkern/OSAtomic.h> @@ -129,10 +130,6 @@ #include <security/mac_framework.h> #endif -#if PF_ALTQ -#include <net/altq/if_altq.h> -#endif /* !PF_ALTQ */ - /* * System initialization */ @@ -246,6 +243,9 @@ static uint32_t if_verbose = 0; SYSCTL_INT(_net_link_generic_system, OID_AUTO, if_verbose, CTLFLAG_RW | CTLFLAG_LOCKED, &if_verbose, 0, ""); +/* Eventhandler context for interface events */ +struct eventhandler_lists_ctxt ifnet_evhdlr_ctxt; + void ifa_init(void) { @@ -384,6 +384,7 @@ if_detach_ifa_common(struct ifnet *ifp, struct ifaddr *ifa, int link) if (ifa->ifa_detached != NULL) (*ifa->ifa_detached)(ifa); + } #define INITIAL_IF_INDEXLIM 8 @@ -1238,7 +1239,7 @@ link_rtrequest(int cmd, struct rtentry *rt, struct sockaddr *sa) struct ifnet *ifp; void (*ifa_rtrequest)(int, struct rtentry *, struct sockaddr *); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK_ASSERT_HELD(rt); if (cmd != RTM_ADD || ((ifa = rt->rt_ifa) == 0) || @@ -1369,12 +1370,6 @@ if_qflush(struct ifnet *ifp, int ifq_locked) if (IFCQ_IS_ENABLED(ifq)) IFCQ_PURGE(ifq); -#if PF_ALTQ - if (IFCQ_IS_DRAINING(ifq)) - ifq->ifcq_drain = 0; - if (ALTQ_IS_ENABLED(IFCQ_ALTQ(ifq))) - ALTQ_PURGE(IFCQ_ALTQ(ifq)); -#endif /* PF_ALTQ */ VERIFY(IFCQ_IS_EMPTY(ifq)); @@ -1398,14 +1393,6 @@ if_qflush_sc(struct ifnet *ifp, mbuf_svc_class_t sc, u_int32_t flow, if (IFCQ_IS_ENABLED(ifq)) IFCQ_PURGE_SC(ifq, sc, flow, cnt, len); -#if PF_ALTQ - if (IFCQ_IS_DRAINING(ifq)) { - VERIFY((signed)(ifq->ifcq_drain - cnt) >= 0); - ifq->ifcq_drain -= cnt; - } - if (ALTQ_IS_ENABLED(IFCQ_ALTQ(ifq))) - ALTQ_PURGE_SC(IFCQ_ALTQ(ifq), sc, flow, a_cnt, a_len); -#endif /* PF_ALTQ */ if (!ifq_locked) IFCQ_UNLOCK(ifq); @@ -1695,16 +1682,10 @@ ifioctl_linkparams(struct ifnet *ifp, u_long cmd, caddr_t data, struct proc *p) u_int64_t tbr_bw = 0, tbr_pct = 0; IFCQ_LOCK(ifq); -#if PF_ALTQ - if (ALTQ_IS_ENABLED(IFCQ_ALTQ(ifq))) { - sched_type = IFCQ_ALTQ(ifq)->altq_type; - flags |= IFLPRF_ALTQ; - } else -#endif /* PF_ALTQ */ - { - if (IFCQ_IS_ENABLED(ifq)) - sched_type = ifq->ifcq_type; - } + + if (IFCQ_IS_ENABLED(ifq)) + sched_type = ifq->ifcq_type; + bcopy(&sched_type, &iflpr->iflpr_output_sched, sizeof (iflpr->iflpr_output_sched)); @@ -1843,6 +1824,116 @@ ifioctl_getnetagents(struct ifnet *ifp, u_int32_t *count, user_addr_t uuid_p) #define IF_MAXAGENTS 64 #define IF_AGENT_INCREMENT 8 +static int +if_add_netagent_locked(struct ifnet *ifp, uuid_t new_agent_uuid) +{ + uuid_t *first_empty_slot = NULL; + u_int32_t index = 0; + bool already_added = FALSE; + + if (ifp->if_agentids != NULL) { + for (index = 0; index < ifp->if_agentcount; index++) { + uuid_t *netagent_uuid = &(ifp->if_agentids[index]); + if (uuid_compare(*netagent_uuid, new_agent_uuid) == 0) { + /* Already present, ignore */ + already_added = TRUE; + break; + } + if (first_empty_slot == NULL && + uuid_is_null(*netagent_uuid)) { + first_empty_slot = netagent_uuid; + } + } + } + if (already_added) { + /* Already added agent, don't return an error */ + return (0); + } + if (first_empty_slot == NULL) { + if (ifp->if_agentcount >= IF_MAXAGENTS) { + /* No room for another netagent UUID, bail */ + return (ENOMEM); + } else { + /* Calculate new array size */ + u_int32_t new_agent_count = + MIN(ifp->if_agentcount + IF_AGENT_INCREMENT, + IF_MAXAGENTS); + + /* Reallocate array */ + uuid_t *new_agent_array = _REALLOC(ifp->if_agentids, + sizeof(uuid_t) * new_agent_count, M_NETAGENT, + M_WAITOK | M_ZERO); + if (new_agent_array == NULL) { + return (ENOMEM); + } + + /* Save new array */ + ifp->if_agentids = new_agent_array; + + /* Set first empty slot */ + first_empty_slot = + &(ifp->if_agentids[ifp->if_agentcount]); + + /* Save new array length */ + ifp->if_agentcount = new_agent_count; + } + } + uuid_copy(*first_empty_slot, new_agent_uuid); + netagent_post_updated_interfaces(new_agent_uuid); + return (0); +} + +int +if_add_netagent(struct ifnet *ifp, uuid_t new_agent_uuid) +{ + VERIFY(ifp != NULL); + + ifnet_lock_exclusive(ifp); + + int error = if_add_netagent_locked(ifp, new_agent_uuid); + + ifnet_lock_done(ifp); + + return (error); +} + +static int +if_delete_netagent_locked(struct ifnet *ifp, uuid_t remove_agent_uuid) +{ + u_int32_t index = 0; + bool removed_agent_id = FALSE; + + if (ifp->if_agentids != NULL) { + for (index = 0; index < ifp->if_agentcount; index++) { + uuid_t *netagent_uuid = &(ifp->if_agentids[index]); + if (uuid_compare(*netagent_uuid, + remove_agent_uuid) == 0) { + uuid_clear(*netagent_uuid); + removed_agent_id = TRUE; + break; + } + } + } + if (removed_agent_id) + netagent_post_updated_interfaces(remove_agent_uuid); + + return (0); +} + +int +if_delete_netagent(struct ifnet *ifp, uuid_t remove_agent_uuid) +{ + VERIFY(ifp != NULL); + + ifnet_lock_exclusive(ifp); + + int error = if_delete_netagent_locked(ifp, remove_agent_uuid); + + ifnet_lock_done(ifp); + + return (error); +} + static __attribute__((noinline)) int ifioctl_netagent(struct ifnet *ifp, u_long cmd, caddr_t data, struct proc *p) { @@ -1852,7 +1943,6 @@ ifioctl_netagent(struct ifnet *ifp, u_long cmd, caddr_t data, struct proc *p) struct if_agentidsreq64 s64; } u; int error = 0; - u_int32_t index = 0; VERIFY(ifp != NULL); @@ -1870,82 +1960,19 @@ ifioctl_netagent(struct ifnet *ifp, u_long cmd, caddr_t data, struct proc *p) switch (cmd) { case SIOCAIFAGENTID: { /* struct if_agentidreq */ - uuid_t *first_empty_slot = NULL; // TODO: Use priv_check_cred() instead of root check if ((error = proc_suser(p)) != 0) { break; } - bool already_added = FALSE; - if (ifp->if_agentids != NULL) { - for (index = 0; index < ifp->if_agentcount; index++) { - uuid_t *netagent_uuid = &(ifp->if_agentids[index]); - if (uuid_compare(*netagent_uuid, ifar->ifar_uuid) == 0) { - /* Already present, ignore */ - already_added = TRUE; - break; - } - if (first_empty_slot == NULL && - uuid_is_null(*netagent_uuid)) { - first_empty_slot = netagent_uuid; - } - } - } - if (already_added) { - /* Already added agent, don't return an error */ - break; - } - if (first_empty_slot == NULL) { - if (ifp->if_agentcount >= IF_MAXAGENTS) { - /* No room for another netagent UUID, bail */ - error = ENOMEM; - break; - } else { - /* Calculate new array size */ - u_int32_t new_agent_count = - MIN(ifp->if_agentcount + IF_AGENT_INCREMENT, IF_MAXAGENTS); - - /* Reallocate array */ - uuid_t *new_agent_array = _REALLOC(ifp->if_agentids, - sizeof(uuid_t) * new_agent_count, M_NETAGENT, - M_WAITOK | M_ZERO); - if (new_agent_array == NULL) { - error = ENOMEM; - break; - } - - /* Save new array */ - ifp->if_agentids = new_agent_array; - - /* Set first empty slot */ - first_empty_slot = &(ifp->if_agentids[ifp->if_agentcount]); - - /* Save new array length */ - ifp->if_agentcount = new_agent_count; - } - } - uuid_copy(*first_empty_slot, ifar->ifar_uuid); - netagent_post_updated_interfaces(ifar->ifar_uuid); + error = if_add_netagent_locked(ifp, ifar->ifar_uuid); break; } case SIOCDIFAGENTID: { /* struct if_agentidreq */ - bool removed_agent_id = FALSE; // TODO: Use priv_check_cred() instead of root check if ((error = proc_suser(p)) != 0) { break; } - if (ifp->if_agentids != NULL) { - for (index = 0; index < ifp->if_agentcount; index++) { - uuid_t *netagent_uuid = &(ifp->if_agentids[index]); - if (uuid_compare(*netagent_uuid, ifar->ifar_uuid) == 0) { - uuid_clear(*netagent_uuid); - removed_agent_id = TRUE; - break; - } - } - } - if (removed_agent_id) { - netagent_post_updated_interfaces(ifar->ifar_uuid); - } + error = if_delete_netagent_locked(ifp, ifar->ifar_uuid); break; } case SIOCGIFAGENTIDS32: { /* struct if_agentidsreq32 */ @@ -2231,6 +2258,34 @@ ifioctl_netsignature(struct ifnet *ifp, u_long cmd, caddr_t data) return (error); } +#if INET6 +static __attribute__((noinline)) int +ifioctl_nat64prefix(struct ifnet *ifp, u_long cmd, caddr_t data) +{ + struct if_nat64req *ifnat64 = (struct if_nat64req *)(void *)data; + int error = 0; + + VERIFY(ifp != NULL); + + switch (cmd) { + case SIOCSIFNAT64PREFIX: /* struct if_nat64req */ + error = ifnet_set_nat64prefix(ifp, ifnat64->ifnat64_prefixes); + break; + + case SIOCGIFNAT64PREFIX: /* struct if_nat64req */ + error = ifnet_get_nat64prefix(ifp, ifnat64->ifnat64_prefixes); + break; + + default: + VERIFY(0); + /* NOTREACHED */ + } + + return (error); +} +#endif + + /* * Interface ioctls. * @@ -2351,19 +2406,13 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct proc *p) case SIOCSIFDISABLEOUTPUT: /* struct ifreq */ #endif /* (DEBUG || DEVELOPMENT) */ case SIOCGECNMODE: /* struct ifreq */ - case SIOCSECNMODE: { /* struct ifreq */ - struct ifreq ifr; - bcopy(data, &ifr, sizeof (ifr)); - ifr.ifr_name[IFNAMSIZ - 1] = '\0'; - bcopy(&ifr.ifr_name, ifname, IFNAMSIZ); - error = ifioctl_ifreq(so, cmd, &ifr, p); - bcopy(&ifr, data, sizeof (ifr)); - goto done; - } + case SIOCSECNMODE: case SIOCSQOSMARKINGMODE: /* struct ifreq */ case SIOCSQOSMARKINGENABLED: /* struct ifreq */ case SIOCGQOSMARKINGMODE: /* struct ifreq */ case SIOCGQOSMARKINGENABLED: /* struct ifreq */ + case SIOCSIFLOWINTERNET: /* struct ifreq */ + case SIOCGIFLOWINTERNET: /* struct ifreq */ { /* struct ifreq */ struct ifreq ifr; bcopy(data, &ifr, sizeof (ifr)); @@ -2548,6 +2597,12 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct proc *p) error = ifioctl_netsignature(ifp, cmd, data); break; +#if INET6 + case SIOCSIFNAT64PREFIX: /* struct if_nsreq */ + case SIOCGIFNAT64PREFIX: /* struct if_nsreq */ + error = ifioctl_nat64prefix(ifp, cmd, data); + break; +#endif default: if (so->so_proto == NULL) { error = EOPNOTSUPP; @@ -2967,10 +3022,10 @@ ifioctl_ifreq(struct socket *so, u_long cmd, struct ifreq *ifr, struct proc *p) case SIOCGIFLINKQUALITYMETRIC: ifnet_lock_shared(ifp); if ((ifp->if_interface_state.valid_bitmask & - IF_INTERFACE_STATE_LQM_STATE_VALID)) + IF_INTERFACE_STATE_LQM_STATE_VALID)) { ifr->ifr_link_quality_metric = ifp->if_interface_state.lqm_state; - else if ((ifp->if_refflags & IFRF_ATTACHED)) { + } else if (IF_FULLY_ATTACHED(ifp)) { ifr->ifr_link_quality_metric = IFNET_LQM_THRESH_UNKNOWN; } else { @@ -3252,6 +3307,33 @@ ifioctl_ifreq(struct socket *so, u_long cmd, struct ifreq *ifr, struct proc *p) error = EINVAL; #endif /* (DEBUG || DEVELOPMENT) */ break; + case SIOCSIFLOWINTERNET: + if ((error = priv_check_cred(kauth_cred_get(), + PRIV_NET_INTERFACE_CONTROL, 0)) != 0) + return (error); + + ifnet_lock_exclusive(ifp); + if (ifr->ifr_low_internet & IFRTYPE_LOW_INTERNET_ENABLE_UL) + ifp->if_xflags |= IFXF_LOW_INTERNET_UL; + else + ifp->if_xflags &= ~(IFXF_LOW_INTERNET_UL); + if (ifr->ifr_low_internet & IFRTYPE_LOW_INTERNET_ENABLE_DL) + ifp->if_xflags |= IFXF_LOW_INTERNET_DL; + else + ifp->if_xflags &= ~(IFXF_LOW_INTERNET_DL); + ifnet_lock_done(ifp); + break; + case SIOCGIFLOWINTERNET: + ifnet_lock_shared(ifp); + ifr->ifr_low_internet = 0; + if (ifp->if_xflags & IFXF_LOW_INTERNET_UL) + ifr->ifr_low_internet |= + IFRTYPE_LOW_INTERNET_ENABLE_UL; + if (ifp->if_xflags & IFXF_LOW_INTERNET_DL) + ifr->ifr_low_internet |= + IFRTYPE_LOW_INTERNET_ENABLE_DL; + ifnet_lock_done(ifp); + break; default: VERIFY(0); /* NOTREACHED */ @@ -4827,3 +4909,62 @@ ifioctl_cassert(void) ; } } + +/* + * XXX: This API is only used by BSD stack and for now will always return 0. + * For Skywalk native drivers, preamble space need not be allocated in mbuf + * as the preamble will be reserved in the translated skywalk packet + * which is transmitted to the driver. + * For Skywalk compat drivers currently headroom is always set to zero. + */ +uint32_t +ifnet_mbuf_packetpreamblelen(struct ifnet *ifp) +{ +#pragma unused(ifp) + return (0); +} + +/* The following is used to enqueue work items for interface events */ +struct intf_event { + struct ifnet *ifp; + union sockaddr_in_4_6 addr; + uint32_t intf_event_code; +}; + +static void +intf_event_callback(void *arg) +{ + struct intf_event *p_intf_ev = (struct intf_event *)arg; + + /* Call this before we walk the tree */ + EVENTHANDLER_INVOKE(&ifnet_evhdlr_ctxt, ifnet_event, p_intf_ev->ifp, + (struct sockaddr *)&(p_intf_ev->addr), p_intf_ev->intf_event_code); +} + +struct intf_event_nwk_wq_entry { + struct nwk_wq_entry nwk_wqe; + struct intf_event intf_ev_arg; +}; + +void +intf_event_enqueue_nwk_wq_entry(struct ifnet *ifp, struct sockaddr *addrp, + uint32_t intf_event_code) +{ +#pragma unused(addrp) + struct intf_event_nwk_wq_entry *p_intf_ev = NULL; + + MALLOC(p_intf_ev, struct intf_event_nwk_wq_entry *, + sizeof(struct intf_event_nwk_wq_entry), + M_NWKWQ, M_WAITOK | M_ZERO); + + p_intf_ev->intf_ev_arg.ifp = ifp; + /* + * XXX Not using addr in the arg. This will be used + * once we need IP address add/delete events + */ + p_intf_ev->intf_ev_arg.intf_event_code = intf_event_code; + p_intf_ev->nwk_wqe.func = intf_event_callback; + p_intf_ev->nwk_wqe.is_arg_managed = TRUE; + p_intf_ev->nwk_wqe.arg = &p_intf_ev->intf_ev_arg; + nwk_wq_enqueue((struct nwk_wq_entry*)p_intf_ev); +} diff --git a/bsd/net/if.h b/bsd/net/if.h index 9d93391f4..2bb7c6655 100644 --- a/bsd/net/if.h +++ b/bsd/net/if.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -137,7 +137,6 @@ struct if_clonereq32 { #define IFEF_VLAN 0x00000200 /* interface has one or more vlans */ #define IFEF_BOND 0x00000400 /* interface is part of bond */ #define IFEF_ARPLL 0x00000800 /* ARP for IPv4LL addresses */ -/* #define IFEF_NOWINDOWSCALE 0x00001000 */ /* Don't scale TCP window on iface */ /* * XXX IFEF_NOAUTOIPV6LL is deprecated and should be done away with. * Configd pretty much manages the interface configuration. @@ -170,7 +169,12 @@ struct if_clonereq32 { * Extra flags */ #define IFXF_WAKE_ON_MAGIC_PACKET 0x00000001 /* wake on magic packet */ -#define IFXF_TIMESTAMP_ENABLED 0x00000002 /* time stamping enabled */ +#define IFXF_TIMESTAMP_ENABLED 0x00000002 /* time stamping enabled */ +#define IFXF_NX_NOAUTO 0x00000004 /* no auto config nexus */ +#define IFXF_MULTISTACK_BPF_TAP 0x00000008 /* multistack bpf tap */ +#define IFXF_LOW_INTERNET_UL 0x00000010 /* Uplink Low Internet is confirmed */ +#define IFXF_LOW_INTERNET_DL 0x00000020 /* Downlink Low Internet is confirmed */ +#define IFXF_ALLOC_KPI 0x00000040 /* Allocated via the ifnet_alloc KPI */ /* * Current requirements for an AWDL interface. Setting/clearing IFEF_AWDL @@ -217,7 +221,7 @@ struct if_clonereq32 { * contains the enabled optional features & capabilites that can be used * individually per packet and are specified in the mbuf pkthdr.csum_flags * field. IFCAP_* and IFNET_* do not match one to one and IFNET_* may be - * more detailed or differenciated than IFCAP_*. + * more detailed or differentiated than IFCAP_*. * IFNET_* hwassist flags have corresponding CSUM_* in sys/mbuf.h */ #define IFCAP_RXCSUM 0x00001 /* can offload checksum on RX */ @@ -233,13 +237,16 @@ struct if_clonereq32 { #define IFCAP_SKYWALK 0x00400 /* Skywalk mode supported/enabled */ #define IFCAP_HW_TIMESTAMP 0x00800 /* Time stamping in hardware */ #define IFCAP_SW_TIMESTAMP 0x01000 /* Time stamping in software */ +#define IFCAP_CSUM_PARTIAL 0x02000 /* can offload partial checksum */ +#define IFCAP_CSUM_ZERO_INVERT 0x04000 /* can invert 0 to -0 (0xffff) */ #define IFCAP_HWCSUM (IFCAP_RXCSUM | IFCAP_TXCSUM) #define IFCAP_TSO (IFCAP_TSO4 | IFCAP_TSO6) #define IFCAP_VALID (IFCAP_HWCSUM | IFCAP_TSO | IFCAP_LRO | IFCAP_VLAN_MTU | \ IFCAP_VLAN_HWTAGGING | IFCAP_JUMBO_MTU | IFCAP_AV | IFCAP_TXSTATUS | \ - IFCAP_SKYWALK | IFCAP_SW_TIMESTAMP | IFCAP_HW_TIMESTAMP) + IFCAP_SKYWALK | IFCAP_SW_TIMESTAMP | IFCAP_HW_TIMESTAMP | \ + IFCAP_CSUM_PARTIAL | IFCAP_CSUM_ZERO_INVERT) #define IFQ_MAXLEN 128 #define IFNET_SLOWHZ 1 /* granularity is 1 second */ @@ -460,6 +467,7 @@ struct ifreq { #define IFRTYPE_SUBFAMILY_RESERVED 5 #define IFRTYPE_SUBFAMILY_INTCOPROC 6 } ifru_type; +#endif /* PRIVATE */ u_int32_t ifru_functional_type; #define IFRTYPE_FUNCTIONAL_UNKNOWN 0 #define IFRTYPE_FUNCTIONAL_LOOPBACK 1 @@ -467,8 +475,9 @@ struct ifreq { #define IFRTYPE_FUNCTIONAL_WIFI_INFRA 3 #define IFRTYPE_FUNCTIONAL_WIFI_AWDL 4 #define IFRTYPE_FUNCTIONAL_CELLULAR 5 -#define IFRTYPE_FUNCTIONAL_INTCOPROC 6 +#define IFRTYPE_FUNCTIONAL_INTCOPROC 6 #define IFRTYPE_FUNCTIONAL_LAST 6 +#ifdef PRIVATE u_int32_t ifru_expensive; u_int32_t ifru_2kcl; struct { @@ -486,6 +495,11 @@ struct ifreq { #define IFRTYPE_QOSMARKING_FASTLANE 1 u_int32_t ifru_qosmarking_enabled; u_int32_t ifru_disable_output; + u_int32_t ifru_low_internet; +#define IFRTYPE_LOW_INTERNET_DISABLE_UL_DL 0x0000 +#define IFRTYPE_LOW_INTERNET_ENABLE_UL 0x0001 +#define IFRTYPE_LOW_INTERNET_ENABLE_DL 0x0002 + #endif /* PRIVATE */ } ifr_ifru; #define ifr_addr ifr_ifru.ifru_addr /* address */ @@ -534,6 +548,7 @@ struct ifreq { #define ifr_qosmarking_enabled ifr_ifru.ifru_qosmarking_enabled #define ifr_fastlane_enabled ifr_qosmarking_enabled #define ifr_disable_output ifr_ifru.ifru_disable_output +#define ifr_low_internet ifr_ifru.ifru_low_internet #endif /* PRIVATE */ }; @@ -696,10 +711,13 @@ struct kev_dl_proto_data { enum { IFNET_LQM_THRESH_OFF = (-2), IFNET_LQM_THRESH_UNKNOWN = (-1), - IFNET_LQM_THRESH_BAD = 10, + IFNET_LQM_THRESH_ABORT = 10, + IFNET_LQM_THRESH_MINIMALLY_VIABLE = 20, IFNET_LQM_THRESH_POOR = 50, IFNET_LQM_THRESH_GOOD = 100 }; +#define IFNET_LQM_THRESH_BAD IFNET_LQM_THRESH_ABORT + #ifdef XNU_KERNEL_PRIVATE #define IFNET_LQM_MIN IFNET_LQM_THRESH_OFF #define IFNET_LQM_MAX IFNET_LQM_THRESH_GOOD @@ -864,6 +882,17 @@ struct if_agentidsreq { uuid_t *ifar_uuids; /* array of agent UUIDs */ }; +/* + * Structure for SIOCGIFNEXUS + */ +struct if_nexusreq { + char ifnr_name[IFNAMSIZ]; /* interface name */ + uint64_t ifnr_flags; /* unused, must be zero */ + uuid_t ifnr_netif; /* netif nexus instance UUID */ + uuid_t ifnr_multistack; /* multistack nexus UUID */ + uint64_t ifnr_reserved[5]; +}; + #ifdef BSD_KERNEL_PRIVATE struct if_agentidsreq32 { char ifar_name[IFNAMSIZ]; @@ -914,6 +943,30 @@ struct if_nsreq { u_int8_t ifnsr_data[IFNET_SIGNATURELEN]; }; + +#define NAT64_PREFIX_LEN_32 4 +#define NAT64_PREFIX_LEN_40 5 +#define NAT64_PREFIX_LEN_48 6 +#define NAT64_PREFIX_LEN_56 7 +#define NAT64_PREFIX_LEN_64 8 +#define NAT64_PREFIX_LEN_96 12 +#define NAT64_PREFIX_LEN_MAX NAT64_PREFIX_LEN_96 + +#define NAT64_MAX_NUM_PREFIXES 4 + +struct ipv6_prefix { + struct in6_addr ipv6_prefix; + uint32_t prefix_len; +}; + +/* + * Structure for SIOC[S/G]IFNAT64PREFIX + */ +struct if_nat64req { + char ifnat64_name[IFNAMSIZ]; + struct ipv6_prefix ifnat64_prefixes[NAT64_MAX_NUM_PREFIXES]; +}; + /* * Structure for SIOC[S/G]IFORDER * diff --git a/bsd/net/if_bond.c b/bsd/net/if_bond.c index 34f6e03d4..202f6c092 100644 --- a/bsd/net/if_bond.c +++ b/bsd/net/if_bond.c @@ -134,14 +134,14 @@ bond_lock_init(void) static __inline__ void bond_assert_lock_held(void) { - lck_mtx_assert(bond_lck_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(bond_lck_mtx, LCK_MTX_ASSERT_OWNED); return; } static __inline__ void bond_assert_lock_not_held(void) { - lck_mtx_assert(bond_lck_mtx, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(bond_lck_mtx, LCK_MTX_ASSERT_NOTOWNED); return; } diff --git a/bsd/net/if_bridge.c b/bsd/net/if_bridge.c index 9a2b34f3d..76de0c2bb 100644 --- a/bsd/net/if_bridge.c +++ b/bsd/net/if_bridge.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2015 Apple Inc. All rights reserved. + * Copyright (c) 2004-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -139,6 +139,7 @@ #include <net/if_types.h> #include <net/if_var.h> #include <net/if_media.h> +#include <net/net_api_stats.h> #include <netinet/in.h> /* for struct arpcom */ #include <netinet/in_systm.h> @@ -175,10 +176,11 @@ #include <netinet/bootp.h> #include <netinet/dhcp.h> + #if BRIDGE_DEBUG #define BR_DBGF_LIFECYCLE 0x0001 #define BR_DBGF_INPUT 0x0002 -#define BR_DBGF_OUTPPUT 0x0004 +#define BR_DBGF_OUTPUT 0x0004 #define BR_DBGF_RT_TABLE 0x0008 #define BR_DBGF_DELAYED_CALL 0x0010 #define BR_DBGF_IOCTL 0x0020 @@ -190,9 +192,9 @@ #define _BRIDGE_LOCK(_sc) lck_mtx_lock(&(_sc)->sc_mtx) #define _BRIDGE_UNLOCK(_sc) lck_mtx_unlock(&(_sc)->sc_mtx) #define BRIDGE_LOCK_ASSERT_HELD(_sc) \ - lck_mtx_assert(&(_sc)->sc_mtx, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_sc)->sc_mtx, LCK_MTX_ASSERT_OWNED) #define BRIDGE_LOCK_ASSERT_NOTHELD(_sc) \ - lck_mtx_assert(&(_sc)->sc_mtx, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_sc)->sc_mtx, LCK_MTX_ASSERT_NOTOWNED) #if BRIDGE_DEBUG @@ -353,14 +355,42 @@ struct bridge_delayed_call { #define BDCF_OUTSTANDING 0x01 /* Delayed call has been scheduled */ #define BDCF_CANCELLING 0x02 /* May be waiting for call completion */ + /* * Software state for each bridge. */ - LIST_HEAD(_bridge_rtnode_list, bridge_rtnode); +typedef struct { + struct _bridge_rtnode_list *bb_rthash; /* our forwarding table */ + struct _bridge_rtnode_list bb_rtlist; /* list version of above */ + uint32_t bb_rthash_key; /* key for hash */ + uint32_t bb_rthash_size; /* size of the hash table */ + struct bridge_delayed_call bb_aging_timer; + struct bridge_delayed_call bb_resize_call; + TAILQ_HEAD(, bridge_iflist) bb_spanlist; /* span ports list */ + struct bstp_state bb_stp; /* STP state */ + bpf_packet_func bb_bpf_input; + bpf_packet_func bb_bpf_output; +} bridge_bsd, *bridge_bsd_t; + +#define sc_rthash sc_u.scu_bsd.bb_rthash +#define sc_rtlist sc_u.scu_bsd.bb_rtlist +#define sc_rthash_key sc_u.scu_bsd.bb_rthash_key +#define sc_rthash_size sc_u.scu_bsd.bb_rthash_size +#define sc_aging_timer sc_u.scu_bsd.bb_aging_timer +#define sc_resize_call sc_u.scu_bsd.bb_resize_call +#define sc_spanlist sc_u.scu_bsd.bb_spanlist +#define sc_stp sc_u.scu_bsd.bb_stp +#define sc_bpf_input sc_u.scu_bsd.bb_bpf_input +#define sc_bpf_output sc_u.scu_bsd.bb_bpf_output + struct bridge_softc { struct ifnet *sc_ifp; /* make this an interface */ + u_int32_t sc_flags; + union { + bridge_bsd scu_bsd; + } sc_u; LIST_ENTRY(bridge_softc) sc_list; decl_lck_mtx_data(, sc_mtx); void *sc_cv; @@ -370,23 +400,11 @@ struct bridge_softc { uint32_t sc_iflist_ref; /* refcount for sc_iflist */ uint32_t sc_iflist_xcnt; /* refcount for sc_iflist */ TAILQ_HEAD(, bridge_iflist) sc_iflist; /* member interface list */ - struct _bridge_rtnode_list *sc_rthash; /* our forwarding table */ - struct _bridge_rtnode_list sc_rtlist; /* list version of above */ - uint32_t sc_rthash_key; /* key for hash */ - uint32_t sc_rthash_size; /* size of the hash table */ - TAILQ_HEAD(, bridge_iflist) sc_spanlist; /* span ports list */ - struct bstp_state sc_stp; /* STP state */ uint32_t sc_brtexceeded; /* # of cache drops */ uint32_t sc_filter_flags; /* ipf and flags */ struct ifnet *sc_ifaddr; /* member mac copied from */ u_char sc_defaddr[6]; /* Default MAC address */ - char sc_if_xname[IFNAMSIZ]; - bpf_packet_func sc_bpf_input; - bpf_packet_func sc_bpf_output; - u_int32_t sc_flags; - struct bridge_delayed_call sc_aging_timer; - struct bridge_delayed_call sc_resize_call; #if BRIDGE_DEBUG /* @@ -402,6 +420,19 @@ struct bridge_softc { #define SCF_DETACHING 0x01 #define SCF_RESIZING 0x02 #define SCF_MEDIA_ACTIVE 0x04 +#define SCF_BSD_MODE 0x08 + +static inline void +bridge_set_bsd_mode(struct bridge_softc * sc) +{ + sc->sc_flags |= SCF_BSD_MODE; +} + +static inline boolean_t +bridge_in_bsd_mode(const struct bridge_softc * sc) +{ + return ((sc->sc_flags & SCF_BSD_MODE) != 0); +} struct bridge_hostfilter_stats bridge_hostfilter_stats; @@ -555,6 +586,7 @@ static void bridge_cancel_delayed_call(struct bridge_delayed_call *); static void bridge_cleanup_delayed_call(struct bridge_delayed_call *); static int bridge_host_filter(struct bridge_iflist *, struct mbuf *); + #define m_copypacket(m, how) m_copym(m, 0, M_COPYALL, how) /* The default bridge vlan is 1 (IEEE 802.1Q-2003 Table 9-2) */ @@ -602,6 +634,14 @@ SYSCTL_INT(_net_link_bridge, OID_AUTO, delayed_callback_delay, "Delay before calling delayed function"); #endif +static int bridge_bsd_mode = 1; +#if (DEVELOPMENT || DEBUG) +SYSCTL_INT(_net_link_bridge, OID_AUTO, bsd_mode, + CTLFLAG_RW|CTLFLAG_LOCKED, + &bridge_bsd_mode, 0, + "Bridge using bsd mode"); +#endif /* (DEVELOPMENT || DEBUG) */ + SYSCTL_STRUCT(_net_link_bridge, OID_AUTO, hostfilterstats, CTLFLAG_RD | CTLFLAG_LOCKED, &bridge_hostfilter_stats, bridge_hostfilter_stats, ""); @@ -863,7 +903,7 @@ static void printf_ether_header(struct ether_header *); static void printf_mbuf_data(mbuf_t, size_t, size_t); static void printf_mbuf_pkthdr(mbuf_t, const char *, const char *); static void printf_mbuf(mbuf_t, const char *, const char *); -static void link_print(struct sockaddr_dl *); +static void link_print(struct bridge_softc * sc); static void bridge_lock(struct bridge_softc *); static void bridge_unlock(struct bridge_softc *); @@ -1047,18 +1087,29 @@ printf_ether_header(struct ether_header *eh) } static void -link_print(struct sockaddr_dl *dl_p) +link_print(struct bridge_softc * sc) { int i; + uint32_t sdl_buffer[offsetof(struct sockaddr_dl, sdl_data) + + IFNAMSIZ + ETHER_ADDR_LEN]; + struct sockaddr_dl *sdl = (struct sockaddr_dl *)sdl_buffer; + + memset(sdl, 0, sizeof (sdl_buffer)); + sdl->sdl_family = AF_LINK; + sdl->sdl_nlen = strlen(sc->sc_if_xname); + sdl->sdl_alen = ETHER_ADDR_LEN; + sdl->sdl_len = offsetof(struct sockaddr_dl, sdl_data); + memcpy(sdl->sdl_data, sc->sc_if_xname, sdl->sdl_nlen); + memcpy(LLADDR(sdl), sc->sc_defaddr, ETHER_ADDR_LEN); #if 1 printf("sdl len %d index %d family %d type 0x%x nlen %d alen %d" - " slen %d addr ", dl_p->sdl_len, dl_p->sdl_index, - dl_p->sdl_family, dl_p->sdl_type, dl_p->sdl_nlen, - dl_p->sdl_alen, dl_p->sdl_slen); + " slen %d addr ", sdl->sdl_len, sdl->sdl_index, + sdl->sdl_family, sdl->sdl_type, sdl->sdl_nlen, + sdl->sdl_alen, sdl->sdl_slen); #endif - for (i = 0; i < dl_p->sdl_alen; i++) - printf("%s%x", i ? ":" : "", (CONST_LLADDR(dl_p))[i]); + for (i = 0; i < sdl->sdl_alen; i++) + printf("%s%x", i ? ":" : "", (CONST_LLADDR(sdl))[i]); printf("\n"); } @@ -1150,6 +1201,38 @@ SYSCTL_PROC(_net_link_bridge, OID_AUTO, ipfw, CTLTYPE_INT|CTLFLAG_RW, &pfil_ipfw, 0, &sysctl_pfil_ipfw, "I", "Layer2 filter with IPFW"); #endif /* PFIL_HOOKS */ +static errno_t +bridge_ifnet_set_attrs(struct ifnet * ifp) +{ + errno_t error; + + error = ifnet_set_mtu(ifp, ETHERMTU); + if (error != 0) { + printf("%s: ifnet_set_mtu failed %d\n", __func__, error); + goto done; + } + error = ifnet_set_addrlen(ifp, ETHER_ADDR_LEN); + if (error != 0) { + printf("%s: ifnet_set_addrlen failed %d\n", __func__, error); + goto done; + } + error = ifnet_set_hdrlen(ifp, ETHER_HDR_LEN); + if (error != 0) { + printf("%s: ifnet_set_hdrlen failed %d\n", __func__, error); + goto done; + } + error = ifnet_set_flags(ifp, + IFF_BROADCAST | IFF_SIMPLEX | IFF_NOTRAILERS | IFF_MULTICAST, + 0xffff); + + if (error != 0) { + printf("%s: ifnet_set_flags failed %d\n", __func__, error); + goto done; + } + done: + return (error); +} + /* * bridge_clone_create: * @@ -1163,9 +1246,6 @@ bridge_clone_create(struct if_clone *ifc, uint32_t unit, void *params) struct bridge_softc *sc, *sc2; struct ifnet_init_eparams init_params; errno_t error = 0; - uint32_t sdl_buffer[offsetof(struct sockaddr_dl, sdl_data) + - IFNAMSIZ + ETHER_ADDR_LEN]; - struct sockaddr_dl *sdl = (struct sockaddr_dl *)sdl_buffer; uint8_t eth_hostid[ETHER_ADDR_LEN]; int fb, retry, has_hostid; @@ -1184,15 +1264,11 @@ bridge_clone_create(struct if_clone *ifc, uint32_t unit, void *params) sc->sc_filter_flags &= ~IFBF_FILT_USEIPF; #endif - /* Initialize our routing table. */ - error = bridge_rtable_init(sc); - if (error != 0) { - printf("%s: bridge_rtable_init failed %d\n", __func__, error); - goto done; + if (bridge_bsd_mode != 0) { + bridge_set_bsd_mode(sc); } TAILQ_INIT(&sc->sc_iflist); - TAILQ_INIT(&sc->sc_spanlist); /* use the interface name as the unique id for ifp recycle */ snprintf(sc->sc_if_xname, sizeof (sc->sc_if_xname), "%s%d", @@ -1200,11 +1276,22 @@ bridge_clone_create(struct if_clone *ifc, uint32_t unit, void *params) bzero(&init_params, sizeof (init_params)); init_params.ver = IFNET_INIT_CURRENT_VERSION; init_params.len = sizeof (init_params); - if (if_bridge_txstart) { - init_params.start = bridge_start; - } else { - init_params.flags = IFNET_INIT_LEGACY; - init_params.output = bridge_output; + if (bridge_in_bsd_mode(sc)) { + /* Initialize our routing table. */ + error = bridge_rtable_init(sc); + if (error != 0) { + printf("%s: bridge_rtable_init failed %d\n", + __func__, error); + goto done; + } + TAILQ_INIT(&sc->sc_spanlist); + if (if_bridge_txstart) { + init_params.start = bridge_start; + } else { + init_params.flags = IFNET_INIT_LEGACY; + init_params.output = bridge_output; + } + init_params.set_bpf_tap = bridge_set_bpf_tap; } init_params.uniqueid = sc->sc_if_xname; init_params.uniqueid_len = strlen(sc->sc_if_xname); @@ -1220,38 +1307,24 @@ bridge_clone_create(struct if_clone *ifc, uint32_t unit, void *params) init_params.framer_extended = ether_frameout_extended; init_params.softc = sc; init_params.ioctl = bridge_ioctl; - init_params.set_bpf_tap = bridge_set_bpf_tap; init_params.detach = bridge_detach; init_params.broadcast_addr = etherbroadcastaddr; init_params.broadcast_len = ETHER_ADDR_LEN; - error = ifnet_allocate_extended(&init_params, &ifp); - if (error != 0) { - printf("%s: ifnet_allocate failed %d\n", __func__, error); - goto done; - } - sc->sc_ifp = ifp; - error = ifnet_set_mtu(ifp, ETHERMTU); - if (error != 0) { - printf("%s: ifnet_set_mtu failed %d\n", __func__, error); - goto done; - } - error = ifnet_set_addrlen(ifp, ETHER_ADDR_LEN); - if (error != 0) { - printf("%s: ifnet_set_addrlen failed %d\n", __func__, error); - goto done; - } - error = ifnet_set_hdrlen(ifp, ETHER_HDR_LEN); - if (error != 0) { - printf("%s: ifnet_set_hdrlen failed %d\n", __func__, error); - goto done; - } - error = ifnet_set_flags(ifp, - IFF_BROADCAST | IFF_SIMPLEX | IFF_NOTRAILERS | IFF_MULTICAST, - 0xffff); - if (error != 0) { - printf("%s: ifnet_set_flags failed %d\n", __func__, error); - goto done; + if (bridge_in_bsd_mode(sc)) { + error = ifnet_allocate_extended(&init_params, &ifp); + if (error != 0) { + printf("%s: ifnet_allocate failed %d\n", + __func__, error); + goto done; + } + sc->sc_ifp = ifp; + error = bridge_ifnet_set_attrs(ifp); + if (error != 0) { + printf("%s: bridge_ifnet_set_attrs failed %d\n", + __func__, error); + goto done; + } } /* @@ -1260,7 +1333,7 @@ bridge_clone_create(struct if_clone *ifc, uint32_t unit, void *params) * Since we are using random ethernet addresses for the bridge, it is * possible that we might have address collisions, so make sure that * this hardware address isn't already in use on another bridge. - * The first try uses the "hostid" and falls back to read_random(); + * The first try uses the "hostid" and falls back to read_frandom(); * for "hostid", we use the MAC address of the first-encountered * Ethernet-type interface that is currently configured. */ @@ -1268,7 +1341,7 @@ bridge_clone_create(struct if_clone *ifc, uint32_t unit, void *params) has_hostid = (uuid_get_ethernet(ð_hostid[0]) == 0); for (retry = 1; retry != 0; ) { if (fb || has_hostid == 0) { - read_random(&sc->sc_defaddr, ETHER_ADDR_LEN); + read_frandom(&sc->sc_defaddr, ETHER_ADDR_LEN); sc->sc_defaddr[0] &= ~1; /* clear multicast bit */ sc->sc_defaddr[0] |= 2; /* set the LAA bit */ } else { @@ -1301,25 +1374,18 @@ bridge_clone_create(struct if_clone *ifc, uint32_t unit, void *params) lck_mtx_unlock(&bridge_list_mtx); } - memset(sdl, 0, sizeof (sdl_buffer)); - sdl->sdl_family = AF_LINK; - sdl->sdl_nlen = strlen(sc->sc_if_xname); - sdl->sdl_alen = ETHER_ADDR_LEN; - sdl->sdl_len = offsetof(struct sockaddr_dl, sdl_data); - memcpy(sdl->sdl_data, sc->sc_if_xname, sdl->sdl_nlen); - memcpy(LLADDR(sdl), sc->sc_defaddr, ETHER_ADDR_LEN); - sc->sc_flags &= ~SCF_MEDIA_ACTIVE; #if BRIDGE_DEBUG if (if_bridge_debug & BR_DBGF_LIFECYCLE) - link_print(sdl); + link_print(sc); #endif - - error = ifnet_attach(ifp, NULL); - if (error != 0) { - printf("%s: ifnet_attach failed %d\n", __func__, error); - goto done; + if (bridge_in_bsd_mode(sc)) { + error = ifnet_attach(ifp, NULL); + if (error != 0) { + printf("%s: ifnet_attach failed %d\n", __func__, error); + goto done; + } } error = ifnet_set_lladdr_and_type(ifp, sc->sc_defaddr, ETHER_ADDR_LEN, @@ -1330,19 +1396,20 @@ bridge_clone_create(struct if_clone *ifc, uint32_t unit, void *params) goto done; } - ifnet_set_offload(ifp, - IFNET_CSUM_IP | IFNET_CSUM_TCP | IFNET_CSUM_UDP | - IFNET_CSUM_TCPIPV6 | IFNET_CSUM_UDPIPV6 | IFNET_MULTIPAGES); - - error = bridge_set_tso(sc); - if (error != 0) { - printf("%s: bridge_set_tso failed %d\n", __func__, error); - goto done; - } - + if (bridge_in_bsd_mode(sc)) { + ifnet_set_offload(ifp, + IFNET_CSUM_IP | IFNET_CSUM_TCP | IFNET_CSUM_UDP | + IFNET_CSUM_TCPIPV6 | IFNET_CSUM_UDPIPV6 | IFNET_MULTIPAGES); + error = bridge_set_tso(sc); + if (error != 0) { + printf("%s: bridge_set_tso failed %d\n", + __func__, error); + goto done; + } #if BRIDGESTP - bstp_attach(&sc->sc_stp, &bridge_ops); + bstp_attach(&sc->sc_stp, &bridge_ops); #endif /* BRIDGESTP */ + } lck_mtx_lock(&bridge_list_mtx); LIST_INSERT_HEAD(&bridge_list, sc, sc_list); @@ -1382,10 +1449,12 @@ bridge_clone_destroy(struct ifnet *ifp) bridge_ifstop(ifp, 1); - bridge_cancel_delayed_call(&sc->sc_resize_call); + if (bridge_in_bsd_mode(sc)) { + bridge_cancel_delayed_call(&sc->sc_resize_call); - bridge_cleanup_delayed_call(&sc->sc_resize_call); - bridge_cleanup_delayed_call(&sc->sc_aging_timer); + bridge_cleanup_delayed_call(&sc->sc_resize_call); + bridge_cleanup_delayed_call(&sc->sc_aging_timer); + } error = ifnet_set_flags(ifp, 0, IFF_UP); if (error != 0) { @@ -1395,24 +1464,18 @@ bridge_clone_destroy(struct ifnet *ifp) while ((bif = TAILQ_FIRST(&sc->sc_iflist)) != NULL) bridge_delete_member(sc, bif, 0); - while ((bif = TAILQ_FIRST(&sc->sc_spanlist)) != NULL) { - bridge_delete_span(sc, bif); + if (bridge_in_bsd_mode(sc)) { + while ((bif = TAILQ_FIRST(&sc->sc_spanlist)) != NULL) { + bridge_delete_span(sc, bif); + } + BRIDGE_UNLOCK(sc); } - BRIDGE_UNLOCK(sc); - error = ifnet_detach(ifp); if (error != 0) { - panic("bridge_clone_destroy: ifnet_detach(%p) failed %d\n", - ifp, error); - if ((sc = (struct bridge_softc *)ifnet_softc(ifp)) != NULL) { - BRIDGE_LOCK(sc); - sc->sc_flags &= ~SCF_DETACHING; - BRIDGE_UNLOCK(sc); - } - return (0); + panic("%s: ifnet_detach(%p) failed %d\n", + __func__, ifp, error); } - return (0); } @@ -1915,7 +1978,7 @@ bridge_iff_output(void *cookie, ifnet_t ifp, protocol_family_t protocol, goto out; #if BRIDGE_DEBUG - if (if_bridge_debug & BR_DBGF_OUTPPUT) { + if (if_bridge_debug & BR_DBGF_OUTPUT) { printf("%s: %s from %s m 0x%llx data 0x%llx\n", __func__, sc->sc_ifp->if_xname, ifp->if_xname, (uint64_t)VM_KERNEL_ADDRPERM(m), @@ -2083,10 +2146,13 @@ bridge_delete_member(struct bridge_softc *sc, struct bridge_iflist *bif, int lladdr_changed = 0, error, filt_attached; uint8_t eaddr[ETHER_ADDR_LEN]; u_int32_t event_code = 0; + boolean_t bsd_mode; BRIDGE_LOCK_ASSERT_HELD(sc); VERIFY(ifs != NULL); + bsd_mode = bridge_in_bsd_mode(sc); + /* * First, remove the member from the list first so it cannot be found anymore * when we release the bridge lock below @@ -2134,8 +2200,9 @@ bridge_delete_member(struct bridge_softc *sc, struct bridge_iflist *bif, BRIDGE_LOCK(sc); } #if BRIDGESTP - if (bif->bif_ifflags & IFBIF_STP) + if (bsd_mode && (bif->bif_ifflags & IFBIF_STP) != 0) { bstp_disable(&bif->bif_stp); + } #endif /* BRIDGESTP */ /* @@ -2167,7 +2234,10 @@ bridge_delete_member(struct bridge_softc *sc, struct bridge_iflist *bif, printf("%s: bridge_set_tso failed %d\n", __func__, error); } - bridge_rtdelete(sc, ifs, IFBF_FLUSHALL); + if (bsd_mode) { + bridge_rtdelete(sc, ifs, IFBF_FLUSHALL); + } + KASSERT(bif->bif_addrcnt == 0, ("%s: %d bridge routes referenced", __func__, bif->bif_addrcnt)); @@ -2178,7 +2248,9 @@ bridge_delete_member(struct bridge_softc *sc, struct bridge_iflist *bif, */ event_code = bridge_updatelinkstatus(sc); - BRIDGE_UNLOCK(sc); + if (bsd_mode) { + BRIDGE_UNLOCK(sc); + } if (lladdr_changed && (error = ifnet_set_lladdr(bifp, eaddr, ETHER_ADDR_LEN)) != 0) @@ -2188,7 +2260,9 @@ bridge_delete_member(struct bridge_softc *sc, struct bridge_iflist *bif, bridge_link_event(bifp, event_code); #if BRIDGESTP - bstp_destroy(&bif->bif_stp); /* prepare to free */ + if (bsd_mode) { + bstp_destroy(&bif->bif_stp); /* prepare to free */ + } #endif /* BRIDGESTP */ if (filt_attached) @@ -2231,6 +2305,7 @@ bridge_ioctl_add(struct bridge_softc *sc, void *arg) uint8_t eaddr[ETHER_ADDR_LEN]; struct iff_filter iff; u_int32_t event_code = 0; + boolean_t bsd_mode = bridge_in_bsd_mode(sc); ifs = ifunit(req->ifbr_ifsname); if (ifs == NULL) @@ -2238,10 +2313,12 @@ bridge_ioctl_add(struct bridge_softc *sc, void *arg) if (ifs->if_ioctl == NULL) /* must be supported */ return (EINVAL); - /* If it's in the span list, it can't be a member. */ - TAILQ_FOREACH(bif, &sc->sc_spanlist, bif_next) - if (ifs == bif->bif_ifp) - return (EBUSY); + if (bsd_mode) { + /* If it's in the span list, it can't be a member. */ + TAILQ_FOREACH(bif, &sc->sc_spanlist, bif_next) + if (ifs == bif->bif_ifp) + return (EBUSY); + } if (ifs->if_bridge == sc) return (EEXIST); @@ -2261,7 +2338,7 @@ bridge_ioctl_add(struct bridge_softc *sc, void *arg) return (EINVAL); } - bif = _MALLOC(sizeof (*bif), M_DEVBUF, M_NOWAIT | M_ZERO); + bif = _MALLOC(sizeof (*bif), M_DEVBUF, M_WAITOK | M_ZERO); if (bif == NULL) return (ENOMEM); @@ -2298,7 +2375,9 @@ bridge_ioctl_add(struct bridge_softc *sc, void *arg) ifs->if_bridge = sc; #if BRIDGESTP - bstp_create(&sc->sc_stp, &bif->bif_stp, bif->bif_ifp); + if (bsd_mode) { + bstp_create(&sc->sc_stp, &bif->bif_stp, bif->bif_ifp); + } #endif /* BRIDGESTP */ /* @@ -2348,7 +2427,9 @@ bridge_ioctl_add(struct bridge_softc *sc, void *arg) /* * Respect lock ordering with DLIL lock for the following operations */ - BRIDGE_UNLOCK(sc); + if (bsd_mode) { + BRIDGE_UNLOCK(sc); + } /* * install an interface filter @@ -2356,13 +2437,16 @@ bridge_ioctl_add(struct bridge_softc *sc, void *arg) memset(&iff, 0, sizeof (struct iff_filter)); iff.iff_cookie = bif; iff.iff_name = "com.apple.kernel.bsd.net.if_bridge"; - iff.iff_input = bridge_iff_input; + if (bsd_mode) { + iff.iff_input = bridge_iff_input; #if BRIDGE_MEMBER_OUT_FILTER - iff.iff_output = bridge_iff_output; + iff.iff_output = bridge_iff_output; #endif /* BRIDGE_MEMBER_OUT_FILTER */ + } iff.iff_event = bridge_iff_event; iff.iff_detached = bridge_iff_detached; - error = dlil_attach_filter(ifs, &iff, &bif->bif_iff_ref, DLIL_IFF_TSO); + error = dlil_attach_filter(ifs, &iff, &bif->bif_iff_ref, + DLIL_IFF_TSO | DLIL_IFF_INTERNAL); if (error != 0) { printf("%s: iflt_attach failed %d\n", __func__, error); BRIDGE_LOCK(sc); @@ -2426,38 +2510,41 @@ bridge_ioctl_gifflags(struct bridge_softc *sc, void *arg) { struct ifbreq *req = arg; struct bridge_iflist *bif; - struct bstp_port *bp; bif = bridge_lookup_member(sc, req->ifbr_ifsname); if (bif == NULL) return (ENOENT); - bp = &bif->bif_stp; + if (bridge_in_bsd_mode(sc)) { + struct bstp_port *bp; + + bp = &bif->bif_stp; + req->ifbr_state = bp->bp_state; + req->ifbr_priority = bp->bp_priority; + req->ifbr_path_cost = bp->bp_path_cost; + req->ifbr_proto = bp->bp_protover; + req->ifbr_role = bp->bp_role; + req->ifbr_stpflags = bp->bp_flags; + /* Copy STP state options as flags */ + if (bp->bp_operedge) + req->ifbr_ifsflags |= IFBIF_BSTP_EDGE; + if (bp->bp_flags & BSTP_PORT_AUTOEDGE) + req->ifbr_ifsflags |= IFBIF_BSTP_AUTOEDGE; + if (bp->bp_ptp_link) + req->ifbr_ifsflags |= IFBIF_BSTP_PTP; + if (bp->bp_flags & BSTP_PORT_AUTOPTP) + req->ifbr_ifsflags |= IFBIF_BSTP_AUTOPTP; + if (bp->bp_flags & BSTP_PORT_ADMEDGE) + req->ifbr_ifsflags |= IFBIF_BSTP_ADMEDGE; + if (bp->bp_flags & BSTP_PORT_ADMCOST) + req->ifbr_ifsflags |= IFBIF_BSTP_ADMCOST; + } req->ifbr_ifsflags = bif->bif_ifflags; - req->ifbr_state = bp->bp_state; - req->ifbr_priority = bp->bp_priority; - req->ifbr_path_cost = bp->bp_path_cost; req->ifbr_portno = bif->bif_ifp->if_index & 0xfff; - req->ifbr_proto = bp->bp_protover; - req->ifbr_role = bp->bp_role; - req->ifbr_stpflags = bp->bp_flags; req->ifbr_addrcnt = bif->bif_addrcnt; req->ifbr_addrmax = bif->bif_addrmax; req->ifbr_addrexceeded = bif->bif_addrexceeded; - /* Copy STP state options as flags */ - if (bp->bp_operedge) - req->ifbr_ifsflags |= IFBIF_BSTP_EDGE; - if (bp->bp_flags & BSTP_PORT_AUTOEDGE) - req->ifbr_ifsflags |= IFBIF_BSTP_AUTOEDGE; - if (bp->bp_ptp_link) - req->ifbr_ifsflags |= IFBIF_BSTP_PTP; - if (bp->bp_flags & BSTP_PORT_AUTOPTP) - req->ifbr_ifsflags |= IFBIF_BSTP_AUTOPTP; - if (bp->bp_flags & BSTP_PORT_ADMEDGE) - req->ifbr_ifsflags |= IFBIF_BSTP_ADMEDGE; - if (bp->bp_flags & BSTP_PORT_ADMCOST) - req->ifbr_ifsflags |= IFBIF_BSTP_ADMCOST; return (0); } @@ -2471,6 +2558,10 @@ bridge_ioctl_sifflags(struct bridge_softc *sc, void *arg) int error; #endif /* BRIDGESTP */ + if (!bridge_in_bsd_mode(sc)) { + return (EINVAL); + } + bif = bridge_lookup_member(sc, req->ifbr_ifsname); if (bif == NULL) return (ENOENT); @@ -2516,8 +2607,9 @@ bridge_ioctl_scache(struct bridge_softc *sc, void *arg) struct ifbrparam *param = arg; sc->sc_brtmax = param->ifbrp_csize; - bridge_rttrim(sc); - + if (bridge_in_bsd_mode(sc)) { + bridge_rttrim(sc); + } return (0); } @@ -2540,8 +2632,10 @@ bridge_ioctl_gcache(struct bridge_softc *sc, void *arg) count = 0; \ TAILQ_FOREACH(bif, &sc->sc_iflist, bif_next) \ count++; \ - TAILQ_FOREACH(bif, &sc->sc_spanlist, bif_next) \ - count++; \ + if (bridge_in_bsd_mode(sc)) { \ + TAILQ_FOREACH(bif, &sc->sc_spanlist, bif_next) \ + count++; \ + } \ \ buflen = sizeof (breq) * count; \ if (bifc->ifbic_len == 0) { \ @@ -2571,18 +2665,22 @@ bridge_ioctl_gcache(struct bridge_softc *sc, void *arg) buf += sizeof (breq); \ len -= sizeof (breq); \ } \ - TAILQ_FOREACH(bif, &sc->sc_spanlist, bif_next) { \ - if (len < sizeof (breq)) \ - break; \ + if (bridge_in_bsd_mode(sc)) { \ + TAILQ_FOREACH(bif, &sc->sc_spanlist, bif_next) { \ + if (len < sizeof (breq)) \ + break; \ \ - snprintf(breq.ifbr_ifsname, sizeof (breq.ifbr_ifsname), \ - "%s", bif->bif_ifp->if_xname); \ - breq.ifbr_ifsflags = bif->bif_ifflags; \ - breq.ifbr_portno = bif->bif_ifp->if_index & 0xfff; \ - memcpy(buf, &breq, sizeof (breq)); \ - count++; \ - buf += sizeof (breq); \ - len -= sizeof (breq); \ + snprintf(breq.ifbr_ifsname, \ + sizeof (breq.ifbr_ifsname), \ + "%s", bif->bif_ifp->if_xname); \ + breq.ifbr_ifsflags = bif->bif_ifflags; \ + breq.ifbr_portno \ + = bif->bif_ifp->if_index & 0xfff; \ + memcpy(buf, &breq, sizeof (breq)); \ + count++; \ + buf += sizeof (breq); \ + len -= sizeof (breq); \ + } \ } \ \ BRIDGE_UNLOCK(sc); \ @@ -2616,14 +2714,19 @@ bridge_ioctl_gifs32(struct bridge_softc *sc, void *arg) #define BRIDGE_IOCTL_RTS do { \ struct bridge_rtnode *brt; \ - char *buf, *outbuf; \ + char *buf; \ + char *outbuf = NULL; \ unsigned int count, buflen, len; \ unsigned long now; \ \ if (bac->ifbac_len == 0) \ return (0); \ \ + bzero(&bareq, sizeof (bareq)); \ count = 0; \ + if (!bridge_in_bsd_mode(sc)) { \ + goto out; \ + } \ LIST_FOREACH(brt, &sc->sc_rtlist, brt_list) \ count++; \ buflen = sizeof (bareq) * count; \ @@ -2635,12 +2738,11 @@ bridge_ioctl_gifs32(struct bridge_softc *sc, void *arg) count = 0; \ buf = outbuf; \ len = min(bac->ifbac_len, buflen); \ - bzero(&bareq, sizeof (bareq)); \ LIST_FOREACH(brt, &sc->sc_rtlist, brt_list) { \ if (len < sizeof (bareq)) \ goto out; \ snprintf(bareq.ifba_ifsname, sizeof (bareq.ifba_ifsname), \ - "%s", brt->brt_ifp->if_xname); \ + "%s", brt->brt_ifp->if_xname); \ memcpy(bareq.ifba_dst, brt->brt_addr, sizeof (brt->brt_addr)); \ bareq.ifba_vlan = brt->brt_vlan; \ if ((brt->brt_flags & IFBAF_TYPEMASK) == IFBAF_DYNAMIC) { \ @@ -2658,11 +2760,13 @@ bridge_ioctl_gifs32(struct bridge_softc *sc, void *arg) len -= sizeof (bareq); \ } \ out: \ - BRIDGE_UNLOCK(sc); \ bac->ifbac_len = sizeof (bareq) * count; \ - error = copyout(outbuf, bac->ifbac_req, bac->ifbac_len); \ - BRIDGE_LOCK(sc); \ - _FREE(outbuf, M_TEMP); \ + if (outbuf != NULL) { \ + BRIDGE_UNLOCK(sc); \ + error = copyout(outbuf, bac->ifbac_req, bac->ifbac_len); \ + _FREE(outbuf, M_TEMP); \ + BRIDGE_LOCK(sc); \ + } \ return (error); \ } while (0) @@ -2674,7 +2778,6 @@ bridge_ioctl_rts64(struct bridge_softc *sc, void *arg) int error = 0; BRIDGE_IOCTL_RTS; - return (error); } @@ -2686,7 +2789,6 @@ bridge_ioctl_rts32(struct bridge_softc *sc, void *arg) int error = 0; BRIDGE_IOCTL_RTS; - return (error); } @@ -2697,6 +2799,10 @@ bridge_ioctl_saddr32(struct bridge_softc *sc, void *arg) struct bridge_iflist *bif; int error; + if (!bridge_in_bsd_mode(sc)) { + return (0); + } + bif = bridge_lookup_member(sc, req->ifba_ifsname); if (bif == NULL) return (ENOENT); @@ -2714,6 +2820,10 @@ bridge_ioctl_saddr64(struct bridge_softc *sc, void *arg) struct bridge_iflist *bif; int error; + if (!bridge_in_bsd_mode(sc)) { + return (0); + } + bif = bridge_lookup_member(sc, req->ifba_ifsname); if (bif == NULL) return (ENOENT); @@ -2747,6 +2857,9 @@ bridge_ioctl_daddr32(struct bridge_softc *sc, void *arg) { struct ifbareq32 *req = arg; + if (!bridge_in_bsd_mode(sc)) { + return (0); + } return (bridge_rtdaddr(sc, req->ifba_dst, req->ifba_vlan)); } @@ -2755,6 +2868,9 @@ bridge_ioctl_daddr64(struct bridge_softc *sc, void *arg) { struct ifbareq64 *req = arg; + if (!bridge_in_bsd_mode(sc)) { + return (0); + } return (bridge_rtdaddr(sc, req->ifba_dst, req->ifba_vlan)); } @@ -2763,6 +2879,9 @@ bridge_ioctl_flush(struct bridge_softc *sc, void *arg) { struct ifbreq *req = arg; + if (!bridge_in_bsd_mode(sc)) { + return (0); + } bridge_rtflush(sc, req->ifbr_ifsflags); return (0); } @@ -2773,6 +2892,9 @@ bridge_ioctl_gpri(struct bridge_softc *sc, void *arg) struct ifbrparam *param = arg; struct bstp_state *bs = &sc->sc_stp; + if (!bridge_in_bsd_mode(sc)) { + return (0); + } param->ifbrp_prio = bs->bs_bridge_priority; return (0); } @@ -2783,6 +2905,9 @@ bridge_ioctl_spri(struct bridge_softc *sc, void *arg) #if BRIDGESTP struct ifbrparam *param = arg; + if (!bridge_in_bsd_mode(sc)) { + return (EOPNOTSUPP); + } return (bstp_set_priority(&sc->sc_stp, param->ifbrp_prio)); #else /* !BRIDGESTP */ #pragma unused(sc, arg) @@ -2796,6 +2921,9 @@ bridge_ioctl_ght(struct bridge_softc *sc, void *arg) struct ifbrparam *param = arg; struct bstp_state *bs = &sc->sc_stp; + if (!bridge_in_bsd_mode(sc)) { + return (0); + } param->ifbrp_hellotime = bs->bs_bridge_htime >> 8; return (0); } @@ -2806,6 +2934,9 @@ bridge_ioctl_sht(struct bridge_softc *sc, void *arg) #if BRIDGESTP struct ifbrparam *param = arg; + if (!bridge_in_bsd_mode(sc)) { + return (EOPNOTSUPP); + } return (bstp_set_htime(&sc->sc_stp, param->ifbrp_hellotime)); #else /* !BRIDGESTP */ #pragma unused(sc, arg) @@ -2816,9 +2947,14 @@ bridge_ioctl_sht(struct bridge_softc *sc, void *arg) static int bridge_ioctl_gfd(struct bridge_softc *sc, void *arg) { - struct ifbrparam *param = arg; - struct bstp_state *bs = &sc->sc_stp; + struct ifbrparam *param; + struct bstp_state *bs; + if (!bridge_in_bsd_mode(sc)) { + return (0); + } + param = arg; + bs = &sc->sc_stp; param->ifbrp_fwddelay = bs->bs_bridge_fdelay >> 8; return (0); } @@ -2829,6 +2965,9 @@ bridge_ioctl_sfd(struct bridge_softc *sc, void *arg) #if BRIDGESTP struct ifbrparam *param = arg; + if (!bridge_in_bsd_mode(sc)) { + return (EOPNOTSUPP); + } return (bstp_set_fdelay(&sc->sc_stp, param->ifbrp_fwddelay)); #else /* !BRIDGESTP */ #pragma unused(sc, arg) @@ -2839,9 +2978,14 @@ bridge_ioctl_sfd(struct bridge_softc *sc, void *arg) static int bridge_ioctl_gma(struct bridge_softc *sc, void *arg) { - struct ifbrparam *param = arg; - struct bstp_state *bs = &sc->sc_stp; + struct ifbrparam *param; + struct bstp_state *bs; + if (!bridge_in_bsd_mode(sc)) { + return (EOPNOTSUPP); + } + param = arg; + bs = &sc->sc_stp; param->ifbrp_maxage = bs->bs_bridge_max_age >> 8; return (0); } @@ -2852,6 +2996,9 @@ bridge_ioctl_sma(struct bridge_softc *sc, void *arg) #if BRIDGESTP struct ifbrparam *param = arg; + if (!bridge_in_bsd_mode(sc)) { + return (EOPNOTSUPP); + } return (bstp_set_maxage(&sc->sc_stp, param->ifbrp_maxage)); #else /* !BRIDGESTP */ #pragma unused(sc, arg) @@ -2866,6 +3013,9 @@ bridge_ioctl_sifprio(struct bridge_softc *sc, void *arg) struct ifbreq *req = arg; struct bridge_iflist *bif; + if (!bridge_in_bsd_mode(sc)) { + return (EOPNOTSUPP); + } bif = bridge_lookup_member(sc, req->ifbr_ifsname); if (bif == NULL) return (ENOENT); @@ -2884,6 +3034,9 @@ bridge_ioctl_sifcost(struct bridge_softc *sc, void *arg) struct ifbreq *req = arg; struct bridge_iflist *bif; + if (!bridge_in_bsd_mode(sc)) { + return (EOPNOTSUPP); + } bif = bridge_lookup_member(sc, req->ifbr_ifsname); if (bif == NULL) return (ENOENT); @@ -2944,6 +3097,9 @@ bridge_ioctl_addspan(struct bridge_softc *sc, void *arg) struct bridge_iflist *bif = NULL; struct ifnet *ifs; + if (!bridge_in_bsd_mode(sc)) { + return (EOPNOTSUPP); + } ifs = ifunit(req->ifbr_ifsname); if (ifs == NULL) return (ENOENT); @@ -2966,7 +3122,7 @@ bridge_ioctl_addspan(struct bridge_softc *sc, void *arg) return (EINVAL); } - bif = _MALLOC(sizeof (*bif), M_DEVBUF, M_NOWAIT | M_ZERO); + bif = _MALLOC(sizeof (*bif), M_DEVBUF, M_WAITOK | M_ZERO); if (bif == NULL) return (ENOMEM); @@ -2987,6 +3143,9 @@ bridge_ioctl_delspan(struct bridge_softc *sc, void *arg) struct bridge_iflist *bif; struct ifnet *ifs; + if (!bridge_in_bsd_mode(sc)) { + return (EOPNOTSUPP); + } ifs = ifunit(req->ifbr_ifsname); if (ifs == NULL) return (ENOENT); @@ -3033,8 +3192,9 @@ bridge_ioctl_gbparam32(struct bridge_softc *sc, void *arg) { struct ifbropreq32 *req = arg; - BRIDGE_IOCTL_GBPARAM; - + if (bridge_in_bsd_mode(sc)) { + BRIDGE_IOCTL_GBPARAM; + } return (0); } @@ -3043,8 +3203,9 @@ bridge_ioctl_gbparam64(struct bridge_softc *sc, void *arg) { struct ifbropreq64 *req = arg; - BRIDGE_IOCTL_GBPARAM; - + if (bridge_in_bsd_mode(sc)) { + BRIDGE_IOCTL_GBPARAM; + } return (0); } @@ -3119,8 +3280,9 @@ bridge_ioctl_gifsstp32(struct bridge_softc *sc, void *arg) struct ifbpstpconf32 *bifstp = arg; int error = 0; - BRIDGE_IOCTL_GIFSSTP; - + if (bridge_in_bsd_mode(sc)) { + BRIDGE_IOCTL_GIFSSTP; + } return (error); } @@ -3130,8 +3292,9 @@ bridge_ioctl_gifsstp64(struct bridge_softc *sc, void *arg) struct ifbpstpconf64 *bifstp = arg; int error = 0; - BRIDGE_IOCTL_GIFSSTP; - + if (bridge_in_bsd_mode(sc)) { + BRIDGE_IOCTL_GIFSSTP; + } return (error); } @@ -3141,6 +3304,9 @@ bridge_ioctl_sproto(struct bridge_softc *sc, void *arg) #if BRIDGESTP struct ifbrparam *param = arg; + if (!bridge_in_bsd_mode(sc)) { + return (EOPNOTSUPP); + } return (bstp_set_protocol(&sc->sc_stp, param->ifbrp_proto)); #else /* !BRIDGESTP */ #pragma unused(sc, arg) @@ -3154,6 +3320,9 @@ bridge_ioctl_stxhc(struct bridge_softc *sc, void *arg) #if BRIDGESTP struct ifbrparam *param = arg; + if (!bridge_in_bsd_mode(sc)) { + return (EOPNOTSUPP); + } return (bstp_set_holdcount(&sc->sc_stp, param->ifbrp_txhc)); #else /* !BRIDGESTP */ #pragma unused(sc, arg) @@ -3192,6 +3361,8 @@ bridge_ioctl_shostfilter(struct bridge_softc *sc, void *arg) if (bif == NULL) return (ENOENT); + INC_ATOMIC_INT64_LIM(net_api_stats.nas_vmnet_total); + if (req->ifbrhf_flags & IFBRHF_ENABLED) { bif->bif_flags |= BIFF_HOST_FILTER; @@ -3247,17 +3418,18 @@ bridge_ifdetach(struct bridge_iflist *bif, struct ifnet *ifp) BRIDGE_UNLOCK(sc); return; } - /* Check if the interface is a span port */ lck_mtx_lock(&bridge_list_mtx); LIST_FOREACH(sc, &bridge_list, sc_list) { - BRIDGE_LOCK(sc); - TAILQ_FOREACH(bif, &sc->sc_spanlist, bif_next) - if (ifp == bif->bif_ifp) { - bridge_delete_span(sc, bif); - break; - } - BRIDGE_UNLOCK(sc); + if (bridge_in_bsd_mode(sc)) { + BRIDGE_LOCK(sc); + TAILQ_FOREACH(bif, &sc->sc_spanlist, bif_next) + if (ifp == bif->bif_ifp) { + bridge_delete_span(sc, bif); + break; + } + BRIDGE_UNLOCK(sc); + } } lck_mtx_unlock(&bridge_list_mtx); } @@ -3546,17 +3718,17 @@ bridge_init(struct ifnet *ifp) error = ifnet_set_flags(ifp, IFF_RUNNING, IFF_RUNNING); - /* - * Calling bridge_aging_timer() is OK as there are no entries to - * age so we're just going to arm the timer - */ - bridge_aging_timer(sc); - + if (bridge_in_bsd_mode(sc)) { + /* + * Calling bridge_aging_timer() is OK as there are no entries to + * age so we're just going to arm the timer + */ + bridge_aging_timer(sc); #if BRIDGESTP - if (error == 0) - bstp_init(&sc->sc_stp); /* Initialize Spanning Tree */ + if (error == 0) + bstp_init(&sc->sc_stp); /* Initialize Spanning Tree */ #endif /* BRIDGESTP */ - + } return (error); } @@ -3576,14 +3748,15 @@ bridge_ifstop(struct ifnet *ifp, int disable) if ((ifnet_flags(ifp) & IFF_RUNNING) == 0) return; - bridge_cancel_delayed_call(&sc->sc_aging_timer); + if (bridge_in_bsd_mode(sc)) { + bridge_cancel_delayed_call(&sc->sc_aging_timer); #if BRIDGESTP - bstp_stop(&sc->sc_stp); + bstp_stop(&sc->sc_stp); #endif /* BRIDGESTP */ - bridge_rtflush(sc, IFBF_FLUSHDYN); - + bridge_rtflush(sc, IFBF_FLUSHDYN); + } (void) ifnet_set_flags(ifp, 0, IFF_RUNNING); } @@ -3720,7 +3893,7 @@ bridge_member_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *sa, uint16_t vlan; #if BRIDGE_DEBUG - if (if_bridge_debug & BR_DBGF_OUTPPUT) + if (if_bridge_debug & BR_DBGF_OUTPUT) printf("%s: ifp %s\n", __func__, ifp->if_xname); #endif /* BRIDGE_DEBUG */ @@ -3851,6 +4024,8 @@ bridge_output(struct ifnet *ifp, struct mbuf *m) dst_if = NULL; BRIDGE_LOCK(sc); + ASSERT(bridge_in_bsd_mode(sc)); + if (!(m->m_flags & (M_BCAST|M_MCAST))) dst_if = bridge_rtlookup(sc, eh->ether_dhost, 0); @@ -3977,9 +4152,10 @@ bridge_forward(struct bridge_softc *sc, struct bridge_iflist *sbif, int error; BRIDGE_LOCK_ASSERT_HELD(sc); + ASSERT(bridge_in_bsd_mode(sc)); #if BRIDGE_DEBUG - if (if_bridge_debug & BR_DBGF_OUTPPUT) + if (if_bridge_debug & BR_DBGF_OUTPUT) printf("%s: %s m 0x%llx\n", __func__, sc->sc_ifp->if_xname, (uint64_t)VM_KERNEL_ADDRPERM(m)); #endif /* BRIDGE_DEBUG */ @@ -4162,6 +4338,7 @@ bridge_input(struct ifnet *ifp, struct mbuf *m, void *frame_header) uint16_t vlan; int error; + ASSERT(bridge_in_bsd_mode(sc)); #if BRIDGE_DEBUG if (if_bridge_debug & BR_DBGF_INPUT) printf("%s: %s from %s m 0x%llx data 0x%llx\n", __func__, @@ -4643,6 +4820,7 @@ bridge_rtupdate(struct bridge_softc *sc, const uint8_t *dst, uint16_t vlan, int error; BRIDGE_LOCK_ASSERT_HELD(sc); + ASSERT(bridge_in_bsd_mode(sc)); /* Check the source address is valid and not multicast. */ if (ETHER_IS_MULTICAST(dst) || @@ -4891,6 +5069,8 @@ bridge_rtable_init(struct bridge_softc *sc) { u_int32_t i; + ASSERT(bridge_in_bsd_mode(sc)); + sc->sc_rthash = _MALLOC(sizeof (*sc->sc_rthash) * BRIDGE_RTHASH_SIZE, M_DEVBUF, M_WAITOK | M_ZERO); if (sc->sc_rthash == NULL) { @@ -5102,6 +5282,7 @@ bridge_rtnode_lookup(struct bridge_softc *sc, const uint8_t *addr, int dir; BRIDGE_LOCK_ASSERT_HELD(sc); + ASSERT(bridge_in_bsd_mode(sc)); hash = bridge_rthash(sc, addr); LIST_FOREACH(brt, &sc->sc_rthash[hash], brt_hash) { @@ -5793,7 +5974,7 @@ bridge_set_bpf_tap(ifnet_t ifp, bpf_tap_mode mode, bpf_packet_func bpf_callback) if (sc == NULL || (sc->sc_flags & SCF_DETACHING)) { return (ENODEV); } - + ASSERT(bridge_in_bsd_mode(sc)); switch (mode) { case BPF_TAP_DISABLE: sc->sc_bpf_input = sc->sc_bpf_output = NULL; @@ -5832,8 +6013,10 @@ bridge_detach(ifnet_t ifp) bstp_detach(&sc->sc_stp); #endif /* BRIDGESTP */ - /* Tear down the routing table. */ - bridge_rtable_fini(sc); + if (bridge_in_bsd_mode(sc)) { + /* Tear down the routing table. */ + bridge_rtable_fini(sc); + } lck_mtx_lock(&bridge_list_mtx); LIST_REMOVE(sc, sc_list); @@ -5856,6 +6039,7 @@ bridge_bpf_input(ifnet_t ifp, struct mbuf *m) { struct bridge_softc *sc = (struct bridge_softc *)ifnet_softc(ifp); + ASSERT(bridge_in_bsd_mode(sc)); if (sc->sc_bpf_input) { if (mbuf_pkthdr_rcvif(m) != ifp) { printf("%s: rcvif: 0x%llx != ifp 0x%llx\n", __func__, @@ -5877,6 +6061,7 @@ bridge_bpf_output(ifnet_t ifp, struct mbuf *m) { struct bridge_softc *sc = (struct bridge_softc *)ifnet_softc(ifp); + ASSERT(bridge_in_bsd_mode(sc)); if (sc->sc_bpf_output) { (*sc->sc_bpf_output)(ifp, m); } @@ -6194,3 +6379,5 @@ done: } return (error); } + + diff --git a/bsd/net/if_dl.h b/bsd/net/if_dl.h index 3d086e402..55d504dc8 100644 --- a/bsd/net/if_dl.h +++ b/bsd/net/if_dl.h @@ -65,6 +65,12 @@ #define _NET_IF_DL_H_ #include <sys/appleapiopts.h> +#include <sys/types.h> + +#ifdef BSD_KERNEL_PRIVATE +#define DLIL_SDLMAXLEN 64 +#endif /* BSD_KERNEL_PRIVATE */ + /* * A Link-Level Sockaddr may specify the interface in one of two * ways: either by means of a system-provided index number (computed diff --git a/bsd/net/if_fake.c b/bsd/net/if_fake.c new file mode 100644 index 000000000..3af936b87 --- /dev/null +++ b/bsd/net/if_fake.c @@ -0,0 +1,1029 @@ +/* + * Copyright (c) 2015-2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +/* + * if_fake.c + * - fake network interface used for testing + * - "feth" (e.g. "feth0", "feth1") is a virtual ethernet interface that allows + * two instances to have their output/input paths "crossed-over" so that + * output on one is input on the other + */ + +/* + * Modification History: + * + * September 9, 2015 Dieter Siegmund (dieter@apple.com) + * - created + */ + +#include <sys/param.h> +#include <sys/kernel.h> +#include <sys/malloc.h> +#include <sys/mbuf.h> +#include <sys/queue.h> +#include <sys/socket.h> +#include <sys/sockio.h> +#include <sys/sysctl.h> +#include <sys/systm.h> +#include <sys/kern_event.h> +#include <sys/mcache.h> +#include <sys/syslog.h> + +#include <net/bpf.h> +#include <net/ethernet.h> +#include <net/if.h> +#include <net/if_vlan_var.h> +#include <net/if_fake_var.h> +#include <net/if_arp.h> +#include <net/if_dl.h> +#include <net/if_ether.h> +#include <net/if_types.h> +#include <libkern/OSAtomic.h> + +#include <net/dlil.h> + +#include <net/kpi_interface.h> +#include <net/kpi_protocol.h> + +#include <kern/locks.h> + +#ifdef INET +#include <netinet/in.h> +#include <netinet/if_ether.h> +#endif + +#include <net/if_media.h> +#include <net/ether_if_module.h> + +#define FAKE_ETHER_NAME "feth" + +SYSCTL_DECL(_net_link); +SYSCTL_NODE(_net_link, OID_AUTO, fake, CTLFLAG_RW|CTLFLAG_LOCKED, 0, + "Fake interface"); + +static int if_fake_txstart = 1; +SYSCTL_INT(_net_link_fake, OID_AUTO, txstart, CTLFLAG_RW | CTLFLAG_LOCKED, + &if_fake_txstart, 0, "Fake interface TXSTART mode"); + +static int if_fake_hwcsum = 0; +SYSCTL_INT(_net_link_fake, OID_AUTO, hwcsum, CTLFLAG_RW | CTLFLAG_LOCKED, + &if_fake_hwcsum, 0, "Fake interface simulate hardware checksum"); + +static int if_fake_nxattach = 0; +SYSCTL_INT(_net_link_fake, OID_AUTO, nxattach, CTLFLAG_RW | CTLFLAG_LOCKED, + &if_fake_nxattach, 0, "Fake interface auto-attach nexus"); + +static int if_fake_bsd_mode = 1; +SYSCTL_INT(_net_link_fake, OID_AUTO, bsd_mode, CTLFLAG_RW | CTLFLAG_LOCKED, + &if_fake_bsd_mode, 0, "Fake interface attach as BSD interface"); + +static int if_fake_debug = 0; +SYSCTL_INT(_net_link_fake, OID_AUTO, debug, CTLFLAG_RW | CTLFLAG_LOCKED, + &if_fake_debug, 0, "Fake interface debug logs"); + +/** + ** virtual ethernet structures, types + **/ + +typedef uint16_t iff_flags_t; +#define IFF_FLAGS_HWCSUM 0x0001 +#define IFF_FLAGS_BSD_MODE 0x0002 +#define IFF_FLAGS_DETACHING 0x0004 + + +struct if_fake { + char iff_name[IFNAMSIZ]; /* our unique id */ + ifnet_t iff_ifp; + iff_flags_t iff_flags; + uint32_t iff_retain_count; + ifnet_t iff_peer; /* the other end */ + int iff_media_current; + int iff_media_active; + uint32_t iff_media_count; + int iff_media_list[IF_FAKE_MEDIA_LIST_MAX]; + struct mbuf * iff_pending_tx_packet; + boolean_t iff_start_busy; +}; + +typedef struct if_fake * if_fake_ref; + +static if_fake_ref +ifnet_get_if_fake(ifnet_t ifp); + +#define FETH_DPRINTF(fmt, ...) \ + { if (if_fake_debug != 0) printf("%s " fmt, __func__, ## __VA_ARGS__); } + +static inline boolean_t +feth_in_bsd_mode(if_fake_ref fakeif) +{ + return ((fakeif->iff_flags & IFF_FLAGS_BSD_MODE) != 0); +} + +static inline void +feth_set_detaching(if_fake_ref fakeif) +{ + fakeif->iff_flags |= IFF_FLAGS_DETACHING; +} + +static inline boolean_t +feth_is_detaching(if_fake_ref fakeif) +{ + return ((fakeif->iff_flags & IFF_FLAGS_DETACHING) != 0); +} + + +#define M_FAKE M_DEVBUF + +static int feth_clone_create(struct if_clone *, u_int32_t, void *); +static int feth_clone_destroy(ifnet_t); +static int feth_output(ifnet_t ifp, struct mbuf *m); +static void feth_start(ifnet_t ifp); +static int feth_ioctl(ifnet_t ifp, u_long cmd, void * addr); +static int feth_config(ifnet_t ifp, ifnet_t peer); +static void feth_if_free(ifnet_t ifp); +static void feth_ifnet_set_attrs(if_fake_ref fakeif, ifnet_t ifp); +static void feth_free(if_fake_ref fakeif); + +static struct if_clone +feth_cloner = IF_CLONE_INITIALIZER(FAKE_ETHER_NAME, + feth_clone_create, + feth_clone_destroy, + 0, + IF_MAXUNIT); +static void interface_link_event(ifnet_t ifp, u_int32_t event_code); + +/* some media words to pretend to be ethernet */ +static int default_media_words[] = { + IFM_MAKEWORD(IFM_ETHER, 0, 0, 0), + IFM_MAKEWORD(IFM_ETHER, IFM_10G_T, IFM_FDX, 0), + IFM_MAKEWORD(IFM_ETHER, IFM_2500_T, IFM_FDX, 0), + IFM_MAKEWORD(IFM_ETHER, IFM_5000_T, IFM_FDX, 0), +}; +#define default_media_words_count (sizeof(default_media_words) \ + / sizeof (default_media_words[0])) + +/** + ** veth locks + **/ +static inline lck_grp_t * +my_lck_grp_alloc_init(const char * grp_name) +{ + lck_grp_t * grp; + lck_grp_attr_t * grp_attrs; + + grp_attrs = lck_grp_attr_alloc_init(); + grp = lck_grp_alloc_init(grp_name, grp_attrs); + lck_grp_attr_free(grp_attrs); + return (grp); +} + +static inline lck_mtx_t * +my_lck_mtx_alloc_init(lck_grp_t * lck_grp) +{ + lck_attr_t * lck_attrs; + lck_mtx_t * lck_mtx; + + lck_attrs = lck_attr_alloc_init(); + lck_mtx = lck_mtx_alloc_init(lck_grp, lck_attrs); + lck_attr_free(lck_attrs); + return (lck_mtx); +} + +static lck_mtx_t * feth_lck_mtx; + +static inline void +feth_lock_init(void) +{ + lck_grp_t * feth_lck_grp; + + feth_lck_grp = my_lck_grp_alloc_init("fake"); + feth_lck_mtx = my_lck_mtx_alloc_init(feth_lck_grp); +} + +#if 0 +static inline void +feth_assert_lock_not_held(void) +{ + LCK_MTX_ASSERT(feth_lck_mtx, LCK_MTX_ASSERT_NOTOWNED); + return; +} +#endif + +static inline void +feth_lock(void) +{ + lck_mtx_lock(feth_lck_mtx); + return; +} + +static inline void +feth_unlock(void) +{ + lck_mtx_unlock(feth_lck_mtx); + return; +} + +static inline int +feth_max_mtu(void) +{ + if (njcl > 0) { + return (M16KCLBYTES - ETHER_HDR_LEN); + } + return (MBIGCLBYTES - ETHER_HDR_LEN); +} + +static void +feth_free(if_fake_ref fakeif) +{ + assert(fakeif->iff_retain_count == 0); + if (feth_in_bsd_mode(fakeif)) { + if (fakeif->iff_pending_tx_packet) { + m_freem(fakeif->iff_pending_tx_packet); + } + } + + FETH_DPRINTF("%s\n", fakeif->iff_name); + FREE(fakeif, M_FAKE); +} + +static void +feth_release(if_fake_ref fakeif) +{ + u_int32_t old_retain_count; + + old_retain_count = OSDecrementAtomic(&fakeif->iff_retain_count); + switch (old_retain_count) { + case 0: + assert(old_retain_count != 0); + break; + case 1: + feth_free(fakeif); + break; + default: + break; + } + return; +} + + +/** + ** feth interface routines + **/ +static void +feth_ifnet_set_attrs(if_fake_ref fakeif, ifnet_t ifp) +{ + (void)ifnet_set_capabilities_enabled(ifp, 0, -1); + ifnet_set_addrlen(ifp, ETHER_ADDR_LEN); + ifnet_set_baudrate(ifp, 0); + ifnet_set_mtu(ifp, ETHERMTU); + ifnet_set_flags(ifp, + IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX, + 0xffff); + ifnet_set_hdrlen(ifp, sizeof(struct ether_header)); + if ((fakeif->iff_flags & IFF_FLAGS_HWCSUM) != 0) { + ifnet_set_offload(ifp, + IFNET_CSUM_IP | IFNET_CSUM_TCP | IFNET_CSUM_UDP | + IFNET_CSUM_TCPIPV6 | IFNET_CSUM_UDPIPV6); + } else { + ifnet_set_offload(ifp, 0); + } +} + +static void +interface_link_event(ifnet_t ifp, u_int32_t event_code) +{ + struct { + struct kern_event_msg header; + u_int32_t unit; + char if_name[IFNAMSIZ]; + } event; + + bzero(&event, sizeof(event)); + event.header.total_size = sizeof(event); + event.header.vendor_code = KEV_VENDOR_APPLE; + event.header.kev_class = KEV_NETWORK_CLASS; + event.header.kev_subclass = KEV_DL_SUBCLASS; + event.header.event_code = event_code; + event.header.event_data[0] = ifnet_family(ifp); + event.unit = (u_int32_t) ifnet_unit(ifp); + strlcpy(event.if_name, ifnet_name(ifp), IFNAMSIZ); + ifnet_event(ifp, &event.header); + return; +} + +static if_fake_ref +ifnet_get_if_fake(ifnet_t ifp) +{ + return ((if_fake_ref)ifnet_softc(ifp)); +} + +static int +feth_clone_create(struct if_clone *ifc, u_int32_t unit, __unused void *params) +{ + int error; + if_fake_ref fakeif; + struct ifnet_init_eparams feth_init; + ifnet_t ifp; + uint8_t mac_address[ETHER_ADDR_LEN]; + + fakeif = _MALLOC(sizeof(struct if_fake), M_FAKE, M_WAITOK | M_ZERO); + if (fakeif == NULL) { + return ENOBUFS; + } + fakeif->iff_retain_count = 1; +#define FAKE_ETHER_NAME_LEN (sizeof(FAKE_ETHER_NAME) - 1) + _CASSERT(FAKE_ETHER_NAME_LEN == 4); + bcopy(FAKE_ETHER_NAME, mac_address, FAKE_ETHER_NAME_LEN); + mac_address[ETHER_ADDR_LEN - 2] = (unit & 0xff00) >> 8; + mac_address[ETHER_ADDR_LEN - 1] = unit & 0xff; + if (if_fake_bsd_mode != 0) { + fakeif->iff_flags |= IFF_FLAGS_BSD_MODE; + } + if (if_fake_hwcsum != 0) { + fakeif->iff_flags |= IFF_FLAGS_HWCSUM; + } + + /* use the interface name as the unique id for ifp recycle */ + if ((unsigned int) + snprintf(fakeif->iff_name, sizeof(fakeif->iff_name), "%s%d", + ifc->ifc_name, unit) >= sizeof(fakeif->iff_name)) { + feth_release(fakeif); + return (EINVAL); + } + bzero(&feth_init, sizeof(feth_init)); + feth_init.ver = IFNET_INIT_CURRENT_VERSION; + feth_init.len = sizeof (feth_init); + if (feth_in_bsd_mode(fakeif)) { + if (if_fake_txstart != 0) { + feth_init.start = feth_start; + } else { + feth_init.flags |= IFNET_INIT_LEGACY; + feth_init.output = feth_output; + } + } + if (if_fake_nxattach == 0) { + feth_init.flags |= IFNET_INIT_NX_NOAUTO; + } + feth_init.uniqueid = fakeif->iff_name; + feth_init.uniqueid_len = strlen(fakeif->iff_name); + feth_init.name = ifc->ifc_name; + feth_init.unit = unit; + feth_init.family = IFNET_FAMILY_ETHERNET; + feth_init.type = IFT_ETHER; + feth_init.demux = ether_demux; + feth_init.add_proto = ether_add_proto; + feth_init.del_proto = ether_del_proto; + feth_init.check_multi = ether_check_multi; + feth_init.framer_extended = ether_frameout_extended; + feth_init.softc = fakeif; + feth_init.ioctl = feth_ioctl; + feth_init.set_bpf_tap = NULL; + feth_init.detach = feth_if_free; + feth_init.broadcast_addr = etherbroadcastaddr; + feth_init.broadcast_len = ETHER_ADDR_LEN; + if (feth_in_bsd_mode(fakeif)) { + error = ifnet_allocate_extended(&feth_init, &ifp); + if (error) { + feth_release(fakeif); + return (error); + } + feth_ifnet_set_attrs(fakeif, ifp); + } + fakeif->iff_media_count = default_media_words_count; + bcopy(default_media_words, fakeif->iff_media_list, + sizeof(default_media_words)); + if (feth_in_bsd_mode(fakeif)) { + error = ifnet_attach(ifp, NULL); + if (error) { + ifnet_release(ifp); + feth_release(fakeif); + return (error); + } + fakeif->iff_ifp = ifp; + } + + ifnet_set_lladdr(ifp, mac_address, sizeof(mac_address)); + + /* attach as ethernet */ + bpfattach(ifp, DLT_EN10MB, sizeof(struct ether_header)); + return (0); +} + +static int +feth_clone_destroy(ifnet_t ifp) +{ + if_fake_ref fakeif; + + feth_lock(); + fakeif = ifnet_get_if_fake(ifp); + if (fakeif == NULL || feth_is_detaching(fakeif)) { + feth_unlock(); + return (0); + } + feth_set_detaching(fakeif); + feth_unlock(); + + feth_config(ifp, NULL); + ifnet_detach(ifp); + return 0; +} + +static void +feth_enqueue_input(ifnet_t ifp, struct mbuf * m) +{ + struct ifnet_stat_increment_param stats = {}; + + stats.packets_in = 1; + stats.bytes_in = (uint32_t)mbuf_pkthdr_len(m) + ETHER_HDR_LEN; + ifnet_input(ifp, m, &stats); +} + +static struct mbuf * +copy_mbuf(struct mbuf *m) +{ + struct mbuf * copy_m; + uint32_t pkt_len; + uint32_t offset; + + if ((m->m_flags & M_PKTHDR) == 0) { + return (NULL); + } + pkt_len = m->m_pkthdr.len; + MGETHDR(copy_m, M_DONTWAIT, MT_DATA); + if (copy_m == NULL) { + goto failed; + } + if (pkt_len > MHLEN) { + if (pkt_len <= MCLBYTES) { + MCLGET(copy_m, M_DONTWAIT); + } else if (pkt_len <= MBIGCLBYTES) { + copy_m = m_mbigget(copy_m, M_DONTWAIT); + } else if (pkt_len <= M16KCLBYTES && njcl > 0) { + copy_m = m_m16kget(copy_m, M_DONTWAIT); + } else { + printf("if_fake: copy_mbuf(): packet too large %d\n", + pkt_len); + goto failed; + } + if (copy_m == NULL || (copy_m->m_flags & M_EXT) == 0) { + goto failed; + } + } + mbuf_setlen(copy_m, pkt_len); + copy_m->m_pkthdr.len = pkt_len; + offset = 0; + while (m != NULL && offset < pkt_len) { + uint32_t frag_len; + + frag_len = m->m_len; + if (frag_len > (pkt_len - offset)) { + printf("if_fake_: Large mbuf fragment %d > %d\n", + frag_len, (pkt_len - offset)); + goto failed; + } + m_copydata(m, 0, frag_len, mtod(copy_m, void *) + offset); + offset += frag_len; + m = m->m_next; + } + return (copy_m); + + failed: + if (copy_m != NULL) { + m_freem(copy_m); + } + return (NULL); +} + +static void +feth_output_common(ifnet_t ifp, struct mbuf * m, ifnet_t peer, + iff_flags_t flags) +{ + void * frame_header; + + frame_header = mbuf_data(m); + if ((flags & IFF_FLAGS_HWCSUM) != 0) { + m->m_pkthdr.csum_data = 0xffff; + m->m_pkthdr.csum_flags = + CSUM_DATA_VALID | CSUM_PSEUDO_HDR | + CSUM_IP_CHECKED | CSUM_IP_VALID; + } + + (void)ifnet_stat_increment_out(ifp, 1, m->m_pkthdr.len, 0); + bpf_tap_out(ifp, DLT_EN10MB, m, NULL, 0); + + (void)mbuf_pkthdr_setrcvif(m, peer); + mbuf_pkthdr_setheader(m, frame_header); + mbuf_pkthdr_adjustlen(m, - ETHER_HDR_LEN); + (void)mbuf_setdata(m, (char *)mbuf_data(m) + ETHER_HDR_LEN, + mbuf_len(m) - ETHER_HDR_LEN); + bpf_tap_in(peer, DLT_EN10MB, m, frame_header, + sizeof(struct ether_header)); + feth_enqueue_input(peer, m); +} + +static void +feth_start(ifnet_t ifp) +{ + struct mbuf * copy_m = NULL; + if_fake_ref fakeif; + iff_flags_t flags = 0; + ifnet_t peer = NULL; + struct mbuf * m; + struct mbuf * save_m; + + feth_lock(); + fakeif = ifnet_get_if_fake(ifp); + if (fakeif->iff_start_busy) { + feth_unlock(); + printf("if_fake: start is busy\n"); + return; + } + if (fakeif != NULL) { + peer = fakeif->iff_peer; + flags = fakeif->iff_flags; + } + + /* check for pending TX */ + m = fakeif->iff_pending_tx_packet; + if (m != NULL) { + if (peer != NULL) { + copy_m = copy_mbuf(m); + if (copy_m == NULL) { + feth_unlock(); + return; + } + } + fakeif->iff_pending_tx_packet = NULL; + m_freem(m); + m = NULL; + } + fakeif->iff_start_busy = TRUE; + feth_unlock(); + save_m = NULL; + for (;;) { + if (copy_m != NULL) { + assert(peer != NULL); + feth_output_common(ifp, copy_m, peer, flags); + copy_m = NULL; + } + if (ifnet_dequeue(ifp, &m) != 0) { + break; + } + if (peer == NULL) { + m_freem(m); + } else { + copy_m = copy_mbuf(m); + if (copy_m == NULL) { + save_m = m; + break; + } + m_freem(m); + } + } + peer = NULL; + feth_lock(); + fakeif = ifnet_get_if_fake(ifp); + if (fakeif != NULL) { + fakeif->iff_start_busy = FALSE; + if (save_m != NULL && fakeif->iff_peer != NULL) { + /* save it for next time */ + fakeif->iff_pending_tx_packet = save_m; + save_m = NULL; + } + } + feth_unlock(); + if (save_m != NULL) { + /* didn't save packet, so free it */ + m_freem(save_m); + } +} + +static int +feth_output(ifnet_t ifp, struct mbuf * m) +{ + struct mbuf * copy_m; + if_fake_ref fakeif; + iff_flags_t flags; + ifnet_t peer = NULL; + + if (m == NULL) { + return (0); + } + copy_m = copy_mbuf(m); + m_freem(m); + m = NULL; + if (copy_m == NULL) { + /* count this as an output error */ + ifnet_stat_increment_out(ifp, 0, 0, 1); + return (0); + } + feth_lock(); + fakeif = ifnet_get_if_fake(ifp); + if (fakeif != NULL) { + peer = fakeif->iff_peer; + flags = fakeif->iff_flags; + } + feth_unlock(); + if (peer == NULL) { + m_freem(copy_m); + ifnet_stat_increment_out(ifp, 0, 0, 1); + return (0); + } + feth_output_common(ifp, copy_m, peer, flags); + return (0); +} + +static int +feth_config(ifnet_t ifp, ifnet_t peer) +{ + int connected = FALSE; + int disconnected = FALSE; + int error = 0; + if_fake_ref fakeif = NULL; + + feth_lock(); + fakeif = ifnet_get_if_fake(ifp); + if (fakeif == NULL) { + error = EINVAL; + goto done; + } + if (peer != NULL) { + /* connect to peer */ + if_fake_ref peer_fakeif; + + peer_fakeif = ifnet_get_if_fake(peer); + if (peer_fakeif == NULL) { + error = EINVAL; + goto done; + } + if (feth_is_detaching(fakeif) || + feth_is_detaching(peer_fakeif) || + peer_fakeif->iff_peer != NULL || + fakeif->iff_peer != NULL) { + error = EBUSY; + goto done; + } + fakeif->iff_peer = peer; + peer_fakeif->iff_peer = ifp; + connected = TRUE; + } + else if (fakeif->iff_peer != NULL) { + /* disconnect from peer */ + if_fake_ref peer_fakeif; + + peer = fakeif->iff_peer; + peer_fakeif = ifnet_get_if_fake(peer); + if (peer_fakeif == NULL) { + /* should not happen */ + error = EINVAL; + goto done; + } + fakeif->iff_peer = NULL; + peer_fakeif->iff_peer = NULL; + disconnected = TRUE; + } + + done: + feth_unlock(); + + /* generate link status event if we connect or disconnect */ + if (connected) { + ifnet_set_flags(ifp, IFF_RUNNING, IFF_RUNNING); + ifnet_set_flags(peer, IFF_RUNNING, IFF_RUNNING); + interface_link_event(ifp, KEV_DL_LINK_ON); + interface_link_event(peer, KEV_DL_LINK_ON); + } + else if (disconnected) { + ifnet_set_flags(ifp, 0, IFF_RUNNING); + ifnet_set_flags(peer, 0, IFF_RUNNING); + interface_link_event(ifp, KEV_DL_LINK_OFF); + interface_link_event(peer, KEV_DL_LINK_OFF); + } + return (error); +} + +static int +feth_set_media(ifnet_t ifp, struct if_fake_request * iffr) +{ + if_fake_ref fakeif; + int error; + + if (iffr->iffr_media.iffm_count > IF_FAKE_MEDIA_LIST_MAX) { + /* list is too long */ + return (EINVAL); + } + feth_lock(); + fakeif = ifnet_get_if_fake(ifp); + if (fakeif == NULL) { + error = EINVAL; + goto done; + } + fakeif->iff_media_count = iffr->iffr_media.iffm_count; + bcopy(iffr->iffr_media.iffm_list, fakeif->iff_media_list, + iffr->iffr_media.iffm_count * sizeof(fakeif->iff_media_list[0])); +#if 0 + /* XXX: "auto-negotiate" active with peer? */ + /* generate link status event? */ + fakeif->iff_media_current = iffr->iffr_media.iffm_current; +#endif + error = 0; + done: + feth_unlock(); + return (error); +} + +static int +if_fake_request_copyin(user_addr_t user_addr, + struct if_fake_request *iffr, u_int32_t len) +{ + int error; + + if (user_addr == USER_ADDR_NULL || len < sizeof(*iffr)) { + error = EINVAL; + goto done; + } + error = copyin(user_addr, iffr, sizeof(*iffr)); + if (error != 0) { + goto done; + } + if (iffr->iffr_reserved[0] != 0 || iffr->iffr_reserved[1] != 0 || + iffr->iffr_reserved[2] != 0 || iffr->iffr_reserved[3] != 0) { + error = EINVAL; + goto done; + } + done: + return (error); +} + +static int +feth_set_drvspec(ifnet_t ifp, uint32_t cmd, u_int32_t len, + user_addr_t user_addr) +{ + int error; + struct if_fake_request iffr; + ifnet_t peer; + + switch (cmd) { + case IF_FAKE_S_CMD_SET_PEER: + error = if_fake_request_copyin(user_addr, &iffr, len); + if (error != 0) { + break; + } + if (iffr.iffr_peer_name[0] == '\0') { + error = feth_config(ifp, NULL); + break; + } + + /* ensure nul termination */ + iffr.iffr_peer_name[IFNAMSIZ - 1] = '\0'; + peer = ifunit(iffr.iffr_peer_name); + if (peer == NULL) { + error = ENXIO; + break; + } + if (ifnet_type(peer) != IFT_ETHER) { + error = EINVAL; + break; + } + if (strcmp(ifnet_name(peer), FAKE_ETHER_NAME) != 0) { + error = EINVAL; + break; + } + error = feth_config(ifp, peer); + break; + case IF_FAKE_S_CMD_SET_MEDIA: + error = if_fake_request_copyin(user_addr, &iffr, len); + if (error != 0) { + break; + } + error = feth_set_media(ifp, &iffr); + break; + default: + error = EOPNOTSUPP; + break; + } + return (error); +} + +static int +feth_get_drvspec(ifnet_t ifp, u_int32_t cmd, u_int32_t len, + user_addr_t user_addr) +{ + int error = EOPNOTSUPP; + if_fake_ref fakeif; + struct if_fake_request iffr; + ifnet_t peer; + + switch (cmd) { + case IF_FAKE_G_CMD_GET_PEER: + if (len < sizeof(iffr)) { + error = EINVAL; + break; + } + feth_lock(); + fakeif = (if_fake_ref)ifnet_softc(ifp); + if (fakeif == NULL) { + feth_unlock(); + error = EOPNOTSUPP; + break; + } + peer = fakeif->iff_peer; + feth_unlock(); + bzero(&iffr, sizeof(iffr)); + if (peer != NULL) { + strlcpy(iffr.iffr_peer_name, + if_name(peer), + sizeof(iffr.iffr_peer_name)); + } + error = copyout(&iffr, user_addr, sizeof(iffr)); + break; + default: + break; + } + return (error); +} + +union ifdrvu { + struct ifdrv32 *ifdrvu_32; + struct ifdrv64 *ifdrvu_64; + void *ifdrvu_p; +}; + +static int +feth_ioctl(ifnet_t ifp, u_long cmd, void * data) +{ + unsigned int count; + struct ifdevmtu * devmtu_p; + union ifdrvu drv; + uint32_t drv_cmd; + uint32_t drv_len; + boolean_t drv_set_command = FALSE; + int error = 0; + struct ifmediareq * ifmr; + struct ifreq * ifr; + if_fake_ref fakeif; + int status; + user_addr_t user_addr; + + ifr = (struct ifreq *)data; + switch (cmd) { + case SIOCSIFADDR: + ifnet_set_flags(ifp, IFF_UP, IFF_UP); + break; + + case SIOCGIFMEDIA32: + case SIOCGIFMEDIA64: + feth_lock(); + fakeif = (if_fake_ref)ifnet_softc(ifp); + if (fakeif == NULL) { + feth_unlock(); + return (EOPNOTSUPP); + } + status = (fakeif->iff_peer != NULL) + ? (IFM_AVALID | IFM_ACTIVE) : IFM_AVALID; + ifmr = (struct ifmediareq *)data; + user_addr = (cmd == SIOCGIFMEDIA64) ? + ((struct ifmediareq64 *)ifmr)->ifmu_ulist : + CAST_USER_ADDR_T(((struct ifmediareq32 *)ifmr)->ifmu_ulist); + count = ifmr->ifm_count; + ifmr->ifm_active = IFM_ETHER; + ifmr->ifm_current = IFM_ETHER; + ifmr->ifm_mask = 0; + ifmr->ifm_status = status; + if (user_addr == USER_ADDR_NULL) { + ifmr->ifm_count = fakeif->iff_media_count; + } + else if (count > 0) { + if (count > fakeif->iff_media_count) { + count = fakeif->iff_media_count; + } + ifmr->ifm_count = count; + error = copyout(&fakeif->iff_media_list, user_addr, + count * sizeof(int)); + } + feth_unlock(); + break; + + case SIOCGIFDEVMTU: + devmtu_p = &ifr->ifr_devmtu; + devmtu_p->ifdm_current = ifnet_mtu(ifp); + devmtu_p->ifdm_max = feth_max_mtu(); + devmtu_p->ifdm_min = IF_MINMTU; + break; + + case SIOCSIFMTU: + if (ifr->ifr_mtu > feth_max_mtu() || ifr->ifr_mtu < IF_MINMTU) { + error = EINVAL; + } else { + error = ifnet_set_mtu(ifp, ifr->ifr_mtu); + } + break; + + case SIOCSDRVSPEC32: + case SIOCSDRVSPEC64: + error = proc_suser(current_proc()); + if (error != 0) { + break; + } + drv_set_command = TRUE; + /* FALL THROUGH */ + case SIOCGDRVSPEC32: + case SIOCGDRVSPEC64: + drv.ifdrvu_p = data; + if (cmd == SIOCGDRVSPEC32 || cmd == SIOCSDRVSPEC32) { + drv_cmd = drv.ifdrvu_32->ifd_cmd; + drv_len = drv.ifdrvu_32->ifd_len; + user_addr = CAST_USER_ADDR_T(drv.ifdrvu_32->ifd_data); + + } else { + drv_cmd = drv.ifdrvu_64->ifd_cmd; + drv_len = drv.ifdrvu_64->ifd_len; + user_addr = drv.ifdrvu_64->ifd_data; + } + if (drv_set_command) { + error = feth_set_drvspec(ifp, drv_cmd, drv_len, + user_addr); + } else { + error = feth_get_drvspec(ifp, drv_cmd, drv_len, + user_addr); + } + break; + + case SIOCSIFLLADDR: + error = ifnet_set_lladdr(ifp, ifr->ifr_addr.sa_data, + ifr->ifr_addr.sa_len); + break; + + case SIOCSIFFLAGS: + error = 0; + break; + + case SIOCADDMULTI: + case SIOCDELMULTI: + error = 0; + break; + default: + error = EOPNOTSUPP; + break; + } + return error; +} + +static void +feth_if_free(ifnet_t ifp) +{ + if_fake_ref fakeif; + + if (ifp == NULL) { + return; + } + feth_lock(); + fakeif = ifnet_get_if_fake(ifp); + if (fakeif == NULL) { + feth_unlock(); + return; + } + ifp->if_softc = NULL; + feth_unlock(); + feth_release(fakeif); + ifnet_release(ifp); + return; +} + +__private_extern__ void +if_fake_init(void) +{ + int error; + + feth_lock_init(); + error = if_clone_attach(&feth_cloner); + if (error != 0) { + return; + } + return; +} diff --git a/bsd/net/if_fake_var.h b/bsd/net/if_fake_var.h new file mode 100644 index 000000000..b6b147070 --- /dev/null +++ b/bsd/net/if_fake_var.h @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2015-2017 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef _NET_IF_FAKE_VAR_H_ +#define _NET_IF_FAKE_VAR_H_ 1 + +#include <stdint.h> + +#ifdef KERNEL_PRIVATE +__private_extern__ void +if_fake_init(void); +#endif /* KERNEL_PRIVATE */ + +/* + * SIOCSDRVSPEC + */ +enum { + IF_FAKE_S_CMD_NONE = 0, + IF_FAKE_S_CMD_SET_PEER = 1, + IF_FAKE_S_CMD_SET_MEDIA = 2, +}; + +/* + * SIOCGDRVSPEC + */ +enum { + IF_FAKE_G_CMD_NONE = 0, + IF_FAKE_G_CMD_GET_PEER = 1, +}; + +#define IF_FAKE_MEDIA_LIST_MAX 27 + +struct if_fake_media { + int32_t iffm_current; + uint32_t iffm_count; + uint32_t iffm_reserved[3]; + int32_t iffm_list[IF_FAKE_MEDIA_LIST_MAX]; +}; + +struct if_fake_request { + uint64_t iffr_reserved[4]; + union { + char iffru_buf[128]; /* stable size */ + struct if_fake_media iffru_media; + char iffru_peer_name[IFNAMSIZ]; /* if name, e.g. "en0" */ + } iffr_u; +#define iffr_peer_name iffr_u.iffru_peer_name +#define iffr_media iffr_u.iffru_media +}; + +#endif /* _NET_IF_FAKE_VAR_H_ */ diff --git a/bsd/net/if_gif.c b/bsd/net/if_gif.c index f144822ba..9e2e6c6ea 100644 --- a/bsd/net/if_gif.c +++ b/bsd/net/if_gif.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2013 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -319,7 +319,7 @@ static int gif_clone_create(struct if_clone *ifc, uint32_t unit, __unused void *params) { struct gif_softc *sc = NULL; - struct ifnet_init_params gif_init_params; + struct ifnet_init_eparams gif_init_params; errno_t error = 0; lck_mtx_lock(gif_mtx); @@ -345,6 +345,9 @@ gif_clone_create(struct if_clone *ifc, uint32_t unit, __unused void *params) lck_mtx_init(&sc->gif_lock, gif_mtx_grp, gif_mtx_attr); bzero(&gif_init_params, sizeof (gif_init_params)); + gif_init_params.ver = IFNET_INIT_CURRENT_VERSION; + gif_init_params.len = sizeof (gif_init_params); + gif_init_params.flags = IFNET_INIT_LEGACY; gif_init_params.uniqueid = sc->gif_ifname; gif_init_params.uniqueid_len = strlen(sc->gif_ifname); gif_init_params.name = GIFNAME; @@ -360,7 +363,7 @@ gif_clone_create(struct if_clone *ifc, uint32_t unit, __unused void *params) gif_init_params.set_bpf_tap = gif_set_bpf_tap; gif_init_params.detach = gif_detach; - error = ifnet_allocate(&gif_init_params, &sc->gif_if); + error = ifnet_allocate_extended(&gif_init_params, &sc->gif_if); if (error != 0) { printf("gif_clone_create, ifnet_allocate failed - %d\n", error); _FREE(sc, M_DEVBUF); diff --git a/bsd/net/if_gif.h b/bsd/net/if_gif.h index 7fe954f2f..619653bee 100644 --- a/bsd/net/if_gif.h +++ b/bsd/net/if_gif.h @@ -99,7 +99,7 @@ struct gif_softc { #define GIF_LOCK(_sc) lck_mtx_lock(&(_sc)->gif_lock) #define GIF_UNLOCK(_sc) lck_mtx_unlock(&(_sc)->gif_lock) -#define GIF_LOCK_ASSERT(_sc) lck_mtx_assert(&(_sc)->gif_lock, \ +#define GIF_LOCK_ASSERT(_sc) LCK_MTX_ASSERT(&(_sc)->gif_lock, \ LCK_MTX_ASSERT_OWNED) #define gif_ro gifsc_gifscr.gifscr_ro diff --git a/bsd/net/if_ipsec.c b/bsd/net/if_ipsec.c index 9e98a05b9..8ffac41fe 100644 --- a/bsd/net/if_ipsec.c +++ b/bsd/net/if_ipsec.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2015 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -45,14 +45,20 @@ #include <sys/kauth.h> #include <netinet6/ipsec.h> #include <netinet6/ipsec6.h> +#include <netinet6/esp.h> +#include <netinet6/esp6.h> #include <netinet/ip.h> #include <net/flowadv.h> #include <net/necp.h> #include <netkey/key.h> #include <net/pktap.h> +#include <kern/zalloc.h> + +#define IPSEC_NEXUS 0 extern int net_qos_policy_restricted; extern int net_qos_policy_restrict_avapps; +extern unsigned int if_enable_netagent; /* Kernel Control functions */ static errno_t ipsec_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac, @@ -67,7 +73,9 @@ static errno_t ipsec_ctl_setopt(kern_ctl_ref kctlref, u_int32_t unit, void *unit int opt, void *data, size_t len); /* Network Interface functions */ +#if !IPSEC_NEXUS static void ipsec_start(ifnet_t interface); +#endif // !IPSEC_NEXUS static errno_t ipsec_output(ifnet_t interface, mbuf_t data); static errno_t ipsec_demux(ifnet_t interface, mbuf_t data, char *frame_header, protocol_family_t *protocol); @@ -86,116 +94,2068 @@ static errno_t ipsec_proto_pre_output(ifnet_t interface, protocol_family_t proto mbuf_t *packet, const struct sockaddr *dest, void *route, char *frame_type, char *link_layer_dest); -static kern_ctl_ref ipsec_kctlref; -static u_int32_t ipsec_family; +static kern_ctl_ref ipsec_kctlref; +static u_int32_t ipsec_family; +static lck_attr_t *ipsec_lck_attr; +static lck_grp_attr_t *ipsec_lck_grp_attr; +static lck_grp_t *ipsec_lck_grp; +static lck_mtx_t ipsec_lock; + +#if IPSEC_NEXUS + +SYSCTL_DECL(_net_ipsec); +SYSCTL_NODE(_net, OID_AUTO, ipsec, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "IPsec"); +static int if_ipsec_verify_interface_creation = 0; +SYSCTL_INT(_net_ipsec, OID_AUTO, verify_interface_creation, CTLFLAG_RW | CTLFLAG_LOCKED, &if_ipsec_verify_interface_creation, 0, ""); + +#define IPSEC_IF_VERIFY(_e) if (unlikely(if_ipsec_verify_interface_creation)) { VERIFY(_e); } + +#define IPSEC_IF_DEFAULT_SLOT_SIZE 4096 +#define IPSEC_IF_DEFAULT_RING_SIZE 64 +#define IPSEC_IF_DEFAULT_TX_FSW_RING_SIZE 64 +#define IPSEC_IF_DEFAULT_RX_FSW_RING_SIZE 128 + +#define IPSEC_IF_MIN_RING_SIZE 16 +#define IPSEC_IF_MAX_RING_SIZE 1024 + +static int sysctl_if_ipsec_ring_size SYSCTL_HANDLER_ARGS; +static int sysctl_if_ipsec_tx_fsw_ring_size SYSCTL_HANDLER_ARGS; +static int sysctl_if_ipsec_rx_fsw_ring_size SYSCTL_HANDLER_ARGS; + +static int if_ipsec_ring_size = IPSEC_IF_DEFAULT_RING_SIZE; +static int if_ipsec_tx_fsw_ring_size = IPSEC_IF_DEFAULT_TX_FSW_RING_SIZE; +static int if_ipsec_rx_fsw_ring_size = IPSEC_IF_DEFAULT_RX_FSW_RING_SIZE; + +SYSCTL_PROC(_net_ipsec, OID_AUTO, ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW, + &if_ipsec_ring_size, IPSEC_IF_DEFAULT_RING_SIZE, &sysctl_if_ipsec_ring_size, "I", ""); +SYSCTL_PROC(_net_ipsec, OID_AUTO, tx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW, + &if_ipsec_tx_fsw_ring_size, IPSEC_IF_DEFAULT_TX_FSW_RING_SIZE, &sysctl_if_ipsec_tx_fsw_ring_size, "I", ""); +SYSCTL_PROC(_net_ipsec, OID_AUTO, rx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW, + &if_ipsec_rx_fsw_ring_size, IPSEC_IF_DEFAULT_RX_FSW_RING_SIZE, &sysctl_if_ipsec_rx_fsw_ring_size, "I", ""); + +static errno_t +ipsec_register_nexus(void); + +typedef struct ipsec_nx { + uuid_t if_provider; + uuid_t if_instance; + uuid_t ms_provider; + uuid_t ms_instance; + uuid_t ms_device; + uuid_t ms_host; + uuid_t ms_agent; +} *ipsec_nx_t; + +static nexus_controller_t ipsec_ncd; +static int ipsec_ncd_refcount; +static uuid_t ipsec_kpipe_uuid; + +#endif // IPSEC_NEXUS + +/* Control block allocated for each kernel control connection */ +struct ipsec_pcb { + TAILQ_ENTRY(ipsec_pcb) ipsec_chain; + kern_ctl_ref ipsec_ctlref; + ifnet_t ipsec_ifp; + u_int32_t ipsec_unit; + u_int32_t ipsec_unique_id; + u_int32_t ipsec_flags; + u_int32_t ipsec_input_frag_size; + bool ipsec_frag_size_set; + int ipsec_ext_ifdata_stats; + mbuf_svc_class_t ipsec_output_service_class; + char ipsec_if_xname[IFXNAMSIZ]; + char ipsec_unique_name[IFXNAMSIZ]; + // PCB lock protects state fields, like ipsec_kpipe_enabled + decl_lck_rw_data(, ipsec_pcb_lock); + bool ipsec_output_disabled; + +#if IPSEC_NEXUS + lck_mtx_t ipsec_input_chain_lock; + struct mbuf * ipsec_input_chain; + struct mbuf * ipsec_input_chain_last; + // Input chain lock protects the list of input mbufs + // The input chain lock must be taken AFTER the PCB lock if both are held + struct ipsec_nx ipsec_nx; + int ipsec_kpipe_enabled; + uuid_t ipsec_kpipe_uuid; + void * ipsec_kpipe_rxring; + void * ipsec_kpipe_txring; + + kern_nexus_t ipsec_netif_nexus; + void * ipsec_netif_rxring; + void * ipsec_netif_txring; + uint64_t ipsec_netif_txring_size; +#endif // IPSEC_NEXUS +}; + +TAILQ_HEAD(ipsec_list, ipsec_pcb) ipsec_head; + +#define IPSEC_PCB_ZONE_MAX 32 +#define IPSEC_PCB_ZONE_NAME "net.if_ipsec" + +static unsigned int ipsec_pcb_size; /* size of zone element */ +static struct zone *ipsec_pcb_zone; /* zone for ipsec_pcb */ + +#define IPSECQ_MAXLEN 256 + +#if IPSEC_NEXUS +static int +sysctl_if_ipsec_ring_size SYSCTL_HANDLER_ARGS +{ +#pragma unused(arg1, arg2) + int value = if_ipsec_ring_size; + + int error = sysctl_handle_int(oidp, &value, 0, req); + if (error || !req->newptr) { + return (error); + } + + if (value < IPSEC_IF_MIN_RING_SIZE || + value > IPSEC_IF_MAX_RING_SIZE) { + return (EINVAL); + } + + if_ipsec_ring_size = value; + + return (0); +} + +static int +sysctl_if_ipsec_tx_fsw_ring_size SYSCTL_HANDLER_ARGS +{ +#pragma unused(arg1, arg2) + int value = if_ipsec_tx_fsw_ring_size; + + int error = sysctl_handle_int(oidp, &value, 0, req); + if (error || !req->newptr) { + return (error); + } + + if (value < IPSEC_IF_MIN_RING_SIZE || + value > IPSEC_IF_MAX_RING_SIZE) { + return (EINVAL); + } + + if_ipsec_tx_fsw_ring_size = value; + + return (0); +} + +static int +sysctl_if_ipsec_rx_fsw_ring_size SYSCTL_HANDLER_ARGS +{ +#pragma unused(arg1, arg2) + int value = if_ipsec_rx_fsw_ring_size; + + int error = sysctl_handle_int(oidp, &value, 0, req); + if (error || !req->newptr) { + return (error); + } + + if (value < IPSEC_IF_MIN_RING_SIZE || + value > IPSEC_IF_MAX_RING_SIZE) { + return (EINVAL); + } + + if_ipsec_rx_fsw_ring_size = value; + + return (0); +} +#endif // IPSEC_NEXUS + +errno_t +ipsec_register_control(void) +{ + struct kern_ctl_reg kern_ctl; + errno_t result = 0; + + /* Find a unique value for our interface family */ + result = mbuf_tag_id_find(IPSEC_CONTROL_NAME, &ipsec_family); + if (result != 0) { + printf("ipsec_register_control - mbuf_tag_id_find_internal failed: %d\n", result); + return result; + } + + ipsec_pcb_size = sizeof(struct ipsec_pcb); + ipsec_pcb_zone = zinit(ipsec_pcb_size, + IPSEC_PCB_ZONE_MAX * ipsec_pcb_size, + 0, IPSEC_PCB_ZONE_NAME); + if (ipsec_pcb_zone == NULL) { + printf("ipsec_register_control - zinit(ipsec_pcb) failed"); + return ENOMEM; + } + +#if IPSEC_NEXUS + ipsec_register_nexus(); +#endif // IPSEC_NEXUS + + TAILQ_INIT(&ipsec_head); + + bzero(&kern_ctl, sizeof(kern_ctl)); + strlcpy(kern_ctl.ctl_name, IPSEC_CONTROL_NAME, sizeof(kern_ctl.ctl_name)); + kern_ctl.ctl_name[sizeof(kern_ctl.ctl_name) - 1] = 0; + kern_ctl.ctl_flags = CTL_FLAG_PRIVILEGED; /* Require root */ + kern_ctl.ctl_sendsize = 64 * 1024; + kern_ctl.ctl_recvsize = 64 * 1024; + kern_ctl.ctl_connect = ipsec_ctl_connect; + kern_ctl.ctl_disconnect = ipsec_ctl_disconnect; + kern_ctl.ctl_send = ipsec_ctl_send; + kern_ctl.ctl_setopt = ipsec_ctl_setopt; + kern_ctl.ctl_getopt = ipsec_ctl_getopt; + + result = ctl_register(&kern_ctl, &ipsec_kctlref); + if (result != 0) { + printf("ipsec_register_control - ctl_register failed: %d\n", result); + return result; + } + + /* Register the protocol plumbers */ + if ((result = proto_register_plumber(PF_INET, ipsec_family, + ipsec_attach_proto, NULL)) != 0) { + printf("ipsec_register_control - proto_register_plumber(PF_INET, %d) failed: %d\n", + ipsec_family, result); + ctl_deregister(ipsec_kctlref); + return result; + } + + /* Register the protocol plumbers */ + if ((result = proto_register_plumber(PF_INET6, ipsec_family, + ipsec_attach_proto, NULL)) != 0) { + proto_unregister_plumber(PF_INET, ipsec_family); + ctl_deregister(ipsec_kctlref); + printf("ipsec_register_control - proto_register_plumber(PF_INET6, %d) failed: %d\n", + ipsec_family, result); + return result; + } + + ipsec_lck_attr = lck_attr_alloc_init(); + ipsec_lck_grp_attr = lck_grp_attr_alloc_init(); + ipsec_lck_grp = lck_grp_alloc_init("ipsec", ipsec_lck_grp_attr); + lck_mtx_init(&ipsec_lock, ipsec_lck_grp, ipsec_lck_attr); + + return 0; +} + +/* Helpers */ +int +ipsec_interface_isvalid (ifnet_t interface) +{ + struct ipsec_pcb *pcb = NULL; + + if (interface == NULL) + return 0; + + pcb = ifnet_softc(interface); + + if (pcb == NULL) + return 0; + + /* When ctl disconnects, ipsec_unit is set to 0 */ + if (pcb->ipsec_unit == 0) + return 0; + + return 1; +} + +static errno_t +ipsec_ifnet_set_attrs(ifnet_t ifp) +{ + /* Set flags and additional information. */ + ifnet_set_mtu(ifp, 1500); + ifnet_set_flags(ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff); + + /* The interface must generate its own IPv6 LinkLocal address, + * if possible following the recommendation of RFC2472 to the 64bit interface ID + */ + ifnet_set_eflags(ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL); + +#if !IPSEC_NEXUS + /* Reset the stats in case as the interface may have been recycled */ + struct ifnet_stats_param stats; + bzero(&stats, sizeof(struct ifnet_stats_param)); + ifnet_set_stat(ifp, &stats); +#endif // !IPSEC_NEXUS + + return (0); +} + +#if IPSEC_NEXUS + +static uuid_t ipsec_nx_dom_prov; + +static errno_t +ipsec_nxdp_init(__unused kern_nexus_domain_provider_t domprov) +{ + return 0; +} + +static void +ipsec_nxdp_fini(__unused kern_nexus_domain_provider_t domprov) +{ + // Ignore +} + +static errno_t +ipsec_register_nexus(void) +{ + const struct kern_nexus_domain_provider_init dp_init = { + .nxdpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION, + .nxdpi_flags = 0, + .nxdpi_init = ipsec_nxdp_init, + .nxdpi_fini = ipsec_nxdp_fini + }; + errno_t err = 0; + + /* ipsec_nxdp_init() is called before this function returns */ + err = kern_nexus_register_domain_provider(NEXUS_TYPE_NET_IF, + (const uint8_t *) "com.apple.ipsec", + &dp_init, sizeof(dp_init), + &ipsec_nx_dom_prov); + if (err != 0) { + printf("%s: failed to register domain provider\n", __func__); + return (err); + } + return (0); +} + +static errno_t +ipsec_netif_prepare(kern_nexus_t nexus, ifnet_t ifp) +{ + struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); + pcb->ipsec_netif_nexus = nexus; + return (ipsec_ifnet_set_attrs(ifp)); +} + +static errno_t +ipsec_nexus_pre_connect(kern_nexus_provider_t nxprov, + proc_t p, kern_nexus_t nexus, + nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx) +{ +#pragma unused(nxprov, p) +#pragma unused(nexus, nexus_port, channel, ch_ctx) + return (0); +} + +static errno_t +ipsec_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel) +{ +#pragma unused(nxprov, channel) + struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); + boolean_t ok = ifnet_is_attached(pcb->ipsec_ifp, 1); + return (ok ? 0 : ENXIO); +} + +static void +ipsec_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel) +{ +#pragma unused(nxprov, nexus, channel) +} + +static void +ipsec_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel) +{ +#pragma unused(nxprov, nexus, channel) +} + +static void +ipsec_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel) +{ +#pragma unused(nxprov, channel) + struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); + if (pcb->ipsec_netif_nexus == nexus) { + pcb->ipsec_netif_nexus = NULL; + } + ifnet_decr_iorefcnt(pcb->ipsec_ifp); +} + +static errno_t +ipsec_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring, + void **ring_ctx) +{ +#pragma unused(nxprov) +#pragma unused(channel) +#pragma unused(ring_ctx) + struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); + if (!is_tx_ring) { + VERIFY(pcb->ipsec_kpipe_rxring == NULL); + pcb->ipsec_kpipe_rxring = ring; + } else { + VERIFY(pcb->ipsec_kpipe_txring == NULL); + pcb->ipsec_kpipe_txring = ring; + } + return 0; +} + +static void +ipsec_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t ring) +{ +#pragma unused(nxprov) + struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); + if (pcb->ipsec_kpipe_rxring == ring) { + pcb->ipsec_kpipe_rxring = NULL; + } else if (pcb->ipsec_kpipe_txring == ring) { + pcb->ipsec_kpipe_txring = NULL; + } +} + +static errno_t +ipsec_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t tx_ring, uint32_t flags) +{ +#pragma unused(nxprov) +#pragma unused(flags) + struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); + + lck_rw_lock_shared(&pcb->ipsec_pcb_lock); + int channel_enabled = pcb->ipsec_kpipe_enabled; + if (!channel_enabled) { + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + return 0; + } + + kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); + if (tx_slot == NULL) { + // Nothing to write, bail + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + return 0; + } + + // Signal the netif ring to read + kern_channel_ring_t rx_ring = pcb->ipsec_netif_rxring; + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + + if (rx_ring != NULL) { + kern_channel_notify(rx_ring, 0); + } + return 0; +} + +static mbuf_t +ipsec_encrypt_mbuf(ifnet_t interface, + mbuf_t data) +{ + struct ipsec_output_state ipsec_state; + int error = 0; + uint32_t af; + + // Make sure this packet isn't looping through the interface + if (necp_get_last_interface_index_from_packet(data) == interface->if_index) { + error = -1; + goto ipsec_output_err; + } + + // Mark the interface so NECP can evaluate tunnel policy + necp_mark_packet_from_interface(data, interface); + + struct ip *ip = mtod(data, struct ip *); + u_int ip_version = ip->ip_v; + + switch (ip_version) { + case 4: { + af = AF_INET; + + memset(&ipsec_state, 0, sizeof(ipsec_state)); + ipsec_state.m = data; + ipsec_state.dst = (struct sockaddr *)&ip->ip_dst; + memset(&ipsec_state.ro, 0, sizeof(ipsec_state.ro)); + + error = ipsec4_interface_output(&ipsec_state, interface); + if (error == 0 && ipsec_state.tunneled == 6) { + // Tunneled in IPv6 - packet is gone + // TODO: Don't lose mbuf + data = NULL; + goto done; + } + + data = ipsec_state.m; + if (error || data == NULL) { + if (error) { + printf("ipsec_encrypt_mbuf: ipsec4_output error %d\n", error); + } + goto ipsec_output_err; + } + goto done; + } + case 6: { + af = AF_INET6; + + data = ipsec6_splithdr(data); + if (data == NULL) { + printf("ipsec_encrypt_mbuf: ipsec6_splithdr returned NULL\n"); + goto ipsec_output_err; + } + + struct ip6_hdr *ip6 = mtod(data, struct ip6_hdr *); + + memset(&ipsec_state, 0, sizeof(ipsec_state)); + ipsec_state.m = data; + ipsec_state.dst = (struct sockaddr *)&ip6->ip6_dst; + memset(&ipsec_state.ro, 0, sizeof(ipsec_state.ro)); + + error = ipsec6_interface_output(&ipsec_state, interface, &ip6->ip6_nxt, ipsec_state.m); + if (error == 0 && ipsec_state.tunneled == 4) { + // Tunneled in IPv4 - packet is gone + // TODO: Don't lose mbuf + data = NULL; + goto done; + } + data = ipsec_state.m; + if (error || data == NULL) { + if (error) { + printf("ipsec_encrypt_mbuf: ipsec6_output error %d\n", error); + } + goto ipsec_output_err; + } + goto done; + } + default: { + printf("ipsec_encrypt_mbuf: Received unknown packet version %d\n", ip_version); + error = -1; + goto ipsec_output_err; + } + } + +done: + return data; + +ipsec_output_err: + if (data) { + mbuf_freem(data); + } + return NULL; +} + +static errno_t +ipsec_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t rx_ring, uint32_t flags) +{ +#pragma unused(nxprov) +#pragma unused(flags) + struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); + struct kern_channel_ring_stat_increment rx_ring_stats; + + lck_rw_lock_shared(&pcb->ipsec_pcb_lock); + + int channel_enabled = pcb->ipsec_kpipe_enabled; + if (!channel_enabled) { + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + return 0; + } + + // Reclaim user-released slots + (void) kern_channel_reclaim(rx_ring); + + uint32_t avail = kern_channel_available_slot_count(rx_ring); + if (avail == 0) { + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + return 0; + } + + kern_channel_ring_t tx_ring = pcb->ipsec_netif_txring; + if (tx_ring == NULL) { + // Net-If TX ring not set up yet, nothing to read + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + return 0; + } + + struct netif_stats *nifs = &NX_NETIF_PRIVATE(pcb->ipsec_netif_nexus)->nif_stats; + + // Unlock ipsec before entering ring + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + + (void)kr_enter(tx_ring, TRUE); + + // Lock again after entering and validate + lck_rw_lock_shared(&pcb->ipsec_pcb_lock); + if (tx_ring != pcb->ipsec_netif_txring) { + // Ring no longer valid + // Unlock first, then exit ring + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + kr_exit(tx_ring); + return 0; + } + + + struct kern_channel_ring_stat_increment tx_ring_stats; + bzero(&tx_ring_stats, sizeof(tx_ring_stats)); + kern_channel_slot_t tx_pslot = NULL; + kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); + if (tx_slot == NULL) { + // Nothing to read, don't bother signalling + // Unlock first, then exit ring + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + kr_exit(tx_ring); + return 0; + } + + struct kern_pbufpool *rx_pp = rx_ring->ckr_pp; + VERIFY(rx_pp != NULL); + bzero(&rx_ring_stats, sizeof(rx_ring_stats)); + kern_channel_slot_t rx_pslot = NULL; + kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL); + + while (rx_slot != NULL && tx_slot != NULL) { + size_t length = 0; + mbuf_t data = NULL; + errno_t error = 0; + + // Allocate rx packet + kern_packet_t rx_ph = 0; + error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph); + if (unlikely(error != 0)) { + printf("ipsec_kpipe_sync_rx %s: failed to allocate packet\n", + pcb->ipsec_ifp->if_xname); + break; + } + + kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot); + + // Advance TX ring + tx_pslot = tx_slot; + tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL); + + if (tx_ph == 0) { + continue; + } + + kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL); + VERIFY(tx_buf != NULL); + uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf); + VERIFY(tx_baddr != NULL); + tx_baddr += kern_buflet_get_data_offset(tx_buf); + + bpf_tap_packet_out(pcb->ipsec_ifp, DLT_RAW, tx_ph, NULL, 0); + + length = MIN(kern_packet_get_data_length(tx_ph), + IPSEC_IF_DEFAULT_SLOT_SIZE); + + // Increment TX stats + tx_ring_stats.kcrsi_slots_transferred++; + tx_ring_stats.kcrsi_bytes_transferred += length; + + if (length > 0) { + error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data); + if (error == 0) { + error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_DONTWAIT); + if (error == 0) { + // Encrypt and send packet + data = ipsec_encrypt_mbuf(pcb->ipsec_ifp, data); + } else { + printf("ipsec_kpipe_sync_rx %s - mbuf_copyback(%zu) error %d\n", pcb->ipsec_ifp->if_xname, length, error); + STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF); + STATS_INC(nifs, NETIF_STATS_DROPPED); + mbuf_freem(data); + data = NULL; + } + } else { + printf("ipsec_kpipe_sync_rx %s - mbuf_gethdr error %d\n", pcb->ipsec_ifp->if_xname, error); + STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF); + STATS_INC(nifs, NETIF_STATS_DROPPED); + } + } else { + printf("ipsec_kpipe_sync_rx %s - 0 length packet\n", pcb->ipsec_ifp->if_xname); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + } + + if (data == NULL) { + printf("ipsec_kpipe_sync_rx %s: no encrypted packet to send\n", pcb->ipsec_ifp->if_xname); + kern_pbufpool_free(rx_pp, rx_ph); + break; + } + + length = mbuf_pkthdr_len(data); + if (length > rx_pp->pp_buflet_size) { + // Flush data + mbuf_freem(data); + kern_pbufpool_free(rx_pp, rx_ph); + printf("ipsec_kpipe_sync_rx %s: encrypted packet length %zu > %u\n", + pcb->ipsec_ifp->if_xname, length, rx_pp->pp_buflet_size); + continue; + } + + // Fillout rx packet + kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL); + VERIFY(rx_buf != NULL); + void *rx_baddr = kern_buflet_get_object_address(rx_buf); + VERIFY(rx_baddr != NULL); + + // Copy-in data from mbuf to buflet + mbuf_copydata(data, 0, length, (void *)rx_baddr); + kern_packet_clear_flow_uuid(rx_ph); // Zero flow id + + // Finalize and attach the packet + error = kern_buflet_set_data_offset(rx_buf, 0); + VERIFY(error == 0); + error = kern_buflet_set_data_length(rx_buf, length); + VERIFY(error == 0); + error = kern_packet_finalize(rx_ph); + VERIFY(error == 0); + error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph); + VERIFY(error == 0); + + STATS_INC(nifs, NETIF_STATS_TXPKTS); + STATS_INC(nifs, NETIF_STATS_TXCOPY_DIRECT); + + rx_ring_stats.kcrsi_slots_transferred++; + rx_ring_stats.kcrsi_bytes_transferred += length; + + if (!pcb->ipsec_ext_ifdata_stats) { + ifnet_stat_increment_out(pcb->ipsec_ifp, 1, length, 0); + } + + mbuf_freem(data); + + rx_pslot = rx_slot; + rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL); + } + + if (rx_pslot) { + kern_channel_advance_slot(rx_ring, rx_pslot); + kern_channel_increment_ring_net_stats(rx_ring, pcb->ipsec_ifp, &rx_ring_stats); + } + + if (tx_pslot) { + kern_channel_advance_slot(tx_ring, tx_pslot); + kern_channel_increment_ring_net_stats(tx_ring, pcb->ipsec_ifp, &tx_ring_stats); + (void)kern_channel_reclaim(tx_ring); + } + + if (pcb->ipsec_output_disabled) { + errno_t error = ifnet_enable_output(pcb->ipsec_ifp); + if (error != 0) { + printf("ipsec_kpipe_sync_rx: ifnet_enable_output returned error %d\n", error); + } else { + pcb->ipsec_output_disabled = false; + } + } + + // Unlock first, then exit ring + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + + if (tx_pslot != NULL) { + kern_channel_notify(tx_ring, 0); + } + kr_exit(tx_ring); + + return 0; +} + +static errno_t +ipsec_netif_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring, + void **ring_ctx) +{ +#pragma unused(nxprov) +#pragma unused(channel) +#pragma unused(ring_ctx) + struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); + if (!is_tx_ring) { + VERIFY(pcb->ipsec_netif_rxring == NULL); + pcb->ipsec_netif_rxring = ring; + } else { + VERIFY(pcb->ipsec_netif_txring == NULL); + pcb->ipsec_netif_txring = ring; + } + return 0; +} + +static void +ipsec_netif_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t ring) +{ +#pragma unused(nxprov) + struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); + if (pcb->ipsec_netif_rxring == ring) { + pcb->ipsec_netif_rxring = NULL; + } else if (pcb->ipsec_netif_txring == ring) { + pcb->ipsec_netif_txring = NULL; + } +} + +static bool +ipsec_netif_check_policy(mbuf_t data) +{ + necp_kernel_policy_result necp_result = 0; + necp_kernel_policy_result_parameter necp_result_parameter = {}; + uint32_t necp_matched_policy_id = 0; + + // This packet has been marked with IP level policy, do not mark again. + if (data && data->m_pkthdr.necp_mtag.necp_policy_id >= NECP_KERNEL_POLICY_ID_FIRST_VALID_IP) { + return (true); + } + + size_t length = mbuf_pkthdr_len(data); + if (length < sizeof(struct ip)) { + return (false); + } + + struct ip *ip = mtod(data, struct ip *); + u_int ip_version = ip->ip_v; + switch (ip_version) { + case 4: { + necp_matched_policy_id = necp_ip_output_find_policy_match(data, 0, NULL, + &necp_result, &necp_result_parameter); + break; + } + case 6: { + necp_matched_policy_id = necp_ip6_output_find_policy_match(data, 0, NULL, + &necp_result, &necp_result_parameter); + break; + } + default: { + return (false); + } + } + + if (necp_result == NECP_KERNEL_POLICY_RESULT_DROP || + necp_result == NECP_KERNEL_POLICY_RESULT_SOCKET_DIVERT) { + /* Drop and flow divert packets should be blocked at the IP layer */ + return (false); + } + + necp_mark_packet_from_ip(data, necp_matched_policy_id); + return (true); +} + +static errno_t +ipsec_netif_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t tx_ring, uint32_t flags) +{ +#pragma unused(nxprov) +#pragma unused(flags) + struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); + + struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats; + + lck_rw_lock_shared(&pcb->ipsec_pcb_lock); + + struct kern_channel_ring_stat_increment tx_ring_stats; + bzero(&tx_ring_stats, sizeof(tx_ring_stats)); + kern_channel_slot_t tx_pslot = NULL; + kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); + + STATS_INC(nifs, NETIF_STATS_TXSYNC); + + if (tx_slot == NULL) { + // Nothing to write, don't bother signalling + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + return 0; + } + + if (pcb->ipsec_kpipe_enabled) { + kern_channel_ring_t rx_ring = pcb->ipsec_kpipe_rxring; + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + + // Signal the kernel pipe ring to read + if (rx_ring != NULL) { + kern_channel_notify(rx_ring, 0); + } + return 0; + } + + // If we're here, we're injecting into the BSD stack + while (tx_slot != NULL) { + size_t length = 0; + mbuf_t data = NULL; + + kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot); + + // Advance TX ring + tx_pslot = tx_slot; + tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL); + + if (tx_ph == 0) { + continue; + } + + kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL); + VERIFY(tx_buf != NULL); + uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf); + VERIFY(tx_baddr != 0); + tx_baddr += kern_buflet_get_data_offset(tx_buf); + + bpf_tap_packet_out(pcb->ipsec_ifp, DLT_RAW, tx_ph, NULL, 0); + + length = MIN(kern_packet_get_data_length(tx_ph), + IPSEC_IF_DEFAULT_SLOT_SIZE); + + if (length > 0) { + errno_t error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data); + if (error == 0) { + error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_DONTWAIT); + if (error == 0) { + // Mark packet from policy + uint32_t policy_id = kern_packet_get_policy_id(tx_ph); + necp_mark_packet_from_ip(data, policy_id); + + // Check policy with NECP + if (!ipsec_netif_check_policy(data)) { + printf("ipsec_netif_sync_tx %s - failed policy check\n", pcb->ipsec_ifp->if_xname); + STATS_INC(nifs, NETIF_STATS_DROPPED); + mbuf_freem(data); + data = NULL; + } else { + // Send through encryption + error = ipsec_output(pcb->ipsec_ifp, data); + if (error != 0) { + printf("ipsec_netif_sync_tx %s - ipsec_output error %d\n", pcb->ipsec_ifp->if_xname, error); + } + } + } else { + printf("ipsec_netif_sync_tx %s - mbuf_copyback(%zu) error %d\n", pcb->ipsec_ifp->if_xname, length, error); + STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF); + STATS_INC(nifs, NETIF_STATS_DROPPED); + mbuf_freem(data); + data = NULL; + } + } else { + printf("ipsec_netif_sync_tx %s - mbuf_gethdr error %d\n", pcb->ipsec_ifp->if_xname, error); + STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF); + STATS_INC(nifs, NETIF_STATS_DROPPED); + } + } else { + printf("ipsec_netif_sync_tx %s - 0 length packet\n", pcb->ipsec_ifp->if_xname); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + } + + if (data == NULL) { + printf("ipsec_netif_sync_tx %s: no encrypted packet to send\n", pcb->ipsec_ifp->if_xname); + break; + } + + STATS_INC(nifs, NETIF_STATS_TXPKTS); + STATS_INC(nifs, NETIF_STATS_TXCOPY_MBUF); + + tx_ring_stats.kcrsi_slots_transferred++; + tx_ring_stats.kcrsi_bytes_transferred += length; + } + + if (tx_pslot) { + kern_channel_advance_slot(tx_ring, tx_pslot); + kern_channel_increment_ring_net_stats(tx_ring, pcb->ipsec_ifp, &tx_ring_stats); + (void)kern_channel_reclaim(tx_ring); + } + + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + + return 0; +} + +static errno_t +ipsec_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t ring, __unused uint32_t flags) +{ +#pragma unused(nxprov) + struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); + + lck_rw_lock_shared(&pcb->ipsec_pcb_lock); + + boolean_t more = false; + errno_t rc = 0; + do { + rc = kern_channel_tx_refill(ring, UINT32_MAX, UINT32_MAX, true, &more); + if (rc != 0 && rc != EAGAIN && rc != EBUSY) { + printf("%s, tx refill failed %d\n", __func__, rc); + } + } while ((rc == 0) && more); + + if (pcb->ipsec_kpipe_enabled && !pcb->ipsec_output_disabled) { + uint32_t tx_available = kern_channel_available_slot_count(ring); + if (pcb->ipsec_netif_txring_size > 0 && + tx_available >= pcb->ipsec_netif_txring_size - 1) { + // No room left in tx ring, disable output for now + errno_t error = ifnet_disable_output(pcb->ipsec_ifp); + if (error != 0) { + printf("ipsec_netif_tx_doorbell: ifnet_disable_output returned error %d\n", error); + } else { + pcb->ipsec_output_disabled = true; + } + } + } + + if (pcb->ipsec_kpipe_enabled && + (((rc != 0) && (rc != EAGAIN)) || pcb->ipsec_output_disabled)) { + kern_channel_ring_t rx_ring = pcb->ipsec_kpipe_rxring; + + // Unlock while calling notify + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + // Signal the kernel pipe ring to read + if (rx_ring != NULL) { + kern_channel_notify(rx_ring, 0); + } + lck_rw_lock_shared(&pcb->ipsec_pcb_lock); + } else { + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + } + + return (0); +} + +static errno_t +ipsec_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t rx_ring, uint32_t flags) +{ +#pragma unused(nxprov) +#pragma unused(flags) + struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); + struct kern_channel_ring_stat_increment rx_ring_stats; + + struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats; + + lck_rw_lock_shared(&pcb->ipsec_pcb_lock); + + // Reclaim user-released slots + (void) kern_channel_reclaim(rx_ring); + + STATS_INC(nifs, NETIF_STATS_RXSYNC); + + uint32_t avail = kern_channel_available_slot_count(rx_ring); + if (avail == 0) { + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + return 0; + } + + struct kern_pbufpool *rx_pp = rx_ring->ckr_pp; + VERIFY(rx_pp != NULL); + bzero(&rx_ring_stats, sizeof(rx_ring_stats)); + kern_channel_slot_t rx_pslot = NULL; + kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL); + + while (rx_slot != NULL) { + // Check for a waiting packet + lck_mtx_lock(&pcb->ipsec_input_chain_lock); + mbuf_t data = pcb->ipsec_input_chain; + if (data == NULL) { + lck_mtx_unlock(&pcb->ipsec_input_chain_lock); + break; + } + + // Allocate rx packet + kern_packet_t rx_ph = 0; + errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph); + if (unlikely(error != 0)) { + STATS_INC(nifs, NETIF_STATS_NOMEM_PKT); + STATS_INC(nifs, NETIF_STATS_DROPPED); + printf("ipsec_netif_sync_rx %s: failed to allocate packet\n", + pcb->ipsec_ifp->if_xname); + lck_mtx_unlock(&pcb->ipsec_input_chain_lock); + break; + } + + // Advance waiting packets + pcb->ipsec_input_chain = data->m_nextpkt; + data->m_nextpkt = NULL; + if (pcb->ipsec_input_chain == NULL) { + pcb->ipsec_input_chain_last = NULL; + } + lck_mtx_unlock(&pcb->ipsec_input_chain_lock); + + size_t length = mbuf_pkthdr_len(data); + + if (length < sizeof(struct ip)) { + // Flush data + mbuf_freem(data); + kern_pbufpool_free(rx_pp, rx_ph); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + printf("ipsec_netif_sync_rx %s: legacy decrypted packet length cannot hold IP %zu < %zu\n", + pcb->ipsec_ifp->if_xname, length, sizeof(struct ip)); + continue; + } + + uint32_t af = 0; + struct ip *ip = mtod(data, struct ip *); + u_int ip_version = ip->ip_v; + switch (ip_version) { + case 4: { + af = AF_INET; + break; + } + case 6: { + af = AF_INET6; + break; + } + default: { + printf("ipsec_netif_sync_rx %s: legacy unknown ip version %u\n", + pcb->ipsec_ifp->if_xname, ip_version); + break; + } + } + + if (length > rx_pp->pp_buflet_size || + (pcb->ipsec_frag_size_set && length > pcb->ipsec_input_frag_size)) { + + // We need to fragment to send up into the netif + + u_int32_t fragment_mtu = rx_pp->pp_buflet_size; + if (pcb->ipsec_frag_size_set && + pcb->ipsec_input_frag_size < rx_pp->pp_buflet_size) { + fragment_mtu = pcb->ipsec_input_frag_size; + } + + mbuf_t fragment_chain = NULL; + switch (af) { + case AF_INET: { + // ip_fragment expects the length in host order + ip->ip_len = ntohs(ip->ip_len); + + // ip_fragment will modify the original data, don't free + int fragment_error = ip_fragment(data, pcb->ipsec_ifp, fragment_mtu, TRUE); + if (fragment_error == 0 && data != NULL) { + fragment_chain = data; + } else { + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + printf("ipsec_netif_sync_rx %s: failed to fragment IPv4 packet of length %zu (%d)\n", + pcb->ipsec_ifp->if_xname, length, fragment_error); + } + break; + } + case AF_INET6: { + if (length < sizeof(struct ip6_hdr)) { + mbuf_freem(data); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + printf("ipsec_netif_sync_rx %s: failed to fragment IPv6 packet of length %zu < %zu\n", + pcb->ipsec_ifp->if_xname, length, sizeof(struct ip6_hdr)); + } else { + + // ip6_do_fragmentation will free the original data on success only + struct ip6_hdr *ip6 = mtod(data, struct ip6_hdr *); + struct ip6_exthdrs exthdrs; + memset(&exthdrs, 0, sizeof(exthdrs)); + + int fragment_error = ip6_do_fragmentation(&data, 0, pcb->ipsec_ifp, sizeof(struct ip6_hdr), + ip6, &exthdrs, fragment_mtu, ip6->ip6_nxt); + if (fragment_error == 0 && data != NULL) { + fragment_chain = data; + } else { + mbuf_freem(data); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + printf("ipsec_netif_sync_rx %s: failed to fragment IPv6 packet of length %zu (%d)\n", + pcb->ipsec_ifp->if_xname, length, fragment_error); + } + } + break; + } + default: { + // Cannot fragment unknown families + mbuf_freem(data); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + printf("ipsec_netif_sync_rx %s: uknown legacy decrypted packet length %zu > %u\n", + pcb->ipsec_ifp->if_xname, length, rx_pp->pp_buflet_size); + break; + } + } + + if (fragment_chain != NULL) { + // Add fragments to chain before continuing + lck_mtx_lock(&pcb->ipsec_input_chain_lock); + if (pcb->ipsec_input_chain != NULL) { + pcb->ipsec_input_chain_last->m_nextpkt = fragment_chain; + } else { + pcb->ipsec_input_chain = fragment_chain; + } + while (fragment_chain->m_nextpkt) { + VERIFY(fragment_chain != fragment_chain->m_nextpkt); + fragment_chain = fragment_chain->m_nextpkt; + } + pcb->ipsec_input_chain_last = fragment_chain; + lck_mtx_unlock(&pcb->ipsec_input_chain_lock); + } + + // Make sure to free unused rx packet + kern_pbufpool_free(rx_pp, rx_ph); + + continue; + } + + mbuf_pkthdr_setrcvif(data, pcb->ipsec_ifp); + + // Fillout rx packet + kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL); + VERIFY(rx_buf != NULL); + void *rx_baddr = kern_buflet_get_object_address(rx_buf); + VERIFY(rx_baddr != NULL); + + // Copy-in data from mbuf to buflet + mbuf_copydata(data, 0, length, (void *)rx_baddr); + kern_packet_clear_flow_uuid(rx_ph); // Zero flow id + + // Finalize and attach the packet + error = kern_buflet_set_data_offset(rx_buf, 0); + VERIFY(error == 0); + error = kern_buflet_set_data_length(rx_buf, length); + VERIFY(error == 0); + error = kern_packet_set_link_header_offset(rx_ph, 0); + VERIFY(error == 0); + error = kern_packet_set_network_header_offset(rx_ph, 0); + VERIFY(error == 0); + error = kern_packet_finalize(rx_ph); + VERIFY(error == 0); + error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph); + VERIFY(error == 0); + + STATS_INC(nifs, NETIF_STATS_RXPKTS); + STATS_INC(nifs, NETIF_STATS_RXCOPY_MBUF); + bpf_tap_packet_in(pcb->ipsec_ifp, DLT_RAW, rx_ph, NULL, 0); + + rx_ring_stats.kcrsi_slots_transferred++; + rx_ring_stats.kcrsi_bytes_transferred += length; + + if (!pcb->ipsec_ext_ifdata_stats) { + ifnet_stat_increment_in(pcb->ipsec_ifp, 1, length, 0); + } + + mbuf_freem(data); + + // Advance ring + rx_pslot = rx_slot; + rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL); + } + + struct kern_channel_ring_stat_increment tx_ring_stats; + bzero(&tx_ring_stats, sizeof(tx_ring_stats)); + kern_channel_ring_t tx_ring = pcb->ipsec_kpipe_txring; + kern_channel_slot_t tx_pslot = NULL; + kern_channel_slot_t tx_slot = NULL; + if (tx_ring == NULL) { + // Net-If TX ring not set up yet, nothing to read + goto done; + } + + + // Unlock ipsec before entering ring + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + + (void)kr_enter(tx_ring, TRUE); + + // Lock again after entering and validate + lck_rw_lock_shared(&pcb->ipsec_pcb_lock); + + if (tx_ring != pcb->ipsec_kpipe_txring) { + goto done; + } + + tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); + if (tx_slot == NULL) { + // Nothing to read, don't bother signalling + goto done; + } + + while (rx_slot != NULL && tx_slot != NULL) { + size_t length = 0; + mbuf_t data = NULL; + errno_t error = 0; + uint32_t af; + + // Allocate rx packet + kern_packet_t rx_ph = 0; + error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph); + if (unlikely(error != 0)) { + STATS_INC(nifs, NETIF_STATS_NOMEM_PKT); + STATS_INC(nifs, NETIF_STATS_DROPPED); + printf("ipsec_netif_sync_rx %s: failed to allocate packet\n", + pcb->ipsec_ifp->if_xname); + break; + } + + kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot); + + // Advance TX ring + tx_pslot = tx_slot; + tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL); + + if (tx_ph == 0) { + continue; + } + + kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL); + VERIFY(tx_buf != NULL); + uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf); + VERIFY(tx_baddr != 0); + tx_baddr += kern_buflet_get_data_offset(tx_buf); + + length = MIN(kern_packet_get_data_length(tx_ph), + IPSEC_IF_DEFAULT_SLOT_SIZE); + + // Increment TX stats + tx_ring_stats.kcrsi_slots_transferred++; + tx_ring_stats.kcrsi_bytes_transferred += length; + + if (length >= sizeof(struct ip)) { + error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data); + if (error == 0) { + error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_DONTWAIT); + if (error == 0) { + struct ip *ip = mtod(data, struct ip *); + u_int ip_version = ip->ip_v; + switch (ip_version) { + case 4: { + af = AF_INET; + ip->ip_len = ntohs(ip->ip_len) - sizeof(struct ip); + ip->ip_off = ntohs(ip->ip_off); + + if (length < ip->ip_len) { + printf("ipsec_netif_sync_rx %s: IPv4 packet length too short (%zu < %u)\n", + pcb->ipsec_ifp->if_xname, length, ip->ip_len); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + mbuf_freem(data); + data = NULL; + } else { + data = esp4_input_extended(data, sizeof(struct ip), pcb->ipsec_ifp); + } + break; + } + case 6: { + if (length < sizeof(struct ip6_hdr)) { + printf("ipsec_netif_sync_rx %s: IPv6 packet length too short for header %zu\n", + pcb->ipsec_ifp->if_xname, length); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + mbuf_freem(data); + data = NULL; + } else { + af = AF_INET6; + struct ip6_hdr *ip6 = mtod(data, struct ip6_hdr *); + const size_t ip6_len = sizeof(*ip6) + ntohs(ip6->ip6_plen); + if (length < ip6_len) { + printf("ipsec_netif_sync_rx %s: IPv6 packet length too short (%zu < %zu)\n", + pcb->ipsec_ifp->if_xname, length, ip6_len); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + mbuf_freem(data); + data = NULL; + } else { + int offset = sizeof(struct ip6_hdr); + esp6_input_extended(&data, &offset, ip6->ip6_nxt, pcb->ipsec_ifp); + } + } + break; + } + default: { + printf("ipsec_netif_sync_rx %s: unknown ip version %u\n", + pcb->ipsec_ifp->if_xname, ip_version); + STATS_INC(nifs, NETIF_STATS_DROPPED); + mbuf_freem(data); + data = NULL; + break; + } + } + } else { + printf("ipsec_netif_sync_rx %s - mbuf_copyback(%zu) error %d\n", pcb->ipsec_ifp->if_xname, length, error); + STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF); + STATS_INC(nifs, NETIF_STATS_DROPPED); + mbuf_freem(data); + data = NULL; + } + } else { + printf("ipsec_netif_sync_rx %s - mbuf_gethdr error %d\n", pcb->ipsec_ifp->if_xname, error); + STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF); + STATS_INC(nifs, NETIF_STATS_DROPPED); + } + } else { + printf("ipsec_netif_sync_rx %s - bad packet length %zu\n", pcb->ipsec_ifp->if_xname, length); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + } + + if (data == NULL) { + // Failed to get decrypted data data + kern_pbufpool_free(rx_pp, rx_ph); + continue; + } + + length = mbuf_pkthdr_len(data); + if (length > rx_pp->pp_buflet_size) { + // Flush data + mbuf_freem(data); + kern_pbufpool_free(rx_pp, rx_ph); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + printf("ipsec_netif_sync_rx %s: decrypted packet length %zu > %u\n", + pcb->ipsec_ifp->if_xname, length, rx_pp->pp_buflet_size); + continue; + } + + mbuf_pkthdr_setrcvif(data, pcb->ipsec_ifp); + + // Fillout rx packet + kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL); + VERIFY(rx_buf != NULL); + void *rx_baddr = kern_buflet_get_object_address(rx_buf); + VERIFY(rx_baddr != NULL); + + // Copy-in data from mbuf to buflet + mbuf_copydata(data, 0, length, (void *)rx_baddr); + kern_packet_clear_flow_uuid(rx_ph); // Zero flow id + + // Finalize and attach the packet + error = kern_buflet_set_data_offset(rx_buf, 0); + VERIFY(error == 0); + error = kern_buflet_set_data_length(rx_buf, length); + VERIFY(error == 0); + error = kern_packet_set_link_header_offset(rx_ph, 0); + VERIFY(error == 0); + error = kern_packet_set_network_header_offset(rx_ph, 0); + VERIFY(error == 0); + error = kern_packet_finalize(rx_ph); + VERIFY(error == 0); + error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph); + VERIFY(error == 0); + + STATS_INC(nifs, NETIF_STATS_RXPKTS); + STATS_INC(nifs, NETIF_STATS_RXCOPY_DIRECT); + bpf_tap_packet_in(pcb->ipsec_ifp, DLT_RAW, rx_ph, NULL, 0); + + rx_ring_stats.kcrsi_slots_transferred++; + rx_ring_stats.kcrsi_bytes_transferred += length; + + if (!pcb->ipsec_ext_ifdata_stats) { + ifnet_stat_increment_in(pcb->ipsec_ifp, 1, length, 0); + } + + mbuf_freem(data); + + rx_pslot = rx_slot; + rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL); + } + +done: + if (rx_pslot) { + kern_channel_advance_slot(rx_ring, rx_pslot); + kern_channel_increment_ring_net_stats(rx_ring, pcb->ipsec_ifp, &rx_ring_stats); + } + + if (tx_pslot) { + kern_channel_advance_slot(tx_ring, tx_pslot); + kern_channel_increment_ring_net_stats(tx_ring, pcb->ipsec_ifp, &tx_ring_stats); + (void)kern_channel_reclaim(tx_ring); + } + + // Unlock first, then exit ring + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + if (tx_ring != NULL) { + if (tx_pslot != NULL) { + kern_channel_notify(tx_ring, 0); + } + kr_exit(tx_ring); + } + + return 0; +} + +static errno_t +ipsec_nexus_ifattach(struct ipsec_pcb *pcb, + struct ifnet_init_eparams *init_params, + struct ifnet **ifp) +{ + errno_t err; + nexus_controller_t controller = kern_nexus_shared_controller(); + struct kern_nexus_net_init net_init; + + nexus_name_t provider_name; + snprintf((char *)provider_name, sizeof(provider_name), + "com.apple.netif.ipsec%d", pcb->ipsec_unit); + + struct kern_nexus_provider_init prov_init = { + .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION, + .nxpi_flags = NXPIF_VIRTUAL_DEVICE, + .nxpi_pre_connect = ipsec_nexus_pre_connect, + .nxpi_connected = ipsec_nexus_connected, + .nxpi_pre_disconnect = ipsec_netif_pre_disconnect, + .nxpi_disconnected = ipsec_nexus_disconnected, + .nxpi_ring_init = ipsec_netif_ring_init, + .nxpi_ring_fini = ipsec_netif_ring_fini, + .nxpi_slot_init = NULL, + .nxpi_slot_fini = NULL, + .nxpi_sync_tx = ipsec_netif_sync_tx, + .nxpi_sync_rx = ipsec_netif_sync_rx, + .nxpi_tx_doorbell = ipsec_netif_tx_doorbell, + }; + + nexus_attr_t nxa = NULL; + err = kern_nexus_attr_create(&nxa); + IPSEC_IF_VERIFY(err == 0); + if (err != 0) { + printf("%s: kern_nexus_attr_create failed: %d\n", + __func__, err); + goto failed; + } + + uint64_t slot_buffer_size = IPSEC_IF_DEFAULT_SLOT_SIZE; + err = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size); + VERIFY(err == 0); + + // Reset ring size for netif nexus to limit memory usage + uint64_t ring_size = if_ipsec_ring_size; + err = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size); + VERIFY(err == 0); + err = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size); + VERIFY(err == 0); + + pcb->ipsec_netif_txring_size = ring_size; + + err = kern_nexus_controller_register_provider(controller, + ipsec_nx_dom_prov, + provider_name, + &prov_init, + sizeof(prov_init), + nxa, + &pcb->ipsec_nx.if_provider); + IPSEC_IF_VERIFY(err == 0); + if (err != 0) { + printf("%s register provider failed, error %d\n", + __func__, err); + goto failed; + } + + bzero(&net_init, sizeof(net_init)); + net_init.nxneti_version = KERN_NEXUS_NET_CURRENT_VERSION; + net_init.nxneti_flags = 0; + net_init.nxneti_eparams = init_params; + net_init.nxneti_lladdr = NULL; + net_init.nxneti_prepare = ipsec_netif_prepare; + err = kern_nexus_controller_alloc_net_provider_instance(controller, + pcb->ipsec_nx.if_provider, + pcb, + &pcb->ipsec_nx.if_instance, + &net_init, + ifp); + IPSEC_IF_VERIFY(err == 0); + if (err != 0) { + printf("%s alloc_net_provider_instance failed, %d\n", + __func__, err); + kern_nexus_controller_deregister_provider(controller, + pcb->ipsec_nx.if_provider); + uuid_clear(pcb->ipsec_nx.if_provider); + goto failed; + } + +failed: + if (nxa) { + kern_nexus_attr_destroy(nxa); + } + return (err); +} + +static void +ipsec_detach_provider_and_instance(uuid_t provider, uuid_t instance) +{ + nexus_controller_t controller = kern_nexus_shared_controller(); + errno_t err; + + if (!uuid_is_null(instance)) { + err = kern_nexus_controller_free_provider_instance(controller, + instance); + if (err != 0) { + printf("%s free_provider_instance failed %d\n", + __func__, err); + } + uuid_clear(instance); + } + if (!uuid_is_null(provider)) { + err = kern_nexus_controller_deregister_provider(controller, + provider); + if (err != 0) { + printf("%s deregister_provider %d\n", __func__, err); + } + uuid_clear(provider); + } + return; +} + +static void +ipsec_nexus_detach(ipsec_nx_t nx) +{ + nexus_controller_t controller = kern_nexus_shared_controller(); + errno_t err; + + if (!uuid_is_null(nx->ms_host)) { + err = kern_nexus_ifdetach(controller, + nx->ms_instance, + nx->ms_host); + if (err != 0) { + printf("%s: kern_nexus_ifdetach ms host failed %d\n", + __func__, err); + } + } + + if (!uuid_is_null(nx->ms_device)) { + err = kern_nexus_ifdetach(controller, + nx->ms_instance, + nx->ms_device); + if (err != 0) { + printf("%s: kern_nexus_ifdetach ms device failed %d\n", + __func__, err); + } + } + + ipsec_detach_provider_and_instance(nx->if_provider, + nx->if_instance); + ipsec_detach_provider_and_instance(nx->ms_provider, + nx->ms_instance); + + memset(nx, 0, sizeof(*nx)); +} + +static errno_t +ipsec_create_fs_provider_and_instance(uint32_t subtype, const char *type_name, + const char *ifname, + uuid_t *provider, uuid_t *instance) +{ + nexus_attr_t attr = NULL; + nexus_controller_t controller = kern_nexus_shared_controller(); + uuid_t dom_prov; + errno_t err; + struct kern_nexus_init init; + nexus_name_t provider_name; + + err = kern_nexus_get_builtin_domain_provider(NEXUS_TYPE_FLOW_SWITCH, + &dom_prov); + IPSEC_IF_VERIFY(err == 0); + if (err != 0) { + printf("%s can't get %s provider, error %d\n", + __func__, type_name, err); + goto failed; + } + + err = kern_nexus_attr_create(&attr); + IPSEC_IF_VERIFY(err == 0); + if (err != 0) { + printf("%s: kern_nexus_attr_create failed: %d\n", + __func__, err); + goto failed; + } + + err = kern_nexus_attr_set(attr, NEXUS_ATTR_EXTENSIONS, subtype); + VERIFY(err == 0); + + uint64_t slot_buffer_size = IPSEC_IF_DEFAULT_SLOT_SIZE; + err = kern_nexus_attr_set(attr, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size); + VERIFY(err == 0); + + // Reset ring size for flowswitch nexus to limit memory usage. Larger RX than netif. + uint64_t tx_ring_size = if_ipsec_tx_fsw_ring_size; + err = kern_nexus_attr_set(attr, NEXUS_ATTR_TX_SLOTS, tx_ring_size); + VERIFY(err == 0); + uint64_t rx_ring_size = if_ipsec_rx_fsw_ring_size; + err = kern_nexus_attr_set(attr, NEXUS_ATTR_RX_SLOTS, rx_ring_size); + VERIFY(err == 0); + + snprintf((char *)provider_name, sizeof(provider_name), + "com.apple.%s.%s", type_name, ifname); + err = kern_nexus_controller_register_provider(controller, + dom_prov, + provider_name, + NULL, + 0, + attr, + provider); + kern_nexus_attr_destroy(attr); + attr = NULL; + IPSEC_IF_VERIFY(err == 0); + if (err != 0) { + printf("%s register %s provider failed, error %d\n", + __func__, type_name, err); + goto failed; + } + bzero(&init, sizeof (init)); + init.nxi_version = KERN_NEXUS_CURRENT_VERSION; + err = kern_nexus_controller_alloc_provider_instance(controller, + *provider, + NULL, + instance, &init); + IPSEC_IF_VERIFY(err == 0); + if (err != 0) { + printf("%s alloc_provider_instance %s failed, %d\n", + __func__, type_name, err); + kern_nexus_controller_deregister_provider(controller, + *provider); + uuid_clear(*provider); + } +failed: + return (err); +} + +static errno_t +ipsec_multistack_attach(struct ipsec_pcb *pcb) +{ + nexus_controller_t controller = kern_nexus_shared_controller(); + errno_t err = 0; + ipsec_nx_t nx = &pcb->ipsec_nx; + + // Allocate multistack flowswitch + err = ipsec_create_fs_provider_and_instance(NEXUS_EXTENSION_FSW_TYPE_MULTISTACK, + "multistack", + pcb->ipsec_ifp->if_xname, + &nx->ms_provider, + &nx->ms_instance); + if (err != 0) { + printf("%s: failed to create bridge provider and instance\n", + __func__); + goto failed; + } + + // Attach multistack to device port + err = kern_nexus_ifattach(controller, nx->ms_instance, + NULL, nx->if_instance, + FALSE, &nx->ms_device); + if (err != 0) { + printf("%s kern_nexus_ifattach ms device %d\n", __func__, err); + goto failed; + } + + // Attach multistack to host port + err = kern_nexus_ifattach(controller, nx->ms_instance, + NULL, nx->if_instance, + TRUE, &nx->ms_host); + if (err != 0) { + printf("%s kern_nexus_ifattach ms host %d\n", __func__, err); + goto failed; + } + + // Extract the agent UUID and save for later + struct kern_nexus *multistack_nx = nx_find(nx->ms_instance, false); + if (multistack_nx != NULL) { + struct nx_flowswitch *flowswitch = NX_FSW_PRIVATE(multistack_nx); + if (flowswitch != NULL) { + FSW_RLOCK(flowswitch); + struct fsw_ms_context *ms_context = (struct fsw_ms_context *)flowswitch->fsw_ops_private; + if (ms_context != NULL) { + uuid_copy(nx->ms_agent, ms_context->mc_agent_uuid); + } else { + printf("ipsec_multistack_attach - fsw_ms_context is NULL\n"); + } + FSW_UNLOCK(flowswitch); + } else { + printf("ipsec_multistack_attach - flowswitch is NULL\n"); + } + nx_release(multistack_nx); + } else { + printf("ipsec_multistack_attach - unable to find multistack nexus\n"); + } + + return (0); + +failed: + ipsec_nexus_detach(nx); + + errno_t detach_error = 0; + if ((detach_error = ifnet_detach(pcb->ipsec_ifp)) != 0) { + panic("ipsec_multistack_attach - ifnet_detach failed: %d\n", detach_error); + /* NOT REACHED */ + } + + return (err); +} + +#pragma mark Kernel Pipe Nexus + +static errno_t +ipsec_register_kernel_pipe_nexus(void) +{ + nexus_attr_t nxa = NULL; + errno_t result; + + lck_mtx_lock(&ipsec_lock); + if (ipsec_ncd_refcount++) { + lck_mtx_unlock(&ipsec_lock); + return 0; + } + + result = kern_nexus_controller_create(&ipsec_ncd); + if (result) { + printf("%s: kern_nexus_controller_create failed: %d\n", + __FUNCTION__, result); + goto done; + } + + uuid_t dom_prov; + result = kern_nexus_get_builtin_domain_provider( + NEXUS_TYPE_KERNEL_PIPE, &dom_prov); + if (result) { + printf("%s: kern_nexus_get_builtin_domain_provider failed: %d\n", + __FUNCTION__, result); + goto done; + } + + struct kern_nexus_provider_init prov_init = { + .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION, + .nxpi_flags = NXPIF_VIRTUAL_DEVICE, + .nxpi_pre_connect = ipsec_nexus_pre_connect, + .nxpi_connected = ipsec_nexus_connected, + .nxpi_pre_disconnect = ipsec_nexus_pre_disconnect, + .nxpi_disconnected = ipsec_nexus_disconnected, + .nxpi_ring_init = ipsec_kpipe_ring_init, + .nxpi_ring_fini = ipsec_kpipe_ring_fini, + .nxpi_slot_init = NULL, + .nxpi_slot_fini = NULL, + .nxpi_sync_tx = ipsec_kpipe_sync_tx, + .nxpi_sync_rx = ipsec_kpipe_sync_rx, + .nxpi_tx_doorbell = NULL, + }; + + result = kern_nexus_attr_create(&nxa); + if (result) { + printf("%s: kern_nexus_attr_create failed: %d\n", + __FUNCTION__, result); + goto done; + } + + uint64_t slot_buffer_size = IPSEC_IF_DEFAULT_SLOT_SIZE; + result = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size); + VERIFY(result == 0); + + // Reset ring size for kernel pipe nexus to limit memory usage + uint64_t ring_size = if_ipsec_ring_size; + result = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size); + VERIFY(result == 0); + result = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size); + VERIFY(result == 0); + + result = kern_nexus_controller_register_provider(ipsec_ncd, + dom_prov, + (const uint8_t *)"com.apple.nexus.ipsec.kpipe", + &prov_init, + sizeof(prov_init), + nxa, + &ipsec_kpipe_uuid); + if (result) { + printf("%s: kern_nexus_controller_register_provider failed: %d\n", + __FUNCTION__, result); + goto done; + } + +done: + if (nxa) { + kern_nexus_attr_destroy(nxa); + } + + if (result) { + if (ipsec_ncd) { + kern_nexus_controller_destroy(ipsec_ncd); + ipsec_ncd = NULL; + } + ipsec_ncd_refcount = 0; + } + + lck_mtx_unlock(&ipsec_lock); + + return result; +} + +static void +ipsec_unregister_kernel_pipe_nexus(void) +{ + lck_mtx_lock(&ipsec_lock); + + VERIFY(ipsec_ncd_refcount > 0); + + if (--ipsec_ncd_refcount == 0) { + kern_nexus_controller_destroy(ipsec_ncd); + ipsec_ncd = NULL; + } + + lck_mtx_unlock(&ipsec_lock); +} + +// For use by socket option, not internally +static errno_t +ipsec_disable_channel(struct ipsec_pcb *pcb) +{ + errno_t result; + int enabled; + uuid_t uuid; + + lck_rw_lock_exclusive(&pcb->ipsec_pcb_lock); + + enabled = pcb->ipsec_kpipe_enabled; + uuid_copy(uuid, pcb->ipsec_kpipe_uuid); + + VERIFY(uuid_is_null(pcb->ipsec_kpipe_uuid) == !enabled); -#define IPSECQ_MAXLEN 256 + pcb->ipsec_kpipe_enabled = 0; + uuid_clear(pcb->ipsec_kpipe_uuid); -errno_t -ipsec_register_control(void) + lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock); + + if (enabled) { + result = kern_nexus_controller_free_provider_instance(ipsec_ncd, uuid); + } else { + result = ENXIO; + } + + if (!result) { + ipsec_unregister_kernel_pipe_nexus(); + } + + return result; +} + +static errno_t +ipsec_enable_channel(struct ipsec_pcb *pcb, struct proc *proc) { - struct kern_ctl_reg kern_ctl; - errno_t result = 0; - - /* Find a unique value for our interface family */ - result = mbuf_tag_id_find(IPSEC_CONTROL_NAME, &ipsec_family); - if (result != 0) { - printf("ipsec_register_control - mbuf_tag_id_find_internal failed: %d\n", result); + struct kern_nexus_init init; + errno_t result; + + result = ipsec_register_kernel_pipe_nexus(); + if (result) { return result; } - - bzero(&kern_ctl, sizeof(kern_ctl)); - strlcpy(kern_ctl.ctl_name, IPSEC_CONTROL_NAME, sizeof(kern_ctl.ctl_name)); - kern_ctl.ctl_name[sizeof(kern_ctl.ctl_name) - 1] = 0; - kern_ctl.ctl_flags = CTL_FLAG_PRIVILEGED; /* Require root */ - kern_ctl.ctl_sendsize = 64 * 1024; - kern_ctl.ctl_recvsize = 64 * 1024; - kern_ctl.ctl_connect = ipsec_ctl_connect; - kern_ctl.ctl_disconnect = ipsec_ctl_disconnect; - kern_ctl.ctl_send = ipsec_ctl_send; - kern_ctl.ctl_setopt = ipsec_ctl_setopt; - kern_ctl.ctl_getopt = ipsec_ctl_getopt; - - result = ctl_register(&kern_ctl, &ipsec_kctlref); - if (result != 0) { - printf("ipsec_register_control - ctl_register failed: %d\n", result); - return result; + + VERIFY(ipsec_ncd); + + lck_rw_lock_exclusive(&pcb->ipsec_pcb_lock); + + if (pcb->ipsec_kpipe_enabled) { + result = EEXIST; // return success instead? + goto done; } - - /* Register the protocol plumbers */ - if ((result = proto_register_plumber(PF_INET, ipsec_family, - ipsec_attach_proto, NULL)) != 0) { - printf("ipsec_register_control - proto_register_plumber(PF_INET, %d) failed: %d\n", - ipsec_family, result); - ctl_deregister(ipsec_kctlref); - return result; + + VERIFY(uuid_is_null(pcb->ipsec_kpipe_uuid)); + bzero(&init, sizeof (init)); + init.nxi_version = KERN_NEXUS_CURRENT_VERSION; + result = kern_nexus_controller_alloc_provider_instance(ipsec_ncd, + ipsec_kpipe_uuid, pcb, &pcb->ipsec_kpipe_uuid, &init); + if (result) { + goto done; } + + nexus_port_t port = NEXUS_PORT_KERNEL_PIPE_CLIENT; + result = kern_nexus_controller_bind_provider_instance(ipsec_ncd, + pcb->ipsec_kpipe_uuid, &port, + proc_pid(proc), NULL, NULL, 0, NEXUS_BIND_PID); + if (result) { + kern_nexus_controller_free_provider_instance(ipsec_ncd, + pcb->ipsec_kpipe_uuid); + uuid_clear(pcb->ipsec_kpipe_uuid); + goto done; + } + + pcb->ipsec_kpipe_enabled = 1; + +done: + lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock); - /* Register the protocol plumbers */ - if ((result = proto_register_plumber(PF_INET6, ipsec_family, - ipsec_attach_proto, NULL)) != 0) { - proto_unregister_plumber(PF_INET, ipsec_family); - ctl_deregister(ipsec_kctlref); - printf("ipsec_register_control - proto_register_plumber(PF_INET6, %d) failed: %d\n", - ipsec_family, result); - return result; + if (result) { + ipsec_unregister_kernel_pipe_nexus(); } - return 0; + return result; } -/* Helpers */ -int -ipsec_interface_isvalid (ifnet_t interface) -{ - struct ipsec_pcb *pcb = NULL; - - if (interface == NULL) - return 0; - - pcb = ifnet_softc(interface); - - if (pcb == NULL) - return 0; - - /* When ctl disconnects, ipsec_unit is set to 0 */ - if (pcb->ipsec_unit == 0) - return 0; - - return 1; -} +#endif // IPSEC_NEXUS + /* Kernel control functions */ +static inline void +ipsec_free_pcb(struct ipsec_pcb *pcb) +{ +#if IPSEC_NEXUS + mbuf_freem_list(pcb->ipsec_input_chain); + lck_mtx_destroy(&pcb->ipsec_input_chain_lock, ipsec_lck_grp); +#endif // IPSEC_NEXUS + lck_rw_destroy(&pcb->ipsec_pcb_lock, ipsec_lck_grp); + lck_mtx_lock(&ipsec_lock); + TAILQ_REMOVE(&ipsec_head, pcb, ipsec_chain); + lck_mtx_unlock(&ipsec_lock); + zfree(ipsec_pcb_zone, pcb); +} + static errno_t -ipsec_ctl_connect(kern_ctl_ref kctlref, - struct sockaddr_ctl *sac, - void **unitinfo) +ipsec_ctl_connect(kern_ctl_ref kctlref, + struct sockaddr_ctl *sac, + void **unitinfo) { - struct ifnet_init_eparams ipsec_init; - struct ipsec_pcb *pcb; - errno_t result; - struct ifnet_stats_param stats; + struct ifnet_init_eparams ipsec_init = {}; + errno_t result = 0; - /* kernel control allocates, interface frees */ - MALLOC(pcb, struct ipsec_pcb *, sizeof(*pcb), M_DEVBUF, M_WAITOK | M_ZERO); + struct ipsec_pcb *pcb = zalloc(ipsec_pcb_zone); + memset(pcb, 0, sizeof(*pcb)); /* Setup the protocol control block */ *unitinfo = pcb; pcb->ipsec_ctlref = kctlref; pcb->ipsec_unit = sac->sc_unit; pcb->ipsec_output_service_class = MBUF_SC_OAM; - - printf("ipsec_ctl_connect: creating interface ipsec%d\n", pcb->ipsec_unit - 1); - + + lck_mtx_lock(&ipsec_lock); + + /* Find some open interface id */ + u_int32_t chosen_unique_id = 1; + struct ipsec_pcb *next_pcb = TAILQ_LAST(&ipsec_head, ipsec_list); + if (next_pcb != NULL) { + /* List was not empty, add one to the last item */ + chosen_unique_id = next_pcb->ipsec_unique_id + 1; + next_pcb = NULL; + + /* + * If this wrapped the id number, start looking at + * the front of the list for an unused id. + */ + if (chosen_unique_id == 0) { + /* Find the next unused ID */ + chosen_unique_id = 1; + TAILQ_FOREACH(next_pcb, &ipsec_head, ipsec_chain) { + if (next_pcb->ipsec_unique_id > chosen_unique_id) { + /* We found a gap */ + break; + } + + chosen_unique_id = next_pcb->ipsec_unique_id + 1; + } + } + } + + pcb->ipsec_unique_id = chosen_unique_id; + + if (next_pcb != NULL) { + TAILQ_INSERT_BEFORE(next_pcb, pcb, ipsec_chain); + } else { + TAILQ_INSERT_TAIL(&ipsec_head, pcb, ipsec_chain); + } + lck_mtx_unlock(&ipsec_lock); + + snprintf(pcb->ipsec_if_xname, sizeof(pcb->ipsec_if_xname), "ipsec%d", pcb->ipsec_unit - 1); + snprintf(pcb->ipsec_unique_name, sizeof(pcb->ipsec_unique_name), "ipsecid%d", pcb->ipsec_unique_id - 1); + printf("ipsec_ctl_connect: creating interface %s (id %s)\n", pcb->ipsec_if_xname, pcb->ipsec_unique_name); + + lck_rw_init(&pcb->ipsec_pcb_lock, ipsec_lck_grp, ipsec_lck_attr); +#if IPSEC_NEXUS + lck_mtx_init(&pcb->ipsec_input_chain_lock, ipsec_lck_grp, ipsec_lck_attr); +#endif // IPSEC_NEXUS + /* Create the interface */ bzero(&ipsec_init, sizeof(ipsec_init)); ipsec_init.ver = IFNET_INIT_CURRENT_VERSION; ipsec_init.len = sizeof (ipsec_init); - ipsec_init.name = "ipsec"; + +#if IPSEC_NEXUS + ipsec_init.flags = (IFNET_INIT_SKYWALK_NATIVE | IFNET_INIT_NX_NOAUTO); +#else // IPSEC_NEXUS + ipsec_init.flags = IFNET_INIT_NX_NOAUTO; ipsec_init.start = ipsec_start; +#endif // IPSEC_NEXUS + ipsec_init.name = "ipsec"; ipsec_init.unit = pcb->ipsec_unit - 1; + ipsec_init.uniqueid = pcb->ipsec_unique_name; + ipsec_init.uniqueid_len = strlen(pcb->ipsec_unique_name); ipsec_init.family = ipsec_family; + ipsec_init.subfamily = IFNET_SUBFAMILY_IPSEC; ipsec_init.type = IFT_OTHER; ipsec_init.demux = ipsec_demux; ipsec_init.add_proto = ipsec_add_proto; @@ -203,44 +2163,51 @@ ipsec_ctl_connect(kern_ctl_ref kctlref, ipsec_init.softc = pcb; ipsec_init.ioctl = ipsec_ioctl; ipsec_init.detach = ipsec_detached; - + +#if IPSEC_NEXUS + result = ipsec_nexus_ifattach(pcb, &ipsec_init, &pcb->ipsec_ifp); + if (result != 0) { + printf("ipsec_ctl_connect - ipsec_nexus_ifattach failed: %d\n", result); + ipsec_free_pcb(pcb); + *unitinfo = NULL; + return result; + } + + result = ipsec_multistack_attach(pcb); + if (result != 0) { + printf("ipsec_ctl_connect - ipsec_multistack_attach failed: %d\n", result); + *unitinfo = NULL; + return result; + } + +#else // IPSEC_NEXUS result = ifnet_allocate_extended(&ipsec_init, &pcb->ipsec_ifp); if (result != 0) { printf("ipsec_ctl_connect - ifnet_allocate failed: %d\n", result); + ipsec_free_pcb(pcb); *unitinfo = NULL; - FREE(pcb, M_DEVBUF); return result; } - - /* Set flags and additional information. */ - ifnet_set_mtu(pcb->ipsec_ifp, 1500); - ifnet_set_flags(pcb->ipsec_ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff); - - /* The interface must generate its own IPv6 LinkLocal address, - * if possible following the recommendation of RFC2472 to the 64bit interface ID - */ - ifnet_set_eflags(pcb->ipsec_ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL); - - /* Reset the stats in case as the interface may have been recycled */ - bzero(&stats, sizeof(struct ifnet_stats_param)); - ifnet_set_stat(pcb->ipsec_ifp, &stats); - + ipsec_ifnet_set_attrs(pcb->ipsec_ifp); + /* Attach the interface */ result = ifnet_attach(pcb->ipsec_ifp, NULL); if (result != 0) { - printf("ipsec_ctl_connect - ifnet_allocate failed: %d\n", result); + printf("ipsec_ctl_connect - ifnet_attach failed: %d\n", result); ifnet_release(pcb->ipsec_ifp); + ipsec_free_pcb(pcb); *unitinfo = NULL; - FREE(pcb, M_DEVBUF); - } else { - /* Attach to bpf */ - bpfattach(pcb->ipsec_ifp, DLT_NULL, 4); - - /* The interfaces resoures allocated, mark it as running */ - ifnet_set_flags(pcb->ipsec_ifp, IFF_RUNNING, IFF_RUNNING); + return (result); } - - return result; +#endif // IPSEC_NEXUS + + /* Attach to bpf */ + bpfattach(pcb->ipsec_ifp, DLT_RAW, 0); + + /* The interfaces resoures allocated, mark it as running */ + ifnet_set_flags(pcb->ipsec_ifp, IFF_RUNNING, IFF_RUNNING); + + return (0); } static errno_t @@ -397,18 +2364,46 @@ ipsec_ctl_disconnect(__unused kern_ctl_ref kctlref, __unused u_int32_t unit, void *unitinfo) { - struct ipsec_pcb *pcb = unitinfo; - ifnet_t ifp = NULL; - errno_t result = 0; + struct ipsec_pcb *pcb = unitinfo; + ifnet_t ifp = NULL; + errno_t result = 0; - if (pcb == NULL) + if (pcb == NULL) { return EINVAL; + } + +#if IPSEC_NEXUS + // Tell the nexus to stop all rings + if (pcb->ipsec_netif_nexus != NULL) { + kern_nexus_stop(pcb->ipsec_netif_nexus); + } +#endif // IPSEC_NEXUS + + lck_rw_lock_exclusive(&pcb->ipsec_pcb_lock); + +#if IPSEC_NEXUS + uuid_t kpipe_uuid; + uuid_copy(kpipe_uuid, pcb->ipsec_kpipe_uuid); + uuid_clear(pcb->ipsec_kpipe_uuid); + pcb->ipsec_kpipe_enabled = FALSE; +#endif // IPSEC_NEXUS ifp = pcb->ipsec_ifp; VERIFY(ifp != NULL); pcb->ipsec_ctlref = NULL; - pcb->ipsec_unit = 0; - + + /* + * Quiesce the interface and flush any pending outbound packets. + */ + if_down(ifp); + + /* Increment refcnt, but detach interface */ + ifnet_incr_iorefcnt(ifp); + if ((result = ifnet_detach(ifp)) != 0) { + panic("ipsec_ctl_disconnect - ifnet_detach failed: %d\n", result); + /* NOT REACHED */ + } + /* * We want to do everything in our power to ensure that the interface * really goes away when the socket is closed. We must remove IP/IPv6 @@ -419,10 +2414,20 @@ ipsec_ctl_disconnect(__unused kern_ctl_ref kctlref, ipsec_cleanup_family(ifp, AF_INET); ipsec_cleanup_family(ifp, AF_INET6); - - if ((result = ifnet_detach(ifp)) != 0) { - printf("ipsec_ctl_disconnect - ifnet_detach failed: %d\n", result); + + lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock); + +#if IPSEC_NEXUS + if (!uuid_is_null(kpipe_uuid)) { + if (kern_nexus_controller_free_provider_instance(ipsec_ncd, kpipe_uuid) == 0) { + ipsec_unregister_kernel_pipe_nexus(); + } } + ipsec_nexus_detach(&pcb->ipsec_nx); +#endif // IPSEC_NEXUS + + /* Decrement refcnt to finish detaching and freeing */ + ifnet_decr_iorefcnt(ifp); return 0; } @@ -540,6 +2545,59 @@ ipsec_ctl_setopt(__unused kern_ctl_ref kctlref, pcb->ipsec_output_service_class); break; } + +#if IPSEC_NEXUS + case IPSEC_OPT_ENABLE_CHANNEL: { + if (len != sizeof(int)) { + result = EMSGSIZE; + break; + } + if (*(int *)data) { + result = ipsec_enable_channel(pcb, current_proc()); + } else { + result = ipsec_disable_channel(pcb); + } + break; + } + + case IPSEC_OPT_ENABLE_FLOWSWITCH: { + if (len != sizeof(int)) { + result = EMSGSIZE; + break; + } + if (!if_enable_netagent) { + result = ENOTSUP; + break; + } + if (*(int *)data) { + if (!uuid_is_null(pcb->ipsec_nx.ms_agent)) { + if_add_netagent(pcb->ipsec_ifp, pcb->ipsec_nx.ms_agent); + } + } else { + if (!uuid_is_null(pcb->ipsec_nx.ms_agent)) { + if_delete_netagent(pcb->ipsec_ifp, pcb->ipsec_nx.ms_agent); + } + } + break; + } + + case IPSEC_OPT_INPUT_FRAG_SIZE: { + if (len != sizeof(u_int32_t)) { + result = EMSGSIZE; + break; + } + u_int32_t input_frag_size = *(u_int32_t *)data; + if (input_frag_size <= sizeof(struct ip6_hdr)) { + pcb->ipsec_frag_size_set = FALSE; + pcb->ipsec_input_frag_size = 0; + } else { + printf("SET FRAG SIZE TO %u\n", input_frag_size); + pcb->ipsec_frag_size_set = TRUE; + pcb->ipsec_input_frag_size = input_frag_size; + } + break; + } +#endif // IPSEC_NEXUS default: result = ENOPROTOOPT; @@ -550,46 +2608,81 @@ ipsec_ctl_setopt(__unused kern_ctl_ref kctlref, } static errno_t -ipsec_ctl_getopt(__unused kern_ctl_ref kctlref, - __unused u_int32_t unit, - void *unitinfo, - int opt, - void *data, - size_t *len) +ipsec_ctl_getopt(__unused kern_ctl_ref kctlref, + __unused u_int32_t unit, + void *unitinfo, + int opt, + void *data, + size_t *len) { - struct ipsec_pcb *pcb = unitinfo; - errno_t result = 0; + struct ipsec_pcb *pcb = unitinfo; + errno_t result = 0; switch (opt) { - case IPSEC_OPT_FLAGS: - if (*len != sizeof(u_int32_t)) + case IPSEC_OPT_FLAGS: { + if (*len != sizeof(u_int32_t)) { result = EMSGSIZE; - else + } else { *(u_int32_t *)data = pcb->ipsec_flags; + } break; + } - case IPSEC_OPT_EXT_IFDATA_STATS: - if (*len != sizeof(int)) + case IPSEC_OPT_EXT_IFDATA_STATS: { + if (*len != sizeof(int)) { result = EMSGSIZE; - else + } else { *(int *)data = (pcb->ipsec_ext_ifdata_stats) ? 1 : 0; + } break; + } - case IPSEC_OPT_IFNAME: - *len = snprintf(data, *len, "%s%d", ifnet_name(pcb->ipsec_ifp), ifnet_unit(pcb->ipsec_ifp)) + 1; + case IPSEC_OPT_IFNAME: { + if (*len < MIN(strlen(pcb->ipsec_if_xname) + 1, sizeof(pcb->ipsec_if_xname))) { + result = EMSGSIZE; + } else { + *len = snprintf(data, *len, "%s", pcb->ipsec_if_xname) + 1; + } break; + } case IPSEC_OPT_OUTPUT_TRAFFIC_CLASS: { if (*len != sizeof(int)) { result = EMSGSIZE; - break; + } else { + *(int *)data = so_svc2tc(pcb->ipsec_output_service_class); } - *(int *)data = so_svc2tc(pcb->ipsec_output_service_class); break; } - default: + +#if IPSEC_NEXUS + case IPSEC_OPT_GET_CHANNEL_UUID: { + lck_rw_lock_shared(&pcb->ipsec_pcb_lock); + if (uuid_is_null(pcb->ipsec_kpipe_uuid)) { + result = ENXIO; + } else if (*len != sizeof(uuid_t)) { + result = EMSGSIZE; + } else { + uuid_copy(data, pcb->ipsec_kpipe_uuid); + } + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + break; + } + + case IPSEC_OPT_INPUT_FRAG_SIZE: { + if (*len != sizeof(u_int32_t)) { + result = EMSGSIZE; + } else { + *(u_int32_t *)data = pcb->ipsec_input_frag_size; + } + break; + } +#endif // IPSEC_NEXUS + + default: { result = ENOPROTOOPT; break; + } } return result; @@ -597,10 +2690,10 @@ ipsec_ctl_getopt(__unused kern_ctl_ref kctlref, /* Network Interface functions */ static errno_t -ipsec_output(ifnet_t interface, - mbuf_t data) +ipsec_output(ifnet_t interface, + mbuf_t data) { - struct ipsec_pcb *pcb = ifnet_softc(interface); + struct ipsec_pcb *pcb = ifnet_softc(interface); struct ipsec_output_state ipsec_state; struct route ro; struct route_in6 ro6; @@ -611,13 +2704,12 @@ ipsec_output(ifnet_t interface, struct ip6_out_args ip6oa; int error = 0; u_int ip_version = 0; - uint32_t af; int flags = 0; struct flowadv *adv = NULL; // Make sure this packet isn't looping through the interface if (necp_get_last_interface_index_from_packet(data) == interface->if_index) { - error = -1; + error = EINVAL; goto ipsec_output_err; } @@ -628,16 +2720,12 @@ ipsec_output(ifnet_t interface, ip_version = ip->ip_v; switch (ip_version) { - case 4: - /* Tap */ - af = AF_INET; - bpf_tap_out(pcb->ipsec_ifp, DLT_NULL, data, &af, sizeof(af)); - + case 4: { /* Apply encryption */ - bzero(&ipsec_state, sizeof(ipsec_state)); + memset(&ipsec_state, 0, sizeof(ipsec_state)); ipsec_state.m = data; ipsec_state.dst = (struct sockaddr *)&ip->ip_dst; - bzero(&ipsec_state.ro, sizeof(ipsec_state.ro)); + memset(&ipsec_state.ro, 0, sizeof(ipsec_state.ro)); error = ipsec4_interface_output(&ipsec_state, interface); /* Tunneled in IPv6 - packet is gone */ @@ -647,7 +2735,9 @@ ipsec_output(ifnet_t interface, data = ipsec_state.m; if (error || data == NULL) { - printf("ipsec_output: ipsec4_output error %d.\n", error); + if (error) { + printf("ipsec_output: ipsec4_output error %d.\n", error); + } goto ipsec_output_err; } @@ -668,12 +2758,12 @@ ipsec_output(ifnet_t interface, ifnet_stat_increment_out(interface, 1, length, 0); /* Send to ip_output */ - bzero(&ro, sizeof(ro)); + memset(&ro, 0, sizeof(ro)); - flags = IP_OUTARGS | /* Passing out args to specify interface */ - IP_NOIPSEC; /* To ensure the packet doesn't go through ipsec twice */ + flags = (IP_OUTARGS | /* Passing out args to specify interface */ + IP_NOIPSEC); /* To ensure the packet doesn't go through ipsec twice */ - bzero(&ipoa, sizeof(ipoa)); + memset(&ipoa, 0, sizeof(ipoa)); ipoa.ipoa_flowadv.code = 0; ipoa.ipoa_flags = IPOAF_SELECT_SRCIF | IPOAF_BOUND_SRCADDR; if (ipsec_state.outgoing_if) { @@ -684,7 +2774,7 @@ ipsec_output(ifnet_t interface, adv = &ipoa.ipoa_flowadv; - (void) ip_output(data, NULL, &ro, flags, NULL, &ipoa); + (void)ip_output(data, NULL, &ro, flags, NULL, &ipoa); data = NULL; if (adv->code == FADV_FLOW_CONTROLLED || adv->code == FADV_SUSPENDED) { @@ -693,10 +2783,8 @@ ipsec_output(ifnet_t interface, } goto done; - case 6: - af = AF_INET6; - bpf_tap_out(pcb->ipsec_ifp, DLT_NULL, data, &af, sizeof(af)); - + } + case 6: { data = ipsec6_splithdr(data); if (data == NULL) { printf("ipsec_output: ipsec6_splithdr returned NULL\n"); @@ -705,17 +2793,20 @@ ipsec_output(ifnet_t interface, ip6 = mtod(data, struct ip6_hdr *); - bzero(&ipsec_state, sizeof(ipsec_state)); + memset(&ipsec_state, 0, sizeof(ipsec_state)); ipsec_state.m = data; ipsec_state.dst = (struct sockaddr *)&ip6->ip6_dst; - bzero(&ipsec_state.ro, sizeof(ipsec_state.ro)); + memset(&ipsec_state.ro, 0, sizeof(ipsec_state.ro)); error = ipsec6_interface_output(&ipsec_state, interface, &ip6->ip6_nxt, ipsec_state.m); - if (error == 0 && ipsec_state.tunneled == 4) /* tunneled in IPv4 - packet is gone */ + if (error == 0 && ipsec_state.tunneled == 4) { /* tunneled in IPv4 - packet is gone */ goto done; + } data = ipsec_state.m; if (error || data == NULL) { - printf("ipsec_output: ipsec6_output error %d.\n", error); + if (error) { + printf("ipsec_output: ipsec6_output error %d\n", error); + } goto ipsec_output_err; } @@ -731,11 +2822,11 @@ ipsec_output(ifnet_t interface, ifnet_stat_increment_out(interface, 1, length, 0); /* Send to ip6_output */ - bzero(&ro6, sizeof(ro6)); + memset(&ro6, 0, sizeof(ro6)); flags = IPV6_OUTARGS; - bzero(&ip6oa, sizeof(ip6oa)); + memset(&ip6oa, 0, sizeof(ip6oa)); ip6oa.ip6oa_flowadv.code = 0; ip6oa.ip6oa_flags = IP6OAF_SELECT_SRCIF | IP6OAF_BOUND_SRCADDR; if (ipsec_state.outgoing_if) { @@ -755,10 +2846,12 @@ ipsec_output(ifnet_t interface, } goto done; - default: + } + default: { printf("ipsec_output: Received unknown packet version %d.\n", ip_version); - error = -1; + error = EINVAL; goto ipsec_output_err; + } } done: @@ -770,11 +2863,14 @@ ipsec_output_err: goto done; } +#if !IPSEC_NEXUS static void ipsec_start(ifnet_t interface) { mbuf_t data; + struct ipsec_pcb *pcb = ifnet_softc(interface); + VERIFY(pcb != NULL); for (;;) { if (ifnet_dequeue(interface, &data) != 0) break; @@ -782,6 +2878,7 @@ ipsec_start(ifnet_t interface) break; } } +#endif // !IPSEC_NEXUS /* Network Interface functions */ static errno_t @@ -843,14 +2940,21 @@ ipsec_del_proto(__unused ifnet_t interface, } static errno_t -ipsec_ioctl(ifnet_t interface, - u_long command, - void *data) +ipsec_ioctl(ifnet_t interface, + u_long command, + void *data) { errno_t result = 0; switch(command) { case SIOCSIFMTU: +#if IPSEC_NEXUS + // Make sure we can fit packets in the channel buffers + if (((uint64_t)((struct ifreq*)data)->ifr_mtu) > IPSEC_IF_DEFAULT_SLOT_SIZE) { + ifnet_set_mtu(interface, IPSEC_IF_DEFAULT_SLOT_SIZE); + break; + } +#endif // IPSEC_NEXUS ifnet_set_mtu(interface, ((struct ifreq*)data)->ifr_mtu); break; @@ -866,12 +2970,11 @@ ipsec_ioctl(ifnet_t interface, } static void -ipsec_detached( - ifnet_t interface) +ipsec_detached(ifnet_t interface) { - struct ipsec_pcb *pcb = ifnet_softc(interface); - - ifnet_release(pcb->ipsec_ifp); + struct ipsec_pcb *pcb = ifnet_softc(interface); + (void)ifnet_release(interface); + ipsec_free_pcb(pcb); } /* Protocol Handlers */ @@ -882,16 +2985,7 @@ ipsec_proto_input(ifnet_t interface, mbuf_t m, __unused char *frame_header) { - struct ip *ip; - uint32_t af = 0; - ip = mtod(m, struct ip *); - if (ip->ip_v == 4) - af = AF_INET; - else if (ip->ip_v == 6) - af = AF_INET6; - mbuf_pkthdr_setrcvif(m, interface); - bpf_tap_in(interface, DLT_NULL, m, &af, sizeof(af)); pktap_input(interface, protocol, m, NULL); if (proto_input(protocol, m) != 0) { @@ -938,6 +3032,38 @@ ipsec_attach_proto(ifnet_t interface, return result; } +#if IPSEC_NEXUS +errno_t +ipsec_inject_inbound_packet(ifnet_t interface, + mbuf_t packet) +{ + struct ipsec_pcb *pcb = ifnet_softc(interface); + + lck_rw_lock_shared(&pcb->ipsec_pcb_lock); + + lck_mtx_lock(&pcb->ipsec_input_chain_lock); + if (pcb->ipsec_input_chain != NULL) { + pcb->ipsec_input_chain_last->m_nextpkt = packet; + } else { + pcb->ipsec_input_chain = packet; + } + while (packet->m_nextpkt) { + VERIFY(packet != packet->m_nextpkt); + packet = packet->m_nextpkt; + } + pcb->ipsec_input_chain_last = packet; + lck_mtx_unlock(&pcb->ipsec_input_chain_lock); + + kern_channel_ring_t rx_ring = pcb->ipsec_netif_rxring; + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + + if (rx_ring != NULL) { + kern_channel_notify(rx_ring, 0); + } + + return (0); +} +#else // IPSEC_NEXUS errno_t ipsec_inject_inbound_packet(ifnet_t interface, mbuf_t packet) @@ -947,9 +3073,10 @@ ipsec_inject_inbound_packet(ifnet_t interface, if ((error = ipsec_demux(interface, packet, NULL, &protocol)) != 0) { return error; } - + return ipsec_proto_input(interface, protocol, packet, NULL); } +#endif // IPSEC_NEXUS void ipsec_set_pkthdr_for_interface(ifnet_t interface, mbuf_t packet, int family) diff --git a/bsd/net/if_ipsec.h b/bsd/net/if_ipsec.h index 31195e7e4..159ef2036 100644 --- a/bsd/net/if_ipsec.h +++ b/bsd/net/if_ipsec.h @@ -35,15 +35,6 @@ #include <sys/kern_control.h> #include <netinet/ip_var.h> -/* Control block allocated for each kernel control connection */ -struct ipsec_pcb { - kern_ctl_ref ipsec_ctlref; - ifnet_t ipsec_ifp; - u_int32_t ipsec_unit; - u_int32_t ipsec_flags; - int ipsec_ext_ifdata_stats; - mbuf_svc_class_t ipsec_output_service_class; -}; errno_t ipsec_register_control(void); @@ -76,6 +67,11 @@ void ipsec_set_ip6oa_for_interface(ifnet_t interface, struct ip6_out_args *ip6oa #define IPSEC_OPT_INC_IFDATA_STATS_OUT 5 /* set to increment stat counters (type struct ipsec_stats_param) */ #define IPSEC_OPT_SET_DELEGATE_INTERFACE 6 /* set the delegate interface (char[]) */ #define IPSEC_OPT_OUTPUT_TRAFFIC_CLASS 7 /* set the traffic class for packets leaving the interface, see sys/socket.h */ +#define IPSEC_OPT_ENABLE_CHANNEL 8 /* enable a kernel pipe nexus that allows the owner to open a channel to act as a driver */ +#define IPSEC_OPT_GET_CHANNEL_UUID 9 /* get the uuid of the kernel pipe nexus instance */ +#define IPSEC_OPT_ENABLE_FLOWSWITCH 10 /* enable a flowswitch nexus that clients can use */ +#define IPSEC_OPT_INPUT_FRAG_SIZE 11 /* set the maximum size of input packets before fragmenting as a uint32_t */ + /* * ipsec stats parameter structure */ diff --git a/bsd/net/if_llatbl.c b/bsd/net/if_llatbl.c new file mode 100644 index 000000000..7b7eab342 --- /dev/null +++ b/bsd/net/if_llatbl.c @@ -0,0 +1,860 @@ +/* + * Copyright (c) 2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * Copyright (c) 2004 Luigi Rizzo, Alessandro Cerri. All rights reserved. + * Copyright (c) 2004-2008 Qing Li. All rights reserved. + * Copyright (c) 2008 Kip Macy. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +#include <sys/cdefs.h> +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/malloc.h> +#include <sys/mbuf.h> +#include <sys/syslog.h> +#include <sys/sysctl.h> +#include <sys/socket.h> +#include <sys/kernel.h> +#include <kern/queue.h> +#include <kern/locks.h> + +#include <netinet/in.h> +#include <net/if_llatbl.h> +#include <net/if.h> +#include <net/if_dl.h> +#include <net/if_var.h> +#include <net/dlil.h> +#include <net/route.h> +#include <netinet/if_ether.h> +#include <netinet6/in6_var.h> +#include <netinet6/nd6.h> + +MALLOC_DEFINE(M_LLTABLE, "lltable", "link level address tables"); + +static SLIST_HEAD(, lltable) lltables = SLIST_HEAD_INITIALIZER(lltables); + +static lck_grp_attr_t *lltable_rwlock_grp_attr; +static lck_grp_t *lltable_rwlock_grp; +static lck_attr_t *lltable_rwlock_attr; + +static lck_grp_attr_t *lle_lock_grp_attr = NULL; +lck_grp_t *lle_lock_grp = NULL; +lck_attr_t *lle_lock_attr = NULL; + +decl_lck_rw_data(, lltable_rwlock_data); +lck_rw_t *lltable_rwlock = &lltable_rwlock_data; + +#if 0 +static void lltable_unlink(struct lltable *llt); +#endif +static void llentries_unlink(struct lltable *llt, struct llentries *head); + +static void htable_unlink_entry(struct llentry *lle); +static void htable_link_entry(struct lltable *llt, struct llentry *lle); +static int htable_foreach_lle(struct lltable *llt, llt_foreach_cb_t *f, + void *farg); + +void +lltable_glbl_init() +{ + lltable_rwlock_grp_attr = lck_grp_attr_alloc_init(); + lltable_rwlock_grp = lck_grp_alloc_init("lltable_rwlock", + lltable_rwlock_grp_attr); + lltable_rwlock_attr = lck_attr_alloc_init(); + lck_rw_init(lltable_rwlock, lltable_rwlock_grp, + lltable_rwlock_attr); + + lle_lock_grp_attr = lck_grp_attr_alloc_init(); + lle_lock_grp = lck_grp_alloc_init("lle locks", lle_lock_grp_attr); + lle_lock_attr = lck_attr_alloc_init(); +} + +/* + * Dump lle state for a specific address family. + */ +static int +lltable_dump_af(struct lltable *llt, struct sysctl_req *wr) +{ + int error; + + LLTABLE_LOCK_ASSERT(); + + if (llt->llt_ifp->if_flags & IFF_LOOPBACK) + return (0); + error = 0; + + IF_AFDATA_RLOCK(llt->llt_ifp, llt->llt_af); + error = lltable_foreach_lle(llt, + (llt_foreach_cb_t *)llt->llt_dump_entry, wr); + IF_AFDATA_RUNLOCK(llt->llt_ifp, llt->llt_af); + + return (error); +} + +/* + * Dump arp state for a specific address family. + */ +int +lltable_sysctl_dumparp(int af, struct sysctl_req *wr) +{ + struct lltable *llt = NULL; + int error = 0; + + LLTABLE_RLOCK(); + SLIST_FOREACH(llt, &lltables, llt_link) { + if (llt->llt_af == af) { + error = lltable_dump_af(llt, wr); + if (error != 0) + goto done; + } + } +done: + LLTABLE_RUNLOCK(); + return (error); +} + +/* + * Common function helpers for chained hash table. + */ + +/* + * Runs specified callback for each entry in @llt. + * Caller does the locking. + * + */ +static int +htable_foreach_lle(struct lltable *llt, llt_foreach_cb_t *f, void *farg) +{ + struct llentry *lle, *next; + int i, error; + + error = 0; + + for (i = 0; i < llt->llt_hsize; i++) { + LIST_FOREACH_SAFE(lle, &llt->lle_head[i], lle_next, next) { + error = f(llt, lle, farg); + if (error != 0) + break; + } + } + + return (error); +} + +static void +htable_link_entry(struct lltable *llt, struct llentry *lle) +{ + struct llentries *lleh; + uint32_t hashidx; + + if ((lle->la_flags & LLE_LINKED) != 0) + return; + + IF_AFDATA_WLOCK_ASSERT(llt->llt_ifp, llt->llt_af); + + hashidx = llt->llt_hash(lle, llt->llt_hsize); + lleh = &llt->lle_head[hashidx]; + + lle->lle_tbl = llt; + lle->lle_head = lleh; + lle->la_flags |= LLE_LINKED; + LIST_INSERT_HEAD(lleh, lle, lle_next); +} + +static void +htable_unlink_entry(struct llentry *lle) +{ + if ((lle->la_flags & LLE_LINKED) != 0) { + IF_AFDATA_WLOCK_ASSERT(lle->lle_tbl->llt_ifp, lle->lle_tbl->llt_af); + LIST_REMOVE(lle, lle_next); + lle->la_flags &= ~(LLE_VALID | LLE_LINKED); +#if 0 + lle->lle_tbl = NULL; + lle->lle_head = NULL; +#endif + } +} + +struct prefix_match_data { + const struct sockaddr *addr; + const struct sockaddr *mask; + struct llentries dchain; + u_int flags; +}; + +static int +htable_prefix_free_cb(struct lltable *llt, struct llentry *lle, void *farg) +{ + struct prefix_match_data *pmd; + + pmd = (struct prefix_match_data *)farg; + + if (llt->llt_match_prefix(pmd->addr, pmd->mask, pmd->flags, lle)) { + LLE_WLOCK(lle); + LIST_INSERT_HEAD(&pmd->dchain, lle, lle_chain); + } + + return (0); +} + +static void +htable_prefix_free(struct lltable *llt, const struct sockaddr *addr, + const struct sockaddr *mask, u_int flags) +{ + struct llentry *lle, *next; + struct prefix_match_data pmd; + + bzero(&pmd, sizeof(pmd)); + pmd.addr = addr; + pmd.mask = mask; + pmd.flags = flags; + LIST_INIT(&pmd.dchain); + + IF_AFDATA_WLOCK(llt->llt_ifp, llt->llt_af); + /* Push matching lles to chain */ + lltable_foreach_lle(llt, htable_prefix_free_cb, &pmd); + + llentries_unlink(llt, &pmd.dchain); + IF_AFDATA_WUNLOCK(llt->llt_ifp, llt->llt_af); + + LIST_FOREACH_SAFE(lle, &pmd.dchain, lle_chain, next) + lltable_free_entry(llt, lle); +} + +static void +htable_free_tbl(struct lltable *llt) +{ + + FREE(llt->lle_head, M_LLTABLE); + FREE(llt, M_LLTABLE); +} + +static void +llentries_unlink(struct lltable *llt, struct llentries *head) +{ + struct llentry *lle, *next; + + LIST_FOREACH_SAFE(lle, head, lle_chain, next) + llt->llt_unlink_entry(lle); +} + +/* + * Helper function used to drop all mbufs in hold queue. + * + * Returns the number of held packets, if any, that were dropped. + */ +size_t +lltable_drop_entry_queue(struct llentry *lle) +{ + size_t pkts_dropped; + struct mbuf *next; + + LLE_WLOCK_ASSERT(lle); + + pkts_dropped = 0; + while ((lle->la_numheld > 0) && (lle->la_hold != NULL)) { + next = lle->la_hold->m_nextpkt; + m_freem(lle->la_hold); + lle->la_hold = next; + lle->la_numheld--; + pkts_dropped++; + } + + KASSERT(lle->la_numheld == 0, + ("%s: la_numheld %d > 0, pkts_droped %zd", __func__, + lle->la_numheld, pkts_dropped)); + + return (pkts_dropped); +} + +void +lltable_set_entry_addr(struct ifnet *ifp, struct llentry *lle, + const char *ll_addr) +{ + bcopy(ll_addr, &lle->ll_addr, ifp->if_addrlen); + lle->la_flags |= LLE_VALID; + lle->r_flags |= RLLE_VALID; +} + +#if 0 +/* + * XXX The following is related to a change to cache destination layer 2 + * header cached in the entry instead of just the destination mac address + * Right now leaving this code out and just storing the destination's mac + * information. + */ +/* + * Tries to update @lle link-level address. + * Since update requires AFDATA WLOCK, function + * drops @lle lock, acquires AFDATA lock and then acquires + * @lle lock to maintain lock order. + * + * Returns 1 on success. + */ +int +lltable_try_set_entry_addr(struct ifnet *ifp, struct llentry *lle, + const char *linkhdr, size_t linkhdrsize, int lladdr_off) +{ + /* Perform real LLE update */ + /* use afdata WLOCK to update fields */ + LLE_WLOCK_ASSERT(lle); + LLE_ADDREF(lle); + LLE_WUNLOCK(lle); + IF_AFDATA_WLOCK(ifp, lle->lle_tbl->llt_af); + LLE_WLOCK(lle); + + /* + * Since we droppped LLE lock, other thread might have deleted + * this lle. Check and return + */ + if ((lle->la_flags & LLE_DELETED) != 0) { + IF_AFDATA_WUNLOCK(ifp, lle->lle_tbl->llt_af); + LLE_FREE_LOCKED(lle); + return (0); + } + + /* Update data */ + lltable_set_entry_addr(ifp, lle, linkhdr, linkhdrsize, lladdr_off); + + IF_AFDATA_WUNLOCK(ifp, lle->lle_tbl->llt_af); + + LLE_REMREF(lle); + + return (1); +} + +/* + * Helper function used to pre-compute full/partial link-layer + * header data suitable for feeding into if_output(). + */ +int +lltable_calc_llheader(struct ifnet *ifp, int family, char *lladdr, + char *buf, size_t *bufsize, int *lladdr_off) +{ + struct if_encap_req ereq; + int error; + + bzero(buf, *bufsize); + bzero(&ereq, sizeof(ereq)); + ereq.buf = buf; + ereq.bufsize = *bufsize; + ereq.rtype = IFENCAP_LL; + ereq.family = family; + ereq.lladdr = lladdr; + ereq.lladdr_len = ifp->if_addrlen; + error = ifp->if_requestencap(ifp, &ereq); + if (error == 0) { + *bufsize = ereq.bufsize; + *lladdr_off = ereq.lladdr_off; + } + + return (error); +} + +/* + * Update link-layer header for given @lle after + * interface lladdr was changed. + */ +static int +llentry_update_ifaddr(struct lltable *llt, struct llentry *lle, void *farg) +{ + struct ifnet *ifp; + u_char linkhdr[LLE_MAX_LINKHDR]; + size_t linkhdrsize; + u_char *lladdr; + int lladdr_off; + + ifp = (struct ifnet *)farg; + + lladdr = (void *)lle->ll_addr; + + LLE_WLOCK(lle); + if ((lle->la_flags & LLE_VALID) == 0) { + LLE_WUNLOCK(lle); + return (0); + } + + if ((lle->la_flags & LLE_IFADDR) != 0) + lladdr = (void *)IF_LLADDR(ifp); + + linkhdrsize = sizeof(linkhdr); + lltable_calc_llheader(ifp, llt->llt_af, (void *)lladdr, (void *)linkhdr, &linkhdrsize, + &lladdr_off); + memcpy(lle->r_linkdata, linkhdr, linkhdrsize); + LLE_WUNLOCK(lle); + + return (0); +} + +/* + * Update all calculated headers for given @llt + */ +void +lltable_update_ifaddr(struct lltable *llt) +{ + + if (llt->llt_ifp->if_flags & IFF_LOOPBACK) + return; + + IF_AFDATA_WLOCK(llt->llt_ifp, llt->llt_af); + lltable_foreach_lle(llt, llentry_update_ifaddr, llt->llt_ifp); + IF_AFDATA_WUNLOCK(llt->llt_ifp, llt->llt_af); +} +#endif + +/* + * + * Performs generic cleanup routines and frees lle. + * + * Called for non-linked entries, with callouts and + * other AF-specific cleanups performed. + * + * @lle must be passed WLOCK'ed + * + * Returns the number of held packets, if any, that were dropped. + */ +size_t +llentry_free(struct llentry *lle) +{ + size_t pkts_dropped; + + LLE_WLOCK_ASSERT(lle); + + KASSERT((lle->la_flags & LLE_LINKED) == 0, ("freeing linked lle")); + + pkts_dropped = lltable_drop_entry_queue(lle); + + LLE_FREE_LOCKED(lle); + + return (pkts_dropped); +} + +/* + * (al)locate an llentry for address dst (equivalent to rtalloc for new-arp). + * + * If found the llentry * is returned referenced and unlocked. + */ +struct llentry * +llentry_alloc(struct ifnet *ifp, struct lltable *lt, + struct sockaddr_storage *dst) +{ + struct llentry *la, *la_tmp; + + IF_AFDATA_RLOCK(ifp, lt->llt_af); + la = lla_lookup(lt, LLE_EXCLUSIVE, (struct sockaddr *)dst); + IF_AFDATA_RUNLOCK(ifp, lt->llt_af); + + if (la != NULL) { + LLE_ADDREF(la); + LLE_WUNLOCK(la); + return (la); + } + + if ((ifp->if_flags & IFF_NOARP) == 0) { + la = lltable_alloc_entry(lt, 0, (struct sockaddr *)dst); + if (la == NULL) + return (NULL); + IF_AFDATA_WLOCK(ifp, lt->llt_af); + LLE_WLOCK(la); + /* Prefer any existing LLE over newly-created one */ + la_tmp = lla_lookup(lt, LLE_EXCLUSIVE, (struct sockaddr *)dst); + if (la_tmp == NULL) + lltable_link_entry(lt, la); + IF_AFDATA_WUNLOCK(ifp, lt->llt_af); + if (la_tmp != NULL) { + lltable_free_entry(lt, la); + la = la_tmp; + } + LLE_ADDREF(la); + LLE_WUNLOCK(la); + } + + return (la); +} + +/* + * Free all entries from given table and free itself. + */ + +static int +lltable_free_cb(struct lltable *llt, struct llentry *lle, void *farg) +{ +#pragma unused(llt) + struct llentries *dchain; + + dchain = (struct llentries *)farg; + + LLE_WLOCK(lle); + LIST_INSERT_HEAD(dchain, lle, lle_chain); + + return (0); +} + +/* + * Free all entries from given table and free itself. + */ +void +lltable_free(struct lltable *llt) +{ + struct llentry *lle, *next; + struct llentries dchain; + + KASSERT(llt != NULL, ("%s: llt is NULL", __func__)); + + //lltable_unlink(llt); + + LIST_INIT(&dchain); + IF_AFDATA_WLOCK(llt->llt_ifp, llt->llt_af); + /* Push all lles to @dchain */ + lltable_foreach_lle(llt, lltable_free_cb, &dchain); + llentries_unlink(llt, &dchain); + IF_AFDATA_WUNLOCK(llt->llt_ifp, llt->llt_af); + + LIST_FOREACH_SAFE(lle, &dchain, lle_chain, next) { +#if 0 + if (thread_call_cancel(lle->lle_timer) == TRUE) + LLE_REMREF(lle); +#endif + llentry_free(lle); + } + + /* XXX We recycle network interfaces so we only purge */ + /* llt->llt_free_tbl(llt); */ +} + +#if 0 +void +lltable_drain(int af) +{ + struct lltable *llt; + struct llentry *lle; + register int i; + + LLTABLE_RLOCK(); + SLIST_FOREACH(llt, &lltables, llt_link) { + if (llt->llt_af != af) + continue; + + for (i=0; i < llt->llt_hsize; i++) { + LIST_FOREACH(lle, &llt->lle_head[i], lle_next) { + LLE_WLOCK(lle); + if (lle->la_hold) { + m_freem(lle->la_hold); + lle->la_hold = NULL; + } + LLE_WUNLOCK(lle); + } + } + } + LLTABLE_RUNLOCK(); +} +#endif + +/* + * Deletes an address from given lltable. + * Used for userland interaction to remove + * individual entries. Skips entries added by OS. + */ +int +lltable_delete_addr(struct lltable *llt, u_int flags, + const struct sockaddr *l3addr) +{ + struct llentry *lle; + struct ifnet *ifp; + + ifp = llt->llt_ifp; + IF_AFDATA_WLOCK(ifp, llt->llt_af); + lle = lla_lookup(llt, LLE_EXCLUSIVE, l3addr); + + if (lle == NULL) { + IF_AFDATA_WUNLOCK(ifp, llt->llt_af); + return (ENOENT); + } + if ((lle->la_flags & LLE_IFADDR) != 0 && (flags & LLE_IFADDR) == 0) { + IF_AFDATA_WUNLOCK(ifp, llt->llt_af); + LLE_WUNLOCK(lle); + return (EPERM); + } + + lltable_unlink_entry(llt, lle); + IF_AFDATA_WUNLOCK(ifp, llt->llt_af); + + llt->llt_delete_entry(llt, lle); + + return (0); +} + +void +lltable_prefix_free(int af, struct sockaddr *addr, struct sockaddr *mask, + u_int flags) +{ + struct lltable *llt; + + LLTABLE_RLOCK(); + SLIST_FOREACH(llt, &lltables, llt_link) { + if (llt->llt_af != af) + continue; + + llt->llt_prefix_free(llt, addr, mask, flags); + } + LLTABLE_RUNLOCK(); +} + +struct lltable * +lltable_allocate_htbl(uint32_t hsize) +{ + struct lltable *llt; + int i; + + MALLOC(llt, struct lltable *, sizeof(struct lltable), M_LLTABLE, M_WAITOK | M_ZERO); + llt->llt_hsize = hsize; + MALLOC(llt->lle_head, struct llentries *, sizeof(struct llentries) * hsize, + M_LLTABLE, M_WAITOK | M_ZERO); + + for (i = 0; i < llt->llt_hsize; i++) + LIST_INIT(&llt->lle_head[i]); + + /* Set some default callbacks */ + llt->llt_link_entry = htable_link_entry; + llt->llt_unlink_entry = htable_unlink_entry; + llt->llt_prefix_free = htable_prefix_free; + llt->llt_foreach_entry = htable_foreach_lle; + llt->llt_free_tbl = htable_free_tbl; + + return (llt); +} + +/* + * Links lltable to global llt list. + */ +void +lltable_link(struct lltable *llt) +{ + LLTABLE_WLOCK(); + SLIST_INSERT_HEAD(&lltables, llt, llt_link); + LLTABLE_WUNLOCK(); +} + +#if 0 +static void +lltable_unlink(struct lltable *llt) +{ + LLTABLE_WLOCK(); + SLIST_REMOVE(&lltables, llt, lltable, llt_link); + LLTABLE_WUNLOCK(); +} +#endif + +/* + * External methods used by lltable consumers + */ + +int +lltable_foreach_lle(struct lltable *llt, llt_foreach_cb_t *f, void *farg) +{ + return (llt->llt_foreach_entry(llt, f, farg)); +} + +struct llentry * +lltable_alloc_entry(struct lltable *llt, u_int flags, + const struct sockaddr *l3addr) +{ + return (llt->llt_alloc_entry(llt, flags, l3addr)); +} + +void +lltable_free_entry(struct lltable *llt, struct llentry *lle) +{ + llt->llt_free_entry(llt, lle); +} + +void +lltable_link_entry(struct lltable *llt, struct llentry *lle) +{ + llt->llt_link_entry(llt, lle); +} + +void +lltable_unlink_entry(struct lltable *llt, struct llentry *lle) +{ + llt->llt_unlink_entry(lle); +} + +void +lltable_fill_sa_entry(const struct llentry *lle, struct sockaddr *sa) +{ + struct lltable *llt; + + llt = lle->lle_tbl; + llt->llt_fill_sa_entry(lle, sa); +} + +struct ifnet * +lltable_get_ifp(const struct lltable *llt) +{ + return (llt->llt_ifp); +} + +int +lltable_get_af(const struct lltable *llt) +{ + return (llt->llt_af); +} + +#define ifnet_byindex(index) ifindex2ifnet[(index)] + +/* + * Called in route_output when rtm_flags contains RTF_LLDATA. + */ +int +lla_rt_output(struct rt_msghdr *rtm, struct rt_addrinfo *info) +{ + struct sockaddr_dl *dl = + (struct sockaddr_dl *)(void *)info->rti_info[RTAX_GATEWAY]; + struct sockaddr *dst = (struct sockaddr *)info->rti_info[RTAX_DST]; + struct ifnet *ifp; + struct lltable *llt; + struct llentry *lle, *lle_tmp; + u_int laflags = 0; + int error; + + KASSERT(dl != NULL && dl->sdl_family == AF_LINK, + ("%s: invalid dl\n", __func__)); + + ifp = ifnet_byindex(dl->sdl_index); + if (ifp == NULL) { + log(LOG_INFO, "%s: invalid ifp (sdl_index %d)\n", + __func__, dl->sdl_index); + return EINVAL; + } + + /* XXX linked list may be too expensive */ + LLTABLE_RLOCK(); + SLIST_FOREACH(llt, &lltables, llt_link) { + if (llt->llt_af == dst->sa_family && + llt->llt_ifp == ifp) + break; + } + LLTABLE_RUNLOCK(); + KASSERT(llt != NULL, ("Yep, ugly hacks are bad\n")); + + error = 0; + + switch (rtm->rtm_type) { + case RTM_ADD: + /* Add static LLE */ + laflags = 0; + if (rtm->rtm_rmx.rmx_expire == 0) + laflags = LLE_STATIC; + lle = lltable_alloc_entry(llt, laflags, dst); + if (lle == NULL) + return (ENOMEM); +#if 0 + linkhdrsize = sizeof(linkhdr); + if (lltable_calc_llheader(ifp, dst->sa_family, LLADDR(dl), + (void *)linkhdr, &linkhdrsize, &lladdr_off) != 0) + return (EINVAL); +#endif + lltable_set_entry_addr(ifp, lle, LLADDR(dl)); + + if (rtm->rtm_flags & RTF_ANNOUNCE) + lle->la_flags |= LLE_PUB; + lle->la_expire = rtm->rtm_rmx.rmx_expire; + + laflags = lle->la_flags; + + /* Try to link new entry */ + lle_tmp = NULL; + IF_AFDATA_WLOCK(ifp, llt->llt_af); + LLE_WLOCK(lle); + lle_tmp = lla_lookup(llt, LLE_EXCLUSIVE, dst); + if (lle_tmp != NULL) { + /* Check if we are trying to replace immutable entry */ + if ((lle_tmp->la_flags & LLE_IFADDR) != 0) { + IF_AFDATA_WUNLOCK(ifp, llt->llt_af); + LLE_WUNLOCK(lle_tmp); + lltable_free_entry(llt, lle); + return (EPERM); + } + /* Unlink existing entry from table */ + lltable_unlink_entry(llt, lle_tmp); + } + lltable_link_entry(llt, lle); + IF_AFDATA_WUNLOCK(ifp, llt->llt_af); + + if (lle_tmp != NULL) { + EVENTHANDLER_INVOKE(NULL, lle_event, lle_tmp, LLENTRY_EXPIRED); + lltable_free_entry(llt, lle_tmp); + } + + /* + * By invoking LLE handler here we might get + * two events on static LLE entry insertion + * in routing socket. However, since we might have + * other subscribers we need to generate this event. + */ + EVENTHANDLER_INVOKE(NULL, lle_event, lle, LLENTRY_RESOLVED); + LLE_WUNLOCK(lle); +#ifdef INET + /* gratuitous ARP */ + if ((laflags & LLE_PUB) && dst->sa_family == AF_INET) + dlil_send_arp(ifp, ARPOP_REQUEST, NULL, dst, NULL, dst, 0); +#endif + + break; + + case RTM_DELETE: + return (lltable_delete_addr(llt, 0, dst)); + + default: + error = EINVAL; + } + + return (error); +} + diff --git a/bsd/net/if_llatbl.h b/bsd/net/if_llatbl.h new file mode 100644 index 000000000..9b4697fb6 --- /dev/null +++ b/bsd/net/if_llatbl.h @@ -0,0 +1,302 @@ +/* + * Copyright (c) 2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * Copyright (c) 2004 Luigi Rizzo, Alessandro Cerri. All rights reserved. + * Copyright (c) 2004-2008 Qing Li. All rights reserved. + * Copyright (c) 2008 Kip Macy. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +#include <sys/cdefs.h> + +#ifndef _NET_IF_LLATBL_H_ +#define _NET_IF_LLATBL_H_ + +#include <netinet/in.h> +#include <net/if_dl.h> +#include <kern/locks.h> +#include <kern/thread_call.h> +#include <sys/eventhandler.h> + +struct ifnet; +struct sysctl_req; +struct rt_msghdr; +struct rt_addrinfo; + +struct llentry; +LIST_HEAD(llentries, llentry); + +extern lck_rw_t *lltable_rwlock; +#define LLTABLE_RLOCK() lck_rw_lock_shared(lltable_rwlock) +#define LLTABLE_RUNLOCK() lck_rw_done(lltable_rwlock) +#define LLTABLE_WLOCK() lck_rw_lock_exclusive(lltable_rwlock) +#define LLTABLE_WUNLOCK() lck_rw_done(lltable_rwlock) +#define LLTABLE_LOCK_ASSERT() LCK_RW_ASSERT(lltable_rwlock, LCK_RW_ASSERT_EXCLUSIVE) + +#define LLE_MAX_LINKHDR 24 /* Full IB header */ +/* + * Code referencing llentry must at least hold + * a shared lock + */ +struct llentry { + LIST_ENTRY(llentry) lle_next; + union { + struct in_addr addr4; + struct in6_addr addr6; + } r_l3addr; +#if 0 + char r_linkdata[LLE_MAX_LINKHDR]; /* L2 data */ + uint8_t r_hdrlen; /* length for LL header */ + uint8_t spare0[3]; +#endif + uint16_t r_flags; /* LLE runtime flags */ + uint16_t r_skip_req; /* feedback from fast path */ + + struct lltable *lle_tbl; + struct llentries *lle_head; + void (*lle_free)(struct llentry *); + struct mbuf *la_hold; + int la_numheld; /* # of packets currently held */ + u_int64_t la_expire; + uint16_t la_flags; + uint16_t la_asked; + uint16_t la_preempt; + int16_t ln_state; /* IPv6 has ND6_LLINFO_NOSTATE == -2 */ + uint16_t ln_router; + time_t ln_ntick; + time_t lle_remtime; /* Real time remaining */ + time_t lle_hittime; /* Time when r_skip_req was unset */ + int lle_refcnt; + union { + uint64_t mac_aligned; + uint16_t mac16[3]; + } ll_addr; + LIST_ENTRY(llentry) lle_chain; /* chain of deleted items */ + thread_call_t lle_timer; + u_int64_t ln_lastused; /* last used timestamp */ + struct if_llreach *ln_llreach; /* link-layer reachability record */ + decl_lck_rw_data(, lle_lock); + decl_lck_mtx_data(, req_mtx); +}; + +extern lck_grp_t *lle_lock_grp; +extern lck_attr_t *lle_lock_attr; + +#define LLE_WLOCK(lle) lck_rw_lock_exclusive(&(lle)->lle_lock) +#define LLE_RLOCK(lle) lck_rw_lock_shared(&(lle)->lle_lock) +#define LLE_WUNLOCK(lle) lck_rw_done(&(lle)->lle_lock) +#define LLE_RUNLOCK(lle) lck_rw_done(&(lle)->lle_lock) +#define LLE_DOWNGRADE(lle) lck_rw_lock_exclusive_to_shared(&(lle)->lle_lock) +#define LLE_TRY_UPGRADE(lle) lck_rw_lock_shared_to_exclusive(&(lle)->lle_lock) +#define LLE_LOCK_INIT(lle) lck_rw_init(&(lle)->lle_lock, lle_lock_grp, lle_lock_attr) +#define LLE_LOCK_DESTROY(lle) lck_rw_destroy(&(lle)->lle_lock, lle_lock_grp) +#define LLE_WLOCK_ASSERT(lle) LCK_RW_ASSERT(&(lle)->lle_lock, LCK_RW_ASSERT_EXCLUSIVE) + +#define LLE_REQ_INIT(lle) lck_mtx_init(&(lle)->req_mtx, lle_lock_grp, lle_lock_attr) +#define LLE_REQ_DESTROY(lle) lck_mtx_destroy(&(lle)->req_mtx, lle_lock_grp) +#define LLE_REQ_LOCK(lle) lck_mtx_lock(&(lle)->req_mtx) +#define LLE_REQ_UNLOCK(lle) lck_mtx_unlock(&(lle)->req_mtx) + +#define LLE_IS_VALID(lle) (((lle) != NULL) && ((lle) != (void *)-1)) + +#define LLE_ADDREF(lle) do { \ + LLE_WLOCK_ASSERT(lle); \ + VERIFY((lle)->lle_refcnt >= 0); \ + (lle)->lle_refcnt++; \ +} while (0) + +#define LLE_REMREF(lle) do { \ + LLE_WLOCK_ASSERT(lle); \ + VERIFY((lle)->lle_refcnt > 0); \ + (lle)->lle_refcnt--; \ +} while (0) + +#define LLE_FREE_LOCKED(lle) do { \ + if ((lle)->lle_refcnt == 1) \ + (lle)->lle_free(lle); \ + else { \ + LLE_REMREF(lle); \ + LLE_WUNLOCK(lle); \ + } \ + /* guard against invalid refs */ \ + (lle) = NULL; \ +} while (0) + +#define LLE_FREE(lle) do { \ + LLE_WLOCK(lle); \ + LLE_FREE_LOCKED(lle); \ +} while (0) + +typedef struct llentry *(llt_lookup_t)(struct lltable *, u_int flags, + const struct sockaddr *l3addr); +typedef struct llentry *(llt_alloc_t)(struct lltable *, u_int flags, + const struct sockaddr *l3addr); +typedef void (llt_delete_t)(struct lltable *, struct llentry *); +typedef void (llt_prefix_free_t)(struct lltable *, + const struct sockaddr *addr, const struct sockaddr *mask, u_int flags); +typedef int (llt_dump_entry_t)(struct lltable *, struct llentry *, + struct sysctl_req *); +typedef uint32_t (llt_hash_t)(const struct llentry *, uint32_t); +typedef int (llt_match_prefix_t)(const struct sockaddr *, + const struct sockaddr *, u_int, struct llentry *); +typedef void (llt_free_entry_t)(struct lltable *, struct llentry *); +typedef void (llt_fill_sa_entry_t)(const struct llentry *, struct sockaddr *); +typedef void (llt_free_tbl_t)(struct lltable *); +typedef void (llt_link_entry_t)(struct lltable *, struct llentry *); +typedef void (llt_unlink_entry_t)(struct llentry *); + +typedef int (llt_foreach_cb_t)(struct lltable *, struct llentry *, void *); +typedef int (llt_foreach_entry_t)(struct lltable *, llt_foreach_cb_t *, void *); + +struct lltable { + SLIST_ENTRY(lltable) llt_link; + int llt_af; + int llt_hsize; + struct llentries *lle_head; + struct ifnet *llt_ifp; + + llt_lookup_t *llt_lookup; + llt_alloc_t *llt_alloc_entry; + llt_delete_t *llt_delete_entry; + llt_prefix_free_t *llt_prefix_free; + llt_dump_entry_t *llt_dump_entry; + llt_hash_t *llt_hash; + llt_match_prefix_t *llt_match_prefix; + llt_free_entry_t *llt_free_entry; + llt_foreach_entry_t *llt_foreach_entry; + llt_link_entry_t *llt_link_entry; + llt_unlink_entry_t *llt_unlink_entry; + llt_fill_sa_entry_t *llt_fill_sa_entry; + llt_free_tbl_t *llt_free_tbl; +}; + +#ifdef MALLOC_DECLARE +MALLOC_DECLARE(M_LLTABLE); +#endif + +/* + * LLentry flags + */ +#define LLE_DELETED 0x0001 /* entry must be deleted */ +#define LLE_STATIC 0x0002 /* entry is static */ +#define LLE_IFADDR 0x0004 /* entry is interface addr */ +#define LLE_VALID 0x0008 /* ll_addr is valid */ +#define LLE_REDIRECT 0x0010 /* installed by redirect; has host rtentry */ +#define LLE_PUB 0x0020 /* publish entry ??? */ +#define LLE_LINKED 0x0040 /* linked to lookup structure */ +/* LLE request flags */ +#define LLE_EXCLUSIVE 0x2000 /* return lle xlocked */ +#define LLE_UNLOCKED 0x4000 /* return lle unlocked */ +#define LLE_ADDRONLY 0x4000 /* return lladdr instead of full header */ +#define LLE_CREATE 0x8000 /* hint to avoid lle lookup */ + +/* LLE flags used by fastpath code */ +#define RLLE_VALID 0x0001 /* entry is valid */ +#define RLLE_IFADDR LLE_IFADDR /* entry is ifaddr */ + +#define LLATBL_HASH(key, mask) \ + (((((((key >> 8) ^ key) >> 8) ^ key) >> 8) ^ key) & mask) + +void lltable_glbl_init(void); +struct lltable *lltable_allocate_htbl(uint32_t hsize); +void lltable_free(struct lltable *); +void lltable_link(struct lltable *llt); +void lltable_prefix_free(int, struct sockaddr *, + struct sockaddr *, u_int); +#if 0 +void lltable_drain(int); +#endif +int lltable_sysctl_dumparp(int, struct sysctl_req *); + +size_t llentry_free(struct llentry *); +struct llentry *llentry_alloc(struct ifnet *, struct lltable *, + struct sockaddr_storage *); + +/* helper functions */ +size_t lltable_drop_entry_queue(struct llentry *); +void lltable_set_entry_addr(struct ifnet *ifp, struct llentry *lle, + const char *ll_addr); +int lltable_try_set_entry_addr(struct ifnet *ifp, struct llentry *lle, + const char *linkhdr, size_t linkhdrsize, int lladdr_off); + +int lltable_calc_llheader(struct ifnet *ifp, int family, char *lladdr, + char *buf, size_t *bufsize, int *lladdr_off); +void lltable_update_ifaddr(struct lltable *llt); +struct llentry *lltable_alloc_entry(struct lltable *llt, u_int flags, + const struct sockaddr *l4addr); +void lltable_free_entry(struct lltable *llt, struct llentry *lle); +int lltable_delete_addr(struct lltable *llt, u_int flags, + const struct sockaddr *l3addr); +void lltable_link_entry(struct lltable *llt, struct llentry *lle); +void lltable_unlink_entry(struct lltable *llt, struct llentry *lle); +void lltable_fill_sa_entry(const struct llentry *lle, struct sockaddr *sa); +struct ifnet *lltable_get_ifp(const struct lltable *llt); +int lltable_get_af(const struct lltable *llt); + +int lltable_foreach_lle(struct lltable *llt, llt_foreach_cb_t *f, + void *farg); +/* + * Generic link layer address lookup function. + */ +static __inline struct llentry * +lla_lookup(struct lltable *llt, u_int flags, const struct sockaddr *l3addr) +{ + return (llt->llt_lookup(llt, flags, l3addr)); +} + +int lla_rt_output(struct rt_msghdr *, struct rt_addrinfo *); + +enum { + LLENTRY_RESOLVED, + LLENTRY_TIMEDOUT, + LLENTRY_DELETED, + LLENTRY_EXPIRED, +}; + +typedef void (*lle_event_fn)(struct eventhandler_entry_arg, struct llentry *, int); +EVENTHANDLER_DECLARE(lle_event, lle_event_fn); +#endif /* _NET_IF_LLATBL_H_ */ diff --git a/bsd/net/if_llreach.c b/bsd/net/if_llreach.c index 2012d8c05..764e72bd8 100644 --- a/bsd/net/if_llreach.c +++ b/bsd/net/if_llreach.c @@ -320,7 +320,7 @@ found: if (!lck_rw_lock_shared_to_exclusive(&ifp->if_llreach_lock)) lck_rw_lock_exclusive(&ifp->if_llreach_lock); - lck_rw_assert(&ifp->if_llreach_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&ifp->if_llreach_lock, LCK_RW_ASSERT_EXCLUSIVE); /* in case things have changed while becoming writer */ lr = RB_FIND(ll_reach_tree, &ifp->if_ll_srcs, &find); diff --git a/bsd/net/if_llreach.h b/bsd/net/if_llreach.h index c27ff9ded..63c86aef4 100644 --- a/bsd/net/if_llreach.h +++ b/bsd/net/if_llreach.h @@ -97,10 +97,10 @@ RB_PROTOTYPE_SC_PREV(__private_extern__, ll_reach_tree, if_llreach, ls_link, ifllr_cmp); #define IFLR_LOCK_ASSERT_HELD(_iflr) \ - lck_mtx_assert(&(_iflr)->lr_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_iflr)->lr_lock, LCK_MTX_ASSERT_OWNED) #define IFLR_LOCK_ASSERT_NOTHELD(_iflr) \ - lck_mtx_assert(&(_iflr)->lr_lock, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_iflr)->lr_lock, LCK_MTX_ASSERT_NOTOWNED) #define IFLR_LOCK(_iflr) \ lck_mtx_lock(&(_iflr)->lr_lock) diff --git a/bsd/net/if_loop.c b/bsd/net/if_loop.c index 3baa27434..00a8345e3 100644 --- a/bsd/net/if_loop.c +++ b/bsd/net/if_loop.c @@ -177,15 +177,6 @@ SYSCTL_DECL(_net_link); SYSCTL_NODE(_net_link, OID_AUTO, loopback, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "loopback interface"); -#define LO_BW_SLEEP 10 -static u_int32_t lo_bw_sleep_usec = LO_BW_SLEEP; -SYSCTL_UINT(_net_link_loopback, OID_AUTO, bw_sleep_usec, - CTLFLAG_RW | CTLFLAG_LOCKED, &lo_bw_sleep_usec, LO_BW_SLEEP, ""); - -static u_int32_t lo_bw_measure = 0; -SYSCTL_UINT(_net_link_loopback, OID_AUTO, bw_measure, - CTLFLAG_RW | CTLFLAG_LOCKED, &lo_bw_measure, 0, ""); - static u_int32_t lo_dequeue_max = LOSNDQ_MAXLEN; SYSCTL_PROC(_net_link_loopback, OID_AUTO, max_dequeue, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &lo_dequeue_max, LOSNDQ_MAXLEN, @@ -405,8 +396,6 @@ lo_start(struct ifnet *ifp) for (;;) { struct mbuf *m = NULL, *m_tail = NULL; u_int32_t cnt, len = 0; - int sleep_chan = 0; - struct timespec ts; if (lo_sched_model == IFNET_SCHED_MODEL_NORMAL) { if (ifnet_dequeue_multi(ifp, lo_dequeue_max, &m, @@ -420,21 +409,6 @@ lo_start(struct ifnet *ifp) } LO_BPF_TAP_OUT_MULTI(m); - - if (lo_bw_measure) { - if (cnt >= if_bw_measure_size) - ifnet_transmit_burst_start(ifp, m); - if (lo_bw_sleep_usec > 0) { - bzero(&ts, sizeof(ts)); - ts.tv_nsec = (lo_bw_sleep_usec << 10) * cnt; - - /* Add msleep with timeout */ - (void) msleep(&sleep_chan, NULL, - PSOCK, "lo_start", &ts); - } - if (cnt >= if_bw_measure_size) - ifnet_transmit_burst_end(ifp, m_tail); - } lo_tx_compl(ifp, m); /* stats are required for extended variant */ @@ -683,6 +657,7 @@ loopattach(void) lo_init.flags = IFNET_INIT_LEGACY; lo_init.output = lo_output; } + lo_init.flags |= IFNET_INIT_NX_NOAUTO; lo_init.name = "lo"; lo_init.unit = 0; lo_init.family = IFNET_FAMILY_LOOPBACK; diff --git a/bsd/net/if_pflog.c b/bsd/net/if_pflog.c index 3bc86c820..1c9113d70 100644 --- a/bsd/net/if_pflog.c +++ b/bsd/net/if_pflog.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2007-2013 Apple Inc. All rights reserved. + * Copyright (c) 2007-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -136,7 +136,7 @@ static int pflog_clone_create(struct if_clone *ifc, u_int32_t unit, __unused void *params) { struct pflog_softc *pflogif; - struct ifnet_init_params pf_init; + struct ifnet_init_eparams pf_init; int error = 0; if (unit >= PFLOGIFS_MAX) { @@ -153,6 +153,9 @@ pflog_clone_create(struct if_clone *ifc, u_int32_t unit, __unused void *params) } bzero(&pf_init, sizeof (pf_init)); + pf_init.ver = IFNET_INIT_CURRENT_VERSION; + pf_init.len = sizeof (pf_init); + pf_init.flags = IFNET_INIT_LEGACY; pf_init.name = ifc->ifc_name; pf_init.unit = unit; pf_init.type = IFT_PFLOG; @@ -168,7 +171,7 @@ pflog_clone_create(struct if_clone *ifc, u_int32_t unit, __unused void *params) bzero(pflogif, sizeof (*pflogif)); pflogif->sc_unit = unit; - error = ifnet_allocate(&pf_init, &pflogif->sc_if); + error = ifnet_allocate_extended(&pf_init, &pflogif->sc_if); if (error != 0) { printf("%s: ifnet_allocate failed - %d\n", __func__, error); _FREE(pflogif, M_DEVBUF); @@ -281,17 +284,18 @@ pflogfree(struct ifnet *ifp) } int -pflog_packet(struct pfi_kif *kif, struct mbuf *m, sa_family_t af, u_int8_t dir, +pflog_packet(struct pfi_kif *kif, pbuf_t *pbuf, sa_family_t af, u_int8_t dir, u_int8_t reason, struct pf_rule *rm, struct pf_rule *am, struct pf_ruleset *ruleset, struct pf_pdesc *pd) { #if NBPFILTER > 0 struct ifnet *ifn; struct pfloghdr hdr; + struct mbuf *m; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); - if (kif == NULL || m == NULL || rm == NULL || pd == NULL) + if (kif == NULL || !pbuf_is_valid(pbuf) || rm == NULL || pd == NULL) return (-1); if (rm->logif >= PFLOGIFS_MAX || @@ -299,6 +303,9 @@ pflog_packet(struct pfi_kif *kif, struct mbuf *m, sa_family_t af, u_int8_t dir, return (0); } + if ((m = pbuf_to_mbuf(pbuf, FALSE)) == NULL) + return (0); + bzero(&hdr, sizeof (hdr)); hdr.length = PFLOG_REAL_HDRLEN; hdr.af = af; diff --git a/bsd/net/if_stf.c b/bsd/net/if_stf.c index 62f662115..ac4950dd1 100644 --- a/bsd/net/if_stf.c +++ b/bsd/net/if_stf.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -312,7 +312,7 @@ stfattach(void) struct stf_softc *sc; int error; const struct encaptab *p; - struct ifnet_init_params stf_init; + struct ifnet_init_eparams stf_init; stfinit(); @@ -338,6 +338,9 @@ stfattach(void) lck_mtx_init(&sc->sc_ro_mtx, stf_mtx_grp, LCK_ATTR_NULL); bzero(&stf_init, sizeof(stf_init)); + stf_init.ver = IFNET_INIT_CURRENT_VERSION; + stf_init.len = sizeof (stf_init); + stf_init.flags = IFNET_INIT_LEGACY; stf_init.name = "stf"; stf_init.unit = 0; stf_init.type = IFT_STF; @@ -350,7 +353,7 @@ stfattach(void) stf_init.ioctl = stf_ioctl; stf_init.set_bpf_tap = stf_set_bpf_tap; - error = ifnet_allocate(&stf_init, &sc->sc_if); + error = ifnet_allocate_extended(&stf_init, &sc->sc_if); if (error != 0) { printf("stfattach, ifnet_allocate failed - %d\n", error); encap_detach(sc->encap_cookie); diff --git a/bsd/net/if_utun.c b/bsd/net/if_utun.c index 35868bbd2..5e01509df 100644 --- a/bsd/net/if_utun.c +++ b/bsd/net/if_utun.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Apple Inc. All rights reserved. + * Copyright (c) 2008-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -48,11 +48,69 @@ This kernel control will register an interface for every client that connects. #include <sys/mbuf.h> #include <sys/sockio.h> #include <netinet/in.h> +#include <netinet/ip.h> #include <netinet6/in6_var.h> #include <netinet6/in6_var.h> #include <sys/kauth.h> - - +#include <net/necp.h> +#include <kern/zalloc.h> + +#define UTUN_NEXUS 0 + +extern unsigned int if_enable_netagent; + +#if UTUN_NEXUS +static nexus_controller_t utun_ncd; +static int utun_ncd_refcount; +static uuid_t utun_kpipe_uuid; +static uuid_t utun_nx_dom_prov; + +typedef struct utun_nx { + uuid_t if_provider; + uuid_t if_instance; + uuid_t ms_provider; + uuid_t ms_instance; + uuid_t ms_device; + uuid_t ms_host; + uuid_t ms_agent; +} *utun_nx_t; + +#endif // UTUN_NEXUS + +/* Control block allocated for each kernel control connection */ +struct utun_pcb { + TAILQ_ENTRY(utun_pcb) utun_chain; + kern_ctl_ref utun_ctlref; + ifnet_t utun_ifp; + u_int32_t utun_unit; + u_int32_t utun_unique_id; + u_int32_t utun_flags; + int utun_ext_ifdata_stats; + u_int32_t utun_max_pending_packets; + char utun_if_xname[IFXNAMSIZ]; + char utun_unique_name[IFXNAMSIZ]; + // PCB lock protects state fields and rings + decl_lck_rw_data(, utun_pcb_lock); + struct mbuf * utun_input_chain; + struct mbuf * utun_input_chain_last; + // Input chain lock protects the list of input mbufs + // The input chain lock must be taken AFTER the PCB lock if both are held + lck_mtx_t utun_input_chain_lock; + bool utun_output_disabled; + +#if UTUN_NEXUS + struct utun_nx utun_nx; + int utun_kpipe_enabled; + uuid_t utun_kpipe_uuid; + void * utun_kpipe_rxring; + void * utun_kpipe_txring; + + kern_nexus_t utun_netif_nexus; + void * utun_netif_rxring; + void * utun_netif_txring; + uint64_t utun_netif_txring_size; +#endif // UTUN_NEXUS +}; /* Kernel Control functions */ static errno_t utun_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac, @@ -69,13 +127,15 @@ static void utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, int flags); /* Network Interface functions */ +#if !UTUN_NEXUS static void utun_start(ifnet_t interface); +static errno_t utun_framer(ifnet_t interface, mbuf_t *packet, + const struct sockaddr *dest, const char *desk_linkaddr, + const char *frame_type, u_int32_t *prepend_len, u_int32_t *postpend_len); +#endif // !UTUN_NEXUS static errno_t utun_output(ifnet_t interface, mbuf_t data); static errno_t utun_demux(ifnet_t interface, mbuf_t data, char *frame_header, protocol_family_t *protocol); -static errno_t utun_framer(ifnet_t interface, mbuf_t *packet, - const struct sockaddr *dest, const char *desk_linkaddr, - const char *frame_type, u_int32_t *prepend_len, u_int32_t *postpend_len); static errno_t utun_add_proto(ifnet_t interface, protocol_family_t protocol, const struct ifnet_demux_desc *demux_array, u_int32_t demux_count); @@ -92,19 +152,1156 @@ static errno_t utun_proto_pre_output(ifnet_t interface, protocol_family_t protoc char *frame_type, char *link_layer_dest); static errno_t utun_pkt_input (struct utun_pcb *pcb, mbuf_t m); +#if UTUN_NEXUS + +#define UTUN_IF_DEFAULT_SLOT_SIZE 4096 +#define UTUN_IF_DEFAULT_RING_SIZE 64 +#define UTUN_IF_DEFAULT_TX_FSW_RING_SIZE 64 +#define UTUN_IF_DEFAULT_RX_FSW_RING_SIZE 128 +#define UTUN_IF_HEADROOM_SIZE 32 + +#define UTUN_IF_MIN_RING_SIZE 16 +#define UTUN_IF_MAX_RING_SIZE 1024 + +static int sysctl_if_utun_ring_size SYSCTL_HANDLER_ARGS; +static int sysctl_if_utun_tx_fsw_ring_size SYSCTL_HANDLER_ARGS; +static int sysctl_if_utun_rx_fsw_ring_size SYSCTL_HANDLER_ARGS; + +static int if_utun_ring_size = UTUN_IF_DEFAULT_RING_SIZE; +static int if_utun_tx_fsw_ring_size = UTUN_IF_DEFAULT_TX_FSW_RING_SIZE; +static int if_utun_rx_fsw_ring_size = UTUN_IF_DEFAULT_RX_FSW_RING_SIZE; + +SYSCTL_DECL(_net_utun); +SYSCTL_NODE(_net, OID_AUTO, utun, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "UTun"); + +SYSCTL_PROC(_net_utun, OID_AUTO, ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW, + &if_utun_ring_size, UTUN_IF_DEFAULT_RING_SIZE, &sysctl_if_utun_ring_size, "I", ""); +SYSCTL_PROC(_net_utun, OID_AUTO, tx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW, + &if_utun_tx_fsw_ring_size, UTUN_IF_DEFAULT_TX_FSW_RING_SIZE, &sysctl_if_utun_tx_fsw_ring_size, "I", ""); +SYSCTL_PROC(_net_utun, OID_AUTO, rx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW, + &if_utun_rx_fsw_ring_size, UTUN_IF_DEFAULT_RX_FSW_RING_SIZE, &sysctl_if_utun_rx_fsw_ring_size, "I", ""); + +static errno_t +utun_register_nexus(void); + +static errno_t +utun_netif_prepare(__unused kern_nexus_t nexus, ifnet_t ifp); +static errno_t +utun_nexus_pre_connect(kern_nexus_provider_t nxprov, + proc_t p, kern_nexus_t nexus, + nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx); +static errno_t +utun_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel); +static void +utun_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel); +static void +utun_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel); +static void +utun_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel); +static errno_t +utun_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring, + void **ring_ctx); +static void +utun_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t ring); +static errno_t +utun_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t ring, uint32_t flags); +static errno_t +utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t ring, uint32_t flags); +#endif // UTUN_NEXUS #define UTUN_DEFAULT_MTU 1500 #define UTUN_HEADER_SIZE(_pcb) (sizeof(u_int32_t) + (((_pcb)->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) ? sizeof(uuid_t) : 0)) static kern_ctl_ref utun_kctlref; static u_int32_t utun_family; +static lck_attr_t *utun_lck_attr; +static lck_grp_attr_t *utun_lck_grp_attr; +static lck_grp_t *utun_lck_grp; +static lck_mtx_t utun_lock; + +TAILQ_HEAD(utun_list, utun_pcb) utun_head; + +#define UTUN_PCB_ZONE_MAX 32 +#define UTUN_PCB_ZONE_NAME "net.if_utun" + +static unsigned int utun_pcb_size; /* size of zone element */ +static struct zone *utun_pcb_zone; /* zone for utun_pcb */ + +#if UTUN_NEXUS + +static int +sysctl_if_utun_ring_size SYSCTL_HANDLER_ARGS +{ +#pragma unused(arg1, arg2) + int value = if_utun_ring_size; + + int error = sysctl_handle_int(oidp, &value, 0, req); + if (error || !req->newptr) { + return (error); + } + + if (value < UTUN_IF_MIN_RING_SIZE || + value > UTUN_IF_MAX_RING_SIZE) { + return (EINVAL); + } + + if_utun_ring_size = value; + + return (0); +} + +static int +sysctl_if_utun_tx_fsw_ring_size SYSCTL_HANDLER_ARGS +{ +#pragma unused(arg1, arg2) + int value = if_utun_tx_fsw_ring_size; + + int error = sysctl_handle_int(oidp, &value, 0, req); + if (error || !req->newptr) { + return (error); + } + + if (value < UTUN_IF_MIN_RING_SIZE || + value > UTUN_IF_MAX_RING_SIZE) { + return (EINVAL); + } + + if_utun_tx_fsw_ring_size = value; + + return (0); +} + +static int +sysctl_if_utun_rx_fsw_ring_size SYSCTL_HANDLER_ARGS +{ +#pragma unused(arg1, arg2) + int value = if_utun_rx_fsw_ring_size; + + int error = sysctl_handle_int(oidp, &value, 0, req); + if (error || !req->newptr) { + return (error); + } + + if (value < UTUN_IF_MIN_RING_SIZE || + value > UTUN_IF_MAX_RING_SIZE) { + return (EINVAL); + } + + if_utun_rx_fsw_ring_size = value; + + return (0); +} + +static errno_t +utun_netif_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring, + void **ring_ctx) +{ +#pragma unused(nxprov) +#pragma unused(channel) +#pragma unused(ring_ctx) + struct utun_pcb *pcb = kern_nexus_get_context(nexus); + if (!is_tx_ring) { + VERIFY(pcb->utun_netif_rxring == NULL); + pcb->utun_netif_rxring = ring; + } else { + VERIFY(pcb->utun_netif_txring == NULL); + pcb->utun_netif_txring = ring; + } + return 0; +} + +static void +utun_netif_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t ring) +{ +#pragma unused(nxprov) + struct utun_pcb *pcb = kern_nexus_get_context(nexus); + if (pcb->utun_netif_rxring == ring) { + pcb->utun_netif_rxring = NULL; + } else if (pcb->utun_netif_txring == ring) { + pcb->utun_netif_txring = NULL; + } +} + +static errno_t +utun_netif_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t tx_ring, uint32_t flags) +{ +#pragma unused(nxprov) +#pragma unused(flags) + struct utun_pcb *pcb = kern_nexus_get_context(nexus); + + struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats; + + lck_rw_lock_shared(&pcb->utun_pcb_lock); + + struct kern_channel_ring_stat_increment tx_ring_stats; + bzero(&tx_ring_stats, sizeof(tx_ring_stats)); + kern_channel_slot_t tx_pslot = NULL; + kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); + + STATS_INC(nifs, NETIF_STATS_TXSYNC); + + if (tx_slot == NULL) { + // Nothing to write, don't bother signalling + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + return 0; + } + + if (pcb->utun_kpipe_enabled) { + kern_channel_ring_t rx_ring = pcb->utun_kpipe_rxring; + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + + // Signal the kernel pipe ring to read + if (rx_ring != NULL) { + kern_channel_notify(rx_ring, 0); + } + return 0; + } + + // If we're here, we're injecting into the utun kernel control socket + while (tx_slot != NULL) { + size_t length = 0; + mbuf_t data = NULL; + + kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot); + + if (tx_ph == 0) { + // Advance TX ring + tx_pslot = tx_slot; + tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL); + continue; + } + (void) kern_channel_slot_detach_packet(tx_ring, tx_slot, tx_ph); + + // Advance TX ring + tx_pslot = tx_slot; + tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL); + + kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL); + VERIFY(tx_buf != NULL); + + /* tx_baddr is the absolute buffer address */ + uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf); + VERIFY(tx_baddr != 0); + + bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0); + + uint16_t tx_offset = kern_buflet_get_data_offset(tx_buf); + uint32_t tx_length = kern_buflet_get_data_length(tx_buf); + + // The offset must be large enough for the headers + VERIFY(tx_offset >= UTUN_HEADER_SIZE(pcb)); + + // Find family + uint32_t af = 0; + uint8_t vhl = *(uint8_t *)(tx_baddr + tx_offset); + u_int ip_version = (vhl >> 4); + switch (ip_version) { + case 4: { + af = AF_INET; + break; + } + case 6: { + af = AF_INET6; + break; + } + default: { + printf("utun_netif_sync_tx %s: unknown ip version %u vhl %u tx_offset %u len %u header_size %zu\n", + pcb->utun_ifp->if_xname, ip_version, vhl, tx_offset, tx_length, + UTUN_HEADER_SIZE(pcb)); + break; + } + } + + tx_offset -= UTUN_HEADER_SIZE(pcb); + tx_length += UTUN_HEADER_SIZE(pcb); + tx_baddr += tx_offset; + + length = MIN(tx_length, UTUN_IF_DEFAULT_SLOT_SIZE); + + // Copy in family + memcpy(tx_baddr, &af, sizeof(af)); + if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) { + kern_packet_get_euuid(tx_ph, (void *)(tx_baddr + sizeof(af))); + } + + if (length > 0) { + errno_t error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data); + if (error == 0) { + error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_DONTWAIT); + if (error == 0) { + error = utun_output(pcb->utun_ifp, data); + if (error != 0) { + printf("utun_netif_sync_tx %s - utun_output error %d\n", pcb->utun_ifp->if_xname, error); + } + } else { + printf("utun_netif_sync_tx %s - mbuf_copyback(%zu) error %d\n", pcb->utun_ifp->if_xname, length, error); + STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF); + STATS_INC(nifs, NETIF_STATS_DROPPED); + mbuf_freem(data); + data = NULL; + } + } else { + printf("utun_netif_sync_tx %s - mbuf_gethdr error %d\n", pcb->utun_ifp->if_xname, error); + STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF); + STATS_INC(nifs, NETIF_STATS_DROPPED); + } + } else { + printf("utun_netif_sync_tx %s - 0 length packet\n", pcb->utun_ifp->if_xname); + STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF); + STATS_INC(nifs, NETIF_STATS_DROPPED); + } + + kern_pbufpool_free(tx_ring->ckr_pp, tx_ph); + + if (data == NULL) { + continue; + } + + STATS_INC(nifs, NETIF_STATS_TXPKTS); + STATS_INC(nifs, NETIF_STATS_TXCOPY_MBUF); + + tx_ring_stats.kcrsi_slots_transferred++; + tx_ring_stats.kcrsi_bytes_transferred += length; + } + + if (tx_pslot) { + kern_channel_advance_slot(tx_ring, tx_pslot); + kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats); + (void)kern_channel_reclaim(tx_ring); + } + + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + + return 0; +} + +static errno_t +utun_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t ring, __unused uint32_t flags) +{ +#pragma unused(nxprov) + struct utun_pcb *pcb = kern_nexus_get_context(nexus); + + lck_rw_lock_shared(&pcb->utun_pcb_lock); + + boolean_t more = false; + errno_t rc = 0; + do { + /* Refill and sync the ring */ + rc = kern_channel_tx_refill(ring, UINT32_MAX, UINT32_MAX, true, &more); + if (rc != 0 && rc != EAGAIN && rc != EBUSY) { + printf("%s, tx refill failed %d\n", __func__, rc); + } + } while ((rc == 0) && more); + + if (pcb->utun_kpipe_enabled && !pcb->utun_output_disabled) { + uint32_t tx_available = kern_channel_available_slot_count(ring); + if (pcb->utun_netif_txring_size > 0 && + tx_available >= pcb->utun_netif_txring_size - 1) { + // No room left in tx ring, disable output for now + errno_t error = ifnet_disable_output(pcb->utun_ifp); + if (error != 0) { + printf("utun_netif_tx_doorbell: ifnet_disable_output returned error %d\n", error); + } else { + pcb->utun_output_disabled = true; + } + } + } + if (pcb->utun_kpipe_enabled && + (((rc != 0) && (rc != EAGAIN)) || pcb->utun_output_disabled)) { + kern_channel_ring_t rx_ring = pcb->utun_kpipe_rxring; + + // Unlock while calling notify + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + // Signal the kernel pipe ring to read + if (rx_ring != NULL) { + kern_channel_notify(rx_ring, 0); + } + } else { + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + } + + return (0); +} + +static errno_t +utun_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t rx_ring, uint32_t flags) +{ +#pragma unused(nxprov) +#pragma unused(flags) + struct utun_pcb *pcb = kern_nexus_get_context(nexus); + struct kern_channel_ring_stat_increment rx_ring_stats; + + struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats; + + lck_rw_lock_shared(&pcb->utun_pcb_lock); + + // Reclaim user-released slots + (void) kern_channel_reclaim(rx_ring); + + STATS_INC(nifs, NETIF_STATS_RXSYNC); + + uint32_t avail = kern_channel_available_slot_count(rx_ring); + if (avail == 0) { + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + return 0; + } + + struct kern_pbufpool *rx_pp = rx_ring->ckr_pp; + VERIFY(rx_pp != NULL); + bzero(&rx_ring_stats, sizeof(rx_ring_stats)); + kern_channel_slot_t rx_pslot = NULL; + kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL); + + while (rx_slot != NULL) { + // Check for a waiting packet + lck_mtx_lock(&pcb->utun_input_chain_lock); + mbuf_t data = pcb->utun_input_chain; + if (data == NULL) { + lck_mtx_unlock(&pcb->utun_input_chain_lock); + break; + } + + // Allocate rx packet + kern_packet_t rx_ph = 0; + errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph); + if (unlikely(error != 0)) { + STATS_INC(nifs, NETIF_STATS_NOMEM_PKT); + STATS_INC(nifs, NETIF_STATS_DROPPED); + printf("utun_netif_sync_rx %s: failed to allocate packet\n", + pcb->utun_ifp->if_xname); + lck_mtx_unlock(&pcb->utun_input_chain_lock); + break; + } + + // Advance waiting packets + pcb->utun_input_chain = data->m_nextpkt; + data->m_nextpkt = NULL; + if (pcb->utun_input_chain == NULL) { + pcb->utun_input_chain_last = NULL; + } + lck_mtx_unlock(&pcb->utun_input_chain_lock); + + size_t header_offset = UTUN_HEADER_SIZE(pcb); + size_t length = mbuf_pkthdr_len(data); + + if (length < header_offset) { + // mbuf is too small + mbuf_freem(data); + kern_pbufpool_free(rx_pp, rx_ph); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + printf("utun_netif_sync_rx %s: legacy packet length too short for header %zu < %zu\n", + pcb->utun_ifp->if_xname, length, header_offset); + continue; + } + + length -= header_offset; + if (length > rx_pp->pp_buflet_size) { + // Flush data + mbuf_freem(data); + kern_pbufpool_free(rx_pp, rx_ph); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + printf("utun_netif_sync_rx %s: legacy packet length %zu > %u\n", + pcb->utun_ifp->if_xname, length, rx_pp->pp_buflet_size); + continue; + } + + mbuf_pkthdr_setrcvif(data, pcb->utun_ifp); + + // Fillout rx packet + kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL); + VERIFY(rx_buf != NULL); + void *rx_baddr = kern_buflet_get_object_address(rx_buf); + VERIFY(rx_baddr != NULL); + + // Copy-in data from mbuf to buflet + mbuf_copydata(data, header_offset, length, (void *)rx_baddr); + kern_packet_clear_flow_uuid(rx_ph); // Zero flow id + + // Finalize and attach the packet + error = kern_buflet_set_data_offset(rx_buf, 0); + VERIFY(error == 0); + error = kern_buflet_set_data_length(rx_buf, length); + VERIFY(error == 0); + error = kern_packet_set_link_header_offset(rx_ph, 0); + VERIFY(error == 0); + error = kern_packet_set_network_header_offset(rx_ph, 0); + VERIFY(error == 0); + error = kern_packet_finalize(rx_ph); + VERIFY(error == 0); + error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph); + VERIFY(error == 0); + + STATS_INC(nifs, NETIF_STATS_RXPKTS); + STATS_INC(nifs, NETIF_STATS_RXCOPY_MBUF); + bpf_tap_packet_in(pcb->utun_ifp, DLT_RAW, rx_ph, NULL, 0); + + rx_ring_stats.kcrsi_slots_transferred++; + rx_ring_stats.kcrsi_bytes_transferred += length; + + mbuf_freem(data); + + // Advance ring + rx_pslot = rx_slot; + rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL); + } + + struct kern_channel_ring_stat_increment tx_ring_stats; + bzero(&tx_ring_stats, sizeof(tx_ring_stats)); + kern_channel_ring_t tx_ring = pcb->utun_kpipe_txring; + kern_channel_slot_t tx_pslot = NULL; + kern_channel_slot_t tx_slot = NULL; + if (tx_ring == NULL) { + // Net-If TX ring not set up yet, nothing to read + goto done; + } + + // Unlock utun before entering ring + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + + (void)kr_enter(tx_ring, TRUE); + + // Lock again after entering and validate + lck_rw_lock_shared(&pcb->utun_pcb_lock); + if (tx_ring != pcb->utun_kpipe_txring) { + goto done; + } + + tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); + if (tx_slot == NULL) { + // Nothing to read, don't bother signalling + goto done; + } + + while (rx_slot != NULL && tx_slot != NULL) { + // Allocate rx packet + kern_packet_t rx_ph = 0; + kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot); + + // Advance TX ring + tx_pslot = tx_slot; + tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL); + + /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */ + if (tx_ph == 0) { + continue; + } + + errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph); + if (unlikely(error != 0)) { + STATS_INC(nifs, NETIF_STATS_NOMEM_PKT); + STATS_INC(nifs, NETIF_STATS_DROPPED); + printf("utun_netif_sync_rx %s: failed to allocate packet\n", + pcb->utun_ifp->if_xname); + break; + } + + kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL); + VERIFY(tx_buf != NULL); + uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf); + VERIFY(tx_baddr != 0); + tx_baddr += kern_buflet_get_data_offset(tx_buf); + + // Check packet length + size_t header_offset = UTUN_HEADER_SIZE(pcb); + uint32_t tx_length = kern_packet_get_data_length(tx_ph); + if (tx_length < header_offset) { + // Packet is too small + kern_pbufpool_free(rx_pp, rx_ph); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + printf("utun_netif_sync_rx %s: packet length too short for header %u < %zu\n", + pcb->utun_ifp->if_xname, tx_length, header_offset); + continue; + } + + size_t length = MIN(tx_length - header_offset, + UTUN_IF_DEFAULT_SLOT_SIZE); + + tx_ring_stats.kcrsi_slots_transferred++; + tx_ring_stats.kcrsi_bytes_transferred += length; + + // Fillout rx packet + kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL); + VERIFY(rx_buf != NULL); + void *rx_baddr = kern_buflet_get_object_address(rx_buf); + VERIFY(rx_baddr != NULL); + + // Copy-in data from tx to rx + memcpy((void *)rx_baddr, (void *)(tx_baddr + header_offset), length); + kern_packet_clear_flow_uuid(rx_ph); // Zero flow id + + // Finalize and attach the packet + error = kern_buflet_set_data_offset(rx_buf, 0); + VERIFY(error == 0); + error = kern_buflet_set_data_length(rx_buf, length); + VERIFY(error == 0); + error = kern_packet_set_link_header_offset(rx_ph, 0); + VERIFY(error == 0); + error = kern_packet_set_network_header_offset(rx_ph, 0); + VERIFY(error == 0); + error = kern_packet_finalize(rx_ph); + VERIFY(error == 0); + error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph); + VERIFY(error == 0); + + STATS_INC(nifs, NETIF_STATS_RXPKTS); + STATS_INC(nifs, NETIF_STATS_RXCOPY_DIRECT); + bpf_tap_packet_in(pcb->utun_ifp, DLT_RAW, rx_ph, NULL, 0); + + rx_ring_stats.kcrsi_slots_transferred++; + rx_ring_stats.kcrsi_bytes_transferred += length; + + rx_pslot = rx_slot; + rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL); + } + +done: + if (rx_pslot) { + kern_channel_advance_slot(rx_ring, rx_pslot); + kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats); + } + + if (tx_pslot) { + kern_channel_advance_slot(tx_ring, tx_pslot); + kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats); + (void)kern_channel_reclaim(tx_ring); + } + + // Unlock first, then exit ring + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + if (tx_ring != NULL) { + if (tx_pslot != NULL) { + kern_channel_notify(tx_ring, 0); + } + kr_exit(tx_ring); + } + + return 0; +} + +static errno_t +utun_nexus_ifattach(struct utun_pcb *pcb, + struct ifnet_init_eparams *init_params, + struct ifnet **ifp) +{ + errno_t err; + nexus_controller_t controller = kern_nexus_shared_controller(); + struct kern_nexus_net_init net_init; + + nexus_name_t provider_name; + snprintf((char *)provider_name, sizeof(provider_name), + "com.apple.netif.utun%d", pcb->utun_unit); + + struct kern_nexus_provider_init prov_init = { + .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION, + .nxpi_flags = NXPIF_VIRTUAL_DEVICE, + .nxpi_pre_connect = utun_nexus_pre_connect, + .nxpi_connected = utun_nexus_connected, + .nxpi_pre_disconnect = utun_netif_pre_disconnect, + .nxpi_disconnected = utun_nexus_disconnected, + .nxpi_ring_init = utun_netif_ring_init, + .nxpi_ring_fini = utun_netif_ring_fini, + .nxpi_slot_init = NULL, + .nxpi_slot_fini = NULL, + .nxpi_sync_tx = utun_netif_sync_tx, + .nxpi_sync_rx = utun_netif_sync_rx, + .nxpi_tx_doorbell = utun_netif_tx_doorbell, + }; + + nexus_attr_t nxa = NULL; + err = kern_nexus_attr_create(&nxa); + if (err != 0) { + printf("%s: kern_nexus_attr_create failed: %d\n", + __func__, err); + goto failed; + } + + uint64_t slot_buffer_size = UTUN_IF_DEFAULT_SLOT_SIZE; + err = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size); + VERIFY(err == 0); + + // Reset ring size for netif nexus to limit memory usage + uint64_t ring_size = if_utun_ring_size; + err = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size); + VERIFY(err == 0); + err = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size); + VERIFY(err == 0); + + pcb->utun_netif_txring_size = ring_size; + + err = kern_nexus_controller_register_provider(controller, + utun_nx_dom_prov, + provider_name, + &prov_init, + sizeof(prov_init), + nxa, + &pcb->utun_nx.if_provider); + if (err != 0) { + printf("%s register provider failed, error %d\n", + __func__, err); + goto failed; + } + + bzero(&net_init, sizeof(net_init)); + net_init.nxneti_version = KERN_NEXUS_NET_CURRENT_VERSION; + net_init.nxneti_flags = 0; + net_init.nxneti_eparams = init_params; + net_init.nxneti_lladdr = NULL; + net_init.nxneti_prepare = utun_netif_prepare; + err = kern_nexus_controller_alloc_net_provider_instance(controller, + pcb->utun_nx.if_provider, + pcb, + &pcb->utun_nx.if_instance, + &net_init, + ifp); + if (err != 0) { + printf("%s alloc_net_provider_instance failed, %d\n", + __func__, err); + kern_nexus_controller_deregister_provider(controller, + pcb->utun_nx.if_provider); + uuid_clear(pcb->utun_nx.if_provider); + goto failed; + } + +failed: + if (nxa) { + kern_nexus_attr_destroy(nxa); + } + return (err); +} + +static void +utun_detach_provider_and_instance(uuid_t provider, uuid_t instance) +{ + nexus_controller_t controller = kern_nexus_shared_controller(); + errno_t err; + + if (!uuid_is_null(instance)) { + err = kern_nexus_controller_free_provider_instance(controller, + instance); + if (err != 0) { + printf("%s free_provider_instance failed %d\n", + __func__, err); + } + uuid_clear(instance); + } + if (!uuid_is_null(provider)) { + err = kern_nexus_controller_deregister_provider(controller, + provider); + if (err != 0) { + printf("%s deregister_provider %d\n", __func__, err); + } + uuid_clear(provider); + } + return; +} + +static void +utun_nexus_detach(utun_nx_t nx) +{ + nexus_controller_t controller = kern_nexus_shared_controller(); + errno_t err; + + if (!uuid_is_null(nx->ms_host)) { + err = kern_nexus_ifdetach(controller, + nx->ms_instance, + nx->ms_host); + if (err != 0) { + printf("%s: kern_nexus_ifdetach ms host failed %d\n", + __func__, err); + } + } + + if (!uuid_is_null(nx->ms_device)) { + err = kern_nexus_ifdetach(controller, + nx->ms_instance, + nx->ms_device); + if (err != 0) { + printf("%s: kern_nexus_ifdetach ms device failed %d\n", + __func__, err); + } + } + + utun_detach_provider_and_instance(nx->if_provider, + nx->if_instance); + utun_detach_provider_and_instance(nx->ms_provider, + nx->ms_instance); + + memset(nx, 0, sizeof(*nx)); +} + +static errno_t +utun_create_fs_provider_and_instance(uint32_t subtype, const char *type_name, + const char *ifname, + uuid_t *provider, uuid_t *instance) +{ + nexus_attr_t attr = NULL; + nexus_controller_t controller = kern_nexus_shared_controller(); + uuid_t dom_prov; + errno_t err; + struct kern_nexus_init init; + nexus_name_t provider_name; + + err = kern_nexus_get_builtin_domain_provider(NEXUS_TYPE_FLOW_SWITCH, + &dom_prov); + if (err != 0) { + printf("%s can't get %s provider, error %d\n", + __func__, type_name, err); + goto failed; + } + + err = kern_nexus_attr_create(&attr); + if (err != 0) { + printf("%s: kern_nexus_attr_create failed: %d\n", + __func__, err); + goto failed; + } + + err = kern_nexus_attr_set(attr, NEXUS_ATTR_EXTENSIONS, subtype); + VERIFY(err == 0); + + uint64_t slot_buffer_size = UTUN_IF_DEFAULT_SLOT_SIZE; + err = kern_nexus_attr_set(attr, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size); + VERIFY(err == 0); + + // Reset ring size for flowswitch nexus to limit memory usage. Larger RX than netif. + uint64_t tx_ring_size = if_utun_tx_fsw_ring_size; + err = kern_nexus_attr_set(attr, NEXUS_ATTR_TX_SLOTS, tx_ring_size); + VERIFY(err == 0); + uint64_t rx_ring_size = if_utun_rx_fsw_ring_size; + err = kern_nexus_attr_set(attr, NEXUS_ATTR_RX_SLOTS, rx_ring_size); + VERIFY(err == 0); + + snprintf((char *)provider_name, sizeof(provider_name), + "com.apple.%s.%s", type_name, ifname); + err = kern_nexus_controller_register_provider(controller, + dom_prov, + provider_name, + NULL, + 0, + attr, + provider); + kern_nexus_attr_destroy(attr); + attr = NULL; + if (err != 0) { + printf("%s register %s provider failed, error %d\n", + __func__, type_name, err); + goto failed; + } + bzero(&init, sizeof (init)); + init.nxi_version = KERN_NEXUS_CURRENT_VERSION; + err = kern_nexus_controller_alloc_provider_instance(controller, + *provider, + NULL, + instance, &init); + if (err != 0) { + printf("%s alloc_provider_instance %s failed, %d\n", + __func__, type_name, err); + kern_nexus_controller_deregister_provider(controller, + *provider); + uuid_clear(*provider); + } +failed: + return (err); +} + +static errno_t +utun_multistack_attach(struct utun_pcb *pcb) +{ + nexus_controller_t controller = kern_nexus_shared_controller(); + errno_t err = 0; + utun_nx_t nx = &pcb->utun_nx; + + // Allocate multistack flowswitch + err = utun_create_fs_provider_and_instance(NEXUS_EXTENSION_FSW_TYPE_MULTISTACK, + "multistack", + pcb->utun_ifp->if_xname, + &nx->ms_provider, + &nx->ms_instance); + if (err != 0) { + printf("%s: failed to create bridge provider and instance\n", + __func__); + goto failed; + } + + // Attach multistack to device port + err = kern_nexus_ifattach(controller, nx->ms_instance, + NULL, nx->if_instance, + FALSE, &nx->ms_device); + if (err != 0) { + printf("%s kern_nexus_ifattach ms device %d\n", __func__, err); + goto failed; + } + + // Attach multistack to host port + err = kern_nexus_ifattach(controller, nx->ms_instance, + NULL, nx->if_instance, + TRUE, &nx->ms_host); + if (err != 0) { + printf("%s kern_nexus_ifattach ms host %d\n", __func__, err); + goto failed; + } + + // Extract the agent UUID and save for later + struct kern_nexus *multistack_nx = nx_find(nx->ms_instance, false); + if (multistack_nx != NULL) { + struct nx_flowswitch *flowswitch = NX_FSW_PRIVATE(multistack_nx); + if (flowswitch != NULL) { + FSW_RLOCK(flowswitch); + struct fsw_ms_context *ms_context = (struct fsw_ms_context *)flowswitch->fsw_ops_private; + if (ms_context != NULL) { + uuid_copy(nx->ms_agent, ms_context->mc_agent_uuid); + } else { + printf("utun_multistack_attach - fsw_ms_context is NULL\n"); + } + FSW_UNLOCK(flowswitch); + } else { + printf("utun_multistack_attach - flowswitch is NULL\n"); + } + nx_release(multistack_nx); + } else { + printf("utun_multistack_attach - unable to find multistack nexus\n"); + } + + return (0); + +failed: + utun_nexus_detach(nx); + + errno_t detach_error = 0; + if ((detach_error = ifnet_detach(pcb->utun_ifp)) != 0) { + panic("utun_multistack_attach - ifnet_detach failed: %d\n", detach_error); + /* NOT REACHED */ + } + + return (err); +} + +static errno_t +utun_register_kernel_pipe_nexus(void) +{ + nexus_attr_t nxa = NULL; + errno_t result; + + lck_mtx_lock(&utun_lock); + if (utun_ncd_refcount++) { + lck_mtx_unlock(&utun_lock); + return 0; + } + + result = kern_nexus_controller_create(&utun_ncd); + if (result) { + printf("%s: kern_nexus_controller_create failed: %d\n", + __FUNCTION__, result); + goto done; + } + + uuid_t dom_prov; + result = kern_nexus_get_builtin_domain_provider( + NEXUS_TYPE_KERNEL_PIPE, &dom_prov); + if (result) { + printf("%s: kern_nexus_get_builtin_domain_provider failed: %d\n", + __FUNCTION__, result); + goto done; + } + + struct kern_nexus_provider_init prov_init = { + .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION, + .nxpi_flags = NXPIF_VIRTUAL_DEVICE, + .nxpi_pre_connect = utun_nexus_pre_connect, + .nxpi_connected = utun_nexus_connected, + .nxpi_pre_disconnect = utun_nexus_pre_disconnect, + .nxpi_disconnected = utun_nexus_disconnected, + .nxpi_ring_init = utun_kpipe_ring_init, + .nxpi_ring_fini = utun_kpipe_ring_fini, + .nxpi_slot_init = NULL, + .nxpi_slot_fini = NULL, + .nxpi_sync_tx = utun_kpipe_sync_tx, + .nxpi_sync_rx = utun_kpipe_sync_rx, + .nxpi_tx_doorbell = NULL, + }; + + result = kern_nexus_attr_create(&nxa); + if (result) { + printf("%s: kern_nexus_attr_create failed: %d\n", + __FUNCTION__, result); + goto done; + } + + uint64_t slot_buffer_size = UTUN_IF_DEFAULT_SLOT_SIZE; + result = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size); + VERIFY(result == 0); + + // Reset ring size for kernel pipe nexus to limit memory usage + uint64_t ring_size = if_utun_ring_size; + result = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size); + VERIFY(result == 0); + result = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size); + VERIFY(result == 0); + + result = kern_nexus_controller_register_provider(utun_ncd, + dom_prov, + (const uint8_t *)"com.apple.nexus.utun.kpipe", + &prov_init, + sizeof(prov_init), + nxa, + &utun_kpipe_uuid); + if (result) { + printf("%s: kern_nexus_controller_register_provider failed: %d\n", + __FUNCTION__, result); + goto done; + } + +done: + if (nxa) { + kern_nexus_attr_destroy(nxa); + } + + if (result) { + if (utun_ncd) { + kern_nexus_controller_destroy(utun_ncd); + utun_ncd = NULL; + } + utun_ncd_refcount = 0; + } + + lck_mtx_unlock(&utun_lock); + + return result; +} + +static void +utun_unregister_kernel_pipe_nexus(void) +{ + lck_mtx_lock(&utun_lock); + + VERIFY(utun_ncd_refcount > 0); + + if (--utun_ncd_refcount == 0) { + kern_nexus_controller_destroy(utun_ncd); + utun_ncd = NULL; + } + + lck_mtx_unlock(&utun_lock); +} + +// For use by socket option, not internally +static errno_t +utun_disable_channel(struct utun_pcb *pcb) +{ + errno_t result; + int enabled; + uuid_t uuid; + + lck_rw_lock_exclusive(&pcb->utun_pcb_lock); + + enabled = pcb->utun_kpipe_enabled; + uuid_copy(uuid, pcb->utun_kpipe_uuid); + + VERIFY(uuid_is_null(pcb->utun_kpipe_uuid) == !enabled); + + pcb->utun_kpipe_enabled = 0; + uuid_clear(pcb->utun_kpipe_uuid); + + lck_rw_unlock_exclusive(&pcb->utun_pcb_lock); + + if (enabled) { + result = kern_nexus_controller_free_provider_instance(utun_ncd, uuid); + } else { + result = ENXIO; + } + + if (!result) { + utun_unregister_kernel_pipe_nexus(); + } + + return result; +} + +static errno_t +utun_enable_channel(struct utun_pcb *pcb, struct proc *proc) +{ + struct kern_nexus_init init; + errno_t result; + + result = utun_register_kernel_pipe_nexus(); + if (result) { + return result; + } + + VERIFY(utun_ncd); + + lck_rw_lock_exclusive(&pcb->utun_pcb_lock); + + if (pcb->utun_kpipe_enabled) { + result = EEXIST; // return success instead? + goto done; + } + + /* + * Make sure we can fit packets in the channel buffers and + * Allow an extra 4 bytes for the protocol number header in the channel + */ + if (pcb->utun_ifp->if_mtu + UTUN_HEADER_SIZE(pcb) > UTUN_IF_DEFAULT_SLOT_SIZE) { + result = EOPNOTSUPP; + goto done; + } + + VERIFY(uuid_is_null(pcb->utun_kpipe_uuid)); + bzero(&init, sizeof (init)); + init.nxi_version = KERN_NEXUS_CURRENT_VERSION; + result = kern_nexus_controller_alloc_provider_instance(utun_ncd, + utun_kpipe_uuid, pcb, &pcb->utun_kpipe_uuid, &init); + if (result) { + goto done; + } + + nexus_port_t port = NEXUS_PORT_KERNEL_PIPE_CLIENT; + result = kern_nexus_controller_bind_provider_instance(utun_ncd, + pcb->utun_kpipe_uuid, &port, + proc_pid(proc), NULL, NULL, 0, NEXUS_BIND_PID); + if (result) { + kern_nexus_controller_free_provider_instance(utun_ncd, + pcb->utun_kpipe_uuid); + uuid_clear(pcb->utun_kpipe_uuid); + goto done; + } + + pcb->utun_kpipe_enabled = 1; + +done: + lck_rw_unlock_exclusive(&pcb->utun_pcb_lock); + + if (result) { + utun_unregister_kernel_pipe_nexus(); + } + + return result; +} + +#endif // UTUN_NEXUS errno_t utun_register_control(void) { - struct kern_ctl_reg kern_ctl; - errno_t result = 0; + struct kern_ctl_reg kern_ctl; + errno_t result = 0; /* Find a unique value for our interface family */ result = mbuf_tag_id_find(UTUN_CONTROL_NAME, &utun_family); @@ -112,6 +1309,21 @@ utun_register_control(void) printf("utun_register_control - mbuf_tag_id_find_internal failed: %d\n", result); return result; } + + utun_pcb_size = sizeof(struct utun_pcb); + utun_pcb_zone = zinit(utun_pcb_size, + UTUN_PCB_ZONE_MAX * utun_pcb_size, + 0, UTUN_PCB_ZONE_NAME); + if (utun_pcb_zone == NULL) { + printf("utun_register_control - zinit(utun_pcb) failed"); + return ENOMEM; + } + +#if UTUN_NEXUS + utun_register_nexus(); +#endif // UTUN_NEXUS + + TAILQ_INIT(&utun_head); bzero(&kern_ctl, sizeof(kern_ctl)); strlcpy(kern_ctl.ctl_name, UTUN_CONTROL_NAME, sizeof(kern_ctl.ctl_name)); @@ -151,51 +1363,137 @@ utun_register_control(void) return result; } - + utun_lck_attr = lck_attr_alloc_init(); + utun_lck_grp_attr = lck_grp_attr_alloc_init(); + utun_lck_grp = lck_grp_alloc_init("utun", utun_lck_grp_attr); + +#if UTUN_NEXUS + lck_mtx_init(&utun_lock, utun_lck_grp, utun_lck_attr); +#endif // UTUN_NEXUS + return 0; } /* Kernel control functions */ +static inline void +utun_free_pcb(struct utun_pcb *pcb) +{ +#ifdef UTUN_NEXUS + mbuf_freem_list(pcb->utun_input_chain); + lck_mtx_destroy(&pcb->utun_input_chain_lock, utun_lck_grp); +#endif // UTUN_NEXUS + lck_rw_destroy(&pcb->utun_pcb_lock, utun_lck_grp); + lck_mtx_lock(&utun_lock); + TAILQ_REMOVE(&utun_head, pcb, utun_chain); + lck_mtx_unlock(&utun_lock); + zfree(utun_pcb_zone, pcb); +} + static errno_t -utun_ctl_connect( - kern_ctl_ref kctlref, - struct sockaddr_ctl *sac, - void **unitinfo) +utun_ctl_connect(kern_ctl_ref kctlref, + struct sockaddr_ctl *sac, + void **unitinfo) { - struct ifnet_init_eparams utun_init; - struct utun_pcb *pcb; - errno_t result; - struct ifnet_stats_param stats; + struct ifnet_init_eparams utun_init = {}; + errno_t result = 0; - /* kernel control allocates, interface frees */ - MALLOC(pcb, struct utun_pcb *, sizeof(*pcb), M_DEVBUF, M_WAITOK | M_ZERO); + struct utun_pcb *pcb = zalloc(utun_pcb_zone); + memset(pcb, 0, sizeof(*pcb)); *unitinfo = pcb; pcb->utun_ctlref = kctlref; pcb->utun_unit = sac->sc_unit; pcb->utun_max_pending_packets = 1; - - printf("utun_ctl_connect: creating interface utun%d\n", pcb->utun_unit - 1); + + lck_mtx_init(&pcb->utun_input_chain_lock, utun_lck_grp, utun_lck_attr); + lck_rw_init(&pcb->utun_pcb_lock, utun_lck_grp, utun_lck_attr); + + lck_mtx_lock(&utun_lock); + + /* Find some open interface id */ + u_int32_t chosen_unique_id = 1; + struct utun_pcb *next_pcb = TAILQ_LAST(&utun_head, utun_list); + if (next_pcb != NULL) { + /* List was not empty, add one to the last item */ + chosen_unique_id = next_pcb->utun_unique_id + 1; + next_pcb = NULL; + + /* + * If this wrapped the id number, start looking at + * the front of the list for an unused id. + */ + if (chosen_unique_id == 0) { + /* Find the next unused ID */ + chosen_unique_id = 1; + TAILQ_FOREACH(next_pcb, &utun_head, utun_chain) { + if (next_pcb->utun_unique_id > chosen_unique_id) { + /* We found a gap */ + break; + } + + chosen_unique_id = next_pcb->utun_unique_id + 1; + } + } + } + + pcb->utun_unique_id = chosen_unique_id; + + if (next_pcb != NULL) { + TAILQ_INSERT_BEFORE(next_pcb, pcb, utun_chain); + } else { + TAILQ_INSERT_TAIL(&utun_head, pcb, utun_chain); + } + lck_mtx_unlock(&utun_lock); + + snprintf(pcb->utun_if_xname, sizeof(pcb->utun_if_xname), "utun%d", pcb->utun_unit - 1); + snprintf(pcb->utun_unique_name, sizeof(pcb->utun_unique_name), "utunid%d", pcb->utun_unique_id - 1); + printf("utun_ctl_connect: creating interface %s (id %s)\n", pcb->utun_if_xname, pcb->utun_unique_name); /* Create the interface */ bzero(&utun_init, sizeof(utun_init)); utun_init.ver = IFNET_INIT_CURRENT_VERSION; utun_init.len = sizeof (utun_init); - utun_init.name = "utun"; + +#if UTUN_NEXUS + utun_init.flags = (IFNET_INIT_SKYWALK_NATIVE | IFNET_INIT_NX_NOAUTO); + utun_init.tx_headroom = UTUN_IF_HEADROOM_SIZE; +#else // UTUN_NEXUS + utun_init.flags = IFNET_INIT_NX_NOAUTO; utun_init.start = utun_start; + utun_init.framer_extended = utun_framer; +#endif // UTUN_NEXUS + utun_init.name = "utun"; utun_init.unit = pcb->utun_unit - 1; + utun_init.uniqueid = pcb->utun_unique_name; + utun_init.uniqueid_len = strlen(pcb->utun_unique_name); utun_init.family = utun_family; utun_init.subfamily = IFNET_SUBFAMILY_UTUN; utun_init.type = IFT_OTHER; utun_init.demux = utun_demux; - utun_init.framer_extended = utun_framer; utun_init.add_proto = utun_add_proto; utun_init.del_proto = utun_del_proto; utun_init.softc = pcb; utun_init.ioctl = utun_ioctl; utun_init.detach = utun_detached; +#if UTUN_NEXUS + result = utun_nexus_ifattach(pcb, &utun_init, &pcb->utun_ifp); + if (result != 0) { + printf("utun_ctl_connect - utun_nexus_ifattach failed: %d\n", result); + utun_free_pcb(pcb); + *unitinfo = NULL; + return result; + } + + result = utun_multistack_attach(pcb); + if (result != 0) { + printf("utun_ctl_connect - utun_multistack_attach failed: %d\n", result); + *unitinfo = NULL; + return result; + } + +#else // UTUN_NEXUS /* * Upon success, this holds an ifnet reference which we will * release via ifnet_release() at final detach time. @@ -203,8 +1501,8 @@ utun_ctl_connect( result = ifnet_allocate_extended(&utun_init, &pcb->utun_ifp); if (result != 0) { printf("utun_ctl_connect - ifnet_allocate failed: %d\n", result); + utun_free_pcb(pcb); *unitinfo = NULL; - FREE(pcb, M_DEVBUF); return result; } @@ -218,31 +1516,34 @@ utun_ctl_connect( ifnet_set_eflags(pcb->utun_ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL); /* Reset the stats in case as the interface may have been recycled */ + struct ifnet_stats_param stats; bzero(&stats, sizeof(struct ifnet_stats_param)); ifnet_set_stat(pcb->utun_ifp, &stats); /* Attach the interface */ result = ifnet_attach(pcb->utun_ifp, NULL); if (result != 0) { - printf("utun_ctl_connect - ifnet_allocate failed: %d\n", result); + printf("utun_ctl_connect - ifnet_attach failed: %d\n", result); /* Release reference now since attach failed */ ifnet_release(pcb->utun_ifp); + utun_free_pcb(pcb); *unitinfo = NULL; - FREE(pcb, M_DEVBUF); - } else { - /* Attach to bpf */ - bpfattach(pcb->utun_ifp, DLT_NULL, UTUN_HEADER_SIZE(pcb)); - /* The interfaces resoures allocated, mark it as running */ - ifnet_set_flags(pcb->utun_ifp, IFF_RUNNING, IFF_RUNNING); + return (result); } +#endif // UTUN_NEXUS + + /* Attach to bpf */ + bpfattach(pcb->utun_ifp, DLT_RAW, 0); + /* The interfaces resoures allocated, mark it as running */ + ifnet_set_flags(pcb->utun_ifp, IFF_RUNNING, IFF_RUNNING); + return result; } static errno_t -utun_detach_ip( - ifnet_t interface, - protocol_family_t protocol, - socket_t pf_socket) +utun_detach_ip(ifnet_t interface, + protocol_family_t protocol, + socket_t pf_socket) { errno_t result = EPROTONOSUPPORT; @@ -255,8 +1556,7 @@ utun_detach_ip( ifnet_name(interface), ifnet_unit(interface)); result = sock_ioctl(pf_socket, SIOCPROTODETACH, &ifr); - } - else if (protocol == PF_INET6) { + } else if (protocol == PF_INET6) { struct in6_ifreq ifr6; bzero(&ifr6, sizeof(ifr6)); @@ -270,17 +1570,16 @@ utun_detach_ip( } static void -utun_remove_address( - ifnet_t interface, - protocol_family_t protocol, - ifaddr_t address, - socket_t pf_socket) +utun_remove_address(ifnet_t interface, + protocol_family_t protocol, + ifaddr_t address, + socket_t pf_socket) { errno_t result = 0; /* Attempt a detach */ if (protocol == PF_INET) { - struct ifreq ifr; + struct ifreq ifr; bzero(&ifr, sizeof(ifr)); snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d", @@ -288,16 +1587,14 @@ utun_remove_address( result = ifaddr_address(address, &ifr.ifr_addr, sizeof(ifr.ifr_addr)); if (result != 0) { printf("utun_remove_address - ifaddr_address failed: %d", result); - } - else { + } else { result = sock_ioctl(pf_socket, SIOCDIFADDR, &ifr); if (result != 0) { printf("utun_remove_address - SIOCDIFADDR failed: %d", result); } } - } - else if (protocol == PF_INET6) { - struct in6_ifreq ifr6; + } else if (protocol == PF_INET6) { + struct in6_ifreq ifr6; bzero(&ifr6, sizeof(ifr6)); snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d", @@ -307,8 +1604,7 @@ utun_remove_address( if (result != 0) { printf("utun_remove_address - ifaddr_address failed (v6): %d", result); - } - else { + } else { result = sock_ioctl(pf_socket, SIOCDIFADDR_IN6, &ifr6); if (result != 0) { printf("utun_remove_address - SIOCDIFADDR_IN6 failed: %d", @@ -319,14 +1615,13 @@ utun_remove_address( } static void -utun_cleanup_family( - ifnet_t interface, - protocol_family_t protocol) -{ - errno_t result = 0; - socket_t pf_socket = NULL; - ifaddr_t *addresses = NULL; - int i; +utun_cleanup_family(ifnet_t interface, + protocol_family_t protocol) +{ + errno_t result = 0; + socket_t pf_socket = NULL; + ifaddr_t *addresses = NULL; + int i; if (protocol != PF_INET && protocol != PF_INET6) { printf("utun_cleanup_family - invalid protocol family %d\n", protocol); @@ -349,8 +1644,7 @@ utun_cleanup_family( if (result == 0 || result == ENXIO) { /* We are done! We either detached or weren't attached. */ goto cleanup; - } - else if (result != EBUSY) { + } else if (result != EBUSY) { /* Uh, not really sure what happened here... */ printf("utun_cleanup_family - utun_detach_ip failed: %d\n", result); goto cleanup; @@ -383,31 +1677,58 @@ utun_cleanup_family( } cleanup: - if (pf_socket != NULL) + if (pf_socket != NULL) { sock_close(pf_socket); + } - if (addresses != NULL) + if (addresses != NULL) { ifnet_free_address_list(addresses); + } } static errno_t -utun_ctl_disconnect( - __unused kern_ctl_ref kctlref, - __unused u_int32_t unit, - void *unitinfo) +utun_ctl_disconnect(__unused kern_ctl_ref kctlref, + __unused u_int32_t unit, + void *unitinfo) { struct utun_pcb *pcb = unitinfo; - ifnet_t ifp = NULL; - errno_t result = 0; + ifnet_t ifp = NULL; + errno_t result = 0; + + if (pcb == NULL) { + return EINVAL; + } + +#if UTUN_NEXUS + // Tell the nexus to stop all rings + if (pcb->utun_netif_nexus != NULL) { + kern_nexus_stop(pcb->utun_netif_nexus); + } +#endif // UTUN_NEXUS - if (pcb == NULL) - return EINVAL; + lck_rw_lock_exclusive(&pcb->utun_pcb_lock); +#if UTUN_NEXUS + uuid_t kpipe_uuid; + uuid_copy(kpipe_uuid, pcb->utun_kpipe_uuid); + uuid_clear(pcb->utun_kpipe_uuid); + pcb->utun_kpipe_enabled = FALSE; +#endif // UTUN_NEXUS ifp = pcb->utun_ifp; VERIFY(ifp != NULL); pcb->utun_ctlref = NULL; - pcb->utun_unit = 0; + + /* + * Quiesce the interface and flush any pending outbound packets. + */ + if_down(ifp); + + /* Increment refcnt, but detach interface */ + ifnet_incr_iorefcnt(ifp); + if ((result = ifnet_detach(ifp)) != 0) { + panic("utun_ctl_disconnect - ifnet_detach failed: %d\n", result); + } /* * We want to do everything in our power to ensure that the interface @@ -418,25 +1739,29 @@ utun_ctl_disconnect( utun_cleanup_family(ifp, AF_INET); utun_cleanup_family(ifp, AF_INET6); - /* - * Detach now; utun_detach() will be called asynchronously once - * the I/O reference count drops to 0. There we will invoke - * ifnet_release(). - */ - if ((result = ifnet_detach(ifp)) != 0) { - printf("utun_ctl_disconnect - ifnet_detach failed: %d\n", result); + lck_rw_unlock_exclusive(&pcb->utun_pcb_lock); + +#if UTUN_NEXUS + if (!uuid_is_null(kpipe_uuid)) { + if (kern_nexus_controller_free_provider_instance(utun_ncd, kpipe_uuid) == 0) { + utun_unregister_kernel_pipe_nexus(); + } } + utun_nexus_detach(&pcb->utun_nx); +#endif // UTUN_NEXUS + + /* Decrement refcnt to finish detaching and freeing */ + ifnet_decr_iorefcnt(ifp); return 0; } static errno_t -utun_ctl_send( - __unused kern_ctl_ref kctlref, - __unused u_int32_t unit, - void *unitinfo, - mbuf_t m, - __unused int flags) +utun_ctl_send(__unused kern_ctl_ref kctlref, + __unused u_int32_t unit, + void *unitinfo, + mbuf_t m, + __unused int flags) { /* * The userland ABI requires the first four bytes have the protocol family @@ -452,16 +1777,15 @@ utun_ctl_send( } static errno_t -utun_ctl_setopt( - __unused kern_ctl_ref kctlref, - __unused u_int32_t unit, - void *unitinfo, - int opt, - void *data, - size_t len) +utun_ctl_setopt(__unused kern_ctl_ref kctlref, + __unused u_int32_t unit, + void *unitinfo, + int opt, + void *data, + size_t len) { - struct utun_pcb *pcb = unitinfo; - errno_t result = 0; + struct utun_pcb *pcb = unitinfo; + errno_t result = 0; /* check for privileges for privileged options */ switch (opt) { case UTUN_OPT_FLAGS: @@ -478,14 +1802,7 @@ utun_ctl_setopt( if (len != sizeof(u_int32_t)) { result = EMSGSIZE; } else { - u_int32_t old_flags = pcb->utun_flags; pcb->utun_flags = *(u_int32_t *)data; - - if (((old_flags ^ pcb->utun_flags) & UTUN_FLAGS_ENABLE_PROC_UUID)) { - // If UTUN_FLAGS_ENABLE_PROC_UUID flag changed, update bpf - bpfdetach(pcb->utun_ifp); - bpfattach(pcb->utun_ifp, DLT_NULL, UTUN_HEADER_SIZE(pcb)); - } } break; @@ -551,6 +1868,41 @@ utun_ctl_setopt( pcb->utun_max_pending_packets = max_pending_packets; break; } +#if UTUN_NEXUS + case UTUN_OPT_ENABLE_CHANNEL: { + if (len != sizeof(int)) { + result = EMSGSIZE; + break; + } + if (*(int *)data) { + result = utun_enable_channel(pcb, current_proc()); + } else { + result = utun_disable_channel(pcb); + } + break; + } + case UTUN_OPT_ENABLE_FLOWSWITCH: { + if (len != sizeof(int)) { + result = EMSGSIZE; + break; + } + if (!if_enable_netagent) { + result = ENOTSUP; + break; + } + if (uuid_is_null(pcb->utun_nx.ms_agent)) { + result = ENOENT; + break; + } + + if (*(int *)data) { + if_add_netagent(pcb->utun_ifp, pcb->utun_nx.ms_agent); + } else { + if_delete_netagent(pcb->utun_ifp, pcb->utun_nx.ms_agent); + } + break; + } +#endif // UTUN_NEXUS default: { result = ENOPROTOOPT; break; @@ -561,42 +1913,63 @@ utun_ctl_setopt( } static errno_t -utun_ctl_getopt( - __unused kern_ctl_ref kctlref, - __unused u_int32_t unit, - void *unitinfo, - int opt, - void *data, - size_t *len) +utun_ctl_getopt(__unused kern_ctl_ref kctlref, + __unused u_int32_t unit, + void *unitinfo, + int opt, + void *data, + size_t *len) { - struct utun_pcb *pcb = unitinfo; - errno_t result = 0; + struct utun_pcb *pcb = unitinfo; + errno_t result = 0; switch (opt) { case UTUN_OPT_FLAGS: - if (*len != sizeof(u_int32_t)) + if (*len != sizeof(u_int32_t)) { result = EMSGSIZE; - else + } else { *(u_int32_t *)data = pcb->utun_flags; + } break; case UTUN_OPT_EXT_IFDATA_STATS: - if (*len != sizeof(int)) + if (*len != sizeof(int)) { result = EMSGSIZE; - else + } else { *(int *)data = (pcb->utun_ext_ifdata_stats) ? 1 : 0; + } break; case UTUN_OPT_IFNAME: - *len = snprintf(data, *len, "%s%d", ifnet_name(pcb->utun_ifp), ifnet_unit(pcb->utun_ifp)) + 1; + if (*len < MIN(strlen(pcb->utun_if_xname) + 1, sizeof(pcb->utun_if_xname))) { + result = EMSGSIZE; + } else { + *len = snprintf(data, *len, "%s", pcb->utun_if_xname) + 1; + } break; case UTUN_OPT_MAX_PENDING_PACKETS: { - *len = sizeof(u_int32_t); - *((u_int32_t *)data) = pcb->utun_max_pending_packets; + if (*len != sizeof(u_int32_t)) { + result = EMSGSIZE; + } else { + *((u_int32_t *)data) = pcb->utun_max_pending_packets; + } break; } +#if UTUN_NEXUS + case UTUN_OPT_GET_CHANNEL_UUID: + lck_rw_lock_shared(&pcb->utun_pcb_lock); + if (uuid_is_null(pcb->utun_kpipe_uuid)) { + result = ENXIO; + } else if (*len != sizeof(uuid_t)) { + result = EMSGSIZE; + } else { + uuid_copy(data, pcb->utun_kpipe_uuid); + } + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + break; +#endif // UTUN_NEXUS default: result = ENOPROTOOPT; @@ -638,6 +2011,7 @@ utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, int flags) } /* Network Interface functions */ +#if !UTUN_NEXUS static void utun_start(ifnet_t interface) { @@ -646,6 +2020,20 @@ utun_start(ifnet_t interface) VERIFY(pcb != NULL); +#if UTUN_NEXUS + lck_rw_lock_shared(&pcb->utun_pcb_lock); + if (pcb->utun_kpipe_enabled) { + /* It's possible to have channels enabled, but not yet have the channel opened, + * in which case the rxring will not be set + */ + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + if (pcb->utun_kpipe_rxring != NULL) { + kern_channel_notify(pcb->utun_kpipe_rxring, 0); + } + return; + } + lck_rw_unlock_shared(&pcb->utun_pcb_lock); +#endif // UTUN_NEXUS for (;;) { bool can_accept_packets = true; @@ -678,26 +2066,25 @@ utun_start(ifnet_t interface) break; } ifnet_lock_done(pcb->utun_ifp); - if (ifnet_dequeue(interface, &data) != 0) + if (ifnet_dequeue(interface, &data) != 0) { break; - if (utun_output(interface, data) != 0) + } + if (utun_output(interface, data) != 0) { break; + } } } +#endif // !UTUN_NEXUS static errno_t utun_output(ifnet_t interface, mbuf_t data) { struct utun_pcb *pcb = ifnet_softc(interface); - errno_t result; + errno_t result; VERIFY(interface == pcb->utun_ifp); - - if (m_pktlen(data) >= (int32_t)UTUN_HEADER_SIZE(pcb)) { - bpf_tap_out(pcb->utun_ifp, DLT_NULL, data, 0, 0); - } - + if (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT) { /* flush data */ mbuf_freem(data); @@ -720,47 +2107,64 @@ utun_output(ifnet_t interface, if (result != 0) { mbuf_freem(data); printf("utun_output - ctl_enqueuembuf failed: %d\n", result); - +#if !UTUN_NEXUS ifnet_stat_increment_out(interface, 0, 0, 1); - } - else { - if (!pcb->utun_ext_ifdata_stats) + } else { + if (!pcb->utun_ext_ifdata_stats) { ifnet_stat_increment_out(interface, 1, length, 0); + } +#endif // !UTUN_NEXUS } - } - else + } else { mbuf_freem(data); + } return 0; } static errno_t -utun_demux( - __unused ifnet_t interface, - mbuf_t data, - __unused char *frame_header, - protocol_family_t *protocol) +utun_demux(__unused ifnet_t interface, + mbuf_t data, + __unused char *frame_header, + protocol_family_t *protocol) { + struct ip *ip; + u_int ip_version; + while (data != NULL && mbuf_len(data) < 1) { data = mbuf_next(data); } - + if (data == NULL) return ENOENT; - - *protocol = *(u_int32_t *)mbuf_data(data); + + ip = mtod(data, struct ip *); + ip_version = ip->ip_v; + + switch(ip_version) { + case 4: + *protocol = PF_INET; + return 0; + case 6: + *protocol = PF_INET6; + return 0; + default: + *protocol = 0; + break; + } + return 0; } +#if !UTUN_NEXUS static errno_t -utun_framer( - __unused ifnet_t interface, - mbuf_t *packet, - __unused const struct sockaddr *dest, +utun_framer(ifnet_t interface, + mbuf_t *packet, + __unused const struct sockaddr *dest, __unused const char *desk_linkaddr, const char *frame_type, - u_int32_t *prepend_len, + u_int32_t *prepend_len, u_int32_t *postpend_len) { struct utun_pcb *pcb = ifnet_softc(interface); @@ -775,10 +2179,12 @@ utun_framer( // just return, because the buffer was freed in mbuf_prepend return EJUSTRETURN; } - if (prepend_len != NULL) + if (prepend_len != NULL) { *prepend_len = header_length; - if (postpend_len != NULL) + } + if (postpend_len != NULL) { *postpend_len = 0; + } // place protocol number at the beginning of the mbuf *(protocol_family_t *)mbuf_data(*packet) = *(protocol_family_t *)(uintptr_t)(size_t)frame_type; @@ -786,13 +2192,13 @@ utun_framer( return 0; } +#endif // !UTUN_NEXUS static errno_t -utun_add_proto( - __unused ifnet_t interface, - protocol_family_t protocol, - __unused const struct ifnet_demux_desc *demux_array, - __unused u_int32_t demux_count) +utun_add_proto(__unused ifnet_t interface, + protocol_family_t protocol, + __unused const struct ifnet_demux_desc *demux_array, + __unused u_int32_t demux_count) { switch(protocol) { case PF_INET: @@ -807,23 +2213,31 @@ utun_add_proto( } static errno_t -utun_del_proto( - __unused ifnet_t interface, - __unused protocol_family_t protocol) +utun_del_proto(__unused ifnet_t interface, + __unused protocol_family_t protocol) { return 0; } static errno_t -utun_ioctl( - ifnet_t interface, - u_long command, - void *data) +utun_ioctl(ifnet_t interface, + u_long command, + void *data) { errno_t result = 0; switch(command) { case SIOCSIFMTU: +#if UTUN_NEXUS + { + // Make sure we can fit packets in the channel buffers + // Allow for the headroom in the slot + if (((uint64_t)((struct ifreq*)data)->ifr_mtu) + UTUN_IF_HEADROOM_SIZE > UTUN_IF_DEFAULT_SLOT_SIZE) { + ifnet_set_mtu(interface, UTUN_IF_DEFAULT_SLOT_SIZE - UTUN_IF_HEADROOM_SIZE); + break; + } + } +#endif // UTUN_NEXUS ifnet_set_mtu(interface, ((struct ifreq*)data)->ifr_mtu); break; @@ -839,64 +2253,57 @@ utun_ioctl( } static void -utun_detached( - ifnet_t interface) +utun_detached(ifnet_t interface) { struct utun_pcb *pcb = ifnet_softc(interface); - - FREE(pcb, M_DEVBUF); - /* Release reference acquired via ifnet_allocate_extended() */ - (void) ifnet_release(interface); + (void)ifnet_release(interface); + utun_free_pcb(pcb); } /* Protocol Handlers */ static errno_t -utun_proto_input( - ifnet_t interface, - protocol_family_t protocol, - mbuf_t m, - __unused char *frame_header) +utun_proto_input(__unused ifnet_t interface, + protocol_family_t protocol, + mbuf_t m, + __unused char *frame_header) { - - // remove protocol family first - struct utun_pcb *pcb = ifnet_softc(interface); - mbuf_adj(m, UTUN_HEADER_SIZE(pcb)); - if (proto_input(protocol, m) != 0) { m_freem(m); +#if !UTUN_NEXUS + ifnet_stat_increment_in(interface, 0, 0, 1); + } else { + ifnet_stat_increment_in(interface, 1, m->m_pkthdr.len, 0); +#endif // UTUN_NEXUS } return 0; } static errno_t -utun_proto_pre_output( - __unused ifnet_t interface, - protocol_family_t protocol, - __unused mbuf_t *packet, - __unused const struct sockaddr *dest, - __unused void *route, - char *frame_type, - __unused char *link_layer_dest) +utun_proto_pre_output(__unused ifnet_t interface, + protocol_family_t protocol, + __unused mbuf_t *packet, + __unused const struct sockaddr *dest, + __unused void *route, + char *frame_type, + __unused char *link_layer_dest) { *(protocol_family_t *)(void *)frame_type = protocol; return 0; } static errno_t -utun_attach_proto( - ifnet_t interface, - protocol_family_t protocol) +utun_attach_proto(ifnet_t interface, + protocol_family_t protocol) { struct ifnet_attach_proto_param proto; - errno_t result; bzero(&proto, sizeof(proto)); proto.input = utun_proto_input; proto.pre_output = utun_proto_pre_output; - result = ifnet_attach_protocol(interface, protocol, &proto); + errno_t result = ifnet_attach_protocol(interface, protocol, &proto); if (result != 0 && result != EEXIST) { printf("utun_attach_inet - ifnet_attach_protocol %d failed: %d\n", protocol, result); @@ -905,6 +2312,35 @@ utun_attach_proto( return result; } +#if UTUN_NEXUS +static errno_t +utun_pkt_input(struct utun_pcb *pcb, mbuf_t packet) +{ + lck_rw_lock_shared(&pcb->utun_pcb_lock); + + lck_mtx_lock(&pcb->utun_input_chain_lock); + if (pcb->utun_input_chain != NULL) { + pcb->utun_input_chain_last->m_nextpkt = packet; + } else { + pcb->utun_input_chain = packet; + } + while (packet->m_nextpkt) { + VERIFY(packet != packet->m_nextpkt); + packet = packet->m_nextpkt; + } + pcb->utun_input_chain_last = packet; + lck_mtx_unlock(&pcb->utun_input_chain_lock); + + kern_channel_ring_t rx_ring = pcb->utun_netif_rxring; + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + + if (rx_ring != NULL) { + kern_channel_notify(rx_ring, 0); + } + + return (0); +} +#else static errno_t utun_pkt_input (struct utun_pcb *pcb, mbuf_t m) { @@ -915,7 +2351,7 @@ utun_pkt_input (struct utun_pcb *pcb, mbuf_t m) if (m_pktlen(m) >= (int32_t)UTUN_HEADER_SIZE(pcb)) { protocol = *(u_int32_t *)mbuf_data(m); - + bpf_tap_in(pcb->utun_ifp, DLT_NULL, m, 0, 0); } if (pcb->utun_flags & UTUN_FLAGS_NO_INPUT) { @@ -926,7 +2362,7 @@ utun_pkt_input (struct utun_pcb *pcb, mbuf_t m) if (!pcb->utun_ext_ifdata_stats) { struct ifnet_stat_increment_param incs; - + bzero(&incs, sizeof(incs)); incs.packets_in = 1; incs.bytes_in = mbuf_pkthdr_len(m); @@ -936,18 +2372,400 @@ utun_pkt_input (struct utun_pcb *pcb, mbuf_t m) } if (result != 0) { ifnet_stat_increment_in(pcb->utun_ifp, 0, 0, 1); - + printf("%s - ifnet_input failed: %d\n", __FUNCTION__, result); mbuf_freem(m); } return 0; } +#endif // UTUN_NEXUS + + +#if UTUN_NEXUS + +static errno_t +utun_nxdp_init(__unused kern_nexus_domain_provider_t domprov) +{ + return 0; +} + +static void +utun_nxdp_fini(__unused kern_nexus_domain_provider_t domprov) +{ + // Ignore +} + +static errno_t +utun_register_nexus(void) +{ + const struct kern_nexus_domain_provider_init dp_init = { + .nxdpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION, + .nxdpi_flags = 0, + .nxdpi_init = utun_nxdp_init, + .nxdpi_fini = utun_nxdp_fini + }; + errno_t err = 0; + + /* utun_nxdp_init() is called before this function returns */ + err = kern_nexus_register_domain_provider(NEXUS_TYPE_NET_IF, + (const uint8_t *) "com.apple.utun", + &dp_init, sizeof(dp_init), + &utun_nx_dom_prov); + if (err != 0) { + printf("%s: failed to register domain provider\n", __func__); + return (err); + } + return (0); +} + +static errno_t +utun_ifnet_set_attrs(ifnet_t ifp) +{ + /* Set flags and additional information. */ + ifnet_set_mtu(ifp, 1500); + ifnet_set_flags(ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff); + + /* The interface must generate its own IPv6 LinkLocal address, + * if possible following the recommendation of RFC2472 to the 64bit interface ID + */ + ifnet_set_eflags(ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL); + + return (0); +} + +static errno_t +utun_netif_prepare(kern_nexus_t nexus, ifnet_t ifp) +{ + struct utun_pcb *pcb = kern_nexus_get_context(nexus); + pcb->utun_netif_nexus = nexus; + return (utun_ifnet_set_attrs(ifp)); +} + +static errno_t +utun_nexus_pre_connect(kern_nexus_provider_t nxprov, + proc_t p, kern_nexus_t nexus, + nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx) +{ +#pragma unused(nxprov, p) +#pragma unused(nexus, nexus_port, channel, ch_ctx) + return (0); +} + +static errno_t +utun_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel) +{ +#pragma unused(nxprov, channel) + struct utun_pcb *pcb = kern_nexus_get_context(nexus); + boolean_t ok = ifnet_is_attached(pcb->utun_ifp, 1); + return (ok ? 0 : ENXIO); +} + +static void +utun_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel) +{ +#pragma unused(nxprov, nexus, channel) +} + +static void +utun_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel) +{ +#pragma unused(nxprov, nexus, channel) +} + +static void +utun_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel) +{ +#pragma unused(nxprov, channel) + struct utun_pcb *pcb = kern_nexus_get_context(nexus); + if (pcb->utun_netif_nexus == nexus) { + pcb->utun_netif_nexus = NULL; + } + ifnet_decr_iorefcnt(pcb->utun_ifp); +} + +static errno_t +utun_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_t channel, kern_channel_ring_t ring, + boolean_t is_tx_ring, void **ring_ctx) +{ +#pragma unused(nxprov) +#pragma unused(channel) +#pragma unused(ring_ctx) + struct utun_pcb *pcb = kern_nexus_get_context(nexus); + if (!is_tx_ring) { + VERIFY(pcb->utun_kpipe_rxring == NULL); + pcb->utun_kpipe_rxring = ring; + } else { + VERIFY(pcb->utun_kpipe_txring == NULL); + pcb->utun_kpipe_txring = ring; + } + return 0; +} + +static void +utun_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t ring) +{ +#pragma unused(nxprov) + struct utun_pcb *pcb = kern_nexus_get_context(nexus); + if (pcb->utun_kpipe_rxring == ring) { + pcb->utun_kpipe_rxring = NULL; + } else if (pcb->utun_kpipe_txring == ring) { + pcb->utun_kpipe_txring = NULL; + } +} + +static errno_t +utun_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t tx_ring, uint32_t flags) +{ +#pragma unused(nxprov) +#pragma unused(flags) + struct utun_pcb *pcb = kern_nexus_get_context(nexus); + + lck_rw_lock_shared(&pcb->utun_pcb_lock); + int channel_enabled = pcb->utun_kpipe_enabled; + if (!channel_enabled) { + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + return 0; + } + + kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); + if (tx_slot == NULL) { + // Nothing to write, bail + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + return 0; + } + + // Signal the netif ring to read + kern_channel_ring_t rx_ring = pcb->utun_netif_rxring; + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + if (rx_ring != NULL) { + kern_channel_notify(rx_ring, 0); + } + + return 0; +} + +static errno_t +utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, + kern_channel_ring_t rx_ring, uint32_t flags) +{ +#pragma unused(nxprov) +#pragma unused(flags) + struct utun_pcb *pcb = kern_nexus_get_context(nexus); + struct kern_channel_ring_stat_increment rx_ring_stats; + + lck_rw_lock_shared(&pcb->utun_pcb_lock); + + int channel_enabled = pcb->utun_kpipe_enabled; + if (!channel_enabled) { + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + return 0; + } + + /* reclaim user-released slots */ + (void) kern_channel_reclaim(rx_ring); + + uint32_t avail = kern_channel_available_slot_count(rx_ring); + if (avail == 0) { + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + return 0; + } + + kern_channel_ring_t tx_ring = pcb->utun_netif_txring; + if (tx_ring == NULL || + pcb->utun_netif_nexus == NULL) { + // Net-If TX ring not set up yet, nothing to read + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + return 0; + } + + struct netif_stats *nifs = &NX_NETIF_PRIVATE(pcb->utun_netif_nexus)->nif_stats; + + // Unlock utun before entering ring + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + + (void)kr_enter(tx_ring, TRUE); + + // Lock again after entering and validate + lck_rw_lock_shared(&pcb->utun_pcb_lock); + if (tx_ring != pcb->utun_netif_txring) { + // Ring no longer valid + // Unlock first, then exit ring + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + kr_exit(tx_ring); + return 0; + } + + struct kern_channel_ring_stat_increment tx_ring_stats; + bzero(&tx_ring_stats, sizeof(tx_ring_stats)); + kern_channel_slot_t tx_pslot = NULL; + kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); + if (tx_slot == NULL) { + // Nothing to read, don't bother signalling + // Unlock first, then exit ring + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + kr_exit(tx_ring); + return 0; + } + + struct kern_pbufpool *rx_pp = rx_ring->ckr_pp; + VERIFY(rx_pp != NULL); + bzero(&rx_ring_stats, sizeof(rx_ring_stats)); + kern_channel_slot_t rx_pslot = NULL; + kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL); + + while (rx_slot != NULL && tx_slot != NULL) { + size_t length; + kern_buflet_t rx_buf; + void *rx_baddr; + + kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot); + + // Advance TX ring + tx_pslot = tx_slot; + tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL); + + /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */ + if (tx_ph == 0) { + continue; + } + + // Allocate rx packet + kern_packet_t rx_ph = 0; + errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph); + if (unlikely(error != 0)) { + printf("utun_kpipe_sync_rx %s: failed to allocate packet\n", + pcb->utun_ifp->if_xname); + break; + } + + kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL); + VERIFY(tx_buf != NULL); + uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf); + VERIFY(tx_baddr != NULL); + tx_baddr += kern_buflet_get_data_offset(tx_buf); + + bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0); + + length = MIN(kern_packet_get_data_length(tx_ph) + UTUN_HEADER_SIZE(pcb), + UTUN_IF_DEFAULT_SLOT_SIZE); + + tx_ring_stats.kcrsi_slots_transferred++; + tx_ring_stats.kcrsi_bytes_transferred += length; + + if (length < UTUN_HEADER_SIZE(pcb) || + length > UTUN_IF_DEFAULT_SLOT_SIZE || + length > rx_pp->pp_buflet_size || + (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT)) { + /* flush data */ + kern_pbufpool_free(rx_pp, rx_ph); + printf("utun_kpipe_sync_rx %s: invalid length %zu header_size %zu\n", + pcb->utun_ifp->if_xname, length, UTUN_HEADER_SIZE(pcb)); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + continue; + } + + /* fillout packet */ + rx_buf = kern_packet_get_next_buflet(rx_ph, NULL); + VERIFY(rx_buf != NULL); + rx_baddr = kern_buflet_get_object_address(rx_buf); + VERIFY(rx_baddr != NULL); + + // Find family + uint32_t af = 0; + uint8_t vhl = *(uint8_t *)(tx_baddr); + u_int ip_version = (vhl >> 4); + switch (ip_version) { + case 4: { + af = AF_INET; + break; + } + case 6: { + af = AF_INET6; + break; + } + default: { + printf("utun_kpipe_sync_rx %s: unknown ip version %u vhl %u header_size %zu\n", + pcb->utun_ifp->if_xname, ip_version, vhl, UTUN_HEADER_SIZE(pcb)); + break; + } + } + + // Copy header + af = htonl(af); + memcpy((void *)rx_baddr, &af, sizeof(af)); + if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) { + kern_packet_get_euuid(tx_ph, (void *)(rx_baddr + sizeof(af))); + } + + // Copy data from tx to rx + memcpy((void *)(rx_baddr + UTUN_HEADER_SIZE(pcb)), (void *)tx_baddr, length - UTUN_HEADER_SIZE(pcb)); + kern_packet_clear_flow_uuid(rx_ph); // zero flow id + + /* finalize and attach the packet */ + error = kern_buflet_set_data_offset(rx_buf, 0); + VERIFY(error == 0); + error = kern_buflet_set_data_length(rx_buf, length); + VERIFY(error == 0); + error = kern_packet_finalize(rx_ph); + VERIFY(error == 0); + error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph); + VERIFY(error == 0); + + STATS_INC(nifs, NETIF_STATS_TXPKTS); + STATS_INC(nifs, NETIF_STATS_TXCOPY_DIRECT); + + rx_ring_stats.kcrsi_slots_transferred++; + rx_ring_stats.kcrsi_bytes_transferred += length; + + rx_pslot = rx_slot; + rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL); + } + + if (rx_pslot) { + kern_channel_advance_slot(rx_ring, rx_pslot); + kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats); + } + + if (tx_pslot) { + kern_channel_advance_slot(tx_ring, tx_pslot); + kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats); + (void)kern_channel_reclaim(tx_ring); + } + + if (pcb->utun_output_disabled) { + errno_t error = ifnet_enable_output(pcb->utun_ifp); + if (error != 0) { + printf("utun_kpipe_sync_rx: ifnet_enable_output returned error %d\n", error); + } else { + pcb->utun_output_disabled = false; + } + } + + // Unlock first, then exit ring + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + + if (tx_pslot != NULL) { + kern_channel_notify(tx_ring, 0); + } + kr_exit(tx_ring); + + return 0; +} +#endif // UTUN_NEXUS /* - * These are place holders until coreTLS kext stops caling them + * These are place holders until coreTLS kext stops calling them */ errno_t utun_ctl_register_dtls (void *reg); int utun_pkt_dtls_input(struct utun_pcb *pcb, mbuf_t *pkt, protocol_family_t family); diff --git a/bsd/net/if_utun.h b/bsd/net/if_utun.h index b75476582..008924154 100644 --- a/bsd/net/if_utun.h +++ b/bsd/net/if_utun.h @@ -34,20 +34,6 @@ #include <sys/kern_control.h> -/* Control block allocated for each kernel control connection */ -struct utun_pcb { - kern_ctl_ref utun_ctlref; - ifnet_t utun_ifp; - u_int32_t utun_unit; - u_int32_t utun_flags; - int utun_ext_ifdata_stats; - u_int32_t utun_max_pending_packets; - int utun_channel_enabled; - uuid_t utun_channel_uuid; - void * utun_channel_rxring; - u_int32_t utun_channel_max_pktlen; -}; - void* utun_alloc(size_t size); void utun_free(void *ptr); errno_t utun_register_control(void); @@ -73,6 +59,7 @@ errno_t utun_register_control(void); from the control socket at a time */ #define UTUN_OPT_ENABLE_CHANNEL 17 #define UTUN_OPT_GET_CHANNEL_UUID 18 +#define UTUN_OPT_ENABLE_FLOWSWITCH 19 /* * Flags for by UTUN_OPT_FLAGS */ diff --git a/bsd/net/if_var.h b/bsd/net/if_var.h index bd3bdeba7..0541857f0 100644 --- a/bsd/net/if_var.h +++ b/bsd/net/if_var.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -75,6 +75,9 @@ #ifdef PRIVATE #include <net/route.h> #endif +#ifdef BSD_KERN_PRIVATE +#include <sys/eventhandler.h> +#endif #ifdef KERNEL #include <net/kpi_interface.h> @@ -366,10 +369,66 @@ struct if_tcp_ecn_stat { u_int64_t ecn_total_conn; u_int64_t ecn_fallback_droprst; u_int64_t ecn_fallback_droprxmt; + u_int64_t ecn_fallback_synrst; struct if_tcp_ecn_perf_stat ecn_on; struct if_tcp_ecn_perf_stat ecn_off; }; +struct if_lim_perf_stat { + u_int64_t lim_dl_max_bandwidth; /* bits per second */ + u_int64_t lim_ul_max_bandwidth; /* bits per second */ + u_int64_t lim_total_txpkts; /* Total transmit packets, count */ + u_int64_t lim_total_rxpkts; /* Total receive packets, count */ + u_int64_t lim_total_retxpkts; /* Total retransmit packets */ + u_int64_t lim_packet_loss_percent; /* Packet loss rate */ + u_int64_t lim_total_oopkts; /* Total out-of-order packets */ + u_int64_t lim_packet_ooo_percent; /* Out-of-order packet rate */ + u_int64_t lim_rtt_variance; /* RTT variance, milliseconds */ + u_int64_t lim_rtt_average; /* RTT average, milliseconds */ + u_int64_t lim_rtt_min; /* RTT minimum, milliseconds */ + u_int64_t lim_conn_timeouts; /* connection timeouts */ + u_int64_t lim_conn_attempts; /* connection attempts */ + u_int64_t lim_conn_timeout_percent; /* Rate of connection timeouts */ + u_int64_t lim_bk_txpkts; /* Transmit packets with BK service class, that use delay based algorithms */ + u_int64_t lim_dl_detected:1, /* Low internet */ + lim_ul_detected:1; +}; + +#define IF_VAR_H_HAS_IFNET_STATS_PER_FLOW 1 +struct ifnet_stats_per_flow { + u_int64_t bk_txpackets; + u_int64_t txpackets; + u_int64_t rxpackets; + u_int32_t txretransmitbytes; + u_int32_t rxoutoforderbytes; + u_int32_t rxmitpkts; + u_int32_t rcvoopack; + u_int32_t pawsdrop; + u_int32_t sack_recovery_episodes; + u_int32_t reordered_pkts; + u_int32_t dsack_sent; + u_int32_t dsack_recvd; + u_int32_t srtt; + u_int32_t rttupdated; + u_int32_t rttvar; + u_int32_t rttmin; + u_int32_t bw_sndbw_max; + u_int32_t bw_rcvbw_max; + u_int32_t ecn_recv_ece; + u_int32_t ecn_recv_ce; + u_int16_t ecn_flags; + u_int16_t ipv4:1, + local:1, + connreset:1, + conntimeout:1, + rxmit_drop:1, + ecn_fallback_synloss:1, + ecn_fallback_droprst:1, + ecn_fallback_droprxmt:1, + ecn_fallback_ce:1, + ecn_fallback_reorder:1; +}; + /* * Interface link status report -- includes statistics related to * the link layer technology sent by the driver. The driver will monitor @@ -572,7 +631,7 @@ struct chain_len_stats { uint64_t cls_three; uint64_t cls_four; uint64_t cls_five_or_more; -}; +} __attribute__((__aligned__(sizeof (uint64_t)))); #endif /* PRIVATE */ @@ -610,7 +669,6 @@ struct if_data_internal { u_int32_t ifi_mtu; /* maximum transmission unit */ u_int32_t ifi_metric; /* routing metric (external only) */ u_int32_t ifi_baudrate; /* linespeed */ - u_int32_t ifi_preamblelen;/* length of the packet preamble */ /* volatile statistics */ u_int64_t ifi_ipackets; /* packets received on interface */ u_int64_t ifi_ierrors; /* input errors on interface */ @@ -634,25 +692,6 @@ struct if_data_internal { u_int32_t ifi_tso_v4_mtu; /* TCP Segment Offload IPv4 maximum segment size */ u_int32_t ifi_tso_v6_mtu; /* TCP Segment Offload IPv6 maximum segment size */ }; - -#if MEASURE_BW -/* - * Fields per interface to measure perceived bandwidth. - */ -struct if_measured_bw { - u_int64_t bw; /* measured bandwidth in bytes per ms */ - u_int64_t bytes; /* XXX not needed */ - u_int64_t ts; /* XXX not needed */ - u_int64_t cur_seq __attribute((aligned(8))); /* current sequence for marking a packet */ - u_int64_t start_ts; /* time at which a measurement started */ - u_int64_t start_seq; /* sequence at which a measurement should start */ - u_int64_t last_seq; /* last recorded seq */ - u_int64_t last_ts; /* last recorded ts */ - u_int32_t flags __attribute__((aligned(4))); /* flags */ -#define IF_MEASURED_BW_INPROGRESS 0x1 -#define IF_MEASURED_BW_CALCULATION 0x2 -}; -#endif /* MEASURE_BW */ #endif /* BSD_KERNEL_PRIVATE */ #ifdef PRIVATE @@ -662,7 +701,6 @@ struct if_measured_bw { #define if_physical if_data.ifi_physical #define if_addrlen if_data.ifi_addrlen #define if_hdrlen if_data.ifi_hdrlen -#define if_preamblelen if_data.ifi_preamblelen #define if_metric if_data.ifi_metric #define if_baudrate if_data.ifi_baudrate #define if_hwassist if_data.ifi_hwassist @@ -720,7 +758,7 @@ TAILQ_HEAD(ddesc_head_name, dlil_demux_desc); #ifdef PRIVATE /* - * All of the following IF_HWASSIST_* flags are defined in kpi_inteface.h as + * All of the following IF_HWASSIST_* flags are defined in kpi_interface.h as * IFNET_* flags. These are redefined here as constants to avoid failures to * build user level programs that can not include kpi_interface.h. It is * important to keep this in sync with the definitions in kpi_interface.h. @@ -737,6 +775,7 @@ TAILQ_HEAD(ddesc_head_name, dlil_demux_desc); #define IF_HWASSIST_CSUM_UDPIPV6 0x0040 /* will csum UDPv6, IFNET_CSUM_UDP */ #define IF_HWASSIST_CSUM_FRAGMENT_IPV6 0x0080 /* will do IPv6 fragmentation, IFNET_IPV6_FRAGMENT */ #define IF_HWASSIST_CSUM_PARTIAL 0x1000 /* simple Sum16 computation, IFNET_CSUM_PARTIAL */ +#define IF_HWASSIST_CSUM_ZERO_INVERT 0x2000 /* capable of inverting csum of 0 to -0 (0xffff) */ #define IF_HWASSIST_CSUM_MASK 0xffff #define IF_HWASSIST_CSUM_FLAGS(hwassist) ((hwassist) & IF_HWASSIST_CSUM_MASK) @@ -767,6 +806,12 @@ TAILQ_HEAD(ddesc_head_name, dlil_demux_desc); RB_HEAD(ll_reach_tree, if_llreach); /* define struct ll_reach_tree */ + +typedef errno_t (*dlil_input_func)(ifnet_t ifp, mbuf_t m_head, + mbuf_t m_tail, const struct ifnet_stat_increment_param *s, + boolean_t poll, struct thread *tp); +typedef errno_t (*dlil_output_func)(ifnet_t interface, mbuf_t data); + #define if_name(ifp) ifp->if_xname /* * Structure defining a network interface. @@ -817,12 +862,12 @@ struct ifnet { ifnet_family_t if_family; /* value assigned by Apple */ ifnet_subfamily_t if_subfamily; /* value assigned by Apple */ uintptr_t if_family_cookie; - ifnet_output_handler_func if_output_handler; + volatile dlil_input_func if_input_dlil; + volatile dlil_output_func if_output_dlil; + volatile ifnet_start_func if_start; ifnet_output_func if_output; ifnet_pre_enqueue_func if_pre_enqueue; - ifnet_start_func if_start; ifnet_ctl_func if_output_ctl; - ifnet_input_handler_func if_input_handler; ifnet_input_poll_func if_input_poll; ifnet_ctl_func if_input_ctl; ifnet_ioctl_func if_ioctl; @@ -882,6 +927,9 @@ struct ifnet { struct dlil_threading_info *if_inp; + /* allocated once along with dlil_ifnet and is never freed */ + thread_call_t if_dt_tcall; + struct { u_int32_t length; union { @@ -923,9 +971,6 @@ struct ifnet { struct mld_ifinfo *if_mli; /* for MLDv2 */ #endif /* INET6 */ -#if MEASURE_BW - struct if_measured_bw if_bw; -#endif /* MEASURE_BW */ struct tcpstat_local *if_tcp_stat; /* TCP specific stats */ struct udpstat_local *if_udp_stat; /* UDP specific stats */ @@ -972,8 +1017,26 @@ struct ifnet { struct if_interface_state if_interface_state; struct if_tcp_ecn_stat *if_ipv4_stat; struct if_tcp_ecn_stat *if_ipv6_stat; + + struct if_lim_perf_stat if_lim_stat; }; +/* Interface event handling declarations */ +extern struct eventhandler_lists_ctxt ifnet_evhdlr_ctxt; + +typedef enum { + INTF_EVENT_CODE_CREATED, + INTF_EVENT_CODE_REMOVED, + INTF_EVENT_CODE_STATUS_UPDATE, + INTF_EVENT_CODE_IPADDR_ATTACHED, + INTF_EVENT_CODE_IPADDR_DETACHED, + INTF_EVENT_CODE_LLADDR_UPDATE, + INTF_EVENT_CODE_MTU_CHANGED, +} intf_event_code_t; + +typedef void (*ifnet_event_fn)(struct eventhandler_entry_arg, struct ifnet *, struct sockaddr *, intf_event_code_t); +EVENTHANDLER_DECLARE(ifnet_event, ifnet_event_fn); + #define IF_TCP_STATINC(_ifp, _s) do { \ if ((_ifp)->if_tcp_stat != NULL) \ atomic_add_64(&(_ifp)->if_tcp_stat->_s, 1); \ @@ -987,9 +1050,14 @@ struct ifnet { /* * Valid values for if_refflags */ -#define IFRF_ATTACHED 0x1 /* ifnet attach is completely done */ -#define IFRF_DETACHING 0x2 /* detach has been requested */ - +#define IFRF_EMBRYONIC 0x1 /* ifnet is allocated; awaiting attach */ +#define IFRF_ATTACHED 0x2 /* ifnet attach is completely done */ +#define IFRF_DETACHING 0x4 /* detach has been requested */ +#define IFRF_ATTACH_MASK \ + (IFRF_EMBRYONIC|IFRF_ATTACHED|IFRF_DETACHING) + +#define IF_FULLY_ATTACHED(_ifp) \ + (((_ifp)->if_refflags & IFRF_ATTACH_MASK) == IFRF_ATTACHED) /* * Valid values for if_start_flags */ @@ -1112,7 +1180,16 @@ struct ifaddr { (struct ifaddr *, int); void (*ifa_attached)(struct ifaddr *); /* callback fn for attaching */ void (*ifa_detached)(struct ifaddr *); /* callback fn for detaching */ +#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4) +/* For the newer ARMv7k ABI where 64-bit types are 64-bit aligned, but pointers + * are 32-bit: + * Align to 64-bit since we cast to this to struct in6_ifaddr, which is + * 64-bit aligned + */ +} __attribute__ ((aligned(8))); +#else }; +#endif /* @@ -1133,10 +1210,10 @@ struct ifaddr { #define IFD_NOTREADY 0x40 /* embryonic; not yet ready */ #define IFA_LOCK_ASSERT_HELD(_ifa) \ - lck_mtx_assert(&(_ifa)->ifa_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_ifa)->ifa_lock, LCK_MTX_ASSERT_OWNED) #define IFA_LOCK_ASSERT_NOTHELD(_ifa) \ - lck_mtx_assert(&(_ifa)->ifa_lock, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_ifa)->ifa_lock, LCK_MTX_ASSERT_NOTOWNED) #define IFA_LOCK(_ifa) \ lck_mtx_lock(&(_ifa)->ifa_lock) @@ -1194,10 +1271,10 @@ struct ifmultiaddr { #define IFMAF_ANONYMOUS 0x1 /* has anonymous request ref(s) held */ #define IFMA_LOCK_ASSERT_HELD(_ifma) \ - lck_mtx_assert(&(_ifma)->ifma_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_ifma)->ifma_lock, LCK_MTX_ASSERT_OWNED) #define IFMA_LOCK_ASSERT_NOTHELD(_ifma) \ - lck_mtx_assert(&(_ifma)->ifma_lock, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_ifma)->ifma_lock, LCK_MTX_ASSERT_NOTOWNED) #define IFMA_LOCK(_ifma) \ lck_mtx_lock(&(_ifma)->ifma_lock) @@ -1323,8 +1400,6 @@ extern lck_grp_t *ifa_mtx_grp; extern lck_grp_t *ifnet_lock_group; extern lck_attr_t *ifnet_lock_attr; extern ifnet_t lo_ifp; -extern uint32_t if_bw_measure_size; -extern u_int32_t if_bw_smoothing_val; extern int if_addmulti(struct ifnet *, const struct sockaddr *, struct ifmultiaddr **); @@ -1452,6 +1527,171 @@ extern int ifnet_set_log(struct ifnet *, int32_t, uint32_t, int32_t, int32_t); extern int ifnet_get_log(struct ifnet *, int32_t *, uint32_t *, int32_t *, int32_t *); extern int ifnet_notify_address(struct ifnet *, int); +extern void ifnet_notify_data_threshold(struct ifnet *); + +#define IF_AFDATA_RLOCK if_afdata_rlock +#define IF_AFDATA_RUNLOCK if_afdata_unlock +#define IF_AFDATA_WLOCK if_afdata_wlock +#define IF_AFDATA_WUNLOCK if_afdata_unlock +#define IF_AFDATA_WLOCK_ASSERT if_afdata_wlock_assert +#define IF_AFDATA_LOCK_ASSERT if_afdata_lock_assert +#define IF_AFDATA_UNLOCK_ASSERT if_afdata_unlock_assert + +static inline void +if_afdata_rlock (struct ifnet *ifp, int af) +{ + switch (af) { +#if INET + case AF_INET: + lck_rw_lock_shared(&ifp->if_inetdata_lock); + break; +#endif +#if INET6 + case AF_INET6: + lck_rw_lock_shared(&ifp->if_inet6data_lock); + break; +#endif + default: + VERIFY(0); + /* NOTREACHED */ + } + return; +} + +static inline void +if_afdata_runlock (struct ifnet *ifp, int af) +{ + switch (af) { +#if INET + case AF_INET: + lck_rw_done(&ifp->if_inetdata_lock); + break; +#endif +#if INET6 + case AF_INET6: + lck_rw_done(&ifp->if_inet6data_lock); + break; +#endif + default: + VERIFY(0); + /* NOTREACHED */ + } + return; +} + +static inline void +if_afdata_wlock (struct ifnet *ifp, int af) +{ + switch (af) { +#if INET + case AF_INET: + lck_rw_lock_exclusive(&ifp->if_inetdata_lock); + break; +#endif +#if INET6 + case AF_INET6: + lck_rw_lock_exclusive(&ifp->if_inet6data_lock); + break; +#endif + default: + VERIFY(0); + /* NOTREACHED */ + } + return; +} + +static inline void +if_afdata_unlock (struct ifnet *ifp, int af) +{ + switch (af) { +#if INET + case AF_INET: + lck_rw_done(&ifp->if_inetdata_lock); + break; +#endif +#if INET6 + case AF_INET6: + lck_rw_done(&ifp->if_inet6data_lock); + break; +#endif + default: + VERIFY(0); + /* NOTREACHED */ + } + return; +} + +static inline void +if_afdata_wlock_assert (struct ifnet *ifp, int af) +{ +#if !MACH_ASSERT +#pragma unused(ifp) +#endif + switch (af) { +#if INET + case AF_INET: + LCK_RW_ASSERT(&ifp->if_inetdata_lock, LCK_RW_ASSERT_EXCLUSIVE); + break; +#endif +#if INET6 + case AF_INET6: + LCK_RW_ASSERT(&ifp->if_inet6data_lock, LCK_RW_ASSERT_EXCLUSIVE); + break; +#endif + default: + VERIFY(0); + /* NOTREACHED */ + } + return; +} + +static inline void +if_afdata_unlock_assert (struct ifnet *ifp, int af) +{ +#if !MACH_ASSERT +#pragma unused(ifp) +#endif + switch (af) { +#if INET + case AF_INET: + LCK_RW_ASSERT(&ifp->if_inetdata_lock, LCK_RW_ASSERT_NOTHELD); + break; +#endif +#if INET6 + case AF_INET6: + LCK_RW_ASSERT(&ifp->if_inet6data_lock, LCK_RW_ASSERT_NOTHELD); + break; +#endif + default: + VERIFY(0); + /* NOTREACHED */ + } + return; +} + +static inline void +if_afdata_lock_assert (struct ifnet *ifp, int af) +{ +#if !MACH_ASSERT +#pragma unused(ifp) +#endif + switch (af) { +#if INET + case AF_INET: + LCK_RW_ASSERT(&ifp->if_inetdata_lock, LCK_RW_ASSERT_HELD); + break; +#endif +#if INET6 + case AF_INET6: + LCK_RW_ASSERT(&ifp->if_inet6data_lock, LCK_RW_ASSERT_HELD); + break; +#endif + default: + VERIFY(0); + /* NOTREACHED */ + } + return; +} #if INET6 struct in6_addr; @@ -1511,17 +1751,39 @@ __private_extern__ int ifnet_set_netsignature(struct ifnet *, uint8_t, __private_extern__ int ifnet_get_netsignature(struct ifnet *, uint8_t, uint8_t *, uint16_t *, uint8_t *); +#if INET6 +struct ipv6_prefix; +__private_extern__ int ifnet_set_nat64prefix(struct ifnet *, + struct ipv6_prefix *); +__private_extern__ int ifnet_get_nat64prefix(struct ifnet *, + struct ipv6_prefix *); +#endif + /* Required exclusive ifnet_head lock */ __private_extern__ void ifnet_remove_from_ordered_list(struct ifnet *); __private_extern__ void ifnet_increment_generation(struct ifnet *); __private_extern__ u_int32_t ifnet_get_generation(struct ifnet *); -extern int if_set_qosmarking_mode(struct ifnet *, u_int32_t); +/* Adding and deleting netagents will take ifnet lock */ +__private_extern__ int if_add_netagent(struct ifnet *, uuid_t); +__private_extern__ int if_delete_netagent(struct ifnet *, uuid_t); +extern int if_set_qosmarking_mode(struct ifnet *, u_int32_t); +__private_extern__ uint32_t ifnet_mbuf_packetpreamblelen(struct ifnet *); +__private_extern__ void intf_event_enqueue_nwk_wq_entry(struct ifnet *ifp, + struct sockaddr *addrp, uint32_t intf_event_code); +__private_extern__ void ifnet_update_stats_per_flow(struct ifnet_stats_per_flow *, + struct ifnet *); +#if !CONFIG_EMBEDDED __private_extern__ errno_t ifnet_framer_stub(struct ifnet *, struct mbuf **, const struct sockaddr *, const char *, const char *, u_int32_t *, u_int32_t *); +#endif /* !CONFIG_EMBEDDED */ +__private_extern__ void ifnet_enqueue_multi_setup(struct ifnet *, uint16_t, + uint16_t); +__private_extern__ errno_t ifnet_enqueue_mbuf(struct ifnet *, struct mbuf *, + boolean_t, boolean_t *); #endif /* BSD_KERNEL_PRIVATE */ #ifdef XNU_KERNEL_PRIVATE /* for uuid.c */ diff --git a/bsd/net/if_vlan.c b/bsd/net/if_vlan.c index 2dabd32b2..737ce40ad 100644 --- a/bsd/net/if_vlan.c +++ b/bsd/net/if_vlan.c @@ -109,9 +109,6 @@ #define VLANNAME "vlan" -typedef int (bpf_callback_func)(struct ifnet *, struct mbuf *); -typedef int (if_set_bpf_tap_func)(struct ifnet *ifp, int mode, bpf_callback_func * func); - /** ** vlan locks **/ @@ -153,14 +150,14 @@ vlan_lock_init(void) static __inline__ void vlan_assert_lock_held(void) { - lck_mtx_assert(vlan_lck_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(vlan_lck_mtx, LCK_MTX_ASSERT_OWNED); return; } static __inline__ void vlan_assert_lock_not_held(void) { - lck_mtx_assert(vlan_lck_mtx, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(vlan_lck_mtx, LCK_MTX_ASSERT_NOTOWNED); return; } @@ -224,8 +221,6 @@ struct ifvlan { #define IFVF_DETACHING 0x2 /* interface is detaching */ #define IFVF_READY 0x4 /* interface is ready */ u_int32_t ifv_flags; - bpf_packet_func ifv_bpf_input; - bpf_packet_func ifv_bpf_output; int32_t ifv_retain_count; u_int32_t ifv_signature; /* IFV_SIGNATURE */ }; @@ -382,8 +377,6 @@ static int vlan_input(ifnet_t ifp, protocol_family_t protocol, mbuf_t m, char *frame_header); static int vlan_output(struct ifnet *ifp, struct mbuf *m); static int vlan_ioctl(ifnet_t ifp, u_long cmd, void * addr); -static int vlan_set_bpf_tap(ifnet_t ifp, bpf_tap_mode mode, - bpf_packet_func func); static int vlan_attach_protocol(struct ifnet *ifp); static int vlan_detach_protocol(struct ifnet *ifp); static int vlan_setmulti(struct ifnet *ifp); @@ -566,39 +559,6 @@ siocsifaltmtu(struct ifnet * ifp, int mtu) return (ifnet_ioctl(ifp, 0, SIOCSIFALTMTU, &ifr)); } -static __inline__ void -vlan_bpf_output(struct ifnet * ifp, struct mbuf * m, - bpf_packet_func func) -{ - if (func != NULL) { - (*func)(ifp, m); - } - return; -} - -static __inline__ void -vlan_bpf_input(struct ifnet * ifp, struct mbuf * m, - bpf_packet_func func, char * frame_header, - int frame_header_len, int encap_len) -{ - if (func != NULL) { - if (encap_len > 0) { - /* present the right header to bpf */ - bcopy(frame_header, frame_header + encap_len, frame_header_len); - } - m->m_data -= frame_header_len; - m->m_len += frame_header_len; - (*func)(ifp, m); - m->m_data += frame_header_len; - m->m_len -= frame_header_len; - if (encap_len > 0) { - /* restore the header */ - bcopy(frame_header + encap_len, frame_header, frame_header_len); - } - } - return; -} - /** ** vlan_parent synchronization routines **/ @@ -1010,7 +970,7 @@ vlan_clone_create(struct if_clone *ifc, u_int32_t unit, __unused void *params) vlan_init.framer_extended = ether_frameout_extended; vlan_init.softc = ifv; vlan_init.ioctl = vlan_ioctl; - vlan_init.set_bpf_tap = vlan_set_bpf_tap; + vlan_init.set_bpf_tap = NULL; vlan_init.detach = vlan_if_free; vlan_init.broadcast_addr = etherbroadcastaddr; vlan_init.broadcast_len = ETHER_ADDR_LEN; @@ -1075,45 +1035,9 @@ vlan_clone_destroy(struct ifnet *ifp) return 0; } -static int -vlan_set_bpf_tap(ifnet_t ifp, bpf_tap_mode mode, bpf_packet_func func) -{ - ifvlan_ref ifv; - - vlan_lock(); - ifv = ifnet_get_ifvlan_retained(ifp); - if (ifv == NULL) { - vlan_unlock(); - return (ENODEV); - } - switch (mode) { - case BPF_TAP_DISABLE: - ifv->ifv_bpf_input = ifv->ifv_bpf_output = NULL; - break; - - case BPF_TAP_INPUT: - ifv->ifv_bpf_input = func; - break; - - case BPF_TAP_OUTPUT: - ifv->ifv_bpf_output = func; - break; - - case BPF_TAP_INPUT_OUTPUT: - ifv->ifv_bpf_input = ifv->ifv_bpf_output = func; - break; - default: - break; - } - vlan_unlock(); - ifvlan_release(ifv); - return 0; -} - static int vlan_output(struct ifnet * ifp, struct mbuf * m) { - bpf_packet_func bpf_func; struct ether_vlan_header * evl; int encaplen; ifvlan_ref ifv; @@ -1143,7 +1067,6 @@ vlan_output(struct ifnet * ifp, struct mbuf * m) p = vlp->vlp_ifp; (void)ifnet_stat_increment_out(ifp, 1, m->m_pkthdr.len, 0); soft_vlan = (ifnet_offload(p) & IF_HWASSIST_VLAN_TAGGING) == 0; - bpf_func = ifv->ifv_bpf_output; tag = ifv->ifv_tag; encaplen = ifv->ifv_encaplen; vlan_unlock(); @@ -1151,7 +1074,7 @@ vlan_output(struct ifnet * ifp, struct mbuf * m) ifvlan_release(ifv); vlan_parent_release(vlp); - vlan_bpf_output(ifp, m, bpf_func); + bpf_tap_out(ifp, DLT_EN10MB, m, NULL, 0); /* do not run parent's if_output() if the parent is not up */ if ((ifnet_flags(p) & (IFF_UP | IFF_RUNNING)) != (IFF_UP | IFF_RUNNING)) { @@ -1230,7 +1153,6 @@ static int vlan_input(ifnet_t p, __unused protocol_family_t protocol, mbuf_t m, char *frame_header) { - bpf_packet_func bpf_func = NULL; struct ether_vlan_header * evl; struct ifnet * ifp = NULL; int soft_vlan = 0; @@ -1294,7 +1216,6 @@ vlan_input(ifnet_t p, __unused protocol_family_t protocol, m_freem(m); return 0; } - bpf_func = ifv->ifv_bpf_input; vlan_unlock(); } if (soft_vlan) { @@ -1313,8 +1234,7 @@ vlan_input(ifnet_t p, __unused protocol_family_t protocol, m->m_pkthdr.pkt_hdr = frame_header; (void)ifnet_stat_increment_in(ifp, 1, m->m_pkthdr.len + ETHER_HDR_LEN, 0); - vlan_bpf_input(ifp, m, bpf_func, frame_header, ETHER_HDR_LEN, - soft_vlan ? ETHER_VLAN_ENCAP_LEN : 0); + bpf_tap_in(ifp, DLT_EN10MB, m, frame_header, ETHER_HDR_LEN); /* We found a vlan interface, inject on that interface. */ dlil_input_packet_list(ifp, m); } else { @@ -1899,6 +1819,8 @@ vlan_ioctl(ifnet_t ifp, u_long cmd, void * data) break; } p = NULL; + /* ensure nul termination */ + vlr.vlr_parent[IFNAMSIZ - 1] = '\0'; if (vlr.vlr_parent[0] != '\0') { if (vlr.vlr_tag & ~EVL_VLID_MASK) { /* diff --git a/bsd/net/iptap.c b/bsd/net/iptap.c index ead29d8f2..a4c2cabdb 100644 --- a/bsd/net/iptap.c +++ b/bsd/net/iptap.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999-2013 Apple Inc. All rights reserved. + * Copyright (c) 1999-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -187,7 +187,7 @@ iptap_clone_create(struct if_clone *ifc, u_int32_t unit, void *params) int error = 0; struct iptap_softc *iptap = NULL; - struct ifnet_init_params if_init; + struct ifnet_init_eparams if_init; iptap = _MALLOC(sizeof(struct iptap_softc), M_DEVBUF, M_WAITOK | M_ZERO); if (iptap == NULL) { @@ -201,7 +201,10 @@ iptap_clone_create(struct if_clone *ifc, u_int32_t unit, void *params) * We do not use a set_bpf_tap() function as we rather rely on the more * accurate callback passed to bpf_attach() */ - bzero(&if_init, sizeof(struct ifnet_init_params)); + bzero(&if_init, sizeof(if_init)); + if_init.ver = IFNET_INIT_CURRENT_VERSION; + if_init.len = sizeof (if_init); + if_init.flags = IFNET_INIT_LEGACY; if_init.name = ifc->ifc_name; if_init.unit = unit; if_init.type = IFT_OTHER; @@ -214,7 +217,7 @@ iptap_clone_create(struct if_clone *ifc, u_int32_t unit, void *params) if_init.ioctl = iptap_ioctl; if_init.detach = iptap_detach; - error = ifnet_allocate(&if_init, &iptap->iptap_ifp); + error = ifnet_allocate_extended(&if_init, &iptap->iptap_ifp); if (error != 0) { printf("%s: ifnet_allocate failed, error %d\n", __func__, error); goto done; @@ -581,6 +584,23 @@ iptap_bpf_tap(struct mbuf *m, u_int32_t proto, int outgoing) struct iptap_softc *iptap; void (*bpf_tap_func)(ifnet_t , u_int32_t , mbuf_t , void * , size_t ) = outgoing ? bpf_tap_out : bpf_tap_in; + uint16_t src_scope_id = 0; + uint16_t dst_scope_id = 0; + + if (proto == AF_INET6) { + struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); + /* + * Clear the embedded scope ID + */ + if (IN6_IS_SCOPE_EMBED(&ip6->ip6_src)) { + src_scope_id = ip6->ip6_src.s6_addr16[1]; + ip6->ip6_src.s6_addr16[1] = 0; + } + if (IN6_IS_SCOPE_EMBED(&ip6->ip6_dst)) { + dst_scope_id = ip6->ip6_dst.s6_addr16[1]; + ip6->ip6_dst.s6_addr16[1] = 0; + } + } iptap_lock_shared(); @@ -625,4 +645,18 @@ iptap_bpf_tap(struct mbuf *m, u_int32_t proto, int outgoing) } iptap_lock_done(); + + if (proto == AF_INET6) { + struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); + + /* + * Restore the embedded scope ID + */ + if (IN6_IS_SCOPE_EMBED(&ip6->ip6_src)) { + ip6->ip6_src.s6_addr16[1] = src_scope_id; + } + if (IN6_IS_SCOPE_EMBED(&ip6->ip6_dst)) { + ip6->ip6_dst.s6_addr16[1] = dst_scope_id; + } + } } diff --git a/bsd/net/kpi_interface.c b/bsd/net/kpi_interface.c index d28af82ac..05e7cfc57 100644 --- a/bsd/net/kpi_interface.c +++ b/bsd/net/kpi_interface.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2016 Apple Inc. All rights reserved. + * Copyright (c) 2004-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -48,6 +48,7 @@ #include <net/if_arp.h> #include <net/if_llreach.h> #include <net/if_ether.h> +#include <net/net_api_stats.h> #include <net/route.h> #include <libkern/libkern.h> #include <libkern/OSAtomic.h> @@ -70,6 +71,7 @@ #include <netinet6/mld6_var.h> #endif #include <netkey/key.h> +#include <stdbool.h> #include "net/net_str_id.h" @@ -78,6 +80,15 @@ #include <security/mac_framework.h> #endif + +#undef ifnet_allocate +errno_t ifnet_allocate(const struct ifnet_init_params *init, + ifnet_t *ifp); + +static errno_t ifnet_allocate_common(const struct ifnet_init_params *init, + ifnet_t *ifp, bool is_internal); + + #define TOUCHLASTCHANGE(__if_lastchange) { \ (__if_lastchange)->tv_sec = net_uptime(); \ (__if_lastchange)->tv_usec = 0; \ @@ -117,7 +128,8 @@ ifnet_kpi_free(ifnet_t ifp) } errno_t -ifnet_allocate(const struct ifnet_init_params *init, ifnet_t *interface) +ifnet_allocate_common(const struct ifnet_init_params *init, + ifnet_t *ifp, bool is_internal) { struct ifnet_init_eparams einit; @@ -125,7 +137,10 @@ ifnet_allocate(const struct ifnet_init_params *init, ifnet_t *interface) einit.ver = IFNET_INIT_CURRENT_VERSION; einit.len = sizeof (einit); - einit.flags = IFNET_INIT_LEGACY; + einit.flags = IFNET_INIT_LEGACY | IFNET_INIT_NX_NOAUTO; + if (!is_internal) { + einit.flags |= IFNET_INIT_ALLOC_KPI; + } einit.uniqueid = init->uniqueid; einit.uniqueid_len = init->uniqueid_len; einit.name = init->name; @@ -146,7 +161,19 @@ ifnet_allocate(const struct ifnet_init_params *init, ifnet_t *interface) einit.broadcast_addr = init->broadcast_addr; einit.broadcast_len = init->broadcast_len; - return (ifnet_allocate_extended(&einit, interface)); + return (ifnet_allocate_extended(&einit, ifp)); +} + +errno_t +ifnet_allocate_internal(const struct ifnet_init_params *init, ifnet_t *ifp) +{ + return (ifnet_allocate_common(init, ifp, true)); +} + +errno_t +ifnet_allocate(const struct ifnet_init_params *init, ifnet_t *ifp) +{ + return (ifnet_allocate_common(init, ifp, false)); } errno_t @@ -169,8 +196,10 @@ ifnet_allocate_extended(const struct ifnet_init_eparams *einit0, (einit.type & 0xFFFFFF00) != 0 || einit.type == 0) return (EINVAL); + if (einit.flags & IFNET_INIT_LEGACY) { - if (einit.output == NULL || einit.flags != IFNET_INIT_LEGACY) + if (einit.output == NULL || + (einit.flags & IFNET_INIT_INPUT_POLL)) return (EINVAL); einit.pre_enqueue = NULL; @@ -198,8 +227,8 @@ ifnet_allocate_extended(const struct ifnet_init_eparams *einit0, if (einit.uniqueid == NULL) { /* Initialize external name (name + unit) */ - snprintf(if_xname, IFXNAMSIZ, - "%s%d", einit.name, einit.unit); + (void) snprintf(if_xname, sizeof (if_xname), "%s%d", + einit.name, einit.unit); einit.uniqueid = if_xname; einit.uniqueid_len = strlen(if_xname); } @@ -268,12 +297,17 @@ ifnet_allocate_extended(const struct ifnet_init_eparams *einit0, * Internally, DLIL will only use the extended callback * variant which is represented by if_framer. */ +#if CONFIG_EMBEDDED + if (ifp->if_framer == NULL && ifp->if_framer_legacy != NULL) + ifp->if_framer = ifp->if_framer_legacy; +#else /* !CONFIG_EMBEDDED */ if (ifp->if_framer == NULL && ifp->if_framer_legacy != NULL) { if (ifp->if_framer_legacy == ether_frameout) ifp->if_framer = ether_frameout_extended; else ifp->if_framer = ifnet_framer_stub; } +#endif /* !CONFIG_EMBEDDED */ if (ifp->if_output_bw.eff_bw > ifp->if_output_bw.max_bw) ifp->if_output_bw.max_bw = ifp->if_output_bw.eff_bw; @@ -313,6 +347,7 @@ ifnet_allocate_extended(const struct ifnet_init_eparams *einit0, if (ifp->if_ioctl == NULL) ifp->if_ioctl = ifp_if_ioctl; + ifp->if_eflags = 0; if (ifp->if_start != NULL) { ifp->if_eflags |= IFEF_TXSTART; if (ifp->if_pre_enqueue == NULL) @@ -327,8 +362,8 @@ ifnet_allocate_extended(const struct ifnet_init_eparams *einit0, else ifp->if_eflags &= ~IFEF_RXPOLL; - ifp->if_output_handler = dlil_output_handler; - ifp->if_input_handler = dlil_input_handler; + ifp->if_output_dlil = dlil_output_handler; + ifp->if_input_dlil = dlil_input_handler; VERIFY(!(einit.flags & IFNET_INIT_LEGACY) || (ifp->if_pre_enqueue == NULL && ifp->if_start == NULL && @@ -359,6 +394,8 @@ ifnet_allocate_extended(const struct ifnet_init_eparams *einit0, bzero(&ifp->if_broadcast, sizeof (ifp->if_broadcast)); } + ifp->if_xflags = 0; + /* * output target queue delay is specified in millisecond * convert it to nanoseconds @@ -367,15 +404,29 @@ ifnet_allocate_extended(const struct ifnet_init_eparams *einit0, einit.output_target_qdelay * 1000 * 1000; IFCQ_MAXLEN(&ifp->if_snd) = einit.sndq_maxlen; - if (einit.start_delay_qlen > 0 && - einit.start_delay_timeout > 0) { - ifp->if_eflags |= IFEF_ENQUEUE_MULTI; - ifp->if_start_delay_qlen = - min(100, einit.start_delay_qlen); - ifp->if_start_delay_timeout = - min(20000, einit.start_delay_timeout); - /* convert timeout to nanoseconds */ - ifp->if_start_delay_timeout *= 1000; + ifnet_enqueue_multi_setup(ifp, einit.start_delay_qlen, + einit.start_delay_timeout); + + IFCQ_PKT_DROP_LIMIT(&ifp->if_snd) = IFCQ_DEFAULT_PKT_DROP_LIMIT; + + /* + * Set embryonic flag; this will be cleared + * later when it is fully attached. + */ + ifp->if_refflags = IFRF_EMBRYONIC; + + /* + * Count the newly allocated ifnet + */ + OSIncrementAtomic64(&net_api_stats.nas_ifnet_alloc_count); + INC_ATOMIC_INT64_LIM(net_api_stats.nas_ifnet_alloc_total); + if (einit.flags & IFNET_INIT_ALLOC_KPI) { + ifp->if_xflags |= IFXF_ALLOC_KPI; + } else { + OSIncrementAtomic64( + &net_api_stats.nas_ifnet_alloc_os_count); + INC_ATOMIC_INT64_LIM( + net_api_stats.nas_ifnet_alloc_os_total); } if (error == 0) { @@ -387,17 +438,6 @@ ifnet_allocate_extended(const struct ifnet_init_eparams *einit0, *interface = NULL; } } - - /* - * Note: We should do something here to indicate that we haven't been - * attached yet. By doing so, we can catch the case in ifnet_release - * where the reference count reaches zero and call the recycle - * function. If the interface is attached, the interface will be - * recycled when the interface's if_free function is called. If the - * interface is never attached, the if_free function will never be - * called and the interface will never be recycled. - */ - return (error); } @@ -617,7 +657,7 @@ ifnet_set_idle_flags_locked(ifnet_t ifp, u_int32_t new_flags, u_int32_t mask) if (ifp == NULL) return (EINVAL); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); ifnet_lock_assert(ifp, IFNET_LCK_ASSERT_EXCLUSIVE); /* @@ -848,9 +888,10 @@ ifnet_capabilities_enabled(ifnet_t ifp) static const ifnet_offload_t offload_mask = (IFNET_CSUM_IP | IFNET_CSUM_TCP | IFNET_CSUM_UDP | IFNET_CSUM_FRAGMENT | IFNET_IP_FRAGMENT | IFNET_CSUM_TCPIPV6 | IFNET_CSUM_UDPIPV6 | - IFNET_IPV6_FRAGMENT | IFNET_CSUM_PARTIAL | IFNET_VLAN_TAGGING | - IFNET_VLAN_MTU | IFNET_MULTIPAGES | IFNET_TSO_IPV4 | IFNET_TSO_IPV6 | - IFNET_TX_STATUS | IFNET_HW_TIMESTAMP | IFNET_SW_TIMESTAMP); + IFNET_IPV6_FRAGMENT | IFNET_CSUM_PARTIAL | IFNET_CSUM_ZERO_INVERT | + IFNET_VLAN_TAGGING | IFNET_VLAN_MTU | IFNET_MULTIPAGES | + IFNET_TSO_IPV4 | IFNET_TSO_IPV6 | IFNET_TX_STATUS | IFNET_HW_TIMESTAMP | + IFNET_SW_TIMESTAMP); static const ifnet_offload_t any_offload_csum = IFNET_CHECKSUMF; @@ -864,6 +905,7 @@ ifnet_set_offload(ifnet_t interface, ifnet_offload_t offload) ifnet_lock_exclusive(interface); interface->if_hwassist = (offload & offload_mask); + /* * Hardware capable of partial checksum offload is * flexible enough to handle any transports utilizing @@ -897,6 +939,10 @@ ifnet_set_offload(ifnet_t interface, ifnet_offload_t offload) ifcaps |= IFCAP_HW_TIMESTAMP; if ((offload & IFNET_SW_TIMESTAMP)) ifcaps |= IFCAP_SW_TIMESTAMP; + if ((offload & IFNET_CSUM_PARTIAL)) + ifcaps |= IFCAP_CSUM_PARTIAL; + if ((offload & IFNET_CSUM_ZERO_INVERT)) + ifcaps |= IFCAP_CSUM_ZERO_INVERT; if (ifcaps != 0) { (void) ifnet_set_capabilities_supported(interface, ifcaps, IFCAP_VALID); @@ -1525,6 +1571,9 @@ ifnet_stat_increment(struct ifnet *ifp, /* Touch the last change time. */ TOUCHLASTCHANGE(&ifp->if_lastchange); + if (ifp->if_data_threshold != 0) + ifnet_notify_data_threshold(ifp); + return (0); } @@ -1544,6 +1593,9 @@ ifnet_stat_increment_in(struct ifnet *ifp, u_int32_t packets_in, TOUCHLASTCHANGE(&ifp->if_lastchange); + if (ifp->if_data_threshold != 0) + ifnet_notify_data_threshold(ifp); + return (0); } @@ -1563,6 +1615,9 @@ ifnet_stat_increment_out(struct ifnet *ifp, u_int32_t packets_out, TOUCHLASTCHANGE(&ifp->if_lastchange); + if (ifp->if_data_threshold != 0) + ifnet_notify_data_threshold(ifp); + return (0); } @@ -1589,6 +1644,9 @@ ifnet_set_stat(struct ifnet *ifp, const struct ifnet_stats_param *s) /* Touch the last change time. */ TOUCHLASTCHANGE(&ifp->if_lastchange); + if (ifp->if_data_threshold != 0) + ifnet_notify_data_threshold(ifp); + return (0); } @@ -1612,6 +1670,9 @@ ifnet_stat(struct ifnet *ifp, struct ifnet_stats_param *s) atomic_get_64(s->dropped, &ifp->if_data.ifi_iqdrops); atomic_get_64(s->no_protocol, &ifp->if_data.ifi_noproto); + if (ifp->if_data_threshold != 0) + ifnet_notify_data_threshold(ifp); + return (0); } @@ -1964,6 +2025,8 @@ ifnet_set_lladdr_internal(ifnet_t interface, const void *lladdr, /* Generate a kernel event */ if (error == 0) { + intf_event_enqueue_nwk_wq_entry(interface, NULL, + INTF_EVENT_CODE_LLADDR_UPDATE); dlil_post_msg(interface, KEV_DL_SUBCLASS, KEV_DL_LINK_ADDRESS_CHANGED, NULL, 0); } @@ -2199,97 +2262,6 @@ ifnet_list_free(ifnet_t *interfaces) FREE(interfaces, M_TEMP); } -void -ifnet_transmit_burst_start(ifnet_t ifp, mbuf_t pkt) -{ -#if MEASURE_BW - uint32_t orig_flags; - - if (ifp == NULL || !(pkt->m_flags & M_PKTHDR)) - return; - - orig_flags = OSBitOrAtomic(IF_MEASURED_BW_INPROGRESS, - &ifp->if_bw.flags); - if (orig_flags & IF_MEASURED_BW_INPROGRESS) { - /* There is already a measurement in progress; skip this one */ - return; - } - - ifp->if_bw.start_seq = pkt->m_pkthdr.pkt_bwseq; - ifp->if_bw.start_ts = mach_absolute_time(); -#else /* !MEASURE_BW */ -#pragma unused(ifp, pkt) -#endif /* !MEASURE_BW */ -} - -void -ifnet_transmit_burst_end(ifnet_t ifp, mbuf_t pkt) -{ -#if MEASURE_BW - uint64_t oseq, ots, bytes, ts, t; - uint32_t flags; - - if (ifp == NULL || !(pkt->m_flags & M_PKTHDR)) - return; - - flags = OSBitOrAtomic(IF_MEASURED_BW_CALCULATION, &ifp->if_bw.flags); - - /* If a calculation is already in progress, just return */ - if (flags & IF_MEASURED_BW_CALCULATION) - return; - - /* Check if a measurement was started at all */ - if (!(flags & IF_MEASURED_BW_INPROGRESS)) { - /* - * It is an error to call burst_end before burst_start. - * Reset the calculation flag and return. - */ - goto done; - } - - oseq = pkt->m_pkthdr.pkt_bwseq; - ots = mach_absolute_time(); - - if (ifp->if_bw.start_seq > 0 && oseq > ifp->if_bw.start_seq) { - ts = ots - ifp->if_bw.start_ts; - if (ts > 0) { - absolutetime_to_nanoseconds(ts, &t); - bytes = oseq - ifp->if_bw.start_seq; - ifp->if_bw.bytes = bytes; - ifp->if_bw.ts = ts; - - if (t > 0) { - uint64_t bw = 0; - - /* Compute bandwidth as bytes/ms */ - bw = (bytes * NSEC_PER_MSEC) / t; - if (bw > 0) { - if (ifp->if_bw.bw > 0) { - u_int32_t shft; - - shft = if_bw_smoothing_val; - /* Compute EWMA of bw */ - ifp->if_bw.bw = (bw + - ((ifp->if_bw.bw << shft) - - ifp->if_bw.bw)) >> shft; - } else { - ifp->if_bw.bw = bw; - } - } - } - ifp->if_bw.last_seq = oseq; - ifp->if_bw.last_ts = ots; - } - } - -done: - flags = ~(IF_MEASURED_BW_INPROGRESS | IF_MEASURED_BW_CALCULATION); - OSBitAndAtomic(flags, &ifp->if_bw.flags); -#else /* !MEASURE_BW */ -#pragma unused(ifp, pkt) -#endif /* !MEASURE_BW */ -} - /*************************************************************************/ /* ifaddr_t accessors */ /*************************************************************************/ @@ -2627,17 +2599,6 @@ ifnet_get_local_ports_extended(ifnet_t ifp, protocol_family_t protocol, u_int32_t ifindex; u_int32_t inp_flags = 0; - inp_flags |= ((flags & IFNET_GET_LOCAL_PORTS_WILDCARDOK) ? - INPCB_GET_PORTS_USED_WILDCARDOK : 0); - inp_flags |= ((flags & IFNET_GET_LOCAL_PORTS_NOWAKEUPOK) ? - INPCB_GET_PORTS_USED_NOWAKEUPOK : 0); - inp_flags |= ((flags & IFNET_GET_LOCAL_PORTS_RECVANYIFONLY) ? - INPCB_GET_PORTS_USED_RECVANYIFONLY : 0); - inp_flags |= ((flags & IFNET_GET_LOCAL_PORTS_EXTBGIDLEONLY) ? - INPCB_GET_PORTS_USED_EXTBGIDLEONLY : 0); - inp_flags |= ((flags & IFNET_GET_LOCAL_PORTS_ACTIVEONLY) ? - INPCB_GET_PORTS_USED_ACTIVEONLY : 0); - if (bitfield == NULL) return (EINVAL); @@ -2653,14 +2614,27 @@ ifnet_get_local_ports_extended(ifnet_t ifp, protocol_family_t protocol, /* bit string is long enough to hold 16-bit port values */ bzero(bitfield, bitstr_size(65536)); - ifindex = (ifp != NULL) ? ifp->if_index : 0; - - if (!(flags & IFNET_GET_LOCAL_PORTS_TCPONLY)) - udp_get_ports_used(ifindex, protocol, inp_flags, bitfield); - - if (!(flags & IFNET_GET_LOCAL_PORTS_UDPONLY)) - tcp_get_ports_used(ifindex, protocol, inp_flags, bitfield); - + inp_flags |= ((flags & IFNET_GET_LOCAL_PORTS_WILDCARDOK) ? + INPCB_GET_PORTS_USED_WILDCARDOK : 0); + inp_flags |= ((flags & IFNET_GET_LOCAL_PORTS_NOWAKEUPOK) ? + INPCB_GET_PORTS_USED_NOWAKEUPOK : 0); + inp_flags |= ((flags & IFNET_GET_LOCAL_PORTS_RECVANYIFONLY) ? + INPCB_GET_PORTS_USED_RECVANYIFONLY : 0); + inp_flags |= ((flags & IFNET_GET_LOCAL_PORTS_EXTBGIDLEONLY) ? + INPCB_GET_PORTS_USED_EXTBGIDLEONLY : 0); + inp_flags |= ((flags & IFNET_GET_LOCAL_PORTS_ACTIVEONLY) ? + INPCB_GET_PORTS_USED_ACTIVEONLY : 0); + + + ifindex = (ifp != NULL) ? ifp->if_index : 0; + + if (!(flags & IFNET_GET_LOCAL_PORTS_TCPONLY)) + udp_get_ports_used(ifindex, protocol, inp_flags, + bitfield); + + if (!(flags & IFNET_GET_LOCAL_PORTS_UDPONLY)) + tcp_get_ports_used(ifindex, protocol, inp_flags, + bitfield); return (0); } @@ -2669,7 +2643,7 @@ ifnet_get_local_ports(ifnet_t ifp, u_int8_t *bitfield) { u_int32_t flags = IFNET_GET_LOCAL_PORTS_WILDCARDOK; return (ifnet_get_local_ports_extended(ifp, PF_UNSPEC, flags, - bitfield)); + bitfield)); } errno_t @@ -2945,6 +2919,9 @@ ifnet_link_status_report(ifnet_t ifp, const void *buffer, atomic_bitset_32(&tcbinfo.ipi_flags, INPCBINFO_UPDATE_MSS); inpcb_timer_sched(&tcbinfo, INPCB_TIMER_FAST); +#if NECP + necp_update_all_clients(); +#endif } /* Finally copy the new information */ @@ -3021,39 +2998,6 @@ done: return (err); } -/*************************************************************************/ -/* Packet preamble */ -/*************************************************************************/ - -#define MAX_IF_PACKET_PREAMBLE_LEN 32 - -errno_t -ifnet_set_packetpreamblelen(ifnet_t interface, u_int32_t len) -{ - errno_t err = 0; - - if (interface == NULL || len > MAX_IF_PACKET_PREAMBLE_LEN) { - err = EINVAL; - goto done; - } - interface->if_data.ifi_preamblelen = len; -done: - return (err); -} - -u_int32_t -ifnet_packetpreamblelen(ifnet_t interface) -{ - return ((interface == NULL) ? 0 : interface->if_data.ifi_preamblelen); -} - -u_int32_t -ifnet_maxpacketpreamblelen(void) -{ - return (MAX_IF_PACKET_PREAMBLE_LEN); -} - - /*************************************************************************/ /* Fastlane QoS Ca */ /*************************************************************************/ @@ -3092,8 +3036,7 @@ ifnet_get_unsent_bytes(ifnet_t interface, int64_t *unsent_bytes) bytes = *unsent_bytes = 0; - if ((interface->if_refflags & (IFRF_ATTACHED | IFRF_DETACHING)) != - IFRF_ATTACHED) + if (!IF_FULLY_ATTACHED(interface)) return (ENXIO); bytes = interface->if_sndbyte_unsent; @@ -3113,15 +3056,15 @@ ifnet_get_buffer_status(const ifnet_t ifp, ifnet_buffer_status_t *buf_status) bzero(buf_status, sizeof (*buf_status)); - if ((ifp->if_refflags & (IFRF_ATTACHED | IFRF_DETACHING)) != - IFRF_ATTACHED) + if (!IF_FULLY_ATTACHED(ifp)) return (ENXIO); - buf_status->buf_sndbuf = ifp->if_sndbyte_unsent; - if (ifp->if_eflags & IFEF_TXSTART) buf_status->buf_interface = IFCQ_BYTES(&ifp->if_snd); + buf_status->buf_sndbuf = ((buf_status->buf_interface != 0) || + (ifp->if_sndbyte_unsent != 0)) ? 1 : 0; + return (0); } @@ -3133,8 +3076,7 @@ ifnet_normalise_unsent_data(void) ifnet_head_lock_shared(); TAILQ_FOREACH(ifp, &ifnet_head, if_link) { ifnet_lock_exclusive(ifp); - if ((ifp->if_refflags & (IFRF_ATTACHED|IFRF_DETACHING)) != - IFRF_ATTACHED) { + if (!IF_FULLY_ATTACHED(ifp)) { ifnet_lock_done(ifp); continue; } diff --git a/bsd/net/kpi_interface.h b/bsd/net/kpi_interface.h index c94f294f0..15b7fd09f 100644 --- a/bsd/net/kpi_interface.h +++ b/bsd/net/kpi_interface.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2016 Apple Inc. All rights reserved. + * Copyright (c) 2004-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -49,10 +49,18 @@ struct if_interface_state; #include <sys/_types/_sa_family_t.h> #ifdef XNU_KERNEL_PRIVATE +#if CONFIG_EMBEDDED +#define KPI_INTERFACE_EMBEDDED 1 +#else #define KPI_INTERFACE_EMBEDDED 0 +#endif +#else +#if TARGET_OS_EMBEDDED +#define KPI_INTERFACE_EMBEDDED 1 #else #define KPI_INTERFACE_EMBEDDED 0 #endif +#endif struct timeval; struct sockaddr; @@ -124,6 +132,7 @@ enum { IFNET_SUBFAMILY_RESERVED = 5, IFNET_SUBFAMILY_INTCOPROC = 6, IFNET_SUBFAMILY_UTUN = 7, + IFNET_SUBFAMILY_IPSEC = 8, }; /* @@ -219,6 +228,7 @@ enum { #ifdef KERNEL_PRIVATE IFNET_CSUM_PARTIAL = 0x00001000, IFNET_CSUM_SUM16 = IFNET_CSUM_PARTIAL, + IFNET_CSUM_ZERO_INVERT = 0x00002000, #endif /* KERNEL_PRIVATE */ IFNET_VLAN_TAGGING = 0x00010000, IFNET_VLAN_MTU = 0x00020000, @@ -238,14 +248,14 @@ typedef u_int32_t ifnet_offload_t; #ifdef KERNEL_PRIVATE #define IFNET_OFFLOADF_BITS \ "\020\1CSUM_IP\2CSUM_TCP\3CSUM_UDP\4CSUM_IP_FRAGS\5IP_FRAGMENT" \ - "\6CSUM_TCPIPV6\7CSUM_UDPIPV6\10IPV6_FRAGMENT\15CSUM_PARTIAL" \ - "\20VLAN_TAGGING\21VLAN_MTU\25MULTIPAGES\26TSO_IPV4\27TSO_IPV6" \ - "\30TXSTATUS\31HW_TIMESTAMP\32SW_TIMESTAMP" + "\6CSUM_TCPIPV6\7CSUM_UDPIPV6\10IPV6_FRAGMENT\15CSUM_PARTIAL" \ + "\16CSUM_ZERO_INVERT\20VLAN_TAGGING\21VLAN_MTU\25MULTIPAGES" \ + "\26TSO_IPV4\27TSO_IPV6\30TXSTATUS\31HW_TIMESTAMP\32SW_TIMESTAMP" #define IFNET_CHECKSUMF \ (IFNET_CSUM_IP | IFNET_CSUM_TCP | IFNET_CSUM_UDP | \ IFNET_CSUM_FRAGMENT | IFNET_CSUM_TCPIPV6 | IFNET_CSUM_UDPIPV6 | \ - IFNET_CSUM_PARTIAL) + IFNET_CSUM_PARTIAL | IFNET_CSUM_ZERO_INVERT) #define IFNET_TSOF \ (IFNET_TSO_IPV4 | IFNET_TSO_IPV6) @@ -698,6 +708,8 @@ struct ifnet_init_params { /* Valid values for flags */ #define IFNET_INIT_LEGACY 0x1 /* legacy network interface model */ #define IFNET_INIT_INPUT_POLL 0x2 /* opportunistic input polling model */ +#define IFNET_INIT_NX_NOAUTO 0x4 /* do not auto config nexus */ +#define IFNET_INIT_ALLOC_KPI 0x8 /* allocated via the ifnet_alloc() KPI */ /* @typedef ifnet_pre_enqueue_func @@ -747,20 +759,6 @@ typedef void (*ifnet_input_poll_func)(ifnet_t interface, u_int32_t flags, u_int32_t max_count, mbuf_t *first_packet, mbuf_t *last_packet, u_int32_t *cnt, u_int32_t *len); -#ifdef BSD_KERNEL_PRIVATE -struct thread; -typedef errno_t (*ifnet_input_handler_func)(ifnet_t ifp, mbuf_t m_head, - mbuf_t m_tail, const struct ifnet_stat_increment_param *s, - boolean_t poll, struct thread *tp); -typedef errno_t (*ifnet_output_handler_func)(ifnet_t interface, mbuf_t data); - -extern errno_t ifnet_set_input_handler(struct ifnet *ifp, - ifnet_input_handler_func fn); -extern errno_t ifnet_set_output_handler(struct ifnet *ifp, - ifnet_output_handler_func fn); -extern void ifnet_reset_input_handler(struct ifnet *ifp); -extern void ifnet_reset_output_handler(struct ifnet *ifp); -#endif /* BSD_KERNEL_PRIVATE */ /* @enum Interface control commands @abstract Constants defining control commands. @@ -1052,6 +1050,13 @@ typedef errno_t (*ifnet_ctl_func)(ifnet_t interface, ifnet_ctl_cmd_t cmd, @field broadcast_addr The link-layer broadcast address for this interface. @field broadcast_len The length of the link-layer broadcast address. + @field tx_headroom The amount of headroom space to be reserved in the + packet being transmitted on the interface, specified in bytes. + Must be a multiple of 8 bytes. + @field tx_trailer The amount of trailer space to be reserved in the + packet being transmitted on the interface, specified in bytes. + @field rx_mit_ival mitigation interval for the rx mitigation logic, + specified in microseconds. */ struct ifnet_init_eparams { u_int32_t ver; /* required */ @@ -1104,11 +1109,14 @@ struct ifnet_init_eparams { u_int32_t broadcast_len; /* required for non point-to-point interfaces */ ifnet_framer_extended_func framer_extended; /* optional */ ifnet_subfamily_t subfamily; /* optional */ + u_int16_t tx_headroom; /* optional */ + u_int16_t tx_trailer; /* optional */ + u_int32_t rx_mit_ival; /* optional */ #if !defined(__LP64__) - u_int64_t _____reserved[3]; /* for future use */ + u_int64_t ____reserved[2]; /* for future use */ #else - u_int32_t ____reserved; /* pad */ - u_int64_t _____reserved[2]; /* for future use */ + u_int32_t ____reserved; /* for future use */ + u_int64_t _____reserved[1]; /* for future use */ #endif /* __LP64__ */ }; #endif /* KERNEL_PRIVATE */ @@ -1236,8 +1244,16 @@ __BEGIN_DECLS if an interface with the same uniqueid and family has already been allocated and is in use. */ +#ifdef KERNEL_PRIVATE +extern errno_t ifnet_allocate_internal(const struct ifnet_init_params *init, + ifnet_t *interface); + +#define ifnet_allocate(init, interface) \ + ifnet_allocate_internal((init), (interface)) +#else extern errno_t ifnet_allocate(const struct ifnet_init_params *init, ifnet_t *interface); +#endif /* KERNEL_PRIVATE */ #ifdef KERNEL_PRIVATE /* @@ -1583,39 +1599,6 @@ extern errno_t ifnet_poll_params(ifnet_t interface, */ extern void ifnet_start(ifnet_t interface); -/* - @function ifnet_transmit_burst_start - @discussion Inform the kernel about the beginning of transmission - of a burst. This function should be called when a burst of - packets are scheduled to get transmitted over the link. The - callback will be used by the system to start measuring - bandwidth available on that link. The driver may choose to - adopt this scheme for uplink bandwidth measurement, in case - the information can't be obtained from the hardware. Else - it may alternatively inform the network stack about the - information using ifnet_set_bandwidths. - @param interface The interface. - @param mbuf_t The first packet in a burst of packets that has been - scheduled to transmit. -*/ -extern void ifnet_transmit_burst_start(ifnet_t interface, mbuf_t pkt); - -/* - @function ifnet_transmit_burst_end - @discussion Inform the kernel about the end of transmission of a burst. - This function should be called when the transmission of a burst - of packets is done. This information will be used by the - system to estimate bandwidth available on that link. The - driver may choose to adopt this scheme for uplink bandwidth - measurement, in case the information can't be obtained from - the hardware. Else it may alternatively inform the network - stack about the information using ifnet_set_bandwidths. - @param interface The interface. - @param mbuf_t The last packet in the burst that has been successfully - transmitted. -*/ -extern void ifnet_transmit_burst_end(ifnet_t interface, mbuf_t pkt); - /* @function ifnet_flowid @discussion Returns the interface flow ID value, which can be used @@ -3237,7 +3220,7 @@ extern errno_t ifnet_get_local_ports(ifnet_t ifp, u_int8_t *bitfield); IFNET_GET_LOCAL_PORTS_EXTBGIDLEONLY: When bit is set, the port is in the list only if the socket has the option SO_EXTENDED_BK_IDLE set - IFNET_GET_LOCAL_PORTS_ACTIVETCPONLY: When bit is set, the + IFNET_GET_LOCAL_PORTS_ACTIVEONLY: When bit is set, the port is in the list only if the socket is not in a final TCP state or the connection is not idle in a final TCP state @param bitfield A pointer to 8192 bytes. @@ -3518,46 +3501,6 @@ extern errno_t ifnet_get_keepalive_offload_frames(ifnet_t ifp, extern errno_t ifnet_link_status_report(ifnet_t ifp, const void *buffer, size_t buffer_len); -/*************************************************************************/ -/* Packet preamble */ -/*************************************************************************/ -/*! - @function ifnet_set_packetpreamblelen - @discussion - Allows a driver to specify a leading space to be - reserved in front of the link layer header. - The preamble is logically adjoining the link layer which - itself is logically contiguous to the network protocol header - (e.g. IP). - There is no guarantee that packets being sent to the - driver has leading space reserved for the preamble. - There is also no guarantee the packet will be laid out in a - contiguous block of memory. - The network protocol header is 32 bit aligned and this dictates - the alignment of the link layer header which in turn affects - the alignment the packet preamble. - This function is intended to be called by the driver. A kext - must not call this function on an interface the kext does not - own. - @param interface The interface. - @param len The length of the packet preamble. - @result 0 on success otherwise the errno error. - */ -extern errno_t ifnet_set_packetpreamblelen(ifnet_t interface, u_int32_t len); - -/*! - @function ifnet_packetpreamblelen - @param interface The interface. - @result The current packet preamble length. - */ -extern u_int32_t ifnet_packetpreamblelen(ifnet_t interface); - -/*! - @function ifnet_maxpacketpreamblelen - @result The maximum packet preamble length supported by the system - */ -extern u_int32_t ifnet_maxpacketpreamblelen(void); - /*************************************************************************/ /* QoS Fastlane */ /*************************************************************************/ diff --git a/bsd/net/kpi_interfacefilter.c b/bsd/net/kpi_interfacefilter.c index 82ccd2d3e..47d03f9d4 100644 --- a/bsd/net/kpi_interfacefilter.c +++ b/bsd/net/kpi_interfacefilter.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003,2013 Apple Inc. All rights reserved. + * Copyright (c) 2003,2013,2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -34,6 +34,26 @@ #include <sys/kern_event.h> #include <net/dlil.h> +#undef iflt_attach +errno_t +iflt_attach( + ifnet_t interface, + const struct iff_filter *filter, + interface_filter_t *filter_ref); + + +errno_t +iflt_attach_internal( + ifnet_t interface, + const struct iff_filter *filter, + interface_filter_t *filter_ref) +{ + if (interface == NULL) return ENOENT; + + return dlil_attach_filter(interface, filter, filter_ref, + DLIL_IFF_INTERNAL); +} + errno_t iflt_attach( ifnet_t interface, diff --git a/bsd/net/kpi_interfacefilter.h b/bsd/net/kpi_interfacefilter.h index e1ea99aef..e5ac569e1 100644 --- a/bsd/net/kpi_interfacefilter.h +++ b/bsd/net/kpi_interfacefilter.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2003,2008,2017 Apple Computer, Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -204,8 +204,16 @@ struct iff_filter { @param filter_ref A reference to the filter used to detach. @result 0 on success otherwise the errno error. */ +#ifdef KERNEL_PRIVATE +extern errno_t iflt_attach_internal(ifnet_t interface, const struct iff_filter *filter, + interface_filter_t *filter_ref); + +#define iflt_attach(interface, filter, filter_ref) \ + iflt_attach_internal((interface), (filter), (filter_ref)) +#else extern errno_t iflt_attach(ifnet_t interface, const struct iff_filter *filter, interface_filter_t *filter_ref); +#endif /* KERNEL_PRIVATE */ /*! @function iflt_detach diff --git a/bsd/net/kpi_protocol.c b/bsd/net/kpi_protocol.c index f35b2b10b..c6314269d 100644 --- a/bsd/net/kpi_protocol.c +++ b/bsd/net/kpi_protocol.c @@ -193,7 +193,7 @@ proto_input_run(void) mbuf_t packet_list; int i, locked = 0; - lck_mtx_assert(&inp->input_lck, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&inp->input_lck, LCK_MTX_ASSERT_NOTOWNED); if (inp->input_waiting & DLIL_PROTO_REGISTER) { lck_mtx_lock_spin(&inp->input_lck); diff --git a/bsd/net/ndrv.c b/bsd/net/ndrv.c index 87b6a7749..41603c5b2 100644 --- a/bsd/net/ndrv.c +++ b/bsd/net/ndrv.c @@ -78,8 +78,6 @@ #endif #include <netinet/if_ether.h> -#include <machine/spl.h> - static unsigned int ndrv_multi_max_count = NDRV_DMUX_MAX_DESCR; SYSCTL_UINT(_net, OID_AUTO, ndrv_multi_max_count, CTLFLAG_RW | CTLFLAG_LOCKED, &ndrv_multi_max_count, 0, "Number of allowed multicast addresses per NRDV socket"); @@ -194,7 +192,7 @@ ndrv_input( return EJUSTRETURN; bcopy(frame_header, m->m_data, ifnet_hdrlen(ifp)); - lck_mtx_assert(ndrvdomain->dom_mtx, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(ndrvdomain->dom_mtx, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(ndrvdomain->dom_mtx); if (sbappendaddr(&(so->so_rcv), (struct sockaddr *)&ndrvsrc, m, (struct mbuf *)0, &error) != 0) { @@ -299,7 +297,7 @@ ndrv_event(struct ifnet *ifp, __unused protocol_family_t protocol, event->kev_class == KEV_NETWORK_CLASS && event->kev_subclass == KEV_DL_SUBCLASS && event->event_code == KEV_DL_IF_DETACHING) { - lck_mtx_assert(ndrvdomain->dom_mtx, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(ndrvdomain->dom_mtx, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(ndrvdomain->dom_mtx); ndrv_handle_ifp_detach(ifnet_family(ifp), ifnet_unit(ifp)); lck_mtx_unlock(ndrvdomain->dom_mtx); @@ -407,7 +405,7 @@ ndrv_disconnect(struct socket *so) static int ndrv_shutdown(struct socket *so) { - lck_mtx_assert(ndrvdomain->dom_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(ndrvdomain->dom_mtx, LCK_MTX_ASSERT_OWNED); socantsendmore(so); return 0; } @@ -866,7 +864,7 @@ ndrv_handle_ifp_detach(u_int32_t family, short unit) so = np->nd_socket; /* Make sure sending returns an error */ - lck_mtx_assert(ndrvdomain->dom_mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(ndrvdomain->dom_mtx, LCK_MTX_ASSERT_OWNED); socantsendmore(so); socantrcvmore(so); } diff --git a/bsd/net/necp.c b/bsd/net/necp.c index cf7e64dba..22f5afbd5 100644 --- a/bsd/net/necp.c +++ b/bsd/net/necp.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2016 Apple Inc. All rights reserved. + * Copyright (c) 2013-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -46,6 +46,7 @@ #include <netinet/ip6.h> #include <netinet/tcp.h> #include <netinet/tcp_var.h> +#include <netinet/tcp_cache.h> #include <netinet/udp.h> #include <netinet/in_pcb.h> #include <netinet/in_tclass.h> @@ -57,6 +58,7 @@ #include <sys/sysproto.h> #include <sys/priv.h> #include <sys/kern_event.h> +#include <sys/file_internal.h> #include <IOKit/IOBSD.h> #include <net/network_agent.h> #include <net/necp.h> @@ -203,6 +205,10 @@ u_int32_t necp_session_count = 0; #define NECP_MAX_POLICY_RESULT_SIZE 512 #define NECP_MAX_ROUTE_RULES_ARRAY_SIZE 1024 #define NECP_MAX_CONDITIONS_ARRAY_SIZE 4096 +#define NECP_MAX_POLICY_LIST_COUNT 1024 + +// Cap the policy size at the max result + conditions size, with room for extra TLVs +#define NECP_MAX_POLICY_SIZE (1024 + NECP_MAX_POLICY_RESULT_SIZE + NECP_MAX_CONDITIONS_ARRAY_SIZE) struct necp_service_registration { LIST_ENTRY(necp_service_registration) session_chain; @@ -211,10 +217,13 @@ struct necp_service_registration { }; struct necp_session { + u_int8_t necp_fd_type; u_int32_t control_unit; u_int32_t session_priority; // Descriptive priority rating u_int32_t session_order; + decl_lck_mtx_data(, lock); + bool proc_locked; // Messages must come from proc_uuid uuid_t proc_uuid; int proc_pid; @@ -223,8 +232,15 @@ struct necp_session { LIST_HEAD(_policies, necp_session_policy) policies; LIST_HEAD(_services, necp_service_registration) services; + + TAILQ_ENTRY(necp_session) chain; }; +#define NECP_SESSION_LOCK(_s) lck_mtx_lock(&_s->lock) +#define NECP_SESSION_UNLOCK(_s) lck_mtx_unlock(&_s->lock) + +static TAILQ_HEAD(_necp_session_list, necp_session) necp_session_list; + struct necp_socket_info { pid_t pid; uid_t uid; @@ -298,16 +314,18 @@ static LIST_HEAD(_necpkernelipoutputpolicies, necp_kernel_ip_output_policy) necp #define NECP_IP_OUTPUT_MAP_ID_TO_BUCKET(id) (id ? (id%(NECP_KERNEL_IP_OUTPUT_POLICIES_MAP_NUM_ID_BUCKETS - 1) + 1) : 0) static struct necp_kernel_ip_output_policy **necp_kernel_ip_output_policies_map[NECP_KERNEL_IP_OUTPUT_POLICIES_MAP_NUM_ID_BUCKETS]; -static struct necp_session *necp_create_session(u_int32_t control_unit); +static struct necp_session *necp_create_session(void); static void necp_delete_session(struct necp_session *session); -static void necp_handle_policy_add(struct necp_session *session, u_int32_t message_id, mbuf_t packet, int offset); +static necp_policy_id necp_handle_policy_add(struct necp_session *session, u_int32_t message_id, mbuf_t packet, + u_int8_t *tlv_buffer, size_t tlv_buffer_length, int offset, int *error); static void necp_handle_policy_get(struct necp_session *session, u_int32_t message_id, mbuf_t packet, int offset); static void necp_handle_policy_delete(struct necp_session *session, u_int32_t message_id, mbuf_t packet, int offset); static void necp_handle_policy_apply_all(struct necp_session *session, u_int32_t message_id, mbuf_t packet, int offset); static void necp_handle_policy_list_all(struct necp_session *session, u_int32_t message_id, mbuf_t packet, int offset); static void necp_handle_policy_delete_all(struct necp_session *session, u_int32_t message_id, mbuf_t packet, int offset); -static void necp_handle_policy_dump_all(struct necp_session *session, u_int32_t message_id, mbuf_t packet, int offset); +static int necp_handle_policy_dump_all(struct necp_session *session, u_int32_t message_id, mbuf_t packet, + user_addr_t out_buffer, size_t out_buffer_length, int offset); static void necp_handle_set_session_priority(struct necp_session *session, u_int32_t message_id, mbuf_t packet, int offset); static void necp_handle_lock_session_to_proc(struct necp_session *session, u_int32_t message_id, mbuf_t packet, int offset); static void necp_handle_register_service(struct necp_session *session, u_int32_t message_id, mbuf_t packet, int offset); @@ -381,6 +399,7 @@ static char *necp_create_trimmed_domain(char *string, size_t length); static inline int necp_count_dots(char *string, size_t length); static char *necp_copy_string(char *string, size_t length); +static bool necp_update_qos_marking(struct ifnet *ifp, u_int32_t route_rule_id); #define ROUTE_RULE_IS_AGGREGATE(ruleid) (ruleid > UINT16_MAX) @@ -436,30 +455,654 @@ necp_allocate_new_session_order(u_int32_t priority, u_int32_t control_unit) priority = NECP_SESSION_PRIORITY_DEFAULT; } - // Use the control unit to decide the offset into the priority list - new_order = (control_unit) + ((priority - 1) * 1000); - - return (new_order); -} - -static inline u_int32_t -necp_get_first_order_for_priority(u_int32_t priority) -{ - return (((priority - 1) * 1000) + 1); -} + // Use the control unit to decide the offset into the priority list + new_order = (control_unit) + ((priority - 1) * 1000); + + return (new_order); +} + +static inline u_int32_t +necp_get_first_order_for_priority(u_int32_t priority) +{ + return (((priority - 1) * 1000) + 1); +} + +// Sysctl handler +static int +sysctl_handle_necp_level SYSCTL_HANDLER_ARGS +{ +#pragma unused(arg1, arg2) + int error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req); + if (necp_drop_all_level == 0) { + necp_drop_all_order = 0; + } else { + necp_drop_all_order = necp_get_first_order_for_priority(necp_drop_all_level); + } + return (error); +} + +// Session fd + +static int noop_read(struct fileproc *, struct uio *, int, vfs_context_t); +static int noop_write(struct fileproc *, struct uio *, int, vfs_context_t); +static int noop_ioctl(struct fileproc *, unsigned long, caddr_t, + vfs_context_t); +static int noop_select(struct fileproc *, int, void *, vfs_context_t); +static int necp_session_op_close(struct fileglob *, vfs_context_t); +static int noop_kqfilter(struct fileproc *, struct knote *, + struct kevent_internal_s *, vfs_context_t); + +static const struct fileops necp_session_fd_ops = { + .fo_type = DTYPE_NETPOLICY, + .fo_read = noop_read, + .fo_write = noop_write, + .fo_ioctl = noop_ioctl, + .fo_select = noop_select, + .fo_close = necp_session_op_close, + .fo_kqfilter = noop_kqfilter, + .fo_drain = NULL, +}; + +static int +noop_read(struct fileproc *fp, struct uio *uio, int flags, vfs_context_t ctx) +{ +#pragma unused(fp, uio, flags, ctx) + return (ENXIO); +} + +static int +noop_write(struct fileproc *fp, struct uio *uio, int flags, + vfs_context_t ctx) +{ +#pragma unused(fp, uio, flags, ctx) + return (ENXIO); +} + +static int +noop_ioctl(struct fileproc *fp, unsigned long com, caddr_t data, + vfs_context_t ctx) +{ +#pragma unused(fp, com, data, ctx) + return (ENOTTY); +} + +static int +noop_select(struct fileproc *fp, int which, void *wql, vfs_context_t ctx) +{ +#pragma unused(fp, which, wql, ctx) + return (ENXIO); +} + +static int +noop_kqfilter(struct fileproc *fp, struct knote *kn, + struct kevent_internal_s *kev, vfs_context_t ctx) +{ +#pragma unused(fp, kn, kev, ctx) + return (ENXIO); +} + +int +necp_session_open(struct proc *p, struct necp_session_open_args *uap, int *retval) +{ +#pragma unused(uap) + int error = 0; + struct necp_session *session = NULL; + struct fileproc *fp = NULL; + int fd = -1; + + uid_t uid = kauth_cred_getuid(proc_ucred(p)); + if (uid != 0 && priv_check_cred(kauth_cred_get(), PRIV_NET_PRIVILEGED_NECP_POLICIES, 0) != 0) { + NECPLOG0(LOG_ERR, "Process does not hold necessary entitlement to open NECP session"); + error = EACCES; + goto done; + } + + error = falloc(p, &fp, &fd, vfs_context_current()); + if (error != 0) { + goto done; + } + + session = necp_create_session(); + if (session == NULL) { + error = ENOMEM; + goto done; + } + + fp->f_fglob->fg_flag = 0; + fp->f_fglob->fg_ops = &necp_session_fd_ops; + fp->f_fglob->fg_data = session; + + proc_fdlock(p); + FDFLAGS_SET(p, fd, (UF_EXCLOSE | UF_FORKCLOSE)); + procfdtbl_releasefd(p, fd, NULL); + fp_drop(p, fd, fp, 1); + proc_fdunlock(p); + + *retval = fd; +done: + if (error != 0) { + if (fp != NULL) { + fp_free(p, fd, fp); + fp = NULL; + } + } + + return (error); +} + +static int +necp_session_op_close(struct fileglob *fg, vfs_context_t ctx) +{ +#pragma unused(ctx) + struct necp_session *session = (struct necp_session *)fg->fg_data; + fg->fg_data = NULL; + + if (session != NULL) { + necp_policy_mark_all_for_deletion(session); + necp_policy_apply_all(session); + necp_delete_session(session); + return (0); + } else { + return (ENOENT); + } +} + +static int +necp_session_find_from_fd(int fd, struct necp_session **session) +{ + proc_t p = current_proc(); + struct fileproc *fp = NULL; + int error = 0; + + proc_fdlock_spin(p); + if ((error = fp_lookup(p, fd, &fp, 1)) != 0) { + goto done; + } + if (fp->f_fglob->fg_ops->fo_type != DTYPE_NETPOLICY) { + fp_drop(p, fd, fp, 1); + error = ENODEV; + goto done; + } + *session = (struct necp_session *)fp->f_fglob->fg_data; + +done: + proc_fdunlock(p); + return (error); +} + +static int +necp_session_add_policy(struct necp_session *session, struct necp_session_action_args *uap, int *retval) +{ + int error = 0; + u_int8_t *tlv_buffer = NULL; + + if (uap->in_buffer_length == 0 || uap->in_buffer_length > NECP_MAX_POLICY_SIZE || uap->in_buffer == 0) { + NECPLOG(LOG_ERR, "necp_session_add_policy invalid input (%zu)", uap->in_buffer_length); + error = EINVAL; + goto done; + } + + if (uap->out_buffer_length < sizeof(necp_policy_id) || uap->out_buffer == 0) { + NECPLOG(LOG_ERR, "necp_session_add_policy invalid output buffer (%zu)", uap->out_buffer_length); + error = EINVAL; + goto done; + } + + if ((tlv_buffer = _MALLOC(uap->in_buffer_length, M_NECP, M_WAITOK | M_ZERO)) == NULL) { + error = ENOMEM; + goto done; + } + + error = copyin(uap->in_buffer, tlv_buffer, uap->in_buffer_length); + if (error != 0) { + NECPLOG(LOG_ERR, "necp_session_add_policy tlv copyin error (%d)", error); + goto done; + } + + necp_policy_id new_policy_id = necp_handle_policy_add(session, 0, NULL, tlv_buffer, uap->in_buffer_length, 0, &error); + if (error != 0) { + NECPLOG(LOG_ERR, "necp_session_add_policy failed to add policy (%d)", error); + goto done; + } + + error = copyout(&new_policy_id, uap->out_buffer, sizeof(new_policy_id)); + if (error != 0) { + NECPLOG(LOG_ERR, "necp_session_add_policy policy_id copyout error (%d)", error); + goto done; + } + +done: + if (tlv_buffer != NULL) { + FREE(tlv_buffer, M_NECP); + tlv_buffer = NULL; + } + *retval = error; + + return (error); +} + +static int +necp_session_get_policy(struct necp_session *session, struct necp_session_action_args *uap, int *retval) +{ + int error = 0; + u_int8_t *response = NULL; + + if (uap->in_buffer_length < sizeof(necp_policy_id) || uap->in_buffer == 0) { + NECPLOG(LOG_ERR, "necp_session_get_policy invalid input (%zu)", uap->in_buffer_length); + error = EINVAL; + goto done; + } + + necp_policy_id policy_id = 0; + error = copyin(uap->in_buffer, &policy_id, sizeof(policy_id)); + if (error != 0) { + NECPLOG(LOG_ERR, "necp_session_get_policy policy_id copyin error (%d)", error); + goto done; + } + + struct necp_session_policy *policy = necp_policy_find(session, policy_id); + if (policy == NULL || policy->pending_deletion) { + NECPLOG(LOG_ERR, "Failed to find policy with id %d", policy_id); + error = ENOENT; + goto done; + } + + u_int32_t order_tlv_size = sizeof(u_int8_t) + sizeof(u_int32_t) + sizeof(necp_policy_order); + u_int32_t result_tlv_size = (policy->result_size ? (sizeof(u_int8_t) + sizeof(u_int32_t) + policy->result_size) : 0); + u_int32_t response_size = order_tlv_size + result_tlv_size + policy->conditions_size; + + if (uap->out_buffer_length < response_size || uap->out_buffer == 0) { + NECPLOG(LOG_ERR, "necp_session_get_policy buffer not large enough (%u < %u)", uap->out_buffer_length, response_size); + error = EINVAL; + goto done; + } + + if (response_size > NECP_MAX_POLICY_SIZE) { + NECPLOG(LOG_ERR, "necp_session_get_policy size too large to copy (%u)", response_size); + error = EINVAL; + goto done; + } + + MALLOC(response, u_int8_t *, response_size, M_NECP, M_WAITOK | M_ZERO); + if (response == NULL) { + error = ENOMEM; + goto done; + } + + u_int8_t *cursor = response; + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_ORDER, sizeof(necp_policy_order), &policy->order, response, response_size); + if (result_tlv_size) { + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_RESULT, policy->result_size, &policy->result, response, response_size); + } + if (policy->conditions_size) { + memcpy(((u_int8_t *)(void *)(cursor)), policy->conditions, policy->conditions_size); + } + + error = copyout(response, uap->out_buffer, response_size); + if (error != 0) { + NECPLOG(LOG_ERR, "necp_session_get_policy TLV copyout error (%d)", error); + goto done; + } + +done: + if (response != NULL) { + FREE(response, M_NECP); + response = NULL; + } + *retval = error; + + return (error); +} + +static int +necp_session_delete_policy(struct necp_session *session, struct necp_session_action_args *uap, int *retval) +{ + int error = 0; + + if (uap->in_buffer_length < sizeof(necp_policy_id) || uap->in_buffer == 0) { + NECPLOG(LOG_ERR, "necp_session_delete_policy invalid input (%zu)", uap->in_buffer_length); + error = EINVAL; + goto done; + } + + necp_policy_id delete_policy_id = 0; + error = copyin(uap->in_buffer, &delete_policy_id, sizeof(delete_policy_id)); + if (error != 0) { + NECPLOG(LOG_ERR, "necp_session_delete_policy policy_id copyin error (%d)", error); + goto done; + } + + struct necp_session_policy *policy = necp_policy_find(session, delete_policy_id); + if (policy == NULL || policy->pending_deletion) { + NECPLOG(LOG_ERR, "necp_session_delete_policy failed to find policy with id %u", delete_policy_id); + error = ENOENT; + goto done; + } + + necp_policy_mark_for_deletion(session, policy); +done: + *retval = error; + return (error); +} + +static int +necp_session_apply_all(struct necp_session *session, struct necp_session_action_args *uap, int *retval) +{ +#pragma unused(uap) + necp_policy_apply_all(session); + *retval = 0; + return (0); +} + +static int +necp_session_list_all(struct necp_session *session, struct necp_session_action_args *uap, int *retval) +{ + u_int32_t tlv_size = (sizeof(u_int8_t) + sizeof(u_int32_t) + sizeof(necp_policy_id)); + u_int32_t response_size = 0; + u_int8_t *response = NULL; + int num_policies = 0; + int cur_policy_index = 0; + int error = 0; + struct necp_session_policy *policy; + + LIST_FOREACH(policy, &session->policies, chain) { + if (!policy->pending_deletion) { + num_policies++; + } + } + + if (num_policies > NECP_MAX_POLICY_LIST_COUNT) { + NECPLOG(LOG_ERR, "necp_session_list_all size too large to copy (%u policies)", num_policies); + error = EINVAL; + goto done; + } + + response_size = num_policies * tlv_size; + if (uap->out_buffer_length < response_size || uap->out_buffer == 0) { + NECPLOG(LOG_ERR, "necp_session_list_all buffer not large enough (%u < %u)", uap->out_buffer_length, response_size); + error = EINVAL; + goto done; + } + + // Create a response with one Policy ID TLV for each policy + MALLOC(response, u_int8_t *, response_size, M_NECP, M_WAITOK | M_ZERO); + if (response == NULL) { + error = ENOMEM; + goto done; + } + + u_int8_t *cursor = response; + LIST_FOREACH(policy, &session->policies, chain) { + if (!policy->pending_deletion && cur_policy_index < num_policies) { + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_ID, sizeof(u_int32_t), &policy->id, response, response_size); + cur_policy_index++; + } + } + + error = copyout(response, uap->out_buffer, response_size); + if (error != 0) { + NECPLOG(LOG_ERR, "necp_session_list_all TLV copyout error (%d)", error); + goto done; + } + +done: + if (response != NULL) { + FREE(response, M_NECP); + response = NULL; + } + *retval = error; + + return (error); +} + + +static int +necp_session_delete_all(struct necp_session *session, struct necp_session_action_args *uap, int *retval) +{ +#pragma unused(uap) + necp_policy_mark_all_for_deletion(session); + *retval = 0; + return (0); +} + +static int +necp_session_set_session_priority(struct necp_session *session, struct necp_session_action_args *uap, int *retval) +{ + int error = 0; + struct necp_session_policy *policy = NULL; + struct necp_session_policy *temp_policy = NULL; + + if (uap->in_buffer_length < sizeof(necp_session_priority) || uap->in_buffer == 0) { + NECPLOG(LOG_ERR, "necp_session_set_session_priority invalid input (%zu)", uap->in_buffer_length); + error = EINVAL; + goto done; + } + + necp_session_priority requested_session_priority = 0; + error = copyin(uap->in_buffer, &requested_session_priority, sizeof(requested_session_priority)); + if (error != 0) { + NECPLOG(LOG_ERR, "necp_session_set_session_priority priority copyin error (%d)", error); + goto done; + } + + // Enforce special session priorities with entitlements + if (requested_session_priority == NECP_SESSION_PRIORITY_CONTROL || + requested_session_priority == NECP_SESSION_PRIORITY_PRIVILEGED_TUNNEL) { + errno_t cred_result = priv_check_cred(kauth_cred_get(), PRIV_NET_PRIVILEGED_NECP_POLICIES, 0); + if (cred_result != 0) { + NECPLOG(LOG_ERR, "Session does not hold necessary entitlement to claim priority level %d", requested_session_priority); + error = EPERM; + goto done; + } + } + + if (session->session_priority != requested_session_priority) { + session->session_priority = requested_session_priority; + session->session_order = necp_allocate_new_session_order(session->session_priority, session->control_unit); + session->dirty = TRUE; + + // Mark all policies as needing updates + LIST_FOREACH_SAFE(policy, &session->policies, chain, temp_policy) { + policy->pending_update = TRUE; + } + } + +done: + *retval = error; + return (error); +} + +static int +necp_session_lock_to_process(struct necp_session *session, struct necp_session_action_args *uap, int *retval) +{ +#pragma unused(uap) + session->proc_locked = TRUE; + *retval = 0; + return (0); +} + +static int +necp_session_register_service(struct necp_session *session, struct necp_session_action_args *uap, int *retval) +{ + int error = 0; + struct necp_service_registration *new_service = NULL; + + if (uap->in_buffer_length < sizeof(uuid_t) || uap->in_buffer == 0) { + NECPLOG(LOG_ERR, "necp_session_register_service invalid input (%zu)", uap->in_buffer_length); + error = EINVAL; + goto done; + } + + uuid_t service_uuid; + error = copyin(uap->in_buffer, service_uuid, sizeof(service_uuid)); + if (error != 0) { + NECPLOG(LOG_ERR, "necp_session_register_service uuid copyin error (%d)", error); + goto done; + } + + MALLOC(new_service, struct necp_service_registration *, sizeof(*new_service), M_NECP, M_WAITOK | M_ZERO); + if (new_service == NULL) { + NECPLOG0(LOG_ERR, "Failed to allocate service registration"); + error = ENOMEM; + goto done; + } + + lck_rw_lock_exclusive(&necp_kernel_policy_lock); + new_service->service_id = necp_create_uuid_service_id_mapping(service_uuid); + LIST_INSERT_HEAD(&session->services, new_service, session_chain); + LIST_INSERT_HEAD(&necp_registered_service_list, new_service, kernel_chain); + lck_rw_done(&necp_kernel_policy_lock); + +done: + *retval = error; + return (error); +} + +static int +necp_session_unregister_service(struct necp_session *session, struct necp_session_action_args *uap, int *retval) +{ + int error = 0; + struct necp_service_registration *service = NULL; + struct necp_service_registration *temp_service = NULL; + struct necp_uuid_id_mapping *mapping = NULL; + + if (uap->in_buffer_length < sizeof(uuid_t) || uap->in_buffer == 0) { + NECPLOG(LOG_ERR, "necp_session_unregister_service invalid input (%zu)", uap->in_buffer_length); + error = EINVAL; + goto done; + } + + uuid_t service_uuid; + error = copyin(uap->in_buffer, service_uuid, sizeof(service_uuid)); + if (error != 0) { + NECPLOG(LOG_ERR, "necp_session_unregister_service uuid copyin error (%d)", error); + goto done; + } + + // Remove all matching services for this session + lck_rw_lock_exclusive(&necp_kernel_policy_lock); + mapping = necp_uuid_lookup_service_id_locked(service_uuid); + if (mapping != NULL) { + LIST_FOREACH_SAFE(service, &session->services, session_chain, temp_service) { + if (service->service_id == mapping->id) { + LIST_REMOVE(service, session_chain); + LIST_REMOVE(service, kernel_chain); + FREE(service, M_NECP); + } + } + necp_remove_uuid_service_id_mapping(service_uuid); + } + lck_rw_done(&necp_kernel_policy_lock); + +done: + *retval = error; + return (error); +} + +static int +necp_session_dump_all(struct necp_session *session, struct necp_session_action_args *uap, int *retval) +{ + int error = 0; + + if (uap->out_buffer_length == 0 || uap->out_buffer == 0) { + NECPLOG(LOG_ERR, "necp_session_dump_all invalid output buffer (%zu)", uap->out_buffer_length); + error = EINVAL; + goto done; + } + + error = necp_handle_policy_dump_all(session, 0, NULL, uap->out_buffer, uap->out_buffer_length, 0); +done: + *retval = error; + return (error); +} + +int +necp_session_action(struct proc *p, struct necp_session_action_args *uap, int *retval) +{ +#pragma unused(p) + int error = 0; + int return_value = 0; + struct necp_session *session = NULL; + error = necp_session_find_from_fd(uap->necp_fd, &session); + if (error != 0) { + NECPLOG(LOG_ERR, "necp_session_action find fd error (%d)", error); + return (error); + } + + NECP_SESSION_LOCK(session); + + if (session->proc_locked) { + // Verify that the calling process is allowed to do actions + uuid_t proc_uuid; + proc_getexecutableuuid(current_proc(), proc_uuid, sizeof(proc_uuid)); + if (uuid_compare(proc_uuid, session->proc_uuid) != 0) { + error = EPERM; + goto done; + } + } else { + // If not locked, update the proc_uuid and proc_pid of the session + proc_getexecutableuuid(current_proc(), session->proc_uuid, sizeof(session->proc_uuid)); + session->proc_pid = proc_pid(current_proc()); + } + + u_int32_t action = uap->action; + switch (action) { + case NECP_SESSION_ACTION_POLICY_ADD: { + return_value = necp_session_add_policy(session, uap, retval); + break; + } + case NECP_SESSION_ACTION_POLICY_GET: { + return_value = necp_session_get_policy(session, uap, retval); + break; + } + case NECP_SESSION_ACTION_POLICY_DELETE: { + return_value = necp_session_delete_policy(session, uap, retval); + break; + } + case NECP_SESSION_ACTION_POLICY_APPLY_ALL: { + return_value = necp_session_apply_all(session, uap, retval); + break; + } + case NECP_SESSION_ACTION_POLICY_LIST_ALL: { + return_value = necp_session_list_all(session, uap, retval); + break; + } + case NECP_SESSION_ACTION_POLICY_DELETE_ALL: { + return_value = necp_session_delete_all(session, uap, retval); + break; + } + case NECP_SESSION_ACTION_SET_SESSION_PRIORITY: { + return_value = necp_session_set_session_priority(session, uap, retval); + break; + } + case NECP_SESSION_ACTION_LOCK_SESSION_TO_PROC: { + return_value = necp_session_lock_to_process(session, uap, retval); + break; + } + case NECP_SESSION_ACTION_REGISTER_SERVICE: { + return_value = necp_session_register_service(session, uap, retval); + break; + } + case NECP_SESSION_ACTION_UNREGISTER_SERVICE: { + return_value = necp_session_unregister_service(session, uap, retval); + break; + } + case NECP_SESSION_ACTION_POLICY_DUMP_ALL: { + return_value = necp_session_dump_all(session, uap, retval); + break; + } + default: { + NECPLOG(LOG_ERR, "necp_session_action unknown action (%u)", action); + return_value = EINVAL; + break; + } + } + +done: + NECP_SESSION_UNLOCK(session); + file_drop(uap->necp_fd); -// Sysctl handler -static int -sysctl_handle_necp_level SYSCTL_HANDLER_ARGS -{ -#pragma unused(arg1, arg2) - int error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req); - if (necp_drop_all_level == 0) { - necp_drop_all_order = 0; - } else { - necp_drop_all_order = necp_get_first_order_for_priority(necp_drop_all_level); - } - return (error); + return (return_value); } // Kernel Control functions @@ -531,6 +1174,8 @@ necp_init(void) necp_client_init(); + TAILQ_INIT(&necp_session_list); + LIST_INIT(&necp_kernel_socket_policies); LIST_INIT(&necp_kernel_ip_output_policies); @@ -664,8 +1309,8 @@ necp_post_change_event(struct kev_necp_policies_changed_data *necp_event_data) static errno_t necp_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac, void **unitinfo) { -#pragma unused(kctlref) - *unitinfo = necp_create_session(sac->sc_unit); +#pragma unused(kctlref, sac) + *unitinfo = necp_create_session(); if (*unitinfo == NULL) { // Could not allocate session return (ENOBUFS); @@ -763,20 +1408,6 @@ necp_packet_get_tlv_at_offset(mbuf_t packet, int tlv_offset, u_int32_t buff_len, return (0); } -static int -necp_packet_get_tlv(mbuf_t packet, int offset, u_int8_t type, u_int32_t buff_len, void *buff, u_int32_t *value_size) -{ - int error = 0; - int tlv_offset; - - tlv_offset = necp_packet_find_tlv(packet, offset, type, &error, 0); - if (tlv_offset < 0) { - return (error); - } - - return (necp_packet_get_tlv_at_offset(packet, tlv_offset, buff_len, buff, value_size)); -} - static u_int8_t * necp_buffer_write_packet_header(u_int8_t *buffer, u_int8_t packet_type, u_int8_t flags, u_int32_t message_id) { @@ -786,41 +1417,66 @@ necp_buffer_write_packet_header(u_int8_t *buffer, u_int8_t packet_type, u_int8_t return (buffer + sizeof(struct necp_packet_header)); } +static inline bool +necp_buffer_write_tlv_validate(u_int8_t *cursor, u_int8_t type, u_int32_t length, + u_int8_t *buffer, u_int32_t buffer_length) +{ + if (cursor < buffer || (uintptr_t)(cursor - buffer) > buffer_length) { + NECPLOG0(LOG_ERR, "Cannot write TLV in buffer (invalid cursor)"); + return (false); + } + u_int8_t *next_tlv = (u_int8_t *)(cursor + sizeof(type) + sizeof(length) + length); + if (next_tlv <= buffer || // make sure the next TLV start doesn't overflow + (uintptr_t)(next_tlv - buffer) > buffer_length) { // make sure the next TLV has enough room in buffer + NECPLOG(LOG_ERR, "Cannot write TLV in buffer (TLV length %u, buffer length %u)", + length, buffer_length); + return (false); + } + return (true); +} u_int8_t * -necp_buffer_write_tlv_if_different(u_int8_t *buffer, const u_int8_t *max, u_int8_t type, - u_int32_t length, const void *value, bool *updated) +necp_buffer_write_tlv_if_different(u_int8_t *cursor, u_int8_t type, + u_int32_t length, const void *value, bool *updated, + u_int8_t *buffer, u_int32_t buffer_length) { - u_int8_t *next_tlv = (u_int8_t *)(buffer + sizeof(type) + sizeof(length) + length); - if (next_tlv <= max) { - if (*updated || *(u_int8_t *)(buffer) != type) { - *(u_int8_t *)(buffer) = type; - *updated = TRUE; - } - if (*updated || *(u_int32_t *)(void *)(buffer + sizeof(type)) != length) { - *(u_int32_t *)(void *)(buffer + sizeof(type)) = length; + if (!necp_buffer_write_tlv_validate(cursor, type, length, buffer, buffer_length)) { + return (NULL); + } + u_int8_t *next_tlv = (u_int8_t *)(cursor + sizeof(type) + sizeof(length) + length); + if (*updated || *(u_int8_t *)(cursor) != type) { + *(u_int8_t *)(cursor) = type; + *updated = TRUE; + } + if (*updated || *(u_int32_t *)(void *)(cursor + sizeof(type)) != length) { + *(u_int32_t *)(void *)(cursor + sizeof(type)) = length; + *updated = TRUE; + } + if (length > 0) { + if (*updated || memcmp((u_int8_t *)(cursor + sizeof(type) + sizeof(length)), value, length) != 0) { + memcpy((u_int8_t *)(cursor + sizeof(type) + sizeof(length)), value, length); *updated = TRUE; } - if (length > 0) { - if (*updated || memcmp((u_int8_t *)(buffer + sizeof(type) + sizeof(length)), value, length) != 0) { - memcpy((u_int8_t *)(buffer + sizeof(type) + sizeof(length)), value, length); - *updated = TRUE; - } - } } return (next_tlv); } u_int8_t * -necp_buffer_write_tlv(u_int8_t *buffer, u_int8_t type, u_int32_t length, const void *value) +necp_buffer_write_tlv(u_int8_t *cursor, u_int8_t type, + u_int32_t length, const void *value, + u_int8_t *buffer, u_int32_t buffer_length) { - *(u_int8_t *)(buffer) = type; - *(u_int32_t *)(void *)(buffer + sizeof(type)) = length; + if (!necp_buffer_write_tlv_validate(cursor, type, length, buffer, buffer_length)) { + return (NULL); + } + u_int8_t *next_tlv = (u_int8_t *)(cursor + sizeof(type) + sizeof(length) + length); + *(u_int8_t *)(cursor) = type; + *(u_int32_t *)(void *)(cursor + sizeof(type)) = length; if (length > 0) { - memcpy((u_int8_t *)(buffer + sizeof(type) + sizeof(length)), value, length); + memcpy((u_int8_t *)(cursor + sizeof(type) + sizeof(length)), value, length); } - return ((u_int8_t *)(buffer + sizeof(type) + sizeof(length) + length)); + return (next_tlv); } u_int8_t @@ -905,6 +1561,90 @@ necp_buffer_find_tlv(u_int8_t *buffer, u_int32_t buffer_length, int offset, u_in } } +static int +necp_find_tlv(mbuf_t packet, u_int8_t *buffer, u_int32_t buffer_length, int offset, u_int8_t type, int *err, int next) +{ + int cursor = -1; + if (packet != NULL) { + cursor = necp_packet_find_tlv(packet, offset, type, err, next); + } else if (buffer != NULL) { + cursor = necp_buffer_find_tlv(buffer, buffer_length, offset, type, next); + } + return (cursor); +} + +static int +necp_get_tlv_at_offset(mbuf_t packet, u_int8_t *buffer, u_int32_t buffer_length, + int tlv_offset, u_int32_t out_buffer_length, void *out_buffer, u_int32_t *value_size) +{ + if (packet != NULL) { + // Handle mbuf parsing + return necp_packet_get_tlv_at_offset(packet, tlv_offset, out_buffer_length, out_buffer, value_size); + } + + if (buffer == NULL) { + NECPLOG0(LOG_ERR, "necp_get_tlv_at_offset buffer is NULL"); + return (EINVAL); + } + + // Handle buffer parsing + + // Validate that buffer has enough room for any TLV + if (tlv_offset + sizeof(u_int8_t) + sizeof(u_int32_t) > buffer_length) { + NECPLOG(LOG_ERR, "necp_get_tlv_at_offset buffer_length is too small for TLV (%u < %u)", + buffer_length, tlv_offset + sizeof(u_int8_t) + sizeof(u_int32_t)); + return (EINVAL); + } + + // Validate that buffer has enough room for this TLV + u_int32_t tlv_length = necp_buffer_get_tlv_length(buffer, tlv_offset); + if (tlv_length > buffer_length - (tlv_offset + sizeof(u_int8_t) + sizeof(u_int32_t))) { + NECPLOG(LOG_ERR, "necp_get_tlv_at_offset buffer_length is too small for TLV of length %u (%u < %u)", + tlv_length, buffer_length, tlv_offset + sizeof(u_int8_t) + sizeof(u_int32_t) + tlv_length); + return (EINVAL); + } + + if (out_buffer != NULL && out_buffer_length > 0) { + // Validate that out buffer is large enough for value + if (out_buffer_length < tlv_length) { + NECPLOG(LOG_ERR, "necp_get_tlv_at_offset out_buffer_length is too small for TLV value (%u < %u)", + out_buffer_length, tlv_length); + return (EINVAL); + } + + // Get value pointer + u_int8_t *tlv_value = necp_buffer_get_tlv_value(buffer, tlv_offset, NULL); + if (tlv_value == NULL) { + NECPLOG0(LOG_ERR, "necp_get_tlv_at_offset tlv_value is NULL"); + return (ENOENT); + } + + // Copy value + memcpy(out_buffer, tlv_value, tlv_length); + } + + // Copy out length + if (value_size != NULL) { + *value_size = tlv_length; + } + + return (0); +} + +static int +necp_get_tlv(mbuf_t packet, u_int8_t *buffer, u_int32_t buffer_length, + int offset, u_int8_t type, u_int32_t buff_len, void *buff, u_int32_t *value_size) +{ + int error = 0; + + int tlv_offset = necp_find_tlv(packet, buffer, buffer_length, offset, type, &error, 0); + if (tlv_offset < 0) { + return (error); + } + + return (necp_get_tlv_at_offset(packet, buffer, buffer_length, tlv_offset, buff_len, buff, value_size)); +} + static bool necp_send_ctl_data(struct necp_session *session, u_int8_t *buffer, size_t buffer_size) { @@ -932,7 +1672,7 @@ necp_send_success_response(struct necp_session *session, u_int8_t packet_type, u } cursor = response; cursor = necp_buffer_write_packet_header(cursor, packet_type, NECP_PACKET_FLAGS_RESPONSE, message_id); - cursor = necp_buffer_write_tlv(cursor, NECP_TLV_NIL, 0, NULL); + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_NIL, 0, NULL, response, response_size); if (!(success = necp_send_ctl_data(session, (u_int8_t *)response, response_size))) { NECPLOG0(LOG_ERR, "Failed to send response"); @@ -955,7 +1695,7 @@ necp_send_error_response(struct necp_session *session, u_int8_t packet_type, u_i } cursor = response; cursor = necp_buffer_write_packet_header(cursor, packet_type, NECP_PACKET_FLAGS_RESPONSE, message_id); - cursor = necp_buffer_write_tlv(cursor, NECP_TLV_ERROR, sizeof(error), &error); + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_ERROR, sizeof(error), &error, response, response_size); if (!(success = necp_send_ctl_data(session, (u_int8_t *)response, response_size))) { NECPLOG0(LOG_ERR, "Failed to send response"); @@ -978,7 +1718,7 @@ necp_send_policy_id_response(struct necp_session *session, u_int8_t packet_type, } cursor = response; cursor = necp_buffer_write_packet_header(cursor, packet_type, NECP_PACKET_FLAGS_RESPONSE, message_id); - cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_ID, sizeof(policy_id), &policy_id); + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_ID, sizeof(policy_id), &policy_id, response, response_size); if (!(success = necp_send_ctl_data(session, (u_int8_t *)response, response_size))) { NECPLOG0(LOG_ERR, "Failed to send response"); @@ -1031,7 +1771,7 @@ necp_ctl_send(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, mbuf_t packe switch (header.packet_type) { case NECP_PACKET_TYPE_POLICY_ADD: { - necp_handle_policy_add(session, header.message_id, packet, sizeof(header)); + necp_handle_policy_add(session, header.message_id, packet, NULL, 0, sizeof(header), NULL); break; } case NECP_PACKET_TYPE_POLICY_GET: { @@ -1055,7 +1795,7 @@ necp_ctl_send(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, mbuf_t packe break; } case NECP_PACKET_TYPE_POLICY_DUMP_ALL: { - necp_handle_policy_dump_all(session, header.message_id, packet, sizeof(header)); + necp_handle_policy_dump_all(session, header.message_id, packet, 0, 0, sizeof(header)); break; } case NECP_PACKET_TYPE_SET_SESSION_PRIORITY: { @@ -1108,29 +1848,55 @@ necp_ctl_setopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, int opt, v } // Session Management + static struct necp_session * -necp_create_session(u_int32_t control_unit) +necp_create_session(void) { struct necp_session *new_session = NULL; - MALLOC(new_session, struct necp_session *, sizeof(*new_session), M_NECP, M_WAITOK); + MALLOC(new_session, struct necp_session *, sizeof(*new_session), M_NECP, M_WAITOK | M_ZERO); if (new_session == NULL) { goto done; } - if (necp_debug) { - NECPLOG(LOG_DEBUG, "Create NECP session, control unit %d", control_unit); - } - memset(new_session, 0, sizeof(*new_session)); + + new_session->necp_fd_type = necp_fd_type_session; new_session->session_priority = NECP_SESSION_PRIORITY_UNKNOWN; - new_session->session_order = necp_allocate_new_session_order(new_session->session_priority, control_unit); - new_session->control_unit = control_unit; new_session->dirty = FALSE; LIST_INIT(&new_session->policies); + lck_mtx_init(&new_session->lock, necp_kernel_policy_mtx_grp, necp_kernel_policy_mtx_attr); + // Take the lock lck_rw_lock_exclusive(&necp_kernel_policy_lock); + + // Find the next available control unit + u_int32_t control_unit = 1; + struct necp_session *next_session = NULL; + TAILQ_FOREACH(next_session, &necp_session_list, chain) { + if (next_session->control_unit > control_unit) { + // Found a gap, grab this control unit + break; + } + + // Try the next control unit, loop around + control_unit = next_session->control_unit + 1; + } + + new_session->control_unit = control_unit; + new_session->session_order = necp_allocate_new_session_order(new_session->session_priority, control_unit); + + if (next_session != NULL) { + TAILQ_INSERT_BEFORE(next_session, new_session, chain); + } else { + TAILQ_INSERT_TAIL(&necp_session_list, new_session, chain); + } + necp_session_count++; lck_rw_done(&necp_kernel_policy_lock); + if (necp_debug) { + NECPLOG(LOG_DEBUG, "Created NECP session, control unit %d", control_unit); + } + done: return (new_session); } @@ -1151,11 +1917,14 @@ necp_delete_session(struct necp_session *session) if (necp_debug) { NECPLOG0(LOG_DEBUG, "Deleted NECP session"); } - FREE(session, M_NECP); lck_rw_lock_exclusive(&necp_kernel_policy_lock); + TAILQ_REMOVE(&necp_session_list, session, chain); necp_session_count--; lck_rw_done(&necp_kernel_policy_lock); + + lck_mtx_destroy(&session->lock, necp_kernel_policy_mtx_grp); + FREE(session, M_NECP); } } @@ -1467,6 +2236,30 @@ necp_policy_route_rule_is_valid(u_int8_t *buffer, u_int32_t length) return (validated); } +static int +necp_get_posix_error_for_necp_error(int response_error) +{ + switch (response_error) { + case NECP_ERROR_UNKNOWN_PACKET_TYPE: + case NECP_ERROR_INVALID_TLV: + case NECP_ERROR_POLICY_RESULT_INVALID: + case NECP_ERROR_POLICY_CONDITIONS_INVALID: + case NECP_ERROR_ROUTE_RULES_INVALID: { + return (EINVAL); + } + case NECP_ERROR_POLICY_ID_NOT_FOUND: { + return (ENOENT); + } + case NECP_ERROR_INVALID_PROCESS: { + return (EPERM); + } + case NECP_ERROR_INTERNAL: + default: { + return (ENOMEM); + } + } +} + static void necp_handle_set_session_priority(struct necp_session *session, u_int32_t message_id, mbuf_t packet, int offset) { @@ -1477,7 +2270,7 @@ necp_handle_set_session_priority(struct necp_session *session, u_int32_t message u_int32_t requested_session_priority = NECP_SESSION_PRIORITY_UNKNOWN; // Read policy id - error = necp_packet_get_tlv(packet, offset, NECP_TLV_SESSION_PRIORITY, sizeof(requested_session_priority), &requested_session_priority, NULL); + error = necp_get_tlv(packet, NULL, 0, offset, NECP_TLV_SESSION_PRIORITY, sizeof(requested_session_priority), &requested_session_priority, NULL); if (error) { NECPLOG(LOG_ERR, "Failed to get session priority: %d", error); response_error = NECP_ERROR_INVALID_TLV; @@ -1550,7 +2343,7 @@ necp_handle_register_service(struct necp_session *session, u_int32_t message_id, } // Read service uuid - error = necp_packet_get_tlv(packet, offset, NECP_TLV_SERVICE_UUID, sizeof(uuid_t), service_uuid, NULL); + error = necp_get_tlv(packet, NULL, 0, offset, NECP_TLV_SERVICE_UUID, sizeof(uuid_t), service_uuid, NULL); if (error) { NECPLOG(LOG_ERR, "Failed to get service UUID: %d", error); response_error = NECP_ERROR_INVALID_TLV; @@ -1595,7 +2388,7 @@ necp_handle_unregister_service(struct necp_session *session, u_int32_t message_i } // Read service uuid - error = necp_packet_get_tlv(packet, offset, NECP_TLV_SERVICE_UUID, sizeof(uuid_t), service_uuid, NULL); + error = necp_get_tlv(packet, NULL, 0, offset, NECP_TLV_SERVICE_UUID, sizeof(uuid_t), service_uuid, NULL); if (error) { NECPLOG(LOG_ERR, "Failed to get service UUID: %d", error); response_error = NECP_ERROR_INVALID_TLV; @@ -1623,8 +2416,9 @@ fail: necp_send_error_response(session, NECP_PACKET_TYPE_UNREGISTER_SERVICE, message_id, response_error); } -static void -necp_handle_policy_add(struct necp_session *session, u_int32_t message_id, mbuf_t packet, int offset) +static necp_policy_id +necp_handle_policy_add(struct necp_session *session, u_int32_t message_id, mbuf_t packet, + u_int8_t *tlv_buffer, size_t tlv_buffer_length, int offset, int *return_error) { bool has_default_condition = FALSE; bool has_non_default_condition = FALSE; @@ -1651,7 +2445,7 @@ necp_handle_policy_add(struct necp_session *session, u_int32_t message_id, mbuf_ u_int32_t policy_result_size = 0; // Read policy order - error = necp_packet_get_tlv(packet, offset, NECP_TLV_POLICY_ORDER, sizeof(order), &order, NULL); + error = necp_get_tlv(packet, tlv_buffer, tlv_buffer_length, offset, NECP_TLV_POLICY_ORDER, sizeof(order), &order, NULL); if (error) { NECPLOG(LOG_ERR, "Failed to get policy order: %d", error); response_error = NECP_ERROR_INVALID_TLV; @@ -1659,8 +2453,8 @@ necp_handle_policy_add(struct necp_session *session, u_int32_t message_id, mbuf_ } // Read policy result - cursor = necp_packet_find_tlv(packet, offset, NECP_TLV_POLICY_RESULT, &error, 0); - error = necp_packet_get_tlv_at_offset(packet, cursor, 0, NULL, &policy_result_size); + cursor = necp_find_tlv(packet, tlv_buffer, tlv_buffer_length, offset, NECP_TLV_POLICY_RESULT, &error, 0); + error = necp_get_tlv_at_offset(packet, tlv_buffer, tlv_buffer_length, cursor, 0, NULL, &policy_result_size); if (error || policy_result_size == 0) { NECPLOG(LOG_ERR, "Failed to get policy result length: %d", error); response_error = NECP_ERROR_INVALID_TLV; @@ -1677,7 +2471,7 @@ necp_handle_policy_add(struct necp_session *session, u_int32_t message_id, mbuf_ response_error = NECP_ERROR_INTERNAL; goto fail; } - error = necp_packet_get_tlv_at_offset(packet, cursor, policy_result_size, policy_result, NULL); + error = necp_get_tlv_at_offset(packet, tlv_buffer, tlv_buffer_length, cursor, policy_result_size, policy_result, NULL); if (error) { NECPLOG(LOG_ERR, "Failed to get policy result: %d", error); response_error = NECP_ERROR_POLICY_RESULT_INVALID; @@ -1691,11 +2485,11 @@ necp_handle_policy_add(struct necp_session *session, u_int32_t message_id, mbuf_ if (necp_policy_result_requires_route_rules(policy_result, policy_result_size)) { // Read route rules conditions - for (cursor = necp_packet_find_tlv(packet, offset, NECP_TLV_ROUTE_RULE, &error, 0); + for (cursor = necp_find_tlv(packet, tlv_buffer, tlv_buffer_length, offset, NECP_TLV_ROUTE_RULE, &error, 0); cursor >= 0; - cursor = necp_packet_find_tlv(packet, cursor, NECP_TLV_ROUTE_RULE, &error, 1)) { + cursor = necp_find_tlv(packet, tlv_buffer, tlv_buffer_length, cursor, NECP_TLV_ROUTE_RULE, &error, 1)) { u_int32_t route_rule_size = 0; - necp_packet_get_tlv_at_offset(packet, cursor, 0, NULL, &route_rule_size); + necp_get_tlv_at_offset(packet, tlv_buffer, tlv_buffer_length, cursor, 0, NULL, &route_rule_size); if (route_rule_size > 0) { route_rules_array_size += (sizeof(u_int8_t) + sizeof(u_int32_t) + route_rule_size); } @@ -1719,12 +2513,12 @@ necp_handle_policy_add(struct necp_session *session, u_int32_t message_id, mbuf_ } route_rules_array_cursor = 0; - for (cursor = necp_packet_find_tlv(packet, offset, NECP_TLV_ROUTE_RULE, &error, 0); + for (cursor = necp_find_tlv(packet, tlv_buffer, tlv_buffer_length, offset, NECP_TLV_ROUTE_RULE, &error, 0); cursor >= 0; - cursor = necp_packet_find_tlv(packet, cursor, NECP_TLV_ROUTE_RULE, &error, 1)) { + cursor = necp_find_tlv(packet, tlv_buffer, tlv_buffer_length, cursor, NECP_TLV_ROUTE_RULE, &error, 1)) { u_int8_t route_rule_type = NECP_TLV_ROUTE_RULE; u_int32_t route_rule_size = 0; - necp_packet_get_tlv_at_offset(packet, cursor, 0, NULL, &route_rule_size); + necp_get_tlv_at_offset(packet, tlv_buffer, tlv_buffer_length, cursor, 0, NULL, &route_rule_size); if (route_rule_size > 0 && route_rule_size <= (route_rules_array_size - route_rules_array_cursor)) { // Add type memcpy((route_rules_array + route_rules_array_cursor), &route_rule_type, sizeof(route_rule_type)); @@ -1735,7 +2529,7 @@ necp_handle_policy_add(struct necp_session *session, u_int32_t message_id, mbuf_ route_rules_array_cursor += sizeof(route_rule_size); // Add value - necp_packet_get_tlv_at_offset(packet, cursor, route_rule_size, (route_rules_array + route_rules_array_cursor), NULL); + necp_get_tlv_at_offset(packet, tlv_buffer, tlv_buffer_length, cursor, route_rule_size, (route_rules_array + route_rules_array_cursor), NULL); if (!necp_policy_route_rule_is_valid((route_rules_array + route_rules_array_cursor), route_rule_size)) { NECPLOG0(LOG_ERR, "Failed to validate policy route rule"); @@ -1758,11 +2552,11 @@ necp_handle_policy_add(struct necp_session *session, u_int32_t message_id, mbuf_ } // Read policy conditions - for (cursor = necp_packet_find_tlv(packet, offset, NECP_TLV_POLICY_CONDITION, &error, 0); + for (cursor = necp_find_tlv(packet, tlv_buffer, tlv_buffer_length, offset, NECP_TLV_POLICY_CONDITION, &error, 0); cursor >= 0; - cursor = necp_packet_find_tlv(packet, cursor, NECP_TLV_POLICY_CONDITION, &error, 1)) { + cursor = necp_find_tlv(packet, tlv_buffer, tlv_buffer_length, cursor, NECP_TLV_POLICY_CONDITION, &error, 1)) { u_int32_t condition_size = 0; - necp_packet_get_tlv_at_offset(packet, cursor, 0, NULL, &condition_size); + necp_get_tlv_at_offset(packet, tlv_buffer, tlv_buffer_length, cursor, 0, NULL, &condition_size); if (condition_size > 0) { conditions_array_size += (sizeof(u_int8_t) + sizeof(u_int32_t) + condition_size); @@ -1787,12 +2581,12 @@ necp_handle_policy_add(struct necp_session *session, u_int32_t message_id, mbuf_ } conditions_array_cursor = 0; - for (cursor = necp_packet_find_tlv(packet, offset, NECP_TLV_POLICY_CONDITION, &error, 0); + for (cursor = necp_find_tlv(packet, tlv_buffer, tlv_buffer_length, offset, NECP_TLV_POLICY_CONDITION, &error, 0); cursor >= 0; - cursor = necp_packet_find_tlv(packet, cursor, NECP_TLV_POLICY_CONDITION, &error, 1)) { + cursor = necp_find_tlv(packet, tlv_buffer, tlv_buffer_length, cursor, NECP_TLV_POLICY_CONDITION, &error, 1)) { u_int8_t condition_type = NECP_TLV_POLICY_CONDITION; u_int32_t condition_size = 0; - necp_packet_get_tlv_at_offset(packet, cursor, 0, NULL, &condition_size); + necp_get_tlv_at_offset(packet, tlv_buffer, tlv_buffer_length, cursor, 0, NULL, &condition_size); if (condition_size > 0 && condition_size <= (conditions_array_size - conditions_array_cursor)) { // Add type memcpy((conditions_array + conditions_array_cursor), &condition_type, sizeof(condition_type)); @@ -1803,7 +2597,7 @@ necp_handle_policy_add(struct necp_session *session, u_int32_t message_id, mbuf_ conditions_array_cursor += sizeof(condition_size); // Add value - necp_packet_get_tlv_at_offset(packet, cursor, condition_size, (conditions_array + conditions_array_cursor), NULL); + necp_get_tlv_at_offset(packet, tlv_buffer, tlv_buffer_length, cursor, condition_size, (conditions_array + conditions_array_cursor), NULL); if (!necp_policy_condition_is_valid((conditions_array + conditions_array_cursor), condition_size, necp_policy_result_get_type_from_buffer(policy_result, policy_result_size))) { NECPLOG0(LOG_ERR, "Failed to validate policy condition"); response_error = NECP_ERROR_POLICY_CONDITIONS_INVALID; @@ -1858,8 +2652,10 @@ necp_handle_policy_add(struct necp_session *session, u_int32_t message_id, mbuf_ goto fail; } - necp_send_policy_id_response(session, NECP_PACKET_TYPE_POLICY_ADD, message_id, policy->id); - return; + if (packet != NULL) { + necp_send_policy_id_response(session, NECP_PACKET_TYPE_POLICY_ADD, message_id, policy->id); + } + return (policy->id); fail: if (policy_result != NULL) { @@ -1872,7 +2668,13 @@ fail: FREE(route_rules_array, M_NECP); } - necp_send_error_response(session, NECP_PACKET_TYPE_POLICY_ADD, message_id, response_error); + if (packet != NULL) { + necp_send_error_response(session, NECP_PACKET_TYPE_POLICY_ADD, message_id, response_error); + } + if (return_error != NULL) { + *return_error = necp_get_posix_error_for_necp_error(response_error); + } + return (0); } static void @@ -1891,7 +2693,7 @@ necp_handle_policy_get(struct necp_session *session, u_int32_t message_id, mbuf_ struct necp_session_policy *policy = NULL; // Read policy id - error = necp_packet_get_tlv(packet, offset, NECP_TLV_POLICY_ID, sizeof(policy_id), &policy_id, NULL); + error = necp_get_tlv(packet, NULL, 0, offset, NECP_TLV_POLICY_ID, sizeof(policy_id), &policy_id, NULL); if (error) { NECPLOG(LOG_ERR, "Failed to get policy id: %d", error); response_error = NECP_ERROR_INVALID_TLV; @@ -1910,16 +2712,16 @@ necp_handle_policy_get(struct necp_session *session, u_int32_t message_id, mbuf_ response_size = sizeof(struct necp_packet_header) + order_tlv_size + result_tlv_size + policy->conditions_size; MALLOC(response, u_int8_t *, response_size, M_NECP, M_WAITOK); if (response == NULL) { - necp_send_error_response(session, NECP_PACKET_TYPE_POLICY_LIST_ALL, message_id, NECP_ERROR_INTERNAL); + necp_send_error_response(session, NECP_PACKET_TYPE_POLICY_GET, message_id, NECP_ERROR_INTERNAL); return; } cursor = response; cursor = necp_buffer_write_packet_header(cursor, NECP_PACKET_TYPE_POLICY_GET, NECP_PACKET_FLAGS_RESPONSE, message_id); - cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_ORDER, sizeof(necp_policy_order), &policy->order); + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_ORDER, sizeof(necp_policy_order), &policy->order, response, response_size); if (result_tlv_size) { - cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_RESULT, policy->result_size, &policy->result); + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_RESULT, policy->result_size, &policy->result, response, response_size); } if (policy->conditions_size) { memcpy(((u_int8_t *)(void *)(cursor)), policy->conditions, policy->conditions_size); @@ -1946,7 +2748,7 @@ necp_handle_policy_delete(struct necp_session *session, u_int32_t message_id, mb struct necp_session_policy *policy = NULL; // Read policy id - error = necp_packet_get_tlv(packet, offset, NECP_TLV_POLICY_ID, sizeof(policy_id), &policy_id, NULL); + error = necp_get_tlv(packet, NULL, 0, offset, NECP_TLV_POLICY_ID, sizeof(policy_id), &policy_id, NULL); if (error) { NECPLOG(LOG_ERR, "Failed to get policy id: %d", error); response_error = NECP_ERROR_INVALID_TLV; @@ -2008,7 +2810,7 @@ necp_handle_policy_list_all(struct necp_session *session, u_int32_t message_id, LIST_FOREACH(policy, &session->policies, chain) { if (!policy->pending_deletion && cur_policy_index < num_policies) { - cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_ID, sizeof(u_int32_t), &policy->id); + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_ID, sizeof(u_int32_t), &policy->id, response, response_size); cur_policy_index++; } } @@ -2147,21 +2949,23 @@ necp_policy_get_new_id(void) * } * ... */ -static void -necp_handle_policy_dump_all(struct necp_session *session, u_int32_t message_id, mbuf_t packet, int offset) +static int +necp_handle_policy_dump_all(struct necp_session *session, u_int32_t message_id, mbuf_t packet, + user_addr_t out_buffer, size_t out_buffer_length, int offset) { -#pragma unused(packet, offset) +#pragma unused(offset) struct necp_kernel_socket_policy *policy = NULL; int policy_i; int policy_count = 0; u_int8_t **tlv_buffer_pointers = NULL; u_int32_t *tlv_buffer_lengths = NULL; - int total_tlv_len = 0; + u_int32_t total_tlv_len = 0; u_int8_t *result_buf = NULL; u_int8_t *result_buf_cursor = result_buf; char result_string[MAX_RESULT_STRING_LEN]; char proc_name_string[MAXCOMLEN + 1]; + int error_code = 0; bool error_occured = false; u_int32_t response_error = NECP_ERROR_INTERNAL; @@ -2181,7 +2985,9 @@ necp_handle_policy_dump_all(struct necp_session *session, u_int32_t message_id, // LOCK lck_rw_lock_shared(&necp_kernel_policy_lock); - NECPLOG0(LOG_DEBUG, "Gathering policies"); + if (necp_debug) { + NECPLOG0(LOG_DEBUG, "Gathering policies"); + } policy_count = necp_kernel_application_policies_count; @@ -2209,7 +3015,9 @@ necp_handle_policy_dump_all(struct necp_session *session, u_int32_t message_id, u_int16_t proc_name_len = strlen(proc_name_string) + 1; u_int16_t result_string_len = strlen(result_string) + 1; - NECPLOG(LOG_DEBUG, "Policy: process: %s, result: %s", proc_name_string, result_string); + if (necp_debug) { + NECPLOG(LOG_DEBUG, "Policy: process: %s, result: %s", proc_name_string, result_string); + } u_int32_t total_allocated_bytes = sizeof(u_int8_t) + sizeof(u_int32_t) + sizeof(policy->id) + // NECP_TLV_POLICY_ID sizeof(u_int8_t) + sizeof(u_int32_t) + sizeof(policy->order) + // NECP_TLV_POLICY_ORDER @@ -2312,11 +3120,11 @@ necp_handle_policy_dump_all(struct necp_session *session, u_int32_t message_id, } u_int8_t *cursor = tlv_buffer; - cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_ID, sizeof(policy->id), &policy->id); - cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_ORDER, sizeof(necp_policy_order), &policy->order); - cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_SESSION_ORDER, sizeof(policy->session_order), &policy->session_order); - cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_RESULT_STRING, result_string_len , result_string); - cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_OWNER, proc_name_len , proc_name_string); + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_ID, sizeof(policy->id), &policy->id, tlv_buffer, total_allocated_bytes); + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_ORDER, sizeof(necp_policy_order), &policy->order, tlv_buffer, total_allocated_bytes); + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_SESSION_ORDER, sizeof(policy->session_order), &policy->session_order, tlv_buffer, total_allocated_bytes); + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_RESULT_STRING, result_string_len, result_string, tlv_buffer, total_allocated_bytes); + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_OWNER, proc_name_len, proc_name_string, tlv_buffer, total_allocated_bytes); #define N_QUICK 256 u_int8_t q_cond_buf[N_QUICK]; // Minor optimization @@ -2336,63 +3144,76 @@ necp_handle_policy_dump_all(struct necp_session *session, u_int32_t message_id, memset(cond_buf, 0, condition_tlv_length); u_int8_t *cond_buf_cursor = cond_buf; if (condition_mask == NECP_POLICY_CONDITION_DEFAULT) { - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_DEFAULT, 0, ""); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_DEFAULT, 0, "", cond_buf, condition_tlv_length); } else { if (condition_mask & NECP_KERNEL_CONDITION_ALL_INTERFACES) { - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_ALL_INTERFACES, 0, ""); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_ALL_INTERFACES, 0, "", cond_buf, condition_tlv_length); } if (condition_mask & NECP_KERNEL_CONDITION_BOUND_INTERFACE) { - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_BOUND_INTERFACE, strlen(if_name) + 1, if_name); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_BOUND_INTERFACE, strlen(if_name) + 1, + if_name, cond_buf, condition_tlv_length); } if (condition_mask & NECP_KERNEL_CONDITION_PROTOCOL) { - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_IP_PROTOCOL, sizeof(policy->cond_protocol), &policy->cond_protocol); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_IP_PROTOCOL, sizeof(policy->cond_protocol), &policy->cond_protocol, + cond_buf, condition_tlv_length); } if (condition_mask & NECP_KERNEL_CONDITION_APP_ID) { struct necp_uuid_id_mapping *entry = necp_uuid_lookup_uuid_with_app_id_locked(policy->cond_app_id); if (entry != NULL) { - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_APPLICATION, sizeof(entry->uuid), entry->uuid); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_APPLICATION, sizeof(entry->uuid), entry->uuid, + cond_buf, condition_tlv_length); } } if (condition_mask & NECP_KERNEL_CONDITION_REAL_APP_ID) { struct necp_uuid_id_mapping *entry = necp_uuid_lookup_uuid_with_app_id_locked(policy->cond_real_app_id); if (entry != NULL) { - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_REAL_APPLICATION, sizeof(entry->uuid), entry->uuid); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_REAL_APPLICATION, sizeof(entry->uuid), entry->uuid, + cond_buf, condition_tlv_length); } } if (condition_mask & NECP_KERNEL_CONDITION_DOMAIN) { - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_DOMAIN, strlen(policy->cond_domain) + 1, policy->cond_domain); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_DOMAIN, strlen(policy->cond_domain) + 1, policy->cond_domain, + cond_buf, condition_tlv_length); } if (condition_mask & NECP_KERNEL_CONDITION_ACCOUNT_ID) { if (account_id_entry != NULL) { - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_ACCOUNT, strlen(account_id_entry->string) + 1, account_id_entry->string); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_ACCOUNT, strlen(account_id_entry->string) + 1, account_id_entry->string, + cond_buf, condition_tlv_length); } } if (condition_mask & NECP_KERNEL_CONDITION_PID) { - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_PID, sizeof(policy->cond_pid), &policy->cond_pid); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_PID, sizeof(policy->cond_pid), &policy->cond_pid, + cond_buf, condition_tlv_length); } if (condition_mask & NECP_KERNEL_CONDITION_UID) { - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_UID, sizeof(policy->cond_uid), &policy->cond_uid); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_UID, sizeof(policy->cond_uid), &policy->cond_uid, + cond_buf, condition_tlv_length); } if (condition_mask & NECP_KERNEL_CONDITION_TRAFFIC_CLASS) { - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_TRAFFIC_CLASS, sizeof(policy->cond_traffic_class), &policy->cond_traffic_class); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_TRAFFIC_CLASS, sizeof(policy->cond_traffic_class), &policy->cond_traffic_class, + cond_buf, condition_tlv_length); } if (condition_mask & NECP_KERNEL_CONDITION_ENTITLEMENT) { - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_ENTITLEMENT, 0, ""); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_ENTITLEMENT, 0, "", + cond_buf, condition_tlv_length); } if (condition_mask & NECP_KERNEL_CONDITION_CUSTOM_ENTITLEMENT) { - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_ENTITLEMENT, strlen(policy->cond_custom_entitlement) + 1, policy->cond_custom_entitlement); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_ENTITLEMENT, strlen(policy->cond_custom_entitlement) + 1, policy->cond_custom_entitlement, + cond_buf, condition_tlv_length); } if (condition_mask & NECP_KERNEL_CONDITION_LOCAL_START) { if (condition_mask & NECP_KERNEL_CONDITION_LOCAL_END) { struct necp_policy_condition_addr_range range; memcpy(&range.start_address, &policy->cond_local_start, sizeof(policy->cond_local_start)); memcpy(&range.end_address, &policy->cond_local_end, sizeof(policy->cond_local_end)); - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_LOCAL_ADDR_RANGE, sizeof(range), &range); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_LOCAL_ADDR_RANGE, sizeof(range), &range, + cond_buf, condition_tlv_length); } else { struct necp_policy_condition_addr addr; addr.prefix = policy->cond_local_prefix; memcpy(&addr.address, &policy->cond_local_start, sizeof(policy->cond_local_start)); - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_LOCAL_ADDR, sizeof(addr), &addr); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_LOCAL_ADDR, sizeof(addr), &addr, + cond_buf, condition_tlv_length); } } if (condition_mask & NECP_KERNEL_CONDITION_REMOTE_START) { @@ -2400,17 +3221,19 @@ necp_handle_policy_dump_all(struct necp_session *session, u_int32_t message_id, struct necp_policy_condition_addr_range range; memcpy(&range.start_address, &policy->cond_remote_start, sizeof(policy->cond_remote_start)); memcpy(&range.end_address, &policy->cond_remote_end, sizeof(policy->cond_remote_end)); - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_REMOTE_ADDR_RANGE, sizeof(range), &range); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_REMOTE_ADDR_RANGE, sizeof(range), &range, + cond_buf, condition_tlv_length); } else { struct necp_policy_condition_addr addr; addr.prefix = policy->cond_remote_prefix; memcpy(&addr.address, &policy->cond_remote_start, sizeof(policy->cond_remote_start)); - cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_REMOTE_ADDR, sizeof(addr), &addr); + cond_buf_cursor = necp_buffer_write_tlv(cond_buf_cursor, NECP_POLICY_CONDITION_REMOTE_ADDR, sizeof(addr), &addr, + cond_buf, condition_tlv_length); } } } - cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_CONDITION, cond_buf_cursor - cond_buf, cond_buf); + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_POLICY_CONDITION, cond_buf_cursor - cond_buf, cond_buf, tlv_buffer, total_allocated_bytes); if (cond_buf != q_cond_buf) { FREE(cond_buf, M_NECP); } @@ -2425,36 +3248,77 @@ necp_handle_policy_dump_all(struct necp_session *session, u_int32_t message_id, // UNLOCK lck_rw_done(&necp_kernel_policy_lock); - u_int32_t total_result_length = sizeof(struct necp_packet_header) + total_tlv_len; - MALLOC(result_buf, u_int8_t *, total_result_length, M_NECP, M_NOWAIT | M_ZERO); - if (result_buf == NULL) { - NECPLOG(LOG_DEBUG, "Failed to allocate result_buffer (%u bytes)", total_result_length); - REPORT_ERROR(NECP_ERROR_INTERNAL); - } + // Send packet + if (packet != NULL) { + u_int32_t total_result_length = sizeof(struct necp_packet_header) + total_tlv_len; + + // Allow malloc to wait, since the total buffer may be large and we are not holding any locks + MALLOC(result_buf, u_int8_t *, total_result_length, M_NECP, M_WAITOK | M_ZERO); + if (result_buf == NULL) { + NECPLOG(LOG_DEBUG, "Failed to allocate result_buffer (%u bytes)", total_result_length); + REPORT_ERROR(NECP_ERROR_INTERNAL); + } - result_buf_cursor = result_buf; - result_buf_cursor = necp_buffer_write_packet_header(result_buf_cursor, NECP_PACKET_TYPE_POLICY_DUMP_ALL, NECP_PACKET_FLAGS_RESPONSE, message_id); + result_buf_cursor = result_buf; + result_buf_cursor = necp_buffer_write_packet_header(result_buf_cursor, NECP_PACKET_TYPE_POLICY_DUMP_ALL, NECP_PACKET_FLAGS_RESPONSE, message_id); + + for (int i = 0; i < policy_count; i++) { + if (tlv_buffer_pointers[i] != NULL) { + result_buf_cursor = necp_buffer_write_tlv(result_buf_cursor, NECP_TLV_POLICY_DUMP, tlv_buffer_lengths[i], tlv_buffer_pointers[i], result_buf, total_result_length); + } + } - for (int i = 0; i < policy_count; i++) { - if (tlv_buffer_pointers[i] != NULL) { - result_buf_cursor = necp_buffer_write_tlv(result_buf_cursor, NECP_TLV_POLICY_DUMP, tlv_buffer_lengths[i], tlv_buffer_pointers[i]); + if (!necp_send_ctl_data(session, result_buf, result_buf_cursor - result_buf)) { + NECPLOG(LOG_ERR, "Failed to send response (%u bytes)", result_buf_cursor - result_buf); + } else { + NECPLOG(LOG_ERR, "Sent data worth %u bytes. Total result buffer length was %u bytes", result_buf_cursor - result_buf, total_result_length); } } - if (!necp_send_ctl_data(session, result_buf, result_buf_cursor - result_buf)) { - NECPLOG(LOG_ERR, "Failed to send response (%u bytes)", result_buf_cursor - result_buf); - } else { - NECPLOG(LOG_ERR, "Sent data worth %u bytes. Total result buffer length was %u bytes", result_buf_cursor - result_buf, total_result_length); + // Copy out + if (out_buffer != 0) { + if (out_buffer_length < total_tlv_len + sizeof(u_int32_t)) { + NECPLOG(LOG_DEBUG, "out_buffer_length too small (%u < %u)", out_buffer_length, total_tlv_len + sizeof(u_int32_t)); + REPORT_ERROR(NECP_ERROR_INVALID_TLV); + } + + // Allow malloc to wait, since the total buffer may be large and we are not holding any locks + MALLOC(result_buf, u_int8_t *, total_tlv_len + sizeof(u_int32_t), M_NECP, M_WAITOK | M_ZERO); + if (result_buf == NULL) { + NECPLOG(LOG_DEBUG, "Failed to allocate result_buffer (%u bytes)", total_tlv_len + sizeof(u_int32_t)); + REPORT_ERROR(NECP_ERROR_INTERNAL); + } + + // Add four bytes for total length at the start + memcpy(result_buf, &total_tlv_len, sizeof(u_int32_t)); + + // Copy the TLVs + result_buf_cursor = result_buf + sizeof(u_int32_t); + for (int i = 0; i < policy_count; i++) { + if (tlv_buffer_pointers[i] != NULL) { + result_buf_cursor = necp_buffer_write_tlv(result_buf_cursor, NECP_TLV_POLICY_DUMP, tlv_buffer_lengths[i], tlv_buffer_pointers[i], + result_buf, total_tlv_len + sizeof(u_int32_t)); + } + } + + int copy_error = copyout(result_buf, out_buffer, total_tlv_len + sizeof(u_int32_t)); + if (copy_error) { + NECPLOG(LOG_DEBUG, "Failed to copy out result_buffer (%u bytes)", total_tlv_len + sizeof(u_int32_t)); + REPORT_ERROR(NECP_ERROR_INTERNAL); + } } done: if (error_occured) { - if(!necp_send_error_response(session, NECP_PACKET_TYPE_POLICY_DUMP_ALL, message_id, response_error)) { - NECPLOG0(LOG_ERR, "Failed to send error response"); - } else { - NECPLOG0(LOG_ERR, "Sent error response"); + if (packet != NULL) { + if(!necp_send_error_response(session, NECP_PACKET_TYPE_POLICY_DUMP_ALL, message_id, response_error)) { + NECPLOG0(LOG_ERR, "Failed to send error response"); + } else { + NECPLOG0(LOG_ERR, "Sent error response"); + } } + error_code = necp_get_posix_error_for_necp_error(response_error); } if (result_buf != NULL) { @@ -2478,6 +3342,8 @@ done: #undef RESET_COND_BUF #undef REPORT_ERROR #undef UNLOCK_AND_REPORT_ERROR + + return (error_code); } static struct necp_session_policy * @@ -2495,7 +3361,7 @@ necp_policy_create(struct necp_session *session, necp_policy_order order, u_int8 goto done; } - memset(new_policy, 0, sizeof(*new_policy)); + memset(new_policy, 0, sizeof(*new_policy)); // M_ZERO is not supported for MALLOC_ZONE new_policy->applied = FALSE; new_policy->pending_deletion = FALSE; new_policy->pending_update = FALSE; @@ -2634,7 +3500,7 @@ necp_policy_unapply(struct necp_session_policy *policy) return (FALSE); } - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); // Release local uuid mappings if (!uuid_is_null(policy->applied_app_uuid)) { @@ -2749,7 +3615,7 @@ necp_policy_apply(struct necp_session *session, struct necp_session_policy *poli return (FALSE); } - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); // Process conditions while (offset < policy->conditions_size) { @@ -3268,18 +4134,30 @@ necp_policy_apply_all(struct necp_session *session) // --------------------- // Kernel policies are derived from session policies static necp_kernel_policy_id -necp_kernel_policy_get_new_id(void) +necp_kernel_policy_get_new_id(bool socket_level) { + static necp_kernel_policy_id necp_last_kernel_socket_policy_id = 0; + static necp_kernel_policy_id necp_last_kernel_ip_policy_id = 0; + necp_kernel_policy_id newid = NECP_KERNEL_POLICY_ID_NONE; - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); - necp_last_kernel_policy_id++; - if (necp_last_kernel_policy_id < NECP_KERNEL_POLICY_ID_FIRST_VALID) { - necp_last_kernel_policy_id = NECP_KERNEL_POLICY_ID_FIRST_VALID; + if (socket_level) { + necp_last_kernel_socket_policy_id++; + if (necp_last_kernel_socket_policy_id < NECP_KERNEL_POLICY_ID_FIRST_VALID_SOCKET || + necp_last_kernel_socket_policy_id >= NECP_KERNEL_POLICY_ID_FIRST_VALID_IP) { + necp_last_kernel_socket_policy_id = NECP_KERNEL_POLICY_ID_FIRST_VALID_SOCKET; + } + newid = necp_last_kernel_socket_policy_id; + } else { + necp_last_kernel_ip_policy_id++; + if (necp_last_kernel_ip_policy_id < NECP_KERNEL_POLICY_ID_FIRST_VALID_IP) { + necp_last_kernel_ip_policy_id = NECP_KERNEL_POLICY_ID_FIRST_VALID_IP; + } + newid = necp_last_kernel_ip_policy_id; } - newid = necp_last_kernel_policy_id; if (newid == NECP_KERNEL_POLICY_ID_NONE) { NECPLOG0(LOG_DEBUG, "Allocate kernel policy id failed.\n"); return (0); @@ -3300,9 +4178,9 @@ necp_kernel_socket_policy_add(necp_policy_id parent_policy_id, necp_policy_order goto done; } - memset(new_kernel_policy, 0, sizeof(*new_kernel_policy)); + memset(new_kernel_policy, 0, sizeof(*new_kernel_policy)); // M_ZERO is not supported for MALLOC_ZONE new_kernel_policy->parent_policy_id = parent_policy_id; - new_kernel_policy->id = necp_kernel_policy_get_new_id(); + new_kernel_policy->id = necp_kernel_policy_get_new_id(true); new_kernel_policy->order = order; new_kernel_policy->session_order = session_order; new_kernel_policy->session_pid = session_pid; @@ -3416,7 +4294,7 @@ necp_kernel_socket_policy_delete(necp_kernel_policy_id policy_id) { struct necp_kernel_socket_policy *policy = NULL; - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); policy = necp_kernel_socket_policy_find(policy_id); if (policy) { @@ -3881,7 +4759,7 @@ necp_kernel_socket_policies_reprocess(void) int app_layer_current_free_index = 0; struct necp_kernel_socket_policy *kernel_policy = NULL; - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); // Reset mask to 0 necp_kernel_application_policies_condition_mask = 0; @@ -4003,7 +4881,7 @@ necp_get_new_string_id(void) { u_int32_t newid = 0; - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); necp_last_string_id++; if (necp_last_string_id < 1) { @@ -4057,7 +4935,7 @@ necp_create_string_to_id_mapping(struct necp_string_id_mapping_list *list, char u_int32_t string_id = 0; struct necp_string_id_mapping *existing_mapping = NULL; - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); existing_mapping = necp_lookup_string_to_id_locked(list, string); if (existing_mapping != NULL) { @@ -4091,7 +4969,7 @@ necp_remove_string_to_id_mapping(struct necp_string_id_mapping_list *list, char { struct necp_string_id_mapping *existing_mapping = NULL; - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); existing_mapping = necp_lookup_string_to_id_locked(list, string); if (existing_mapping != NULL) { @@ -4111,7 +4989,7 @@ necp_get_new_route_rule_id(void) { u_int32_t newid = 0; - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); necp_last_route_rule_id++; if (necp_last_route_rule_id < 1 || necp_last_route_rule_id > UINT16_MAX) { @@ -4132,7 +5010,7 @@ necp_get_new_aggregate_route_rule_id(void) { u_int32_t newid = 0; - lck_rw_assert(&necp_route_rule_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_route_rule_lock, LCK_RW_ASSERT_EXCLUSIVE); necp_last_aggregate_route_rule_id++; if (necp_last_aggregate_route_rule_id <= UINT16_MAX) { @@ -4232,7 +5110,7 @@ necp_create_route_rule(struct necp_route_rule_list *list, u_int8_t *route_rules_ u_int8_t if_actions[MAX_ROUTE_RULE_INTERFACES]; memset(&if_actions, 0, sizeof(if_actions)); - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); if (route_rules_array == NULL || route_rules_array_size == 0) { return (0); @@ -4346,7 +5224,7 @@ necp_remove_route_rule(struct necp_route_rule_list *list, u_int32_t route_rule_i { struct necp_route_rule *existing_rule = NULL; - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); existing_rule = necp_lookup_route_rule_locked(list, route_rule_id); if (existing_rule != NULL) { @@ -4423,7 +5301,7 @@ necp_get_new_uuid_id(void) { u_int32_t newid = 0; - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); necp_last_uuid_id++; if (necp_last_uuid_id < (NECP_NULL_SERVICE_ID + 1)) { @@ -4480,7 +5358,7 @@ necp_create_uuid_app_id_mapping(uuid_t uuid, bool *allocated_mapping, bool uuid_ u_int32_t local_id = 0; struct necp_uuid_id_mapping *existing_mapping = NULL; - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); if (allocated_mapping) { *allocated_mapping = FALSE; @@ -4524,7 +5402,7 @@ necp_remove_uuid_app_id_mapping(uuid_t uuid, bool *removed_mapping, bool uuid_po { struct necp_uuid_id_mapping *existing_mapping = NULL; - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); if (removed_mapping) { *removed_mapping = FALSE; @@ -4608,7 +5486,7 @@ necp_create_uuid_service_id_mapping(uuid_t uuid) return (NECP_NULL_SERVICE_ID); } - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); existing_mapping = necp_uuid_lookup_service_id_locked(uuid); if (existing_mapping != NULL) { @@ -4640,7 +5518,7 @@ necp_remove_uuid_service_id_mapping(uuid_t uuid) return (TRUE); } - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); existing_mapping = necp_uuid_lookup_app_id_locked(uuid); if (existing_mapping != NULL) { @@ -4658,7 +5536,7 @@ necp_remove_uuid_service_id_mapping(uuid_t uuid) static bool necp_kernel_socket_policies_update_uuid_table(void) { - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); if (necp_uuid_app_id_mappings_dirty) { if (proc_uuid_policy_kernel(PROC_UUID_POLICY_OPERATION_CLEAR, NULL, PROC_UUID_NECP_APP_POLICY) < 0) { @@ -4697,9 +5575,9 @@ necp_kernel_ip_output_policy_add(necp_policy_id parent_policy_id, necp_policy_or goto done; } - memset(new_kernel_policy, 0, sizeof(*new_kernel_policy)); + memset(new_kernel_policy, 0, sizeof(*new_kernel_policy)); // M_ZERO is not supported for MALLOC_ZONE new_kernel_policy->parent_policy_id = parent_policy_id; - new_kernel_policy->id = necp_kernel_policy_get_new_id(); + new_kernel_policy->id = necp_kernel_policy_get_new_id(false); new_kernel_policy->suborder = suborder; new_kernel_policy->order = order; new_kernel_policy->session_order = session_order; @@ -4788,7 +5666,7 @@ necp_kernel_ip_output_policy_delete(necp_kernel_policy_id policy_id) { struct necp_kernel_ip_output_policy *policy = NULL; - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); policy = necp_kernel_ip_output_policy_find(policy_id); if (policy) { @@ -4971,7 +5849,7 @@ necp_kernel_ip_output_policies_reprocess(void) int bucket_current_free_index[NECP_KERNEL_IP_OUTPUT_POLICIES_MAP_NUM_ID_BUCKETS]; struct necp_kernel_ip_output_policy *kernel_policy = NULL; - lck_rw_assert(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&necp_kernel_policy_lock, LCK_RW_ASSERT_EXCLUSIVE); // Reset mask to 0 necp_kernel_ip_output_policies_condition_mask = 0; @@ -5261,7 +6139,10 @@ necp_application_find_policy_match_internal(proc_t proc, u_int32_t parameters_size, struct necp_aggregate_result *returned_result, u_int32_t *flags, - u_int required_interface_index) + u_int required_interface_index, + const union necp_sockaddr_union *override_local_addr, + const union necp_sockaddr_union *override_remote_addr, + struct rtentry **returned_route, bool ignore_address) { int error = 0; size_t offset = 0; @@ -5276,14 +6157,23 @@ necp_application_find_policy_match_internal(proc_t proc, u_int16_t protocol = 0; u_int32_t bound_interface_index = required_interface_index; u_int32_t traffic_class = 0; + u_int32_t client_flags = 0; union necp_sockaddr_union local_addr; union necp_sockaddr_union remote_addr; bool no_remote_addr = FALSE; u_int8_t remote_family = 0; bool no_local_addr = FALSE; - memset(&local_addr, 0, sizeof(local_addr)); - memset(&remote_addr, 0, sizeof(remote_addr)); + if (override_local_addr) { + memcpy(&local_addr, override_local_addr, sizeof(local_addr)); + } else { + memset(&local_addr, 0, sizeof(local_addr)); + } + if (override_remote_addr) { + memcpy(&remote_addr, override_remote_addr, sizeof(remote_addr)); + } else { + memset(&remote_addr, 0, sizeof(remote_addr)); + } // Initialize UID, PID, and UUIDs to the current process uid_t uid = kauth_cred_getuid(proc_ucred(proc)); @@ -5423,6 +6313,10 @@ necp_application_find_policy_match_internal(proc_t proc, break; } case NECP_CLIENT_PARAMETER_LOCAL_ADDRESS: { + if (ignore_address) { + break; + } + if (length >= sizeof(struct necp_policy_condition_addr)) { struct necp_policy_condition_addr *address_struct = (struct necp_policy_condition_addr *)(void *)value; if (necp_address_is_valid(&address_struct->address.sa)) { @@ -5432,6 +6326,10 @@ necp_application_find_policy_match_internal(proc_t proc, break; } case NECP_CLIENT_PARAMETER_REMOTE_ADDRESS: { + if (ignore_address) { + break; + } + if (length >= sizeof(struct necp_policy_condition_addr)) { struct necp_policy_condition_addr *address_struct = (struct necp_policy_condition_addr *)(void *)value; if (necp_address_is_valid(&address_struct->address.sa)) { @@ -5440,6 +6338,11 @@ necp_application_find_policy_match_internal(proc_t proc, } break; } + case NECP_CLIENT_PARAMETER_FLAGS: { + if (length >= sizeof(client_flags)) { + memcpy(&client_flags, value, sizeof(client_flags)); + } + } default: { break; } @@ -5459,6 +6362,10 @@ necp_application_find_policy_match_internal(proc_t proc, returned_result->policy_id = matched_policy->id; returned_result->routing_result = matched_policy->result; memcpy(&returned_result->routing_result_parameter, &matched_policy->result_parameter, sizeof(returned_result->routing_result_parameter)); + } else if (necp_drop_all_order > 0) { + // Mark socket as a drop if drop_all is set + returned_result->policy_id = NECP_KERNEL_POLICY_ID_NO_MATCH; + returned_result->routing_result = NECP_KERNEL_POLICY_RESULT_DROP; } else { returned_result->policy_id = 0; returned_result->routing_result = NECP_KERNEL_POLICY_RESULT_NONE; @@ -5522,70 +6429,76 @@ necp_application_find_policy_match_internal(proc_t proc, remote_family = remote_addr.sa.sa_family; } - if (no_remote_addr) { - memset(&remote_addr, 0, sizeof(remote_addr)); - if (remote_family == AF_INET6) { - // Reset address to :: - remote_addr.sa.sa_family = AF_INET6; - remote_addr.sa.sa_len = sizeof(struct sockaddr_in6); - } else { - // Reset address to 0.0.0.0 - remote_addr.sa.sa_family = AF_INET; - remote_addr.sa.sa_len = sizeof(struct sockaddr_in); - } - } - + returned_result->routed_interface_index = 0; struct rtentry *rt = NULL; - rt = rtalloc1_scoped((struct sockaddr *)&remote_addr, 0, 0, - output_bound_interface); + if (!no_local_addr && (client_flags & NECP_CLIENT_PARAMETER_FLAG_LISTENER) != 0) { + // Treat the output bound interface as the routed interface for local address + // validation later. + returned_result->routed_interface_index = output_bound_interface; + } else { + if (no_remote_addr) { + memset(&remote_addr, 0, sizeof(remote_addr)); + if (remote_family == AF_INET6) { + // Reset address to :: + remote_addr.sa.sa_family = AF_INET6; + remote_addr.sa.sa_len = sizeof(struct sockaddr_in6); + } else { + // Reset address to 0.0.0.0 + remote_addr.sa.sa_family = AF_INET; + remote_addr.sa.sa_len = sizeof(struct sockaddr_in); + } + } - if (no_remote_addr && remote_family == 0 && - (rt == NULL || rt->rt_ifp == NULL)) { - // Route lookup for default IPv4 failed, try IPv6 + rt = rtalloc1_scoped((struct sockaddr *)&remote_addr, 0, 0, + output_bound_interface); - // Cleanup old route if necessary - if (rt != NULL) { - rtfree(rt); - rt = NULL; - } + if (no_remote_addr && remote_family == 0 && + (rt == NULL || rt->rt_ifp == NULL)) { + // Route lookup for default IPv4 failed, try IPv6 - // Reset address to :: - memset(&remote_addr, 0, sizeof(remote_addr)); - remote_addr.sa.sa_family = AF_INET6; - remote_addr.sa.sa_len = sizeof(struct sockaddr_in6); + // Cleanup old route if necessary + if (rt != NULL) { + rtfree(rt); + rt = NULL; + } - // Get route - rt = rtalloc1_scoped((struct sockaddr *)&remote_addr, 0, 0, - output_bound_interface); - } + // Reset address to :: + memset(&remote_addr, 0, sizeof(remote_addr)); + remote_addr.sa.sa_family = AF_INET6; + remote_addr.sa.sa_len = sizeof(struct sockaddr_in6); - returned_result->routed_interface_index = 0; - if (rt != NULL && - rt->rt_ifp != NULL) { - returned_result->routed_interface_index = rt->rt_ifp->if_index; - /* - * For local addresses, we allow the interface scope to be - * either the loopback interface or the interface hosting the - * local address. - */ - if (bound_interface_index != IFSCOPE_NONE && - rt->rt_ifa != NULL && rt->rt_ifa->ifa_ifp && - (output_bound_interface == lo_ifp->if_index || - rt->rt_ifp->if_index == lo_ifp->if_index || - rt->rt_ifa->ifa_ifp->if_index == bound_interface_index)) { - struct sockaddr_storage dst; - unsigned int ifscope = bound_interface_index; + // Get route + rt = rtalloc1_scoped((struct sockaddr *)&remote_addr, 0, 0, + output_bound_interface); + } + if (rt != NULL && + rt->rt_ifp != NULL) { + returned_result->routed_interface_index = rt->rt_ifp->if_index; /* - * Transform dst into the internal routing table form + * For local addresses, we allow the interface scope to be + * either the loopback interface or the interface hosting the + * local address. */ - (void) sa_copy((struct sockaddr *)&remote_addr, - &dst, &ifscope); + if (bound_interface_index != IFSCOPE_NONE && + rt->rt_ifa != NULL && rt->rt_ifa->ifa_ifp && + (output_bound_interface == lo_ifp->if_index || + rt->rt_ifp->if_index == lo_ifp->if_index || + rt->rt_ifa->ifa_ifp->if_index == bound_interface_index)) { + struct sockaddr_storage dst; + unsigned int ifscope = bound_interface_index; - if ((rt->rt_ifp->if_index == lo_ifp->if_index) || - rt_ifa_is_dst((struct sockaddr *)&dst, rt->rt_ifa)) - returned_result->routed_interface_index = - bound_interface_index; + /* + * Transform dst into the internal routing table form + */ + (void) sa_copy((struct sockaddr *)&remote_addr, + &dst, &ifscope); + + if ((rt->rt_ifp->if_index == lo_ifp->if_index) || + rt_ifa_is_dst((struct sockaddr *)&dst, rt->rt_ifa)) + returned_result->routed_interface_index = + bound_interface_index; + } } } @@ -5619,35 +6532,80 @@ necp_application_find_policy_match_internal(proc_t proc, } if (flags != NULL) { - // Check for local/direct - bool is_local = FALSE; - if (rt != NULL && (rt->rt_flags & RTF_LOCAL)) { - is_local = TRUE; - } else if (returned_result->routed_interface_index != 0 && - !no_remote_addr) { - // Check if remote address is an interface address - struct ifaddr *ifa = ifa_ifwithaddr(&remote_addr.sa); - if (ifa != NULL && ifa->ifa_ifp != NULL) { - u_int if_index_for_remote_addr = ifa->ifa_ifp->if_index; - if (if_index_for_remote_addr == returned_result->routed_interface_index || - if_index_for_remote_addr == lo_ifp->if_index) { - is_local = TRUE; + if ((client_flags & NECP_CLIENT_PARAMETER_FLAG_LISTENER) == 0) { + // Check for local/direct + bool is_local = FALSE; + if (rt != NULL && (rt->rt_flags & RTF_LOCAL)) { + is_local = TRUE; + } else if (returned_result->routed_interface_index != 0 && + !no_remote_addr) { + // Check if remote address is an interface address + struct ifaddr *ifa = ifa_ifwithaddr(&remote_addr.sa); + if (ifa != NULL && ifa->ifa_ifp != NULL) { + u_int if_index_for_remote_addr = ifa->ifa_ifp->if_index; + if (if_index_for_remote_addr == returned_result->routed_interface_index || + if_index_for_remote_addr == lo_ifp->if_index) { + is_local = TRUE; + } + } + if (ifa != NULL) { + ifaddr_release(ifa); + ifa = NULL; } } - if (ifa != NULL) { - ifaddr_release(ifa); - ifa = NULL; + + if (is_local) { + *flags |= (NECP_CLIENT_RESULT_FLAG_IS_LOCAL | NECP_CLIENT_RESULT_FLAG_IS_DIRECT); + } else { + if (rt != NULL && + !(rt->rt_flags & RTF_GATEWAY) && + (rt->rt_ifa && rt->rt_ifa->ifa_ifp && !(rt->rt_ifa->ifa_ifp->if_flags & IFF_POINTOPOINT))) { + // Route is directly accessible + *flags |= NECP_CLIENT_RESULT_FLAG_IS_DIRECT; + } } - } - if (is_local) { - *flags |= (NECP_CLIENT_RESULT_FLAG_IS_LOCAL | NECP_CLIENT_RESULT_FLAG_IS_DIRECT); - } else { if (rt != NULL && - !(rt->rt_flags & RTF_GATEWAY) && - (rt->rt_ifa && rt->rt_ifa->ifa_ifp && !(rt->rt_ifa->ifa_ifp->if_flags & IFF_POINTOPOINT))) { - // Route is directly accessible - *flags |= NECP_CLIENT_RESULT_FLAG_IS_DIRECT; + rt->rt_ifp != NULL) { + // Check probe status + if (rt->rt_ifp->if_eflags & IFEF_PROBE_CONNECTIVITY) { + *flags |= NECP_CLIENT_RESULT_FLAG_PROBE_CONNECTIVITY; + } + + if (rt->rt_ifp->if_type == IFT_CELLULAR) { + struct if_cellular_status_v1 *ifsr; + + ifnet_lock_shared(rt->rt_ifp); + lck_rw_lock_exclusive(&rt->rt_ifp->if_link_status_lock); + + if (rt->rt_ifp->if_link_status != NULL) { + ifsr = &rt->rt_ifp->if_link_status->ifsr_u.ifsr_cell.if_cell_u.if_status_v1; + + if (ifsr->valid_bitmask & IF_CELL_UL_MSS_RECOMMENDED_VALID) { + if (ifsr->mss_recommended == IF_CELL_UL_MSS_RECOMMENDED_NONE) { + returned_result->mss_recommended = NECP_CLIENT_RESULT_RECOMMENDED_MSS_NONE; + } else if (ifsr->mss_recommended == IF_CELL_UL_MSS_RECOMMENDED_MEDIUM) { + returned_result->mss_recommended = NECP_CLIENT_RESULT_RECOMMENDED_MSS_MEDIUM; + } else if (ifsr->mss_recommended == IF_CELL_UL_MSS_RECOMMENDED_LOW) { + returned_result->mss_recommended = NECP_CLIENT_RESULT_RECOMMENDED_MSS_LOW; + } + } + } + lck_rw_done(&rt->rt_ifp->if_link_status_lock); + ifnet_lock_done(rt->rt_ifp); + } + + // Check link quality + if ((client_flags & NECP_CLIENT_PARAMETER_FLAG_DISCRETIONARY) && + (rt->rt_ifp->if_interface_state.valid_bitmask & IF_INTERFACE_STATE_LQM_STATE_VALID) && + rt->rt_ifp->if_interface_state.lqm_state == IFNET_LQM_THRESH_ABORT) { + *flags |= NECP_CLIENT_RESULT_FLAG_LINK_QUALITY_ABORT; + } + + // Check QoS marking (fastlane) + if (necp_update_qos_marking(rt->rt_ifp, route_rule_id)) { + *flags |= NECP_CLIENT_RESULT_FLAG_ALLOW_QOS_MARKING; + } } } @@ -5701,7 +6659,11 @@ necp_application_find_policy_match_internal(proc_t proc, } if (rt != NULL) { - rtfree(rt); + if (returned_route != NULL) { + *returned_route = rt; + } else { + rtfree(rt); + } rt = NULL; } // Unlock @@ -7109,7 +8071,7 @@ necp_buffer_compare_with_bit_prefix(u_int8_t *p1, u_int8_t *p2, u_int32_t bits) } static bool -necp_socket_update_qos_marking_inner(struct ifnet *ifp, u_int32_t route_rule_id) +necp_update_qos_marking(struct ifnet *ifp, u_int32_t route_rule_id) { bool qos_marking = FALSE; int exception_index = 0; @@ -7170,8 +8132,9 @@ necp_socket_update_qos_marking(struct inpcb *inp, struct rtentry *route, struct bool qos_marking = FALSE; struct ifnet *ifp = interface = NULL; - ASSERT(net_qos_policy_restricted != 0); - + if (net_qos_policy_restricted == 0) { + return; + } if (inp->inp_socket == NULL) { return; } @@ -7207,14 +8170,14 @@ necp_socket_update_qos_marking(struct inpcb *inp, struct rtentry *route, struct if (sub_route_rule_id == 0) { break; } - qos_marking = necp_socket_update_qos_marking_inner(ifp, sub_route_rule_id); + qos_marking = necp_update_qos_marking(ifp, sub_route_rule_id); if (qos_marking == TRUE) { break; } } } } else { - qos_marking = necp_socket_update_qos_marking_inner(ifp, route_rule_id); + qos_marking = necp_update_qos_marking(ifp, route_rule_id); } /* * Now that we have an interface we remember the gencount @@ -7452,16 +8415,18 @@ necp_socket_is_allowed_to_send_recv_internal(struct inpcb *inp, struct sockaddr if ((necp_socket_is_connected(inp) || (override_local_addr == NULL && override_remote_addr == NULL)) && inp->inp_policyresult.policy_id != NECP_KERNEL_POLICY_ID_NONE) { bool policies_have_changed = FALSE; bool route_allowed = TRUE; - lck_rw_lock_shared(&necp_kernel_policy_lock); + if (inp->inp_policyresult.policy_gencount != necp_kernel_socket_policies_gencount) { policies_have_changed = TRUE; } else { - if (inp->inp_policyresult.results.route_rule_id != 0 && - !necp_route_is_allowed(route, interface, inp->inp_policyresult.results.route_rule_id, &interface_type_denied)) { - route_allowed = FALSE; + if (inp->inp_policyresult.results.route_rule_id != 0) { + lck_rw_lock_shared(&necp_kernel_policy_lock); + if (!necp_route_is_allowed(route, interface, inp->inp_policyresult.results.route_rule_id, &interface_type_denied)) { + route_allowed = FALSE; + } + lck_rw_done(&necp_kernel_policy_lock); } } - lck_rw_done(&necp_kernel_policy_lock); if (!policies_have_changed) { if (!route_allowed || diff --git a/bsd/net/necp.h b/bsd/net/necp.h index ec33fb120..63a9fe967 100644 --- a/bsd/net/necp.h +++ b/bsd/net/necp.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2016 Apple Inc. All rights reserved. + * Copyright (c) 2013-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -33,6 +33,7 @@ #ifdef PRIVATE #include <netinet/in.h> +#include <netinet/in_stat.h> #include <sys/socket.h> #include <net/if.h> @@ -64,6 +65,21 @@ struct necp_packet_header { #define NECP_PACKET_TYPE_UNREGISTER_SERVICE 10 #define NECP_PACKET_TYPE_POLICY_DUMP_ALL 11 +/* + * Session actions + */ +#define NECP_SESSION_ACTION_POLICY_ADD 1 // In: Policy TLVs Out: necp_policy_id +#define NECP_SESSION_ACTION_POLICY_GET 2 // In: necp_policy_id Out: Policy TLVs +#define NECP_SESSION_ACTION_POLICY_DELETE 3 // In: necp_policy_id Out: None +#define NECP_SESSION_ACTION_POLICY_APPLY_ALL 4 // In: None Out: None +#define NECP_SESSION_ACTION_POLICY_LIST_ALL 5 // In: None Out: TLVs of IDs +#define NECP_SESSION_ACTION_POLICY_DELETE_ALL 6 // In: None Out: None +#define NECP_SESSION_ACTION_SET_SESSION_PRIORITY 7 // In: necp_session_priority Out: None +#define NECP_SESSION_ACTION_LOCK_SESSION_TO_PROC 8 // In: None Out: None +#define NECP_SESSION_ACTION_REGISTER_SERVICE 9 // In: uuid_t Out: None +#define NECP_SESSION_ACTION_UNREGISTER_SERVICE 10 // In: uuid_t Out: None +#define NECP_SESSION_ACTION_POLICY_DUMP_ALL 11 // In: None Out: uint32_t bytes length, then Policy TLVs + /* * Control message flags */ @@ -207,6 +223,7 @@ struct necp_policy_condition_addr_range { typedef u_int32_t necp_policy_id; typedef u_int32_t necp_policy_order; +typedef u_int32_t necp_session_priority; typedef u_int32_t necp_kernel_policy_result; typedef u_int32_t necp_kernel_policy_filter; @@ -220,6 +237,8 @@ typedef union { #define NECP_SERVICE_FLAGS_REGISTERED 0x01 #define NECP_MAX_NETAGENTS 8 + +#define NECP_TFO_COOKIE_LEN_MAX 16 struct necp_aggregate_result { necp_kernel_policy_result routing_result; necp_kernel_policy_routing_result_parameter routing_result_parameter; @@ -232,6 +251,7 @@ struct necp_aggregate_result { u_int32_t policy_id; uuid_t netagents[NECP_MAX_NETAGENTS]; u_int32_t netagent_flags[NECP_MAX_NETAGENTS]; + u_int8_t mss_recommended; }; /* @@ -239,7 +259,7 @@ struct necp_aggregate_result { * but they get entangled with #defines for v4 etc in pfvar.h and it may be better practice * to have separate definitions here. */ -typedef struct necp_stat_counts +struct necp_stat_counts { /* Counters */ u_int64_t necp_stat_rxpackets __attribute__((aligned(8))); @@ -258,13 +278,7 @@ typedef struct necp_stat_counts u_int32_t necp_stat_avg_rtt; u_int32_t necp_stat_var_rtt; - u_int64_t necp_stat_cell_rxbytes __attribute__((aligned(8))); - u_int64_t necp_stat_cell_txbytes __attribute__((aligned(8))); - u_int64_t necp_stat_wifi_rxbytes __attribute__((aligned(8))); - u_int64_t necp_stat_wifi_txbytes __attribute__((aligned(8))); - u_int64_t necp_stat_wired_rxbytes __attribute__((aligned(8))); - u_int64_t necp_stat_wired_txbytes __attribute__((aligned(8))); -} necp_stat_counts; +}; // Note, some metadata is implicit in the necp client itself: // From the process itself : pid, upid, uuid, proc name. @@ -272,17 +286,11 @@ typedef struct necp_stat_counts // // The following may well be supplied via future necp client parameters, // but they are here so they don't get forgotten. -typedef struct necp_basic_metadata +struct necp_basic_metadata { u_int32_t rcvbufsize; u_int32_t rcvbufused; - - u_int64_t eupid; // Not currently used by NetworkStatistics, could skip. - u_int32_t epid; - - uuid_t vuuid; // Effective UUID as given from voucher - uint16_t ifnet_properties; -} necp_basic_metadata; +}; struct necp_tcp_probe_status { unsigned int probe_activated : 1; @@ -291,7 +299,7 @@ struct necp_tcp_probe_status { unsigned int conn_probe_failed : 1; }; -typedef struct necp_extra_tcp_metadata +struct necp_extra_tcp_metadata { struct necp_tcp_probe_status probestatus; @@ -300,42 +308,106 @@ typedef struct necp_extra_tcp_metadata u_int32_t txunacked; u_int32_t txwindow; u_int32_t txcwindow; + u_int32_t flags; // use SOF_* + u_int32_t flags1; // use SOF1_* u_int32_t traffic_mgt_flags; u_int32_t cc_alg_index; u_int32_t state; -} necp_extra_tcp_metadata; + activity_bitmap_t activity_bitmap; +}; -typedef struct necp_stats_hdr { +struct necp_stats_hdr { u_int32_t necp_stats_type __attribute__((aligned(8))); u_int32_t necp_stats_ver; - u_int64_t necp_stats_event; -} necp_stats_hdr; + u_int64_t __necp_stats_reserved; // Pad the field for future use +}; #define NECP_CLIENT_STATISTICS_TYPE_TCP 1 // Identifies use of necp_tcp_stats #define NECP_CLIENT_STATISTICS_TYPE_UDP 2 // Identifies use of necp_udp_stats #define NECP_CLIENT_STATISTICS_TYPE_TCP_VER_1 1 // Currently supported version for TCP #define NECP_CLIENT_STATISTICS_TYPE_UDP_VER_1 1 // Currently supported version for UDP -typedef struct necp_tcp_stats { - necp_stats_hdr necp_tcp_hdr; - necp_stat_counts necp_tcp_counts; - necp_basic_metadata necp_tcp_basic; - necp_extra_tcp_metadata necp_tcp_extra; -} necp_tcp_stats; +#define NECP_CLIENT_STATISTICS_TYPE_TCP_CURRENT_VER NECP_CLIENT_STATISTICS_TYPE_TCP_VER_1 +#define NECP_CLIENT_STATISTICS_TYPE_UDP_CURRENT_VER NECP_CLIENT_STATISTICS_TYPE_UDP_VER_1 + +#define NECP_CLIENT_STATISTICS_EVENT_INIT 0x00000000 // Register the flow +#define NECP_CLIENT_STATISTICS_EVENT_TIME_WAIT 0x00000001 // The flow is effectively finished but waiting on timer + +struct necp_tcp_stats { + struct necp_stats_hdr necp_tcp_hdr; + struct necp_stat_counts necp_tcp_counts; + struct necp_basic_metadata necp_tcp_basic; + struct necp_extra_tcp_metadata necp_tcp_extra; +}; -typedef struct necp_udp_stats { - necp_stats_hdr necp_udp_hdr; - necp_stat_counts necp_udp_counts; - necp_basic_metadata necp_udp_basic; -} necp_udp_stats; +struct necp_udp_stats { + struct necp_stats_hdr necp_udp_hdr; + struct necp_stat_counts necp_udp_counts; + struct necp_basic_metadata necp_udp_basic; +}; typedef struct necp_all_stats { union { - necp_tcp_stats tcp_stats; - necp_udp_stats udp_stats; + struct necp_tcp_stats tcp_stats; + struct necp_udp_stats udp_stats; } all_stats_u; } necp_all_stats; +// Memory for statistics is requested via a necp_stats_bufreq +// +struct necp_stats_bufreq { + u_int32_t necp_stats_bufreq_id __attribute__((aligned(8))); + u_int32_t necp_stats_bufreq_type; // NECP_CLIENT_STATISTICS_TYPE_* + u_int32_t necp_stats_bufreq_ver; // NECP_CLIENT_STATISTICS_TYPE_*_VER + u_int32_t necp_stats_bufreq_size; + union { + void *necp_stats_bufreq_addr; + mach_vm_address_t necp_stats_bufreq_uaddr; + }; +}; + +#define NECP_CLIENT_STATISTICS_BUFREQ_ID 0xbf // Distinguishes from statistics actions taking a necp_all_stats struct + +// There is a limit to the number of statistics structures that may be allocated per process, subject to change +// +#define NECP_MAX_PER_PROCESS_CLIENT_STATISTICS_STRUCTS 512 + +#define NECP_TCP_ECN_HEURISTICS_SYN_RST 1 +typedef struct necp_tcp_ecn_cache { + u_int8_t necp_tcp_ecn_heuristics_success:1; + u_int8_t necp_tcp_ecn_heuristics_loss:1; + u_int8_t necp_tcp_ecn_heuristics_drop_rst:1; + u_int8_t necp_tcp_ecn_heuristics_drop_rxmt:1; + u_int8_t necp_tcp_ecn_heuristics_aggressive:1; + u_int8_t necp_tcp_ecn_heuristics_syn_rst:1; +} necp_tcp_ecn_cache; + +#define NECP_TCP_TFO_HEURISTICS_RST 1 +typedef struct necp_tcp_tfo_cache { + u_int8_t necp_tcp_tfo_cookie[NECP_TFO_COOKIE_LEN_MAX]; + u_int8_t necp_tcp_tfo_cookie_len; + u_int8_t necp_tcp_tfo_heuristics_success:1; // TFO succeeded with data in the SYN + u_int8_t necp_tcp_tfo_heuristics_loss:1; // TFO SYN-loss with data + u_int8_t necp_tcp_tfo_heuristics_middlebox:1; // TFO middlebox detected + u_int8_t necp_tcp_tfo_heuristics_success_req:1; // TFO succeeded with the TFO-option in the SYN + u_int8_t necp_tcp_tfo_heuristics_loss_req:1; // TFO SYN-loss with the TFO-option + u_int8_t necp_tcp_tfo_heuristics_rst_data:1; // Recevied RST upon SYN with data in the SYN + u_int8_t necp_tcp_tfo_heuristics_rst_req:1; // Received RST upon SYN with the TFO-option +} necp_tcp_tfo_cache; + +#define NECP_CLIENT_CACHE_TYPE_ECN 1 // Identifies use of necp_tcp_ecn_cache +#define NECP_CLIENT_CACHE_TYPE_TFO 2 // Identifies use of necp_tcp_tfo_cache + +#define NECP_CLIENT_CACHE_TYPE_ECN_VER_1 1 // Currently supported version for ECN +#define NECP_CLIENT_CACHE_TYPE_TFO_VER_1 1 // Currently supported version for TFO + +typedef struct necp_cache_buffer { + u_int8_t necp_cache_buf_type; // NECP_CLIENT_CACHE_TYPE_* + u_int8_t necp_cache_buf_ver; // NECP_CLIENT_CACHE_TYPE_*_VER + u_int32_t necp_cache_buf_size; + mach_vm_address_t necp_cache_buf_addr; +} necp_cache_buffer; + /* * NECP Client definitions */ @@ -343,18 +415,27 @@ typedef struct necp_all_stats { #define NECP_MAX_CLIENT_RESULT_SIZE 512 #define NECP_OPEN_FLAG_OBSERVER 0x01 // Observers can query clients they don't own +#define NECP_OPEN_FLAG_BACKGROUND 0x02 // Mark this fd as backgrounded +#define NECP_OPEN_FLAG_PUSH_OBSERVER 0x04 // When used with the OBSERVER flag, allows updates to be pushed. Adding clients is not allowed in this mode. + +#define NECP_FD_SUPPORTS_GUARD 1 #define NECP_CLIENT_ACTION_ADD 1 // Register a new client. Input: parameters in buffer; Output: client_id -#define NECP_CLIENT_ACTION_REMOVE 2 // Unregister a client. Input: client_id +#define NECP_CLIENT_ACTION_REMOVE 2 // Unregister a client. Input: client_id, optional struct ifnet_stats_per_flow #define NECP_CLIENT_ACTION_COPY_PARAMETERS 3 // Copy client parameters. Input: client_id; Output: parameters in buffer #define NECP_CLIENT_ACTION_COPY_RESULT 4 // Copy client result. Input: client_id; Output: result in buffer #define NECP_CLIENT_ACTION_COPY_LIST 5 // Copy all client IDs. Output: struct necp_client_list in buffer -#define NECP_CLIENT_ACTION_REQUEST_NEXUS_INSTANCE 6 // Request a nexus instance from a nexus provider +#define NECP_CLIENT_ACTION_REQUEST_NEXUS_INSTANCE 6 // Request a nexus instance from a nexus provider, optional struct necp_stats_bufreq #define NECP_CLIENT_ACTION_AGENT 7 // Interact with agent. Input: client_id, agent parameters #define NECP_CLIENT_ACTION_COPY_AGENT 8 // Copy agent content. Input: agent UUID; Output: struct netagent #define NECP_CLIENT_ACTION_COPY_INTERFACE 9 // Copy interface details. Input: ifindex cast to UUID; Output: struct necp_interface_details -#define NECP_CLIENT_ACTION_SET_STATISTICS 10 // Start/update/complete per-flow statistics. Input: client_id, statistics area +#define NECP_CLIENT_ACTION_SET_STATISTICS 10 // Deprecated +#define NECP_CLIENT_ACTION_COPY_ROUTE_STATISTICS 11 // Get route statistics. Input: client_id; Output: struct necp_stat_counts #define NECP_CLIENT_ACTION_AGENT_USE 12 // Return the use count and increment the use count. Input/Output: struct necp_agent_use_parameters +#define NECP_CLIENT_ACTION_MAP_SYSCTLS 13 // Get the read-only sysctls memory location. Output: mach_vm_address_t +#define NECP_CLIENT_ACTION_UPDATE_CACHE 14 // Update heuristics and cache +#define NECP_CLIENT_ACTION_COPY_CLIENT_UPDATE 15 // Fetch an updated client for push-mode observer. Output: Client id, struct necp_client_observer_update in buffer +#define NECP_CLIENT_ACTION_COPY_UPDATED_RESULT 16 // Copy client result only if changed. Input: client_id; Output: result in buffer #define NECP_CLIENT_PARAMETER_APPLICATION NECP_POLICY_CONDITION_APPLICATION // Requires entitlement #define NECP_CLIENT_PARAMETER_REAL_APPLICATION NECP_POLICY_CONDITION_REAL_APPLICATION // Requires entitlement @@ -367,6 +448,7 @@ typedef struct necp_all_stats { #define NECP_CLIENT_PARAMETER_IP_PROTOCOL NECP_POLICY_CONDITION_IP_PROTOCOL #define NECP_CLIENT_PARAMETER_LOCAL_ADDRESS NECP_POLICY_CONDITION_LOCAL_ADDR #define NECP_CLIENT_PARAMETER_REMOTE_ADDRESS NECP_POLICY_CONDITION_REMOTE_ADDR +#define NECP_CLIENT_PARAMETER_NEXUS_KEY 102 // "Prohibit" will never choose an interface with that property #define NECP_CLIENT_PARAMETER_PROHIBIT_INTERFACE 100 // String, interface name @@ -390,7 +472,18 @@ typedef struct necp_all_stats { #define NECP_CLIENT_PARAMETER_LOCAL_ENDPOINT 200 // struct necp_client_endpoint #define NECP_CLIENT_PARAMETER_REMOTE_ENDPOINT 201 // struct necp_client_endpoint -#define NECP_CLIENT_PARAMETER_RESERVED_START 1000 // Parameters 1000 and higher are reserved for custom userspace options +#define NECP_CLIENT_PARAMETER_BROWSE_CATEGORY 202 // struct necp_client_endpoint + +#define NECP_CLIENT_PARAMETER_FLAGS 250 // u_int32_t, see NECP_CLIENT_PAREMETER_FLAG_* values + +#define NECP_CLIENT_PARAMETER_FLAG_MULTIPATH 0x0001 // Get multipath interface results +#define NECP_CLIENT_PARAMETER_FLAG_BROWSE 0x0002 // Agent assertions on nexuses are requests to browse +#define NECP_CLIENT_PARAMETER_FLAG_PROHIBIT_EXPENSIVE 0x0004 // Prohibit expensive interfaces +#define NECP_CLIENT_PARAMETER_FLAG_LISTENER 0x0008 // Client is interested in listening for inbound connections +#define NECP_CLIENT_PARAMETER_FLAG_DISCRETIONARY 0x0010 // Client's traffic is discretionary, and eligible for early defuncting +#define NECP_CLIENT_PARAMETER_FLAG_ECN_ENABLE 0x0020 // Client is requesting to enable ECN +#define NECP_CLIENT_PARAMETER_FLAG_ECN_DISABLE 0x0040 // Client is requesting to disable ECN +#define NECP_CLIENT_PARAMETER_FLAG_TFO_ENABLE 0x0080 // Client is requesting to enable TFO #define NECP_CLIENT_RESULT_CLIENT_ID 1 // uuid_t #define NECP_CLIENT_RESULT_POLICY_RESULT 2 // u_int32_t @@ -400,17 +493,50 @@ typedef struct necp_all_stats { #define NECP_CLIENT_RESULT_NETAGENT 6 // struct necp_client_result_netagent #define NECP_CLIENT_RESULT_FLAGS 7 // u_int32_t, see NECP_CLIENT_RESULT_FLAG_* values #define NECP_CLIENT_RESULT_INTERFACE 8 // struct necp_client_result_interface +#define NECP_CLIENT_RESULT_MULTIPATH_INTERFACE 9 // struct necp_client_result_interface +#define NECP_CLIENT_RESULT_EFFECTIVE_MTU 10 // u_int32_t +#define NECP_CLIENT_RESULT_FLOW 11 // TLV array of a single flow's state +#define NECP_CLIENT_RESULT_PROTO_CTL_EVENT 12 +#define NECP_CLIENT_RESULT_TFO_COOKIE 13 // NECP_TFO_COOKIE_LEN_MAX +#define NECP_CLIENT_RESULT_TFO_FLAGS 14 // u_int8_t +#define NECP_CLIENT_RESULT_RECOMMENDED_MSS 15 // u_int8_t #define NECP_CLIENT_RESULT_NEXUS_INSTANCE 100 // uuid_t #define NECP_CLIENT_RESULT_NEXUS_PORT 101 // u_int16_t +#define NECP_CLIENT_RESULT_NEXUS_KEY 102 // uuid_t +#define NECP_CLIENT_RESULT_NEXUS_PORT_FLOW_INDEX 103 // u_int32_t #define NECP_CLIENT_RESULT_LOCAL_ENDPOINT 200 // struct necp_client_endpoint #define NECP_CLIENT_RESULT_REMOTE_ENDPOINT 201 // struct necp_client_endpoint +#define NECP_CLIENT_RESULT_DISCOVERED_ENDPOINT 202 // struct necp_client_endpoint, result of browse +#define NECP_CLIENT_RESULT_EFFECTIVE_TRAFFIC_CLASS 210 // u_int32_t +#define NECP_CLIENT_RESULT_TRAFFIC_MGMT_BG 211 // u_int32_t, 1: background, 0: not background #define NECP_CLIENT_RESULT_FLAG_IS_LOCAL 0x0001 // Routes to this device #define NECP_CLIENT_RESULT_FLAG_IS_DIRECT 0x0002 // Routes to directly accessible peer #define NECP_CLIENT_RESULT_FLAG_HAS_IPV4 0x0004 // Supports IPv4 #define NECP_CLIENT_RESULT_FLAG_HAS_IPV6 0x0008 // Supports IPv6 +#define NECP_CLIENT_RESULT_FLAG_DEFUNCT 0x0010 // Defunct +#define NECP_CLIENT_RESULT_FLAG_SATISFIED 0x0020 // Satisfied path +#define NECP_CLIENT_RESULT_FLAG_FLOW_ASSIGNED 0x0040 // Assigned, the flow is active +#define NECP_CLIENT_RESULT_FLAG_FLOW_VIABLE 0x0080 // Viable, the flow has a valid route +#define NECP_CLIENT_RESULT_FLAG_PROBE_CONNECTIVITY 0x0100 // Flow should probe connectivity +#define NECP_CLIENT_RESULT_FLAG_ECN_ENABLED 0x0200 // ECN should be used +#define NECP_CLIENT_RESULT_FLAG_FAST_OPEN_BLOCKED 0x0400 // Fast open should not be used +#define NECP_CLIENT_RESULT_FLAG_LINK_QUALITY_ABORT 0x0800 // Link quality is very bad, recommend close connections +#define NECP_CLIENT_RESULT_FLAG_ALLOW_QOS_MARKING 0x1000 // QoS marking is allowed + +#define NECP_CLIENT_RESULT_FAST_OPEN_SND_PROBE 0x01 // DEPRECATED - Fast open send probe +#define NECP_CLIENT_RESULT_FAST_OPEN_RCV_PROBE 0x02 // DEPRECATED - Fast open receive probe + +#define NECP_CLIENT_RESULT_RECOMMENDED_MSS_NONE 0x01 +#define NECP_CLIENT_RESULT_RECOMMENDED_MSS_LOW 0x02 +#define NECP_CLIENT_RESULT_RECOMMENDED_MSS_MEDIUM 0x04 + +struct necp_interface_signature { + u_int8_t signature[IFNET_SIGNATURELEN]; + u_int8_t signature_len; +}; struct necp_interface_details { char name[IFXNAMSIZ]; @@ -420,11 +546,13 @@ struct necp_interface_details { u_int32_t delegate_index; u_int32_t flags; // see NECP_INTERFACE_FLAG_* u_int32_t mtu; - u_int8_t ipv4_signature[IFNET_SIGNATURELEN]; - u_int8_t ipv6_signature[IFNET_SIGNATURELEN]; + struct necp_interface_signature ipv4_signature; + struct necp_interface_signature ipv6_signature; }; #define NECP_INTERFACE_FLAG_EXPENSIVE 0x0001 +#define NECP_INTERFACE_FLAG_TXSTART 0X0002 +#define NECP_INTERFACE_FLAG_NOACKPRI 0x0004 struct necp_client_parameter_netagent_type { char netagent_domain[32]; @@ -470,16 +598,35 @@ struct necp_agent_use_parameters { uint64_t out_use_count; }; +struct necp_client_flow_protoctl_event { + uint32_t protoctl_event_code; + uint32_t protoctl_event_val; + /* TCP seq number is in host byte order */ + uint32_t protoctl_event_tcp_seq_num; +}; + +#define NECP_CLIENT_UPDATE_TYPE_PARAMETERS 1 // Parameters, for a new client +#define NECP_CLIENT_UPDATE_TYPE_RESULT 2 // Result, for a udpated client +#define NECP_CLIENT_UPDATE_TYPE_REMOVE 3 // Empty, for a removed client + +struct necp_client_observer_update { + u_int32_t update_type; // NECP_CLIENT_UPDATE_TYPE_* + u_int8_t tlv_buffer[0]; // Parameters or result as TLVs, based on type +}; + #ifdef BSD_KERNEL_PRIVATE #include <stdbool.h> #include <sys/socketvar.h> #include <sys/kern_control.h> #include <netinet/ip_var.h> +#include <netinet/mp_pcb.h> #include <netinet6/ip6_var.h> #include <net/if_var.h> #include <sys/syslog.h> #include <net/network_agent.h> +SYSCTL_DECL(_net_necp); + #define NECPLOG(level, format, ...) do { \ log((level > LOG_NOTICE ? LOG_NOTICE : level), "%s: " format "\n", __FUNCTION__, __VA_ARGS__); \ } while (0) @@ -488,19 +635,53 @@ struct necp_agent_use_parameters { log((level > LOG_NOTICE ? LOG_NOTICE : level), "%s: %s\n", __FUNCTION__, msg); \ } while (0) +enum necp_fd_type_t { + necp_fd_type_invalid = 0, + necp_fd_type_session = 1, + necp_fd_type_client = 2, +}; + +union necp_sockaddr_union { + struct sockaddr sa; + struct sockaddr_in sin; + struct sockaddr_in6 sin6; +}; + +/* + * kstats + * The ustats and kstats region are mirrored. So when we allocate with + * skmem_cache from kstats region, we also get an ustats object. To tie them + * together, kstats has an extra *necp_stats_ustats pointer pointing to the + * ustats object + */ +struct necp_all_kstats { + struct necp_all_stats necp_stats_comm; /* kernel private stats snapshot */ + struct necp_all_stats *necp_stats_ustats; /* points to user-visible stats (in shared ustats region) */ +}; + extern errno_t necp_client_init(void); extern int necp_application_find_policy_match_internal(proc_t proc, u_int8_t *parameters, u_int32_t parameters_size, - struct necp_aggregate_result *returned_result, - u_int32_t *flags, u_int required_interface_index); + struct necp_aggregate_result *returned_result, + u_int32_t *flags, u_int required_interface_index, + const union necp_sockaddr_union *override_local_addr, + const union necp_sockaddr_union *override_remote_addr, + struct rtentry **returned_route, bool ignore_address); /* * TLV utilities * * Note that these functions (other than necp_buffer_find_tlv) do not check the length of the entire buffer, * so the caller must be sure that the entire TLV is within bounds. */ -extern u_int8_t *necp_buffer_write_tlv(u_int8_t *buffer, u_int8_t type, u_int32_t length, const void *value); -extern u_int8_t *necp_buffer_write_tlv_if_different(u_int8_t *buffer, const u_int8_t *max, u_int8_t type, - u_int32_t length, const void *value, bool *updated); +struct necp_tlv_header { + u_int8_t type; + u_int32_t length; +} __attribute__((__packed__)); + +extern u_int8_t *necp_buffer_write_tlv(u_int8_t *cursor, u_int8_t type, u_int32_t length, const void *value, + u_int8_t *buffer, u_int32_t buffer_length); +extern u_int8_t *necp_buffer_write_tlv_if_different(u_int8_t *cursor, u_int8_t type, + u_int32_t length, const void *value, bool *updated, + u_int8_t *buffer, u_int32_t buffer_length); extern u_int8_t necp_buffer_get_tlv_type(u_int8_t *buffer, int tlv_offset); extern u_int32_t necp_buffer_get_tlv_length(u_int8_t *buffer, int tlv_offset); extern u_int8_t *necp_buffer_get_tlv_value(u_int8_t *buffer, int tlv_offset, u_int32_t *value_size); @@ -514,6 +695,15 @@ extern int necp_buffer_find_tlv(u_int8_t *buffer, u_int32_t buffer_length, int o #define NECPCTL_SOCKET_NON_APP_POLICY_COUNT 6 /* Count of non-per-app socket-level policies */ #define NECPCTL_IP_POLICY_COUNT 7 /* Count of all ip-level policies */ #define NECPCTL_SESSION_COUNT 8 /* Count of NECP sessions */ +#define NECPCTL_CLIENT_FD_COUNT 9 /* Count of NECP client fds */ +#define NECPCTL_CLIENT_COUNT 10 /* Count of NECP clients */ +#define NECPCTL_ARENA_COUNT 11 /* Count of NECP arenas (stats, etc) */ +#define NECPCTL_NEXUS_FLOW_COUNT 12 /* Count of NECP nexus flows */ +#define NECPCTL_SOCKET_FLOW_COUNT 13 /* Count of NECP socket flows */ +#define NECPCTL_IF_FLOW_COUNT 14 /* Count of NECP socket flows */ +#define NECPCTL_OBSERVER_FD_COUNT 15 /* Count of NECP observer fds */ +#define NECPCTL_OBSERVER_MESSAGE_LIMIT 16 /* Number of of NECP observer messages allowed to be queued */ +#define NECPCTL_SYSCTL_ARENA_COUNT 17 /* Count of sysctl arenas */ #define NECPCTL_NAMES { \ { 0, 0 }, \ @@ -526,7 +716,8 @@ extern int necp_buffer_find_tlv(u_int8_t *buffer, u_int32_t buffer_length, int o typedef u_int32_t necp_kernel_policy_id; #define NECP_KERNEL_POLICY_ID_NONE 0 #define NECP_KERNEL_POLICY_ID_NO_MATCH 1 -#define NECP_KERNEL_POLICY_ID_FIRST_VALID 2 +#define NECP_KERNEL_POLICY_ID_FIRST_VALID_SOCKET 2 +#define NECP_KERNEL_POLICY_ID_FIRST_VALID_IP UINT16_MAX typedef u_int32_t necp_app_id; @@ -562,12 +753,6 @@ typedef union { necp_kernel_policy_service service; } necp_kernel_policy_result_parameter; -union necp_sockaddr_union { - struct sockaddr sa; - struct sockaddr_in sin; - struct sockaddr_in6 sin6; -}; - enum necp_boolean_state { necp_boolean_state_unknown = 0, necp_boolean_state_false = 1, @@ -652,7 +837,6 @@ struct necp_session_policy { uuid_t applied_app_uuid; uuid_t applied_real_app_uuid; - char *applied_domain; char *applied_account; uuid_t applied_result_uuid; @@ -683,7 +867,9 @@ extern errno_t necp_init(void); extern errno_t necp_set_socket_attributes(struct socket *so, struct sockopt *sopt); extern errno_t necp_get_socket_attributes(struct socket *so, struct sockopt *sopt); +extern void necp_inpcb_remove_cb(struct inpcb *inp); extern void necp_inpcb_dispose(struct inpcb *inp); +extern void necp_mppcb_dispose(struct mppcb *mpp); extern u_int32_t necp_socket_get_content_filter_control_unit(struct socket *so); @@ -737,9 +923,82 @@ extern void necp_update_all_clients(void); // Handle general re-evaluate event extern void necp_force_update_client(uuid_t client_id, uuid_t remove_netagent_uuid); // Cause a single client to get an update event +extern void necp_client_early_close(uuid_t client_id); // Cause a single client to close stats, etc + +extern void necp_set_client_as_background(proc_t proc, struct fileproc *fp, bool background); // Set all clients for an fp as background or not + +extern void necp_defunct_client(proc_t proc, struct fileproc *fp); // Set all clients for an fp as defunct + +extern int necp_client_register_socket_flow(pid_t pid, uuid_t client_id, struct inpcb *inp); + +extern int necp_client_register_multipath_cb(pid_t pid, uuid_t client_id, struct mppcb *mpp); + +extern int necp_client_assign_from_socket(pid_t pid, uuid_t client_id, struct inpcb *inp); + extern int necp_assign_client_result(uuid_t netagent_uuid, uuid_t client_id, u_int8_t *assigned_results, size_t assigned_results_length); +struct skmem_obj_info; // forward declaration +extern int necp_stats_ctor(struct skmem_obj_info *oi, struct skmem_obj_info *oim, void *arg, uint32_t skmflag); +extern int necp_stats_dtor(void *addr, void *arg); + +/* value to denote invalid flow advisory index */ +struct netagent_session; +extern int +necp_update_flow_protoctl_event(uuid_t netagent_uuid, uuid_t client_id, + uint32_t protoctl_event_code, uint32_t protoctl_event_val, + uint32_t protoctl_event_tcp_seq_num); + +#define NECP_FLOWADV_IDX_INVALID UINT32_MAX +extern void *necp_create_nexus_assign_message(uuid_t nexus_instance, u_int32_t nexus_port, void *key, uint32_t key_length, + struct necp_client_endpoint *local_endpoint, struct necp_client_endpoint *remote_endpoint, + u_int32_t flow_adv_index, size_t *message_length); + +struct necp_client_nexus_parameters { + pid_t pid; + pid_t epid; + uuid_t euuid; + union necp_sockaddr_union local_addr; + union necp_sockaddr_union remote_addr; + u_int16_t ip_protocol; + u_int32_t traffic_class; + necp_policy_id policy_id; + unsigned is_listener:1; + unsigned allow_qos_marking:1; +}; +extern int necp_client_copy_parameters(uuid_t client_uuid, struct necp_client_nexus_parameters *parameters); + +#define NECP_CLIENT_CBACTION_NONVIABLE 1 +#define NECP_CLIENT_CBACTION_VIABLE 2 +#define NECP_CLIENT_CBACTION_INITIAL 3 + +struct necp_client_flow { + LIST_ENTRY(necp_client_flow) flow_chain; + unsigned invalid : 1; + unsigned nexus : 1; // If true, flow is a nexus; if false, flow is attached to socket + unsigned socket : 1; + unsigned viable : 1; + unsigned requested_nexus : 1; + unsigned assigned : 1; + unsigned has_protoctl_event : 1; + unsigned check_tcp_heuristics : 1; + union { + uuid_t nexus_agent; + struct { + void *socket_handle; + void (*cb)(void *handle, int action, struct necp_client_flow *flow); + }; + } u; + uint32_t interface_index; + uint16_t interface_flags; + uint32_t necp_flow_flags; + struct necp_client_flow_protoctl_event protoctl_event; + union necp_sockaddr_union local_addr; + union necp_sockaddr_union remote_addr; + + size_t assigned_results_length; + u_int8_t *assigned_results; +}; #endif /* BSD_KERNEL_PRIVATE */ #ifndef KERNEL @@ -750,6 +1009,12 @@ extern int necp_open(int flags); extern int necp_client_action(int necp_fd, uint32_t action, uuid_t client_id, size_t client_id_len, uint8_t *buffer, size_t buffer_size); +extern int necp_session_open(int flags); + +extern int necp_session_action(int necp_fd, uint32_t action, + uint8_t *in_buffer, size_t in_buffer_length, + uint8_t *out_buffer, size_t out_buffer_length); + #endif /* !KERNEL */ #endif /* PRIVATE */ diff --git a/bsd/net/necp_client.c b/bsd/net/necp_client.c index 33adf5a22..0cb79a5b7 100644 --- a/bsd/net/necp_client.c +++ b/bsd/net/necp_client.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016 Apple Inc. All rights reserved. + * Copyright (c) 2015-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -27,31 +27,45 @@ */ #include <string.h> -#include <sys/systm.h> -#include <sys/types.h> -#include <sys/queue.h> -#include <sys/malloc.h> + +#include <kern/thread_call.h> +#include <kern/zalloc.h> + #include <libkern/OSMalloc.h> -#include <sys/kernel.h> + #include <net/if.h> -#include <sys/domain.h> -#include <sys/protosw.h> -#include <sys/socket.h> -#include <sys/socketvar.h> +#include <net/if_var.h> +#include <net/net_api_stats.h> +#include <net/necp.h> +#include <net/network_agent.h> +#include <net/ntstat.h> + +#include <netinet/in_pcb.h> #include <netinet/ip.h> #include <netinet/ip6.h> -#include <netinet/in_pcb.h> -#include <net/if_var.h> +#include <netinet/mp_pcb.h> #include <netinet/tcp_cc.h> -#include <net/ntstat.h> -#include <sys/kauth.h> -#include <sys/sysproto.h> -#include <sys/priv.h> -#include <net/network_agent.h> -#include <net/necp.h> +#include <netinet/tcp_fsm.h> +#include <netinet/tcp_cache.h> +#include <netinet6/in6_var.h> + +#include <sys/domain.h> #include <sys/file_internal.h> +#include <sys/kauth.h> +#include <sys/kernel.h> +#include <sys/malloc.h> #include <sys/poll.h> -#include <kern/thread_call.h> +#include <sys/priv.h> +#include <sys/protosw.h> +#include <sys/queue.h> +#include <sys/socket.h> +#include <sys/socketvar.h> +#include <sys/sysproto.h> +#include <sys/systm.h> +#include <sys/types.h> +#include <sys/codesign.h> +#include <libkern/section_keywords.h> + /* * NECP Client Architecture @@ -131,41 +145,71 @@ extern u_int32_t necp_debug; +// proc_best_name() is declared here in advance of it landing in a header file. +// See comment in kern_proc.c +extern char *proc_best_name(proc_t p); + static int noop_read(struct fileproc *, struct uio *, int, vfs_context_t); static int noop_write(struct fileproc *, struct uio *, int, vfs_context_t); static int noop_ioctl(struct fileproc *, unsigned long, caddr_t, - vfs_context_t); + vfs_context_t); static int necpop_select(struct fileproc *, int, void *, vfs_context_t); static int necpop_close(struct fileglob *, vfs_context_t); -static int necpop_kqfilter(struct fileproc *, struct knote *, vfs_context_t); +static int necpop_kqfilter(struct fileproc *, struct knote *, + struct kevent_internal_s *kev, vfs_context_t); // Timer functions static int necp_timeout_microseconds = 1000 * 100; // 100ms static int necp_timeout_leeway_microseconds = 1000 * 500; // 500ms + +static int necp_client_fd_count = 0; +static int necp_observer_fd_count = 0; +static int necp_client_count = 0; +static int necp_socket_flow_count = 0; +static int necp_if_flow_count = 0; +static int necp_observer_message_limit = 256; + +SYSCTL_INT(_net_necp, NECPCTL_CLIENT_FD_COUNT, client_fd_count, CTLFLAG_LOCKED | CTLFLAG_RD, &necp_client_fd_count, 0, ""); +SYSCTL_INT(_net_necp, NECPCTL_OBSERVER_FD_COUNT, observer_fd_count, CTLFLAG_LOCKED | CTLFLAG_RD, &necp_observer_fd_count, 0, ""); +SYSCTL_INT(_net_necp, NECPCTL_CLIENT_COUNT, client_count, CTLFLAG_LOCKED | CTLFLAG_RD, &necp_client_count, 0, ""); +SYSCTL_INT(_net_necp, NECPCTL_SOCKET_FLOW_COUNT, socket_flow_count, CTLFLAG_LOCKED | CTLFLAG_RD, &necp_socket_flow_count, 0, ""); +SYSCTL_INT(_net_necp, NECPCTL_IF_FLOW_COUNT, if_flow_count, CTLFLAG_LOCKED | CTLFLAG_RD, &necp_if_flow_count, 0, ""); +SYSCTL_INT(_net_necp, NECPCTL_OBSERVER_MESSAGE_LIMIT, observer_message_limit, CTLFLAG_LOCKED | CTLFLAG_RW, &necp_observer_message_limit, 256, ""); + +#define NECP_MAX_CLIENT_LIST_SIZE 1024 * 1024 // 1MB + extern int tvtohz(struct timeval *); +extern unsigned int get_maxmtu(struct rtentry *); // Parsed parameters -#define NECP_PARSED_PARAMETERS_FIELD_LOCAL_ADDR 0x0001 -#define NECP_PARSED_PARAMETERS_FIELD_REMOTE_ADDR 0x0002 -#define NECP_PARSED_PARAMETERS_FIELD_REQUIRED_IF 0x0004 -#define NECP_PARSED_PARAMETERS_FIELD_PROHIBITED_IF 0x0008 -#define NECP_PARSED_PARAMETERS_FIELD_REQUIRED_IFTYPE 0x0010 -#define NECP_PARSED_PARAMETERS_FIELD_PROHIBITED_IFTYPE 0x0020 -#define NECP_PARSED_PARAMETERS_FIELD_REQUIRED_AGENT 0x0040 -#define NECP_PARSED_PARAMETERS_FIELD_PROHIBITED_AGENT 0x0080 -#define NECP_PARSED_PARAMETERS_FIELD_PREFERRED_AGENT 0x0100 -#define NECP_PARSED_PARAMETERS_FIELD_REQUIRED_AGENT_TYPE 0x0200 -#define NECP_PARSED_PARAMETERS_FIELD_PROHIBITED_AGENT_TYPE 0x0400 -#define NECP_PARSED_PARAMETERS_FIELD_PREFERRED_AGENT_TYPE 0x0800 +#define NECP_PARSED_PARAMETERS_FIELD_LOCAL_ADDR 0x00001 +#define NECP_PARSED_PARAMETERS_FIELD_REMOTE_ADDR 0x00002 +#define NECP_PARSED_PARAMETERS_FIELD_REQUIRED_IF 0x00004 +#define NECP_PARSED_PARAMETERS_FIELD_PROHIBITED_IF 0x00008 +#define NECP_PARSED_PARAMETERS_FIELD_REQUIRED_IFTYPE 0x00010 +#define NECP_PARSED_PARAMETERS_FIELD_PROHIBITED_IFTYPE 0x00020 +#define NECP_PARSED_PARAMETERS_FIELD_REQUIRED_AGENT 0x00040 +#define NECP_PARSED_PARAMETERS_FIELD_PROHIBITED_AGENT 0x00080 +#define NECP_PARSED_PARAMETERS_FIELD_PREFERRED_AGENT 0x00100 +#define NECP_PARSED_PARAMETERS_FIELD_REQUIRED_AGENT_TYPE 0x00200 +#define NECP_PARSED_PARAMETERS_FIELD_PROHIBITED_AGENT_TYPE 0x00400 +#define NECP_PARSED_PARAMETERS_FIELD_PREFERRED_AGENT_TYPE 0x00800 +#define NECP_PARSED_PARAMETERS_FIELD_FLAGS 0x01000 +#define NECP_PARSED_PARAMETERS_FIELD_IP_PROTOCOL 0x02000 +#define NECP_PARSED_PARAMETERS_FIELD_EFFECTIVE_PID 0x04000 +#define NECP_PARSED_PARAMETERS_FIELD_EFFECTIVE_UUID 0x08000 +#define NECP_PARSED_PARAMETERS_FIELD_TRAFFIC_CLASS 0x10000 +#define NECP_PARSED_PARAMETERS_FIELD_LOCAL_PORT 0x20000 #define NECP_MAX_PARSED_PARAMETERS 16 struct necp_client_parsed_parameters { u_int32_t valid_fields; + u_int32_t flags; union necp_sockaddr_union local_addr; union necp_sockaddr_union remote_addr; u_int32_t required_interface_index; char prohibited_interfaces[IFXNAMSIZ][NECP_MAX_PARSED_PARAMETERS]; - u_int8_t required_interface_types[NECP_MAX_PARSED_PARAMETERS]; + u_int8_t required_interface_type; u_int8_t prohibited_interface_types[NECP_MAX_PARSED_PARAMETERS]; struct necp_client_parameter_netagent_type required_netagent_types[NECP_MAX_PARSED_PARAMETERS]; struct necp_client_parameter_netagent_type prohibited_netagent_types[NECP_MAX_PARSED_PARAMETERS]; @@ -173,9 +217,23 @@ struct necp_client_parsed_parameters { uuid_t required_netagents[NECP_MAX_PARSED_PARAMETERS]; uuid_t prohibited_netagents[NECP_MAX_PARSED_PARAMETERS]; uuid_t preferred_netagents[NECP_MAX_PARSED_PARAMETERS]; + u_int16_t ip_protocol; + pid_t effective_pid; + uuid_t effective_uuid; + u_int32_t traffic_class; }; -static bool necp_find_matching_interface_index(struct necp_client_parsed_parameters *parsed_parameters, u_int *return_ifindex); +static bool +necp_find_matching_interface_index(struct necp_client_parsed_parameters *parsed_parameters, + u_int *return_ifindex); + +static bool +necp_ifnet_matches_local_address(struct ifnet *ifp, struct sockaddr *sa); + +static bool +necp_ifnet_matches_parameters(struct ifnet *ifp, + struct necp_client_parsed_parameters *parsed_parameters, + u_int32_t *preferred_count, bool ignore_require_if); static const struct fileops necp_fd_ops = { .fo_type = DTYPE_NETPOLICY, @@ -193,48 +251,194 @@ struct necp_client_assertion { uuid_t asserted_netagent; }; +struct necp_client_flow_header { + struct necp_tlv_header outer_header; + struct necp_tlv_header flags_tlv_header; + u_int32_t flags_value; + struct necp_tlv_header interface_tlv_header; + struct necp_client_result_interface interface_value; +} __attribute__((__packed__)); + +struct necp_client_flow_protoctl_event_header { + struct necp_tlv_header protoctl_tlv_header; + struct necp_client_flow_protoctl_event protoctl_event; +} __attribute__((__packed__)); + +struct necp_client_nexus_flow_header { + struct necp_client_flow_header flow_header; + struct necp_tlv_header agent_tlv_header; + struct necp_client_result_netagent agent_value; + struct necp_tlv_header tfo_cookie_tlv_header; + u_int8_t tfo_cookie_value[NECP_TFO_COOKIE_LEN_MAX]; +} __attribute__((__packed__)); + struct necp_client { - LIST_ENTRY(necp_client) chain; + RB_ENTRY(necp_client) link; + RB_ENTRY(necp_client) global_link; + LIST_ENTRY(necp_client) collect_stats_chain; + + decl_lck_mtx_data(, lock); + decl_lck_mtx_data(, route_lock); + uint32_t reference_count; uuid_t client_id; - bool result_read; - bool assigned_result_read; + unsigned result_read : 1; + unsigned flow_result_read : 1; + unsigned allow_multiple_flows : 1; + + unsigned defunct : 1; + unsigned background : 1; + unsigned background_update : 1; + unsigned platform_binary : 1; size_t result_length; u_int8_t result[NECP_MAX_CLIENT_RESULT_SIZE]; - uuid_t nexus_agent; - size_t assigned_results_length; - u_int8_t *assigned_results; + necp_policy_id policy_id; + + u_int16_t ip_protocol; + int proc_pid; + LIST_HEAD(_necp_client_flow_list, necp_client_flow) flow_list; LIST_HEAD(_necp_client_assertion_list, necp_client_assertion) assertion_list; - user_addr_t stats_uaddr; - user_size_t stats_ulen; - nstat_userland_context stats_handler_context; - necp_stats_hdr *stats_area; + struct rtentry *current_route; + + void *interface_handle; + void (*interface_cb)(void *handle, int action, struct necp_client_flow *flow); size_t parameters_length; u_int8_t parameters[0]; }; +#define NECP_CLIENT_LOCK(_c) lck_mtx_lock(&_c->lock) +#define NECP_CLIENT_UNLOCK(_c) lck_mtx_unlock(&_c->lock) +#define NECP_CLIENT_ASSERT_LOCKED(_c) LCK_MTX_ASSERT(&_c->lock, LCK_MTX_ASSERT_OWNED) +#define NECP_CLIENT_ASSERT_UNLOCKED(_c) LCK_MTX_ASSERT(&_c->lock, LCK_MTX_ASSERT_NOTOWNED) + +#define NECP_CLIENT_ROUTE_LOCK(_c) lck_mtx_lock(&_c->route_lock) +#define NECP_CLIENT_ROUTE_UNLOCK(_c) lck_mtx_unlock(&_c->route_lock) + +static void necp_client_retain_locked(struct necp_client *client); +static void necp_client_retain(struct necp_client *client); +static bool necp_client_release_locked(struct necp_client *client); + +static void +necp_client_add_assertion(struct necp_client *client, uuid_t netagent_uuid); + +static bool +necp_client_remove_assertion(struct necp_client *client, uuid_t netagent_uuid); + +LIST_HEAD(_necp_client_list, necp_client); +static struct _necp_client_list necp_collect_stats_client_list; + +struct necp_client_defunct { + LIST_ENTRY(necp_client_defunct) chain; + + uuid_t client_id; + uuid_t nexus_agent; + int proc_pid; +}; + +LIST_HEAD(_necp_client_defunct_list, necp_client_defunct); + +static int necp_client_id_cmp(struct necp_client *client0, struct necp_client *client1); + +RB_HEAD(_necp_client_tree, necp_client); +RB_PROTOTYPE_PREV(_necp_client_tree, necp_client, link, necp_client_id_cmp); +RB_GENERATE_PREV(_necp_client_tree, necp_client, link, necp_client_id_cmp); + +RB_HEAD(_necp_client_global_tree, necp_client); +RB_PROTOTYPE_PREV(_necp_client_global_tree, necp_client, global_link, necp_client_id_cmp); +RB_GENERATE_PREV(_necp_client_global_tree, necp_client, global_link, necp_client_id_cmp); + +static struct _necp_client_global_tree necp_client_global_tree; + +struct necp_client_update { + TAILQ_ENTRY(necp_client_update) chain; + + uuid_t client_id; + + size_t update_length; + struct necp_client_observer_update update; +}; + struct necp_fd_data { + u_int8_t necp_fd_type; LIST_ENTRY(necp_fd_data) chain; - LIST_HEAD(_clients, necp_client) clients; + struct _necp_client_tree clients; + TAILQ_HEAD(_necp_client_update_list, necp_client_update) update_list; + int update_count; int flags; int proc_pid; decl_lck_mtx_data(, fd_lock); struct selinfo si; }; +#define NECP_FD_LOCK(_f) lck_mtx_lock(&_f->fd_lock) +#define NECP_FD_UNLOCK(_f) lck_mtx_unlock(&_f->fd_lock) +#define NECP_FD_ASSERT_LOCKED(_f) LCK_MTX_ASSERT(&_f->fd_lock, LCK_MTX_ASSERT_OWNED) +#define NECP_FD_ASSERT_UNLOCKED(_f) LCK_MTX_ASSERT(&_f->fd_lock, LCK_MTX_ASSERT_NOTOWNED) + static LIST_HEAD(_necp_fd_list, necp_fd_data) necp_fd_list; +static LIST_HEAD(_necp_fd_observer_list, necp_fd_data) necp_fd_observer_list; + +#define NECP_CLIENT_FD_ZONE_MAX 128 +#define NECP_CLIENT_FD_ZONE_NAME "necp.clientfd" + +static unsigned int necp_client_fd_size; /* size of zone element */ +static struct zone *necp_client_fd_zone; /* zone for necp_fd_data */ + +#define NECP_FLOW_ZONE_MAX 512 +#define NECP_FLOW_ZONE_NAME "necp.flow" + +static unsigned int necp_flow_size; /* size of zone element */ +static struct zone *necp_flow_zone; /* zone for necp_client_flow */ static lck_grp_attr_t *necp_fd_grp_attr = NULL; static lck_attr_t *necp_fd_mtx_attr = NULL; static lck_grp_t *necp_fd_mtx_grp = NULL; + decl_lck_rw_data(static, necp_fd_lock); +decl_lck_rw_data(static, necp_observer_lock); +decl_lck_rw_data(static, necp_client_tree_lock); +decl_lck_rw_data(static, necp_collect_stats_list_lock); + +#define NECP_STATS_LIST_LOCK_EXCLUSIVE() lck_rw_lock_exclusive(&necp_collect_stats_list_lock) +#define NECP_STATS_LIST_LOCK_SHARED() lck_rw_lock_shared(&necp_collect_stats_list_lock) +#define NECP_STATS_LIST_UNLOCK() lck_rw_done(&necp_collect_stats_list_lock) + +#define NECP_CLIENT_TREE_LOCK_EXCLUSIVE() lck_rw_lock_exclusive(&necp_client_tree_lock) +#define NECP_CLIENT_TREE_LOCK_SHARED() lck_rw_lock_shared(&necp_client_tree_lock) +#define NECP_CLIENT_TREE_UNLOCK() lck_rw_done(&necp_client_tree_lock) + +#define NECP_FD_LIST_LOCK_EXCLUSIVE() lck_rw_lock_exclusive(&necp_fd_lock) +#define NECP_FD_LIST_LOCK_SHARED() lck_rw_lock_shared(&necp_fd_lock) +#define NECP_FD_LIST_UNLOCK() lck_rw_done(&necp_fd_lock) + +#define NECP_OBSERVER_LIST_LOCK_EXCLUSIVE() lck_rw_lock_exclusive(&necp_observer_lock) +#define NECP_OBSERVER_LIST_LOCK_SHARED() lck_rw_lock_shared(&necp_observer_lock) +#define NECP_OBSERVER_LIST_UNLOCK() lck_rw_done(&necp_observer_lock) + +// Locking Notes + +// Take NECP_FD_LIST_LOCK when accessing or modifying the necp_fd_list +// Take NECP_CLIENT_TREE_LOCK when accessing or modifying the necp_client_global_tree +// Take NECP_STATS_LIST_LOCK when accessing or modifying the necp_collect_stats_client_list +// Take NECP_FD_LOCK when accessing or modifying an necp_fd_data entry +// Take NECP_CLIENT_LOCK when accessing or modifying a single necp_client +// Take NECP_CLIENT_ROUTE_LOCK when accessing or modifying a client's route + +// Precedence, where 1 is the first lock that must be taken +// 1. NECP_FD_LIST_LOCK +// 2. NECP_FD_LOCK (any) +// 3. NECP_CLIENT_TREE_LOCK +// 4. NECP_CLIENT_LOCK (any) +// 5. NECP_STATS_LIST_LOCK +// 6. NECP_CLIENT_ROUTE_LOCK (any) + +static thread_call_t necp_client_update_tcall; -static thread_call_t necp_client_tcall; /// NECP file descriptor functions @@ -267,7 +471,7 @@ necp_fd_notify(struct necp_fd_data *fd_data, bool locked) struct selinfo *si = &fd_data->si; if (!locked) { - lck_mtx_lock(&fd_data->fd_lock); + NECP_FD_LOCK(fd_data); } selwakeup(si); @@ -277,7 +481,7 @@ necp_fd_notify(struct necp_fd_data *fd_data, bool locked) KNOTE(&si->si_note, 1); // notification if (!locked) { - lck_mtx_unlock(&fd_data->fd_lock); + NECP_FD_UNLOCK(fd_data); } } @@ -286,27 +490,59 @@ necp_fd_poll(struct necp_fd_data *fd_data, int events, void *wql, struct proc *p { #pragma unused(wql, p, is_kevent) u_int revents = 0; - struct necp_client *client = NULL; - bool has_unread_clients = FALSE; u_int want_rx = events & (POLLIN | POLLRDNORM); if (want_rx) { - - LIST_FOREACH(client, &fd_data->clients, chain) { - if (!client->result_read || !client->assigned_result_read) { - has_unread_clients = TRUE; - break; + if (fd_data->flags & NECP_OPEN_FLAG_PUSH_OBSERVER) { + // Push-mode observers are readable when they have a new update + if (!TAILQ_EMPTY(&fd_data->update_list)) { + revents |= want_rx; + } + } else { + // Standard fds are readable when some client is unread + struct necp_client *client = NULL; + bool has_unread_clients = FALSE; + RB_FOREACH(client, _necp_client_tree, &fd_data->clients) { + NECP_CLIENT_LOCK(client); + if (!client->result_read || !client->flow_result_read) { + has_unread_clients = TRUE; + } + NECP_CLIENT_UNLOCK(client); + if (has_unread_clients) { + break; + } } - } - if (has_unread_clients) { - revents |= want_rx; + if (has_unread_clients) { + revents |= want_rx; + } } } return (revents); } +static struct necp_client * +necp_client_fd_find_client_and_lock(struct necp_fd_data *client_fd, uuid_t client_id) +{ + struct necp_client find; + NECP_FD_ASSERT_LOCKED(client_fd); + uuid_copy(find.client_id, client_id); + struct necp_client *client = RB_FIND(_necp_client_tree, &client_fd->clients, &find); + + if (client != NULL) { + NECP_CLIENT_LOCK(client); + } + + return (client); +} + +static inline int +necp_client_id_cmp(struct necp_client *client0, struct necp_client *client1) +{ + return (uuid_compare(client0->client_id, client1->client_id)); +} + static int necpop_select(struct fileproc *fp, int which, void *wql, vfs_context_t ctx) { @@ -335,9 +571,9 @@ necpop_select(struct fileproc *fp, int which, void *wql, vfs_context_t ctx) } } - lck_mtx_lock(&fd_data->fd_lock); + NECP_FD_LOCK(fd_data); revents = necp_fd_poll(fd_data, events, wql, procp, 0); - lck_mtx_unlock(&fd_data->fd_lock); + NECP_FD_UNLOCK(fd_data); return ((events & revents) ? 1 : 0); } @@ -348,9 +584,9 @@ necp_fd_knrdetach(struct knote *kn) struct necp_fd_data *fd_data = (struct necp_fd_data *)kn->kn_hook; struct selinfo *si = &fd_data->si; - lck_mtx_lock(&fd_data->fd_lock); + NECP_FD_LOCK(fd_data); KNOTE_DETACH(&si->si_note, kn); - lck_mtx_unlock(&fd_data->fd_lock); + NECP_FD_UNLOCK(fd_data); } static int @@ -370,17 +606,17 @@ necp_fd_knrprocess(struct knote *kn, struct filt_process_s *data, struct kevent_ fd_data = (struct necp_fd_data *)kn->kn_hook; - lck_mtx_lock(&fd_data->fd_lock); + NECP_FD_LOCK(fd_data); revents = necp_fd_poll(fd_data, POLLIN, NULL, current_proc(), 1); res = ((revents & POLLIN) != 0); if (res) { *kev = kn->kn_kevent; } - lck_mtx_unlock(&fd_data->fd_lock); + NECP_FD_UNLOCK(fd_data); return (res); } -static int +static int necp_fd_knrtouch(struct knote *kn, struct kevent_internal_s *kev) { #pragma unused(kev) @@ -389,16 +625,16 @@ necp_fd_knrtouch(struct knote *kn, struct kevent_internal_s *kev) fd_data = (struct necp_fd_data *)kn->kn_hook; - lck_mtx_lock(&fd_data->fd_lock); + NECP_FD_LOCK(fd_data); if ((kn->kn_status & KN_UDATA_SPECIFIC) == 0) kn->kn_udata = kev->udata; revents = necp_fd_poll(fd_data, POLLIN, NULL, current_proc(), 1); - lck_mtx_unlock(&fd_data->fd_lock); + NECP_FD_UNLOCK(fd_data); return ((revents & POLLIN) != 0); } -struct filterops necp_fd_rfiltops = { +SECURITY_READ_ONLY_EARLY(struct filterops) necp_fd_rfiltops = { .f_isfd = 1, .f_detach = necp_fd_knrdetach, .f_event = necp_fd_knread, @@ -407,7 +643,8 @@ struct filterops necp_fd_rfiltops = { }; static int -necpop_kqfilter(struct fileproc *fp, struct knote *kn, vfs_context_t ctx) +necpop_kqfilter(struct fileproc *fp, struct knote *kn, + __unused struct kevent_internal_s *kev, vfs_context_t ctx) { #pragma unused(fp, ctx) struct necp_fd_data *fd_data = NULL; @@ -428,77 +665,326 @@ necpop_kqfilter(struct fileproc *fp, struct knote *kn, vfs_context_t ctx) return (0); } - lck_mtx_lock(&fd_data->fd_lock); + NECP_FD_LOCK(fd_data); kn->kn_filtid = EVFILTID_NECP_FD; kn->kn_hook = fd_data; KNOTE_ATTACH(&fd_data->si.si_note, kn); revents = necp_fd_poll(fd_data, POLLIN, NULL, current_proc(), 1); - lck_mtx_unlock(&fd_data->fd_lock); + NECP_FD_UNLOCK(fd_data); return ((revents & POLLIN) != 0); } + +static bool +necp_set_client_defunct(struct necp_client *client) +{ + bool updated = FALSE; + u_int32_t flags = 0; + u_int32_t value_size = 0; + + client->defunct = TRUE; + + u_int8_t *flags_pointer = necp_buffer_get_tlv_value(client->result, 0, &value_size); + if (flags_pointer && value_size == sizeof(flags)) { + memcpy(&flags, flags_pointer, value_size); + + flags |= NECP_CLIENT_RESULT_FLAG_DEFUNCT; + + (void)necp_buffer_write_tlv_if_different(client->result, NECP_CLIENT_RESULT_FLAGS, + sizeof(flags), &flags, &updated, client->result, sizeof(client->result)); + } + + return (updated); +} + static void -necp_destroy_client_stats(struct necp_client *client) +necp_defunct_client_for_policy(struct necp_client *client, + struct _necp_client_defunct_list *defunct_list) { - if ((client->stats_area != NULL) && - (client->stats_handler_context != NULL) && - (client->stats_uaddr != 0)) { - // Close old stats if required. - int error = copyin(client->stats_uaddr, client->stats_area, client->stats_ulen); - if (error) { - NECPLOG(LOG_ERR, "necp_destroy_client_stats copyin error on close (%d)", error); - // Not much we can for an error on an obsolete address + if (!client->defunct) { + bool needs_defunct = false; + struct necp_client_flow *search_flow = NULL; + LIST_FOREACH(search_flow, &client->flow_list, flow_chain) { + if (search_flow->nexus && + !uuid_is_null(search_flow->u.nexus_agent) && + search_flow->requested_nexus) { + + // Save defunct values for the nexus + if (defunct_list != NULL) { + // Sleeping alloc won't fail; copy only what's necessary + struct necp_client_defunct *client_defunct = _MALLOC(sizeof (struct necp_client_defunct), + M_NECP, M_WAITOK | M_ZERO); + uuid_copy(client_defunct->nexus_agent, search_flow->u.nexus_agent); + uuid_copy(client_defunct->client_id, client->client_id); + client_defunct->proc_pid = client->proc_pid; + + // Add to the list provided by caller + LIST_INSERT_HEAD(defunct_list, client_defunct, chain); + } + + needs_defunct = true; + } + } + + if (needs_defunct) { + // Only set defunct if there was some assigned flow + client->defunct = true; } - ntstat_userland_stats_close(client->stats_handler_context); - FREE(client->stats_area, M_NECP); - client->stats_area = NULL; - client->stats_handler_context = NULL; - client->stats_uaddr = 0; - client->stats_ulen = 0; } } static void -necp_destroy_client(struct necp_client *client) +necp_client_free(struct necp_client *client) +{ + NECP_CLIENT_ASSERT_LOCKED(client); + + NECP_CLIENT_UNLOCK(client); + + lck_mtx_destroy(&client->route_lock, necp_fd_mtx_grp); + lck_mtx_destroy(&client->lock, necp_fd_mtx_grp); + + FREE(client, M_NECP); +} + +static void +necp_client_retain_locked(struct necp_client *client) +{ + NECP_CLIENT_ASSERT_LOCKED(client); + + client->reference_count++; + ASSERT(client->reference_count != 0); +} + +static void +necp_client_retain(struct necp_client *client) +{ + NECP_CLIENT_LOCK(client); + necp_client_retain_locked(client); + NECP_CLIENT_UNLOCK(client); +} + +static bool +necp_client_release_locked(struct necp_client *client) +{ + NECP_CLIENT_ASSERT_LOCKED(client); + + uint32_t old_ref = client->reference_count; + + ASSERT(client->reference_count != 0); + if (--client->reference_count == 0) { + necp_client_free(client); + } + + return (old_ref == 1); +} + + +static void +necp_client_update_observer_add_internal(struct necp_fd_data *observer_fd, struct necp_client *client) +{ + NECP_FD_LOCK(observer_fd); + + if (observer_fd->update_count >= necp_observer_message_limit) { + NECP_FD_UNLOCK(observer_fd); + return; + } + + struct necp_client_update *client_update = _MALLOC(sizeof(struct necp_client_update) + client->parameters_length, + M_NECP, M_WAITOK | M_ZERO); + if (client_update != NULL) { + client_update->update_length = sizeof(struct necp_client_observer_update) + client->parameters_length; + uuid_copy(client_update->client_id, client->client_id); + client_update->update.update_type = NECP_CLIENT_UPDATE_TYPE_PARAMETERS; + memcpy(client_update->update.tlv_buffer, client->parameters, client->parameters_length); + TAILQ_INSERT_TAIL(&observer_fd->update_list, client_update, chain); + observer_fd->update_count++; + + necp_fd_notify(observer_fd, true); + } + + NECP_FD_UNLOCK(observer_fd); +} + +static void +necp_client_update_observer_update_internal(struct necp_fd_data *observer_fd, struct necp_client *client) +{ + NECP_FD_LOCK(observer_fd); + + if (observer_fd->update_count >= necp_observer_message_limit) { + NECP_FD_UNLOCK(observer_fd); + return; + } + + struct necp_client_update *client_update = _MALLOC(sizeof(struct necp_client_update) + client->result_length, + M_NECP, M_WAITOK | M_ZERO); + if (client_update != NULL) { + client_update->update_length = sizeof(struct necp_client_observer_update) + client->result_length; + uuid_copy(client_update->client_id, client->client_id); + client_update->update.update_type = NECP_CLIENT_UPDATE_TYPE_RESULT; + memcpy(client_update->update.tlv_buffer, client->result, client->result_length); + TAILQ_INSERT_TAIL(&observer_fd->update_list, client_update, chain); + observer_fd->update_count++; + + necp_fd_notify(observer_fd, true); + } + + NECP_FD_UNLOCK(observer_fd); +} + +static void +necp_client_update_observer_remove_internal(struct necp_fd_data *observer_fd, struct necp_client *client) +{ + NECP_FD_LOCK(observer_fd); + + if (observer_fd->update_count >= necp_observer_message_limit) { + NECP_FD_UNLOCK(observer_fd); + return; + } + + struct necp_client_update *client_update = _MALLOC(sizeof(struct necp_client_update), + M_NECP, M_WAITOK | M_ZERO); + if (client_update != NULL) { + client_update->update_length = sizeof(struct necp_client_observer_update); + uuid_copy(client_update->client_id, client->client_id); + client_update->update.update_type = NECP_CLIENT_UPDATE_TYPE_REMOVE; + TAILQ_INSERT_TAIL(&observer_fd->update_list, client_update, chain); + observer_fd->update_count++; + + necp_fd_notify(observer_fd, true); + } + + NECP_FD_UNLOCK(observer_fd); +} + +static void +necp_client_update_observer_add(struct necp_client *client) +{ + NECP_OBSERVER_LIST_LOCK_SHARED(); + + if (LIST_EMPTY(&necp_fd_observer_list)) { + // No observers, bail + NECP_OBSERVER_LIST_UNLOCK(); + return; + } + + struct necp_fd_data *observer_fd = NULL; + LIST_FOREACH(observer_fd, &necp_fd_observer_list, chain) { + necp_client_update_observer_add_internal(observer_fd, client); + } + + NECP_OBSERVER_LIST_UNLOCK(); +} + +static void +necp_client_update_observer_update(struct necp_client *client) +{ + NECP_OBSERVER_LIST_LOCK_SHARED(); + + if (LIST_EMPTY(&necp_fd_observer_list)) { + // No observers, bail + NECP_OBSERVER_LIST_UNLOCK(); + return; + } + + struct necp_fd_data *observer_fd = NULL; + LIST_FOREACH(observer_fd, &necp_fd_observer_list, chain) { + necp_client_update_observer_update_internal(observer_fd, client); + } + + NECP_OBSERVER_LIST_UNLOCK(); +} + +static void +necp_client_update_observer_remove(struct necp_client *client) { - // Remove from list - LIST_REMOVE(client, chain); + NECP_OBSERVER_LIST_LOCK_SHARED(); + + if (LIST_EMPTY(&necp_fd_observer_list)) { + // No observers, bail + NECP_OBSERVER_LIST_UNLOCK(); + return; + } + + struct necp_fd_data *observer_fd = NULL; + LIST_FOREACH(observer_fd, &necp_fd_observer_list, chain) { + necp_client_update_observer_remove_internal(observer_fd, client); + } - // Remove nexus assignment - if (client->assigned_results != NULL) { - if (!uuid_is_null(client->nexus_agent)) { - int netagent_error = netagent_client_message(client->nexus_agent, client->client_id, - NETAGENT_MESSAGE_TYPE_CLOSE_NEXUS); - if (netagent_error != 0) { + NECP_OBSERVER_LIST_UNLOCK(); +} + +static void +necp_destroy_client(struct necp_client *client, pid_t pid, bool abort) +{ + NECP_CLIENT_ASSERT_UNLOCKED(client); + + necp_client_update_observer_remove(client); + + NECP_CLIENT_LOCK(client); + + // Free route + NECP_CLIENT_ROUTE_LOCK(client); + if (client->current_route != NULL) { + rtfree(client->current_route); + client->current_route = NULL; + } + NECP_CLIENT_ROUTE_UNLOCK(client); + + // Remove flow assignments + struct necp_client_flow *search_flow = NULL; + struct necp_client_flow *temp_flow = NULL; + LIST_FOREACH_SAFE(search_flow, &client->flow_list, flow_chain, temp_flow) { + if (search_flow->nexus && + !uuid_is_null(search_flow->u.nexus_agent) && + search_flow->requested_nexus) { + // Note that if we had defuncted the client earlier, this would result in a harmless ENOENT + int netagent_error = netagent_client_message(search_flow->u.nexus_agent, client->client_id, pid, + abort ? NETAGENT_MESSAGE_TYPE_ABORT_NEXUS : NETAGENT_MESSAGE_TYPE_CLOSE_NEXUS); + if (netagent_error != 0 && netagent_error != ENOENT) { NECPLOG(LOG_ERR, "necp_client_remove close nexus error (%d)", netagent_error); } + uuid_clear(search_flow->u.nexus_agent); } - FREE(client->assigned_results, M_NETAGENT); + if (search_flow->assigned_results != NULL) { + FREE(search_flow->assigned_results, M_NETAGENT); + search_flow->assigned_results = NULL; + } + LIST_REMOVE(search_flow, flow_chain); + if (search_flow->socket) { + OSDecrementAtomic(&necp_socket_flow_count); + } else { + OSDecrementAtomic(&necp_if_flow_count); + } + zfree(necp_flow_zone, search_flow); } // Remove agent assertions struct necp_client_assertion *search_assertion = NULL; struct necp_client_assertion *temp_assertion = NULL; LIST_FOREACH_SAFE(search_assertion, &client->assertion_list, assertion_chain, temp_assertion) { - int netagent_error = netagent_client_message(search_assertion->asserted_netagent, client->client_id, NETAGENT_MESSAGE_TYPE_CLIENT_UNASSERT); + int netagent_error = netagent_client_message(search_assertion->asserted_netagent, client->client_id, pid, NETAGENT_MESSAGE_TYPE_CLIENT_UNASSERT); if (netagent_error != 0) { - NECPLOG(LOG_ERR, "necp_client_remove unassert agent error (%d)", netagent_error); + NECPLOG((netagent_error == ENOENT ? LOG_DEBUG : LOG_ERR), + "necp_client_remove unassert agent error (%d)", netagent_error); } LIST_REMOVE(search_assertion, assertion_chain); FREE(search_assertion, M_NECP); } - necp_destroy_client_stats(client); - FREE(client, M_NECP); + if (!necp_client_release_locked(client)) { + NECP_CLIENT_UNLOCK(client); + } + + OSDecrementAtomic(&necp_client_count); } static int necpop_close(struct fileglob *fg, vfs_context_t ctx) { -#pragma unused(fg, ctx) +#pragma unused(ctx) struct necp_fd_data *fd_data = NULL; int error = 0; @@ -506,26 +992,61 @@ necpop_close(struct fileglob *fg, vfs_context_t ctx) fg->fg_data = NULL; if (fd_data != NULL) { - lck_rw_lock_exclusive(&necp_fd_lock); + struct _necp_client_tree clients_to_close; + RB_INIT(&clients_to_close); + + // Remove from list quickly + if (fd_data->flags & NECP_OPEN_FLAG_PUSH_OBSERVER) { + NECP_OBSERVER_LIST_LOCK_EXCLUSIVE(); + LIST_REMOVE(fd_data, chain); + NECP_OBSERVER_LIST_UNLOCK(); + } else { + NECP_FD_LIST_LOCK_EXCLUSIVE(); + LIST_REMOVE(fd_data, chain); + NECP_FD_LIST_UNLOCK(); + } - lck_mtx_lock(&fd_data->fd_lock); + NECP_FD_LOCK(fd_data); + pid_t pid = fd_data->proc_pid; struct necp_client *client = NULL; struct necp_client *temp_client = NULL; - LIST_FOREACH_SAFE(client, &fd_data->clients, chain, temp_client) { - necp_destroy_client(client); + RB_FOREACH_SAFE(client, _necp_client_tree, &fd_data->clients, temp_client) { + NECP_CLIENT_TREE_LOCK_EXCLUSIVE(); + RB_REMOVE(_necp_client_global_tree, &necp_client_global_tree, client); + NECP_CLIENT_TREE_UNLOCK(); + RB_REMOVE(_necp_client_tree, &fd_data->clients, client); + RB_INSERT(_necp_client_tree, &clients_to_close, client); } - lck_mtx_unlock(&fd_data->fd_lock); + + struct necp_client_update *client_update = NULL; + struct necp_client_update *temp_update = NULL; + TAILQ_FOREACH_SAFE(client_update, &fd_data->update_list, chain, temp_update) { + // Flush pending updates + TAILQ_REMOVE(&fd_data->update_list, client_update, chain); + FREE(client_update, M_NECP); + } + fd_data->update_count = 0; + + + NECP_FD_UNLOCK(fd_data); selthreadclear(&fd_data->si); lck_mtx_destroy(&fd_data->fd_lock, necp_fd_mtx_grp); - LIST_REMOVE(fd_data, chain); - - lck_rw_done(&necp_fd_lock); + if (fd_data->flags & NECP_OPEN_FLAG_PUSH_OBSERVER) { + OSDecrementAtomic(&necp_observer_fd_count); + } else { + OSDecrementAtomic(&necp_client_fd_count); + } - FREE(fd_data, M_NECP); + zfree(necp_client_fd_zone, fd_data); fd_data = NULL; + + RB_FOREACH_SAFE(client, _necp_client_tree, &clients_to_close, temp_client) { + RB_REMOVE(_necp_client_tree, &clients_to_close, client); + necp_destroy_client(client, pid, true); + } } return (error); @@ -533,6 +1054,13 @@ necpop_close(struct fileglob *fg, vfs_context_t ctx) /// NECP client utilities +static inline bool +necp_address_is_wildcard(const union necp_sockaddr_union * const addr) +{ + return ((addr->sa.sa_family == AF_INET && addr->sin.sin_addr.s_addr == INADDR_ANY) || + (addr->sa.sa_family == AF_INET6 && IN6_IS_ADDR_UNSPECIFIED(&addr->sin6.sin6_addr))); +} + static int necp_find_fd_data(int fd, struct necp_fd_data **fd_data) { @@ -556,15 +1084,183 @@ done: return (error); } -static bool -necp_netagent_applies_to_client(__unused struct necp_client *client, struct necp_client_parsed_parameters *parameters, uuid_t netagent_uuid) -{ - bool applies = FALSE; - u_int32_t flags = netagent_get_flags(netagent_uuid); - if (!(flags & NETAGENT_FLAG_REGISTERED)) { - // Unregistered agents never apply - return (applies); - } + +static void +necp_client_add_socket_flow(struct necp_client *client, struct inpcb *inp) +{ + struct necp_client_flow *new_flow = zalloc(necp_flow_zone); + if (new_flow == NULL) { + NECPLOG0(LOG_ERR, "Failed to allocate socket flow"); + return; + } + + memset(new_flow, 0, sizeof(*new_flow)); + + new_flow->socket = TRUE; + new_flow->u.socket_handle = inp; + new_flow->u.cb = inp->necp_cb; + + OSIncrementAtomic(&necp_socket_flow_count); + + LIST_INSERT_HEAD(&client->flow_list, new_flow, flow_chain); +} + +static void +necp_client_add_interface_flow(struct necp_client *client, uint32_t interface_index) +{ + struct necp_client_flow *new_flow = zalloc(necp_flow_zone); + if (new_flow == NULL) { + NECPLOG0(LOG_ERR, "Failed to allocate interface flow"); + return; + } + + memset(new_flow, 0, sizeof(*new_flow)); + + // Neither nexus nor socket + new_flow->interface_index = interface_index; + new_flow->u.socket_handle = client->interface_handle; + new_flow->u.cb = client->interface_cb; + + OSIncrementAtomic(&necp_if_flow_count); + + LIST_INSERT_HEAD(&client->flow_list, new_flow, flow_chain); +} + +static void +necp_client_add_interface_flow_if_needed(struct necp_client *client, uint32_t interface_index) +{ + if (!client->allow_multiple_flows || + interface_index == IFSCOPE_NONE) { + // Interface not set, or client not allowed to use this mode + return; + } + + struct necp_client_flow *flow = NULL; + LIST_FOREACH(flow, &client->flow_list, flow_chain) { + if (!flow->nexus && !flow->socket && flow->interface_index == interface_index) { + // Already have the flow + flow->invalid = FALSE; + + flow->u.socket_handle = client->interface_handle; + flow->u.cb = client->interface_cb; + return; + } + } + + necp_client_add_interface_flow(client, interface_index); +} + +static bool +necp_client_flow_is_viable(proc_t proc, struct necp_client *client, + struct necp_client_flow *flow) +{ + struct necp_aggregate_result result; + bool ignore_address = (client->allow_multiple_flows && !flow->nexus && !flow->socket); + + flow->necp_flow_flags = 0; + int error = necp_application_find_policy_match_internal(proc, client->parameters, + (u_int32_t)client->parameters_length, + &result, &flow->necp_flow_flags, + flow->interface_index, + &flow->local_addr, &flow->remote_addr, NULL, ignore_address); + + return (error == 0 && + result.routed_interface_index != IFSCOPE_NONE && + result.routing_result != NECP_KERNEL_POLICY_RESULT_DROP); +} + +static bool +necp_client_update_flows(proc_t proc, struct necp_client *client) +{ + bool client_updated = FALSE; + struct necp_client_flow *flow = NULL; + struct necp_client_flow *temp_flow = NULL; + LIST_FOREACH_SAFE(flow, &client->flow_list, flow_chain, temp_flow) { + // Check policy result for flow + int old_flags = flow->necp_flow_flags; + bool viable = necp_client_flow_is_viable(proc, client, flow); + + // TODO: Defunct nexus flows that are blocked by policy + + if (flow->viable != viable) { + flow->viable = viable; + client_updated = TRUE; + } + + if ((old_flags & (NECP_CLIENT_RESULT_FLAG_HAS_IPV4 | NECP_CLIENT_RESULT_FLAG_HAS_IPV6)) != + (flow->necp_flow_flags & (NECP_CLIENT_RESULT_FLAG_HAS_IPV4 | NECP_CLIENT_RESULT_FLAG_HAS_IPV6))) { + client_updated = TRUE; + } + + if (flow->viable && client_updated && (flow->socket || (!flow->socket && !flow->nexus)) && flow->u.cb) { + flow->u.cb(flow->u.socket_handle, NECP_CLIENT_CBACTION_VIABLE, flow); + } + + if (!flow->viable || flow->invalid) { + if (client_updated && (flow->socket || (!flow->socket && !flow->nexus)) && flow->u.cb) { + flow->u.cb(flow->u.socket_handle, NECP_CLIENT_CBACTION_NONVIABLE, flow); + } + // The callback might change the viable-flag of the + // flow depending on its policy. Thus, we need to + // check again the flags after the callback. + } + + + // Handle flows that no longer match + if (!flow->viable || flow->invalid) { + // Drop them as long as they aren't assigned data + if (!flow->requested_nexus && !flow->assigned) { + if (flow->assigned_results != NULL) { + FREE(flow->assigned_results, M_NETAGENT); + flow->assigned_results = NULL; + client_updated = TRUE; + } + LIST_REMOVE(flow, flow_chain); + if (flow->socket) { + OSDecrementAtomic(&necp_socket_flow_count); + } else { + OSDecrementAtomic(&necp_if_flow_count); + } + zfree(necp_flow_zone, flow); + } + } + } + + return (client_updated); +} + +static void +necp_client_mark_all_nonsocket_flows_as_invalid(struct necp_client *client) +{ + struct necp_client_flow *flow = NULL; + LIST_FOREACH(flow, &client->flow_list, flow_chain) { + if (!flow->socket) { // Socket flows are not marked as invalid + flow->invalid = TRUE; + } + } +} + +static bool +necp_netagent_applies_to_client(__unused struct necp_client *client, + const struct necp_client_parsed_parameters *parameters, + uuid_t netagent_uuid, bool allow_nexus, + uint32_t interface_index, u_int16_t interface_flags) +{ +#pragma unused(interface_index, interface_flags) + bool applies = FALSE; + u_int32_t flags = netagent_get_flags(netagent_uuid); + if (!(flags & NETAGENT_FLAG_REGISTERED)) { + // Unregistered agents never apply + return (applies); + } + + if (!allow_nexus && + (flags & NETAGENT_FLAG_NEXUS_PROVIDER)) { + // Hide nexus providers unless allowed + // Direct interfaces and direct policies are allowed to use a nexus + // Delegate interfaces or re-scoped interfaces are not allowed + return (applies); + } if (flags & NETAGENT_FLAG_SPECIFIC_USE_ONLY) { // Specific use agents only apply when required @@ -619,15 +1315,39 @@ necp_netagent_applies_to_client(__unused struct necp_client *client, struct necp applies = TRUE; } - if (applies && - (flags & NETAGENT_FLAG_NEXUS_PROVIDER) && - uuid_is_null(client->nexus_agent)) { - uuid_copy(client->nexus_agent, netagent_uuid); - } return (applies); } +static void +necp_client_add_agent_flows_for_interface(struct necp_client *client, + const struct necp_client_parsed_parameters *parsed_parameters, + ifnet_t ifp) +{ + if (ifp != NULL && ifp->if_agentids != NULL) { + for (u_int32_t i = 0; i < ifp->if_agentcount; i++) { + if (uuid_is_null(ifp->if_agentids[i])) { + continue; + } + u_int16_t if_flags = nstat_ifnet_to_flags(ifp); + // Relies on the side effect that nexus agents that apply will create flows + (void)necp_netagent_applies_to_client(client, parsed_parameters, ifp->if_agentids[i], TRUE, ifp->if_index, if_flags); + } + } +} + +static inline bool +necp_client_address_is_valid(struct sockaddr *address) +{ + if (address->sa_family == AF_INET) { + return (address->sa_len == sizeof(struct sockaddr_in)); + } else if (address->sa_family == AF_INET6) { + return (address->sa_len == sizeof(struct sockaddr_in6)); + } else { + return (FALSE); + } +} + static int necp_client_parse_parameters(u_int8_t *parameters, u_int32_t parameters_size, @@ -637,7 +1357,6 @@ necp_client_parse_parameters(u_int8_t *parameters, size_t offset = 0; u_int32_t num_prohibited_interfaces = 0; - u_int32_t num_required_interface_types = 0; u_int32_t num_prohibited_interface_types = 0; u_int32_t num_required_agents = 0; u_int32_t num_prohibited_agents = 0; @@ -652,11 +1371,11 @@ necp_client_parse_parameters(u_int8_t *parameters, memset(parsed_parameters, 0, sizeof(struct necp_client_parsed_parameters)); - while ((offset + sizeof(u_int8_t) + sizeof(u_int32_t)) <= parameters_size) { + while ((offset + sizeof(struct necp_tlv_header)) <= parameters_size) { u_int8_t type = necp_buffer_get_tlv_type(parameters, offset); u_int32_t length = necp_buffer_get_tlv_length(parameters, offset); - if (length > (parameters_size - (offset + sizeof(u_int8_t) + sizeof(u_int32_t)))) { + if (length > (parameters_size - (offset + sizeof(struct necp_tlv_header)))) { // If the length is larger than what can fit in the remaining parameters size, bail NECPLOG(LOG_ERR, "Invalid TLV length (%u)", length); break; @@ -683,11 +1402,15 @@ necp_client_parse_parameters(u_int8_t *parameters, case NECP_CLIENT_PARAMETER_LOCAL_ADDRESS: { if (length >= sizeof(struct necp_policy_condition_addr)) { struct necp_policy_condition_addr *address_struct = (struct necp_policy_condition_addr *)(void *)value; - if ((address_struct->address.sa.sa_family == AF_INET || - address_struct->address.sa.sa_family == AF_INET6) && - address_struct->address.sa.sa_len <= length) { + if (necp_client_address_is_valid(&address_struct->address.sa)) { memcpy(&parsed_parameters->local_addr, &address_struct->address, sizeof(address_struct->address)); - parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_LOCAL_ADDR; + if (!necp_address_is_wildcard(&parsed_parameters->local_addr)) { + parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_LOCAL_ADDR; + } + if ((parsed_parameters->local_addr.sa.sa_family == AF_INET && parsed_parameters->local_addr.sin.sin_port) || + (parsed_parameters->local_addr.sa.sa_family == AF_INET6 && parsed_parameters->local_addr.sin6.sin6_port)) { + parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_LOCAL_PORT; + } } } break; @@ -695,11 +1418,15 @@ necp_client_parse_parameters(u_int8_t *parameters, case NECP_CLIENT_PARAMETER_LOCAL_ENDPOINT: { if (length >= sizeof(struct necp_client_endpoint)) { struct necp_client_endpoint *endpoint = (struct necp_client_endpoint *)(void *)value; - if ((endpoint->u.endpoint.endpoint_family == AF_INET || - endpoint->u.endpoint.endpoint_family == AF_INET6) && - endpoint->u.endpoint.endpoint_length <= length) { + if (necp_client_address_is_valid(&endpoint->u.sa)) { memcpy(&parsed_parameters->local_addr, &endpoint->u.sa, sizeof(union necp_sockaddr_union)); - parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_LOCAL_ADDR; + if (!necp_address_is_wildcard(&parsed_parameters->local_addr)) { + parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_LOCAL_ADDR; + } + if ((parsed_parameters->local_addr.sa.sa_family == AF_INET && parsed_parameters->local_addr.sin.sin_port) || + (parsed_parameters->local_addr.sa.sa_family == AF_INET6 && parsed_parameters->local_addr.sin6.sin6_port)) { + parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_LOCAL_PORT; + } } } break; @@ -707,9 +1434,7 @@ necp_client_parse_parameters(u_int8_t *parameters, case NECP_CLIENT_PARAMETER_REMOTE_ADDRESS: { if (length >= sizeof(struct necp_policy_condition_addr)) { struct necp_policy_condition_addr *address_struct = (struct necp_policy_condition_addr *)(void *)value; - if ((address_struct->address.sa.sa_family == AF_INET || - address_struct->address.sa.sa_family == AF_INET6) && - address_struct->address.sa.sa_len <= length) { + if (necp_client_address_is_valid(&address_struct->address.sa)) { memcpy(&parsed_parameters->remote_addr, &address_struct->address, sizeof(address_struct->address)); parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_REMOTE_ADDR; } @@ -719,9 +1444,7 @@ necp_client_parse_parameters(u_int8_t *parameters, case NECP_CLIENT_PARAMETER_REMOTE_ENDPOINT: { if (length >= sizeof(struct necp_client_endpoint)) { struct necp_client_endpoint *endpoint = (struct necp_client_endpoint *)(void *)value; - if ((endpoint->u.endpoint.endpoint_family == AF_INET || - endpoint->u.endpoint.endpoint_family == AF_INET6) && - endpoint->u.endpoint.endpoint_length <= length) { + if (necp_client_address_is_valid(&endpoint->u.sa)) { memcpy(&parsed_parameters->remote_addr, &endpoint->u.sa, sizeof(union necp_sockaddr_union)); parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_REMOTE_ADDR; } @@ -741,13 +1464,14 @@ necp_client_parse_parameters(u_int8_t *parameters, break; } case NECP_CLIENT_PARAMETER_REQUIRE_IF_TYPE: { - if (num_required_interface_types >= NECP_MAX_PARSED_PARAMETERS) { + if (parsed_parameters->valid_fields & NECP_PARSED_PARAMETERS_FIELD_REQUIRED_IFTYPE) { break; } if (length >= sizeof(u_int8_t)) { - memcpy(&parsed_parameters->required_interface_types[num_required_interface_types], value, sizeof(u_int8_t)); - num_required_interface_types++; - parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_REQUIRED_IFTYPE; + memcpy(&parsed_parameters->required_interface_type, value, sizeof(u_int8_t)); + if (parsed_parameters->required_interface_type) { + parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_REQUIRED_IFTYPE; + } } break; } @@ -828,6 +1552,89 @@ necp_client_parse_parameters(u_int8_t *parameters, } break; } + case NECP_CLIENT_PARAMETER_FLAGS: { + if (length >= sizeof(u_int32_t)) { + memcpy(&parsed_parameters->flags, value, sizeof(parsed_parameters->flags)); + parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_FLAGS; + } + break; + } + case NECP_CLIENT_PARAMETER_IP_PROTOCOL: { + if (length >= sizeof(parsed_parameters->ip_protocol)) { + memcpy(&parsed_parameters->ip_protocol, value, sizeof(parsed_parameters->ip_protocol)); + parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_IP_PROTOCOL; + } + break; + } + case NECP_CLIENT_PARAMETER_PID: { + if (length >= sizeof(parsed_parameters->effective_pid)) { + memcpy(&parsed_parameters->effective_pid, value, sizeof(parsed_parameters->effective_pid)); + parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_EFFECTIVE_PID; + } + break; + } + case NECP_CLIENT_PARAMETER_APPLICATION: { + if (length >= sizeof(parsed_parameters->effective_uuid)) { + memcpy(&parsed_parameters->effective_uuid, value, sizeof(parsed_parameters->effective_uuid)); + parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_EFFECTIVE_UUID; + } + break; + } + case NECP_CLIENT_PARAMETER_TRAFFIC_CLASS: { + if (length >= sizeof(parsed_parameters->traffic_class)) { + memcpy(&parsed_parameters->traffic_class, value, sizeof(parsed_parameters->traffic_class)); + parsed_parameters->valid_fields |= NECP_PARSED_PARAMETERS_FIELD_TRAFFIC_CLASS; + } + break; + } + default: { + break; + } + } + } + } + + offset += sizeof(struct necp_tlv_header) + length; + } + + return (error); +} + +static int +necp_client_parse_result(u_int8_t *result, + u_int32_t result_size, + union necp_sockaddr_union *local_address, + union necp_sockaddr_union *remote_address) +{ + int error = 0; + size_t offset = 0; + + while ((offset + sizeof(struct necp_tlv_header)) <= result_size) { + u_int8_t type = necp_buffer_get_tlv_type(result, offset); + u_int32_t length = necp_buffer_get_tlv_length(result, offset); + + if (length > 0 && (offset + sizeof(struct necp_tlv_header) + length) <= result_size) { + u_int8_t *value = necp_buffer_get_tlv_value(result, offset, NULL); + if (value != NULL) { + switch (type) { + case NECP_CLIENT_RESULT_LOCAL_ENDPOINT: { + if (length >= sizeof(struct necp_client_endpoint)) { + struct necp_client_endpoint *endpoint = (struct necp_client_endpoint *)(void *)value; + if (local_address != NULL && necp_client_address_is_valid(&endpoint->u.sa)) { + memcpy(local_address, &endpoint->u.sa, endpoint->u.sa.sa_len); + } + } + break; + } + case NECP_CLIENT_RESULT_REMOTE_ENDPOINT: { + if (length >= sizeof(struct necp_client_endpoint)) { + struct necp_client_endpoint *endpoint = (struct necp_client_endpoint *)(void *)value; + if (remote_address != NULL && necp_client_address_is_valid(&endpoint->u.sa)) { + memcpy(remote_address, &endpoint->u.sa, endpoint->u.sa.sa_len); + } + } + break; + } default: { break; } @@ -835,56 +1642,462 @@ necp_client_parse_parameters(u_int8_t *parameters, } } - offset += sizeof(u_int8_t) + sizeof(u_int32_t) + length; + offset += sizeof(struct necp_tlv_header) + length; } return (error); } int -necp_assign_client_result(uuid_t netagent_uuid, uuid_t client_id, - u_int8_t *assigned_results, size_t assigned_results_length) +necp_client_register_socket_flow(pid_t pid, uuid_t client_id, struct inpcb *inp) +{ + int error = 0; + bool found_client = FALSE; + + NECP_CLIENT_TREE_LOCK_SHARED(); + + struct necp_client find; + uuid_copy(find.client_id, client_id); + struct necp_client *client = RB_FIND(_necp_client_global_tree, &necp_client_global_tree, &find); + if (client != NULL) { + NECP_CLIENT_LOCK(client); + + if (!pid || client->proc_pid == pid) { + // Found the right client! + found_client = TRUE; + necp_client_add_socket_flow(client, inp); + } + + NECP_CLIENT_UNLOCK(client); + } + + NECP_CLIENT_TREE_UNLOCK(); + + if (!found_client) { + error = ENOENT; + } else { + /* + * Count the sockets that have the NECP client UUID set + */ + struct socket *so = inp->inp_socket; + if (!(so->so_flags1 & SOF1_HAS_NECP_CLIENT_UUID)) { + so->so_flags1 |= SOF1_HAS_NECP_CLIENT_UUID; + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_necp_clientuuid_total); + } + } + + return (error); +} + +static void +necp_client_add_multipath_cb(struct necp_client *client, struct mppcb *mpp) +{ + struct necp_client_flow *flow = NULL; + + client->interface_handle = mpp; + client->interface_cb = mpp->necp_cb; + + LIST_FOREACH(flow, &client->flow_list, flow_chain) { + if (flow->nexus || flow->socket) { + continue; + } + + flow->u.socket_handle = mpp; + flow->u.cb = mpp->necp_cb; + + if (flow->viable && flow->u.cb) { + flow->u.cb(mpp, NECP_CLIENT_CBACTION_INITIAL, flow); + } + } +} + +int +necp_client_register_multipath_cb(pid_t pid, uuid_t client_id, struct mppcb *mpp) +{ + int error = 0; + bool found_client = FALSE; + + NECP_CLIENT_TREE_LOCK_SHARED(); + + struct necp_client find; + uuid_copy(find.client_id, client_id); + struct necp_client *client = RB_FIND(_necp_client_global_tree, &necp_client_global_tree, &find); + if (client != NULL) { + NECP_CLIENT_LOCK(client); + + if (!pid || client->proc_pid == pid) { + // Found the right client! + found_client = TRUE; + necp_client_add_multipath_cb(client, mpp); + } + + NECP_CLIENT_UNLOCK(client); + } + + NECP_CLIENT_TREE_UNLOCK(); + + if (!found_client) { + error = ENOENT; + } + + return (error); +} + +#define NETAGENT_DOMAIN_NETEXT "NetworkExtension" +#define NETAGENT_TYPE_PATHCTRL "PathController" + +static int +necp_client_unregister_socket_flow(uuid_t client_id, void *handle) { int error = 0; struct necp_fd_data *client_fd = NULL; bool found_client = FALSE; bool client_updated = FALSE; - lck_rw_lock_shared(&necp_fd_lock); + NECP_FD_LIST_LOCK_SHARED(); + LIST_FOREACH(client_fd, &necp_fd_list, chain) { + NECP_FD_LOCK(client_fd); + + struct necp_client *client = necp_client_fd_find_client_and_lock(client_fd, client_id); + if (client != NULL) { + // Found the right client! + found_client = TRUE; + + // Remove flow assignment + struct necp_client_flow *search_flow = NULL; + struct necp_client_flow *temp_flow = NULL; + LIST_FOREACH_SAFE(search_flow, &client->flow_list, flow_chain, temp_flow) { + if (search_flow->socket && search_flow->u.socket_handle == handle) { + if (search_flow->assigned_results != NULL) { + FREE(search_flow->assigned_results, M_NETAGENT); + search_flow->assigned_results = NULL; + } + client_updated = TRUE; + LIST_REMOVE(search_flow, flow_chain); + OSDecrementAtomic(&necp_socket_flow_count); + zfree(necp_flow_zone, search_flow); + } + } + + NECP_CLIENT_UNLOCK(client); + } + + if (client_updated) { + client->flow_result_read = FALSE; + necp_fd_notify(client_fd, true); + } + NECP_FD_UNLOCK(client_fd); + + if (found_client) { + break; + } + } + NECP_FD_LIST_UNLOCK(); + + if (!found_client) { + error = ENOENT; + } + + return (error); +} + +static int +necp_client_unregister_multipath_cb(uuid_t client_id, void *handle) +{ + int error = 0; + bool found_client = FALSE; + + NECP_CLIENT_TREE_LOCK_SHARED(); + + struct necp_client find; + uuid_copy(find.client_id, client_id); + struct necp_client *client = RB_FIND(_necp_client_global_tree, &necp_client_global_tree, &find); + if (client != NULL) { + NECP_CLIENT_LOCK(client); + + // Found the right client! + found_client = TRUE; + + // Remove flow assignment + struct necp_client_flow *search_flow = NULL; + struct necp_client_flow *temp_flow = NULL; + LIST_FOREACH_SAFE(search_flow, &client->flow_list, flow_chain, temp_flow) { + if (!search_flow->socket && !search_flow->nexus && + search_flow->u.socket_handle == handle) { + search_flow->u.socket_handle = NULL; + search_flow->u.cb = NULL; + } + } + client->interface_handle = NULL; + client->interface_cb = NULL; + + NECP_CLIENT_UNLOCK(client); + } + + NECP_CLIENT_TREE_UNLOCK(); + + if (!found_client) { + error = ENOENT; + } + + return (error); +} + +int +necp_client_assign_from_socket(pid_t pid, uuid_t client_id, struct inpcb *inp) +{ + int error = 0; + struct necp_fd_data *client_fd = NULL; + bool found_client = FALSE; + bool client_updated = FALSE; + + NECP_FD_LIST_LOCK_SHARED(); LIST_FOREACH(client_fd, &necp_fd_list, chain) { - struct necp_client *client = NULL; - lck_mtx_lock(&client_fd->fd_lock); - LIST_FOREACH(client, &client_fd->clients, chain) { - if (uuid_compare(client->client_id, client_id) == 0) { - // Found the right client! - found_client = TRUE; + if (pid && client_fd->proc_pid != pid) { + continue; + } - if (uuid_compare(client->nexus_agent, netagent_uuid) == 0) { - // Verify that the client nexus agent matches - if (client->assigned_results != NULL) { - // Release prior result - FREE(client->assigned_results, M_NETAGENT); + proc_t proc = proc_find(client_fd->proc_pid); + if (proc == PROC_NULL) { + continue; + } + + NECP_FD_LOCK(client_fd); + + struct necp_client *client = necp_client_fd_find_client_and_lock(client_fd, client_id); + if (client != NULL) { + // Found the right client! + found_client = TRUE; + + struct necp_client_flow *flow = NULL; + LIST_FOREACH(flow, &client->flow_list, flow_chain) { + if (flow->socket && flow->u.socket_handle == inp) { + // Release prior results and route + if (flow->assigned_results != NULL) { + FREE(flow->assigned_results, M_NETAGENT); + flow->assigned_results = NULL; + } + + ifnet_t ifp = NULL; + if ((inp->inp_flags & INP_BOUND_IF) && inp->inp_boundifp) { + ifp = inp->inp_boundifp; + } else { + ifp = inp->inp_last_outifp; + } + + if (ifp != NULL) { + flow->interface_index = ifp->if_index; + } else { + flow->interface_index = IFSCOPE_NONE; } - client->assigned_results = assigned_results; - client->assigned_results_length = assigned_results_length; - client->assigned_result_read = FALSE; + + if (inp->inp_vflag & INP_IPV4) { + flow->local_addr.sin.sin_family = AF_INET; + flow->local_addr.sin.sin_len = sizeof(struct sockaddr_in); + flow->local_addr.sin.sin_port = inp->inp_lport; + memcpy(&flow->local_addr.sin.sin_addr, &inp->inp_laddr, sizeof(struct in_addr)); + + flow->remote_addr.sin.sin_family = AF_INET; + flow->remote_addr.sin.sin_len = sizeof(struct sockaddr_in); + flow->remote_addr.sin.sin_port = inp->inp_fport; + memcpy(&flow->remote_addr.sin.sin_addr, &inp->inp_faddr, sizeof(struct in_addr)); + } else if (inp->inp_vflag & INP_IPV6) { + in6_ip6_to_sockaddr(&inp->in6p_laddr, inp->inp_lport, &flow->local_addr.sin6, sizeof(flow->local_addr)); + in6_ip6_to_sockaddr(&inp->in6p_faddr, inp->inp_fport, &flow->remote_addr.sin6, sizeof(flow->remote_addr)); + } + + flow->viable = necp_client_flow_is_viable(proc, client, flow); + + uuid_t empty_uuid; + uuid_clear(empty_uuid); + flow->assigned = TRUE; + flow->assigned_results = necp_create_nexus_assign_message(empty_uuid, 0, NULL, 0, + (struct necp_client_endpoint *)&flow->local_addr, + (struct necp_client_endpoint *)&flow->remote_addr, + 0, &flow->assigned_results_length); + client->flow_result_read = FALSE; client_updated = TRUE; + break; } } + + NECP_CLIENT_UNLOCK(client); } if (client_updated) { necp_fd_notify(client_fd, true); } - lck_mtx_unlock(&client_fd->fd_lock); + NECP_FD_UNLOCK(client_fd); + + proc_rele(proc); + proc = PROC_NULL; if (found_client) { break; } } + NECP_FD_LIST_UNLOCK(); - lck_rw_done(&necp_fd_lock); + if (!found_client) { + error = ENOENT; + } else if (!client_updated) { + error = EINVAL; + } + + return (error); +} + +int +necp_update_flow_protoctl_event(uuid_t netagent_uuid, uuid_t client_id, + uint32_t protoctl_event_code, uint32_t protoctl_event_val, + uint32_t protoctl_event_tcp_seq_number) +{ + int error = 0; + struct necp_fd_data *client_fd = NULL; + bool found_client = FALSE; + bool client_updated = FALSE; + + NECP_FD_LIST_LOCK_SHARED(); + LIST_FOREACH(client_fd, &necp_fd_list, chain) { + proc_t proc = proc_find(client_fd->proc_pid); + if (proc == PROC_NULL) { + continue; + } + + NECP_FD_LOCK(client_fd); + + struct necp_client *client = necp_client_fd_find_client_and_lock(client_fd, client_id); + if (client != NULL) { + /* Found the right client! */ + found_client = TRUE; + + struct necp_client_flow *flow = NULL; + LIST_FOREACH(flow, &client->flow_list, flow_chain) { + // Verify that the client nexus agent matches + if (flow->nexus && + uuid_compare(flow->u.nexus_agent, + netagent_uuid) == 0) { + flow->has_protoctl_event = TRUE; + flow->protoctl_event.protoctl_event_code = protoctl_event_code; + flow->protoctl_event.protoctl_event_val = protoctl_event_val; + flow->protoctl_event.protoctl_event_tcp_seq_num = protoctl_event_tcp_seq_number; + client->flow_result_read = FALSE; + client_updated = TRUE; + break; + } + } + + NECP_CLIENT_UNLOCK(client); + } + + if (client_updated) { + necp_fd_notify(client_fd, true); + } + + NECP_FD_UNLOCK(client_fd); + proc_rele(proc); + proc = PROC_NULL; + + if (found_client) { + break; + } + } + NECP_FD_LIST_UNLOCK(); + if (!found_client) { + error = ENOENT; + } else if (!client_updated) { + error = EINVAL; + } + return (error); +} + +static bool +necp_assign_client_result_locked(struct proc *proc, struct necp_fd_data *client_fd, struct necp_client *client, uuid_t netagent_uuid, u_int8_t *assigned_results, size_t assigned_results_length) +{ + bool client_updated = FALSE; + + NECP_FD_ASSERT_LOCKED(client_fd); + NECP_CLIENT_ASSERT_LOCKED(client); + + struct necp_client_flow *flow = NULL; + LIST_FOREACH(flow, &client->flow_list, flow_chain) { + // Verify that the client nexus agent matches + if (flow->nexus && + uuid_compare(flow->u.nexus_agent, netagent_uuid) == 0) { + // Release prior results and route + if (flow->assigned_results != NULL) { + FREE(flow->assigned_results, M_NETAGENT); + flow->assigned_results = NULL; + } + + if (assigned_results != NULL && assigned_results_length > 0) { + int error = necp_client_parse_result(assigned_results, (u_int32_t)assigned_results_length, + &flow->local_addr, &flow->remote_addr); + VERIFY(error == 0); + } + + flow->viable = necp_client_flow_is_viable(proc, client, flow); + + flow->assigned = TRUE; + flow->assigned_results = assigned_results; + flow->assigned_results_length = assigned_results_length; + client->flow_result_read = FALSE; + client_updated = TRUE; + break; + } + } + + if (client_updated) { + necp_fd_notify(client_fd, true); + } + + // if not updated, client must free assigned_results + return (client_updated); +} + +int +necp_assign_client_result(uuid_t netagent_uuid, uuid_t client_id, + u_int8_t *assigned_results, size_t assigned_results_length) +{ + int error = 0; + struct necp_fd_data *client_fd = NULL; + bool found_client = FALSE; + bool client_updated = FALSE; + + NECP_FD_LIST_LOCK_SHARED(); + + LIST_FOREACH(client_fd, &necp_fd_list, chain) { + proc_t proc = proc_find(client_fd->proc_pid); + if (proc == PROC_NULL) { + continue; + } + + NECP_FD_LOCK(client_fd); + struct necp_client *client = necp_client_fd_find_client_and_lock(client_fd, client_id); + if (client != NULL) { + // Found the right client! + found_client = TRUE; + + if (necp_assign_client_result_locked(proc, client_fd, client, netagent_uuid, assigned_results, assigned_results_length)) { + client_updated = TRUE; + } + + NECP_CLIENT_UNLOCK(client); + } + NECP_FD_UNLOCK(client_fd); + + proc_rele(proc); + proc = PROC_NULL; + + if (found_client) { + break; + } + } + + NECP_FD_LIST_UNLOCK(); + + // upon error, client must free assigned_results if (!found_client) { error = ENOENT; } else if (!client_updated) { @@ -898,81 +2111,177 @@ necp_assign_client_result(uuid_t netagent_uuid, uuid_t client_id, static bool necp_update_client_result(proc_t proc, - struct necp_client *client) + struct necp_fd_data *client_fd, + struct necp_client *client, + struct _necp_client_defunct_list *defunct_list) { struct necp_client_result_netagent netagent; struct necp_aggregate_result result; - struct necp_client_parsed_parameters parsed_parameters; + struct necp_client_parsed_parameters *parsed_parameters = NULL; u_int32_t flags = 0; + struct rtentry *route = NULL; - uuid_clear(client->nexus_agent); + NECP_CLIENT_ASSERT_LOCKED(client); - int error = necp_client_parse_parameters(client->parameters, (u_int32_t)client->parameters_length, &parsed_parameters); + MALLOC(parsed_parameters, struct necp_client_parsed_parameters *, sizeof(*parsed_parameters), M_NECP, (M_WAITOK | M_ZERO)); + if (parsed_parameters == NULL) { + NECPLOG0(LOG_ERR, "Failed to allocate parsed parameters"); + return (FALSE); + } + + // Nexus flows will be brought back if they are still valid + necp_client_mark_all_nonsocket_flows_as_invalid(client); + + int error = necp_client_parse_parameters(client->parameters, (u_int32_t)client->parameters_length, parsed_parameters); if (error != 0) { + FREE(parsed_parameters, M_NECP); return (FALSE); } + // Update saved IP protocol + client->ip_protocol = parsed_parameters->ip_protocol; + // Check parameters to find best interface u_int matching_if_index = 0; - if (necp_find_matching_interface_index(&parsed_parameters, &matching_if_index)) { + if (necp_find_matching_interface_index(parsed_parameters, &matching_if_index)) { if (matching_if_index != 0) { - parsed_parameters.required_interface_index = matching_if_index; + parsed_parameters->required_interface_index = matching_if_index; } // Interface found or not needed, match policy. - error = necp_application_find_policy_match_internal(proc, client->parameters, (u_int32_t)client->parameters_length, &result, &flags, matching_if_index); + error = necp_application_find_policy_match_internal(proc, client->parameters, + (u_int32_t)client->parameters_length, + &result, &flags, matching_if_index, + NULL, NULL, &route, false); if (error != 0) { + if (route != NULL) { + rtfree(route); + } + FREE(parsed_parameters, M_NECP); return (FALSE); } + + // Reset current route + NECP_CLIENT_ROUTE_LOCK(client); + if (client->current_route != NULL) { + rtfree(client->current_route); + } + client->current_route = route; + NECP_CLIENT_ROUTE_UNLOCK(client); } else { // Interface not found. Clear out the whole result, make everything fail. memset(&result, 0, sizeof(result)); } + // Save the last policy id on the client + client->policy_id = result.policy_id; + + if ((parsed_parameters->flags & NECP_CLIENT_PARAMETER_FLAG_MULTIPATH) || + ((parsed_parameters->flags & NECP_CLIENT_PARAMETER_FLAG_LISTENER) && + result.routing_result != NECP_KERNEL_POLICY_RESULT_SOCKET_SCOPED)) { + client->allow_multiple_flows = TRUE; + } else { + client->allow_multiple_flows = FALSE; + } + // If the original request was scoped, and the policy result matches, make sure the result is scoped if ((result.routing_result == NECP_KERNEL_POLICY_RESULT_NONE || result.routing_result == NECP_KERNEL_POLICY_RESULT_PASS) && result.routed_interface_index != IFSCOPE_NONE && - parsed_parameters.required_interface_index == result.routed_interface_index) { + parsed_parameters->required_interface_index == result.routed_interface_index) { result.routing_result = NECP_KERNEL_POLICY_RESULT_SOCKET_SCOPED; result.routing_result_parameter.scoped_interface_index = result.routed_interface_index; } + if (defunct_list != NULL && + result.routing_result == NECP_KERNEL_POLICY_RESULT_DROP) { + // If we are forced to drop the client, defunct it if it has flows + necp_defunct_client_for_policy(client, defunct_list); + } + + // Recalculate flags + if (client->defunct) { + flags |= NECP_CLIENT_RESULT_FLAG_DEFUNCT; + } + if (parsed_parameters->flags & NECP_CLIENT_PARAMETER_FLAG_LISTENER) { + // Listeners are valid as long as they aren't dropped + if (result.routing_result != NECP_KERNEL_POLICY_RESULT_DROP) { + flags |= NECP_CLIENT_RESULT_FLAG_SATISFIED; + } + } else if (result.routed_interface_index != 0) { + // Clients without flows determine viability based on having some routable interface + flags |= NECP_CLIENT_RESULT_FLAG_SATISFIED; + } + bool updated = FALSE; u_int8_t *cursor = client->result; - const u_int8_t *max = client->result + NECP_MAX_CLIENT_RESULT_SIZE; - cursor = necp_buffer_write_tlv_if_different(cursor, max, NECP_CLIENT_RESULT_CLIENT_ID, sizeof(uuid_t), client->client_id, &updated); - cursor = necp_buffer_write_tlv_if_different(cursor, max, NECP_CLIENT_RESULT_POLICY_RESULT, sizeof(result.routing_result), &result.routing_result, &updated); + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_FLAGS, sizeof(flags), &flags, &updated, client->result, sizeof(client->result)); + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_CLIENT_ID, sizeof(uuid_t), client->client_id, &updated, + client->result, sizeof(client->result)); + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_POLICY_RESULT, sizeof(result.routing_result), &result.routing_result, &updated, + client->result, sizeof(client->result)); if (result.routing_result_parameter.tunnel_interface_index != 0) { - cursor = necp_buffer_write_tlv_if_different(cursor, max, NECP_CLIENT_RESULT_POLICY_RESULT_PARAMETER, - sizeof(result.routing_result_parameter), &result.routing_result_parameter, &updated); + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_POLICY_RESULT_PARAMETER, + sizeof(result.routing_result_parameter), &result.routing_result_parameter, &updated, + client->result, sizeof(client->result)); } if (result.filter_control_unit != 0) { - cursor = necp_buffer_write_tlv_if_different(cursor, max, NECP_CLIENT_RESULT_FILTER_CONTROL_UNIT, - sizeof(result.filter_control_unit), &result.filter_control_unit, &updated); + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_FILTER_CONTROL_UNIT, + sizeof(result.filter_control_unit), &result.filter_control_unit, &updated, + client->result, sizeof(client->result)); } if (result.routed_interface_index != 0) { u_int routed_interface_index = result.routed_interface_index; if (result.routing_result == NECP_KERNEL_POLICY_RESULT_IP_TUNNEL && - parsed_parameters.required_interface_index != IFSCOPE_NONE && - parsed_parameters.required_interface_index != result.routed_interface_index) { - routed_interface_index = parsed_parameters.required_interface_index; + parsed_parameters->required_interface_index != IFSCOPE_NONE && + parsed_parameters->required_interface_index != result.routed_interface_index) { + routed_interface_index = parsed_parameters->required_interface_index; } - cursor = necp_buffer_write_tlv_if_different(cursor, max, NECP_CLIENT_RESULT_INTERFACE_INDEX, - sizeof(routed_interface_index), &routed_interface_index, &updated); + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_INTERFACE_INDEX, + sizeof(routed_interface_index), &routed_interface_index, &updated, + client->result, sizeof(client->result)); + } + if (client_fd && client_fd->flags & NECP_OPEN_FLAG_BACKGROUND) { + u_int32_t effective_traffic_class = SO_TC_BK_SYS; + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_EFFECTIVE_TRAFFIC_CLASS, + sizeof(effective_traffic_class), &effective_traffic_class, &updated, + client->result, sizeof(client->result)); + } + if (client->background_update) { + u_int32_t background = client->background; + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_TRAFFIC_MGMT_BG, + sizeof(background), &background, &updated, + client->result, sizeof(client->result)); + if (updated) { + client->background_update = 0; + } } - if (flags != 0) { - cursor = necp_buffer_write_tlv_if_different(cursor, max, NECP_CLIENT_RESULT_FLAGS, - sizeof(flags), &flags, &updated); + NECP_CLIENT_ROUTE_LOCK(client); + if (client->current_route != NULL) { + const u_int32_t route_mtu = get_maxmtu(client->current_route); + if (route_mtu != 0) { + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_EFFECTIVE_MTU, + sizeof(route_mtu), &route_mtu, &updated, + client->result, sizeof(client->result)); + } } + NECP_CLIENT_ROUTE_UNLOCK(client); + + if (result.mss_recommended != 0) { + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_RECOMMENDED_MSS, + sizeof(result.mss_recommended), &result.mss_recommended, &updated, + client->result, sizeof(client->result)); + } + for (int i = 0; i < NECP_MAX_NETAGENTS; i++) { if (uuid_is_null(result.netagents[i])) { break; } uuid_copy(netagent.netagent_uuid, result.netagents[i]); netagent.generation = netagent_get_generation(netagent.netagent_uuid); - if (necp_netagent_applies_to_client(client, &parsed_parameters, netagent.netagent_uuid)) { - cursor = necp_buffer_write_tlv_if_different(cursor, max, NECP_CLIENT_RESULT_NETAGENT, sizeof(netagent), &netagent, &updated); + if (necp_netagent_applies_to_client(client, parsed_parameters, netagent.netagent_uuid, TRUE, 0, 0)) { + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_NETAGENT, sizeof(netagent), &netagent, &updated, + client->result, sizeof(client->result)); } } @@ -983,10 +2292,10 @@ necp_update_client_result(proc_t proc, if (result.routed_interface_index != IFSCOPE_NONE && result.routed_interface_index <= (u_int32_t)if_index) { direct_interface = ifindex2ifnet[result.routed_interface_index]; - } else if (parsed_parameters.required_interface_index != IFSCOPE_NONE && - parsed_parameters.required_interface_index <= (u_int32_t)if_index) { + } else if (parsed_parameters->required_interface_index != IFSCOPE_NONE && + parsed_parameters->required_interface_index <= (u_int32_t)if_index) { // If the request was scoped, but the route didn't match, still grab the agents - direct_interface = ifindex2ifnet[parsed_parameters.required_interface_index]; + direct_interface = ifindex2ifnet[parsed_parameters->required_interface_index]; } else if (result.routed_interface_index == IFSCOPE_NONE && result.routing_result == NECP_KERNEL_POLICY_RESULT_SOCKET_SCOPED && result.routing_result_parameter.scoped_interface_index != IFSCOPE_NONE) { @@ -996,30 +2305,59 @@ necp_update_client_result(proc_t proc, delegate_interface = direct_interface->if_delegated.ifp; } if (result.routing_result == NECP_KERNEL_POLICY_RESULT_IP_TUNNEL && - parsed_parameters.required_interface_index != IFSCOPE_NONE && - parsed_parameters.required_interface_index != result.routing_result_parameter.tunnel_interface_index && - parsed_parameters.required_interface_index <= (u_int32_t)if_index) { - original_scoped_interface = ifindex2ifnet[parsed_parameters.required_interface_index]; + parsed_parameters->required_interface_index != IFSCOPE_NONE && + parsed_parameters->required_interface_index != result.routing_result_parameter.tunnel_interface_index && + parsed_parameters->required_interface_index <= (u_int32_t)if_index) { + original_scoped_interface = ifindex2ifnet[parsed_parameters->required_interface_index]; } // Add interfaces if (original_scoped_interface != NULL) { struct necp_client_result_interface interface_struct; interface_struct.index = original_scoped_interface->if_index; interface_struct.generation = ifnet_get_generation(original_scoped_interface); - cursor = necp_buffer_write_tlv_if_different(cursor, max, NECP_CLIENT_RESULT_INTERFACE, sizeof(interface_struct), &interface_struct, &updated); + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_INTERFACE, sizeof(interface_struct), &interface_struct, &updated, + client->result, sizeof(client->result)); } if (direct_interface != NULL) { struct necp_client_result_interface interface_struct; interface_struct.index = direct_interface->if_index; interface_struct.generation = ifnet_get_generation(direct_interface); - cursor = necp_buffer_write_tlv_if_different(cursor, max, NECP_CLIENT_RESULT_INTERFACE, sizeof(interface_struct), &interface_struct, &updated); + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_INTERFACE, sizeof(interface_struct), &interface_struct, &updated, + client->result, sizeof(client->result)); } if (delegate_interface != NULL) { struct necp_client_result_interface interface_struct; interface_struct.index = delegate_interface->if_index; interface_struct.generation = ifnet_get_generation(delegate_interface); - cursor = necp_buffer_write_tlv_if_different(cursor, max, NECP_CLIENT_RESULT_INTERFACE, sizeof(interface_struct), &interface_struct, &updated); + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_INTERFACE, sizeof(interface_struct), &interface_struct, &updated, + client->result, sizeof(client->result)); + } + + // Update multipath/listener interface flows + if (parsed_parameters->flags & NECP_CLIENT_PARAMETER_FLAG_MULTIPATH) { + // Get multipath interface options from ordered list + struct ifnet *multi_interface = NULL; + TAILQ_FOREACH(multi_interface, &ifnet_ordered_head, if_ordered_link) { + if (necp_ifnet_matches_parameters(multi_interface, parsed_parameters, NULL, true)) { + // Add multipath interface flows for kernel MPTCP + necp_client_add_interface_flow_if_needed(client, multi_interface->if_index); + + // Add nexus agents for multipath + necp_client_add_agent_flows_for_interface(client, parsed_parameters, multi_interface); + } + } + } else if ((parsed_parameters->flags & NECP_CLIENT_PARAMETER_FLAG_LISTENER) && + result.routing_result != NECP_KERNEL_POLICY_RESULT_SOCKET_SCOPED) { + // Get listener interface options from global list + struct ifnet *listen_interface = NULL; + TAILQ_FOREACH(listen_interface, &ifnet_head, if_link) { + if (necp_ifnet_matches_parameters(listen_interface, parsed_parameters, NULL, false)) { + // Add nexus agents for listeners + necp_client_add_agent_flows_for_interface(client, parsed_parameters, listen_interface); + } + } } + // Add agents if (original_scoped_interface != NULL) { ifnet_lock_shared(original_scoped_interface); @@ -1028,10 +2366,12 @@ necp_update_client_result(proc_t proc, if (uuid_is_null(original_scoped_interface->if_agentids[i])) { continue; } + u_int16_t if_flags = nstat_ifnet_to_flags(original_scoped_interface); uuid_copy(netagent.netagent_uuid, original_scoped_interface->if_agentids[i]); netagent.generation = netagent_get_generation(netagent.netagent_uuid); - if (necp_netagent_applies_to_client(client, &parsed_parameters, netagent.netagent_uuid)) { - cursor = necp_buffer_write_tlv_if_different(cursor, max, NECP_CLIENT_RESULT_NETAGENT, sizeof(netagent), &netagent, &updated); + if (necp_netagent_applies_to_client(client, parsed_parameters, netagent.netagent_uuid, FALSE, original_scoped_interface->if_index, if_flags)) { + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_NETAGENT, sizeof(netagent), &netagent, &updated, + client->result, sizeof(client->result)); } } } @@ -1044,10 +2384,12 @@ necp_update_client_result(proc_t proc, if (uuid_is_null(direct_interface->if_agentids[i])) { continue; } + u_int16_t if_flags = nstat_ifnet_to_flags(direct_interface); uuid_copy(netagent.netagent_uuid, direct_interface->if_agentids[i]); netagent.generation = netagent_get_generation(netagent.netagent_uuid); - if (necp_netagent_applies_to_client(client, &parsed_parameters, netagent.netagent_uuid)) { - cursor = necp_buffer_write_tlv_if_different(cursor, max, NECP_CLIENT_RESULT_NETAGENT, sizeof(netagent), &netagent, &updated); + if (necp_netagent_applies_to_client(client, parsed_parameters, netagent.netagent_uuid, TRUE, direct_interface->if_index, if_flags)) { + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_NETAGENT, sizeof(netagent), &netagent, &updated, + client->result, sizeof(client->result)); } } } @@ -1060,10 +2402,12 @@ necp_update_client_result(proc_t proc, if (uuid_is_null(delegate_interface->if_agentids[i])) { continue; } + u_int16_t if_flags = nstat_ifnet_to_flags(delegate_interface); uuid_copy(netagent.netagent_uuid, delegate_interface->if_agentids[i]); netagent.generation = netagent_get_generation(netagent.netagent_uuid); - if (necp_netagent_applies_to_client(client, &parsed_parameters, netagent.netagent_uuid)) { - cursor = necp_buffer_write_tlv_if_different(cursor, max, NECP_CLIENT_RESULT_NETAGENT, sizeof(netagent), &netagent, &updated); + if (necp_netagent_applies_to_client(client, parsed_parameters, netagent.netagent_uuid, FALSE, delegate_interface->if_index, if_flags)) { + cursor = necp_buffer_write_tlv_if_different(cursor, NECP_CLIENT_RESULT_NETAGENT, sizeof(netagent), &netagent, &updated, + client->result, sizeof(client->result)); } } } @@ -1076,62 +2420,262 @@ necp_update_client_result(proc_t proc, client->result_length = new_result_length; updated = TRUE; } + + // Update flow viability/flags + if (necp_client_update_flows(proc, client)) { + updated = TRUE; + } + if (updated) { client->result_read = FALSE; + necp_client_update_observer_update(client); } + FREE(parsed_parameters, M_NECP); return (updated); } +static inline void +necp_defunct_client_fd_locked(struct necp_fd_data *client_fd, struct _necp_client_defunct_list *defunct_list) +{ + bool updated_result = FALSE; + struct necp_client *client = NULL; + + NECP_FD_ASSERT_LOCKED(client_fd); + RB_FOREACH(client, _necp_client_tree, &client_fd->clients) { + NECP_CLIENT_LOCK(client); + if (!client->defunct) { + updated_result = necp_set_client_defunct(client); + + // Prepare close events to be sent to the nexus to effectively remove the flows + struct necp_client_flow *search_flow = NULL; + LIST_FOREACH(search_flow, &client->flow_list, flow_chain) { + if (search_flow->nexus && + !uuid_is_null(search_flow->u.nexus_agent) && + search_flow->requested_nexus) { + + struct necp_client_defunct *client_defunct; + + // Sleeping alloc won't fail; copy only what's necessary + client_defunct = _MALLOC(sizeof (struct necp_client_defunct), M_NECP, M_WAITOK | M_ZERO); + uuid_copy(client_defunct->nexus_agent, search_flow->u.nexus_agent); + uuid_copy(client_defunct->client_id, client->client_id); + client_defunct->proc_pid = client->proc_pid; + + // Add to the list provided by caller + LIST_INSERT_HEAD(defunct_list, client_defunct, chain); + } + } + } + NECP_CLIENT_UNLOCK(client); + } + if (updated_result) { + necp_fd_notify(client_fd, true); + } +} + +static inline void +necp_update_client_fd_locked(struct necp_fd_data *client_fd, + proc_t proc, + struct _necp_client_defunct_list *defunct_list) +{ + struct necp_client *client = NULL; + bool updated_result = FALSE; + NECP_FD_ASSERT_LOCKED(client_fd); + RB_FOREACH(client, _necp_client_tree, &client_fd->clients) { + NECP_CLIENT_LOCK(client); + if (necp_update_client_result(proc, client_fd, client, defunct_list)) { + updated_result = TRUE; + } + NECP_CLIENT_UNLOCK(client); + } + if (updated_result) { + necp_fd_notify(client_fd, true); + } +} + + static void necp_update_all_clients_callout(__unused thread_call_param_t dummy, __unused thread_call_param_t arg) { -#pragma unused(arg) struct necp_fd_data *client_fd = NULL; - lck_rw_lock_shared(&necp_fd_lock); + struct _necp_client_defunct_list defunct_list; + LIST_INIT(&defunct_list); + + NECP_FD_LIST_LOCK_SHARED(); LIST_FOREACH(client_fd, &necp_fd_list, chain) { - bool updated_result = FALSE; - struct necp_client *client = NULL; proc_t proc = proc_find(client_fd->proc_pid); - if (proc == NULL) { + if (proc == PROC_NULL) { continue; } - lck_mtx_lock(&client_fd->fd_lock); - LIST_FOREACH(client, &client_fd->clients, chain) { - if (necp_update_client_result(proc, client)) { - updated_result = TRUE; + // Update all clients on one fd + NECP_FD_LOCK(client_fd); + necp_update_client_fd_locked(client_fd, proc, &defunct_list); + NECP_FD_UNLOCK(client_fd); + + proc_rele(proc); + proc = PROC_NULL; + } + + NECP_FD_LIST_UNLOCK(); + + // Handle the case in which some clients became newly defunct + if (!LIST_EMPTY(&defunct_list)) { + struct necp_client_defunct *client_defunct = NULL; + struct necp_client_defunct *temp_client_defunct = NULL; + + // For each newly defunct client, send a message to the nexus to remove the flow + LIST_FOREACH_SAFE(client_defunct, &defunct_list, chain, temp_client_defunct) { + if (!uuid_is_null(client_defunct->nexus_agent)) { + int netagent_error = netagent_client_message(client_defunct->nexus_agent, + client_defunct->client_id, + client_defunct->proc_pid, + NETAGENT_MESSAGE_TYPE_ABORT_NEXUS); + if (netagent_error != 0) { + NECPLOG((netagent_error == ENOENT ? LOG_DEBUG : LOG_ERR), "necp_update_client abort nexus error (%d)", netagent_error); + } } + LIST_REMOVE(client_defunct, chain); + FREE(client_defunct, M_NECP); } - if (updated_result) { - necp_fd_notify(client_fd, true); + } + ASSERT(LIST_EMPTY(&defunct_list)); +} + +void +necp_update_all_clients(void) +{ + if (necp_client_update_tcall == NULL) { + // Don't try to update clients if the module is not initialized + return; + } + + uint64_t deadline = 0; + uint64_t leeway = 0; + clock_interval_to_deadline(necp_timeout_microseconds, NSEC_PER_USEC, &deadline); + clock_interval_to_absolutetime_interval(necp_timeout_leeway_microseconds, NSEC_PER_USEC, &leeway); + + thread_call_enter_delayed_with_leeway(necp_client_update_tcall, NULL, + deadline, leeway, THREAD_CALL_DELAY_LEEWAY); +} + +void +necp_set_client_as_background(proc_t proc, + struct fileproc *fp, + bool background) +{ + bool updated_result = FALSE; + struct necp_client *client = NULL; + + if (proc == PROC_NULL) { + NECPLOG0(LOG_ERR, "NULL proc"); + return; + } + + if (fp == NULL) { + NECPLOG0(LOG_ERR, "NULL fp"); + return; + } + + struct necp_fd_data *client_fd = (struct necp_fd_data *)fp->f_fglob->fg_data; + if (client_fd == NULL) { + NECPLOG0(LOG_ERR, "Could not find client structure for backgrounded client"); + return; + } + + if (client_fd->necp_fd_type != necp_fd_type_client) { + // Not a client fd, ignore + NECPLOG0(LOG_ERR, "Not a client fd, ignore"); + return; + } + + NECP_FD_LOCK(client_fd); + + RB_FOREACH(client, _necp_client_tree, &client_fd->clients) { + NECP_CLIENT_LOCK(client); + + bool has_assigned_flow = FALSE; + struct necp_client_flow *search_flow = NULL; + LIST_FOREACH(search_flow, &client->flow_list, flow_chain) { + if (search_flow->assigned) { + has_assigned_flow = TRUE; + break; + } } - lck_mtx_unlock(&client_fd->fd_lock); - proc_rele(proc); + if (has_assigned_flow) { + client->background = background; + client->background_update = TRUE; + updated_result = TRUE; + } + + NECP_CLIENT_UNLOCK(client); + } + if (updated_result) { + necp_update_client_fd_locked(client_fd, proc, NULL); + } + NECP_FD_UNLOCK(client_fd); +} + +void +necp_defunct_client(proc_t proc, + struct fileproc *fp) +{ + struct _necp_client_defunct_list defunct_list; + + if (proc == PROC_NULL) { + NECPLOG0(LOG_ERR, "NULL proc passed to set as background"); + return; + } + + if (fp == NULL) { + NECPLOG0(LOG_ERR, "NULL fp passed to set as background"); + return; } - lck_rw_done(&necp_fd_lock); -} + struct necp_fd_data *client_fd = (struct necp_fd_data *)fp->f_fglob->fg_data; + if (client_fd == NULL) { + NECPLOG0(LOG_ERR, "Could not find client structure for backgrounded client"); + return; + } -void -necp_update_all_clients(void) -{ - if (necp_client_tcall == NULL) { - // Don't try to update clients if the module is not initialized + if (client_fd->necp_fd_type != necp_fd_type_client) { + // Not a client fd, ignore return; } - uint64_t deadline = 0; - uint64_t leeway = 0; - clock_interval_to_deadline(necp_timeout_microseconds, NSEC_PER_USEC, &deadline); - clock_interval_to_absolutetime_interval(necp_timeout_leeway_microseconds, NSEC_PER_USEC, &leeway); + // Our local temporary list + LIST_INIT(&defunct_list); - thread_call_enter_delayed_with_leeway(necp_client_tcall, NULL, - deadline, leeway, THREAD_CALL_DELAY_LEEWAY); + // Need to hold lock so ntstats defunct the same set of clients + NECP_FD_LOCK(client_fd); + necp_defunct_client_fd_locked(client_fd, &defunct_list); + NECP_FD_UNLOCK(client_fd); + + if (!LIST_EMPTY(&defunct_list)) { + struct necp_client_defunct *client_defunct = NULL; + struct necp_client_defunct *temp_client_defunct = NULL; + + // For each defunct client, remove flow from the nexus + LIST_FOREACH_SAFE(client_defunct, &defunct_list, chain, temp_client_defunct) { + if (!uuid_is_null(client_defunct->nexus_agent)) { + int netagent_error = netagent_client_message(client_defunct->nexus_agent, + client_defunct->client_id, + client_defunct->proc_pid, + NETAGENT_MESSAGE_TYPE_ABORT_NEXUS); + if (netagent_error != 0) { + NECPLOG((netagent_error == ENOENT ? LOG_DEBUG : LOG_ERR), "necp_defunct_client abort nexus error (%d)", netagent_error); + } + } + LIST_REMOVE(client_defunct, chain); + FREE(client_defunct, M_NECP); + } + } + ASSERT(LIST_EMPTY(&defunct_list)); } static void @@ -1140,11 +2684,11 @@ necp_client_remove_agent_from_result(struct necp_client *client, uuid_t netagent size_t offset = 0; u_int8_t *result_buffer = client->result; - while ((offset + sizeof(u_int8_t) + sizeof(u_int32_t)) <= client->result_length) { + while ((offset + sizeof(struct necp_tlv_header)) <= client->result_length) { u_int8_t type = necp_buffer_get_tlv_type(result_buffer, offset); u_int32_t length = necp_buffer_get_tlv_length(result_buffer, offset); - size_t tlv_total_length = (sizeof(u_int8_t) + sizeof(u_int32_t) + length); + size_t tlv_total_length = (sizeof(struct necp_tlv_header) + length); if (type == NECP_CLIENT_RESULT_NETAGENT && length == sizeof(struct necp_client_result_netagent) && (offset + tlv_total_length) <= client->result_length) { @@ -1158,7 +2702,7 @@ necp_client_remove_agent_from_result(struct necp_client *client, uuid_t netagent result_buffer + offset + tlv_total_length, client->result_length - (offset + tlv_total_length)); client->result_length -= tlv_total_length; - memset(result_buffer + client->result_length, 0, NECP_MAX_CLIENT_RESULT_SIZE - client->result_length); + memset(result_buffer + client->result_length, 0, sizeof(client->result) - client->result_length); continue; } } @@ -1172,36 +2716,35 @@ necp_force_update_client(uuid_t client_id, uuid_t remove_netagent_uuid) { struct necp_fd_data *client_fd = NULL; - lck_rw_lock_shared(&necp_fd_lock); + NECP_FD_LIST_LOCK_SHARED(); LIST_FOREACH(client_fd, &necp_fd_list, chain) { bool updated_result = FALSE; - struct necp_client *client = NULL; - lck_mtx_lock(&client_fd->fd_lock); - LIST_FOREACH(client, &client_fd->clients, chain) { - if (uuid_compare(client->client_id, client_id) == 0) { - if (!uuid_is_null(remove_netagent_uuid)) { - necp_client_remove_agent_from_result(client, remove_netagent_uuid); - } - client->assigned_result_read = FALSE; - updated_result = TRUE; - // Found the client, break - break; + NECP_FD_LOCK(client_fd); + struct necp_client *client = necp_client_fd_find_client_and_lock(client_fd, client_id); + if (client != NULL) { + if (!uuid_is_null(remove_netagent_uuid)) { + necp_client_remove_agent_from_result(client, remove_netagent_uuid); } + client->flow_result_read = FALSE; + // Found the client, break + updated_result = TRUE; + NECP_CLIENT_UNLOCK(client); } if (updated_result) { necp_fd_notify(client_fd, true); } - lck_mtx_unlock(&client_fd->fd_lock); + NECP_FD_UNLOCK(client_fd); if (updated_result) { // Found the client, break break; } } - lck_rw_done(&necp_fd_lock); + NECP_FD_LIST_UNLOCK(); } + /// Interface matching #define NECP_PARSED_PARAMETERS_INTERESTING_IFNET_FIELDS (NECP_PARSED_PARAMETERS_FIELD_LOCAL_ADDR | \ @@ -1346,10 +2889,28 @@ necp_ifnet_matches_local_address(struct ifnet *ifp, struct sockaddr *sa) return (matched_local_address); } +static bool +necp_interface_type_is_primary_eligible(u_int8_t interface_type) +{ + switch (interface_type) { + // These types can never be primary, so a client requesting these types is allowed + // to match an interface that isn't currently eligible to be primary (has default + // route, dns, etc) + case IFRTYPE_FUNCTIONAL_WIFI_AWDL: + case IFRTYPE_FUNCTIONAL_INTCOPROC: + return false; + default: + break; + } + return true; +} + +#define NECP_IFP_IS_ON_ORDERED_LIST(_ifp) ((_ifp)->if_ordered_link.tqe_next != NULL || (_ifp)->if_ordered_link.tqe_prev != NULL) + static bool necp_ifnet_matches_parameters(struct ifnet *ifp, struct necp_client_parsed_parameters *parsed_parameters, - u_int32_t *preferred_count) + u_int32_t *preferred_count, bool ignore_require_if) { if (preferred_count) { *preferred_count = 0; @@ -1361,18 +2922,19 @@ necp_ifnet_matches_parameters(struct ifnet *ifp, } } - if (parsed_parameters->valid_fields & NECP_PARSED_PARAMETERS_FIELD_REQUIRED_IFTYPE) { - for (int i = 0; i < NECP_MAX_PARSED_PARAMETERS; i++) { - if (parsed_parameters->required_interface_types[i] == 0) { - break; - } - - if (!necp_ifnet_matches_type(ifp, parsed_parameters->required_interface_types[i], FALSE)) { - return (FALSE); - } + if (parsed_parameters->valid_fields & NECP_PARSED_PARAMETERS_FIELD_FLAGS) { + if ((parsed_parameters->flags & NECP_CLIENT_PARAMETER_FLAG_PROHIBIT_EXPENSIVE) && + IFNET_IS_EXPENSIVE(ifp)) { + return (FALSE); } } + if (!ignore_require_if && + (parsed_parameters->valid_fields & NECP_PARSED_PARAMETERS_FIELD_REQUIRED_IFTYPE) && + !necp_ifnet_matches_type(ifp, parsed_parameters->required_interface_type, FALSE)) { + return (FALSE); + } + if (parsed_parameters->valid_fields & NECP_PARSED_PARAMETERS_FIELD_PROHIBITED_IFTYPE) { for (int i = 0; i < NECP_MAX_PARSED_PARAMETERS; i++) { if (parsed_parameters->prohibited_interface_types[i] == 0) { @@ -1502,9 +3064,9 @@ necp_find_matching_interface_index(struct necp_client_parsed_parameters *parsed_ if (!(parsed_parameters->valid_fields & NECP_PARSED_PARAMETERS_SCOPED_IFNET_FIELDS)) { // We do have fields to match, but they are only prohibitory - // If the first interface in the list matches, we don't need to scope + // If the first interface in the list matches, or there are no ordered interfaces, we don't need to scope ifp = TAILQ_FIRST(&ifnet_ordered_head); - if (ifp && necp_ifnet_matches_parameters(ifp, parsed_parameters, NULL)) { + if (ifp == NULL || necp_ifnet_matches_parameters(ifp, parsed_parameters, NULL, false)) { // Don't set return_ifindex, so the client doesn't need to scope ifnet_head_done(); return (TRUE); @@ -1514,7 +3076,7 @@ necp_find_matching_interface_index(struct necp_client_parsed_parameters *parsed_ // First check the ordered interface list TAILQ_FOREACH(ifp, &ifnet_ordered_head, if_ordered_link) { u_int32_t preferred_count = 0; - if (necp_ifnet_matches_parameters(ifp, parsed_parameters, &preferred_count)) { + if (necp_ifnet_matches_parameters(ifp, parsed_parameters, &preferred_count, false)) { if (preferred_count > best_preferred_count || *return_ifindex == 0) { @@ -1531,16 +3093,16 @@ necp_find_matching_interface_index(struct necp_client_parsed_parameters *parsed_ // Then check the remaining interfaces if ((parsed_parameters->valid_fields & NECP_PARSED_PARAMETERS_SCOPED_IFNET_FIELDS) && - !(parsed_parameters->valid_fields & NECP_PARSED_PARAMETERS_FIELD_REQUIRED_IFTYPE) && + ((!(parsed_parameters->valid_fields & NECP_PARSED_PARAMETERS_FIELD_REQUIRED_IFTYPE)) || + !necp_interface_type_is_primary_eligible(parsed_parameters->required_interface_type)) && *return_ifindex == 0) { TAILQ_FOREACH(ifp, &ifnet_head, if_link) { u_int32_t preferred_count = 0; - if (ifp->if_ordered_link.tqe_next != NULL || - ifp->if_ordered_link.tqe_prev != NULL) { + if (NECP_IFP_IS_ON_ORDERED_LIST(ifp)) { // This interface was in the ordered list, skip continue; } - if (necp_ifnet_matches_parameters(ifp, parsed_parameters, &preferred_count)) { + if (necp_ifnet_matches_parameters(ifp, parsed_parameters, &preferred_count, false)) { if (preferred_count > best_preferred_count || *return_ifindex == 0) { @@ -1568,186 +3130,6 @@ necp_find_matching_interface_index(struct necp_client_parsed_parameters *parsed_ return (*return_ifindex != 0); } -static void -necp_find_netstat_data(struct necp_client *client, union necp_sockaddr_union *local, union necp_sockaddr_union *remote, u_int32_t *ifindex, uuid_t euuid, u_int32_t *traffic_class) -{ - size_t offset = 0; - u_int8_t *parameters; - u_int32_t parameters_size; - - parameters = client->parameters; - parameters_size = (u_int32_t)client->parameters_length; - - while ((offset + sizeof(u_int8_t) + sizeof(u_int32_t)) <= parameters_size) { - u_int8_t type = necp_buffer_get_tlv_type(parameters, offset); - u_int32_t length = necp_buffer_get_tlv_length(parameters, offset); - - if (length > (parameters_size - (offset + sizeof(u_int8_t) + sizeof(u_int32_t)))) { - // If the length is larger than what can fit in the remaining parameters size, bail - NECPLOG(LOG_ERR, "Invalid TLV length (%u)", length); - break; - } - - if (length > 0) { - u_int8_t *value = necp_buffer_get_tlv_value(parameters, offset, NULL); - if (value != NULL) { - switch (type) { - case NECP_CLIENT_PARAMETER_REAL_APPLICATION: { - if (length >= sizeof(uuid_t)) { - uuid_copy(euuid, value); - } - break; - } - case NECP_CLIENT_PARAMETER_TRAFFIC_CLASS: { - if (length >= sizeof(u_int32_t)) { - memcpy(traffic_class, value, sizeof(u_int32_t)); - } - break; - } - case NECP_CLIENT_PARAMETER_BOUND_INTERFACE: { - if (length <= IFXNAMSIZ && length > 0) { - ifnet_t bound_interface = NULL; - char interface_name[IFXNAMSIZ]; - memcpy(interface_name, value, length); - interface_name[length - 1] = 0; // Make sure the string is NULL terminated - if (ifnet_find_by_name(interface_name, &bound_interface) == 0) { - *ifindex = bound_interface->if_index; - ifnet_release(bound_interface); - } - } - break; - } - case NECP_CLIENT_PARAMETER_LOCAL_ADDRESS: { - if (length >= sizeof(struct necp_policy_condition_addr)) { - struct necp_policy_condition_addr *address_struct = (struct necp_policy_condition_addr *)(void *)value; - memcpy(local, &address_struct->address, sizeof(address_struct->address)); - } - break; - } - case NECP_CLIENT_PARAMETER_REMOTE_ADDRESS: { - if (length >= sizeof(struct necp_policy_condition_addr)) { - struct necp_policy_condition_addr *address_struct = (struct necp_policy_condition_addr *)(void *)value; - memcpy(remote, &address_struct->address, sizeof(address_struct->address)); - } - break; - } - default: { - break; - } - } - } - } - offset += sizeof(u_int8_t) + sizeof(u_int32_t) + length; - } -} - -static void -necp_fillout_current_process_details(u_int32_t *pid, u_int64_t *upid, unsigned char *uuid, char *pname, size_t len) -{ - *pid = proc_selfpid(); - *upid = proc_uniqueid(current_proc()); - proc_selfname(pname, (int) len); - proc_getexecutableuuid(current_proc(), uuid, sizeof(uuid_t)); -} - -// Called from NetworkStatistics when it wishes to collect latest information for a TCP flow. -// It is a responsibility of NetworkStatistics to have previously zeroed any supplied memory. -static bool -necp_request_tcp_netstats(userland_stats_provider_context *ctx, - nstat_counts *countsp, - void *metadatap) -{ - if (ctx == NULL) { - return false; - } - - struct necp_client *client = (struct necp_client *)ctx; - struct necp_tcp_stats *tcpstats = (struct necp_tcp_stats *)client->stats_area; - if (tcpstats == NULL) { - return false; - } - - if (countsp) { - *countsp = *((struct nstat_counts *)&tcpstats->necp_tcp_counts); - } - - if (metadatap) { - nstat_tcp_descriptor *desc = (nstat_tcp_descriptor *)metadatap; - - // Metadata for the process - necp_fillout_current_process_details(&desc->pid, &desc->upid, desc->uuid, desc->pname, sizeof(desc->pname)); - - // Metadata that the necp client should have in TLV format. - necp_find_netstat_data(client, (union necp_sockaddr_union *)&desc->local, (union necp_sockaddr_union *)&desc->remote, &desc->ifindex, desc->euuid, &desc->traffic_class); - - // Basic metadata - desc->rcvbufsize = tcpstats->necp_tcp_basic.rcvbufsize; - desc->rcvbufused = tcpstats->necp_tcp_basic.rcvbufused; - desc->eupid = tcpstats->necp_tcp_basic.eupid; - desc->epid = tcpstats->necp_tcp_basic.epid; - memcpy(desc->vuuid, tcpstats->necp_tcp_basic.vuuid, sizeof(desc->vuuid)); - desc->ifnet_properties = tcpstats->necp_tcp_basic.ifnet_properties; - - // Additional TCP specific data - desc->sndbufsize = tcpstats->necp_tcp_extra.sndbufsize; - desc->sndbufused = tcpstats->necp_tcp_extra.sndbufused; - desc->txunacked = tcpstats->necp_tcp_extra.txunacked; - desc->txwindow = tcpstats->necp_tcp_extra.txwindow; - desc->txcwindow = tcpstats->necp_tcp_extra.txcwindow; - desc->traffic_mgt_flags = tcpstats->necp_tcp_extra.traffic_mgt_flags; - - if (tcpstats->necp_tcp_extra.cc_alg_index < TCP_CC_ALGO_COUNT) { - strlcpy(desc->cc_algo, tcp_cc_algo_list[tcpstats->necp_tcp_extra.cc_alg_index]->name, sizeof(desc->cc_algo)); - } else { - strlcpy(desc->cc_algo, "unknown", sizeof(desc->cc_algo)); - } - - desc->connstatus.write_probe_failed = tcpstats->necp_tcp_extra.probestatus.write_probe_failed; - desc->connstatus.read_probe_failed = tcpstats->necp_tcp_extra.probestatus.read_probe_failed; - desc->connstatus.conn_probe_failed = tcpstats->necp_tcp_extra.probestatus.conn_probe_failed; - } - return true; -} - -// Called from NetworkStatistics when it wishes to collect latest information for a UDP flow. -static bool -necp_request_udp_netstats(userland_stats_provider_context *ctx, - nstat_counts *countsp, - void *metadatap) -{ - if (ctx == NULL) { - return false; - } - - struct necp_client *client = (struct necp_client *)ctx; - struct necp_udp_stats *udpstats = (struct necp_udp_stats *)client->stats_area; - if (udpstats == NULL) { - return false; - } - - if (countsp) { - *countsp = *((struct nstat_counts *)&udpstats->necp_udp_counts); - } - - if (metadatap) { - nstat_udp_descriptor *desc = (nstat_udp_descriptor *)metadatap; - - // Metadata for the process - necp_fillout_current_process_details(&desc->pid, &desc->upid, desc->uuid, desc->pname, sizeof(desc->pname)); - - // Metadata that the necp client should have in TLV format. - necp_find_netstat_data(client, (union necp_sockaddr_union *)&desc->local, (union necp_sockaddr_union *)&desc->remote, &desc->ifindex, desc->euuid, &desc->traffic_class); - - // Basic metadata is all that is required for UDP - desc->rcvbufsize = udpstats->necp_udp_basic.rcvbufsize; - desc->rcvbufused = udpstats->necp_udp_basic.rcvbufused; - desc->eupid = udpstats->necp_udp_basic.eupid; - desc->epid = udpstats->necp_udp_basic.epid; - memcpy(desc->vuuid, udpstats->necp_udp_basic.vuuid, sizeof(desc->euuid)); - desc->ifnet_properties = udpstats->necp_udp_basic.ifnet_properties; - } - return true; -} static int necp_skywalk_priv_check_cred(proc_t p, kauth_cred_t cred) @@ -1769,7 +3151,7 @@ necp_open(struct proc *p, struct necp_open_args *uap, int *retval) if (uap->flags & NECP_OPEN_FLAG_OBSERVER) { if (necp_skywalk_priv_check_cred(p, kauth_cred_get()) != 0 && - priv_check_cred(kauth_cred_get(), PRIV_NET_PRIVILEGED_NETWORK_STATISTICS, 0) != 0) { + priv_check_cred(kauth_cred_get(), PRIV_NET_PRIVILEGED_NETWORK_STATISTICS, 0) != 0) { NECPLOG0(LOG_ERR, "Client does not hold necessary entitlement to observe other NECP clients"); error = EACCES; goto done; @@ -1781,14 +3163,17 @@ necp_open(struct proc *p, struct necp_open_args *uap, int *retval) goto done; } - if ((fd_data = _MALLOC(sizeof(struct necp_fd_data), M_NECP, - M_WAITOK | M_ZERO)) == NULL) { + if ((fd_data = zalloc(necp_client_fd_zone)) == NULL) { error = ENOMEM; goto done; } + memset(fd_data, 0, sizeof(*fd_data)); + + fd_data->necp_fd_type = necp_fd_type_client; fd_data->flags = uap->flags; - LIST_INIT(&fd_data->clients); + RB_INIT(&fd_data->clients); + TAILQ_INIT(&fd_data->update_list); lck_mtx_init(&fd_data->fd_lock, necp_fd_mtx_grp, necp_fd_mtx_attr); klist_init(&fd_data->si.si_note); fd_data->proc_pid = proc_pid(p); @@ -1805,9 +3190,28 @@ necp_open(struct proc *p, struct necp_open_args *uap, int *retval) *retval = fd; - lck_rw_lock_exclusive(&necp_fd_lock); - LIST_INSERT_HEAD(&necp_fd_list, fd_data, chain); - lck_rw_done(&necp_fd_lock); + if (fd_data->flags & NECP_OPEN_FLAG_PUSH_OBSERVER) { + NECP_OBSERVER_LIST_LOCK_EXCLUSIVE(); + LIST_INSERT_HEAD(&necp_fd_observer_list, fd_data, chain); + OSIncrementAtomic(&necp_observer_fd_count); + NECP_OBSERVER_LIST_UNLOCK(); + + // Walk all existing clients and add them + NECP_CLIENT_TREE_LOCK_SHARED(); + struct necp_client *existing_client = NULL; + RB_FOREACH(existing_client, _necp_client_global_tree, &necp_client_global_tree) { + NECP_CLIENT_LOCK(existing_client); + necp_client_update_observer_add_internal(fd_data, existing_client); + necp_client_update_observer_update_internal(fd_data, existing_client); + NECP_CLIENT_UNLOCK(existing_client); + } + NECP_CLIENT_TREE_UNLOCK(); + } else { + NECP_FD_LIST_LOCK_EXCLUSIVE(); + LIST_INSERT_HEAD(&necp_fd_list, fd_data, chain); + OSIncrementAtomic(&necp_client_fd_count); + NECP_FD_LIST_UNLOCK(); + } proc_fdunlock(p); @@ -1818,7 +3222,7 @@ done: fp = NULL; } if (fd_data != NULL) { - FREE(fd_data, M_NECP); + zfree(necp_client_fd_zone, fd_data); fd_data = NULL; } } @@ -1827,15 +3231,19 @@ done: } static int -necp_client_add(struct necp_fd_data *fd_data, struct necp_client_action_args *uap, int *retval) +necp_client_add(struct proc *p, struct necp_fd_data *fd_data, struct necp_client_action_args *uap, int *retval) { int error = 0; struct necp_client *client = NULL; + if (fd_data->flags & NECP_OPEN_FLAG_PUSH_OBSERVER) { + NECPLOG0(LOG_ERR, "NECP client observers with push enabled may not add their own clients"); + return (EINVAL); + } + if (uap->client_id == 0 || uap->client_id_len != sizeof(uuid_t) || uap->buffer_size == 0 || uap->buffer_size > NECP_MAX_CLIENT_PARAMETERS_SIZE || uap->buffer == 0) { - error = EINVAL; - goto done; + return (EINVAL); } if ((client = _MALLOC(sizeof(struct necp_client) + uap->buffer_size, M_NECP, @@ -1850,10 +3258,17 @@ necp_client_add(struct necp_fd_data *fd_data, struct necp_client_action_args *ua goto done; } + lck_mtx_init(&client->lock, necp_fd_mtx_grp, necp_fd_mtx_attr); + lck_mtx_init(&client->route_lock, necp_fd_mtx_grp, necp_fd_mtx_attr); + necp_client_retain(client); // Hold our reference until close + client->parameters_length = uap->buffer_size; + client->proc_pid = fd_data->proc_pid; // Save off proc pid in case the client will persist past fd + client->platform_binary = ((csproc_get_platform_binary(p) == 0) ? 0 : 1); uuid_generate_random(client->client_id); LIST_INIT(&client->assertion_list); + LIST_INIT(&client->flow_list); error = copyout(client->client_id, uap->client_id, sizeof(uuid_t)); if (error) { @@ -1861,12 +3276,20 @@ necp_client_add(struct necp_fd_data *fd_data, struct necp_client_action_args *ua goto done; } - lck_mtx_lock(&fd_data->fd_lock); - LIST_INSERT_HEAD(&fd_data->clients, client, chain); + necp_client_update_observer_add(client); + + NECP_FD_LOCK(fd_data); + RB_INSERT(_necp_client_tree, &fd_data->clients, client); + OSIncrementAtomic(&necp_client_count); + NECP_CLIENT_TREE_LOCK_EXCLUSIVE(); + RB_INSERT(_necp_client_global_tree, &necp_client_global_tree, client); + NECP_CLIENT_TREE_UNLOCK(); // Prime the client result - (void)necp_update_client_result(current_proc(), client); - lck_mtx_unlock(&fd_data->fd_lock); + NECP_CLIENT_LOCK(client); + (void)necp_update_client_result(current_proc(), fd_data, client, NULL); + NECP_CLIENT_UNLOCK(client); + NECP_FD_UNLOCK(fd_data); done: if (error != 0) { if (client != NULL) { @@ -1885,7 +3308,8 @@ necp_client_remove(struct necp_fd_data *fd_data, struct necp_client_action_args int error = 0; struct necp_client *client = NULL; struct necp_client *temp_client = NULL; - uuid_t client_id; + uuid_t client_id = {}; + struct ifnet_stats_per_flow flow_ifnet_stats = {}; if (uap->client_id == 0 || uap->client_id_len != sizeof(uuid_t)) { error = EINVAL; @@ -1898,19 +3322,128 @@ necp_client_remove(struct necp_fd_data *fd_data, struct necp_client_action_args goto done; } - lck_mtx_lock(&fd_data->fd_lock); - LIST_FOREACH_SAFE(client, &fd_data->clients, chain, temp_client) { + if (uap->buffer != 0 && uap->buffer_size == sizeof(flow_ifnet_stats)) { + error = copyin(uap->buffer, &flow_ifnet_stats, uap->buffer_size); + if (error) { + NECPLOG(LOG_ERR, "necp_client_remove flow_ifnet_stats copyin error (%d)", error); + // Not fatal + } + } else if (uap->buffer != 0) { + NECPLOG(LOG_ERR, "necp_client_remove unexpected parameters length (%zu)", uap->buffer_size); + } + + struct _necp_client_tree clients_to_close; + RB_INIT(&clients_to_close); + NECP_FD_LOCK(fd_data); + pid_t pid = fd_data->proc_pid; + RB_FOREACH_SAFE(client, _necp_client_tree, &fd_data->clients, temp_client) { if (uuid_compare(client->client_id, client_id) == 0) { - necp_destroy_client(client); + NECP_CLIENT_TREE_LOCK_EXCLUSIVE(); + RB_REMOVE(_necp_client_global_tree, &necp_client_global_tree, client); + NECP_CLIENT_TREE_UNLOCK(); + RB_REMOVE(_necp_client_tree, &fd_data->clients, client); + RB_INSERT(_necp_client_tree, &clients_to_close, client); } } - lck_mtx_unlock(&fd_data->fd_lock); + + + NECP_FD_UNLOCK(fd_data); + + RB_FOREACH_SAFE(client, _necp_client_tree, &clients_to_close, temp_client) { + RB_REMOVE(_necp_client_tree, &clients_to_close, client); + necp_destroy_client(client, pid, true); + } done: *retval = error; return (error); } +static int +necp_client_check_tcp_heuristics(struct necp_client *client, struct necp_client_flow *flow, u_int32_t *flags, u_int8_t *tfo_cookie, u_int8_t *tfo_cookie_len) +{ + struct necp_client_parsed_parameters parsed_parameters; + int error = 0; + + error = necp_client_parse_parameters(client->parameters, + (u_int32_t)client->parameters_length, + &parsed_parameters); + if (error) { + NECPLOG(LOG_ERR, "necp_client_parse_parameters error (%d)", error); + return (error); + } + + if ((flow->remote_addr.sa.sa_family != AF_INET && + flow->remote_addr.sa.sa_family != AF_INET6) || + (flow->local_addr.sa.sa_family != AF_INET && + flow->local_addr.sa.sa_family != AF_INET6)) { + return (EINVAL); + } + + NECP_CLIENT_ROUTE_LOCK(client); + + if (client->current_route == NULL) { + error = ENOENT; + goto do_unlock; + } + + bool check_ecn = false; + do { + if ((parsed_parameters.flags & NECP_CLIENT_PARAMETER_FLAG_ECN_ENABLE) == + NECP_CLIENT_PARAMETER_FLAG_ECN_ENABLE) { + check_ecn = true; + break; + } + + if ((parsed_parameters.flags & NECP_CLIENT_PARAMETER_FLAG_ECN_DISABLE) == + NECP_CLIENT_PARAMETER_FLAG_ECN_DISABLE) { + break; + } + + if (client->current_route != NULL) { + if (client->current_route->rt_ifp->if_eflags & IFEF_ECN_ENABLE) { + check_ecn = true; + break; + } + if (client->current_route->rt_ifp->if_eflags & IFEF_ECN_DISABLE) { + break; + } + } + + bool inbound = ((parsed_parameters.flags & NECP_CLIENT_PARAMETER_FLAG_LISTENER) == 0); + if ((inbound && tcp_ecn_inbound == 1) || + (!inbound && tcp_ecn_outbound == 1)) { + check_ecn = true; + } + } while (false); + + if (check_ecn) { + if (tcp_heuristic_do_ecn_with_address(client->current_route->rt_ifp, + (union sockaddr_in_4_6 *)&flow->local_addr)) { + *flags |= NECP_CLIENT_RESULT_FLAG_ECN_ENABLED; + } + } + + if ((parsed_parameters.flags & NECP_CLIENT_PARAMETER_FLAG_TFO_ENABLE) == + NECP_CLIENT_PARAMETER_FLAG_TFO_ENABLE) { + + if (!tcp_heuristic_do_tfo_with_address(client->current_route->rt_ifp, + (union sockaddr_in_4_6 *)&flow->local_addr, + (union sockaddr_in_4_6 *)&flow->remote_addr, + tfo_cookie, tfo_cookie_len)) { + *flags |= NECP_CLIENT_RESULT_FLAG_FAST_OPEN_BLOCKED; + *tfo_cookie_len = 0; + } + } else { + *flags |= NECP_CLIENT_RESULT_FLAG_FAST_OPEN_BLOCKED; + *tfo_cookie_len = 0; + } +do_unlock: + NECP_CLIENT_ROUTE_UNLOCK(client); + + return (error); +} + static int necp_client_copy_internal(struct necp_client *client, bool client_is_observed, struct necp_client_action_args *uap, int *retval) { @@ -1927,8 +3460,31 @@ necp_client_copy_internal(struct necp_client *client, bool client_is_observed, s goto done; } *retval = client->parameters_length; - } else if (uap->action == NECP_CLIENT_ACTION_COPY_RESULT) { - if (uap->buffer_size < (client->result_length + client->assigned_results_length)) { + } else if (uap->action == NECP_CLIENT_ACTION_COPY_UPDATED_RESULT && + client->result_read && client->flow_result_read) { + // Copy updates only, but nothing to read + // Just return 0 for bytes read + *retval = 0; + } else if (uap->action == NECP_CLIENT_ACTION_COPY_RESULT || + uap->action == NECP_CLIENT_ACTION_COPY_UPDATED_RESULT) { + size_t assigned_results_size = 0; + struct necp_client_flow *flow = NULL; + LIST_FOREACH(flow, &client->flow_list, flow_chain) { + if (flow->nexus || (flow->socket && flow->assigned)) { + size_t header_length = 0; + if (flow->nexus) { + header_length = sizeof(struct necp_client_nexus_flow_header); + } else { + header_length = sizeof(struct necp_client_flow_header); + } + assigned_results_size += (header_length + flow->assigned_results_length); + + if (flow->has_protoctl_event) { + assigned_results_size += sizeof(struct necp_client_flow_protoctl_event_header); + } + } + } + if (uap->buffer_size < (client->result_length + assigned_results_size)) { error = EINVAL; goto done; } @@ -1937,20 +3493,148 @@ necp_client_copy_internal(struct necp_client *client, bool client_is_observed, s NECPLOG(LOG_ERR, "necp_client_copy result copyout error (%d)", error); goto done; } - if (client->assigned_results_length && client->assigned_results) { - error = copyout(client->assigned_results, uap->buffer + client->result_length, client->assigned_results_length); - if (error) { - NECPLOG(LOG_ERR, "necp_client_copy assigned results copyout error (%d)", error); - goto done; + + size_t assigned_results_cursor = 0; + + flow = NULL; + LIST_FOREACH(flow, &client->flow_list, flow_chain) { + if (flow->nexus || (flow->socket && flow->assigned)) { + // Write TLV headers + struct necp_client_nexus_flow_header header; + u_int32_t length = 0; + u_int32_t flags = 0; + u_int8_t tfo_cookie_len = 0; + u_int8_t type = 0; + + if (flow->nexus) { + if (flow->check_tcp_heuristics) { + u_int8_t tfo_cookie[NECP_TFO_COOKIE_LEN_MAX]; + tfo_cookie_len = NECP_TFO_COOKIE_LEN_MAX; + + if (necp_client_check_tcp_heuristics(client, flow, &flags, + tfo_cookie, &tfo_cookie_len) != 0) { + tfo_cookie_len = 0; + } else { + flow->check_tcp_heuristics = FALSE; + + if (tfo_cookie_len != 0) { + type = NECP_CLIENT_RESULT_TFO_COOKIE; + length = tfo_cookie_len; + memcpy(&header.tfo_cookie_tlv_header.type, &type, sizeof(type)); + memcpy(&header.tfo_cookie_tlv_header.length, &length, sizeof(length)); + memcpy(&header.tfo_cookie_value, tfo_cookie, tfo_cookie_len); + } + } + } + } + + size_t header_length = 0; + if (flow->nexus) { + if (tfo_cookie_len != 0) { + header_length = sizeof(struct necp_client_nexus_flow_header) - (NECP_TFO_COOKIE_LEN_MAX - tfo_cookie_len); + } else { + header_length = sizeof(struct necp_client_nexus_flow_header) - sizeof(struct necp_tlv_header) - NECP_TFO_COOKIE_LEN_MAX; + } + } else { + header_length = sizeof(struct necp_client_flow_header); + } + + type = NECP_CLIENT_RESULT_FLAGS; + length = sizeof(header.flow_header.flags_value); + memcpy(&header.flow_header.flags_tlv_header.type, &type, sizeof(type)); + memcpy(&header.flow_header.flags_tlv_header.length, &length, sizeof(length)); + if (flow->assigned) { + flags |= NECP_CLIENT_RESULT_FLAG_FLOW_ASSIGNED; + } + if (flow->viable) { + flags |= NECP_CLIENT_RESULT_FLAG_FLOW_VIABLE; + } + memcpy(&header.flow_header.flags_value, &flags, sizeof(flags)); + + type = NECP_CLIENT_RESULT_INTERFACE; + length = sizeof(header.flow_header.interface_value); + memcpy(&header.flow_header.interface_tlv_header.type, &type, sizeof(type)); + memcpy(&header.flow_header.interface_tlv_header.length, &length, sizeof(length)); + + struct necp_client_result_interface interface_struct; + interface_struct.generation = 0; + interface_struct.index = flow->interface_index; + + memcpy(&header.flow_header.interface_value, &interface_struct, sizeof(interface_struct)); + if (flow->nexus) { + type = NECP_CLIENT_RESULT_NETAGENT; + length = sizeof(header.agent_value); + memcpy(&header.agent_tlv_header.type, &type, sizeof(type)); + memcpy(&header.agent_tlv_header.length, &length, sizeof(length)); + + struct necp_client_result_netagent agent_struct; + agent_struct.generation = 0; + uuid_copy(agent_struct.netagent_uuid, flow->u.nexus_agent); + + memcpy(&header.agent_value, &agent_struct, sizeof(agent_struct)); + } + + // Don't include outer TLV header in length field + type = NECP_CLIENT_RESULT_FLOW; + length = (header_length - sizeof(struct necp_tlv_header) + flow->assigned_results_length); + if (flow->has_protoctl_event) { + length += sizeof(struct necp_client_flow_protoctl_event_header); + } + memcpy(&header.flow_header.outer_header.type, &type, sizeof(type)); + memcpy(&header.flow_header.outer_header.length, &length, sizeof(length)); + + error = copyout(&header, uap->buffer + client->result_length + assigned_results_cursor, header_length); + if (error) { + NECPLOG(LOG_ERR, "necp_client_copy assigned results tlv_header copyout error (%d)", error); + goto done; + } + assigned_results_cursor += header_length; + + if (flow->assigned_results && flow->assigned_results_length) { + // Write inner TLVs + error = copyout(flow->assigned_results, uap->buffer + client->result_length + assigned_results_cursor, + flow->assigned_results_length); + if (error) { + NECPLOG(LOG_ERR, "necp_client_copy assigned results copyout error (%d)", error); + goto done; + } + } + assigned_results_cursor += flow->assigned_results_length; + + /* Read the protocol event and reset it */ + if (flow->has_protoctl_event) { + struct necp_client_flow_protoctl_event_header protoctl_event_header; + + type = NECP_CLIENT_RESULT_PROTO_CTL_EVENT; + length = sizeof(protoctl_event_header.protoctl_event); + + memcpy(&protoctl_event_header.protoctl_tlv_header.type, &type, sizeof(type)); + memcpy(&protoctl_event_header.protoctl_tlv_header.length, &length, sizeof(length)); + memcpy(&protoctl_event_header.protoctl_event, &flow->protoctl_event, + sizeof(flow->protoctl_event)); + + error = copyout(&protoctl_event_header, uap->buffer + client->result_length + assigned_results_cursor, + sizeof(protoctl_event_header)); + + if (error) { + NECPLOG(LOG_ERR, "necp_client_copy protocol control event results" + " tlv_header copyout error (%d)", error); + goto done; + } + assigned_results_cursor += sizeof(protoctl_event_header); + flow->has_protoctl_event = FALSE; + flow->protoctl_event.protoctl_event_code = 0; + flow->protoctl_event.protoctl_event_val = 0; + flow->protoctl_event.protoctl_event_tcp_seq_num = 0; + } } - *retval = client->result_length + client->assigned_results_length; - } else { - *retval = client->result_length; } + *retval = client->result_length + assigned_results_cursor; + if (!client_is_observed) { client->result_read = TRUE; - client->assigned_result_read = TRUE; + client->flow_result_read = TRUE; } } @@ -1975,7 +3659,8 @@ necp_client_copy(struct necp_fd_data *fd_data, struct necp_client_action_args *u } if (uap->action != NECP_CLIENT_ACTION_COPY_PARAMETERS && - uap->action != NECP_CLIENT_ACTION_COPY_RESULT) { + uap->action != NECP_CLIENT_ACTION_COPY_RESULT && + uap->action != NECP_CLIENT_ACTION_COPY_UPDATED_RESULT) { error = EINVAL; goto done; } @@ -1994,16 +3679,19 @@ necp_client_copy(struct necp_fd_data *fd_data, struct necp_client_action_args *u } } - lck_mtx_lock(&fd_data->fd_lock); - LIST_FOREACH(find_client, &fd_data->clients, chain) { - if (uap->action == NECP_CLIENT_ACTION_COPY_RESULT && + NECP_FD_LOCK(fd_data); + RB_FOREACH(find_client, _necp_client_tree, &fd_data->clients) { + NECP_CLIENT_LOCK(find_client); + if ((uap->action == NECP_CLIENT_ACTION_COPY_RESULT || uap->action == NECP_CLIENT_ACTION_COPY_UPDATED_RESULT) && uuid_is_null(client_id)) { - if (!find_client->result_read || !find_client->assigned_result_read) { + if (!find_client->result_read || !find_client->flow_result_read) { client = find_client; - break; } } else if (uuid_compare(find_client->client_id, client_id) == 0) { client = find_client; + } + NECP_CLIENT_UNLOCK(find_client); + if (client != NULL) { break; } } @@ -2013,44 +3701,35 @@ necp_client_copy(struct necp_fd_data *fd_data, struct necp_client_action_args *u } // Unlock our own client before moving on or returning - lck_mtx_unlock(&fd_data->fd_lock); + NECP_FD_UNLOCK(fd_data); if (client == NULL) { if (fd_data->flags & NECP_OPEN_FLAG_OBSERVER) { // Observers are allowed to lookup clients on other fds - // Lock list - lck_rw_lock_shared(&necp_fd_lock); - struct necp_fd_data *client_fd = NULL; - LIST_FOREACH(client_fd, &necp_fd_list, chain) { - // Lock client - lck_mtx_lock(&client_fd->fd_lock); - find_client = NULL; - LIST_FOREACH(find_client, &client_fd->clients, chain) { - if (uuid_compare(find_client->client_id, client_id) == 0) { - client = find_client; - break; - } - } + // Lock tree + NECP_CLIENT_TREE_LOCK_SHARED(); - if (client != NULL) { - // Matched, copy out data - error = necp_client_copy_internal(client, TRUE, uap, retval); - } + bool found_client = FALSE; - // Unlock client - lck_mtx_unlock(&client_fd->fd_lock); + struct necp_client find; + uuid_copy(find.client_id, client_id); + client = RB_FIND(_necp_client_global_tree, &necp_client_global_tree, &find); + if (client != NULL) { + NECP_CLIENT_LOCK(client); - if (client != NULL) { - break; - } + // Matched, copy out data + found_client = TRUE; + error = necp_client_copy_internal(client, TRUE, uap, retval); + + NECP_CLIENT_UNLOCK(client); } - // Unlock list - lck_rw_done(&necp_fd_lock); + // Unlock tree + NECP_CLIENT_TREE_UNLOCK(); // No client found, fail - if (client == NULL) { + if (!found_client) { error = ENOENT; goto done; } @@ -2065,6 +3744,137 @@ done: return (error); } +static int +necp_client_copy_client_update(struct necp_fd_data *fd_data, struct necp_client_action_args *uap, int *retval) +{ + int error = 0; + + *retval = 0; + + if (!(fd_data->flags & NECP_OPEN_FLAG_PUSH_OBSERVER)) { + NECPLOG0(LOG_ERR, "NECP fd is not observer, cannot copy client update"); + return (EINVAL); + } + + if (uap->client_id_len != sizeof(uuid_t) || uap->client_id == 0) { + NECPLOG0(LOG_ERR, "Client id invalid, cannot copy client update"); + return (EINVAL); + } + + if (uap->buffer_size == 0 || uap->buffer == 0) { + NECPLOG0(LOG_ERR, "Buffer invalid, cannot copy client update"); + return (EINVAL); + } + + NECP_FD_LOCK(fd_data); + struct necp_client_update *client_update = TAILQ_FIRST(&fd_data->update_list); + if (client_update != NULL) { + TAILQ_REMOVE(&fd_data->update_list, client_update, chain); + VERIFY(fd_data->update_count > 0); + fd_data->update_count--; + } + NECP_FD_UNLOCK(fd_data); + + if (client_update != NULL) { + error = copyout(client_update->client_id, uap->client_id, sizeof(uuid_t)); + if (error) { + NECPLOG(LOG_ERR, "Copy client update copyout client id error (%d)", error); + } else { + if (uap->buffer_size < client_update->update_length) { + NECPLOG(LOG_ERR, "Buffer size cannot hold update (%zu < %zu)", uap->buffer_size, client_update->update_length); + error = EINVAL; + } else { + error = copyout(&client_update->update, uap->buffer, client_update->update_length); + if (error) { + NECPLOG(LOG_ERR, "Copy client update copyout error (%d)", error); + } else { + *retval = client_update->update_length; + } + } + } + + FREE(client_update, M_NECP); + client_update = NULL; + } else { + error = ENOENT; + } + + return (error); +} + +static int +necp_client_copy_parameters_locked(struct necp_client *client, struct necp_client_nexus_parameters *parameters) +{ + VERIFY(parameters != NULL); + + struct necp_client_parsed_parameters parsed_parameters = {}; + int error = necp_client_parse_parameters(client->parameters, (u_int32_t)client->parameters_length, &parsed_parameters); + + parameters->pid = client->proc_pid; + if (parsed_parameters.valid_fields & NECP_PARSED_PARAMETERS_FIELD_EFFECTIVE_PID) { + parameters->epid = parsed_parameters.effective_pid; + } else { + parameters->epid = parameters->pid; + } + memcpy(¶meters->local_addr, &parsed_parameters.local_addr, sizeof(parameters->local_addr)); + memcpy(¶meters->remote_addr, &parsed_parameters.remote_addr, sizeof(parameters->remote_addr)); + parameters->ip_protocol = parsed_parameters.ip_protocol; + parameters->traffic_class = parsed_parameters.traffic_class; + uuid_copy(parameters->euuid, parsed_parameters.effective_uuid); + parameters->is_listener = (parsed_parameters.flags & NECP_CLIENT_PARAMETER_FLAG_LISTENER) ? 1 : 0; + parameters->policy_id = client->policy_id; + + // parse client result flag + u_int32_t client_result_flags = 0; + u_int32_t value_size = 0; + u_int8_t *flags_pointer = NULL; + flags_pointer = necp_buffer_get_tlv_value(client->result, 0, &value_size); + if (flags_pointer && value_size == sizeof(client_result_flags)) { + memcpy(&client_result_flags, flags_pointer, value_size); + } + parameters->allow_qos_marking = (client_result_flags & NECP_CLIENT_RESULT_FLAG_ALLOW_QOS_MARKING) ? 1 : 0; + + return (error); +} + +int +necp_client_copy_parameters(uuid_t client_id, struct necp_client_nexus_parameters *parameters) +{ + int error = 0; + struct necp_client *client = NULL; + + if (parameters == NULL) { + return EINVAL; + } + + // Lock tree + NECP_CLIENT_TREE_LOCK_SHARED(); + + bool found_client = FALSE; + struct necp_client find; + uuid_copy(find.client_id, client_id); + client = RB_FIND(_necp_client_global_tree, &necp_client_global_tree, &find); + if (client != NULL) { + NECP_CLIENT_LOCK(client); + + // Matched, parse parameters + found_client = TRUE; + error = necp_client_copy_parameters_locked(client, parameters); + + NECP_CLIENT_UNLOCK(client); + } + + // Unlock tree + NECP_CLIENT_TREE_UNLOCK(); + + // No client found, fail + if (!found_client) { + return ENOENT; + } + + return error; +} + static int necp_client_list(struct necp_fd_data *fd_data, struct necp_client_action_args *uap, int *retval) { @@ -2073,6 +3883,7 @@ necp_client_list(struct necp_fd_data *fd_data, struct necp_client_action_args *u uuid_t *list = NULL; u_int32_t requested_client_count = 0; u_int32_t client_count = 0; + size_t copy_buffer_size = 0; if (uap->buffer_size < sizeof(requested_client_count) || uap->buffer == 0) { error = EINVAL; @@ -2090,38 +3901,45 @@ necp_client_list(struct necp_fd_data *fd_data, struct necp_client_action_args *u goto done; } - if (uap->buffer_size != (sizeof(requested_client_count) + requested_client_count * sizeof(uuid_t))) { + if (os_mul_overflow(sizeof(uuid_t), requested_client_count, ©_buffer_size)) { + error = ERANGE; + goto done; + } + + if (uap->buffer_size - sizeof(requested_client_count) != copy_buffer_size) { + error = EINVAL; + goto done; + } + + if (copy_buffer_size > NECP_MAX_CLIENT_LIST_SIZE) { error = EINVAL; goto done; } if (requested_client_count > 0) { - if ((list = _MALLOC(requested_client_count * sizeof(uuid_t), M_NECP, M_WAITOK | M_ZERO)) == NULL) { + if ((list = _MALLOC(copy_buffer_size, M_NECP, M_WAITOK | M_ZERO)) == NULL) { error = ENOMEM; goto done; } } - // Lock list - lck_rw_lock_shared(&necp_fd_lock); - struct necp_fd_data *client_fd = NULL; - LIST_FOREACH(client_fd, &necp_fd_list, chain) { - // Lock client - lck_mtx_lock(&client_fd->fd_lock); - find_client = NULL; - LIST_FOREACH(find_client, &client_fd->clients, chain) { - if (!uuid_is_null(find_client->client_id)) { - if (client_count < requested_client_count) { - uuid_copy(list[client_count], find_client->client_id); - } - client_count++; + // Lock tree + NECP_CLIENT_TREE_LOCK_SHARED(); + + find_client = NULL; + RB_FOREACH(find_client, _necp_client_global_tree, &necp_client_global_tree) { + NECP_CLIENT_LOCK(find_client); + if (!uuid_is_null(find_client->client_id)) { + if (client_count < requested_client_count) { + uuid_copy(list[client_count], find_client->client_id); } + client_count++; } - lck_mtx_unlock(&client_fd->fd_lock); + NECP_CLIENT_UNLOCK(find_client); } - // Unlock list - lck_rw_done(&necp_fd_lock); + // Unlock tree + NECP_CLIENT_TREE_UNLOCK(); error = copyout(&client_count, uap->buffer, sizeof(client_count)); if (error) { @@ -2132,7 +3950,7 @@ necp_client_list(struct necp_fd_data *fd_data, struct necp_client_action_args *u if (requested_client_count > 0 && client_count > 0 && list != NULL) { - error = copyout(list, uap->buffer + sizeof(client_count), requested_client_count * sizeof(uuid_t)); + error = copyout(list, uap->buffer + sizeof(client_count), copy_buffer_size); if (error) { NECPLOG(LOG_ERR, "necp_client_list client count copyout error (%d)", error); goto done; @@ -2143,55 +3961,11 @@ done: FREE(list, M_NECP); } *retval = error; - - return (error); -} - -static int -necp_client_request_nexus(struct necp_fd_data *fd_data, struct necp_client_action_args *uap, int *retval) -{ - int error = 0; - struct necp_client *client = NULL; - uuid_t client_id; - bool requested_nexus = FALSE; - - if (uap->client_id == 0 || uap->client_id_len != sizeof(uuid_t)) { - error = EINVAL; - goto done; - } - - error = copyin(uap->client_id, client_id, sizeof(uuid_t)); - if (error) { - NECPLOG(LOG_ERR, "necp_client_request_nexus copyin client_id error (%d)", error); - goto done; - } - - lck_mtx_lock(&fd_data->fd_lock); - LIST_FOREACH(client, &fd_data->clients, chain) { - if (uuid_compare(client->client_id, client_id) == 0) { - // Request from nexus agent - if (!uuid_is_null(client->nexus_agent)) { - error = netagent_client_message(client->nexus_agent, client->client_id, - NETAGENT_MESSAGE_TYPE_REQUEST_NEXUS); - if (error == 0) { - requested_nexus = TRUE; - } - } - break; - } - } - lck_mtx_unlock(&fd_data->fd_lock); - - if (!requested_nexus && - error == 0) { - error = ENOENT; - } -done: - *retval = error; return (error); } + static void necp_client_add_assertion(struct necp_client *client, uuid_t netagent_uuid) { @@ -2234,7 +4008,6 @@ static int necp_client_agent_action(struct necp_fd_data *fd_data, struct necp_client_action_args *uap, int *retval) { int error = 0; - struct necp_client *matched_client = NULL; struct necp_client *client = NULL; uuid_t client_id; bool acted_on_agent = FALSE; @@ -2264,20 +4037,15 @@ necp_client_agent_action(struct necp_fd_data *fd_data, struct necp_client_action goto done; } - lck_mtx_lock(&fd_data->fd_lock); - LIST_FOREACH(client, &fd_data->clients, chain) { - if (uuid_compare(client->client_id, client_id) == 0) { - matched_client = client; - break; - } - } - if (matched_client) { + NECP_FD_LOCK(fd_data); + client = necp_client_fd_find_client_and_lock(fd_data, client_id); + if (client != NULL) { size_t offset = 0; - while ((offset + sizeof(u_int8_t) + sizeof(u_int32_t)) <= parameters_size) { + while ((offset + sizeof(struct necp_tlv_header)) <= parameters_size) { u_int8_t type = necp_buffer_get_tlv_type(parameters, offset); u_int32_t length = necp_buffer_get_tlv_length(parameters, offset); - if (length > (parameters_size - (offset + sizeof(u_int8_t) + sizeof(u_int32_t)))) { + if (length > (parameters_size - (offset + sizeof(struct necp_tlv_header)))) { // If the length is larger than what can fit in the remaining parameters size, bail NECPLOG(LOG_ERR, "Invalid TLV length (%u)", length); break; @@ -2310,8 +4078,15 @@ necp_client_agent_action(struct necp_fd_data *fd_data, struct necp_client_action } } - error = netagent_client_message(agent_uuid, client_id, - netagent_message_type); + struct necp_client_nexus_parameters parsed_parameters = {}; + necp_client_copy_parameters_locked(client, &parsed_parameters); + + error = netagent_client_message_with_params(agent_uuid, + client_id, + fd_data->proc_pid, + netagent_message_type, + &parsed_parameters, + NULL, NULL); if (error == 0) { acted_on_agent = TRUE; } else { @@ -2325,10 +4100,12 @@ necp_client_agent_action(struct necp_fd_data *fd_data, struct necp_client_action } } - offset += sizeof(u_int8_t) + sizeof(u_int32_t) + length; + offset += sizeof(struct necp_tlv_header) + length; } + + NECP_CLIENT_UNLOCK(client); } - lck_mtx_unlock(&fd_data->fd_lock); + NECP_FD_UNLOCK(fd_data); if (!acted_on_agent && error == 0) { @@ -2340,7 +4117,7 @@ done: FREE(parameters, M_NECP); parameters = NULL; } - + return (error); } @@ -2365,12 +4142,12 @@ necp_client_copy_agent(__unused struct necp_fd_data *fd_data, struct necp_client error = netagent_copyout(agent_uuid, uap->buffer, uap->buffer_size); if (error) { - NECPLOG(LOG_ERR, "necp_client_copy_agent netagent_copyout error (%d)", error); + // netagent_copyout already logs appropriate errors goto done; } done: *retval = error; - + return (error); } @@ -2378,7 +4155,6 @@ static int necp_client_agent_use(struct necp_fd_data *fd_data, struct necp_client_action_args *uap, int *retval) { int error = 0; - struct necp_client *matched_client = NULL; struct necp_client *client = NULL; uuid_t client_id; struct necp_agent_use_parameters parameters; @@ -2401,21 +4177,16 @@ necp_client_agent_use(struct necp_fd_data *fd_data, struct necp_client_action_ar goto done; } - lck_mtx_lock(&fd_data->fd_lock); - LIST_FOREACH(client, &fd_data->clients, chain) { - if (uuid_compare(client->client_id, client_id) == 0) { - matched_client = client; - break; - } - } - - if (matched_client) { + NECP_FD_LOCK(fd_data); + client = necp_client_fd_find_client_and_lock(fd_data, client_id); + if (client != NULL) { error = netagent_use(parameters.agent_uuid, ¶meters.out_use_count); + NECP_CLIENT_UNLOCK(client); } else { error = ENOENT; } - lck_mtx_unlock(&fd_data->fd_lock); + NECP_FD_UNLOCK(fd_data); if (error == 0) { error = copyout(¶meters, uap->buffer, uap->buffer_size); @@ -2478,17 +4249,29 @@ necp_client_copy_interface(__unused struct necp_fd_data *fd_data, struct necp_cl if (IFNET_IS_EXPENSIVE(interface)) { interface_details.flags |= NECP_INTERFACE_FLAG_EXPENSIVE; } + if ((interface->if_eflags & IFEF_TXSTART) == IFEF_TXSTART) { + interface_details.flags |= NECP_INTERFACE_FLAG_TXSTART; + } + if ((interface->if_eflags & IFEF_NOACKPRI) == IFEF_NOACKPRI) { + interface_details.flags |= NECP_INTERFACE_FLAG_NOACKPRI; + } interface_details.mtu = interface->if_mtu; - u_int8_t ipv4_signature_len = sizeof(interface_details.ipv4_signature); + u_int8_t ipv4_signature_len = sizeof(interface_details.ipv4_signature.signature); u_int16_t ipv4_signature_flags; - ifnet_get_netsignature(interface, AF_INET, &ipv4_signature_len, &ipv4_signature_flags, - (u_int8_t *)&interface_details.ipv4_signature); + if (ifnet_get_netsignature(interface, AF_INET, &ipv4_signature_len, &ipv4_signature_flags, + (u_int8_t *)&interface_details.ipv4_signature) != 0) { + ipv4_signature_len = 0; + } + interface_details.ipv4_signature.signature_len = ipv4_signature_len; - u_int8_t ipv6_signature_len = sizeof(interface_details.ipv6_signature); + u_int8_t ipv6_signature_len = sizeof(interface_details.ipv6_signature.signature); u_int16_t ipv6_signature_flags; - ifnet_get_netsignature(interface, AF_INET6, &ipv6_signature_len, &ipv6_signature_flags, - (u_int8_t *)&interface_details.ipv6_signature); + if (ifnet_get_netsignature(interface, AF_INET6, &ipv6_signature_len, &ipv6_signature_flags, + (u_int8_t *)&interface_details.ipv6_signature) != 0) { + ipv6_signature_len = 0; + } + interface_details.ipv6_signature.signature_len = ipv6_signature_len; } ifnet_head_done(); @@ -2504,106 +4287,74 @@ done: return (error); } + static int -necp_client_stats_action(struct necp_client *client, user_addr_t buffer, user_size_t buffer_size) +necp_client_copy_route_statistics(__unused struct necp_fd_data *fd_data, struct necp_client_action_args *uap, int *retval) { int error = 0; - struct necp_stats_hdr *stats_hdr = NULL; - - if (client->stats_area) { - // Close old stats if required. - if ((client->stats_uaddr != buffer) || (client->stats_ulen != buffer_size)) { - necp_destroy_client_stats(client); - } - } - - if ((buffer == 0) || (buffer_size == 0)) { - goto done; - } - - if (client->stats_area) { - // An update - error = copyin(client->stats_uaddr, client->stats_area, client->stats_ulen); - if (error) { - NECPLOG(LOG_ERR, "necp_client_stats_action copyin error on update (%d)", error); - } else { - // Future use - check - stats_hdr = (necp_stats_hdr *)client->stats_area; - if (stats_hdr->necp_stats_event != 0) { - ntstat_userland_stats_event(client->stats_handler_context, (userland_stats_event_t)stats_hdr->necp_stats_event); - } - } - goto done; - } + struct necp_client *client = NULL; + uuid_t client_id; - // A create - if ((buffer_size > sizeof(necp_all_stats)) || (buffer_size < sizeof(necp_stats_hdr))) { + if (uap->client_id == 0 || uap->client_id_len != sizeof(uuid_t) || + uap->buffer_size < sizeof(struct necp_stat_counts) || uap->buffer == 0) { + NECPLOG0(LOG_ERR, "necp_client_copy_route_statistics bad input"); error = EINVAL; goto done; } - if ((stats_hdr = _MALLOC(buffer_size, M_NECP, M_WAITOK | M_ZERO)) == NULL) { - error = ENOMEM; - goto done; - } - - client->stats_handler_context = NULL; - client->stats_uaddr = buffer; - client->stats_ulen = buffer_size; - client->stats_area = stats_hdr; - error = copyin(client->stats_uaddr, client->stats_area, client->stats_ulen); + error = copyin(uap->client_id, client_id, sizeof(uuid_t)); if (error) { - NECPLOG(LOG_ERR, "necp_client_stats_action copyin error on create (%d)", error); + NECPLOG(LOG_ERR, "necp_client_copy_route_statistics copyin client_id error (%d)", error); goto done; } - switch (stats_hdr->necp_stats_type) { - case NECP_CLIENT_STATISTICS_TYPE_TCP: { - if (stats_hdr->necp_stats_ver == NECP_CLIENT_STATISTICS_TYPE_TCP_VER_1) { - client->stats_handler_context = ntstat_userland_stats_open((userland_stats_provider_context *)client, - NSTAT_PROVIDER_TCP_USERLAND, 0, necp_request_tcp_netstats); - if (client->stats_handler_context == NULL) { - error = EIO; - } - } else { - error = ENOTSUP; - } - break; - } - case NECP_CLIENT_STATISTICS_TYPE_UDP: { - if (stats_hdr->necp_stats_ver != NECP_CLIENT_STATISTICS_TYPE_UDP_VER_1) { - client->stats_handler_context = ntstat_userland_stats_open((userland_stats_provider_context *)client, - NSTAT_PROVIDER_UDP_USERLAND, 0, necp_request_udp_netstats); - if (client->stats_handler_context == NULL) { - error = EIO; - } - } else { - error = ENOTSUP; - } - break; + // Lock + NECP_FD_LOCK(fd_data); + client = necp_client_fd_find_client_and_lock(fd_data, client_id); + if (client != NULL) { + NECP_CLIENT_ROUTE_LOCK(client); + struct nstat_counts route_stats = {}; + if (client->current_route != NULL && client->current_route->rt_stats != NULL) { + struct nstat_counts *rt_stats = client->current_route->rt_stats; + atomic_get_64(route_stats.nstat_rxpackets, &rt_stats->nstat_rxpackets); + atomic_get_64(route_stats.nstat_rxbytes, &rt_stats->nstat_rxbytes); + atomic_get_64(route_stats.nstat_txpackets, &rt_stats->nstat_txpackets); + atomic_get_64(route_stats.nstat_txbytes, &rt_stats->nstat_txbytes); + route_stats.nstat_rxduplicatebytes = rt_stats->nstat_rxduplicatebytes; + route_stats.nstat_rxoutoforderbytes = rt_stats->nstat_rxoutoforderbytes; + route_stats.nstat_txretransmit = rt_stats->nstat_txretransmit; + route_stats.nstat_connectattempts = rt_stats->nstat_connectattempts; + route_stats.nstat_connectsuccesses = rt_stats->nstat_connectsuccesses; + route_stats.nstat_min_rtt = rt_stats->nstat_min_rtt; + route_stats.nstat_avg_rtt = rt_stats->nstat_avg_rtt; + route_stats.nstat_var_rtt = rt_stats->nstat_var_rtt; } - default: { - error = ENOTSUP; - break; + + // Unlock before copying out + NECP_CLIENT_ROUTE_UNLOCK(client); + NECP_CLIENT_UNLOCK(client); + NECP_FD_UNLOCK(fd_data); + + error = copyout(&route_stats, uap->buffer, sizeof(route_stats)); + if (error) { + NECPLOG(LOG_ERR, "necp_client_copy_route_statistics copyout error (%d)", error); } + } else { + // Unlock + NECP_FD_UNLOCK(fd_data); + error = ENOENT; } -done: - if ((error) && (stats_hdr != NULL)) { - FREE(stats_hdr, M_NECP); - client->stats_area = NULL; - client->stats_handler_context = NULL; - client->stats_uaddr = 0; - client->stats_ulen = 0; - } + +done: + *retval = error; return (error); } static int -necp_client_set_statistics(__unused struct necp_fd_data *fd_data, struct necp_client_action_args *uap, int *retval) +necp_client_update_cache(struct necp_fd_data *fd_data, struct necp_client_action_args *uap, int *retval) { int error = 0; - struct necp_client *find_client = NULL; struct necp_client *client = NULL; uuid_t client_id; @@ -2614,24 +4365,102 @@ necp_client_set_statistics(__unused struct necp_fd_data *fd_data, struct necp_cl error = copyin(uap->client_id, client_id, sizeof(uuid_t)); if (error) { - NECPLOG(LOG_ERR, "necp_client_set_statistics copyin client_id error (%d)", error); + NECPLOG(LOG_ERR, "necp_client_update_cache copyin client_id error (%d)", error); goto done; } - lck_mtx_lock(&fd_data->fd_lock); - LIST_FOREACH(find_client, &fd_data->clients, chain) { - if (uuid_compare(find_client->client_id, client_id) == 0) { - client = find_client; - break; - } + NECP_FD_LOCK(fd_data); + client = necp_client_fd_find_client_and_lock(fd_data, client_id); + if (client == NULL) { + NECP_FD_UNLOCK(fd_data); + error = ENOENT; + goto done; + } + + NECP_CLIENT_ROUTE_LOCK(client); + // This needs to be changed when TFO/ECN is supported by multiple flows + struct necp_client_flow *flow = LIST_FIRST(&client->flow_list); + if (flow == NULL || + (flow->remote_addr.sa.sa_family != AF_INET && + flow->remote_addr.sa.sa_family != AF_INET6) || + (flow->local_addr.sa.sa_family != AF_INET && + flow->local_addr.sa.sa_family != AF_INET6)) { + error = EINVAL; + NECPLOG(LOG_ERR, "necp_client_update_cache no flow error (%d)", error); + goto done_unlock; + } + + necp_cache_buffer cache_buffer; + memset(&cache_buffer, 0, sizeof(cache_buffer)); + + if (uap->buffer_size != sizeof(necp_cache_buffer) || + uap->buffer == USER_ADDR_NULL) { + error = EINVAL; + goto done_unlock; } - if (client) { - error = necp_client_stats_action(client, uap->buffer, uap->buffer_size); + error = copyin(uap->buffer, &cache_buffer, sizeof(cache_buffer)); + if (error) { + NECPLOG(LOG_ERR, "necp_client_update_cache copyin cache buffer error (%d)", error); + goto done_unlock; + } + + if (cache_buffer.necp_cache_buf_type == NECP_CLIENT_CACHE_TYPE_ECN && + cache_buffer.necp_cache_buf_ver == NECP_CLIENT_CACHE_TYPE_ECN_VER_1) { + if (cache_buffer.necp_cache_buf_size != sizeof(necp_tcp_ecn_cache) || + cache_buffer.necp_cache_buf_addr == USER_ADDR_NULL) { + error = EINVAL; + goto done_unlock; + } + + necp_tcp_ecn_cache ecn_cache_buffer; + memset(&ecn_cache_buffer, 0, sizeof(ecn_cache_buffer)); + + error = copyin(cache_buffer.necp_cache_buf_addr, &ecn_cache_buffer, sizeof(necp_tcp_ecn_cache)); + if (error) { + NECPLOG(LOG_ERR, "necp_client_update_cache copyin ecn cache buffer error (%d)", error); + goto done_unlock; + } + + if (client->current_route != NULL && client->current_route->rt_ifp != NULL) { + if (!client->platform_binary) { + ecn_cache_buffer.necp_tcp_ecn_heuristics_success = 0; + } + tcp_heuristics_ecn_update(&ecn_cache_buffer, client->current_route->rt_ifp, + (union sockaddr_in_4_6 *)&flow->local_addr); + } + } else if (cache_buffer.necp_cache_buf_type == NECP_CLIENT_CACHE_TYPE_TFO && + cache_buffer.necp_cache_buf_ver == NECP_CLIENT_CACHE_TYPE_TFO_VER_1) { + if (cache_buffer.necp_cache_buf_size != sizeof(necp_tcp_tfo_cache) || + cache_buffer.necp_cache_buf_addr == USER_ADDR_NULL) { + error = EINVAL; + goto done_unlock; + } + + necp_tcp_tfo_cache tfo_cache_buffer; + memset(&tfo_cache_buffer, 0, sizeof(tfo_cache_buffer)); + + error = copyin(cache_buffer.necp_cache_buf_addr, &tfo_cache_buffer, sizeof(necp_tcp_tfo_cache)); + if (error) { + NECPLOG(LOG_ERR, "necp_client_update_cache copyin tfo cache buffer error (%d)", error); + goto done_unlock; + } + + if (client->current_route != NULL && client->current_route->rt_ifp != NULL) { + if (!client->platform_binary) { + tfo_cache_buffer.necp_tcp_tfo_heuristics_success = 0; + } + tcp_heuristics_tfo_update(&tfo_cache_buffer, client->current_route->rt_ifp, + (union sockaddr_in_4_6 *)&flow->local_addr, + (union sockaddr_in_4_6 *)&flow->remote_addr); + } } else { - error = ENOENT; + error = EINVAL; } - lck_mtx_unlock(&fd_data->fd_lock); +done_unlock: + NECP_CLIENT_ROUTE_UNLOCK(client); + NECP_CLIENT_UNLOCK(client); + NECP_FD_UNLOCK(fd_data); done: *retval = error; return (error); @@ -2653,7 +4482,7 @@ necp_client_action(struct proc *p, struct necp_client_action_args *uap, int *ret u_int32_t action = uap->action; switch (action) { case NECP_CLIENT_ACTION_ADD: { - return_value = necp_client_add(fd_data, uap, retval); + return_value = necp_client_add(p, fd_data, uap, retval); break; } case NECP_CLIENT_ACTION_REMOVE: { @@ -2661,7 +4490,8 @@ necp_client_action(struct proc *p, struct necp_client_action_args *uap, int *ret break; } case NECP_CLIENT_ACTION_COPY_PARAMETERS: - case NECP_CLIENT_ACTION_COPY_RESULT: { + case NECP_CLIENT_ACTION_COPY_RESULT: + case NECP_CLIENT_ACTION_COPY_UPDATED_RESULT: { return_value = necp_client_copy(fd_data, uap, retval); break; } @@ -2669,10 +4499,6 @@ necp_client_action(struct proc *p, struct necp_client_action_args *uap, int *ret return_value = necp_client_list(fd_data, uap, retval); break; } - case NECP_CLIENT_ACTION_REQUEST_NEXUS_INSTANCE: { - return_value = necp_client_request_nexus(fd_data, uap, retval); - break; - } case NECP_CLIENT_ACTION_AGENT: { return_value = necp_client_agent_action(fd_data, uap, retval); break; @@ -2689,8 +4515,16 @@ necp_client_action(struct proc *p, struct necp_client_action_args *uap, int *ret return_value = necp_client_copy_interface(fd_data, uap, retval); break; } - case NECP_CLIENT_ACTION_SET_STATISTICS: { - return_value = necp_client_set_statistics(fd_data, uap, retval); + case NECP_CLIENT_ACTION_COPY_ROUTE_STATISTICS: { + return_value = necp_client_copy_route_statistics(fd_data, uap, retval); + break; + } + case NECP_CLIENT_ACTION_UPDATE_CACHE: { + return_value = necp_client_update_cache(fd_data, uap, retval); + break; + } + case NECP_CLIENT_ACTION_COPY_CLIENT_UPDATE: { + return_value = necp_client_copy_client_update(fd_data, uap, retval); break; } default: { @@ -2736,7 +4570,8 @@ necp_match_policy(struct proc *p, struct necp_match_policy_args *uap, int32_t *r goto done; } - error = necp_application_find_policy_match_internal(p, parameters, uap->parameters_size, &returned_result, NULL, 0); + error = necp_application_find_policy_match_internal(p, parameters, uap->parameters_size, + &returned_result, NULL, 0, NULL, NULL, NULL, false); if (error) { goto done; } @@ -2827,7 +4662,7 @@ necp_set_socket_attributes(struct socket *so, struct sockopt *sopt) size_t valsize = sopt->sopt_valsize; if (valsize == 0 || - valsize > ((sizeof(u_int8_t) + sizeof(u_int32_t) + NECP_MAX_SOCKET_ATTRIBUTE_STRING_LENGTH) * 2)) { + valsize > ((sizeof(struct necp_tlv_header) + NECP_MAX_SOCKET_ATTRIBUTE_STRING_LENGTH) * 2)) { goto done; } @@ -2874,10 +4709,10 @@ necp_get_socket_attributes(struct socket *so, struct sockopt *sopt) struct inpcb *inp = sotoinpcb(so); if (inp->inp_necp_attributes.inp_domain != NULL) { - valsize += sizeof(u_int8_t) + sizeof(u_int32_t) + strlen(inp->inp_necp_attributes.inp_domain); + valsize += sizeof(struct necp_tlv_header) + strlen(inp->inp_necp_attributes.inp_domain); } if (inp->inp_necp_attributes.inp_account != NULL) { - valsize += sizeof(u_int8_t) + sizeof(u_int32_t) + strlen(inp->inp_necp_attributes.inp_account); + valsize += sizeof(struct necp_tlv_header) + strlen(inp->inp_necp_attributes.inp_account); } if (valsize == 0) { goto done; @@ -2890,11 +4725,13 @@ necp_get_socket_attributes(struct socket *so, struct sockopt *sopt) cursor = buffer; if (inp->inp_necp_attributes.inp_domain != NULL) { - cursor = necp_buffer_write_tlv(cursor, NECP_TLV_ATTRIBUTE_DOMAIN, strlen(inp->inp_necp_attributes.inp_domain), inp->inp_necp_attributes.inp_domain); + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_ATTRIBUTE_DOMAIN, strlen(inp->inp_necp_attributes.inp_domain), inp->inp_necp_attributes.inp_domain, + buffer, valsize); } if (inp->inp_necp_attributes.inp_account != NULL) { - cursor = necp_buffer_write_tlv(cursor, NECP_TLV_ATTRIBUTE_ACCOUNT, strlen(inp->inp_necp_attributes.inp_account), inp->inp_necp_attributes.inp_account); + cursor = necp_buffer_write_tlv(cursor, NECP_TLV_ATTRIBUTE_ACCOUNT, strlen(inp->inp_necp_attributes.inp_account), inp->inp_necp_attributes.inp_account, + buffer, valsize); } error = sooptcopyout(sopt, buffer, valsize); @@ -2905,10 +4742,79 @@ done: if (buffer != NULL) { FREE(buffer, M_NECP); } - + return (error); } +void * +necp_create_nexus_assign_message(uuid_t nexus_instance, u_int32_t nexus_port, void *key, uint32_t key_length, + struct necp_client_endpoint *local_endpoint, struct necp_client_endpoint *remote_endpoint, + u_int32_t flow_adv_index, size_t *message_length) +{ + u_int8_t *buffer = NULL; + u_int8_t *cursor = NULL; + size_t valsize = 0; + bool has_nexus_assignment = FALSE; + + + if (!uuid_is_null(nexus_instance)) { + has_nexus_assignment = TRUE; + valsize += sizeof(struct necp_tlv_header) + sizeof(uuid_t); + valsize += sizeof(struct necp_tlv_header) + sizeof(u_int32_t); + } + if (flow_adv_index != NECP_FLOWADV_IDX_INVALID) { + valsize += sizeof(struct necp_tlv_header) + sizeof(u_int32_t); + } + if (key != NULL && key_length > 0) { + valsize += sizeof(struct necp_tlv_header) + key_length; + } + if (local_endpoint != NULL) { + valsize += sizeof(struct necp_tlv_header) + sizeof(struct necp_client_endpoint); + } + if (remote_endpoint != NULL) { + valsize += sizeof(struct necp_tlv_header) + sizeof(struct necp_client_endpoint); + } + if (valsize == 0) { + return (NULL); + } + + MALLOC(buffer, u_int8_t *, valsize, M_NETAGENT, M_WAITOK | M_ZERO); // Use M_NETAGENT area, since it is expected upon free + if (buffer == NULL) { + return (NULL); + } + + cursor = buffer; + if (has_nexus_assignment) { + cursor = necp_buffer_write_tlv(cursor, NECP_CLIENT_RESULT_NEXUS_INSTANCE, sizeof(uuid_t), nexus_instance, buffer, valsize); + cursor = necp_buffer_write_tlv(cursor, NECP_CLIENT_RESULT_NEXUS_PORT, sizeof(u_int32_t), &nexus_port, buffer, valsize); + } + if (flow_adv_index != NECP_FLOWADV_IDX_INVALID) { + cursor = necp_buffer_write_tlv(cursor, NECP_CLIENT_RESULT_NEXUS_PORT_FLOW_INDEX, sizeof(u_int32_t), &flow_adv_index, buffer, valsize); + } + if (key != NULL && key_length > 0) { + cursor = necp_buffer_write_tlv(cursor, NECP_CLIENT_PARAMETER_NEXUS_KEY, key_length, key, buffer, valsize); + } + if (local_endpoint != NULL) { + cursor = necp_buffer_write_tlv(cursor, NECP_CLIENT_RESULT_LOCAL_ENDPOINT, sizeof(struct necp_client_endpoint), local_endpoint, buffer, valsize); + } + if (remote_endpoint != NULL) { + cursor = necp_buffer_write_tlv(cursor, NECP_CLIENT_RESULT_REMOTE_ENDPOINT, sizeof(struct necp_client_endpoint), remote_endpoint, buffer, valsize); + } + + *message_length = valsize; + + return (buffer); +} + +void +necp_inpcb_remove_cb(struct inpcb *inp) +{ + if (!uuid_is_null(inp->necp_client_uuid)) { + necp_client_unregister_socket_flow(inp->necp_client_uuid, inp); + uuid_clear(inp->necp_client_uuid); + } +} + void necp_inpcb_dispose(struct inpcb *inp) { @@ -2922,6 +4828,15 @@ necp_inpcb_dispose(struct inpcb *inp) } } +void +necp_mppcb_dispose(struct mppcb *mpp) +{ + if (!uuid_is_null(mpp->necp_client_uuid)) { + necp_client_unregister_multipath_cb(mpp->necp_client_uuid, mpp); + uuid_clear(mpp->necp_client_uuid); + } +} + /// Module init errno_t @@ -2950,16 +4865,40 @@ necp_client_init(void) goto done; } - necp_client_tcall = thread_call_allocate(necp_update_all_clients_callout, NULL); - if (necp_client_tcall == NULL) { - NECPLOG0(LOG_ERR, "thread_call_allocate failed"); + necp_client_fd_size = sizeof(struct necp_fd_data); + necp_client_fd_zone = zinit(necp_client_fd_size, + NECP_CLIENT_FD_ZONE_MAX * necp_client_fd_size, + 0, NECP_CLIENT_FD_ZONE_NAME); + if (necp_client_fd_zone == NULL) { + NECPLOG0(LOG_ERR, "zinit(necp_client_fd) failed"); + result = ENOMEM; + goto done; + } + + necp_flow_size = sizeof(struct necp_client_flow); + necp_flow_zone = zinit(necp_flow_size, + NECP_FLOW_ZONE_MAX * necp_flow_size, + 0, NECP_FLOW_ZONE_NAME); + if (necp_flow_zone == NULL) { + NECPLOG0(LOG_ERR, "zinit(necp_flow) failed"); result = ENOMEM; goto done; } + necp_client_update_tcall = thread_call_allocate_with_options(necp_update_all_clients_callout, NULL, + THREAD_CALL_PRIORITY_KERNEL, THREAD_CALL_OPTIONS_ONCE); + VERIFY(necp_client_update_tcall != NULL); + lck_rw_init(&necp_fd_lock, necp_fd_mtx_grp, necp_fd_mtx_attr); + lck_rw_init(&necp_observer_lock, necp_fd_mtx_grp, necp_fd_mtx_attr); + lck_rw_init(&necp_client_tree_lock, necp_fd_mtx_grp, necp_fd_mtx_attr); + lck_rw_init(&necp_collect_stats_list_lock, necp_fd_mtx_grp, necp_fd_mtx_attr); LIST_INIT(&necp_fd_list); + LIST_INIT(&necp_fd_observer_list); + LIST_INIT(&necp_collect_stats_client_list); + + RB_INIT(&necp_client_global_tree); done: if (result != 0) { diff --git a/bsd/net/net_api_stats.h b/bsd/net/net_api_stats.h new file mode 100644 index 000000000..af986b05e --- /dev/null +++ b/bsd/net/net_api_stats.h @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef __NET_API_STATS__ +#define __NET_API_STATS__ + +#ifdef PRIVATE +#include <stdint.h> + +/* + * net_api_stats counts the usage of the networking APIs + * + * Note: we are using signed 64 bit values to detect and prevent wrap around + */ +struct net_api_stats { + /* + * Interface Filters + */ + int64_t nas_iflt_attach_count; // Currently attached + int64_t nas_iflt_attach_total; // Total number of attachments + int64_t nas_iflt_attach_os_total; + + /* + * IP Filters + */ + int64_t nas_ipf_add_count; // Currently attached + int64_t nas_ipf_add_total; // Total number of attachments + int64_t nas_ipf_add_os_total; + + /* + * Socket Filters + */ + int64_t nas_sfltr_register_count; // Currently attached + int64_t nas_sfltr_register_total; // Total number of attachments + int64_t nas_sfltr_register_os_total; + + /* + * Sockets + */ + int64_t nas_socket_alloc_total; + int64_t nas_socket_in_kernel_total; + int64_t nas_socket_in_kernel_os_total; + int64_t nas_socket_necp_clientuuid_total; + + /* + * Sockets per protocol domains + */ + int64_t nas_socket_domain_local_total; + int64_t nas_socket_domain_route_total; + int64_t nas_socket_domain_inet_total; + int64_t nas_socket_domain_inet6_total; + int64_t nas_socket_domain_system_total; + int64_t nas_socket_domain_multipath_total; + int64_t nas_socket_domain_key_total; + int64_t nas_socket_domain_ndrv_total; + int64_t nas_socket_domain_other_total; + + /* + * Sockets per domain and type + */ + int64_t nas_socket_inet_stream_total; + int64_t nas_socket_inet_dgram_total; + int64_t nas_socket_inet_dgram_connected; + int64_t nas_socket_inet_dgram_dns; // port 53 + int64_t nas_socket_inet_dgram_no_data; // typically for interface ioctl + + int64_t nas_socket_inet6_stream_total; + int64_t nas_socket_inet6_dgram_total; + int64_t nas_socket_inet6_dgram_connected; + int64_t nas_socket_inet6_dgram_dns; // port 53 + int64_t nas_socket_inet6_dgram_no_data; // typically for interface ioctl + + /* + * Multicast join + */ + int64_t nas_socket_mcast_join_total; + int64_t nas_socket_mcast_join_os_total; + + /* + * IPv6 Extension Header Socket API + */ + int64_t nas_sock_inet6_stream_exthdr_in; + int64_t nas_sock_inet6_stream_exthdr_out; + int64_t nas_sock_inet6_dgram_exthdr_in; + int64_t nas_sock_inet6_dgram_exthdr_out; + + /* + * Nexus flows + */ + int64_t nas_nx_flow_inet_stream_total; + int64_t nas_nx_flow_inet_dgram_total; + + int64_t nas_nx_flow_inet6_stream_total; + int64_t nas_nx_flow_inet6_dgram_total; + + /* + * Interfaces + */ + int64_t nas_ifnet_alloc_count; + int64_t nas_ifnet_alloc_total; + int64_t nas_ifnet_alloc_os_count; + int64_t nas_ifnet_alloc_os_total; + + /* + * PF + */ + int64_t nas_pf_addrule_total; + int64_t nas_pf_addrule_os; + + /* + * vmnet API + */ + int64_t nas_vmnet_total; +}; + +#ifdef XNU_KERNEL_PRIVATE +extern struct net_api_stats net_api_stats; + +/* + * Increment up to the max value of int64_t + */ +#define INC_ATOMIC_INT64_LIM(counter) { \ + int64_t val; \ + do { \ + val = counter; \ + if (val >= INT64_MAX) { \ + break; \ + } \ + } while (!OSCompareAndSwap64(val, val + 1, &(counter))); \ +} +#endif /* XNU_KERNEL_PRIVATE */ + +#endif /* PRIVATE */ + +#endif /* __NET_API_STATS__ */ diff --git a/bsd/net/net_kev.h b/bsd/net/net_kev.h index ba1de1cbe..366b801a3 100644 --- a/bsd/net/net_kev.h +++ b/bsd/net/net_kev.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 Apple Inc. All rights reserved. + * Copyright (c) 2016-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -101,6 +101,7 @@ #define KEV_INET6_NEW_LL_ADDR 4 /* Autoconf LL address appeared */ #define KEV_INET6_NEW_RTADV_ADDR 5 /* Autoconf address has appeared */ #define KEV_INET6_DEFROUTER 6 /* Default router detected */ +#define KEV_INET6_REQUEST_NAT64_PREFIX 7 /* Asking for the NAT64-prefix */ #ifdef PRIVATE #define KEV_ND6_SUBCLASS 7 /* IPv6 NDP subclass */ @@ -108,6 +109,13 @@ #define KEV_ND6_RA 1 #define KEV_ND6_NDFAILURE 2 /* IPv6 neighbor cache entry expiry */ #define KEV_ND6_NDALIVE 3 /* IPv6 neighbor reachable */ +#define KEV_ND6_DAD_FAILURE 4 /* IPv6 address failed DAD */ +#define KEV_ND6_DAD_SUCCESS 5 /* IPv6 address completed DAD */ +#define KEV_ND6_ADDR_DETACHED 6 /* IPv6 address is deemed detached */ +#define KEV_ND6_ADDR_DEPRECATED 7 /* IPv6 address's preferred lifetime expired */ +#define KEV_ND6_ADDR_EXPIRED 8 /* IPv6 address has expired */ +#define KEV_ND6_RTR_EXPIRED 9 /* IPv6 default router has expired */ +#define KEV_ND6_PFX_EXPIRED 10 /* IPv6 prefix has expired */ #define KEV_NECP_SUBCLASS 8 /* NECP subclasss */ /* KEV_NECP_SUBCLASS event codes */ @@ -127,8 +135,11 @@ #define KEV_NETEVENT_SUBCLASS 11 /* Generic Net events subclass */ /* KEV_NETEVENT_SUBCLASS event codes */ #define KEV_NETEVENT_APNFALLBACK 1 -#endif /* PRIVATE */ -#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */ +#define KEV_MPTCP_SUBCLASS 12 /* Global MPTCP events subclass */ +/* KEV_MPTCP_SUBCLASS event codes */ +#define KEV_MPTCP_CELLUSE 1 +#endif /* PRIVATE */ +#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */ #endif /* _NET_NETKEV_H_ */ diff --git a/bsd/net/net_stubs.c b/bsd/net/net_stubs.c index 31ac0627a..137682285 100644 --- a/bsd/net/net_stubs.c +++ b/bsd/net/net_stubs.c @@ -86,6 +86,7 @@ STUB(ifmaddr_release); STUB(ifnet_add_multicast); STUB(ifnet_addrlen); STUB(ifnet_allocate); +STUB(ifnet_allocate_internal); STUB(ifnet_attach); STUB(ifnet_attach_protocol); STUB(ifnet_baudrate); @@ -337,8 +338,6 @@ STUB(ifnet_set_poll_params); STUB(ifnet_set_rcvq_maxlen); STUB(ifnet_set_sndq_maxlen); STUB(ifnet_start); -STUB(ifnet_transmit_burst_end); -STUB(ifnet_transmit_burst_start); STUB(ifnet_tx_compl_status); STUB(ifnet_tx_compl); STUB(ifnet_flowid); @@ -346,9 +345,6 @@ STUB(ifnet_enable_output); STUB(ifnet_disable_output); STUB(ifnet_get_keepalive_offload_frames); STUB(ifnet_link_status_report); -STUB(ifnet_set_packetpreamblelen); -STUB(ifnet_packetpreamblelen); -STUB(ifnet_maxpacketpreamblelen); STUB(ifnet_set_fastlane_capable); STUB(ifnet_get_fastlane_capable); STUB(ifnet_get_unsent_bytes); @@ -454,6 +450,12 @@ STUB(arp_ifinit); STUB(arp_lookup_ip); STUB(ip_gre_register_input); STUB(sock_iskernel); +STUB(iflt_attach_internal); +STUB(ipf_addv4_internal); +STUB(ipf_addv6_internal); +STUB(sflt_register_internal); +STUB(sock_accept_internal); +STUB(sock_socket_internal); #undef STUB /* diff --git a/bsd/net/netsrc.c b/bsd/net/netsrc.c index 052858985..17f00fead 100644 --- a/bsd/net/netsrc.c +++ b/bsd/net/netsrc.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2013 Apple Inc. All rights reserved. + * Copyright (c) 2011-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -26,6 +26,19 @@ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ +// Include netinet/in.h first. net/netsrc.h depends on netinet/in.h but +// netinet/in.h doesn't work with -Wpadded, -Wpacked. +#include <netinet/in.h> + +#pragma clang diagnostic push +#pragma clang diagnostic error "-Wpadded" +#pragma clang diagnostic error "-Wpacked" +// This header defines structures shared with user space, so we need to ensure there is +// no compiler inserted padding in case the user space process isn't using the same +// architecture as the kernel (example: i386 process with x86_64 kernel). +#include <net/netsrc.h> +#pragma clang diagnostic pop + #include <sys/param.h> #include <sys/types.h> #include <sys/kpi_mbuf.h> @@ -48,29 +61,7 @@ #include <netinet/ip6.h> #include <netinet6/ip6_var.h> -#include <net/netsrc.h> - -static errno_t netsrc_ctlsend(kern_ctl_ref, uint32_t, void *, mbuf_t, int); -static errno_t netsrc_ctlconnect(kern_ctl_ref, struct sockaddr_ctl *, void **); -static errno_t netsrc_ipv4(kern_ctl_ref, uint32_t, struct netsrc_req *); -static errno_t netsrc_ipv6(kern_ctl_ref, uint32_t, struct netsrc_req *); - -static kern_ctl_ref netsrc_ctlref = NULL; - -__private_extern__ void -netsrc_init(void) -{ - errno_t error; - struct kern_ctl_reg netsrc_ctl = { - .ctl_connect = netsrc_ctlconnect, - .ctl_send = netsrc_ctlsend, - }; - - strlcpy(netsrc_ctl.ctl_name, NETSRC_CTLNAME, sizeof(NETSRC_CTLNAME)); - - if ((error = ctl_register(&netsrc_ctl, &netsrc_ctlref))) - printf("%s: ctl_register failed %d\n", __func__, error); -} +#include <net/ntstat.h> static errno_t netsrc_ctlconnect(kern_ctl_ref kctl, struct sockaddr_ctl *sac, void **uinfo) @@ -84,6 +75,198 @@ netsrc_ctlconnect(kern_ctl_ref kctl, struct sockaddr_ctl *sac, void **uinfo) return (0); } +static errno_t +netsrc_reply(kern_ctl_ref kctl, uint32_t unit, uint16_t version, + struct netsrc_rep *reply) +{ + switch (version) { + case NETSRC_CURVERS: + return ctl_enqueuedata(kctl, unit, reply, + sizeof(*reply), CTL_DATA_EOR); + case NETSRC_VERSION1: { + if ((reply->nrp_flags & NETSRC_FLAG_ROUTEABLE) == 0) { + return EHOSTUNREACH; + } +#define NETSRC_FLAG_V1_MASK (NETSRC_IP6_FLAG_TENTATIVE | \ + NETSRC_IP6_FLAG_TEMPORARY | \ + NETSRC_IP6_FLAG_DEPRECATED | \ + NETSRC_IP6_FLAG_OPTIMISTIC | \ + NETSRC_IP6_FLAG_SECURED) + struct netsrc_repv1 v1 = { + .nrp_src = reply->nrp_src, + .nrp_flags = (reply->nrp_flags & NETSRC_FLAG_V1_MASK), + .nrp_label = reply->nrp_label, + .nrp_precedence = reply->nrp_precedence, + .nrp_dstlabel = reply->nrp_dstlabel, + .nrp_dstprecedence = reply->nrp_dstprecedence + }; + return ctl_enqueuedata(kctl, unit, &v1, sizeof(v1), CTL_DATA_EOR); + } + } + return EINVAL; +} + +static void +netsrc_common(struct rtentry *rt, struct netsrc_rep *reply) +{ + if (!rt) { + return; + } + + // Gather statistics information + struct nstat_counts *rt_stats = rt->rt_stats; + if (rt_stats) { + reply->nrp_min_rtt = rt_stats->nstat_min_rtt; + reply->nrp_connection_attempts = rt_stats->nstat_connectattempts; + reply->nrp_connection_successes = rt_stats->nstat_connectsuccesses; + } + + // If this route didn't have any stats, check its parent + if (reply->nrp_min_rtt == 0) { + // Is this lock necessary? + RT_LOCK(rt); + if (rt->rt_parent) { + rt_stats = rt->rt_parent->rt_stats; + if (rt_stats) { + reply->nrp_min_rtt = rt_stats->nstat_min_rtt; + reply->nrp_connection_attempts = rt_stats->nstat_connectattempts; + reply->nrp_connection_successes = rt_stats->nstat_connectsuccesses; + } + } + RT_UNLOCK(rt); + } + reply->nrp_ifindex = rt->rt_ifp ? rt->rt_ifp->if_index : 0; + + if (rt->rt_ifp->if_eflags & IFEF_AWDL) { + reply->nrp_flags |= NETSRC_FLAG_AWDL; + } + if (rt->rt_flags & RTF_LOCAL) { + reply->nrp_flags |= NETSRC_FLAG_DIRECT; + } else if (!(rt->rt_flags & RTF_GATEWAY) && + (rt->rt_ifa && rt->rt_ifa->ifa_ifp && + !(rt->rt_ifa->ifa_ifp->if_flags & IFF_POINTOPOINT))) { + reply->nrp_flags |= NETSRC_FLAG_DIRECT; + } +} + +static struct in6_addrpolicy * +lookup_policy(struct sockaddr* sa) +{ + // alignment fun - if sa_family is AF_INET or AF_INET6, this is one of those + // addresses and it should be aligned, so this should be safe. + union sockaddr_in_4_6 *addr = (union sockaddr_in_4_6 *)(void*)sa; + if (addr->sa.sa_family == AF_INET6) { + return in6_addrsel_lookup_policy(&addr->sin6); + } else if (sa->sa_family == AF_INET) { + struct sockaddr_in6 mapped = { + .sin6_family = AF_INET6, + .sin6_len = sizeof(mapped), + .sin6_addr = IN6ADDR_V4MAPPED_INIT, + }; + mapped.sin6_addr.s6_addr32[3] = addr->sin.sin_addr.s_addr; + return in6_addrsel_lookup_policy(&mapped); + } + return NULL; +} + +static void +netsrc_policy_common(struct netsrc_req *request, struct netsrc_rep *reply) +{ + // Destination policy + struct in6_addrpolicy *policy = lookup_policy(&request->nrq_dst.sa); + if (policy != NULL && policy->label != -1) { + reply->nrp_dstlabel = policy->label; + reply->nrp_dstprecedence = policy->preced; + } + + // Source policy + policy = lookup_policy(&reply->nrp_src.sa); + if (policy != NULL && policy->label != -1) { + reply->nrp_label = policy->label; + reply->nrp_precedence = policy->preced; + } +} + +static errno_t +netsrc_ipv6(kern_ctl_ref kctl, uint32_t unit, struct netsrc_req *request) +{ + struct route_in6 ro = { + .ro_dst = request->nrq_sin6, + }; + + int error = 0; + struct in6_addr storage, *in6 = in6_selectsrc(&request->nrq_sin6, NULL, + NULL, &ro, NULL, &storage, + request->nrq_ifscope, &error); + struct netsrc_rep reply = { + .nrp_sin6.sin6_family = AF_INET6, + .nrp_sin6.sin6_len = sizeof(reply.nrp_sin6), + .nrp_sin6.sin6_addr = in6 ? *in6 : (struct in6_addr){}, + }; + netsrc_common(ro.ro_rt, &reply); + if (ro.ro_srcia == NULL && in6 != NULL) { + ro.ro_srcia = (struct ifaddr *)ifa_foraddr6_scoped(in6, reply.nrp_ifindex); + } + if (ro.ro_srcia) { + struct in6_ifaddr *ia = (struct in6_ifaddr *)ro.ro_srcia; +#define IA_TO_NRP_FLAG(flag) \ + if (ia->ia6_flags & IN6_IFF_##flag) { \ + reply.nrp_flags |= NETSRC_FLAG_IP6_##flag; \ + } + IA_TO_NRP_FLAG(TENTATIVE); + IA_TO_NRP_FLAG(TEMPORARY); + IA_TO_NRP_FLAG(DEPRECATED); + IA_TO_NRP_FLAG(OPTIMISTIC); + IA_TO_NRP_FLAG(SECURED); + IA_TO_NRP_FLAG(DYNAMIC); + IA_TO_NRP_FLAG(AUTOCONF); +#undef IA_TO_NRP_FLAG + reply.nrp_flags |= NETSRC_FLAG_ROUTEABLE; + } + ROUTE_RELEASE(&ro); + netsrc_policy_common(request, &reply); + return netsrc_reply(kctl, unit, request->nrq_ver, &reply); +} + +static errno_t +netsrc_ipv4(kern_ctl_ref kctl, uint32_t unit, struct netsrc_req *request) +{ + // Unfortunately, IPv4 doesn't have a function like in6_selectsrc + // Look up the route + lck_mtx_lock(rnh_lock); + struct rtentry *rt = rt_lookup(TRUE, &request->nrq_dst.sa, + NULL, rt_tables[AF_INET], + request->nrq_ifscope); + lck_mtx_unlock(rnh_lock); + + // Look up the ifa + struct netsrc_rep reply = {}; + if (rt) { + struct in_ifaddr *ia = NULL; + lck_rw_lock_shared(in_ifaddr_rwlock); + TAILQ_FOREACH(ia, &in_ifaddrhead, ia_link) { + IFA_LOCK_SPIN(&ia->ia_ifa); + if (ia->ia_ifp == rt->rt_ifp) { + IFA_ADDREF_LOCKED(&ia->ia_ifa); + break; + } + IFA_UNLOCK(&ia->ia_ifa); + } + lck_rw_done(in_ifaddr_rwlock); + + if (ia) { + reply.nrp_sin = *IA_SIN(ia); + IFA_REMREF_LOCKED(&ia->ia_ifa); + IFA_UNLOCK(&ia->ia_ifa); + reply.nrp_flags |= NETSRC_FLAG_ROUTEABLE; + } + netsrc_common(rt, &reply); + rtfree(rt); + } + netsrc_policy_common(request, &reply); + return netsrc_reply(kctl, unit, request->nrq_ver, &reply); +} + static errno_t netsrc_ctlsend(kern_ctl_ref kctl, uint32_t unit, void *uinfo, mbuf_t m, int flags) @@ -102,17 +285,26 @@ netsrc_ctlsend(kern_ctl_ref kctl, uint32_t unit, void *uinfo, mbuf_t m, mbuf_copydata(m, 0, sizeof(storage), &storage); nrq = &storage; } - /* We only have one version right now. */ - if (nrq->nrq_ver != NETSRC_VERSION1) { + if (nrq->nrq_ver > NETSRC_CURVERS) { error = EINVAL; goto out; } switch (nrq->nrq_sin.sin_family) { case AF_INET: - error = netsrc_ipv4(kctl, unit, nrq); + if (nrq->nrq_sin.sin_len < sizeof (nrq->nrq_sin) || + nrq->nrq_sin.sin_addr.s_addr == INADDR_ANY) { + error = EINVAL; + } else { + error = netsrc_ipv4(kctl, unit, nrq); + } break; case AF_INET6: - error = netsrc_ipv6(kctl, unit, nrq); + if (nrq->nrq_sin6.sin6_len < sizeof(nrq->nrq_sin6) || + IN6_IS_ADDR_UNSPECIFIED(&nrq->nrq_sin6.sin6_addr)) { + error = EINVAL; + } else { + error = netsrc_ipv6(kctl, unit, nrq); + } break; default: printf("%s: invalid family\n", __func__); @@ -125,132 +317,19 @@ out: } -static errno_t -netsrc_ipv4(kern_ctl_ref kctl, uint32_t unit, struct netsrc_req *nrq) +__private_extern__ void +netsrc_init(void) { - errno_t error = EHOSTUNREACH; - struct sockaddr_in *dstsin; - struct rtentry *rt; - struct in_ifaddr *ia; - struct netsrc_rep nrp; - struct sockaddr_in6 v4entry = { - .sin6_family = AF_INET6, - .sin6_len = sizeof(struct sockaddr_in6), - .sin6_addr = IN6ADDR_V4MAPPED_INIT, + struct kern_ctl_reg netsrc_ctl = { + .ctl_connect = netsrc_ctlconnect, + .ctl_send = netsrc_ctlsend, }; - struct in6_addrpolicy *policy; - dstsin = &nrq->nrq_sin; + strlcpy(netsrc_ctl.ctl_name, NETSRC_CTLNAME, sizeof(netsrc_ctl.ctl_name)); - if (dstsin->sin_len < sizeof (*dstsin) || - dstsin->sin_addr.s_addr == INADDR_ANY) - return (EINVAL); - - lck_mtx_lock(rnh_lock); - rt = rt_lookup(TRUE, (struct sockaddr *)dstsin, NULL, - rt_tables[AF_INET], nrq->nrq_ifscope); - lck_mtx_unlock(rnh_lock); - if (!rt) - return (EHOSTUNREACH); - lck_rw_lock_shared(in_ifaddr_rwlock); - TAILQ_FOREACH(ia, &in_ifaddrhead, ia_link) { - IFA_LOCK_SPIN(&ia->ia_ifa); - if (ia->ia_ifp == rt->rt_ifp) { - memset(&nrp, 0, sizeof(nrp)); - memcpy(&nrp.nrp_sin, IA_SIN(ia), sizeof(nrp.nrp_sin)); - IFA_UNLOCK(&ia->ia_ifa); - v4entry.sin6_addr.s6_addr32[3] = - nrp.nrp_sin.sin_addr.s_addr; - policy = in6_addrsel_lookup_policy(&v4entry); - if (policy->label != -1) { - nrp.nrp_label = policy->label; - nrp.nrp_precedence = policy->preced; - /* XXX might not be true */ - nrp.nrp_dstlabel = policy->label; - nrp.nrp_dstprecedence = policy->preced; - } - error = ctl_enqueuedata(kctl, unit, &nrp, - sizeof(nrp), CTL_DATA_EOR); - break; - } - IFA_UNLOCK(&ia->ia_ifa); - } - lck_rw_done(in_ifaddr_rwlock); - if (rt) - rtfree(rt); - - return (error); -} - -static errno_t -netsrc_ipv6(kern_ctl_ref kctl, uint32_t unit, struct netsrc_req *nrq) -{ - struct sockaddr_in6 *dstsin6; - struct in6_addr *in6, storage; - struct in6_ifaddr *ia; - struct route_in6 ro; - int error = EHOSTUNREACH; - struct netsrc_rep nrp; - - dstsin6 = &nrq->nrq_sin6; - - if (dstsin6->sin6_len < sizeof (*dstsin6) || - IN6_IS_ADDR_UNSPECIFIED(&dstsin6->sin6_addr)) - return (EINVAL); - - memset(&ro, 0, sizeof(ro)); - lck_mtx_lock(rnh_lock); - ro.ro_rt = rt_lookup(TRUE, (struct sockaddr *)dstsin6, NULL, - rt_tables[AF_INET6], nrq->nrq_ifscope); - lck_mtx_unlock(rnh_lock); - if (!ro.ro_rt) - return (EHOSTUNREACH); - in6 = in6_selectsrc(dstsin6, NULL, NULL, &ro, NULL, &storage, - nrq->nrq_ifscope, &error); - ROUTE_RELEASE(&ro); - if (!in6 || error) - return (error); - memset(&nrp, 0, sizeof(nrp)); - nrp.nrp_sin6.sin6_family = AF_INET6; - nrp.nrp_sin6.sin6_len = sizeof(nrp.nrp_sin6); - memcpy(&nrp.nrp_sin6.sin6_addr, in6, sizeof(nrp.nrp_sin6.sin6_addr)); - lck_rw_lock_shared(&in6_ifaddr_rwlock); - for (ia = in6_ifaddrs; ia; ia = ia->ia_next) { - if (memcmp(&ia->ia_addr.sin6_addr, in6, sizeof(*in6)) == 0) { - struct sockaddr_in6 sin6; - struct in6_addrpolicy *policy; - - if (ia->ia6_flags & IN6_IFF_TEMPORARY) - nrp.nrp_flags |= NETSRC_IP6_FLAG_TEMPORARY; - if (ia->ia6_flags & IN6_IFF_TENTATIVE) - nrp.nrp_flags |= NETSRC_IP6_FLAG_TENTATIVE; - if (ia->ia6_flags & IN6_IFF_DEPRECATED) - nrp.nrp_flags |= NETSRC_IP6_FLAG_DEPRECATED; - if (ia->ia6_flags & IN6_IFF_OPTIMISTIC) - nrp.nrp_flags |= NETSRC_IP6_FLAG_OPTIMISTIC; - if (ia->ia6_flags & IN6_IFF_SECURED) - nrp.nrp_flags |= NETSRC_IP6_FLAG_SECURED; - sin6.sin6_family = AF_INET6; - sin6.sin6_len = sizeof(sin6); - memcpy(&sin6.sin6_addr, in6, sizeof(*in6)); - policy = in6_addrsel_lookup_policy(&sin6); - if (policy->label != -1) { - nrp.nrp_label = policy->label; - nrp.nrp_precedence = policy->preced; - } - memcpy(&sin6.sin6_addr, &dstsin6->sin6_addr, - sizeof(dstsin6->sin6_addr)); - policy = in6_addrsel_lookup_policy(&sin6); - if (policy->label != -1) { - nrp.nrp_dstlabel = policy->label; - nrp.nrp_dstprecedence = policy->preced; - } - break; - } + static kern_ctl_ref netsrc_ctlref = NULL; + errno_t error = ctl_register(&netsrc_ctl, &netsrc_ctlref); + if (error != 0) { + printf("%s: ctl_register failed %d\n", __func__, error); } - lck_rw_done(&in6_ifaddr_rwlock); - error = ctl_enqueuedata(kctl, unit, &nrp, sizeof(nrp), - CTL_DATA_EOR); - - return (error); } diff --git a/bsd/net/netsrc.h b/bsd/net/netsrc.h index 13fcd456b..02d43fa19 100644 --- a/bsd/net/netsrc.h +++ b/bsd/net/netsrc.h @@ -28,28 +28,31 @@ #ifndef __NET_NETSRC_H__ +#include <netinet/in.h> + #define NETSRC_CTLNAME "com.apple.netsrc" #define NETSRC_VERSION1 1 -#define NETSRC_CURVERS NETSRC_VERSION1 +#define NETSRC_VERSION2 2 +#define NETSRC_CURVERS NETSRC_VERSION2 struct netsrc_req { unsigned int nrq_ver; unsigned int nrq_ifscope; union { - struct sockaddr_in _usin; - struct sockaddr_in6 _usin6; - } _usa; + union sockaddr_in_4_6 nrq_dst; + union sockaddr_in_4_6 _usa; + }; }; -#define nrq_sin _usa._usin -#define nrq_sin6 _usa._usin6 +#define nrq_sin _usa.sin +#define nrq_sin6 _usa.sin6 -struct netsrc_rep { +struct netsrc_repv1 { union { - struct sockaddr_in _usin; - struct sockaddr_in6 _usin6; - } _usa; + union sockaddr_in_4_6 nrp_src; + union sockaddr_in_4_6 _usa; + }; #define NETSRC_IP6_FLAG_TENTATIVE 0x0001 #define NETSRC_IP6_FLAG_TEMPORARY 0x0002 #define NETSRC_IP6_FLAG_DEPRECATED 0x0004 @@ -60,10 +63,41 @@ struct netsrc_rep { uint16_t nrp_precedence; uint16_t nrp_dstlabel; uint16_t nrp_dstprecedence; + uint16_t nrp_unused; // Padding }; -#define nrp_sin _usa._usin -#define nrp_sin6 _usa._usin6 +struct netsrc_repv2 { + union { + union sockaddr_in_4_6 nrp_src; + union sockaddr_in_4_6 _usa; + }; + uint32_t nrp_min_rtt; + uint32_t nrp_connection_attempts; + uint32_t nrp_connection_successes; + // Continues from above, fixes naming +#define NETSRC_FLAG_IP6_TENTATIVE NETSRC_IP6_FLAG_TENTATIVE +#define NETSRC_FLAG_IP6_TEMPORARY NETSRC_IP6_FLAG_TEMPORARY +#define NETSRC_FLAG_IP6_DEPRECATED NETSRC_IP6_FLAG_DEPRECATED +#define NETSRC_FLAG_IP6_OPTIMISTIC NETSRC_IP6_FLAG_OPTIMISTIC +#define NETSRC_FLAG_IP6_SECURED NETSRC_IP6_FLAG_SECURED +#define NETSRC_FLAG_ROUTEABLE 0x00000020 +#define NETSRC_FLAG_DIRECT 0x00000040 +#define NETSRC_FLAG_AWDL 0x00000080 +#define NETSRC_FLAG_IP6_DYNAMIC 0x00000100 +#define NETSRC_FLAG_IP6_AUTOCONF 0x00000200 + uint32_t nrp_flags; + uint16_t nrp_label; + uint16_t nrp_precedence; + uint16_t nrp_dstlabel; + uint16_t nrp_dstprecedence; + uint16_t nrp_ifindex; + uint16_t nrp_unused; // Padding +}; + +#define netsrc_rep netsrc_repv2 + +#define nrp_sin nrp_src.sin +#define nrp_sin6 nrp_src.sin6 #ifdef KERNEL_PRIVATE __private_extern__ void netsrc_init(void); diff --git a/bsd/net/network_agent.c b/bsd/net/network_agent.c index a51eacfb8..14fe6dc99 100644 --- a/bsd/net/network_agent.c +++ b/bsd/net/network_agent.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2016, 2017 Apple Inc. All rights reserved. + * Copyright (c) 2014-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -80,6 +80,8 @@ LIST_HEAD(netagent_client_list_s, netagent_client); struct netagent_wrapper { LIST_ENTRY(netagent_wrapper) master_chain; u_int32_t control_unit; + netagent_event_f event_handler; + void *event_context; u_int32_t generation; u_int64_t use_count; struct netagent_client_list_s pending_triggers_list; @@ -87,8 +89,10 @@ struct netagent_wrapper { }; struct netagent_session { - u_int32_t control_unit; + u_int32_t control_unit; // A control unit of 0 indicates an agent owned by the kernel struct netagent_wrapper *wrapper; + netagent_event_f event_handler; + void *event_context; }; typedef enum { @@ -608,6 +612,18 @@ done: return (new_session); } +netagent_session_t netagent_create(netagent_event_f event_handler, void *context) +{ + struct netagent_session *session = netagent_create_session(0); + if (session == NULL) { + return NULL; + } + + session->event_handler = event_handler; + session->event_context = context; + return session; +} + static void netagent_free_wrapper(struct netagent_wrapper *wrapper) { @@ -668,6 +684,11 @@ netagent_delete_session(struct netagent_session *session) } } +void netagent_destroy(netagent_session_t session) +{ + return netagent_delete_session((struct netagent_session *)session); +} + static int netagent_packet_get_netagent_data_size(mbuf_t packet, int offset, int *err) { @@ -693,6 +714,8 @@ netagent_handle_register_inner(struct netagent_session *session, struct netagent lck_rw_lock_exclusive(&netagent_lock); new_wrapper->control_unit = session->control_unit; + new_wrapper->event_handler = session->event_handler; + new_wrapper->event_context = session->event_context; new_wrapper->generation = g_next_generation++; session->wrapper = new_wrapper; @@ -710,6 +733,56 @@ netagent_handle_register_inner(struct netagent_session *session, struct netagent return 0; } +errno_t +netagent_register(netagent_session_t _session, struct netagent *agent) +{ + int data_size = 0; + struct netagent_wrapper *new_wrapper = NULL; + + struct netagent_session *session = (struct netagent_session *)_session; + if (session == NULL) { + NETAGENTLOG0(LOG_ERR, "Cannot register agent on NULL session"); + return EINVAL; + } + + if (agent == NULL) { + NETAGENTLOG0(LOG_ERR, "Cannot register NULL agent"); + return EINVAL; + } + + if (session->wrapper != NULL) { + NETAGENTLOG0(LOG_ERR, "Session already has a registered agent"); + return EINVAL; + } + + data_size = agent->netagent_data_size; + if (data_size < 0 || data_size > NETAGENT_MAX_DATA_SIZE) { + NETAGENTLOG(LOG_ERR, "Register message size could not be read, data_size %d", + data_size); + return EINVAL; + } + + MALLOC(new_wrapper, struct netagent_wrapper *, sizeof(*new_wrapper) + data_size, M_NETAGENT, M_WAITOK); + if (new_wrapper == NULL) { + NETAGENTLOG0(LOG_ERR, "Failed to allocate agent"); + return ENOMEM; + } + + memset(new_wrapper, 0, sizeof(*new_wrapper) + data_size); + memcpy(&new_wrapper->netagent, agent, sizeof(struct netagent) + data_size); + + int error = netagent_handle_register_inner(session, new_wrapper); + if (error != 0) { + FREE(new_wrapper, M_NETAGENT); + return error; + } + + NETAGENTLOG0(LOG_DEBUG, "Registered new agent"); + netagent_post_event(new_wrapper->netagent.netagent_uuid, KEV_NETAGENT_REGISTERED, TRUE); + + return 0; +} + static errno_t netagent_handle_register_setopt(struct netagent_session *session, u_int8_t *payload, u_int32_t payload_length) @@ -846,6 +919,19 @@ fail: netagent_send_error_response(session, NETAGENT_MESSAGE_TYPE_REGISTER, message_id, response_error); } +errno_t +netagent_unregister(netagent_session_t _session) +{ + struct netagent_session *session = (struct netagent_session *)_session; + if (session == NULL) { + NETAGENTLOG0(LOG_ERR, "Cannot unregister NULL session"); + return EINVAL; + } + + netagent_unregister_session_wrapper(session); + return 0; +} + static errno_t netagent_handle_unregister_setopt(struct netagent_session *session, u_int8_t *payload, u_int32_t payload_length) @@ -978,6 +1064,8 @@ netagent_handle_update_inner(struct netagent_session *session, struct netagent_w netagent_free_wrapper(session->wrapper); session->wrapper = new_wrapper; new_wrapper->control_unit = session->control_unit; + new_wrapper->event_handler = session->event_handler; + new_wrapper->event_context = session->event_context; LIST_INSERT_HEAD(&master_netagent_list, new_wrapper, master_chain); LIST_INIT(&new_wrapper->pending_triggers_list); @@ -989,6 +1077,59 @@ netagent_handle_update_inner(struct netagent_session *session, struct netagent_w return response_error; } +errno_t +netagent_update(netagent_session_t _session, struct netagent *agent) +{ + u_int8_t agent_changed; + int data_size = 0; + struct netagent_wrapper *new_wrapper = NULL; + + struct netagent_session *session = (struct netagent_session *)_session; + if (session == NULL) { + NETAGENTLOG0(LOG_ERR, "Cannot update agent on NULL session"); + return EINVAL; + } + + if (agent == NULL) { + NETAGENTLOG0(LOG_ERR, "Cannot register NULL agent"); + return EINVAL; + } + + if (session->wrapper == NULL) { + NETAGENTLOG0(LOG_ERR, "Session has no agent to update"); + return EINVAL; + } + + data_size = agent->netagent_data_size; + if (data_size > NETAGENT_MAX_DATA_SIZE) { + NETAGENTLOG(LOG_ERR, "Update message size (%u > %u) too large", data_size, NETAGENT_MAX_DATA_SIZE); + return EINVAL; + } + + MALLOC(new_wrapper, struct netagent_wrapper *, sizeof(*new_wrapper) + data_size, M_NETAGENT, M_WAITOK); + if (new_wrapper == NULL) { + NETAGENTLOG0(LOG_ERR, "Failed to allocate agent"); + return ENOMEM; + } + + memset(new_wrapper, 0, sizeof(*new_wrapper) + data_size); + memcpy(&new_wrapper->netagent, agent, sizeof(struct netagent) + data_size); + + int error = netagent_handle_update_inner(session, new_wrapper, data_size, &agent_changed, kNetagentErrorDomainPOSIX); + if (error == 0) { + netagent_post_event(session->wrapper->netagent.netagent_uuid, KEV_NETAGENT_UPDATED, agent_changed); + if (agent_changed == FALSE) { + // The session wrapper does not need the "new_wrapper" as nothing changed + FREE(new_wrapper, M_NETAGENT); + } + } else { + FREE(new_wrapper, M_NETAGENT); + return error; + } + + return 0; +} + static errno_t netagent_handle_update_setopt(struct netagent_session *session, u_int8_t *payload, u_int32_t payload_length) { @@ -1182,6 +1323,57 @@ fail: netagent_send_error_response(session, NETAGENT_MESSAGE_TYPE_GET, message_id, response_error); } +errno_t +netagent_assign_nexus(netagent_session_t _session, uuid_t necp_client_uuid, + void *assign_message, size_t assigned_results_length) +{ + struct netagent_session *session = (struct netagent_session *)_session; + if (session == NULL) { + NETAGENTLOG0(LOG_ERR, "Cannot assign nexus from NULL session"); + return EINVAL; + } + + if (session->wrapper == NULL) { + NETAGENTLOG0(LOG_ERR, "Session has no agent"); + return ENOENT; + } + + // Note that if the error is 0, NECP has taken over our malloc'ed buffer + int error = necp_assign_client_result(session->wrapper->netagent.netagent_uuid, necp_client_uuid, assign_message, assigned_results_length); + if (error) { + // necp_assign_client_result returns POSIX errors; don't error for ENOENT + NETAGENTLOG((error == ENOENT ? LOG_DEBUG : LOG_ERR), "Client assignment failed: %d", error); + return error; + } + + NETAGENTLOG0(LOG_DEBUG, "Agent assigned nexus properties to client"); + return 0; +} + +errno_t +netagent_update_flow_protoctl_event(netagent_session_t _session, + uuid_t client_id, uint32_t protoctl_event_code, + uint32_t protoctl_event_val, uint32_t protoctl_event_tcp_seq_number) +{ + struct netagent_session *session = (struct netagent_session *)_session; + int error = 0; + + if (session == NULL) { + NETAGENTLOG0(LOG_ERR, "Cannot assign nexus from NULL session"); + return (EINVAL); + } + + if (session->wrapper == NULL) { + NETAGENTLOG0(LOG_ERR, "Session has no agent"); + return (ENOENT); + } + + error = necp_update_flow_protoctl_event(session->wrapper->netagent.netagent_uuid, + client_id, protoctl_event_code, protoctl_event_val, protoctl_event_tcp_seq_number); + + return (error); +} + static errno_t netagent_handle_assign_nexus_setopt(struct netagent_session *session, u_int8_t *payload, u_int32_t payload_length) @@ -1676,7 +1868,13 @@ done: } int -netagent_client_message(uuid_t agent_uuid, uuid_t necp_client_uuid, u_int8_t message_type) +netagent_client_message_with_params(uuid_t agent_uuid, + uuid_t necp_client_uuid, + pid_t pid, + u_int8_t message_type, + struct necp_client_nexus_parameters *parameters, + void **assigned_results, + size_t *assigned_results_length) { int error = 0; @@ -1684,7 +1882,8 @@ netagent_client_message(uuid_t agent_uuid, uuid_t necp_client_uuid, u_int8_t mes message_type != NETAGENT_MESSAGE_TYPE_CLIENT_ASSERT && message_type != NETAGENT_MESSAGE_TYPE_CLIENT_UNASSERT && message_type != NETAGENT_MESSAGE_TYPE_REQUEST_NEXUS && - message_type != NETAGENT_MESSAGE_TYPE_CLOSE_NEXUS) { + message_type != NETAGENT_MESSAGE_TYPE_CLOSE_NEXUS && + message_type != NETAGENT_MESSAGE_TYPE_ABORT_NEXUS) { NETAGENTLOG(LOG_ERR, "Client netagent message type (%d) is invalid", message_type); return(EINVAL); } @@ -1693,7 +1892,7 @@ netagent_client_message(uuid_t agent_uuid, uuid_t necp_client_uuid, u_int8_t mes bool should_unlock = TRUE; struct netagent_wrapper *wrapper = netagent_find_agent_with_uuid(agent_uuid); if (wrapper == NULL) { - NETAGENTLOG0(LOG_ERR, "Requested netagent for nexus instance could not be found"); + NETAGENTLOG0(LOG_DEBUG, "Requested netagent for nexus instance could not be found"); error = ENOENT; goto done; } @@ -1704,19 +1903,25 @@ netagent_client_message(uuid_t agent_uuid, uuid_t necp_client_uuid, u_int8_t mes // Don't log, since this is a common case used to trigger events that cellular data is blocked, etc. error = ENOTSUP; - struct proc *p = current_proc(); - pid_t current_pid = 0; - uuid_t current_proc_uuid; - uuid_clear(current_proc_uuid); - if (p != NULL) { - current_pid = proc_pid(p); - proc_getexecutableuuid(p, current_proc_uuid, sizeof(current_proc_uuid)); + + pid_t report_pid = 0; + uuid_t report_proc_uuid = {}; + if (parameters != NULL) { + report_pid = parameters->epid; + uuid_copy(report_proc_uuid, parameters->euuid); + } else { + struct proc *p = current_proc(); + if (p != NULL) { + report_pid = proc_pid(p); + proc_getexecutableuuid(p, report_proc_uuid, sizeof(report_proc_uuid)); + } } - netagent_send_cellular_failed_event(wrapper, current_pid, current_proc_uuid); + netagent_send_cellular_failed_event(wrapper, report_pid, report_proc_uuid); goto done; } } else if (message_type == NETAGENT_MESSAGE_TYPE_REQUEST_NEXUS || - message_type == NETAGENT_MESSAGE_TYPE_CLOSE_NEXUS) { + message_type == NETAGENT_MESSAGE_TYPE_CLOSE_NEXUS || + message_type == NETAGENT_MESSAGE_TYPE_ABORT_NEXUS) { if ((wrapper->netagent.netagent_flags & NETAGENT_FLAG_NEXUS_PROVIDER) == 0) { NETAGENTLOG0(LOG_ERR, "Requested netagent for nexus instance is not a nexus provider"); // Agent is not a nexus provider @@ -1732,29 +1937,54 @@ netagent_client_message(uuid_t agent_uuid, uuid_t necp_client_uuid, u_int8_t mes } } - error = netagent_send_client_message(wrapper, necp_client_uuid, message_type); - if (error == 0 && message_type == NETAGENT_MESSAGE_TYPE_CLIENT_TRIGGER) { - if (lck_rw_lock_shared_to_exclusive(&netagent_lock)) { - // Grab the lock exclusively to add a pending client to the list - struct netagent_client *new_pending_client = NULL; - MALLOC(new_pending_client, struct netagent_client *, sizeof(*new_pending_client), M_NETAGENT, M_WAITOK); - if (new_pending_client == NULL) { - NETAGENTLOG0(LOG_ERR, "Failed to allocate client for trigger"); - } else { - uuid_copy(new_pending_client->client_id, necp_client_uuid); - struct proc *p = current_proc(); - if (p != NULL) { - new_pending_client->client_pid = proc_pid(p); - proc_getexecutableuuid(p, new_pending_client->client_proc_uuid, sizeof(new_pending_client->client_proc_uuid)); + if (wrapper->control_unit == 0) { + should_unlock = FALSE; + lck_rw_done(&netagent_lock); + if (wrapper->event_handler == NULL) { + // No event handler registered for kernel agent + error = EINVAL; + } else { + error = wrapper->event_handler(message_type, necp_client_uuid, pid, wrapper->event_context, parameters, assigned_results, assigned_results_length); + if (error != 0) { + VERIFY(assigned_results == NULL || *assigned_results == NULL); + VERIFY(assigned_results_length == NULL || *assigned_results_length == 0); + } + } + } else { + // ABORT_NEXUS is kernel-private, so translate it for userspace nexus + if (message_type == NETAGENT_MESSAGE_TYPE_ABORT_NEXUS) { + message_type = NETAGENT_MESSAGE_TYPE_CLOSE_NEXUS; + } + + error = netagent_send_client_message(wrapper, necp_client_uuid, message_type); + if (error == 0 && message_type == NETAGENT_MESSAGE_TYPE_CLIENT_TRIGGER) { + if (lck_rw_lock_shared_to_exclusive(&netagent_lock)) { + // Grab the lock exclusively to add a pending client to the list + struct netagent_client *new_pending_client = NULL; + MALLOC(new_pending_client, struct netagent_client *, sizeof(*new_pending_client), M_NETAGENT, M_WAITOK); + if (new_pending_client == NULL) { + NETAGENTLOG0(LOG_ERR, "Failed to allocate client for trigger"); + } else { + uuid_copy(new_pending_client->client_id, necp_client_uuid); + if (parameters != NULL) { + new_pending_client->client_pid = parameters->epid; + uuid_copy(new_pending_client->client_proc_uuid, parameters->euuid); + } else { + struct proc *p = current_proc(); + if (p != NULL) { + new_pending_client->client_pid = proc_pid(p); + proc_getexecutableuuid(p, new_pending_client->client_proc_uuid, sizeof(new_pending_client->client_proc_uuid)); + } + } + LIST_INSERT_HEAD(&wrapper->pending_triggers_list, new_pending_client, client_chain); } - LIST_INSERT_HEAD(&wrapper->pending_triggers_list, new_pending_client, client_chain); + } else { + // If lck_rw_lock_shared_to_exclusive fails, it unlocks automatically + should_unlock = FALSE; } - } else { - // If lck_rw_lock_shared_to_exclusive fails, it unlocks automatically - should_unlock = FALSE; } } - NETAGENTLOG((error ? LOG_ERR : LOG_INFO), "Send message %d for client (error %d)", message_type, error); + NETAGENTLOG(((error && error != ENOENT) ? LOG_ERR : LOG_INFO), "Send message %d for client (error %d)", message_type, error); done: if (should_unlock) { lck_rw_done(&netagent_lock); @@ -1762,6 +1992,12 @@ done: return (error); } +int +netagent_client_message(uuid_t agent_uuid, uuid_t necp_client_uuid, pid_t pid, u_int8_t message_type) +{ + return (netagent_client_message_with_params(agent_uuid, necp_client_uuid, pid, message_type, NULL, NULL, NULL)); +} + int netagent_use(uuid_t agent_uuid, uint64_t *out_use_count) { @@ -1795,7 +2031,7 @@ netagent_copyout(uuid_t agent_uuid, user_addr_t user_addr, u_int32_t user_size) lck_rw_lock_shared(&netagent_lock); struct netagent_wrapper *wrapper = netagent_find_agent_with_uuid(agent_uuid); if (wrapper == NULL) { - NETAGENTLOG0(LOG_ERR, "Requested netagent for nexus instance could not be found"); + NETAGENTLOG0(LOG_DEBUG, "Requested netagent for nexus instance could not be found"); error = ENOENT; goto done; } diff --git a/bsd/net/network_agent.h b/bsd/net/network_agent.h index f459e2f6d..3e2c86417 100644 --- a/bsd/net/network_agent.h +++ b/bsd/net/network_agent.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2016 Apple Inc. All rights reserved. + * Copyright (c) 2014-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -89,6 +89,7 @@ struct netagent_assign_nexus_message { #define NETAGENT_OPTION_TYPE_UPDATE NETAGENT_MESSAGE_TYPE_UPDATE // Pass netagent to update, no return value #define NETAGENT_OPTION_TYPE_ASSIGN_NEXUS NETAGENT_MESSAGE_TYPE_ASSIGN_NEXUS // Pass struct netagent_assign_nexus_message #define NETAGENT_OPTION_TYPE_USE_COUNT 16 // Pass use count to set, get current use count +#define NETAGENT_MESSAGE_TYPE_ABORT_NEXUS 17 // Kernel private #define NETAGENT_MESSAGE_FLAGS_RESPONSE 0x01 // Used for acks, errors, and query responses @@ -115,6 +116,7 @@ struct netagent_assign_nexus_message { #define NETAGENT_FLAG_SPECIFIC_USE_ONLY 0x0020 // Agent should only be used and activated when specifically required #define NETAGENT_FLAG_NETWORK_PROVIDER 0x0040 // Agent provides network access #define NETAGENT_FLAG_NEXUS_PROVIDER 0x0080 // Agent provides a skywalk nexus +#define NETAGENT_FLAG_SUPPORTS_BROWSE 0x0100 // Assertions will cause agent to fill in browse endpoints #define NETAGENT_NEXUS_MAX_REQUEST_TYPES 16 #define NETAGENT_NEXUS_MAX_RESOLUTION_TYPE_PAIRS 16 @@ -125,11 +127,19 @@ struct netagent_assign_nexus_message { #define NETAGENT_NEXUS_FRAME_TYPE_TRANSPORT 3 #define NETAGENT_NEXUS_FRAME_TYPE_APPLICATION 4 +#define NETAGENT_NEXUS_ENDPOINT_TYPE_ADDRESS 1 +#define NETAGENT_NEXUS_ENDPOINT_TYPE_HOST 2 +#define NETAGENT_NEXUS_ENDPOINT_TYPE_BONJOUR 3 + +#define NETAGENT_NEXUS_FLAG_SUPPORTS_USER_PACKET_POOL 0x1 +#define NETAGENT_NEXUS_FLAG_ASSERT_UNSUPPORTED 0x2 // No calls to assert the agent are required + struct netagent_nexus { u_int32_t frame_type; u_int32_t endpoint_assignment_type; u_int32_t endpoint_request_types[NETAGENT_NEXUS_MAX_REQUEST_TYPES]; u_int32_t endpoint_resolution_type_pairs[NETAGENT_NEXUS_MAX_RESOLUTION_TYPE_PAIRS * 2]; + u_int32_t nexus_flags; }; #define NETAGENT_TRIGGER_FLAG_USER 0x0001 // Userspace triggered agent @@ -196,6 +206,8 @@ struct netagentlist_req64 { user64_addr_t data __attribute__((aligned(8))); }; +struct necp_client_nexus_parameters; + // Kernel accessors extern void netagent_post_updated_interfaces(uuid_t uuid); // To be called from interface ioctls @@ -207,11 +219,60 @@ extern bool netagent_get_agent_domain_and_type(uuid_t uuid, char *domain, char * extern int netagent_kernel_trigger(uuid_t uuid); -extern int netagent_client_message(uuid_t agent_uuid, uuid_t necp_client_uuid, u_int8_t message_type); +extern int netagent_client_message(uuid_t agent_uuid, uuid_t necp_client_uuid, pid_t pid, u_int8_t message_type); + +extern int netagent_client_message_with_params(uuid_t agent_uuid, + uuid_t necp_client_uuid, + pid_t pid, + u_int8_t message_type, + struct necp_client_nexus_parameters *parameters, + void **assigned_results, + size_t *assigned_results_length); extern int netagent_copyout(uuid_t uuid, user_addr_t user_addr, u_int32_t user_size); + +// Kernel agent management + +typedef void * netagent_session_t; + +struct netagent_nexus_agent { + struct netagent agent; + struct netagent_nexus nexus_data; +}; + +#define NETAGENT_EVENT_TRIGGER NETAGENT_MESSAGE_TYPE_CLIENT_TRIGGER +#define NETAGENT_EVENT_ASSERT NETAGENT_MESSAGE_TYPE_CLIENT_ASSERT +#define NETAGENT_EVENT_UNASSERT NETAGENT_MESSAGE_TYPE_CLIENT_UNASSERT +#define NETAGENT_EVENT_NEXUS_FLOW_INSERT NETAGENT_MESSAGE_TYPE_REQUEST_NEXUS +#define NETAGENT_EVENT_NEXUS_FLOW_REMOVE NETAGENT_MESSAGE_TYPE_CLOSE_NEXUS +#define NETAGENT_EVENT_NEXUS_FLOW_ABORT NETAGENT_MESSAGE_TYPE_ABORT_NEXUS + +typedef errno_t (*netagent_event_f)(u_int8_t event, uuid_t necp_client_uuid, pid_t pid, void *context, struct necp_client_nexus_parameters *parameters, void **assigned_results, size_t *assigned_results_length); + +extern netagent_session_t netagent_create(netagent_event_f event_handler, void *handle); + +extern void netagent_destroy(netagent_session_t session); + +extern errno_t netagent_register(netagent_session_t session, struct netagent *agent); + +extern errno_t netagent_update(netagent_session_t session, struct netagent *agent); + +extern errno_t netagent_unregister(netagent_session_t session); + +extern errno_t netagent_assign_nexus(netagent_session_t _session, + uuid_t necp_client_uuid, + void *assign_message, + size_t assigned_results_length); // Length of assigned_results_length + +extern errno_t netagent_update_flow_protoctl_event(netagent_session_t _session, + uuid_t client_id, + uint32_t protoctl_event_code, + uint32_t protoctl_event_val, + uint32_t protoctl_event_tcp_seq_number); + extern int netagent_use(uuid_t agent_uuid, uint64_t *out_use_count); + #endif /* BSD_KERNEL_PRIVATE */ #ifndef KERNEL diff --git a/bsd/net/ntstat.c b/bsd/net/ntstat.c index 1135bc55e..1d51a92e7 100644 --- a/bsd/net/ntstat.c +++ b/bsd/net/ntstat.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2016 Apple Inc. All rights reserved. + * Copyright (c) 2010-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -50,12 +50,25 @@ #include <net/if_var.h> #include <net/if_types.h> #include <net/route.h> + +// These includes appear in ntstat.h but we include them here first so they won't trigger +// any clang diagnostic errors. +#include <netinet/in.h> +#include <netinet/in_stat.h> +#include <netinet/tcp.h> + +#pragma clang diagnostic push +#pragma clang diagnostic error "-Wpadded" +#pragma clang diagnostic error "-Wpacked" +// This header defines structures shared with user space, so we need to ensure there is +// no compiler inserted padding in case the user space process isn't using the same +// architecture as the kernel (example: i386 process with x86_64 kernel). #include <net/ntstat.h> +#pragma clang diagnostic pop #include <netinet/ip_var.h> #include <netinet/in_pcb.h> #include <netinet/in_var.h> -#include <netinet/tcp.h> #include <netinet/tcp_var.h> #include <netinet/tcp_fsm.h> #include <netinet/tcp_cc.h> @@ -64,6 +77,8 @@ #include <netinet6/in6_pcb.h> #include <netinet6/in6_var.h> +extern unsigned int if_enable_netagent; + __private_extern__ int nstat_collect = 1; #if (DEBUG || DEVELOPMENT) @@ -71,7 +86,11 @@ SYSCTL_INT(_net, OID_AUTO, statistics, CTLFLAG_RW | CTLFLAG_LOCKED, &nstat_collect, 0, "Collect detailed statistics"); #endif /* (DEBUG || DEVELOPMENT) */ +#if CONFIG_EMBEDDED +static int nstat_privcheck = 1; +#else static int nstat_privcheck = 0; +#endif SYSCTL_INT(_net, OID_AUTO, statistics_privcheck, CTLFLAG_RW | CTLFLAG_LOCKED, &nstat_privcheck, 0, "Entitlement check"); @@ -94,6 +113,33 @@ static struct nstat_stats nstat_stats; SYSCTL_STRUCT(_net_stats, OID_AUTO, stats, CTLFLAG_RD | CTLFLAG_LOCKED, &nstat_stats, nstat_stats, ""); +static u_int32_t nstat_lim_interval = 30 * 60; /* Report interval, seconds */ +static u_int32_t nstat_lim_min_tx_pkts = 100; +static u_int32_t nstat_lim_min_rx_pkts = 100; +#if (DEBUG || DEVELOPMENT) +SYSCTL_INT(_net_stats, OID_AUTO, lim_report_interval, + CTLFLAG_RW | CTLFLAG_LOCKED, &nstat_lim_interval, 0, + "Low internet stat report interval"); + +SYSCTL_INT(_net_stats, OID_AUTO, lim_min_tx_pkts, + CTLFLAG_RW | CTLFLAG_LOCKED, &nstat_lim_min_tx_pkts, 0, + "Low Internet, min transmit packets threshold"); + +SYSCTL_INT(_net_stats, OID_AUTO, lim_min_rx_pkts, + CTLFLAG_RW | CTLFLAG_LOCKED, &nstat_lim_min_rx_pkts, 0, + "Low Internet, min receive packets threshold"); +#endif /* DEBUG || DEVELOPMENT */ + +static struct net_api_stats net_api_stats_before; +static u_int64_t net_api_stats_last_report_time; +#define NET_API_STATS_REPORT_INTERVAL (12 * 60 * 60) /* 12 hours, in seconds */ +static u_int32_t net_api_stats_report_interval = NET_API_STATS_REPORT_INTERVAL; + +#if (DEBUG || DEVELOPMENT) +SYSCTL_UINT(_net_stats, OID_AUTO, api_report_interval, + CTLFLAG_RW | CTLFLAG_LOCKED, &net_api_stats_report_interval, 0, ""); +#endif /* DEBUG || DEVELOPMENT */ + enum { NSTAT_FLAG_CLEANUP = (1 << 0), @@ -102,7 +148,14 @@ enum NSTAT_FLAG_SYSINFO_SUBSCRIBED = (1 << 3), }; +#if CONFIG_EMBEDDED +#define QUERY_CONTINUATION_SRC_COUNT 50 +#else #define QUERY_CONTINUATION_SRC_COUNT 100 +#endif + +typedef TAILQ_HEAD(, nstat_src) tailq_head_nstat_src; +typedef TAILQ_ENTRY(nstat_src) tailq_entry_nstat_src; typedef struct nstat_provider_filter { @@ -117,11 +170,11 @@ typedef struct nstat_control_state { struct nstat_control_state *ncs_next; u_int32_t ncs_watching; - decl_lck_mtx_data(, mtx); + decl_lck_mtx_data(, ncs_mtx); kern_ctl_ref ncs_kctl; u_int32_t ncs_unit; nstat_src_ref_t ncs_next_srcref; - struct nstat_src *ncs_srcs; + tailq_head_nstat_src ncs_src_queue; mbuf_t ncs_accumulated; u_int32_t ncs_flags; nstat_provider_filter ncs_provider_filters[NSTAT_PROVIDER_COUNT]; @@ -138,7 +191,7 @@ typedef struct nstat_provider errno_t (*nstat_lookup)(const void *data, u_int32_t length, nstat_provider_cookie_t *out_cookie); int (*nstat_gone)(nstat_provider_cookie_t cookie); errno_t (*nstat_counts)(nstat_provider_cookie_t cookie, struct nstat_counts *out_counts, int *out_gone); - errno_t (*nstat_watcher_add)(nstat_control_state *state); + errno_t (*nstat_watcher_add)(nstat_control_state *state, nstat_msg_add_all_srcs *req); void (*nstat_watcher_remove)(nstat_control_state *state); errno_t (*nstat_copy_descriptor)(nstat_provider_cookie_t cookie, void *data, u_int32_t len); void (*nstat_release)(nstat_provider_cookie_t cookie, boolean_t locked); @@ -151,9 +204,13 @@ typedef STAILQ_ENTRY(nstat_src) stailq_entry_nstat_src; typedef TAILQ_HEAD(, nstat_tu_shadow) tailq_head_tu_shadow; typedef TAILQ_ENTRY(nstat_tu_shadow) tailq_entry_tu_shadow; +typedef TAILQ_HEAD(, nstat_procdetails) tailq_head_procdetails; +typedef TAILQ_ENTRY(nstat_procdetails) tailq_entry_procdetails; + typedef struct nstat_src { - struct nstat_src *next; + tailq_entry_nstat_src ns_control_link; // All sources for the nstat_control_state, for iterating over. + nstat_control_state *ns_control; // The nstat_control_state that this is a source for nstat_src_ref_t srcref; nstat_provider *provider; nstat_provider_cookie_t cookie; @@ -172,11 +229,12 @@ static bool nstat_control_reporting_allowed(nstat_control_state *state, nstat_s static boolean_t nstat_control_begin_query(nstat_control_state *state, const nstat_msg_hdr *hdrp); static u_int16_t nstat_control_end_query(nstat_control_state *state, nstat_src *last_src, boolean_t partial); static void nstat_ifnet_report_ecn_stats(void); +static void nstat_ifnet_report_lim_stats(void); +static void nstat_net_api_report_stats(void); +static errno_t nstat_set_provider_filter( nstat_control_state *state, nstat_msg_add_all_srcs *req); static u_int32_t nstat_udp_watchers = 0; -static u_int32_t nstat_userland_udp_watchers = 0; static u_int32_t nstat_tcp_watchers = 0; -static u_int32_t nstat_userland_tcp_watchers = 0; static void nstat_control_register(void); @@ -185,7 +243,7 @@ static void nstat_control_register(void); * * socket_lock (inpcb) * nstat_mtx - * state->mtx + * state->ncs_mtx */ static volatile OSMallocTag nstat_malloc_tag = NULL; static nstat_control_state *nstat_controls = NULL; @@ -234,28 +292,7 @@ nstat_ip_to_sockaddr( sin->sin_addr = *ip; } -static void -nstat_ip6_to_sockaddr( - const struct in6_addr *ip6, - u_int16_t port, - struct sockaddr_in6 *sin6, - u_int32_t maxlen) -{ - if (maxlen < sizeof(struct sockaddr_in6)) - return; - - sin6->sin6_family = AF_INET6; - sin6->sin6_len = sizeof(*sin6); - sin6->sin6_port = port; - sin6->sin6_addr = *ip6; - if (IN6_IS_SCOPE_EMBED(&sin6->sin6_addr)) - { - sin6->sin6_scope_id = ntohs(sin6->sin6_addr.s6_addr16[1]); - sin6->sin6_addr.s6_addr16[1] = 0; - } -} - -static u_int16_t +u_int16_t nstat_ifnet_to_flags( struct ifnet *ifp) { @@ -362,9 +399,7 @@ nstat_lookup_entry( static void nstat_init_route_provider(void); static void nstat_init_tcp_provider(void); -static void nstat_init_userland_tcp_provider(void); static void nstat_init_udp_provider(void); -static void nstat_init_userland_udp_provider(void); static void nstat_init_ifnet_provider(void); __private_extern__ void @@ -383,9 +418,7 @@ nstat_init(void) // we need to initialize other things, we do it here as this code path will only be hit once; nstat_init_route_provider(); nstat_init_tcp_provider(); - nstat_init_userland_tcp_provider(); nstat_init_udp_provider(); - nstat_init_userland_udp_provider(); nstat_init_ifnet_provider(); nstat_control_register(); } @@ -555,7 +588,7 @@ nstat_route_walktree_add( struct rtentry *rt = (struct rtentry *)rn; nstat_control_state *state = (nstat_control_state*)context; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); /* RTF_UP can't change while rnh_lock is held */ if ((rt->rt_flags & RTF_UP) != 0) @@ -584,23 +617,34 @@ nstat_route_walktree_add( static errno_t nstat_route_add_watcher( - nstat_control_state *state) + nstat_control_state *state, + nstat_msg_add_all_srcs *req) { int i; errno_t result = 0; - OSIncrementAtomic(&nstat_route_watchers); lck_mtx_lock(rnh_lock); - for (i = 1; i < AF_MAX; i++) + + result = nstat_set_provider_filter(state, req); + if (result == 0) { - struct radix_node_head *rnh; - rnh = rt_tables[i]; - if (!rnh) continue; + OSIncrementAtomic(&nstat_route_watchers); - result = rnh->rnh_walktree(rnh, nstat_route_walktree_add, state); - if (result != 0) + for (i = 1; i < AF_MAX; i++) { - break; + struct radix_node_head *rnh; + rnh = rt_tables[i]; + if (!rnh) continue; + + result = rnh->rnh_walktree(rnh, nstat_route_walktree_add, state); + if (result != 0) + { + // This is probably resource exhaustion. + // There currently isn't a good way to recover from this. + // Least bad seems to be to give up on the add-all but leave + // the watcher in place. + break; + } } } lck_mtx_unlock(rnh_lock); @@ -726,7 +770,7 @@ nstat_init_route_provider(void) #pragma mark -- Route Collection -- -static struct nstat_counts* +__private_extern__ struct nstat_counts* nstat_route_attach( struct rtentry *rte) { @@ -850,68 +894,99 @@ nstat_route_rx( } } +/* atomically average current value at _val_addr with _new_val and store */ +#define NSTAT_EWMA_ATOMIC(_val_addr, _new_val, _decay) do { \ + volatile uint32_t _old_val; \ + volatile uint32_t _avg; \ + do { \ + _old_val = *_val_addr; \ + if (_old_val == 0) \ + { \ + _avg = _new_val; \ + } \ + else \ + { \ + _avg = _old_val - (_old_val >> _decay) + (_new_val >> _decay); \ + } \ + if (_old_val == _avg) break; \ + } while (!OSCompareAndSwap(_old_val, _avg, _val_addr)); \ +} while (0); + +/* atomically compute minimum of current value at _val_addr with _new_val and store */ +#define NSTAT_MIN_ATOMIC(_val_addr, _new_val) do { \ + volatile uint32_t _old_val; \ + do { \ + _old_val = *_val_addr; \ + if (_old_val != 0 && _old_val < _new_val) \ + { \ + break; \ + } \ + } while (!OSCompareAndSwap(_old_val, _new_val, _val_addr)); \ +} while (0); + __private_extern__ void nstat_route_rtt( struct rtentry *rte, u_int32_t rtt, u_int32_t rtt_var) { - const int32_t factor = 8; + const uint32_t decay = 3; while (rte) { struct nstat_counts* stats = nstat_route_attach(rte); if (stats) { - int32_t oldrtt; - int32_t newrtt; - - // average - do - { - oldrtt = stats->nstat_avg_rtt; - if (oldrtt == 0) - { - newrtt = rtt; - } - else - { - newrtt = oldrtt - (oldrtt - (int32_t)rtt) / factor; - } - if (oldrtt == newrtt) break; - } while (!OSCompareAndSwap(oldrtt, newrtt, &stats->nstat_avg_rtt)); + NSTAT_EWMA_ATOMIC(&stats->nstat_avg_rtt, rtt, decay); + NSTAT_MIN_ATOMIC(&stats->nstat_min_rtt, rtt); + NSTAT_EWMA_ATOMIC(&stats->nstat_var_rtt, rtt_var, decay); + } + rte = rte->rt_parent; + } +} - // minimum - do - { - oldrtt = stats->nstat_min_rtt; - if (oldrtt != 0 && oldrtt < (int32_t)rtt) - { - break; - } - } while (!OSCompareAndSwap(oldrtt, rtt, &stats->nstat_min_rtt)); +__private_extern__ void +nstat_route_update( + struct rtentry *rte, + uint32_t connect_attempts, + uint32_t connect_successes, + uint32_t rx_packets, + uint32_t rx_bytes, + uint32_t rx_duplicatebytes, + uint32_t rx_outoforderbytes, + uint32_t tx_packets, + uint32_t tx_bytes, + uint32_t tx_retransmit, + uint32_t rtt, + uint32_t rtt_var) +{ + const uint32_t decay = 3; - // variance - do - { - oldrtt = stats->nstat_var_rtt; - if (oldrtt == 0) - { - newrtt = rtt_var; - } - else - { - newrtt = oldrtt - (oldrtt - (int32_t)rtt_var) / factor; - } - if (oldrtt == newrtt) break; - } while (!OSCompareAndSwap(oldrtt, newrtt, &stats->nstat_var_rtt)); + while (rte) + { + struct nstat_counts* stats = nstat_route_attach(rte); + if (stats) + { + OSAddAtomic(connect_attempts, &stats->nstat_connectattempts); + OSAddAtomic(connect_successes, &stats->nstat_connectsuccesses); + OSAddAtomic64((SInt64)tx_packets, (SInt64*)&stats->nstat_txpackets); + OSAddAtomic64((SInt64)tx_bytes, (SInt64*)&stats->nstat_txbytes); + OSAddAtomic(tx_retransmit, &stats->nstat_txretransmit); + OSAddAtomic64((SInt64)rx_packets, (SInt64*)&stats->nstat_rxpackets); + OSAddAtomic64((SInt64)rx_bytes, (SInt64*)&stats->nstat_rxbytes); + OSAddAtomic(rx_outoforderbytes, &stats->nstat_rxoutoforderbytes); + OSAddAtomic(rx_duplicatebytes, &stats->nstat_rxduplicatebytes); + + if (rtt != 0) { + NSTAT_EWMA_ATOMIC(&stats->nstat_avg_rtt, rtt, decay); + NSTAT_MIN_ATOMIC(&stats->nstat_min_rtt, rtt); + NSTAT_EWMA_ATOMIC(&stats->nstat_var_rtt, rtt_var, decay); + } } - rte = rte->rt_parent; } } - #pragma mark -- TCP Kernel Provider -- /* @@ -955,7 +1030,7 @@ nstat_tucookie_alloc_internal( if (cookie == NULL) return NULL; if (!locked) - lck_mtx_assert(&nstat_mtx, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&nstat_mtx, LCK_MTX_ASSERT_NOTOWNED); if (ref && in_pcb_checkstate(inp, WNT_ACQUIRE, locked) == WNT_STOPUSING) { OSFree(cookie, sizeof(*cookie), nstat_malloc_tag); @@ -1188,31 +1263,46 @@ nstat_tcp_release( static errno_t nstat_tcp_add_watcher( - nstat_control_state *state) -{ - OSIncrementAtomic(&nstat_tcp_watchers); + nstat_control_state *state, + nstat_msg_add_all_srcs *req) +{ + // There is a tricky issue around getting all TCP sockets added once + // and only once. nstat_tcp_new_pcb() is called prior to the new item + // being placed on any lists where it might be found. + // By locking the tcbinfo.ipi_lock prior to marking the state as a watcher, + // it should be impossible for a new socket to be added twice. + // On the other hand, there is still a timing issue where a new socket + // results in a call to nstat_tcp_new_pcb() before this watcher + // is instantiated and yet the socket doesn't make it into ipi_listhead + // prior to the scan. <rdar://problem/30361716> - lck_rw_lock_shared(tcbinfo.ipi_lock); + errno_t result; - // Add all current tcp inpcbs. Ignore those in timewait - struct inpcb *inp; - struct nstat_tucookie *cookie; - LIST_FOREACH(inp, tcbinfo.ipi_listhead, inp_list) - { - cookie = nstat_tucookie_alloc_ref(inp); - if (cookie == NULL) - continue; - if (nstat_control_source_add(0, state, &nstat_tcp_provider, - cookie) != 0) + lck_rw_lock_shared(tcbinfo.ipi_lock); + result = nstat_set_provider_filter(state, req); + if (result == 0) { + OSIncrementAtomic(&nstat_tcp_watchers); + + // Add all current tcp inpcbs. Ignore those in timewait + struct inpcb *inp; + struct nstat_tucookie *cookie; + LIST_FOREACH(inp, tcbinfo.ipi_listhead, inp_list) { - nstat_tucookie_release(cookie); - break; + cookie = nstat_tucookie_alloc_ref(inp); + if (cookie == NULL) + continue; + if (nstat_control_source_add(0, state, &nstat_tcp_provider, + cookie) != 0) + { + nstat_tucookie_release(cookie); + break; + } } } lck_rw_done(tcbinfo.ipi_lock); - return 0; + return result; } static void @@ -1228,6 +1318,8 @@ nstat_tcp_new_pcb( { struct nstat_tucookie *cookie; + inp->inp_start_timestamp = mach_continuous_time(); + if (nstat_tcp_watchers == 0) return; @@ -1260,20 +1352,20 @@ __private_extern__ void nstat_pcb_detach(struct inpcb *inp) { nstat_control_state *state; - nstat_src *src, *prevsrc; - nstat_src *dead_list = NULL; + nstat_src *src; + tailq_head_nstat_src dead_list; struct nstat_tucookie *tucookie; errno_t result; if (inp == NULL || (nstat_tcp_watchers == 0 && nstat_udp_watchers == 0)) return; + TAILQ_INIT(&dead_list); lck_mtx_lock(&nstat_mtx); for (state = nstat_controls; state; state = state->ncs_next) { - lck_mtx_lock(&state->mtx); - for (prevsrc = NULL, src = state->ncs_srcs; src; - prevsrc = src, src = src->next) + lck_mtx_lock(&state->ncs_mtx); + TAILQ_FOREACH(src, &state->ncs_src_queue, ns_control_link) { nstat_provider_id_t provider_id = src->provider->nstat_provider_id; if (provider_id == NSTAT_PROVIDER_TCP_KERNEL || provider_id == NSTAT_PROVIDER_UDP_KERNEL) @@ -1288,22 +1380,16 @@ nstat_pcb_detach(struct inpcb *inp) { result = nstat_control_send_goodbye(state, src); - if (prevsrc) - prevsrc->next = src->next; - else - state->ncs_srcs = src->next; - - src->next = dead_list; - dead_list = src; + TAILQ_REMOVE(&state->ncs_src_queue, src, ns_control_link); + TAILQ_INSERT_TAIL(&dead_list, src, ns_control_link); } - lck_mtx_unlock(&state->mtx); + lck_mtx_unlock(&state->ncs_mtx); } lck_mtx_unlock(&nstat_mtx); - while (dead_list) { - src = dead_list; - dead_list = src->next; - + while ((src = TAILQ_FIRST(&dead_list))) + { + TAILQ_REMOVE(&dead_list, src, ns_control_link); nstat_control_cleanup_source(NULL, src, TRUE); } } @@ -1321,19 +1407,19 @@ nstat_pcb_cache(struct inpcb *inp) VERIFY(SOCK_PROTO(inp->inp_socket) == IPPROTO_UDP); lck_mtx_lock(&nstat_mtx); for (state = nstat_controls; state; state = state->ncs_next) { - lck_mtx_lock(&state->mtx); - for (src = state->ncs_srcs; src; src = src->next) + lck_mtx_lock(&state->ncs_mtx); + TAILQ_FOREACH(src, &state->ncs_src_queue, ns_control_link) { tucookie = (struct nstat_tucookie *)src->cookie; if (tucookie->inp == inp) { if (inp->inp_vflag & INP_IPV6) { - nstat_ip6_to_sockaddr(&inp->in6p_laddr, + in6_ip6_to_sockaddr(&inp->in6p_laddr, inp->inp_lport, &tucookie->local.v6, sizeof(tucookie->local)); - nstat_ip6_to_sockaddr(&inp->in6p_faddr, + in6_ip6_to_sockaddr(&inp->in6p_faddr, inp->inp_fport, &tucookie->remote.v6, sizeof(tucookie->remote)); @@ -1358,7 +1444,7 @@ nstat_pcb_cache(struct inpcb *inp) break; } } - lck_mtx_unlock(&state->mtx); + lck_mtx_unlock(&state->ncs_mtx); } lck_mtx_unlock(&nstat_mtx); } @@ -1376,8 +1462,8 @@ nstat_pcb_invalidate_cache(struct inpcb *inp) VERIFY(SOCK_PROTO(inp->inp_socket) == IPPROTO_UDP); lck_mtx_lock(&nstat_mtx); for (state = nstat_controls; state; state = state->ncs_next) { - lck_mtx_lock(&state->mtx); - for (src = state->ncs_srcs; src; src = src->next) + lck_mtx_lock(&state->ncs_mtx); + TAILQ_FOREACH(src, &state->ncs_src_queue, ns_control_link) { tucookie = (struct nstat_tucookie *)src->cookie; if (tucookie->inp == inp) @@ -1386,7 +1472,7 @@ nstat_pcb_invalidate_cache(struct inpcb *inp) break; } } - lck_mtx_unlock(&state->mtx); + lck_mtx_unlock(&state->ncs_mtx); } lck_mtx_unlock(&nstat_mtx); } @@ -1414,9 +1500,9 @@ nstat_tcp_copy_descriptor( if (inp->inp_vflag & INP_IPV6) { - nstat_ip6_to_sockaddr(&inp->in6p_laddr, inp->inp_lport, + in6_ip6_to_sockaddr(&inp->in6p_laddr, inp->inp_lport, &desc->local.v6, sizeof(desc->local)); - nstat_ip6_to_sockaddr(&inp->in6p_faddr, inp->inp_fport, + in6_ip6_to_sockaddr(&inp->in6p_faddr, inp->inp_fport, &desc->remote.v6, sizeof(desc->remote)); } else if (inp->inp_vflag & INP_IPV4) @@ -1484,6 +1570,9 @@ nstat_tcp_copy_descriptor( tcp_get_connectivity_status(tp, &desc->connstatus); desc->ifnet_properties = nstat_inpcb_to_flags(inp); + inp_get_activity_bitmap(inp, &desc->activity_bitmap); + desc->start_timestamp = inp->inp_start_timestamp; + desc->timestamp = mach_continuous_time(); return 0; } @@ -1667,31 +1756,48 @@ nstat_udp_release( static errno_t nstat_udp_add_watcher( - nstat_control_state *state) -{ - struct inpcb *inp; - struct nstat_tucookie *cookie; + nstat_control_state *state, + nstat_msg_add_all_srcs *req) +{ + // There is a tricky issue around getting all UDP sockets added once + // and only once. nstat_udp_new_pcb() is called prior to the new item + // being placed on any lists where it might be found. + // By locking the udpinfo.ipi_lock prior to marking the state as a watcher, + // it should be impossible for a new socket to be added twice. + // On the other hand, there is still a timing issue where a new socket + // results in a call to nstat_udp_new_pcb() before this watcher + // is instantiated and yet the socket doesn't make it into ipi_listhead + // prior to the scan. <rdar://problem/30361716> - OSIncrementAtomic(&nstat_udp_watchers); + errno_t result; lck_rw_lock_shared(udbinfo.ipi_lock); - // Add all current UDP inpcbs. - LIST_FOREACH(inp, udbinfo.ipi_listhead, inp_list) - { - cookie = nstat_tucookie_alloc_ref(inp); - if (cookie == NULL) - continue; - if (nstat_control_source_add(0, state, &nstat_udp_provider, - cookie) != 0) + result = nstat_set_provider_filter(state, req); + + if (result == 0) { + struct inpcb *inp; + struct nstat_tucookie *cookie; + + OSIncrementAtomic(&nstat_udp_watchers); + + // Add all current UDP inpcbs. + LIST_FOREACH(inp, udbinfo.ipi_listhead, inp_list) { - nstat_tucookie_release(cookie); - break; + cookie = nstat_tucookie_alloc_ref(inp); + if (cookie == NULL) + continue; + if (nstat_control_source_add(0, state, &nstat_udp_provider, + cookie) != 0) + { + nstat_tucookie_release(cookie); + break; + } } } lck_rw_done(udbinfo.ipi_lock); - return 0; + return result; } static void @@ -1707,6 +1813,8 @@ nstat_udp_new_pcb( { struct nstat_tucookie *cookie; + inp->inp_start_timestamp = mach_continuous_time(); + if (nstat_udp_watchers == 0) return; @@ -1759,9 +1867,9 @@ nstat_udp_copy_descriptor( if (tucookie->cached == false) { if (inp->inp_vflag & INP_IPV6) { - nstat_ip6_to_sockaddr(&inp->in6p_laddr, inp->inp_lport, + in6_ip6_to_sockaddr(&inp->in6p_laddr, inp->inp_lport, &desc->local.v6, sizeof(desc->local.v6)); - nstat_ip6_to_sockaddr(&inp->in6p_faddr, inp->inp_fport, + in6_ip6_to_sockaddr(&inp->in6p_faddr, inp->inp_fport, &desc->remote.v6, sizeof(desc->remote.v6)); } else if (inp->inp_vflag & INP_IPV4) @@ -1830,6 +1938,9 @@ nstat_udp_copy_descriptor( desc->rcvbufsize = so->so_rcv.sb_hiwat; desc->rcvbufused = so->so_rcv.sb_cc; desc->traffic_class = so->so_traffic_class; + inp_get_activity_bitmap(inp, &desc->activity_bitmap); + desc->start_timestamp = inp->inp_start_timestamp; + desc->timestamp = mach_continuous_time(); } return 0; @@ -1850,435 +1961,18 @@ nstat_init_udp_provider(void) nstat_udp_provider.nstat_descriptor_length = sizeof(nstat_udp_descriptor); nstat_udp_provider.nstat_lookup = nstat_udp_lookup; nstat_udp_provider.nstat_gone = nstat_udp_gone; - nstat_udp_provider.nstat_counts = nstat_udp_counts; - nstat_udp_provider.nstat_watcher_add = nstat_udp_add_watcher; - nstat_udp_provider.nstat_watcher_remove = nstat_udp_remove_watcher; - nstat_udp_provider.nstat_copy_descriptor = nstat_udp_copy_descriptor; - nstat_udp_provider.nstat_release = nstat_udp_release; - nstat_udp_provider.nstat_reporting_allowed = nstat_udp_reporting_allowed; - nstat_udp_provider.next = nstat_providers; - nstat_providers = &nstat_udp_provider; -} - -#pragma mark -- TCP/UDP Userland - -// Almost all of this infrastucture is common to both TCP and UDP - -static nstat_provider nstat_userland_tcp_provider; -static nstat_provider nstat_userland_udp_provider; - - -struct nstat_tu_shadow { - tailq_entry_tu_shadow shad_link; - userland_stats_request_vals_fn *shad_getvals_fn; - userland_stats_provider_context *shad_provider_context; - u_int64_t shad_properties; - int shad_provider; - uint32_t shad_magic; -}; - -// Magic number checking should remain in place until the userland provider has been fully proven -#define TU_SHADOW_MAGIC 0xfeedf00d -#define TU_SHADOW_UNMAGIC 0xdeaddeed - -static tailq_head_tu_shadow nstat_userprot_shad_head = TAILQ_HEAD_INITIALIZER(nstat_userprot_shad_head); - -static errno_t -nstat_userland_tu_lookup( - __unused const void *data, - __unused u_int32_t length, - __unused nstat_provider_cookie_t *out_cookie) -{ - // Looking up a specific connection is not supported - return ENOTSUP; -} - -static int -nstat_userland_tu_gone( - __unused nstat_provider_cookie_t cookie) -{ - // Returns non-zero if the source has gone. - // We don't keep a source hanging around, so the answer is always 0 - return 0; -} - -static errno_t -nstat_userland_tu_counts( - nstat_provider_cookie_t cookie, - struct nstat_counts *out_counts, - int *out_gone) - { - struct nstat_tu_shadow *shad = (struct nstat_tu_shadow *)cookie; - assert(shad->shad_magic == TU_SHADOW_MAGIC); - - bool result = (*shad->shad_getvals_fn)(shad->shad_provider_context, out_counts, NULL); - - if (out_gone) *out_gone = 0; - - return (result)? 0 : EIO; -} - - -static errno_t -nstat_userland_tu_copy_descriptor( - nstat_provider_cookie_t cookie, - void *data, - __unused u_int32_t len) -{ - struct nstat_tu_shadow *shad = (struct nstat_tu_shadow *)cookie; - assert(shad->shad_magic == TU_SHADOW_MAGIC); - - bool result = (*shad->shad_getvals_fn)(shad->shad_provider_context, NULL, data); - - return (result)? 0 : EIO; -} - -static void -nstat_userland_tu_release( - __unused nstat_provider_cookie_t cookie, - __unused int locked) -{ - // Called when a nstat_src is detached. - // We don't reference count or ask for delayed release so nothing to do here. -} - -static bool -check_reporting_for_user(nstat_provider_filter *filter, pid_t pid, pid_t epid, uuid_t *uuid, uuid_t *euuid) -{ - bool retval = true; - - if ((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER) != 0) - { - retval = false; - - if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER_BY_PID) != 0) && - (filter->npf_pid == pid)) - { - retval = true; - } - else if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER_BY_EPID) != 0) && - (filter->npf_pid == epid)) - { - retval = true; - } - else if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER_BY_UUID) != 0) && - (memcmp(filter->npf_uuid, uuid, sizeof(*uuid)) == 0)) - { - retval = true; - } - else if (((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER_BY_EUUID) != 0) && - (memcmp(filter->npf_uuid, euuid, sizeof(*euuid)) == 0)) - { - retval = true; - } - } - return retval; -} - -static bool -nstat_userland_tcp_reporting_allowed(nstat_provider_cookie_t cookie, nstat_provider_filter *filter) -{ - bool retval = true; - - if ((filter->npf_flags & (NSTAT_FILTER_IFNET_FLAGS|NSTAT_FILTER_SPECIFIC_USER)) != 0) - { - nstat_tcp_descriptor tcp_desc; // Stack allocation - OK or pushing the limits too far? - struct nstat_tu_shadow *shad = (struct nstat_tu_shadow *)cookie; - - assert(shad->shad_magic == TU_SHADOW_MAGIC); - - if ((*shad->shad_getvals_fn)(shad->shad_provider_context, NULL, &tcp_desc)) - { - if ((filter->npf_flags & NSTAT_FILTER_IFNET_FLAGS) != 0) - { - if ((filter->npf_flags & tcp_desc.ifnet_properties) == 0) - { - return false; - } - } - if ((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER) != 0) - { - retval = check_reporting_for_user(filter, (pid_t)tcp_desc.pid, (pid_t)tcp_desc.epid, - &tcp_desc.uuid, &tcp_desc.euuid); - } - } - else - { - retval = false; // No further information, so might as well give up now. - } - } - return retval; -} - -static bool -nstat_userland_udp_reporting_allowed(nstat_provider_cookie_t cookie, nstat_provider_filter *filter) -{ - bool retval = true; - - if ((filter->npf_flags & (NSTAT_FILTER_IFNET_FLAGS|NSTAT_FILTER_SPECIFIC_USER)) != 0) - { - nstat_udp_descriptor udp_desc; // Stack allocation - OK or pushing the limits too far? - struct nstat_tu_shadow *shad = (struct nstat_tu_shadow *)cookie; - - assert(shad->shad_magic == TU_SHADOW_MAGIC); - - if ((*shad->shad_getvals_fn)(shad->shad_provider_context, NULL, &udp_desc)) - { - if ((filter->npf_flags & NSTAT_FILTER_IFNET_FLAGS) != 0) - { - if ((filter->npf_flags & udp_desc.ifnet_properties) == 0) - { - return false; - } - } - if ((filter->npf_flags & NSTAT_FILTER_SPECIFIC_USER) != 0) - { - retval = check_reporting_for_user(filter, (pid_t)udp_desc.pid, (pid_t)udp_desc.epid, - &udp_desc.uuid, &udp_desc.euuid); - } - } - else - { - retval = false; // No further information, so might as well give up now. - } - } - return retval; -} - - - -static errno_t -nstat_userland_tcp_add_watcher( - nstat_control_state *state) -{ - struct nstat_tu_shadow *shad; - - OSIncrementAtomic(&nstat_userland_tcp_watchers); - - lck_mtx_lock(&nstat_mtx); - - TAILQ_FOREACH(shad, &nstat_userprot_shad_head, shad_link) { - assert(shad->shad_magic == TU_SHADOW_MAGIC); - - if (shad->shad_provider == NSTAT_PROVIDER_TCP_USERLAND) - { - int result = nstat_control_source_add(0, state, &nstat_userland_tcp_provider, shad); - if (result != 0) - { - printf("%s - nstat_control_source_add returned %d\n", __func__, result); - } - } - } - lck_mtx_unlock(&nstat_mtx); - - return 0; -} - -static errno_t -nstat_userland_udp_add_watcher( - nstat_control_state *state) -{ - struct nstat_tu_shadow *shad; - - OSIncrementAtomic(&nstat_userland_udp_watchers); - - lck_mtx_lock(&nstat_mtx); - - TAILQ_FOREACH(shad, &nstat_userprot_shad_head, shad_link) { - assert(shad->shad_magic == TU_SHADOW_MAGIC); - - if (shad->shad_provider == NSTAT_PROVIDER_UDP_USERLAND) - { - int result = nstat_control_source_add(0, state, &nstat_userland_udp_provider, shad); - if (result != 0) - { - printf("%s - nstat_control_source_add returned %d\n", __func__, result); - } - } - } - lck_mtx_unlock(&nstat_mtx); - - return 0; -} - - -static void -nstat_userland_tcp_remove_watcher( - __unused nstat_control_state *state) -{ - OSDecrementAtomic(&nstat_userland_tcp_watchers); -} - -static void -nstat_userland_udp_remove_watcher( - __unused nstat_control_state *state) -{ - OSDecrementAtomic(&nstat_userland_udp_watchers); -} - -static void -nstat_init_userland_tcp_provider(void) -{ - bzero(&nstat_userland_tcp_provider, sizeof(nstat_tcp_provider)); - nstat_userland_tcp_provider.nstat_descriptor_length = sizeof(nstat_tcp_descriptor); - nstat_userland_tcp_provider.nstat_provider_id = NSTAT_PROVIDER_TCP_USERLAND; - nstat_userland_tcp_provider.nstat_lookup = nstat_userland_tu_lookup; - nstat_userland_tcp_provider.nstat_gone = nstat_userland_tu_gone; - nstat_userland_tcp_provider.nstat_counts = nstat_userland_tu_counts; - nstat_userland_tcp_provider.nstat_release = nstat_userland_tu_release; - nstat_userland_tcp_provider.nstat_watcher_add = nstat_userland_tcp_add_watcher; - nstat_userland_tcp_provider.nstat_watcher_remove = nstat_userland_tcp_remove_watcher; - nstat_userland_tcp_provider.nstat_copy_descriptor = nstat_userland_tu_copy_descriptor; - nstat_userland_tcp_provider.nstat_reporting_allowed = nstat_userland_tcp_reporting_allowed; - nstat_userland_tcp_provider.next = nstat_providers; - nstat_providers = &nstat_userland_tcp_provider; -} - - -static void -nstat_init_userland_udp_provider(void) -{ - bzero(&nstat_userland_udp_provider, sizeof(nstat_udp_provider)); - nstat_userland_udp_provider.nstat_descriptor_length = sizeof(nstat_udp_descriptor); - nstat_userland_udp_provider.nstat_provider_id = NSTAT_PROVIDER_UDP_USERLAND; - nstat_userland_udp_provider.nstat_lookup = nstat_userland_tu_lookup; - nstat_userland_udp_provider.nstat_gone = nstat_userland_tu_gone; - nstat_userland_udp_provider.nstat_counts = nstat_userland_tu_counts; - nstat_userland_udp_provider.nstat_release = nstat_userland_tu_release; - nstat_userland_udp_provider.nstat_watcher_add = nstat_userland_udp_add_watcher; - nstat_userland_udp_provider.nstat_watcher_remove = nstat_userland_udp_remove_watcher; - nstat_userland_udp_provider.nstat_copy_descriptor = nstat_userland_tu_copy_descriptor; - nstat_userland_udp_provider.nstat_reporting_allowed = nstat_userland_udp_reporting_allowed; - nstat_userland_udp_provider.next = nstat_providers; - nstat_providers = &nstat_userland_udp_provider; -} - - - -// Things get started with a call to netstats to say that thereâs a new connection: -__private_extern__ nstat_userland_context -ntstat_userland_stats_open(userland_stats_provider_context *ctx, - int provider_id, - u_int64_t properties, - userland_stats_request_vals_fn req_fn) -{ - struct nstat_tu_shadow *shad; - - if ((provider_id != NSTAT_PROVIDER_TCP_USERLAND) && (provider_id != NSTAT_PROVIDER_UDP_USERLAND)) - { - printf("%s - incorrect provider is supplied, %d\n", __func__, provider_id); - return NULL; - } - - shad = OSMalloc(sizeof(*shad), nstat_malloc_tag); - if (shad == NULL) - return NULL; - - shad->shad_getvals_fn = req_fn; - shad->shad_provider_context = ctx; - shad->shad_provider = provider_id; - shad->shad_properties = properties; - shad->shad_magic = TU_SHADOW_MAGIC; - - lck_mtx_lock(&nstat_mtx); - nstat_control_state *state; - - // Even if there are no watchers, we save the shadow structure - TAILQ_INSERT_HEAD(&nstat_userprot_shad_head, shad, shad_link); - - for (state = nstat_controls; state; state = state->ncs_next) - { - if ((state->ncs_watching & (1 << provider_id)) != 0) - { - // this client is watching tcp/udp userland - // Link to it. - int result = nstat_control_source_add(0, state, &nstat_userland_tcp_provider, shad); - if (result != 0) - { - printf("%s - nstat_control_source_add returned %d\n", __func__, result); - } - } - } - lck_mtx_unlock(&nstat_mtx); - - return (nstat_userland_context)shad; -} - - -__private_extern__ void -ntstat_userland_stats_close(nstat_userland_context nstat_ctx) -{ - struct nstat_tu_shadow *shad = (struct nstat_tu_shadow *)nstat_ctx; - nstat_src *dead_list = NULL; - - if (shad == NULL) - return; - - assert(shad->shad_magic == TU_SHADOW_MAGIC); - - lck_mtx_lock(&nstat_mtx); - if (nstat_userland_udp_watchers != 0 || nstat_userland_tcp_watchers != 0) - { - nstat_control_state *state; - nstat_src *src, *prevsrc; - errno_t result; - - for (state = nstat_controls; state; state = state->ncs_next) - { - lck_mtx_lock(&state->mtx); - for (prevsrc = NULL, src = state->ncs_srcs; src; - prevsrc = src, src = src->next) - { - if (shad == (struct nstat_tu_shadow *)src->cookie) - break; - } - - if (src) - { - result = nstat_control_send_goodbye(state, src); - - if (prevsrc) - prevsrc->next = src->next; - else - state->ncs_srcs = src->next; - - src->next = dead_list; - dead_list = src; - } - lck_mtx_unlock(&state->mtx); - } - } - TAILQ_REMOVE(&nstat_userprot_shad_head, shad, shad_link); - - lck_mtx_unlock(&nstat_mtx); - - while (dead_list) - { - nstat_src *src; - src = dead_list; - dead_list = src->next; - - nstat_control_cleanup_source(NULL, src, TRUE); - } - - shad->shad_magic = TU_SHADOW_UNMAGIC; - - OSFree(shad, sizeof(*shad), nstat_malloc_tag); -} - - -__private_extern__ void -ntstat_userland_stats_event( - __unused nstat_userland_context context, - __unused userland_stats_event_t event) -{ - // This is a dummy for when we hook up event reporting to NetworkStatistics. - // See <rdar://problem/23022832> NetworkStatistics should provide opt-in notifications + nstat_udp_provider.nstat_counts = nstat_udp_counts; + nstat_udp_provider.nstat_watcher_add = nstat_udp_add_watcher; + nstat_udp_provider.nstat_watcher_remove = nstat_udp_remove_watcher; + nstat_udp_provider.nstat_copy_descriptor = nstat_udp_copy_descriptor; + nstat_udp_provider.nstat_release = nstat_udp_release; + nstat_udp_provider.nstat_reporting_allowed = nstat_udp_reporting_allowed; + nstat_udp_provider.next = nstat_providers; + nstat_providers = &nstat_udp_provider; } - #pragma mark -- ifnet Provider -- static nstat_provider nstat_ifnet_provider; @@ -2353,14 +2047,14 @@ nstat_ifnet_lookup( lck_mtx_lock(&nstat_mtx); for (state = nstat_controls; state; state = state->ncs_next) { - lck_mtx_lock(&state->mtx); - for (src = state->ncs_srcs; src; src = src->next) + lck_mtx_lock(&state->ncs_mtx); + TAILQ_FOREACH(src, &state->ncs_src_queue, ns_control_link) { if (src->provider != &nstat_ifnet_provider) continue; nstat_control_send_description(state, src, 0, 0); } - lck_mtx_unlock(&state->mtx); + lck_mtx_unlock(&state->ncs_mtx); } lck_mtx_unlock(&nstat_mtx); } @@ -2435,8 +2129,8 @@ nstat_ifnet_release( lck_mtx_lock(&nstat_mtx); for (state = nstat_controls; state; state = state->ncs_next) { - lck_mtx_lock(&state->mtx); - for (src = state->ncs_srcs; src; src = src->next) + lck_mtx_lock(&state->ncs_mtx); + TAILQ_FOREACH(src, &state->ncs_src_queue, ns_control_link) { /* Skip the provider we are about to detach. */ if (src->provider != &nstat_ifnet_provider || @@ -2446,7 +2140,7 @@ nstat_ifnet_release( if (ifcookie->threshold < minthreshold) minthreshold = ifcookie->threshold; } - lck_mtx_unlock(&state->mtx); + lck_mtx_unlock(&state->ncs_mtx); } lck_mtx_unlock(&nstat_mtx); /* @@ -2778,8 +2472,7 @@ nstat_ifnet_report_ecn_stats(void) if (ifp->if_ipv4_stat == NULL || ifp->if_ipv6_stat == NULL) continue; - if ((ifp->if_refflags & (IFRF_ATTACHED | IFRF_DETACHING)) != - IFRF_ATTACHED) + if (!IF_FULLY_ATTACHED(ifp)) continue; /* Limit reporting to Wifi, Ethernet and cellular. */ @@ -2831,6 +2524,133 @@ v6: } +/* Some thresholds to determine Low Iternet mode */ +#define NSTAT_LIM_DL_MAX_BANDWIDTH_THRESHOLD 1000000 /* 1 Mbps */ +#define NSTAT_LIM_UL_MAX_BANDWIDTH_THRESHOLD 500000 /* 500 Kbps */ +#define NSTAT_LIM_UL_MIN_RTT_THRESHOLD 1000 /* 1 second */ +#define NSTAT_LIM_CONN_TIMEOUT_PERCENT_THRESHOLD (10 << 10) /* 10 percent connection timeouts */ +#define NSTAT_LIM_PACKET_LOSS_PERCENT_THRESHOLD (2 << 10) /* 2 percent packet loss rate */ + +static boolean_t +nstat_lim_activity_check(struct if_lim_perf_stat *st) +{ + /* check that the current activity is enough to report stats */ + if (st->lim_total_txpkts < nstat_lim_min_tx_pkts || + st->lim_total_rxpkts < nstat_lim_min_rx_pkts || + st->lim_conn_attempts == 0) + return (FALSE); + + /* + * Compute percentages if there was enough activity. Use + * shift-left by 10 to preserve precision. + */ + st->lim_packet_loss_percent = ((st->lim_total_retxpkts << 10) / + st->lim_total_txpkts) * 100; + + st->lim_packet_ooo_percent = ((st->lim_total_oopkts << 10) / + st->lim_total_rxpkts) * 100; + + st->lim_conn_timeout_percent = ((st->lim_conn_timeouts << 10) / + st->lim_conn_attempts) * 100; + + /* + * Is Low Internet detected? First order metrics are bandwidth + * and RTT. If these metrics are below the minimum thresholds + * defined then the network attachment can be classified as + * having Low Internet capacity. + * + * High connection timeout rate also indicates Low Internet + * capacity. + */ + if (st->lim_dl_max_bandwidth > 0 && + st->lim_dl_max_bandwidth <= NSTAT_LIM_DL_MAX_BANDWIDTH_THRESHOLD) + st->lim_dl_detected = 1; + + if ((st->lim_ul_max_bandwidth > 0 && + st->lim_ul_max_bandwidth <= NSTAT_LIM_UL_MAX_BANDWIDTH_THRESHOLD) || + st->lim_rtt_min >= NSTAT_LIM_UL_MIN_RTT_THRESHOLD) + st->lim_ul_detected = 1; + + if (st->lim_conn_attempts > 20 && + st->lim_conn_timeout_percent >= + NSTAT_LIM_CONN_TIMEOUT_PERCENT_THRESHOLD) + st->lim_ul_detected = 1; + /* + * Second order metrics: If there was high packet loss even after + * using delay based algorithms then we classify it as Low Internet + * again + */ + if (st->lim_bk_txpkts >= nstat_lim_min_tx_pkts && + st->lim_packet_loss_percent >= + NSTAT_LIM_PACKET_LOSS_PERCENT_THRESHOLD) + st->lim_ul_detected = 1; + return (TRUE); +} + +static u_int64_t nstat_lim_last_report_time = 0; +static void +nstat_ifnet_report_lim_stats(void) +{ + u_int64_t uptime; + struct nstat_sysinfo_data data; + struct nstat_sysinfo_lim_stats *st; + struct ifnet *ifp; + int err; + + uptime = net_uptime(); + + if ((u_int32_t)(uptime - nstat_lim_last_report_time) < + nstat_lim_interval) + return; + + nstat_lim_last_report_time = uptime; + data.flags = NSTAT_SYSINFO_LIM_STATS; + st = &data.u.lim_stats; + data.unsent_data_cnt = 0; + + ifnet_head_lock_shared(); + TAILQ_FOREACH(ifp, &ifnet_head, if_link) { + if (!IF_FULLY_ATTACHED(ifp)) + continue; + + /* Limit reporting to Wifi, Ethernet and cellular */ + if (!(IFNET_IS_ETHERNET(ifp) || IFNET_IS_CELLULAR(ifp))) + continue; + + if (!nstat_lim_activity_check(&ifp->if_lim_stat)) + continue; + + bzero(st, sizeof(*st)); + st->ifnet_siglen = sizeof (st->ifnet_signature); + err = ifnet_get_netsignature(ifp, AF_INET, + (u_int8_t *)&st->ifnet_siglen, NULL, + st->ifnet_signature); + if (err != 0) { + err = ifnet_get_netsignature(ifp, AF_INET6, + (u_int8_t *)&st->ifnet_siglen, NULL, + st->ifnet_signature); + if (err != 0) + continue; + } + ifnet_lock_shared(ifp); + if (IFNET_IS_CELLULAR(ifp)) { + st->ifnet_type = NSTAT_IFNET_DESC_LINK_STATUS_TYPE_CELLULAR; + } else if (IFNET_IS_WIFI(ifp)) { + st->ifnet_type = NSTAT_IFNET_DESC_LINK_STATUS_TYPE_WIFI; + } else { + st->ifnet_type = NSTAT_IFNET_DESC_LINK_STATUS_TYPE_ETHERNET; + } + bcopy(&ifp->if_lim_stat, &st->lim_stat, + sizeof(st->lim_stat)); + + /* Zero the stats in ifp */ + bzero(&ifp->if_lim_stat, sizeof(ifp->if_lim_stat)); + ifnet_lock_done(ifp); + nstat_sysinfo_send_data(&data); + } + ifnet_head_done(); +} + static errno_t nstat_ifnet_copy_descriptor( nstat_provider_cookie_t cookie, @@ -2890,8 +2710,8 @@ nstat_ifnet_threshold_reached(unsigned int ifindex) lck_mtx_lock(&nstat_mtx); for (state = nstat_controls; state; state = state->ncs_next) { - lck_mtx_lock(&state->mtx); - for (src = state->ncs_srcs; src; src = src->next) + lck_mtx_lock(&state->ncs_mtx); + TAILQ_FOREACH(src, &state->ncs_src_queue, ns_control_link) { if (src->provider != &nstat_ifnet_provider) continue; @@ -2901,7 +2721,7 @@ nstat_ifnet_threshold_reached(unsigned int ifindex) continue; nstat_control_send_counts(state, src, 0, 0, NULL); } - lck_mtx_unlock(&state->mtx); + lck_mtx_unlock(&state->ncs_mtx); } lck_mtx_unlock(&nstat_mtx); } @@ -2913,6 +2733,18 @@ nstat_set_keyval_scalar(nstat_sysinfo_keyval *kv, int key, u_int32_t val) kv->nstat_sysinfo_key = key; kv->nstat_sysinfo_flags = NSTAT_SYSINFO_FLAG_SCALAR; kv->u.nstat_sysinfo_scalar = val; + kv->nstat_sysinfo_valsize = sizeof(kv->u.nstat_sysinfo_scalar); +} + +static void +nstat_set_keyval_string(nstat_sysinfo_keyval *kv, int key, u_int8_t *buf, + u_int32_t len) +{ + kv->nstat_sysinfo_key = key; + kv->nstat_sysinfo_flags = NSTAT_SYSINFO_FLAG_STRING; + kv->nstat_sysinfo_valsize = min(len, + NSTAT_SYSINFO_KEYVAL_STRING_MAXSIZE); + bcopy(buf, kv->u.nstat_sysinfo_string, kv->nstat_sysinfo_valsize); } static void @@ -2938,8 +2770,7 @@ nstat_sysinfo_send_data_internal( sizeof(u_int32_t); break; case NSTAT_SYSINFO_TCP_STATS: - nkeyvals = sizeof(struct nstat_sysinfo_tcp_stats) / - sizeof(u_int32_t); + nkeyvals = NSTAT_SYSINFO_TCP_STATS_COUNT; break; case NSTAT_SYSINFO_IFNET_ECN_STATS: nkeyvals = (sizeof(struct if_tcp_ecn_stat) / @@ -2951,6 +2782,12 @@ nstat_sysinfo_send_data_internal( /* One key for unsent data. */ nkeyvals++; break; + case NSTAT_SYSINFO_LIM_STATS: + nkeyvals = NSTAT_LIM_STAT_KEYVAL_COUNT; + break; + case NSTAT_SYSINFO_NET_API_STATS: + nkeyvals = NSTAT_NET_API_STAT_KEYVAL_COUNT; + break; default: return; } @@ -3125,6 +2962,90 @@ nstat_sysinfo_send_data_internal( nstat_set_keyval_scalar(&kv[i++], NSTAT_SYSINFO_TFO_SEND_BLACKHOLE, data->u.tcp_stats.tfo_sndblackhole); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_HANDOVER_ATTEMPT, + data->u.tcp_stats.mptcp_handover_attempt); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_INTERACTIVE_ATTEMPT, + data->u.tcp_stats.mptcp_interactive_attempt); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_AGGREGATE_ATTEMPT, + data->u.tcp_stats.mptcp_aggregate_attempt); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_FP_HANDOVER_ATTEMPT, + data->u.tcp_stats.mptcp_fp_handover_attempt); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_FP_INTERACTIVE_ATTEMPT, + data->u.tcp_stats.mptcp_fp_interactive_attempt); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_FP_AGGREGATE_ATTEMPT, + data->u.tcp_stats.mptcp_fp_aggregate_attempt); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_HEURISTIC_FALLBACK, + data->u.tcp_stats.mptcp_heuristic_fallback); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_FP_HEURISTIC_FALLBACK, + data->u.tcp_stats.mptcp_fp_heuristic_fallback); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_HANDOVER_SUCCESS_WIFI, + data->u.tcp_stats.mptcp_handover_success_wifi); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_HANDOVER_SUCCESS_CELL, + data->u.tcp_stats.mptcp_handover_success_cell); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_INTERACTIVE_SUCCESS, + data->u.tcp_stats.mptcp_interactive_success); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_AGGREGATE_SUCCESS, + data->u.tcp_stats.mptcp_aggregate_success); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_FP_HANDOVER_SUCCESS_WIFI, + data->u.tcp_stats.mptcp_fp_handover_success_wifi); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_FP_HANDOVER_SUCCESS_CELL, + data->u.tcp_stats.mptcp_fp_handover_success_cell); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_FP_INTERACTIVE_SUCCESS, + data->u.tcp_stats.mptcp_fp_interactive_success); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_FP_AGGREGATE_SUCCESS, + data->u.tcp_stats.mptcp_fp_aggregate_success); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_HANDOVER_CELL_FROM_WIFI, + data->u.tcp_stats.mptcp_handover_cell_from_wifi); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_HANDOVER_WIFI_FROM_CELL, + data->u.tcp_stats.mptcp_handover_wifi_from_cell); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_INTERACTIVE_CELL_FROM_WIFI, + data->u.tcp_stats.mptcp_interactive_cell_from_wifi); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_HANDOVER_CELL_BYTES, + data->u.tcp_stats.mptcp_handover_cell_bytes); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_INTERACTIVE_CELL_BYTES, + data->u.tcp_stats.mptcp_interactive_cell_bytes); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_AGGREGATE_CELL_BYTES, + data->u.tcp_stats.mptcp_aggregate_cell_bytes); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_HANDOVER_ALL_BYTES, + data->u.tcp_stats.mptcp_handover_all_bytes); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_INTERACTIVE_ALL_BYTES, + data->u.tcp_stats.mptcp_interactive_all_bytes); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_AGGREGATE_ALL_BYTES, + data->u.tcp_stats.mptcp_aggregate_all_bytes); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_BACK_TO_WIFI, + data->u.tcp_stats.mptcp_back_to_wifi); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_WIFI_PROXY, + data->u.tcp_stats.mptcp_wifi_proxy); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_MPTCP_CELL_PROXY, + data->u.tcp_stats.mptcp_cell_proxy); VERIFY(i == nkeyvals); break; } @@ -3271,6 +3192,191 @@ nstat_sysinfo_send_data_internal( nstat_set_keyval_scalar(&kv[i++], NSTAT_SYSINFO_ECN_IFNET_FALLBACK_DROPRXMT, data->u.ifnet_ecn_stats.ecn_stat.ecn_fallback_droprxmt); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_ECN_IFNET_FALLBACK_SYNRST, + data->u.ifnet_ecn_stats.ecn_stat.ecn_fallback_synrst); + break; + } + case NSTAT_SYSINFO_LIM_STATS: + { + nstat_set_keyval_string(&kv[i++], + NSTAT_SYSINFO_LIM_IFNET_SIGNATURE, + data->u.lim_stats.ifnet_signature, + data->u.lim_stats.ifnet_siglen); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_LIM_IFNET_DL_MAX_BANDWIDTH, + data->u.lim_stats.lim_stat.lim_dl_max_bandwidth); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_LIM_IFNET_UL_MAX_BANDWIDTH, + data->u.lim_stats.lim_stat.lim_ul_max_bandwidth); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_LIM_IFNET_PACKET_LOSS_PERCENT, + data->u.lim_stats.lim_stat.lim_packet_loss_percent); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_LIM_IFNET_PACKET_OOO_PERCENT, + data->u.lim_stats.lim_stat.lim_packet_ooo_percent); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_LIM_IFNET_RTT_VARIANCE, + data->u.lim_stats.lim_stat.lim_rtt_variance); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_LIM_IFNET_RTT_MIN, + data->u.lim_stats.lim_stat.lim_rtt_min); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_LIM_IFNET_RTT_AVG, + data->u.lim_stats.lim_stat.lim_rtt_average); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_LIM_IFNET_CONN_TIMEOUT_PERCENT, + data->u.lim_stats.lim_stat.lim_conn_timeout_percent); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_LIM_IFNET_DL_DETECTED, + data->u.lim_stats.lim_stat.lim_dl_detected); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_LIM_IFNET_UL_DETECTED, + data->u.lim_stats.lim_stat.lim_ul_detected); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_LIM_IFNET_TYPE, + data->u.lim_stats.ifnet_type); + break; + } + case NSTAT_SYSINFO_NET_API_STATS: + { + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_IF_FLTR_ATTACH, + data->u.net_api_stats.net_api_stats.nas_iflt_attach_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_IF_FLTR_ATTACH_OS, + data->u.net_api_stats.net_api_stats.nas_iflt_attach_os_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_IP_FLTR_ADD, + data->u.net_api_stats.net_api_stats.nas_ipf_add_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_IP_FLTR_ADD_OS, + data->u.net_api_stats.net_api_stats.nas_ipf_add_os_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_FLTR_ATTACH, + data->u.net_api_stats.net_api_stats.nas_sfltr_register_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_FLTR_ATTACH_OS, + data->u.net_api_stats.net_api_stats.nas_sfltr_register_os_total); + + + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_ALLOC_TOTAL, + data->u.net_api_stats.net_api_stats.nas_socket_alloc_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_ALLOC_KERNEL, + data->u.net_api_stats.net_api_stats.nas_socket_in_kernel_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_ALLOC_KERNEL_OS, + data->u.net_api_stats.net_api_stats.nas_socket_in_kernel_os_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_NECP_CLIENTUUID, + data->u.net_api_stats.net_api_stats.nas_socket_necp_clientuuid_total); + + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_DOMAIN_LOCAL, + data->u.net_api_stats.net_api_stats.nas_socket_domain_local_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_DOMAIN_ROUTE, + data->u.net_api_stats.net_api_stats.nas_socket_domain_route_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_DOMAIN_INET, + data->u.net_api_stats.net_api_stats.nas_socket_domain_inet_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_DOMAIN_INET6, + data->u.net_api_stats.net_api_stats.nas_socket_domain_inet6_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_DOMAIN_SYSTEM, + data->u.net_api_stats.net_api_stats.nas_socket_domain_system_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_DOMAIN_MULTIPATH, + data->u.net_api_stats.net_api_stats.nas_socket_domain_multipath_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_DOMAIN_KEY, + data->u.net_api_stats.net_api_stats.nas_socket_domain_key_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_DOMAIN_NDRV, + data->u.net_api_stats.net_api_stats.nas_socket_domain_ndrv_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_DOMAIN_OTHER, + data->u.net_api_stats.net_api_stats.nas_socket_domain_other_total); + + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_INET_STREAM, + data->u.net_api_stats.net_api_stats.nas_socket_inet_stream_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_INET_DGRAM, + data->u.net_api_stats.net_api_stats.nas_socket_inet_dgram_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_INET_DGRAM_CONNECTED, + data->u.net_api_stats.net_api_stats.nas_socket_inet_dgram_connected); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_INET_DGRAM_DNS, + data->u.net_api_stats.net_api_stats.nas_socket_inet_dgram_dns); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_INET_DGRAM_NO_DATA, + data->u.net_api_stats.net_api_stats.nas_socket_inet_dgram_no_data); + + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_INET6_STREAM, + data->u.net_api_stats.net_api_stats.nas_socket_inet6_stream_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_INET6_DGRAM, + data->u.net_api_stats.net_api_stats.nas_socket_inet6_dgram_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_INET6_DGRAM_CONNECTED, + data->u.net_api_stats.net_api_stats.nas_socket_inet6_dgram_connected); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_INET6_DGRAM_DNS, + data->u.net_api_stats.net_api_stats.nas_socket_inet6_dgram_dns); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_INET6_DGRAM_NO_DATA, + data->u.net_api_stats.net_api_stats.nas_socket_inet6_dgram_no_data); + + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_INET_MCAST_JOIN, + data->u.net_api_stats.net_api_stats.nas_socket_mcast_join_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_SOCK_INET_MCAST_JOIN_OS, + data->u.net_api_stats.net_api_stats.nas_socket_mcast_join_os_total); + + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_NEXUS_FLOW_INET_STREAM, + data->u.net_api_stats.net_api_stats.nas_nx_flow_inet_stream_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_NEXUS_FLOW_INET_DATAGRAM, + data->u.net_api_stats.net_api_stats.nas_nx_flow_inet_dgram_total); + + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_NEXUS_FLOW_INET6_STREAM, + data->u.net_api_stats.net_api_stats.nas_nx_flow_inet6_stream_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_NEXUS_FLOW_INET6_DATAGRAM, + data->u.net_api_stats.net_api_stats.nas_nx_flow_inet6_dgram_total); + + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_IFNET_ALLOC, + data->u.net_api_stats.net_api_stats.nas_ifnet_alloc_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_IFNET_ALLOC_OS, + data->u.net_api_stats.net_api_stats.nas_ifnet_alloc_os_total); + + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_PF_ADDRULE, + data->u.net_api_stats.net_api_stats.nas_pf_addrule_total); + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_PF_ADDRULE_OS, + data->u.net_api_stats.net_api_stats.nas_pf_addrule_os); + + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_VMNET_START, + data->u.net_api_stats.net_api_stats.nas_vmnet_total); + + + nstat_set_keyval_scalar(&kv[i++], + NSTAT_SYSINFO_API_REPORT_INTERVAL, + data->u.net_api_stats.report_interval); + break; } } @@ -3303,14 +3409,12 @@ nstat_sysinfo_send_data( nstat_control_state *control; lck_mtx_lock(&nstat_mtx); - for (control = nstat_controls; control; control = control->ncs_next) - { - lck_mtx_lock(&control->mtx); - if ((control->ncs_flags & NSTAT_FLAG_SYSINFO_SUBSCRIBED) != 0) - { + for (control = nstat_controls; control; control = control->ncs_next) { + lck_mtx_lock(&control->ncs_mtx); + if ((control->ncs_flags & NSTAT_FLAG_SYSINFO_SUBSCRIBED) != 0) { nstat_sysinfo_send_data_internal(control, data); } - lck_mtx_unlock(&control->mtx); + lck_mtx_unlock(&control->ncs_mtx); } lck_mtx_unlock(&nstat_mtx); } @@ -3321,8 +3425,124 @@ nstat_sysinfo_generate_report(void) mbuf_report_peak_usage(); tcp_report_stats(); nstat_ifnet_report_ecn_stats(); + nstat_ifnet_report_lim_stats(); + nstat_net_api_report_stats(); +} + +#pragma mark -- net_api -- + +static struct net_api_stats net_api_stats_before; +static u_int64_t net_api_stats_last_report_time; + +static void +nstat_net_api_report_stats(void) +{ + struct nstat_sysinfo_data data; + struct nstat_sysinfo_net_api_stats *st = &data.u.net_api_stats; + u_int64_t uptime; + + uptime = net_uptime(); + + if ((u_int32_t)(uptime - net_api_stats_last_report_time) < + net_api_stats_report_interval) + return; + + st->report_interval = uptime - net_api_stats_last_report_time; + net_api_stats_last_report_time = uptime; + + data.flags = NSTAT_SYSINFO_NET_API_STATS; + data.unsent_data_cnt = 0; + + /* + * Some of the fields in the report are the current value and + * other fields are the delta from the last report: + * - Report difference for the per flow counters as they increase + * with time + * - Report current value for other counters as they tend not to change + * much with time + */ +#define STATCOPY(f) \ + (st->net_api_stats.f = net_api_stats.f) +#define STATDIFF(f) \ + (st->net_api_stats.f = net_api_stats.f - net_api_stats_before.f) + + STATCOPY(nas_iflt_attach_count); + STATCOPY(nas_iflt_attach_total); + STATCOPY(nas_iflt_attach_os_total); + + STATCOPY(nas_ipf_add_count); + STATCOPY(nas_ipf_add_total); + STATCOPY(nas_ipf_add_os_total); + + STATCOPY(nas_sfltr_register_count); + STATCOPY(nas_sfltr_register_total); + STATCOPY(nas_sfltr_register_os_total); + + STATDIFF(nas_socket_alloc_total); + STATDIFF(nas_socket_in_kernel_total); + STATDIFF(nas_socket_in_kernel_os_total); + STATDIFF(nas_socket_necp_clientuuid_total); + + STATDIFF(nas_socket_domain_local_total); + STATDIFF(nas_socket_domain_route_total); + STATDIFF(nas_socket_domain_inet_total); + STATDIFF(nas_socket_domain_inet6_total); + STATDIFF(nas_socket_domain_system_total); + STATDIFF(nas_socket_domain_multipath_total); + STATDIFF(nas_socket_domain_key_total); + STATDIFF(nas_socket_domain_ndrv_total); + STATDIFF(nas_socket_domain_other_total); + + STATDIFF(nas_socket_inet_stream_total); + STATDIFF(nas_socket_inet_dgram_total); + STATDIFF(nas_socket_inet_dgram_connected); + STATDIFF(nas_socket_inet_dgram_dns); + STATDIFF(nas_socket_inet_dgram_no_data); + + STATDIFF(nas_socket_inet6_stream_total); + STATDIFF(nas_socket_inet6_dgram_total); + STATDIFF(nas_socket_inet6_dgram_connected); + STATDIFF(nas_socket_inet6_dgram_dns); + STATDIFF(nas_socket_inet6_dgram_no_data); + + STATDIFF(nas_socket_mcast_join_total); + STATDIFF(nas_socket_mcast_join_os_total); + + STATDIFF(nas_sock_inet6_stream_exthdr_in); + STATDIFF(nas_sock_inet6_stream_exthdr_out); + STATDIFF(nas_sock_inet6_dgram_exthdr_in); + STATDIFF(nas_sock_inet6_dgram_exthdr_out); + + STATDIFF(nas_nx_flow_inet_stream_total); + STATDIFF(nas_nx_flow_inet_dgram_total); + + STATDIFF(nas_nx_flow_inet6_stream_total); + STATDIFF(nas_nx_flow_inet6_dgram_total); + + STATCOPY(nas_ifnet_alloc_count); + STATCOPY(nas_ifnet_alloc_total); + STATCOPY(nas_ifnet_alloc_os_count); + STATCOPY(nas_ifnet_alloc_os_total); + + STATCOPY(nas_pf_addrule_total); + STATCOPY(nas_pf_addrule_os); + + STATCOPY(nas_vmnet_total); + +#undef STATCOPY +#undef STATDIFF + + nstat_sysinfo_send_data(&data); + + /* + * Save a copy of the current fields so we can diff them the next time + */ + memcpy(&net_api_stats_before, &net_api_stats, + sizeof(struct net_api_stats)); + _CASSERT(sizeof (net_api_stats_before) == sizeof (net_api_stats)); } + #pragma mark -- Kernel Control Socket -- static kern_ctl_ref nstat_ctlref = NULL; @@ -3491,44 +3711,38 @@ nstat_idle_check( __unused thread_call_param_t p0, __unused thread_call_param_t p1) { + nstat_control_state *control; + nstat_src *src, *tmpsrc; + tailq_head_nstat_src dead_list; + TAILQ_INIT(&dead_list); + lck_mtx_lock(&nstat_mtx); nstat_idle_time = 0; - nstat_control_state *control; - nstat_src *dead = NULL; - nstat_src *dead_list = NULL; for (control = nstat_controls; control; control = control->ncs_next) { - lck_mtx_lock(&control->mtx); - nstat_src **srcpp = &control->ncs_srcs; - + lck_mtx_lock(&control->ncs_mtx); if (!(control->ncs_flags & NSTAT_FLAG_REQCOUNTS)) { - while(*srcpp != NULL) + TAILQ_FOREACH_SAFE(src, &control->ncs_src_queue, ns_control_link, tmpsrc) { - if ((*srcpp)->provider->nstat_gone((*srcpp)->cookie)) + if (src->provider->nstat_gone(src->cookie)) { errno_t result; // Pull it off the list - dead = *srcpp; - *srcpp = (*srcpp)->next; + TAILQ_REMOVE(&control->ncs_src_queue, src, ns_control_link); - result = nstat_control_send_goodbye(control, dead); + result = nstat_control_send_goodbye(control, src); // Put this on the list to release later - dead->next = dead_list; - dead_list = dead; - } - else - { - srcpp = &(*srcpp)->next; + TAILQ_INSERT_TAIL(&dead_list, src, ns_control_link); } } } control->ncs_flags &= ~NSTAT_FLAG_REQCOUNTS; - lck_mtx_unlock(&control->mtx); + lck_mtx_unlock(&control->ncs_mtx); } if (nstat_controls) @@ -3543,14 +3757,13 @@ nstat_idle_check( nstat_sysinfo_generate_report(); // Release the sources now that we aren't holding lots of locks - while (dead_list) + while ((src = TAILQ_FIRST(&dead_list))) { - dead = dead_list; - dead_list = dead->next; - - nstat_control_cleanup_source(NULL, dead, FALSE); + TAILQ_REMOVE(&dead_list, src, ns_control_link); + nstat_control_cleanup_source(NULL, src, FALSE); } + return NULL; } @@ -3629,7 +3842,7 @@ nstat_control_connect( if (state == NULL) return ENOMEM; bzero(state, sizeof(*state)); - lck_mtx_init(&state->mtx, nstat_lck_grp, NULL); + lck_mtx_init(&state->ncs_mtx, nstat_lck_grp, NULL); state->ncs_kctl = kctl; state->ncs_unit = sac->sc_unit; state->ncs_flags = NSTAT_FLAG_REQCOUNTS; @@ -3658,6 +3871,10 @@ nstat_control_disconnect( { u_int32_t watching; nstat_control_state *state = (nstat_control_state*)uinfo; + tailq_head_nstat_src cleanup_list; + nstat_src *src; + + TAILQ_INIT(&cleanup_list); // pull it out of the global list of states lck_mtx_lock(&nstat_mtx); @@ -3672,7 +3889,7 @@ nstat_control_disconnect( } lck_mtx_unlock(&nstat_mtx); - lck_mtx_lock(&state->mtx); + lck_mtx_lock(&state->ncs_mtx); // Stop watching for sources nstat_provider *provider; watching = state->ncs_watching; @@ -3696,22 +3913,16 @@ nstat_control_disconnect( } // Copy out the list of sources - nstat_src *srcs = state->ncs_srcs; - state->ncs_srcs = NULL; - lck_mtx_unlock(&state->mtx); + TAILQ_CONCAT(&cleanup_list, &state->ncs_src_queue, ns_control_link); + lck_mtx_unlock(&state->ncs_mtx); - while (srcs) + while ((src = TAILQ_FIRST(&cleanup_list))) { - nstat_src *src; - - // pull it out of the list - src = srcs; - srcs = src->next; - - // clean it up + TAILQ_REMOVE(&cleanup_list, src, ns_control_link); nstat_control_cleanup_source(NULL, src, FALSE); } - lck_mtx_destroy(&state->mtx, nstat_lck_grp); + + lck_mtx_destroy(&state->ncs_mtx, nstat_lck_grp); OSFree(state, sizeof(*state), nstat_malloc_tag); return 0; @@ -4071,8 +4282,8 @@ nstat_control_handle_add_request( return EINVAL; } - nstat_provider *provider; - nstat_provider_cookie_t cookie; + nstat_provider *provider = NULL; + nstat_provider_cookie_t cookie = NULL; nstat_msg_add_src_req *req = mbuf_data(m); if (mbuf_pkthdr_len(m) > mbuf_len(m)) { @@ -4102,6 +4313,26 @@ nstat_control_handle_add_request( return result; } +static errno_t +nstat_set_provider_filter( + nstat_control_state *state, + nstat_msg_add_all_srcs *req) +{ + nstat_provider_id_t provider_id = req->provider; + + u_int32_t prev_ncs_watching = atomic_or_32_ov(&state->ncs_watching, (1 << provider_id)); + + if ((prev_ncs_watching & (1 << provider_id)) != 0) + return EALREADY; + + state->ncs_watching |= (1 << provider_id); + state->ncs_provider_filters[provider_id].npf_flags = req->filter; + state->ncs_provider_filters[provider_id].npf_events = req->events; + state->ncs_provider_filters[provider_id].npf_pid = req->target_pid; + uuid_copy(state->ncs_provider_filters[provider_id].npf_uuid, req->target_uuid); + return 0; +} + static errno_t nstat_control_handle_add_all( nstat_control_state *state, @@ -4115,7 +4346,6 @@ nstat_control_handle_add_all( return EINVAL; } - nstat_msg_add_all_srcs *req = mbuf_data(m); if (req->provider > NSTAT_PROVIDER_LAST) return ENOENT; @@ -4131,33 +4361,21 @@ nstat_control_handle_add_all( return result; } - // Make sure we don't add the provider twice - lck_mtx_lock(&state->mtx); - if ((state->ncs_watching & (1 << provider->nstat_provider_id)) != 0) - result = EALREADY; - state->ncs_watching |= (1 << provider->nstat_provider_id); - lck_mtx_unlock(&state->mtx); - if (result != 0) return result; + lck_mtx_lock(&state->ncs_mtx); + if (req->filter & NSTAT_FILTER_SUPPRESS_SRC_ADDED) + { + // Suppression of source messages implicitly requires the use of update messages + state->ncs_flags |= NSTAT_FLAG_SUPPORTS_UPDATES; + } + lck_mtx_unlock(&state->ncs_mtx); - state->ncs_provider_filters[req->provider].npf_flags = req->filter; - state->ncs_provider_filters[req->provider].npf_events = req->events; - state->ncs_provider_filters[req->provider].npf_pid = req->target_pid; - memcpy(state->ncs_provider_filters[req->provider].npf_uuid, req->target_uuid, - sizeof(state->ncs_provider_filters[req->provider].npf_uuid)); + // rdar://problem/30301300 Different providers require different synchronization + // to ensure that a new entry does not get double counted due to being added prior + // to all current provider entries being added. Hence pass the provider the details + // in the original request for this to be applied atomically - result = provider->nstat_watcher_add(state); - if (result != 0) - { - state->ncs_provider_filters[req->provider].npf_flags = 0; - state->ncs_provider_filters[req->provider].npf_events = 0; - state->ncs_provider_filters[req->provider].npf_pid = 0; - bzero(state->ncs_provider_filters[req->provider].npf_uuid, - sizeof(state->ncs_provider_filters[req->provider].npf_uuid)); + result = provider->nstat_watcher_add(state, req); - lck_mtx_lock(&state->mtx); - state->ncs_watching &= ~(1 << provider->nstat_provider_id); - lck_mtx_unlock(&state->mtx); - } if (result == 0) nstat_enqueue_success(req->hdr.context, state, 0); @@ -4183,6 +4401,11 @@ nstat_control_source_add( (provider_filter_flagss & NSTAT_FILTER_PROVIDER_NOZEROBYTES) ? NSTAT_FILTER_NOZEROBYTES : 0; + if (provider_filter_flagss & NSTAT_FILTER_TCP_NO_EARLY_CLOSE) + { + src_filter |= NSTAT_FILTER_TCP_NO_EARLY_CLOSE; + } + if (tell_user) { unsigned int one = 1; @@ -4211,7 +4434,7 @@ nstat_control_source_add( } // Fill in the source, including picking an unused source ref - lck_mtx_lock(&state->mtx); + lck_mtx_lock(&state->ncs_mtx); src->srcref = nstat_control_next_src_ref(state); if (srcrefp) @@ -4219,7 +4442,7 @@ nstat_control_source_add( if (state->ncs_flags & NSTAT_FLAG_CLEANUP || src->srcref == NSTAT_SRC_REF_INVALID) { - lck_mtx_unlock(&state->mtx); + lck_mtx_unlock(&state->ncs_mtx); OSFree(src, sizeof(*src), nstat_malloc_tag); if (msg) mbuf_freem(msg); return EINVAL; @@ -4237,17 +4460,17 @@ nstat_control_source_add( if (result != 0) { nstat_stats.nstat_srcaddedfailures += 1; - lck_mtx_unlock(&state->mtx); + lck_mtx_unlock(&state->ncs_mtx); OSFree(src, sizeof(*src), nstat_malloc_tag); mbuf_freem(msg); return result; } } // Put the source in the list - src->next = state->ncs_srcs; - state->ncs_srcs = src; + TAILQ_INSERT_HEAD(&state->ncs_src_queue, src, ns_control_link); + src->ns_control = state; - lck_mtx_unlock(&state->mtx); + lck_mtx_unlock(&state->ncs_mtx); return 0; } @@ -4257,29 +4480,30 @@ nstat_control_handle_remove_request( nstat_control_state *state, mbuf_t m) { - nstat_src_ref_t srcref = NSTAT_SRC_REF_INVALID; + nstat_src_ref_t srcref = NSTAT_SRC_REF_INVALID; + nstat_src *src; if (mbuf_copydata(m, offsetof(nstat_msg_rem_src_req, srcref), sizeof(srcref), &srcref) != 0) { return EINVAL; } - lck_mtx_lock(&state->mtx); + lck_mtx_lock(&state->ncs_mtx); // Remove this source as we look for it - nstat_src **nextp; - nstat_src *src = NULL; - for (nextp = &state->ncs_srcs; *nextp; nextp = &(*nextp)->next) + TAILQ_FOREACH(src, &state->ncs_src_queue, ns_control_link) { - if ((*nextp)->srcref == srcref) + if (src->srcref == srcref) { - src = *nextp; - *nextp = src->next; break; } } + if (src) + { + TAILQ_REMOVE(&state->ncs_src_queue, src, ns_control_link); + } - lck_mtx_unlock(&state->mtx); + lck_mtx_unlock(&state->ncs_mtx); if (src) nstat_control_cleanup_source(state, src, FALSE); @@ -4299,7 +4523,7 @@ nstat_control_handle_query_request( // using this socket, one for read and one for write. Two threads probably // won't work with this code anyhow since we don't have proper locking in // place yet. - nstat_src *dead_srcs = NULL; + tailq_head_nstat_src dead_list; errno_t result = ENOENT; nstat_msg_query_src_req req; @@ -4309,14 +4533,15 @@ nstat_control_handle_query_request( } const boolean_t all_srcs = (req.srcref == NSTAT_SRC_REF_ALL); + TAILQ_INIT(&dead_list); - lck_mtx_lock(&state->mtx); + lck_mtx_lock(&state->ncs_mtx); if (all_srcs) { state->ncs_flags |= NSTAT_FLAG_REQCOUNTS; } - nstat_src **srcpp = &state->ncs_srcs; + nstat_src *src, *tmpsrc; u_int64_t src_count = 0; boolean_t partial = FALSE; @@ -4326,14 +4551,11 @@ nstat_control_handle_query_request( */ partial = nstat_control_begin_query(state, &req.hdr); - while (*srcpp != NULL - && (!partial || src_count < QUERY_CONTINUATION_SRC_COUNT)) + + TAILQ_FOREACH_SAFE(src, &state->ncs_src_queue, ns_control_link, tmpsrc) { - nstat_src *src = NULL; - int gone; + int gone = 0; - src = *srcpp; - gone = 0; // XXX ignore IFACE types? if (all_srcs || src->srcref == req.srcref) { @@ -4380,7 +4602,7 @@ nstat_control_handle_query_request( // send one last descriptor message so client may see last state // If we can't send the notification now, it // will be sent in the idle cleanup. - result = nstat_control_send_description(state, *srcpp, 0, 0); + result = nstat_control_send_description(state, src, 0, 0); if (result != 0) { nstat_stats.nstat_control_send_description_failures++; @@ -4391,28 +4613,30 @@ nstat_control_handle_query_request( } // pull src out of the list - *srcpp = src->next; - - src->next = dead_srcs; - dead_srcs = src; + TAILQ_REMOVE(&state->ncs_src_queue, src, ns_control_link); + TAILQ_INSERT_TAIL(&dead_list, src, ns_control_link); } - else + + if (all_srcs) { - srcpp = &(*srcpp)->next; + if (src_count >= QUERY_CONTINUATION_SRC_COUNT) + { + break; + } } - - if (!all_srcs && req.srcref == src->srcref) + else if (req.srcref == src->srcref) { break; } } + nstat_flush_accumulated_msgs(state); u_int16_t flags = 0; if (req.srcref == NSTAT_SRC_REF_ALL) - flags = nstat_control_end_query(state, *srcpp, partial); + flags = nstat_control_end_query(state, src, partial); - lck_mtx_unlock(&state->mtx); + lck_mtx_unlock(&state->ncs_mtx); /* * If an error occurred enqueueing data, then allow the error to @@ -4425,14 +4649,9 @@ nstat_control_handle_query_request( result = 0; } - while (dead_srcs) + while ((src = TAILQ_FIRST(&dead_list))) { - nstat_src *src; - - src = dead_srcs; - dead_srcs = src->next; - - // release src and send notification + TAILQ_REMOVE(&dead_list, src, ns_control_link); nstat_control_cleanup_source(state, src, FALSE); } @@ -4453,7 +4672,7 @@ nstat_control_handle_get_src_description( return EINVAL; } - lck_mtx_lock(&state->mtx); + lck_mtx_lock(&state->ncs_mtx); u_int64_t src_count = 0; boolean_t partial = FALSE; const boolean_t all_srcs = (req.srcref == NSTAT_SRC_REF_ALL); @@ -4464,9 +4683,7 @@ nstat_control_handle_get_src_description( */ partial = nstat_control_begin_query(state, &req.hdr); - for (src = state->ncs_srcs; - src && (!partial || src_count < QUERY_CONTINUATION_SRC_COUNT); - src = src->next) + TAILQ_FOREACH(src, &state->ncs_src_queue, ns_control_link) { if (all_srcs || src->srcref == req.srcref) { @@ -4498,6 +4715,10 @@ nstat_control_handle_get_src_description( */ src->seq = state->ncs_seq; src_count++; + if (src_count >= QUERY_CONTINUATION_SRC_COUNT) + { + break; + } } } @@ -4513,7 +4734,7 @@ nstat_control_handle_get_src_description( if (req.srcref == NSTAT_SRC_REF_ALL) flags = nstat_control_end_query(state, src, partial); - lck_mtx_unlock(&state->mtx); + lck_mtx_unlock(&state->ncs_mtx); /* * If an error occurred enqueueing data, then allow the error to * propagate to nstat_control_send. This way, the error is sent to @@ -4542,14 +4763,16 @@ nstat_control_handle_set_filter( req.srcref == NSTAT_SRC_REF_INVALID) return EINVAL; - lck_mtx_lock(&state->mtx); - for (src = state->ncs_srcs; src; src = src->next) + lck_mtx_lock(&state->ncs_mtx); + TAILQ_FOREACH(src, &state->ncs_src_queue, ns_control_link) + { if (req.srcref == src->srcref) { src->filter = req.filter; break; } - lck_mtx_unlock(&state->mtx); + } + lck_mtx_unlock(&state->ncs_mtx); if (src == NULL) return ENOENT; @@ -4645,16 +4868,16 @@ nstat_control_handle_get_update( return EINVAL; } - lck_mtx_lock(&state->mtx); + lck_mtx_lock(&state->ncs_mtx); state->ncs_flags |= NSTAT_FLAG_SUPPORTS_UPDATES; errno_t result = ENOENT; - nstat_src *src; - nstat_src *dead_srcs = NULL; - nstat_src **srcpp = &state->ncs_srcs; + nstat_src *src, *tmpsrc; + tailq_head_nstat_src dead_list; u_int64_t src_count = 0; boolean_t partial = FALSE; + TAILQ_INIT(&dead_list); /* * Error handling policy and sequence number generation is folded into @@ -4662,14 +4885,11 @@ nstat_control_handle_get_update( */ partial = nstat_control_begin_query(state, &req.hdr); - while (*srcpp != NULL - && (FALSE == partial - || src_count < QUERY_CONTINUATION_SRC_COUNT)) + TAILQ_FOREACH_SAFE(src, &state->ncs_src_queue, ns_control_link, tmpsrc) { int gone; gone = 0; - src = *srcpp; if (nstat_control_reporting_allowed(state, src)) { /* skip this source if it has the current state @@ -4706,20 +4926,18 @@ nstat_control_handle_get_update( if (gone) { // pull src out of the list - *srcpp = src->next; - - src->next = dead_srcs; - dead_srcs = src; - } - else - { - srcpp = &(*srcpp)->next; + TAILQ_REMOVE(&state->ncs_src_queue, src, ns_control_link); + TAILQ_INSERT_TAIL(&dead_list, src, ns_control_link); } if (req.srcref != NSTAT_SRC_REF_ALL && req.srcref == src->srcref) { break; } + if (src_count >= QUERY_CONTINUATION_SRC_COUNT) + { + break; + } } nstat_flush_accumulated_msgs(state); @@ -4727,9 +4945,9 @@ nstat_control_handle_get_update( u_int16_t flags = 0; if (req.srcref == NSTAT_SRC_REF_ALL) - flags = nstat_control_end_query(state, *srcpp, partial); + flags = nstat_control_end_query(state, src, partial); - lck_mtx_unlock(&state->mtx); + lck_mtx_unlock(&state->ncs_mtx); /* * If an error occurred enqueueing data, then allow the error to * propagate to nstat_control_send. This way, the error is sent to @@ -4741,11 +4959,9 @@ nstat_control_handle_get_update( result = 0; } - while (dead_srcs) + while ((src = TAILQ_FIRST(&dead_list))) { - src = dead_srcs; - dead_srcs = src->next; - + TAILQ_REMOVE(&dead_list, src, ns_control_link); // release src and send notification nstat_control_cleanup_source(state, src, FALSE); } @@ -4764,9 +4980,9 @@ nstat_control_handle_subscribe_sysinfo( return result; } - lck_mtx_lock(&state->mtx); + lck_mtx_lock(&state->ncs_mtx); state->ncs_flags |= NSTAT_FLAG_SYSINFO_SUBSCRIBED; - lck_mtx_unlock(&state->mtx); + lck_mtx_unlock(&state->ncs_mtx); return 0; } @@ -4890,3 +5106,5 @@ nstat_control_send( return result; } + + diff --git a/bsd/net/ntstat.h b/bsd/net/ntstat.h index e686d877d..af474f67a 100644 --- a/bsd/net/ntstat.h +++ b/bsd/net/ntstat.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2016 Apple Inc. All rights reserved. + * Copyright (c) 2010-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -30,13 +30,14 @@ #include <netinet/in.h> #include <net/if.h> #include <net/if_var.h> +#include <net/net_api_stats.h> +#include <netinet/in_stat.h> #include <netinet/tcp.h> #ifdef PRIVATE -#pragma pack(push, 4) #pragma mark -- Common Data Structures -- -#define __NSTAT_REVISION__ 8 +#define __NSTAT_REVISION__ 9 typedef u_int32_t nstat_provider_id_t; typedef u_int64_t nstat_src_ref_t; @@ -60,10 +61,17 @@ enum typedef struct nstat_counts { /* Counters */ - u_int64_t nstat_rxpackets __attribute__((aligned(8))); - u_int64_t nstat_rxbytes __attribute__((aligned(8))); - u_int64_t nstat_txpackets __attribute__((aligned(8))); - u_int64_t nstat_txbytes __attribute__((aligned(8))); + u_int64_t nstat_rxpackets __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t nstat_rxbytes __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t nstat_txpackets __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t nstat_txbytes __attribute__((aligned(sizeof(u_int64_t)))); + + u_int64_t nstat_cell_rxbytes __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t nstat_cell_txbytes __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t nstat_wifi_rxbytes __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t nstat_wifi_txbytes __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t nstat_wired_rxbytes __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t nstat_wired_txbytes __attribute__((aligned(sizeof(u_int64_t)))); u_int32_t nstat_rxduplicatebytes; u_int32_t nstat_rxoutoforderbytes; @@ -75,15 +83,9 @@ typedef struct nstat_counts u_int32_t nstat_min_rtt; u_int32_t nstat_avg_rtt; u_int32_t nstat_var_rtt; - - u_int64_t nstat_cell_rxbytes __attribute__((aligned(8))); - u_int64_t nstat_cell_txbytes __attribute__((aligned(8))); - u_int64_t nstat_wifi_rxbytes __attribute__((aligned(8))); - u_int64_t nstat_wifi_txbytes __attribute__((aligned(8))); - u_int64_t nstat_wired_rxbytes __attribute__((aligned(8))); - u_int64_t nstat_wired_txbytes __attribute__((aligned(8))); } nstat_counts; +#define NSTAT_SYSINFO_KEYVAL_STRING_MAXSIZE 24 typedef struct nstat_sysinfo_keyval { u_int32_t nstat_sysinfo_key; @@ -91,11 +93,15 @@ typedef struct nstat_sysinfo_keyval union { int64_t nstat_sysinfo_scalar; double nstat_sysinfo_distribution; + u_int8_t nstat_sysinfo_string[NSTAT_SYSINFO_KEYVAL_STRING_MAXSIZE]; } u; -} __attribute__((packed)) nstat_sysinfo_keyval; + u_int32_t nstat_sysinfo_valsize; + u_int8_t reserved[4]; +} nstat_sysinfo_keyval; #define NSTAT_SYSINFO_FLAG_SCALAR 0x0001 #define NSTAT_SYSINFO_FLAG_DISTRIBUTION 0x0002 +#define NSTAT_SYSINFO_FLAG_STRING 0x0004 #define NSTAT_MAX_MSG_SIZE 4096 @@ -105,7 +111,7 @@ typedef struct nstat_sysinfo_counts u_int32_t nstat_sysinfo_len; u_int32_t pad; u_int8_t nstat_sysinfo_keyvals[]; -} __attribute__((packed)) nstat_sysinfo_counts; +} nstat_sysinfo_counts; enum { @@ -208,10 +214,116 @@ enum ,NSTAT_SYSINFO_IFNET_UNSENT_DATA = 97 ,NSTAT_SYSINFO_ECN_IFNET_FALLBACK_DROPRST = 98 ,NSTAT_SYSINFO_ECN_IFNET_FALLBACK_DROPRXMT = 99 + ,NSTAT_SYSINFO_LIM_IFNET_SIGNATURE = 100 + ,NSTAT_SYSINFO_LIM_IFNET_DL_MAX_BANDWIDTH = 101 + ,NSTAT_SYSINFO_LIM_IFNET_UL_MAX_BANDWIDTH = 102 + ,NSTAT_SYSINFO_LIM_IFNET_PACKET_LOSS_PERCENT = 103 + ,NSTAT_SYSINFO_LIM_IFNET_PACKET_OOO_PERCENT = 104 + ,NSTAT_SYSINFO_LIM_IFNET_RTT_VARIANCE = 105 + ,NSTAT_SYSINFO_LIM_IFNET_RTT_MIN = 106 + ,NSTAT_SYSINFO_LIM_IFNET_RTT_AVG = 107 + ,NSTAT_SYSINFO_LIM_IFNET_CONN_TIMEOUT_PERCENT = 108 + ,NSTAT_SYSINFO_LIM_IFNET_DL_DETECTED = 109 + ,NSTAT_SYSINFO_LIM_IFNET_UL_DETECTED = 110 + ,NSTAT_SYSINFO_LIM_IFNET_TYPE = 111 + + ,NSTAT_SYSINFO_API_IF_FLTR_ATTACH = 112 + ,NSTAT_SYSINFO_API_IF_FLTR_ATTACH_OS = 113 + ,NSTAT_SYSINFO_API_IP_FLTR_ADD = 114 + ,NSTAT_SYSINFO_API_IP_FLTR_ADD_OS = 115 + ,NSTAT_SYSINFO_API_SOCK_FLTR_ATTACH = 116 + ,NSTAT_SYSINFO_API_SOCK_FLTR_ATTACH_OS = 117 + + ,NSTAT_SYSINFO_API_SOCK_ALLOC_TOTAL = 118 + ,NSTAT_SYSINFO_API_SOCK_ALLOC_KERNEL = 119 + ,NSTAT_SYSINFO_API_SOCK_ALLOC_KERNEL_OS = 120 + ,NSTAT_SYSINFO_API_SOCK_NECP_CLIENTUUID = 121 + + ,NSTAT_SYSINFO_API_SOCK_DOMAIN_LOCAL = 122 + ,NSTAT_SYSINFO_API_SOCK_DOMAIN_ROUTE = 123 + ,NSTAT_SYSINFO_API_SOCK_DOMAIN_INET = 124 + ,NSTAT_SYSINFO_API_SOCK_DOMAIN_INET6 = 125 + ,NSTAT_SYSINFO_API_SOCK_DOMAIN_SYSTEM = 126 + ,NSTAT_SYSINFO_API_SOCK_DOMAIN_MULTIPATH = 127 + ,NSTAT_SYSINFO_API_SOCK_DOMAIN_KEY = 128 + ,NSTAT_SYSINFO_API_SOCK_DOMAIN_NDRV = 129 + ,NSTAT_SYSINFO_API_SOCK_DOMAIN_OTHER = 130 + + ,NSTAT_SYSINFO_API_SOCK_INET_STREAM= 131 + ,NSTAT_SYSINFO_API_SOCK_INET_DGRAM = 132 + ,NSTAT_SYSINFO_API_SOCK_INET_DGRAM_CONNECTED = 133 + ,NSTAT_SYSINFO_API_SOCK_INET_DGRAM_DNS = 134 + ,NSTAT_SYSINFO_API_SOCK_INET_DGRAM_NO_DATA = 135 + + ,NSTAT_SYSINFO_API_SOCK_INET6_STREAM= 136 + ,NSTAT_SYSINFO_API_SOCK_INET6_DGRAM = 137 + ,NSTAT_SYSINFO_API_SOCK_INET6_DGRAM_CONNECTED = 138 + ,NSTAT_SYSINFO_API_SOCK_INET6_DGRAM_DNS = 139 + ,NSTAT_SYSINFO_API_SOCK_INET6_DGRAM_NO_DATA = 140 + + ,NSTAT_SYSINFO_API_SOCK_INET_MCAST_JOIN = 141 + ,NSTAT_SYSINFO_API_SOCK_INET_MCAST_JOIN_OS = 142 + + ,NSTAT_SYSINFO_API_SOCK_INET6_STREAM_EXTHDR_IN = 143 + ,NSTAT_SYSINFO_API_SOCK_INET6_STREAM_EXTHDR_OUT = 144 + ,NSTAT_SYSINFO_API_SOCK_INET6_DGRAM_EXTHDR_IN = 145 + ,NSTAT_SYSINFO_API_SOCK_INET6_DGRAM_EXTHDR_OUT = 146 + + ,NSTAT_SYSINFO_API_NEXUS_FLOW_INET_STREAM = 147 + ,NSTAT_SYSINFO_API_NEXUS_FLOW_INET_DATAGRAM = 148 + + ,NSTAT_SYSINFO_API_NEXUS_FLOW_INET6_STREAM = 149 + ,NSTAT_SYSINFO_API_NEXUS_FLOW_INET6_DATAGRAM = 150 + + ,NSTAT_SYSINFO_API_IFNET_ALLOC = 151 + ,NSTAT_SYSINFO_API_IFNET_ALLOC_OS = 152 + + ,NSTAT_SYSINFO_API_PF_ADDRULE = 153 + ,NSTAT_SYSINFO_API_PF_ADDRULE_OS = 154 + + ,NSTAT_SYSINFO_API_VMNET_START = 155 + + ,NSTAT_SYSINFO_API_IF_NETAGENT_ENABLED = 156 + + ,NSTAT_SYSINFO_API_REPORT_INTERVAL = 157 + + ,NSTAT_SYSINFO_MPTCP_HANDOVER_ATTEMPT = 158 + ,NSTAT_SYSINFO_MPTCP_INTERACTIVE_ATTEMPT = 159 + ,NSTAT_SYSINFO_MPTCP_AGGREGATE_ATTEMPT = 160 + ,NSTAT_SYSINFO_MPTCP_FP_HANDOVER_ATTEMPT = 161 /* _FP_ stands for first-party */ + ,NSTAT_SYSINFO_MPTCP_FP_INTERACTIVE_ATTEMPT = 162 + ,NSTAT_SYSINFO_MPTCP_FP_AGGREGATE_ATTEMPT = 163 + ,NSTAT_SYSINFO_MPTCP_HEURISTIC_FALLBACK = 164 + ,NSTAT_SYSINFO_MPTCP_FP_HEURISTIC_FALLBACK = 165 + ,NSTAT_SYSINFO_MPTCP_HANDOVER_SUCCESS_WIFI = 166 + ,NSTAT_SYSINFO_MPTCP_HANDOVER_SUCCESS_CELL = 167 + ,NSTAT_SYSINFO_MPTCP_INTERACTIVE_SUCCESS = 168 + ,NSTAT_SYSINFO_MPTCP_AGGREGATE_SUCCESS = 169 + ,NSTAT_SYSINFO_MPTCP_FP_HANDOVER_SUCCESS_WIFI = 170 + ,NSTAT_SYSINFO_MPTCP_FP_HANDOVER_SUCCESS_CELL = 171 + ,NSTAT_SYSINFO_MPTCP_FP_INTERACTIVE_SUCCESS = 172 + ,NSTAT_SYSINFO_MPTCP_FP_AGGREGATE_SUCCESS = 173 + ,NSTAT_SYSINFO_MPTCP_HANDOVER_CELL_FROM_WIFI = 174 + ,NSTAT_SYSINFO_MPTCP_HANDOVER_WIFI_FROM_CELL = 175 + ,NSTAT_SYSINFO_MPTCP_INTERACTIVE_CELL_FROM_WIFI = 176 + ,NSTAT_SYSINFO_MPTCP_HANDOVER_CELL_BYTES = 177 + ,NSTAT_SYSINFO_MPTCP_INTERACTIVE_CELL_BYTES = 178 + ,NSTAT_SYSINFO_MPTCP_AGGREGATE_CELL_BYTES = 179 + ,NSTAT_SYSINFO_MPTCP_HANDOVER_ALL_BYTES = 180 + ,NSTAT_SYSINFO_MPTCP_INTERACTIVE_ALL_BYTES = 181 + ,NSTAT_SYSINFO_MPTCP_AGGREGATE_ALL_BYTES = 182 + ,NSTAT_SYSINFO_MPTCP_BACK_TO_WIFI = 183 + ,NSTAT_SYSINFO_MPTCP_WIFI_PROXY = 184 + ,NSTAT_SYSINFO_MPTCP_CELL_PROXY = 185 + ,NSTAT_SYSINFO_ECN_IFNET_FALLBACK_SYNRST = 186 + // NSTAT_SYSINFO_ENUM_VERSION must be updated any time a value is added -#define NSTAT_SYSINFO_ENUM_VERSION 20160715 +#define NSTAT_SYSINFO_ENUM_VERSION 20170623 }; +#define NSTAT_SYSINFO_API_FIRST NSTAT_SYSINFO_API_IF_FLTR_ATTACH +#define NSTAT_SYSINFO_API_LAST NSTAT_SYSINFO_API_REPORT_INTERVAL + #pragma mark -- Network Statistics Providers -- @@ -226,6 +338,9 @@ enum #define NSTAT_IFNET_IS_EXPENSIVE 0x40 #define NSTAT_IFNET_IS_VPN 0x80 #define NSTAT_IFNET_VIA_CELLFALLBACK 0x100 +// Temporary properties of use for bringing up userland providers +#define NSTAT_IFNET_ROUTE_VALUE_UNOBTAINABLE 0x1000 +#define NSTAT_IFNET_FLOWSWITCH_VALUE_UNOBTAINABLE 0x2000 enum @@ -273,20 +388,14 @@ typedef struct nstat_tcp_add_param typedef struct nstat_tcp_descriptor { - union - { - struct sockaddr_in v4; - struct sockaddr_in6 v6; - } local; + u_int64_t upid __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t eupid __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t start_timestamp __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t timestamp __attribute__((aligned(sizeof(u_int64_t)))); - union - { - struct sockaddr_in v4; - struct sockaddr_in6 v6; - } remote; + activity_bitmap_t activity_bitmap; u_int32_t ifindex; - u_int32_t state; u_int32_t sndbufsize; @@ -298,25 +407,49 @@ typedef struct nstat_tcp_descriptor u_int32_t txcwindow; u_int32_t traffic_class; u_int32_t traffic_mgt_flags; - char cc_algo[16]; - u_int64_t upid; u_int32_t pid; - char pname[64]; - u_int64_t eupid; u_int32_t epid; + union + { + struct sockaddr_in v4; + struct sockaddr_in6 v6; + } local; + + union + { + struct sockaddr_in v4; + struct sockaddr_in6 v6; + } remote; + + char cc_algo[16]; + char pname[64]; + uuid_t uuid; uuid_t euuid; uuid_t vuuid; - struct tcp_conn_status connstatus; + union { + struct tcp_conn_status connstatus; + // On armv7k, tcp_conn_status is 1 byte instead of 4 + uint8_t __pad_connstatus[4]; + }; uint16_t ifnet_properties __attribute__((aligned(4))); + + u_int8_t reserved[6]; } nstat_tcp_descriptor; typedef struct nstat_tcp_add_param nstat_udp_add_param; typedef struct nstat_udp_descriptor { + u_int64_t upid __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t eupid __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t start_timestamp __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t timestamp __attribute__((aligned(sizeof(u_int64_t)))); + + activity_bitmap_t activity_bitmap; + union { struct sockaddr_in v4; @@ -335,23 +468,23 @@ typedef struct nstat_udp_descriptor u_int32_t rcvbufused; u_int32_t traffic_class; - u_int64_t upid; u_int32_t pid; char pname[64]; - u_int64_t eupid; u_int32_t epid; uuid_t uuid; uuid_t euuid; uuid_t vuuid; uint16_t ifnet_properties; + + u_int8_t reserved[6]; } nstat_udp_descriptor; typedef struct nstat_route_descriptor { - u_int64_t id; - u_int64_t parent_id; - u_int64_t gateway_id; + u_int64_t id __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t parent_id __attribute__((aligned(sizeof(u_int64_t)))); + u_int64_t gateway_id __attribute__((aligned(sizeof(u_int64_t)))); union { @@ -377,12 +510,15 @@ typedef struct nstat_route_descriptor u_int32_t ifindex; u_int32_t flags; + u_int8_t reserved[4]; } nstat_route_descriptor; typedef struct nstat_ifnet_add_param { + u_int64_t threshold __attribute__((aligned(sizeof(u_int64_t)))); u_int32_t ifindex; - u_int64_t threshold; + + u_int8_t reserved[4]; } nstat_ifnet_add_param; typedef struct nstat_ifnet_desc_cellular_status @@ -436,6 +572,7 @@ typedef struct nstat_ifnet_desc_cellular_status #define NSTAT_IFNET_DESC_MSS_RECOMMENDED_MEDIUM 0x1 #define NSTAT_IFNET_DESC_MSS_RECOMMENDED_LOW 0x2 u_int16_t mss_recommended; /* recommended MSS */ + u_int8_t reserved[2]; } nstat_ifnet_desc_cellular_status; typedef struct nstat_ifnet_desc_wifi_status { @@ -508,6 +645,7 @@ enum NSTAT_IFNET_DESC_LINK_STATUS_TYPE_NONE = 0 ,NSTAT_IFNET_DESC_LINK_STATUS_TYPE_CELLULAR = 1 ,NSTAT_IFNET_DESC_LINK_STATUS_TYPE_WIFI = 2 + ,NSTAT_IFNET_DESC_LINK_STATUS_TYPE_ETHERNET = 3 }; typedef struct nstat_ifnet_desc_link_status @@ -524,12 +662,13 @@ typedef struct nstat_ifnet_desc_link_status #endif typedef struct nstat_ifnet_descriptor { - char name[IFNAMSIZ+1]; + u_int64_t threshold __attribute__((aligned(sizeof(u_int64_t)))); u_int32_t ifindex; - u_int64_t threshold; - unsigned int type; - char description[IF_DESCSIZE]; nstat_ifnet_desc_link_status link_status; + unsigned int type; + char description[IF_DESCSIZE]; + char name[IFNAMSIZ+1]; + u_int8_t reserved[3]; } nstat_ifnet_descriptor; typedef struct nstat_sysinfo_descriptor @@ -545,7 +684,9 @@ typedef struct nstat_sysinfo_add_param #define NSTAT_SYSINFO_MBUF_STATS 0x0001 #define NSTAT_SYSINFO_TCP_STATS 0x0002 -#define NSTAT_SYSINFO_IFNET_ECN_STATS 0x0003 +#define NSTAT_SYSINFO_IFNET_ECN_STATS 0x0003 +#define NSTAT_SYSINFO_LIM_STATS 0x0004 /* Low Internet mode stats */ +#define NSTAT_SYSINFO_NET_API_STATS 0x0005 /* API and KPI stats */ #pragma mark -- Network Statistics User Client -- @@ -604,6 +745,7 @@ enum ,NSTAT_FILTER_TCP_NO_LISTENER = 0x00001000 ,NSTAT_FILTER_TCP_ONLY_LISTENER = 0x00002000 ,NSTAT_FILTER_TCP_INTERFACE_ATTACH = 0x00004000 + ,NSTAT_FILTER_TCP_NO_EARLY_CLOSE = 0x00008000 ,NSTAT_FILTER_TCP_FLAGS = 0x0000F000 ,NSTAT_FILTER_UDP_INTERFACE_ATTACH = 0x00010000 @@ -629,7 +771,7 @@ enum typedef struct nstat_msg_hdr { - u_int64_t context; + u_int64_t context __attribute__((aligned(sizeof(u_int64_t)))); u_int32_t type; u_int16_t length; u_int16_t flags; @@ -639,21 +781,45 @@ typedef struct nstat_msg_error { nstat_msg_hdr hdr; u_int32_t error; // errno error + u_int8_t reserved[4]; } nstat_msg_error; +#define NSTAT_ADD_SRC_FIELDS \ + nstat_msg_hdr hdr; \ + nstat_provider_id_t provider; \ + u_int8_t reserved[4] \ + typedef struct nstat_msg_add_src { - nstat_msg_hdr hdr; - nstat_provider_id_t provider; - u_int8_t param[]; + NSTAT_ADD_SRC_FIELDS; + u_int8_t param[]; } nstat_msg_add_src_req; +typedef struct nstat_msg_add_src_header +{ + NSTAT_ADD_SRC_FIELDS; +} nstat_msg_add_src_header; + +typedef struct nstat_msg_add_src_convenient +{ + nstat_msg_add_src_header hdr; + union { + nstat_route_add_param route; + nstat_tcp_add_param tcp; + nstat_udp_add_param udp; + nstat_ifnet_add_param ifnet; + nstat_sysinfo_add_param sysinfo; + }; +} nstat_msg_add_src_convenient; + +#undef NSTAT_ADD_SRC_FIELDS + typedef struct nstat_msg_add_all_srcs { nstat_msg_hdr hdr; + u_int64_t filter __attribute__((aligned(sizeof(u_int64_t)))); + nstat_event_flags_t events __attribute__((aligned(sizeof(u_int64_t)))); nstat_provider_id_t provider; - u_int64_t filter; - nstat_event_flags_t events; pid_t target_pid; uuid_t target_uuid; } nstat_msg_add_all_srcs; @@ -661,76 +827,122 @@ typedef struct nstat_msg_add_all_srcs typedef struct nstat_msg_src_added { nstat_msg_hdr hdr; + nstat_src_ref_t srcref __attribute__((aligned(sizeof(u_int64_t)))); nstat_provider_id_t provider; - nstat_src_ref_t srcref; + u_int8_t reserved[4]; } nstat_msg_src_added; typedef struct nstat_msg_rem_src { nstat_msg_hdr hdr; - nstat_src_ref_t srcref; + nstat_src_ref_t srcref __attribute__((aligned(sizeof(u_int64_t)))); } nstat_msg_rem_src_req; typedef struct nstat_msg_get_src_description { nstat_msg_hdr hdr; - nstat_src_ref_t srcref; + nstat_src_ref_t srcref __attribute__((aligned(sizeof(u_int64_t)))); } nstat_msg_get_src_description; typedef struct nstat_msg_set_filter { nstat_msg_hdr hdr; - nstat_src_ref_t srcref; - u_int32_t filter; + nstat_src_ref_t srcref __attribute__((aligned(sizeof(u_int64_t)))); + u_int32_t filter; + u_int8_t reserved[4]; } nstat_msg_set_filter; +#define NSTAT_SRC_DESCRIPTION_FIELDS \ + nstat_msg_hdr hdr; \ + nstat_src_ref_t srcref __attribute__((aligned(sizeof(u_int64_t)))); \ + nstat_event_flags_t event_flags __attribute__((aligned(sizeof(u_int64_t)))); \ + nstat_provider_id_t provider; \ + u_int8_t reserved[4] + typedef struct nstat_msg_src_description { - nstat_msg_hdr hdr; - nstat_src_ref_t srcref; - nstat_event_flags_t event_flags; - nstat_provider_id_t provider; - u_int8_t data[]; + NSTAT_SRC_DESCRIPTION_FIELDS; + u_int8_t data[]; } nstat_msg_src_description; +typedef struct nstat_msg_src_description_header +{ + NSTAT_SRC_DESCRIPTION_FIELDS; +} nstat_msg_src_description_header; + +typedef struct nstat_msg_src_description_convenient +{ + nstat_msg_src_description_header hdr; + union { + nstat_tcp_descriptor tcp; + nstat_udp_descriptor udp; + nstat_route_descriptor route; + nstat_ifnet_descriptor ifnet; + nstat_sysinfo_descriptor sysinfo; + }; +} nstat_msg_src_description_convenient; + +#undef NSTAT_SRC_DESCRIPTION_FIELDS + typedef struct nstat_msg_query_src { nstat_msg_hdr hdr; - nstat_src_ref_t srcref; + nstat_src_ref_t srcref __attribute__((aligned(sizeof(u_int64_t)))); } nstat_msg_query_src_req; typedef struct nstat_msg_src_counts { nstat_msg_hdr hdr; - nstat_src_ref_t srcref; - nstat_event_flags_t event_flags; + nstat_src_ref_t srcref __attribute__((aligned(sizeof(u_int64_t)))); + nstat_event_flags_t event_flags __attribute__((aligned(sizeof(u_int64_t)))); nstat_counts counts; } nstat_msg_src_counts; +#define NSTAT_SRC_UPDATE_FIELDS \ + nstat_msg_hdr hdr; \ + nstat_src_ref_t srcref __attribute__((aligned(sizeof(u_int64_t)))); \ + nstat_event_flags_t event_flags __attribute__((aligned(sizeof(u_int64_t)))); \ + nstat_counts counts; \ + nstat_provider_id_t provider; \ + u_int8_t reserved[4] + typedef struct nstat_msg_src_update { - nstat_msg_hdr hdr; - nstat_src_ref_t srcref; - nstat_event_flags_t event_flags; - nstat_counts counts; - nstat_provider_id_t provider; - u_int8_t data[]; + NSTAT_SRC_UPDATE_FIELDS; + u_int8_t data[]; } nstat_msg_src_update; +typedef struct nstat_msg_src_update_hdr +{ + NSTAT_SRC_UPDATE_FIELDS; +} nstat_msg_src_update_hdr; + +typedef struct nstat_msg_src_update_convenient +{ + nstat_msg_src_update_hdr hdr; + union { + nstat_tcp_descriptor tcp; + nstat_udp_descriptor udp; + nstat_route_descriptor route; + nstat_ifnet_descriptor ifnet; + nstat_sysinfo_descriptor sysinfo; + }; +} nstat_msg_src_update_convenient; + +#undef NSTAT_SRC_UPDATE_FIELDS + typedef struct nstat_msg_src_removed { nstat_msg_hdr hdr; - nstat_src_ref_t srcref; + nstat_src_ref_t srcref __attribute__((aligned(sizeof(u_int64_t)))); } nstat_msg_src_removed; typedef struct nstat_msg_sysinfo_counts { nstat_msg_hdr hdr; - nstat_src_ref_t srcref; + nstat_src_ref_t srcref __attribute__((aligned(sizeof(u_int64_t)))); nstat_sysinfo_counts counts; -} __attribute__((packed)) nstat_msg_sysinfo_counts; - -#pragma pack(pop) +} nstat_msg_sysinfo_counts; #pragma mark -- Statitiscs about Network Statistics -- @@ -775,6 +987,7 @@ typedef struct nstat_sysinfo_mbuf_stats typedef struct nstat_sysinfo_tcp_stats { + /* When adding/removing here, also adjust NSTAT_SYSINFO_TCP_STATS_COUNT */ u_int32_t ipv4_avgrtt; /* Average RTT for IPv4 */ u_int32_t ipv6_avgrtt; /* Average RTT for IPv6 */ u_int32_t send_plr; /* Average uplink packet loss rate */ @@ -817,7 +1030,37 @@ typedef struct nstat_sysinfo_tcp_stats u_int32_t tfo_no_cookie_rcv; /* We asked for a cookie but didn't get one */ u_int32_t tfo_heuristics_disable; /* TFO got disabled due to heuristics */ u_int32_t tfo_sndblackhole; /* TFO got blackholed in the sending direction */ + u_int32_t mptcp_handover_attempt; /* Total number of MPTCP-attempts using handover mode */ + u_int32_t mptcp_interactive_attempt; /* Total number of MPTCP-attempts using interactive mode */ + u_int32_t mptcp_aggregate_attempt; /* Total number of MPTCP-attempts using aggregate mode */ + u_int32_t mptcp_fp_handover_attempt; /* Same as previous three but only for first-party apps */ + u_int32_t mptcp_fp_interactive_attempt; + u_int32_t mptcp_fp_aggregate_attempt; + u_int32_t mptcp_heuristic_fallback; /* Total number of MPTCP-connections that fell back due to heuristics */ + u_int32_t mptcp_fp_heuristic_fallback; /* Same as previous but for first-party apps */ + u_int32_t mptcp_handover_success_wifi; /* Total number of successfull handover-mode connections that *started* on WiFi */ + u_int32_t mptcp_handover_success_cell; /* Total number of successfull handover-mode connections that *started* on Cell */ + u_int32_t mptcp_interactive_success; /* Total number of interactive-mode connections that negotiated MPTCP */ + u_int32_t mptcp_aggregate_success; /* Same as previous but for aggregate */ + u_int32_t mptcp_fp_handover_success_wifi; /* Same as previous four, but for first-party apps */ + u_int32_t mptcp_fp_handover_success_cell; + u_int32_t mptcp_fp_interactive_success; + u_int32_t mptcp_fp_aggregate_success; + u_int32_t mptcp_handover_cell_from_wifi; /* Total number of connections that use cell in handover-mode (coming from WiFi) */ + u_int32_t mptcp_handover_wifi_from_cell; /* Total number of connections that use WiFi in handover-mode (coming from cell) */ + u_int32_t mptcp_interactive_cell_from_wifi; /* Total number of connections that use cell in interactive mode (coming from WiFi) */ + u_int32_t mptcp_back_to_wifi; /* Total number of connections that succeed to move traffic away from cell (when starting on cell) */ + u_int64_t mptcp_handover_cell_bytes; /* Total number of bytes sent on cell in handover-mode (on new subflows, ignoring initial one) */ + u_int64_t mptcp_interactive_cell_bytes; /* Same as previous but for interactive */ + u_int64_t mptcp_aggregate_cell_bytes; + u_int64_t mptcp_handover_all_bytes; /* Total number of bytes sent in handover */ + u_int64_t mptcp_interactive_all_bytes; + u_int64_t mptcp_aggregate_all_bytes; + u_int32_t mptcp_wifi_proxy; /* Total number of new subflows that fell back to regular TCP on cell */ + u_int32_t mptcp_cell_proxy; /* Total number of new subflows that fell back to regular TCP on WiFi */ + /* When adding/removing here, also adjust NSTAT_SYSINFO_TCP_STATS_COUNT */ } nstat_sysinfo_tcp_stats; +#define NSTAT_SYSINFO_TCP_STATS_COUNT 70 enum { NSTAT_IFNET_ECN_PROTO_IPV4 = 1 @@ -836,15 +1079,33 @@ typedef struct nstat_sysinfo_ifnet_ecn_stats { struct if_tcp_ecn_stat ecn_stat; } nstat_sysinfo_ifnet_ecn_stats; +/* Total number of Low Internet stats that will be reported */ +#define NSTAT_LIM_STAT_KEYVAL_COUNT 12 +typedef struct nstat_sysinfo_lim_stats { + u_int8_t ifnet_signature[NSTAT_SYSINFO_KEYVAL_STRING_MAXSIZE]; + u_int32_t ifnet_siglen; + u_int32_t ifnet_type; + struct if_lim_perf_stat lim_stat; +} nstat_sysinfo_lim_stats; + +#define NSTAT_NET_API_STAT_KEYVAL_COUNT (NSTAT_SYSINFO_API_LAST - NSTAT_SYSINFO_API_FIRST + 1) +typedef struct nstat_sysinfo_net_api_stats { + u_int32_t report_interval; + u_int32_t _padding; + struct net_api_stats net_api_stats; +} nstat_sysinfo_net_api_stats; + typedef struct nstat_sysinfo_data { - u_int32_t flags; + uint32_t flags; + uint32_t unsent_data_cnt; /* Before sleeping */ union { nstat_sysinfo_mbuf_stats mb_stats; nstat_sysinfo_tcp_stats tcp_stats; nstat_sysinfo_ifnet_ecn_stats ifnet_ecn_stats; + nstat_sysinfo_lim_stats lim_stats; + nstat_sysinfo_net_api_stats net_api_stats; } u; - uint32_t unsent_data_cnt; /* Before sleeping */ } nstat_sysinfo_data; #pragma mark -- Generic Network Statistics Provider -- @@ -876,6 +1137,11 @@ void nstat_route_connect_success(struct rtentry *rte); void nstat_route_tx(struct rtentry *rte, u_int32_t packets, u_int32_t bytes, u_int32_t flags); void nstat_route_rx(struct rtentry *rte, u_int32_t packets, u_int32_t bytes, u_int32_t flags); void nstat_route_rtt(struct rtentry *rte, u_int32_t rtt, u_int32_t rtt_var); +void nstat_route_update(struct rtentry *rte, uint32_t connect_attempts, uint32_t connect_successes, + uint32_t rx_packets, uint32_t rx_bytes, uint32_t rx_duplicatebytes, uint32_t rx_outoforderbytes, + uint32_t tx_packets, uint32_t tx_bytes, uint32_t tx_retransmit, + uint32_t rtt, uint32_t rtt_var); +struct nstat_counts* nstat_route_attach(struct rtentry *rte); void nstat_route_detach(struct rtentry *rte); // watcher support @@ -892,37 +1158,9 @@ void nstat_ifnet_threshold_reached(unsigned int ifindex); void nstat_sysinfo_send_data(struct nstat_sysinfo_data *); -// Userland stats reporting - -// Each side, NetworkStatistics and the kernel provider for userland, -// pass opaque references. -typedef void *userland_stats_provider_context; -typedef void *nstat_userland_context; - -// When things have been set up, Netstats can request a refresh of its data. -typedef bool (userland_stats_request_vals_fn)(userland_stats_provider_context *ctx, - nstat_counts *countsp, - void *metadatap); - -// Things get started with a call to netstats to say that thereâs a new connection: -nstat_userland_context ntstat_userland_stats_open(userland_stats_provider_context *ctx, - int provider_id, - u_int64_t properties, - userland_stats_request_vals_fn req_fn); - -void ntstat_userland_stats_close(nstat_userland_context nstat_ctx); - - -// There may be other occasions where the stats have changed and NECP should push the new values. -// This is provisional, ahead of full implementation. - -typedef enum { - USERLAND_STATS_WILL_UPDATE, - USERLAND_STATS_DID_UPDATE -} userland_stats_event_t; - -void ntstat_userland_stats_event(nstat_userland_context nstat_ctx, userland_stats_event_t event); +// Utilities for userland stats reporting +u_int16_t nstat_ifnet_to_flags(struct ifnet *ifp); // locked_add_64 uses atomic operations on 32bit so the 64bit // value can be properly read. The values are only ever incremented diff --git a/bsd/net/nwk_wq.c b/bsd/net/nwk_wq.c new file mode 100644 index 000000000..400fd1312 --- /dev/null +++ b/bsd/net/nwk_wq.c @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include <stddef.h> +#include <kern/debug.h> +#include <kern/locks.h> +#include <kern/thread.h> +#include <kern/thread_call.h> +#include <net/nwk_wq.h> +#include <sys/proc_internal.h> +#include <sys/systm.h> +#include <sys/mcache.h> + +MALLOC_DEFINE(M_NWKWQ, "nwkwq", "Network work-queue"); + +static TAILQ_HEAD(, nwk_wq_entry) nwk_wq_head; +decl_lck_mtx_data(static, nwk_wq_lock); + +/* Lock group and attributes */ +static lck_grp_attr_t *nwk_wq_lock_grp_attributes = NULL; +static lck_grp_t *nwk_wq_lock_group = NULL; + +/* Lock and lock attributes */ +static lck_attr_t *nwk_wq_lock_attributes = NULL; +decl_lck_mtx_data(static, nwk_wq_lock); + +/* Wait channel for Network work queue */ +static void *nwk_wq_waitch = NULL; +static void nwk_wq_thread_func(void *, wait_result_t); + +static int nwk_wq_thread_cont(int err); +static void nwk_wq_thread_func(void *v, wait_result_t w); + +void +nwk_wq_init (void) +{ + thread_t nwk_wq_thread = THREAD_NULL; + + TAILQ_INIT(&nwk_wq_head); + nwk_wq_lock_grp_attributes = lck_grp_attr_alloc_init(); + nwk_wq_lock_group = lck_grp_alloc_init("Network work queue lock", + nwk_wq_lock_grp_attributes); + + nwk_wq_lock_attributes = lck_attr_alloc_init(); + lck_mtx_init(&nwk_wq_lock, nwk_wq_lock_group, nwk_wq_lock_attributes); + if (kernel_thread_start(nwk_wq_thread_func, + NULL, &nwk_wq_thread) != KERN_SUCCESS) { + panic_plain("%s: couldn't create network work queue thread", __func__); + /* NOTREACHED */ + } + thread_deallocate(nwk_wq_thread); +} + +static int +nwk_wq_thread_cont(int err) +{ + TAILQ_HEAD(, nwk_wq_entry) temp_nwk_wq_head; + struct nwk_wq_entry *nwk_item; + struct nwk_wq_entry *nwk_item_next; + +#pragma unused(err) + for (;;) { + nwk_item = NULL; + nwk_item_next = NULL; + TAILQ_INIT(&temp_nwk_wq_head); + + LCK_MTX_ASSERT(&nwk_wq_lock, LCK_MTX_ASSERT_OWNED); + while (TAILQ_FIRST(&nwk_wq_head) == NULL) { + (void) msleep0(&nwk_wq_waitch, &nwk_wq_lock, + (PZERO - 1), "nwk_wq_thread_cont", 0, + nwk_wq_thread_cont); + /* NOTREACHED */ + } + + TAILQ_SWAP(&temp_nwk_wq_head, &nwk_wq_head, nwk_wq_entry, nwk_wq_link); + VERIFY(TAILQ_EMPTY(&nwk_wq_head)); + lck_mtx_unlock(&nwk_wq_lock); + + VERIFY(TAILQ_FIRST(&temp_nwk_wq_head) != NULL); + TAILQ_FOREACH_SAFE(nwk_item, &temp_nwk_wq_head, nwk_wq_link, nwk_item_next) { + nwk_item->func(nwk_item->arg); + if (nwk_item->is_arg_managed == FALSE) + FREE(nwk_item->arg, M_NWKWQ); + FREE(nwk_item, M_NWKWQ); + } + lck_mtx_lock(&nwk_wq_lock); + } +} + +static void +nwk_wq_thread_func(void *v, wait_result_t w) +{ +#pragma unused(v, w) + lck_mtx_lock(&nwk_wq_lock); + (void) msleep0(&nwk_wq_waitch, &nwk_wq_lock, + (PZERO - 1), "nwk_wq_thread_func", 0, nwk_wq_thread_cont); + /* + * msleep0() shouldn't have returned as PCATCH was not set; + * therefore assert in this case. + */ + lck_mtx_unlock(&nwk_wq_lock); + VERIFY(0); +} + +void +nwk_wq_enqueue(struct nwk_wq_entry *nwk_item) +{ + lck_mtx_lock(&nwk_wq_lock); + TAILQ_INSERT_TAIL(&nwk_wq_head, nwk_item, nwk_wq_link); + lck_mtx_unlock(&nwk_wq_lock); + wakeup((caddr_t)&nwk_wq_waitch); +} + diff --git a/bsd/net/nwk_wq.h b/bsd/net/nwk_wq.h new file mode 100644 index 000000000..80d8e4851 --- /dev/null +++ b/bsd/net/nwk_wq.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2016-2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef NWK_WQ_H +#define NWK_WQ_H +#include <sys/queue.h> +#include <kern/kern_types.h> + +#ifdef BSD_KERNEL_PRIVATE +struct nwk_wq_entry { + void (* func) (void *); + void *arg; + boolean_t is_arg_managed; + TAILQ_ENTRY(nwk_wq_entry) nwk_wq_link; +}; + +void nwk_wq_init (void); +void nwk_wq_enqueue(struct nwk_wq_entry *nwk_item); +#endif /* BSD_KERNEL_PRIVATE */ +#endif /* NWK_WQ_H */ + diff --git a/bsd/net/packet_mangler.c b/bsd/net/packet_mangler.c index 5a1776d74..d6c8b9dbd 100644 --- a/bsd/net/packet_mangler.c +++ b/bsd/net/packet_mangler.c @@ -822,7 +822,7 @@ static errno_t pktmnglr_ipfilter_output(void *cookie, mbuf_t *data, ipf_pktopts_ if (ip.ip_v != 4) { PKT_MNGLR_LOG(LOG_INFO, "%s:%d Not handling IP version %d\n", - __FILE__, __LINE__, ip.ip_v); + __func__, __LINE__, ip.ip_v); goto output_done; } diff --git a/bsd/net/pf.c b/bsd/net/pf.c index 70ea3cf33..6ce23690e 100644 --- a/bsd/net/pf.c +++ b/bsd/net/pf.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2007-2016 Apple Inc. All rights reserved. + * Copyright (c) 2007-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -68,7 +68,6 @@ #include <machine/endian.h> #include <sys/param.h> #include <sys/systm.h> -#include <sys/mbuf.h> #include <sys/filio.h> #include <sys/socket.h> #include <sys/socketvar.h> @@ -83,6 +82,7 @@ #include <libkern/libkern.h> #include <mach/thread_act.h> +#include <mach/branch_predicates.h> #include <net/if.h> #include <net/if_types.h> @@ -162,14 +162,6 @@ struct pf_state_tree_ext_gwy pf_statetbl_ext_gwy; struct pf_palist pf_pabuf; struct pf_status pf_status; -#if PF_ALTQ -struct pf_altqqueue pf_altqs[2]; -struct pf_altqqueue *pf_altqs_active; -struct pf_altqqueue *pf_altqs_inactive; -u_int32_t ticket_altqs_active; -u_int32_t ticket_altqs_inactive; -int altqs_inactive_open; -#endif /* PF_ALTQ */ u_int32_t ticket_pabuf; static MD5_CTX pf_tcp_secret_ctx; @@ -186,9 +178,6 @@ static struct pf_anchor_stackframe { struct pool pf_src_tree_pl, pf_rule_pl, pf_pooladdr_pl; struct pool pf_state_pl, pf_state_key_pl; -#if PF_ALTQ -struct pool pf_altq_pl; -#endif /* PF_ALTQ */ typedef void (*hook_fn_t)(void *); @@ -218,11 +207,11 @@ static void pf_init_threshold(struct pf_threshold *, u_int32_t, static void pf_add_threshold(struct pf_threshold *); static int pf_check_threshold(struct pf_threshold *); -static void pf_change_ap(int, struct mbuf *, struct pf_addr *, +static void pf_change_ap(int, pbuf_t *, struct pf_addr *, u_int16_t *, u_int16_t *, u_int16_t *, struct pf_addr *, u_int16_t, u_int8_t, sa_family_t, sa_family_t, int); -static int pf_modulate_sack(struct mbuf *, int, struct pf_pdesc *, +static int pf_modulate_sack(pbuf_t *, int, struct pf_pdesc *, struct tcphdr *, struct pf_state_peer *); #if INET6 static void pf_change_a6(struct pf_addr *, u_int16_t *, @@ -240,46 +229,47 @@ static void pf_send_tcp(const struct pf_rule *, sa_family_t, u_int16_t, u_int16_t, u_int32_t, u_int32_t, u_int8_t, u_int16_t, u_int16_t, u_int8_t, int, u_int16_t, struct ether_header *, struct ifnet *); -static void pf_send_icmp(struct mbuf *, u_int8_t, u_int8_t, +static void pf_send_icmp(pbuf_t *, u_int8_t, u_int8_t, sa_family_t, struct pf_rule *); -static struct pf_rule *pf_match_translation(struct pf_pdesc *, struct mbuf *, +static struct pf_rule *pf_match_translation(struct pf_pdesc *, pbuf_t *, int, int, struct pfi_kif *, struct pf_addr *, union pf_state_xport *, struct pf_addr *, union pf_state_xport *, int); static struct pf_rule *pf_get_translation_aux(struct pf_pdesc *, - struct mbuf *, int, int, struct pfi_kif *, + pbuf_t *, int, int, struct pfi_kif *, struct pf_src_node **, struct pf_addr *, union pf_state_xport *, struct pf_addr *, - union pf_state_xport *, union pf_state_xport *); + union pf_state_xport *, union pf_state_xport * + ); static void pf_attach_state(struct pf_state_key *, struct pf_state *, int); static void pf_detach_state(struct pf_state *, int); static u_int32_t pf_tcp_iss(struct pf_pdesc *); static int pf_test_rule(struct pf_rule **, struct pf_state **, - int, struct pfi_kif *, struct mbuf *, int, + int, struct pfi_kif *, pbuf_t *, int, void *, struct pf_pdesc *, struct pf_rule **, struct pf_ruleset **, struct ifqueue *); #if DUMMYNET static int pf_test_dummynet(struct pf_rule **, int, - struct pfi_kif *, struct mbuf **, + struct pfi_kif *, pbuf_t **, struct pf_pdesc *, struct ip_fw_args *); #endif /* DUMMYNET */ static int pf_test_fragment(struct pf_rule **, int, - struct pfi_kif *, struct mbuf *, void *, + struct pfi_kif *, pbuf_t *, void *, struct pf_pdesc *, struct pf_rule **, struct pf_ruleset **); static int pf_test_state_tcp(struct pf_state **, int, - struct pfi_kif *, struct mbuf *, int, + struct pfi_kif *, pbuf_t *, int, void *, struct pf_pdesc *, u_short *); static int pf_test_state_udp(struct pf_state **, int, - struct pfi_kif *, struct mbuf *, int, + struct pfi_kif *, pbuf_t *, int, void *, struct pf_pdesc *, u_short *); static int pf_test_state_icmp(struct pf_state **, int, - struct pfi_kif *, struct mbuf *, int, + struct pfi_kif *, pbuf_t *, int, void *, struct pf_pdesc *, u_short *); static int pf_test_state_other(struct pf_state **, int, struct pfi_kif *, struct pf_pdesc *); -static int pf_match_tag(struct mbuf *, struct pf_rule *, +static int pf_match_tag(struct pf_rule *, struct pf_mtag *, int *); static void pf_hash(struct pf_addr *, struct pf_addr *, struct pf_poolhashkey *, sa_family_t); @@ -290,24 +280,25 @@ static int pf_get_sport(struct pf_pdesc *, struct pfi_kif *, struct pf_rule *, struct pf_addr *, union pf_state_xport *, struct pf_addr *, union pf_state_xport *, struct pf_addr *, - union pf_state_xport *, struct pf_src_node **); -static void pf_route(struct mbuf **, struct pf_rule *, int, + union pf_state_xport *, struct pf_src_node ** + ); +static void pf_route(pbuf_t **, struct pf_rule *, int, struct ifnet *, struct pf_state *, struct pf_pdesc *); #if INET6 -static void pf_route6(struct mbuf **, struct pf_rule *, int, +static void pf_route6(pbuf_t **, struct pf_rule *, int, struct ifnet *, struct pf_state *, struct pf_pdesc *); #endif /* INET6 */ -static u_int8_t pf_get_wscale(struct mbuf *, int, u_int16_t, +static u_int8_t pf_get_wscale(pbuf_t *, int, u_int16_t, sa_family_t); -static u_int16_t pf_get_mss(struct mbuf *, int, u_int16_t, +static u_int16_t pf_get_mss(pbuf_t *, int, u_int16_t, sa_family_t); static u_int16_t pf_calc_mss(struct pf_addr *, sa_family_t, u_int16_t); static void pf_set_rt_ifp(struct pf_state *, struct pf_addr *, sa_family_t af); -static int pf_check_proto_cksum(struct mbuf *, int, int, +static int pf_check_proto_cksum(pbuf_t *, int, int, u_int8_t, sa_family_t); static int pf_addr_wrap_neq(struct pf_addr_wrap *, struct pf_addr_wrap *); @@ -345,35 +336,37 @@ struct pf_pool_limit pf_pool_limits[PF_LIMIT_MAX] = { { &pfr_kentry_pl, PFR_KENTRY_HIWAT }, }; -struct mbuf * -pf_lazy_makewritable(struct pf_pdesc *pd, struct mbuf *m, int len) +void * +pf_lazy_makewritable(struct pf_pdesc *pd, pbuf_t *pbuf, int len) { + void *p; + if (pd->lmw < 0) - return (0); + return (NULL); - VERIFY(m == pd->mp); + VERIFY(pbuf == pd->mp); + p = pbuf->pb_data; if (len > pd->lmw) { - if (m_makewritable(&m, 0, len, M_DONTWAIT)) + if ((p = pbuf_ensure_writable(pbuf, len)) == NULL) len = -1; pd->lmw = len; - if (len >= 0 && m != pd->mp) { - pd->mp = m; - pd->pf_mtag = pf_find_mtag(m); + if (len >= 0) { + pd->pf_mtag = pf_find_mtag_pbuf(pbuf); switch (pd->af) { case AF_INET: { - struct ip *h = mtod(m, struct ip *); - pd->src = (struct pf_addr *)&h->ip_src; - pd->dst = (struct pf_addr *)&h->ip_dst; + struct ip *h = p; + pd->src = (struct pf_addr *)(uintptr_t)&h->ip_src; + pd->dst = (struct pf_addr *)(uintptr_t)&h->ip_dst; pd->ip_sum = &h->ip_sum; break; } #if INET6 case AF_INET6: { - struct ip6_hdr *h = mtod(m, struct ip6_hdr *); - pd->src = (struct pf_addr *)&h->ip6_src; - pd->dst = (struct pf_addr *)&h->ip6_dst; + struct ip6_hdr *h = p; + pd->src = (struct pf_addr *)(uintptr_t)&h->ip6_src; + pd->dst = (struct pf_addr *)(uintptr_t)&h->ip6_dst; break; } #endif /* INET6 */ @@ -381,7 +374,7 @@ pf_lazy_makewritable(struct pf_pdesc *pd, struct mbuf *m, int len) } } - return (len < 0 ? 0 : m); + return (len < 0 ? NULL : p); } static const int * @@ -1222,13 +1215,13 @@ pf_src_connlimit(struct pf_state **state) #if INET case AF_INET: p.pfra_net = 32; - p.pfra_ip4addr = (*state)->src_node->addr.v4; + p.pfra_ip4addr = (*state)->src_node->addr.v4addr; break; #endif /* INET */ #if INET6 case AF_INET6: p.pfra_net = 128; - p.pfra_ip6addr = (*state)->src_node->addr.v6; + p.pfra_ip6addr = (*state)->src_node->addr.v6addr; break; #endif /* INET6 */ } @@ -1532,7 +1525,7 @@ pf_state_expires(const struct pf_state *state) u_int32_t end; u_int32_t states; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); /* handle all PFTM_* > PFTM_MAX here */ if (state->timeout == PFTM_PURGE) @@ -1567,7 +1560,7 @@ pf_purge_expired_src_nodes(void) { struct pf_src_node *cur, *next; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); for (cur = RB_MIN(pf_src_tree, &tree_src_tracking); cur; cur = next) { next = RB_NEXT(pf_src_tree, &tree_src_tracking, cur); @@ -1592,7 +1585,7 @@ pf_src_tree_remove_state(struct pf_state *s) { u_int32_t t; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (s->src_node != NULL) { if (s->src.tcp_est) { @@ -1622,7 +1615,7 @@ pf_src_tree_remove_state(struct pf_state *s) void pf_unlink_state(struct pf_state *cur) { - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (cur->src.state == PF_TCPS_PROXY_DST) { pf_send_tcp(cur->rule.ptr, cur->state_key->af_lan, @@ -1649,7 +1642,7 @@ pf_unlink_state(struct pf_state *cur) void pf_free_state(struct pf_state *cur) { - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); #if NPFSYNC if (pfsyncif != NULL && (pfsyncif->sc_bulk_send_next == cur || @@ -1689,7 +1682,7 @@ pf_purge_expired_states(u_int32_t maxcheck) static struct pf_state *cur = NULL; struct pf_state *next; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); while (maxcheck--) { /* wrap to start of list when we hit the end */ @@ -1716,7 +1709,7 @@ pf_purge_expired_states(u_int32_t maxcheck) int pf_tbladdr_setup(struct pf_ruleset *rs, struct pf_addr_wrap *aw) { - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (aw->type != PF_ADDR_TABLE) return (0); @@ -1728,7 +1721,7 @@ pf_tbladdr_setup(struct pf_ruleset *rs, struct pf_addr_wrap *aw) void pf_tbladdr_remove(struct pf_addr_wrap *aw) { - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (aw->type != PF_ADDR_TABLE || aw->p.tbl == NULL) return; @@ -1741,7 +1734,7 @@ pf_tbladdr_copyout(struct pf_addr_wrap *aw) { struct pfr_ktable *kt = aw->p.tbl; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (aw->type != PF_ADDR_TABLE || kt == NULL) return; @@ -2096,7 +2089,7 @@ pf_cksum_fixup(u_int16_t cksum, u_int16_t old, u_int16_t new, u_int8_t udp) * only the checksum is recalculated & updated. */ static void -pf_change_ap(int dir, struct mbuf *m, struct pf_addr *a, u_int16_t *p, +pf_change_ap(int dir, pbuf_t *pbuf, struct pf_addr *a, u_int16_t *p, u_int16_t *ic, u_int16_t *pc, struct pf_addr *an, u_int16_t pn, u_int8_t u, sa_family_t af, sa_family_t afn, int ua) { @@ -2126,9 +2119,8 @@ pf_change_ap(int dir, struct mbuf *m, struct pf_addr *a, u_int16_t *p, * will have UDP/TCP CSUM flag set (gets set in protocol * output). */ - if (dir == PF_OUT && m != NULL && - (m->m_flags & M_PKTHDR) && - (m->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP))) { + if (dir == PF_OUT && pbuf != NULL && + (*pbuf->pb_csum_flags & (CSUM_TCP | CSUM_UDP))) { /* Pseudo-header checksum does not include ports */ *pc = ~pf_cksum_fixup(pf_cksum_fixup(~*pc, ao.addr16[0], an->addr16[0], u), @@ -2175,9 +2167,8 @@ pf_change_ap(int dir, struct mbuf *m, struct pf_addr *a, u_int16_t *p, * will have UDP/TCP CSUM flag set (gets set in protocol * output). */ - if (dir == PF_OUT && m != NULL && - (m->m_flags & M_PKTHDR) && - (m->m_pkthdr.csum_flags & (CSUM_TCPIPV6 | + if (dir == PF_OUT && pbuf != NULL && + (*pbuf->pb_csum_flags & (CSUM_TCPIPV6 | CSUM_UDPIPV6))) { /* Pseudo-header checksum does not include ports */ *pc = @@ -2280,7 +2271,7 @@ pf_change_addr(struct pf_addr *a, u_int16_t *c, struct pf_addr *an, u_int8_t u, case AF_INET: switch (afn) { case AF_INET: - pf_change_a(a, c, an->v4.s_addr, u); + pf_change_a(a, c, an->v4addr.s_addr, u); break; case AF_INET6: *c = pf_cksum_fixup(pf_cksum_fixup(pf_cksum_fixup( @@ -2413,7 +2404,7 @@ pf_change_icmp(struct pf_addr *ia, u_int16_t *ip, struct pf_addr *oa, * (credits to Krzysztof Pfaff for report and patch) */ static int -pf_modulate_sack(struct mbuf *m, int off, struct pf_pdesc *pd, +pf_modulate_sack(pbuf_t *pbuf, int off, struct pf_pdesc *pd, struct tcphdr *th, struct pf_state_peer *dst) { int hlen = (th->th_off << 2) - sizeof (*th), thoptlen = hlen; @@ -2423,7 +2414,7 @@ pf_modulate_sack(struct mbuf *m, int off, struct pf_pdesc *pd, #define TCPOLEN_SACKLEN (TCPOLEN_SACK + 2) if (hlen < TCPOLEN_SACKLEN || - !pf_pull_hdr(m, off + sizeof (*th), opts, hlen, NULL, NULL, pd->af)) + !pf_pull_hdr(pbuf, off + sizeof (*th), opts, hlen, NULL, NULL, pd->af)) return (0); while (hlen >= TCPOLEN_SACKLEN) { @@ -2461,14 +2452,25 @@ pf_modulate_sack(struct mbuf *m, int off, struct pf_pdesc *pd, } if (copyback) { - m = pf_lazy_makewritable(pd, m, copyback); - if (!m) + if (pf_lazy_makewritable(pd, pbuf, copyback) == NULL) return (-1); - m_copyback(m, off + sizeof (*th), thoptlen, opts); + pbuf_copy_back(pbuf, off + sizeof (*th), thoptlen, opts); } return (copyback); } +/* + * XXX + * + * The following functions (pf_send_tcp and pf_send_icmp) are somewhat + * special in that they originate "spurious" packets rather than + * filter/NAT existing packets. As such, they're not a great fit for + * the 'pbuf' shim, which assumes the underlying packet buffers are + * allocated elsewhere. + * + * Since these functions are rarely used, we'll carry on allocating mbufs + * and passing them to the IP stack for eventual routing. + */ static void pf_send_tcp(const struct pf_rule *r, sa_family_t af, const struct pf_addr *saddr, const struct pf_addr *daddr, @@ -2515,10 +2517,8 @@ pf_send_tcp(const struct pf_rule *r, sa_family_t af, if (m == NULL) return; - if ((pf_mtag = pf_get_mtag(m)) == NULL) { - m_free(m); + if ((pf_mtag = pf_get_mtag(m)) == NULL) return; - } if (tag) pf_mtag->pftag_flags |= PF_TAG_GENERATED; @@ -2527,11 +2527,6 @@ pf_send_tcp(const struct pf_rule *r, sa_family_t af, if (r != NULL && PF_RTABLEID_IS_VALID(r->rtableid)) pf_mtag->pftag_rtableid = r->rtableid; -#if PF_ALTQ - if (altq_allowed && r != NULL && r->qid) - pf_mtag->pftag_qid = r->qid; -#endif /* PF_ALTQ */ - #if PF_ECN /* add hints for ecn */ pf_mtag->pftag_hdr = mtod(m, struct ip *); @@ -2567,8 +2562,8 @@ pf_send_tcp(const struct pf_rule *r, sa_family_t af, /* IP header fields included in the TCP checksum */ h->ip_p = IPPROTO_TCP; h->ip_len = htons(tlen); - h->ip_src.s_addr = saddr->v4.s_addr; - h->ip_dst.s_addr = daddr->v4.s_addr; + h->ip_src.s_addr = saddr->v4addr.s_addr; + h->ip_dst.s_addr = daddr->v4addr.s_addr; th = (struct tcphdr *)(void *)((caddr_t)h + sizeof (struct ip)); break; @@ -2580,8 +2575,8 @@ pf_send_tcp(const struct pf_rule *r, sa_family_t af, /* IP header fields included in the TCP checksum */ h6->ip6_nxt = IPPROTO_TCP; h6->ip6_plen = htons(tlen); - memcpy(&h6->ip6_src, &saddr->v6, sizeof (struct in6_addr)); - memcpy(&h6->ip6_dst, &daddr->v6, sizeof (struct in6_addr)); + memcpy(&h6->ip6_src, &saddr->v6addr, sizeof (struct in6_addr)); + memcpy(&h6->ip6_dst, &daddr->v6addr, sizeof (struct in6_addr)); th = (struct tcphdr *)(void *) ((caddr_t)h6 + sizeof (struct ip6_hdr)); @@ -2655,13 +2650,13 @@ pf_send_tcp(const struct pf_rule *r, sa_family_t af, } static void -pf_send_icmp(struct mbuf *m, u_int8_t type, u_int8_t code, sa_family_t af, +pf_send_icmp(pbuf_t *pbuf, u_int8_t type, u_int8_t code, sa_family_t af, struct pf_rule *r) { struct mbuf *m0; struct pf_mtag *pf_mtag; - m0 = m_copy(m, 0, M_COPYALL); + m0 = pbuf_clone_to_mbuf(pbuf); if (m0 == NULL) return; @@ -2673,11 +2668,6 @@ pf_send_icmp(struct mbuf *m, u_int8_t type, u_int8_t code, sa_family_t af, if (PF_RTABLEID_IS_VALID(r->rtableid)) pf_mtag->pftag_rtableid = r->rtableid; -#if PF_ALTQ - if (altq_allowed && r->qid) - pf_mtag->pftag_qid = r->qid; -#endif /* PF_ALTQ */ - #if PF_ECN /* add hints for ecn */ pf_mtag->pftag_hdr = mtod(m0, struct ip *); @@ -2886,10 +2876,9 @@ pf_match_gid(u_int8_t op, gid_t a1, gid_t a2, gid_t g) } static int -pf_match_tag(struct mbuf *m, struct pf_rule *r, struct pf_mtag *pf_mtag, +pf_match_tag(struct pf_rule *r, struct pf_mtag *pf_mtag, int *tag) { -#pragma unused(m) if (*tag == -1) *tag = pf_mtag->pftag_tag; @@ -2898,14 +2887,14 @@ pf_match_tag(struct mbuf *m, struct pf_rule *r, struct pf_mtag *pf_mtag, } int -pf_tag_packet(struct mbuf *m, struct pf_mtag *pf_mtag, int tag, +pf_tag_packet(pbuf_t *pbuf, struct pf_mtag *pf_mtag, int tag, unsigned int rtableid, struct pf_pdesc *pd) { if (tag <= 0 && !PF_RTABLEID_IS_VALID(rtableid) && (pd == NULL || !(pd->pktflags & PKTF_FLOW_ID))) return (0); - if (pf_mtag == NULL && (pf_mtag = pf_get_mtag(m)) == NULL) + if (pf_mtag == NULL && (pf_mtag = pf_get_mtag_pbuf(pbuf)) == NULL) return (1); if (tag > 0) @@ -2913,10 +2902,10 @@ pf_tag_packet(struct mbuf *m, struct pf_mtag *pf_mtag, int tag, if (PF_RTABLEID_IS_VALID(rtableid)) pf_mtag->pftag_rtableid = rtableid; if (pd != NULL && (pd->pktflags & PKTF_FLOW_ID)) { - m->m_pkthdr.pkt_flowsrc = pd->flowsrc; - m->m_pkthdr.pkt_flowid = pd->flowhash; - m->m_pkthdr.pkt_flags |= pd->pktflags; - m->m_pkthdr.pkt_proto = pd->proto; + *pbuf->pb_flowsrc = pd->flowsrc; + *pbuf->pb_flowid = pd->flowhash; + *pbuf->pb_flags |= pd->pktflags; + *pbuf->pb_proto = pd->proto; } return (0); @@ -3310,7 +3299,8 @@ static int pf_get_sport(struct pf_pdesc *pd, struct pfi_kif *kif, struct pf_rule *r, struct pf_addr *saddr, union pf_state_xport *sxport, struct pf_addr *daddr, union pf_state_xport *dxport, struct pf_addr *naddr, - union pf_state_xport *nxport, struct pf_src_node **sn) + union pf_state_xport *nxport, struct pf_src_node **sn + ) { #pragma unused(kif) struct pf_state_key_cmp key; @@ -3423,11 +3413,14 @@ pf_get_sport(struct pf_pdesc *pd, struct pfi_kif *kif, struct pf_rule *r, return (0); } else if (low == 0 && high == 0) { key.gwy.xport = *nxport; - if (pf_find_state_all(&key, PF_IN, NULL) == NULL) + if (pf_find_state_all(&key, PF_IN, NULL) == NULL + ) { return (0); + } } else if (low == high) { key.gwy.xport.port = htons(low); - if (pf_find_state_all(&key, PF_IN, NULL) == NULL) { + if (pf_find_state_all(&key, PF_IN, NULL) == NULL + ) { nxport->port = htons(low); return (0); } @@ -3443,16 +3436,16 @@ pf_get_sport(struct pf_pdesc *pd, struct pfi_kif *kif, struct pf_rule *r, /* low <= cut <= high */ for (tmp = cut; tmp <= high; ++(tmp)) { key.gwy.xport.port = htons(tmp); - if (pf_find_state_all(&key, PF_IN, NULL) == - NULL) { + if (pf_find_state_all(&key, PF_IN, NULL) == NULL + ) { nxport->port = htons(tmp); return (0); } } for (tmp = cut - 1; tmp >= low; --(tmp)) { key.gwy.xport.port = htons(tmp); - if (pf_find_state_all(&key, PF_IN, NULL) == - NULL) { + if (pf_find_state_all(&key, PF_IN, NULL) == NULL + ) { nxport->port = htons(tmp); return (0); } @@ -3477,7 +3470,7 @@ pf_get_sport(struct pf_pdesc *pd, struct pfi_kif *kif, struct pf_rule *r, } static struct pf_rule * -pf_match_translation(struct pf_pdesc *pd, struct mbuf *m, int off, +pf_match_translation(struct pf_pdesc *pd, pbuf_t *pbuf, int off, int direction, struct pfi_kif *kif, struct pf_addr *saddr, union pf_state_xport *sxport, struct pf_addr *daddr, union pf_state_xport *dxport, int rs_num) @@ -3544,10 +3537,10 @@ pf_match_translation(struct pf_pdesc *pd, struct mbuf *m, int off, else if (dst && !pf_match_xport(r->proto, r->proto_variant, &dst->xport, dxport)) r = r->skip[PF_SKIP_DST_PORT].ptr; - else if (r->match_tag && !pf_match_tag(m, r, pd->pf_mtag, &tag)) + else if (r->match_tag && !pf_match_tag(r, pd->pf_mtag, &tag)) r = TAILQ_NEXT(r, entries); else if (r->os_fingerprint != PF_OSFP_ANY && (pd->proto != - IPPROTO_TCP || !pf_osfp_match(pf_osfp_fingerprint(pd, m, + IPPROTO_TCP || !pf_osfp_match(pf_osfp_fingerprint(pd, pbuf, off, pd->hdr.tcp), r->os_fingerprint))) r = TAILQ_NEXT(r, entries); else { @@ -3565,7 +3558,7 @@ pf_match_translation(struct pf_pdesc *pd, struct mbuf *m, int off, pf_step_out_of_anchor(&asd, &ruleset, rs_num, &r, NULL, NULL); } - if (pf_tag_packet(m, pd->pf_mtag, tag, rtableid, NULL)) + if (pf_tag_packet(pbuf, pd->pf_mtag, tag, rtableid, NULL)) return (NULL); if (rm != NULL && (rm->action == PF_NONAT || rm->action == PF_NORDR || rm->action == PF_NOBINAT || @@ -3577,7 +3570,7 @@ pf_match_translation(struct pf_pdesc *pd, struct mbuf *m, int off, /* * Get address translation information for NAT/BINAT/RDR * pd : pf packet descriptor - * m : mbuf holding the packet + * pbuf : pbuf holding the packet * off : offset to protocol header * direction : direction of packet * kif : pf interface info obtained from the packet's recv interface @@ -3592,28 +3585,29 @@ pf_match_translation(struct pf_pdesc *pd, struct mbuf *m, int off, * pd->ndaddr */ static struct pf_rule * -pf_get_translation_aux(struct pf_pdesc *pd, struct mbuf *m, int off, +pf_get_translation_aux(struct pf_pdesc *pd, pbuf_t *pbuf, int off, int direction, struct pfi_kif *kif, struct pf_src_node **sn, struct pf_addr *saddr, union pf_state_xport *sxport, struct pf_addr *daddr, - union pf_state_xport *dxport, union pf_state_xport *nsxport) + union pf_state_xport *dxport, union pf_state_xport *nsxport + ) { struct pf_rule *r = NULL; pd->naf = pd->af; if (direction == PF_OUT) { - r = pf_match_translation(pd, m, off, direction, kif, saddr, + r = pf_match_translation(pd, pbuf, off, direction, kif, saddr, sxport, daddr, dxport, PF_RULESET_BINAT); if (r == NULL) - r = pf_match_translation(pd, m, off, direction, kif, + r = pf_match_translation(pd, pbuf, off, direction, kif, saddr, sxport, daddr, dxport, PF_RULESET_RDR); if (r == NULL) - r = pf_match_translation(pd, m, off, direction, kif, + r = pf_match_translation(pd, pbuf, off, direction, kif, saddr, sxport, daddr, dxport, PF_RULESET_NAT); } else { - r = pf_match_translation(pd, m, off, direction, kif, saddr, + r = pf_match_translation(pd, pbuf, off, direction, kif, saddr, sxport, daddr, dxport, PF_RULESET_RDR); if (r == NULL) - r = pf_match_translation(pd, m, off, direction, kif, + r = pf_match_translation(pd, pbuf, off, direction, kif, saddr, sxport, daddr, dxport, PF_RULESET_BINAT); } @@ -3643,7 +3637,9 @@ pf_get_translation_aux(struct pf_pdesc *pd, struct mbuf *m, int off, return (NULL); if (pf_get_sport(pd, kif, r, saddr, sxport, daddr, - dxport, nsaddr, nsxport, sn)) { + dxport, nsaddr, nsxport, sn + )) + { DPFPRINTF(PF_DEBUG_MISC, ("pf: NAT proxy port allocation " "(%u-%u) failed\n", @@ -3656,7 +3652,7 @@ pf_get_translation_aux(struct pf_pdesc *pd, struct mbuf *m, int off, * from the last 32 bits of synthesized IPv6 address */ if (r->action == PF_NAT64) { - ndaddr->v4.s_addr = daddr->addr32[3]; + ndaddr->v4addr.s_addr = daddr->addr32[3]; pd->naf = AF_INET; } break; @@ -3877,7 +3873,7 @@ pf_socket_lookup(int direction, struct pf_pdesc *pd) switch (pd->af) { #if INET case AF_INET: - inp = in_pcblookup_hash_exists(pi, saddr->v4, sport, daddr->v4, dport, + inp = in_pcblookup_hash_exists(pi, saddr->v4addr, sport, daddr->v4addr, dport, 0, &pd->lookup.uid, &pd->lookup.gid, NULL); #if INET6 if (inp == 0) { @@ -3885,19 +3881,19 @@ pf_socket_lookup(int direction, struct pf_pdesc *pd) memset(&s6, 0, sizeof (s6)); s6.s6_addr16[5] = htons(0xffff); - memcpy(&s6.s6_addr32[3], &saddr->v4, - sizeof (saddr->v4)); + memcpy(&s6.s6_addr32[3], &saddr->v4addr, + sizeof (saddr->v4addr)); memset(&d6, 0, sizeof (d6)); d6.s6_addr16[5] = htons(0xffff); - memcpy(&d6.s6_addr32[3], &daddr->v4, - sizeof (daddr->v4)); + memcpy(&d6.s6_addr32[3], &daddr->v4addr, + sizeof (daddr->v4addr)); inp = in6_pcblookup_hash_exists(pi, &s6, sport, &d6, dport, 0, &pd->lookup.uid, &pd->lookup.gid, NULL); if (inp == 0) { - inp = in_pcblookup_hash_exists(pi, saddr->v4, sport, - daddr->v4, dport, INPLOOKUP_WILDCARD, &pd->lookup.uid, &pd->lookup.gid, NULL); + inp = in_pcblookup_hash_exists(pi, saddr->v4addr, sport, + daddr->v4addr, dport, INPLOOKUP_WILDCARD, &pd->lookup.uid, &pd->lookup.gid, NULL); if (inp == 0) { inp = in6_pcblookup_hash_exists(pi, &s6, sport, &d6, dport, INPLOOKUP_WILDCARD, @@ -3909,8 +3905,8 @@ pf_socket_lookup(int direction, struct pf_pdesc *pd) } #else if (inp == 0) { - inp = in_pcblookup_hash_exists(pi, saddr->v4, sport, - daddr->v4, dport, INPLOOKUP_WILDCARD, + inp = in_pcblookup_hash_exists(pi, saddr->v4addr, sport, + daddr->v4addr, dport, INPLOOKUP_WILDCARD, &pd->lookup.uid, &pd->lookup.gid, NULL); if (inp == 0) return (-1); @@ -3920,11 +3916,11 @@ pf_socket_lookup(int direction, struct pf_pdesc *pd) #endif /* INET */ #if INET6 case AF_INET6: - inp = in6_pcblookup_hash_exists(pi, &saddr->v6, sport, &daddr->v6, + inp = in6_pcblookup_hash_exists(pi, &saddr->v6addr, sport, &daddr->v6addr, dport, 0, &pd->lookup.uid, &pd->lookup.gid, NULL); if (inp == 0) { - inp = in6_pcblookup_hash_exists(pi, &saddr->v6, sport, - &daddr->v6, dport, INPLOOKUP_WILDCARD, + inp = in6_pcblookup_hash_exists(pi, &saddr->v6addr, sport, + &daddr->v6addr, dport, INPLOOKUP_WILDCARD, &pd->lookup.uid, &pd->lookup.gid, NULL); if (inp == 0) return (-1); @@ -3940,7 +3936,7 @@ pf_socket_lookup(int direction, struct pf_pdesc *pd) } static u_int8_t -pf_get_wscale(struct mbuf *m, int off, u_int16_t th_off, sa_family_t af) +pf_get_wscale(pbuf_t *pbuf, int off, u_int16_t th_off, sa_family_t af) { int hlen; u_int8_t hdr[60]; @@ -3950,7 +3946,7 @@ pf_get_wscale(struct mbuf *m, int off, u_int16_t th_off, sa_family_t af) hlen = th_off << 2; /* hlen <= sizeof (hdr) */ if (hlen <= (int)sizeof (struct tcphdr)) return (0); - if (!pf_pull_hdr(m, off, hdr, hlen, NULL, NULL, af)) + if (!pf_pull_hdr(pbuf, off, hdr, hlen, NULL, NULL, af)) return (0); opt = hdr + sizeof (struct tcphdr); hlen -= sizeof (struct tcphdr); @@ -3980,7 +3976,7 @@ pf_get_wscale(struct mbuf *m, int off, u_int16_t th_off, sa_family_t af) } static u_int16_t -pf_get_mss(struct mbuf *m, int off, u_int16_t th_off, sa_family_t af) +pf_get_mss(pbuf_t *pbuf, int off, u_int16_t th_off, sa_family_t af) { int hlen; u_int8_t hdr[60]; @@ -3990,7 +3986,7 @@ pf_get_mss(struct mbuf *m, int off, u_int16_t th_off, sa_family_t af) hlen = th_off << 2; /* hlen <= sizeof (hdr) */ if (hlen <= (int)sizeof (struct tcphdr)) return (0); - if (!pf_pull_hdr(m, off, hdr, hlen, NULL, NULL, af)) + if (!pf_pull_hdr(pbuf, off, hdr, hlen, NULL, NULL, af)) return (0); opt = hdr + sizeof (struct tcphdr); hlen -= sizeof (struct tcphdr); @@ -4042,7 +4038,7 @@ pf_calc_mss(struct pf_addr *addr, sa_family_t af, u_int16_t offer) dst = (struct sockaddr_in *)(void *)&ro.ro_dst; dst->sin_family = AF_INET; dst->sin_len = sizeof (*dst); - dst->sin_addr = addr->v4; + dst->sin_addr = addr->v4addr; rtalloc(&ro); rt = ro.ro_rt; break; @@ -4054,7 +4050,7 @@ pf_calc_mss(struct pf_addr *addr, sa_family_t af, u_int16_t offer) dst6 = (struct sockaddr_in6 *)(void *)&ro6.ro_dst; dst6->sin6_family = AF_INET6; dst6->sin6_len = sizeof (*dst6); - dst6->sin6_addr = addr->v6; + dst6->sin6_addr = addr->v6addr; rtalloc((struct route *)&ro); rt = ro6.ro_rt; break; @@ -4166,7 +4162,7 @@ pf_tcp_iss(struct pf_pdesc *pd) u_int32_t digest[4]; if (pf_tcp_secret_init == 0) { - read_random(pf_tcp_secret, sizeof (pf_tcp_secret)); + read_frandom(pf_tcp_secret, sizeof (pf_tcp_secret)); MD5Init(&pf_tcp_secret_ctx); MD5Update(&pf_tcp_secret_ctx, pf_tcp_secret, sizeof (pf_tcp_secret)); @@ -4177,11 +4173,11 @@ pf_tcp_iss(struct pf_pdesc *pd) MD5Update(&ctx, (char *)&pd->hdr.tcp->th_sport, sizeof (u_short)); MD5Update(&ctx, (char *)&pd->hdr.tcp->th_dport, sizeof (u_short)); if (pd->af == AF_INET6) { - MD5Update(&ctx, (char *)&pd->src->v6, sizeof (struct in6_addr)); - MD5Update(&ctx, (char *)&pd->dst->v6, sizeof (struct in6_addr)); + MD5Update(&ctx, (char *)&pd->src->v6addr, sizeof (struct in6_addr)); + MD5Update(&ctx, (char *)&pd->dst->v6addr, sizeof (struct in6_addr)); } else { - MD5Update(&ctx, (char *)&pd->src->v4, sizeof (struct in_addr)); - MD5Update(&ctx, (char *)&pd->dst->v4, sizeof (struct in_addr)); + MD5Update(&ctx, (char *)&pd->src->v4addr, sizeof (struct in_addr)); + MD5Update(&ctx, (char *)&pd->dst->v4addr, sizeof (struct in_addr)); } MD5Final((u_char *)digest, &ctx); pf_tcp_iss_off += 4096; @@ -4190,60 +4186,54 @@ pf_tcp_iss(struct pf_pdesc *pd) /* * This routine is called to perform address family translation on the - * inner IP header (that may come as payload) of an ICMP(v4/6) error + * inner IP header (that may come as payload) of an ICMP(v4addr/6) error * response. */ static int -pf_change_icmp_af(struct mbuf *m, int off, +pf_change_icmp_af(pbuf_t *pbuf, int off, struct pf_pdesc *pd, struct pf_pdesc *pd2, struct pf_addr *src, struct pf_addr *dst, sa_family_t af, sa_family_t naf) { - struct mbuf *n = NULL; struct ip *ip4 = NULL; struct ip6_hdr *ip6 = NULL; - int hlen, olen, mlen; + void *hdr; + int hlen, olen; if (af == naf || (af != AF_INET && af != AF_INET6) || (naf != AF_INET && naf != AF_INET6)) return (-1); - /* split the mbuf chain on the inner ip/ip6 header boundary */ - if ((n = m_split(m, off, M_DONTWAIT)) == NULL) - return (-1); - /* old header */ olen = pd2->off - off; /* new header */ hlen = naf == AF_INET ? sizeof(*ip4) : sizeof(*ip6); - /* trim old header */ - m_adj(n, olen); - - /* prepend a new one */ - if (M_PREPEND(n, hlen, M_DONTWAIT, 0) == NULL) + /* Modify the pbuf to accommodate the new header */ + hdr = pbuf_resize_segment(pbuf, off, olen, hlen); + if (hdr == NULL) return (-1); /* translate inner ip/ip6 header */ switch (naf) { case AF_INET: - ip4 = mtod(n, struct ip *); + ip4 = hdr; bzero(ip4, sizeof(*ip4)); ip4->ip_v = IPVERSION; ip4->ip_hl = sizeof(*ip4) >> 2; ip4->ip_len = htons(sizeof(*ip4) + pd2->tot_len - olen); - ip4->ip_id = htons(ip_randomid()); + ip4->ip_id = rfc6864 ? 0 : htons(ip_randomid()); ip4->ip_off = htons(IP_DF); ip4->ip_ttl = pd2->ttl; if (pd2->proto == IPPROTO_ICMPV6) ip4->ip_p = IPPROTO_ICMP; else ip4->ip_p = pd2->proto; - ip4->ip_src = src->v4; - ip4->ip_dst = dst->v4; - ip4->ip_sum = in_cksum(n, ip4->ip_hl << 2); + ip4->ip_src = src->v4addr; + ip4->ip_dst = dst->v4addr; + ip4->ip_sum = pbuf_inet_cksum(pbuf, 0, 0, ip4->ip_hl << 2); break; case AF_INET6: - ip6 = mtod(n, struct ip6_hdr *); + ip6 = hdr; bzero(ip6, sizeof(*ip6)); ip6->ip6_vfc = IPV6_VERSION; ip6->ip6_plen = htons(pd2->tot_len - olen); @@ -4255,8 +4245,8 @@ pf_change_icmp_af(struct mbuf *m, int off, ip6->ip6_hlim = IPV6_DEFHLIM; else ip6->ip6_hlim = pd2->ttl; - ip6->ip6_src = src->v6; - ip6->ip6_dst = dst->v6; + ip6->ip6_src = src->v6addr; + ip6->ip6_dst = dst->v6addr; break; } @@ -4264,11 +4254,6 @@ pf_change_icmp_af(struct mbuf *m, int off, pd2->off += hlen - olen; pd->tot_len += hlen - olen; - /* merge modified inner packet with the original header */ - mlen = n->m_pkthdr.len; - m_cat(m, n); - m->m_pkthdr.len += mlen; - return (0); } @@ -4471,10 +4456,12 @@ pf_translate_icmp_af(int af, void *arg) return (0); } +/* Note: frees pbuf if PF_NAT64 is returned */ static int -pf_nat64_ipv6(struct mbuf *m, int off, struct pf_pdesc *pd) +pf_nat64_ipv6(pbuf_t *pbuf, int off, struct pf_pdesc *pd) { struct ip *ip4; + struct mbuf *m; /* * ip_input asserts for rcvif to be not NULL @@ -4484,17 +4471,13 @@ pf_nat64_ipv6(struct mbuf *m, int off, struct pf_pdesc *pd) * 2. If IPv6 stack in kernel internally generates a * message destined for a synthesized IPv6 end-point. */ - if (m->m_pkthdr.rcvif == NULL) + if (pbuf->pb_ifp == NULL) return (PF_DROP); - /* trim the old header */ - m_adj(m, off); - - /* prepend the new one */ - if (M_PREPEND(m, sizeof(*ip4), M_DONTWAIT, 0) == NULL) + ip4 = (struct ip *)pbuf_resize_segment(pbuf, 0, off, sizeof(*ip4)); + if (ip4 == NULL) return (PF_DROP); - ip4 = mtod(m, struct ip *); ip4->ip_v = 4; ip4->ip_hl = 5; ip4->ip_tos = pd->tos & htonl(0x0ff00000); @@ -4504,81 +4487,88 @@ pf_nat64_ipv6(struct mbuf *m, int off, struct pf_pdesc *pd) ip4->ip_ttl = pd->ttl; ip4->ip_p = pd->proto; ip4->ip_sum = 0; - ip4->ip_src = pd->naddr.v4; - ip4->ip_dst = pd->ndaddr.v4; - ip4->ip_sum = in_cksum(m, ip4->ip_hl << 2); + ip4->ip_src = pd->naddr.v4addr; + ip4->ip_dst = pd->ndaddr.v4addr; + ip4->ip_sum = pbuf_inet_cksum(pbuf, 0, 0, ip4->ip_hl << 2); /* recalculate icmp checksums */ if (pd->proto == IPPROTO_ICMP) { - struct mbuf *mp; struct icmp *icmp; - int moff, hlen = sizeof(*ip4); + int hlen = sizeof(*ip4); - if ((mp = m_pulldown(m, hlen, ICMP_MINLEN, &moff)) == NULL) - return (PF_DROP); + icmp = (struct icmp *)pbuf_contig_segment(pbuf, hlen, + ICMP_MINLEN); + if (icmp == NULL) + return (PF_NAT64); - icmp = (struct icmp *)(void *)(mtod(mp, char *) + moff); icmp->icmp_cksum = 0; - icmp->icmp_cksum = inet_cksum(m, 0, hlen, + icmp->icmp_cksum = pbuf_inet_cksum(pbuf, 0, hlen, ntohs(ip4->ip_len) - hlen); } - ip_input(m); + if ((m = pbuf_to_mbuf(pbuf, TRUE)) != NULL) + ip_input(m); + return (PF_NAT64); } static int -pf_nat64_ipv4(struct mbuf *m, int off, struct pf_pdesc *pd) +pf_nat64_ipv4(pbuf_t *pbuf, int off, struct pf_pdesc *pd) { struct ip6_hdr *ip6; + struct mbuf *m; - if (m->m_pkthdr.rcvif == NULL) + if (pbuf->pb_ifp == NULL) return (PF_DROP); - m_adj(m, off); - if (M_PREPEND(m, sizeof(*ip6), M_DONTWAIT, 0) == NULL) + ip6 = (struct ip6_hdr *)pbuf_resize_segment(pbuf, 0, off, sizeof(*ip6)); + if (ip6 == NULL) return (PF_DROP); - ip6 = mtod(m, struct ip6_hdr *); ip6->ip6_vfc = htonl((6 << 28) | (pd->tos << 20)); ip6->ip6_plen = htons(pd->tot_len - off); ip6->ip6_nxt = pd->proto; ip6->ip6_hlim = pd->ttl; - ip6->ip6_src = pd->naddr.v6; - ip6->ip6_dst = pd->ndaddr.v6; + ip6->ip6_src = pd->naddr.v6addr; + ip6->ip6_dst = pd->ndaddr.v6addr; /* recalculate icmp6 checksums */ if (pd->proto == IPPROTO_ICMPV6) { - struct mbuf *mp; struct icmp6_hdr *icmp6; - int moff, hlen = sizeof(*ip6); + int hlen = sizeof(*ip6); - if ((mp = m_pulldown(m, hlen, sizeof(*icmp6), &moff)) == NULL) + icmp6 = (struct icmp6_hdr *)pbuf_contig_segment(pbuf, hlen, + sizeof(*icmp6)); + if (icmp6 == NULL) return (PF_DROP); - icmp6 = (struct icmp6_hdr *)(void *)(mtod(mp, char *) + moff); icmp6->icmp6_cksum = 0; - icmp6->icmp6_cksum = inet6_cksum(m, IPPROTO_ICMPV6, hlen, - ntohs(ip6->ip6_plen)); + icmp6->icmp6_cksum = pbuf_inet6_cksum(pbuf, + IPPROTO_ICMPV6, hlen, + ntohs(ip6->ip6_plen)); } else if (pd->proto == IPPROTO_UDP) { - struct mbuf *mp; struct udphdr *uh; - int moff, hlen = sizeof(*ip6); - if ((mp = m_pulldown(m, hlen, sizeof(*uh), &moff)) == NULL) + int hlen = sizeof(*ip6); + + uh = (struct udphdr *)pbuf_contig_segment(pbuf, hlen, + sizeof(*uh)); + if (uh == NULL) return (PF_DROP); - uh = (struct udphdr *)(void *)(mtod(mp, char *) + moff); + if (uh->uh_sum == 0) - uh->uh_sum = inet6_cksum(m, IPPROTO_UDP, hlen, - ntohs(ip6->ip6_plen)); + uh->uh_sum = pbuf_inet6_cksum(pbuf, IPPROTO_UDP, + hlen, ntohs(ip6->ip6_plen)); } - ip6_input(m); + if ((m = pbuf_to_mbuf(pbuf, TRUE)) != NULL) + ip6_input(m); + return (PF_NAT64); } static int pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, - struct pfi_kif *kif, struct mbuf *m, int off, void *h, + struct pfi_kif *kif, pbuf_t *pbuf, int off, void *h, struct pf_pdesc *pd, struct pf_rule **am, struct pf_ruleset **rsm, struct ifqueue *ifq) { @@ -4605,7 +4595,7 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, union pf_state_xport bxport, bdxport, nxport, sxport, dxport; struct pf_state_key psk; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (direction == PF_IN && pf_check_congestion(ifq)) { REASON_SET(&reason, PFRES_CONGEST); @@ -4686,9 +4676,9 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, nxport = dxport; /* check packet for BINAT/NAT/RDR */ - if ((nr = pf_get_translation_aux(pd, m, off, direction, kif, &nsn, - saddr, &sxport, daddr, &dxport, &nxport)) != - NULL) { + if ((nr = pf_get_translation_aux(pd, pbuf, off, direction, kif, &nsn, + saddr, &sxport, daddr, &dxport, &nxport + )) != NULL) { int ua; u_int16_t dport; @@ -4765,8 +4755,8 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, * Will cross the bridge when it comes. */ if (PF_ANEQ(saddr, &pd->naddr, pd->af)) { - pf_change_a(&saddr->v4.s_addr, pd->ip_sum, - pd->naddr.v4.s_addr, 0); + pf_change_a(&saddr->v4addr.s_addr, pd->ip_sum, + pd->naddr.v4addr.s_addr, 0); pd->hdr.icmp->icmp_cksum = pf_cksum_fixup( pd->hdr.icmp->icmp_cksum, sxport.port, nxport.port, 0); @@ -4774,8 +4764,8 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, } if (PF_ANEQ(daddr, &pd->ndaddr, pd->af)) { - pf_change_a(&daddr->v4.s_addr, pd->ip_sum, - pd->ndaddr.v4.s_addr, 0); + pf_change_a(&daddr->v4addr.s_addr, pd->ip_sum, + pd->ndaddr.v4addr.s_addr, 0); } ++rewrite; break; @@ -4817,14 +4807,14 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, #if INET case AF_INET: if (PF_ANEQ(saddr, &pd->naddr, pd->af)) { - pf_change_a(&saddr->v4.s_addr, + pf_change_a(&saddr->v4addr.s_addr, pd->ip_sum, - pd->naddr.v4.s_addr, 0); + pd->naddr.v4addr.s_addr, 0); } if (PF_ANEQ(daddr, &pd->ndaddr, pd->af)) { - pf_change_a(&daddr->v4.s_addr, + pf_change_a(&daddr->v4addr.s_addr, pd->ip_sum, - pd->ndaddr.v4.s_addr, 0); + pd->ndaddr.v4addr.s_addr, 0); } break; #endif /* INET */ @@ -4847,13 +4837,13 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, #if INET case AF_INET: if (PF_ANEQ(saddr, &pd->naddr, pd->af)) { - pf_change_a(&saddr->v4.s_addr, - pd->ip_sum, pd->naddr.v4.s_addr, 0); + pf_change_a(&saddr->v4addr.s_addr, + pd->ip_sum, pd->naddr.v4addr.s_addr, 0); } if (PF_ANEQ(daddr, &pd->ndaddr, pd->af)) { - pf_change_a(&daddr->v4.s_addr, + pf_change_a(&daddr->v4addr.s_addr, pd->ip_sum, - pd->ndaddr.v4.s_addr, 0); + pd->ndaddr.v4addr.s_addr, 0); } break; #endif /* INET */ @@ -4900,6 +4890,7 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, r = NULL; pd->nat_rule = nr; pd->af = pd->naf; + } else { } if (nr && nr->tag > 0) @@ -4971,11 +4962,11 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, r = TAILQ_NEXT(r, entries); else if (r->prob && r->prob <= (RandomULong() % (UINT_MAX - 1) + 1)) r = TAILQ_NEXT(r, entries); - else if (r->match_tag && !pf_match_tag(m, r, pd->pf_mtag, &tag)) + else if (r->match_tag && !pf_match_tag(r, pd->pf_mtag, &tag)) r = TAILQ_NEXT(r, entries); else if (r->os_fingerprint != PF_OSFP_ANY && (pd->proto != IPPROTO_TCP || !pf_osfp_match( - pf_osfp_fingerprint(pd, m, off, th), + pf_osfp_fingerprint(pd, pbuf, off, th), r->os_fingerprint))) r = TAILQ_NEXT(r, entries); else { @@ -5010,15 +5001,14 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, if (rewrite < off + hdrlen) rewrite = off + hdrlen; - m = pf_lazy_makewritable(pd, m, rewrite); - if (!m) { + if (pf_lazy_makewritable(pd, pbuf, rewrite) == NULL) { REASON_SET(&reason, PFRES_MEMORY); return (PF_DROP); } - m_copyback(m, off, hdrlen, pd->hdr.any); + pbuf_copy_back(pbuf, off, hdrlen, pd->hdr.any); } - PFLOG_PACKET(kif, h, m, pd->af, direction, reason, + PFLOG_PACKET(kif, h, pbuf, pd->af, direction, reason, r->log ? r : nr, a, ruleset, pd); } @@ -5060,9 +5050,9 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, switch (af) { #if INET case AF_INET: - pf_change_a(&saddr->v4.s_addr, + pf_change_a(&saddr->v4addr.s_addr, pd->ip_sum, - pd->baddr.v4.s_addr, 0); + pd->baddr.v4addr.s_addr, 0); break; #endif /* INET */ #if INET6 @@ -5078,9 +5068,9 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, switch (af) { #if INET case AF_INET: - pf_change_a(&saddr->v4.s_addr, + pf_change_a(&saddr->v4addr.s_addr, pd->ip_sum, - pd->baddr.v4.s_addr, 0); + pd->baddr.v4addr.s_addr, 0); break; #endif /* INET */ #if INET6 @@ -5094,9 +5084,9 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, default: switch (af) { case AF_INET: - pf_change_a(&saddr->v4.s_addr, + pf_change_a(&saddr->v4addr.s_addr, pd->ip_sum, - pd->baddr.v4.s_addr, 0); + pd->baddr.v4addr.s_addr, 0); break; case AF_INET6: PF_ACPY(saddr, &pd->baddr, af); @@ -5136,9 +5126,9 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, switch (af) { #if INET case AF_INET: - pf_change_a(&daddr->v4.s_addr, + pf_change_a(&daddr->v4addr.s_addr, pd->ip_sum, - pd->bdaddr.v4.s_addr, 0); + pd->bdaddr.v4addr.s_addr, 0); break; #endif /* INET */ #if INET6 @@ -5153,9 +5143,9 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, switch (af) { #if INET case AF_INET: - pf_change_a(&daddr->v4.s_addr, + pf_change_a(&daddr->v4addr.s_addr, pd->ip_sum, - pd->bdaddr.v4.s_addr, 0); + pd->bdaddr.v4addr.s_addr, 0); break; #endif /* INET */ #if INET6 @@ -5169,9 +5159,9 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, default: switch (af) { case AF_INET: - pf_change_a(&daddr->v4.s_addr, + pf_change_a(&daddr->v4addr.s_addr, pd->ip_sum, - pd->bdaddr.v4.s_addr, 0); + pd->bdaddr.v4addr.s_addr, 0); break; #if INET6 case AF_INET6: @@ -5195,19 +5185,19 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, switch (pd->af) { case AF_INET: - h4 = mtod(m, struct ip *); + h4 = pbuf->pb_data; len = ntohs(h4->ip_len) - off; break; #if INET6 case AF_INET6: - h6 = mtod(m, struct ip6_hdr *); + h6 = pbuf->pb_data; len = ntohs(h6->ip6_plen) - (off - sizeof (*h6)); break; #endif /* INET6 */ } - if (pf_check_proto_cksum(m, off, len, IPPROTO_TCP, + if (pf_check_proto_cksum(pbuf, off, len, IPPROTO_TCP, pd->af)) REASON_SET(&reason, PFRES_PROTCKSUM); else { @@ -5223,17 +5213,18 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, } else if (pd->proto != IPPROTO_ICMP && pd->af == AF_INET && pd->proto != IPPROTO_ESP && pd->proto != IPPROTO_AH && r->return_icmp) - pf_send_icmp(m, r->return_icmp >> 8, + pf_send_icmp(pbuf, r->return_icmp >> 8, r->return_icmp & 255, pd->af, r); else if (pd->proto != IPPROTO_ICMPV6 && af == AF_INET6 && pd->proto != IPPROTO_ESP && pd->proto != IPPROTO_AH && r->return_icmp6) - pf_send_icmp(m, r->return_icmp6 >> 8, + pf_send_icmp(pbuf, r->return_icmp6 >> 8, r->return_icmp6 & 255, pd->af, r); } - if (r->action == PF_DROP) + if (r->action == PF_DROP) { return (PF_DROP); + } /* prepare state key, for flowhash and/or the state (if created) */ bzero(&psk, sizeof (psk)); @@ -5268,7 +5259,7 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, * NAT64 requires protocol translation between ICMPv4 * and ICMPv6. TCP and UDP do not require protocol * translation. To avoid adding complexity just to - * handle ICMP(v4/v6), we always lookup for + * handle ICMP(v4addr/v6addr), we always lookup for * proto = IPPROTO_ICMP on both LAN and WAN side */ psk.proto = IPPROTO_ICMP; @@ -5310,7 +5301,7 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, * NAT64 requires protocol translation between ICMPv4 * and ICMPv6. TCP and UDP do not require protocol * translation. To avoid adding complexity just to - * handle ICMP(v4/v6), we always lookup for + * handle ICMP(v4addr/v6addr), we always lookup for * proto = IPPROTO_ICMP on both LAN and WAN side */ psk.proto = IPPROTO_ICMP; @@ -5382,7 +5373,7 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, pd->pktflags &= ~PKTF_FLOW_ADV; } - if (pf_tag_packet(m, pd->pf_mtag, tag, rtableid, pd)) { + if (pf_tag_packet(pbuf, pd->pf_mtag, tag, rtableid, pd)) { REASON_SET(&reason, PFRES_MEMORY); return (PF_DROP); } @@ -5396,14 +5387,15 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction, struct pf_ike_hdr ike; if (pd->proto == IPPROTO_UDP) { - size_t plen = m->m_pkthdr.len - off - sizeof (*uh); + size_t plen = pbuf->pb_packet_len - off - sizeof(*uh); if (ntohs(uh->uh_sport) == PF_IKE_PORT && ntohs(uh->uh_dport) == PF_IKE_PORT && plen >= PF_IKE_PACKET_MINSIZE) { if (plen > PF_IKE_PACKET_MINSIZE) plen = PF_IKE_PACKET_MINSIZE; - m_copydata(m, off + sizeof (*uh), plen, &ike); + pbuf_copy_data(pbuf, off + sizeof (*uh), plen, + &ike); } } @@ -5507,7 +5499,7 @@ cleanup: s->src.seqdiff = 0; if (th->th_flags & TH_SYN) { s->src.seqhi++; - s->src.wscale = pf_get_wscale(m, off, + s->src.wscale = pf_get_wscale(pbuf, off, th->th_off, af); } s->src.max_win = MAX(ntohs(th->th_win), 1); @@ -5569,7 +5561,7 @@ cleanup: } if (pd->proto == IPPROTO_TCP) { if ((pd->flags & PFDESC_TCP_NORM) && - pf_normalize_tcp_init(m, off, pd, th, &s->src, + pf_normalize_tcp_init(pbuf, off, pd, th, &s->src, &s->dst)) { REASON_SET(&reason, PFRES_MEMORY); pf_src_tree_remove_state(s); @@ -5578,7 +5570,7 @@ cleanup: return (PF_DROP); } if ((pd->flags & PFDESC_TCP_NORM) && s->src.scrub && - pf_normalize_tcp_stateful(m, off, pd, &reason, + pf_normalize_tcp_stateful(pbuf, off, pd, &reason, th, s, &s->src, &s->dst, &rewrite)) { /* This really shouldn't happen!!! */ DPFPRINTF(PF_DEBUG_URGENT, @@ -5595,12 +5587,17 @@ cleanup: /* allocate state key and import values from psk */ if ((sk = pf_alloc_state_key(s, &psk)) == NULL) { REASON_SET(&reason, PFRES_MEMORY); + /* + * XXXSCW: This will leak the freshly-allocated + * state structure 's'. Although it should + * eventually be aged-out and removed. + */ goto cleanup; } pf_set_rt_ifp(s, saddr, af); /* needs s->state_key set */ - m = pd->mp; + pbuf = pd->mp; // XXXSCW: Why? if (sk->app_state == 0) { switch (pd->proto) { @@ -5668,8 +5665,9 @@ cleanup: STATE_DEC_COUNTERS(s); pool_put(&pf_state_pl, s); return (PF_DROP); - } else + } else { *sm = s; + } if (tag > 0) { pf_tag_ref(tag); s->tag = tag; @@ -5696,7 +5694,7 @@ cleanup: } s->src.seqhi = htonl(random()); /* Find mss option */ - mss = pf_get_mss(m, off, th->th_off, af); + mss = pf_get_mss(pbuf, off, th->th_off, af); mss = pf_calc_mss(saddr, af, mss); mss = pf_calc_mss(daddr, af, mss); s->src.mss = mss; @@ -5729,7 +5727,7 @@ cleanup: REASON_SET(&reason, PFRES_MEMORY); return (PF_DROP); } - m = pd->mp; + pbuf = pd->mp; // XXXSCW: Why? } } } @@ -5739,23 +5737,60 @@ cleanup: if (rewrite < off + hdrlen) rewrite = off + hdrlen; - m = pf_lazy_makewritable(pd, pd->mp, rewrite); - if (!m) { + if (pf_lazy_makewritable(pd, pd->mp, rewrite) == NULL) { REASON_SET(&reason, PFRES_MEMORY); return (PF_DROP); } - m_copyback(m, off, hdrlen, pd->hdr.any); + pbuf_copy_back(pbuf, off, hdrlen, pd->hdr.any); if (af == AF_INET6 && pd->naf == AF_INET) - return pf_nat64_ipv6(m, off, pd); + return pf_nat64_ipv6(pbuf, off, pd); else if (af == AF_INET && pd->naf == AF_INET6) - return pf_nat64_ipv4(m, off, pd); + return pf_nat64_ipv4(pbuf, off, pd); } return (PF_PASS); } +boolean_t is_nlc_enabled_glb = FALSE; + +static inline boolean_t +pf_is_dummynet_enabled(void) +{ +#if DUMMYNET + if (__probable(!PF_IS_ENABLED)) + return (FALSE); + + if (__probable(!DUMMYNET_LOADED)) + return (FALSE); + + if (__probable(TAILQ_EMPTY(pf_main_ruleset. + rules[PF_RULESET_DUMMYNET].active.ptr))) + return (FALSE); + + return (TRUE); +#else + return (FALSE); +#endif /* DUMMYNET */ +} + +boolean_t +pf_is_nlc_enabled(void) +{ +#if DUMMYNET + if (__probable(!pf_is_dummynet_enabled())) + return (FALSE); + + if (__probable(!is_nlc_enabled_glb)) + return (FALSE); + + return (TRUE); +#else + return (FALSE); +#endif /* DUMMYNET */ +} + #if DUMMYNET /* * When pf_test_dummynet() returns PF_PASS, the rule matching parameter "rm" @@ -5766,9 +5801,9 @@ cleanup: */ static int pf_test_dummynet(struct pf_rule **rm, int direction, struct pfi_kif *kif, - struct mbuf **m0, struct pf_pdesc *pd, struct ip_fw_args *fwa) + pbuf_t **pbuf0, struct pf_pdesc *pd, struct ip_fw_args *fwa) { - struct mbuf *m = *m0; + pbuf_t *pbuf = *pbuf0; struct pf_rule *am = NULL; struct pf_ruleset *rsm = NULL; struct pf_addr *saddr = pd->src, *daddr = pd->dst; @@ -5787,12 +5822,9 @@ pf_test_dummynet(struct pf_rule **rm, int direction, struct pfi_kif *kif, struct pf_rule *prev_matching_rule = fwa ? fwa->fwa_pf_rule : NULL; int found_prev_rule = (prev_matching_rule) ? 0 : 1; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); - if (!DUMMYNET_LOADED) - return (PF_PASS); - - if (TAILQ_EMPTY(pf_main_ruleset.rules[PF_RULESET_DUMMYNET].active.ptr)) + if (!pf_is_dummynet_enabled()) return (PF_PASS); bzero(&dnflow, sizeof(dnflow)); @@ -5896,7 +5928,7 @@ pf_test_dummynet(struct pf_rule **rm, int direction, struct pfi_kif *kif, r = TAILQ_NEXT(r, entries); else if (r->prob && r->prob <= (RandomULong() % (UINT_MAX - 1) + 1)) r = TAILQ_NEXT(r, entries); - else if (r->match_tag && !pf_match_tag(m, r, pd->pf_mtag, &tag)) + else if (r->match_tag && !pf_match_tag(r, pd->pf_mtag, &tag)) r = TAILQ_NEXT(r, entries); else { /* @@ -5937,7 +5969,7 @@ pf_test_dummynet(struct pf_rule **rm, int direction, struct pfi_kif *kif, REASON_SET(&reason, PFRES_DUMMYNET); if (r->log) { - PFLOG_PACKET(kif, h, m, af, direction, reason, r, + PFLOG_PACKET(kif, h, pbuf, af, direction, reason, r, a, ruleset, pd); } @@ -5949,13 +5981,14 @@ pf_test_dummynet(struct pf_rule **rm, int direction, struct pfi_kif *kif, return (PF_PASS); } - if (pf_tag_packet(m, pd->pf_mtag, tag, rtableid, pd)) { + if (pf_tag_packet(pbuf, pd->pf_mtag, tag, rtableid, pd)) { REASON_SET(&reason, PFRES_MEMORY); return (PF_DROP); } if (r->dnpipe && ip_dn_io_ptr != NULL) { + struct mbuf *m; int dirndx = (direction == PF_OUT); r->packets[dirndx]++; @@ -5968,13 +6001,13 @@ pf_test_dummynet(struct pf_rule **rm, int direction, struct pfi_kif *kif, switch (af) { case AF_INET: dnflow.fwa_id.addr_type = 4; - dnflow.fwa_id.src_ip = ntohl(saddr->v4.s_addr); - dnflow.fwa_id.dst_ip = ntohl(daddr->v4.s_addr); + dnflow.fwa_id.src_ip = ntohl(saddr->v4addr.s_addr); + dnflow.fwa_id.dst_ip = ntohl(daddr->v4addr.s_addr); break; case AF_INET6: dnflow.fwa_id.addr_type = 6; - dnflow.fwa_id.src_ip6 = saddr->v6; - dnflow.fwa_id.dst_ip6 = saddr->v6; + dnflow.fwa_id.src_ip6 = saddr->v6addr; + dnflow.fwa_id.dst_ip6 = saddr->v6addr; break; } @@ -5998,7 +6031,7 @@ pf_test_dummynet(struct pf_rule **rm, int direction, struct pfi_kif *kif, } if (af == AF_INET) { - struct ip *iphdr = mtod(m, struct ip *); + struct ip *iphdr = pbuf->pb_data; NTOHS(iphdr->ip_len); NTOHS(iphdr->ip_off); } @@ -6006,18 +6039,20 @@ pf_test_dummynet(struct pf_rule **rm, int direction, struct pfi_kif *kif, * Don't need to unlock pf_lock as NET_THREAD_HELD_PF * allows for recursive behavior */ - ip_dn_io_ptr(m, - dnflow.fwa_cookie, - af == AF_INET ? - direction == PF_IN ? DN_TO_IP_IN : DN_TO_IP_OUT : - direction == PF_IN ? DN_TO_IP6_IN : DN_TO_IP6_OUT, - &dnflow, DN_CLIENT_PF); + m = pbuf_to_mbuf(pbuf, TRUE); + if (m != NULL) { + ip_dn_io_ptr(m, + dnflow.fwa_cookie, (af == AF_INET) ? + ((direction==PF_IN) ? DN_TO_IP_IN : DN_TO_IP_OUT) : + ((direction==PF_IN) ? DN_TO_IP6_IN : DN_TO_IP6_OUT), + &dnflow, DN_CLIENT_PF); + } /* * The packet is siphoned out by dummynet so return a NULL - * mbuf so the caller can still return success. + * pbuf so the caller can still return success. */ - *m0 = NULL; + *pbuf0 = NULL; return (PF_PASS); } @@ -6028,7 +6063,7 @@ pf_test_dummynet(struct pf_rule **rm, int direction, struct pfi_kif *kif, static int pf_test_fragment(struct pf_rule **rm, int direction, struct pfi_kif *kif, - struct mbuf *m, void *h, struct pf_pdesc *pd, struct pf_rule **am, + pbuf_t *pbuf, void *h, struct pf_pdesc *pd, struct pf_rule **am, struct pf_ruleset **rsm) { #pragma unused(h) @@ -6081,7 +6116,7 @@ pf_test_fragment(struct pf_rule **rm, int direction, struct pfi_kif *kif, r = TAILQ_NEXT(r, entries); else if (r->prob && r->prob <= (RandomULong() % (UINT_MAX - 1) + 1)) r = TAILQ_NEXT(r, entries); - else if (r->match_tag && !pf_match_tag(m, r, pd->pf_mtag, &tag)) + else if (r->match_tag && !pf_match_tag(r, pd->pf_mtag, &tag)) r = TAILQ_NEXT(r, entries); else { if (r->anchor == NULL) { @@ -6107,13 +6142,13 @@ pf_test_fragment(struct pf_rule **rm, int direction, struct pfi_kif *kif, REASON_SET(&reason, PFRES_MATCH); if (r->log) - PFLOG_PACKET(kif, h, m, af, direction, reason, r, a, ruleset, + PFLOG_PACKET(kif, h, pbuf, af, direction, reason, r, a, ruleset, pd); if (r->action != PF_PASS) return (PF_DROP); - if (pf_tag_packet(m, pd->pf_mtag, tag, -1, NULL)) { + if (pf_tag_packet(pbuf, pd->pf_mtag, tag, -1, NULL)) { REASON_SET(&reason, PFRES_MEMORY); return (PF_DROP); } @@ -6138,7 +6173,7 @@ pf_pptp_handler(struct pf_state *s, int direction, int off, u_int8_t *pac_state; u_int8_t *pns_state; enum { PF_PPTP_PASS, PF_PPTP_INSERT_GRE, PF_PPTP_REMOVE_GRE } op; - struct mbuf *m; + pbuf_t *pbuf; struct pf_state_key *sk; struct pf_state_key *gsk; struct pf_app_state *gas; @@ -6150,12 +6185,12 @@ pf_pptp_handler(struct pf_state *s, int direction, int off, if (gs) gs->expire = pf_time_second(); - m = pd->mp; - plen = min(sizeof (cm), m->m_pkthdr.len - off); + pbuf = pd->mp; + plen = min(sizeof (cm), pbuf->pb_packet_len - off); if (plen < PF_PPTP_CTRL_MSG_MINSIZE) return; tlen = plen - PF_PPTP_CTRL_MSG_MINSIZE; - m_copydata(m, off, plen, &cm); + pbuf_copy_data(pbuf, off, plen, &cm); if (ntohl(cm.hdr.magic) != PF_PPTP_MAGIC_NUMBER) return; @@ -6399,14 +6434,13 @@ pf_pptp_handler(struct pf_state *s, int direction, int off, gsk->lan.xport.call_id, gsk->gwy.xport.call_id, 0); } - m = pf_lazy_makewritable(pd, m, off + plen); - if (!m) { + if (pf_lazy_makewritable(pd, pbuf, off + plen) == NULL) { pptps->grev1_state = NULL; STATE_DEC_COUNTERS(gs); pool_put(&pf_state_pl, gs); return; } - m_copyback(m, off, plen, &cm); + pbuf_copy_back(pbuf, off, plen, &cm); } switch (op) { @@ -6499,7 +6533,7 @@ pf_ike_compare(struct pf_app_state *a, struct pf_app_state *b) } static int -pf_do_nat64(struct pf_state_key *sk, struct pf_pdesc *pd, struct mbuf *m, +pf_do_nat64(struct pf_state_key *sk, struct pf_pdesc *pd, pbuf_t *pbuf, int off) { if (pd->af == AF_INET) { @@ -6510,7 +6544,7 @@ pf_do_nat64(struct pf_state_key *sk, struct pf_pdesc *pd, struct mbuf *m, pd->naddr = sk->gwy.addr; pd->ndaddr = sk->ext_gwy.addr; } - return (pf_nat64_ipv4(m, off, pd)); + return (pf_nat64_ipv4(pbuf, off, pd)); } else if (pd->af == AF_INET6) { if (pd->af != sk->af_lan) { @@ -6520,14 +6554,14 @@ pf_do_nat64(struct pf_state_key *sk, struct pf_pdesc *pd, struct mbuf *m, pd->naddr = sk->gwy.addr; pd->ndaddr = sk->ext_gwy.addr; } - return (pf_nat64_ipv6(m, off, pd)); + return (pf_nat64_ipv6(pbuf, off, pd)); } return (PF_DROP); } static int pf_test_state_tcp(struct pf_state **state, int direction, struct pfi_kif *kif, - struct mbuf *m, int off, void *h, struct pf_pdesc *pd, + pbuf_t *pbuf, int off, void *h, struct pf_pdesc *pd, u_short *reason) { #pragma unused(h) @@ -6710,7 +6744,7 @@ pf_test_state_tcp(struct pf_state **state, int direction, struct pfi_kif *kif, if ((pd->flags & PFDESC_TCP_NORM || dst->scrub) && src->scrub == NULL) { - if (pf_normalize_tcp_init(m, off, pd, th, src, dst)) { + if (pf_normalize_tcp_init(pbuf, off, pd, th, src, dst)) { REASON_SET(reason, PFRES_MEMORY); return (PF_DROP); } @@ -6734,8 +6768,8 @@ pf_test_state_tcp(struct pf_state **state, int direction, struct pfi_kif *kif, if (th->th_flags & TH_SYN) { end++; if (dst->wscale & PF_WSCALE_FLAG) { - src->wscale = pf_get_wscale(m, off, th->th_off, - pd->af); + src->wscale = pf_get_wscale(pbuf, off, + th->th_off, pd->af); if (src->wscale & PF_WSCALE_FLAG) { /* * Remove scale factor from initial @@ -6836,13 +6870,13 @@ pf_test_state_tcp(struct pf_state **state, int direction, struct pfi_kif *kif, * options anyway. */ if (dst->seqdiff && (th->th_off << 2) > (int)sizeof (struct tcphdr)) { - copyback = pf_modulate_sack(m, off, pd, th, dst); + copyback = pf_modulate_sack(pbuf, off, pd, th, dst); if (copyback == -1) { REASON_SET(reason, PFRES_MEMORY); return (PF_DROP); } - m = pd->mp; + pbuf = pd->mp; // XXXSCW: Why? } @@ -6861,11 +6895,11 @@ pf_test_state_tcp(struct pf_state **state, int direction, struct pfi_kif *kif, /* Require an exact/+1 sequence match on resets when possible */ if (dst->scrub || src->scrub) { - if (pf_normalize_tcp_stateful(m, off, pd, reason, th, + if (pf_normalize_tcp_stateful(pbuf, off, pd, reason, th, *state, src, dst, ©back)) return (PF_DROP); - m = pd->mp; + pbuf = pd->mp; // XXXSCW: Why? } /* update max window */ @@ -6961,10 +6995,10 @@ pf_test_state_tcp(struct pf_state **state, int direction, struct pfi_kif *kif, } if (dst->scrub || src->scrub) { - if (pf_normalize_tcp_stateful(m, off, pd, reason, th, + if (pf_normalize_tcp_stateful(pbuf, off, pd, reason, th, *state, src, dst, ©back)) return (PF_DROP); - m = pd->mp; + pbuf = pd->mp; // XXXSCW: Why? } /* update max window */ @@ -7040,7 +7074,7 @@ pf_test_state_tcp(struct pf_state **state, int direction, struct pfi_kif *kif, REASON_SET(reason, PFRES_MEMORY); return (PF_DROP); } - m = pd->mp; + pbuf = pd->mp; // XXXSCW: Why? } /* translate source/destination address, if necessary */ @@ -7092,24 +7126,23 @@ pf_test_state_tcp(struct pf_state **state, int direction, struct pfi_kif *kif, } if (copyback) { - m = pf_lazy_makewritable(pd, m, copyback); - if (!m) { + if (pf_lazy_makewritable(pd, pbuf, copyback) == NULL) { REASON_SET(reason, PFRES_MEMORY); return (PF_DROP); } /* Copyback sequence modulation or stateful scrub changes */ - m_copyback(m, off, sizeof (*th), th); + pbuf_copy_back(pbuf, off, sizeof (*th), th); if (sk->af_lan != sk->af_gwy) - return (pf_do_nat64(sk, pd, m, off)); + return (pf_do_nat64(sk, pd, pbuf, off)); } return (PF_PASS); } static int pf_test_state_udp(struct pf_state **state, int direction, struct pfi_kif *kif, - struct mbuf *m, int off, void *h, struct pf_pdesc *pd, u_short *reason) + pbuf_t *pbuf, int off, void *h, struct pf_pdesc *pd, u_short *reason) { #pragma unused(h) struct pf_state_peer *src, *dst; @@ -7150,7 +7183,7 @@ pf_test_state_udp(struct pf_state **state, int direction, struct pfi_kif *kif, if (ntohs(uh->uh_sport) == PF_IKE_PORT && ntohs(uh->uh_dport) == PF_IKE_PORT) { struct pf_ike_hdr ike; - size_t plen = m->m_pkthdr.len - off - sizeof (*uh); + size_t plen = pbuf->pb_packet_len - off - sizeof (*uh); if (plen < PF_IKE_PACKET_MINSIZE) { DPFPRINTF(PF_DEBUG_MISC, ("pf: IKE message too small.\n")); @@ -7159,7 +7192,7 @@ pf_test_state_udp(struct pf_state **state, int direction, struct pfi_kif *kif, if (plen > sizeof (ike)) plen = sizeof (ike); - m_copydata(m, off + sizeof (*uh), plen, &ike); + pbuf_copy_data(pbuf, off + sizeof (*uh), plen, &ike); if (ike.initiator_cookie) { key.app_state = &as; @@ -7254,13 +7287,12 @@ pf_test_state_udp(struct pf_state **state, int direction, struct pfi_kif *kif, REASON_SET(reason, PFRES_MEMORY); return (PF_DROP); } - m = pd->mp; + pbuf = pd->mp; // XXXSCW: Why? } /* translate source/destination address, if necessary */ if (STATE_TRANSLATE(sk)) { - m = pf_lazy_makewritable(pd, m, off + sizeof (*uh)); - if (!m) { + if (pf_lazy_makewritable(pd, pbuf, off + sizeof (*uh)) == NULL) { REASON_SET(reason, PFRES_MEMORY); return (PF_DROP); } @@ -7309,22 +7341,22 @@ pf_test_state_udp(struct pf_state **state, int direction, struct pfi_kif *kif, } } - m_copyback(m, off, sizeof (*uh), uh); + pbuf_copy_back(pbuf, off, sizeof (*uh), uh); if (sk->af_lan != sk->af_gwy) - return (pf_do_nat64(sk, pd, m, off)); + return (pf_do_nat64(sk, pd, pbuf, off)); } return (PF_PASS); } static int pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, - struct mbuf *m, int off, void *h, struct pf_pdesc *pd, u_short *reason) + pbuf_t *pbuf, int off, void *h, struct pf_pdesc *pd, u_short *reason) { #pragma unused(h) struct pf_addr *saddr = pd->src, *daddr = pd->dst; - struct in_addr srcv4_inaddr = saddr->v4; - u_int16_t icmpid = 0, *icmpsum; - u_int8_t icmptype; + struct in_addr srcv4_inaddr = saddr->v4addr; + u_int16_t icmpid = 0, *icmpsum = NULL; + u_int8_t icmptype = 0; int state_icmp = 0; struct pf_state_key_cmp key; struct pf_state_key *sk; @@ -7374,7 +7406,7 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, * NAT64 requires protocol translation between ICMPv4 * and ICMPv6. TCP and UDP do not require protocol * translation. To avoid adding complexity just to - * handle ICMP(v4/v6), we always lookup for + * handle ICMP(v4addr/v6addr), we always lookup for * proto = IPPROTO_ICMP on both LAN and WAN side */ key.proto = IPPROTO_ICMP; @@ -7404,20 +7436,19 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, switch (pd->af) { #if INET case AF_INET: - pf_change_a(&saddr->v4.s_addr, + pf_change_a(&saddr->v4addr.s_addr, pd->ip_sum, - sk->gwy.addr.v4.s_addr, 0); + sk->gwy.addr.v4addr.s_addr, 0); pd->hdr.icmp->icmp_cksum = pf_cksum_fixup( pd->hdr.icmp->icmp_cksum, icmpid, sk->gwy.xport.port, 0); pd->hdr.icmp->icmp_id = sk->gwy.xport.port; - m = pf_lazy_makewritable(pd, m, - off + ICMP_MINLEN); - if (!m) + if (pf_lazy_makewritable(pd, pbuf, + off + ICMP_MINLEN) == NULL) return (PF_DROP); - m_copyback(m, off, ICMP_MINLEN, + pbuf_copy_back(pbuf, off, ICMP_MINLEN, pd->hdr.icmp); break; #endif /* INET */ @@ -7426,11 +7457,11 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, pf_change_a6(saddr, &pd->hdr.icmp6->icmp6_cksum, &sk->gwy.addr, 0); - m = pf_lazy_makewritable(pd, m, - off + sizeof (struct icmp6_hdr)); - if (!m) + if (pf_lazy_makewritable(pd, NULL, + off + sizeof (struct icmp6_hdr)) == + NULL) return (PF_DROP); - m_copyback(m, off, + pbuf_copy_back(pbuf, off, sizeof (struct icmp6_hdr), pd->hdr.icmp6); break; @@ -7449,9 +7480,9 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, } else { - pf_change_a(&daddr->v4.s_addr, + pf_change_a(&daddr->v4addr.s_addr, pd->ip_sum, - sk->lan.addr.v4.s_addr, 0); + sk->lan.addr.v4addr.s_addr, 0); pd->hdr.icmp->icmp_cksum = pf_cksum_fixup( @@ -7462,15 +7493,14 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, sk->lan.xport.port; } - m = pf_lazy_makewritable(pd, m, - off + ICMP_MINLEN); - if (!m) + if (pf_lazy_makewritable(pd, pbuf, + off + ICMP_MINLEN) == NULL) return (PF_DROP); - m_copyback(m, off, ICMP_MINLEN, + pbuf_copy_back(pbuf, off, ICMP_MINLEN, pd->hdr.icmp); if (sk->af_lan != sk->af_gwy) - return (pf_do_nat64(sk, pd, m, - off)); + return (pf_do_nat64(sk, pd, + pbuf, off)); break; #endif /* INET */ #if INET6 @@ -7486,16 +7516,16 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, &pd->hdr.icmp6->icmp6_cksum, &sk->lan.addr, 0); } - m = pf_lazy_makewritable(pd, m, - off + sizeof (struct icmp6_hdr)); - if (!m) + if (pf_lazy_makewritable(pd, pbuf, + off + sizeof (struct icmp6_hdr)) == + NULL) return (PF_DROP); - m_copyback(m, off, + pbuf_copy_back(pbuf, off, sizeof (struct icmp6_hdr), pd->hdr.icmp6); if (sk->af_lan != sk->af_gwy) - return (pf_do_nat64(sk, pd, m, - off)); + return (pf_do_nat64(sk, pd, + pbuf, off)); break; #endif /* INET6 */ } @@ -7529,7 +7559,7 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, /* offset of h2 in mbuf chain */ ipoff2 = off + ICMP_MINLEN; - if (!pf_pull_hdr(m, ipoff2, &h2, sizeof (h2), + if (!pf_pull_hdr(pbuf, ipoff2, &h2, sizeof (h2), NULL, reason, pd2.af)) { DPFPRINTF(PF_DEBUG_MISC, ("pf: ICMP error message too short " @@ -7560,7 +7590,7 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, case AF_INET6: ipoff2 = off + sizeof (struct icmp6_hdr); - if (!pf_pull_hdr(m, ipoff2, &h2_6, sizeof (h2_6), + if (!pf_pull_hdr(pbuf, ipoff2, &h2_6, sizeof (h2_6), NULL, reason, pd2.af)) { DPFPRINTF(PF_DEBUG_MISC, ("pf: ICMP error message too short " @@ -7568,8 +7598,8 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, return (PF_DROP); } pd2.proto = h2_6.ip6_nxt; - pd2.src = (struct pf_addr *)&h2_6.ip6_src; - pd2.dst = (struct pf_addr *)&h2_6.ip6_dst; + pd2.src = (struct pf_addr *)(uintptr_t)&h2_6.ip6_src; + pd2.dst = (struct pf_addr *)(uintptr_t)&h2_6.ip6_dst; pd2.ip_sum = NULL; off2 = ipoff2 + sizeof (h2_6); do { @@ -7588,7 +7618,7 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, /* get next header and header length */ struct ip6_ext opt6; - if (!pf_pull_hdr(m, off2, &opt6, + if (!pf_pull_hdr(pbuf, off2, &opt6, sizeof (opt6), NULL, reason, pd2.af)) { DPFPRINTF(PF_DEBUG_MISC, @@ -7627,7 +7657,7 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, * expected. Don't access any TCP header fields after * th_seq, an ackskew test is not possible. */ - if (!pf_pull_hdr(m, off2, &th, 8, NULL, reason, + if (!pf_pull_hdr(pbuf, off2, &th, 8, NULL, reason, pd2.af)) { DPFPRINTF(PF_DEBUG_MISC, ("pf: ICMP error message too short " @@ -7711,12 +7741,12 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, if (pf_translate_icmp_af(pd->naf, pd->hdr.icmp)) return (PF_DROP); - m = - pf_lazy_makewritable(pd, m, off2 + 8); - if (!m) + + if (pf_lazy_makewritable(pd, pbuf, + off2 + 8) == NULL) return (PF_DROP); - m_copyback(m, pd->off, + pbuf_copy_back(pbuf, pd->off, sizeof(struct icmp6_hdr), pd->hdr.icmp6); @@ -7724,7 +7754,7 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, * translate inner ip header within the * ICMP message */ - if (pf_change_icmp_af(m, ipoff2, pd, + if (pf_change_icmp_af(pbuf, ipoff2, pd, &pd2, &saddr2->addr, &daddr2->addr, pd->af, pd->naf)) return (PF_DROP); @@ -7750,7 +7780,7 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, daddr2->xport.port, 0, pd2.af, pd2.naf, 0); - m_copyback(m, pd2.off, 8, &th); + pbuf_copy_back(pbuf, pd2.off, 8, &th); /* translate outer ip header */ PF_ACPY(&pd->naddr, &daddr2->addr, @@ -7761,10 +7791,10 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, memcpy(&pd->naddr.addr32[3], &srcv4_inaddr, sizeof(pd->naddr.addr32[3])); - return (pf_nat64_ipv4(m, off, + return (pf_nat64_ipv4(pbuf, off, pd)); } else { - return (pf_nat64_ipv6(m, off, + return (pf_nat64_ipv6(pbuf, off, pd)); } } @@ -7785,29 +7815,29 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, } if (copyback) { - m = pf_lazy_makewritable(pd, m, off2 + 8); - if (!m) + if (pf_lazy_makewritable(pd, pbuf, off2 + 8) == + NULL) return (PF_DROP); switch (pd2.af) { #if INET case AF_INET: - m_copyback(m, off, ICMP_MINLEN, + pbuf_copy_back(pbuf, off, ICMP_MINLEN, pd->hdr.icmp); - m_copyback(m, ipoff2, sizeof (h2), + pbuf_copy_back(pbuf, ipoff2, sizeof(h2), &h2); break; #endif /* INET */ #if INET6 case AF_INET6: - m_copyback(m, off, + pbuf_copy_back(pbuf, off, sizeof (struct icmp6_hdr), pd->hdr.icmp6); - m_copyback(m, ipoff2, sizeof (h2_6), - &h2_6); + pbuf_copy_back(pbuf, ipoff2, + sizeof (h2_6), &h2_6); break; #endif /* INET6 */ } - m_copyback(m, off2, 8, &th); + pbuf_copy_back(pbuf, off2, 8, &th); } return (PF_PASS); @@ -7815,7 +7845,7 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, case IPPROTO_UDP: { struct udphdr uh; int dx, action; - if (!pf_pull_hdr(m, off2, &uh, sizeof (uh), + if (!pf_pull_hdr(pbuf, off2, &uh, sizeof (uh), NULL, reason, pd2.af)) { DPFPRINTF(PF_DEBUG_MISC, ("pf: ICMP error message too short " @@ -7842,8 +7872,8 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, if (ntohs(uh.uh_sport) == PF_IKE_PORT && ntohs(uh.uh_dport) == PF_IKE_PORT) { struct pf_ike_hdr ike; - size_t plen = - m->m_pkthdr.len - off2 - sizeof (uh); + size_t plen = pbuf->pb_packet_len - off2 - + sizeof (uh); if (direction == PF_IN && plen < 8 /* PF_IKE_PACKET_MINSIZE */) { DPFPRINTF(PF_DEBUG_MISC, ("pf: " @@ -7854,7 +7884,8 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, if (plen > sizeof (ike)) plen = sizeof (ike); - m_copydata(m, off + sizeof (uh), plen, &ike); + pbuf_copy_data(pbuf, off + sizeof (uh), plen, + &ike); key.app_state = &as; as.compare_lan_ext = pf_ike_compare; @@ -7914,12 +7945,11 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, if (pf_translate_icmp_af(pd->naf, pd->hdr.icmp)) return (PF_DROP); - m = - pf_lazy_makewritable(pd, m, off2 + 8); - if (!m) + if (pf_lazy_makewritable(pd, pbuf, + off2 + 8) == NULL) return (PF_DROP); - m_copyback(m, pd->off, + pbuf_copy_back(pbuf, pd->off, sizeof(struct icmp6_hdr), pd->hdr.icmp6); @@ -7927,7 +7957,7 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, * translate inner ip header within the * ICMP message */ - if (pf_change_icmp_af(m, ipoff2, pd, + if (pf_change_icmp_af(pbuf, ipoff2, pd, &pd2, &saddr2->addr, &daddr2->addr, pd->af, pd->naf)) return (PF_DROP); @@ -7953,7 +7983,8 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, daddr2->xport.port, 0, pd2.af, pd2.naf, 0); - m_copyback(m, pd2.off, sizeof(uh), &uh); + pbuf_copy_back(pbuf, pd2.off, + sizeof(uh), &uh); /* translate outer ip header */ PF_ACPY(&pd->naddr, &daddr2->addr, @@ -7964,10 +7995,10 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, memcpy(&pd->naddr.addr32[3], &srcv4_inaddr, sizeof(pd->naddr.addr32[3])); - return (pf_nat64_ipv4(m, off, + return (pf_nat64_ipv4(pbuf, off, pd)); } else { - return (pf_nat64_ipv6(m, off, + return (pf_nat64_ipv6(pbuf, off, pd)); } } @@ -7984,29 +8015,29 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, pd2.ip_sum, icmpsum, pd->ip_sum, 1, pd2.af); } - m = pf_lazy_makewritable(pd, m, - off2 + sizeof (uh)); - if (!m) + if (pf_lazy_makewritable(pd, pbuf, + off2 + sizeof (uh)) == NULL) return (PF_DROP); switch (pd2.af) { #if INET case AF_INET: - m_copyback(m, off, ICMP_MINLEN, + pbuf_copy_back(pbuf, off, ICMP_MINLEN, pd->hdr.icmp); - m_copyback(m, ipoff2, sizeof (h2), &h2); + pbuf_copy_back(pbuf, ipoff2, + sizeof (h2), &h2); break; #endif /* INET */ #if INET6 case AF_INET6: - m_copyback(m, off, + pbuf_copy_back(pbuf, off, sizeof (struct icmp6_hdr), pd->hdr.icmp6); - m_copyback(m, ipoff2, sizeof (h2_6), - &h2_6); + pbuf_copy_back(pbuf, ipoff2, + sizeof (h2_6), &h2_6); break; #endif /* INET6 */ } - m_copyback(m, off2, sizeof (uh), &uh); + pbuf_copy_back(pbuf, off2, sizeof (uh), &uh); } return (PF_PASS); @@ -8015,7 +8046,7 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, case IPPROTO_ICMP: { struct icmp iih; - if (!pf_pull_hdr(m, off2, &iih, ICMP_MINLEN, + if (!pf_pull_hdr(pbuf, off2, &iih, ICMP_MINLEN, NULL, reason, pd2.af)) { DPFPRINTF(PF_DEBUG_MISC, ("pf: ICMP error message too short i" @@ -8055,13 +8086,13 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, pd2.ip_sum, icmpsum, pd->ip_sum, 0, AF_INET); } - m = pf_lazy_makewritable(pd, m, - off2 + ICMP_MINLEN); - if (!m) + if (pf_lazy_makewritable(pd, pbuf, + off2 + ICMP_MINLEN) == NULL) return (PF_DROP); - m_copyback(m, off, ICMP_MINLEN, pd->hdr.icmp); - m_copyback(m, ipoff2, sizeof (h2), &h2); - m_copyback(m, off2, ICMP_MINLEN, &iih); + pbuf_copy_back(pbuf, off, ICMP_MINLEN, + pd->hdr.icmp); + pbuf_copy_back(pbuf, ipoff2, sizeof (h2), &h2); + pbuf_copy_back(pbuf, off2, ICMP_MINLEN, &iih); } return (PF_PASS); @@ -8071,7 +8102,7 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, case IPPROTO_ICMPV6: { struct icmp6_hdr iih; - if (!pf_pull_hdr(m, off2, &iih, + if (!pf_pull_hdr(pbuf, off2, &iih, sizeof (struct icmp6_hdr), NULL, reason, pd2.af)) { DPFPRINTF(PF_DEBUG_MISC, ("pf: ICMP error message too short " @@ -8111,15 +8142,15 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, pd2.ip_sum, icmpsum, pd->ip_sum, 0, AF_INET6); } - m = pf_lazy_makewritable(pd, m, off2 + - sizeof (struct icmp6_hdr)); - if (!m) + if (pf_lazy_makewritable(pd, pbuf, off2 + + sizeof (struct icmp6_hdr)) == NULL) return (PF_DROP); - m_copyback(m, off, sizeof (struct icmp6_hdr), - pd->hdr.icmp6); - m_copyback(m, ipoff2, sizeof (h2_6), &h2_6); - m_copyback(m, off2, sizeof (struct icmp6_hdr), - &iih); + pbuf_copy_back(pbuf, off, + sizeof (struct icmp6_hdr), pd->hdr.icmp6); + pbuf_copy_back(pbuf, ipoff2, sizeof (h2_6), + &h2_6); + pbuf_copy_back(pbuf, off2, + sizeof (struct icmp6_hdr), &iih); } return (PF_PASS); @@ -8159,22 +8190,32 @@ pf_test_state_icmp(struct pf_state **state, int direction, struct pfi_kif *kif, switch (pd2.af) { #if INET case AF_INET: - m = pf_lazy_makewritable(pd, m, - ipoff2 + sizeof (h2)); - if (!m) + if (pf_lazy_makewritable(pd, pbuf, + ipoff2 + sizeof (h2)) == NULL) return (PF_DROP); + /* + * <XXXSCW> + * Xnu was missing the following... + */ + pbuf_copy_back(pbuf, off, ICMP_MINLEN, + pd->hdr.icmp); + pbuf_copy_back(pbuf, ipoff2, + sizeof(h2), &h2); + break; + /* + * </XXXSCW> + */ #endif /* INET */ #if INET6 case AF_INET6: - m = pf_lazy_makewritable(pd, m, - ipoff2 + sizeof (h2_6)); - if (!m) + if (pf_lazy_makewritable(pd, pbuf, + ipoff2 + sizeof (h2_6)) == NULL) return (PF_DROP); - m_copyback(m, off, - sizeof (struct icmp6_hdr), - pd->hdr.icmp6); - m_copyback(m, ipoff2, sizeof (h2_6), - &h2_6); + pbuf_copy_back(pbuf, off, + sizeof (struct icmp6_hdr), + pd->hdr.icmp6); + pbuf_copy_back(pbuf, ipoff2, + sizeof (h2_6), &h2_6); break; #endif /* INET6 */ } @@ -8194,7 +8235,6 @@ pf_test_state_grev1(struct pf_state **state, int direction, struct pf_state_peer *dst; struct pf_state_key_cmp key; struct pf_grev1_hdr *grev1 = pd->hdr.grev1; - struct mbuf *m; key.app_state = 0; key.proto = IPPROTO_GRE; @@ -8247,9 +8287,9 @@ pf_test_state_grev1(struct pf_state **state, int direction, switch (pd->af) { #if INET case AF_INET: - pf_change_a(&pd->src->v4.s_addr, + pf_change_a(&pd->src->v4addr.s_addr, pd->ip_sum, - (*state)->state_key->gwy.addr.v4.s_addr, 0); + (*state)->state_key->gwy.addr.v4addr.s_addr, 0); break; #endif /* INET */ #if INET6 @@ -8265,9 +8305,9 @@ pf_test_state_grev1(struct pf_state **state, int direction, switch (pd->af) { #if INET case AF_INET: - pf_change_a(&pd->dst->v4.s_addr, + pf_change_a(&pd->dst->v4addr.s_addr, pd->ip_sum, - (*state)->state_key->lan.addr.v4.s_addr, 0); + (*state)->state_key->lan.addr.v4addr.s_addr, 0); break; #endif /* INET */ #if INET6 @@ -8279,10 +8319,10 @@ pf_test_state_grev1(struct pf_state **state, int direction, } } - m = pf_lazy_makewritable(pd, pd->mp, off + sizeof (*grev1)); - if (!m) + if (pf_lazy_makewritable(pd, pd->mp, off + sizeof (*grev1)) == + NULL) return (PF_DROP); - m_copyback(m, off, sizeof (*grev1), grev1); + pbuf_copy_back(pd->mp, off, sizeof (*grev1), grev1); } return (PF_PASS); @@ -8418,9 +8458,9 @@ pf_test_state_esp(struct pf_state **state, int direction, struct pfi_kif *kif, switch (pd->af) { #if INET case AF_INET: - pf_change_a(&pd->src->v4.s_addr, + pf_change_a(&pd->src->v4addr.s_addr, pd->ip_sum, - (*state)->state_key->gwy.addr.v4.s_addr, 0); + (*state)->state_key->gwy.addr.v4addr.s_addr, 0); break; #endif /* INET */ #if INET6 @@ -8434,9 +8474,9 @@ pf_test_state_esp(struct pf_state **state, int direction, struct pfi_kif *kif, switch (pd->af) { #if INET case AF_INET: - pf_change_a(&pd->dst->v4.s_addr, + pf_change_a(&pd->dst->v4addr.s_addr, pd->ip_sum, - (*state)->state_key->lan.addr.v4.s_addr, 0); + (*state)->state_key->lan.addr.v4addr.s_addr, 0); break; #endif /* INET */ #if INET6 @@ -8504,9 +8544,9 @@ pf_test_state_other(struct pf_state **state, int direction, struct pfi_kif *kif, switch (pd->af) { #if INET case AF_INET: - pf_change_a(&pd->src->v4.s_addr, + pf_change_a(&pd->src->v4addr.s_addr, pd->ip_sum, - (*state)->state_key->gwy.addr.v4.s_addr, + (*state)->state_key->gwy.addr.v4addr.s_addr, 0); break; #endif /* INET */ @@ -8521,9 +8561,9 @@ pf_test_state_other(struct pf_state **state, int direction, struct pfi_kif *kif, switch (pd->af) { #if INET case AF_INET: - pf_change_a(&pd->dst->v4.s_addr, + pf_change_a(&pd->dst->v4addr.s_addr, pd->ip_sum, - (*state)->state_key->lan.addr.v4.s_addr, + (*state)->state_key->lan.addr.v4addr.s_addr, 0); break; #endif /* INET */ @@ -8545,13 +8585,13 @@ pf_test_state_other(struct pf_state **state, int direction, struct pfi_kif *kif, * h must be at "ipoff" on the mbuf chain. */ void * -pf_pull_hdr(struct mbuf *m, int off, void *p, int len, +pf_pull_hdr(pbuf_t *pbuf, int off, void *p, int len, u_short *actionp, u_short *reasonp, sa_family_t af) { switch (af) { #if INET case AF_INET: { - struct ip *h = mtod(m, struct ip *); + struct ip *h = pbuf->pb_data; u_int16_t fragoff = (ntohs(h->ip_off) & IP_OFFMASK) << 3; if (fragoff) { @@ -8563,7 +8603,7 @@ pf_pull_hdr(struct mbuf *m, int off, void *p, int len, } return (NULL); } - if (m->m_pkthdr.len < off + len || + if (pbuf->pb_packet_len < (unsigned)(off + len) || ntohs(h->ip_len) < off + len) { ACTION_SET(actionp, PF_DROP); REASON_SET(reasonp, PFRES_SHORT); @@ -8574,9 +8614,9 @@ pf_pull_hdr(struct mbuf *m, int off, void *p, int len, #endif /* INET */ #if INET6 case AF_INET6: { - struct ip6_hdr *h = mtod(m, struct ip6_hdr *); + struct ip6_hdr *h = pbuf->pb_data; - if (m->m_pkthdr.len < off + len || + if (pbuf->pb_packet_len < (unsigned)(off + len) || (ntohs(h->ip6_plen) + sizeof (struct ip6_hdr)) < (unsigned)(off + len)) { ACTION_SET(actionp, PF_DROP); @@ -8587,7 +8627,7 @@ pf_pull_hdr(struct mbuf *m, int off, void *p, int len, } #endif /* INET6 */ } - m_copydata(m, off, len, p); + pbuf_copy_data(pbuf, off, len, p); return (p); } @@ -8610,14 +8650,14 @@ pf_routable(struct pf_addr *addr, sa_family_t af, struct pfi_kif *kif) dst = satosin(&ro.ro_dst); dst->sin_family = AF_INET; dst->sin_len = sizeof (*dst); - dst->sin_addr = addr->v4; + dst->sin_addr = addr->v4addr; break; #if INET6 case AF_INET6: dst6 = (struct sockaddr_in6 *)&ro.ro_dst; dst6->sin6_family = AF_INET6; dst6->sin6_len = sizeof (*dst6); - dst6->sin6_addr = addr->v6; + dst6->sin6_addr = addr->v6addr; break; #endif /* INET6 */ default: @@ -8656,14 +8696,14 @@ pf_rtlabel_match(struct pf_addr *addr, sa_family_t af, struct pf_addr_wrap *aw) dst = satosin(&ro.ro_dst); dst->sin_family = AF_INET; dst->sin_len = sizeof (*dst); - dst->sin_addr = addr->v4; + dst->sin_addr = addr->v4addr; break; #if INET6 case AF_INET6: dst6 = (struct sockaddr_in6 *)&ro.ro_dst; dst6->sin6_family = AF_INET6; dst6->sin6_len = sizeof (*dst6); - dst6->sin6_addr = addr->v6; + dst6->sin6_addr = addr->v6addr; break; #endif /* INET6 */ default: @@ -8680,7 +8720,7 @@ pf_rtlabel_match(struct pf_addr *addr, sa_family_t af, struct pf_addr_wrap *aw) #if INET static void -pf_route(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp, +pf_route(pbuf_t **pbufp, struct pf_rule *r, int dir, struct ifnet *oifp, struct pf_state *s, struct pf_pdesc *pd) { #pragma unused(pd) @@ -8697,28 +8737,41 @@ pf_route(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp, bzero(&iproute, sizeof (iproute)); - if (m == NULL || *m == NULL || r == NULL || + if (pbufp == NULL || !pbuf_is_valid(*pbufp) || r == NULL || (dir != PF_IN && dir != PF_OUT) || oifp == NULL) panic("pf_route: invalid parameters"); if (pd->pf_mtag->pftag_routed++ > 3) { - m0 = *m; - *m = NULL; + pbuf_destroy(*pbufp); + *pbufp = NULL; + m0 = NULL; goto bad; } - if (r->rt == PF_DUPTO) { - if ((m0 = m_copym(*m, 0, M_COPYALL, M_NOWAIT)) == NULL) - return; - } else { - if ((r->rt == PF_REPLYTO) == (r->direction == dir)) - return; - m0 = *m; + /* + * Since this is something of an edge case and may involve the + * host stack (for routing, at least for now), we convert the + * incoming pbuf into an mbuf. + */ + if (r->rt == PF_DUPTO) + m0 = pbuf_clone_to_mbuf(*pbufp); + else + if ((r->rt == PF_REPLYTO) == (r->direction == dir)) + return; + else { + /* We're going to consume this packet */ + m0 = pbuf_to_mbuf(*pbufp, TRUE); + *pbufp = NULL; } + if (m0 == NULL) + goto bad; + + /* We now have the packet in an mbuf (m0) */ + if (m0->m_len < (int)sizeof (struct ip)) { DPFPRINTF(PF_DEBUG_URGENT, - ("pf_route: m0->m_len < sizeof (struct ip)\n")); + ("pf_route: packet length < sizeof (struct ip)\n")); goto bad; } @@ -8753,13 +8806,13 @@ pf_route(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp, pf_map_addr(AF_INET, r, (struct pf_addr *)&ip->ip_src, &naddr, NULL, &sn); if (!PF_AZERO(&naddr, AF_INET)) - dst->sin_addr.s_addr = naddr.v4.s_addr; + dst->sin_addr.s_addr = naddr.v4addr.s_addr; ifp = r->rpool.cur->kif ? r->rpool.cur->kif->pfik_ifp : NULL; } else { if (!PF_AZERO(&s->rt_addr, AF_INET)) dst->sin_addr.s_addr = - s->rt_addr.v4.s_addr; + s->rt_addr.v4addr.s_addr; ifp = s->rt_kif ? s->rt_kif->pfik_ifp : NULL; } } @@ -8767,13 +8820,13 @@ pf_route(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp, goto bad; if (oifp != ifp) { - if (pf_test(PF_OUT, ifp, &m0, NULL, NULL) != PF_PASS) + if (pf_test_mbuf(PF_OUT, ifp, &m0, NULL, NULL) != PF_PASS) goto bad; else if (m0 == NULL) goto done; if (m0->m_len < (int)sizeof (struct ip)) { DPFPRINTF(PF_DEBUG_URGENT, - ("pf_route: m0->m_len < sizeof (struct ip)\n")); + ("pf_route: packet length < sizeof (struct ip)\n")); goto bad; } ip = mtod(m0, struct ip *); @@ -8840,21 +8893,19 @@ pf_route(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp, ipstat.ips_fragmented++; done: - if (r->rt != PF_DUPTO) - *m = NULL; - ROUTE_RELEASE(&iproute); return; bad: - m_freem(m0); + if (m0) + m_freem(m0); goto done; } #endif /* INET */ #if INET6 static void -pf_route6(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp, +pf_route6(pbuf_t **pbufp, struct pf_rule *r, int dir, struct ifnet *oifp, struct pf_state *s, struct pf_pdesc *pd) { #pragma unused(pd) @@ -8868,25 +8919,36 @@ pf_route6(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp, struct pf_src_node *sn = NULL; int error = 0; - if (m == NULL || *m == NULL || r == NULL || + if (pbufp == NULL || !pbuf_is_valid(*pbufp) || r == NULL || (dir != PF_IN && dir != PF_OUT) || oifp == NULL) panic("pf_route6: invalid parameters"); if (pd->pf_mtag->pftag_routed++ > 3) { - m0 = *m; - *m = NULL; + pbuf_destroy(*pbufp); + *pbufp = NULL; + m0 = NULL; goto bad; } + /* + * Since this is something of an edge case and may involve the + * host stack (for routing, at least for now), we convert the + * incoming pbuf into an mbuf. + */ if (r->rt == PF_DUPTO) { - if ((m0 = m_copym(*m, 0, M_COPYALL, M_NOWAIT)) == NULL) - return; - } else { - if ((r->rt == PF_REPLYTO) == (r->direction == dir)) - return; - m0 = *m; + m0 = pbuf_clone_to_mbuf(*pbufp); + } else + if ((r->rt == PF_REPLYTO) == (r->direction == dir)) + return; + else { + /* We're about to consume this packet */ + m0 = pbuf_to_mbuf(*pbufp, TRUE); + *pbufp = NULL; } + if (m0 == NULL) + goto bad; + if (m0->m_len < (int)sizeof (struct ip6_hdr)) { DPFPRINTF(PF_DEBUG_URGENT, ("pf_route6: m0->m_len < sizeof (struct ip6_hdr)\n")); @@ -8901,7 +8963,7 @@ pf_route6(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp, dst->sin6_len = sizeof (*dst); dst->sin6_addr = ip6->ip6_dst; - /* Cheat. XXX why only in the v6 case??? */ + /* Cheat. XXX why only in the v6addr case??? */ if (r->rt == PF_FASTROUTE) { struct pf_mtag *pf_mtag; @@ -8918,7 +8980,7 @@ pf_route6(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp, goto bad; } if (s == NULL) { - pf_map_addr(AF_INET6, r, (struct pf_addr *)&ip6->ip6_src, + pf_map_addr(AF_INET6, r, (struct pf_addr *)(uintptr_t)&ip6->ip6_src, &naddr, NULL, &sn); if (!PF_AZERO(&naddr, AF_INET6)) PF_ACPY((struct pf_addr *)&dst->sin6_addr, @@ -8934,7 +8996,7 @@ pf_route6(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp, goto bad; if (oifp != ifp) { - if (pf_test6(PF_OUT, ifp, &m0, NULL, NULL) != PF_PASS) + if (pf_test6_mbuf(PF_OUT, ifp, &m0, NULL, NULL) != PF_PASS) goto bad; else if (m0 == NULL) goto done; @@ -8963,12 +9025,11 @@ pf_route6(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp, } done: - if (r->rt != PF_DUPTO) - *m = NULL; return; bad: - m_freem(m0); + if (m0) + m_freem(m0); goto done; } #endif /* INET6 */ @@ -8981,7 +9042,7 @@ bad: * returns 0 when the checksum is valid, otherwise returns 1. */ static int -pf_check_proto_cksum(struct mbuf *m, int off, int len, u_int8_t p, +pf_check_proto_cksum(pbuf_t *pbuf, int off, int len, u_int8_t p, sa_family_t af) { u_int16_t sum; @@ -8995,10 +9056,10 @@ pf_check_proto_cksum(struct mbuf *m, int off, int len, u_int8_t p, * is partially-computed (only 16-bit summation), do it in * software below. */ - if ((m->m_pkthdr.csum_flags & + if ((*pbuf->pb_csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == (CSUM_DATA_VALID | CSUM_PSEUDO_HDR) && - (m->m_pkthdr.csum_data ^ 0xffff) == 0) { + (*pbuf->pb_csum_data ^ 0xffff) == 0) { return (0); } break; @@ -9012,12 +9073,13 @@ pf_check_proto_cksum(struct mbuf *m, int off, int len, u_int8_t p, } if (off < (int)sizeof (struct ip) || len < (int)sizeof (struct udphdr)) return (1); - if (m->m_pkthdr.len < off + len) + if (pbuf->pb_packet_len < (unsigned)(off + len)) return (1); switch (af) { #if INET case AF_INET: if (p == IPPROTO_ICMP) { +#if 0 if (m->m_len < off) return (1); m->m_data += off; @@ -9025,18 +9087,23 @@ pf_check_proto_cksum(struct mbuf *m, int off, int len, u_int8_t p, sum = in_cksum(m, len); m->m_data -= off; m->m_len += off; +#else + if (pbuf->pb_contig_len < (unsigned)off) + return (1); + sum = pbuf_inet_cksum(pbuf, 0, off, len); +#endif } else { - if (m->m_len < (int)sizeof (struct ip)) + if (pbuf->pb_contig_len < (int)sizeof (struct ip)) return (1); - sum = inet_cksum(m, p, off, len); + sum = pbuf_inet_cksum(pbuf, p, off, len); } break; #endif /* INET */ #if INET6 case AF_INET6: - if (m->m_len < (int)sizeof (struct ip6_hdr)) + if (pbuf->pb_contig_len < (int)sizeof (struct ip6_hdr)) return (1); - sum = inet6_cksum(m, p, off, len); + sum = pbuf_inet6_cksum(pbuf, p, off, len); break; #endif /* INET6 */ default: @@ -9067,15 +9134,37 @@ pf_check_proto_cksum(struct mbuf *m, int off, int len, u_int8_t p, #if INET #define PF_APPLE_UPDATE_PDESC_IPv4() \ do { \ - if (m && pd.mp && m != pd.mp) { \ - m = pd.mp; \ - h = mtod(m, struct ip *); \ - pd.pf_mtag = pf_get_mtag(m); \ + if (pbuf && pd.mp && pbuf != pd.mp) { \ + pbuf = pd.mp; \ + h = pbuf->pb_data; \ + pd.pf_mtag = pf_get_mtag_pbuf(pbuf); \ } \ } while (0) int -pf_test(int dir, struct ifnet *ifp, struct mbuf **m0, +pf_test_mbuf(int dir, struct ifnet *ifp, struct mbuf **m0, + struct ether_header *eh, struct ip_fw_args *fwa) +{ + pbuf_t pbuf_store, *pbuf; + int rv; + + pbuf_init_mbuf(&pbuf_store, *m0, (*m0)->m_pkthdr.rcvif); + pbuf = &pbuf_store; + + rv = pf_test(dir, ifp, &pbuf, eh, fwa); + + if (pbuf_is_valid(pbuf)) { + *m0 = pbuf->pb_mbuf; + pbuf->pb_mbuf = NULL; + pbuf_destroy(pbuf); + } else + *m0 = NULL; + + return (rv); +} + +int +pf_test(int dir, struct ifnet *ifp, pbuf_t **pbufp, struct ether_header *eh, struct ip_fw_args *fwa) { #if !DUMMYNET @@ -9083,7 +9172,7 @@ pf_test(int dir, struct ifnet *ifp, struct mbuf **m0, #endif struct pfi_kif *kif; u_short action = PF_PASS, reason = 0, log = 0; - struct mbuf *m = *m0; + pbuf_t *pbuf = *pbufp; struct ip *h = 0; struct pf_rule *a = NULL, *r = &pf_default_rule, *tr, *nr; struct pf_state *s = NULL; @@ -9092,16 +9181,16 @@ pf_test(int dir, struct ifnet *ifp, struct mbuf **m0, struct pf_pdesc pd; int off, dirndx, pqid = 0; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (!pf_status.running) return (PF_PASS); memset(&pd, 0, sizeof (pd)); - if ((pd.pf_mtag = pf_get_mtag(m)) == NULL) { + if ((pd.pf_mtag = pf_get_mtag_pbuf(pbuf)) == NULL) { DPFPRINTF(PF_DEBUG_URGENT, - ("pf_test: pf_get_mtag returned NULL\n")); + ("pf_test: pf_get_mtag_pbuf returned NULL\n")); return (PF_DROP); } @@ -9118,13 +9207,11 @@ pf_test(int dir, struct ifnet *ifp, struct mbuf **m0, if (kif->pfik_flags & PFI_IFLAG_SKIP) return (PF_PASS); - VERIFY(m->m_flags & M_PKTHDR); - /* initialize enough of pd for the done label */ - h = mtod(m, struct ip *); - pd.mp = m; + h = pbuf->pb_data; + pd.mp = pbuf; pd.lmw = 0; - pd.pf_mtag = pf_get_mtag(m); + pd.pf_mtag = pf_get_mtag_pbuf(pbuf); pd.src = (struct pf_addr *)&h->ip_src; pd.dst = (struct pf_addr *)&h->ip_dst; PF_ACPY(&pd.baddr, pd.src, AF_INET); @@ -9138,7 +9225,7 @@ pf_test(int dir, struct ifnet *ifp, struct mbuf **m0, pd.tot_len = ntohs(h->ip_len); pd.eh = eh; - if (m->m_pkthdr.len < (int)sizeof (*h)) { + if (pbuf->pb_packet_len < (int)sizeof (*h)) { action = PF_DROP; REASON_SET(&reason, PFRES_SHORT); log = 1; @@ -9151,8 +9238,7 @@ pf_test(int dir, struct ifnet *ifp, struct mbuf **m0, #endif /* DUMMYNET */ /* We do IP header normalization and packet reassembly here */ - action = pf_normalize_ip(m0, dir, kif, &reason, &pd); - pd.mp = m = *m0; + action = pf_normalize_ip(pbuf, dir, kif, &reason, &pd); if (action != PF_PASS || pd.lmw < 0) { action = PF_DROP; goto done; @@ -9161,8 +9247,8 @@ pf_test(int dir, struct ifnet *ifp, struct mbuf **m0, #if DUMMYNET nonormalize: #endif /* DUMMYNET */ - m = *m0; /* pf_normalize messes with m0 */ - h = mtod(m, struct ip *); + /* pf_normalize can mess with pb_data */ + h = pbuf->pb_data; off = h->ip_hl << 2; if (off < (int)sizeof (*h)) { @@ -9179,20 +9265,20 @@ nonormalize: pd.ip_sum = &h->ip_sum; pd.proto = h->ip_p; pd.proto_variant = 0; - pd.mp = m; + pd.mp = pbuf; pd.lmw = 0; - pd.pf_mtag = pf_get_mtag(m); + pd.pf_mtag = pf_get_mtag_pbuf(pbuf); pd.af = AF_INET; pd.tos = h->ip_tos; pd.ttl = h->ip_ttl; - pd.sc = MBUF_SCIDX(mbuf_get_service_class(m)); + pd.sc = MBUF_SCIDX(pbuf_get_service_class(pbuf)); pd.tot_len = ntohs(h->ip_len); pd.eh = eh; - if (m->m_pkthdr.pkt_flags & PKTF_FLOW_ID) { - pd.flowsrc = m->m_pkthdr.pkt_flowsrc; - pd.flowhash = m->m_pkthdr.pkt_flowid; - pd.pktflags = (m->m_pkthdr.pkt_flags & PKTF_FLOW_MASK); + if (*pbuf->pb_flags & PKTF_FLOW_ID) { + pd.flowsrc = *pbuf->pb_flowsrc; + pd.flowhash = *pbuf->pb_flowid; + pd.pktflags = *pbuf->pb_flags & PKTF_FLOW_MASK; } /* handle fragments that didn't get reassembled by normalization */ @@ -9200,13 +9286,13 @@ nonormalize: pd.flags |= PFDESC_IP_FRAG; #if DUMMYNET /* Traffic goes through dummynet first */ - action = pf_test_dummynet(&r, dir, kif, &m, &pd, fwa); - if (action == PF_DROP || m == NULL) { - *m0 = NULL; + action = pf_test_dummynet(&r, dir, kif, &pbuf, &pd, fwa); + if (action == PF_DROP || pbuf == NULL) { + *pbufp = NULL; return (action); } #endif /* DUMMYNET */ - action = pf_test_fragment(&r, dir, kif, m, h, + action = pf_test_fragment(&r, dir, kif, pbuf, h, &pd, &a, &ruleset); goto done; } @@ -9216,7 +9302,7 @@ nonormalize: case IPPROTO_TCP: { struct tcphdr th; pd.hdr.tcp = &th; - if (!pf_pull_hdr(m, off, &th, sizeof (th), + if (!pf_pull_hdr(pbuf, off, &th, sizeof (th), &action, &reason, AF_INET)) { log = action != PF_PASS; goto done; @@ -9226,19 +9312,19 @@ nonormalize: pqid = 1; #if DUMMYNET /* Traffic goes through dummynet first */ - action = pf_test_dummynet(&r, dir, kif, &m, &pd, fwa); - if (action == PF_DROP || m == NULL) { - *m0 = NULL; + action = pf_test_dummynet(&r, dir, kif, &pbuf, &pd, fwa); + if (action == PF_DROP || pbuf == NULL) { + *pbufp = NULL; return (action); } #endif /* DUMMYNET */ - action = pf_normalize_tcp(dir, kif, m, 0, off, h, &pd); + action = pf_normalize_tcp(dir, kif, pbuf, 0, off, h, &pd); if (pd.lmw < 0) goto done; PF_APPLE_UPDATE_PDESC_IPv4(); if (action == PF_DROP) goto done; - action = pf_test_state_tcp(&s, dir, kif, m, off, h, &pd, + action = pf_test_state_tcp(&s, dir, kif, pbuf, off, h, &pd, &reason); if (action == PF_NAT64) goto done; @@ -9254,7 +9340,7 @@ nonormalize: log = s->log; } else if (s == NULL) action = pf_test_rule(&r, &s, dir, kif, - m, off, h, &pd, &a, &ruleset, NULL); + pbuf, off, h, &pd, &a, &ruleset, NULL); break; } @@ -9262,13 +9348,13 @@ nonormalize: struct udphdr uh; pd.hdr.udp = &uh; - if (!pf_pull_hdr(m, off, &uh, sizeof (uh), + if (!pf_pull_hdr(pbuf, off, &uh, sizeof (uh), &action, &reason, AF_INET)) { log = action != PF_PASS; goto done; } if (uh.uh_dport == 0 || - ntohs(uh.uh_ulen) > m->m_pkthdr.len - off || + ntohs(uh.uh_ulen) > pbuf->pb_packet_len - off || ntohs(uh.uh_ulen) < sizeof (struct udphdr)) { action = PF_DROP; REASON_SET(&reason, PFRES_SHORT); @@ -9276,13 +9362,13 @@ nonormalize: } #if DUMMYNET /* Traffic goes through dummynet first */ - action = pf_test_dummynet(&r, dir, kif, &m, &pd, fwa); - if (action == PF_DROP || m == NULL) { - *m0 = NULL; + action = pf_test_dummynet(&r, dir, kif, &pbuf, &pd, fwa); + if (action == PF_DROP || pbuf == NULL) { + *pbufp = NULL; return (action); } #endif /* DUMMYNET */ - action = pf_test_state_udp(&s, dir, kif, m, off, h, &pd, + action = pf_test_state_udp(&s, dir, kif, pbuf, off, h, &pd, &reason); if (action == PF_NAT64) goto done; @@ -9298,7 +9384,7 @@ nonormalize: log = s->log; } else if (s == NULL) action = pf_test_rule(&r, &s, dir, kif, - m, off, h, &pd, &a, &ruleset, NULL); + pbuf, off, h, &pd, &a, &ruleset, NULL); break; } @@ -9306,20 +9392,20 @@ nonormalize: struct icmp ih; pd.hdr.icmp = &ih; - if (!pf_pull_hdr(m, off, &ih, ICMP_MINLEN, + if (!pf_pull_hdr(pbuf, off, &ih, ICMP_MINLEN, &action, &reason, AF_INET)) { log = action != PF_PASS; goto done; } #if DUMMYNET /* Traffic goes through dummynet first */ - action = pf_test_dummynet(&r, dir, kif, &m, &pd, fwa); - if (action == PF_DROP || m == NULL) { - *m0 = NULL; + action = pf_test_dummynet(&r, dir, kif, &pbuf, &pd, fwa); + if (action == PF_DROP || pbuf == NULL) { + *pbufp = NULL; return (action); } #endif /* DUMMYNET */ - action = pf_test_state_icmp(&s, dir, kif, m, off, h, &pd, + action = pf_test_state_icmp(&s, dir, kif, pbuf, off, h, &pd, &reason); if (action == PF_NAT64) goto done; @@ -9335,7 +9421,7 @@ nonormalize: log = s->log; } else if (s == NULL) action = pf_test_rule(&r, &s, dir, kif, - m, off, h, &pd, &a, &ruleset, NULL); + pbuf, off, h, &pd, &a, &ruleset, NULL); break; } @@ -9343,16 +9429,16 @@ nonormalize: struct pf_esp_hdr esp; pd.hdr.esp = &esp; - if (!pf_pull_hdr(m, off, &esp, sizeof (esp), &action, &reason, + if (!pf_pull_hdr(pbuf, off, &esp, sizeof (esp), &action, &reason, AF_INET)) { log = action != PF_PASS; goto done; } #if DUMMYNET /* Traffic goes through dummynet first */ - action = pf_test_dummynet(&r, dir, kif, &m, &pd, fwa); - if (action == PF_DROP || m == NULL) { - *m0 = NULL; + action = pf_test_dummynet(&r, dir, kif, &pbuf, &pd, fwa); + if (action == PF_DROP || pbuf == NULL) { + *pbufp = NULL; return (action); } #endif /* DUMMYNET */ @@ -9369,30 +9455,30 @@ nonormalize: log = s->log; } else if (s == NULL) action = pf_test_rule(&r, &s, dir, kif, - m, off, h, &pd, &a, &ruleset, NULL); + pbuf, off, h, &pd, &a, &ruleset, NULL); break; } case IPPROTO_GRE: { struct pf_grev1_hdr grev1; pd.hdr.grev1 = &grev1; - if (!pf_pull_hdr(m, off, &grev1, sizeof (grev1), &action, + if (!pf_pull_hdr(pbuf, off, &grev1, sizeof (grev1), &action, &reason, AF_INET)) { log = (action != PF_PASS); goto done; } #if DUMMYNET /* Traffic goes through dummynet first */ - action = pf_test_dummynet(&r, dir, kif, &m, &pd, fwa); - if (action == PF_DROP || m == NULL) { - *m0 = NULL; + action = pf_test_dummynet(&r, dir, kif, &pbuf, &pd, fwa); + if (action == PF_DROP || pbuf == NULL) { + *pbufp = NULL; return (action); } #endif /* DUMMYNET */ if ((ntohs(grev1.flags) & PF_GRE_FLAG_VERSION_MASK) == 1 && ntohs(grev1.protocol_type) == PF_GRE_PPP_ETHERTYPE) { if (ntohs(grev1.payload_length) > - m->m_pkthdr.len - off) { + pbuf->pb_packet_len - off) { action = PF_DROP; REASON_SET(&reason, PFRES_SHORT); goto done; @@ -9410,8 +9496,8 @@ nonormalize: log = s->log; break; } else if (s == NULL) { - action = pf_test_rule(&r, &s, dir, kif, m, off, - h, &pd, &a, &ruleset, NULL); + action = pf_test_rule(&r, &s, dir, kif, pbuf, + off, h, &pd, &a, &ruleset, NULL); if (action == PF_PASS) break; } @@ -9423,9 +9509,9 @@ nonormalize: default: #if DUMMYNET /* Traffic goes through dummynet first */ - action = pf_test_dummynet(&r, dir, kif, &m, &pd, fwa); - if (action == PF_DROP || m == NULL) { - *m0 = NULL; + action = pf_test_dummynet(&r, dir, kif, &pbuf, &pd, fwa); + if (action == PF_DROP || pbuf == NULL) { + *pbufp = NULL; return (action); } #endif /* DUMMYNET */ @@ -9441,18 +9527,18 @@ nonormalize: a = s->anchor.ptr; log = s->log; } else if (s == NULL) - action = pf_test_rule(&r, &s, dir, kif, m, off, h, + action = pf_test_rule(&r, &s, dir, kif, pbuf, off, h, &pd, &a, &ruleset, NULL); break; } done: if (action == PF_NAT64) { - *m0 = NULL; + *pbufp = NULL; return (action); } - *m0 = pd.mp; + *pbufp = pd.mp; PF_APPLE_UPDATE_PDESC_IPv4(); if (action != PF_DROP) { @@ -9468,18 +9554,10 @@ done: if ((s && s->tag) || PF_RTABLEID_IS_VALID(r->rtableid) || (pd.pktflags & PKTF_FLOW_ID)) - (void) pf_tag_packet(m, pd.pf_mtag, s ? s->tag : 0, + (void) pf_tag_packet(pbuf, pd.pf_mtag, s ? s->tag : 0, r->rtableid, &pd); if (action == PF_PASS) { -#if PF_ALTQ - if (altq_allowed && r->qid) { - if (pqid || (pd.tos & IPTOS_LOWDELAY)) - pd.pf_mtag->pftag_qid = r->pqid; - else - pd.pf_mtag->pftag_qid = r->qid; - } -#endif /* PF_ALTQ */ #if PF_ECN /* add hints for ecn */ pd.pf_mtag->pftag_hdr = h; @@ -9488,7 +9566,7 @@ done: pd.pf_mtag->pftag_flags |= PF_TAG_HDR_INET; #endif /* PF_ECN */ /* record protocol */ - m->m_pkthdr.pkt_proto = pd.proto; + *pbuf->pb_proto = pd.proto; /* * connections redirected to loopback should not match sockets @@ -9500,7 +9578,7 @@ done: s->nat_rule.ptr != NULL && (s->nat_rule.ptr->action == PF_RDR || s->nat_rule.ptr->action == PF_BINAT) && - (ntohl(pd.dst->v4.s_addr) >> IN_CLASSA_NSHIFT) + (ntohl(pd.dst->v4addr.s_addr) >> IN_CLASSA_NSHIFT) == IN_LOOPBACKNET) pd.pf_mtag->pftag_flags |= PF_TAG_TRANSLATE_LOCALHOST; } @@ -9514,7 +9592,7 @@ done: lr = s->nat_rule.ptr; else lr = r; - PFLOG_PACKET(kif, h, m, AF_INET, dir, reason, lr, a, ruleset, + PFLOG_PACKET(kif, h, pbuf, AF_INET, dir, reason, lr, a, ruleset, &pd); } @@ -9584,30 +9662,30 @@ done: tr->dst.neg); } - VERIFY(m == NULL || pd.mp == NULL || pd.mp == m); + VERIFY(pbuf == NULL || pd.mp == NULL || pd.mp == pbuf); - if (*m0) { + if (*pbufp) { if (pd.lmw < 0) { REASON_SET(&reason, PFRES_MEMORY); action = PF_DROP; } if (action == PF_DROP) { - m_freem(*m0); - *m0 = NULL; + pbuf_destroy(*pbufp); + *pbufp = NULL; return (PF_DROP); } - *m0 = m; + *pbufp = pbuf; } if (action == PF_SYNPROXY_DROP) { - m_freem(*m0); - *m0 = NULL; + pbuf_destroy(*pbufp); + *pbufp = NULL; action = PF_PASS; } else if (r->rt) - /* pf_route can free the mbuf causing *m0 to become NULL */ - pf_route(m0, r, dir, kif->pfik_ifp, s, &pd); + /* pf_route can free the pbuf causing *pbufp to become NULL */ + pf_route(pbufp, r, dir, kif->pfik_ifp, s, &pd); return (action); } @@ -9616,16 +9694,36 @@ done: #if INET6 #define PF_APPLE_UPDATE_PDESC_IPv6() \ do { \ - if (m && pd.mp && m != pd.mp) { \ - if (n == m) \ - n = pd.mp; \ - m = pd.mp; \ - h = mtod(m, struct ip6_hdr *); \ + if (pbuf && pd.mp && pbuf != pd.mp) { \ + pbuf = pd.mp; \ } \ + h = pbuf->pb_data; \ } while (0) int -pf_test6(int dir, struct ifnet *ifp, struct mbuf **m0, +pf_test6_mbuf(int dir, struct ifnet *ifp, struct mbuf **m0, + struct ether_header *eh, struct ip_fw_args *fwa) +{ + pbuf_t pbuf_store, *pbuf; + int rv; + + pbuf_init_mbuf(&pbuf_store, *m0, (*m0)->m_pkthdr.rcvif); + pbuf = &pbuf_store; + + rv = pf_test6(dir, ifp, &pbuf, eh, fwa); + + if (pbuf_is_valid(pbuf)) { + *m0 = pbuf->pb_mbuf; + pbuf->pb_mbuf = NULL; + pbuf_destroy(pbuf); + } else + *m0 = NULL; + + return (rv); +} + +int +pf_test6(int dir, struct ifnet *ifp, pbuf_t **pbufp, struct ether_header *eh, struct ip_fw_args *fwa) { #if !DUMMYNET @@ -9633,7 +9731,7 @@ pf_test6(int dir, struct ifnet *ifp, struct mbuf **m0, #endif struct pfi_kif *kif; u_short action = PF_PASS, reason = 0, log = 0; - struct mbuf *m = *m0, *n = NULL; + pbuf_t *pbuf = *pbufp; struct ip6_hdr *h; struct pf_rule *a = NULL, *r = &pf_default_rule, *tr, *nr; struct pf_state *s = NULL; @@ -9643,16 +9741,16 @@ pf_test6(int dir, struct ifnet *ifp, struct mbuf **m0, int off, terminal = 0, dirndx, rh_cnt = 0; u_int8_t nxt; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (!pf_status.running) return (PF_PASS); memset(&pd, 0, sizeof (pd)); - if ((pd.pf_mtag = pf_get_mtag(m)) == NULL) { + if ((pd.pf_mtag = pf_get_mtag_pbuf(pbuf)) == NULL) { DPFPRINTF(PF_DEBUG_URGENT, - ("pf_test6: pf_get_mtag returned NULL\n")); + ("pf_test6: pf_get_mtag_pbuf returned NULL\n")); return (PF_DROP); } @@ -9669,17 +9767,15 @@ pf_test6(int dir, struct ifnet *ifp, struct mbuf **m0, if (kif->pfik_flags & PFI_IFLAG_SKIP) return (PF_PASS); - VERIFY(m->m_flags & M_PKTHDR); - - h = mtod(m, struct ip6_hdr *); + h = pbuf->pb_data; nxt = h->ip6_nxt; - off = ((caddr_t)h - m->m_data) + sizeof(struct ip6_hdr); - pd.mp = m; + off = ((caddr_t)h - (caddr_t)pbuf->pb_data) + sizeof(struct ip6_hdr); + pd.mp = pbuf; pd.lmw = 0; - pd.pf_mtag = pf_get_mtag(m); - pd.src = (struct pf_addr *)&h->ip6_src; - pd.dst = (struct pf_addr *)&h->ip6_dst; + pd.pf_mtag = pf_get_mtag_pbuf(pbuf); + pd.src = (struct pf_addr *)(uintptr_t)&h->ip6_src; + pd.dst = (struct pf_addr *)(uintptr_t)&h->ip6_dst; PF_ACPY(&pd.baddr, pd.src, AF_INET6); PF_ACPY(&pd.bdaddr, pd.dst, AF_INET6); pd.ip_sum = NULL; @@ -9688,17 +9784,17 @@ pf_test6(int dir, struct ifnet *ifp, struct mbuf **m0, pd.proto_variant = 0; pd.tos = 0; pd.ttl = h->ip6_hlim; - pd.sc = MBUF_SCIDX(mbuf_get_service_class(m)); + pd.sc = MBUF_SCIDX(pbuf_get_service_class(pbuf)); pd.tot_len = ntohs(h->ip6_plen) + sizeof(struct ip6_hdr); pd.eh = eh; - if (m->m_pkthdr.pkt_flags & PKTF_FLOW_ID) { - pd.flowsrc = m->m_pkthdr.pkt_flowsrc; - pd.flowhash = m->m_pkthdr.pkt_flowid; - pd.pktflags = (m->m_pkthdr.pkt_flags & PKTF_FLOW_MASK); + if (*pbuf->pb_flags & PKTF_FLOW_ID) { + pd.flowsrc = *pbuf->pb_flowsrc; + pd.flowhash = *pbuf->pb_flowid; + pd.pktflags = (*pbuf->pb_flags & PKTF_FLOW_MASK); } - if (m->m_pkthdr.len < (int)sizeof (*h)) { + if (pbuf->pb_packet_len < (int)sizeof (*h)) { action = PF_DROP; REASON_SET(&reason, PFRES_SHORT); log = 1; @@ -9711,8 +9807,7 @@ pf_test6(int dir, struct ifnet *ifp, struct mbuf **m0, #endif /* DUMMYNET */ /* We do IP header normalization and packet reassembly here */ - action = pf_normalize_ip6(m0, dir, kif, &reason, &pd); - pd.mp = m = *m0; + action = pf_normalize_ip6(pbuf, dir, kif, &reason, &pd); if (action != PF_PASS || pd.lmw < 0) { action = PF_DROP; goto done; @@ -9721,7 +9816,7 @@ pf_test6(int dir, struct ifnet *ifp, struct mbuf **m0, #if DUMMYNET nonormalize: #endif /* DUMMYNET */ - h = mtod(m, struct ip6_hdr *); + h = pbuf->pb_data; #if 1 /* @@ -9735,8 +9830,8 @@ nonormalize: } #endif - pd.src = (struct pf_addr *)&h->ip6_src; - pd.dst = (struct pf_addr *)&h->ip6_dst; + pd.src = (struct pf_addr *)(uintptr_t)&h->ip6_src; + pd.dst = (struct pf_addr *)(uintptr_t)&h->ip6_dst; PF_ACPY(&pd.baddr, pd.src, AF_INET6); PF_ACPY(&pd.bdaddr, pd.dst, AF_INET6); pd.ip_sum = NULL; @@ -9746,12 +9841,12 @@ nonormalize: pd.tot_len = ntohs(h->ip6_plen) + sizeof (struct ip6_hdr); pd.eh = eh; - off = ((caddr_t)h - m->m_data) + sizeof (struct ip6_hdr); + off = ((caddr_t)h - (caddr_t)pbuf->pb_data) + sizeof (struct ip6_hdr); pd.proto = h->ip6_nxt; pd.proto_variant = 0; - pd.mp = m; + pd.mp = pbuf; pd.lmw = 0; - pd.pf_mtag = pf_get_mtag(m); + pd.pf_mtag = pf_get_mtag_pbuf(pbuf); do { switch (nxt) { @@ -9759,7 +9854,7 @@ nonormalize: struct ip6_frag ip6f; pd.flags |= PFDESC_IP_FRAG; - if (!pf_pull_hdr(m, off, &ip6f, sizeof ip6f, NULL, + if (!pf_pull_hdr(pbuf, off, &ip6f, sizeof ip6f, NULL, &reason, pd.af)) { DPFPRINTF(PF_DEBUG_MISC, ("pf: IPv6 short fragment header\n")); @@ -9771,14 +9866,15 @@ nonormalize: pd.proto = nxt = ip6f.ip6f_nxt; #if DUMMYNET /* Traffic goes through dummynet first */ - action = pf_test_dummynet(&r, dir, kif, &m, &pd, fwa); - if (action == PF_DROP || m == NULL) { - *m0 = NULL; + action = pf_test_dummynet(&r, dir, kif, &pbuf, &pd, + fwa); + if (action == PF_DROP || pbuf == NULL) { + *pbufp = NULL; return (action); } #endif /* DUMMYNET */ - action = pf_test_fragment(&r, dir, kif, m, h, &pd, &a, - &ruleset); + action = pf_test_fragment(&r, dir, kif, pbuf, h, &pd, + &a, &ruleset); if (action == PF_DROP) { REASON_SET(&reason, PFRES_FRAG); log = 1; @@ -9795,7 +9891,7 @@ nonormalize: /* get next header and header length */ struct ip6_ext opt6; - if (!pf_pull_hdr(m, off, &opt6, sizeof(opt6), + if (!pf_pull_hdr(pbuf, off, &opt6, sizeof(opt6), NULL, &reason, pd.af)) { DPFPRINTF(PF_DEBUG_MISC, ("pf: IPv6 short opt\n")); @@ -9817,9 +9913,6 @@ nonormalize: } } while (!terminal); - /* if there's no routing header, use unmodified mbuf for checksumming */ - if (!n) - n = m; switch (pd.proto) { @@ -9827,7 +9920,7 @@ nonormalize: struct tcphdr th; pd.hdr.tcp = &th; - if (!pf_pull_hdr(m, off, &th, sizeof (th), + if (!pf_pull_hdr(pbuf, off, &th, sizeof (th), &action, &reason, AF_INET6)) { log = action != PF_PASS; goto done; @@ -9835,19 +9928,19 @@ nonormalize: pd.p_len = pd.tot_len - off - (th.th_off << 2); #if DUMMYNET /* Traffic goes through dummynet first */ - action = pf_test_dummynet(&r, dir, kif, &m, &pd, fwa); - if (action == PF_DROP || m == NULL) { - *m0 = NULL; + action = pf_test_dummynet(&r, dir, kif, &pbuf, &pd, fwa); + if (action == PF_DROP || pbuf == NULL) { + *pbufp = NULL; return (action); } #endif /* DUMMYNET */ - action = pf_normalize_tcp(dir, kif, m, 0, off, h, &pd); + action = pf_normalize_tcp(dir, kif, pbuf, 0, off, h, &pd); if (pd.lmw < 0) goto done; PF_APPLE_UPDATE_PDESC_IPv6(); if (action == PF_DROP) goto done; - action = pf_test_state_tcp(&s, dir, kif, m, off, h, &pd, + action = pf_test_state_tcp(&s, dir, kif, pbuf, off, h, &pd, &reason); if (action == PF_NAT64) goto done; @@ -9863,7 +9956,7 @@ nonormalize: log = s->log; } else if (s == NULL) action = pf_test_rule(&r, &s, dir, kif, - m, off, h, &pd, &a, &ruleset, NULL); + pbuf, off, h, &pd, &a, &ruleset, NULL); break; } @@ -9871,13 +9964,13 @@ nonormalize: struct udphdr uh; pd.hdr.udp = &uh; - if (!pf_pull_hdr(m, off, &uh, sizeof (uh), + if (!pf_pull_hdr(pbuf, off, &uh, sizeof (uh), &action, &reason, AF_INET6)) { log = action != PF_PASS; goto done; } if (uh.uh_dport == 0 || - ntohs(uh.uh_ulen) > m->m_pkthdr.len - off || + ntohs(uh.uh_ulen) > pbuf->pb_packet_len - off || ntohs(uh.uh_ulen) < sizeof (struct udphdr)) { action = PF_DROP; REASON_SET(&reason, PFRES_SHORT); @@ -9885,13 +9978,13 @@ nonormalize: } #if DUMMYNET /* Traffic goes through dummynet first */ - action = pf_test_dummynet(&r, dir, kif, &m, &pd, fwa); - if (action == PF_DROP || m == NULL) { - *m0 = NULL; + action = pf_test_dummynet(&r, dir, kif, &pbuf, &pd, fwa); + if (action == PF_DROP || pbuf == NULL) { + *pbufp = NULL; return (action); } #endif /* DUMMYNET */ - action = pf_test_state_udp(&s, dir, kif, m, off, h, &pd, + action = pf_test_state_udp(&s, dir, kif, pbuf, off, h, &pd, &reason); if (action == PF_NAT64) goto done; @@ -9907,7 +10000,7 @@ nonormalize: log = s->log; } else if (s == NULL) action = pf_test_rule(&r, &s, dir, kif, - m, off, h, &pd, &a, &ruleset, NULL); + pbuf, off, h, &pd, &a, &ruleset, NULL); break; } @@ -9915,21 +10008,21 @@ nonormalize: struct icmp6_hdr ih; pd.hdr.icmp6 = &ih; - if (!pf_pull_hdr(m, off, &ih, sizeof (ih), + if (!pf_pull_hdr(pbuf, off, &ih, sizeof (ih), &action, &reason, AF_INET6)) { log = action != PF_PASS; goto done; } #if DUMMYNET /* Traffic goes through dummynet first */ - action = pf_test_dummynet(&r, dir, kif, &m, &pd, fwa); - if (action == PF_DROP || m == NULL) { - *m0 = NULL; + action = pf_test_dummynet(&r, dir, kif, &pbuf, &pd, fwa); + if (action == PF_DROP || pbuf == NULL) { + *pbufp = NULL; return (action); } #endif /* DUMMYNET */ action = pf_test_state_icmp(&s, dir, kif, - m, off, h, &pd, &reason); + pbuf, off, h, &pd, &reason); if (action == PF_NAT64) goto done; if (pd.lmw < 0) @@ -9944,7 +10037,7 @@ nonormalize: log = s->log; } else if (s == NULL) action = pf_test_rule(&r, &s, dir, kif, - m, off, h, &pd, &a, &ruleset, NULL); + pbuf, off, h, &pd, &a, &ruleset, NULL); break; } @@ -9952,16 +10045,16 @@ nonormalize: struct pf_esp_hdr esp; pd.hdr.esp = &esp; - if (!pf_pull_hdr(m, off, &esp, sizeof (esp), &action, &reason, - AF_INET6)) { + if (!pf_pull_hdr(pbuf, off, &esp, sizeof (esp), &action, + &reason, AF_INET6)) { log = action != PF_PASS; goto done; } #if DUMMYNET /* Traffic goes through dummynet first */ - action = pf_test_dummynet(&r, dir, kif, &m, &pd, fwa); - if (action == PF_DROP || m == NULL) { - *m0 = NULL; + action = pf_test_dummynet(&r, dir, kif, &pbuf, &pd, fwa); + if (action == PF_DROP || pbuf == NULL) { + *pbufp = NULL; return (action); } #endif /* DUMMYNET */ @@ -9978,7 +10071,7 @@ nonormalize: log = s->log; } else if (s == NULL) action = pf_test_rule(&r, &s, dir, kif, - m, off, h, &pd, &a, &ruleset, NULL); + pbuf, off, h, &pd, &a, &ruleset, NULL); break; } @@ -9986,23 +10079,23 @@ nonormalize: struct pf_grev1_hdr grev1; pd.hdr.grev1 = &grev1; - if (!pf_pull_hdr(m, off, &grev1, sizeof (grev1), &action, + if (!pf_pull_hdr(pbuf, off, &grev1, sizeof (grev1), &action, &reason, AF_INET6)) { log = (action != PF_PASS); goto done; } #if DUMMYNET /* Traffic goes through dummynet first */ - action = pf_test_dummynet(&r, dir, kif, &m, &pd, fwa); - if (action == PF_DROP || m == NULL) { - *m0 = NULL; + action = pf_test_dummynet(&r, dir, kif, &pbuf, &pd, fwa); + if (action == PF_DROP || pbuf == NULL) { + *pbufp = NULL; return (action); } #endif /* DUMMYNET */ if ((ntohs(grev1.flags) & PF_GRE_FLAG_VERSION_MASK) == 1 && ntohs(grev1.protocol_type) == PF_GRE_PPP_ETHERTYPE) { if (ntohs(grev1.payload_length) > - m->m_pkthdr.len - off) { + pbuf->pb_packet_len - off) { action = PF_DROP; REASON_SET(&reason, PFRES_SHORT); goto done; @@ -10020,8 +10113,8 @@ nonormalize: log = s->log; break; } else if (s == NULL) { - action = pf_test_rule(&r, &s, dir, kif, m, off, - h, &pd, &a, &ruleset, NULL); + action = pf_test_rule(&r, &s, dir, kif, pbuf, + off, h, &pd, &a, &ruleset, NULL); if (action == PF_PASS) break; } @@ -10033,9 +10126,9 @@ nonormalize: default: #if DUMMYNET /* Traffic goes through dummynet first */ - action = pf_test_dummynet(&r, dir, kif, &m, &pd, fwa); - if (action == PF_DROP || m == NULL) { - *m0 = NULL; + action = pf_test_dummynet(&r, dir, kif, &pbuf, &pd, fwa); + if (action == PF_DROP || pbuf == NULL) { + *pbufp = NULL; return (action); } #endif /* DUMMYNET */ @@ -10051,25 +10144,20 @@ nonormalize: a = s->anchor.ptr; log = s->log; } else if (s == NULL) - action = pf_test_rule(&r, &s, dir, kif, m, off, h, + action = pf_test_rule(&r, &s, dir, kif, pbuf, off, h, &pd, &a, &ruleset, NULL); break; } done: if (action == PF_NAT64) { - *m0 = NULL; + *pbufp = NULL; return (action); } - *m0 = pd.mp; + *pbufp = pd.mp; PF_APPLE_UPDATE_PDESC_IPv6(); - if (n != m) { - m_freem(n); - n = NULL; - } - /* handle dangerous IPv6 extension headers. */ if (action != PF_DROP) { if (action == PF_PASS && rh_cnt && @@ -10078,23 +10166,15 @@ done: REASON_SET(&reason, PFRES_IPOPTIONS); log = 1; DPFPRINTF(PF_DEBUG_MISC, - ("pf: dropping packet with dangerous v6 headers\n")); + ("pf: dropping packet with dangerous v6addr headers\n")); } if ((s && s->tag) || PF_RTABLEID_IS_VALID(r->rtableid) || (pd.pktflags & PKTF_FLOW_ID)) - (void) pf_tag_packet(m, pd.pf_mtag, s ? s->tag : 0, + (void) pf_tag_packet(pbuf, pd.pf_mtag, s ? s->tag : 0, r->rtableid, &pd); if (action == PF_PASS) { -#if PF_ALTQ - if (altq_allowed && r->qid) { - if (pd.tos & IPTOS_LOWDELAY) - pd.pf_mtag->pftag_qid = r->pqid; - else - pd.pf_mtag->pftag_qid = r->qid; - } -#endif /* PF_ALTQ */ #if PF_ECN /* add hints for ecn */ pd.pf_mtag->pftag_hdr = h; @@ -10103,13 +10183,13 @@ done: pd.pf_mtag->pftag_flags |= PF_TAG_HDR_INET6; #endif /* PF_ECN */ /* record protocol */ - m->m_pkthdr.pkt_proto = pd.proto; + *pbuf->pb_proto = pd.proto; if (dir == PF_IN && (pd.proto == IPPROTO_TCP || pd.proto == IPPROTO_UDP) && s != NULL && s->nat_rule.ptr != NULL && (s->nat_rule.ptr->action == PF_RDR || s->nat_rule.ptr->action == PF_BINAT) && - IN6_IS_ADDR_LOOPBACK(&pd.dst->v6)) + IN6_IS_ADDR_LOOPBACK(&pd.dst->v6addr)) pd.pf_mtag->pftag_flags |= PF_TAG_TRANSLATE_LOCALHOST; } } @@ -10123,7 +10203,7 @@ done: lr = s->nat_rule.ptr; else lr = r; - PFLOG_PACKET(kif, h, m, AF_INET6, dir, reason, lr, a, ruleset, + PFLOG_PACKET(kif, h, pbuf, AF_INET6, dir, reason, lr, a, ruleset, &pd); } @@ -10201,35 +10281,30 @@ done: /* pf_route6 can free the mbuf causing *m0 to become NULL */ pf_route6(m0, r, dir, kif->pfik_ifp, s, &pd); #else - VERIFY(m == NULL || pd.mp == NULL || pd.mp == m); + VERIFY(pbuf == NULL || pd.mp == NULL || pd.mp == pbuf); - if (*m0) { + if (*pbufp) { if (pd.lmw < 0) { REASON_SET(&reason, PFRES_MEMORY); action = PF_DROP; } if (action == PF_DROP) { - m_freem(*m0); - *m0 = NULL; + pbuf_destroy(*pbufp); + *pbufp = NULL; return (PF_DROP); } - *m0 = m; + *pbufp = pbuf; } if (action == PF_SYNPROXY_DROP) { - m_freem(*m0); - *m0 = NULL; + pbuf_destroy(*pbufp); + *pbufp = NULL; action = PF_PASS; } else if (r->rt) { - if (action == PF_PASS) { - m = *m0; - h = mtod(m, struct ip6_hdr *); - } - /* pf_route6 can free the mbuf causing *m0 to become NULL */ - pf_route6(m0, r, dir, kif->pfik_ifp, s, &pd); + pf_route6(pbufp, r, dir, kif->pfik_ifp, s, &pd); } #endif /* 0 */ @@ -10284,7 +10359,7 @@ pool_get(struct pool *pp, int flags) { void *buf; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (pp->pool_count > pp->pool_limit) { DPFPRINTF(PF_DEBUG_NOISY, @@ -10306,18 +10381,23 @@ pool_get(struct pool *pp, int flags) void pool_put(struct pool *pp, void *v) { - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); zfree(pp->pool_zone, v); VERIFY(pp->pool_count != 0); pp->pool_count--; } +struct pf_mtag * +pf_find_mtag_pbuf(pbuf_t *pbuf) +{ + + return (pbuf->pb_pftag); +} + struct pf_mtag * pf_find_mtag(struct mbuf *m) { - if (!(m->m_flags & M_PKTHDR)) - return (NULL); return (m_pftag(m)); } @@ -10328,6 +10408,12 @@ pf_get_mtag(struct mbuf *m) return (pf_find_mtag(m)); } +struct pf_mtag * +pf_get_mtag_pbuf(pbuf_t *pbuf) +{ + return (pf_find_mtag_pbuf(pbuf)); +} + uint64_t pf_time_second(void) { diff --git a/bsd/net/pf_if.c b/bsd/net/pf_if.c index 05f265677..f67d06a5d 100644 --- a/bsd/net/pf_if.c +++ b/bsd/net/pf_if.c @@ -231,7 +231,7 @@ pfi_attach_ifnet(struct ifnet *ifp) { struct pfi_kif *kif; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); pfi_update++; if ((kif = pfi_kif_get(if_name(ifp))) == NULL) @@ -253,7 +253,7 @@ pfi_detach_ifnet(struct ifnet *ifp) { struct pfi_kif *kif; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if ((kif = (struct pfi_kif *)ifp->if_pf_kif) == NULL) return; @@ -310,7 +310,7 @@ pfi_dynaddr_setup(struct pf_addr_wrap *aw, sa_family_t af) struct pf_ruleset *ruleset = NULL; int rv = 0; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (aw->type != PF_ADDR_DYNIFTL) return (0); @@ -378,7 +378,7 @@ pfi_kif_update(struct pfi_kif *kif) { struct pfi_dynaddr *p; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); /* update all dynaddr */ TAILQ_FOREACH(p, &kif->pfik_dynaddrs, entry) @@ -576,7 +576,7 @@ pfi_kifaddr_update(void *v) { struct pfi_kif *kif = (struct pfi_kif *)v; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); pfi_update++; pfi_kif_update(kif); @@ -595,7 +595,7 @@ pfi_update_status(const char *name, struct pf_status *pfs) struct pfi_kif_cmp key; int i, j, k; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); strlcpy(key.pfik_name, name, sizeof (key.pfik_name)); p = RB_FIND(pfi_ifhead, &pfi_ifs, (struct pfi_kif *)(void *)&key); @@ -627,7 +627,7 @@ pfi_get_ifaces(const char *name, user_addr_t buf, int *size) struct pfi_kif *p, *nextp; int n = 0; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); for (p = RB_MIN(pfi_ifhead, &pfi_ifs); p; p = nextp) { nextp = RB_NEXT(pfi_ifhead, &pfi_ifs, p); @@ -689,7 +689,7 @@ pfi_set_flags(const char *name, int flags) { struct pfi_kif *p; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); RB_FOREACH(p, pfi_ifhead, &pfi_ifs) { if (pfi_skip_if(name, p)) @@ -704,7 +704,7 @@ pfi_clear_flags(const char *name, int flags) { struct pfi_kif *p; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); RB_FOREACH(p, pfi_ifhead, &pfi_ifs) { if (pfi_skip_if(name, p)) diff --git a/bsd/net/pf_ioctl.c b/bsd/net/pf_ioctl.c index 517ee6856..05bb21b56 100644 --- a/bsd/net/pf_ioctl.c +++ b/bsd/net/pf_ioctl.c @@ -86,6 +86,7 @@ #include <net/dlil.h> #include <net/if.h> #include <net/if_types.h> +#include <net/net_api_stats.h> #include <net/route.h> #include <netinet/in.h> @@ -123,15 +124,6 @@ struct ip_fw_args; #include <netinet/in_pcb.h> #endif /* INET6 */ -#if PF_ALTQ -#include <net/altq/altq.h> -#include <net/altq/altq_cbq.h> -#include <net/classq/classq_red.h> -#include <net/classq/classq_rio.h> -#include <net/classq/classq_blue.h> -#include <net/classq/classq_sfb.h> -#endif /* PF_ALTQ */ - #include <dev/random/randomdev.h> #if 0 @@ -167,15 +159,6 @@ static struct pf_pool *pf_get_pool(char *, u_int32_t, u_int8_t, u_int32_t, u_int8_t, u_int8_t, u_int8_t); static void pf_mv_pool(struct pf_palist *, struct pf_palist *); static void pf_empty_pool(struct pf_palist *); -#if PF_ALTQ -static int pf_begin_altq(u_int32_t *); -static int pf_rollback_altq(u_int32_t); -static int pf_commit_altq(u_int32_t); -static int pf_enable_altq(struct pf_altq *); -static int pf_disable_altq(struct pf_altq *); -static void pf_altq_copyin(struct pf_altq *, struct pf_altq *); -static void pf_altq_copyout(struct pf_altq *, struct pf_altq *); -#endif /* PF_ALTQ */ static int pf_begin_rules(u_int32_t *, int, const char *); static int pf_rollback_rules(u_int32_t, int, char *); static int pf_setup_pfsync_matching(struct pf_ruleset *); @@ -235,10 +218,6 @@ static void pf_detach_hooks(void); */ int pf_is_enabled = 0; -#if PF_ALTQ -u_int32_t altq_allowed = 0; -#endif /* PF_ALTQ */ - u_int32_t pf_hash_seed; int16_t pf_nat64_configured = 0; @@ -254,19 +233,10 @@ SLIST_HEAD(list_head, pfioc_kernel_token); static struct list_head token_list_head; struct pf_rule pf_default_rule; -#if PF_ALTQ -static int pf_altq_running; -#endif /* PF_ALTQ */ #define TAGID_MAX 50000 -#if !PF_ALTQ static TAILQ_HEAD(pf_tags, pf_tagname) pf_tags = TAILQ_HEAD_INITIALIZER(pf_tags); -#else /* PF_ALTQ */ -static TAILQ_HEAD(pf_tags, pf_tagname) - pf_tags = TAILQ_HEAD_INITIALIZER(pf_tags), - pf_qids = TAILQ_HEAD_INITIALIZER(pf_qids); -#endif /* PF_ALTQ */ #if (PF_QNAME_SIZE != PF_TAG_NAME_SIZE) #error PF_QNAME_SIZE must be equal to PF_TAG_NAME_SIZE @@ -377,7 +347,7 @@ generate_token(struct proc *p) new_token = _MALLOC(sizeof (struct pfioc_kernel_token), M_TEMP, M_WAITOK|M_ZERO); - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (new_token == NULL) { /* malloc failed! bail! */ @@ -404,7 +374,7 @@ remove_token(struct pfioc_remove_token *tok) { struct pfioc_kernel_token *entry, *tmp; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); SLIST_FOREACH_SAFE(entry, &token_list_head, next, tmp) { if (tok->token_value == entry->token.token_value) { @@ -425,7 +395,7 @@ invalidate_all_tokens(void) { struct pfioc_kernel_token *entry, *tmp; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); SLIST_FOREACH_SAFE(entry, &token_list_head, next, tmp) { SLIST_REMOVE(&token_list_head, entry, pfioc_kernel_token, next); @@ -462,10 +432,6 @@ pfinit(void) "pfstatekeypl", NULL); pool_init(&pf_app_state_pl, sizeof (struct pf_app_state), 0, 0, 0, "pfappstatepl", NULL); -#if PF_ALTQ - pool_init(&pf_altq_pl, sizeof (struct pf_altq), 0, 0, 0, "pfaltqpl", - NULL); -#endif /* PF_ALTQ */ pool_init(&pf_pooladdr_pl, sizeof (struct pf_pooladdr), 0, 0, 0, "pfpooladdrpl", NULL); pfr_initialize(); @@ -484,21 +450,6 @@ pfinit(void) pf_init_ruleset(&pf_main_ruleset); TAILQ_INIT(&pf_pabuf); TAILQ_INIT(&state_list); -#if PF_ALTQ - TAILQ_INIT(&pf_altqs[0]); - TAILQ_INIT(&pf_altqs[1]); - pf_altqs_active = &pf_altqs[0]; - pf_altqs_inactive = &pf_altqs[1]; - - PE_parse_boot_argn("altq", &altq_allowed, sizeof (altq_allowed)); - - _CASSERT(ALTRQ_PURGE == CLASSQRQ_PURGE); - _CASSERT(ALTRQ_PURGE_SC == CLASSQRQ_PURGE_SC); - _CASSERT(ALTRQ_EVENT == CLASSQRQ_EVENT); - - _CASSERT(ALTDQ_REMOVE == CLASSQDQ_REMOVE); - _CASSERT(ALTDQ_POLL == CLASSQDQ_POLL); -#endif /* PF_ALTQ */ _CASSERT((SC_BE & SCIDX_MASK) == SCIDX_BE); _CASSERT((SC_BK_SYS & SCIDX_MASK) == SCIDX_BK_SYS); @@ -571,6 +522,9 @@ pfinit(void) UID_ROOT, GID_WHEEL, 0600, "pfm", 0); pf_attach_hooks(); +#if DUMMYNET + dummynet_init(); +#endif } #if 0 @@ -594,10 +548,6 @@ pfdetach(void) for (i = 0; i < PF_RULESET_MAX; i++) if (pf_begin_rules(&ticket, i, &r) == 0) pf_commit_rules(ticket, i, &r); -#if PF_ALTQ - if (pf_begin_altq(&ticket) == 0) - pf_commit_altq(ticket); -#endif /* PF_ALTQ */ /* clear states */ RB_FOREACH(state, pf_state_tree_id, &tree_id) { @@ -639,9 +589,6 @@ pfdetach(void) /* destroy the pools */ pool_destroy(&pf_pooladdr_pl); -#if PF_ALTQ - pool_destroy(&pf_altq_pl); -#endif /* PF_ALTQ */ pool_destroy(&pf_state_pl); pool_destroy(&pf_rule_pl); pool_destroy(&pf_src_tree_pl); @@ -783,13 +730,6 @@ pf_rm_rule(struct pf_rulequeue *rulequeue, struct pf_rule *rule) return; pf_tag_unref(rule->tag); pf_tag_unref(rule->match_tag); -#if PF_ALTQ - if (altq_allowed) { - if (rule->pqid != rule->qid) - pf_qid_unref(rule->pqid); - pf_qid_unref(rule->qid); - } -#endif /* PF_ALTQ */ pf_rtlabel_remove(&rule->src.addr); pf_rtlabel_remove(&rule->dst.addr); pfi_dynaddr_remove(&rule->src.addr); @@ -930,236 +870,6 @@ pf_rtlabel_copyout(struct pf_addr_wrap *a) #pragma unused(a) } -#if PF_ALTQ -u_int32_t -pf_qname2qid(char *qname) -{ - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - return ((u_int32_t)tagname2tag(&pf_qids, qname)); -} - -void -pf_qid2qname(u_int32_t qid, char *p) -{ - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - tag2tagname(&pf_qids, (u_int16_t)qid, p); -} - -void -pf_qid_unref(u_int32_t qid) -{ - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - tag_unref(&pf_qids, (u_int16_t)qid); -} - -static int -pf_begin_altq(u_int32_t *ticket) -{ - struct pf_altq *altq; - int error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - /* Purge the old altq list */ - while ((altq = TAILQ_FIRST(pf_altqs_inactive)) != NULL) { - TAILQ_REMOVE(pf_altqs_inactive, altq, entries); - if (altq->qname[0] == '\0') { - /* detach and destroy the discipline */ - error = altq_remove(altq); - } else - pf_qid_unref(altq->qid); - pool_put(&pf_altq_pl, altq); - } - if (error) - return (error); - *ticket = ++ticket_altqs_inactive; - altqs_inactive_open = 1; - return (0); -} - -static int -pf_rollback_altq(u_int32_t ticket) -{ - struct pf_altq *altq; - int error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if (!altqs_inactive_open || ticket != ticket_altqs_inactive) - return (0); - /* Purge the old altq list */ - while ((altq = TAILQ_FIRST(pf_altqs_inactive)) != NULL) { - TAILQ_REMOVE(pf_altqs_inactive, altq, entries); - if (altq->qname[0] == '\0') { - /* detach and destroy the discipline */ - error = altq_remove(altq); - } else - pf_qid_unref(altq->qid); - pool_put(&pf_altq_pl, altq); - } - altqs_inactive_open = 0; - return (error); -} - -static int -pf_commit_altq(u_int32_t ticket) -{ - struct pf_altqqueue *old_altqs; - struct pf_altq *altq; - int err, error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if (!altqs_inactive_open || ticket != ticket_altqs_inactive) - return (EBUSY); - - /* swap altqs, keep the old. */ - old_altqs = pf_altqs_active; - pf_altqs_active = pf_altqs_inactive; - pf_altqs_inactive = old_altqs; - ticket_altqs_active = ticket_altqs_inactive; - - /* Attach new disciplines */ - TAILQ_FOREACH(altq, pf_altqs_active, entries) { - if (altq->qname[0] == '\0') { - /* attach the discipline */ - error = altq_pfattach(altq); - if (error == 0 && pf_altq_running) - error = pf_enable_altq(altq); - if (error != 0) { - return (error); - } - } - } - - /* Purge the old altq list */ - while ((altq = TAILQ_FIRST(pf_altqs_inactive)) != NULL) { - TAILQ_REMOVE(pf_altqs_inactive, altq, entries); - if (altq->qname[0] == '\0') { - /* detach and destroy the discipline */ - if (pf_altq_running) - error = pf_disable_altq(altq); - err = altq_pfdetach(altq); - if (err != 0 && error == 0) - error = err; - err = altq_remove(altq); - if (err != 0 && error == 0) - error = err; - } else - pf_qid_unref(altq->qid); - pool_put(&pf_altq_pl, altq); - } - - altqs_inactive_open = 0; - return (error); -} - -static int -pf_enable_altq(struct pf_altq *altq) -{ - struct ifnet *ifp; - struct ifclassq *ifq; - int error = 0; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((ifp = ifunit(altq->ifname)) == NULL) - return (EINVAL); - - ifq = &ifp->if_snd; - IFCQ_LOCK(ifq); - if (IFCQ_ALTQ(ifq)->altq_type != ALTQT_NONE) - error = altq_enable(IFCQ_ALTQ(ifq)); - - /* set or clear tokenbucket regulator */ - if (error == 0 && ifp != NULL && ALTQ_IS_ENABLED(IFCQ_ALTQ(ifq))) { - struct tb_profile tb = { 0, 0, 0 }; - - if (altq->aflags & PF_ALTQF_TBR) { - if (altq->bwtype != PF_ALTQ_BW_ABSOLUTE && - altq->bwtype != PF_ALTQ_BW_PERCENT) { - error = EINVAL; - } else { - if (altq->bwtype == PF_ALTQ_BW_ABSOLUTE) - tb.rate = altq->ifbandwidth; - else - tb.percent = altq->ifbandwidth; - tb.depth = altq->tbrsize; - error = ifclassq_tbr_set(ifq, &tb, TRUE); - } - } else if (IFCQ_TBR_IS_ENABLED(ifq)) { - error = ifclassq_tbr_set(ifq, &tb, TRUE); - } - } - IFCQ_UNLOCK(ifq); - - return (error); -} - -static int -pf_disable_altq(struct pf_altq *altq) -{ - struct ifnet *ifp; - struct ifclassq *ifq; - int error; - - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); - - if ((ifp = ifunit(altq->ifname)) == NULL) - return (EINVAL); - - /* - * when the discipline is no longer referenced, it was overridden - * by a new one. if so, just return. - */ - ifq = &ifp->if_snd; - IFCQ_LOCK(ifq); - if (altq->altq_disc != IFCQ_ALTQ(ifq)->altq_disc) { - IFCQ_UNLOCK(ifq); - return (0); - } - - error = altq_disable(IFCQ_ALTQ(ifq)); - - if (error == 0 && IFCQ_TBR_IS_ENABLED(ifq)) { - /* clear tokenbucket regulator */ - struct tb_profile tb = { 0, 0, 0 }; - error = ifclassq_tbr_set(ifq, &tb, TRUE); - } - IFCQ_UNLOCK(ifq); - - return (error); -} - -static void -pf_altq_copyin(struct pf_altq *src, struct pf_altq *dst) -{ - bcopy(src, dst, sizeof (struct pf_altq)); - - dst->ifname[sizeof (dst->ifname) - 1] = '\0'; - dst->qname[sizeof (dst->qname) - 1] = '\0'; - dst->parent[sizeof (dst->parent) - 1] = '\0'; - dst->altq_disc = NULL; - dst->entries.tqe_next = NULL; - dst->entries.tqe_prev = NULL; -} - -static void -pf_altq_copyout(struct pf_altq *src, struct pf_altq *dst) -{ - struct pf_altq pa; - - bcopy(src, &pa, sizeof (struct pf_altq)); - pa.altq_disc = NULL; - pa.entries.tqe_next = NULL; - pa.entries.tqe_prev = NULL; - bcopy(&pa, dst, sizeof (struct pf_altq)); -} -#endif /* PF_ALTQ */ - static int pf_begin_rules(u_int32_t *ticket, int rs_num, const char *anchor) { @@ -1301,7 +1011,7 @@ pf_commit_rules(u_int32_t ticket, int rs_num, char *anchor) int error; u_int32_t old_rcount; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (rs_num < 0 || rs_num >= PF_RULESET_MAX) return (EINVAL); @@ -1572,7 +1282,7 @@ pf_setup_pfsync_matching(struct pf_ruleset *rs) static void pf_start(void) { - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); VERIFY(pf_is_enabled == 0); @@ -1590,7 +1300,7 @@ pf_start(void) static void pf_stop(void) { - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); VERIFY(pf_is_enabled); @@ -1739,22 +1449,6 @@ pfioctl(dev_t dev, u_long cmd, caddr_t addr, int flags, struct proc *p) return (EACCES); } -#if PF_ALTQ - switch (cmd) { - case DIOCSTARTALTQ: - case DIOCSTOPALTQ: - case DIOCADDALTQ: - case DIOCGETALTQS: - case DIOCGETALTQ: - case DIOCCHANGEALTQ: - case DIOCGETQSTATS: - /* fail if ALTQ is disabled */ - if (!altq_allowed) - return (ENODEV); - break; - } -#endif /* PF_ALTQ */ - if (flags & FWRITE) lck_rw_lock_exclusive(pf_perim_lock); else @@ -2006,171 +1700,6 @@ pfioctl(dev_t dev, u_long cmd, caddr_t addr, int flags, struct proc *p) break; } -#if PF_ALTQ - case DIOCSTARTALTQ: { - struct pf_altq *altq; - - VERIFY(altq_allowed); - /* enable all altq interfaces on active list */ - TAILQ_FOREACH(altq, pf_altqs_active, entries) { - if (altq->qname[0] == '\0') { - error = pf_enable_altq(altq); - if (error != 0) - break; - } - } - if (error == 0) - pf_altq_running = 1; - DPFPRINTF(PF_DEBUG_MISC, ("altq: started\n")); - break; - } - - case DIOCSTOPALTQ: { - struct pf_altq *altq; - - VERIFY(altq_allowed); - /* disable all altq interfaces on active list */ - TAILQ_FOREACH(altq, pf_altqs_active, entries) { - if (altq->qname[0] == '\0') { - error = pf_disable_altq(altq); - if (error != 0) - break; - } - } - if (error == 0) - pf_altq_running = 0; - DPFPRINTF(PF_DEBUG_MISC, ("altq: stopped\n")); - break; - } - - case DIOCADDALTQ: { /* struct pfioc_altq */ - struct pfioc_altq *pa = (struct pfioc_altq *)(void *)addr; - struct pf_altq *altq, *a; - u_int32_t ticket; - - VERIFY(altq_allowed); - bcopy(&pa->ticket, &ticket, sizeof (ticket)); - if (ticket != ticket_altqs_inactive) { - error = EBUSY; - break; - } - altq = pool_get(&pf_altq_pl, PR_WAITOK); - if (altq == NULL) { - error = ENOMEM; - break; - } - pf_altq_copyin(&pa->altq, altq); - - /* - * if this is for a queue, find the discipline and - * copy the necessary fields - */ - if (altq->qname[0] != '\0') { - if ((altq->qid = pf_qname2qid(altq->qname)) == 0) { - error = EBUSY; - pool_put(&pf_altq_pl, altq); - break; - } - altq->altq_disc = NULL; - TAILQ_FOREACH(a, pf_altqs_inactive, entries) { - if (strncmp(a->ifname, altq->ifname, - IFNAMSIZ) == 0 && a->qname[0] == '\0') { - altq->altq_disc = a->altq_disc; - break; - } - } - } - - error = altq_add(altq); - if (error) { - pool_put(&pf_altq_pl, altq); - break; - } - - TAILQ_INSERT_TAIL(pf_altqs_inactive, altq, entries); - pf_altq_copyout(altq, &pa->altq); - break; - } - - case DIOCGETALTQS: { - struct pfioc_altq *pa = (struct pfioc_altq *)(void *)addr; - struct pf_altq *altq; - u_int32_t nr; - - VERIFY(altq_allowed); - nr = 0; - TAILQ_FOREACH(altq, pf_altqs_active, entries) - nr++; - bcopy(&nr, &pa->nr, sizeof (nr)); - bcopy(&ticket_altqs_active, &pa->ticket, sizeof (pa->ticket)); - break; - } - - case DIOCGETALTQ: { - struct pfioc_altq *pa = (struct pfioc_altq *)(void *)addr; - struct pf_altq *altq; - u_int32_t nr, pa_nr, ticket; - - VERIFY(altq_allowed); - bcopy(&pa->ticket, &ticket, sizeof (ticket)); - if (ticket != ticket_altqs_active) { - error = EBUSY; - break; - } - bcopy(&pa->nr, &pa_nr, sizeof (pa_nr)); - nr = 0; - altq = TAILQ_FIRST(pf_altqs_active); - while ((altq != NULL) && (nr < pa_nr)) { - altq = TAILQ_NEXT(altq, entries); - nr++; - } - if (altq == NULL) { - error = EBUSY; - break; - } - pf_altq_copyout(altq, &pa->altq); - break; - } - - case DIOCCHANGEALTQ: - VERIFY(altq_allowed); - /* CHANGEALTQ not supported yet! */ - error = ENODEV; - break; - - case DIOCGETQSTATS: { - struct pfioc_qstats *pq = (struct pfioc_qstats *)(void *)addr; - struct pf_altq *altq; - u_int32_t nr, pq_nr, ticket; - int nbytes; - - VERIFY(altq_allowed); - bcopy(&pq->ticket, &ticket, sizeof (ticket)); - if (ticket != ticket_altqs_active) { - error = EBUSY; - break; - } - bcopy(&pq->nr, &pq_nr, sizeof (pq_nr)); - nr = 0; - altq = TAILQ_FIRST(pf_altqs_active); - while ((altq != NULL) && (nr < pq_nr)) { - altq = TAILQ_NEXT(altq, entries); - nr++; - } - if (altq == NULL) { - error = EBUSY; - break; - } - bcopy(&pq->nbytes, &nbytes, sizeof (nbytes)); - error = altq_getqstats(altq, pq->buf, &nbytes); - if (error == 0) { - pq->scheduler = altq->scheduler; - bcopy(&nbytes, &pq->nbytes, sizeof (nbytes)); - } - break; - } -#endif /* PF_ALTQ */ - case DIOCBEGINADDRS: /* struct pfioc_pooladdr */ case DIOCADDADDR: /* struct pfioc_pooladdr */ case DIOCGETADDRS: /* struct pfioc_pooladdr */ @@ -3037,19 +2566,6 @@ pf_rule_setup(struct pfioc_rule *pr, struct pf_rule *rule, } pfi_kif_ref(rule->kif, PFI_KIF_REF_RULE); } -#if PF_ALTQ - /* set queue IDs */ - if (altq_allowed && rule->qname[0] != '\0') { - if ((rule->qid = pf_qname2qid(rule->qname)) == 0) - error = EBUSY; - else if (rule->pqname[0] != '\0') { - if ((rule->pqid = - pf_qname2qid(rule->pqname)) == 0) - error = EBUSY; - } else - rule->pqid = rule->qid; - } -#endif /* PF_ALTQ */ if (rule->tagname[0]) if ((rule->tag = pf_tagname2tag(rule->tagname)) == 0) error = EBUSY; @@ -3191,6 +2707,38 @@ pfioctl_ioc_rule(u_long cmd, int minordev, struct pfioc_rule *pr, struct proc *p if (rule->action == PF_NAT64) atomic_add_16(&pf_nat64_configured, 1); + + if (pr->anchor_call[0] == '\0') { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_pf_addrule_total); + if (rule->rule_flag & PFRULE_PFM) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_pf_addrule_os); + } + } + +#if DUMMYNET + if (rule->action == PF_DUMMYNET) { + struct dummynet_event dn_event; + uint32_t direction = DN_INOUT;; + bzero(&dn_event, sizeof(dn_event)); + + dn_event.dn_event_code = DUMMYNET_RULE_CONFIG; + + if (rule->direction == PF_IN) + direction = DN_IN; + else if (rule->direction == PF_OUT) + direction = DN_OUT; + + dn_event.dn_event_rule_config.dir = direction; + dn_event.dn_event_rule_config.af = rule->af; + dn_event.dn_event_rule_config.proto = rule->proto; + dn_event.dn_event_rule_config.src_port = rule->src.xport.range.port[0]; + dn_event.dn_event_rule_config.dst_port = rule->dst.xport.range.port[0]; + strlcpy(dn_event.dn_event_rule_config.ifname, rule->ifname, + sizeof(dn_event.dn_event_rule_config.ifname)); + + dummynet_event_enqueue_nwk_wq_entry(&dn_event); + } +#endif break; } @@ -3355,20 +2903,6 @@ pfioctl_ioc_rule(u_long cmd, int minordev, struct pfioc_rule *pr, struct proc *p } else newrule->kif = NULL; -#if PF_ALTQ - /* set queue IDs */ - if (altq_allowed && newrule->qname[0] != '\0') { - if ((newrule->qid = - pf_qname2qid(newrule->qname)) == 0) - error = EBUSY; - else if (newrule->pqname[0] != '\0') { - if ((newrule->pqid = - pf_qname2qid(newrule->pqname)) == 0) - error = EBUSY; - } else - newrule->pqid = newrule->qid; - } -#endif /* PF_ALTQ */ if (newrule->tagname[0]) if ((newrule->tag = pf_tagname2tag(newrule->tagname)) == 0) @@ -3593,6 +3127,13 @@ pfioctl_ioc_rule(u_long cmd, int minordev, struct pfioc_rule *pr, struct proc *p pffwrules++; if (rule->action == PF_NAT64) atomic_add_16(&pf_nat64_configured, 1); + + if (pr->anchor_call[0] == '\0') { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_pf_addrule_total); + if (rule->rule_flag & PFRULE_PFM) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_pf_addrule_os); + } + } break; } @@ -4401,22 +3942,6 @@ pfioctl_ioc_trans(u_long cmd, struct pfioc_trans_32 *io32, ioe->anchor[sizeof (ioe->anchor) - 1] = '\0'; switch (ioe->rs_num) { case PF_RULESET_ALTQ: -#if PF_ALTQ - if (altq_allowed) { - if (ioe->anchor[0]) { - _FREE(table, M_TEMP); - _FREE(ioe, M_TEMP); - error = EINVAL; - goto fail; - } - error = pf_begin_altq(&ioe->ticket); - if (error != 0) { - _FREE(table, M_TEMP); - _FREE(ioe, M_TEMP); - goto fail; - } - } -#endif /* PF_ALTQ */ break; case PF_RULESET_TABLE: bzero(table, sizeof (*table)); @@ -4471,22 +3996,6 @@ pfioctl_ioc_trans(u_long cmd, struct pfioc_trans_32 *io32, ioe->anchor[sizeof (ioe->anchor) - 1] = '\0'; switch (ioe->rs_num) { case PF_RULESET_ALTQ: -#if PF_ALTQ - if (altq_allowed) { - if (ioe->anchor[0]) { - _FREE(table, M_TEMP); - _FREE(ioe, M_TEMP); - error = EINVAL; - goto fail; - } - error = pf_rollback_altq(ioe->ticket); - if (error != 0) { - _FREE(table, M_TEMP); - _FREE(ioe, M_TEMP); - goto fail; /* really bad */ - } - } -#endif /* PF_ALTQ */ break; case PF_RULESET_TABLE: bzero(table, sizeof (*table)); @@ -4538,24 +4047,6 @@ pfioctl_ioc_trans(u_long cmd, struct pfioc_trans_32 *io32, ioe->anchor[sizeof (ioe->anchor) - 1] = '\0'; switch (ioe->rs_num) { case PF_RULESET_ALTQ: -#if PF_ALTQ - if (altq_allowed) { - if (ioe->anchor[0]) { - _FREE(table, M_TEMP); - _FREE(ioe, M_TEMP); - error = EINVAL; - goto fail; - } - if (!altqs_inactive_open || - ioe->ticket != - ticket_altqs_inactive) { - _FREE(table, M_TEMP); - _FREE(ioe, M_TEMP); - error = EBUSY; - goto fail; - } - } -#endif /* PF_ALTQ */ break; case PF_RULESET_TABLE: rs = pf_find_ruleset(ioe->anchor); @@ -4600,14 +4091,6 @@ pfioctl_ioc_trans(u_long cmd, struct pfioc_trans_32 *io32, ioe->anchor[sizeof (ioe->anchor) - 1] = '\0'; switch (ioe->rs_num) { case PF_RULESET_ALTQ: -#if PF_ALTQ - if (altq_allowed && - (error = pf_commit_altq(ioe->ticket))) { - _FREE(table, M_TEMP); - _FREE(ioe, M_TEMP); - goto fail; /* really bad */ - } -#endif /* PF_ALTQ */ break; case PF_RULESET_TABLE: bzero(table, sizeof (*table)); @@ -4967,7 +4450,7 @@ pf_inet_hook(struct ifnet *ifp, struct mbuf **mp, int input, HTONS(ip->ip_len); HTONS(ip->ip_off); #endif - if (pf_test(input ? PF_IN : PF_OUT, ifp, mp, NULL, fwa) != PF_PASS) { + if (pf_test_mbuf(input ? PF_IN : PF_OUT, ifp, mp, NULL, fwa) != PF_PASS) { if (*mp != NULL) { m_freem(*mp); *mp = NULL; @@ -5018,7 +4501,7 @@ pf_inet6_hook(struct ifnet *ifp, struct mbuf **mp, int input, } } - if (pf_test6(input ? PF_IN : PF_OUT, ifp, mp, NULL, fwa) != PF_PASS) { + if (pf_test6_mbuf(input ? PF_IN : PF_OUT, ifp, mp, NULL, fwa) != PF_PASS) { if (*mp != NULL) { m_freem(*mp); *mp = NULL; diff --git a/bsd/net/pf_norm.c b/bsd/net/pf_norm.c index 547ff705f..3cb22e9ce 100644 --- a/bsd/net/pf_norm.c +++ b/bsd/net/pf_norm.c @@ -134,6 +134,8 @@ struct pf_fragment { LIST_HEAD(pf_fragq, pf_frent) fru_queue; /* buffering */ LIST_HEAD(pf_cacheq, pf_frcache) fru_cache; /* non-buf */ } fr_u; + uint32_t fr_csum_flags; /* checksum flags */ + uint32_t fr_csum; /* partial checksum value */ }; static TAILQ_HEAD(pf_fragqueue, pf_fragment) pf_fragqueue; @@ -160,7 +162,7 @@ static __inline struct pf_fragment * static __inline struct pf_fragment * pf_find_fragment_by_ipv6_header(struct ip6_hdr *, struct ip6_frag *, struct pf_frag_tree *); -static struct mbuf *pf_reassemble(struct mbuf **, struct pf_fragment **, +static struct mbuf *pf_reassemble(struct mbuf *, struct pf_fragment **, struct pf_frent *, int); static struct mbuf *pf_fragcache(struct mbuf **, struct ip *, struct pf_fragment **, int, int, int *); @@ -169,7 +171,7 @@ static struct mbuf *pf_reassemble6(struct mbuf **, struct pf_fragment **, static struct mbuf *pf_frag6cache(struct mbuf **, struct ip6_hdr*, struct ip6_frag *, struct pf_fragment **, int, int, int, int *); static int pf_normalize_tcpopt(struct pf_rule *, int, struct pfi_kif *, - struct pf_pdesc *, struct mbuf *, struct tcphdr *, int, int *); + struct pf_pdesc *, pbuf_t *, struct tcphdr *, int, int *); #define DPFPRINTF(x) do { \ if (pf_status.debug >= PF_DEBUG_MISC) { \ @@ -246,13 +248,13 @@ pf_frag_compare(struct pf_fragment *a, struct pf_fragment *b) case AF_INET: if ((diff = a->fr_id - b->fr_id)) return (diff); - else if (sa->v4.s_addr < sb->v4.s_addr) + else if (sa->v4addr.s_addr < sb->v4addr.s_addr) return (-1); - else if (sa->v4.s_addr > sb->v4.s_addr) + else if (sa->v4addr.s_addr > sb->v4addr.s_addr) return (1); - else if (da->v4.s_addr < db->v4.s_addr) + else if (da->v4addr.s_addr < db->v4addr.s_addr) return (-1); - else if (da->v4.s_addr > db->v4.s_addr) + else if (da->v4addr.s_addr > db->v4addr.s_addr) return (1); break; #endif @@ -432,8 +434,8 @@ pf_ip6hdr2key(struct pf_fragment *key, struct ip6_hdr *ip6, key->fr_p = fh->ip6f_nxt; key->fr_id6 = fh->ip6f_ident; key->fr_af = AF_INET6; - key->fr_srcx.v6 = ip6->ip6_src; - key->fr_dstx.v6 = ip6->ip6_dst; + key->fr_srcx.v6addr = ip6->ip6_src; + key->fr_dstx.v6addr = ip6->ip6_dst; } static void @@ -442,8 +444,8 @@ pf_ip2key(struct pf_fragment *key, struct ip *ip) key->fr_p = ip->ip_p; key->fr_id = ip->ip_id; key->fr_af = AF_INET; - key->fr_srcx.v4.s_addr = ip->ip_src.s_addr; - key->fr_dstx.v4.s_addr = ip->ip_dst.s_addr; + key->fr_srcx.v4addr.s_addr = ip->ip_src.s_addr; + key->fr_dstx.v4addr.s_addr = ip->ip_dst.s_addr; } static struct pf_fragment * @@ -502,20 +504,79 @@ pf_remove_fragment(struct pf_fragment *frag) #define FR_IP_OFF(fr) ((ntohs((fr)->fr_ip->ip_off) & IP_OFFMASK) << 3) static struct mbuf * -pf_reassemble(struct mbuf **m0, struct pf_fragment **frag, +pf_reassemble(struct mbuf *m0, struct pf_fragment **frag, struct pf_frent *frent, int mff) { - struct mbuf *m = *m0, *m2; + struct mbuf *m = m0, *m2; struct pf_frent *frea, *next; struct pf_frent *frep = NULL; struct ip *ip = frent->fr_ip; - int hlen = ip->ip_hl << 2; + uint32_t hlen = ip->ip_hl << 2; u_int16_t off = (ntohs(ip->ip_off) & IP_OFFMASK) << 3; u_int16_t ip_len = ntohs(ip->ip_len) - ip->ip_hl * 4; u_int16_t fr_max = ip_len + off; + uint32_t csum, csum_flags; VERIFY(*frag == NULL || BUFFER_FRAGMENTS(*frag)); + /* + * Leverage partial checksum offload for IP fragments. Narrow down + * the scope to cover only UDP without IP options, as that is the + * most common case. + * + * Perform 1's complement adjustment of octets that got included/ + * excluded in the hardware-calculated checksum value. Ignore cases + * where the value includes the entire IPv4 header span, as the sum + * for those octets would already be 0 by the time we get here; IP + * has already performed its header checksum validation. Also take + * care of any trailing bytes and subtract out their partial sum. + */ + if (ip->ip_p == IPPROTO_UDP && hlen == sizeof (struct ip) && + (m->m_pkthdr.csum_flags & + (CSUM_DATA_VALID | CSUM_PARTIAL | CSUM_PSEUDO_HDR)) == + (CSUM_DATA_VALID | CSUM_PARTIAL)) { + uint32_t start = m->m_pkthdr.csum_rx_start; + int32_t trailer = (m_pktlen(m) - ntohs(ip->ip_len)); + uint32_t swbytes = (uint32_t)trailer; + + csum = m->m_pkthdr.csum_rx_val; + + ASSERT(trailer >= 0); + if ((start != 0 && start != hlen) || trailer != 0) { +#if BYTE_ORDER != BIG_ENDIAN + if (start < hlen) { + HTONS(ip->ip_len); + HTONS(ip->ip_off); + } +#endif /* BYTE_ORDER != BIG_ENDIAN */ + /* callee folds in sum */ + csum = m_adj_sum16(m, start, hlen, + (ip->ip_len - hlen), csum); + if (hlen > start) + swbytes += (hlen - start); + else + swbytes += (start - hlen); +#if BYTE_ORDER != BIG_ENDIAN + if (start < hlen) { + NTOHS(ip->ip_off); + NTOHS(ip->ip_len); + } +#endif /* BYTE_ORDER != BIG_ENDIAN */ + } + csum_flags = m->m_pkthdr.csum_flags; + + if (swbytes != 0) + udp_in_cksum_stats(swbytes); + if (trailer != 0) + m_adj(m, -trailer); + } else { + csum = 0; + csum_flags = 0; + } + + /* Invalidate checksum */ + m->m_pkthdr.csum_flags &= ~CSUM_DATA_VALID; + /* Strip off ip header */ m->m_data += hlen; m->m_len -= hlen; @@ -533,11 +594,15 @@ pf_reassemble(struct mbuf **m0, struct pf_fragment **frag, (*frag)->fr_flags = 0; (*frag)->fr_max = 0; (*frag)->fr_af = AF_INET; - (*frag)->fr_srcx.v4 = frent->fr_ip->ip_src; - (*frag)->fr_dstx.v4 = frent->fr_ip->ip_dst; + (*frag)->fr_srcx.v4addr = frent->fr_ip->ip_src; + (*frag)->fr_dstx.v4addr = frent->fr_ip->ip_dst; (*frag)->fr_p = frent->fr_ip->ip_p; (*frag)->fr_id = frent->fr_ip->ip_id; (*frag)->fr_timeout = pf_time_second(); + if (csum_flags != 0) { + (*frag)->fr_csum_flags = csum_flags; + (*frag)->fr_csum = csum; + } LIST_INIT(&(*frag)->fr_queue); RB_INSERT(pf_frag_tree, &pf_frag_tree, *frag); @@ -548,6 +613,16 @@ pf_reassemble(struct mbuf **m0, struct pf_fragment **frag, goto insert; } + /* + * If this fragment contains similar checksum offload info + * as that of the existing ones, accumulate checksum. Otherwise, + * invalidate checksum offload info for the entire datagram. + */ + if (csum_flags != 0 && csum_flags == (*frag)->fr_csum_flags) + (*frag)->fr_csum += csum; + else if ((*frag)->fr_csum_flags != 0) + (*frag)->fr_csum_flags = 0; + /* * Find a fragment after the current one: * - off contains the real shifted offset. @@ -665,8 +740,26 @@ insert: m_cat(m, m2); } - ip->ip_src = (*frag)->fr_srcx.v4; - ip->ip_dst = (*frag)->fr_dstx.v4; + ip->ip_src = (*frag)->fr_srcx.v4addr; + ip->ip_dst = (*frag)->fr_dstx.v4addr; + + if ((*frag)->fr_csum_flags != 0) { + csum = (*frag)->fr_csum; + + ADDCARRY(csum); + + m->m_pkthdr.csum_rx_val = csum; + m->m_pkthdr.csum_rx_start = sizeof (struct ip); + m->m_pkthdr.csum_flags = (*frag)->fr_csum_flags; + } else if ((m->m_pkthdr.rcvif->if_flags & IFF_LOOPBACK) || + (m->m_pkthdr.pkt_flags & PKTF_LOOP)) { + /* loopback checksums are always OK */ + m->m_pkthdr.csum_data = 0xffff; + m->m_pkthdr.csum_flags &= ~CSUM_PARTIAL; + m->m_pkthdr.csum_flags = + CSUM_DATA_VALID | CSUM_PSEUDO_HDR | + CSUM_IP_CHECKED | CSUM_IP_VALID; + } /* Remove from fragment queue */ pf_remove_fragment(*frag); @@ -733,8 +826,8 @@ pf_fragcache(struct mbuf **m0, struct ip *h, struct pf_fragment **frag, int mff, (*frag)->fr_flags = PFFRAG_NOBUFFER; (*frag)->fr_max = 0; (*frag)->fr_af = AF_INET; - (*frag)->fr_srcx.v4 = h->ip_src; - (*frag)->fr_dstx.v4 = h->ip_dst; + (*frag)->fr_srcx.v4addr = h->ip_src; + (*frag)->fr_dstx.v4addr = h->ip_dst; (*frag)->fr_p = h->ip_p; (*frag)->fr_id = h->ip_id; (*frag)->fr_timeout = pf_time_second(); @@ -1008,19 +1101,84 @@ pf_reassemble6(struct mbuf **m0, struct pf_fragment **frag, struct mbuf *m, *m2; struct pf_frent *frea, *frep, *next; struct ip6_hdr *ip6; + struct ip6_frag *ip6f; int plen, off, fr_max; + uint32_t uoff, csum, csum_flags; VERIFY(*frag == NULL || BUFFER_FRAGMENTS(*frag)); m = *m0; frep = NULL; ip6 = frent->fr_ip6; + ip6f = &frent->fr_ip6f_opt; off = FR_IP6_OFF(frent); + uoff = frent->fr_ip6f_hlen; plen = FR_IP6_PLEN(frent); fr_max = off + plen - (frent->fr_ip6f_hlen - sizeof *ip6); DPFPRINTF(("0x%llx IPv6 frag plen %u off %u fr_ip6f_hlen %u " "fr_max %u m_len %u\n", (uint64_t)VM_KERNEL_ADDRPERM(m), plen, off, frent->fr_ip6f_hlen, fr_max, m->m_len)); + + /* + * Leverage partial checksum offload for simple UDP/IP fragments, + * as that is the most common case. + * + * Perform 1's complement adjustment of octets that got included/ + * excluded in the hardware-calculated checksum value. Also take + * care of any trailing bytes and subtract out their partial sum. + */ + if (ip6f->ip6f_nxt == IPPROTO_UDP && + uoff == (sizeof (*ip6) + sizeof (*ip6f)) && + (m->m_pkthdr.csum_flags & + (CSUM_DATA_VALID | CSUM_PARTIAL | CSUM_PSEUDO_HDR)) == + (CSUM_DATA_VALID | CSUM_PARTIAL)) { + uint32_t start = m->m_pkthdr.csum_rx_start; + uint32_t ip_len = (sizeof (*ip6) + ntohs(ip6->ip6_plen)); + int32_t trailer = (m_pktlen(m) - ip_len); + uint32_t swbytes = (uint32_t)trailer; + + csum = m->m_pkthdr.csum_rx_val; + + ASSERT(trailer >= 0); + if (start != uoff || trailer != 0) { + uint16_t s = 0, d = 0; + + if (IN6_IS_SCOPE_EMBED(&ip6->ip6_src)) { + s = ip6->ip6_src.s6_addr16[1]; + ip6->ip6_src.s6_addr16[1] = 0 ; + } + if (IN6_IS_SCOPE_EMBED(&ip6->ip6_dst)) { + d = ip6->ip6_dst.s6_addr16[1]; + ip6->ip6_dst.s6_addr16[1] = 0; + } + + /* callee folds in sum */ + csum = m_adj_sum16(m, start, uoff, + (ip_len - uoff), csum); + if (uoff > start) + swbytes += (uoff - start); + else + swbytes += (start - uoff); + + if (IN6_IS_SCOPE_EMBED(&ip6->ip6_src)) + ip6->ip6_src.s6_addr16[1] = s; + if (IN6_IS_SCOPE_EMBED(&ip6->ip6_dst)) + ip6->ip6_dst.s6_addr16[1] = d; + + } + csum_flags = m->m_pkthdr.csum_flags; + + if (swbytes != 0) + udp_in6_cksum_stats(swbytes); + if (trailer != 0) + m_adj(m, -trailer); + } else { + csum = 0; + csum_flags = 0; + } + + /* Invalidate checksum */ + m->m_pkthdr.csum_flags &= ~CSUM_DATA_VALID; /* strip off headers up to the fragment payload */ m->m_data += frent->fr_ip6f_hlen; @@ -1039,11 +1197,15 @@ pf_reassemble6(struct mbuf **m0, struct pf_fragment **frag, (*frag)->fr_flags = 0; (*frag)->fr_max = 0; (*frag)->fr_af = AF_INET6; - (*frag)->fr_srcx.v6 = frent->fr_ip6->ip6_src; - (*frag)->fr_dstx.v6 = frent->fr_ip6->ip6_dst; + (*frag)->fr_srcx.v6addr = frent->fr_ip6->ip6_src; + (*frag)->fr_dstx.v6addr = frent->fr_ip6->ip6_dst; (*frag)->fr_p = frent->fr_ip6f_opt.ip6f_nxt; (*frag)->fr_id6 = frent->fr_ip6f_opt.ip6f_ident; (*frag)->fr_timeout = pf_time_second(); + if (csum_flags != 0) { + (*frag)->fr_csum_flags = csum_flags; + (*frag)->fr_csum = csum; + } LIST_INIT(&(*frag)->fr_queue); RB_INSERT(pf_frag_tree, &pf_frag_tree, *frag); @@ -1053,6 +1215,16 @@ pf_reassemble6(struct mbuf **m0, struct pf_fragment **frag, frep = NULL; goto insert; } + + /* + * If this fragment contains similar checksum offload info + * as that of the existing ones, accumulate checksum. Otherwise, + * invalidate checksum offload info for the entire datagram. + */ + if (csum_flags != 0 && csum_flags == (*frag)->fr_csum_flags) + (*frag)->fr_csum += csum; + else if ((*frag)->fr_csum_flags != 0) + (*frag)->fr_csum_flags = 0; /* * Find a fragment after the current one: @@ -1159,8 +1331,24 @@ pf_reassemble6(struct mbuf **m0, struct pf_fragment **frag, ip6 = frent->fr_ip6; ip6->ip6_nxt = (*frag)->fr_p; ip6->ip6_plen = htons(off); - ip6->ip6_src = (*frag)->fr_srcx.v6; - ip6->ip6_dst = (*frag)->fr_dstx.v6; + ip6->ip6_src = (*frag)->fr_srcx.v6addr; + ip6->ip6_dst = (*frag)->fr_dstx.v6addr; + + if ((*frag)->fr_csum_flags != 0) { + csum = (*frag)->fr_csum; + + ADDCARRY(csum); + + m->m_pkthdr.csum_rx_val = csum; + m->m_pkthdr.csum_rx_start = sizeof (struct ip6_hdr); + m->m_pkthdr.csum_flags = (*frag)->fr_csum_flags; + } else if ((m->m_pkthdr.rcvif->if_flags & IFF_LOOPBACK) || + (m->m_pkthdr.pkt_flags & PKTF_LOOP)) { + /* loopback checksums are always OK */ + m->m_pkthdr.csum_data = 0xffff; + m->m_pkthdr.csum_flags &= ~CSUM_PARTIAL; + m->m_pkthdr.csum_flags = CSUM_DATA_VALID | CSUM_PSEUDO_HDR; + } /* Remove from fragment queue */ pf_remove_fragment(*frag); @@ -1263,8 +1451,8 @@ pf_frag6cache(struct mbuf **m0, struct ip6_hdr *h, struct ip6_frag *fh, (*frag)->fr_flags = PFFRAG_NOBUFFER; (*frag)->fr_max = 0; (*frag)->fr_af = AF_INET6; - (*frag)->fr_srcx.v6 = h->ip6_src; - (*frag)->fr_dstx.v6 = h->ip6_dst; + (*frag)->fr_srcx.v6addr = h->ip6_src; + (*frag)->fr_dstx.v6addr = h->ip6_dst; (*frag)->fr_p = fh->ip6f_nxt; (*frag)->fr_id6 = fh->ip6f_ident; (*frag)->fr_timeout = pf_time_second(); @@ -1530,14 +1718,14 @@ pf_frag6cache(struct mbuf **m0, struct ip6_hdr *h, struct ip6_frag *fh, } int -pf_normalize_ip(struct mbuf **m0, int dir, struct pfi_kif *kif, u_short *reason, +pf_normalize_ip(pbuf_t *pbuf, int dir, struct pfi_kif *kif, u_short *reason, struct pf_pdesc *pd) { - struct mbuf *m = *m0; + struct mbuf *m; struct pf_rule *r; struct pf_frent *frent; struct pf_fragment *frag = NULL; - struct ip *h = mtod(m, struct ip *); + struct ip *h = pbuf->pb_data; int mff = (ntohs(h->ip_off) & IP_MF); int hlen = h->ip_hl << 2; u_int16_t fragoff = (ntohs(h->ip_off) & IP_OFFMASK) << 3; @@ -1546,6 +1734,7 @@ pf_normalize_ip(struct mbuf **m0, int dir, struct pfi_kif *kif, u_short *reason, int ip_off; int asd = 0; struct pf_ruleset *ruleset = NULL; + struct ifnet *ifp = pbuf->pb_ifp; r = TAILQ_FIRST(pf_main_ruleset.rules[PF_RULESET_SCRUB].active.ptr); while (r != NULL) { @@ -1639,10 +1828,21 @@ pf_normalize_ip(struct mbuf **m0, int dir, struct pfi_kif *kif, u_short *reason, fr_max > frag->fr_max) goto bad; + if ((m = pbuf_to_mbuf(pbuf, TRUE)) == NULL) { + REASON_SET(reason, PFRES_MEMORY); + return (PF_DROP); + } + + VERIFY(!pbuf_is_valid(pbuf)); + + /* Restore iph pointer after pbuf_to_mbuf() */ + h = mtod(m, struct ip *); + /* Get an entry for the fragment queue */ frent = pool_get(&pf_frent_pl, PR_NOWAIT); if (frent == NULL) { REASON_SET(reason, PFRES_MEMORY); + m_freem(m); return (PF_DROP); } pf_nfrents++; @@ -1652,29 +1852,34 @@ pf_normalize_ip(struct mbuf **m0, int dir, struct pfi_kif *kif, u_short *reason, /* Might return a completely reassembled mbuf, or NULL */ DPFPRINTF(("reass IPv4 frag %d @ %d-%d\n", ntohs(h->ip_id), fragoff, fr_max)); - *m0 = m = pf_reassemble(m0, &frag, frent, mff); + m = pf_reassemble(m, &frag, frent, mff); if (m == NULL) return (PF_DROP); VERIFY(m->m_flags & M_PKTHDR); + pbuf_init_mbuf(pbuf, m, ifp); /* use mtag from concatenated mbuf chain */ - pd->pf_mtag = pf_find_mtag(m); + pd->pf_mtag = pf_find_mtag_pbuf(pbuf); +#if 0 +// SCW: This check is superfluous #if DIAGNOSTIC if (pd->pf_mtag == NULL) { printf("%s: pf_find_mtag returned NULL(1)\n", __func__); if ((pd->pf_mtag = pf_get_mtag(m)) == NULL) { m_freem(m); - m = *m0 = NULL; + m = NULL; goto no_mem; } } #endif - if (frag != NULL && (frag->fr_flags & PFFRAG_DROP)) - goto drop; +#endif h = mtod(m, struct ip *); + + if (frag != NULL && (frag->fr_flags & PFFRAG_DROP)) + goto drop; } else { /* non-buffering fragment cache (drops or masks overlaps) */ int nomem = 0; @@ -1698,33 +1903,49 @@ pf_normalize_ip(struct mbuf **m0, int dir, struct pfi_kif *kif, u_short *reason, goto bad; } - *m0 = m = pf_fragcache(m0, h, &frag, mff, + if ((m = pbuf_to_mbuf(pbuf, TRUE)) == NULL) { + REASON_SET(reason, PFRES_MEMORY); + goto bad; + } + + VERIFY(!pbuf_is_valid(pbuf)); + + /* Restore iph pointer after pbuf_to_mbuf() */ + h = mtod(m, struct ip *); + + m = pf_fragcache(&m, h, &frag, mff, (r->rule_flag & PFRULE_FRAGDROP) ? 1 : 0, &nomem); if (m == NULL) { + // Note: pf_fragcache() has already m_freem'd the mbuf if (nomem) goto no_mem; goto drop; } VERIFY(m->m_flags & M_PKTHDR); + pbuf_init_mbuf(pbuf, m, ifp); /* use mtag from copied and trimmed mbuf chain */ - pd->pf_mtag = pf_find_mtag(m); + pd->pf_mtag = pf_find_mtag_pbuf(pbuf); +#if 0 +// SCW: This check is superfluous #if DIAGNOSTIC if (pd->pf_mtag == NULL) { printf("%s: pf_find_mtag returned NULL(2)\n", __func__); if ((pd->pf_mtag = pf_get_mtag(m)) == NULL) { m_freem(m); - m = *m0 = NULL; + m = NULL; goto no_mem; } } +#endif #endif if (dir == PF_IN) pd->pf_mtag->pftag_flags |= PF_TAG_FRAGCACHE; if (frag != NULL && (frag->fr_flags & PFFRAG_DROP)) goto drop; + goto fragment_pass; } @@ -1747,7 +1968,11 @@ no_fragment: if (r->rule_flag & PFRULE_RANDOMID) { u_int16_t oip_id = h->ip_id; - h->ip_id = ip_randomid(); + if (rfc6864 && IP_OFF_IS_ATOMIC(ntohs(h->ip_off))) { + h->ip_id = 0; + } else { + h->ip_id = ip_randomid(); + } h->ip_sum = pf_cksum_fixup(h->ip_sum, oip_id, h->ip_id, 0); } if ((r->rule_flag & (PFRULE_FRAGCROP|PFRULE_FRAGDROP)) == 0) @@ -1769,15 +1994,15 @@ fragment_pass: no_mem: REASON_SET(reason, PFRES_MEMORY); - if (r != NULL && r->log) - PFLOG_PACKET(kif, h, m, AF_INET, dir, *reason, r, + if (r != NULL && r->log && pbuf_is_valid(pbuf)) + PFLOG_PACKET(kif, h, pbuf, AF_INET, dir, *reason, r, NULL, NULL, pd); return (PF_DROP); drop: REASON_SET(reason, PFRES_NORM); - if (r != NULL && r->log) - PFLOG_PACKET(kif, h, m, AF_INET, dir, *reason, r, + if (r != NULL && r->log && pbuf_is_valid(pbuf)) + PFLOG_PACKET(kif, h, pbuf, AF_INET, dir, *reason, r, NULL, NULL, pd); return (PF_DROP); @@ -1789,20 +2014,20 @@ bad: pf_free_fragment(frag); REASON_SET(reason, PFRES_FRAG); - if (r != NULL && r->log) - PFLOG_PACKET(kif, h, m, AF_INET, dir, *reason, r, NULL, NULL, pd); + if (r != NULL && r->log && pbuf_is_valid(pbuf)) + PFLOG_PACKET(kif, h, pbuf, AF_INET, dir, *reason, r, NULL, NULL, pd); return (PF_DROP); } #if INET6 int -pf_normalize_ip6(struct mbuf **m0, int dir, struct pfi_kif *kif, +pf_normalize_ip6(pbuf_t *pbuf, int dir, struct pfi_kif *kif, u_short *reason, struct pf_pdesc *pd) { - struct mbuf *m = *m0; + struct mbuf *m; struct pf_rule *r; - struct ip6_hdr *h = mtod(m, struct ip6_hdr *); + struct ip6_hdr *h = pbuf->pb_data; int off; struct ip6_ext ext; /* adi XXX */ @@ -1823,6 +2048,7 @@ pf_normalize_ip6(struct mbuf **m0, int dir, struct pfi_kif *kif, u_int16_t fr_max; int asd = 0; struct pf_ruleset *ruleset = NULL; + struct ifnet *ifp = pbuf->pb_ifp; r = TAILQ_FIRST(pf_main_ruleset.rules[PF_RULESET_SCRUB].active.ptr); while (r != NULL) { @@ -1838,11 +2064,11 @@ pf_normalize_ip6(struct mbuf **m0, int dir, struct pfi_kif *kif, r = r->skip[PF_SKIP_PROTO].ptr; #endif else if (PF_MISMATCHAW(&r->src.addr, - (struct pf_addr *)&h->ip6_src, AF_INET6, + (struct pf_addr *)(uintptr_t)&h->ip6_src, AF_INET6, r->src.neg, kif)) r = r->skip[PF_SKIP_SRC_ADDR].ptr; else if (PF_MISMATCHAW(&r->dst.addr, - (struct pf_addr *)&h->ip6_dst, AF_INET6, + (struct pf_addr *)(uintptr_t)&h->ip6_dst, AF_INET6, r->dst.neg, NULL)) r = r->skip[PF_SKIP_DST_ADDR].ptr; else { @@ -1865,7 +2091,8 @@ pf_normalize_ip6(struct mbuf **m0, int dir, struct pfi_kif *kif, } /* Check for illegal packets */ - if ((int)(sizeof (struct ip6_hdr) + IPV6_MAXPACKET) < m->m_pkthdr.len) + if ((uint32_t)(sizeof (struct ip6_hdr) + IPV6_MAXPACKET) < + pbuf->pb_packet_len) goto drop; off = sizeof (struct ip6_hdr); @@ -1879,7 +2106,7 @@ pf_normalize_ip6(struct mbuf **m0, int dir, struct pfi_kif *kif, case IPPROTO_AH: case IPPROTO_ROUTING: case IPPROTO_DSTOPTS: - if (!pf_pull_hdr(m, off, &ext, sizeof (ext), NULL, + if (!pf_pull_hdr(pbuf, off, &ext, sizeof (ext), NULL, NULL, AF_INET6)) goto shortpkt; /* @@ -1964,7 +2191,7 @@ pf_normalize_ip6(struct mbuf **m0, int dir, struct pfi_kif *kif, plen = ntohs(h->ip6_plen); if (plen == 0) goto drop; - if ((int)(sizeof (struct ip6_hdr) + plen) > m->m_pkthdr.len) + if ((uint32_t)(sizeof (struct ip6_hdr) + plen) > pbuf->pb_packet_len) goto shortpkt; /* Enforce a minimum ttl, may cause endless packet loops */ @@ -1978,7 +2205,7 @@ fragment: goto drop; plen = ntohs(h->ip6_plen); - if (!pf_pull_hdr(m, off, &frag, sizeof (frag), NULL, NULL, AF_INET6)) + if (!pf_pull_hdr(pbuf, off, &frag, sizeof (frag), NULL, NULL, AF_INET6)) goto shortpkt; fragoff = ntohs(frag.ip6f_offlg & IP6F_OFF_MASK); pd->proto = frag.ip6f_nxt; @@ -1988,9 +2215,10 @@ fragment: goto badfrag; fr_max = fragoff + plen - (off - sizeof(struct ip6_hdr)); - DPFPRINTF(("0x%llx IPv6 frag plen %u mff %d off %u fragoff %u " - "fr_max %u\n", (uint64_t)VM_KERNEL_ADDRPERM(m), plen, mff, off, - fragoff, fr_max)); +// XXX SCW: mbuf-specific +// DPFPRINTF(("0x%llx IPv6 frag plen %u mff %d off %u fragoff %u " +// "fr_max %u\n", (uint64_t)VM_KERNEL_ADDRPERM(m), plen, mff, off, +// fragoff, fr_max)); if ((r->rule_flag & (PFRULE_FRAGCROP|PFRULE_FRAGDROP)) == 0) { /* Fully buffer all of the fragments */ @@ -2003,13 +2231,22 @@ fragment: if (pff != NULL && (pff->fr_flags & PFFRAG_SEENLAST) && fr_max > pff->fr_max) goto badfrag; + + if ((m = pbuf_to_mbuf(pbuf, TRUE)) == NULL) { + REASON_SET(reason, PFRES_MEMORY); + return (PF_DROP); + } + /* Restore iph pointer after pbuf_to_mbuf() */ + h = mtod(m, struct ip6_hdr *); + /* Get an entry for the fragment queue */ frent = pool_get(&pf_frent_pl, PR_NOWAIT); if (frent == NULL) { REASON_SET(reason, PFRES_MEMORY); return (PF_DROP); } + pf_nfrents++; frent->fr_ip6 = h; frent->fr_m = m; @@ -2019,15 +2256,16 @@ fragment: /* Might return a completely reassembled mbuf, or NULL */ DPFPRINTF(("reass IPv6 frag %d @ %d-%d\n", ntohl(frag.ip6f_ident), fragoff, fr_max)); - *m0 = m = pf_reassemble6(m0, &pff, frent, mff); + m = pf_reassemble6(&m, &pff, frent, mff); if (m == NULL) return (PF_DROP); + + pbuf_init_mbuf(pbuf, m, ifp); + h = pbuf->pb_data; if (pff != NULL && (pff->fr_flags & PFFRAG_DROP)) goto drop; - - h = mtod(m, struct ip6_hdr *); } else if (dir == PF_IN || !(pd->pf_mtag->pftag_flags & PF_TAG_FRAGCACHE)) { /* non-buffering fragment cache (overlaps: see RFC 5722) */ @@ -2044,14 +2282,26 @@ fragment: goto badfrag; } - *m0 = m = pf_frag6cache(m0, h, &frag, &pff, off, mff, + if ((m = pbuf_to_mbuf(pbuf, TRUE)) == NULL) { + goto no_mem; + } + + /* Restore iph pointer after pbuf_to_mbuf() */ + h = mtod(m, struct ip6_hdr *); + + m = pf_frag6cache(&m, h, &frag, &pff, off, mff, (r->rule_flag & PFRULE_FRAGDROP) ? 1 : 0, &nomem); if (m == NULL) { + // Note: pf_frag6cache() has already m_freem'd the mbuf if (nomem) goto no_mem; goto drop; } + pbuf_init_mbuf(pbuf, m, ifp); + pd->pf_mtag = pf_find_mtag_pbuf(pbuf); + h = pbuf->pb_data; + if (dir == PF_IN) pd->pf_mtag->pftag_flags |= PF_TAG_FRAGCACHE; @@ -2084,14 +2334,14 @@ fragment: dropout: if (pff != NULL) pf_free_fragment(pff); - if (r != NULL && r->log) - PFLOG_PACKET(kif, h, m, AF_INET6, dir, *reason, r, NULL, NULL, pd); + if (r != NULL && r->log && pbuf_is_valid(pbuf)) + PFLOG_PACKET(kif, h, pbuf, AF_INET6, dir, *reason, r, NULL, NULL, pd); return (PF_DROP); } #endif /* INET6 */ int -pf_normalize_tcp(int dir, struct pfi_kif *kif, struct mbuf *m, int ipoff, +pf_normalize_tcp(int dir, struct pfi_kif *kif, pbuf_t *pbuf, int ipoff, int off, void *h, struct pf_pdesc *pd) { #pragma unused(ipoff, h) @@ -2134,7 +2384,7 @@ pf_normalize_tcp(int dir, struct pfi_kif *kif, struct mbuf *m, int ipoff, &r->dst.xport, &dxport)) r = r->skip[PF_SKIP_DST_PORT].ptr; else if (r->os_fingerprint != PF_OSFP_ANY && - !pf_osfp_match(pf_osfp_fingerprint(pd, m, off, th), + !pf_osfp_match(pf_osfp_fingerprint(pd, pbuf, off, th), r->os_fingerprint)) r = TAILQ_NEXT(r, entries); else { @@ -2208,25 +2458,24 @@ pf_normalize_tcp(int dir, struct pfi_kif *kif, struct mbuf *m, int ipoff, /* copy back packet headers if we sanitized */ /* Process options */ if (r->max_mss) { - int rv = pf_normalize_tcpopt(r, dir, kif, pd, m, th, off, + int rv = pf_normalize_tcpopt(r, dir, kif, pd, pbuf, th, off, &rewrite); if (rv == PF_DROP) return rv; - m = pd->mp; + pbuf = pd->mp; } if (rewrite) { - struct mbuf *mw = pf_lazy_makewritable(pd, m, - off + sizeof (*th)); - if (!mw) { + if (pf_lazy_makewritable(pd, pbuf, + off + sizeof (*th)) == NULL) { REASON_SET(&reason, PFRES_MEMORY); if (r->log) - PFLOG_PACKET(kif, h, m, AF_INET, dir, reason, + PFLOG_PACKET(kif, h, pbuf, AF_INET, dir, reason, r, 0, 0, pd); return PF_DROP; } - m_copyback(mw, off, sizeof (*th), th); + pbuf_copy_back(pbuf, off, sizeof (*th), th); } return (PF_PASS); @@ -2234,12 +2483,12 @@ pf_normalize_tcp(int dir, struct pfi_kif *kif, struct mbuf *m, int ipoff, tcp_drop: REASON_SET(&reason, PFRES_NORM); if (rm != NULL && r->log) - PFLOG_PACKET(kif, h, m, AF_INET, dir, reason, r, NULL, NULL, pd); + PFLOG_PACKET(kif, h, pbuf, AF_INET, dir, reason, r, NULL, NULL, pd); return (PF_DROP); } int -pf_normalize_tcp_init(struct mbuf *m, int off, struct pf_pdesc *pd, +pf_normalize_tcp_init(pbuf_t *pbuf, int off, struct pf_pdesc *pd, struct tcphdr *th, struct pf_state_peer *src, struct pf_state_peer *dst) { #pragma unused(dst) @@ -2257,14 +2506,14 @@ pf_normalize_tcp_init(struct mbuf *m, int off, struct pf_pdesc *pd, switch (pd->af) { #if INET case AF_INET: { - struct ip *h = mtod(m, struct ip *); + struct ip *h = pbuf->pb_data; src->scrub->pfss_ttl = h->ip_ttl; break; } #endif /* INET */ #if INET6 case AF_INET6: { - struct ip6_hdr *h = mtod(m, struct ip6_hdr *); + struct ip6_hdr *h = pbuf->pb_data; src->scrub->pfss_ttl = h->ip6_hlim; break; } @@ -2281,7 +2530,7 @@ pf_normalize_tcp_init(struct mbuf *m, int off, struct pf_pdesc *pd, if (th->th_off > (sizeof (struct tcphdr) >> 2) && src->scrub && - pf_pull_hdr(m, off, hdr, th->th_off << 2, NULL, NULL, pd->af)) { + pf_pull_hdr(pbuf, off, hdr, th->th_off << 2, NULL, NULL, pd->af)) { /* Diddle with TCP options */ int hlen; opt = hdr + sizeof (struct tcphdr); @@ -2334,12 +2583,12 @@ pf_normalize_tcp_cleanup(struct pf_state *state) } int -pf_normalize_tcp_stateful(struct mbuf *m, int off, struct pf_pdesc *pd, +pf_normalize_tcp_stateful(pbuf_t *pbuf, int off, struct pf_pdesc *pd, u_short *reason, struct tcphdr *th, struct pf_state *state, struct pf_state_peer *src, struct pf_state_peer *dst, int *writeback) { struct timeval uptime; - u_int32_t tsval, tsecr; + u_int32_t tsval = 0, tsecr = 0; u_int tsval_from_last; u_int8_t hdr[60]; u_int8_t *opt; @@ -2357,7 +2606,7 @@ pf_normalize_tcp_stateful(struct mbuf *m, int off, struct pf_pdesc *pd, #if INET case AF_INET: { if (src->scrub) { - struct ip *h = mtod(m, struct ip *); + struct ip *h = pbuf->pb_data; if (h->ip_ttl > src->scrub->pfss_ttl) src->scrub->pfss_ttl = h->ip_ttl; h->ip_ttl = src->scrub->pfss_ttl; @@ -2368,7 +2617,7 @@ pf_normalize_tcp_stateful(struct mbuf *m, int off, struct pf_pdesc *pd, #if INET6 case AF_INET6: { if (src->scrub) { - struct ip6_hdr *h = mtod(m, struct ip6_hdr *); + struct ip6_hdr *h = pbuf->pb_data; if (h->ip6_hlim > src->scrub->pfss_ttl) src->scrub->pfss_ttl = h->ip6_hlim; h->ip6_hlim = src->scrub->pfss_ttl; @@ -2381,7 +2630,7 @@ pf_normalize_tcp_stateful(struct mbuf *m, int off, struct pf_pdesc *pd, if (th->th_off > (sizeof (struct tcphdr) >> 2) && ((src->scrub && (src->scrub->pfss_flags & PFSS_TIMESTAMP)) || (dst->scrub && (dst->scrub->pfss_flags & PFSS_TIMESTAMP))) && - pf_pull_hdr(m, off, hdr, th->th_off << 2, NULL, NULL, pd->af)) { + pf_pull_hdr(pbuf, off, hdr, th->th_off << 2, NULL, NULL, pd->af)) { /* Diddle with TCP options */ int hlen; opt = hdr + sizeof (struct tcphdr); @@ -2451,13 +2700,13 @@ pf_normalize_tcp_stateful(struct mbuf *m, int off, struct pf_pdesc *pd, /* Copyback the options, caller copys back header */ int optoff = off + sizeof (*th); int optlen = (th->th_off << 2) - sizeof (*th); - m = pf_lazy_makewritable(pd, m, optoff + optlen); - if (!m) { + if (pf_lazy_makewritable(pd, pbuf, optoff + optlen) == + NULL) { REASON_SET(reason, PFRES_MEMORY); return PF_DROP; } *writeback = optoff + optlen; - m_copyback(m, optoff, optlen, hdr + sizeof (*th)); + pbuf_copy_back(pbuf, optoff, optlen, hdr + sizeof(*th)); } } @@ -2735,7 +2984,7 @@ pf_normalize_tcp_stateful(struct mbuf *m, int off, struct pf_pdesc *pd, static int pf_normalize_tcpopt(struct pf_rule *r, int dir, struct pfi_kif *kif, - struct pf_pdesc *pd, struct mbuf *m, struct tcphdr *th, int off, + struct pf_pdesc *pd, pbuf_t *pbuf, struct tcphdr *th, int off, int *rewrptr) { #pragma unused(dir, kif) @@ -2750,7 +2999,7 @@ pf_normalize_tcpopt(struct pf_rule *r, int dir, struct pfi_kif *kif, thoff = th->th_off << 2; cnt = thoff - sizeof (struct tcphdr); - if (cnt > 0 && !pf_pull_hdr(m, off + sizeof (*th), opts, cnt, + if (cnt > 0 && !pf_pull_hdr(pbuf, off + sizeof (*th), opts, cnt, NULL, NULL, af)) return PF_DROP; @@ -2776,8 +3025,8 @@ pf_normalize_tcpopt(struct pf_rule *r, int dir, struct pfi_kif *kif, * Only do the TCP checksum fixup if delayed * checksum calculation will not be performed. */ - if (m->m_pkthdr.rcvif || - !(m->m_pkthdr.csum_flags & CSUM_TCP)) + if (pbuf->pb_ifp || + !(*pbuf->pb_csum_flags & CSUM_TCP)) th->th_sum = pf_cksum_fixup(th->th_sum, *mss, htons(r->max_mss), 0); *mss = htons(r->max_mss); @@ -2790,21 +3039,21 @@ pf_normalize_tcpopt(struct pf_rule *r, int dir, struct pfi_kif *kif, } if (rewrite) { - struct mbuf *mw; u_short reason; - mw = pf_lazy_makewritable(pd, pd->mp, - off + sizeof (*th) + thoff); - if (!mw) { + VERIFY(pbuf == pd->mp); + + if (pf_lazy_makewritable(pd, pd->mp, + off + sizeof (*th) + thoff) == NULL) { REASON_SET(&reason, PFRES_MEMORY); if (r->log) - PFLOG_PACKET(kif, h, m, AF_INET, dir, reason, + PFLOG_PACKET(kif, h, pbuf, AF_INET, dir, reason, r, 0, 0, pd); return PF_DROP; } *rewrptr = 1; - m_copyback(mw, off + sizeof (*th), thoff - sizeof (*th), opts); + pbuf_copy_back(pd->mp, off + sizeof (*th), thoff - sizeof (*th), opts); } return PF_PASS; diff --git a/bsd/net/pf_osfp.c b/bsd/net/pf_osfp.c index e04a94e0f..2dc1e241c 100644 --- a/bsd/net/pf_osfp.c +++ b/bsd/net/pf_osfp.c @@ -86,7 +86,7 @@ static void pf_osfp_insert(struct pf_osfp_list *, struct pf_os_fingerprint *); * Returns the list of possible OSes. */ struct pf_osfp_enlist * -pf_osfp_fingerprint(struct pf_pdesc *pd, struct mbuf *m, int off, +pf_osfp_fingerprint(struct pf_pdesc *pd, pbuf_t *pbuf, int off, const struct tcphdr *tcp) { struct ip *ip; @@ -99,13 +99,13 @@ pf_osfp_fingerprint(struct pf_pdesc *pd, struct mbuf *m, int off, return (NULL); if (pd->af == PF_INET) { - ip = mtod(m, struct ip *); + ip = pbuf->pb_data; ip6 = (struct ip6_hdr *)NULL; } else { ip = (struct ip *)NULL; - ip6 = mtod(m, struct ip6_hdr *); + ip6 = pbuf->pb_data; } - if (!pf_pull_hdr(m, off, hdr, tcp->th_off << 2, NULL, NULL, + if (!pf_pull_hdr(pbuf, off, hdr, tcp->th_off << 2, NULL, NULL, pd->af)) return (NULL); diff --git a/bsd/net/pf_pbuf.c b/bsd/net/pf_pbuf.c new file mode 100644 index 000000000..a5b69b226 --- /dev/null +++ b/bsd/net/pf_pbuf.c @@ -0,0 +1,410 @@ +/* + * Copyright (c) 2016-2017 Apple Inc. All rights reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ + +#include <sys/cdefs.h> +#include <sys/systm.h> +#include <sys/param.h> +#include <sys/types.h> +#include <sys/mcache.h> +#include <kern/kern_types.h> +#include <net/pf_pbuf.h> +#include <netinet/in.h> + +void +pbuf_init_mbuf(pbuf_t *pbuf, struct mbuf *m, struct ifnet *ifp) +{ + + VERIFY((m->m_flags & M_PKTHDR) != 0); + + pbuf->pb_type = PBUF_TYPE_MBUF; + pbuf->pb_mbuf = m; + pbuf->pb_ifp = ifp; + pbuf->pb_next = NULL; + pbuf_sync(pbuf); +} + +void +pbuf_init_memory(pbuf_t *pbuf, const struct pbuf_memory *mp, struct ifnet *ifp) +{ + + pbuf->pb_type = PBUF_TYPE_MEMORY; + pbuf->pb_memory = *mp; + pbuf->pb_ifp = ifp; + pbuf->pb_next = NULL; + pbuf_sync(pbuf); +} + +void +pbuf_destroy(pbuf_t *pbuf) +{ + + if (pbuf->pb_type == PBUF_TYPE_MBUF) { + if (pbuf->pb_mbuf) { + m_freem(pbuf->pb_mbuf); + pbuf->pb_mbuf = NULL; + } + } else + if (pbuf->pb_type == PBUF_TYPE_MEMORY) { + VERIFY(pbuf->pb_memory.pm_buffer != NULL); + (void) (pbuf->pb_memory.pm_action)(&pbuf->pb_memory, + PBUF_ACTION_DESTROY); + } else { + VERIFY(pbuf->pb_type == PBUF_TYPE_ZOMBIE); + } + + memset(pbuf, 0, sizeof(*pbuf)); +} + +void +pbuf_sync(pbuf_t *pbuf) +{ + + if (pbuf->pb_type == PBUF_TYPE_MBUF) { + struct mbuf *m = pbuf->pb_mbuf; + + VERIFY(m != NULL); + VERIFY(m->m_flags & M_PKTHDR); + + pbuf->pb_data = mtod(m, void *); + pbuf->pb_packet_len = m->m_pkthdr.len; + pbuf->pb_contig_len = m->m_len; + pbuf->pb_csum_flags = &m->m_pkthdr.csum_flags; + pbuf->pb_csum_data = &m->m_pkthdr.csum_data; + pbuf->pb_proto = &m->m_pkthdr.pkt_proto; + pbuf->pb_flowsrc = &m->m_pkthdr.pkt_flowsrc; + pbuf->pb_flowid = &m->m_pkthdr.pkt_flowid; + pbuf->pb_flags = &m->m_pkthdr.pkt_flags; + pbuf->pb_pftag = m_pftag(m); + } else + if (pbuf->pb_type == PBUF_TYPE_MEMORY) { + struct pbuf_memory *nm = &pbuf->pb_memory; + + VERIFY(nm->pm_buffer != NULL); + VERIFY(nm->pm_buffer_len != 0); + VERIFY(nm->pm_len != 0); + VERIFY(nm->pm_len <= nm->pm_buffer_len); + VERIFY(nm->pm_offset < nm->pm_len); + + pbuf->pb_data = &nm->pm_buffer[nm->pm_offset]; + pbuf->pb_packet_len = nm->pm_len; + pbuf->pb_contig_len = nm->pm_len; + pbuf->pb_csum_flags = &nm->pm_csum_flags; + pbuf->pb_csum_data = &nm->pm_csum_data; + pbuf->pb_proto = &nm->pm_proto; + pbuf->pb_flowsrc = &nm->pm_flowsrc; + pbuf->pb_flowid = &nm->pm_flowid; + pbuf->pb_flags = &nm->pm_flags; + pbuf->pb_pftag = &nm->pm_pftag; + } else + panic("%s: bad pb_type: %d", __func__, pbuf->pb_type); +} + +struct mbuf * +pbuf_to_mbuf(pbuf_t *pbuf, boolean_t release_ptr) +{ + struct mbuf *m = NULL; + + pbuf_sync(pbuf); + + if (pbuf->pb_type == PBUF_TYPE_MBUF) { + m = pbuf->pb_mbuf; + if (release_ptr) { + pbuf->pb_mbuf = NULL; + pbuf_destroy(pbuf); + } + } else + if (pbuf->pb_type == PBUF_TYPE_MEMORY) { + if (pbuf->pb_packet_len > (u_int)MHLEN) { + if (pbuf->pb_packet_len > (u_int)MCLBYTES) { + printf("%s: packet too big for cluster (%u)\n", + __func__, pbuf->pb_packet_len); + return (NULL); + } + m = m_getcl(M_WAITOK, MT_DATA, M_PKTHDR); + } else { + m = m_gethdr(M_DONTWAIT, MT_DATA); + } + if (m == NULL) + return (NULL); + + m_copyback(m, 0, pbuf->pb_packet_len, pbuf->pb_data); + m->m_pkthdr.csum_flags = *pbuf->pb_csum_flags; + m->m_pkthdr.csum_data = *pbuf->pb_csum_data; + m->m_pkthdr.pkt_proto = *pbuf->pb_proto; + m->m_pkthdr.pkt_flowsrc = *pbuf->pb_flowsrc; + m->m_pkthdr.pkt_flowid = *pbuf->pb_flowid; + m->m_pkthdr.pkt_flags = *pbuf->pb_flags; + + if (pbuf->pb_pftag != NULL) { + struct pf_mtag *pftag = m_pftag(m); + + if (pftag != NULL) + *pftag = *pbuf->pb_pftag; + } + + if (release_ptr) + pbuf_destroy(pbuf); + } + + return (m); +} + +struct mbuf * +pbuf_clone_to_mbuf(pbuf_t *pbuf) +{ + struct mbuf *m = NULL; + + pbuf_sync(pbuf); + + if (pbuf->pb_type == PBUF_TYPE_MBUF) + m = m_copy(pbuf->pb_mbuf, 0, M_COPYALL); + else + if (pbuf->pb_type == PBUF_TYPE_MEMORY) + m = pbuf_to_mbuf(pbuf, FALSE); + else + panic("%s: bad pb_type: %d", __func__, pbuf->pb_type); + + return (m); +} + +void * +pbuf_ensure_writable(pbuf_t *pbuf, size_t len) +{ + + if (pbuf->pb_type == PBUF_TYPE_MBUF) { + struct mbuf *m = pbuf->pb_mbuf; + + if (m_makewritable(&pbuf->pb_mbuf, 0, len, M_DONTWAIT)) + return (NULL); + + if (pbuf->pb_mbuf == NULL) { + pbuf_destroy(pbuf); + return (NULL); + } + + if (m != pbuf->pb_mbuf) + pbuf_sync(pbuf); + + } else + if (pbuf->pb_type != PBUF_TYPE_MEMORY) + panic("%s: bad pb_type: %d", __func__, pbuf->pb_type); + + return (pbuf->pb_data); +} + +void * +pbuf_resize_segment(pbuf_t *pbuf, int off, int olen, int nlen) +{ + void *rv = NULL; + + VERIFY(off >= 0); + VERIFY((u_int)off <= pbuf->pb_packet_len); + + if (pbuf->pb_type == PBUF_TYPE_MBUF) { + struct mbuf *m, *n; + + VERIFY(pbuf->pb_mbuf != NULL); + + m = pbuf->pb_mbuf; + + if (off > 0) { + /* Split the mbuf chain at the specified boundary */ + if ((n = m_split(m, off, M_DONTWAIT)) == NULL) + return (NULL); + } else { + n = m; + } + + /* Trim old length */ + m_adj(n, olen); + + /* Prepend new length */ + if (M_PREPEND(n, nlen, M_DONTWAIT, 0) == NULL) + return (NULL); + + rv = mtod(n, void *); + + if (off > 0) { + /* Merge the two chains */ + int mlen; + + mlen = n->m_pkthdr.len; + m_cat(m, n); + m->m_pkthdr.len += mlen; + } else { + /* The new mbuf becomes the packet header */ + pbuf->pb_mbuf = n; + } + + pbuf_sync(pbuf); + } else + if (pbuf->pb_type == PBUF_TYPE_MEMORY) { + struct pbuf_memory *nm = &pbuf->pb_memory; + u_int true_offset, move_len; + int delta_len; + uint8_t *psrc, *pdst; + + delta_len = nlen - olen; + VERIFY(nm->pm_offset + (nm->pm_len + delta_len) <= + nm->pm_buffer_len); + + true_offset = (u_int)off + nm->pm_offset; + rv = &nm->pm_buffer[true_offset]; + psrc = &nm->pm_buffer[true_offset + (u_int)olen]; + pdst = &nm->pm_buffer[true_offset + (u_int)nlen]; + move_len = pbuf->pb_packet_len - ((u_int)off + olen); + memmove(pdst, psrc, move_len); + + nm->pm_len += delta_len; + + VERIFY((nm->pm_len + nm->pm_offset) <= nm->pm_buffer_len); + + pbuf_sync(pbuf); + } else + panic("pbuf_csum_flags_get: bad pb_type: %d", pbuf->pb_type); + + return (rv); +} + +void * +pbuf_contig_segment(pbuf_t *pbuf, int off, int len) +{ + void *rv = NULL; + + VERIFY(off >= 0); + VERIFY(len >= 0); + VERIFY((u_int)(off + len) < pbuf->pb_packet_len); + + /* + * Note: If this fails, then the pbuf is destroyed. This is a + * side-effect of m_pulldown(). + * + * PF expects this behaviour so it's not a real problem. + */ + + if (pbuf->pb_type == PBUF_TYPE_MBUF) { + struct mbuf *n; + int moff; + + n = m_pulldown(pbuf->pb_mbuf, off, len, &moff); + if (n == NULL) { + /* mbuf is freed by m_pulldown() in this case */ + pbuf->pb_mbuf = NULL; + pbuf_destroy(pbuf); + return (NULL); + } + + pbuf_sync(pbuf); + + rv = (void *)(mtod(n, uint8_t *) + moff); + } else + if (pbuf->pb_type == PBUF_TYPE_MEMORY) { + /* + * This always succeeds since memory pbufs are fully contig. + */ + rv = (void *)(uintptr_t)(((uint8_t *)pbuf->pb_data)[off]); + } else + panic("%s: bad pb_type: %d", __func__, pbuf->pb_type); + + return (rv); +} + +void +pbuf_copy_back(pbuf_t *pbuf, int off, int len, void *src) +{ + + VERIFY(off >= 0); + VERIFY(len >= 0); + VERIFY((u_int)(off + len) <= pbuf->pb_packet_len); + + if (pbuf->pb_type == PBUF_TYPE_MBUF) + m_copyback(pbuf->pb_mbuf, off, len, src); + else + if (pbuf->pb_type == PBUF_TYPE_MBUF) { + if (len) + memcpy(&((uint8_t *)pbuf->pb_data)[off], src, len); + } else + panic("%s: bad pb_type: %d", __func__, pbuf->pb_type); +} + +void +pbuf_copy_data(pbuf_t *pbuf, int off, int len, void *dst) +{ + + VERIFY(off >= 0); + VERIFY(len >= 0); + VERIFY((u_int)(off + len) <= pbuf->pb_packet_len); + + if (pbuf->pb_type == PBUF_TYPE_MBUF) + m_copydata(pbuf->pb_mbuf, off, len, dst); + else + if (pbuf->pb_type == PBUF_TYPE_MBUF) { + if (len) + memcpy(dst, &((uint8_t *)pbuf->pb_data)[off], len); + } else + panic("%s: bad pb_type: %d", __func__, pbuf->pb_type); +} + +uint16_t +pbuf_inet_cksum(const pbuf_t *pbuf, uint32_t nxt, uint32_t off, uint32_t len) +{ + uint16_t sum = 0; + + if (pbuf->pb_type == PBUF_TYPE_MBUF) + sum = inet_cksum(pbuf->pb_mbuf, nxt, off, len); + else + if (pbuf->pb_type == PBUF_TYPE_MEMORY) + sum = inet_cksum_buffer(pbuf->pb_data, nxt, off, len); + else + panic("%s: bad pb_type: %d", __func__, pbuf->pb_type); + + return (sum); +} + +uint16_t +pbuf_inet6_cksum(const pbuf_t *pbuf, uint32_t nxt, uint32_t off, uint32_t len) +{ + uint16_t sum = 0; + + if (pbuf->pb_type == PBUF_TYPE_MBUF) + sum = inet6_cksum(pbuf->pb_mbuf, nxt, off, len); + else + if (pbuf->pb_type == PBUF_TYPE_MEMORY) + sum = inet6_cksum_buffer(pbuf->pb_data, nxt, off, len); + else + panic("%s: bad pb_type: %d", __func__, pbuf->pb_type); + + return (sum); +} + +mbuf_svc_class_t +pbuf_get_service_class(const pbuf_t *pbuf) +{ + + if (pbuf->pb_type == PBUF_TYPE_MBUF) + return m_get_service_class(pbuf->pb_mbuf); + + VERIFY(pbuf->pb_type == PBUF_TYPE_MEMORY); + + return (MBUF_SC_BE); +} diff --git a/bsd/net/pf_pbuf.h b/bsd/net/pf_pbuf.h new file mode 100644 index 000000000..55c7f0aa8 --- /dev/null +++ b/bsd/net/pf_pbuf.h @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2016 Apple Inc. All rights reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ + +#ifndef __PBUF_H__ +#define __PBUF_H__ + +#include <sys/mbuf.h> + +enum pbuf_type { + PBUF_TYPE_ZOMBIE = 0, + PBUF_TYPE_MBUF, + PBUF_TYPE_MEMORY +}; + +enum pbuf_action { + PBUF_ACTION_DESTROY +}; + +#define PBUF_ACTION_RV_SUCCESS 0 +#define PBUF_ACTION_RV_FAILURE (-1) + +struct pbuf_memory { + uint8_t *pm_buffer; // Pointer to start of buffer + u_int pm_buffer_len; // Total length of buffer + u_int pm_offset; // Offset to start of payload + u_int pm_len; // Length of payload + uint32_t pm_csum_flags; + uint32_t pm_csum_data; + uint8_t pm_proto; + uint8_t pm_flowsrc; + uint32_t pm_flowid; + uint32_t pm_flags; + struct pf_mtag pm_pftag; + int (*pm_action)(struct pbuf_memory *, enum pbuf_action); + void *pm_action_cookie; +}; + +typedef struct pbuf { + enum pbuf_type pb_type; + union { + struct mbuf *pbu_mbuf; + struct pbuf_memory pbu_memory; + } pb_u; +#define pb_mbuf pb_u.pbu_mbuf +#define pb_memory pb_u.pbu_memory + + void *pb_data; + uint32_t pb_packet_len; + uint32_t pb_contig_len; + uint32_t *pb_csum_flags; + uint32_t *pb_csum_data; + uint8_t *pb_proto; + uint8_t *pb_flowsrc; + uint32_t *pb_flowid; + uint32_t *pb_flags; + struct pf_mtag *pb_pftag; + struct ifnet *pb_ifp; + struct pbuf *pb_next; +} pbuf_t; + +#define pbuf_is_valid(pb) (!((pb) == NULL || (pb)->pb_type == PBUF_TYPE_ZOMBIE)) + +void pbuf_init_mbuf(pbuf_t *, struct mbuf *, struct ifnet *); +void pbuf_init_memory(pbuf_t *, const struct pbuf_memory *, + struct ifnet *); +void pbuf_destroy(pbuf_t *); +void pbuf_sync(pbuf_t *); + +struct mbuf *pbuf_to_mbuf(pbuf_t *, boolean_t); +struct mbuf *pbuf_clone_to_mbuf(pbuf_t *); + +void * pbuf_ensure_contig(pbuf_t *, size_t); +void * pbuf_ensure_writable(pbuf_t *, size_t); + +void * pbuf_resize_segment(pbuf_t *, int off, int olen, int nlen); +void * pbuf_contig_segment(pbuf_t *, int off, int len); + +void pbuf_copy_data(pbuf_t *, int, int, void *); +void pbuf_copy_back(pbuf_t *, int, int, void *); + +uint16_t pbuf_inet_cksum(const pbuf_t *, uint32_t, uint32_t, uint32_t); +uint16_t pbuf_inet6_cksum(const pbuf_t *, uint32_t, uint32_t, uint32_t); + +mbuf_svc_class_t pbuf_get_service_class(const pbuf_t *); + +#endif /* __PBUF_H__ */ diff --git a/bsd/net/pf_ruleset.c b/bsd/net/pf_ruleset.c index f03f3297f..ff4c3f904 100644 --- a/bsd/net/pf_ruleset.c +++ b/bsd/net/pf_ruleset.c @@ -74,6 +74,7 @@ #endif /* KERNEL */ #include <sys/mbuf.h> +#include <netinet/ip_dummynet.h> #include <netinet/in.h> #include <netinet/in_systm.h> #include <netinet/ip.h> @@ -245,7 +246,7 @@ pf_find_ruleset_with_owner(const char *path, const char *owner, int is_anchor, struct pf_ruleset * pf_find_or_create_ruleset(const char *path) { - char *p, *q, *r; + char *p, *q = NULL, *r; struct pf_ruleset *ruleset; struct pf_anchor *anchor = 0, *dup, *parent = NULL; @@ -329,6 +330,11 @@ pf_find_or_create_ruleset(const char *path) q = r + 1; else *q = 0; +#if DUMMYNET + if(strncmp("com.apple.nlc", anchor->name, + sizeof("com.apple.nlc")) == 0) + is_nlc_enabled_glb = TRUE; +#endif } rs_free(p); return (anchor ? &anchor->ruleset : 0); @@ -352,6 +358,16 @@ pf_remove_if_empty_ruleset(struct pf_ruleset *ruleset) ruleset->rules[i].inactive.open) return; RB_REMOVE(pf_anchor_global, &pf_anchors, ruleset->anchor); +#if DUMMYNET + if(strncmp("com.apple.nlc", ruleset->anchor->name, + sizeof("com.apple.nlc")) == 0) { + struct dummynet_event dn_event; + bzero(&dn_event, sizeof(dn_event)); + dn_event.dn_event_code = DUMMYNET_NLC_DISABLED; + dummynet_event_enqueue_nwk_wq_entry(&dn_event); + is_nlc_enabled_glb = FALSE; + } +#endif if ((parent = ruleset->anchor->parent) != NULL) RB_REMOVE(pf_anchor_node, &parent->children, ruleset->anchor); diff --git a/bsd/net/pf_table.c b/bsd/net/pf_table.c index 17ff0ab1a..5ccaf0426 100644 --- a/bsd/net/pf_table.c +++ b/bsd/net/pf_table.c @@ -793,7 +793,7 @@ pfr_lookup_addr(struct pfr_ktable *kt, struct pfr_addr *ad, int exact) struct radix_node_head *head; struct pfr_kentry *ke; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); bzero(&sa, sizeof (sa)); if (ad->pfra_af == AF_INET) { @@ -938,7 +938,7 @@ pfr_clstats_kentries(struct pfr_kentryworkq *workq, u_int64_t tzero, { struct pfr_kentry *p; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); SLIST_FOREACH(p, workq, pfrke_workq) { if (negchange) @@ -996,7 +996,7 @@ pfr_route_kentry(struct pfr_ktable *kt, struct pfr_kentry *ke) struct radix_node *rn; struct radix_node_head *head; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); bzero(ke->pfrke_node, sizeof (ke->pfrke_node)); if (ke->pfrke_af == AF_INET) @@ -1022,7 +1022,7 @@ pfr_unroute_kentry(struct pfr_ktable *kt, struct pfr_kentry *ke) struct radix_node *rn; struct radix_node_head *head; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (ke->pfrke_af == AF_INET) head = kt->pfrkt_ip4; @@ -1066,7 +1066,7 @@ pfr_walktree(struct radix_node *rn, void *arg) struct pfr_walktree *w = arg; int flags = w->pfrw_flags; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); switch (w->pfrw_op) { case PFRW_MARK: @@ -1150,7 +1150,7 @@ pfr_clr_tables(struct pfr_table *filter, int *ndel, int flags) struct pfr_ktable *p; int xdel = 0; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); ACCEPT_FLAGS(flags, PFR_FLAG_ATOMIC | PFR_FLAG_DUMMY | PFR_FLAG_ALLRSETS); @@ -1187,7 +1187,7 @@ pfr_add_tables(user_addr_t tbl, int size, int *nadd, int flags) int i, rv, xadd = 0; u_int64_t tzero = pf_calendar_time_second(); - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); ACCEPT_FLAGS(flags, PFR_FLAG_ATOMIC | PFR_FLAG_DUMMY); SLIST_INIT(&addq); @@ -1265,7 +1265,7 @@ pfr_del_tables(user_addr_t tbl, int size, int *ndel, int flags) struct pfr_ktable *p, *q, key; int i, xdel = 0; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); ACCEPT_FLAGS(flags, PFR_FLAG_ATOMIC | PFR_FLAG_DUMMY); SLIST_INIT(&workq); @@ -1340,7 +1340,7 @@ pfr_get_tstats(struct pfr_table *filter, user_addr_t tbl, int *size, int n, nn; u_int64_t tzero = pf_calendar_time_second(); - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); /* XXX PFR_FLAG_CLSTATS disabled */ ACCEPT_FLAGS(flags, PFR_FLAG_ATOMIC | PFR_FLAG_ALLRSETS); @@ -1384,7 +1384,7 @@ pfr_clr_tstats(user_addr_t tbl, int size, int *nzero, int flags) int i, xzero = 0; u_int64_t tzero = pf_calendar_time_second(); - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); ACCEPT_FLAGS(flags, PFR_FLAG_ATOMIC | PFR_FLAG_DUMMY | PFR_FLAG_ADDRSTOO); @@ -1417,7 +1417,7 @@ pfr_set_tflags(user_addr_t tbl, int size, int setflag, int clrflag, struct pfr_ktable *p, *q, key; int i, xchange = 0, xdel = 0; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); ACCEPT_FLAGS(flags, PFR_FLAG_ATOMIC | PFR_FLAG_DUMMY); if ((setflag & ~PFR_TFLAG_USRMASK) || @@ -1470,7 +1470,7 @@ pfr_ina_begin(struct pfr_table *trs, u_int32_t *ticket, int *ndel, int flags) struct pf_ruleset *rs; int xdel = 0; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); ACCEPT_FLAGS(flags, PFR_FLAG_DUMMY); rs = pf_find_or_create_ruleset(trs->pfrt_anchor); @@ -1509,7 +1509,7 @@ pfr_ina_define(struct pfr_table *tbl, user_addr_t addr, int size, struct pf_ruleset *rs; int i, rv, xadd = 0, xaddr = 0; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); ACCEPT_FLAGS(flags, PFR_FLAG_DUMMY | PFR_FLAG_ADDRSTOO); if (size && !(flags & PFR_FLAG_ADDRSTOO)) @@ -1608,7 +1608,7 @@ pfr_ina_rollback(struct pfr_table *trs, u_int32_t ticket, int *ndel, int flags) struct pf_ruleset *rs; int xdel = 0; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); ACCEPT_FLAGS(flags, PFR_FLAG_DUMMY); rs = pf_find_ruleset(trs->pfrt_anchor); @@ -1643,7 +1643,7 @@ pfr_ina_commit(struct pfr_table *trs, u_int32_t ticket, int *nadd, int xadd = 0, xchange = 0; u_int64_t tzero = pf_calendar_time_second(); - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); ACCEPT_FLAGS(flags, PFR_FLAG_ATOMIC | PFR_FLAG_DUMMY); rs = pf_find_ruleset(trs->pfrt_anchor); @@ -1684,7 +1684,7 @@ pfr_commit_ktable(struct pfr_ktable *kt, u_int64_t tzero) struct pfr_ktable *shadow = kt->pfrkt_shadow; int nflags; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (shadow->pfrkt_cnt == NO_ADDRESSES) { if (!(kt->pfrkt_flags & PFR_TFLAG_ACTIVE)) @@ -1825,7 +1825,7 @@ pfr_insert_ktables(struct pfr_ktableworkq *workq) { struct pfr_ktable *p; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); SLIST_FOREACH(p, workq, pfrkt_workq) pfr_insert_ktable(p); @@ -1834,7 +1834,7 @@ pfr_insert_ktables(struct pfr_ktableworkq *workq) static void pfr_insert_ktable(struct pfr_ktable *kt) { - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); RB_INSERT(pfr_ktablehead, &pfr_ktables, kt); pfr_ktable_cnt++; @@ -1849,7 +1849,7 @@ pfr_setflags_ktables(struct pfr_ktableworkq *workq) { struct pfr_ktable *p, *q; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); for (p = SLIST_FIRST(workq); p; p = q) { q = SLIST_NEXT(p, pfrkt_workq); @@ -1862,7 +1862,7 @@ pfr_setflags_ktable(struct pfr_ktable *kt, int newf) { struct pfr_kentryworkq addrq; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (!(newf & PFR_TFLAG_REFERENCED) && !(newf & PFR_TFLAG_PERSIST)) @@ -1896,7 +1896,7 @@ pfr_clstats_ktables(struct pfr_ktableworkq *workq, u_int64_t tzero, int recurse) { struct pfr_ktable *p; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); SLIST_FOREACH(p, workq, pfrkt_workq) pfr_clstats_ktable(p, tzero, recurse); @@ -1907,7 +1907,7 @@ pfr_clstats_ktable(struct pfr_ktable *kt, u_int64_t tzero, int recurse) { struct pfr_kentryworkq addrq; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (recurse) { pfr_enqueue_addrs(kt, &addrq, NULL, 0); @@ -1925,7 +1925,7 @@ pfr_create_ktable(struct pfr_table *tbl, u_int64_t tzero, int attachruleset) struct pfr_ktable *kt; struct pf_ruleset *rs; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); kt = pool_get(&pfr_ktable_pl, PR_WAITOK); if (kt == NULL) @@ -1960,7 +1960,7 @@ pfr_destroy_ktables(struct pfr_ktableworkq *workq, int flushaddr) { struct pfr_ktable *p, *q; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); for (p = SLIST_FIRST(workq); p; p = q) { q = SLIST_NEXT(p, pfrkt_workq); @@ -1973,7 +1973,7 @@ pfr_destroy_ktable(struct pfr_ktable *kt, int flushaddr) { struct pfr_kentryworkq addrq; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (flushaddr) { pfr_enqueue_addrs(kt, &addrq, NULL, 0); @@ -2006,7 +2006,7 @@ pfr_ktable_compare(struct pfr_ktable *p, struct pfr_ktable *q) static struct pfr_ktable * pfr_lookup_table(struct pfr_table *tbl) { - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); /* struct pfr_ktable start like a struct pfr_table */ return (RB_FIND(pfr_ktablehead, &pfr_ktables, @@ -2019,7 +2019,7 @@ pfr_match_addr(struct pfr_ktable *kt, struct pf_addr *a, sa_family_t af) struct pfr_kentry *ke = NULL; int match; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (!(kt->pfrkt_flags & PFR_TFLAG_ACTIVE) && kt->pfrkt_root != NULL) kt = kt->pfrkt_root; @@ -2058,7 +2058,7 @@ pfr_update_stats(struct pfr_ktable *kt, struct pf_addr *a, sa_family_t af, { struct pfr_kentry *ke = NULL; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (!(kt->pfrkt_flags & PFR_TFLAG_ACTIVE) && kt->pfrkt_root != NULL) kt = kt->pfrkt_root; @@ -2105,7 +2105,7 @@ pfr_attach_table(struct pf_ruleset *rs, char *name) struct pfr_table tbl; struct pf_anchor *ac = rs->anchor; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); bzero(&tbl, sizeof (tbl)); strlcpy(tbl.pfrt_name, name, sizeof (tbl.pfrt_name)); @@ -2139,7 +2139,7 @@ pfr_attach_table(struct pf_ruleset *rs, char *name) void pfr_detach_table(struct pfr_ktable *kt) { - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (kt->pfrkt_refcnt[PFR_REFCNT_RULE] <= 0) printf("pfr_detach_table: refcount = %d.\n", @@ -2157,7 +2157,7 @@ pfr_pool_get(struct pfr_ktable *kt, int *pidx, struct pf_addr *counter, union sockaddr_union mask; int idx = -1, use_counter = 0; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); if (af == AF_INET) addr = (struct pf_addr *)&pfr_sin.sin_addr; @@ -2247,7 +2247,7 @@ pfr_kentry_byidx(struct pfr_ktable *kt, int idx, int af) { struct pfr_walktree w; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); bzero(&w, sizeof (w)); w.pfrw_op = PFRW_POOL_GET; @@ -2276,7 +2276,7 @@ pfr_dynaddr_update(struct pfr_ktable *kt, struct pfi_dynaddr *dyn) { struct pfr_walktree w; - lck_mtx_assert(pf_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(pf_lock, LCK_MTX_ASSERT_OWNED); bzero(&w, sizeof (w)); w.pfrw_op = PFRW_DYNADDR_UPDATE; diff --git a/bsd/net/pfkeyv2.h b/bsd/net/pfkeyv2.h index 97d6280fa..74ec44859 100644 --- a/bsd/net/pfkeyv2.h +++ b/bsd/net/pfkeyv2.h @@ -432,6 +432,7 @@ struct sadb_sastat { #define SADB_X_EALG_AESCBC 12 #define SADB_X_EALG_AES 12 #define SADB_X_EALG_AES_GCM 13 +#define SADB_X_EALG_CHACHA20POLY1305 14 /* private allocations should use 249-255 (RFC2407) */ #if 1 /*nonstandard */ @@ -470,6 +471,8 @@ struct sadb_sastat { #define SADB_X_EXT_PZERO 0x0200 /* zero padding for ESP */ #define SADB_X_EXT_PMASK 0x0300 /* mask for padding flag */ +#define SADB_X_EXT_IIV 0x0400 /* Implicit IV */ + #ifdef PRIVATE #define SADB_X_EXT_NATT_DETECTED_PEER 0x1000 #define SADB_X_EXT_ESP_KEEPALIVE 0x2000 diff --git a/bsd/net/pfvar.h b/bsd/net/pfvar.h index 11e771bf5..d7ea4c6d1 100644 --- a/bsd/net/pfvar.h +++ b/bsd/net/pfvar.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2007-2015 Apple Inc. All rights reserved. + * Copyright (c) 2007-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -91,6 +91,8 @@ extern "C" { #include <machine/endian.h> #include <sys/systm.h> +#include <net/pf_pbuf.h> + #if BYTE_ORDER == BIG_ENDIAN #define htobe64(x) (x) @@ -231,17 +233,17 @@ enum { PF_ADDR_ADDRMASK, PF_ADDR_NOROUTE, PF_ADDR_DYNIFTL, struct pf_addr { union { - struct in_addr v4; - struct in6_addr v6; - u_int8_t addr8[16]; - u_int16_t addr16[8]; - u_int32_t addr32[4]; + struct in_addr _v4addr; + struct in6_addr _v6addr; + u_int8_t _addr8[16]; + u_int16_t _addr16[8]; + u_int32_t _addr32[4]; } pfa; /* 128-bit address */ -#define v4 pfa.v4 -#define v6 pfa.v6 -#define addr8 pfa.addr8 -#define addr16 pfa.addr16 -#define addr32 pfa.addr32 +#define v4addr pfa._v4addr +#define v6addr pfa._v6addr +#define addr8 pfa._addr8 +#define addr16 pfa._addr16 +#define addr32 pfa._addr32 }; #define PF_TABLE_NAME_SIZE 32 @@ -1431,7 +1433,7 @@ struct pf_pdesc { struct pf_addr *dst; struct ether_header *eh; - struct mbuf *mp; + pbuf_t *mp; int lmw; /* lazy writable offset */ struct pf_mtag *pf_mtag; u_int16_t *ip_sum; @@ -2165,15 +2167,6 @@ TAILQ_HEAD(pf_poolqueue, pf_pool); extern struct pf_poolqueue pf_pools[2]; extern struct pf_palist pf_pabuf; extern u_int32_t ticket_pabuf; -#if PF_ALTQ -TAILQ_HEAD(pf_altqqueue, pf_altq); -extern struct pf_altqqueue pf_altqs[2]; -extern u_int32_t ticket_altqs_active; -extern u_int32_t ticket_altqs_inactive; -extern int altqs_inactive_open; -extern struct pf_altqqueue *pf_altqs_active; -extern struct pf_altqqueue *pf_altqs_inactive; -#endif /* PF_ALTQ */ extern struct pf_poolqueue *pf_pools_active; extern struct pf_poolqueue *pf_pools_inactive; @@ -2187,9 +2180,6 @@ __private_extern__ u_int32_t pf_calc_state_key_flowhash(struct pf_state_key *); extern struct pool pf_src_tree_pl, pf_rule_pl; extern struct pool pf_state_pl, pf_state_key_pl, pf_pooladdr_pl; extern struct pool pf_state_scrub_pl; -#if PF_ALTQ -extern struct pool pf_altq_pl; -#endif /* PF_ALTQ */ extern struct pool pf_app_state_pl; extern struct thread *pf_purge_thread; @@ -2219,25 +2209,33 @@ __private_extern__ void pf_addrcpy(struct pf_addr *, struct pf_addr *, __private_extern__ void pf_rm_rule(struct pf_rulequeue *, struct pf_rule *); struct ip_fw_args; + +extern boolean_t is_nlc_enabled_glb; +extern boolean_t pf_is_nlc_enabled(void); + #if INET -__private_extern__ int pf_test(int, struct ifnet *, struct mbuf **, +__private_extern__ int pf_test(int, struct ifnet *, pbuf_t **, + struct ether_header *, struct ip_fw_args *); +__private_extern__ int pf_test_mbuf(int, struct ifnet *, struct mbuf **, struct ether_header *, struct ip_fw_args *); #endif /* INET */ #if INET6 -__private_extern__ int pf_test6(int, struct ifnet *, struct mbuf **, +__private_extern__ int pf_test6(int, struct ifnet *, pbuf_t **, + struct ether_header *, struct ip_fw_args *); +__private_extern__ int pf_test6_mbuf(int, struct ifnet *, struct mbuf **, struct ether_header *, struct ip_fw_args *); __private_extern__ void pf_poolmask(struct pf_addr *, struct pf_addr *, struct pf_addr *, struct pf_addr *, u_int8_t); __private_extern__ void pf_addr_inc(struct pf_addr *, sa_family_t); #endif /* INET6 */ -__private_extern__ struct mbuf *pf_lazy_makewritable(struct pf_pdesc *, - struct mbuf *, int); -__private_extern__ void *pf_pull_hdr(struct mbuf *, int, void *, int, +__private_extern__ void *pf_lazy_makewritable(struct pf_pdesc *, + pbuf_t *, int); +__private_extern__ void *pf_pull_hdr(pbuf_t *, int, void *, int, u_short *, u_short *, sa_family_t); __private_extern__ void pf_change_a(void *, u_int16_t *, u_int32_t, u_int8_t); -__private_extern__ int pflog_packet(struct pfi_kif *, struct mbuf *, +__private_extern__ int pflog_packet(struct pfi_kif *, pbuf_t *, sa_family_t, u_int8_t, u_int8_t, struct pf_rule *, struct pf_rule *, struct pf_ruleset *, struct pf_pdesc *); __private_extern__ int pf_match_addr(u_int8_t, struct pf_addr *, @@ -2253,17 +2251,17 @@ __private_extern__ int pf_match_gid(u_int8_t, gid_t, gid_t, gid_t); __private_extern__ void pf_normalize_init(void); __private_extern__ int pf_normalize_isempty(void); -__private_extern__ int pf_normalize_ip(struct mbuf **, int, struct pfi_kif *, +__private_extern__ int pf_normalize_ip(pbuf_t *, int, struct pfi_kif *, u_short *, struct pf_pdesc *); -__private_extern__ int pf_normalize_ip6(struct mbuf **, int, struct pfi_kif *, +__private_extern__ int pf_normalize_ip6(pbuf_t *, int, struct pfi_kif *, u_short *, struct pf_pdesc *); -__private_extern__ int pf_normalize_tcp(int, struct pfi_kif *, struct mbuf *, +__private_extern__ int pf_normalize_tcp(int, struct pfi_kif *, pbuf_t *, int, int, void *, struct pf_pdesc *); __private_extern__ void pf_normalize_tcp_cleanup(struct pf_state *); -__private_extern__ int pf_normalize_tcp_init(struct mbuf *, int, +__private_extern__ int pf_normalize_tcp_init(pbuf_t *, int, struct pf_pdesc *, struct tcphdr *, struct pf_state_peer *, struct pf_state_peer *); -__private_extern__ int pf_normalize_tcp_stateful(struct mbuf *, int, +__private_extern__ int pf_normalize_tcp_stateful(pbuf_t *, int, struct pf_pdesc *, u_short *, struct tcphdr *, struct pf_state *, struct pf_state_peer *, struct pf_state_peer *, int *); __private_extern__ u_int64_t pf_state_expires(const struct pf_state *); @@ -2347,7 +2345,7 @@ __private_extern__ u_int16_t pf_tagname2tag(char *); __private_extern__ void pf_tag2tagname(u_int16_t, char *); __private_extern__ void pf_tag_ref(u_int16_t); __private_extern__ void pf_tag_unref(u_int16_t); -__private_extern__ int pf_tag_packet(struct mbuf *, struct pf_mtag *, +__private_extern__ int pf_tag_packet(pbuf_t *, struct pf_mtag *, int, unsigned int, struct pf_pdesc *); __private_extern__ void pf_step_into_anchor(int *, struct pf_ruleset **, int, struct pf_rule **, struct pf_rule **, int *); @@ -2384,10 +2382,6 @@ extern int16_t pf_nat64_configured; #define PF_IS_ENABLED (pf_is_enabled != 0) extern u_int32_t pf_hash_seed; -#if PF_ALTQ -extern u_int32_t altq_allowed; -#endif /* PF_ALTQ */ - /* these ruleset functions can be linked into userland programs (pfctl) */ __private_extern__ int pf_get_ruleset_number(u_int8_t); __private_extern__ void pf_init_ruleset(struct pf_ruleset *); @@ -2406,7 +2400,7 @@ __private_extern__ void pf_rs_initialize(void); __private_extern__ int pf_osfp_add(struct pf_osfp_ioctl *); __private_extern__ struct pf_osfp_enlist *pf_osfp_fingerprint(struct pf_pdesc *, - struct mbuf *, int, const struct tcphdr *); + pbuf_t *, int, const struct tcphdr *); __private_extern__ struct pf_osfp_enlist *pf_osfp_fingerprint_hdr( const struct ip *, const struct ip6_hdr *, const struct tcphdr *); __private_extern__ void pf_osfp_flush(void); @@ -2415,7 +2409,9 @@ __private_extern__ void pf_osfp_initialize(void); __private_extern__ int pf_osfp_match(struct pf_osfp_enlist *, pf_osfp_t); __private_extern__ struct pf_os_fingerprint *pf_osfp_validate(void); __private_extern__ struct pf_mtag *pf_find_mtag(struct mbuf *); +__private_extern__ struct pf_mtag *pf_find_mtag_pbuf(pbuf_t *); __private_extern__ struct pf_mtag *pf_get_mtag(struct mbuf *); +__private_extern__ struct pf_mtag *pf_get_mtag_pbuf(pbuf_t *); #else /* !KERNEL */ extern struct pf_anchor_global pf_anchors; extern struct pf_anchor pf_main_anchor; diff --git a/bsd/net/pktap.c b/bsd/net/pktap.c index 2b5c5e0ce..6608e9df3 100644 --- a/bsd/net/pktap.c +++ b/bsd/net/pktap.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -76,7 +76,7 @@ struct pktap_softc { }; #ifndef PKTAP_DEBUG -#define PKTAP_DEBUG 1 +#define PKTAP_DEBUG 0 #endif /* PKTAP_DEBUG */ #define PKTAP_FILTER_OK 0 /* Packet passes filter checks */ @@ -88,8 +88,8 @@ SYSCTL_DECL(_net_link); SYSCTL_NODE(_net_link, IFT_PKTAP, pktap, CTLFLAG_RW |CTLFLAG_LOCKED, 0, "pktap virtual interface"); -static int pktap_total_tap_count = 0; -SYSCTL_INT(_net_link_pktap, OID_AUTO, total_tap_count, +uint32_t pktap_total_tap_count = 0; +SYSCTL_UINT(_net_link_pktap, OID_AUTO, total_tap_count, CTLFLAG_RD | CTLFLAG_LOCKED, &pktap_total_tap_count, 0, ""); static u_int64_t pktap_count_unknown_if_type = 0; @@ -208,7 +208,7 @@ pktap_clone_create(struct if_clone *ifc, u_int32_t unit, __unused void *params) { int error = 0; struct pktap_softc *pktap = NULL; - struct ifnet_init_params if_init; + struct ifnet_init_eparams if_init; PKTAP_LOG(PKTP_LOG_FUNC, "unit %u\n", unit); @@ -228,9 +228,15 @@ pktap_clone_create(struct if_clone *ifc, u_int32_t unit, __unused void *params) pktap->pktp_filters[0].filter_param = PKTAP_FILTER_PARAM_IF_TYPE; pktap->pktp_filters[0].filter_param_if_type = IFT_ETHER; +#if CONFIG_EMBEDDED + pktap->pktp_filters[1].filter_op = PKTAP_FILTER_OP_PASS; + pktap->pktp_filters[1].filter_param = PKTAP_FILTER_PARAM_IF_TYPE; + pktap->pktp_filters[1].filter_param_if_type = IFT_CELLULAR; +#else /* CONFIG_EMBEDDED */ pktap->pktp_filters[1].filter_op = PKTAP_FILTER_OP_PASS; pktap->pktp_filters[1].filter_param = PKTAP_FILTER_PARAM_IF_TYPE; pktap->pktp_filters[1].filter_param_if_type = IFT_IEEE1394; +#endif /* CONFIG_EMBEDDED */ #if (DEVELOPMENT || DEBUG) pktap->pktp_filters[2].filter_op = PKTAP_FILTER_OP_PASS; @@ -242,7 +248,10 @@ pktap_clone_create(struct if_clone *ifc, u_int32_t unit, __unused void *params) * We do not use a set_bpf_tap() function as we rather rely on the more * accurate callback passed to bpf_attach() */ - bzero(&if_init, sizeof(struct ifnet_init_params)); + bzero(&if_init, sizeof(if_init)); + if_init.ver = IFNET_INIT_CURRENT_VERSION; + if_init.len = sizeof (if_init); + if_init.flags = IFNET_INIT_LEGACY; if_init.name = ifc->ifc_name; if_init.unit = unit; if_init.type = IFT_PKTAP; @@ -255,7 +264,7 @@ pktap_clone_create(struct if_clone *ifc, u_int32_t unit, __unused void *params) if_init.ioctl = pktap_ioctl; if_init.detach = pktap_detach; - error = ifnet_allocate(&if_init, &pktap->pktp_ifp); + error = ifnet_allocate_extended(&if_init, &pktap->pktp_ifp); if (error != 0) { printf("%s: ifnet_allocate failed, error %d\n", __func__, error); @@ -835,7 +844,7 @@ pktap_fill_proc_info(struct pktap_header *hdr, protocol_family_t proto, errno_t error; size_t hlen; struct in_addr faddr, laddr; - u_short fport, lport; + u_short fport = 0, lport = 0; struct inpcbinfo *pcbinfo = NULL; int wildcard = 0; @@ -899,7 +908,7 @@ pktap_fill_proc_info(struct pktap_header *hdr, protocol_family_t proto, errno_t error; struct in6_addr *faddr; struct in6_addr *laddr; - u_short fport, lport; + u_short fport = 0, lport = 0; struct inpcbinfo *pcbinfo = NULL; int wildcard = 0; @@ -1082,15 +1091,22 @@ pktap_bpf_tap(struct ifnet *ifp, protocol_family_t proto, struct mbuf *m, goto done; if (proto == AF_INET6 && (size_t) m_pktlen(m) - 4 < sizeof(struct ip6_hdr)) goto done; + /* - * Skip the protocol in the mbuf as it's in network order + * Handle two cases: + * - The old utun encapsulation with the protocol family in network order + * - A raw IPv4 or IPv6 packet */ - pre = 4; - data_adjust = 4; - hdr->pth_dlt = DLT_NULL; - hdr_buffer.proto = proto; - hdr_size = sizeof(hdr_buffer); - break; + uint8_t data = *(uint8_t *)mbuf_data(m); + if ((data >> 4) == 4 || (data >> 4) == 6) { + pre = 4; + } else { + /* + * Skip the protocol in the mbuf as it's in network order + */ + pre = 4; + data_adjust = 4; + } } hdr->pth_dlt = DLT_NULL; hdr_buffer.proto = proto; @@ -1109,8 +1125,8 @@ pktap_bpf_tap(struct ifnet *ifp, protocol_family_t proto, struct mbuf *m, ifp->if_type, ifp->if_xname); pktap_count_unknown_if_type += 1; } else { - snprintf(hdr->pth_ifname, sizeof(hdr->pth_ifname), "%s", - ifp->if_xname); + strlcpy(hdr->pth_ifname, ifp->if_xname, + sizeof(hdr->pth_ifname)); hdr->pth_flags |= outgoing ? PTH_FLAG_DIR_OUT : PTH_FLAG_DIR_IN; hdr->pth_protocol_family = proto; hdr->pth_frame_pre_length = pre + pre_adjust; @@ -1146,13 +1162,15 @@ __private_extern__ void pktap_input(struct ifnet *ifp, protocol_family_t proto, struct mbuf *m, char *frame_header) { - char *hdr = (char *)mbuf_data(m); - char *start = (char *)mbuf_datastart(m); + char *hdr; + char *start; /* Fast path */ if (pktap_total_tap_count == 0) return; + hdr = (char *)mbuf_data(m); + start = (char *)mbuf_datastart(m); /* Make sure the frame header is fully contained in the mbuf */ if (frame_header != NULL && frame_header >= start && frame_header <= hdr) { size_t o_len = m->m_len; @@ -1186,3 +1204,4 @@ pktap_output(struct ifnet *ifp, protocol_family_t proto, struct mbuf *m, pktap_bpf_tap(ifp, proto, m, pre, post, 1); } + diff --git a/bsd/net/pktap.h b/bsd/net/pktap.h index ef5ec8c7c..74b0b5bd1 100644 --- a/bsd/net/pktap.h +++ b/bsd/net/pktap.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2014 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -137,9 +137,11 @@ struct pktap_header { #endif /* BSD_KERNEL_PRIVATE */ #define PTH_FLAG_TSTAMP 0x2000 /* Has time stamp */ #define PTH_FLAG_NEW_FLOW 0x4000 /* Packet from a new flow */ - +#define PTH_FLAG_MSFSW 0x8000 /* Multi stack flow switch */ #ifdef BSD_KERNEL_PRIVATE +extern uint32_t pktap_total_tap_count; + extern void pktap_init(void); extern void pktap_input(struct ifnet *, protocol_family_t, struct mbuf *, char *); extern void pktap_output(struct ifnet *, protocol_family_t, struct mbuf *, diff --git a/bsd/net/pktsched/pktsched.c b/bsd/net/pktsched/pktsched.c index 451aa1709..e757dc893 100644 --- a/bsd/net/pktsched/pktsched.c +++ b/bsd/net/pktsched/pktsched.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 Apple Inc. All rights reserved. + * Copyright (c) 2011-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -46,21 +46,10 @@ #include <net/pktsched/pktsched_tcq.h> #include <net/pktsched/pktsched_qfq.h> #include <net/pktsched/pktsched_fq_codel.h> -#if PKTSCHED_PRIQ -#include <net/pktsched/pktsched_priq.h> -#endif /* PKTSCHED_PRIQ */ -#if PKTSCHED_FAIRQ -#include <net/pktsched/pktsched_fairq.h> -#endif /* PKTSCHED_FAIRQ */ -#if PKTSCHED_CBQ -#include <net/pktsched/pktsched_cbq.h> -#endif /* PKTSCHED_CBQ */ -#if PKTSCHED_HFSC -#include <net/pktsched/pktsched_hfsc.h> -#endif /* PKTSCHED_HFSC */ #include <pexpert/pexpert.h> + u_int32_t machclk_freq = 0; u_int64_t machclk_per_sec = 0; u_int32_t pktsched_verbose; /* more noise if greater than 1 */ @@ -83,18 +72,6 @@ pktsched_init(void) tcq_init(); qfq_init(); -#if PKTSCHED_PRIQ - priq_init(); -#endif /* PKTSCHED_PRIQ */ -#if PKTSCHED_FAIRQ - fairq_init(); -#endif /* PKTSCHED_FAIRQ */ -#if PKTSCHED_CBQ - cbq_init(); -#endif /* PKTSCHED_CBQ */ -#if PKTSCHED_HFSC - hfsc_init(); -#endif /* PKTSCHED_HFSC */ } static void @@ -129,10 +106,10 @@ pktsched_nsecs_to_abstime(u_int64_t nsecs) } int -pktsched_setup(struct ifclassq *ifq, u_int32_t scheduler, u_int32_t sflags) +pktsched_setup(struct ifclassq *ifq, u_int32_t scheduler, u_int32_t sflags, + classq_pkt_type_t ptype) { int error = 0; - u_int32_t qflags = sflags; u_int32_t rflags; IFCQ_LOCK_ASSERT_HELD(ifq); @@ -143,17 +120,6 @@ pktsched_setup(struct ifclassq *ifq, u_int32_t scheduler, u_int32_t sflags) if (ifq->ifcq_type == scheduler) return (0); - qflags &= (PKTSCHEDF_QALG_RED | PKTSCHEDF_QALG_RIO | - PKTSCHEDF_QALG_BLUE | PKTSCHEDF_QALG_SFB); - - /* These are mutually exclusive */ - if (qflags != 0 && - qflags != PKTSCHEDF_QALG_RED && qflags != PKTSCHEDF_QALG_RIO && - qflags != PKTSCHEDF_QALG_BLUE && qflags != PKTSCHEDF_QALG_SFB) { - panic("%s: RED|RIO|BLUE|SFB mutually exclusive\n", __func__); - /* NOTREACHED */ - } - /* * Remember the flags that need to be restored upon success, as * they may be cleared when we tear down existing scheduler. @@ -173,21 +139,15 @@ pktsched_setup(struct ifclassq *ifq, u_int32_t scheduler, u_int32_t sflags) } switch (scheduler) { -#if PKTSCHED_PRIQ - case PKTSCHEDT_PRIQ: - error = priq_setup_ifclassq(ifq, sflags); - break; -#endif /* PKTSCHED_PRIQ */ - case PKTSCHEDT_TCQ: - error = tcq_setup_ifclassq(ifq, sflags); + error = tcq_setup_ifclassq(ifq, sflags, ptype); break; case PKTSCHEDT_QFQ: - error = qfq_setup_ifclassq(ifq, sflags); + error = qfq_setup_ifclassq(ifq, sflags, ptype); break; case PKTSCHEDT_FQ_CODEL: - error = fq_if_setup_ifclassq(ifq, sflags); + error = fq_if_setup_ifclassq(ifq, sflags, ptype); break; default: error = ENXIO; @@ -216,12 +176,6 @@ pktsched_teardown(struct ifclassq *ifq) case PKTSCHEDT_NONE: break; -#if PKTSCHED_PRIQ - case PKTSCHEDT_PRIQ: - error = priq_teardown_ifclassq(ifq); - break; -#endif /* PKTSCHED_PRIQ */ - case PKTSCHEDT_TCQ: error = tcq_teardown_ifclassq(ifq); break; @@ -249,12 +203,6 @@ pktsched_getqstats(struct ifclassq *ifq, u_int32_t qid, IFCQ_LOCK_ASSERT_HELD(ifq); switch (ifq->ifcq_type) { -#if PKTSCHED_PRIQ - case PKTSCHEDT_PRIQ: - error = priq_getqstats_ifclassq(ifq, qid, ifqs); - break; -#endif /* PKTSCHED_PRIQ */ - case PKTSCHEDT_TCQ: error = tcq_getqstats_ifclassq(ifq, qid, ifqs); break; @@ -273,3 +221,161 @@ pktsched_getqstats(struct ifclassq *ifq, u_int32_t qid, return (error); } + +void +pktsched_pkt_encap(pktsched_pkt_t *pkt, classq_pkt_type_t ptype, void *pp) +{ + pkt->pktsched_ptype = ptype; + pkt->pktsched_pkt = pp; + + switch (ptype) { + case QP_MBUF: + pkt->pktsched_plen = + (uint32_t)m_pktlen((struct mbuf *)pkt->pktsched_pkt); + break; + + + default: + VERIFY(0); + /* NOTREACHED */ + } +} + +void +pktsched_free_pkt(pktsched_pkt_t *pkt) +{ + switch (pkt->pktsched_ptype) { + case QP_MBUF: + m_freem(pkt->pktsched_pkt); + break; + + + default: + VERIFY(0); + /* NOTREACHED */ + } + + pkt->pktsched_pkt = NULL; + pkt->pktsched_plen = 0; + pkt->pktsched_ptype = 0; +} + +uint32_t +pktsched_get_pkt_len(pktsched_pkt_t *pkt) +{ + return (pkt->pktsched_plen); +} + +mbuf_svc_class_t +pktsched_get_pkt_svc(pktsched_pkt_t *pkt) +{ + mbuf_svc_class_t svc = MBUF_SC_UNSPEC; + + switch (pkt->pktsched_ptype) { + case QP_MBUF: + svc = m_get_service_class((mbuf_t)pkt->pktsched_pkt); + break; + + + default: + VERIFY(0); + /* NOTREACHED */ + } + + return (svc); +} + +void +pktsched_get_pkt_vars(pktsched_pkt_t *pkt, uint32_t **flags, + uint64_t **timestamp, uint32_t *flowid, uint8_t *flowsrc, uint8_t *proto, + uint32_t *tcp_start_seq) +{ + switch (pkt->pktsched_ptype) { + case QP_MBUF: { + struct mbuf *m = (struct mbuf *)pkt->pktsched_pkt; + struct pkthdr *pkth = &m->m_pkthdr; + + if (flags != NULL) + *flags = &pkth->pkt_flags; + if (timestamp != NULL) + *timestamp = &pkth->pkt_timestamp; + if (flowid != NULL) + *flowid = pkth->pkt_flowid; + if (flowsrc != NULL) + *flowsrc = pkth->pkt_flowsrc; + if (proto != NULL) + *proto = pkth->pkt_proto; + /* + * caller should use this value only if PKTF_START_SEQ + * is set in the mbuf packet flags + */ + if (tcp_start_seq != NULL) + *tcp_start_seq = pkth->tx_start_seq; + + break; + } + + + default: + VERIFY(0); + /* NOTREACHED */ + } +} + +struct flowadv_fcentry * +pktsched_alloc_fcentry(pktsched_pkt_t *pkt, struct ifnet *ifp, int how) +{ +#pragma unused(ifp) + struct flowadv_fcentry *fce = NULL; + + switch (pkt->pktsched_ptype) { + case QP_MBUF: { + struct mbuf *m = (struct mbuf *)pkt->pktsched_pkt; + + fce = flowadv_alloc_entry(how); + if (fce == NULL) + break; + + _CASSERT(sizeof (m->m_pkthdr.pkt_flowid) == + sizeof (fce->fce_flowid)); + + fce->fce_flowsrc_type = m->m_pkthdr.pkt_flowsrc; + fce->fce_flowid = m->m_pkthdr.pkt_flowid; + break; + } + + + default: + VERIFY(0); + /* NOTREACHED */ + } + + return (fce); +} + +uint32_t * +pktsched_get_pkt_sfb_vars(pktsched_pkt_t *pkt, uint32_t **sfb_flags) +{ + uint32_t *hashp = NULL; + + switch (pkt->pktsched_ptype) { + case QP_MBUF: { + struct mbuf *m = (struct mbuf *)pkt->pktsched_pkt; + struct pkthdr *pkth = &m->m_pkthdr; + + _CASSERT(sizeof (pkth->pkt_mpriv_hash) == sizeof (uint32_t)); + _CASSERT(sizeof (pkth->pkt_mpriv_flags) == sizeof (uint32_t)); + + *sfb_flags = &pkth->pkt_mpriv_flags; + hashp = &pkth->pkt_mpriv_hash; + break; + } + + + default: + VERIFY(0); + /* NOTREACHED */ + } + + return (hashp); +} diff --git a/bsd/net/pktsched/pktsched.h b/bsd/net/pktsched/pktsched.h index fb25dfcef..6c2a8eaae 100644 --- a/bsd/net/pktsched/pktsched.h +++ b/bsd/net/pktsched/pktsched.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2012 Apple Inc. All rights reserved. + * Copyright (c) 2011-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -51,14 +51,26 @@ extern "C" { #include <libkern/libkern.h> /* flags for pktsched_setup */ -#define PKTSCHEDF_QALG_RED 0x1 /* use RED */ -#define PKTSCHEDF_QALG_RIO 0x2 /* use RIO */ -#define PKTSCHEDF_QALG_BLUE 0x4 /* use BLUE */ -#define PKTSCHEDF_QALG_SFB 0x8 /* use SFB */ -#define PKTSCHEDF_QALG_ECN 0x10 /* enable ECN */ -#define PKTSCHEDF_QALG_FLOWCTL 0x20 /* enable flow control advisories */ -#define PKTSCHEDF_QALG_DELAYBASED 0x40 /* Delay based queueing */ -#define PKTSCHEDF_QALG_FQ_CODEL 0x80 /* Flow queueing with Codel */ +#define PKTSCHEDF_QALG_SFB 0x01 /* use SFB */ +#define PKTSCHEDF_QALG_ECN 0x02 /* enable ECN */ +#define PKTSCHEDF_QALG_FLOWCTL 0x04 /* enable flow control advisories */ +#define PKTSCHEDF_QALG_DELAYBASED 0x08 /* Delay based queueing */ +#define PKTSCHEDF_QALG_DRIVER_MANAGED 0x10 /* driver managed */ + +typedef struct _pktsched_pkt_ { + classq_pkt_type_t __ptype; + uint32_t __plen; + void *__pkt; +#define pktsched_ptype __ptype +#define pktsched_plen __plen +#define pktsched_pkt __pkt +} pktsched_pkt_t; + +#define _PKTSCHED_PKT_INIT(_p) do { \ + (_p)->pktsched_ptype = QP_INVALID; \ + (_p)->pktsched_plen = 0; \ + (_p)->pktsched_pkt = NULL; \ +} while (0) /* macro for timeout/untimeout */ /* use old-style timeout/untimeout */ @@ -144,12 +156,22 @@ SYSCTL_DECL(_net_pktsched); struct if_ifclassq_stats; extern void pktsched_init(void); -extern int pktsched_setup(struct ifclassq *, u_int32_t, u_int32_t); +extern int pktsched_setup(struct ifclassq *, u_int32_t, u_int32_t, + classq_pkt_type_t); extern int pktsched_teardown(struct ifclassq *); extern int pktsched_getqstats(struct ifclassq *, u_int32_t, struct if_ifclassq_stats *); extern u_int64_t pktsched_abs_to_nsecs(u_int64_t); extern u_int64_t pktsched_nsecs_to_abstime(u_int64_t); +extern void pktsched_free_pkt(pktsched_pkt_t *); +extern uint32_t pktsched_get_pkt_len(pktsched_pkt_t *); +extern void pktsched_get_pkt_vars(pktsched_pkt_t *, uint32_t **, uint64_t **, + uint32_t *, uint8_t *, uint8_t *, uint32_t *); +extern uint32_t *pktsched_get_pkt_sfb_vars(pktsched_pkt_t *, uint32_t **); +extern void pktsched_pkt_encap(pktsched_pkt_t *, classq_pkt_type_t, void *); +extern mbuf_svc_class_t pktsched_get_pkt_svc(pktsched_pkt_t *); +extern struct flowadv_fcentry *pktsched_alloc_fcentry(pktsched_pkt_t *, + struct ifnet *, int); #endif /* BSD_KERNEL_PRIVATE */ #ifdef __cplusplus diff --git a/bsd/net/pktsched/pktsched_cbq.c b/bsd/net/pktsched/pktsched_cbq.c deleted file mode 100644 index 41b1f8ede..000000000 --- a/bsd/net/pktsched/pktsched_cbq.c +++ /dev/null @@ -1,705 +0,0 @@ -/* - * Copyright (c) 2007-2012 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* $OpenBSD: altq_cbq.c,v 1.23 2007/09/13 20:40:02 chl Exp $ */ -/* $KAME: altq_cbq.c,v 1.9 2000/12/14 08:12:45 thorpej Exp $ */ - -/* - * Copyright (c) Sun Microsystems, Inc. 1993-1998 All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the SMCC Technology - * Development Group at Sun Microsystems, Inc. - * - * 4. The name of the Sun Microsystems, Inc nor may not be used to endorse or - * promote products derived from this software without specific prior - * written permission. - * - * SUN MICROSYSTEMS DOES NOT CLAIM MERCHANTABILITY OF THIS SOFTWARE OR THE - * SUITABILITY OF THIS SOFTWARE FOR ANY PARTICULAR PURPOSE. The software is - * provided "as is" without express or implied warranty of any kind. - * - * These notices must be retained in any copies of any part of this software. - */ - -#if PKTSCHED_CBQ - -#include <sys/cdefs.h> -#include <sys/param.h> -#include <sys/malloc.h> -#include <sys/mbuf.h> -#include <sys/systm.h> -#include <sys/errno.h> -#include <sys/kernel.h> -#include <sys/syslog.h> - -#include <kern/zalloc.h> - -#include <net/if.h> -#include <net/net_osdep.h> - -#include <net/pktsched/pktsched_cbq.h> -#include <netinet/in.h> - -/* - * Forward Declarations. - */ -#if 0 -static int cbq_enqueue_ifclassq(struct ifclassq *, struct mbuf *); -static struct mbuf *cbq_dequeue_ifclassq(struct ifclassq *, cqdq_op_t); -static int cbq_request_ifclassq(struct ifclassq *, cqrq_t, void *); -#endif -static int cbq_class_destroy(cbq_state_t *, struct rm_class *); -static int cbq_destroy_locked(cbq_state_t *); -static struct rm_class *cbq_clh_to_clp(cbq_state_t *, u_int32_t); -static const char *cbq_style(cbq_state_t *); -static int cbq_clear_interface(cbq_state_t *); -static void cbqrestart(struct ifclassq *); - -#define CBQ_ZONE_MAX 32 /* maximum elements in zone */ -#define CBQ_ZONE_NAME "pktsched_cbq" /* zone name */ - -static unsigned int cbq_size; /* size of zone element */ -static struct zone *cbq_zone; /* zone for cbq */ - -void -cbq_init(void) -{ - _CASSERT(CBQCLF_RED == RMCF_RED); - _CASSERT(CBQCLF_ECN == RMCF_ECN); - _CASSERT(CBQCLF_RIO == RMCF_RIO); - _CASSERT(CBQCLF_FLOWVALVE == RMCF_FLOWVALVE); - _CASSERT(CBQCLF_CLEARDSCP == RMCF_CLEARDSCP); - _CASSERT(CBQCLF_WRR == RMCF_WRR); - _CASSERT(CBQCLF_EFFICIENT == RMCF_EFFICIENT); - _CASSERT(CBQCLF_BLUE == RMCF_BLUE); - _CASSERT(CBQCLF_SFB == RMCF_SFB); - _CASSERT(CBQCLF_FLOWCTL == RMCF_FLOWCTL); - _CASSERT(CBQCLF_LAZY == RMCF_LAZY); - - cbq_size = sizeof (cbq_state_t); - cbq_zone = zinit(cbq_size, CBQ_ZONE_MAX * cbq_size, 0, CBQ_ZONE_NAME); - if (cbq_zone == NULL) { - panic("%s: failed allocating %s", __func__, CBQ_ZONE_NAME); - /* NOTREACHED */ - } - zone_change(cbq_zone, Z_EXPAND, TRUE); - zone_change(cbq_zone, Z_CALLERACCT, TRUE); - - rmclass_init(); -} - -cbq_state_t * -cbq_alloc(struct ifnet *ifp, int how, boolean_t altq) -{ - cbq_state_t *cbqp; - - /* allocate and initialize cbq_state_t */ - cbqp = (how == M_WAITOK) ? zalloc(cbq_zone) : zalloc_noblock(cbq_zone); - if (cbqp == NULL) - return (NULL); - - bzero(cbqp, cbq_size); - CALLOUT_INIT(&cbqp->cbq_callout); - cbqp->cbq_qlen = 0; - cbqp->ifnp.ifq_ = &ifp->if_snd; /* keep the ifclassq */ - if (altq) - cbqp->cbq_flags |= CBQSF_ALTQ; - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s scheduler allocated\n", - if_name(ifp), cbq_style(cbqp)); - } - - return (cbqp); -} - -int -cbq_destroy(cbq_state_t *cbqp) -{ - struct ifclassq *ifq = cbqp->ifnp.ifq_; - int err; - - IFCQ_LOCK(ifq); - err = cbq_destroy_locked(cbqp); - IFCQ_UNLOCK(ifq); - - return (err); -} - -static int -cbq_destroy_locked(cbq_state_t *cbqp) -{ - IFCQ_LOCK_ASSERT_HELD(cbqp->ifnp.ifq_); - - (void) cbq_clear_interface(cbqp); - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s scheduler destroyed\n", - if_name(CBQS_IFP(cbqp)), cbq_style(cbqp)); - } - - if (cbqp->ifnp.default_) - cbq_class_destroy(cbqp, cbqp->ifnp.default_); - if (cbqp->ifnp.root_) - cbq_class_destroy(cbqp, cbqp->ifnp.root_); - - /* deallocate cbq_state_t */ - zfree(cbq_zone, cbqp); - - return (0); -} - -int -cbq_add_queue(cbq_state_t *cbqp, u_int32_t qlimit, u_int32_t priority, - u_int32_t minburst, u_int32_t maxburst, u_int32_t pktsize, - u_int32_t maxpktsize, u_int32_t ns_per_byte, u_int32_t maxidle, int minidle, - u_int32_t offtime, u_int32_t flags, u_int32_t parent_qid, u_int32_t qid, - struct rm_class **clp) -{ -#pragma unused(minburst, maxburst, maxpktsize) - struct rm_class *borrow, *parent; - struct rm_class *cl; - int i, error; - - IFCQ_LOCK_ASSERT_HELD(cbqp->ifnp.ifq_); - - /* Sanitize flags unless internally configured */ - if (cbqp->cbq_flags & CBQSF_ALTQ) - flags &= CBQCLF_USERFLAGS; - - /* - * find a free slot in the class table. if the slot matching - * the lower bits of qid is free, use this slot. otherwise, - * use the first free slot. - */ - i = qid % CBQ_MAX_CLASSES; - if (cbqp->cbq_class_tbl[i] != NULL) { - for (i = 0; i < CBQ_MAX_CLASSES; i++) - if (cbqp->cbq_class_tbl[i] == NULL) - break; - if (i == CBQ_MAX_CLASSES) - return (EINVAL); - } - - /* check parameters */ - if (priority >= CBQ_MAXPRI) - return (EINVAL); - - if (ns_per_byte == 0) { - log(LOG_ERR, "%s: %s invalid inverse data rate\n", - if_name(CBQS_IFP(cbqp)), cbq_style(cbqp)); - return (EINVAL); - } - - /* Get pointers to parent and borrow classes. */ - parent = cbq_clh_to_clp(cbqp, parent_qid); - if (flags & CBQCLF_BORROW) - borrow = parent; - else - borrow = NULL; - - /* - * A class must borrow from its parent or it can not - * borrow at all. Hence, borrow can be null. - */ - if (parent == NULL && (flags & CBQCLF_ROOTCLASS) == 0) { - log(LOG_ERR, "%s: %s no parent class!\n", - if_name(CBQS_IFP(cbqp)), cbq_style(cbqp)); - return (EINVAL); - } - - if ((borrow != parent) && (borrow != NULL)) { - log(LOG_ERR, "%s: %s borrow class != parent\n", - if_name(CBQS_IFP(cbqp)), cbq_style(cbqp)); - return (EINVAL); - } - - /* - * check parameters - */ - switch (flags & CBQCLF_CLASSMASK) { - case CBQCLF_ROOTCLASS: - if (parent != NULL) { - log(LOG_ERR, "%s: %s parent exists\n", - if_name(CBQS_IFP(cbqp)), cbq_style(cbqp)); - return (EINVAL); - } - if (cbqp->ifnp.root_) { - log(LOG_ERR, "%s: %s root class exists\n", - if_name(CBQS_IFP(cbqp)), cbq_style(cbqp)); - return (EINVAL); - } - break; - case CBQCLF_DEFCLASS: - if (cbqp->ifnp.default_) { - log(LOG_ERR, "%s: %s default class exists\n", - if_name(CBQS_IFP(cbqp)), cbq_style(cbqp)); - return (EINVAL); - } - break; - case 0: - break; - default: - /* more than two flags bits set */ - log(LOG_ERR, "%s: %s invalid class flags 0x%x\n", - if_name(CBQS_IFP(cbqp)), cbq_style(cbqp), - (flags & CBQCLF_CLASSMASK)); - return (EINVAL); - } - - /* - * create a class. if this is a root class, initialize the - * interface. - */ - if ((flags & CBQCLF_CLASSMASK) == CBQCLF_ROOTCLASS) { - error = rmc_init(cbqp->ifnp.ifq_, &cbqp->ifnp, ns_per_byte, - cbqrestart, qid, qlimit, RM_MAXQUEUED, maxidle, minidle, - offtime, flags); - if (error != 0) - return (error); - cl = cbqp->ifnp.root_; - } else { - cl = rmc_newclass(priority, &cbqp->ifnp, ns_per_byte, - rmc_delay_action, qid, qlimit, parent, borrow, maxidle, - minidle, offtime, pktsize, flags); - } - if (cl == NULL) - return (ENOMEM); - - /* return handle to user space. */ - cl->stats_.handle = qid; - cl->stats_.depth = cl->depth_; - - /* save the allocated class */ - cbqp->cbq_class_tbl[i] = cl; - - if ((flags & CBQCLF_CLASSMASK) == CBQCLF_DEFCLASS) - cbqp->ifnp.default_ = cl; - - if (clp != NULL) - *clp = cl; - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s created qid=%d pri=%d qlimit=%d " - "flags=%b\n", if_name(CBQS_IFP(cbqp)), cbq_style(cbqp), - qid, priority, qlimit, flags, CBQCLF_BITS); - } - - return (0); -} - -int -cbq_remove_queue(cbq_state_t *cbqp, u_int32_t qid) -{ - struct rm_class *cl; - int i; - - IFCQ_LOCK_ASSERT_HELD(cbqp->ifnp.ifq_); - - if ((cl = cbq_clh_to_clp(cbqp, qid)) == NULL) - return (EINVAL); - - /* if we are a parent class, then return an error. */ - if (RMC_IS_A_PARENT_CLASS(cl)) - return (EINVAL); - - /* delete the class */ - rmc_delete_class(&cbqp->ifnp, cl); - - /* - * free the class handle - */ - for (i = 0; i < CBQ_MAX_CLASSES; i++) { - if (cbqp->cbq_class_tbl[i] == cl) { - cbqp->cbq_class_tbl[i] = NULL; - if (cl == cbqp->ifnp.root_) - cbqp->ifnp.root_ = NULL; - if (cl == cbqp->ifnp.default_) - cbqp->ifnp.default_ = NULL; - break; - } - } - return (0); -} - -/* - * int - * cbq_class_destroy(cbq_mod_state_t *, struct rm_class *) - This - * function destroys a given traffic class. Before destroying - * the class, all traffic for that class is released. - */ -static int -cbq_class_destroy(cbq_state_t *cbqp, struct rm_class *cl) -{ - int i; - - IFCQ_LOCK_ASSERT_HELD(cbqp->ifnp.ifq_); - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s destroyed qid=%d pri=%d\n", - if_name(CBQS_IFP(cbqp)), cbq_style(cbqp), - cl->stats_.handle, cl->pri_); - } - - /* delete the class */ - rmc_delete_class(&cbqp->ifnp, cl); - - /* - * free the class handle - */ - for (i = 0; i < CBQ_MAX_CLASSES; i++) - if (cbqp->cbq_class_tbl[i] == cl) - cbqp->cbq_class_tbl[i] = NULL; - - if (cl == cbqp->ifnp.root_) - cbqp->ifnp.root_ = NULL; - if (cl == cbqp->ifnp.default_) - cbqp->ifnp.default_ = NULL; - - return (0); -} - -/* convert class handle to class pointer */ -static struct rm_class * -cbq_clh_to_clp(cbq_state_t *cbqp, u_int32_t chandle) -{ - int i; - struct rm_class *cl; - - IFCQ_LOCK_ASSERT_HELD(cbqp->ifnp.ifq_); - - /* - * first, try optimistically the slot matching the lower bits of - * the handle. if it fails, do the linear table search. - */ - i = chandle % CBQ_MAX_CLASSES; - if ((cl = cbqp->cbq_class_tbl[i]) != NULL && - cl->stats_.handle == chandle) - return (cl); - for (i = 0; i < CBQ_MAX_CLASSES; i++) - if ((cl = cbqp->cbq_class_tbl[i]) != NULL && - cl->stats_.handle == chandle) - return (cl); - return (NULL); -} - -static const char * -cbq_style(cbq_state_t *cbqp) -{ - return ((cbqp->cbq_flags & CBQSF_ALTQ) ? "ALTQ_CBQ" : "CBQ"); -} - -static int -cbq_clear_interface(cbq_state_t *cbqp) -{ - int again, i; - struct rm_class *cl; - - IFCQ_LOCK_ASSERT_HELD(cbqp->ifnp.ifq_); - - /* clear out the classes now */ - do { - again = 0; - for (i = 0; i < CBQ_MAX_CLASSES; i++) { - if ((cl = cbqp->cbq_class_tbl[i]) != NULL) { - if (RMC_IS_A_PARENT_CLASS(cl)) - again++; - else { - cbq_class_destroy(cbqp, cl); - cbqp->cbq_class_tbl[i] = NULL; - if (cl == cbqp->ifnp.root_) - cbqp->ifnp.root_ = NULL; - if (cl == cbqp->ifnp.default_) - cbqp->ifnp.default_ = NULL; - } - } - } - } while (again); - - return (0); -} - -/* copy the stats info in rm_class to class_states_t */ -int -cbq_get_class_stats(cbq_state_t *cbqp, u_int32_t qid, class_stats_t *statsp) -{ - struct rm_class *cl; - - IFCQ_LOCK_ASSERT_HELD(cbqp->ifnp.ifq_); - - if ((cl = cbq_clh_to_clp(cbqp, qid)) == NULL) - return (EINVAL); - - statsp->xmit_cnt = cl->stats_.xmit_cnt; - statsp->drop_cnt = cl->stats_.drop_cnt; - statsp->over = cl->stats_.over; - statsp->borrows = cl->stats_.borrows; - statsp->overactions = cl->stats_.overactions; - statsp->delays = cl->stats_.delays; - - statsp->depth = cl->depth_; - statsp->priority = cl->pri_; - statsp->maxidle = cl->maxidle_; - statsp->minidle = cl->minidle_; - statsp->offtime = cl->offtime_; - statsp->qmax = qlimit(&cl->q_); - statsp->ns_per_byte = cl->ns_per_byte_; - statsp->wrr_allot = cl->w_allotment_; - statsp->qcnt = qlen(&cl->q_); - statsp->avgidle = cl->avgidle_; - - statsp->qtype = qtype(&cl->q_); - statsp->qstate = qstate(&cl->q_); -#if CLASSQ_RED - if (q_is_red(&cl->q_)) - red_getstats(cl->red_, &statsp->red[0]); -#endif /* CLASSQ_RED */ -#if CLASSQ_RIO - if (q_is_rio(&cl->q_)) - rio_getstats(cl->rio_, &statsp->red[0]); -#endif /* CLASSQ_RIO */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->q_)) - blue_getstats(cl->blue_, &statsp->blue); -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->q_) && cl->sfb_ != NULL) - sfb_getstats(cl->sfb_, &statsp->sfb); - - return (0); -} - -int -cbq_enqueue(cbq_state_t *cbqp, struct rm_class *cl, struct mbuf *m, - struct pf_mtag *t) -{ - struct ifclassq *ifq = cbqp->ifnp.ifq_; - int len, ret; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - /* grab class set by classifier */ - if (!(m->m_flags & M_PKTHDR)) { - /* should not happen */ - log(LOG_ERR, "%s: packet for %s does not have pkthdr\n", - if_name(ifq->ifcq_ifp)); - IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (ENOBUFS); - } - - if (cl == NULL) { -#if PF_ALTQ - cl = cbq_clh_to_clp(cbqp, t->pftag_qid); -#else /* !PF_ALTQ */ - cl = cbq_clh_to_clp(cbqp, 0); -#endif /* !PF_ALTQ */ - if (cl == NULL) { - cl = cbqp->ifnp.default_; - if (cl == NULL) { - IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (ENOBUFS); - } - } - } - - len = m_pktlen(m); - - ret = rmc_queue_packet(cl, m, t); - if (ret != 0) { - if (ret == CLASSQEQ_SUCCESS_FC) { - /* packet enqueued, return advisory feedback */ - ret = EQFULL; - } else { - VERIFY(ret == CLASSQEQ_DROPPED || - ret == CLASSQEQ_DROPPED_FC || - ret == CLASSQEQ_DROPPED_SP); - /* packet has been freed in rmc_queue_packet */ - PKTCNTR_ADD(&cl->stats_.drop_cnt, 1, len); - IFCQ_DROP_ADD(ifq, 1, len); - switch (ret) { - case CLASSQEQ_DROPPED: - return (ENOBUFS); - case CLASSQEQ_DROPPED_FC: - return (EQFULL); - case CLASSQEQ_DROPPED_SP: - return (EQSUSPENDED); - } - /* NOT REACHED */ - } - } - - /* successfully queued. */ - ++cbqp->cbq_qlen; - IFCQ_INC_LEN(ifq); - IFCQ_INC_BYTES(ifq, len); - - return (ret); -} - -struct mbuf * -cbq_dequeue(cbq_state_t *cbqp, cqdq_op_t op) -{ - struct ifclassq *ifq = cbqp->ifnp.ifq_; - struct mbuf *m; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - m = rmc_dequeue_next(&cbqp->ifnp, op); - - if (m && op == CLASSQDQ_REMOVE) { - --cbqp->cbq_qlen; /* decrement # of packets in cbq */ - IFCQ_DEC_LEN(ifq); - IFCQ_DEC_BYTES(ifq, m_pktlen(m)); - IFCQ_XMIT_ADD(ifq, 1, m_pktlen(m)); - - /* Update the class. */ - rmc_update_class_util(&cbqp->ifnp); - } - return (m); -} - -/* - * void - * cbqrestart(queue_t *) - Restart sending of data. - * called from rmc_restart via timeout after waking up - * a suspended class. - * Returns: NONE - */ - -static void -cbqrestart(struct ifclassq *ifq) -{ - u_int32_t qlen; - - IFCQ_LOCK(ifq); - qlen = IFCQ_LEN(ifq); - IFCQ_UNLOCK(ifq); - - if (qlen > 0) - ifnet_start(ifq->ifcq_ifp); -} - -void -cbq_purge(cbq_state_t *cbqp) -{ - struct rm_class *cl; - int i; - - IFCQ_LOCK_ASSERT_HELD(cbqp->ifnp.ifq_); - - for (i = 0; i < CBQ_MAX_CLASSES; i++) { - if ((cl = cbqp->cbq_class_tbl[i]) != NULL) { - if (!qempty(&cl->q_) && pktsched_verbose) { - log(LOG_DEBUG, "%s: %s purge qid=%d pri=%d " - "qlen=%d\n", if_name(CBQS_IFP(cbqp)), - cbq_style(cbqp), cl->stats_.handle, - cl->pri_, qlen(&cl->q_)); - } - rmc_dropall(cl); - } - } -} - -void -cbq_event(cbq_state_t *cbqp, cqev_t ev) -{ - struct rm_class *cl; - int i; - - IFCQ_LOCK_ASSERT_HELD(cbqp->ifnp.ifq_); - - for (i = 0; i < CBQ_MAX_CLASSES; i++) { - if ((cl = cbqp->cbq_class_tbl[i]) != NULL) { - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s update qid=%d pri=%d " - "event=%s\n", if_name(CBQS_IFP(cbqp)), - cbq_style(cbqp), cl->stats_.handle, - cl->pri_, ifclassq_ev2str(ev)); - } - rmc_updateq(cl, ev); - } - } -} - -int -cqb_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) -{ -#pragma unused(ifq, flags) - return (ENXIO); /* not yet */ -} - -int -cbq_teardown_ifclassq(struct ifclassq *ifq) -{ - cbq_state_t *cbqp = ifq->ifcq_disc; - int i; - - IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(cbqp != NULL && ifq->ifcq_type == PKTSCHEDT_CBQ); - - (void) cbq_destroy_locked(cbqp); - - ifq->ifcq_disc = NULL; - for (i = 0; i < IFCQ_SC_MAX; i++) { - ifq->ifcq_disc_slots[i].qid = 0; - ifq->ifcq_disc_slots[i].cl = NULL; - } - - return (ifclassq_detach(ifq)); -} - -int -cbq_getqstats_ifclassq(struct ifclassq *ifq, u_int32_t slot, - struct if_ifclassq_stats *ifqs) -{ - cbq_state_t *cbqp = ifq->ifcq_disc; - - IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(ifq->ifcq_type == PKTSCHEDT_CBQ); - - if (slot >= IFCQ_SC_MAX) - return (EINVAL); - - return (cbq_get_class_stats(cbqp, ifq->ifcq_disc_slots[slot].qid, - &ifqs->ifqs_cbq_stats)); -} -#endif /* PKTSCHED_CBQ */ diff --git a/bsd/net/pktsched/pktsched_cbq.h b/bsd/net/pktsched/pktsched_cbq.h index 15fe1b0b3..0553397d7 100644 --- a/bsd/net/pktsched/pktsched_cbq.h +++ b/bsd/net/pktsched/pktsched_cbq.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2012 Apple Inc. All rights reserved. + * Copyright (c) 2011-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -152,51 +152,6 @@ typedef struct cbq_classstats { classq_state_t qstate; } class_stats_t; -#ifdef BSD_KERNEL_PRIVATE -/* - * Define macros only good for kernel drivers and modules. - */ -#define CBQ_WATCHDOG (hz / 20) -#define CBQ_TIMEOUT 10 -#define CBQ_LS_TIMEOUT (20 * hz / 1000) - -#define CBQ_MAX_CLASSES 256 - -/* cbqstate flags */ -#define CBQSF_ALTQ 0x1 /* configured via PF/ALTQ */ - -/* - * Define State structures. - */ -typedef struct cbqstate { - int cbq_qlen; /* # of packets in cbq */ - u_int32_t cbq_flags; /* flags */ - struct rm_class *cbq_class_tbl[CBQ_MAX_CLASSES]; - - struct rm_ifdat ifnp; - struct callout cbq_callout; /* for timeouts */ -} cbq_state_t; - -#define CBQS_IFP(_cs) ((_cs)->ifnp.ifq_->ifcq_ifp) - -extern void cbq_init(void); -extern cbq_state_t *cbq_alloc(struct ifnet *, int, boolean_t); -extern int cbq_destroy(cbq_state_t *); -extern void cbq_purge(cbq_state_t *); -extern void cbq_event(cbq_state_t *, cqev_t); -extern int cbq_add_queue(cbq_state_t *, u_int32_t, u_int32_t, u_int32_t, - u_int32_t, u_int32_t, u_int32_t, u_int32_t, u_int32_t, int, u_int32_t, - u_int32_t, u_int32_t, u_int32_t, struct rm_class **); -extern int cbq_remove_queue(cbq_state_t *, u_int32_t); -extern int cbq_get_class_stats(cbq_state_t *, u_int32_t, class_stats_t *); -extern int cbq_enqueue(cbq_state_t *, struct rm_class *, struct mbuf *, - struct pf_mtag *); -extern struct mbuf *cbq_dequeue(cbq_state_t *, cqdq_op_t); -extern int cqb_setup_ifclassq(struct ifclassq *, u_int32_t); -extern int cbq_teardown_ifclassq(struct ifclassq *); -extern int cbq_getqstats_ifclassq(struct ifclassq *, u_int32_t, - struct if_ifclassq_stats *); -#endif /* BSD_KERNEL_PRIVATE */ #ifdef __cplusplus } #endif diff --git a/bsd/net/pktsched/pktsched_fairq.c b/bsd/net/pktsched/pktsched_fairq.c deleted file mode 100644 index 7e61e04c1..000000000 --- a/bsd/net/pktsched/pktsched_fairq.c +++ /dev/null @@ -1,1300 +0,0 @@ -/* - * Copyright (c) 2011-2013 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* - * Copyright (c) 2008 The DragonFly Project. All rights reserved. - * - * This code is derived from software contributed to The DragonFly Project - * by Matthew Dillon <dillon@backplane.com> - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * 3. Neither the name of The DragonFly Project nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific, prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * $DragonFly: src/sys/net/altq/altq_fairq.c,v 1.2 2008/05/14 11:59:23 sephe Exp $ - */ -/* - * Matt: I gutted altq_priq.c and used it as a skeleton on which to build - * fairq. The fairq algorithm is completely different then priq, of course, - * but because I used priq's skeleton I believe I should include priq's - * copyright. - * - * Copyright (C) 2000-2003 - * Sony Computer Science Laboratories Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY SONY CSL AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL SONY CSL OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -/* - * FAIRQ - take traffic classified by keep state (hashed into - * pf->pftag_flowhash) and bucketize it. Fairly extract - * the first packet from each bucket in a round-robin fashion. - * - * TODO - better overall qlimit support (right now it is per-bucket). - * - NOTE: red etc is per bucket, not overall. - * - better service curve support. - * - * EXAMPLE: - * - * altq on em0 fairq bandwidth 650Kb queue { std, bulk } - * queue std priority 3 bandwidth 200Kb \ - * fairq (buckets 64, default, hogs 1Kb) qlimit 50 - * queue bulk priority 2 bandwidth 100Kb \ - * fairq (buckets 64, hogs 1Kb) qlimit 50 - * - * NOTE: When the aggregate bandwidth is less than the link bandwidth - * any remaining bandwidth is dynamically assigned using the - * existing bandwidth specs as weightings. - * - * pass out on em0 from any to any keep state queue std - * pass out on em0 inet proto tcp ..... port ... keep state queue bulk - */ - -#if PKTSCHED_FAIRQ - -#include <sys/cdefs.h> -#include <sys/param.h> -#include <sys/malloc.h> -#include <sys/mbuf.h> -#include <sys/systm.h> -#include <sys/errno.h> -#include <sys/kernel.h> -#include <sys/syslog.h> - -#include <kern/zalloc.h> - -#include <net/if.h> -#include <net/net_osdep.h> - -#include <net/pktsched/pktsched_fairq.h> -#include <netinet/in.h> - -/* - * function prototypes - */ -#if 0 -static int fairq_enqueue_ifclassq(struct ifclassq *, struct mbuf *); -static struct mbuf *fairq_dequeue_ifclassq(struct ifclassq *, cqdq_op_t); -static int fairq_request_ifclassq(struct ifclassq *, cqrq_t, void *); -#endif -static int fairq_clear_interface(struct fairq_if *); -static inline int fairq_addq(struct fairq_class *, struct mbuf *, - struct pf_mtag *); -static inline struct mbuf *fairq_getq(struct fairq_class *, u_int64_t); -static inline struct mbuf *fairq_pollq(struct fairq_class *, u_int64_t, int *); -static fairq_bucket_t *fairq_selectq(struct fairq_class *, int); -static void fairq_purgeq(struct fairq_if *, struct fairq_class *, u_int32_t, - u_int32_t *, u_int32_t *); -static void fairq_updateq(struct fairq_if *, struct fairq_class *, cqev_t); -static struct fairq_class *fairq_class_create(struct fairq_if *, int, u_int32_t, - u_int64_t, u_int32_t, int, u_int64_t, u_int64_t, u_int64_t, u_int64_t, - u_int32_t); -static int fairq_class_destroy(struct fairq_if *, struct fairq_class *); -static int fairq_destroy_locked(struct fairq_if *); -static inline struct fairq_class *fairq_clh_to_clp(struct fairq_if *, - u_int32_t); -static const char *fairq_style(struct fairq_if *); - -#define FAIRQ_ZONE_MAX 32 /* maximum elements in zone */ -#define FAIRQ_ZONE_NAME "pktsched_fairq" /* zone name */ - -static unsigned int fairq_size; /* size of zone element */ -static struct zone *fairq_zone; /* zone for fairq */ - -#define FAIRQ_CL_ZONE_MAX 32 /* maximum elements in zone */ -#define FAIRQ_CL_ZONE_NAME "pktsched_fairq_cl" /* zone name */ - -static unsigned int fairq_cl_size; /* size of zone element */ -static struct zone *fairq_cl_zone; /* zone for fairq */ - -void -fairq_init(void) -{ - fairq_size = sizeof (struct fairq_if); - fairq_zone = zinit(fairq_size, FAIRQ_ZONE_MAX * fairq_size, - 0, FAIRQ_ZONE_NAME); - if (fairq_zone == NULL) { - panic("%s: failed allocating %s", __func__, FAIRQ_ZONE_NAME); - /* NOTREACHED */ - } - zone_change(fairq_zone, Z_EXPAND, TRUE); - zone_change(fairq_zone, Z_CALLERACCT, TRUE); - - fairq_cl_size = sizeof (struct fairq_class); - fairq_cl_zone = zinit(fairq_cl_size, FAIRQ_CL_ZONE_MAX * fairq_cl_size, - 0, FAIRQ_CL_ZONE_NAME); - if (fairq_cl_zone == NULL) { - panic("%s: failed allocating %s", __func__, FAIRQ_CL_ZONE_NAME); - /* NOTREACHED */ - } - zone_change(fairq_cl_zone, Z_EXPAND, TRUE); - zone_change(fairq_cl_zone, Z_CALLERACCT, TRUE); -} - -struct fairq_if * -fairq_alloc(struct ifnet *ifp, int how, boolean_t altq) -{ - struct fairq_if *fif; - - fif = (how == M_WAITOK) ? - zalloc(fairq_zone) : zalloc_noblock(fairq_zone); - if (fif == NULL) - return (NULL); - - bzero(fif, fairq_size); - fif->fif_maxpri = -1; - fif->fif_ifq = &ifp->if_snd; - if (altq) - fif->fif_flags |= FAIRQIFF_ALTQ; - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s scheduler allocated\n", - if_name(ifp), fairq_style(fif)); - } - - return (fif); -} - -int -fairq_destroy(struct fairq_if *fif) -{ - struct ifclassq *ifq = fif->fif_ifq; - int err; - - IFCQ_LOCK(ifq); - err = fairq_destroy_locked(fif); - IFCQ_UNLOCK(ifq); - - return (err); -} - -static int -fairq_destroy_locked(struct fairq_if *fif) -{ - IFCQ_LOCK_ASSERT_HELD(fif->fif_ifq); - - (void) fairq_clear_interface(fif); - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s scheduler destroyed\n", - if_name(FAIRQIF_IFP(fif)), fairq_style(fif)); - } - - zfree(fairq_zone, fif); - - return (0); -} - -/* - * bring the interface back to the initial state by discarding - * all the filters and classes. - */ -static int -fairq_clear_interface(struct fairq_if *fif) -{ - struct fairq_class *cl; - int pri; - - IFCQ_LOCK_ASSERT_HELD(fif->fif_ifq); - - /* clear out the classes */ - for (pri = 0; pri <= fif->fif_maxpri; pri++) - if ((cl = fif->fif_classes[pri]) != NULL) - fairq_class_destroy(fif, cl); - - return (0); -} - -/* discard all the queued packets on the interface */ -void -fairq_purge(struct fairq_if *fif) -{ - struct fairq_class *cl; - int pri; - - IFCQ_LOCK_ASSERT_HELD(fif->fif_ifq); - - for (pri = 0; pri <= fif->fif_maxpri; pri++) { - if ((cl = fif->fif_classes[pri]) != NULL && cl->cl_head) - fairq_purgeq(fif, cl, 0, NULL, NULL); - } -#if !PF_ALTQ - /* - * This assertion is safe to be made only when PF_ALTQ is not - * configured; otherwise, IFCQ_LEN represents the sum of the - * packets managed by ifcq_disc and altq_disc instances, which - * is possible when transitioning between the two. - */ - VERIFY(IFCQ_LEN(fif->fif_ifq) == 0); -#endif /* !PF_ALTQ */ -} - -void -fairq_event(struct fairq_if *fif, cqev_t ev) -{ - struct fairq_class *cl; - int pri; - - IFCQ_LOCK_ASSERT_HELD(fif->fif_ifq); - - for (pri = 0; pri <= fif->fif_maxpri; pri++) - if ((cl = fif->fif_classes[pri]) != NULL) - fairq_updateq(fif, cl, ev); -} - -int -fairq_add_queue(struct fairq_if *fif, int priority, u_int32_t qlimit, - u_int64_t bandwidth, u_int32_t nbuckets, int flags, u_int64_t hogs_m1, - u_int64_t lssc_m1, u_int64_t lssc_d, u_int64_t lssc_m2, u_int32_t qid, - struct fairq_class **clp) -{ - struct fairq_class *cl; - - IFCQ_LOCK_ASSERT_HELD(fif->fif_ifq); - - /* check parameters */ - if (priority >= FAIRQ_MAXPRI) - return (EINVAL); - if (bandwidth == 0 || (bandwidth / 8) == 0) - return (EINVAL); - if (fif->fif_classes[priority] != NULL) - return (EBUSY); - if (fairq_clh_to_clp(fif, qid) != NULL) - return (EBUSY); - - cl = fairq_class_create(fif, priority, qlimit, bandwidth, - nbuckets, flags, hogs_m1, lssc_m1, lssc_d, lssc_m2, qid); - if (cl == NULL) - return (ENOMEM); - - if (clp != NULL) - *clp = cl; - - return (0); -} - -static struct fairq_class * -fairq_class_create(struct fairq_if *fif, int pri, u_int32_t qlimit, - u_int64_t bandwidth, u_int32_t nbuckets, int flags, u_int64_t hogs_m1, - u_int64_t lssc_m1, u_int64_t lssc_d, u_int64_t lssc_m2, u_int32_t qid) -{ -#pragma unused(lssc_d, lssc_m2) - struct ifnet *ifp; - struct ifclassq *ifq; - struct fairq_class *cl; - u_int32_t i; - - IFCQ_LOCK_ASSERT_HELD(fif->fif_ifq); - - /* Sanitize flags unless internally configured */ - if (fif->fif_flags & FAIRQIFF_ALTQ) - flags &= FARF_USERFLAGS; - -#if !CLASSQ_RED - if (flags & FARF_RED) { - log(LOG_ERR, "%s: %s RED not available!\n", - if_name(FAIRQIF_IFP(fif)), fairq_style(fif)); - return (NULL); - } -#endif /* !CLASSQ_RED */ - -#if !CLASSQ_RIO - if (flags & FARF_RIO) { - log(LOG_ERR, "%s: %s RIO not available!\n", - if_name(FAIRQIF_IFP(fif)), fairq_style(fif)); - return (NULL); - } -#endif /* CLASSQ_RIO */ - -#if !CLASSQ_BLUE - if (flags & FARF_BLUE) { - log(LOG_ERR, "%s: %s BLUE not available!\n", - if_name(FAIRQIF_IFP(fif)), fairq_style(fif)); - return (NULL); - } -#endif /* CLASSQ_BLUE */ - - /* These are mutually exclusive */ - if ((flags & (FARF_RED|FARF_RIO|FARF_BLUE|FARF_SFB)) && - (flags & (FARF_RED|FARF_RIO|FARF_BLUE|FARF_SFB)) != FARF_RED && - (flags & (FARF_RED|FARF_RIO|FARF_BLUE|FARF_SFB)) != FARF_RIO && - (flags & (FARF_RED|FARF_RIO|FARF_BLUE|FARF_SFB)) != FARF_BLUE && - (flags & (FARF_RED|FARF_RIO|FARF_BLUE|FARF_SFB)) != FARF_SFB) { - log(LOG_ERR, "%s: %s more than one RED|RIO|BLUE|SFB\n", - if_name(FAIRQIF_IFP(fif)), fairq_style(fif)); - return (NULL); - } - - if (bandwidth == 0 || (bandwidth / 8) == 0) { - log(LOG_ERR, "%s: %s invalid data rate %llu\n", - if_name(FAIRQIF_IFP(fif)), fairq_style(fif), bandwidth); - return (NULL); - } - - if (nbuckets == 0) - nbuckets = 256; - if (nbuckets > FAIRQ_MAX_BUCKETS) - nbuckets = FAIRQ_MAX_BUCKETS; - /* enforce power-of-2 size */ - while ((nbuckets ^ (nbuckets - 1)) != ((nbuckets << 1) - 1)) - ++nbuckets; - - ifq = fif->fif_ifq; - ifp = FAIRQIF_IFP(fif); - - if ((cl = fif->fif_classes[pri]) != NULL) { - /* modify the class instead of creating a new one */ - if (cl->cl_head) - fairq_purgeq(fif, cl, 0, NULL, NULL); -#if CLASSQ_RIO - if (cl->cl_qtype == Q_RIO) - rio_destroy(cl->cl_rio); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (cl->cl_qtype == Q_RED) - red_destroy(cl->cl_red); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (cl->cl_qtype == Q_BLUE) - blue_destroy(cl->cl_blue); -#endif /* CLASSQ_BLUE */ - if (cl->cl_qtype == Q_SFB && cl->cl_sfb != NULL) - sfb_destroy(cl->cl_sfb); - cl->cl_qalg.ptr = NULL; - cl->cl_qtype = Q_DROPTAIL; - cl->cl_qstate = QS_RUNNING; - } else { - cl = zalloc(fairq_cl_zone); - if (cl == NULL) - goto err_ret; - bzero(cl, fairq_cl_size); - cl->cl_nbuckets = nbuckets; - cl->cl_nbucket_mask = nbuckets - 1; - - cl->cl_buckets = _MALLOC(sizeof (struct fairq_bucket) * - cl->cl_nbuckets, M_DEVBUF, M_WAITOK|M_ZERO); - if (cl->cl_buckets == NULL) - goto err_buckets; - cl->cl_head = NULL; - } - - fif->fif_classes[pri] = cl; - if (flags & FARF_DEFAULTCLASS) - fif->fif_default = cl; - if (qlimit == 0 || qlimit > IFCQ_MAXLEN(ifq)) { - qlimit = IFCQ_MAXLEN(ifq); - if (qlimit == 0) - qlimit = DEFAULT_QLIMIT; /* use default */ - } - cl->cl_qlimit = qlimit; - for (i = 0; i < cl->cl_nbuckets; ++i) { - _qinit(&cl->cl_buckets[i].queue, Q_DROPTAIL, qlimit); - } - cl->cl_bandwidth = bandwidth / 8; /* cvt to bytes per second */ - cl->cl_qtype = Q_DROPTAIL; - cl->cl_qstate = QS_RUNNING; - cl->cl_flags = flags; - cl->cl_pri = pri; - if (pri > fif->fif_maxpri) - fif->fif_maxpri = pri; - cl->cl_fif = fif; - cl->cl_handle = qid; - cl->cl_hogs_m1 = hogs_m1 / 8; - cl->cl_lssc_m1 = lssc_m1 / 8; /* NOT YET USED */ - cl->cl_bw_current = 0; - - if (flags & (FARF_RED|FARF_RIO|FARF_BLUE|FARF_SFB)) { -#if CLASSQ_RED || CLASSQ_RIO - u_int64_t ifbandwidth = ifnet_output_linkrate(ifp); - int pkttime; -#endif /* CLASSQ_RED || CLASSQ_RIO */ - - cl->cl_qflags = 0; - if (flags & FARF_ECN) { - if (flags & FARF_BLUE) - cl->cl_qflags |= BLUEF_ECN; - else if (flags & FARF_SFB) - cl->cl_qflags |= SFBF_ECN; - else if (flags & FARF_RED) - cl->cl_qflags |= REDF_ECN; - else if (flags & FARF_RIO) - cl->cl_qflags |= RIOF_ECN; - } - if (flags & FARF_FLOWCTL) { - if (flags & FARF_SFB) - cl->cl_qflags |= SFBF_FLOWCTL; - } - if (flags & FARF_CLEARDSCP) { - if (flags & FARF_RIO) - cl->cl_qflags |= RIOF_CLEARDSCP; - } -#if CLASSQ_RED || CLASSQ_RIO - /* - * XXX: RED & RIO should be watching link speed and MTU - * events and recompute pkttime accordingly. - */ - if (ifbandwidth < 8) - pkttime = 1000 * 1000 * 1000; /* 1 sec */ - else - pkttime = (int64_t)ifp->if_mtu * 1000 * 1000 * 1000 / - (ifbandwidth / 8); - - /* Test for exclusivity {RED,RIO,BLUE,SFB} was done above */ -#if CLASSQ_RIO - if (flags & FARF_RIO) { - cl->cl_rio = - rio_alloc(ifp, 0, NULL, cl->cl_qflags, pkttime); - if (cl->cl_rio != NULL) - cl->cl_qtype = Q_RIO; - } -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (flags & FARF_RED) { - cl->cl_red = red_alloc(ifp, 0, 0, - cl->cl_qlimit * 10/100, - cl->cl_qlimit * 30/100, - cl->cl_qflags, pkttime); - if (cl->cl_red != NULL) - cl->cl_qtype = Q_RED; - } -#endif /* CLASSQ_RED */ -#endif /* CLASSQ_RED || CLASSQ_RIO */ -#if CLASSQ_BLUE - if (flags & FARF_BLUE) { - cl->cl_blue = blue_alloc(ifp, 0, 0, cl->cl_qflags); - if (cl->cl_blue != NULL) - cl->cl_qtype = Q_BLUE; - } -#endif /* CLASSQ_BLUE */ - if (flags & FARF_SFB) { - if (!(cl->cl_flags & FARF_LAZY)) - cl->cl_sfb = sfb_alloc(ifp, cl->cl_handle, - cl->cl_qlimit, cl->cl_qflags); - if (cl->cl_sfb != NULL || (cl->cl_flags & FARF_LAZY)) - cl->cl_qtype = Q_SFB; - } - } - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s created qid=%d pri=%d qlimit=%d " - "flags=%b\n", if_name(ifp), fairq_style(fif), - cl->cl_handle, cl->cl_pri, cl->cl_qlimit, flags, FARF_BITS); - } - - return (cl); - -err_buckets: - if (cl->cl_buckets != NULL) - _FREE(cl->cl_buckets, M_DEVBUF); -err_ret: - if (cl != NULL) { - if (cl->cl_qalg.ptr != NULL) { -#if CLASSQ_RIO - if (cl->cl_qtype == Q_RIO) - rio_destroy(cl->cl_rio); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (cl->cl_qtype == Q_RED) - red_destroy(cl->cl_red); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (cl->cl_qtype == Q_BLUE) - blue_destroy(cl->cl_blue); -#endif /* CLASSQ_BLUE */ - if (cl->cl_qtype == Q_SFB && cl->cl_sfb != NULL) - sfb_destroy(cl->cl_sfb); - cl->cl_qalg.ptr = NULL; - cl->cl_qtype = Q_DROPTAIL; - cl->cl_qstate = QS_RUNNING; - } - zfree(fairq_cl_zone, cl); - } - return (NULL); -} - -int -fairq_remove_queue(struct fairq_if *fif, u_int32_t qid) -{ - struct fairq_class *cl; - - IFCQ_LOCK_ASSERT_HELD(fif->fif_ifq); - - if ((cl = fairq_clh_to_clp(fif, qid)) == NULL) - return (EINVAL); - - return (fairq_class_destroy(fif, cl)); -} - -static int -fairq_class_destroy(struct fairq_if *fif, struct fairq_class *cl) -{ - struct ifclassq *ifq = fif->fif_ifq; - int pri; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - if (cl->cl_head) - fairq_purgeq(fif, cl, 0, NULL, NULL); - - fif->fif_classes[cl->cl_pri] = NULL; - if (fif->fif_poll_cache == cl) - fif->fif_poll_cache = NULL; - if (fif->fif_maxpri == cl->cl_pri) { - for (pri = cl->cl_pri; pri >= 0; pri--) - if (fif->fif_classes[pri] != NULL) { - fif->fif_maxpri = pri; - break; - } - if (pri < 0) - fif->fif_maxpri = -1; - } - - if (cl->cl_qalg.ptr != NULL) { -#if CLASSQ_RIO - if (cl->cl_qtype == Q_RIO) - rio_destroy(cl->cl_rio); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (cl->cl_qtype == Q_RED) - red_destroy(cl->cl_red); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (cl->cl_qtype == Q_BLUE) - blue_destroy(cl->cl_blue); -#endif /* CLASSQ_BLUE */ - if (cl->cl_qtype == Q_SFB && cl->cl_sfb != NULL) - sfb_destroy(cl->cl_sfb); - cl->cl_qalg.ptr = NULL; - cl->cl_qtype = Q_DROPTAIL; - cl->cl_qstate = QS_RUNNING; - } - - if (fif->fif_default == cl) - fif->fif_default = NULL; - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s destroyed qid=%d pri=%d\n", - if_name(FAIRQIF_IFP(fif)), fairq_style(fif), - cl->cl_handle, cl->cl_pri); - } - - _FREE(cl->cl_buckets, M_DEVBUF); - cl->cl_head = NULL; /* sanity */ - cl->cl_polled = NULL; /* sanity */ - cl->cl_buckets = NULL; /* sanity */ - - zfree(fairq_cl_zone, cl); - - return (0); -} - -int -fairq_enqueue(struct fairq_if *fif, struct fairq_class *cl, struct mbuf *m, - struct pf_mtag *t) -{ - struct ifclassq *ifq = fif->fif_ifq; - int len, ret; - - IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(cl == NULL || cl->cl_fif == fif); - - if (cl == NULL) { -#if PF_ALTQ - cl = fairq_clh_to_clp(fif, t->pftag_qid); -#else /* !PF_ALTQ */ - cl = fairq_clh_to_clp(fif, 0); -#endif /* !PF_ALTQ */ - if (cl == NULL) { - cl = fif->fif_default; - if (cl == NULL) { - IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (ENOBUFS); - } - } - } - - cl->cl_flags |= FARF_HAS_PACKETS; - len = m_pktlen(m); - - ret = fairq_addq(cl, m, t); - if (ret != 0) { - if (ret == CLASSQEQ_SUCCESS_FC) { - /* packet enqueued, return advisory feedback */ - ret = EQFULL; - } else { - VERIFY(ret == CLASSQEQ_DROPPED || - ret == CLASSQEQ_DROPPED_FC || - ret == CLASSQEQ_DROPPED_SP); - - /* packet has been freed in fairq_addq */ - PKTCNTR_ADD(&cl->cl_dropcnt, 1, len); - IFCQ_DROP_ADD(ifq, 1, len); - switch (ret) { - case CLASSQEQ_DROPPED: - return (ENOBUFS); - case CLASSQEQ_DROPPED_FC: - return (EQFULL); - case CLASSQEQ_DROPPED_SP: - return (EQSUSPENDED); - } - /* NOT REACHED */ - } - } - IFCQ_INC_LEN(ifq); - IFCQ_INC_BYTES(ifq, len); - - /* successfully queued. */ - return (ret); -} - -/* - * note: CLASSQDQ_POLL returns the next packet without removing the packet - * from the queue. CLASSQDQ_REMOVE is a normal dequeue operation. - * CLASSQDQ_REMOVE must return the same packet if called immediately - * after CLASSQDQ_POLL. - */ -struct mbuf * -fairq_dequeue(struct fairq_if *fif, cqdq_op_t op) -{ - struct ifclassq *ifq = fif->fif_ifq; - struct fairq_class *cl; - struct fairq_class *best_cl; - struct mbuf *best_m; - struct mbuf *m; - u_int64_t cur_time = read_machclk(); - u_int32_t best_scale; - u_int32_t scale; - int pri; - int hit_limit; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - if (IFCQ_IS_EMPTY(ifq)) { - /* no packet in the queue */ - return (NULL); - } - - if (fif->fif_poll_cache && op == CLASSQDQ_REMOVE) { - best_cl = fif->fif_poll_cache; - m = fairq_getq(best_cl, cur_time); - fif->fif_poll_cache = NULL; - if (m != NULL) { - IFCQ_DEC_LEN(ifq); - IFCQ_DEC_BYTES(ifq, m_pktlen(m)); - IFCQ_XMIT_ADD(ifq, 1, m_pktlen(m)); - PKTCNTR_ADD(&best_cl->cl_xmitcnt, 1, m_pktlen(m)); - } - } else { - best_cl = NULL; - best_m = NULL; - best_scale = 0xFFFFFFFFU; - - for (pri = fif->fif_maxpri; pri >= 0; pri--) { - if ((cl = fif->fif_classes[pri]) == NULL) - continue; - if ((cl->cl_flags & FARF_HAS_PACKETS) == 0) - continue; - m = fairq_pollq(cl, cur_time, &hit_limit); - if (m == NULL) { - cl->cl_flags &= ~FARF_HAS_PACKETS; - continue; - } - - /* - * We can halt the search immediately if the queue - * did not hit its bandwidth limit. - */ - if (hit_limit == 0) { - best_cl = cl; - best_m = m; - break; - } - - /* - * Otherwise calculate the scale factor and select - * the queue with the lowest scale factor. This - * apportions any unused bandwidth weighted by - * the relative bandwidth specification. - */ - scale = cl->cl_bw_current * 100 / cl->cl_bandwidth; - if (scale < best_scale) { - best_cl = cl; - best_m = m; - best_scale = scale; - } - } - - if (op == CLASSQDQ_POLL) { - fif->fif_poll_cache = best_cl; - m = best_m; - } else if (best_cl != NULL) { - m = fairq_getq(best_cl, cur_time); - if (m != NULL) { - IFCQ_DEC_LEN(ifq); - IFCQ_DEC_BYTES(ifq, m_pktlen(m)); - IFCQ_XMIT_ADD(ifq, 1, m_pktlen(m)); - PKTCNTR_ADD(&best_cl->cl_xmitcnt, 1, - m_pktlen(m)); - } - } else { - m = NULL; - } - } - return (m); -} - -static inline int -fairq_addq(struct fairq_class *cl, struct mbuf *m, struct pf_mtag *t) -{ - struct ifclassq *ifq = cl->cl_fif->fif_ifq; - fairq_bucket_t *b; - u_int32_t hash = m->m_pkthdr.pkt_flowid; - u_int32_t hindex; - u_int64_t bw; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - /* - * If the packet doesn't have any keep state put it on the end of - * our queue. XXX this can result in out of order delivery. - */ - if (hash == 0) { - if (cl->cl_head) - b = cl->cl_head->prev; - else - b = &cl->cl_buckets[0]; - } else { - hindex = (hash & cl->cl_nbucket_mask); - b = &cl->cl_buckets[hindex]; - } - - /* - * Add the bucket to the end of the circular list of active buckets. - * - * As a special case we add the bucket to the beginning of the list - * instead of the end if it was not previously on the list and if - * its traffic is less then the hog level. - */ - if (b->in_use == 0) { - b->in_use = 1; - if (cl->cl_head == NULL) { - cl->cl_head = b; - b->next = b; - b->prev = b; - } else { - b->next = cl->cl_head; - b->prev = cl->cl_head->prev; - b->prev->next = b; - b->next->prev = b; - - if (b->bw_delta && cl->cl_hogs_m1) { - bw = b->bw_bytes * machclk_freq / b->bw_delta; - if (bw < cl->cl_hogs_m1) - cl->cl_head = b; - } - } - } - -#if CLASSQ_RIO - if (cl->cl_qtype == Q_RIO) - return (rio_addq(cl->cl_rio, &b->queue, m, t)); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (cl->cl_qtype == Q_RED) - return (red_addq(cl->cl_red, &b->queue, m, t)); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (cl->cl_qtype == Q_BLUE) - return (blue_addq(cl->cl_blue, &b->queue, m, t)); - else -#endif /* CLASSQ_BLUE */ - if (cl->cl_qtype == Q_SFB) { - if (cl->cl_sfb == NULL) { - struct ifnet *ifp = FAIRQIF_IFP(cl->cl_fif); - - VERIFY(cl->cl_flags & FARF_LAZY); - IFCQ_CONVERT_LOCK(ifq); - - cl->cl_sfb = sfb_alloc(ifp, cl->cl_handle, - cl->cl_qlimit, cl->cl_qflags); - if (cl->cl_sfb == NULL) { - /* fall back to droptail */ - cl->cl_qtype = Q_DROPTAIL; - cl->cl_flags &= ~FARF_SFB; - cl->cl_qflags &= ~(SFBF_ECN | SFBF_FLOWCTL); - - log(LOG_ERR, "%s: %s SFB lazy allocation " - "failed for qid=%d pri=%d, falling back " - "to DROPTAIL\n", if_name(ifp), - fairq_style(cl->cl_fif), cl->cl_handle, - cl->cl_pri); - } - } - if (cl->cl_sfb != NULL) - return (sfb_addq(cl->cl_sfb, &b->queue, m, t)); - } else if (qlen(&b->queue) >= qlimit(&b->queue)) { - IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (CLASSQEQ_DROPPED); - } - -#if PF_ECN - if (cl->cl_flags & FARF_CLEARDSCP) - write_dsfield(m, t, 0); -#endif /* PF_ECN */ - - _addq(&b->queue, m); - - return (0); -} - -static inline struct mbuf * -fairq_getq(struct fairq_class *cl, u_int64_t cur_time) -{ - fairq_bucket_t *b; - struct mbuf *m; - - IFCQ_LOCK_ASSERT_HELD(cl->cl_fif->fif_ifq); - - b = fairq_selectq(cl, 0); - if (b == NULL) - m = NULL; -#if CLASSQ_RIO - else if (cl->cl_qtype == Q_RIO) - m = rio_getq(cl->cl_rio, &b->queue); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - else if (cl->cl_qtype == Q_RED) - m = red_getq(cl->cl_red, &b->queue); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - else if (cl->cl_qtype == Q_BLUE) - m = blue_getq(cl->cl_blue, &b->queue); -#endif /* CLASSQ_BLUE */ - else if (cl->cl_qtype == Q_SFB && cl->cl_sfb != NULL) - m = sfb_getq(cl->cl_sfb, &b->queue); - else - m = _getq(&b->queue); - - /* - * Calculate the BW change - */ - if (m != NULL) { - u_int64_t delta; - - /* - * Per-class bandwidth calculation - */ - delta = (cur_time - cl->cl_last_time); - if (delta > machclk_freq * 8) - delta = machclk_freq * 8; - cl->cl_bw_delta += delta; - cl->cl_bw_bytes += m->m_pkthdr.len; - cl->cl_last_time = cur_time; - if (cl->cl_bw_delta > machclk_freq) { - cl->cl_bw_delta -= cl->cl_bw_delta >> 2; - cl->cl_bw_bytes -= cl->cl_bw_bytes >> 2; - } - - /* - * Per-bucket bandwidth calculation - */ - delta = (cur_time - b->last_time); - if (delta > machclk_freq * 8) - delta = machclk_freq * 8; - b->bw_delta += delta; - b->bw_bytes += m->m_pkthdr.len; - b->last_time = cur_time; - if (b->bw_delta > machclk_freq) { - b->bw_delta -= b->bw_delta >> 2; - b->bw_bytes -= b->bw_bytes >> 2; - } - } - return (m); -} - -/* - * Figure out what the next packet would be if there were no limits. If - * this class hits its bandwidth limit *hit_limit is set to no-zero, otherwise - * it is set to 0. A non-NULL mbuf is returned either way. - */ -static inline struct mbuf * -fairq_pollq(struct fairq_class *cl, u_int64_t cur_time, int *hit_limit) -{ - fairq_bucket_t *b; - struct mbuf *m; - u_int64_t delta; - u_int64_t bw; - - IFCQ_LOCK_ASSERT_HELD(cl->cl_fif->fif_ifq); - - *hit_limit = 0; - b = fairq_selectq(cl, 1); - if (b == NULL) - return (NULL); - m = qhead(&b->queue); - - /* - * Did this packet exceed the class bandwidth? Calculate the - * bandwidth component of the packet. - * - * - Calculate bytes per second - */ - delta = cur_time - cl->cl_last_time; - if (delta > machclk_freq * 8) - delta = machclk_freq * 8; - cl->cl_bw_delta += delta; - cl->cl_last_time = cur_time; - if (cl->cl_bw_delta) { - bw = cl->cl_bw_bytes * machclk_freq / cl->cl_bw_delta; - - if (bw > cl->cl_bandwidth) - *hit_limit = 1; - cl->cl_bw_current = bw; -#if 0 - printf("BW %6lld relative to %6u %d queue 0x%llx\n", - bw, cl->cl_bandwidth, *hit_limit, - (uint64_t)VM_KERNEL_ADDRPERM(b)); -#endif - } - return (m); -} - -/* - * Locate the next queue we want to pull a packet out of. This code - * is also responsible for removing empty buckets from the circular list. - */ -static fairq_bucket_t * -fairq_selectq(struct fairq_class *cl, int ispoll) -{ - fairq_bucket_t *b; - u_int64_t bw; - - IFCQ_LOCK_ASSERT_HELD(cl->cl_fif->fif_ifq); - - if (ispoll == 0 && cl->cl_polled) { - b = cl->cl_polled; - cl->cl_polled = NULL; - return (b); - } - - while ((b = cl->cl_head) != NULL) { - /* - * Remove empty queues from consideration - */ - if (qempty(&b->queue)) { - b->in_use = 0; - cl->cl_head = b->next; - if (cl->cl_head == b) { - cl->cl_head = NULL; - } else { - b->next->prev = b->prev; - b->prev->next = b->next; - } - continue; - } - - /* - * Advance the round robin. Queues with bandwidths less - * then the hog bandwidth are allowed to burst. - */ - if (cl->cl_hogs_m1 == 0) { - cl->cl_head = b->next; - } else if (b->bw_delta) { - bw = b->bw_bytes * machclk_freq / b->bw_delta; - if (bw >= cl->cl_hogs_m1) { - cl->cl_head = b->next; - } - /* - * XXX TODO - - */ - } - - /* - * Return bucket b. - */ - break; - } - if (ispoll) - cl->cl_polled = b; - return (b); -} - -static void -fairq_purgeq(struct fairq_if *fif, struct fairq_class *cl, u_int32_t flow, - u_int32_t *packets, u_int32_t *bytes) -{ - struct ifclassq *ifq = fif->fif_ifq; - u_int32_t _cnt = 0, _len = 0; - fairq_bucket_t *b; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - /* become regular mutex before freeing mbufs */ - IFCQ_CONVERT_LOCK(ifq); - - while ((b = fairq_selectq(cl, 0)) != NULL) { - u_int32_t cnt, len, qlen; - - if ((qlen = qlen(&b->queue)) == 0) - continue; - -#if CLASSQ_RIO - if (cl->cl_qtype == Q_RIO) - rio_purgeq(cl->cl_rio, &b->queue, flow, &cnt, &len); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (cl->cl_qtype == Q_RED) - red_purgeq(cl->cl_red, &b->queue, flow, &cnt, &len); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (cl->cl_qtype == Q_BLUE) - blue_purgeq(cl->cl_blue, &b->queue, flow, &cnt, &len); - else -#endif /* CLASSQ_BLUE */ - if (cl->cl_qtype == Q_SFB && cl->cl_sfb != NULL) - sfb_purgeq(cl->cl_sfb, &b->queue, flow, &cnt, &len); - else - _flushq_flow(&b->queue, flow, &cnt, &len); - - if (cnt == 0) - continue; - - VERIFY(qlen(&b->queue) == (qlen - cnt)); - - PKTCNTR_ADD(&cl->cl_dropcnt, cnt, len); - IFCQ_DROP_ADD(ifq, cnt, len); - - VERIFY(((signed)IFCQ_LEN(ifq) - cnt) >= 0); - IFCQ_LEN(ifq) -= cnt; - - _cnt += cnt; - _len += len; - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s purge qid=%d pri=%d " - "qlen=[%d,%d] cnt=%d len=%d flow=0x%x\n", - if_name(FAIRQIF_IFP(fif)), fairq_style(fif), - cl->cl_handle, cl->cl_pri, qlen, qlen(&b->queue), - cnt, len, flow); - } - } - - if (packets != NULL) - *packets = _cnt; - if (bytes != NULL) - *bytes = _len; -} - -static void -fairq_updateq(struct fairq_if *fif, struct fairq_class *cl, cqev_t ev) -{ - IFCQ_LOCK_ASSERT_HELD(fif->fif_ifq); - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s update qid=%d pri=%d event=%s\n", - if_name(FAIRQIF_IFP(fif)), fairq_style(fif), - cl->cl_handle, cl->cl_pri, ifclassq_ev2str(ev)); - } - -#if CLASSQ_RIO - if (cl->cl_qtype == Q_RIO) - return (rio_updateq(cl->cl_rio, ev)); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (cl->cl_qtype == Q_RED) - return (red_updateq(cl->cl_red, ev)); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (cl->cl_qtype == Q_BLUE) - return (blue_updateq(cl->cl_blue, ev)); -#endif /* CLASSQ_BLUE */ - if (cl->cl_qtype == Q_SFB && cl->cl_sfb != NULL) - return (sfb_updateq(cl->cl_sfb, ev)); -} - -int -fairq_get_class_stats(struct fairq_if *fif, u_int32_t qid, - struct fairq_classstats *sp) -{ - struct fairq_class *cl; - fairq_bucket_t *b; - - IFCQ_LOCK_ASSERT_HELD(fif->fif_ifq); - - if ((cl = fairq_clh_to_clp(fif, qid)) == NULL) - return (EINVAL); - - sp->class_handle = cl->cl_handle; - sp->priority = cl->cl_pri; - sp->qlimit = cl->cl_qlimit; - sp->xmit_cnt = cl->cl_xmitcnt; - sp->drop_cnt = cl->cl_dropcnt; - sp->qtype = cl->cl_qtype; - sp->qstate = cl->cl_qstate; - sp->qlength = 0; - - if (cl->cl_head) { - b = cl->cl_head; - do { - sp->qlength += qlen(&b->queue); - b = b->next; - } while (b != cl->cl_head); - } - -#if CLASSQ_RED - if (cl->cl_qtype == Q_RED) - red_getstats(cl->cl_red, &sp->red[0]); -#endif /* CLASSQ_RED */ -#if CLASSQ_RIO - if (cl->cl_qtype == Q_RIO) - rio_getstats(cl->cl_rio, &sp->red[0]); -#endif /* CLASSQ_RIO */ -#if CLASSQ_BLUE - if (cl->cl_qtype == Q_BLUE) - blue_getstats(cl->cl_blue, &sp->blue); -#endif /* CLASSQ_BLUE */ - if (cl->cl_qtype == Q_SFB && cl->cl_sfb != NULL) - sfb_getstats(cl->cl_sfb, &sp->sfb); - - return (0); -} - -/* convert a class handle to the corresponding class pointer */ -static inline struct fairq_class * -fairq_clh_to_clp(struct fairq_if *fif, u_int32_t chandle) -{ - struct fairq_class *cl; - int idx; - - IFCQ_LOCK_ASSERT_HELD(fif->fif_ifq); - - for (idx = fif->fif_maxpri; idx >= 0; idx--) - if ((cl = fif->fif_classes[idx]) != NULL && - cl->cl_handle == chandle) - return (cl); - - return (NULL); -} - -static const char * -fairq_style(struct fairq_if *fif) -{ - return ((fif->fif_flags & FAIRQIFF_ALTQ) ? "ALTQ_FAIRQ" : "FAIRQ"); -} - -int -fairq_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) -{ -#pragma unused(ifq, flags) - return (ENXIO); /* not yet */ -} - -int -fairq_teardown_ifclassq(struct ifclassq *ifq) -{ - struct fairq_if *fif = ifq->ifcq_disc; - int i; - - IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(fif != NULL && ifq->ifcq_type == PKTSCHEDT_FAIRQ); - - (void) fairq_destroy_locked(fif); - - ifq->ifcq_disc = NULL; - for (i = 0; i < IFCQ_SC_MAX; i++) { - ifq->ifcq_disc_slots[i].qid = 0; - ifq->ifcq_disc_slots[i].cl = NULL; - } - - return (ifclassq_detach(ifq)); -} - -int -fairq_getqstats_ifclassq(struct ifclassq *ifq, u_int32_t slot, - struct if_ifclassq_stats *ifqs) -{ - struct fairq_if *fif = ifq->ifcq_disc; - - IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(ifq->ifcq_type == PKTSCHEDT_FAIRQ); - - if (slot >= IFCQ_SC_MAX) - return (EINVAL); - - return (fairq_get_class_stats(fif, ifq->ifcq_disc_slots[slot].qid, - &ifqs->ifqs_fairq_stats)); -} -#endif /* PKTSCHED_FAIRQ */ diff --git a/bsd/net/pktsched/pktsched_fairq.h b/bsd/net/pktsched/pktsched_fairq.h index 910172950..bbe88c451 100644 --- a/bsd/net/pktsched/pktsched_fairq.h +++ b/bsd/net/pktsched/pktsched_fairq.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2012 Apple Inc. All rights reserved. + * Copyright (c) 2011-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -133,97 +133,6 @@ struct fairq_classstats { classq_state_t qstate; }; -#ifdef BSD_KERNEL_PRIVATE - -typedef struct fairq_bucket { - struct fairq_bucket *next; /* circular list */ - struct fairq_bucket *prev; /* circular list */ - class_queue_t queue; /* the actual queue */ - u_int64_t bw_bytes; /* statistics used to calculate bw */ - u_int64_t bw_delta; /* statistics used to calculate bw */ - u_int64_t last_time; - int in_use; -} fairq_bucket_t; - -struct fairq_class { - u_int32_t cl_handle; /* class handle */ - u_int32_t cl_nbuckets; /* (power of 2) */ - u_int32_t cl_nbucket_mask; /* bucket mask */ - u_int32_t cl_qflags; /* class queue flags */ - fairq_bucket_t *cl_buckets; - fairq_bucket_t *cl_head; /* head of circular bucket list */ - fairq_bucket_t *cl_polled; - union { - void *ptr; - struct red *red; /* RED state */ - struct rio *rio; /* RIO state */ - struct blue *blue; /* BLUE state */ - struct sfb *sfb; /* SFB state */ - } cl_qalg; - u_int64_t cl_hogs_m1; - u_int64_t cl_lssc_m1; - u_int64_t cl_bandwidth; - u_int64_t cl_bw_current; - u_int64_t cl_bw_bytes; - u_int64_t cl_bw_delta; - u_int64_t cl_last_time; - classq_type_t cl_qtype; /* rollup */ - classq_state_t cl_qstate; /* state */ - int cl_qlimit; - int cl_pri; /* priority */ - int cl_flags; /* class flags */ - struct fairq_if *cl_fif; /* back pointer to fif */ - - /* round robin index */ - - /* statistics */ - struct pktcntr cl_xmitcnt; /* transmitted packet counter */ - struct pktcntr cl_dropcnt; /* dropped packet counter */ -}; - -#define cl_red cl_qalg.red -#define cl_rio cl_qalg.rio -#define cl_blue cl_qalg.blue -#define cl_sfb cl_qalg.sfb - -/* fairq_if flags */ -#define FAIRQIFF_ALTQ 0x1 /* configured via PF/ALTQ */ - -/* - * fairq interface state - */ -struct fairq_if { - struct ifclassq *fif_ifq; /* backpointer to ifclassq */ - int fif_maxpri; /* max priority in use */ - u_int32_t fif_flags; /* flags */ - struct fairq_class *fif_poll_cache; /* cached poll */ - struct fairq_class *fif_default; /* default class */ - struct fairq_class *fif_classes[FAIRQ_MAXPRI]; /* classes */ -}; - -#define FAIRQIF_IFP(_fif) ((_fif)->fif_ifq->ifcq_ifp) - -struct if_ifclassq_stats; - -extern void fairq_init(void); -extern struct fairq_if *fairq_alloc(struct ifnet *, int, boolean_t); -extern int fairq_destroy(struct fairq_if *); -extern void fairq_purge(struct fairq_if *); -extern void fairq_event(struct fairq_if *, cqev_t); -extern int fairq_add_queue(struct fairq_if *, int, u_int32_t, u_int64_t, - u_int32_t, int, u_int64_t, u_int64_t, u_int64_t, u_int64_t, u_int32_t, - struct fairq_class **); -extern int fairq_remove_queue(struct fairq_if *, u_int32_t); -extern int fairq_get_class_stats(struct fairq_if *, u_int32_t, - struct fairq_classstats *); -extern int fairq_enqueue(struct fairq_if *, struct fairq_class *, - struct mbuf *, struct pf_mtag *); -extern struct mbuf *fairq_dequeue(struct fairq_if *, cqdq_op_t); -extern int fairq_setup_ifclassq(struct ifclassq *, u_int32_t); -extern int fairq_teardown_ifclassq(struct ifclassq *ifq); -extern int fairq_getqstats_ifclassq(struct ifclassq *, u_int32_t, - struct if_ifclassq_stats *); -#endif /* BSD_KERNEL_PRIVATE */ #ifdef __cplusplus } #endif diff --git a/bsd/net/pktsched/pktsched_fq_codel.c b/bsd/net/pktsched/pktsched_fq_codel.c index f7eef8378..c30dc2eb1 100644 --- a/bsd/net/pktsched/pktsched_fq_codel.c +++ b/bsd/net/pktsched/pktsched_fq_codel.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 Apple Inc. All rights reserved. + * Copyright (c) 2016-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -35,21 +35,26 @@ #include <net/classq/classq_fq_codel.h> #include <net/pktsched/pktsched_fq_codel.h> - static size_t fq_if_size; static struct zone *fq_if_zone; -static fq_if_t *fq_if_alloc(struct ifnet *ifp, int how); +static fq_if_t *fq_if_alloc(struct ifnet *, classq_pkt_type_t); static void fq_if_destroy(fq_if_t *fqs); static void fq_if_classq_init(fq_if_t *fqs, u_int32_t priority, u_int32_t quantum, u_int32_t drr_max, u_int32_t svc_class); -static int fq_if_enqueue_classq(struct ifclassq *ifq, struct mbuf *m); -static struct mbuf *fq_if_dequeue_classq(struct ifclassq *ifq, cqdq_op_t); -static int fq_if_dequeue_classq_multi(struct ifclassq *, cqdq_op_t, - u_int32_t, u_int32_t, struct mbuf **, struct mbuf **, u_int32_t *, - u_int32_t *); +static int fq_if_enqueue_classq(struct ifclassq *ifq, void *p, + classq_pkt_type_t ptype, boolean_t *pdrop); +static void *fq_if_dequeue_classq(struct ifclassq *, classq_pkt_type_t *); +static int fq_if_dequeue_classq_multi(struct ifclassq *, u_int32_t, + u_int32_t, void **, void **, u_int32_t *, u_int32_t *, classq_pkt_type_t *); +static void *fq_if_dequeue_sc_classq(struct ifclassq *, mbuf_svc_class_t, + classq_pkt_type_t *); +static int fq_if_dequeue_sc_classq_multi(struct ifclassq *, + mbuf_svc_class_t, u_int32_t, u_int32_t, void **, + void **, u_int32_t *, u_int32_t *, classq_pkt_type_t *); static void fq_if_dequeue(fq_if_t *, fq_if_classq_t *, u_int32_t, - u_int32_t, struct mbuf **, struct mbuf **, u_int32_t *, u_int32_t *); + u_int32_t, void **, void **, u_int32_t *, u_int32_t *, + boolean_t drvmgmt, classq_pkt_type_t *); static int fq_if_request_classq(struct ifclassq *ifq, cqrq_t op, void *arg); void fq_if_stat_sc(fq_if_t *fqs, cqrq_stat_sc_t *stat); static void fq_if_purge(fq_if_t *); @@ -59,8 +64,6 @@ static void fq_if_empty_new_flow(fq_t *fq, fq_if_classq_t *fq_cl, bool add_to_old); static void fq_if_empty_old_flow(fq_if_t *fqs, fq_if_classq_t *fq_cl, fq_t *fq, bool remove_hash); -static void fq_if_destroy_flow(fq_if_t *fqs, fq_if_classq_t *fq_cl, - fq_t *fq); #define FQ_IF_ZONE_MAX 32 /* Maximum elements in zone */ #define FQ_IF_ZONE_NAME "pktsched_fq_if" /* zone for fq_if class */ @@ -72,6 +75,68 @@ static void fq_if_destroy_flow(fq_if_t *fqs, fq_if_classq_t *fq_cl, (STAILQ_EMPTY(&(_fcl_)->fcl_new_flows) && \ STAILQ_EMPTY(&(_fcl_)->fcl_old_flows)) +typedef void (* fq_if_append_pkt_t)(void *, void *); +typedef boolean_t (* fq_getq_flow_t)(fq_if_t *, fq_if_classq_t *, fq_t *, + u_int32_t, u_int32_t, void **, void **, u_int32_t *, u_int32_t *, + boolean_t *, u_int32_t); + +static void +fq_if_append_mbuf(void *pkt, void *next_pkt) +{ + ((mbuf_t)pkt)->m_nextpkt = (mbuf_t)next_pkt; +} + + + +static boolean_t +fq_getq_flow_mbuf(fq_if_t *fqs, fq_if_classq_t *fq_cl, fq_t *fq, + u_int32_t byte_limit, u_int32_t pkt_limit, void **top, void **last, + u_int32_t *byte_cnt, u_int32_t *pkt_cnt, boolean_t *qempty, + u_int32_t pflags) +{ + struct mbuf *m; + u_int32_t plen; + pktsched_pkt_t pkt; + boolean_t limit_reached = FALSE; + struct ifclassq *ifq = fqs->fqs_ifq; + struct ifnet *ifp = ifq->ifcq_ifp; + + while (fq->fq_deficit > 0 && limit_reached == FALSE && + !MBUFQ_EMPTY(&fq->fq_mbufq)) { + + _PKTSCHED_PKT_INIT(&pkt); + m = fq_getq_flow(fqs, fq, &pkt); + ASSERT(pkt.pktsched_ptype == QP_MBUF); + + plen = pktsched_get_pkt_len(&pkt); + fq->fq_deficit -= plen; + m->m_pkthdr.pkt_flags |= pflags; + + if (*top == NULL) { + *top = m; + } else { + ASSERT(*last != NULL); + ASSERT((*(struct mbuf **)last)->m_nextpkt == NULL); + (*(struct mbuf **)last)->m_nextpkt = m; + } + *last = m; + (*(mbuf_t *)last)->m_nextpkt = NULL; + fq_cl->fcl_stat.fcl_dequeue++; + fq_cl->fcl_stat.fcl_dequeue_bytes += plen; + *pkt_cnt += 1; + *byte_cnt += plen; + + ifclassq_set_packet_metadata(ifq, ifp, m, QP_MBUF); + + /* Check if the limit is reached */ + if (*pkt_cnt >= pkt_limit || *byte_cnt >= byte_limit) + limit_reached = TRUE; + } + + *qempty = MBUFQ_EMPTY(&fq->fq_mbufq); + return (limit_reached); +} + void fq_codel_scheduler_init(void) { @@ -91,23 +156,25 @@ fq_codel_scheduler_init(void) } fq_if_t * -fq_if_alloc(struct ifnet *ifp, int how) +fq_if_alloc(struct ifnet *ifp, classq_pkt_type_t ptype) { fq_if_t *fqs; - fqs = (how == M_WAITOK) ? zalloc(fq_if_zone) : - zalloc_noblock(fq_if_zone); + fqs = zalloc(fq_if_zone); if (fqs == NULL) return (NULL); bzero(fqs, fq_if_size); fqs->fqs_ifq = &ifp->if_snd; + fqs->fqs_ptype = ptype; /* Calculate target queue delay */ ifclassq_calc_target_qdelay(ifp, &fqs->fqs_target_qdelay); /* Calculate update interval */ ifclassq_calc_update_interval(&fqs->fqs_update_interval); - fqs->fqs_pkt_droplimit = FQ_IF_MAX_PKT_LIMIT; + + /* Configure packet drop limit across all queues */ + fqs->fqs_pkt_droplimit = IFCQ_PKT_DROP_LIMIT(&ifp->if_snd); STAILQ_INIT(&fqs->fqs_fclist); return (fqs); } @@ -115,17 +182,44 @@ fq_if_alloc(struct ifnet *ifp, int how) void fq_if_destroy(fq_if_t *fqs) { - IFCQ_LOCK_ASSERT_HELD(fqs->fqs_ifq); fq_if_purge(fqs); fqs->fqs_ifq = NULL; zfree(fq_if_zone, fqs); } static inline u_int32_t -fq_if_service_to_priority(mbuf_svc_class_t svc) +fq_if_service_to_priority(fq_if_t *fqs, mbuf_svc_class_t svc) { u_int32_t pri; + if (fqs->fqs_flags & FQS_DRIVER_MANAGED) { + switch (svc) { + case MBUF_SC_BK_SYS: + case MBUF_SC_BK: + pri = FQ_IF_BK_INDEX; + break; + case MBUF_SC_BE: + case MBUF_SC_RD: + case MBUF_SC_OAM: + pri = FQ_IF_BE_INDEX; + break; + case MBUF_SC_AV: + case MBUF_SC_RV: + case MBUF_SC_VI: + pri = FQ_IF_VI_INDEX; + break; + case MBUF_SC_VO: + case MBUF_SC_CTL: + pri = FQ_IF_VO_INDEX; + break; + default: + pri = FQ_IF_BE_INDEX; /* Use best effort by default */ + break; + } + return (pri); + } + + /* scheduler is not managed by the driver */ switch (svc) { case MBUF_SC_BK_SYS: pri = FQ_IF_BK_SYS_INDEX; @@ -183,24 +277,28 @@ fq_if_classq_init(fq_if_t *fqs, u_int32_t pri, u_int32_t quantum, } int -fq_if_enqueue_classq(struct ifclassq *ifq, struct mbuf *m) +fq_if_enqueue_classq(struct ifclassq *ifq, void *p, classq_pkt_type_t ptype, + boolean_t *pdrop) { u_int32_t pri; fq_if_t *fqs; fq_if_classq_t *fq_cl; int ret, len; mbuf_svc_class_t svc; + pktsched_pkt_t pkt; IFCQ_LOCK_ASSERT_HELD(ifq); - if (!(m->m_flags & M_PKTHDR)) { + if ((ptype == QP_MBUF) && !(((mbuf_t)p)->m_flags & M_PKTHDR)) { IFCQ_CONVERT_LOCK(ifq); - m_freem(m); + m_freem((mbuf_t)p); + *pdrop = TRUE; return (ENOBUFS); } + pktsched_pkt_encap(&pkt, ptype, p); fqs = (fq_if_t *)ifq->ifcq_disc; - svc = mbuf_get_service_class(m); - pri = fq_if_service_to_priority(svc); + svc = pktsched_get_pkt_svc(&pkt); + pri = fq_if_service_to_priority(fqs, svc); VERIFY(pri >= 0 && pri < FQ_IF_MAX_CLASSES); fq_cl = &fqs->fqs_classq[pri]; @@ -208,13 +306,15 @@ fq_if_enqueue_classq(struct ifclassq *ifq, struct mbuf *m) /* BK_SYS is currently throttled */ fq_cl->fcl_stat.fcl_throttle_drops++; IFCQ_CONVERT_LOCK(ifq); - m_freem(m); + pktsched_free_pkt(&pkt); + *pdrop = TRUE; return (EQSUSPENDED); } - len = m_length(m); - ret = fq_addq(fqs, m, fq_cl); - if (!FQ_IF_CLASSQ_IDLE(fq_cl)) { + len = pktsched_get_pkt_len(&pkt); + ret = fq_addq(fqs, &pkt, fq_cl); + if (!(fqs->fqs_flags & FQS_DRIVER_MANAGED) && + !FQ_IF_CLASSQ_IDLE(fq_cl)) { if (((fqs->fqs_bitmaps[FQ_IF_ER] | fqs->fqs_bitmaps[FQ_IF_EB]) & (1 << pri)) == 0) { /* @@ -229,56 +329,91 @@ fq_if_enqueue_classq(struct ifclassq *ifq, struct mbuf *m) if (ret == CLASSQEQ_SUCCESS_FC) { /* packet enqueued, return advisory feedback */ ret = EQFULL; + *pdrop = FALSE; } else { - VERIFY(ret == CLASSQEQ_DROPPED || - ret == CLASSQEQ_DROPPED_FC || - ret == CLASSQEQ_DROPPED_SP); + *pdrop = TRUE; + VERIFY(ret == CLASSQEQ_DROP || + ret == CLASSQEQ_DROP_FC || + ret == CLASSQEQ_DROP_SP); + pktsched_free_pkt(&pkt); switch (ret) { - case CLASSQEQ_DROPPED: + case CLASSQEQ_DROP: return (ENOBUFS); - case CLASSQEQ_DROPPED_FC: + case CLASSQEQ_DROP_FC: return (EQFULL); - case CLASSQEQ_DROPPED_SP: + case CLASSQEQ_DROP_SP: return (EQSUSPENDED); } } + } else { + *pdrop = FALSE; } IFCQ_INC_LEN(ifq); IFCQ_INC_BYTES(ifq, len); return (ret); } -struct mbuf * -fq_if_dequeue_classq(struct ifclassq *ifq, cqdq_op_t op) +static void * +fq_if_dequeue_classq(struct ifclassq *ifq, classq_pkt_type_t *ptype) { - struct mbuf *top; + void *top; - (void) fq_if_dequeue_classq_multi(ifq, op, 1, - CLASSQ_DEQUEUE_MAX_BYTE_LIMIT, &top, NULL, NULL, NULL); + (void) fq_if_dequeue_classq_multi(ifq, 1, + CLASSQ_DEQUEUE_MAX_BYTE_LIMIT, &top, NULL, NULL, NULL, ptype); + return (top); +} +static void * +fq_if_dequeue_sc_classq(struct ifclassq *ifq, mbuf_svc_class_t svc, + classq_pkt_type_t *ptype) +{ + void *top; + fq_if_t *fqs = (fq_if_t *)ifq->ifcq_disc; + fq_if_classq_t *fq_cl; + u_int32_t pri; + + pri = fq_if_service_to_priority(fqs, svc); + fq_cl = &fqs->fqs_classq[pri]; + + fq_if_dequeue(fqs, fq_cl, 1, CLASSQ_DEQUEUE_MAX_BYTE_LIMIT, + &top, NULL, NULL, NULL, TRUE, ptype); return (top); } int -fq_if_dequeue_classq_multi(struct ifclassq *ifq, cqdq_op_t op, - u_int32_t maxpktcnt, u_int32_t maxbytecnt, struct mbuf **first_packet, - struct mbuf **last_packet, u_int32_t *retpktcnt, u_int32_t *retbytecnt) +fq_if_dequeue_classq_multi(struct ifclassq *ifq, u_int32_t maxpktcnt, + u_int32_t maxbytecnt, void **first_packet, + void **last_packet, u_int32_t *retpktcnt, u_int32_t *retbytecnt, + classq_pkt_type_t *ptype) { -#pragma unused(op) - struct mbuf *top = NULL, *tail = NULL, *first, *last; + void *top = NULL, *tail = NULL, *first, *last; u_int32_t pktcnt = 0, bytecnt = 0, total_pktcnt, total_bytecnt; fq_if_t *fqs; fq_if_classq_t *fq_cl; int pri; + fq_if_append_pkt_t append_pkt; IFCQ_LOCK_ASSERT_HELD(ifq); fqs = (fq_if_t *)ifq->ifcq_disc; + switch (fqs->fqs_ptype) { + case QP_MBUF: + append_pkt = fq_if_append_mbuf; + break; + + + default: + VERIFY(0); + /* NOTREACHED */ + } + first = last = NULL; total_pktcnt = total_bytecnt = 0; + *ptype = fqs->fqs_ptype; for (;;) { + classq_pkt_type_t tmp_ptype; if (fqs->fqs_bitmaps[FQ_IF_ER] == 0 && fqs->fqs_bitmaps[FQ_IF_EB] == 0) { fqs->fqs_bitmaps[FQ_IF_EB] = fqs->fqs_bitmaps[FQ_IF_IB]; @@ -313,21 +448,22 @@ fq_if_dequeue_classq_multi(struct ifclassq *ifq, cqdq_op_t op, } fq_if_dequeue(fqs, fq_cl, (maxpktcnt - total_pktcnt), (maxbytecnt - total_bytecnt), &top, &tail, &pktcnt, - &bytecnt); + &bytecnt, FALSE, &tmp_ptype); if (top != NULL) { - VERIFY(pktcnt > 0 && bytecnt > 0); + ASSERT(tmp_ptype == *ptype); + ASSERT(pktcnt > 0 && bytecnt > 0); if (first == NULL) { first = top; last = tail; total_pktcnt = pktcnt; total_bytecnt = bytecnt; } else { - last->m_nextpkt = top; + append_pkt(last, top); last = tail; total_pktcnt += pktcnt; total_bytecnt += bytecnt; } - last->m_nextpkt = NULL; + append_pkt(last, NULL); fq_cl->fcl_budget -= bytecnt; pktcnt = 0; bytecnt = 0; @@ -378,21 +514,94 @@ state_change: return (0); } +int +fq_if_dequeue_sc_classq_multi(struct ifclassq *ifq, mbuf_svc_class_t svc, + u_int32_t maxpktcnt, u_int32_t maxbytecnt, void **first_packet, + void **last_packet, u_int32_t *retpktcnt, u_int32_t *retbytecnt, + classq_pkt_type_t *ptype) +{ +#pragma unused(maxpktcnt, maxbytecnt, first_packet, last_packet, retpktcnt, retbytecnt) + fq_if_t *fqs = (fq_if_t *)ifq->ifcq_disc; + u_int32_t pri; + u_int32_t total_pktcnt = 0, total_bytecnt = 0; + fq_if_classq_t *fq_cl; + void *first = NULL, *last = NULL; + fq_if_append_pkt_t append_pkt; + + switch (fqs->fqs_ptype) { + case QP_MBUF: + append_pkt = fq_if_append_mbuf; + break; + + + default: + VERIFY(0); + /* NOTREACHED */ + } + + pri = fq_if_service_to_priority(fqs, svc); + fq_cl = &fqs->fqs_classq[pri]; + + /* + * Now we have the queue for a particular service class. We need + * to dequeue as many packets as needed, first from the new flows + * and then from the old flows. + */ + while (total_pktcnt < maxpktcnt && total_bytecnt < maxbytecnt && + fq_cl->fcl_stat.fcl_pkt_cnt > 0) { + void *top, *tail; + u_int32_t pktcnt = 0, bytecnt = 0; + fq_if_dequeue(fqs, fq_cl, (maxpktcnt - total_pktcnt), + (maxbytecnt - total_bytecnt), &top, &tail, &pktcnt, + &bytecnt, TRUE, ptype); + if (first == NULL) { + first = top; + total_pktcnt = pktcnt; + total_bytecnt = bytecnt; + } else { + append_pkt(last, top); + total_pktcnt += pktcnt; + total_bytecnt += bytecnt; + } + last = tail; + } + if (first != NULL) { + if (first_packet != NULL) + *first_packet = first; + if (last_packet != NULL) + *last_packet = last; + if (retpktcnt != NULL) + *retpktcnt = total_pktcnt; + if (retbytecnt != NULL) + *retbytecnt = total_bytecnt; + } else { + if (first_packet != NULL) + *first_packet = NULL; + if (last_packet != NULL) + *last_packet = NULL; + if (retpktcnt != NULL) + *retpktcnt = 0; + if (retbytecnt != NULL) + *retbytecnt = 0; + } + return (0); +} + static void fq_if_purge_flow(fq_if_t *fqs, fq_t *fq, u_int32_t *pktsp, u_int32_t *bytesp) { fq_if_classq_t *fq_cl; u_int32_t pkts, bytes; - struct mbuf *m; + pktsched_pkt_t pkt; fq_cl = &fqs->fqs_classq[fq->fq_sc_index]; pkts = bytes = 0; - while ((m = fq_getq_flow(fqs, fq)) != NULL) { + _PKTSCHED_PKT_INIT(&pkt); + while (fq_getq_flow(fqs, fq, &pkt) != NULL) { pkts++; - bytes += m_length(m); - m_freem(m); - m = NULL; + bytes += pktsched_get_pkt_len(&pkt); + pktsched_free_pkt(&pkt); } IFCQ_DROP_ADD(fqs->fqs_ifq, pkts, bytes); @@ -471,7 +680,8 @@ fq_if_purge_sc(fq_if_t *fqs, cqrq_purge_sc_t *req) req->packets = req->bytes = 0; VERIFY(req->flow != 0); - fq = fq_if_hash_pkt(fqs, req->flow, req->sc, 0, FALSE); + /* packet type is needed only if we want to create a flow queue */ + fq = fq_if_hash_pkt(fqs, req->flow, req->sc, 0, FALSE, QP_INVALID); if (fq != NULL) fq_if_purge_flow(fqs, fq, &req->packets, &req->bytes); @@ -514,7 +724,9 @@ fq_if_throttle(fq_if_t *fqs, cqrq_throttle_t *tr) { struct ifclassq *ifq = fqs->fqs_ifq; int index; - +#if !MACH_ASSERT +#pragma unused(ifq) +#endif IFCQ_LOCK_ASSERT_HELD(ifq); if (!tr->set) { @@ -526,7 +738,7 @@ fq_if_throttle(fq_if_t *fqs, cqrq_throttle_t *tr) return (EALREADY); /* Throttling is allowed on BK_SYS class only */ - index = fq_if_service_to_priority(MBUF_SC_BK_SYS); + index = fq_if_service_to_priority(fqs, MBUF_SC_BK_SYS); switch (tr->level) { case IFNET_THROTTLE_OFF: fq_if_classq_resume(fqs, &fqs->fqs_classq[index]); @@ -549,7 +761,7 @@ fq_if_stat_sc(fq_if_t *fqs, cqrq_stat_sc_t *stat) if (stat == NULL) return; - pri = fq_if_service_to_priority(stat->sc); + pri = fq_if_service_to_priority(fqs, stat->sc); fq_cl = &fqs->fqs_classq[pri]; stat->packets = fq_cl->fcl_stat.fcl_pkt_cnt; stat->bytes = fq_cl->fcl_stat.fcl_byte_cnt; @@ -588,7 +800,8 @@ fq_if_request_classq(struct ifclassq *ifq, cqrq_t rq, void *arg) } int -fq_if_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) +fq_if_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags, + classq_pkt_type_t ptype) { #pragma unused(flags) struct ifnet *ifp = ifq->ifcq_ifp; @@ -599,24 +812,47 @@ fq_if_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) VERIFY(ifq->ifcq_disc == NULL); VERIFY(ifq->ifcq_type == PKTSCHEDT_NONE); - fqs = fq_if_alloc(ifp, M_WAITOK); + fqs = fq_if_alloc(ifp, ptype); if (fqs == NULL) return (ENOMEM); - fq_if_classq_init(fqs, FQ_IF_BK_SYS_INDEX, 1500, 2, MBUF_SC_BK_SYS); - fq_if_classq_init(fqs, FQ_IF_BK_INDEX, 1500, 2, MBUF_SC_BK); - fq_if_classq_init(fqs, FQ_IF_BE_INDEX, 1500, 4, MBUF_SC_BE); - fq_if_classq_init(fqs, FQ_IF_RD_INDEX, 1500, 4, MBUF_SC_RD); - fq_if_classq_init(fqs, FQ_IF_OAM_INDEX, 1500, 4, MBUF_SC_OAM); - fq_if_classq_init(fqs, FQ_IF_AV_INDEX, 3000, 6, MBUF_SC_AV); - fq_if_classq_init(fqs, FQ_IF_RV_INDEX, 3000, 6, MBUF_SC_RV); - fq_if_classq_init(fqs, FQ_IF_VI_INDEX, 3000, 6, MBUF_SC_VI); - fq_if_classq_init(fqs, FQ_IF_VO_INDEX, 600, 8, MBUF_SC_VO); - fq_if_classq_init(fqs, FQ_IF_CTL_INDEX, 600, 8, MBUF_SC_CTL); + if (flags & PKTSCHEDF_QALG_DRIVER_MANAGED) { + fqs->fqs_flags |= FQS_DRIVER_MANAGED; + fq_if_classq_init(fqs, FQ_IF_BK_INDEX, 1500, + 2, MBUF_SC_BK); + fq_if_classq_init(fqs, FQ_IF_BE_INDEX, 1500, + 4, MBUF_SC_BE); + fq_if_classq_init(fqs, FQ_IF_VI_INDEX, 3000, + 6, MBUF_SC_VI); + fq_if_classq_init(fqs, FQ_IF_VO_INDEX, 600, + 8, MBUF_SC_VO); + } else { + fq_if_classq_init(fqs, FQ_IF_BK_SYS_INDEX, 1500, + 2, MBUF_SC_BK_SYS); + fq_if_classq_init(fqs, FQ_IF_BK_INDEX, 1500, + 2, MBUF_SC_BK); + fq_if_classq_init(fqs, FQ_IF_BE_INDEX, 1500, + 4, MBUF_SC_BE); + fq_if_classq_init(fqs, FQ_IF_RD_INDEX, 1500, + 4, MBUF_SC_RD); + fq_if_classq_init(fqs, FQ_IF_OAM_INDEX, 1500, + 4, MBUF_SC_OAM); + fq_if_classq_init(fqs, FQ_IF_AV_INDEX, 3000, + 6, MBUF_SC_AV); + fq_if_classq_init(fqs, FQ_IF_RV_INDEX, 3000, + 6, MBUF_SC_RV); + fq_if_classq_init(fqs, FQ_IF_VI_INDEX, 3000, + 6, MBUF_SC_VI); + fq_if_classq_init(fqs, FQ_IF_VO_INDEX, 600, + 8, MBUF_SC_VO); + fq_if_classq_init(fqs, FQ_IF_CTL_INDEX, 600, + 8, MBUF_SC_CTL); + } err = ifclassq_attach(ifq, PKTSCHEDT_FQ_CODEL, fqs, - fq_if_enqueue_classq, fq_if_dequeue_classq, NULL, - fq_if_dequeue_classq_multi, fq_if_request_classq); + fq_if_enqueue_classq, fq_if_dequeue_classq, + fq_if_dequeue_sc_classq, fq_if_dequeue_classq_multi, + fq_if_dequeue_sc_classq_multi, fq_if_request_classq); if (err != 0) { printf("%s: error from ifclassq_attach, " @@ -628,7 +864,7 @@ fq_if_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) fq_t * fq_if_hash_pkt(fq_if_t *fqs, u_int32_t flowid, mbuf_svc_class_t svc_class, - u_int64_t now, boolean_t create) + u_int64_t now, boolean_t create, classq_pkt_type_t ptype) { fq_t *fq = NULL; flowq_list_t *fq_list; @@ -636,7 +872,7 @@ fq_if_hash_pkt(fq_if_t *fqs, u_int32_t flowid, mbuf_svc_class_t svc_class, u_int8_t fqs_hash_id; u_int8_t scidx; - scidx = fq_if_service_to_priority(svc_class); + scidx = fq_if_service_to_priority(fqs, svc_class); fqs_hash_id = FQ_IF_FLOW_HASH_ID(flowid); @@ -648,15 +884,16 @@ fq_if_hash_pkt(fq_if_t *fqs, u_int32_t flowid, mbuf_svc_class_t svc_class, break; } if (fq == NULL && create == TRUE) { + ASSERT(ptype == QP_MBUF); + /* If the flow is not already on the list, allocate it */ IFCQ_CONVERT_LOCK(fqs->fqs_ifq); - fq = fq_alloc(M_WAITOK); + fq = fq_alloc(ptype); if (fq != NULL) { fq->fq_flowhash = flowid; fq->fq_sc_index = scidx; fq->fq_updatetime = now + fqs->fqs_update_interval; fq_cl = &fqs->fqs_classq[scidx]; - fq->fq_flags = FQF_FLOWCTL_CAPABLE; SLIST_INSERT_HEAD(fq_list, fq, fq_hashlink); fq_cl->fcl_stat.fcl_flows_cnt++; @@ -667,13 +904,13 @@ fq_if_hash_pkt(fq_if_t *fqs, u_int32_t flowid, mbuf_svc_class_t svc_class, * If getq time is not set because this is the first packet or after * idle time, set it now so that we can detect a stall. */ - if (fq->fq_getqtime == 0) + if (fq != NULL && fq->fq_getqtime == 0) fq->fq_getqtime = now; return (fq); } -static void +void fq_if_destroy_flow(fq_if_t *fqs, fq_if_classq_t *fq_cl, fq_t *fq) { u_int8_t hash_id; @@ -734,20 +971,30 @@ inline void fq_if_drop_packet(fq_if_t *fqs) { fq_t *fq = fqs->fqs_large_flow; - struct mbuf *m; fq_if_classq_t *fq_cl; + pktsched_pkt_t pkt; + uint32_t *pkt_flags; + uint64_t *pkt_timestamp; if (fq == NULL) return; - /* mbufq can not be empty on the largest flow */ - VERIFY(!MBUFQ_EMPTY(&fq->fq_mbufq)); + /* queue can not be empty on the largest flow */ + VERIFY(!fq_empty(fq)); fq_cl = &fqs->fqs_classq[fq->fq_sc_index]; + _PKTSCHED_PKT_INIT(&pkt); + (void)fq_getq_flow_internal(fqs, fq, &pkt); - m = fq_getq_flow(fqs, fq); + pktsched_get_pkt_vars(&pkt, &pkt_flags, &pkt_timestamp, NULL, NULL, + NULL, NULL); IFCQ_CONVERT_LOCK(fqs->fqs_ifq); - if (MBUFQ_EMPTY(&fq->fq_mbufq)) { + *pkt_timestamp = 0; + if (pkt.pktsched_ptype == QP_MBUF) + *pkt_flags &= ~PKTF_PRIV_GUARDED; + + if (fq_empty(fq)) { + fqs->fqs_large_flow = NULL; if (fq->fq_flags & FQF_OLD_FLOW) { fq_if_empty_old_flow(fqs, fq_cl, fq, true); } else { @@ -755,18 +1002,28 @@ fq_if_drop_packet(fq_if_t *fqs) fq_if_empty_new_flow(fq, fq_cl, true); } } - IFCQ_DROP_ADD(fqs->fqs_ifq, 1, m_length(m)); + IFCQ_DROP_ADD(fqs->fqs_ifq, 1, pktsched_get_pkt_len(&pkt)); - m_freem(m); + pktsched_free_pkt(&pkt); fq_cl->fcl_stat.fcl_drop_overflow++; } inline void fq_if_is_flow_heavy(fq_if_t *fqs, fq_t *fq) { - fq_t *prev_fq = fqs->fqs_large_flow; - if (prev_fq == NULL && !MBUFQ_EMPTY(&fq->fq_mbufq)) { - fqs->fqs_large_flow = fq; + fq_t *prev_fq; + + if (fqs->fqs_large_flow != NULL && + fqs->fqs_large_flow->fq_bytes < FQ_IF_LARGE_FLOW_BYTE_LIMIT) + fqs->fqs_large_flow = NULL; + + if (fq == NULL || fq->fq_bytes < FQ_IF_LARGE_FLOW_BYTE_LIMIT) + return; + + prev_fq = fqs->fqs_large_flow; + if (prev_fq == NULL) { + if (!fq_empty(fq)) + fqs->fqs_large_flow = fq; return; } else if (fq->fq_bytes > prev_fq->fq_bytes) { fqs->fqs_large_flow = fq; @@ -774,27 +1031,21 @@ fq_if_is_flow_heavy(fq_if_t *fqs, fq_t *fq) } boolean_t -fq_if_add_fcentry(fq_if_t *fqs, struct pkthdr *pkt, fq_if_classq_t *fq_cl) +fq_if_add_fcentry(fq_if_t *fqs, pktsched_pkt_t *pkt, uint32_t flowid, + uint8_t flowsrc, fq_if_classq_t *fq_cl) { struct flowadv_fcentry *fce; - u_int32_t flowsrc, flowid; - - flowsrc = pkt->pkt_flowsrc; - flowid = pkt->pkt_flowid; STAILQ_FOREACH(fce, &fqs->fqs_fclist, fce_link) { - if (fce->fce_flowsrc == flowsrc && + if ((uint8_t)fce->fce_flowsrc_type == flowsrc && fce->fce_flowid == flowid) { /* Already on flowcontrol list */ return (TRUE); } } - IFCQ_CONVERT_LOCK(fqs->fqs_ifq); - fce = flowadv_alloc_entry(M_WAITOK); + fce = pktsched_alloc_fcentry(pkt, fqs->fqs_ifq->ifcq_ifp, M_WAITOK); if (fce != NULL) { - fce->fce_flowsrc = flowsrc; - fce->fce_flowid = flowid; /* XXX Add number of bytes in the queue */ STAILQ_INSERT_TAIL(&fqs->fqs_fclist, fce, fce_link); fq_cl->fcl_stat.fcl_flow_control++; @@ -824,92 +1075,65 @@ fq_if_flow_feedback(fq_if_t *fqs, fq_t *fq, fq_if_classq_t *fq_cl) void fq_if_dequeue(fq_if_t *fqs, fq_if_classq_t *fq_cl, u_int32_t pktlimit, - u_int32_t bytelimit, struct mbuf **top, struct mbuf **tail, - u_int32_t *retpktcnt, u_int32_t *retbytecnt) + u_int32_t bytelimit, void **top, void **tail, + u_int32_t *retpktcnt, u_int32_t *retbytecnt, boolean_t drvmgmt, + classq_pkt_type_t *ptype) { fq_t *fq = NULL, *tfq = NULL; - struct mbuf *m = NULL, *last = NULL; flowq_stailq_t temp_stailq; - u_int32_t pktcnt, bytecnt, mlen; - boolean_t limit_reached = FALSE; + u_int32_t pktcnt, bytecnt; + boolean_t qempty, limit_reached = FALSE; + void *last = NULL; + fq_getq_flow_t fq_getq_flow_fn; + + switch (fqs->fqs_ptype) { + case QP_MBUF: + fq_getq_flow_fn = fq_getq_flow_mbuf; + break; + + + default: + VERIFY(0); + /* NOTREACHED */ + } /* * maximum byte limit should not be greater than the budget for * this class */ - if ((int32_t)bytelimit > fq_cl->fcl_budget) + if ((int32_t)bytelimit > fq_cl->fcl_budget && !drvmgmt) bytelimit = fq_cl->fcl_budget; VERIFY(pktlimit > 0 && bytelimit > 0 && top != NULL); *top = NULL; + *ptype = fqs->fqs_ptype; pktcnt = bytecnt = 0; STAILQ_INIT(&temp_stailq); STAILQ_FOREACH_SAFE(fq, &fq_cl->fcl_new_flows, fq_actlink, tfq) { - VERIFY((fq->fq_flags & (FQF_NEW_FLOW|FQF_OLD_FLOW)) == + ASSERT((fq->fq_flags & (FQF_NEW_FLOW|FQF_OLD_FLOW)) == FQF_NEW_FLOW); - while (fq->fq_deficit > 0 && limit_reached == FALSE && - !MBUFQ_EMPTY(&fq->fq_mbufq)) { - - m = fq_getq_flow(fqs, fq); - m->m_pkthdr.pkt_flags |= PKTF_NEW_FLOW; - mlen = m_length(m); - fq->fq_deficit -= mlen; - - if (*top == NULL) { - *top = m; - } else { - last->m_nextpkt = m; - } - last = m; - last->m_nextpkt = NULL; - fq_cl->fcl_stat.fcl_dequeue++; - fq_cl->fcl_stat.fcl_dequeue_bytes += mlen; - pktcnt++; - bytecnt += mlen; + limit_reached = fq_getq_flow_fn(fqs, fq_cl, fq, bytelimit, + pktlimit, top, &last, &bytecnt, &pktcnt, &qempty, + PKTF_NEW_FLOW); - /* Check if the limit is reached */ - if (pktcnt >= pktlimit || bytecnt >= bytelimit) - limit_reached = TRUE; - } - - if (fq->fq_deficit <= 0 || MBUFQ_EMPTY(&fq->fq_mbufq)) { + if (fq->fq_deficit <= 0 || qempty) fq_if_empty_new_flow(fq, fq_cl, true); - fq->fq_deficit += fq_cl->fcl_quantum; - } - if (limit_reached == TRUE) + fq->fq_deficit += fq_cl->fcl_quantum; + if (limit_reached) goto done; } STAILQ_FOREACH_SAFE(fq, &fq_cl->fcl_old_flows, fq_actlink, tfq) { VERIFY((fq->fq_flags & (FQF_NEW_FLOW|FQF_OLD_FLOW)) == FQF_OLD_FLOW); - while (fq->fq_deficit > 0 && !MBUFQ_EMPTY(&fq->fq_mbufq) && - limit_reached == FALSE) { - m = fq_getq_flow(fqs, fq); - mlen = m_length(m); - fq->fq_deficit -= mlen; - if (*top == NULL) { - *top = m; - } else { - last->m_nextpkt = m; - } - last = m; - last->m_nextpkt = NULL; - fq_cl->fcl_stat.fcl_dequeue++; - fq_cl->fcl_stat.fcl_dequeue_bytes += mlen; - pktcnt++; - bytecnt += mlen; + limit_reached = fq_getq_flow_fn(fqs, fq_cl, fq, bytelimit, + pktlimit, top, &last, &bytecnt, &pktcnt, &qempty, 0); - /* Check if the limit is reached */ - if (pktcnt >= pktlimit || bytecnt >= bytelimit) - limit_reached = TRUE; - } - - if (MBUFQ_EMPTY(&fq->fq_mbufq)) { + if (qempty) { fq_if_empty_old_flow(fqs, fq_cl, fq, true); } else if (fq->fq_deficit <= 0) { STAILQ_REMOVE(&fq_cl->fcl_old_flows, fq, @@ -922,8 +1146,7 @@ fq_if_dequeue(fq_if_t *fqs, fq_if_classq_t *fq_cl, u_int32_t pktlimit, STAILQ_INSERT_TAIL(&temp_stailq, fq, fq_actlink); fq->fq_deficit += fq_cl->fcl_quantum; } - - if (limit_reached == TRUE) + if (limit_reached) break; } @@ -955,10 +1178,29 @@ fq_if_teardown_ifclassq(struct ifclassq *ifq) fq_if_destroy(fqs); ifq->ifcq_disc = NULL; - return (ifclassq_detach(ifq)); } +static void +fq_export_flowstats(fq_if_t *fqs, fq_t *fq, + struct fq_codel_flowstats *flowstat) +{ + bzero(flowstat, sizeof (*flowstat)); + flowstat->fqst_min_qdelay = fq->fq_min_qdelay; + flowstat->fqst_bytes = fq->fq_bytes; + flowstat->fqst_flowhash = fq->fq_flowhash; + if (fq->fq_flags & FQF_NEW_FLOW) + flowstat->fqst_flags |= FQ_FLOWSTATS_NEW_FLOW; + if (fq->fq_flags & FQF_OLD_FLOW) + flowstat->fqst_flags |= FQ_FLOWSTATS_OLD_FLOW; + if (fq->fq_flags & FQF_DELAY_HIGH) + flowstat->fqst_flags |= FQ_FLOWSTATS_DELAY_HIGH; + if (fq->fq_flags & FQF_FLOWCTL_ON) + flowstat->fqst_flags |= FQ_FLOWSTATS_FLOWCTL_ON; + if (fqs->fqs_large_flow == fq) + flowstat->fqst_flags |= FQ_FLOWSTATS_LARGE_FLOW; +} + int fq_if_getqstats_ifclassq(struct ifclassq *ifq, u_int32_t qid, struct if_ifclassq_stats *ifqs) @@ -966,6 +1208,8 @@ fq_if_getqstats_ifclassq(struct ifclassq *ifq, u_int32_t qid, struct fq_codel_classstats *fcls; fq_if_classq_t *fq_cl; fq_if_t *fqs; + fq_t *fq = NULL; + u_int32_t i, flowstat_cnt; if (qid >= FQ_IF_MAX_CLASSES) return (EINVAL); @@ -1002,5 +1246,28 @@ fq_if_getqstats_ifclassq(struct ifclassq *ifq, u_int32_t qid, fcls->fcls_throttle_drops = fq_cl->fcl_stat.fcl_throttle_drops; fcls->fcls_dup_rexmts = fq_cl->fcl_stat.fcl_dup_rexmts; + /* Gather per flow stats */ + flowstat_cnt = min((fcls->fcls_newflows_cnt + + fcls->fcls_oldflows_cnt), FQ_IF_MAX_FLOWSTATS); + i = 0; + STAILQ_FOREACH(fq, &fq_cl->fcl_new_flows, fq_actlink) { + if (i >= fcls->fcls_newflows_cnt || i >= flowstat_cnt) + break; + + /* leave space for a few old flows */ + if ((flowstat_cnt - i) < fcls->fcls_oldflows_cnt && + i >= (FQ_IF_MAX_FLOWSTATS >> 1)) + break; + fq_export_flowstats(fqs, fq, &fcls->fcls_flowstats[i]); + i++; + } + STAILQ_FOREACH(fq, &fq_cl->fcl_old_flows, fq_actlink) { + if (i >= flowstat_cnt) + break; + fq_export_flowstats(fqs, fq, &fcls->fcls_flowstats[i]); + i++; + } + VERIFY(i <= flowstat_cnt); + fcls->fcls_flowstats_cnt = i; return (0); } diff --git a/bsd/net/pktsched/pktsched_fq_codel.h b/bsd/net/pktsched/pktsched_fq_codel.h index 8d760a409..be7629a71 100644 --- a/bsd/net/pktsched/pktsched_fq_codel.h +++ b/bsd/net/pktsched/pktsched_fq_codel.h @@ -73,15 +73,14 @@ struct fcl_stat { #define FQ_IF_HASH_TAG_MASK 0xFF #define FQ_IF_HASH_TABLE_SIZE (1 << FQ_IF_HASH_TAG_SIZE) -/* maximum number f packets stored across all queues */ -#define FQ_IF_MAX_PKT_LIMIT 2048 - /* Set the quantum to be one MTU */ #define FQ_IF_DEFAULT_QUANTUM 1500 /* Max number of service classes currently supported */ #define FQ_IF_MAX_CLASSES 10 +#define FQ_IF_LARGE_FLOW_BYTE_LIMIT 15000 + struct flowq; typedef u_int32_t pktsched_bitmap_t; struct if_ifclassq_stats; @@ -130,13 +129,30 @@ typedef struct fq_codel_sched_data { pktsched_bitmap_t fqs_bitmaps[FQ_IF_MAX_STATE]; u_int32_t fqs_pkt_droplimit; /* drop limit */ u_int8_t fqs_throttle; /* throttle on or off */ + u_int8_t fqs_flags; /* flags */ +#define FQS_DRIVER_MANAGED 0x1 fq_if_classq_t fqs_classq[FQ_IF_MAX_CLASSES]; /* class queues */ struct flowadv_fclist fqs_fclist; /* flow control state */ struct flowq *fqs_large_flow; /* flow has highest number of bytes */ + classq_pkt_type_t fqs_ptype; } fq_if_t; #endif /* BSD_KERNEL_PRIVATE */ +struct fq_codel_flowstats { + u_int32_t fqst_min_qdelay; +#define FQ_FLOWSTATS_OLD_FLOW 0x1 +#define FQ_FLOWSTATS_NEW_FLOW 0x2 +#define FQ_FLOWSTATS_LARGE_FLOW 0x4 +#define FQ_FLOWSTATS_DELAY_HIGH 0x8 +#define FQ_FLOWSTATS_FLOWCTL_ON 0x10 + u_int32_t fqst_flags; + u_int32_t fqst_bytes; + u_int32_t fqst_flowhash; +}; + +#define FQ_IF_MAX_FLOWSTATS 20 + struct fq_codel_classstats { u_int32_t fcls_pri; u_int32_t fcls_service_class; @@ -163,23 +179,28 @@ struct fq_codel_classstats { u_int32_t fcls_throttle_off; u_int32_t fcls_throttle_drops; u_int32_t fcls_dup_rexmts; + u_int32_t fcls_flowstats_cnt; + struct fq_codel_flowstats fcls_flowstats[FQ_IF_MAX_FLOWSTATS]; }; #ifdef BSD_KERNEL_PRIVATE extern void fq_codel_scheduler_init(void); extern struct flowq *fq_if_hash_pkt(fq_if_t *, u_int32_t, mbuf_svc_class_t, - u_int64_t, boolean_t); + u_int64_t, boolean_t, classq_pkt_type_t); extern boolean_t fq_if_at_drop_limit(fq_if_t *); extern void fq_if_drop_packet(fq_if_t *); extern void fq_if_is_flow_heavy(fq_if_t *, struct flowq *); -extern boolean_t fq_if_add_fcentry(fq_if_t *, struct pkthdr *, - fq_if_classq_t *); +extern boolean_t fq_if_add_fcentry(fq_if_t *, pktsched_pkt_t *, uint32_t, + uint8_t, fq_if_classq_t *); extern void fq_if_flow_feedback(fq_if_t *, struct flowq *, fq_if_classq_t *); -extern int fq_if_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags); +extern int fq_if_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags, + classq_pkt_type_t ptype); extern int fq_if_teardown_ifclassq(struct ifclassq *ifq); extern int fq_if_getqstats_ifclassq(struct ifclassq *ifq, u_int32_t qid, struct if_ifclassq_stats *ifqs); +extern void fq_if_destroy_flow(fq_if_t *, fq_if_classq_t *, + struct flowq *); #endif /* BSD_KERNEL_PRIVATE */ diff --git a/bsd/net/pktsched/pktsched_hfsc.c b/bsd/net/pktsched/pktsched_hfsc.c deleted file mode 100644 index 365d16f01..000000000 --- a/bsd/net/pktsched/pktsched_hfsc.c +++ /dev/null @@ -1,2065 +0,0 @@ -/* - * Copyright (c) 2007-2013 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* $OpenBSD: altq_hfsc.c,v 1.25 2007/09/13 20:40:02 chl Exp $ */ -/* $KAME: altq_hfsc.c,v 1.17 2002/11/29 07:48:33 kjc Exp $ */ - -/* - * Copyright (c) 1997-1999 Carnegie Mellon University. All Rights Reserved. - * - * Permission to use, copy, modify, and distribute this software and - * its documentation is hereby granted (including for commercial or - * for-profit use), provided that both the copyright notice and this - * permission notice appear in all copies of the software, derivative - * works, or modified versions, and any portions thereof. - * - * THIS SOFTWARE IS EXPERIMENTAL AND IS KNOWN TO HAVE BUGS, SOME OF - * WHICH MAY HAVE SERIOUS CONSEQUENCES. CARNEGIE MELLON PROVIDES THIS - * SOFTWARE IN ITS ``AS IS'' CONDITION, AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT - * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR - * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE - * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH - * DAMAGE. - * - * Carnegie Mellon encourages (but does not require) users of this - * software to return any improvements or extensions that they make, - * and to grant Carnegie Mellon the rights to redistribute these - * changes without encumbrance. - */ -/* - * H-FSC is described in Proceedings of SIGCOMM'97, - * "A Hierarchical Fair Service Curve Algorithm for Link-Sharing, - * Real-Time and Priority Service" - * by Ion Stoica, Hui Zhang, and T. S. Eugene Ng. - * - * Oleg Cherevko <olwi@aq.ml.com.ua> added the upperlimit for link-sharing. - * when a class has an upperlimit, the fit-time is computed from the - * upperlimit service curve. the link-sharing scheduler does not schedule - * a class whose fit-time exceeds the current time. - */ - -#if PKTSCHED_HFSC - -#include <sys/cdefs.h> -#include <sys/param.h> -#include <sys/malloc.h> -#include <sys/mbuf.h> -#include <sys/systm.h> -#include <sys/errno.h> -#include <sys/kernel.h> -#include <sys/syslog.h> - -#include <kern/zalloc.h> - -#include <net/if.h> -#include <net/net_osdep.h> - -#include <net/pktsched/pktsched_hfsc.h> -#include <netinet/in.h> - -/* - * function prototypes - */ -#if 0 -static int hfsc_enqueue_ifclassq(struct ifclassq *, struct mbuf *); -static struct mbuf *hfsc_dequeue_ifclassq(struct ifclassq *, cqdq_op_t); -static int hfsc_request_ifclassq(struct ifclassq *, cqrq_t, void *); -#endif -static int hfsc_addq(struct hfsc_class *, struct mbuf *, struct pf_mtag *); -static struct mbuf *hfsc_getq(struct hfsc_class *); -static struct mbuf *hfsc_pollq(struct hfsc_class *); -static void hfsc_purgeq(struct hfsc_if *, struct hfsc_class *, u_int32_t, - u_int32_t *, u_int32_t *); -static void hfsc_print_sc(struct hfsc_if *, u_int32_t, u_int64_t, - struct service_curve *, struct internal_sc *, const char *); -static void hfsc_updateq_linkrate(struct hfsc_if *, struct hfsc_class *); -static void hfsc_updateq(struct hfsc_if *, struct hfsc_class *, cqev_t); - -static int hfsc_clear_interface(struct hfsc_if *); -static struct hfsc_class *hfsc_class_create(struct hfsc_if *, - struct service_curve *, struct service_curve *, struct service_curve *, - struct hfsc_class *, u_int32_t, int, u_int32_t); -static int hfsc_class_destroy(struct hfsc_if *, struct hfsc_class *); -static int hfsc_destroy_locked(struct hfsc_if *); -static struct hfsc_class *hfsc_nextclass(struct hfsc_class *); -static struct hfsc_class *hfsc_clh_to_clp(struct hfsc_if *, u_int32_t); -static const char *hfsc_style(struct hfsc_if *); - -static void set_active(struct hfsc_class *, u_int32_t); -static void set_passive(struct hfsc_class *); - -static void init_ed(struct hfsc_class *, u_int32_t); -static void update_ed(struct hfsc_class *, u_int32_t); -static void update_d(struct hfsc_class *, u_int32_t); -static void init_vf(struct hfsc_class *, u_int32_t); -static void update_vf(struct hfsc_class *, u_int32_t, u_int64_t); -static void update_cfmin(struct hfsc_class *); -static void ellist_insert(struct hfsc_class *); -static void ellist_remove(struct hfsc_class *); -static void ellist_update(struct hfsc_class *); -static struct hfsc_class *ellist_get_mindl(ellist_t *, u_int64_t); -static void actlist_insert(struct hfsc_class *); -static void actlist_remove(struct hfsc_class *); -static void actlist_update(struct hfsc_class *); -static struct hfsc_class *actlist_firstfit(struct hfsc_class *, u_int64_t); - -static inline u_int64_t seg_x2y(u_int64_t, u_int64_t); -static inline u_int64_t seg_y2x(u_int64_t, u_int64_t); -static inline u_int64_t m2sm(u_int64_t); -static inline u_int64_t m2ism(u_int64_t); -static inline u_int64_t d2dx(u_int64_t); -static u_int64_t sm2m(u_int64_t); -static u_int64_t dx2d(u_int64_t); - -static boolean_t sc2isc(struct hfsc_class *, struct service_curve *, - struct internal_sc *, u_int64_t); -static void rtsc_init(struct runtime_sc *, struct internal_sc *, - u_int64_t, u_int64_t); -static u_int64_t rtsc_y2x(struct runtime_sc *, u_int64_t); -static u_int64_t rtsc_x2y(struct runtime_sc *, u_int64_t); -static void rtsc_min(struct runtime_sc *, struct internal_sc *, - u_int64_t, u_int64_t); - -#define HFSC_ZONE_MAX 32 /* maximum elements in zone */ -#define HFSC_ZONE_NAME "pktsched_hfsc" /* zone name */ - -static unsigned int hfsc_size; /* size of zone element */ -static struct zone *hfsc_zone; /* zone for hfsc_if */ - -#define HFSC_CL_ZONE_MAX 32 /* maximum elements in zone */ -#define HFSC_CL_ZONE_NAME "pktsched_hfsc_cl" /* zone name */ - -static unsigned int hfsc_cl_size; /* size of zone element */ -static struct zone *hfsc_cl_zone; /* zone for hfsc_class */ - -/* - * macros - */ -#define HFSC_IS_A_PARENT_CLASS(cl) ((cl)->cl_children != NULL) - -#define HT_INFINITY 0xffffffffffffffffLL /* infinite time value */ - -void -hfsc_init(void) -{ - hfsc_size = sizeof (struct hfsc_if); - hfsc_zone = zinit(hfsc_size, HFSC_ZONE_MAX * hfsc_size, - 0, HFSC_ZONE_NAME); - if (hfsc_zone == NULL) { - panic("%s: failed allocating %s", __func__, HFSC_ZONE_NAME); - /* NOTREACHED */ - } - zone_change(hfsc_zone, Z_EXPAND, TRUE); - zone_change(hfsc_zone, Z_CALLERACCT, TRUE); - - hfsc_cl_size = sizeof (struct hfsc_class); - hfsc_cl_zone = zinit(hfsc_cl_size, HFSC_CL_ZONE_MAX * hfsc_cl_size, - 0, HFSC_CL_ZONE_NAME); - if (hfsc_cl_zone == NULL) { - panic("%s: failed allocating %s", __func__, HFSC_CL_ZONE_NAME); - /* NOTREACHED */ - } - zone_change(hfsc_cl_zone, Z_EXPAND, TRUE); - zone_change(hfsc_cl_zone, Z_CALLERACCT, TRUE); -} - -struct hfsc_if * -hfsc_alloc(struct ifnet *ifp, int how, boolean_t altq) -{ - struct hfsc_if *hif; - - hif = (how == M_WAITOK) ? zalloc(hfsc_zone) : zalloc_noblock(hfsc_zone); - if (hif == NULL) - return (NULL); - - bzero(hif, hfsc_size); - TAILQ_INIT(&hif->hif_eligible); - hif->hif_ifq = &ifp->if_snd; - if (altq) { - hif->hif_maxclasses = HFSC_MAX_CLASSES; - hif->hif_flags |= HFSCIFF_ALTQ; - } else { - hif->hif_maxclasses = IFCQ_SC_MAX + 1; /* incl. root class */ - } - - if ((hif->hif_class_tbl = _MALLOC(sizeof (struct hfsc_class *) * - hif->hif_maxclasses, M_DEVBUF, M_WAITOK|M_ZERO)) == NULL) { - log(LOG_ERR, "%s: %s unable to allocate class table array\n", - if_name(ifp), hfsc_style(hif)); - goto error; - } - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s scheduler allocated\n", - if_name(ifp), hfsc_style(hif)); - } - - return (hif); - -error: - if (hif->hif_class_tbl != NULL) { - _FREE(hif->hif_class_tbl, M_DEVBUF); - hif->hif_class_tbl = NULL; - } - zfree(hfsc_zone, hif); - - return (NULL); -} - -int -hfsc_destroy(struct hfsc_if *hif) -{ - struct ifclassq *ifq = hif->hif_ifq; - int err; - - IFCQ_LOCK(ifq); - err = hfsc_destroy_locked(hif); - IFCQ_UNLOCK(ifq); - - return (err); -} - -static int -hfsc_destroy_locked(struct hfsc_if *hif) -{ - IFCQ_LOCK_ASSERT_HELD(hif->hif_ifq); - - (void) hfsc_clear_interface(hif); - (void) hfsc_class_destroy(hif, hif->hif_rootclass); - - VERIFY(hif->hif_class_tbl != NULL); - _FREE(hif->hif_class_tbl, M_DEVBUF); - hif->hif_class_tbl = NULL; - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s scheduler destroyed\n", - if_name(HFSCIF_IFP(hif)), hfsc_style(hif)); - } - - zfree(hfsc_zone, hif); - - return (0); -} - -/* - * bring the interface back to the initial state by discarding - * all the filters and classes except the root class. - */ -static int -hfsc_clear_interface(struct hfsc_if *hif) -{ - struct hfsc_class *cl; - - IFCQ_LOCK_ASSERT_HELD(hif->hif_ifq); - - /* clear out the classes */ - while (hif->hif_rootclass != NULL && - (cl = hif->hif_rootclass->cl_children) != NULL) { - /* - * remove the first leaf class found in the hierarchy - * then start over - */ - for (; cl != NULL; cl = hfsc_nextclass(cl)) { - if (!HFSC_IS_A_PARENT_CLASS(cl)) { - (void) hfsc_class_destroy(hif, cl); - break; - } - } - } - - return (0); -} - -/* discard all the queued packets on the interface */ -void -hfsc_purge(struct hfsc_if *hif) -{ - struct hfsc_class *cl; - - IFCQ_LOCK_ASSERT_HELD(hif->hif_ifq); - - for (cl = hif->hif_rootclass; cl != NULL; cl = hfsc_nextclass(cl)) { - if (!qempty(&cl->cl_q)) - hfsc_purgeq(hif, cl, 0, NULL, NULL); - } -#if !PF_ALTQ - /* - * This assertion is safe to be made only when PF_ALTQ is not - * configured; otherwise, IFCQ_LEN represents the sum of the - * packets managed by ifcq_disc and altq_disc instances, which - * is possible when transitioning between the two. - */ - VERIFY(IFCQ_LEN(hif->hif_ifq) == 0); -#endif /* !PF_ALTQ */ -} - -void -hfsc_event(struct hfsc_if *hif, cqev_t ev) -{ - struct hfsc_class *cl; - - IFCQ_LOCK_ASSERT_HELD(hif->hif_ifq); - - for (cl = hif->hif_rootclass; cl != NULL; cl = hfsc_nextclass(cl)) - hfsc_updateq(hif, cl, ev); -} - -int -hfsc_add_queue(struct hfsc_if *hif, struct service_curve *rtsc, - struct service_curve *lssc, struct service_curve *ulsc, - u_int32_t qlimit, int flags, u_int32_t parent_qid, u_int32_t qid, - struct hfsc_class **clp) -{ - struct hfsc_class *cl = NULL, *parent; - - IFCQ_LOCK_ASSERT_HELD(hif->hif_ifq); - - if (parent_qid == HFSC_NULLCLASS_HANDLE && hif->hif_rootclass == NULL) - parent = NULL; - else if ((parent = hfsc_clh_to_clp(hif, parent_qid)) == NULL) - return (EINVAL); - - if (hfsc_clh_to_clp(hif, qid) != NULL) - return (EBUSY); - - cl = hfsc_class_create(hif, rtsc, lssc, ulsc, parent, - qlimit, flags, qid); - if (cl == NULL) - return (ENOMEM); - - if (clp != NULL) - *clp = cl; - - return (0); -} - -static struct hfsc_class * -hfsc_class_create(struct hfsc_if *hif, struct service_curve *rsc, - struct service_curve *fsc, struct service_curve *usc, - struct hfsc_class *parent, u_int32_t qlimit, int flags, u_int32_t qid) -{ - struct ifnet *ifp; - struct ifclassq *ifq; - struct hfsc_class *cl, *p; - u_int64_t eff_rate; - u_int32_t i; - - IFCQ_LOCK_ASSERT_HELD(hif->hif_ifq); - - /* Sanitize flags unless internally configured */ - if (hif->hif_flags & HFSCIFF_ALTQ) - flags &= HFCF_USERFLAGS; - - if (hif->hif_classes >= hif->hif_maxclasses) { - log(LOG_ERR, "%s: %s out of classes! (max %d)\n", - if_name(HFSCIF_IFP(hif)), hfsc_style(hif), - hif->hif_maxclasses); - return (NULL); - } - -#if !CLASSQ_RED - if (flags & HFCF_RED) { - log(LOG_ERR, "%s: %s RED not available!\n", - if_name(HFSCIF_IFP(hif)), hfsc_style(hif)); - return (NULL); - } -#endif /* !CLASSQ_RED */ - -#if !CLASSQ_RIO - if (flags & HFCF_RIO) { - log(LOG_ERR, "%s: %s RIO not available!\n", - if_name(HFSCIF_IFP(hif)), hfsc_style(hif)); - return (NULL); - } -#endif /* CLASSQ_RIO */ - -#if !CLASSQ_BLUE - if (flags & HFCF_BLUE) { - log(LOG_ERR, "%s: %s BLUE not available!\n", - if_name(HFSCIF_IFP(hif)), hfsc_style(hif)); - return (NULL); - } -#endif /* CLASSQ_BLUE */ - - /* These are mutually exclusive */ - if ((flags & (HFCF_RED|HFCF_RIO|HFCF_BLUE|HFCF_SFB)) && - (flags & (HFCF_RED|HFCF_RIO|HFCF_BLUE|HFCF_SFB)) != HFCF_RED && - (flags & (HFCF_RED|HFCF_RIO|HFCF_BLUE|HFCF_SFB)) != HFCF_RIO && - (flags & (HFCF_RED|HFCF_RIO|HFCF_BLUE|HFCF_SFB)) != HFCF_BLUE && - (flags & (HFCF_RED|HFCF_RIO|HFCF_BLUE|HFCF_SFB)) != HFCF_SFB) { - log(LOG_ERR, "%s: %s more than one RED|RIO|BLUE|SFB\n", - if_name(HFSCIF_IFP(hif)), hfsc_style(hif)); - return (NULL); - } - - cl = zalloc(hfsc_cl_zone); - if (cl == NULL) - return (NULL); - - bzero(cl, hfsc_cl_size); - TAILQ_INIT(&cl->cl_actc); - ifq = hif->hif_ifq; - ifp = HFSCIF_IFP(hif); - - if (qlimit == 0 || qlimit > IFCQ_MAXLEN(ifq)) { - qlimit = IFCQ_MAXLEN(ifq); - if (qlimit == 0) - qlimit = DEFAULT_QLIMIT; /* use default */ - } - _qinit(&cl->cl_q, Q_DROPTAIL, qlimit); - - cl->cl_flags = flags; - if (flags & (HFCF_RED|HFCF_RIO|HFCF_BLUE|HFCF_SFB)) { -#if CLASSQ_RED || CLASSQ_RIO - int pkttime; -#endif /* CLASSQ_RED || CLASSQ_RIO */ - u_int64_t m2; - - m2 = 0; - if (rsc != NULL && rsc->m2 > m2) - m2 = rsc->m2; - if (fsc != NULL && fsc->m2 > m2) - m2 = fsc->m2; - if (usc != NULL && usc->m2 > m2) - m2 = usc->m2; - - cl->cl_qflags = 0; - if (flags & HFCF_ECN) { - if (flags & HFCF_BLUE) - cl->cl_qflags |= BLUEF_ECN; - else if (flags & HFCF_SFB) - cl->cl_qflags |= SFBF_ECN; - else if (flags & HFCF_RED) - cl->cl_qflags |= REDF_ECN; - else if (flags & HFCF_RIO) - cl->cl_qflags |= RIOF_ECN; - } - if (flags & HFCF_FLOWCTL) { - if (flags & HFCF_SFB) - cl->cl_qflags |= SFBF_FLOWCTL; - } - if (flags & HFCF_CLEARDSCP) { - if (flags & HFCF_RIO) - cl->cl_qflags |= RIOF_CLEARDSCP; - } -#if CLASSQ_RED || CLASSQ_RIO - /* - * XXX: RED & RIO should be watching link speed and MTU - * events and recompute pkttime accordingly. - */ - if (m2 < 8) - pkttime = 1000 * 1000 * 1000; /* 1 sec */ - else - pkttime = (int64_t)ifp->if_mtu * 1000 * 1000 * 1000 / - (m2 / 8); - - /* Test for exclusivity {RED,RIO,BLUE,SFB} was done above */ -#if CLASSQ_RED - if (flags & HFCF_RED) { - cl->cl_red = red_alloc(ifp, 0, 0, - qlimit(&cl->cl_q) * 10/100, - qlimit(&cl->cl_q) * 30/100, - cl->cl_qflags, pkttime); - if (cl->cl_red != NULL) - qtype(&cl->cl_q) = Q_RED; - } -#endif /* CLASSQ_RED */ -#if CLASSQ_RIO - if (flags & HFCF_RIO) { - cl->cl_rio = - rio_alloc(ifp, 0, NULL, cl->cl_qflags, pkttime); - if (cl->cl_rio != NULL) - qtype(&cl->cl_q) = Q_RIO; - } -#endif /* CLASSQ_RIO */ -#endif /* CLASSQ_RED || CLASSQ_RIO */ -#if CLASSQ_BLUE - if (flags & HFCF_BLUE) { - cl->cl_blue = blue_alloc(ifp, 0, 0, cl->cl_qflags); - if (cl->cl_blue != NULL) - qtype(&cl->cl_q) = Q_BLUE; - } -#endif /* CLASSQ_BLUE */ - if (flags & HFCF_SFB) { - if (!(cl->cl_flags & HFCF_LAZY)) - cl->cl_sfb = sfb_alloc(ifp, qid, - qlimit(&cl->cl_q), cl->cl_qflags); - if (cl->cl_sfb != NULL || (cl->cl_flags & HFCF_LAZY)) - qtype(&cl->cl_q) = Q_SFB; - } - } - - cl->cl_id = hif->hif_classid++; - cl->cl_handle = qid; - cl->cl_hif = hif; - cl->cl_parent = parent; - - eff_rate = ifnet_output_linkrate(HFSCIF_IFP(hif)); - hif->hif_eff_rate = eff_rate; - - if (rsc != NULL && (rsc->m1 != 0 || rsc->m2 != 0) && - (!(rsc->fl & HFSCF_M1_PCT) || (rsc->m1 > 0 && rsc->m1 <= 100)) && - (!(rsc->fl & HFSCF_M2_PCT) || (rsc->m2 > 0 && rsc->m2 <= 100))) { - rsc->fl &= HFSCF_USERFLAGS; - cl->cl_flags |= HFCF_RSC; - cl->cl_rsc0 = *rsc; - (void) sc2isc(cl, &cl->cl_rsc0, &cl->cl_rsc, eff_rate); - rtsc_init(&cl->cl_deadline, &cl->cl_rsc, 0, 0); - rtsc_init(&cl->cl_eligible, &cl->cl_rsc, 0, 0); - } - if (fsc != NULL && (fsc->m1 != 0 || fsc->m2 != 0) && - (!(fsc->fl & HFSCF_M1_PCT) || (fsc->m1 > 0 && fsc->m1 <= 100)) && - (!(fsc->fl & HFSCF_M2_PCT) || (fsc->m2 > 0 && fsc->m2 <= 100))) { - fsc->fl &= HFSCF_USERFLAGS; - cl->cl_flags |= HFCF_FSC; - cl->cl_fsc0 = *fsc; - (void) sc2isc(cl, &cl->cl_fsc0, &cl->cl_fsc, eff_rate); - rtsc_init(&cl->cl_virtual, &cl->cl_fsc, 0, 0); - } - if (usc != NULL && (usc->m1 != 0 || usc->m2 != 0) && - (!(usc->fl & HFSCF_M1_PCT) || (usc->m1 > 0 && usc->m1 <= 100)) && - (!(usc->fl & HFSCF_M2_PCT) || (usc->m2 > 0 && usc->m2 <= 100))) { - usc->fl &= HFSCF_USERFLAGS; - cl->cl_flags |= HFCF_USC; - cl->cl_usc0 = *usc; - (void) sc2isc(cl, &cl->cl_usc0, &cl->cl_usc, eff_rate); - rtsc_init(&cl->cl_ulimit, &cl->cl_usc, 0, 0); - } - - /* - * find a free slot in the class table. if the slot matching - * the lower bits of qid is free, use this slot. otherwise, - * use the first free slot. - */ - i = qid % hif->hif_maxclasses; - if (hif->hif_class_tbl[i] == NULL) { - hif->hif_class_tbl[i] = cl; - } else { - for (i = 0; i < hif->hif_maxclasses; i++) - if (hif->hif_class_tbl[i] == NULL) { - hif->hif_class_tbl[i] = cl; - break; - } - if (i == hif->hif_maxclasses) { - goto err_ret; - } - } - hif->hif_classes++; - - if (flags & HFCF_DEFAULTCLASS) - hif->hif_defaultclass = cl; - - if (parent == NULL) { - /* this is root class */ - hif->hif_rootclass = cl; - } else { - /* add this class to the children list of the parent */ - if ((p = parent->cl_children) == NULL) - parent->cl_children = cl; - else { - while (p->cl_siblings != NULL) - p = p->cl_siblings; - p->cl_siblings = cl; - } - } - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s created qid=%d pqid=%d qlimit=%d " - "flags=%b\n", if_name(ifp), hfsc_style(hif), cl->cl_handle, - (cl->cl_parent != NULL) ? cl->cl_parent->cl_handle : 0, - qlimit(&cl->cl_q), cl->cl_flags, HFCF_BITS); - if (cl->cl_flags & HFCF_RSC) { - hfsc_print_sc(hif, cl->cl_handle, eff_rate, - &cl->cl_rsc0, &cl->cl_rsc, "rsc"); - } - if (cl->cl_flags & HFCF_FSC) { - hfsc_print_sc(hif, cl->cl_handle, eff_rate, - &cl->cl_fsc0, &cl->cl_fsc, "fsc"); - } - if (cl->cl_flags & HFCF_USC) { - hfsc_print_sc(hif, cl->cl_handle, eff_rate, - &cl->cl_usc0, &cl->cl_usc, "usc"); - } - } - - return (cl); - -err_ret: - if (cl->cl_qalg.ptr != NULL) { -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_destroy(cl->cl_rio); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_destroy(cl->cl_red); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_destroy(cl->cl_blue); -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - sfb_destroy(cl->cl_sfb); - cl->cl_qalg.ptr = NULL; - qtype(&cl->cl_q) = Q_DROPTAIL; - qstate(&cl->cl_q) = QS_RUNNING; - } - zfree(hfsc_cl_zone, cl); - return (NULL); -} - -int -hfsc_remove_queue(struct hfsc_if *hif, u_int32_t qid) -{ - struct hfsc_class *cl; - - IFCQ_LOCK_ASSERT_HELD(hif->hif_ifq); - - if ((cl = hfsc_clh_to_clp(hif, qid)) == NULL) - return (EINVAL); - - return (hfsc_class_destroy(hif, cl)); -} - -static int -hfsc_class_destroy(struct hfsc_if *hif, struct hfsc_class *cl) -{ - u_int32_t i; - - if (cl == NULL) - return (0); - - if (HFSC_IS_A_PARENT_CLASS(cl)) - return (EBUSY); - - IFCQ_LOCK_ASSERT_HELD(hif->hif_ifq); - - if (!qempty(&cl->cl_q)) - hfsc_purgeq(hif, cl, 0, NULL, NULL); - - if (cl->cl_parent == NULL) { - /* this is root class */ - } else { - struct hfsc_class *p = cl->cl_parent->cl_children; - - if (p == cl) - cl->cl_parent->cl_children = cl->cl_siblings; - else do { - if (p->cl_siblings == cl) { - p->cl_siblings = cl->cl_siblings; - break; - } - } while ((p = p->cl_siblings) != NULL); - VERIFY(p != NULL); - } - - for (i = 0; i < hif->hif_maxclasses; i++) - if (hif->hif_class_tbl[i] == cl) { - hif->hif_class_tbl[i] = NULL; - break; - } - - hif->hif_classes--; - - if (cl->cl_qalg.ptr != NULL) { -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_destroy(cl->cl_rio); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_destroy(cl->cl_red); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_destroy(cl->cl_blue); -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - sfb_destroy(cl->cl_sfb); - cl->cl_qalg.ptr = NULL; - qtype(&cl->cl_q) = Q_DROPTAIL; - qstate(&cl->cl_q) = QS_RUNNING; - } - - if (cl == hif->hif_rootclass) - hif->hif_rootclass = NULL; - if (cl == hif->hif_defaultclass) - hif->hif_defaultclass = NULL; - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s destroyed qid=%d slot=%d\n", - if_name(HFSCIF_IFP(hif)), hfsc_style(hif), - cl->cl_handle, cl->cl_id); - } - - zfree(hfsc_cl_zone, cl); - - return (0); -} - -/* - * hfsc_nextclass returns the next class in the tree. - * usage: - * for (cl = hif->hif_rootclass; cl != NULL; cl = hfsc_nextclass(cl)) - * do_something; - */ -static struct hfsc_class * -hfsc_nextclass(struct hfsc_class *cl) -{ - IFCQ_LOCK_ASSERT_HELD(cl->cl_hif->hif_ifq); - - if (cl->cl_children != NULL) - cl = cl->cl_children; - else if (cl->cl_siblings != NULL) - cl = cl->cl_siblings; - else { - while ((cl = cl->cl_parent) != NULL) - if (cl->cl_siblings) { - cl = cl->cl_siblings; - break; - } - } - - return (cl); -} - -int -hfsc_enqueue(struct hfsc_if *hif, struct hfsc_class *cl, struct mbuf *m, - struct pf_mtag *t) -{ - struct ifclassq *ifq = hif->hif_ifq; - u_int32_t len; - int ret; - - IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(cl == NULL || cl->cl_hif == hif); - - if (cl == NULL) { -#if PF_ALTQ - cl = hfsc_clh_to_clp(hif, t->pftag_qid); -#else /* !PF_ALTQ */ - cl = hfsc_clh_to_clp(hif, 0); -#endif /* !PF_ALTQ */ - if (cl == NULL || HFSC_IS_A_PARENT_CLASS(cl)) { - cl = hif->hif_defaultclass; - if (cl == NULL) { - IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (ENOBUFS); - } - } - } - - len = m_pktlen(m); - - ret = hfsc_addq(cl, m, t); - if (ret != 0) { - if (ret == CLASSQEQ_SUCCESS_FC) { - /* packet enqueued, return advisory feedback */ - ret = EQFULL; - } else { - VERIFY(ret == CLASSQEQ_DROPPED || - ret == CLASSQEQ_DROPPED_FC || - ret == CLASSQEQ_DROPPED_SP); - /* packet has been freed in hfsc_addq */ - PKTCNTR_ADD(&cl->cl_stats.drop_cnt, 1, len); - IFCQ_DROP_ADD(ifq, 1, len); - switch (ret) { - case CLASSQEQ_DROPPED: - return (ENOBUFS); - case CLASSQEQ_DROPPED_FC: - return (EQFULL); - case CLASSQEQ_DROPPED_SP: - return (EQSUSPENDED); - } - /* NOT_REACHED */ - } - } - IFCQ_INC_LEN(ifq); - IFCQ_INC_BYTES(ifq, len); - cl->cl_hif->hif_packets++; - - /* successfully queued. */ - if (qlen(&cl->cl_q) == 1) - set_active(cl, len); - - return (ret); -} - -/* - * note: CLASSQDQ_POLL returns the next packet without removing the packet - * from the queue. CLASSQDQ_REMOVE is a normal dequeue operation. - * CLASSQDQ_REMOVE must return the same packet if called immediately - * after CLASSQDQ_POLL. - */ -struct mbuf * -hfsc_dequeue(struct hfsc_if *hif, cqdq_op_t op) -{ - struct ifclassq *ifq = hif->hif_ifq; - struct hfsc_class *cl; - struct mbuf *m; - u_int32_t len, next_len; - int realtime = 0; - u_int64_t cur_time; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - if (hif->hif_packets == 0) - /* no packet in the tree */ - return (NULL); - - cur_time = read_machclk(); - - if (op == CLASSQDQ_REMOVE && hif->hif_pollcache != NULL) { - - cl = hif->hif_pollcache; - hif->hif_pollcache = NULL; - /* check if the class was scheduled by real-time criteria */ - if (cl->cl_flags & HFCF_RSC) - realtime = (cl->cl_e <= cur_time); - } else { - /* - * if there are eligible classes, use real-time criteria. - * find the class with the minimum deadline among - * the eligible classes. - */ - if ((cl = ellist_get_mindl(&hif->hif_eligible, cur_time)) - != NULL) { - realtime = 1; - } else { - int fits = 0; - /* - * use link-sharing criteria - * get the class with the minimum vt in the hierarchy - */ - cl = hif->hif_rootclass; - while (HFSC_IS_A_PARENT_CLASS(cl)) { - - cl = actlist_firstfit(cl, cur_time); - if (cl == NULL) { - if (fits > 0) - log(LOG_ERR, "%s: %s " - "%d fit but none found\n", - if_name(HFSCIF_IFP(hif)), - hfsc_style(hif), fits); - return (NULL); - } - /* - * update parent's cl_cvtmin. - * don't update if the new vt is smaller. - */ - if (cl->cl_parent->cl_cvtmin < cl->cl_vt) - cl->cl_parent->cl_cvtmin = cl->cl_vt; - fits++; - } - } - - if (op == CLASSQDQ_POLL) { - hif->hif_pollcache = cl; - m = hfsc_pollq(cl); - return (m); - } - } - - m = hfsc_getq(cl); - VERIFY(m != NULL); - len = m_pktlen(m); - cl->cl_hif->hif_packets--; - IFCQ_DEC_LEN(ifq); - IFCQ_DEC_BYTES(ifq, len); - IFCQ_XMIT_ADD(ifq, 1, len); - PKTCNTR_ADD(&cl->cl_stats.xmit_cnt, 1, len); - - update_vf(cl, len, cur_time); - if (realtime) - cl->cl_cumul += len; - - if (!qempty(&cl->cl_q)) { - if (cl->cl_flags & HFCF_RSC) { - /* update ed */ - next_len = m_pktlen(qhead(&cl->cl_q)); - - if (realtime) - update_ed(cl, next_len); - else - update_d(cl, next_len); - } - } else { - /* the class becomes passive */ - set_passive(cl); - } - - return (m); - -} - -static int -hfsc_addq(struct hfsc_class *cl, struct mbuf *m, struct pf_mtag *t) -{ - struct ifclassq *ifq = cl->cl_hif->hif_ifq; - - IFCQ_LOCK_ASSERT_HELD(ifq); - -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - return (rio_addq(cl->cl_rio, &cl->cl_q, m, t)); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - return (red_addq(cl->cl_red, &cl->cl_q, m, t)); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - return (blue_addq(cl->cl_blue, &cl->cl_q, m, t)); - else -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q)) { - if (cl->cl_sfb == NULL) { - struct ifnet *ifp = HFSCIF_IFP(cl->cl_hif); - - VERIFY(cl->cl_flags & HFCF_LAZY); - IFCQ_CONVERT_LOCK(ifq); - - cl->cl_sfb = sfb_alloc(ifp, cl->cl_handle, - qlimit(&cl->cl_q), cl->cl_qflags); - if (cl->cl_sfb == NULL) { - /* fall back to droptail */ - qtype(&cl->cl_q) = Q_DROPTAIL; - cl->cl_flags &= ~HFCF_SFB; - cl->cl_qflags &= ~(SFBF_ECN | SFBF_FLOWCTL); - - log(LOG_ERR, "%s: %s SFB lazy allocation " - "failed for qid=%d slot=%d, falling back " - "to DROPTAIL\n", if_name(ifp), - hfsc_style(cl->cl_hif), cl->cl_handle, - cl->cl_id); - } - } - if (cl->cl_sfb != NULL) - return (sfb_addq(cl->cl_sfb, &cl->cl_q, m, t)); - } else if (qlen(&cl->cl_q) >= qlimit(&cl->cl_q)) { - IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (CLASSQEQ_DROPPED); - } - -#if PF_ECN - if (cl->cl_flags & HFCF_CLEARDSCP) - write_dsfield(m, t, 0); -#endif /* PF_ECN */ - - _addq(&cl->cl_q, m); - - return (0); -} - -static struct mbuf * -hfsc_getq(struct hfsc_class *cl) -{ - IFCQ_LOCK_ASSERT_HELD(cl->cl_hif->hif_ifq); - -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - return (rio_getq(cl->cl_rio, &cl->cl_q)); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - return (red_getq(cl->cl_red, &cl->cl_q)); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - return (blue_getq(cl->cl_blue, &cl->cl_q)); - else -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - return (sfb_getq(cl->cl_sfb, &cl->cl_q)); - - return (_getq(&cl->cl_q)); -} - -static struct mbuf * -hfsc_pollq(struct hfsc_class *cl) -{ - IFCQ_LOCK_ASSERT_HELD(cl->cl_hif->hif_ifq); - - return (qhead(&cl->cl_q)); -} - -static void -hfsc_purgeq(struct hfsc_if *hif, struct hfsc_class *cl, u_int32_t flow, - u_int32_t *packets, u_int32_t *bytes) -{ - struct ifclassq *ifq = hif->hif_ifq; - u_int32_t cnt = 0, len = 0, qlen; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - if ((qlen = qlen(&cl->cl_q)) == 0) { - VERIFY(hif->hif_packets == 0); - goto done; - } - - /* become regular mutex before freeing mbufs */ - IFCQ_CONVERT_LOCK(ifq); - -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_purgeq(cl->cl_rio, &cl->cl_q, flow, &cnt, &len); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_purgeq(cl->cl_red, &cl->cl_q, flow, &cnt, &len); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_purgeq(cl->cl_blue, &cl->cl_q, flow, &cnt, &len); - else -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - sfb_purgeq(cl->cl_sfb, &cl->cl_q, flow, &cnt, &len); - else - _flushq_flow(&cl->cl_q, flow, &cnt, &len); - - if (cnt > 0) { - VERIFY(qlen(&cl->cl_q) == (qlen - cnt)); - - PKTCNTR_ADD(&cl->cl_stats.drop_cnt, cnt, len); - IFCQ_DROP_ADD(ifq, cnt, len); - - VERIFY(hif->hif_packets >= cnt); - hif->hif_packets -= cnt; - - VERIFY(((signed)IFCQ_LEN(ifq) - cnt) >= 0); - IFCQ_LEN(ifq) -= cnt; - - if (qempty(&cl->cl_q)) { - update_vf(cl, 0, 0); /* remove cl from the actlist */ - set_passive(cl); - } - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s purge qid=%d slot=%d " - "qlen=[%d,%d] cnt=%d len=%d flow=0x%x\n", - if_name(HFSCIF_IFP(hif)), hfsc_style(hif), - cl->cl_handle, cl->cl_id, qlen, qlen(&cl->cl_q), - cnt, len, flow); - } - } -done: - if (packets != NULL) - *packets = cnt; - if (bytes != NULL) - *bytes = len; -} - -static void -hfsc_print_sc(struct hfsc_if *hif, u_int32_t qid, u_int64_t eff_rate, - struct service_curve *sc, struct internal_sc *isc, const char *which) -{ - struct ifnet *ifp = HFSCIF_IFP(hif); - - log(LOG_DEBUG, "%s: %s qid=%d {%s_m1=%llu%s [%llu], " - "%s_d=%u msec, %s_m2=%llu%s [%llu]} linkrate=%llu bps\n", - if_name(ifp), hfsc_style(hif), qid, - which, sc->m1, (sc->fl & HFSCF_M1_PCT) ? "%" : " bps", isc->sm1, - which, sc->d, - which, sc->m2, (sc->fl & HFSCF_M2_PCT) ? "%" : " bps", isc->sm2, - eff_rate); -} - -static void -hfsc_updateq_linkrate(struct hfsc_if *hif, struct hfsc_class *cl) -{ - u_int64_t eff_rate = ifnet_output_linkrate(HFSCIF_IFP(hif)); - struct service_curve *sc; - struct internal_sc *isc; - - /* Update parameters only if rate has changed */ - if (eff_rate == hif->hif_eff_rate) - return; - - sc = &cl->cl_rsc0; - isc = &cl->cl_rsc; - if ((cl->cl_flags & HFCF_RSC) && sc2isc(cl, sc, isc, eff_rate)) { - rtsc_init(&cl->cl_deadline, isc, 0, 0); - rtsc_init(&cl->cl_eligible, isc, 0, 0); - if (pktsched_verbose) { - hfsc_print_sc(hif, cl->cl_handle, eff_rate, - sc, isc, "rsc"); - } - } - sc = &cl->cl_fsc0; - isc = &cl->cl_fsc; - if ((cl->cl_flags & HFCF_FSC) && sc2isc(cl, sc, isc, eff_rate)) { - rtsc_init(&cl->cl_virtual, isc, 0, 0); - if (pktsched_verbose) { - hfsc_print_sc(hif, cl->cl_handle, eff_rate, - sc, isc, "fsc"); - } - } - sc = &cl->cl_usc0; - isc = &cl->cl_usc; - if ((cl->cl_flags & HFCF_USC) && sc2isc(cl, sc, isc, eff_rate)) { - rtsc_init(&cl->cl_ulimit, isc, 0, 0); - if (pktsched_verbose) { - hfsc_print_sc(hif, cl->cl_handle, eff_rate, - sc, isc, "usc"); - } - } -} - -static void -hfsc_updateq(struct hfsc_if *hif, struct hfsc_class *cl, cqev_t ev) -{ - IFCQ_LOCK_ASSERT_HELD(hif->hif_ifq); - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s update qid=%d slot=%d event=%s\n", - if_name(HFSCIF_IFP(hif)), hfsc_style(hif), - cl->cl_handle, cl->cl_id, ifclassq_ev2str(ev)); - } - - if (ev == CLASSQ_EV_LINK_BANDWIDTH) - hfsc_updateq_linkrate(hif, cl); - -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - return (rio_updateq(cl->cl_rio, ev)); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - return (red_updateq(cl->cl_red, ev)); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - return (blue_updateq(cl->cl_blue, ev)); -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - return (sfb_updateq(cl->cl_sfb, ev)); -} - -static void -set_active(struct hfsc_class *cl, u_int32_t len) -{ - if (cl->cl_flags & HFCF_RSC) - init_ed(cl, len); - if (cl->cl_flags & HFCF_FSC) - init_vf(cl, len); - - cl->cl_stats.period++; -} - -static void -set_passive(struct hfsc_class *cl) -{ - if (cl->cl_flags & HFCF_RSC) - ellist_remove(cl); - - /* - * actlist is now handled in update_vf() so that update_vf(cl, 0, 0) - * needs to be called explicitly to remove a class from actlist - */ -} - -static void -init_ed(struct hfsc_class *cl, u_int32_t next_len) -{ - u_int64_t cur_time; - - cur_time = read_machclk(); - - /* update the deadline curve */ - rtsc_min(&cl->cl_deadline, &cl->cl_rsc, cur_time, cl->cl_cumul); - - /* - * update the eligible curve. - * for concave, it is equal to the deadline curve. - * for convex, it is a linear curve with slope m2. - */ - cl->cl_eligible = cl->cl_deadline; - if (cl->cl_rsc.sm1 <= cl->cl_rsc.sm2) { - cl->cl_eligible.dx = 0; - cl->cl_eligible.dy = 0; - } - - /* compute e and d */ - cl->cl_e = rtsc_y2x(&cl->cl_eligible, cl->cl_cumul); - cl->cl_d = rtsc_y2x(&cl->cl_deadline, cl->cl_cumul + next_len); - - ellist_insert(cl); -} - -static void -update_ed(struct hfsc_class *cl, u_int32_t next_len) -{ - cl->cl_e = rtsc_y2x(&cl->cl_eligible, cl->cl_cumul); - cl->cl_d = rtsc_y2x(&cl->cl_deadline, cl->cl_cumul + next_len); - - ellist_update(cl); -} - -static void -update_d(struct hfsc_class *cl, u_int32_t next_len) -{ - cl->cl_d = rtsc_y2x(&cl->cl_deadline, cl->cl_cumul + next_len); -} - -static void -init_vf(struct hfsc_class *cl, u_int32_t len) -{ -#pragma unused(len) - struct hfsc_class *max_cl, *p; - u_int64_t vt, f, cur_time; - int go_active; - - cur_time = 0; - go_active = 1; - for (; cl->cl_parent != NULL; cl = cl->cl_parent) { - - if (go_active && cl->cl_nactive++ == 0) - go_active = 1; - else - go_active = 0; - - if (go_active) { - max_cl = actlist_last(&cl->cl_parent->cl_actc); - if (max_cl != NULL) { - /* - * set vt to the average of the min and max - * classes. if the parent's period didn't - * change, don't decrease vt of the class. - */ - vt = max_cl->cl_vt; - if (cl->cl_parent->cl_cvtmin != 0) - vt = (cl->cl_parent->cl_cvtmin + vt)/2; - - if (cl->cl_parent->cl_vtperiod != - cl->cl_parentperiod || vt > cl->cl_vt) - cl->cl_vt = vt; - } else { - /* - * first child for a new parent backlog period. - * add parent's cvtmax to vtoff of children - * to make a new vt (vtoff + vt) larger than - * the vt in the last period for all children. - */ - vt = cl->cl_parent->cl_cvtmax; - for (p = cl->cl_parent->cl_children; p != NULL; - p = p->cl_siblings) - p->cl_vtoff += vt; - cl->cl_vt = 0; - cl->cl_parent->cl_cvtmax = 0; - cl->cl_parent->cl_cvtmin = 0; - } - cl->cl_initvt = cl->cl_vt; - - /* update the virtual curve */ - vt = cl->cl_vt + cl->cl_vtoff; - rtsc_min(&cl->cl_virtual, &cl->cl_fsc, - vt, cl->cl_total); - if (cl->cl_virtual.x == vt) { - cl->cl_virtual.x -= cl->cl_vtoff; - cl->cl_vtoff = 0; - } - cl->cl_vtadj = 0; - - cl->cl_vtperiod++; /* increment vt period */ - cl->cl_parentperiod = cl->cl_parent->cl_vtperiod; - if (cl->cl_parent->cl_nactive == 0) - cl->cl_parentperiod++; - cl->cl_f = 0; - - actlist_insert(cl); - - if (cl->cl_flags & HFCF_USC) { - /* class has upper limit curve */ - if (cur_time == 0) - cur_time = read_machclk(); - - /* update the ulimit curve */ - rtsc_min(&cl->cl_ulimit, &cl->cl_usc, cur_time, - cl->cl_total); - /* compute myf */ - cl->cl_myf = rtsc_y2x(&cl->cl_ulimit, - cl->cl_total); - cl->cl_myfadj = 0; - } - } - - if (cl->cl_myf > cl->cl_cfmin) - f = cl->cl_myf; - else - f = cl->cl_cfmin; - if (f != cl->cl_f) { - cl->cl_f = f; - update_cfmin(cl->cl_parent); - } - } -} - -static void -update_vf(struct hfsc_class *cl, u_int32_t len, u_int64_t cur_time) -{ -#pragma unused(cur_time) -#if 0 - u_int64_t myf_bound, delta; -#endif - u_int64_t f; - int go_passive; - - go_passive = (qempty(&cl->cl_q) && (cl->cl_flags & HFCF_FSC)); - - for (; cl->cl_parent != NULL; cl = cl->cl_parent) { - - cl->cl_total += len; - - if (!(cl->cl_flags & HFCF_FSC) || cl->cl_nactive == 0) - continue; - - if (go_passive && --cl->cl_nactive == 0) - go_passive = 1; - else - go_passive = 0; - - if (go_passive) { - /* no more active child, going passive */ - - /* update cvtmax of the parent class */ - if (cl->cl_vt > cl->cl_parent->cl_cvtmax) - cl->cl_parent->cl_cvtmax = cl->cl_vt; - - /* remove this class from the vt list */ - actlist_remove(cl); - - update_cfmin(cl->cl_parent); - - continue; - } - - /* - * update vt and f - */ - cl->cl_vt = rtsc_y2x(&cl->cl_virtual, cl->cl_total) - - cl->cl_vtoff + cl->cl_vtadj; - - /* - * if vt of the class is smaller than cvtmin, - * the class was skipped in the past due to non-fit. - * if so, we need to adjust vtadj. - */ - if (cl->cl_vt < cl->cl_parent->cl_cvtmin) { - cl->cl_vtadj += cl->cl_parent->cl_cvtmin - cl->cl_vt; - cl->cl_vt = cl->cl_parent->cl_cvtmin; - } - - /* update the vt list */ - actlist_update(cl); - - if (cl->cl_flags & HFCF_USC) { - cl->cl_myf = cl->cl_myfadj + - rtsc_y2x(&cl->cl_ulimit, cl->cl_total); -#if 0 - /* - * if myf lags behind by more than one clock tick - * from the current time, adjust myfadj to prevent - * a rate-limited class from going greedy. - * in a steady state under rate-limiting, myf - * fluctuates within one clock tick. - */ - myf_bound = cur_time - machclk_per_tick; - if (cl->cl_myf < myf_bound) { - delta = cur_time - cl->cl_myf; - cl->cl_myfadj += delta; - cl->cl_myf += delta; - } -#endif - } - - /* cl_f is max(cl_myf, cl_cfmin) */ - if (cl->cl_myf > cl->cl_cfmin) - f = cl->cl_myf; - else - f = cl->cl_cfmin; - if (f != cl->cl_f) { - cl->cl_f = f; - update_cfmin(cl->cl_parent); - } - } -} - -static void -update_cfmin(struct hfsc_class *cl) -{ - struct hfsc_class *p; - u_int64_t cfmin; - - if (TAILQ_EMPTY(&cl->cl_actc)) { - cl->cl_cfmin = 0; - return; - } - cfmin = HT_INFINITY; - TAILQ_FOREACH(p, &cl->cl_actc, cl_actlist) { - if (p->cl_f == 0) { - cl->cl_cfmin = 0; - return; - } - if (p->cl_f < cfmin) - cfmin = p->cl_f; - } - cl->cl_cfmin = cfmin; -} - -/* - * TAILQ based ellist and actlist implementation - * (ion wanted to make a calendar queue based implementation) - */ -/* - * eligible list holds backlogged classes being sorted by their eligible times. - * there is one eligible list per interface. - */ - -static void -ellist_insert(struct hfsc_class *cl) -{ - struct hfsc_if *hif = cl->cl_hif; - struct hfsc_class *p; - - /* check the last entry first */ - if ((p = TAILQ_LAST(&hif->hif_eligible, _eligible)) == NULL || - p->cl_e <= cl->cl_e) { - TAILQ_INSERT_TAIL(&hif->hif_eligible, cl, cl_ellist); - return; - } - - TAILQ_FOREACH(p, &hif->hif_eligible, cl_ellist) { - if (cl->cl_e < p->cl_e) { - TAILQ_INSERT_BEFORE(p, cl, cl_ellist); - return; - } - } - VERIFY(0); /* should not reach here */ -} - -static void -ellist_remove(struct hfsc_class *cl) -{ - struct hfsc_if *hif = cl->cl_hif; - - TAILQ_REMOVE(&hif->hif_eligible, cl, cl_ellist); -} - -static void -ellist_update(struct hfsc_class *cl) -{ - struct hfsc_if *hif = cl->cl_hif; - struct hfsc_class *p, *last; - - /* - * the eligible time of a class increases monotonically. - * if the next entry has a larger eligible time, nothing to do. - */ - p = TAILQ_NEXT(cl, cl_ellist); - if (p == NULL || cl->cl_e <= p->cl_e) - return; - - /* check the last entry */ - last = TAILQ_LAST(&hif->hif_eligible, _eligible); - VERIFY(last != NULL); - if (last->cl_e <= cl->cl_e) { - TAILQ_REMOVE(&hif->hif_eligible, cl, cl_ellist); - TAILQ_INSERT_TAIL(&hif->hif_eligible, cl, cl_ellist); - return; - } - - /* - * the new position must be between the next entry - * and the last entry - */ - while ((p = TAILQ_NEXT(p, cl_ellist)) != NULL) { - if (cl->cl_e < p->cl_e) { - TAILQ_REMOVE(&hif->hif_eligible, cl, cl_ellist); - TAILQ_INSERT_BEFORE(p, cl, cl_ellist); - return; - } - } - VERIFY(0); /* should not reach here */ -} - -/* find the class with the minimum deadline among the eligible classes */ -static struct hfsc_class * -ellist_get_mindl(ellist_t *head, u_int64_t cur_time) -{ - struct hfsc_class *p, *cl = NULL; - - TAILQ_FOREACH(p, head, cl_ellist) { - if (p->cl_e > cur_time) - break; - if (cl == NULL || p->cl_d < cl->cl_d) - cl = p; - } - return (cl); -} - -/* - * active children list holds backlogged child classes being sorted - * by their virtual time. - * each intermediate class has one active children list. - */ - -static void -actlist_insert(struct hfsc_class *cl) -{ - struct hfsc_class *p; - - /* check the last entry first */ - if ((p = TAILQ_LAST(&cl->cl_parent->cl_actc, _active)) == NULL || - p->cl_vt <= cl->cl_vt) { - TAILQ_INSERT_TAIL(&cl->cl_parent->cl_actc, cl, cl_actlist); - return; - } - - TAILQ_FOREACH(p, &cl->cl_parent->cl_actc, cl_actlist) { - if (cl->cl_vt < p->cl_vt) { - TAILQ_INSERT_BEFORE(p, cl, cl_actlist); - return; - } - } - VERIFY(0); /* should not reach here */ -} - -static void -actlist_remove(struct hfsc_class *cl) -{ - TAILQ_REMOVE(&cl->cl_parent->cl_actc, cl, cl_actlist); -} - -static void -actlist_update(struct hfsc_class *cl) -{ - struct hfsc_class *p, *last; - - /* - * the virtual time of a class increases monotonically during its - * backlogged period. - * if the next entry has a larger virtual time, nothing to do. - */ - p = TAILQ_NEXT(cl, cl_actlist); - if (p == NULL || cl->cl_vt < p->cl_vt) - return; - - /* check the last entry */ - last = TAILQ_LAST(&cl->cl_parent->cl_actc, _active); - VERIFY(last != NULL); - if (last->cl_vt <= cl->cl_vt) { - TAILQ_REMOVE(&cl->cl_parent->cl_actc, cl, cl_actlist); - TAILQ_INSERT_TAIL(&cl->cl_parent->cl_actc, cl, cl_actlist); - return; - } - - /* - * the new position must be between the next entry - * and the last entry - */ - while ((p = TAILQ_NEXT(p, cl_actlist)) != NULL) { - if (cl->cl_vt < p->cl_vt) { - TAILQ_REMOVE(&cl->cl_parent->cl_actc, cl, cl_actlist); - TAILQ_INSERT_BEFORE(p, cl, cl_actlist); - return; - } - } - VERIFY(0); /* should not reach here */ -} - -static struct hfsc_class * -actlist_firstfit(struct hfsc_class *cl, u_int64_t cur_time) -{ - struct hfsc_class *p; - - TAILQ_FOREACH(p, &cl->cl_actc, cl_actlist) { - if (p->cl_f <= cur_time) - return (p); - } - return (NULL); -} - -/* - * service curve support functions - * - * external service curve parameters - * m: bits/sec - * d: msec - * internal service curve parameters - * sm: (bytes/tsc_interval) << SM_SHIFT - * ism: (tsc_count/byte) << ISM_SHIFT - * dx: tsc_count - * - * SM_SHIFT and ISM_SHIFT are scaled in order to keep effective digits. - * we should be able to handle 100K-1Gbps linkspeed with 200Hz-1GHz CPU - * speed. SM_SHIFT and ISM_SHIFT are selected to have at least 3 effective - * digits in decimal using the following table. - * - * bits/sec 100Kbps 1Mbps 10Mbps 100Mbps 1Gbps - * ----------+------------------------------------------------------- - * bytes/nsec 12.5e-6 125e-6 1250e-6 12500e-6 125000e-6 - * sm(500MHz) 25.0e-6 250e-6 2500e-6 25000e-6 250000e-6 - * sm(200MHz) 62.5e-6 625e-6 6250e-6 62500e-6 625000e-6 - * - * nsec/byte 80000 8000 800 80 8 - * ism(500MHz) 40000 4000 400 40 4 - * ism(200MHz) 16000 1600 160 16 1.6 - */ -#define SM_SHIFT 24 -#define ISM_SHIFT 10 - -#define SM_MASK ((1LL << SM_SHIFT) - 1) -#define ISM_MASK ((1LL << ISM_SHIFT) - 1) - -static inline u_int64_t -seg_x2y(u_int64_t x, u_int64_t sm) -{ - u_int64_t y; - - /* - * compute - * y = x * sm >> SM_SHIFT - * but divide it for the upper and lower bits to avoid overflow - */ - y = (x >> SM_SHIFT) * sm + (((x & SM_MASK) * sm) >> SM_SHIFT); - return (y); -} - -static inline u_int64_t -seg_y2x(u_int64_t y, u_int64_t ism) -{ - u_int64_t x; - - if (y == 0) - x = 0; - else if (ism == HT_INFINITY) - x = HT_INFINITY; - else { - x = (y >> ISM_SHIFT) * ism - + (((y & ISM_MASK) * ism) >> ISM_SHIFT); - } - return (x); -} - -static inline u_int64_t -m2sm(u_int64_t m) -{ - u_int64_t sm; - - sm = (m << SM_SHIFT) / 8 / machclk_freq; - return (sm); -} - -static inline u_int64_t -m2ism(u_int64_t m) -{ - u_int64_t ism; - - if (m == 0) - ism = HT_INFINITY; - else - ism = ((u_int64_t)machclk_freq << ISM_SHIFT) * 8 / m; - return (ism); -} - -static inline u_int64_t -d2dx(u_int64_t d) -{ - u_int64_t dx; - - dx = (d * machclk_freq) / 1000; - return (dx); -} - -static u_int64_t -sm2m(u_int64_t sm) -{ - u_int64_t m; - - m = (sm * 8 * machclk_freq) >> SM_SHIFT; - return (m); -} - -static u_int64_t -dx2d(u_int64_t dx) -{ - u_int64_t d; - - d = dx * 1000 / machclk_freq; - return (d); -} - -static boolean_t -sc2isc(struct hfsc_class *cl, struct service_curve *sc, struct internal_sc *isc, - u_int64_t eff_rate) -{ - struct hfsc_if *hif = cl->cl_hif; - struct internal_sc oisc = *isc; - u_int64_t m1, m2; - - if (eff_rate == 0 && (sc->fl & (HFSCF_M1_PCT | HFSCF_M2_PCT))) { - /* - * If service curve is configured with percentage and the - * effective uplink rate is not known, assume this is a - * transient case, and that the rate will be updated in - * the near future via CLASSQ_EV_LINK_SPEED. Pick a - * reasonable number for now, e.g. 10 Mbps. - */ - eff_rate = (10 * 1000 * 1000); - - log(LOG_WARNING, "%s: %s qid=%d slot=%d eff_rate unknown; " - "using temporary rate %llu bps\n", if_name(HFSCIF_IFP(hif)), - hfsc_style(hif), cl->cl_handle, cl->cl_id, eff_rate); - } - - m1 = sc->m1; - if (sc->fl & HFSCF_M1_PCT) { - VERIFY(m1 > 0 && m1 <= 100); - m1 = (eff_rate * m1) / 100; - } - - m2 = sc->m2; - if (sc->fl & HFSCF_M2_PCT) { - VERIFY(m2 > 0 && m2 <= 100); - m2 = (eff_rate * m2) / 100; - } - - isc->sm1 = m2sm(m1); - isc->ism1 = m2ism(m1); - isc->dx = d2dx(sc->d); - isc->dy = seg_x2y(isc->dx, isc->sm1); - isc->sm2 = m2sm(m2); - isc->ism2 = m2ism(m2); - - /* return non-zero if there's any change */ - return (bcmp(&oisc, isc, sizeof (*isc))); -} - -/* - * initialize the runtime service curve with the given internal - * service curve starting at (x, y). - */ -static void -rtsc_init(struct runtime_sc *rtsc, struct internal_sc *isc, u_int64_t x, - u_int64_t y) -{ - rtsc->x = x; - rtsc->y = y; - rtsc->sm1 = isc->sm1; - rtsc->ism1 = isc->ism1; - rtsc->dx = isc->dx; - rtsc->dy = isc->dy; - rtsc->sm2 = isc->sm2; - rtsc->ism2 = isc->ism2; -} - -/* - * calculate the y-projection of the runtime service curve by the - * given x-projection value - */ -static u_int64_t -rtsc_y2x(struct runtime_sc *rtsc, u_int64_t y) -{ - u_int64_t x; - - if (y < rtsc->y) - x = rtsc->x; - else if (y <= rtsc->y + rtsc->dy) { - /* x belongs to the 1st segment */ - if (rtsc->dy == 0) - x = rtsc->x + rtsc->dx; - else - x = rtsc->x + seg_y2x(y - rtsc->y, rtsc->ism1); - } else { - /* x belongs to the 2nd segment */ - x = rtsc->x + rtsc->dx - + seg_y2x(y - rtsc->y - rtsc->dy, rtsc->ism2); - } - return (x); -} - -static u_int64_t -rtsc_x2y(struct runtime_sc *rtsc, u_int64_t x) -{ - u_int64_t y; - - if (x <= rtsc->x) - y = rtsc->y; - else if (x <= rtsc->x + rtsc->dx) - /* y belongs to the 1st segment */ - y = rtsc->y + seg_x2y(x - rtsc->x, rtsc->sm1); - else - /* y belongs to the 2nd segment */ - y = rtsc->y + rtsc->dy - + seg_x2y(x - rtsc->x - rtsc->dx, rtsc->sm2); - return (y); -} - -/* - * update the runtime service curve by taking the minimum of the current - * runtime service curve and the service curve starting at (x, y). - */ -static void -rtsc_min(struct runtime_sc *rtsc, struct internal_sc *isc, u_int64_t x, - u_int64_t y) -{ - u_int64_t y1, y2, dx, dy; - - if (isc->sm1 <= isc->sm2) { - /* service curve is convex */ - y1 = rtsc_x2y(rtsc, x); - if (y1 < y) - /* the current rtsc is smaller */ - return; - rtsc->x = x; - rtsc->y = y; - return; - } - - /* - * service curve is concave - * compute the two y values of the current rtsc - * y1: at x - * y2: at (x + dx) - */ - y1 = rtsc_x2y(rtsc, x); - if (y1 <= y) { - /* rtsc is below isc, no change to rtsc */ - return; - } - - y2 = rtsc_x2y(rtsc, x + isc->dx); - if (y2 >= y + isc->dy) { - /* rtsc is above isc, replace rtsc by isc */ - rtsc->x = x; - rtsc->y = y; - rtsc->dx = isc->dx; - rtsc->dy = isc->dy; - return; - } - - /* - * the two curves intersect - * compute the offsets (dx, dy) using the reverse - * function of seg_x2y() - * seg_x2y(dx, sm1) == seg_x2y(dx, sm2) + (y1 - y) - */ - dx = ((y1 - y) << SM_SHIFT) / (isc->sm1 - isc->sm2); - /* - * check if (x, y1) belongs to the 1st segment of rtsc. - * if so, add the offset. - */ - if (rtsc->x + rtsc->dx > x) - dx += rtsc->x + rtsc->dx - x; - dy = seg_x2y(dx, isc->sm1); - - rtsc->x = x; - rtsc->y = y; - rtsc->dx = dx; - rtsc->dy = dy; -} - -int -hfsc_get_class_stats(struct hfsc_if *hif, u_int32_t qid, - struct hfsc_classstats *sp) -{ - struct hfsc_class *cl; - - IFCQ_LOCK_ASSERT_HELD(hif->hif_ifq); - - if ((cl = hfsc_clh_to_clp(hif, qid)) == NULL) - return (EINVAL); - - sp->class_id = cl->cl_id; - sp->class_handle = cl->cl_handle; - - if (cl->cl_flags & HFCF_RSC) { - sp->rsc.m1 = sm2m(cl->cl_rsc.sm1); - sp->rsc.d = dx2d(cl->cl_rsc.dx); - sp->rsc.m2 = sm2m(cl->cl_rsc.sm2); - } else { - sp->rsc.m1 = 0; - sp->rsc.d = 0; - sp->rsc.m2 = 0; - } - if (cl->cl_flags & HFCF_FSC) { - sp->fsc.m1 = sm2m(cl->cl_fsc.sm1); - sp->fsc.d = dx2d(cl->cl_fsc.dx); - sp->fsc.m2 = sm2m(cl->cl_fsc.sm2); - } else { - sp->fsc.m1 = 0; - sp->fsc.d = 0; - sp->fsc.m2 = 0; - } - if (cl->cl_flags & HFCF_USC) { - sp->usc.m1 = sm2m(cl->cl_usc.sm1); - sp->usc.d = dx2d(cl->cl_usc.dx); - sp->usc.m2 = sm2m(cl->cl_usc.sm2); - } else { - sp->usc.m1 = 0; - sp->usc.d = 0; - sp->usc.m2 = 0; - } - - sp->total = cl->cl_total; - sp->cumul = cl->cl_cumul; - - sp->d = cl->cl_d; - sp->e = cl->cl_e; - sp->vt = cl->cl_vt; - sp->f = cl->cl_f; - - sp->initvt = cl->cl_initvt; - sp->vtperiod = cl->cl_vtperiod; - sp->parentperiod = cl->cl_parentperiod; - sp->nactive = cl->cl_nactive; - sp->vtoff = cl->cl_vtoff; - sp->cvtmax = cl->cl_cvtmax; - sp->myf = cl->cl_myf; - sp->cfmin = cl->cl_cfmin; - sp->cvtmin = cl->cl_cvtmin; - sp->myfadj = cl->cl_myfadj; - sp->vtadj = cl->cl_vtadj; - - sp->cur_time = read_machclk(); - sp->machclk_freq = machclk_freq; - - sp->qlength = qlen(&cl->cl_q); - sp->qlimit = qlimit(&cl->cl_q); - sp->xmit_cnt = cl->cl_stats.xmit_cnt; - sp->drop_cnt = cl->cl_stats.drop_cnt; - sp->period = cl->cl_stats.period; - - sp->qtype = qtype(&cl->cl_q); - sp->qstate = qstate(&cl->cl_q); -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_getstats(cl->cl_red, &sp->red[0]); -#endif /* CLASSQ_RED */ -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_getstats(cl->cl_rio, &sp->red[0]); -#endif /* CLASSQ_RIO */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_getstats(cl->cl_blue, &sp->blue); -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - sfb_getstats(cl->cl_sfb, &sp->sfb); - - return (0); -} - -/* convert a class handle to the corresponding class pointer */ -static struct hfsc_class * -hfsc_clh_to_clp(struct hfsc_if *hif, u_int32_t chandle) -{ - u_int32_t i; - struct hfsc_class *cl; - - IFCQ_LOCK_ASSERT_HELD(hif->hif_ifq); - - /* - * first, try optimistically the slot matching the lower bits of - * the handle. if it fails, do the linear table search. - */ - i = chandle % hif->hif_maxclasses; - if ((cl = hif->hif_class_tbl[i]) != NULL && cl->cl_handle == chandle) - return (cl); - for (i = 0; i < hif->hif_maxclasses; i++) - if ((cl = hif->hif_class_tbl[i]) != NULL && - cl->cl_handle == chandle) - return (cl); - return (NULL); -} - -static const char * -hfsc_style(struct hfsc_if *hif) -{ - return ((hif->hif_flags & HFSCIFF_ALTQ) ? "ALTQ_HFSC" : "HFSC"); -} - -int -hfsc_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) -{ -#pragma unused(ifq, flags) - return (ENXIO); /* not yet */ -} - -int -hfsc_teardown_ifclassq(struct ifclassq *ifq) -{ - struct hfsc_if *hif = ifq->ifcq_disc; - int i; - - IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(hif != NULL && ifq->ifcq_type == PKTSCHEDT_HFSC); - - (void) hfsc_destroy_locked(hif); - - ifq->ifcq_disc = NULL; - for (i = 0; i < IFCQ_SC_MAX; i++) { - ifq->ifcq_disc_slots[i].qid = 0; - ifq->ifcq_disc_slots[i].cl = NULL; - } - - return (ifclassq_detach(ifq)); -} - -int -hfsc_getqstats_ifclassq(struct ifclassq *ifq, u_int32_t slot, - struct if_ifclassq_stats *ifqs) -{ - struct hfsc_if *hif = ifq->ifcq_disc; - - IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(ifq->ifcq_type == PKTSCHEDT_HFSC); - - if (slot >= IFCQ_SC_MAX) - return (EINVAL); - - return (hfsc_get_class_stats(hif, ifq->ifcq_disc_slots[slot].qid, - &ifqs->ifqs_hfsc_stats)); -} -#endif /* PKTSCHED_HFSC */ diff --git a/bsd/net/pktsched/pktsched_hfsc.h b/bsd/net/pktsched/pktsched_hfsc.h index d22b95380..7f14cdcb9 100644 --- a/bsd/net/pktsched/pktsched_hfsc.h +++ b/bsd/net/pktsched/pktsched_hfsc.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2012 Apple Inc. All rights reserved. + * Copyright (c) 2011-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -173,181 +173,6 @@ struct hfsc_classstats { classq_state_t qstate; }; -#ifdef BSD_KERNEL_PRIVATE -#include <sys/queue.h> -/* - * kernel internal service curve representation - * coordinates are given by 64 bit unsigned integers. - * x-axis: unit is clock count. for the intel x86 architecture, - * the raw Pentium TSC (Timestamp Counter) value is used. - * virtual time is also calculated in this time scale. - * y-axis: unit is byte. - * - * the service curve parameters are converted to the internal - * representation. - * the slope values are scaled to avoid overflow. - * the inverse slope values as well as the y-projection of the 1st - * segment are kept in order to to avoid 64-bit divide operations - * that are expensive on 32-bit architectures. - * - * note: Intel Pentium TSC never wraps around in several thousands of years. - * x-axis doesn't wrap around for 1089 years with 1GHz clock. - * y-axis doesn't wrap around for 4358 years with 1Gbps bandwidth. - */ - -/* kernel internal representation of a service curve */ -struct internal_sc { - u_int64_t sm1; /* scaled slope of the 1st segment */ - u_int64_t ism1; /* scaled inverse-slope of the 1st segment */ - u_int64_t dx; /* the x-projection of the 1st segment */ - u_int64_t dy; /* the y-projection of the 1st segment */ - u_int64_t sm2; /* scaled slope of the 2nd segment */ - u_int64_t ism2; /* scaled inverse-slope of the 2nd segment */ -}; - -/* runtime service curve */ -struct runtime_sc { - u_int64_t x; /* current starting position on x-axis */ - u_int64_t y; /* current starting position on x-axis */ - u_int64_t sm1; /* scaled slope of the 1st segment */ - u_int64_t ism1; /* scaled inverse-slope of the 1st segment */ - u_int64_t dx; /* the x-projection of the 1st segment */ - u_int64_t dy; /* the y-projection of the 1st segment */ - u_int64_t sm2; /* scaled slope of the 2nd segment */ - u_int64_t ism2; /* scaled inverse-slope of the 2nd segment */ -}; - -/* for TAILQ based ellist and actlist implementation */ -struct hfsc_class; -typedef TAILQ_HEAD(_eligible, hfsc_class) ellist_t; -typedef TAILQ_ENTRY(hfsc_class) elentry_t; -typedef TAILQ_HEAD(_active, hfsc_class) actlist_t; -typedef TAILQ_ENTRY(hfsc_class) actentry_t; -#define ellist_first(s) TAILQ_FIRST(s) -#define actlist_first(s) TAILQ_FIRST(s) -#define actlist_last(s) TAILQ_LAST(s, _active) - -struct hfsc_class { - u_int32_t cl_id; /* class id (just for debug) */ - u_int32_t cl_handle; /* class handle */ - struct hfsc_if *cl_hif; /* back pointer to struct hfsc_if */ - u_int32_t cl_flags; /* misc flags */ - - struct hfsc_class *cl_parent; /* parent class */ - struct hfsc_class *cl_siblings; /* sibling classes */ - struct hfsc_class *cl_children; /* child classes */ - - class_queue_t cl_q; /* class queue structure */ - u_int32_t cl_qflags; /* class queue flags */ - union { - void *ptr; - struct red *red; /* RED state */ - struct rio *rio; /* RIO state */ - struct blue *blue; /* BLUE state */ - struct sfb *sfb; /* SFB state */ - } cl_qalg; - - u_int64_t cl_total; /* total work in bytes */ - u_int64_t cl_cumul; /* cumulative work in bytes */ - /* done by real-time criteria */ - u_int64_t cl_d; /* deadline */ - u_int64_t cl_e; /* eligible time */ - u_int64_t cl_vt; /* virtual time */ - u_int64_t cl_f; /* time when this class will fit for */ - /* link-sharing, max(myf, cfmin) */ - u_int64_t cl_myf; /* my fit-time (as calculated from */ - /* this class's own upperlimit */ - /* curve) */ - u_int64_t cl_myfadj; /* my fit-time adjustment */ - /* (to cancel history dependence) */ - u_int64_t cl_cfmin; /* earliest children's fit-time (used */ - /* with cl_myf to obtain cl_f) */ - u_int64_t cl_cvtmin; /* minimal virtual time among the */ - /* children fit for link-sharing */ - /* (monotonic within a period) */ - u_int64_t cl_vtadj; /* intra-period cumulative vt */ - /* adjustment */ - u_int64_t cl_vtoff; /* inter-period cumulative vt offset */ - u_int64_t cl_cvtmax; /* max child's vt in the last period */ - - u_int64_t cl_initvt; /* init virtual time (for debugging) */ - - struct service_curve cl_rsc0; /* external real-time service curve */ - struct service_curve cl_fsc0; /* external fair service curve */ - struct service_curve cl_usc0; /* external uppperlimit service curve */ - struct internal_sc cl_rsc; /* internal real-time service curve */ - struct internal_sc cl_fsc; /* internal fair service curve */ - struct internal_sc cl_usc; /* internal upperlimit service curve */ - struct runtime_sc cl_deadline; /* deadline curve */ - struct runtime_sc cl_eligible; /* eligible curve */ - struct runtime_sc cl_virtual; /* virtual curve */ - struct runtime_sc cl_ulimit; /* upperlimit curve */ - - u_int32_t cl_vtperiod; /* vt period sequence no */ - u_int32_t cl_parentperiod; /* parent's vt period seqno */ - u_int32_t cl_nactive; /* number of active children */ - actlist_t cl_actc; /* active children list */ - - actentry_t cl_actlist; /* active children list entry */ - elentry_t cl_ellist; /* eligible list entry */ - - struct { - struct pktcntr xmit_cnt; - struct pktcntr drop_cnt; - u_int32_t period; - } cl_stats; -}; - -#define cl_red cl_qalg.red -#define cl_rio cl_qalg.rio -#define cl_blue cl_qalg.blue -#define cl_sfb cl_qalg.sfb - -/* hfsc_if flags */ -#define HFSCIFF_ALTQ 0x1 /* configured via PF/ALTQ */ - -/* - * hfsc interface state - */ -struct hfsc_if { - struct ifclassq *hif_ifq; /* backpointer to ifclassq */ - struct hfsc_class *hif_rootclass; /* root class */ - struct hfsc_class *hif_defaultclass; /* default class */ - struct hfsc_class **hif_class_tbl; - struct hfsc_class *hif_pollcache; /* cache for poll operation */ - - u_int32_t hif_flags; /* flags */ - u_int32_t hif_maxclasses; /* max # of classes in table */ - u_int32_t hif_classes; /* # of classes in the tree */ - u_int32_t hif_packets; /* # of packets in the tree */ - u_int32_t hif_classid; /* class id sequence number */ - u_int64_t hif_eff_rate; /* last known effective rate */ - - ellist_t hif_eligible; /* eligible list */ -}; - -#define HFSCIF_IFP(_hif) ((_hif)->hif_ifq->ifcq_ifp) - -extern void hfsc_init(void); -extern struct hfsc_if *hfsc_alloc(struct ifnet *, int, boolean_t); -extern int hfsc_destroy(struct hfsc_if *); -extern void hfsc_purge(struct hfsc_if *); -extern void hfsc_event(struct hfsc_if *, cqev_t); -extern int hfsc_add_queue(struct hfsc_if *, struct service_curve *, - struct service_curve *, struct service_curve *, u_int32_t, int, - u_int32_t, u_int32_t, struct hfsc_class **); -extern int hfsc_remove_queue(struct hfsc_if *, u_int32_t); -extern int hfsc_get_class_stats(struct hfsc_if *, u_int32_t, - struct hfsc_classstats *); -extern int hfsc_enqueue(struct hfsc_if *, struct hfsc_class *, - struct mbuf *, struct pf_mtag *); -extern struct mbuf *hfsc_dequeue(struct hfsc_if *, cqdq_op_t); -extern int hfsc_setup_ifclassq(struct ifclassq *, u_int32_t); -extern int hfsc_teardown_ifclassq(struct ifclassq *); -extern int hfsc_getqstats_ifclassq(struct ifclassq *, u_int32_t, - struct if_ifclassq_stats *); -#endif /* BSD_KERNEL_PRIVATE */ - #ifdef __cplusplus } #endif diff --git a/bsd/net/pktsched/pktsched_priq.c b/bsd/net/pktsched/pktsched_priq.c deleted file mode 100644 index ffbf5cf28..000000000 --- a/bsd/net/pktsched/pktsched_priq.c +++ /dev/null @@ -1,1309 +0,0 @@ -/* - * Copyright (c) 2007-2013 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* $OpenBSD: altq_priq.c,v 1.21 2007/09/13 20:40:02 chl Exp $ */ -/* $KAME: altq_priq.c,v 1.1 2000/10/18 09:15:23 kjc Exp $ */ - -/* - * Copyright (C) 2000-2003 - * Sony Computer Science Laboratories Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY SONY CSL AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL SONY CSL OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -/* - * priority queue - */ - -#if PKTSCHED_PRIQ - -#include <sys/cdefs.h> -#include <sys/param.h> -#include <sys/malloc.h> -#include <sys/mbuf.h> -#include <sys/systm.h> -#include <sys/errno.h> -#include <sys/kernel.h> -#include <sys/syslog.h> - -#include <kern/zalloc.h> - -#include <net/if.h> -#include <net/net_osdep.h> - -#include <net/pktsched/pktsched_priq.h> -#include <netinet/in.h> - -/* - * function prototypes - */ -static int priq_enqueue_ifclassq(struct ifclassq *, struct mbuf *); -static struct mbuf *priq_dequeue_ifclassq(struct ifclassq *, cqdq_op_t); -static int priq_request_ifclassq(struct ifclassq *, cqrq_t, void *); -static int priq_clear_interface(struct priq_if *); -static struct priq_class *priq_class_create(struct priq_if *, int, u_int32_t, - int, u_int32_t); -static int priq_class_destroy(struct priq_if *, struct priq_class *); -static int priq_destroy_locked(struct priq_if *); -static inline int priq_addq(struct priq_class *, struct mbuf *, - struct pf_mtag *); -static inline struct mbuf *priq_getq(struct priq_class *); -static inline struct mbuf *priq_pollq(struct priq_class *); -static void priq_purgeq(struct priq_if *, struct priq_class *, u_int32_t, - u_int32_t *, u_int32_t *); -static void priq_purge_sc(struct priq_if *, cqrq_purge_sc_t *); -static void priq_updateq(struct priq_if *, struct priq_class *, cqev_t); -static int priq_throttle(struct priq_if *, cqrq_throttle_t *); -static int priq_resumeq(struct priq_if *, struct priq_class *); -static int priq_suspendq(struct priq_if *, struct priq_class *); -static int priq_stat_sc(struct priq_if *, cqrq_stat_sc_t *); -static inline struct priq_class *priq_clh_to_clp(struct priq_if *, u_int32_t); -static const char *priq_style(struct priq_if *); - -#define PRIQ_ZONE_MAX 32 /* maximum elements in zone */ -#define PRIQ_ZONE_NAME "pktsched_priq" /* zone name */ - -static unsigned int priq_size; /* size of zone element */ -static struct zone *priq_zone; /* zone for priq */ - -#define PRIQ_CL_ZONE_MAX 32 /* maximum elements in zone */ -#define PRIQ_CL_ZONE_NAME "pktsched_priq_cl" /* zone name */ - -static unsigned int priq_cl_size; /* size of zone element */ -static struct zone *priq_cl_zone; /* zone for priq_class */ - -void -priq_init(void) -{ - priq_size = sizeof (struct priq_if); - priq_zone = zinit(priq_size, PRIQ_ZONE_MAX * priq_size, - 0, PRIQ_ZONE_NAME); - if (priq_zone == NULL) { - panic("%s: failed allocating %s", __func__, PRIQ_ZONE_NAME); - /* NOTREACHED */ - } - zone_change(priq_zone, Z_EXPAND, TRUE); - zone_change(priq_zone, Z_CALLERACCT, TRUE); - - priq_cl_size = sizeof (struct priq_class); - priq_cl_zone = zinit(priq_cl_size, PRIQ_CL_ZONE_MAX * priq_cl_size, - 0, PRIQ_CL_ZONE_NAME); - if (priq_cl_zone == NULL) { - panic("%s: failed allocating %s", __func__, PRIQ_CL_ZONE_NAME); - /* NOTREACHED */ - } - zone_change(priq_cl_zone, Z_EXPAND, TRUE); - zone_change(priq_cl_zone, Z_CALLERACCT, TRUE); -} - -struct priq_if * -priq_alloc(struct ifnet *ifp, int how, boolean_t altq) -{ - struct priq_if *pif; - - pif = (how == M_WAITOK) ? zalloc(priq_zone) : zalloc_noblock(priq_zone); - if (pif == NULL) - return (NULL); - - bzero(pif, priq_size); - pif->pif_maxpri = -1; - pif->pif_ifq = &ifp->if_snd; - if (altq) - pif->pif_flags |= PRIQIFF_ALTQ; - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s scheduler allocated\n", - if_name(ifp), priq_style(pif)); - } - - return (pif); -} - -int -priq_destroy(struct priq_if *pif) -{ - struct ifclassq *ifq = pif->pif_ifq; - int err; - - IFCQ_LOCK(ifq); - err = priq_destroy_locked(pif); - IFCQ_UNLOCK(ifq); - - return (err); -} - -static int -priq_destroy_locked(struct priq_if *pif) -{ - IFCQ_LOCK_ASSERT_HELD(pif->pif_ifq); - - (void) priq_clear_interface(pif); - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s scheduler destroyed\n", - if_name(PRIQIF_IFP(pif)), priq_style(pif)); - } - - zfree(priq_zone, pif); - - return (0); -} - -/* - * bring the interface back to the initial state by discarding - * all the filters and classes. - */ -static int -priq_clear_interface(struct priq_if *pif) -{ - struct priq_class *cl; - int pri; - - IFCQ_LOCK_ASSERT_HELD(pif->pif_ifq); - - /* clear out the classes */ - for (pri = 0; pri <= pif->pif_maxpri; pri++) - if ((cl = pif->pif_classes[pri]) != NULL) - priq_class_destroy(pif, cl); - - return (0); -} - -/* discard all the queued packets on the interface */ -void -priq_purge(struct priq_if *pif) -{ - struct priq_class *cl; - int pri; - - IFCQ_LOCK_ASSERT_HELD(pif->pif_ifq); - - for (pri = 0; pri <= pif->pif_maxpri; pri++) { - if ((cl = pif->pif_classes[pri]) != NULL && !qempty(&cl->cl_q)) - priq_purgeq(pif, cl, 0, NULL, NULL); - } -#if !PF_ALTQ - /* - * This assertion is safe to be made only when PF_ALTQ is not - * configured; otherwise, IFCQ_LEN represents the sum of the - * packets managed by ifcq_disc and altq_disc instances, which - * is possible when transitioning between the two. - */ - VERIFY(IFCQ_LEN(pif->pif_ifq) == 0); -#endif /* !PF_ALTQ */ -} - -static void -priq_purge_sc(struct priq_if *pif, cqrq_purge_sc_t *pr) -{ - struct ifclassq *ifq = pif->pif_ifq; - u_int32_t i; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - VERIFY(pr->sc == MBUF_SC_UNSPEC || MBUF_VALID_SC(pr->sc)); - VERIFY(pr->flow != 0); - - if (pr->sc != MBUF_SC_UNSPEC) { - i = MBUF_SCIDX(pr->sc); - VERIFY(i < IFCQ_SC_MAX); - - priq_purgeq(pif, ifq->ifcq_disc_slots[i].cl, - pr->flow, &pr->packets, &pr->bytes); - } else { - u_int32_t cnt, len; - - pr->packets = 0; - pr->bytes = 0; - - for (i = 0; i < IFCQ_SC_MAX; i++) { - priq_purgeq(pif, ifq->ifcq_disc_slots[i].cl, - pr->flow, &cnt, &len); - pr->packets += cnt; - pr->bytes += len; - } - } -} - -void -priq_event(struct priq_if *pif, cqev_t ev) -{ - struct priq_class *cl; - int pri; - - IFCQ_LOCK_ASSERT_HELD(pif->pif_ifq); - - for (pri = 0; pri <= pif->pif_maxpri; pri++) - if ((cl = pif->pif_classes[pri]) != NULL) - priq_updateq(pif, cl, ev); -} - -int -priq_add_queue(struct priq_if *pif, int priority, u_int32_t qlimit, - int flags, u_int32_t qid, struct priq_class **clp) -{ - struct priq_class *cl; - - IFCQ_LOCK_ASSERT_HELD(pif->pif_ifq); - - /* check parameters */ - if (priority >= PRIQ_MAXPRI) - return (EINVAL); - if (pif->pif_classes[priority] != NULL) - return (EBUSY); - if (priq_clh_to_clp(pif, qid) != NULL) - return (EBUSY); - - cl = priq_class_create(pif, priority, qlimit, flags, qid); - if (cl == NULL) - return (ENOMEM); - - if (clp != NULL) - *clp = cl; - - return (0); -} - -static struct priq_class * -priq_class_create(struct priq_if *pif, int pri, u_int32_t qlimit, - int flags, u_int32_t qid) -{ - struct ifnet *ifp; - struct ifclassq *ifq; - struct priq_class *cl; - - IFCQ_LOCK_ASSERT_HELD(pif->pif_ifq); - - /* Sanitize flags unless internally configured */ - if (pif->pif_flags & PRIQIFF_ALTQ) - flags &= PRCF_USERFLAGS; - -#if !CLASSQ_RED - if (flags & PRCF_RED) { - log(LOG_ERR, "%s: %s RED not available!\n", - if_name(PRIQIF_IFP(pif)), priq_style(pif)); - return (NULL); - } -#endif /* !CLASSQ_RED */ - -#if !CLASSQ_RIO - if (flags & PRCF_RIO) { - log(LOG_ERR, "%s: %s RIO not available!\n", - if_name(PRIQIF_IFP(pif)), priq_style(pif)); - return (NULL); - } -#endif /* CLASSQ_RIO */ - -#if !CLASSQ_BLUE - if (flags & PRCF_BLUE) { - log(LOG_ERR, "%s: %s BLUE not available!\n", - if_name(PRIQIF_IFP(pif)), priq_style(pif)); - return (NULL); - } -#endif /* CLASSQ_BLUE */ - - /* These are mutually exclusive */ - if ((flags & (PRCF_RED|PRCF_RIO|PRCF_BLUE|PRCF_SFB)) && - (flags & (PRCF_RED|PRCF_RIO|PRCF_BLUE|PRCF_SFB)) != PRCF_RED && - (flags & (PRCF_RED|PRCF_RIO|PRCF_BLUE|PRCF_SFB)) != PRCF_RIO && - (flags & (PRCF_RED|PRCF_RIO|PRCF_BLUE|PRCF_SFB)) != PRCF_BLUE && - (flags & (PRCF_RED|PRCF_RIO|PRCF_BLUE|PRCF_SFB)) != PRCF_SFB) { - log(LOG_ERR, "%s: %s more than one RED|RIO|BLUE|SFB\n", - if_name(PRIQIF_IFP(pif)), priq_style(pif)); - return (NULL); - } - - ifq = pif->pif_ifq; - ifp = PRIQIF_IFP(pif); - - if ((cl = pif->pif_classes[pri]) != NULL) { - /* modify the class instead of creating a new one */ - if (!qempty(&cl->cl_q)) - priq_purgeq(pif, cl, 0, NULL, NULL); -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_destroy(cl->cl_rio); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_destroy(cl->cl_red); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_destroy(cl->cl_blue); -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - sfb_destroy(cl->cl_sfb); - cl->cl_qalg.ptr = NULL; - qtype(&cl->cl_q) = Q_DROPTAIL; - qstate(&cl->cl_q) = QS_RUNNING; - } else { - cl = zalloc(priq_cl_zone); - if (cl == NULL) - return (NULL); - - bzero(cl, priq_cl_size); - } - - pif->pif_classes[pri] = cl; - if (flags & PRCF_DEFAULTCLASS) - pif->pif_default = cl; - if (qlimit == 0 || qlimit > IFCQ_MAXLEN(ifq)) { - qlimit = IFCQ_MAXLEN(ifq); - if (qlimit == 0) - qlimit = DEFAULT_QLIMIT; /* use default */ - } - _qinit(&cl->cl_q, Q_DROPTAIL, qlimit); - cl->cl_flags = flags; - cl->cl_pri = pri; - if (pri > pif->pif_maxpri) - pif->pif_maxpri = pri; - cl->cl_pif = pif; - cl->cl_handle = qid; - - if (flags & (PRCF_RED|PRCF_RIO|PRCF_BLUE|PRCF_SFB)) { -#if CLASSQ_RED || CLASSQ_RIO - u_int64_t ifbandwidth = ifnet_output_linkrate(ifp); - int pkttime; -#endif /* CLASSQ_RED || CLASSQ_RIO */ - - cl->cl_qflags = 0; - if (flags & PRCF_ECN) { - if (flags & PRCF_BLUE) - cl->cl_qflags |= BLUEF_ECN; - else if (flags & PRCF_SFB) - cl->cl_qflags |= SFBF_ECN; - else if (flags & PRCF_RED) - cl->cl_qflags |= REDF_ECN; - else if (flags & PRCF_RIO) - cl->cl_qflags |= RIOF_ECN; - } - if (flags & PRCF_FLOWCTL) { - if (flags & PRCF_SFB) - cl->cl_qflags |= SFBF_FLOWCTL; - } - if (flags & PRCF_CLEARDSCP) { - if (flags & PRCF_RIO) - cl->cl_qflags |= RIOF_CLEARDSCP; - } -#if CLASSQ_RED || CLASSQ_RIO - /* - * XXX: RED & RIO should be watching link speed and MTU - * events and recompute pkttime accordingly. - */ - if (ifbandwidth < 8) - pkttime = 1000 * 1000 * 1000; /* 1 sec */ - else - pkttime = (int64_t)ifp->if_mtu * 1000 * 1000 * 1000 / - (ifbandwidth / 8); - - /* Test for exclusivity {RED,RIO,BLUE,SFB} was done above */ -#if CLASSQ_RED - if (flags & PRCF_RED) { - cl->cl_red = red_alloc(ifp, 0, 0, - qlimit(&cl->cl_q) * 10/100, - qlimit(&cl->cl_q) * 30/100, - cl->cl_qflags, pkttime); - if (cl->cl_red != NULL) - qtype(&cl->cl_q) = Q_RED; - } -#endif /* CLASSQ_RED */ -#if CLASSQ_RIO - if (flags & PRCF_RIO) { - cl->cl_rio = - rio_alloc(ifp, 0, NULL, cl->cl_qflags, pkttime); - if (cl->cl_rio != NULL) - qtype(&cl->cl_q) = Q_RIO; - } -#endif /* CLASSQ_RIO */ -#endif /* CLASSQ_RED || CLASSQ_RIO */ -#if CLASSQ_BLUE - if (flags & PRCF_BLUE) { - cl->cl_blue = blue_alloc(ifp, 0, 0, cl->cl_qflags); - if (cl->cl_blue != NULL) - qtype(&cl->cl_q) = Q_BLUE; - } -#endif /* CLASSQ_BLUE */ - if (flags & PRCF_SFB) { - if (!(cl->cl_flags & PRCF_LAZY)) - cl->cl_sfb = sfb_alloc(ifp, cl->cl_handle, - qlimit(&cl->cl_q), cl->cl_qflags); - if (cl->cl_sfb != NULL || (cl->cl_flags & PRCF_LAZY)) - qtype(&cl->cl_q) = Q_SFB; - } - } - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s created qid=%d pri=%d qlimit=%d " - "flags=%b\n", if_name(ifp), priq_style(pif), - cl->cl_handle, cl->cl_pri, qlimit, flags, PRCF_BITS); - } - - return (cl); -} - -int -priq_remove_queue(struct priq_if *pif, u_int32_t qid) -{ - struct priq_class *cl; - - IFCQ_LOCK_ASSERT_HELD(pif->pif_ifq); - - if ((cl = priq_clh_to_clp(pif, qid)) == NULL) - return (EINVAL); - - return (priq_class_destroy(pif, cl)); -} - -static int -priq_class_destroy(struct priq_if *pif, struct priq_class *cl) -{ - struct ifclassq *ifq = pif->pif_ifq; - int pri; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - if (!qempty(&cl->cl_q)) - priq_purgeq(pif, cl, 0, NULL, NULL); - - VERIFY(cl->cl_pri < PRIQ_MAXPRI); - VERIFY(!pktsched_bit_tst(cl->cl_pri, &pif->pif_bitmap)); - - pif->pif_classes[cl->cl_pri] = NULL; - if (pif->pif_maxpri == cl->cl_pri) { - for (pri = cl->cl_pri; pri >= 0; pri--) - if (pif->pif_classes[pri] != NULL) { - pif->pif_maxpri = pri; - break; - } - if (pri < 0) - pif->pif_maxpri = -1; - } - - if (pif->pif_default == cl) - pif->pif_default = NULL; - - if (cl->cl_qalg.ptr != NULL) { -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_destroy(cl->cl_rio); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_destroy(cl->cl_red); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_destroy(cl->cl_blue); -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - sfb_destroy(cl->cl_sfb); - cl->cl_qalg.ptr = NULL; - qtype(&cl->cl_q) = Q_DROPTAIL; - qstate(&cl->cl_q) = QS_RUNNING; - } - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s destroyed qid=%d pri=%d\n", - if_name(PRIQIF_IFP(pif)), priq_style(pif), - cl->cl_handle, cl->cl_pri); - } - - zfree(priq_cl_zone, cl); - - return (0); -} - -int -priq_enqueue(struct priq_if *pif, struct priq_class *cl, struct mbuf *m, - struct pf_mtag *t) -{ - struct ifclassq *ifq = pif->pif_ifq; - u_int32_t pri; - int len, ret; - - IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(cl == NULL || cl->cl_pif == pif); - - if (cl == NULL) { -#if PF_ALTQ - cl = priq_clh_to_clp(pif, t->pftag_qid); -#else /* !PF_ALTQ */ - cl = priq_clh_to_clp(pif, 0); -#endif /* !PF_ALTQ */ - if (cl == NULL) { - cl = pif->pif_default; - if (cl == NULL) { - IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (ENOBUFS); - } - } - } - pri = cl->cl_pri; - VERIFY(pri < PRIQ_MAXPRI); - - len = m_pktlen(m); - - ret = priq_addq(cl, m, t); - if (ret != 0) { - if (ret == CLASSQEQ_SUCCESS_FC) { - /* packet enqueued, return advisory feedback */ - ret = EQFULL; - } else { - VERIFY(ret == CLASSQEQ_DROPPED || - ret == CLASSQEQ_DROPPED_FC || - ret == CLASSQEQ_DROPPED_SP); - /* packet has been freed in priq_addq */ - PKTCNTR_ADD(&cl->cl_dropcnt, 1, len); - IFCQ_DROP_ADD(ifq, 1, len); - switch (ret) { - case CLASSQEQ_DROPPED: - return (ENOBUFS); - case CLASSQEQ_DROPPED_FC: - return (EQFULL); - case CLASSQEQ_DROPPED_SP: - return (EQSUSPENDED); - } - /* NOT REACHED */ - } - } - IFCQ_INC_LEN(ifq); - IFCQ_INC_BYTES(ifq, len); - - /* class is now active; indicate it as such */ - if (!pktsched_bit_tst(pri, &pif->pif_bitmap)) - pktsched_bit_set(pri, &pif->pif_bitmap); - - /* successfully queued. */ - return (ret); -} - -/* - * note: CLASSQDQ_POLL returns the next packet without removing the packet - * from the queue. CLASSQDQ_REMOVE is a normal dequeue operation. - * CLASSQDQ_REMOVE must return the same packet if called immediately - * after CLASSQDQ_POLL. - */ -struct mbuf * -priq_dequeue(struct priq_if *pif, cqdq_op_t op) -{ - struct ifclassq *ifq = pif->pif_ifq; - struct priq_class *cl; - struct mbuf *m; - u_int32_t pri, len; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - if (pif->pif_bitmap == 0) { - /* no active class; nothing to dequeue */ - return (NULL); - } - VERIFY(!IFCQ_IS_EMPTY(ifq)); - - pri = pktsched_fls(pif->pif_bitmap) - 1; /* zero based */ - VERIFY(pri < PRIQ_MAXPRI); - cl = pif->pif_classes[pri]; - VERIFY(cl != NULL && !qempty(&cl->cl_q)); - - if (op == CLASSQDQ_POLL) - return (priq_pollq(cl)); - - m = priq_getq(cl); - VERIFY(m != NULL); /* qalg must be work conserving */ - len = m_pktlen(m); - - IFCQ_DEC_LEN(ifq); - IFCQ_DEC_BYTES(ifq, len); - if (qempty(&cl->cl_q)) { - cl->cl_period++; - /* class is now inactive; indicate it as such */ - pktsched_bit_clr(pri, &pif->pif_bitmap); - } - PKTCNTR_ADD(&cl->cl_xmitcnt, 1, len); - IFCQ_XMIT_ADD(ifq, 1, len); - - return (m); -} - -static inline int -priq_addq(struct priq_class *cl, struct mbuf *m, struct pf_mtag *t) -{ - struct priq_if *pif = cl->cl_pif; - struct ifclassq *ifq = pif->pif_ifq; - - IFCQ_LOCK_ASSERT_HELD(ifq); - -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - return (rio_addq(cl->cl_rio, &cl->cl_q, m, t)); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - return (red_addq(cl->cl_red, &cl->cl_q, m, t)); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - return (blue_addq(cl->cl_blue, &cl->cl_q, m, t)); - else -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q)) { - if (cl->cl_sfb == NULL) { - struct ifnet *ifp = PRIQIF_IFP(pif); - - VERIFY(cl->cl_flags & PRCF_LAZY); - cl->cl_flags &= ~PRCF_LAZY; - IFCQ_CONVERT_LOCK(ifq); - - cl->cl_sfb = sfb_alloc(ifp, cl->cl_handle, - qlimit(&cl->cl_q), cl->cl_qflags); - if (cl->cl_sfb == NULL) { - /* fall back to droptail */ - qtype(&cl->cl_q) = Q_DROPTAIL; - cl->cl_flags &= ~PRCF_SFB; - cl->cl_qflags &= ~(SFBF_ECN | SFBF_FLOWCTL); - - log(LOG_ERR, "%s: %s SFB lazy allocation " - "failed for qid=%d pri=%d, falling back " - "to DROPTAIL\n", if_name(ifp), - priq_style(pif), cl->cl_handle, - cl->cl_pri); - } else if (pif->pif_throttle != IFNET_THROTTLE_OFF) { - /* if there's pending throttling, set it */ - cqrq_throttle_t tr = { 1, pif->pif_throttle }; - int err = priq_throttle(pif, &tr); - - if (err == EALREADY) - err = 0; - if (err != 0) { - tr.level = IFNET_THROTTLE_OFF; - (void) priq_throttle(pif, &tr); - } - } - } - if (cl->cl_sfb != NULL) - return (sfb_addq(cl->cl_sfb, &cl->cl_q, m, t)); - } else if (qlen(&cl->cl_q) >= qlimit(&cl->cl_q)) { - IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (CLASSQEQ_DROPPED); - } - -#if PF_ECN - if (cl->cl_flags & PRCF_CLEARDSCP) - write_dsfield(m, t, 0); -#endif /* PF_ECN */ - - _addq(&cl->cl_q, m); - - return (0); -} - -static inline struct mbuf * -priq_getq(struct priq_class *cl) -{ - IFCQ_LOCK_ASSERT_HELD(cl->cl_pif->pif_ifq); - -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - return (rio_getq(cl->cl_rio, &cl->cl_q)); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - return (red_getq(cl->cl_red, &cl->cl_q)); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - return (blue_getq(cl->cl_blue, &cl->cl_q)); - else -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - return (sfb_getq(cl->cl_sfb, &cl->cl_q)); - - return (_getq(&cl->cl_q)); -} - -static inline struct mbuf * -priq_pollq(struct priq_class *cl) -{ - IFCQ_LOCK_ASSERT_HELD(cl->cl_pif->pif_ifq); - - return (qhead(&cl->cl_q)); -} - -static void -priq_purgeq(struct priq_if *pif, struct priq_class *cl, u_int32_t flow, - u_int32_t *packets, u_int32_t *bytes) -{ - struct ifclassq *ifq = pif->pif_ifq; - u_int32_t cnt = 0, len = 0, qlen; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - if ((qlen = qlen(&cl->cl_q)) == 0) { - VERIFY(!pktsched_bit_tst(cl->cl_pri, &pif->pif_bitmap)); - goto done; - } - - /* become regular mutex before freeing mbufs */ - IFCQ_CONVERT_LOCK(ifq); - -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_purgeq(cl->cl_rio, &cl->cl_q, flow, &cnt, &len); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_purgeq(cl->cl_red, &cl->cl_q, flow, &cnt, &len); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_purgeq(cl->cl_blue, &cl->cl_q, flow, &cnt, &len); - else -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - sfb_purgeq(cl->cl_sfb, &cl->cl_q, flow, &cnt, &len); - else - _flushq_flow(&cl->cl_q, flow, &cnt, &len); - - if (cnt > 0) { - VERIFY(qlen(&cl->cl_q) == (qlen - cnt)); - - PKTCNTR_ADD(&cl->cl_dropcnt, cnt, len); - IFCQ_DROP_ADD(ifq, cnt, len); - - VERIFY(((signed)IFCQ_LEN(ifq) - cnt) >= 0); - IFCQ_LEN(ifq) -= cnt; - - if (qempty(&cl->cl_q)) - pktsched_bit_clr(cl->cl_pri, &pif->pif_bitmap); - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s purge qid=%d pri=%d " - "qlen=[%d,%d] cnt=%d len=%d flow=0x%x\n", - if_name(PRIQIF_IFP(pif)), priq_style(pif), - cl->cl_handle, cl->cl_pri, qlen, qlen(&cl->cl_q), - cnt, len, flow); - } - } -done: - if (packets != NULL) - *packets = cnt; - if (bytes != NULL) - *bytes = len; -} - -static void -priq_updateq(struct priq_if *pif, struct priq_class *cl, cqev_t ev) -{ - IFCQ_LOCK_ASSERT_HELD(pif->pif_ifq); - - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s update qid=%d pri=%d event=%s\n", - if_name(PRIQIF_IFP(pif)), priq_style(pif), - cl->cl_handle, cl->cl_pri, ifclassq_ev2str(ev)); - } - -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - return (rio_updateq(cl->cl_rio, ev)); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - return (red_updateq(cl->cl_red, ev)); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - return (blue_updateq(cl->cl_blue, ev)); -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - return (sfb_updateq(cl->cl_sfb, ev)); -} - -int -priq_get_class_stats(struct priq_if *pif, u_int32_t qid, - struct priq_classstats *sp) -{ - struct priq_class *cl; - - IFCQ_LOCK_ASSERT_HELD(pif->pif_ifq); - - if ((cl = priq_clh_to_clp(pif, qid)) == NULL) - return (EINVAL); - - sp->class_handle = cl->cl_handle; - sp->priority = cl->cl_pri; - sp->qlength = qlen(&cl->cl_q); - sp->qlimit = qlimit(&cl->cl_q); - sp->period = cl->cl_period; - sp->xmitcnt = cl->cl_xmitcnt; - sp->dropcnt = cl->cl_dropcnt; - - sp->qtype = qtype(&cl->cl_q); - sp->qstate = qstate(&cl->cl_q); -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_getstats(cl->cl_red, &sp->red[0]); -#endif /* CLASSQ_RED */ -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_getstats(cl->cl_rio, &sp->red[0]); -#endif /* CLASSQ_RIO */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_getstats(cl->cl_blue, &sp->blue); -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - sfb_getstats(cl->cl_sfb, &sp->sfb); - - return (0); -} - -static int -priq_stat_sc(struct priq_if *pif, cqrq_stat_sc_t *sr) -{ - struct ifclassq *ifq = pif->pif_ifq; - struct priq_class *cl; - u_int32_t i; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - VERIFY(sr->sc == MBUF_SC_UNSPEC || MBUF_VALID_SC(sr->sc)); - - i = MBUF_SCIDX(sr->sc); - VERIFY(i < IFCQ_SC_MAX); - - cl = ifq->ifcq_disc_slots[i].cl; - sr->packets = qlen(&cl->cl_q); - sr->bytes = qsize(&cl->cl_q); - - return (0); -} - -/* convert a class handle to the corresponding class pointer */ -static inline struct priq_class * -priq_clh_to_clp(struct priq_if *pif, u_int32_t chandle) -{ - struct priq_class *cl; - int idx; - - IFCQ_LOCK_ASSERT_HELD(pif->pif_ifq); - - for (idx = pif->pif_maxpri; idx >= 0; idx--) - if ((cl = pif->pif_classes[idx]) != NULL && - cl->cl_handle == chandle) - return (cl); - - return (NULL); -} - -static const char * -priq_style(struct priq_if *pif) -{ - return ((pif->pif_flags & PRIQIFF_ALTQ) ? "ALTQ_PRIQ" : "PRIQ"); -} - -/* - * priq_enqueue_ifclassq is an enqueue function to be registered to - * (*ifcq_enqueue) in struct ifclassq. - */ -static int -priq_enqueue_ifclassq(struct ifclassq *ifq, struct mbuf *m) -{ - u_int32_t i; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - if (!(m->m_flags & M_PKTHDR)) { - /* should not happen */ - log(LOG_ERR, "%s: packet does not have pkthdr\n", - if_name(ifq->ifcq_ifp)); - IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (ENOBUFS); - } - - i = MBUF_SCIDX(mbuf_get_service_class(m)); - VERIFY((u_int32_t)i < IFCQ_SC_MAX); - - return (priq_enqueue(ifq->ifcq_disc, - ifq->ifcq_disc_slots[i].cl, m, m_pftag(m))); -} - -/* - * priq_dequeue_ifclassq is a dequeue function to be registered to - * (*ifcq_dequeue) in struct ifclass. - * - * note: CLASSQDQ_POLL returns the next packet without removing the packet - * from the queue. CLASSQDQ_REMOVE is a normal dequeue operation. - * CLASSQDQ_REMOVE must return the same packet if called immediately - * after CLASSQDQ_POLL. - */ -static struct mbuf * -priq_dequeue_ifclassq(struct ifclassq *ifq, cqdq_op_t op) -{ - return (priq_dequeue(ifq->ifcq_disc, op)); -} - -static int -priq_request_ifclassq(struct ifclassq *ifq, cqrq_t req, void *arg) -{ - struct priq_if *pif = (struct priq_if *)ifq->ifcq_disc; - int err = 0; - - IFCQ_LOCK_ASSERT_HELD(ifq); - - switch (req) { - case CLASSQRQ_PURGE: - priq_purge(pif); - break; - - case CLASSQRQ_PURGE_SC: - priq_purge_sc(pif, (cqrq_purge_sc_t *)arg); - break; - - case CLASSQRQ_EVENT: - priq_event(pif, (cqev_t)arg); - break; - - case CLASSQRQ_THROTTLE: - err = priq_throttle(pif, (cqrq_throttle_t *)arg); - break; - - case CLASSQRQ_STAT_SC: - err = priq_stat_sc(pif, (cqrq_stat_sc_t *)arg); - break; - } - return (err); -} - -int -priq_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) -{ - struct ifnet *ifp = ifq->ifcq_ifp; - struct priq_class *cl0, *cl1, *cl2, *cl3, *cl4; - struct priq_class *cl5, *cl6, *cl7, *cl8, *cl9; - struct priq_if *pif; - u_int32_t maxlen = 0, qflags = 0; - int err = 0; - - IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(ifq->ifcq_disc == NULL); - VERIFY(ifq->ifcq_type == PKTSCHEDT_NONE); - - if (flags & PKTSCHEDF_QALG_RED) - qflags |= PRCF_RED; - if (flags & PKTSCHEDF_QALG_RIO) - qflags |= PRCF_RIO; - if (flags & PKTSCHEDF_QALG_BLUE) - qflags |= PRCF_BLUE; - if (flags & PKTSCHEDF_QALG_SFB) - qflags |= PRCF_SFB; - if (flags & PKTSCHEDF_QALG_ECN) - qflags |= PRCF_ECN; - if (flags & PKTSCHEDF_QALG_FLOWCTL) - qflags |= PRCF_FLOWCTL; - - pif = priq_alloc(ifp, M_WAITOK, FALSE); - if (pif == NULL) - return (ENOMEM); - - if ((maxlen = IFCQ_MAXLEN(ifq)) == 0) - maxlen = if_sndq_maxlen; - - if ((err = priq_add_queue(pif, 0, maxlen, - qflags | PRCF_LAZY, SCIDX_BK_SYS, &cl0)) != 0) - goto cleanup; - - if ((err = priq_add_queue(pif, 1, maxlen, - qflags | PRCF_LAZY, SCIDX_BK, &cl1)) != 0) - goto cleanup; - - if ((err = priq_add_queue(pif, 2, maxlen, - qflags | PRCF_DEFAULTCLASS, SCIDX_BE, &cl2)) != 0) - goto cleanup; - - if ((err = priq_add_queue(pif, 3, maxlen, - qflags | PRCF_LAZY, SCIDX_RD, &cl3)) != 0) - goto cleanup; - - if ((err = priq_add_queue(pif, 4, maxlen, - qflags | PRCF_LAZY, SCIDX_OAM, &cl4)) != 0) - goto cleanup; - - if ((err = priq_add_queue(pif, 5, maxlen, - qflags | PRCF_LAZY, SCIDX_AV, &cl5)) != 0) - goto cleanup; - - if ((err = priq_add_queue(pif, 6, maxlen, - qflags | PRCF_LAZY, SCIDX_RV, &cl6)) != 0) - goto cleanup; - - if ((err = priq_add_queue(pif, 7, maxlen, - qflags | PRCF_LAZY, SCIDX_VI, &cl7)) != 0) - goto cleanup; - - if ((err = priq_add_queue(pif, 8, maxlen, - qflags | PRCF_LAZY, SCIDX_VO, &cl8)) != 0) - goto cleanup; - - if ((err = priq_add_queue(pif, 9, maxlen, - qflags, SCIDX_CTL, &cl9)) != 0) - goto cleanup; - - err = ifclassq_attach(ifq, PKTSCHEDT_PRIQ, pif, - priq_enqueue_ifclassq, priq_dequeue_ifclassq, NULL, - NULL, priq_request_ifclassq); - - /* cache these for faster lookup */ - if (err == 0) { - ifq->ifcq_disc_slots[SCIDX_BK_SYS].qid = SCIDX_BK_SYS; - ifq->ifcq_disc_slots[SCIDX_BK_SYS].cl = cl0; - - ifq->ifcq_disc_slots[SCIDX_BK].qid = SCIDX_BK; - ifq->ifcq_disc_slots[SCIDX_BK].cl = cl1; - - ifq->ifcq_disc_slots[SCIDX_BE].qid = SCIDX_BE; - ifq->ifcq_disc_slots[SCIDX_BE].cl = cl2; - - ifq->ifcq_disc_slots[SCIDX_RD].qid = SCIDX_RD; - ifq->ifcq_disc_slots[SCIDX_RD].cl = cl3; - - ifq->ifcq_disc_slots[SCIDX_OAM].qid = SCIDX_OAM; - ifq->ifcq_disc_slots[SCIDX_OAM].cl = cl4; - - ifq->ifcq_disc_slots[SCIDX_AV].qid = SCIDX_AV; - ifq->ifcq_disc_slots[SCIDX_AV].cl = cl5; - - ifq->ifcq_disc_slots[SCIDX_RV].qid = SCIDX_RV; - ifq->ifcq_disc_slots[SCIDX_RV].cl = cl6; - - ifq->ifcq_disc_slots[SCIDX_VI].qid = SCIDX_VI; - ifq->ifcq_disc_slots[SCIDX_VI].cl = cl7; - - ifq->ifcq_disc_slots[SCIDX_VO].qid = SCIDX_VO; - ifq->ifcq_disc_slots[SCIDX_VO].cl = cl8; - - ifq->ifcq_disc_slots[SCIDX_CTL].qid = SCIDX_CTL; - ifq->ifcq_disc_slots[SCIDX_CTL].cl = cl9; - } - -cleanup: - if (err != 0) - (void) priq_destroy_locked(pif); - - return (err); -} - -int -priq_teardown_ifclassq(struct ifclassq *ifq) -{ - struct priq_if *pif = ifq->ifcq_disc; - int i; - - IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(pif != NULL && ifq->ifcq_type == PKTSCHEDT_PRIQ); - - (void) priq_destroy_locked(pif); - - ifq->ifcq_disc = NULL; - for (i = 0; i < IFCQ_SC_MAX; i++) { - ifq->ifcq_disc_slots[i].qid = 0; - ifq->ifcq_disc_slots[i].cl = NULL; - } - - return (ifclassq_detach(ifq)); -} - -int -priq_getqstats_ifclassq(struct ifclassq *ifq, u_int32_t slot, - struct if_ifclassq_stats *ifqs) -{ - struct priq_if *pif = ifq->ifcq_disc; - - IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(ifq->ifcq_type == PKTSCHEDT_PRIQ); - - if (slot >= IFCQ_SC_MAX) - return (EINVAL); - - return (priq_get_class_stats(pif, ifq->ifcq_disc_slots[slot].qid, - &ifqs->ifqs_priq_stats)); -} - -static int -priq_throttle(struct priq_if *pif, cqrq_throttle_t *tr) -{ - struct ifclassq *ifq = pif->pif_ifq; - struct priq_class *cl; - int err = 0; - - IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(!(pif->pif_flags & PRIQIFF_ALTQ)); - - if (!tr->set) { - tr->level = pif->pif_throttle; - return (0); - } - - if (tr->level == pif->pif_throttle) - return (EALREADY); - - /* Current throttling levels only involve BK_SYS class */ - cl = ifq->ifcq_disc_slots[SCIDX_BK_SYS].cl; - - switch (tr->level) { - case IFNET_THROTTLE_OFF: - err = priq_resumeq(pif, cl); - break; - - case IFNET_THROTTLE_OPPORTUNISTIC: - err = priq_suspendq(pif, cl); - break; - - default: - VERIFY(0); - /* NOTREACHED */ - } - - if (err == 0 || err == ENXIO) { - if (pktsched_verbose) { - log(LOG_DEBUG, "%s: %s throttling level %sset %d->%d\n", - if_name(PRIQIF_IFP(pif)), priq_style(pif), - (err == 0) ? "" : "lazy ", pif->pif_throttle, - tr->level); - } - pif->pif_throttle = tr->level; - if (err != 0) - err = 0; - else - priq_purgeq(pif, cl, 0, NULL, NULL); - } else { - log(LOG_ERR, "%s: %s unable to set throttling level " - "%d->%d [error=%d]\n", if_name(PRIQIF_IFP(pif)), - priq_style(pif), pif->pif_throttle, tr->level, err); - } - - return (err); -} - -static int -priq_resumeq(struct priq_if *pif, struct priq_class *cl) -{ - struct ifclassq *ifq = pif->pif_ifq; - int err = 0; - - IFCQ_LOCK_ASSERT_HELD(ifq); - -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - err = rio_suspendq(cl->cl_rio, &cl->cl_q, FALSE); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - err = red_suspendq(cl->cl_red, &cl->cl_q, FALSE); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - err = blue_suspendq(cl->cl_blue, &cl->cl_q, FALSE); - else -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - err = sfb_suspendq(cl->cl_sfb, &cl->cl_q, FALSE); - - if (err == 0) - qstate(&cl->cl_q) = QS_RUNNING; - - return (err); -} - -static int -priq_suspendq(struct priq_if *pif, struct priq_class *cl) -{ - struct ifclassq *ifq = pif->pif_ifq; - int err = 0; - - IFCQ_LOCK_ASSERT_HELD(ifq); - -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - err = rio_suspendq(cl->cl_rio, &cl->cl_q, TRUE); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - err = red_suspendq(cl->cl_red, &cl->cl_q, TRUE); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - err = blue_suspendq(cl->cl_blue, &cl->cl_q, TRUE); - else -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q)) { - if (cl->cl_sfb != NULL) { - err = sfb_suspendq(cl->cl_sfb, &cl->cl_q, TRUE); - } else { - VERIFY(cl->cl_flags & PRCF_LAZY); - err = ENXIO; /* delayed throttling */ - } - } - - if (err == 0 || err == ENXIO) - qstate(&cl->cl_q) = QS_SUSPENDED; - - return (err); -} -#endif /* PKTSCHED_PRIQ */ diff --git a/bsd/net/pktsched/pktsched_priq.h b/bsd/net/pktsched/pktsched_priq.h index 4dc9b74bc..858cf9ef3 100644 --- a/bsd/net/pktsched/pktsched_priq.h +++ b/bsd/net/pktsched/pktsched_priq.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2012 Apple Inc. All rights reserved. + * Copyright (c) 2011-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -118,73 +118,6 @@ struct priq_classstats { classq_state_t qstate; }; -#ifdef BSD_KERNEL_PRIVATE -struct priq_class { - u_int32_t cl_handle; /* class handle */ - class_queue_t cl_q; /* class queue structure */ - u_int32_t cl_qflags; /* class queue flags */ - union { - void *ptr; - struct red *red; /* RED state */ - struct rio *rio; /* RIO state */ - struct blue *blue; /* BLUE state */ - struct sfb *sfb; /* SFB state */ - } cl_qalg; - int32_t cl_pri; /* priority */ - u_int32_t cl_flags; /* class flags */ - struct priq_if *cl_pif; /* back pointer to pif */ - - /* statistics */ - u_int32_t cl_period; /* backlog period */ - struct pktcntr cl_xmitcnt; /* transmitted packet counter */ - struct pktcntr cl_dropcnt; /* dropped packet counter */ -}; - -#define cl_red cl_qalg.red -#define cl_rio cl_qalg.rio -#define cl_blue cl_qalg.blue -#define cl_sfb cl_qalg.sfb - -/* priq_if flags */ -#define PRIQIFF_ALTQ 0x1 /* configured via PF/ALTQ */ - -/* - * priq interface state - */ -struct priq_if { - struct ifclassq *pif_ifq; /* backpointer to ifclassq */ - int pif_maxpri; /* max priority in use */ - u_int32_t pif_flags; /* flags */ - u_int32_t pif_throttle; /* throttling level */ - pktsched_bitmap_t pif_bitmap; /* active class bitmap */ - struct priq_class *pif_default; /* default class */ - struct priq_class *pif_classes[PRIQ_MAXPRI]; /* classes */ -}; - -#define PRIQIF_IFP(_pif) ((_pif)->pif_ifq->ifcq_ifp) - -struct if_ifclassq_stats; - -extern void priq_init(void); -extern struct priq_if *priq_alloc(struct ifnet *, int, boolean_t); -extern int priq_destroy(struct priq_if *); -extern void priq_purge(struct priq_if *); -extern void priq_event(struct priq_if *, cqev_t); -extern int priq_add_queue(struct priq_if *, int, u_int32_t, int, u_int32_t, - struct priq_class **); -extern int priq_remove_queue(struct priq_if *, u_int32_t); -extern int priq_get_class_stats(struct priq_if *, u_int32_t, - struct priq_classstats *); -extern int priq_enqueue(struct priq_if *, struct priq_class *, struct mbuf *, - struct pf_mtag *); -extern struct mbuf *priq_dequeue(struct priq_if *, cqdq_op_t); -extern int priq_setup_ifclassq(struct ifclassq *, u_int32_t); -extern int priq_teardown_ifclassq(struct ifclassq *ifq); -extern int priq_getqstats_ifclassq(struct ifclassq *, u_int32_t, - struct if_ifclassq_stats *); -extern int priq_set_throttle(struct ifclassq *, u_int32_t); -extern int priq_get_throttle(struct ifclassq *, u_int32_t *); -#endif /* BSD_KERNEL_PRIVATE */ #ifdef __cplusplus } #endif diff --git a/bsd/net/pktsched/pktsched_qfq.c b/bsd/net/pktsched/pktsched_qfq.c index b1a88d435..862ee8711 100644 --- a/bsd/net/pktsched/pktsched_qfq.c +++ b/bsd/net/pktsched/pktsched_qfq.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2012 Apple Inc. All rights reserved. + * Copyright (c) 2011-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -78,20 +78,22 @@ #include <net/pktsched/pktsched_qfq.h> #include <netinet/in.h> + /* * function prototypes */ -static int qfq_enqueue_ifclassq(struct ifclassq *, struct mbuf *); -static struct mbuf *qfq_dequeue_ifclassq(struct ifclassq *, cqdq_op_t); +static int qfq_enqueue_ifclassq(struct ifclassq *, void *, classq_pkt_type_t, + boolean_t *); +static void *qfq_dequeue_ifclassq(struct ifclassq *, classq_pkt_type_t *); static int qfq_request_ifclassq(struct ifclassq *, cqrq_t, void *); static int qfq_clear_interface(struct qfq_if *); static struct qfq_class *qfq_class_create(struct qfq_if *, u_int32_t, - u_int32_t, u_int32_t, u_int32_t, u_int32_t); + u_int32_t, u_int32_t, u_int32_t, u_int32_t, classq_pkt_type_t); static int qfq_class_destroy(struct qfq_if *, struct qfq_class *); static int qfq_destroy_locked(struct qfq_if *); -static inline int qfq_addq(struct qfq_class *, struct mbuf *, struct pf_mtag *); -static inline struct mbuf *qfq_getq(struct qfq_class *); -static inline struct mbuf *qfq_pollq(struct qfq_class *); +static inline int qfq_addq(struct qfq_class *, pktsched_pkt_t *, + struct pf_mtag *); +static inline void qfq_getq(struct qfq_class *, pktsched_pkt_t *); static void qfq_purgeq(struct qfq_if *, struct qfq_class *, u_int32_t, u_int32_t *, u_int32_t *); static void qfq_purge_sc(struct qfq_if *, cqrq_purge_sc_t *); @@ -179,7 +181,7 @@ qfq_init(void) } struct qfq_if * -qfq_alloc(struct ifnet *ifp, int how, boolean_t altq) +qfq_alloc(struct ifnet *ifp, int how) { struct qfq_if *qif; @@ -189,22 +191,17 @@ qfq_alloc(struct ifnet *ifp, int how, boolean_t altq) bzero(qif, qfq_size); qif->qif_ifq = &ifp->if_snd; - if (altq) { - qif->qif_maxclasses = QFQ_MAX_CLASSES; - qif->qif_maxslots = QFQ_MAX_SLOTS; - qif->qif_flags |= QFQIFF_ALTQ; - } else { - qif->qif_maxclasses = IFCQ_SC_MAX; - /* - * TODO: adi@apple.com - * - * Ideally I would like to have the following - * but QFQ needs further modifications. - * - * qif->qif_maxslots = IFCQ_SC_MAX; - */ - qif->qif_maxslots = QFQ_MAX_SLOTS; - } + + qif->qif_maxclasses = IFCQ_SC_MAX; + /* + * TODO: adi@apple.com + * + * Ideally I would like to have the following + * but QFQ needs further modifications. + * + * qif->qif_maxslots = IFCQ_SC_MAX; + */ + qif->qif_maxslots = QFQ_MAX_SLOTS; if ((qif->qif_class_tbl = _MALLOC(sizeof (struct qfq_class *) * qif->qif_maxclasses, M_DEVBUF, M_WAITOK|M_ZERO)) == NULL) { @@ -325,15 +322,7 @@ qfq_purge(struct qfq_if *qif) if ((cl = qif->qif_class_tbl[i]) != NULL) qfq_purgeq(qif, cl, 0, NULL, NULL); } -#if !PF_ALTQ - /* - * This assertion is safe to be made only when PF_ALTQ is not - * configured; otherwise, IFCQ_LEN represents the sum of the - * packets managed by ifcq_disc and altq_disc instances, which - * is possible when transitioning between the two. - */ VERIFY(IFCQ_LEN(qif->qif_ifq) == 0); -#endif /* !PF_ALTQ */ } static void @@ -383,7 +372,8 @@ qfq_event(struct qfq_if *qif, cqev_t ev) int qfq_add_queue(struct qfq_if *qif, u_int32_t qlimit, u_int32_t weight, - u_int32_t maxsz, u_int32_t flags, u_int32_t qid, struct qfq_class **clp) + u_int32_t maxsz, u_int32_t flags, u_int32_t qid, struct qfq_class **clp, + classq_pkt_type_t ptype) { struct qfq_class *cl; u_int32_t w; @@ -404,7 +394,7 @@ qfq_add_queue(struct qfq_if *qif, u_int32_t qlimit, u_int32_t weight, if (maxsz == 0 || maxsz > (1 << QFQ_MTU_SHIFT)) return (EINVAL); - cl = qfq_class_create(qif, weight, qlimit, flags, maxsz, qid); + cl = qfq_class_create(qif, weight, qlimit, flags, maxsz, qid, ptype); if (cl == NULL) return (ENOMEM); @@ -416,7 +406,7 @@ qfq_add_queue(struct qfq_if *qif, u_int32_t qlimit, u_int32_t weight, static struct qfq_class * qfq_class_create(struct qfq_if *qif, u_int32_t weight, u_int32_t qlimit, - u_int32_t flags, u_int32_t maxsz, u_int32_t qid) + u_int32_t flags, u_int32_t maxsz, u_int32_t qid, classq_pkt_type_t ptype) { struct ifnet *ifp; struct ifclassq *ifq; @@ -427,10 +417,6 @@ qfq_class_create(struct qfq_if *qif, u_int32_t weight, u_int32_t qlimit, IFCQ_LOCK_ASSERT_HELD(qif->qif_ifq); - /* Sanitize flags unless internally configured */ - if (qif->qif_flags & QFQIFF_ALTQ) - flags &= QFCF_USERFLAGS; - if (qif->qif_classes >= qif->qif_maxclasses) { log(LOG_ERR, "%s: %s out of classes! (max %d)\n", if_name(QFQIF_IFP(qif)), qfq_style(qif), @@ -438,41 +424,6 @@ qfq_class_create(struct qfq_if *qif, u_int32_t weight, u_int32_t qlimit, return (NULL); } -#if !CLASSQ_RED - if (flags & QFCF_RED) { - log(LOG_ERR, "%s: %s RED not available!\n", - if_name(QFQIF_IFP(qif)), qfq_style(qif)); - return (NULL); - } -#endif /* !CLASSQ_RED */ - -#if !CLASSQ_RIO - if (flags & QFCF_RIO) { - log(LOG_ERR, "%s: %s RIO not available!\n", - if_name(QFQIF_IFP(qif)), qfq_style(qif)); - return (NULL); - } -#endif /* CLASSQ_RIO */ - -#if !CLASSQ_BLUE - if (flags & QFCF_BLUE) { - log(LOG_ERR, "%s: %s BLUE not available!\n", - if_name(QFQIF_IFP(qif)), qfq_style(qif)); - return (NULL); - } -#endif /* CLASSQ_BLUE */ - - /* These are mutually exclusive */ - if ((flags & (QFCF_RED|QFCF_RIO|QFCF_BLUE|QFCF_SFB)) && - (flags & (QFCF_RED|QFCF_RIO|QFCF_BLUE|QFCF_SFB)) != QFCF_RED && - (flags & (QFCF_RED|QFCF_RIO|QFCF_BLUE|QFCF_SFB)) != QFCF_RIO && - (flags & (QFCF_RED|QFCF_RIO|QFCF_BLUE|QFCF_SFB)) != QFCF_BLUE && - (flags & (QFCF_RED|QFCF_RIO|QFCF_BLUE|QFCF_SFB)) != QFCF_SFB) { - log(LOG_ERR, "%s: %s more than one RED|RIO|BLUE|SFB\n", - if_name(QFQIF_IFP(qif)), qfq_style(qif)); - return (NULL); - } - ifq = qif->qif_ifq; ifp = QFQIF_IFP(qif); @@ -487,7 +438,7 @@ qfq_class_create(struct qfq_if *qif, u_int32_t weight, u_int32_t qlimit, if (qlimit == 0) qlimit = DEFAULT_QLIMIT; /* use default */ } - _qinit(&cl->cl_q, Q_DROPTAIL, qlimit); + _qinit(&cl->cl_q, Q_DROPTAIL, qlimit, ptype); cl->cl_qif = qif; cl->cl_flags = flags; cl->cl_handle = qid; @@ -561,80 +512,22 @@ qfq_class_create(struct qfq_if *qif, u_int32_t weight, u_int32_t qlimit, if (flags & QFCF_DEFAULTCLASS) qif->qif_default = cl; - if (flags & (QFCF_RED|QFCF_RIO|QFCF_BLUE|QFCF_SFB)) { -#if CLASSQ_RED || CLASSQ_RIO - u_int64_t ifbandwidth = ifnet_output_linkrate(ifp); - int pkttime; -#endif /* CLASSQ_RED || CLASSQ_RIO */ - + if (flags & QFCF_SFB) { cl->cl_qflags = 0; if (flags & QFCF_ECN) { - if (flags & QFCF_BLUE) - cl->cl_qflags |= BLUEF_ECN; - else if (flags & QFCF_SFB) - cl->cl_qflags |= SFBF_ECN; - else if (flags & QFCF_RED) - cl->cl_qflags |= REDF_ECN; - else if (flags & QFCF_RIO) - cl->cl_qflags |= RIOF_ECN; + cl->cl_qflags |= SFBF_ECN; } if (flags & QFCF_FLOWCTL) { - if (flags & QFCF_SFB) - cl->cl_qflags |= SFBF_FLOWCTL; + cl->cl_qflags |= SFBF_FLOWCTL; } if (flags & QFCF_DELAYBASED) { - if (flags & QFCF_SFB) - cl->cl_qflags |= SFBF_DELAYBASED; - } - if (flags & QFCF_CLEARDSCP) { - if (flags & QFCF_RIO) - cl->cl_qflags |= RIOF_CLEARDSCP; - } -#if CLASSQ_RED || CLASSQ_RIO - /* - * XXX: RED & RIO should be watching link speed and MTU - * events and recompute pkttime accordingly. - */ - if (ifbandwidth < 8) - pkttime = 1000 * 1000 * 1000; /* 1 sec */ - else - pkttime = (int64_t)ifp->if_mtu * 1000 * 1000 * 1000 / - (ifbandwidth / 8); - - /* Test for exclusivity {RED,RIO,BLUE,SFB} was done above */ -#if CLASSQ_RED - if (flags & QFCF_RED) { - cl->cl_red = red_alloc(ifp, 0, 0, - qlimit(&cl->cl_q) * 10/100, - qlimit(&cl->cl_q) * 30/100, - cl->cl_qflags, pkttime); - if (cl->cl_red != NULL) - qtype(&cl->cl_q) = Q_RED; - } -#endif /* CLASSQ_RED */ -#if CLASSQ_RIO - if (flags & QFCF_RIO) { - cl->cl_rio = - rio_alloc(ifp, 0, NULL, cl->cl_qflags, pkttime); - if (cl->cl_rio != NULL) - qtype(&cl->cl_q) = Q_RIO; - } -#endif /* CLASSQ_RIO */ -#endif /* CLASSQ_RED || CLASSQ_RIO */ -#if CLASSQ_BLUE - if (flags & QFCF_BLUE) { - cl->cl_blue = blue_alloc(ifp, 0, 0, cl->cl_qflags); - if (cl->cl_blue != NULL) - qtype(&cl->cl_q) = Q_BLUE; - } -#endif /* CLASSQ_BLUE */ - if (flags & QFCF_SFB) { - if (!(cl->cl_flags & QFCF_LAZY)) - cl->cl_sfb = sfb_alloc(ifp, cl->cl_handle, - qlimit(&cl->cl_q), cl->cl_qflags); - if (cl->cl_sfb != NULL || (cl->cl_flags & QFCF_LAZY)) - qtype(&cl->cl_q) = Q_SFB; + cl->cl_qflags |= SFBF_DELAYBASED; } + if (!(cl->cl_flags & QFCF_LAZY)) + cl->cl_sfb = sfb_alloc(ifp, cl->cl_handle, + qlimit(&cl->cl_q), cl->cl_qflags); + if (cl->cl_sfb != NULL || (cl->cl_flags & QFCF_LAZY)) + qtype(&cl->cl_q) = Q_SFB; } if (pktsched_verbose) { @@ -665,6 +558,9 @@ qfq_class_destroy(struct qfq_if *qif, struct qfq_class *cl) { struct ifclassq *ifq = qif->qif_ifq; int i; +#if !MACH_ASSERT +#pragma unused(ifq) +#endif IFCQ_LOCK_ASSERT_HELD(ifq); @@ -684,18 +580,6 @@ qfq_class_destroy(struct qfq_if *qif, struct qfq_class *cl) qif->qif_classes--; if (cl->cl_qalg.ptr != NULL) { -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_destroy(cl->cl_rio); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_destroy(cl->cl_red); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_destroy(cl->cl_blue); -#endif /* CLASSQ_BLUE */ if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) sfb_destroy(cl->cl_sfb); cl->cl_qalg.ptr = NULL; @@ -911,7 +795,7 @@ qfq_update_class(struct qfq_if *qif, struct qfq_group *grp, u_int32_t len; u_int64_t roundedS; - len = m_pktlen(qhead(&cl->cl_q)); + len = m_pktlen((struct mbuf *)qhead(&cl->cl_q)); cl->cl_F = cl->cl_S + (u_int64_t)len * cl->cl_inv_w; roundedS = qfq_round_down(cl->cl_S, grp->qfg_slot_shift); if (roundedS == grp->qfg_S) @@ -929,19 +813,20 @@ qfq_update_class(struct qfq_if *qif, struct qfq_group *grp, * CLASSQDQ_REMOVE must return the same packet if called immediately * after CLASSQDQ_POLL. */ -struct mbuf * -qfq_dequeue(struct qfq_if *qif, cqdq_op_t op) +void +qfq_dequeue(struct qfq_if *qif, pktsched_pkt_t *pkt) { pktsched_bitmap_t er_bits = qif->qif_bitmaps[ER]; struct ifclassq *ifq = qif->qif_ifq; struct qfq_group *grp; struct qfq_class *cl; - struct mbuf *m; u_int64_t old_V; u_int32_t len; IFCQ_LOCK_ASSERT_HELD(ifq); + pkt->pktsched_pkt = NULL; + for (;;) { if (er_bits == 0) { #if QFQ_DEBUG @@ -949,7 +834,7 @@ qfq_dequeue(struct qfq_if *qif, cqdq_op_t op) qfq_dump_sched(qif, "start dequeue"); #endif /* QFQ_DEBUG */ /* no eligible and ready packet */ - return (NULL); + return; } grp = qfq_ffs(qif, er_bits); /* if group is non-empty, use it */ @@ -965,12 +850,9 @@ qfq_dequeue(struct qfq_if *qif, cqdq_op_t op) cl = grp->qfg_slots[grp->qfg_front]; VERIFY(cl != NULL && !qempty(&cl->cl_q)); - if (op == CLASSQDQ_POLL) - return (qfq_pollq(cl)); - - m = qfq_getq(cl); - VERIFY(m != NULL); /* qalg must be work conserving */ - len = m_pktlen(m); + qfq_getq(cl, pkt); + VERIFY(pkt->pktsched_pkt != NULL); /* qalg must be work conserving */ + len = pktsched_get_pkt_len(pkt); #if QFQ_DEBUG qif->qif_queued--; @@ -987,9 +869,10 @@ qfq_dequeue(struct qfq_if *qif, cqdq_op_t op) qif->qif_V += (u_int64_t)len * QFQ_IWSUM; if (pktsched_verbose > 2) { - log(LOG_DEBUG, "%s: %s qid=%d dequeue m=0x%llx F=0x%llx " + log(LOG_DEBUG, "%s: %s qid=%d dequeue pkt=0x%llx F=0x%llx " "V=0x%llx", if_name(QFQIF_IFP(qif)), qfq_style(qif), - cl->cl_handle, (uint64_t)VM_KERNEL_ADDRPERM(m), cl->cl_F, + cl->cl_handle, + (uint64_t)VM_KERNEL_ADDRPERM(pkt->pktsched_pkt), cl->cl_F, qif->qif_V); } @@ -1026,8 +909,6 @@ skip_unblock: if (!qif->qif_bitmaps[ER] && qif->qif_queued && pktsched_verbose > 1) qfq_dump_sched(qif, "end dequeue"); #endif /* QFQ_DEBUG */ - - return (m); } /* @@ -1070,7 +951,7 @@ qfq_update_start(struct qfq_if *qif, struct qfq_class *cl) } int -qfq_enqueue(struct qfq_if *qif, struct qfq_class *cl, struct mbuf *m, +qfq_enqueue(struct qfq_if *qif, struct qfq_class *cl, pktsched_pkt_t *pkt, struct pf_mtag *t) { struct ifclassq *ifq = qif->qif_ifq; @@ -1082,45 +963,27 @@ qfq_enqueue(struct qfq_if *qif, struct qfq_class *cl, struct mbuf *m, VERIFY(cl == NULL || cl->cl_qif == qif); if (cl == NULL) { -#if PF_ALTQ - cl = qfq_clh_to_clp(qif, t->pftag_qid); -#else /* !PF_ALTQ */ cl = qfq_clh_to_clp(qif, 0); -#endif /* !PF_ALTQ */ if (cl == NULL) { cl = qif->qif_default; if (cl == NULL) { IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (ENOBUFS); + return (CLASSQEQ_DROP); } } } - len = m_pktlen(m); - - ret = qfq_addq(cl, m, t); - if (ret != 0) { - if (ret == CLASSQEQ_SUCCESS_FC) { - /* packet enqueued, return advisory feedback */ - ret = EQFULL; - } else { - VERIFY(ret == CLASSQEQ_DROPPED || - ret == CLASSQEQ_DROPPED_FC || - ret == CLASSQEQ_DROPPED_SP); - /* packet has been freed in qfq_addq */ - PKTCNTR_ADD(&cl->cl_dropcnt, 1, len); - IFCQ_DROP_ADD(ifq, 1, len); - switch (ret) { - case CLASSQEQ_DROPPED: - return (ENOBUFS); - case CLASSQEQ_DROPPED_FC: - return (EQFULL); - case CLASSQEQ_DROPPED_SP: - return (EQSUSPENDED); - } - /* NOT REACHED */ - } + VERIFY(pkt->pktsched_ptype == qptype(&cl->cl_q)); + len = pktsched_get_pkt_len(pkt); + + ret = qfq_addq(cl, pkt, t); + if ((ret != 0) && (ret != CLASSQEQ_SUCCESS_FC)) { + VERIFY(ret == CLASSQEQ_DROP || + ret == CLASSQEQ_DROP_FC || + ret == CLASSQEQ_DROP_SP); + PKTCNTR_ADD(&cl->cl_dropcnt, 1, len); + IFCQ_DROP_ADD(ifq, 1, len); + return (ret); } IFCQ_INC_LEN(ifq); IFCQ_INC_BYTES(ifq, len); @@ -1174,7 +1037,8 @@ qfq_enqueue(struct qfq_if *qif, struct qfq_class *cl, struct mbuf *m, log(LOG_DEBUG, "%s: %s qid=%d enqueue m=0x%llx state=%s 0x%x " "S=0x%llx F=0x%llx V=0x%llx\n", if_name(QFQIF_IFP(qif)), qfq_style(qif), cl->cl_handle, - (uint64_t)VM_KERNEL_ADDRPERM(m), qfq_state2str(s), + (uint64_t)VM_KERNEL_ADDRPERM(pkt->pktsched_pkt), + qfq_state2str(s), qif->qif_bitmaps[s], cl->cl_S, cl->cl_F, qif->qif_V); } @@ -1311,36 +1175,21 @@ qfq_state2str(int s) } static inline int -qfq_addq(struct qfq_class *cl, struct mbuf *m, struct pf_mtag *t) +qfq_addq(struct qfq_class *cl, pktsched_pkt_t *pkt, struct pf_mtag *t) { struct qfq_if *qif = cl->cl_qif; struct ifclassq *ifq = qif->qif_ifq; IFCQ_LOCK_ASSERT_HELD(ifq); -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - return (rio_addq(cl->cl_rio, &cl->cl_q, m, t)); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - return (red_addq(cl->cl_red, &cl->cl_q, m, t)); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - return (blue_addq(cl->cl_blue, &cl->cl_q, m, t)); - else -#endif /* CLASSQ_BLUE */ if (q_is_sfb(&cl->cl_q)) { if (cl->cl_sfb == NULL) { struct ifnet *ifp = QFQIF_IFP(qif); VERIFY(cl->cl_flags & QFCF_LAZY); cl->cl_flags &= ~QFCF_LAZY; - IFCQ_CONVERT_LOCK(ifq); + IFCQ_CONVERT_LOCK(ifq); cl->cl_sfb = sfb_alloc(ifp, cl->cl_handle, qlimit(&cl->cl_q), cl->cl_qflags); if (cl->cl_sfb == NULL) { @@ -1368,55 +1217,34 @@ qfq_addq(struct qfq_class *cl, struct mbuf *m, struct pf_mtag *t) } } if (cl->cl_sfb != NULL) - return (sfb_addq(cl->cl_sfb, &cl->cl_q, m, t)); + return (sfb_addq(cl->cl_sfb, &cl->cl_q, pkt, t)); } else if (qlen(&cl->cl_q) >= qlimit(&cl->cl_q)) { IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (CLASSQEQ_DROPPED); + return (CLASSQEQ_DROP); } #if PF_ECN - if (cl->cl_flags & QFCF_CLEARDSCP) + if (cl->cl_flags & QFCF_CLEARDSCP) { + /* not supported for non-mbuf type packets */ + VERIFY(pkt->pktsched_ptype == QP_MBUF); write_dsfield(m, t, 0); + } #endif /* PF_ECN */ - _addq(&cl->cl_q, m); - + VERIFY(pkt->pktsched_ptype == qptype(&cl->cl_q)); + _addq(&cl->cl_q, pkt->pktsched_pkt); return (0); } -static inline struct mbuf * -qfq_getq(struct qfq_class *cl) +static inline void +qfq_getq(struct qfq_class *cl, pktsched_pkt_t *pkt) { IFCQ_LOCK_ASSERT_HELD(cl->cl_qif->qif_ifq); -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - return (rio_getq(cl->cl_rio, &cl->cl_q)); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - return (red_getq(cl->cl_red, &cl->cl_q)); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - return (blue_getq(cl->cl_blue, &cl->cl_q)); - else -#endif /* CLASSQ_BLUE */ if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - return (sfb_getq(cl->cl_sfb, &cl->cl_q)); - - return (_getq(&cl->cl_q)); -} - -static inline struct mbuf * -qfq_pollq(struct qfq_class *cl) -{ - IFCQ_LOCK_ASSERT_HELD(cl->cl_qif->qif_ifq); + return (sfb_getq(cl->cl_sfb, &cl->cl_q, pkt)); - return (qhead(&cl->cl_q)); + return (pktsched_pkt_encap(pkt, qptype(&cl->cl_q), _getq(&cl->cl_q))); } static void @@ -1431,24 +1259,7 @@ qfq_purgeq(struct qfq_if *qif, struct qfq_class *cl, u_int32_t flow, if ((qlen = qlen(&cl->cl_q)) == 0) goto done; - /* become regular mutex before freeing mbufs */ IFCQ_CONVERT_LOCK(ifq); - -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_purgeq(cl->cl_rio, &cl->cl_q, flow, &cnt, &len); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_purgeq(cl->cl_red, &cl->cl_q, flow, &cnt, &len); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_purgeq(cl->cl_blue, &cl->cl_q, flow, &cnt, &len); - else -#endif /* CLASSQ_BLUE */ if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) sfb_purgeq(cl->cl_sfb, &cl->cl_q, flow, &cnt, &len); else @@ -1498,18 +1309,6 @@ qfq_updateq(struct qfq_if *qif, struct qfq_class *cl, cqev_t ev) ifclassq_ev2str(ev)); } -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - return (rio_updateq(cl->cl_rio, ev)); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - return (red_updateq(cl->cl_red, ev)); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - return (blue_updateq(cl->cl_blue, ev)); -#endif /* CLASSQ_BLUE */ if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) return (sfb_updateq(cl->cl_sfb, ev)); } @@ -1537,18 +1336,7 @@ qfq_get_class_stats(struct qfq_if *qif, u_int32_t qid, sp->qtype = qtype(&cl->cl_q); sp->qstate = qstate(&cl->cl_q); -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_getstats(cl->cl_red, &sp->red[0]); -#endif /* CLASSQ_RED */ -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_getstats(cl->cl_rio, &sp->red[0]); -#endif /* CLASSQ_RIO */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_getstats(cl->cl_blue, &sp->blue); -#endif /* CLASSQ_BLUE */ + if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) sfb_getstats(cl->cl_sfb, &sp->sfb); @@ -1603,7 +1391,8 @@ qfq_clh_to_clp(struct qfq_if *qif, u_int32_t chandle) static const char * qfq_style(struct qfq_if *qif) { - return ((qif->qif_flags & QFQIFF_ALTQ) ? "ALTQ_QFQ" : "QFQ"); +#pragma unused(qif) + return ("QFQ"); } /* @@ -1726,26 +1515,73 @@ qfq_dump_sched(struct qfq_if *qif, const char *msg) * (*ifcq_enqueue) in struct ifclassq. */ static int -qfq_enqueue_ifclassq(struct ifclassq *ifq, struct mbuf *m) +qfq_enqueue_ifclassq(struct ifclassq *ifq, void *p, classq_pkt_type_t ptype, + boolean_t *pdrop) { - u_int32_t i; + u_int32_t i = 0; + int ret; + pktsched_pkt_t pkt; + struct pf_mtag *t = NULL; IFCQ_LOCK_ASSERT_HELD(ifq); - if (!(m->m_flags & M_PKTHDR)) { - /* should not happen */ - log(LOG_ERR, "%s: packet does not have pkthdr\n", - if_name(ifq->ifcq_ifp)); - IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (ENOBUFS); + switch (ptype) { + case QP_MBUF: { + struct mbuf *m = p; + if (!(m->m_flags & M_PKTHDR)) { + /* should not happen */ + log(LOG_ERR, "%s: packet does not have pkthdr\n", + if_name(ifq->ifcq_ifp)); + IFCQ_CONVERT_LOCK(ifq); + m_freem(m); + *pdrop = TRUE; + return (ENOBUFS); + } + i = MBUF_SCIDX(mbuf_get_service_class(m)); + t = m_pftag(m); + break; + } + + + default: + VERIFY(0); + /* NOTREACHED */ } - i = MBUF_SCIDX(mbuf_get_service_class(m)); VERIFY((u_int32_t)i < IFCQ_SC_MAX); - return (qfq_enqueue(ifq->ifcq_disc, - ifq->ifcq_disc_slots[i].cl, m, m_pftag(m))); + pktsched_pkt_encap(&pkt, ptype, p); + + ret = qfq_enqueue(ifq->ifcq_disc, + ifq->ifcq_disc_slots[i].cl, &pkt, t); + + if ((ret != 0) && (ret != CLASSQEQ_SUCCESS_FC)) { + pktsched_free_pkt(&pkt); + *pdrop = TRUE; + } else { + *pdrop = FALSE; + } + + switch (ret) { + case CLASSQEQ_DROP: + ret = ENOBUFS; + break; + case CLASSQEQ_DROP_FC: + ret = EQFULL; + break; + case CLASSQEQ_DROP_SP: + ret = EQSUSPENDED; + break; + case CLASSQEQ_SUCCESS_FC: + ret = EQFULL; + break; + case CLASSQEQ_SUCCESS: + ret = 0; + break; + default: + VERIFY(0); + } + return (ret); } /* @@ -1757,10 +1593,14 @@ qfq_enqueue_ifclassq(struct ifclassq *ifq, struct mbuf *m) * CLASSQDQ_REMOVE must return the same packet if called immediately * after CLASSQDQ_POLL. */ -static struct mbuf * -qfq_dequeue_ifclassq(struct ifclassq *ifq, cqdq_op_t op) +static void * +qfq_dequeue_ifclassq(struct ifclassq *ifq, classq_pkt_type_t *ptype) { - return (qfq_dequeue(ifq->ifcq_disc, op)); + pktsched_pkt_t pkt; + bzero(&pkt, sizeof (pkt)); + qfq_dequeue(ifq->ifcq_disc, &pkt); + *ptype = pkt.pktsched_ptype; + return (pkt.pktsched_pkt); } static int @@ -1795,7 +1635,8 @@ qfq_request_ifclassq(struct ifclassq *ifq, cqrq_t req, void *arg) } int -qfq_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) +qfq_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags, + classq_pkt_type_t ptype) { struct ifnet *ifp = ifq->ifcq_ifp; struct qfq_class *cl0, *cl1, *cl2, *cl3, *cl4; @@ -1808,12 +1649,6 @@ qfq_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) VERIFY(ifq->ifcq_disc == NULL); VERIFY(ifq->ifcq_type == PKTSCHEDT_NONE); - if (flags & PKTSCHEDF_QALG_RED) - qflags |= QFCF_RED; - if (flags & PKTSCHEDF_QALG_RIO) - qflags |= QFCF_RIO; - if (flags & PKTSCHEDF_QALG_BLUE) - qflags |= QFCF_BLUE; if (flags & PKTSCHEDF_QALG_SFB) qflags |= QFCF_SFB; if (flags & PKTSCHEDF_QALG_ECN) @@ -1823,7 +1658,7 @@ qfq_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) if (flags & PKTSCHEDF_QALG_DELAYBASED) qflags |= QFCF_DELAYBASED; - qif = qfq_alloc(ifp, M_WAITOK, FALSE); + qif = qfq_alloc(ifp, M_WAITOK); if (qif == NULL) return (ENOMEM); @@ -1831,48 +1666,48 @@ qfq_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) maxlen = if_sndq_maxlen; if ((err = qfq_add_queue(qif, maxlen, 300, 1200, - qflags | QFCF_LAZY, SCIDX_BK_SYS, &cl0)) != 0) + qflags | QFCF_LAZY, SCIDX_BK_SYS, &cl0, ptype)) != 0) goto cleanup; if ((err = qfq_add_queue(qif, maxlen, 600, 1400, - qflags | QFCF_LAZY, SCIDX_BK, &cl1)) != 0) + qflags | QFCF_LAZY, SCIDX_BK, &cl1, ptype)) != 0) goto cleanup; if ((err = qfq_add_queue(qif, maxlen, 2400, 600, - qflags | QFCF_DEFAULTCLASS, SCIDX_BE, &cl2)) != 0) + qflags | QFCF_DEFAULTCLASS, SCIDX_BE, &cl2, ptype)) != 0) goto cleanup; if ((err = qfq_add_queue(qif, maxlen, 2700, 600, - qflags | QFCF_LAZY, SCIDX_RD, &cl3)) != 0) + qflags | QFCF_LAZY, SCIDX_RD, &cl3, ptype)) != 0) goto cleanup; if ((err = qfq_add_queue(qif, maxlen, 3000, 400, - qflags | QFCF_LAZY, SCIDX_OAM, &cl4)) != 0) + qflags | QFCF_LAZY, SCIDX_OAM, &cl4, ptype)) != 0) goto cleanup; if ((err = qfq_add_queue(qif, maxlen, 8000, 1000, - qflags | QFCF_LAZY, SCIDX_AV, &cl5)) != 0) + qflags | QFCF_LAZY, SCIDX_AV, &cl5, ptype)) != 0) goto cleanup; if ((err = qfq_add_queue(qif, maxlen, 15000, 1200, - qflags | QFCF_LAZY, SCIDX_RV, &cl6)) != 0) + qflags | QFCF_LAZY, SCIDX_RV, &cl6, ptype)) != 0) goto cleanup; if ((err = qfq_add_queue(qif, maxlen, 20000, 1400, - qflags | QFCF_LAZY, SCIDX_VI, &cl7)) != 0) + qflags | QFCF_LAZY, SCIDX_VI, &cl7, ptype)) != 0) goto cleanup; if ((err = qfq_add_queue(qif, maxlen, 23000, 200, - qflags | QFCF_LAZY, SCIDX_VO, &cl8)) != 0) + qflags | QFCF_LAZY, SCIDX_VO, &cl8, ptype)) != 0) goto cleanup; if ((err = qfq_add_queue(qif, maxlen, 25000, 200, - qflags, SCIDX_CTL, &cl9)) != 0) + qflags, SCIDX_CTL, &cl9, ptype)) != 0) goto cleanup; err = ifclassq_attach(ifq, PKTSCHEDT_QFQ, qif, qfq_enqueue_ifclassq, qfq_dequeue_ifclassq, NULL, - NULL, qfq_request_ifclassq); + NULL, NULL, qfq_request_ifclassq); /* cache these for faster lookup */ if (err == 0) { @@ -1958,7 +1793,6 @@ qfq_throttle(struct qfq_if *qif, cqrq_throttle_t *tr) int err = 0; IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(!(qif->qif_flags & QFQIFF_ALTQ)); if (!tr->set) { tr->level = qif->qif_throttle; @@ -2011,24 +1845,11 @@ qfq_resumeq(struct qfq_if *qif, struct qfq_class *cl) { struct ifclassq *ifq = qif->qif_ifq; int err = 0; - +#if !MACH_ASSERT +#pragma unused(ifq) +#endif IFCQ_LOCK_ASSERT_HELD(ifq); -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - err = rio_suspendq(cl->cl_rio, &cl->cl_q, FALSE); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - err = red_suspendq(cl->cl_red, &cl->cl_q, FALSE); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - err = blue_suspendq(cl->cl_blue, &cl->cl_q, FALSE); - else -#endif /* CLASSQ_BLUE */ if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) err = sfb_suspendq(cl->cl_sfb, &cl->cl_q, FALSE); @@ -2043,24 +1864,11 @@ qfq_suspendq(struct qfq_if *qif, struct qfq_class *cl) { struct ifclassq *ifq = qif->qif_ifq; int err = 0; - +#if !MACH_ASSERT +#pragma unused(ifq) +#endif IFCQ_LOCK_ASSERT_HELD(ifq); -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - err = rio_suspendq(cl->cl_rio, &cl->cl_q, TRUE); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - err = red_suspendq(cl->cl_red, &cl->cl_q, TRUE); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - err = blue_suspendq(cl->cl_blue, &cl->cl_q, TRUE); - else -#endif /* CLASSQ_BLUE */ if (q_is_sfb(&cl->cl_q)) { if (cl->cl_sfb != NULL) { err = sfb_suspendq(cl->cl_sfb, &cl->cl_q, TRUE); diff --git a/bsd/net/pktsched/pktsched_qfq.h b/bsd/net/pktsched/pktsched_qfq.h index ca3a2c4c3..15ce5a323 100644 --- a/bsd/net/pktsched/pktsched_qfq.h +++ b/bsd/net/pktsched/pktsched_qfq.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2012 Apple Inc. All rights reserved. + * Copyright (c) 2011-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -204,9 +204,6 @@ struct qfq_class { u_int32_t cl_qflags; /* class queue flags */ union { void *ptr; - struct red *red; /* RED state */ - struct rio *rio; /* RIO state */ - struct blue *blue; /* BLUE state */ struct sfb *sfb; /* SFB state */ } cl_qalg; struct qfq_if *cl_qif; /* back pointer to qif */ @@ -229,9 +226,6 @@ struct qfq_class { struct pktcntr cl_dropcnt; /* dropped packet counter */ }; -#define cl_red cl_qalg.red -#define cl_rio cl_qalg.rio -#define cl_blue cl_qalg.blue #define cl_sfb cl_qalg.sfb /* @@ -249,15 +243,11 @@ struct qfq_group { struct qfq_class **qfg_slots; }; -/* qfq_if flags */ -#define QFQIFF_ALTQ 0x1 /* configured via PF/ALTQ */ - /* * qfq interface state */ struct qfq_if { struct ifclassq *qif_ifq; /* backpointer to ifclassq */ - u_int32_t qif_flags; /* flags */ u_int32_t qif_throttle; /* throttling level */ u_int8_t qif_classes; /* # of classes in table */ u_int8_t qif_maxclasses; /* max # of classes in table */ @@ -281,19 +271,19 @@ struct qfq_if { struct if_ifclassq_stats; extern void qfq_init(void); -extern struct qfq_if *qfq_alloc(struct ifnet *, int, boolean_t); +extern struct qfq_if *qfq_alloc(struct ifnet *, int); extern int qfq_destroy(struct qfq_if *); extern void qfq_purge(struct qfq_if *); extern void qfq_event(struct qfq_if *, cqev_t); extern int qfq_add_queue(struct qfq_if *, u_int32_t, u_int32_t, u_int32_t, - u_int32_t, u_int32_t, struct qfq_class **); + u_int32_t, u_int32_t, struct qfq_class **, classq_pkt_type_t); extern int qfq_remove_queue(struct qfq_if *, u_int32_t); extern int qfq_get_class_stats(struct qfq_if *, u_int32_t, struct qfq_classstats *); -extern int qfq_enqueue(struct qfq_if *, struct qfq_class *, struct mbuf *, +extern int qfq_enqueue(struct qfq_if *, struct qfq_class *, pktsched_pkt_t *, struct pf_mtag *); -extern struct mbuf *qfq_dequeue(struct qfq_if *, cqdq_op_t); -extern int qfq_setup_ifclassq(struct ifclassq *, u_int32_t); +extern void qfq_dequeue(struct qfq_if *, pktsched_pkt_t *); +extern int qfq_setup_ifclassq(struct ifclassq *, u_int32_t, classq_pkt_type_t); extern int qfq_teardown_ifclassq(struct ifclassq *ifq); extern int qfq_getqstats_ifclassq(struct ifclassq *, u_int32_t, struct if_ifclassq_stats *); diff --git a/bsd/net/pktsched/pktsched_rmclass.c b/bsd/net/pktsched/pktsched_rmclass.c deleted file mode 100644 index 89b98e14b..000000000 --- a/bsd/net/pktsched/pktsched_rmclass.c +++ /dev/null @@ -1,1852 +0,0 @@ -/* - * Copyright (c) 2007-2012 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* $OpenBSD: altq_rmclass.c,v 1.13 2007/09/13 20:40:02 chl Exp $ */ -/* $KAME: altq_rmclass.c,v 1.10 2001/02/09 07:20:40 kjc Exp $ */ - -/* - * Copyright (c) 1991-1997 Regents of the University of California. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the Network Research - * Group at Lawrence Berkeley Laboratory. - * 4. Neither the name of the University nor of the Laboratory may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * LBL code modified by speer@eng.sun.com, May 1977. - * For questions and/or comments, please send mail to cbq@ee.lbl.gov - */ - -#include <sys/cdefs.h> - -#ident "@(#)rm_class.c 1.48 97/12/05 SMI" - -#if PKTSCHED_CBQ - -#include <sys/param.h> -#include <sys/malloc.h> -#include <sys/mbuf.h> -#include <sys/socket.h> -#include <sys/systm.h> -#include <sys/errno.h> -#include <sys/time.h> -#include <sys/kernel.h> -#include <sys/kernel_types.h> -#include <sys/syslog.h> - -#include <kern/zalloc.h> - -#include <net/if.h> -#include <net/net_osdep.h> -#include <net/pktsched/pktsched.h> -#include <net/pktsched/pktsched_rmclass.h> -#include <net/pktsched/pktsched_rmclass_debug.h> -#include <net/classq/classq_red.h> -#include <net/classq/classq_rio.h> -#include <net/classq/classq_blue.h> -#include <net/classq/classq_sfb.h> - -/* - * Local Macros - */ - -#define reset_cutoff(ifd) { ifd->cutoff_ = RM_MAXDEPTH; } - -/* - * Local routines. - */ - -static int rmc_satisfied(struct rm_class *, struct timeval *); -static void rmc_wrr_set_weights(struct rm_ifdat *); -static void rmc_depth_compute(struct rm_class *); -static void rmc_depth_recompute(rm_class_t *); - -static struct mbuf *_rmc_wrr_dequeue_next(struct rm_ifdat *, cqdq_op_t); -static struct mbuf *_rmc_prr_dequeue_next(struct rm_ifdat *, cqdq_op_t); - -static int _rmc_addq(rm_class_t *, struct mbuf *, struct pf_mtag *); -static void _rmc_dropq(rm_class_t *); -static struct mbuf *_rmc_getq(rm_class_t *); -static struct mbuf *_rmc_pollq(rm_class_t *); - -static int rmc_under_limit(struct rm_class *, struct timeval *); -static void rmc_tl_satisfied(struct rm_ifdat *, struct timeval *); -static void rmc_drop_action(struct rm_class *); -static void rmc_restart(struct rm_class *); -static void rmc_root_overlimit(rm_class_t *, rm_class_t *); - -#define RMC_ZONE_MAX 32 /* maximum elements in zone */ -#define RMC_ZONE_NAME "pktsched_cbq_cl" /* zone name (CBQ for now) */ - -static unsigned int rmc_size; /* size of zone element */ -static struct zone *rmc_zone; /* zone for rm_class */ - -void -rmclass_init(void) -{ - if (rmc_zone != NULL) - return; - - rmc_size = sizeof (struct rm_class); - rmc_zone = zinit(rmc_size, RMC_ZONE_MAX * rmc_size, 0, RMC_ZONE_NAME); - if (rmc_zone == NULL) { - panic("%s: failed allocating %s", __func__, RMC_ZONE_NAME); - /* NOTREACHED */ - } - zone_change(rmc_zone, Z_EXPAND, TRUE); - zone_change(rmc_zone, Z_CALLERACCT, TRUE); -} - -#define BORROW_OFFTIME -/* - * BORROW_OFFTIME (experimental): - * borrow the offtime of the class borrowing from. - * the reason is that when its own offtime is set, the class is unable - * to borrow much, especially when cutoff is taking effect. - * but when the borrowed class is overloaded (advidle is close to minidle), - * use the borrowing class's offtime to avoid overload. - */ -#define ADJUST_CUTOFF -/* - * ADJUST_CUTOFF (experimental): - * if no underlimit class is found due to cutoff, increase cutoff and - * retry the scheduling loop. - * also, don't invoke delay_actions while cutoff is taking effect, - * since a sleeping class won't have a chance to be scheduled in the - * next loop. - * - * now heuristics for setting the top-level variable (cutoff_) becomes: - * 1. if a packet arrives for a not-overlimit class, set cutoff - * to the depth of the class. - * 2. if cutoff is i, and a packet arrives for an overlimit class - * with an underlimit ancestor at a lower level than i (say j), - * then set cutoff to j. - * 3. at scheduling a packet, if there is no underlimit class - * due to the current cutoff level, increase cutoff by 1 and - * then try to schedule again. - */ - -/* - * rm_class_t * - * rmc_newclass(...) - Create a new resource management class at priority - * 'pri' on the interface given by 'ifd'. - * - * nsecPerByte is the data rate of the interface in nanoseconds/byte. - * E.g., 800 for a 10Mb/s ethernet. If the class gets less - * than 100% of the bandwidth, this number should be the - * 'effective' rate for the class. Let f be the - * bandwidth fraction allocated to this class, and let - * nsPerByte be the data rate of the output link in - * nanoseconds/byte. Then nsecPerByte is set to - * nsPerByte / f. E.g., 1600 (= 800 / .5) - * for a class that gets 50% of an ethernet's bandwidth. - * - * action the routine to call when the class is over limit. - * - * maxq max allowable queue size for class (in packets). - * - * parent parent class pointer. - * - * borrow class to borrow from (should be either 'parent' or null). - * - * maxidle max value allowed for class 'idle' time estimate (this - * parameter determines how large an initial burst of packets - * can be before overlimit action is invoked. - * - * offtime how long 'delay' action will delay when class goes over - * limit (this parameter determines the steady-state burst - * size when a class is running over its limit). - * - * Maxidle and offtime have to be computed from the following: If the - * average packet size is s, the bandwidth fraction allocated to this - * class is f, we want to allow b packet bursts, and the gain of the - * averaging filter is g (= 1 - 2^(-RM_FILTER_GAIN)), then: - * - * ptime = s * nsPerByte * (1 - f) / f - * maxidle = ptime * (1 - g^b) / g^b - * minidle = -ptime * (1 / (f - 1)) - * offtime = ptime * (1 + 1/(1 - g) * (1 - g^(b - 1)) / g^(b - 1) - * - * Operationally, it's convenient to specify maxidle & offtime in units - * independent of the link bandwidth so the maxidle & offtime passed to - * this routine are the above values multiplied by 8*f/(1000*nsPerByte). - * (The constant factor is a scale factor needed to make the parameters - * integers. This scaling also means that the 'unscaled' values of - * maxidle*nsecPerByte/8 and offtime*nsecPerByte/8 will be in microseconds, - * not nanoseconds.) Also note that the 'idle' filter computation keeps - * an estimate scaled upward by 2^RM_FILTER_GAIN so the passed value of - * maxidle also must be scaled upward by this value. Thus, the passed - * values for maxidle and offtime can be computed as follows: - * - * maxidle = maxidle * 2^RM_FILTER_GAIN * 8 / (1000 * nsecPerByte) - * offtime = offtime * 8 / (1000 * nsecPerByte) - * - * When USE_HRTIME is employed, then maxidle and offtime become: - * maxidle = maxilde * (8.0 / nsecPerByte); - * offtime = offtime * (8.0 / nsecPerByte); - */ -struct rm_class * -rmc_newclass(int pri, struct rm_ifdat *ifd, u_int32_t nsecPerByte, - void (*action)(rm_class_t *, rm_class_t *), u_int32_t qid, u_int32_t maxq, - struct rm_class *parent, struct rm_class *borrow, u_int32_t maxidle, - int minidle, u_int32_t offtime, int pktsize, int flags) -{ - struct ifnet *ifp; - struct ifclassq *ifq; - struct rm_class *cl; - struct rm_class *peer; - - if (nsecPerByte == 0) { - log(LOG_ERR, "%s: invalid inverse data rate\n", __func__); - return (NULL); - } - - if (pri >= RM_MAXPRIO) { - log(LOG_ERR, "%s: priority %d out of range! (max %d)\n", - __func__, pri, RM_MAXPRIO - 1); - return (NULL); - } - -#if !CLASSQ_RED - if (flags & RMCF_RED) { - log(LOG_ERR, "%s: RED not configured for CBQ!\n", __func__); - return (NULL); - } -#endif /* !CLASSQ_RED */ - -#if !CLASSQ_RIO - if (flags & RMCF_RIO) { - log(LOG_ERR, "%s: RIO not configured for CBQ!\n", __func__); - return (NULL); - } -#endif /* CLASSQ_RIO */ - -#if !CLASSQ_BLUE - if (flags & RMCF_BLUE) { - log(LOG_ERR, "%s: BLUE not configured for CBQ!\n", __func__); - return (NULL); - } -#endif /* CLASSQ_BLUE */ - - /* These are mutually exclusive */ - if ((flags & (RMCF_RED|RMCF_RIO|RMCF_BLUE|RMCF_SFB)) && - (flags & (RMCF_RED|RMCF_RIO|RMCF_BLUE|RMCF_SFB)) != RMCF_RED && - (flags & (RMCF_RED|RMCF_RIO|RMCF_BLUE|RMCF_SFB)) != RMCF_RIO && - (flags & (RMCF_RED|RMCF_RIO|RMCF_BLUE|RMCF_SFB)) != RMCF_BLUE && - (flags & (RMCF_RED|RMCF_RIO|RMCF_BLUE|RMCF_SFB)) != RMCF_SFB) { - log(LOG_ERR, "%s: RED|RIO|BLUE|SFB mutually exclusive\n", - __func__); - return (NULL); - } - - cl = zalloc(rmc_zone); - if (cl == NULL) - return (NULL); - - bzero(cl, rmc_size); - CALLOUT_INIT(&cl->callout_); - - /* - * Class initialization. - */ - cl->children_ = NULL; - cl->parent_ = parent; - cl->borrow_ = borrow; - cl->leaf_ = 1; - cl->ifdat_ = ifd; - cl->pri_ = pri; - cl->allotment_ = RM_NS_PER_SEC / nsecPerByte; /* Bytes per sec */ - cl->depth_ = 0; - cl->qthresh_ = 0; - cl->ns_per_byte_ = nsecPerByte; - - ifq = ifd->ifq_; - ifp = ifq->ifcq_ifp; - - if (maxq == 0 || maxq > IFCQ_MAXLEN(ifq)) { - maxq = IFCQ_MAXLEN(ifq); - if (maxq == 0) - maxq = DEFAULT_QLIMIT; /* use default */ - } - _qinit(&cl->q_, Q_DROPHEAD, maxq); - - cl->flags_ = flags; - - cl->minidle_ = (minidle * (int)nsecPerByte) / 8; - if (cl->minidle_ > 0) - cl->minidle_ = 0; - - cl->maxidle_ = (maxidle * nsecPerByte) / 8; - if (cl->maxidle_ == 0) - cl->maxidle_ = 1; - - cl->avgidle_ = cl->maxidle_; - cl->offtime_ = ((offtime * nsecPerByte) / 8) >> RM_FILTER_GAIN; - if (cl->offtime_ == 0) - cl->offtime_ = 1; - - cl->overlimit = action; - - if (flags & (RMCF_RED|RMCF_RIO|RMCF_BLUE|RMCF_SFB)) { - int pkttime; - - cl->qflags_ = 0; - if (flags & RMCF_ECN) { - if (flags & RMCF_BLUE) - cl->qflags_ |= BLUEF_ECN; - else if (flags & RMCF_SFB) - cl->qflags_ |= SFBF_ECN; - else if (flags & RMCF_RED) - cl->qflags_ |= REDF_ECN; - else if (flags & RMCF_RIO) - cl->qflags_ |= RIOF_ECN; - } - if (flags & RMCF_FLOWCTL) { - if (flags & RMCF_SFB) - cl->qflags_ |= SFBF_FLOWCTL; - } - if (flags & RMCF_FLOWVALVE) { - if (flags & RMCF_RED) - cl->qflags_ |= REDF_FLOWVALVE; - } - if (flags & RMCF_CLEARDSCP) { - if (flags & RMCF_RIO) - cl->qflags_ |= RIOF_CLEARDSCP; - } - pkttime = nsecPerByte * pktsize / 1000; - - /* Test for exclusivity {RED,RIO,BLUE,SFB} was done above */ -#if CLASSQ_RED - if (flags & RMCF_RED) { - cl->red_ = red_alloc(ifp, 0, 0, - qlimit(&cl->q_) * 10/100, - qlimit(&cl->q_) * 30/100, - cl->qflags_, pkttime); - if (cl->red_ != NULL) - qtype(&cl->q_) = Q_RED; - } -#endif /* CLASSQ_RED */ -#if CLASSQ_RIO - if (flags & RMCF_RIO) { - cl->rio_ = - rio_alloc(ifp, 0, NULL, cl->qflags_, pkttime); - if (cl->rio_ != NULL) - qtype(&cl->q_) = Q_RIO; - } -#endif /* CLASSQ_RIO */ -#if CLASSQ_BLUE - if (flags & RMCF_BLUE) { - cl->blue_ = blue_alloc(ifp, 0, 0, cl->qflags_); - if (cl->blue_ != NULL) - qtype(&cl->q_) = Q_BLUE; - } -#endif /* CLASSQ_BLUE */ - if (flags & RMCF_SFB) { - if (!(cl->flags_ & RMCF_LAZY)) - cl->sfb_ = sfb_alloc(ifp, qid, - qlimit(&cl->q_), cl->qflags_); - if (cl->sfb_ != NULL || (cl->flags_ & RMCF_LAZY)) - qtype(&cl->q_) = Q_SFB; - } - } - - /* - * put the class into the class tree - */ - if ((peer = ifd->active_[pri]) != NULL) { - /* find the last class at this pri */ - cl->peer_ = peer; - while (peer->peer_ != ifd->active_[pri]) - peer = peer->peer_; - peer->peer_ = cl; - } else { - ifd->active_[pri] = cl; - cl->peer_ = cl; - } - - if (cl->parent_) { - cl->next_ = parent->children_; - parent->children_ = cl; - parent->leaf_ = 0; - } - - /* - * Compute the depth of this class and its ancestors in the class - * hierarchy. - */ - rmc_depth_compute(cl); - - /* - * If CBQ's WRR is enabled, then initialize the class WRR state. - */ - if (ifd->wrr_) { - ifd->num_[pri]++; - ifd->alloc_[pri] += cl->allotment_; - rmc_wrr_set_weights(ifd); - } - return (cl); -} - -int -rmc_modclass(struct rm_class *cl, u_int32_t nsecPerByte, int maxq, - u_int32_t maxidle, int minidle, u_int32_t offtime, int pktsize) -{ -#pragma unused(pktsize) - struct rm_ifdat *ifd; - u_int32_t old_allotment; - - ifd = cl->ifdat_; - old_allotment = cl->allotment_; - - cl->allotment_ = RM_NS_PER_SEC / nsecPerByte; /* Bytes per sec */ - cl->qthresh_ = 0; - cl->ns_per_byte_ = nsecPerByte; - - qlimit(&cl->q_) = maxq; - - cl->minidle_ = (minidle * nsecPerByte) / 8; - if (cl->minidle_ > 0) - cl->minidle_ = 0; - - cl->maxidle_ = (maxidle * nsecPerByte) / 8; - if (cl->maxidle_ == 0) - cl->maxidle_ = 1; - - cl->avgidle_ = cl->maxidle_; - cl->offtime_ = ((offtime * nsecPerByte) / 8) >> RM_FILTER_GAIN; - if (cl->offtime_ == 0) - cl->offtime_ = 1; - - /* - * If CBQ's WRR is enabled, then initialize the class WRR state. - */ - if (ifd->wrr_) { - ifd->alloc_[cl->pri_] += cl->allotment_ - old_allotment; - rmc_wrr_set_weights(ifd); - } - return (0); -} - -/* - * static void - * rmc_wrr_set_weights(struct rm_ifdat *ifdat) - This function computes - * the appropriate run robin weights for the CBQ weighted round robin - * algorithm. - * - * Returns: NONE - */ - -static void -rmc_wrr_set_weights(struct rm_ifdat *ifd) -{ - int i; - struct rm_class *cl, *clh; - - for (i = 0; i < RM_MAXPRIO; i++) { - /* - * This is inverted from that of the simulator to - * maintain precision. - */ - if (ifd->num_[i] == 0) { - ifd->M_[i] = 0; - } else { - ifd->M_[i] = - ifd->alloc_[i] / (ifd->num_[i] * ifd->maxpkt_); - } - /* - * Compute the weighted allotment for each class. - * This takes the expensive div instruction out - * of the main loop for the wrr scheduling path. - * These only get recomputed when a class comes or - * goes. - */ - if (ifd->active_[i] != NULL) { - clh = cl = ifd->active_[i]; - do { - /* safe-guard for slow link or alloc_ == 0 */ - if (ifd->M_[i] == 0) { - cl->w_allotment_ = 0; - } else { - cl->w_allotment_ = - cl->allotment_ / ifd->M_[i]; - } - cl = cl->peer_; - } while ((cl != NULL) && (cl != clh)); - } - } -} - -int -rmc_get_weight(struct rm_ifdat *ifd, int pri) -{ - if ((pri >= 0) && (pri < RM_MAXPRIO)) - return (ifd->M_[pri]); - else - return (0); -} - -/* - * static void - * rmc_depth_compute(struct rm_class *cl) - This function computes the - * appropriate depth of class 'cl' and its ancestors. - * - * Returns: NONE - */ - -static void -rmc_depth_compute(struct rm_class *cl) -{ - rm_class_t *t = cl, *p; - - /* - * Recompute the depth for the branch of the tree. - */ - while (t != NULL) { - p = t->parent_; - if (p && (t->depth_ >= p->depth_)) { - p->depth_ = t->depth_ + 1; - t = p; - } else - t = NULL; - } -} - -/* - * static void - * rmc_depth_recompute(struct rm_class *cl) - This function re-computes - * the depth of the tree after a class has been deleted. - * - * Returns: NONE - */ - -static void -rmc_depth_recompute(rm_class_t *cl) -{ - rm_class_t *p, *t; - - p = cl; - while (p != NULL) { - if ((t = p->children_) == NULL) { - p->depth_ = 0; - } else { - int cdepth = 0; - - while (t != NULL) { - if (t->depth_ > cdepth) - cdepth = t->depth_; - t = t->next_; - } - - if (p->depth_ == cdepth + 1) - /* no change to this parent */ - return; - - p->depth_ = cdepth + 1; - } - - p = p->parent_; - } -} - -/* - * void - * rmc_delete_class(struct rm_ifdat *ifdat, struct rm_class *cl) - This - * function deletes a class from the link-sharing structure and frees - * all resources associated with the class. - * - * Returns: NONE - */ - -void -rmc_delete_class(struct rm_ifdat *ifd, struct rm_class *cl) -{ - struct rm_class *p, *head, *previous; - - VERIFY(cl->children_ == NULL); - - if (cl->sleeping_) - CALLOUT_STOP(&cl->callout_); - - /* - * Free packets in the packet queue. - * XXX - this may not be a desired behavior. Packets should be - * re-queued. - */ - rmc_dropall(cl); - - /* - * If the class has a parent, then remove the class from the - * class from the parent's children chain. - */ - if (cl->parent_ != NULL) { - head = cl->parent_->children_; - p = previous = head; - if (head->next_ == NULL) { - VERIFY(head == cl); - cl->parent_->children_ = NULL; - cl->parent_->leaf_ = 1; - } else while (p != NULL) { - if (p == cl) { - if (cl == head) - cl->parent_->children_ = cl->next_; - else - previous->next_ = cl->next_; - cl->next_ = NULL; - p = NULL; - } else { - previous = p; - p = p->next_; - } - } - } - - /* - * Delete class from class priority peer list. - */ - if ((p = ifd->active_[cl->pri_]) != NULL) { - /* - * If there is more than one member of this priority - * level, then look for class(cl) in the priority level. - */ - if (p != p->peer_) { - while (p->peer_ != cl) - p = p->peer_; - p->peer_ = cl->peer_; - - if (ifd->active_[cl->pri_] == cl) - ifd->active_[cl->pri_] = cl->peer_; - } else { - VERIFY(p == cl); - ifd->active_[cl->pri_] = NULL; - } - } - - /* - * Recompute the WRR weights. - */ - if (ifd->wrr_) { - ifd->alloc_[cl->pri_] -= cl->allotment_; - ifd->num_[cl->pri_]--; - rmc_wrr_set_weights(ifd); - } - - /* - * Re-compute the depth of the tree. - */ - rmc_depth_recompute(cl->parent_); - - /* - * Free the class structure. - */ - if (cl->qalg_.ptr != NULL) { -#if CLASSQ_RIO - if (q_is_rio(&cl->q_)) - rio_destroy(cl->rio_); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->q_)) - red_destroy(cl->red_); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->q_)) - blue_destroy(cl->blue_); -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->q_) && cl->sfb_ != NULL) - sfb_destroy(cl->sfb_); - cl->qalg_.ptr = NULL; - qtype(&cl->q_) = Q_DROPTAIL; - qstate(&cl->q_) = QS_RUNNING; - } - zfree(rmc_zone, cl); -} - - -/* - * int - * rmc_init(...) - Initialize the resource management data structures - * associated with the output portion of interface 'ifp'. 'ifd' is - * where the structures will be built (for backwards compatibility, the - * structures aren't kept in the ifnet struct). 'nsecPerByte' - * gives the link speed (inverse of bandwidth) in nanoseconds/byte. - * 'restart' is the driver-specific routine that the generic 'delay - * until under limit' action will call to restart output. `maxq' - * is the queue size of the 'link' & 'default' classes. 'maxqueued' - * is the maximum number of packets that the resource management - * code will allow to be queued 'downstream' (this is typically 1). - * - * Returns: 0 on success - */ - -int -rmc_init(struct ifclassq *ifq, struct rm_ifdat *ifd, u_int32_t nsecPerByte, - void (*restart)(struct ifclassq *), u_int32_t qid, int maxq, int maxqueued, - u_int32_t maxidle, int minidle, u_int32_t offtime, int flags) -{ - struct ifnet *ifp = ifq->ifcq_ifp; - int i, mtu; - - /* - * Initialize the CBQ tracing/debug facility. - */ - CBQTRACEINIT(); - - if (nsecPerByte == 0) { - log(LOG_ERR, "%s: %s: invalid inverse data rate)\n", - __func__, if_name(ifp)); - return (EINVAL); - } - - mtu = ifp->if_mtu; - if (mtu < 1) { - log(LOG_ERR, "%s: %s: invalid MTU (interface not " - "initialized?)\n", __func__, if_name(ifp)); - return (EINVAL); - } - bzero((char *)ifd, sizeof (*ifd)); - - ifd->ifq_ = ifq; - ifd->restart = restart; - ifd->maxqueued_ = maxqueued; - ifd->ns_per_byte_ = nsecPerByte; - ifd->maxpkt_ = mtu; - ifd->wrr_ = (flags & RMCF_WRR) ? 1 : 0; - ifd->efficient_ = (flags & RMCF_EFFICIENT) ? 1 : 0; -#if 1 - ifd->maxiftime_ = mtu * nsecPerByte / 1000 * 16; - if (mtu * nsecPerByte > 10 * 1000000) - ifd->maxiftime_ /= 4; -#endif - - reset_cutoff(ifd); - CBQTRACE(rmc_init, 'INIT', ifd->cutoff_); - - /* - * Initialize the CBQ's WRR state. - */ - for (i = 0; i < RM_MAXPRIO; i++) { - ifd->alloc_[i] = 0; - ifd->M_[i] = 0; - ifd->num_[i] = 0; - ifd->na_[i] = 0; - ifd->active_[i] = NULL; - } - - /* - * Initialize current packet state. - */ - ifd->qi_ = 0; - ifd->qo_ = 0; - for (i = 0; i < RM_MAXQUEUED; i++) { - ifd->class_[i] = NULL; - ifd->curlen_[i] = 0; - ifd->borrowed_[i] = NULL; - } - - /* - * Create the root class of the link-sharing structure. - */ - if ((ifd->root_ = rmc_newclass(0, ifd, nsecPerByte, - rmc_root_overlimit, qid, maxq, 0, 0, maxidle, minidle, offtime, - 0, 0)) == NULL) { - log(LOG_ERR, "rmc_init: root class not allocated\n"); - return (ENOMEM); - } - ifd->root_->depth_ = 0; - - return (0); -} - -/* - * void - * rmc_queue_packet(struct rm_class *cl, struct mbuf *m) - Add packet given by - * mbuf 'm' to queue for resource class 'cl'. This routine is called - * by a driver's if_output routine. This routine must be called with - * output packet completion interrupts locked out (to avoid racing with - * rmc_dequeue_next). - * - * Returns: 0 on successful queueing - * CLASSQEQ_DROPPED when packet drop occurs - */ -int -rmc_queue_packet(struct rm_class *cl, struct mbuf *m, struct pf_mtag *t) -{ - struct timeval now; - struct rm_ifdat *ifd = cl->ifdat_; - int cpri = cl->pri_; - int is_empty = qempty(&cl->q_); - int ret = 0; - - RM_GETTIME(now); - if (ifd->cutoff_ > 0) { - if (TV_LT(&cl->undertime_, &now)) { - if (ifd->cutoff_ > cl->depth_) - ifd->cutoff_ = cl->depth_; - CBQTRACE(rmc_queue_packet, 'ffoc', cl->depth_); - } else { - /* - * the class is overlimit. if the class has - * underlimit ancestors, set cutoff to the lowest - * depth among them. - */ - struct rm_class *borrow = cl->borrow_; - - while (borrow != NULL && - borrow->depth_ < ifd->cutoff_) { - if (TV_LT(&borrow->undertime_, &now)) { - ifd->cutoff_ = borrow->depth_; - CBQTRACE(rmc_queue_packet, 'ffob', - ifd->cutoff_); - break; - } - borrow = borrow->borrow_; - } - } - } - - ret = _rmc_addq(cl, m, t); - if (ret != 0 && - (ret == CLASSQEQ_DROPPED || ret == CLASSQEQ_DROPPED_FC || - ret == CLASSQEQ_DROPPED_SP)) { - /* failed */ - return (ret); - } - VERIFY(ret == 0 || ret == CLASSQEQ_SUCCESS_FC); - if (is_empty) { - CBQTRACE(rmc_queue_packet, 'type', cl->stats_.handle); - ifd->na_[cpri]++; - } - - if (qlen(&cl->q_) > qlimit(&cl->q_)) { - /* note: qlimit can be set to 0 or 1 */ - rmc_drop_action(cl); - return (CLASSQEQ_DROPPED); - } - return (ret); -} - -/* - * void - * rmc_tl_satisfied(struct rm_ifdat *ifd, struct timeval *now) - Check all - * classes to see if there are satified. - */ - -static void -rmc_tl_satisfied(struct rm_ifdat *ifd, struct timeval *now) -{ - int i; - rm_class_t *p, *bp; - - for (i = RM_MAXPRIO - 1; i >= 0; i--) { - if ((bp = ifd->active_[i]) != NULL) { - p = bp; - do { - if (!rmc_satisfied(p, now)) { - ifd->cutoff_ = p->depth_; - return; - } - p = p->peer_; - } while (p != bp); - } - } - - reset_cutoff(ifd); -} - -/* - * rmc_satisfied - Return 1 of the class is satisfied. O, otherwise. - */ - -static int -rmc_satisfied(struct rm_class *cl, struct timeval *now) -{ - rm_class_t *p; - - if (cl == NULL) - return (1); - if (TV_LT(now, &cl->undertime_)) - return (1); - if (cl->depth_ == 0) { - if (!cl->sleeping_ && (qlen(&cl->q_) > cl->qthresh_)) - return (0); - else - return (1); - } - if (cl->children_ != NULL) { - p = cl->children_; - while (p != NULL) { - if (!rmc_satisfied(p, now)) - return (0); - p = p->next_; - } - } - - return (1); -} - -/* - * Return 1 if class 'cl' is under limit or can borrow from a parent, - * 0 if overlimit. As a side-effect, this routine will invoke the - * class overlimit action if the class if overlimit. - */ - -static int -rmc_under_limit(struct rm_class *cl, struct timeval *now) -{ - rm_class_t *p = cl; - rm_class_t *top; - struct rm_ifdat *ifd = cl->ifdat_; - - ifd->borrowed_[ifd->qi_] = NULL; - /* - * If cl is the root class, then always return that it is - * underlimit. Otherwise, check to see if the class is underlimit. - */ - if (cl->parent_ == NULL) - return (1); - - if (cl->sleeping_) { - if (TV_LT(now, &cl->undertime_)) - return (0); - - CALLOUT_STOP(&cl->callout_); - cl->sleeping_ = 0; - cl->undertime_.tv_sec = 0; - return (1); - } - - top = NULL; - while (cl->undertime_.tv_sec && TV_LT(now, &cl->undertime_)) { - if (((cl = cl->borrow_) == NULL) || - (cl->depth_ > ifd->cutoff_)) { -#ifdef ADJUST_CUTOFF - if (cl != NULL) - /* - * cutoff is taking effect, just - * return false without calling - * the delay action. - */ - return (0); -#endif -#ifdef BORROW_OFFTIME - /* - * check if the class can borrow offtime too. - * borrow offtime from the top of the borrow - * chain if the top class is not overloaded. - */ - if (cl != NULL) { - /* - * cutoff is taking effect, use this - * class as top. - */ - top = cl; - CBQTRACE(rmc_under_limit, 'ffou', ifd->cutoff_); - } - if (top != NULL && top->avgidle_ == top->minidle_) - top = NULL; - p->overtime_ = *now; - (p->overlimit)(p, top); -#else - p->overtime_ = *now; - (p->overlimit)(p, NULL); -#endif - return (0); - } - top = cl; - } - - if (cl != p) - ifd->borrowed_[ifd->qi_] = cl; - return (1); -} - -/* - * _rmc_wrr_dequeue_next() - This is scheduler for WRR as opposed to - * Packet-by-packet round robin. - * - * The heart of the weighted round-robin scheduler, which decides which - * class next gets to send a packet. Highest priority first, then - * weighted round-robin within priorites. - * - * Each able-to-send class gets to send until its byte allocation is - * exhausted. Thus, the active pointer is only changed after a class has - * exhausted its allocation. - * - * If the scheduler finds no class that is underlimit or able to borrow, - * then the first class found that had a nonzero queue and is allowed to - * borrow gets to send. - */ - -static struct mbuf * -_rmc_wrr_dequeue_next(struct rm_ifdat *ifd, cqdq_op_t op) -{ - struct rm_class *cl = NULL, *first = NULL; - u_int32_t deficit; - int cpri; - struct mbuf *m; - struct timeval now; - - RM_GETTIME(now); - - /* - * if the driver polls the top of the queue and then removes - * the polled packet, we must return the same packet. - */ - if (op == CLASSQDQ_REMOVE && ifd->pollcache_) { - cl = ifd->pollcache_; - cpri = cl->pri_; - if (ifd->efficient_) { - /* check if this class is overlimit */ - if (cl->undertime_.tv_sec != 0 && - rmc_under_limit(cl, &now) == 0) - first = cl; - } - ifd->pollcache_ = NULL; - goto _wrr_out; - } else { - /* mode == CLASSQDQ_POLL || pollcache == NULL */ - ifd->pollcache_ = NULL; - ifd->borrowed_[ifd->qi_] = NULL; - } -#ifdef ADJUST_CUTOFF -_again: -#endif - for (cpri = RM_MAXPRIO - 1; cpri >= 0; cpri--) { - if (ifd->na_[cpri] == 0) - continue; - deficit = 0; - /* - * Loop through twice for a priority level, if some class - * was unable to send a packet the first round because - * of the weighted round-robin mechanism. - * During the second loop at this level, deficit==2. - * (This second loop is not needed if for every class, - * "M[cl->pri_])" times "cl->allotment" is greater than - * the byte size for the largest packet in the class.) - */ -_wrr_loop: - cl = ifd->active_[cpri]; - VERIFY(cl != NULL); - do { - if ((deficit < 2) && (cl->bytes_alloc_ <= 0)) - cl->bytes_alloc_ += cl->w_allotment_; - if (!qempty(&cl->q_)) { - if ((cl->undertime_.tv_sec == 0) || - rmc_under_limit(cl, &now)) { - if (cl->bytes_alloc_ > 0 || deficit > 1) - goto _wrr_out; - - /* underlimit but no alloc */ - deficit = 1; -#if 1 - ifd->borrowed_[ifd->qi_] = NULL; -#endif - } else if (first == NULL && cl->borrow_ != NULL) - first = cl; /* borrowing candidate */ - } - - cl->bytes_alloc_ = 0; - cl = cl->peer_; - } while (cl != ifd->active_[cpri]); - - if (deficit == 1) { - /* first loop found an underlimit class with deficit */ - /* Loop on same priority level, with new deficit. */ - deficit = 2; - goto _wrr_loop; - } - } - -#ifdef ADJUST_CUTOFF - /* - * no underlimit class found. if cutoff is taking effect, - * increase cutoff and try again. - */ - if (first != NULL && ifd->cutoff_ < ifd->root_->depth_) { - ifd->cutoff_++; - CBQTRACE(_rmc_wrr_dequeue_next, 'ojda', ifd->cutoff_); - goto _again; - } -#endif /* ADJUST_CUTOFF */ - /* - * If LINK_EFFICIENCY is turned on, then the first overlimit - * class we encounter will send a packet if all the classes - * of the link-sharing structure are overlimit. - */ - reset_cutoff(ifd); - CBQTRACE(_rmc_wrr_dequeue_next, 'otsr', ifd->cutoff_); - - if (!ifd->efficient_ || first == NULL) - return (NULL); - - cl = first; - cpri = cl->pri_; -#if 0 /* too time-consuming for nothing */ - if (cl->sleeping_) - CALLOUT_STOP(&cl->callout_); - cl->sleeping_ = 0; - cl->undertime_.tv_sec = 0; -#endif - ifd->borrowed_[ifd->qi_] = cl->borrow_; - ifd->cutoff_ = cl->borrow_->depth_; - - /* - * Deque the packet and do the book keeping... - */ -_wrr_out: - if (op == CLASSQDQ_REMOVE) { - m = _rmc_getq(cl); - if (m == NULL) - return (NULL); - - if (qempty(&cl->q_)) - ifd->na_[cpri]--; - - /* - * Update class statistics and link data. - */ - if (cl->bytes_alloc_ > 0) - cl->bytes_alloc_ -= m_pktlen(m); - - if ((cl->bytes_alloc_ <= 0) || first == cl) - ifd->active_[cl->pri_] = cl->peer_; - else - ifd->active_[cl->pri_] = cl; - - ifd->class_[ifd->qi_] = cl; - ifd->curlen_[ifd->qi_] = m_pktlen(m); - ifd->now_[ifd->qi_] = now; - ifd->qi_ = (ifd->qi_ + 1) % ifd->maxqueued_; - ifd->queued_++; - } else { - /* mode == ALTDQ_PPOLL */ - m = _rmc_pollq(cl); - ifd->pollcache_ = cl; - } - return (m); -} - -/* - * Dequeue & return next packet from the highest priority class that - * has a packet to send & has enough allocation to send it. This - * routine is called by a driver whenever it needs a new packet to - * output. - */ -static struct mbuf * -_rmc_prr_dequeue_next(struct rm_ifdat *ifd, cqdq_op_t op) -{ - struct mbuf *m; - int cpri; - struct rm_class *cl, *first = NULL; - struct timeval now; - - RM_GETTIME(now); - - /* - * if the driver polls the top of the queue and then removes - * the polled packet, we must return the same packet. - */ - if (op == CLASSQDQ_REMOVE && ifd->pollcache_) { - cl = ifd->pollcache_; - cpri = cl->pri_; - ifd->pollcache_ = NULL; - goto _prr_out; - } else { - /* mode == CLASSQDQ_POLL || pollcache == NULL */ - ifd->pollcache_ = NULL; - ifd->borrowed_[ifd->qi_] = NULL; - } -#ifdef ADJUST_CUTOFF -_again: -#endif - for (cpri = RM_MAXPRIO - 1; cpri >= 0; cpri--) { - if (ifd->na_[cpri] == 0) - continue; - cl = ifd->active_[cpri]; - VERIFY(cl != NULL); - do { - if (!qempty(&cl->q_)) { - if ((cl->undertime_.tv_sec == 0) || - rmc_under_limit(cl, &now)) - goto _prr_out; - if (first == NULL && cl->borrow_ != NULL) - first = cl; - } - cl = cl->peer_; - } while (cl != ifd->active_[cpri]); - } - -#ifdef ADJUST_CUTOFF - /* - * no underlimit class found. if cutoff is taking effect, increase - * cutoff and try again. - */ - if (first != NULL && ifd->cutoff_ < ifd->root_->depth_) { - ifd->cutoff_++; - goto _again; - } -#endif /* ADJUST_CUTOFF */ - /* - * If LINK_EFFICIENCY is turned on, then the first overlimit - * class we encounter will send a packet if all the classes - * of the link-sharing structure are overlimit. - */ - reset_cutoff(ifd); - if (!ifd->efficient_ || first == NULL) - return (NULL); - - cl = first; - cpri = cl->pri_; -#if 0 /* too time-consuming for nothing */ - if (cl->sleeping_) - CALLOUT_STOP(&cl->callout_); - cl->sleeping_ = 0; - cl->undertime_.tv_sec = 0; -#endif - ifd->borrowed_[ifd->qi_] = cl->borrow_; - ifd->cutoff_ = cl->borrow_->depth_; - - /* - * Deque the packet and do the book keeping... - */ -_prr_out: - if (op == CLASSQDQ_REMOVE) { - m = _rmc_getq(cl); - if (m == NULL) - return (NULL); - - if (qempty(&cl->q_)) - ifd->na_[cpri]--; - - ifd->active_[cpri] = cl->peer_; - - ifd->class_[ifd->qi_] = cl; - ifd->curlen_[ifd->qi_] = m_pktlen(m); - ifd->now_[ifd->qi_] = now; - ifd->qi_ = (ifd->qi_ + 1) % ifd->maxqueued_; - ifd->queued_++; - } else { - /* mode == CLASSQDQ_POLL */ - m = _rmc_pollq(cl); - ifd->pollcache_ = cl; - } - return (m); -} - -/* - * struct mbuf * - * rmc_dequeue_next(struct rm_ifdat *ifd, struct timeval *now) - this function - * is invoked by the packet driver to get the next packet to be - * dequeued and output on the link. If WRR is enabled, then the - * WRR dequeue next routine will determine the next packet to sent. - * Otherwise, packet-by-packet round robin is invoked. - * - * Returns: NULL, if a packet is not available or if all - * classes are overlimit. - * - * Otherwise, Pointer to the next packet. - */ - -struct mbuf * -rmc_dequeue_next(struct rm_ifdat *ifd, cqdq_op_t mode) -{ - if (ifd->queued_ >= ifd->maxqueued_) - return (NULL); - else if (ifd->wrr_) - return (_rmc_wrr_dequeue_next(ifd, mode)); - else - return (_rmc_prr_dequeue_next(ifd, mode)); -} - -/* - * Update the utilization estimate for the packet that just completed. - * The packet's class & the parent(s) of that class all get their - * estimators updated. This routine is called by the driver's output- - * packet-completion interrupt service routine. - */ - -/* - * a macro to approximate "divide by 1000" that gives 0.000999, - * if a value has enough effective digits. - * (on pentium, mul takes 9 cycles but div takes 46!) - */ -#define NSEC_TO_USEC(t) (((t) >> 10) + ((t) >> 16) + ((t) >> 17)) -void -rmc_update_class_util(struct rm_ifdat *ifd) -{ - int idle, avgidle, pktlen; - int pkt_time, tidle; - rm_class_t *cl, *borrowed; - rm_class_t *borrows; - struct timeval *nowp; - - /* - * Get the most recent completed class. - */ - if ((cl = ifd->class_[ifd->qo_]) == NULL) - return; - - pktlen = ifd->curlen_[ifd->qo_]; - borrowed = ifd->borrowed_[ifd->qo_]; - borrows = borrowed; - - PKTCNTR_ADD(&cl->stats_.xmit_cnt, 1, pktlen); - - /* - * Run estimator on class and its ancestors. - */ - /* - * rm_update_class_util is designed to be called when the - * transfer is completed from a xmit complete interrupt, - * but most drivers don't implement an upcall for that. - * so, just use estimated completion time. - * as a result, ifd->qi_ and ifd->qo_ are always synced. - */ - nowp = &ifd->now_[ifd->qo_]; - /* get pkt_time (for link) in usec */ -#if 1 /* use approximation */ - pkt_time = ifd->curlen_[ifd->qo_] * ifd->ns_per_byte_; - pkt_time = NSEC_TO_USEC(pkt_time); -#else - pkt_time = ifd->curlen_[ifd->qo_] * ifd->ns_per_byte_ / 1000; -#endif -#if 1 /* ALTQ4PPP */ - if (TV_LT(nowp, &ifd->ifnow_)) { - int iftime; - - /* - * make sure the estimated completion time does not go - * too far. it can happen when the link layer supports - * data compression or the interface speed is set to - * a much lower value. - */ - TV_DELTA(&ifd->ifnow_, nowp, iftime); - if (iftime+pkt_time < ifd->maxiftime_) { - TV_ADD_DELTA(&ifd->ifnow_, pkt_time, &ifd->ifnow_); - } else { - TV_ADD_DELTA(nowp, ifd->maxiftime_, &ifd->ifnow_); - } - } else { - TV_ADD_DELTA(nowp, pkt_time, &ifd->ifnow_); - } -#else - if (TV_LT(nowp, &ifd->ifnow_)) { - TV_ADD_DELTA(&ifd->ifnow_, pkt_time, &ifd->ifnow_); - } else { - TV_ADD_DELTA(nowp, pkt_time, &ifd->ifnow_); - } -#endif - - while (cl != NULL) { - TV_DELTA(&ifd->ifnow_, &cl->last_, idle); - if (idle >= 2000000) - /* - * this class is idle enough, reset avgidle. - * (TV_DELTA returns 2000000 us when delta is large.) - */ - cl->avgidle_ = cl->maxidle_; - - /* get pkt_time (for class) in usec */ -#if 1 /* use approximation */ - pkt_time = pktlen * cl->ns_per_byte_; - pkt_time = NSEC_TO_USEC(pkt_time); -#else - pkt_time = pktlen * cl->ns_per_byte_ / 1000; -#endif - idle -= pkt_time; - - avgidle = cl->avgidle_; - avgidle += idle - (avgidle >> RM_FILTER_GAIN); - cl->avgidle_ = avgidle; - - /* Are we overlimit ? */ - if (avgidle <= 0) { - CBQTRACE(rmc_update_class_util, 'milo', - cl->stats_.handle); - /* - * need some lower bound for avgidle, otherwise - * a borrowing class gets unbounded penalty. - */ - if (avgidle < cl->minidle_) - avgidle = cl->avgidle_ = cl->minidle_; - - /* set next idle to make avgidle 0 */ - tidle = pkt_time + - (((1 - RM_POWER) * avgidle) >> RM_FILTER_GAIN); - TV_ADD_DELTA(nowp, tidle, &cl->undertime_); - ++cl->stats_.over; - } else { - cl->avgidle_ = - (avgidle > cl->maxidle_) ? cl->maxidle_ : avgidle; - cl->undertime_.tv_sec = 0; - if (cl->sleeping_) { - CALLOUT_STOP(&cl->callout_); - cl->sleeping_ = 0; - } - } - - if (borrows != NULL) { - if (borrows != cl) - ++cl->stats_.borrows; - else - borrows = NULL; - } - cl->last_ = ifd->ifnow_; - cl->last_pkttime_ = pkt_time; - -#if 1 - if (cl->parent_ == NULL) { - /* take stats of root class */ - PKTCNTR_ADD(&cl->stats_.xmit_cnt, 1, pktlen); - } -#endif - - cl = cl->parent_; - } - - /* - * Check to see if cutoff needs to set to a new level. - */ - cl = ifd->class_[ifd->qo_]; - if (borrowed && (ifd->cutoff_ >= borrowed->depth_)) { - if ((qlen(&cl->q_) <= 0) || - TV_LT(nowp, &borrowed->undertime_)) { - rmc_tl_satisfied(ifd, nowp); - CBQTRACE(rmc_update_class_util, 'broe', ifd->cutoff_); - } else { - ifd->cutoff_ = borrowed->depth_; - CBQTRACE(rmc_update_class_util, 'ffob', - borrowed->depth_); - } - } - - /* - * Release class slot - */ - ifd->borrowed_[ifd->qo_] = NULL; - ifd->class_[ifd->qo_] = NULL; - ifd->qo_ = (ifd->qo_ + 1) % ifd->maxqueued_; - ifd->queued_--; -} - -/* - * void - * rmc_drop_action(struct rm_class *cl) - Generic (not protocol-specific) - * over-limit action routines. These get invoked by rmc_under_limit() - * if a class with packets to send if over its bandwidth limit & can't - * borrow from a parent class. - * - * Returns: NONE - */ - -static void -rmc_drop_action(struct rm_class *cl) -{ - struct rm_ifdat *ifd = cl->ifdat_; - - VERIFY(qlen(&cl->q_) > 0); - IFCQ_CONVERT_LOCK(ifd->ifq_); - _rmc_dropq(cl); - if (qempty(&cl->q_)) - ifd->na_[cl->pri_]--; -} - -void -rmc_drop(struct rm_class *cl, u_int32_t flow, u_int32_t *packets, - u_int32_t *bytes) -{ - struct rm_ifdat *ifd = cl->ifdat_; - struct ifclassq *ifq = ifd->ifq_; - u_int32_t pkt = 0, len = 0, qlen; - - if ((qlen = qlen(&cl->q_)) != 0) { - IFCQ_CONVERT_LOCK(ifq); -#if CLASSQ_RIO - if (q_is_rio(&cl->q_)) - rio_purgeq(cl->rio_, &cl->q_, flow, &pkt, &len); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->q_)) - red_purgeq(cl->red_, &cl->q_, flow, &pkt, &len); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->q_)) - blue_purgeq(cl->blue_, &cl->q_, flow, &pkt, &len); - else -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->q_) && cl->sfb_ != NULL) - sfb_purgeq(cl->sfb_, &cl->q_, flow, &pkt, &len); - else - _flushq_flow(&cl->q_, flow, &pkt, &len); - - if (pkt > 0) { - VERIFY(qlen(&cl->q_) == (qlen - pkt)); - - PKTCNTR_ADD(&cl->stats_.drop_cnt, pkt, len); - IFCQ_DROP_ADD(ifq, pkt, len); - - VERIFY(((signed)IFCQ_LEN(ifq) - pkt) >= 0); - IFCQ_LEN(ifq) -= pkt; - - if (qempty(&cl->q_)) - ifd->na_[cl->pri_]--; - } - } - if (packets != NULL) - *packets = pkt; - if (bytes != NULL) - *bytes = len; -} - -void -rmc_dropall(struct rm_class *cl) -{ - rmc_drop(cl, 0, NULL, NULL); -} - -/* - * void - * rmc_delay_action(struct rm_class *cl) - This function is the generic CBQ - * delay action routine. It is invoked via rmc_under_limit when the - * packet is discoverd to be overlimit. - * - * If the delay action is result of borrow class being overlimit, then - * delay for the offtime of the borrowing class that is overlimit. - * - * Returns: NONE - */ - -void -rmc_delay_action(struct rm_class *cl, struct rm_class *borrow) -{ - int ndelay, t, extradelay; - - cl->stats_.overactions++; - TV_DELTA(&cl->undertime_, &cl->overtime_, ndelay); -#ifndef BORROW_OFFTIME - ndelay += cl->offtime_; -#endif - - if (!cl->sleeping_) { - CBQTRACE(rmc_delay_action, 'yled', cl->stats_.handle); -#ifdef BORROW_OFFTIME - if (borrow != NULL) - extradelay = borrow->offtime_; - else -#endif - extradelay = cl->offtime_; - - /* - * XXX recalculate suspend time: - * current undertime is (tidle + pkt_time) calculated - * from the last transmission. - * tidle: time required to bring avgidle back to 0 - * pkt_time: target waiting time for this class - * we need to replace pkt_time by offtime - */ - extradelay -= cl->last_pkttime_; - if (extradelay > 0) { - TV_ADD_DELTA(&cl->undertime_, extradelay, - &cl->undertime_); - ndelay += extradelay; - } - - cl->sleeping_ = 1; - cl->stats_.delays++; - - /* - * Since packets are phased randomly with respect to the - * clock, 1 tick (the next clock tick) can be an arbitrarily - * short time so we have to wait for at least two ticks. - * NOTE: If there's no other traffic, we need the timer as - * a 'backstop' to restart this class. - */ - if (ndelay > tick * 2) { - /* - * FreeBSD rounds up the tick; - * other BSDs round down the tick. - */ - t = hzto(&cl->undertime_) + 1; - } else { - t = 2; - } - CALLOUT_RESET(&cl->callout_, t, - (timeout_t *)rmc_restart, (caddr_t)cl); - } -} - -/* - * void - * rmc_restart() - is just a helper routine for rmc_delay_action -- it is - * called by the system timer code & is responsible checking if the - * class is still sleeping (it might have been restarted as a side - * effect of the queue scan on a packet arrival) and, if so, restarting - * output for the class. Inspecting the class state & restarting output - * require locking the class structure. In general the driver is - * responsible for locking but this is the only routine that is not - * called directly or indirectly from the interface driver so it has - * know about system locking conventions. - * - * Returns: NONE - */ - -static void -rmc_restart(struct rm_class *cl) -{ - struct rm_ifdat *ifd = cl->ifdat_; - - if (cl->sleeping_) { - cl->sleeping_ = 0; - cl->undertime_.tv_sec = 0; - - if (ifd->queued_ < ifd->maxqueued_ && ifd->restart != NULL) { - CBQTRACE(rmc_restart, 'trts', cl->stats_.handle); - (ifd->restart)(ifd->ifq_); - } - } -} - -/* - * void - * rmc_root_overlimit(struct rm_class *cl) - This the generic overlimit - * handling routine for the root class of the link sharing structure. - * - * Returns: NONE - */ -static void -rmc_root_overlimit(struct rm_class *cl, - struct rm_class *borrow) -{ -#pragma unused(cl, borrow) - panic("rmc_root_overlimit"); -} - -/* - * Packet Queue handling routines. Eventually, this is to localize the - * effects on the code whether queues are red queues or droptail - * queues. - */ - -static int -_rmc_addq(rm_class_t *cl, struct mbuf *m, struct pf_mtag *t) -{ -#if CLASSQ_RIO - if (q_is_rio(&cl->q_)) - return (rio_addq(cl->rio_, &cl->q_, m, t)); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->q_)) - return (red_addq(cl->red_, &cl->q_, m, t)); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->q_)) - return (blue_addq(cl->blue_, &cl->q_, m, t)); - else -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->q_)) { - if (cl->sfb_ == NULL) { - struct ifclassq *ifq = cl->ifdat_->ifq_; - struct ifnet *ifp = ifq->ifcq_ifp; - - VERIFY(cl->flags_ & RMCF_LAZY); - IFCQ_CONVERT_LOCK(ifq); - - cl->sfb_ = sfb_alloc(ifp, cl->stats_.handle, - qlimit(&cl->q_), cl->qflags_); - if (cl->sfb_ == NULL) { - /* fall back to droptail */ - qtype(&cl->q_) = Q_DROPTAIL; - cl->flags_ &= ~RMCF_SFB; - cl->qflags_ &= ~(SFBF_ECN | SFBF_FLOWCTL); - - log(LOG_ERR, "%s: CBQ SFB lazy allocation " - "failed for qid=%d pri=%d, falling back " - "to DROPTAIL\n", if_name(ifp), - cl->stats_.handle, cl->pri_); - } - } - if (cl->sfb_ != NULL) - return (sfb_addq(cl->sfb_, &cl->q_, m, t)); - } -#if PF_ECN - else if (cl->flags_ & RMCF_CLEARDSCP) - write_dsfield(m, t, 0); -#endif /* PF_ECN */ - - /* test for qlen > qlimit is done by caller */ - _addq(&cl->q_, m); - return (0); -} - -/* note: _rmc_dropq is not called for red */ -static void -_rmc_dropq(rm_class_t *cl) -{ - struct mbuf *m; - - if ((m = _rmc_getq(cl)) != NULL) - m_freem(m); -} - -static struct mbuf * -_rmc_getq(rm_class_t *cl) -{ -#if CLASSQ_RIO - if (q_is_rio(&cl->q_)) - return (rio_getq(cl->rio_, &cl->q_)); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->q_)) - return (red_getq(cl->red_, &cl->q_)); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->q_)) - return (blue_getq(cl->blue_, &cl->q_)); - else -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->q_) && cl->sfb_ != NULL) - return (sfb_getq(cl->sfb_, &cl->q_)); - - return (_getq(&cl->q_)); -} - -static struct mbuf * -_rmc_pollq(rm_class_t *cl) -{ - return (qhead(&cl->q_)); -} - -void -rmc_updateq(rm_class_t *cl, cqev_t ev) -{ -#if CLASSQ_RIO - if (q_is_rio(&cl->q_)) - return (rio_updateq(cl->rio_, ev)); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->q_)) - return (red_updateq(cl->red_, ev)); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->q_)) - return (blue_updateq(cl->blue_, ev)); -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->q_) && cl->sfb_ != NULL) - return (sfb_updateq(cl->sfb_, ev)); -} - -#ifdef CBQ_TRACE - -struct cbqtrace cbqtrace_buffer[NCBQTRACE+1]; -struct cbqtrace *cbqtrace_ptr = NULL; -int cbqtrace_count; - -/* - * DDB hook to trace cbq events: - * the last 1024 events are held in a circular buffer. - * use "call cbqtrace_dump(N)" to display 20 events from Nth event. - */ -void cbqtrace_dump(int); -static char *rmc_funcname(void *); - -static struct rmc_funcs { - void *func; - char *name; -} rmc_funcs[] = -{ - rmc_init, "rmc_init", - rmc_queue_packet, "rmc_queue_packet", - rmc_under_limit, "rmc_under_limit", - rmc_update_class_util, "rmc_update_class_util", - rmc_delay_action, "rmc_delay_action", - rmc_restart, "rmc_restart", - _rmc_wrr_dequeue_next, "_rmc_wrr_dequeue_next", - NULL, NULL -}; - -static char * -rmc_funcname(void *func) -{ - struct rmc_funcs *fp; - - for (fp = rmc_funcs; fp->func != NULL; fp++) - if (fp->func == func) - return (fp->name); - return ("unknown"); -} - -void -cbqtrace_dump(int counter) -{ - int i, *p; - char *cp; - - counter = counter % NCBQTRACE; - p = (int *)&cbqtrace_buffer[counter]; - - for (i = 0; i < 20; i++) { - log(LOG_DEBUG, "[0x%x] ", *p++); - log(LOG_DEBUG, "%s: ", rmc_funcname((void *)*p++)); - cp = (char *)p++; - log(LOG_DEBUG, "%c%c%c%c: ", cp[0], cp[1], cp[2], cp[3]); - log(LOG_DEBUG, "%d\n", *p++); - - if (p >= (int *)&cbqtrace_buffer[NCBQTRACE]) - p = (int *)cbqtrace_buffer; - } -} -#endif /* CBQ_TRACE */ -#endif /* PKTSCHED_CBQ */ diff --git a/bsd/net/pktsched/pktsched_rmclass.h b/bsd/net/pktsched/pktsched_rmclass.h index d5f6b13b2..b467fa835 100644 --- a/bsd/net/pktsched/pktsched_rmclass.h +++ b/bsd/net/pktsched/pktsched_rmclass.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 Apple Inc. All rights reserved. + * Copyright (c) 2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -90,225 +90,6 @@ extern "C" { #define RMCF_BLUE 0x10000 /* use BLUE */ #define RMCF_SFB 0x20000 /* use SFB */ #define RMCF_FLOWCTL 0x40000 /* enable flow control advisories */ -#ifdef BSD_KERNEL_PRIVATE -#define RMCF_LAZY 0x10000000 /* on-demand resource allocation */ - -typedef struct rm_ifdat rm_ifdat_t; -typedef struct rm_class rm_class_t; - -struct red; -struct rio; -struct blue; -struct sfb; - -/* - * Macros for dealing with time values. We assume all times are - * 'timevals'. `microuptime' is used to get the best available clock - * resolution. If `microuptime' *doesn't* return a value that's about - * ten times smaller than the average packet time on the fastest - * link that will use these routines, a slightly different clock - * scheme than this one should be used. - * (Bias due to truncation error in this scheme will overestimate utilization - * and discriminate against high bandwidth classes. To remove this bias an - * integrator needs to be added. The simplest integrator uses a history of - * 10 * avg.packet.time / min.tick.time packet completion entries. This is - * straight forward to add but we don't want to pay the extra memory - * traffic to maintain it if it's not necessary (occasionally a vendor - * accidentally builds a workstation with a decent clock - e.g., Sun & HP).) - */ - -#define RM_GETTIME(now) microuptime(&now) - -#define TV_LT(a, b) (((a)->tv_sec < (b)->tv_sec) || \ - (((a)->tv_usec < (b)->tv_usec) && ((a)->tv_sec <= (b)->tv_sec))) - -#define TV_DELTA(a, b, delta) { \ - int xxs; \ - \ - delta = (a)->tv_usec - (b)->tv_usec; \ - if ((xxs = (a)->tv_sec - (b)->tv_sec)) { \ - switch (xxs) { \ - default: \ - /* \ - * if (xxs < 0) \ - * printf("rm_class: bogus time values\n"); \ - */ \ - delta = 0; \ - /* fall through */ \ - case 2: \ - delta += 1000000; \ - /* fall through */ \ - case 1: \ - delta += 1000000; \ - break; \ - } \ - } \ -} - -#define TV_ADD_DELTA(a, delta, res) { \ - int xxus = (a)->tv_usec + (delta); \ - \ - (res)->tv_sec = (a)->tv_sec; \ - while (xxus >= 1000000) { \ - ++((res)->tv_sec); \ - xxus -= 1000000; \ - } \ - (res)->tv_usec = xxus; \ -} - -#define RM_TIMEOUT 2 /* 1 Clock tick. */ - -#if 1 -#define RM_MAXQUEUED 1 /* this isn't used in ALTQ/CBQ */ -#else -#define RM_MAXQUEUED 16 /* Max number of packets downstream of CBQ */ -#endif -#define RM_MAXQUEUE 64 /* Max queue length */ -#define RM_FILTER_GAIN 5 /* log2 of gain, e.g., 5 => 31/32 */ -#define RM_POWER (1 << RM_FILTER_GAIN) -#define RM_MAXDEPTH 32 -#define RM_NS_PER_SEC (1000000000) - -typedef struct _rm_class_stats_ { - u_int32_t handle; - u_int32_t depth; - - struct pktcntr xmit_cnt; /* packets sent in this class */ - struct pktcntr drop_cnt; /* dropped packets */ - u_int32_t over; /* # times went over limit */ - u_int32_t borrows; /* # times tried to borrow */ - u_int32_t overactions; /* # times invoked overlimit action */ - u_int32_t delays; /* # times invoked delay actions */ -} rm_class_stats_t; - -/* - * CBQ Class state structure - */ -struct rm_class { - class_queue_t q_; /* Queue of packets */ - rm_ifdat_t *ifdat_; - int pri_; /* Class priority. */ - int depth_; /* Class depth */ - u_int32_t ns_per_byte_; /* NanoSeconds per byte. */ - u_int32_t maxrate_; /* Bytes per second for this class. */ - u_int32_t allotment_; /* Fraction of link bandwidth. */ - u_int32_t w_allotment_; /* Weighted allotment for WRR */ - int bytes_alloc_; /* Allocation for round of WRR */ - - int avgidle_; - int maxidle_; - int minidle_; - int offtime_; - int sleeping_; /* != 0 if delaying */ - u_int32_t qthresh_; /* Threshold for formal link sharing */ - int leaf_; /* Note whether leaf class or not */ - - rm_class_t *children_; /* Children of this class */ - rm_class_t *next_; /* Next pointer, used if child */ - - rm_class_t *peer_; /* Peer class */ - rm_class_t *borrow_; /* Borrow class */ - rm_class_t *parent_; /* Parent class */ - - void (*overlimit)(struct rm_class *, struct rm_class *); - void (*drop)(struct rm_class *); /* Class drop action. */ - - union { - void *ptr; - struct red *red; /* RED state */ - struct rio *rio; /* RIO state */ - struct blue *blue; /* BLUE state */ - struct sfb *sfb; /* SFB state */ - } qalg_; - int flags_; - u_int32_t qflags_; - - int last_pkttime_; /* saved pkt_time */ - struct timeval undertime_; /* time can next send */ - struct timeval last_; /* time last packet sent */ - struct timeval overtime_; - struct callout callout_; /* for timeout() calls */ - - rm_class_stats_t stats_; /* Class Statistics */ -}; - -#define red_ qalg_.red -#define rio_ qalg_.rio -#define blue_ qalg_.blue -#define sfb_ qalg_.sfb - -/* - * CBQ Interface state - */ -struct rm_ifdat { - int queued_; /* # pkts queued downstream */ - int efficient_; /* Link Efficency bit */ - int wrr_; /* Enable Weighted Round-Robin */ - u_long ns_per_byte_; /* Link byte speed. */ - int maxqueued_; /* Max packets to queue */ - int maxpkt_; /* Max packet size. */ - int qi_; /* In/out pointers for downstream */ - int qo_; /* packets */ - - /* - * Active class state and WRR state. - */ - rm_class_t *active_[RM_MAXPRIO]; /* Active cl's in each pri */ - int na_[RM_MAXPRIO]; /* # of active cl's in a pri */ - int num_[RM_MAXPRIO]; /* # of cl's per pri */ - int alloc_[RM_MAXPRIO]; /* Byte Allocation */ - u_long M_[RM_MAXPRIO]; /* WRR weights. */ - - /* - * Network Interface/Solaris Queue state pointer. - */ - struct ifclassq *ifq_; - rm_class_t *default_; /* Default Pkt class, BE */ - rm_class_t *root_; /* Root Link class. */ - rm_class_t *ctl_; /* Control Traffic class. */ - void (*restart)(struct ifclassq *); /* Restart routine. */ - - /* - * Current packet downstream packet state and dynamic state. - */ - rm_class_t *borrowed_[RM_MAXQUEUED]; /* Class borrowed last */ - rm_class_t *class_[RM_MAXQUEUED]; /* class sending */ - int curlen_[RM_MAXQUEUED]; /* Current pktlen */ - struct timeval now_[RM_MAXQUEUED]; /* Current packet time */ - int is_overlimit_[RM_MAXQUEUED]; /* Current packet time */ - - int cutoff_; /* Cut-off depth for borrowing */ - - struct timeval ifnow_; /* expected xmit completion time */ -#if 1 /* ALTQ4PPP */ - int maxiftime_; /* max delay inside interface */ -#endif - rm_class_t *pollcache_; /* cached rm_class by poll operation */ -}; - -#define RMC_IS_A_PARENT_CLASS(cl) ((cl)->children_ != NULL) - -extern void rmclass_init(void); -extern rm_class_t *rmc_newclass(int, struct rm_ifdat *, u_int32_t, - void (*)(struct rm_class *, struct rm_class *), u_int32_t, - u_int32_t, struct rm_class *, struct rm_class *, - u_int32_t, int, u_int32_t, int, int); -extern void rmc_delete_class(struct rm_ifdat *, struct rm_class *); -extern int rmc_modclass(struct rm_class *, u_int32_t, int, u_int32_t, - int, u_int32_t, int); -extern int rmc_init(struct ifclassq *, struct rm_ifdat *, u_int32_t, - void (*)(struct ifclassq *), u_int32_t, int, int, u_int32_t, - int, u_int32_t, int); -extern int rmc_queue_packet(struct rm_class *, struct mbuf *, struct pf_mtag *); -extern struct mbuf *rmc_dequeue_next(struct rm_ifdat *, cqdq_op_t); -extern void rmc_update_class_util(struct rm_ifdat *); -extern void rmc_delay_action(struct rm_class *, struct rm_class *); -extern void rmc_drop(struct rm_class *, u_int32_t, u_int32_t *, u_int32_t *); -extern void rmc_dropall(struct rm_class *); -extern int rmc_get_weight(struct rm_ifdat *, int); -extern void rmc_updateq(struct rm_class *, cqev_t); - -#endif /* BSD_KERNEL_PRIVATE */ #ifdef __cplusplus } diff --git a/bsd/net/pktsched/pktsched_rmclass_debug.h b/bsd/net/pktsched/pktsched_rmclass_debug.h deleted file mode 100644 index dd3f364f5..000000000 --- a/bsd/net/pktsched/pktsched_rmclass_debug.h +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright (c) 2011 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -/* $NetBSD: altq_rmclass_debug.h,v 1.7 2006/10/12 19:59:08 peter Exp $ */ -/* $KAME: altq_rmclass_debug.h,v 1.3 2002/11/29 04:36:24 kjc Exp $ */ - -/* - * Copyright (c) Sun Microsystems, Inc. 1998 All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the SMCC Technology - * Development Group at Sun Microsystems, Inc. - * - * 4. The name of the Sun Microsystems, Inc nor may not be used to endorse or - * promote products derived from this software without specific prior - * written permission. - * - * SUN MICROSYSTEMS DOES NOT CLAIM MERCHANTABILITY OF THIS SOFTWARE OR THE - * SUITABILITY OF THIS SOFTWARE FOR ANY PARTICULAR PURPOSE. The software is - * provided "as is" without express or implied warranty of any kind. - * - * These notices must be retained in any copies of any part of this software. - */ - -#ifndef _NET_PKTSCHED_PKTSCHED_RMCLASS_DEBUG_H_ -#define _NET_PKTSCHED_PKTSCHED_RMCLASS_DEBUG_H_ - -/* #pragma ident "@(#)rm_class_debug.h 1.7 98/05/04 SMI" */ - -/* - * Cbq debugging macros - */ - -#ifdef __cplusplus -extern "C" { -#endif - -#ifdef BSD_KERNEL_PRIVATE - -#ifdef CBQ_TRACE -#ifndef NCBQTRACE -#define NCBQTRACE (16 * 1024) -#endif - -/* - * To view the trace output, using adb, type: - * adb -k /dev/ksyms /dev/mem <cr>, then type - * cbqtrace_count/D to get the count, then type - * cbqtrace_buffer,0tcount/Dp4C" "Xn - * This will dump the trace buffer from 0 to count. - */ -/* - * in ALTQ, "call cbqtrace_dump(N)" from DDB to display 20 events - * from Nth event in the circular buffer. - */ - -struct cbqtrace { - int count; - int function; /* address of function */ - int trace_action; /* descriptive 4 characters */ - int object; /* object operated on */ -}; - -extern struct cbqtrace cbqtrace_buffer[]; -extern struct cbqtrace *cbqtrace_ptr; -extern int cbqtrace_count; - -#define CBQTRACEINIT() { \ - if (cbqtrace_ptr == NULL) \ - cbqtrace_ptr = cbqtrace_buffer; \ - else { \ - cbqtrace_ptr = cbqtrace_buffer; \ - bzero((void *)cbqtrace_ptr, sizeof (cbqtrace_buffer)); \ - cbqtrace_count = 0; \ - } \ -} - -#define CBQTRACE(func, act, obj) { \ - int *_p = &cbqtrace_ptr->count; \ - *_p++ = ++cbqtrace_count; \ - *_p++ = (int)(func); \ - *_p++ = (int)(act); \ - *_p++ = (int)(obj); \ - if ((struct cbqtrace *)(void *)_p >= &cbqtrace_buffer[NCBQTRACE]) \ - cbqtrace_ptr = cbqtrace_buffer; \ - else \ - cbqtrace_ptr = (struct cbqtrace *)(void *)_p; \ - } -#else - -/* If no tracing, define no-ops */ -#define CBQTRACEINIT() -#define CBQTRACE(a, b, c) - -#endif /* !CBQ_TRACE */ - -#endif /* BSD_KERNEL_PRIVATE */ - -#ifdef __cplusplus -} -#endif - -#endif /* _NET_PKTSCHED_PKTSCHED_RMCLASS_DEBUG_H_ */ diff --git a/bsd/net/pktsched/pktsched_tcq.c b/bsd/net/pktsched/pktsched_tcq.c index d3e64b5e1..d83fbb253 100644 --- a/bsd/net/pktsched/pktsched_tcq.c +++ b/bsd/net/pktsched/pktsched_tcq.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2012 Apple Inc. All rights reserved. + * Copyright (c) 2011-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -47,22 +47,23 @@ #include <net/pktsched/pktsched_tcq.h> #include <netinet/in.h> + /* * function prototypes */ -static int tcq_enqueue_ifclassq(struct ifclassq *, struct mbuf *); -static struct mbuf *tcq_dequeue_tc_ifclassq(struct ifclassq *, - mbuf_svc_class_t, cqdq_op_t); +static int tcq_enqueue_ifclassq(struct ifclassq *, void *, classq_pkt_type_t, + boolean_t *); +static void *tcq_dequeue_tc_ifclassq(struct ifclassq *, mbuf_svc_class_t, + classq_pkt_type_t *); static int tcq_request_ifclassq(struct ifclassq *, cqrq_t, void *); static int tcq_clear_interface(struct tcq_if *); static struct tcq_class *tcq_class_create(struct tcq_if *, int, u_int32_t, - int, u_int32_t); + int, u_int32_t, classq_pkt_type_t); static int tcq_class_destroy(struct tcq_if *, struct tcq_class *); static int tcq_destroy_locked(struct tcq_if *); -static inline int tcq_addq(struct tcq_class *, struct mbuf *, +static inline int tcq_addq(struct tcq_class *, pktsched_pkt_t *, struct pf_mtag *); -static inline struct mbuf *tcq_getq(struct tcq_class *); -static inline struct mbuf *tcq_pollq(struct tcq_class *); +static inline void tcq_getq(struct tcq_class *, pktsched_pkt_t *); static void tcq_purgeq(struct tcq_if *, struct tcq_class *, u_int32_t, u_int32_t *, u_int32_t *); static void tcq_purge_sc(struct tcq_if *, cqrq_purge_sc_t *); @@ -71,8 +72,8 @@ static int tcq_throttle(struct tcq_if *, cqrq_throttle_t *); static int tcq_resumeq(struct tcq_if *, struct tcq_class *); static int tcq_suspendq(struct tcq_if *, struct tcq_class *); static int tcq_stat_sc(struct tcq_if *, cqrq_stat_sc_t *); -static struct mbuf *tcq_dequeue_cl(struct tcq_if *, struct tcq_class *, - mbuf_svc_class_t, cqdq_op_t); +static void tcq_dequeue_cl(struct tcq_if *, struct tcq_class *, + mbuf_svc_class_t, pktsched_pkt_t *); static inline struct tcq_class *tcq_clh_to_clp(struct tcq_if *, u_int32_t); static const char *tcq_style(struct tcq_if *); @@ -113,7 +114,7 @@ tcq_init(void) } struct tcq_if * -tcq_alloc(struct ifnet *ifp, int how, boolean_t altq) +tcq_alloc(struct ifnet *ifp, int how) { struct tcq_if *tif; @@ -124,8 +125,6 @@ tcq_alloc(struct ifnet *ifp, int how, boolean_t altq) bzero(tif, tcq_size); tif->tif_maxpri = -1; tif->tif_ifq = &ifp->if_snd; - if (altq) - tif->tif_flags |= TCQIFF_ALTQ; if (pktsched_verbose) { log(LOG_DEBUG, "%s: %s scheduler allocated\n", @@ -198,15 +197,7 @@ tcq_purge(struct tcq_if *tif) if ((cl = tif->tif_classes[pri]) != NULL && !qempty(&cl->cl_q)) tcq_purgeq(tif, cl, 0, NULL, NULL); } -#if !PF_ALTQ - /* - * This assertion is safe to be made only when PF_ALTQ is not - * configured; otherwise, IFCQ_LEN represents the sum of the - * packets managed by ifcq_disc and altq_disc instances, which - * is possible when transitioning between the two. - */ VERIFY(IFCQ_LEN(tif->tif_ifq) == 0); -#endif /* !PF_ALTQ */ } static void @@ -256,7 +247,7 @@ tcq_event(struct tcq_if *tif, cqev_t ev) int tcq_add_queue(struct tcq_if *tif, int priority, u_int32_t qlimit, - int flags, u_int32_t qid, struct tcq_class **clp) + int flags, u_int32_t qid, struct tcq_class **clp, classq_pkt_type_t ptype) { struct tcq_class *cl; @@ -270,7 +261,7 @@ tcq_add_queue(struct tcq_if *tif, int priority, u_int32_t qlimit, if (tcq_clh_to_clp(tif, qid) != NULL) return (EBUSY); - cl = tcq_class_create(tif, priority, qlimit, flags, qid); + cl = tcq_class_create(tif, priority, qlimit, flags, qid, ptype); if (cl == NULL) return (ENOMEM); @@ -282,7 +273,7 @@ tcq_add_queue(struct tcq_if *tif, int priority, u_int32_t qlimit, static struct tcq_class * tcq_class_create(struct tcq_if *tif, int pri, u_int32_t qlimit, - int flags, u_int32_t qid) + int flags, u_int32_t qid, classq_pkt_type_t ptype) { struct ifnet *ifp; struct ifclassq *ifq; @@ -290,45 +281,6 @@ tcq_class_create(struct tcq_if *tif, int pri, u_int32_t qlimit, IFCQ_LOCK_ASSERT_HELD(tif->tif_ifq); - /* Sanitize flags unless internally configured */ - if (tif->tif_flags & TCQIFF_ALTQ) - flags &= TQCF_USERFLAGS; - -#if !CLASSQ_RED - if (flags & TQCF_RED) { - log(LOG_ERR, "%s: %s RED not available!\n", - if_name(TCQIF_IFP(tif)), tcq_style(tif)); - return (NULL); - } -#endif /* !CLASSQ_RED */ - -#if !CLASSQ_RIO - if (flags & TQCF_RIO) { - log(LOG_ERR, "%s: %s RIO not available!\n", - if_name(TCQIF_IFP(tif)), tcq_style(tif)); - return (NULL); - } -#endif /* CLASSQ_RIO */ - -#if !CLASSQ_BLUE - if (flags & TQCF_BLUE) { - log(LOG_ERR, "%s: %s BLUE not available!\n", - if_name(TCQIF_IFP(tif)), tcq_style(tif)); - return (NULL); - } -#endif /* CLASSQ_BLUE */ - - /* These are mutually exclusive */ - if ((flags & (TQCF_RED|TQCF_RIO|TQCF_BLUE|TQCF_SFB)) && - (flags & (TQCF_RED|TQCF_RIO|TQCF_BLUE|TQCF_SFB)) != TQCF_RED && - (flags & (TQCF_RED|TQCF_RIO|TQCF_BLUE|TQCF_SFB)) != TQCF_RIO && - (flags & (TQCF_RED|TQCF_RIO|TQCF_BLUE|TQCF_SFB)) != TQCF_BLUE && - (flags & (TQCF_RED|TQCF_RIO|TQCF_BLUE|TQCF_SFB)) != TQCF_SFB) { - log(LOG_ERR, "%s: %s more than one RED|RIO|BLUE|SFB\n", - if_name(TCQIF_IFP(tif)), tcq_style(tif)); - return (NULL); - } - ifq = tif->tif_ifq; ifp = TCQIF_IFP(tif); @@ -336,23 +288,13 @@ tcq_class_create(struct tcq_if *tif, int pri, u_int32_t qlimit, /* modify the class instead of creating a new one */ if (!qempty(&cl->cl_q)) tcq_purgeq(tif, cl, 0, NULL, NULL); -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_destroy(cl->cl_rio); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_destroy(cl->cl_red); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_destroy(cl->cl_blue); -#endif /* CLASSQ_BLUE */ + if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) sfb_destroy(cl->cl_sfb); cl->cl_qalg.ptr = NULL; qtype(&cl->cl_q) = Q_DROPTAIL; qstate(&cl->cl_q) = QS_RUNNING; + VERIFY(qptype(&cl->cl_q) == ptype); } else { cl = zalloc(tcq_cl_zone); if (cl == NULL) @@ -369,7 +311,7 @@ tcq_class_create(struct tcq_if *tif, int pri, u_int32_t qlimit, if (qlimit == 0) qlimit = DEFAULT_QLIMIT; /* use default */ } - _qinit(&cl->cl_q, Q_DROPTAIL, qlimit); + _qinit(&cl->cl_q, Q_DROPTAIL, qlimit, ptype); cl->cl_flags = flags; cl->cl_pri = pri; if (pri > tif->tif_maxpri) @@ -377,80 +319,22 @@ tcq_class_create(struct tcq_if *tif, int pri, u_int32_t qlimit, cl->cl_tif = tif; cl->cl_handle = qid; - if (flags & (TQCF_RED|TQCF_RIO|TQCF_BLUE|TQCF_SFB)) { -#if CLASSQ_RED || CLASSQ_RIO - u_int64_t ifbandwidth = ifnet_output_linkrate(ifp); - int pkttime; -#endif /* CLASSQ_RED || CLASSQ_RIO */ - + if (flags & TQCF_SFB) { cl->cl_qflags = 0; if (flags & TQCF_ECN) { - if (flags & TQCF_BLUE) - cl->cl_qflags |= BLUEF_ECN; - else if (flags & TQCF_SFB) - cl->cl_qflags |= SFBF_ECN; - else if (flags & TQCF_RED) - cl->cl_qflags |= REDF_ECN; - else if (flags & TQCF_RIO) - cl->cl_qflags |= RIOF_ECN; + cl->cl_qflags |= SFBF_ECN; } if (flags & TQCF_FLOWCTL) { - if (flags & TQCF_SFB) - cl->cl_qflags |= SFBF_FLOWCTL; + cl->cl_qflags |= SFBF_FLOWCTL; } if (flags & TQCF_DELAYBASED) { - if (flags & TQCF_SFB) - cl->cl_qflags |= SFBF_DELAYBASED; - } - if (flags & TQCF_CLEARDSCP) { - if (flags & TQCF_RIO) - cl->cl_qflags |= RIOF_CLEARDSCP; - } -#if CLASSQ_RED || CLASSQ_RIO - /* - * XXX: RED & RIO should be watching link speed and MTU - * events and recompute pkttime accordingly. - */ - if (ifbandwidth < 8) - pkttime = 1000 * 1000 * 1000; /* 1 sec */ - else - pkttime = (int64_t)ifp->if_mtu * 1000 * 1000 * 1000 / - (ifbandwidth / 8); - - /* Test for exclusivity {RED,RIO,BLUE,SFB} was done above */ -#if CLASSQ_RED - if (flags & TQCF_RED) { - cl->cl_red = red_alloc(ifp, 0, 0, - qlimit(&cl->cl_q) * 10/100, - qlimit(&cl->cl_q) * 30/100, - cl->cl_qflags, pkttime); - if (cl->cl_red != NULL) - qtype(&cl->cl_q) = Q_RED; - } -#endif /* CLASSQ_RED */ -#if CLASSQ_RIO - if (flags & TQCF_RIO) { - cl->cl_rio = - rio_alloc(ifp, 0, NULL, cl->cl_qflags, pkttime); - if (cl->cl_rio != NULL) - qtype(&cl->cl_q) = Q_RIO; - } -#endif /* CLASSQ_RIO */ -#endif /* CLASSQ_RED || CLASSQ_RIO */ -#if CLASSQ_BLUE - if (flags & TQCF_BLUE) { - cl->cl_blue = blue_alloc(ifp, 0, 0, cl->cl_qflags); - if (cl->cl_blue != NULL) - qtype(&cl->cl_q) = Q_BLUE; - } -#endif /* CLASSQ_BLUE */ - if (flags & TQCF_SFB) { - if (!(cl->cl_flags & TQCF_LAZY)) - cl->cl_sfb = sfb_alloc(ifp, cl->cl_handle, - qlimit(&cl->cl_q), cl->cl_qflags); - if (cl->cl_sfb != NULL || (cl->cl_flags & TQCF_LAZY)) - qtype(&cl->cl_q) = Q_SFB; + cl->cl_qflags |= SFBF_DELAYBASED; } + if (!(cl->cl_flags & TQCF_LAZY)) + cl->cl_sfb = sfb_alloc(ifp, cl->cl_handle, + qlimit(&cl->cl_q), cl->cl_qflags); + if (cl->cl_sfb != NULL || (cl->cl_flags & TQCF_LAZY)) + qtype(&cl->cl_q) = Q_SFB; } if (pktsched_verbose) { @@ -480,7 +364,9 @@ tcq_class_destroy(struct tcq_if *tif, struct tcq_class *cl) { struct ifclassq *ifq = tif->tif_ifq; int pri; - +#if !MACH_ASSERT +#pragma unused(ifq) +#endif IFCQ_LOCK_ASSERT_HELD(ifq); if (!qempty(&cl->cl_q)) @@ -501,18 +387,6 @@ tcq_class_destroy(struct tcq_if *tif, struct tcq_class *cl) tif->tif_default = NULL; if (cl->cl_qalg.ptr != NULL) { -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_destroy(cl->cl_rio); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_destroy(cl->cl_red); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_destroy(cl->cl_blue); -#endif /* CLASSQ_BLUE */ if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) sfb_destroy(cl->cl_sfb); cl->cl_qalg.ptr = NULL; @@ -531,7 +405,7 @@ tcq_class_destroy(struct tcq_if *tif, struct tcq_class *cl) } int -tcq_enqueue(struct tcq_if *tif, struct tcq_class *cl, struct mbuf *m, +tcq_enqueue(struct tcq_if *tif, struct tcq_class *cl, pktsched_pkt_t *pkt, struct pf_mtag *t) { struct ifclassq *ifq = tif->tif_ifq; @@ -541,45 +415,27 @@ tcq_enqueue(struct tcq_if *tif, struct tcq_class *cl, struct mbuf *m, VERIFY(cl == NULL || cl->cl_tif == tif); if (cl == NULL) { -#if PF_ALTQ - cl = tcq_clh_to_clp(tif, t->pftag_qid); -#else /* !PF_ALTQ */ cl = tcq_clh_to_clp(tif, 0); -#endif /* !PF_ALTQ */ if (cl == NULL) { cl = tif->tif_default; if (cl == NULL) { IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (ENOBUFS); + return (CLASSQEQ_DROP); } } } - len = m_pktlen(m); - - ret = tcq_addq(cl, m, t); - if (ret != 0) { - if (ret == CLASSQEQ_SUCCESS_FC) { - /* packet enqueued, return advisory feedback */ - ret = EQFULL; - } else { - VERIFY(ret == CLASSQEQ_DROPPED || - ret == CLASSQEQ_DROPPED_FC || - ret == CLASSQEQ_DROPPED_SP); - /* packet has been freed in tcq_addq */ - PKTCNTR_ADD(&cl->cl_dropcnt, 1, len); - IFCQ_DROP_ADD(ifq, 1, len); - switch (ret) { - case CLASSQEQ_DROPPED: - return (ENOBUFS); - case CLASSQEQ_DROPPED_FC: - return (EQFULL); - case CLASSQEQ_DROPPED_SP: - return (EQSUSPENDED); - } - /* NOT REACHED */ - } + VERIFY(pkt->pktsched_ptype == qptype(&cl->cl_q)); + len = pktsched_get_pkt_len(pkt); + + ret = tcq_addq(cl, pkt, t); + if ((ret != 0) && (ret != CLASSQEQ_SUCCESS_FC)) { + VERIFY(ret == CLASSQEQ_DROP || + ret == CLASSQEQ_DROP_FC || + ret == CLASSQEQ_DROP_SP); + PKTCNTR_ADD(&cl->cl_dropcnt, 1, len); + IFCQ_DROP_ADD(ifq, 1, len); + return (ret); } IFCQ_INC_LEN(ifq); IFCQ_INC_BYTES(ifq, len); @@ -594,70 +450,56 @@ tcq_enqueue(struct tcq_if *tif, struct tcq_class *cl, struct mbuf *m, * CLASSQDQ_REMOVE must return the same packet if called immediately * after CLASSQDQ_POLL. */ -struct mbuf * -tcq_dequeue_tc(struct tcq_if *tif, mbuf_svc_class_t sc, cqdq_op_t op) +void +tcq_dequeue_tc(struct tcq_if *tif, mbuf_svc_class_t sc, pktsched_pkt_t *pkt) { - return (tcq_dequeue_cl(tif, NULL, sc, op)); + tcq_dequeue_cl(tif, NULL, sc, pkt); } -static struct mbuf * -tcq_dequeue_cl(struct tcq_if *tif, struct tcq_class *cl, - mbuf_svc_class_t sc, cqdq_op_t op) +static void +tcq_dequeue_cl(struct tcq_if *tif, struct tcq_class *cl, mbuf_svc_class_t sc, + pktsched_pkt_t *pkt) { struct ifclassq *ifq = tif->tif_ifq; - struct mbuf *m; + uint32_t len; IFCQ_LOCK_ASSERT_HELD(ifq); if (cl == NULL) { cl = tcq_clh_to_clp(tif, MBUF_SCIDX(sc)); - if (cl == NULL) - return (NULL); + if (cl == NULL) { + pkt->pktsched_pkt = NULL; + return; + } } - if (qempty(&cl->cl_q)) - return (NULL); + if (qempty(&cl->cl_q)) { + pkt->pktsched_pkt = NULL; + return; + } VERIFY(!IFCQ_IS_EMPTY(ifq)); - if (op == CLASSQDQ_POLL) - return (tcq_pollq(cl)); - - m = tcq_getq(cl); - if (m != NULL) { + tcq_getq(cl, pkt); + if (pkt->pktsched_pkt != NULL) { + len = pktsched_get_pkt_len(pkt); IFCQ_DEC_LEN(ifq); - IFCQ_DEC_BYTES(ifq, m_pktlen(m)); + IFCQ_DEC_BYTES(ifq, len); if (qempty(&cl->cl_q)) cl->cl_period++; - PKTCNTR_ADD(&cl->cl_xmitcnt, 1, m_pktlen(m)); - IFCQ_XMIT_ADD(ifq, 1, m_pktlen(m)); + PKTCNTR_ADD(&cl->cl_xmitcnt, 1, len); + IFCQ_XMIT_ADD(ifq, 1, len); } - return (m); } static inline int -tcq_addq(struct tcq_class *cl, struct mbuf *m, struct pf_mtag *t) +tcq_addq(struct tcq_class *cl, pktsched_pkt_t *pkt, struct pf_mtag *t) { struct tcq_if *tif = cl->cl_tif; struct ifclassq *ifq = tif->tif_ifq; IFCQ_LOCK_ASSERT_HELD(ifq); -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - return (rio_addq(cl->cl_rio, &cl->cl_q, m, t)); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - return (red_addq(cl->cl_red, &cl->cl_q, m, t)); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - return (blue_addq(cl->cl_blue, &cl->cl_q, m, t)); - else -#endif /* CLASSQ_BLUE */ if (q_is_sfb(&cl->cl_q)) { if (cl->cl_sfb == NULL) { struct ifnet *ifp = TCQIF_IFP(tif); @@ -693,55 +535,35 @@ tcq_addq(struct tcq_class *cl, struct mbuf *m, struct pf_mtag *t) } } if (cl->cl_sfb != NULL) - return (sfb_addq(cl->cl_sfb, &cl->cl_q, m, t)); + return (sfb_addq(cl->cl_sfb, &cl->cl_q, pkt, t)); } else if (qlen(&cl->cl_q) >= qlimit(&cl->cl_q)) { IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (CLASSQEQ_DROPPED); + return (CLASSQEQ_DROP); } #if PF_ECN if (cl->cl_flags & TQCF_CLEARDSCP) + /* not supported for skywalk packets */ + VERIFY(pkt->pktsched_ptype == QP_MBUF); write_dsfield(m, t, 0); #endif /* PF_ECN */ - _addq(&cl->cl_q, m); + VERIFY(pkt->pktsched_ptype == qptype(&cl->cl_q)); + _addq(&cl->cl_q, pkt->pktsched_pkt); return (0); } -static inline struct mbuf * -tcq_getq(struct tcq_class *cl) +static inline void +tcq_getq(struct tcq_class *cl, pktsched_pkt_t *pkt) { IFCQ_LOCK_ASSERT_HELD(cl->cl_tif->tif_ifq); -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - return (rio_getq(cl->cl_rio, &cl->cl_q)); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - return (red_getq(cl->cl_red, &cl->cl_q)); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - return (blue_getq(cl->cl_blue, &cl->cl_q)); - else -#endif /* CLASSQ_BLUE */ - if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) - return (sfb_getq(cl->cl_sfb, &cl->cl_q)); - - return (_getq(&cl->cl_q)); -} - -static inline struct mbuf * -tcq_pollq(struct tcq_class *cl) -{ - IFCQ_LOCK_ASSERT_HELD(cl->cl_tif->tif_ifq); + if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) { + return (sfb_getq(cl->cl_sfb, &cl->cl_q, pkt)); + } - return (qhead(&cl->cl_q)); + return (pktsched_pkt_encap(pkt, qptype(&cl->cl_q), _getq(&cl->cl_q))); } static void @@ -756,24 +578,7 @@ tcq_purgeq(struct tcq_if *tif, struct tcq_class *cl, u_int32_t flow, if ((qlen = qlen(&cl->cl_q)) == 0) goto done; - /* become regular mutex before freeing mbufs */ IFCQ_CONVERT_LOCK(ifq); - -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_purgeq(cl->cl_rio, &cl->cl_q, flow, &cnt, &len); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_purgeq(cl->cl_red, &cl->cl_q, flow, &cnt, &len); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_purgeq(cl->cl_blue, &cl->cl_q, flow, &cnt, &len); - else -#endif /* CLASSQ_BLUE */ if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) sfb_purgeq(cl->cl_sfb, &cl->cl_q, flow, &cnt, &len); else @@ -814,18 +619,6 @@ tcq_updateq(struct tcq_if *tif, struct tcq_class *cl, cqev_t ev) cl->cl_handle, cl->cl_pri, ifclassq_ev2str(ev)); } -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - return (rio_updateq(cl->cl_rio, ev)); -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - return (red_updateq(cl->cl_red, ev)); -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - return (blue_updateq(cl->cl_blue, ev)); -#endif /* CLASSQ_BLUE */ if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) return (sfb_updateq(cl->cl_sfb, ev)); } @@ -851,18 +644,7 @@ tcq_get_class_stats(struct tcq_if *tif, u_int32_t qid, sp->qtype = qtype(&cl->cl_q); sp->qstate = qstate(&cl->cl_q); -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - red_getstats(cl->cl_red, &sp->red[0]); -#endif /* CLASSQ_RED */ -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - rio_getstats(cl->cl_rio, &sp->red[0]); -#endif /* CLASSQ_RIO */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - blue_getstats(cl->cl_blue, &sp->blue); -#endif /* CLASSQ_BLUE */ + if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) sfb_getstats(cl->cl_sfb, &sp->sfb); @@ -910,7 +692,8 @@ tcq_clh_to_clp(struct tcq_if *tif, u_int32_t chandle) static const char * tcq_style(struct tcq_if *tif) { - return ((tif->tif_flags & TCQIFF_ALTQ) ? "ALTQ_TCQ" : "TCQ"); +#pragma unused(tif) + return ("TCQ"); } /* @@ -918,26 +701,64 @@ tcq_style(struct tcq_if *tif) * (*ifcq_enqueue) in struct ifclassq. */ static int -tcq_enqueue_ifclassq(struct ifclassq *ifq, struct mbuf *m) +tcq_enqueue_ifclassq(struct ifclassq *ifq, void *p, classq_pkt_type_t ptype, + boolean_t *pdrop) { - u_int32_t i; + u_int32_t i = 0; + int ret; + pktsched_pkt_t pkt; + struct pf_mtag *t = NULL; IFCQ_LOCK_ASSERT_HELD(ifq); - if (!(m->m_flags & M_PKTHDR)) { - /* should not happen */ - log(LOG_ERR, "%s: packet does not have pkthdr\n", - if_name(ifq->ifcq_ifp)); - IFCQ_CONVERT_LOCK(ifq); - m_freem(m); - return (ENOBUFS); + if (ptype == QP_MBUF) { + struct mbuf *m = p; + if (!(m->m_flags & M_PKTHDR)) { + /* should not happen */ + log(LOG_ERR, "%s: packet does not have pkthdr\n", + if_name(ifq->ifcq_ifp)); + IFCQ_CONVERT_LOCK(ifq); + m_freem(m); + *pdrop = TRUE; + return (ENOBUFS); + } + t = m_pftag(m); + i = MBUF_SCIDX(mbuf_get_service_class(m)); } - - i = MBUF_SCIDX(mbuf_get_service_class(m)); VERIFY((u_int32_t)i < IFCQ_SC_MAX); - return (tcq_enqueue(ifq->ifcq_disc, - ifq->ifcq_disc_slots[i].cl, m, m_pftag(m))); + pktsched_pkt_encap(&pkt, ptype, p); + + ret = tcq_enqueue(ifq->ifcq_disc, + ifq->ifcq_disc_slots[i].cl, &pkt, t); + + if ((ret != 0) && (ret != CLASSQEQ_SUCCESS_FC)) { + pktsched_free_pkt(&pkt); + *pdrop = TRUE; + } else { + *pdrop = FALSE; + } + + switch (ret) { + case CLASSQEQ_DROP: + ret = ENOBUFS; + break; + case CLASSQEQ_DROP_FC: + ret = EQFULL; + break; + case CLASSQEQ_DROP_SP: + ret = EQSUSPENDED; + break; + case CLASSQEQ_SUCCESS_FC: + ret = EQFULL; + break; + case CLASSQEQ_SUCCESS: + ret = 0; + break; + default: + VERIFY(0); + } + return (ret); } /* @@ -949,16 +770,19 @@ tcq_enqueue_ifclassq(struct ifclassq *ifq, struct mbuf *m) * CLASSQDQ_REMOVE must return the same packet if called immediately * after CLASSQDQ_POLL. */ -static struct mbuf * +static void * tcq_dequeue_tc_ifclassq(struct ifclassq *ifq, mbuf_svc_class_t sc, - cqdq_op_t op) + classq_pkt_type_t *ptype) { + pktsched_pkt_t pkt; u_int32_t i = MBUF_SCIDX(sc); VERIFY((u_int32_t)i < IFCQ_SC_MAX); - return (tcq_dequeue_cl(ifq->ifcq_disc, - ifq->ifcq_disc_slots[i].cl, sc, op)); + bzero(&pkt, sizeof (pkt)); + (tcq_dequeue_cl(ifq->ifcq_disc, ifq->ifcq_disc_slots[i].cl, sc, &pkt)); + *ptype = pkt.pktsched_ptype; + return (pkt.pktsched_pkt); } static int @@ -994,7 +818,8 @@ tcq_request_ifclassq(struct ifclassq *ifq, cqrq_t req, void *arg) } int -tcq_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) +tcq_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags, + classq_pkt_type_t ptype) { struct ifnet *ifp = ifq->ifcq_ifp; struct tcq_class *cl0, *cl1, *cl2, *cl3; @@ -1006,12 +831,6 @@ tcq_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) VERIFY(ifq->ifcq_disc == NULL); VERIFY(ifq->ifcq_type == PKTSCHEDT_NONE); - if (flags & PKTSCHEDF_QALG_RED) - qflags |= TQCF_RED; - if (flags & PKTSCHEDF_QALG_RIO) - qflags |= TQCF_RIO; - if (flags & PKTSCHEDF_QALG_BLUE) - qflags |= TQCF_BLUE; if (flags & PKTSCHEDF_QALG_SFB) qflags |= TQCF_SFB; if (flags & PKTSCHEDF_QALG_ECN) @@ -1021,7 +840,7 @@ tcq_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) if (flags & PKTSCHEDF_QALG_DELAYBASED) qflags |= TQCF_DELAYBASED; - tif = tcq_alloc(ifp, M_WAITOK, FALSE); + tif = tcq_alloc(ifp, M_WAITOK); if (tif == NULL) return (ENOMEM); @@ -1029,24 +848,24 @@ tcq_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags) maxlen = if_sndq_maxlen; if ((err = tcq_add_queue(tif, 0, maxlen, - qflags | PRCF_LAZY, SCIDX_BK, &cl0)) != 0) + qflags | TQCF_LAZY, SCIDX_BK, &cl0, ptype)) != 0) goto cleanup; if ((err = tcq_add_queue(tif, 1, maxlen, - qflags | TQCF_DEFAULTCLASS, SCIDX_BE, &cl1)) != 0) + qflags | TQCF_DEFAULTCLASS, SCIDX_BE, &cl1, ptype)) != 0) goto cleanup; if ((err = tcq_add_queue(tif, 2, maxlen, - qflags | PRCF_LAZY, SCIDX_VI, &cl2)) != 0) + qflags | TQCF_LAZY, SCIDX_VI, &cl2, ptype)) != 0) goto cleanup; if ((err = tcq_add_queue(tif, 3, maxlen, - qflags, SCIDX_VO, &cl3)) != 0) + qflags, SCIDX_VO, &cl3, ptype)) != 0) goto cleanup; err = ifclassq_attach(ifq, PKTSCHEDT_TCQ, tif, tcq_enqueue_ifclassq, NULL, tcq_dequeue_tc_ifclassq, - NULL, tcq_request_ifclassq); + NULL, NULL, tcq_request_ifclassq); /* cache these for faster lookup */ if (err == 0) { @@ -1136,7 +955,6 @@ tcq_throttle(struct tcq_if *tif, cqrq_throttle_t *tr) int err = 0; IFCQ_LOCK_ASSERT_HELD(ifq); - VERIFY(!(tif->tif_flags & TCQIFF_ALTQ)); if (!tr->set) { tr->level = tif->tif_throttle; @@ -1189,24 +1007,11 @@ tcq_resumeq(struct tcq_if *tif, struct tcq_class *cl) { struct ifclassq *ifq = tif->tif_ifq; int err = 0; - +#if !MACH_ASSERT +#pragma unused(ifq) +#endif IFCQ_LOCK_ASSERT_HELD(ifq); -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - err = rio_suspendq(cl->cl_rio, &cl->cl_q, FALSE); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - err = red_suspendq(cl->cl_red, &cl->cl_q, FALSE); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - err = blue_suspendq(cl->cl_blue, &cl->cl_q, FALSE); - else -#endif /* CLASSQ_BLUE */ if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL) err = sfb_suspendq(cl->cl_sfb, &cl->cl_q, FALSE); @@ -1221,24 +1026,11 @@ tcq_suspendq(struct tcq_if *tif, struct tcq_class *cl) { struct ifclassq *ifq = tif->tif_ifq; int err = 0; - +#if !MACH_ASSERT +#pragma unused(ifq) +#endif IFCQ_LOCK_ASSERT_HELD(ifq); -#if CLASSQ_RIO - if (q_is_rio(&cl->cl_q)) - err = rio_suspendq(cl->cl_rio, &cl->cl_q, TRUE); - else -#endif /* CLASSQ_RIO */ -#if CLASSQ_RED - if (q_is_red(&cl->cl_q)) - err = red_suspendq(cl->cl_red, &cl->cl_q, TRUE); - else -#endif /* CLASSQ_RED */ -#if CLASSQ_BLUE - if (q_is_blue(&cl->cl_q)) - err = blue_suspendq(cl->cl_blue, &cl->cl_q, TRUE); - else -#endif /* CLASSQ_BLUE */ if (q_is_sfb(&cl->cl_q)) { if (cl->cl_sfb != NULL) { err = sfb_suspendq(cl->cl_sfb, &cl->cl_q, TRUE); diff --git a/bsd/net/pktsched/pktsched_tcq.h b/bsd/net/pktsched/pktsched_tcq.h index 57bb9fea0..91c2f71e8 100644 --- a/bsd/net/pktsched/pktsched_tcq.h +++ b/bsd/net/pktsched/pktsched_tcq.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2012 Apple Inc. All rights reserved. + * Copyright (c) 2011-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -52,7 +52,7 @@ extern "C" { #define TQCF_SFB 0x0200 /* use SFB */ #define TQCF_FLOWCTL 0x0400 /* enable flow control advisories */ #define TQCF_DEFAULTCLASS 0x1000 /* default class */ -#define TQCF_DELAYBASED 0x2000 /* queue sizing is delay based */ +#define TQCF_DELAYBASED 0x2000 /* queue sizing is delay based */ #ifdef BSD_KERNEL_PRIVATE #define TQCF_LAZY 0x10000000 /* on-demand resource allocation */ #endif /* BSD_KERNEL_PRIVATE */ @@ -98,9 +98,6 @@ struct tcq_class { u_int32_t cl_qflags; /* class queue flags */ union { void *ptr; - struct red *red; /* RED state */ - struct rio *rio; /* RIO state */ - struct blue *blue; /* BLUE state */ struct sfb *sfb; /* SFB state */ } cl_qalg; int32_t cl_pri; /* priority */ @@ -113,21 +110,14 @@ struct tcq_class { struct pktcntr cl_dropcnt; /* dropped packet counter */ }; -#define cl_red cl_qalg.red -#define cl_rio cl_qalg.rio -#define cl_blue cl_qalg.blue #define cl_sfb cl_qalg.sfb -/* tcq_if flags */ -#define TCQIFF_ALTQ 0x1 /* configured via PF/ALTQ */ - /* * tcq interface state */ struct tcq_if { struct ifclassq *tif_ifq; /* backpointer to ifclassq */ int tif_maxpri; /* max priority in use */ - u_int32_t tif_flags; /* flags */ u_int32_t tif_throttle; /* throttling level */ struct tcq_class *tif_default; /* default class */ struct tcq_class *tif_classes[TCQ_MAXPRI]; /* classes */ @@ -138,20 +128,19 @@ struct tcq_if { struct if_ifclassq_stats; extern void tcq_init(void); -extern struct tcq_if *tcq_alloc(struct ifnet *, int, boolean_t); +extern struct tcq_if *tcq_alloc(struct ifnet *, int); extern int tcq_destroy(struct tcq_if *); extern void tcq_purge(struct tcq_if *); extern void tcq_event(struct tcq_if *, cqev_t); extern int tcq_add_queue(struct tcq_if *, int, u_int32_t, int, u_int32_t, - struct tcq_class **); + struct tcq_class **, classq_pkt_type_t); extern int tcq_remove_queue(struct tcq_if *, u_int32_t); extern int tcq_get_class_stats(struct tcq_if *, u_int32_t, struct tcq_classstats *); -extern int tcq_enqueue(struct tcq_if *, struct tcq_class *, struct mbuf *, +extern int tcq_enqueue(struct tcq_if *, struct tcq_class *, pktsched_pkt_t *, struct pf_mtag *); -extern struct mbuf *tcq_dequeue_tc(struct tcq_if *, mbuf_svc_class_t, - cqdq_op_t); -extern int tcq_setup_ifclassq(struct ifclassq *, u_int32_t); +extern void tcq_dequeue_tc(struct tcq_if *, mbuf_svc_class_t, pktsched_pkt_t *); +extern int tcq_setup_ifclassq(struct ifclassq *, u_int32_t, classq_pkt_type_t); extern int tcq_teardown_ifclassq(struct ifclassq *ifq); extern int tcq_getqstats_ifclassq(struct ifclassq *, u_int32_t qid, struct if_ifclassq_stats *); diff --git a/bsd/net/radix.h b/bsd/net/radix.h index d48399aae..78f251b22 100644 --- a/bsd/net/radix.h +++ b/bsd/net/radix.h @@ -102,7 +102,15 @@ struct radix_node { struct radix_node *rn_ybro; #endif +#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4) +/* For the newer ARMv7k ABI where 64-bit types are 64-bit aligned, but pointers + * are 32-bit: + * Aligned to 64-bit since this is cast to rtentry, which is 64-bit aligned. + */ +} __attribute__ ((aligned(8))); +#else }; +#endif #define rn_dupedkey rn_u.rn_leaf.rn_Dupedkey #define rn_key rn_u.rn_leaf.rn_Key diff --git a/bsd/net/raw_cb.h b/bsd/net/raw_cb.h index c944ec005..39a63716c 100644 --- a/bsd/net/raw_cb.h +++ b/bsd/net/raw_cb.h @@ -90,7 +90,7 @@ extern LIST_HEAD(rawcb_list_head, rawcb) rawcb_list; __BEGIN_DECLS extern int raw_attach(struct socket *, int); -extern void raw_ctlinput(int, struct sockaddr *, void *); +extern void raw_ctlinput(int, struct sockaddr *, void *, struct ifnet *); extern void raw_detach(struct rawcb *); extern void raw_disconnect(struct rawcb *); extern void raw_init(struct protosw *, struct domain *); diff --git a/bsd/net/raw_usrreq.c b/bsd/net/raw_usrreq.c index 657f47d06..462842129 100644 --- a/bsd/net/raw_usrreq.c +++ b/bsd/net/raw_usrreq.c @@ -175,7 +175,8 @@ raw_input(struct mbuf *m0, struct sockproto *proto, struct sockaddr *src, /*ARGSUSED*/ void -raw_ctlinput(int cmd, __unused struct sockaddr *arg, __unused void *dummy) +raw_ctlinput(int cmd, __unused struct sockaddr *arg, __unused void *dummy, + __unused struct ifnet *ifp) { if (cmd < 0 || cmd >= PRC_NCMDS) @@ -193,7 +194,7 @@ raw_uabort(struct socket *so) mutex_held = (*so->so_proto->pr_getlock)(so, 0); else mutex_held = so->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); if (rp == 0) return EINVAL; @@ -242,7 +243,7 @@ raw_udetach(struct socket *so) mutex_held = (*so->so_proto->pr_getlock)(so, 0); else mutex_held = so->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); if (rp == 0) return EINVAL; @@ -296,7 +297,7 @@ raw_usend(struct socket *so, int flags, struct mbuf *m, mutex_held = (*so->so_proto->pr_getlock)(so, 0); else mutex_held = so->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); if (rp == 0) { error = EINVAL; @@ -348,7 +349,7 @@ raw_ushutdown(struct socket *so) mutex_held = (*so->so_proto->pr_getlock)(so, 0); else mutex_held = so->so_proto->pr_domain->dom_mtx; - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); if (rp == 0) return EINVAL; diff --git a/bsd/net/route.c b/bsd/net/route.c index 96633f833..80d336a40 100644 --- a/bsd/net/route.c +++ b/bsd/net/route.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -83,11 +83,16 @@ #include <net/if.h> #include <net/route.h> #include <net/ntstat.h> +#include <net/nwk_wq.h> +#if NECP +#include <net/necp.h> +#endif /* NECP */ #include <netinet/in.h> #include <netinet/in_var.h> #include <netinet/ip_var.h> #include <netinet/ip6.h> +#include <netinet/in_arp.h> #if INET6 #include <netinet6/ip6_var.h> @@ -208,7 +213,7 @@ extern void kdp_set_gateway_mac(void *gatewaymac); -__private_extern__ struct rtstat rtstat = { 0, 0, 0, 0, 0 }; +__private_extern__ struct rtstat rtstat = { 0, 0, 0, 0, 0, 0 }; struct radix_node_head *rt_tables[AF_MAX+1]; decl_lck_mtx_data(, rnh_lock_data); /* global routing tables mutex */ @@ -224,7 +229,7 @@ static lck_grp_attr_t *rte_mtx_grp_attr; int rttrash = 0; /* routes not in table but not freed */ -unsigned int rte_debug; +unsigned int rte_debug = 0; /* Possible flags for rte_debug */ #define RTD_DEBUG 0x1 /* enable or disable rtentry debug facility */ @@ -319,6 +324,7 @@ static void rtfree_common(struct rtentry *, boolean_t); static void rte_if_ref(struct ifnet *, int); static void rt_set_idleref(struct rtentry *); static void rt_clear_idleref(struct rtentry *); +static void route_event_callback(void *); static void rt_str4(struct rtentry *, char *, uint32_t, char *, uint32_t); #if INET6 static void rt_str6(struct rtentry *, char *, uint32_t, char *, uint32_t); @@ -413,6 +419,8 @@ route_init(void) #if INET6 _CASSERT(offsetof(struct route, ro_rt) == offsetof(struct route_in6, ro_rt)); + _CASSERT(offsetof(struct route, ro_lle) == + offsetof(struct route_in6, ro_lle)); _CASSERT(offsetof(struct route, ro_srcia) == offsetof(struct route_in6, ro_srcia)); _CASSERT(offsetof(struct route, ro_flags) == @@ -699,7 +707,7 @@ sa_trim(struct sockaddr *sa, int skip) */ struct sockaddr * rtm_scrub(int type, int idx, struct sockaddr *hint, struct sockaddr *sa, - void *buf, uint32_t buflen, kauth_cred_t *credp, uint32_t rtm_hint_flags) + void *buf, uint32_t buflen, kauth_cred_t *credp) { struct sockaddr_storage *ss = (struct sockaddr_storage *)buf; struct sockaddr *ret = sa; @@ -782,8 +790,7 @@ rtm_scrub(int type, int idx, struct sockaddr *hint, struct sockaddr *sa, /* fallthrough */ } case RTAX_IFP: { - if (sa->sa_family == AF_LINK && credp && - (rtm_hint_flags & RTMF_HIDE_LLADDR)) { + if (sa->sa_family == AF_LINK && credp) { struct sockaddr_dl *sdl = SDL(buf); const void *bytes; size_t size; @@ -893,7 +900,7 @@ rtalloc_ign_common_locked(struct route *ro, uint32_t ignore, void rtalloc_ign(struct route *ro, uint32_t ignore) { - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(rnh_lock); rtalloc_ign_common_locked(ro, ignore, IFSCOPE_NONE); lck_mtx_unlock(rnh_lock); @@ -902,7 +909,7 @@ rtalloc_ign(struct route *ro, uint32_t ignore) void rtalloc_scoped_ign(struct route *ro, uint32_t ignore, unsigned int ifscope) { - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(rnh_lock); rtalloc_ign_common_locked(ro, ignore, ifscope); lck_mtx_unlock(rnh_lock); @@ -1039,7 +1046,7 @@ struct rtentry * rtalloc1(struct sockaddr *dst, int report, uint32_t ignflags) { struct rtentry *entry; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(rnh_lock); entry = rtalloc1_locked(dst, report, ignflags); lck_mtx_unlock(rnh_lock); @@ -1051,7 +1058,7 @@ rtalloc1_scoped(struct sockaddr *dst, int report, uint32_t ignflags, unsigned int ifscope) { struct rtentry *entry; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(rnh_lock); entry = rtalloc1_scoped_locked(dst, report, ignflags, ifscope); lck_mtx_unlock(rnh_lock); @@ -1073,7 +1080,7 @@ rtfree_common(struct rtentry *rt, boolean_t locked) { struct radix_node_head *rnh; - lck_mtx_assert(rnh_lock, locked ? + LCK_MTX_ASSERT(rnh_lock, locked ? LCK_MTX_ASSERT_OWNED : LCK_MTX_ASSERT_NOTOWNED); /* @@ -1119,7 +1126,7 @@ rtfree_common(struct rtentry *rt, boolean_t locked) */ RT_CONVERT_LOCK(rt); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); /* Negative refcnt must never happen */ if (rt->rt_refcnt != 0) { @@ -1154,6 +1161,7 @@ rtfree_common(struct rtentry *rt, boolean_t locked) struct rtentry *rt_parent; struct ifaddr *rt_ifa; + rt->rt_flags |= RTF_DEAD; if (rt->rt_nodes->rn_flags & (RNF_ACTIVE | RNF_ROOT)) { panic("rt %p freed while in radix tree\n", rt); /* NOTREACHED */ @@ -1189,11 +1197,15 @@ rtfree_common(struct rtentry *rt, boolean_t locked) rt->rt_llinfo = NULL; } + /* Destroy eventhandler lists context */ + eventhandler_lists_ctxt_destroy(&rt->rt_evhdlr_ctxt); + /* * Route is no longer in the tree and refcnt is 0; * we have exclusive access, so destroy it. */ RT_UNLOCK(rt); + rte_lock_destroy(rt); if (rt_parent != NULL) rtfree_locked(rt_parent); @@ -1216,7 +1228,6 @@ rtfree_common(struct rtentry *rt, boolean_t locked) /* * and the rtentry itself of course */ - rte_lock_destroy(rt); rte_free(rt); } else { /* @@ -1289,6 +1300,7 @@ rtref(struct rtentry *p) { RT_LOCK_ASSERT_HELD(p); + VERIFY((p->rt_flags & RTF_DEAD) == 0); if (++p->rt_refcnt == 0) { panic("%s(%p) bad refcnt\n", __func__, p); /* NOTREACHED */ @@ -1321,7 +1333,7 @@ rtref_audit(struct rtentry_dbg *rte) void rtsetifa(struct rtentry *rt, struct ifaddr *ifa) { - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK_ASSERT_HELD(rt); @@ -1363,7 +1375,7 @@ rtredirect(struct ifnet *ifp, struct sockaddr *dst, struct sockaddr *gateway, struct sockaddr_storage ss; int af = src->sa_family; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(rnh_lock); /* @@ -1554,7 +1566,7 @@ ifa_ifwithroute_common_locked(int flags, const struct sockaddr *dst, struct rtentry *rt = NULL; struct sockaddr_storage dst_ss, gw_ss; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); /* * Just in case the sockaddr passed in by the caller @@ -1734,7 +1746,7 @@ rtrequest_common_locked(int req, struct sockaddr *dst0, #define senderr(x) { error = x; goto bad; } - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); /* * Find the correct routing tree to use for this Address Family */ @@ -1785,6 +1797,8 @@ rtrequest_common_locked(int req, struct sockaddr *dst0, switch (req) { case RTM_DELETE: { struct rtentry *gwrt = NULL; + boolean_t was_router = FALSE; + uint32_t old_rt_refcnt = 0; /* * Remove the item from the tree and return it. * Complain if it is not there and do no more processing. @@ -1798,6 +1812,7 @@ rtrequest_common_locked(int req, struct sockaddr *dst0, rt = (struct rtentry *)rn; RT_LOCK(rt); + old_rt_refcnt = rt->rt_refcnt; rt->rt_flags &= ~RTF_UP; /* * Release any idle reference count held on the interface @@ -1824,10 +1839,21 @@ rtrequest_common_locked(int req, struct sockaddr *dst0, * Clear RTF_ROUTER if it's set. */ if (rt->rt_flags & RTF_ROUTER) { + was_router = TRUE; VERIFY(rt->rt_flags & RTF_HOST); rt->rt_flags &= ~RTF_ROUTER; } + /* + * Enqueue work item to invoke callback for this route entry + * + * If the old count is 0, it implies that last reference is being + * removed and there's no one listening for this route event. + */ + if (old_rt_refcnt != 0) + route_event_enqueue_nwk_wq_entry(rt, NULL, + ROUTE_ENTRY_DELETED, NULL, TRUE); + /* * Now search what's left of the subtree for any cloned * routes which might have been formed from this node. @@ -1840,6 +1866,15 @@ rtrequest_common_locked(int req, struct sockaddr *dst0, RT_LOCK(rt); } + if (was_router) { + struct route_event rt_ev; + route_event_init(&rt_ev, rt, NULL, ROUTE_LLENTRY_DELETED); + RT_UNLOCK(rt); + (void) rnh->rnh_walktree(rnh, + route_event_walktree, (void *)&rt_ev); + RT_LOCK(rt); + } + /* * Remove any external references we may have. */ @@ -1878,6 +1913,16 @@ rtrequest_common_locked(int req, struct sockaddr *dst0, IFSCOPE_NONE); } +#if NECP + /* + * If this is a change in a default route, update + * necp client watchers to re-evaluate + */ + if (SA_DEFAULT(rt_key(rt))) { + necp_update_all_clients(); + } +#endif /* NECP */ + RT_UNLOCK(rt); /* @@ -2008,10 +2053,15 @@ rtrequest_common_locked(int req, struct sockaddr *dst0, if (ifa == NULL) senderr(ENETUNREACH); makeroute: + /* + * We land up here for both RTM_RESOLVE and RTM_ADD + * when we decide to create a route. + */ if ((rt = rte_alloc()) == NULL) senderr(ENOBUFS); Bzero(rt, sizeof(*rt)); rte_lock_init(rt); + eventhandler_lists_ctxt_init(&rt->rt_evhdlr_ctxt); getmicrotime(&caltime); rt->base_calendartime = caltime.tv_sec; rt->base_uptime = net_uptime(); @@ -2172,6 +2222,16 @@ makeroute: rt->rt_ifp->if_index); } +#if NECP + /* + * If this is a change in a default route, update + * necp client watchers to re-evaluate + */ + if (SA_DEFAULT(rt_key(rt))) { + necp_update_all_clients(); + } +#endif /* NECP */ + /* * actually return a resultant rtentry and * give the caller a single reference. @@ -2225,7 +2285,7 @@ rtrequest(int req, struct sockaddr *dst, struct sockaddr *gateway, struct sockaddr *netmask, int flags, struct rtentry **ret_nrt) { int error; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(rnh_lock); error = rtrequest_locked(req, dst, gateway, netmask, flags, ret_nrt); lck_mtx_unlock(rnh_lock); @@ -2238,7 +2298,7 @@ rtrequest_scoped(int req, struct sockaddr *dst, struct sockaddr *gateway, unsigned int ifscope) { int error; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(rnh_lock); error = rtrequest_scoped_locked(req, dst, gateway, netmask, flags, ret_nrt, ifscope); @@ -2259,7 +2319,7 @@ rt_fixdelete(struct radix_node *rn, void *vp) struct rtentry *rt = (struct rtentry *)rn; struct rtentry *rt0 = vp; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK(rt); if (rt->rt_parent == rt0 && @@ -2300,7 +2360,7 @@ rt_fixchange(struct radix_node *rn, void *vp) u_char *xk1, *xm1, *xk2, *xmp; int i, len; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK(rt); @@ -2394,7 +2454,7 @@ rt_setgate(struct rtentry *rt, struct sockaddr *dst, struct sockaddr *gate) } rnh = rt_tables[dst->sa_family]; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK_ASSERT_HELD(rt); /* @@ -2546,6 +2606,16 @@ rt_setgate(struct rtentry *rt, struct sockaddr *dst, struct sockaddr *gate) rt->rt_ifp->if_index); } +#if NECP + /* + * If this is a change in a default route, update + * necp client watchers to re-evaluate + */ + if (SA_DEFAULT(dst)) { + necp_update_all_clients(); + } +#endif /* NECP */ + /* * Tell the kernel debugger about the new default gateway * if the gateway route uses the primary interface, or @@ -2647,7 +2717,7 @@ rt_set_gwroute(struct rtentry *rt, struct sockaddr *dst, struct rtentry *gwrt) { boolean_t gwrt_isrouter; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK_ASSERT_HELD(rt); if (gwrt != NULL) @@ -2833,7 +2903,7 @@ rt_lookup_common(boolean_t lookup_only, boolean_t coarse, struct sockaddr *dst, #endif VERIFY(!coarse || ifscope == IFSCOPE_NONE); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); #if INET6 /* * While we have rnh_lock held, see if we need to schedule the timer. @@ -3109,7 +3179,7 @@ rtinit(struct ifaddr *ifa, int cmd, int flags) { int error; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(rnh_lock); error = rtinit_locked(ifa, cmd, flags); @@ -3137,7 +3207,7 @@ rtinit_locked(struct ifaddr *ifa, int cmd, int flags) * changing (e.g. in_ifinit), so it is safe to access its * ifa_{dst}addr (here and down below) without locking. */ - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); if (flags & RTF_HOST) { dst = ifa->ifa_dstaddr; @@ -3681,6 +3751,10 @@ route_copyout(struct route *dst, const struct route *src, size_t length) if (dst->ro_rt != NULL) RT_ADDREF(dst->ro_rt); + /* Hold one reference for the local copy of struct lle */ + if (dst->ro_lle != NULL) + LLE_ADDREF(dst->ro_lle); + /* Hold one reference for the local copy of struct ifaddr */ if (dst->ro_srcia != NULL) IFA_ADDREF(dst->ro_srcia); @@ -3689,8 +3763,18 @@ route_copyout(struct route *dst, const struct route *src, size_t length) void route_copyin(struct route *src, struct route *dst, size_t length) { - /* No cached route at the destination? */ + /* + * No cached route at the destination? + * If none, then remove old references if present + * and copy entire src route. + */ if (dst->ro_rt == NULL) { + /* + * Ditch the cached link layer reference (dst) + * since we're about to take everything there is in src + */ + if (dst->ro_lle != NULL) + LLE_REMREF(dst->ro_lle); /* * Ditch the address in the cached copy (dst) since * we're about to take everything there is in src. @@ -3698,42 +3782,78 @@ route_copyin(struct route *src, struct route *dst, size_t length) if (dst->ro_srcia != NULL) IFA_REMREF(dst->ro_srcia); /* - * Copy everything (rt, srcia, flags, dst) from src; the + * Copy everything (rt, ro_lle, srcia, flags, dst) from src; the * references to rt and/or srcia were held at the time * of storage and are kept intact. */ bcopy(src, dst, length); - } else if (src->ro_rt != NULL) { - /* - * If the same, update srcia and flags, and ditch the route - * in the local copy. Else ditch the one that is currently - * cached, and cache the new route. - */ - if (dst->ro_rt == src->ro_rt) { - dst->ro_flags = src->ro_flags; - if (dst->ro_srcia != src->ro_srcia) { - if (dst->ro_srcia != NULL) - IFA_REMREF(dst->ro_srcia); - dst->ro_srcia = src->ro_srcia; - } else if (src->ro_srcia != NULL) { - IFA_REMREF(src->ro_srcia); - } - rtfree(src->ro_rt); - } else { - rtfree(dst->ro_rt); + goto done; + } + + /* + * We know dst->ro_rt is not NULL here. + * If the src->ro_rt is the same, update ro_lle, srcia and flags + * and ditch the route in the local copy. + */ + if (dst->ro_rt == src->ro_rt) { + dst->ro_flags = src->ro_flags; + + if (dst->ro_lle != src->ro_lle) { + if (dst->ro_lle != NULL) + LLE_REMREF(dst->ro_lle); + dst->ro_lle = src->ro_lle; + } else if (src->ro_lle != NULL) { + LLE_REMREF(src->ro_lle); + } + + if (dst->ro_srcia != src->ro_srcia) { if (dst->ro_srcia != NULL) IFA_REMREF(dst->ro_srcia); - bcopy(src, dst, length); + dst->ro_srcia = src->ro_srcia; + } else if (src->ro_srcia != NULL) { + IFA_REMREF(src->ro_srcia); } - } else if (src->ro_srcia != NULL) { + rtfree(src->ro_rt); + goto done; + } + + /* + * If they are dst's ro_rt is not equal to src's, + * and src'd rt is not NULL, then remove old references + * if present and copy entire src route. + */ + if (src->ro_rt != NULL) { + rtfree(dst->ro_rt); + + if (dst->ro_lle != NULL) + LLE_REMREF(dst->ro_lle); + if (dst->ro_srcia != NULL) + IFA_REMREF(dst->ro_srcia); + bcopy(src, dst, length); + goto done; + } + + /* + * Here, dst's cached route is not NULL but source's is. + * Just get rid of all the other cached reference in src. + */ + if (src->ro_srcia != NULL) { /* * Ditch src address in the local copy (src) since we're * not caching the route entry anyway (ro_rt is NULL). */ IFA_REMREF(src->ro_srcia); } - + if (src->ro_lle != NULL) { + /* + * Ditch cache lle in the local copy (src) since we're + * not caching the route anyway (ro_rt is NULL). + */ + LLE_REMREF(src->ro_lle); + } +done: /* This function consumes the references on src */ + src->ro_lle = NULL; src->ro_rt = NULL; src->ro_srcia = NULL; } @@ -4075,3 +4195,173 @@ rt_str(struct rtentry *rt, char *ds, uint32_t dslen, char *gs, uint32_t gslen) break; } } + +void route_event_init(struct route_event *p_route_ev, struct rtentry *rt, + struct rtentry *gwrt, int route_ev_code) +{ + VERIFY(p_route_ev != NULL); + bzero(p_route_ev, sizeof(*p_route_ev)); + + p_route_ev->rt = rt; + p_route_ev->gwrt = gwrt; + p_route_ev->route_event_code = route_ev_code; +} + +static void +route_event_callback(void *arg) +{ + struct route_event *p_rt_ev = (struct route_event *)arg; + struct rtentry *rt = p_rt_ev->rt; + eventhandler_tag evtag = p_rt_ev->evtag; + int route_ev_code = p_rt_ev->route_event_code; + + if (route_ev_code == ROUTE_EVHDLR_DEREGISTER) { + VERIFY(evtag != NULL); + EVENTHANDLER_DEREGISTER(&rt->rt_evhdlr_ctxt, route_event, + evtag); + rtfree(rt); + return; + } + + EVENTHANDLER_INVOKE(&rt->rt_evhdlr_ctxt, route_event, rt_key(rt), + route_ev_code, (struct sockaddr *)&p_rt_ev->rt_addr, + rt->rt_flags); + + /* The code enqueuing the route event held a reference */ + rtfree(rt); + /* XXX No reference is taken on gwrt */ +} + +int +route_event_walktree(struct radix_node *rn, void *arg) +{ + struct route_event *p_route_ev = (struct route_event *)arg; + struct rtentry *rt = (struct rtentry *)rn; + struct rtentry *gwrt = p_route_ev->rt; + + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); + + RT_LOCK(rt); + + /* Return if the entry is pending cleanup */ + if (rt->rt_flags & RTPRF_OURS) { + RT_UNLOCK(rt); + return (0); + } + + /* Return if it is not an indirect route */ + if (!(rt->rt_flags & RTF_GATEWAY)) { + RT_UNLOCK(rt); + return (0); + } + + if (rt->rt_gwroute != gwrt) { + RT_UNLOCK(rt); + return (0); + } + + route_event_enqueue_nwk_wq_entry(rt, gwrt, p_route_ev->route_event_code, + NULL, TRUE); + RT_UNLOCK(rt); + + return (0); +} + +struct route_event_nwk_wq_entry +{ + struct nwk_wq_entry nwk_wqe; + struct route_event rt_ev_arg; +}; + +void +route_event_enqueue_nwk_wq_entry(struct rtentry *rt, struct rtentry *gwrt, + uint32_t route_event_code, eventhandler_tag evtag, boolean_t rt_locked) +{ + struct route_event_nwk_wq_entry *p_rt_ev = NULL; + struct sockaddr *p_gw_saddr = NULL; + + MALLOC(p_rt_ev, struct route_event_nwk_wq_entry *, + sizeof(struct route_event_nwk_wq_entry), + M_NWKWQ, M_WAITOK | M_ZERO); + + /* + * If the intent is to de-register, don't take + * reference, route event registration already takes + * a reference on route. + */ + if (route_event_code != ROUTE_EVHDLR_DEREGISTER) { + /* The reference is released by route_event_callback */ + if (rt_locked) + RT_ADDREF_LOCKED(rt); + else + RT_ADDREF(rt); + } + + p_rt_ev->rt_ev_arg.rt = rt; + p_rt_ev->rt_ev_arg.gwrt = gwrt; + p_rt_ev->rt_ev_arg.evtag = evtag; + + if (gwrt != NULL) + p_gw_saddr = gwrt->rt_gateway; + else + p_gw_saddr = rt->rt_gateway; + + VERIFY(p_gw_saddr->sa_len <= sizeof(p_rt_ev->rt_ev_arg.rt_addr)); + bcopy(p_gw_saddr, &(p_rt_ev->rt_ev_arg.rt_addr), p_gw_saddr->sa_len); + + p_rt_ev->rt_ev_arg.route_event_code = route_event_code; + p_rt_ev->nwk_wqe.func = route_event_callback; + p_rt_ev->nwk_wqe.is_arg_managed = TRUE; + p_rt_ev->nwk_wqe.arg = &p_rt_ev->rt_ev_arg; + nwk_wq_enqueue((struct nwk_wq_entry*)p_rt_ev); +} + +const char * +route_event2str(int route_event) +{ + const char *route_event_str = "ROUTE_EVENT_UNKNOWN"; + switch (route_event) { + case ROUTE_STATUS_UPDATE: + route_event_str = "ROUTE_STATUS_UPDATE"; + break; + case ROUTE_ENTRY_REFRESH: + route_event_str = "ROUTE_ENTRY_REFRESH"; + break; + case ROUTE_ENTRY_DELETED: + route_event_str = "ROUTE_ENTRY_DELETED"; + break; + case ROUTE_LLENTRY_RESOLVED: + route_event_str = "ROUTE_LLENTRY_RESOLVED"; + break; + case ROUTE_LLENTRY_UNREACH: + route_event_str = "ROUTE_LLENTRY_UNREACH"; + break; + case ROUTE_LLENTRY_CHANGED: + route_event_str = "ROUTE_LLENTRY_CHANGED"; + break; + case ROUTE_LLENTRY_STALE: + route_event_str = "ROUTE_LLENTRY_STALE"; + break; + case ROUTE_LLENTRY_TIMEDOUT: + route_event_str = "ROUTE_LLENTRY_TIMEDOUT"; + break; + case ROUTE_LLENTRY_DELETED: + route_event_str = "ROUTE_LLENTRY_DELETED"; + break; + case ROUTE_LLENTRY_EXPIRED: + route_event_str = "ROUTE_LLENTRY_EXPIRED"; + break; + case ROUTE_LLENTRY_PROBED: + route_event_str = "ROUTE_LLENTRY_PROBED"; + break; + case ROUTE_EVHDLR_DEREGISTER: + route_event_str = "ROUTE_EVHDLR_DEREGISTER"; + break; + default: + /* Init'd to ROUTE_EVENT_UNKNOWN */ + break; + } + return route_event_str; +} + + diff --git a/bsd/net/route.h b/bsd/net/route.h index e02d052d9..1e01812ec 100644 --- a/bsd/net/route.h +++ b/bsd/net/route.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -83,7 +83,8 @@ struct rt_metrics { u_int32_t rmx_rtt; /* estimated round trip time */ u_int32_t rmx_rttvar; /* estimated rtt variance */ u_int32_t rmx_pksent; /* packets sent using this route */ - u_int32_t rmx_filler[4]; /* will be used for T/TCP later */ + u_int32_t rmx_state; /* route state */ + u_int32_t rmx_filler[3]; /* will be used for T/TCP later */ }; /* @@ -102,6 +103,9 @@ struct route_old { #ifdef BSD_KERNEL_PRIVATE #include <kern/locks.h> #include <net/radix.h> +#include <net/if_llatbl.h> +#include <sys/eventhandler.h> +#include <net/if_dl.h> /* * Kernel resident routing tables. @@ -123,17 +127,30 @@ struct rt_reach_info; */ struct route { /* - * N.B: struct route must begin with ro_{rt,srcia,flags} + * N.B: struct route must begin with ro_{rt, lle, srcia, flags} * because the code does some casts of a 'struct route_in6 *' * to a 'struct route *'. */ struct rtentry *ro_rt; + struct llentry *ro_lle; + struct ifaddr *ro_srcia; uint32_t ro_flags; /* route flags (see below) */ struct sockaddr ro_dst; }; -#define ROF_SRCIF_SELECTED 0x1 /* source interface was selected */ +#define ROF_SRCIF_SELECTED 0x0001 /* source interface was selected */ +#if 0 +/* XXX These will be used in the changes coming in later */ +#define ROF_NORTREF 0x0002 /* doesn't hold reference on ro_rt */ +#define ROF_L2_ME 0x0004 /* dst L2 addr is our address */ +#define ROF_MAY_LOOP 0x0008 /* dst may require loop copy */ +#define ROF_HAS_HEADER 0x0010 /* mbuf already have its header prepended */ +#define ROF_REJECT 0x0020 /* Destination is reject */ +#define ROF_BLACKHOLE 0x0040 /* Destination is blackhole */ +#define ROF_HAS_GW 0x0080 /* Destination has GW */ +#endif +#define ROF_LLE_CACHE 0x0100 /* Cache link layer */ #define ROUTE_UNUSABLE(_ro) \ ((_ro)->ro_rt == NULL || \ @@ -154,6 +171,11 @@ struct route { (_ro)->ro_srcia = NULL; \ (_ro)->ro_flags &= ~ROF_SRCIF_SELECTED; \ } \ + if ((_ro)->ro_lle != NULL) { \ + LLE_REMREF((_ro)->ro_lle); \ + (_ro)->ro_lle = NULL; \ + (_ro)->ro_flags &= ~ROF_LLE_CACHE; \ + } \ } while (0) #define ROUTE_RELEASE_LOCKED(_ro) _ROUTE_RELEASE_COMMON(_ro, TRUE) @@ -210,8 +232,31 @@ struct rtentry { u_int32_t rtt_min; /* minimum RTT computed from history */ u_int32_t rtt_expire_ts; /* RTT history expire timestamp */ u_int8_t rtt_index; /* Index into RTT history */ + /* Event handler context for the rtentrt */ + struct eventhandler_lists_ctxt rt_evhdlr_ctxt; +}; + +enum { + ROUTE_STATUS_UPDATE = 1, + ROUTE_ENTRY_REFRESH, + ROUTE_ENTRY_DELETED, + ROUTE_LLENTRY_RESOLVED, + ROUTE_LLENTRY_UNREACH, + ROUTE_LLENTRY_CHANGED, + ROUTE_LLENTRY_STALE, + ROUTE_LLENTRY_TIMEDOUT, + ROUTE_LLENTRY_DELETED, + ROUTE_LLENTRY_EXPIRED, + ROUTE_LLENTRY_PROBED, + ROUTE_EVHDLR_DEREGISTER, }; +extern const char * route_event2str(int route_event); + +typedef void (*route_event_fn) (struct eventhandler_entry_arg, + struct sockaddr *, int, struct sockaddr *, int); +EVENTHANDLER_DECLARE(route_event, route_event_fn); + /* * Synchronize route entry's generation ID with the tree's. */ @@ -239,7 +284,9 @@ struct rtentry { #define RTF_DELCLONE 0x80 /* delete cloned route */ #define RTF_CLONING 0x100 /* generate new routes on use */ #define RTF_XRESOLVE 0x200 /* external daemon resolves name */ -#define RTF_LLINFO 0x400 /* generated by link layer (e.g. ARP) */ +#define RTF_LLINFO 0x400 /* DEPRECATED - exists ONLY for backward + compatibility */ +#define RTF_LLDATA 0x400 /* used by apps to add/del L2 entries */ #define RTF_STATIC 0x800 /* manually added */ #define RTF_BLACKHOLE 0x1000 /* just discard pkts (during updates) */ #define RTF_NOIFREF 0x2000 /* not eligible for RTF_IFREF */ @@ -259,8 +306,10 @@ struct rtentry { #define RTF_IFREF 0x4000000 /* route holds a ref to interface */ #define RTF_PROXY 0x8000000 /* proxying, no interface scope */ #define RTF_ROUTER 0x10000000 /* host is a router */ - /* 0x20000000 and up unassigned */ +#define RTF_DEAD 0x20000000 /* Route entry is being freed */ + /* 0x40000000 and up unassigned */ +#define RTPRF_OURS RTF_PROTO3 /* set on routes we manage */ #define RTF_BITS \ "\020\1UP\2GATEWAY\3HOST\4REJECT\5DYNAMIC\6MODIFIED\7DONE" \ "\10DELCLONE\11CLONING\12XRESOLVE\13LLINFO\14STATIC\15BLACKHOLE" \ @@ -268,6 +317,8 @@ struct rtentry { "\25PINNED\26LOCAL\27BROADCAST\30MULTICAST\31IFSCOPE\32CONDEMNED" \ "\33IFREF\34PROXY\35ROUTER" +#define IS_DIRECT_HOSTROUTE(rt) \ + (((rt)->rt_flags & (RTF_HOST | RTF_GATEWAY)) == RTF_HOST) /* * Routing statistics. */ @@ -277,6 +328,7 @@ struct rtstat { short rts_newgateway; /* routes modified by redirects */ short rts_unreach; /* lookups which failed */ short rts_wildcard; /* lookups satisfied by a wildcard */ + short rts_badrtgwroute; /* route to gateway is not direct */ }; /* @@ -384,10 +436,6 @@ struct rt_msghdr_ext { #define RTM_GET_EXT 0x15 #endif /* PRIVATE */ -#ifdef BSD_KERNEL_PRIVATE -/* RTM flags */ -#define RTMF_HIDE_LLADDR 0x00000001 -#endif /* BSD_KERNEL_PRIVATE */ /* * Bitmask values for rtm_inits and rmx_locks. */ @@ -454,10 +502,10 @@ typedef struct ctrace { extern void ctrace_record(ctrace_t *); #define RT_LOCK_ASSERT_HELD(_rt) \ - lck_mtx_assert(&(_rt)->rt_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_rt)->rt_lock, LCK_MTX_ASSERT_OWNED) #define RT_LOCK_ASSERT_NOTHELD(_rt) \ - lck_mtx_assert(&(_rt)->rt_lock, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_rt)->rt_lock, LCK_MTX_ASSERT_NOTOWNED) #define RT_LOCK(_rt) do { \ rt_lock(_rt, FALSE); \ @@ -581,7 +629,7 @@ extern unsigned int sin6_get_ifscope(struct sockaddr *); extern void rt_lock(struct rtentry *, boolean_t); extern void rt_unlock(struct rtentry *); extern struct sockaddr *rtm_scrub(int, int, struct sockaddr *, - struct sockaddr *, void *, uint32_t, kauth_cred_t *, uint32_t); + struct sockaddr *, void *, uint32_t, kauth_cred_t *); extern boolean_t rt_validate(struct rtentry *); extern void rt_set_proxy(struct rtentry *, boolean_t); extern void rt_set_gwroute(struct rtentry *, struct sockaddr *, @@ -597,5 +645,38 @@ extern void route_copyout(struct route *, const struct route *, size_t); extern boolean_t rt_ifa_is_dst(struct sockaddr *, struct ifaddr *); extern struct sockaddr *sa_copy(struct sockaddr *, struct sockaddr_storage *, unsigned int *); + +/* + * The following is used to enqueue work items for route events + * and also used to pass route event while walking the tree + */ +struct route_event { + struct rtentry *rt; + /* + * There's no reference taken on gwrt. + * We only use it to check whether we should + * point to rt_gateway or the embedded rt_addr + * structure. + */ + struct rtentry *gwrt; + union { + union sockaddr_in_4_6 _rtev_ipaddr; + struct sockaddr_dl _rtev_lladdr; + char _rtev_addr_bytes[DLIL_SDLMAXLEN]; + } rt_addr; + uint32_t route_event_code; + eventhandler_tag evtag; +}; + +#define rtev_ipaddr rt_addr._rtev_ipaddr +#define rtev_lladdr rt_addr._rtev_lladdr +#define rtev_addr_bytes rt_addr._rtev_addr_bytes + +extern void route_event_init(struct route_event *p_route_ev, struct rtentry *rt, + struct rtentry *gwrt, int route_ev_code); +extern int route_event_walktree(struct radix_node *rn, void *arg); +extern void route_event_enqueue_nwk_wq_entry(struct rtentry *, struct rtentry *, + uint32_t, eventhandler_tag, boolean_t); + #endif /* BSD_KERNEL_PRIVATE */ #endif /* _NET_ROUTE_H_ */ diff --git a/bsd/net/rtsock.c b/bsd/net/rtsock.c index f66639877..457eaf4eb 100644 --- a/bsd/net/rtsock.c +++ b/bsd/net/rtsock.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -75,6 +75,7 @@ #include <sys/syslog.h> #include <sys/mcache.h> #include <kern/locks.h> +#include <sys/codesign.h> #include <net/if.h> #include <net/route.h> @@ -131,7 +132,7 @@ static void rt_setif(struct rtentry *, struct sockaddr *, struct sockaddr *, static int rt_xaddrs(caddr_t, caddr_t, struct rt_addrinfo *); static struct mbuf *rt_msg1(int, struct rt_addrinfo *); static int rt_msg2(int, struct rt_addrinfo *, caddr_t, struct walkarg *, - kauth_cred_t *, uint32_t); + kauth_cred_t *); static int sysctl_dumpentry(struct radix_node *rn, void *vw); static int sysctl_dumpentry_ext(struct radix_node *rn, void *vw); static int sysctl_iflist(int af, struct walkarg *w); @@ -145,6 +146,11 @@ SYSCTL_NODE(_net, PF_ROUTE, routetable, CTLFLAG_RD | CTLFLAG_LOCKED, SYSCTL_NODE(_net, OID_AUTO, route, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "routing"); +/* Align x to 1024 (only power of 2) assuming x is positive */ +#define ALIGN_BYTES(x) do { \ + x = P2ALIGN(x, 1024); \ +} while(0) + #define ROUNDUP32(a) \ ((a) > 0 ? (1 + (((a) - 1) | (sizeof (uint32_t) - 1))) : \ sizeof (uint32_t)) @@ -307,7 +313,7 @@ route_output(struct mbuf *m, struct socket *so) int sendonlytoself = 0; unsigned int ifscope = IFSCOPE_NONE; struct rawcb *rp = NULL; - uint32_t rtm_hint_flags = 0; + boolean_t is_router = FALSE; #define senderr(e) { error = (e); goto flush; } if (m == NULL || ((m->m_len < sizeof (intptr_t)) && (m = m_pullup(m, sizeof (intptr_t))) == NULL)) @@ -535,9 +541,6 @@ route_output(struct mbuf *m, struct socket *so) senderr(ESRCH); RT_LOCK(rt); - if (rt->rt_ifp == lo_ifp) - rtm_hint_flags |= RTMF_HIDE_LLADDR; - /* * Holding rnh_lock here prevents the possibility of * ifa from changing (e.g. in_ifinit), so it is safe @@ -577,9 +580,7 @@ report: } if (ifa2 != NULL) IFA_LOCK(ifa2); - - len = rt_msg2(rtm->rtm_type, &info, NULL, NULL, &cred, rtm_hint_flags); - + len = rt_msg2(rtm->rtm_type, &info, NULL, NULL, &cred); if (ifa2 != NULL) IFA_UNLOCK(ifa2); if (len > rtm->rtm_msglen) { @@ -596,10 +597,8 @@ report: } if (ifa2 != NULL) IFA_LOCK(ifa2); - (void) rt_msg2(rtm->rtm_type, &info, (caddr_t)rtm, - NULL, &cred, rtm_hint_flags); - + NULL, &cred); if (ifa2 != NULL) IFA_UNLOCK(ifa2); rtm->rtm_flags = rt->rt_flags; @@ -607,10 +606,14 @@ report: rtm->rtm_addrs = info.rti_addrs; if (ifa2 != NULL) IFA_REMREF(ifa2); + + kauth_cred_unref(&cred); break; } case RTM_CHANGE: + is_router = (rt->rt_flags & RTF_ROUTER) ? TRUE : FALSE; + if (info.rti_info[RTAX_GATEWAY] != NULL && (error = rt_setgate(rt, rt_key(rt), info.rti_info[RTAX_GATEWAY]))) { @@ -623,7 +626,7 @@ report: * the required gateway, then just use the old one. * This can happen if the user tries to change the * flags on the default route without changing the - * default gateway. Changing flags still doesn't work. + * default gateway. Changing flags still doesn't work. */ if ((rt->rt_flags & RTF_GATEWAY) && info.rti_info[RTAX_GATEWAY] == NULL) @@ -646,6 +649,24 @@ report: } if (info.rti_info[RTAX_GENMASK]) rt->rt_genmask = info.rti_info[RTAX_GENMASK]; + + /* + * Enqueue work item to invoke callback for this route entry + * This may not be needed always, but for now issue it anytime + * RTM_CHANGE gets called. + */ + route_event_enqueue_nwk_wq_entry(rt, NULL, ROUTE_ENTRY_REFRESH, NULL, TRUE); + /* + * If the route is for a router, walk the tree to send refresh + * event to protocol cloned entries + */ + if (is_router) { + struct route_event rt_ev; + route_event_init(&rt_ev, rt, NULL, ROUTE_ENTRY_REFRESH); + RT_UNLOCK(rt); + (void) rnh->rnh_walktree(rnh, route_event_walktree, (void *)&rt_ev); + RT_LOCK(rt); + } /* FALLTHRU */ case RTM_LOCK: rt->rt_rmx.rmx_locks &= ~(rtm->rtm_inits); @@ -822,7 +843,7 @@ rt_setif(struct rtentry *rt, struct sockaddr *Ifpaddr, struct sockaddr *Ifaaddr, struct ifnet *ifp = NULL; void (*ifa_rtrequest)(int, struct rtentry *, struct sockaddr *); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK_ASSERT_HELD(rt); @@ -1082,7 +1103,7 @@ rt_msg1(int type, struct rt_addrinfo *rtinfo) /* Scrub away any trace of embedded interface scope */ sa = rtm_scrub(type, i, hint, sa, &ssbuf, - sizeof (ssbuf), NULL, 0); + sizeof (ssbuf), NULL); break; default: @@ -1107,7 +1128,7 @@ rt_msg1(int type, struct rt_addrinfo *rtinfo) static int rt_msg2(int type, struct rt_addrinfo *rtinfo, caddr_t cp, struct walkarg *w, - kauth_cred_t* credp, uint32_t rtm_hint_flags) + kauth_cred_t* credp) { int i; int len, dlen, rlen, second_time = 0; @@ -1173,12 +1194,12 @@ again: /* Scrub away any trace of embedded interface scope */ sa = rtm_scrub(type, i, hint, sa, &ssbuf, - sizeof (ssbuf), NULL, rtm_hint_flags); + sizeof (ssbuf), NULL); break; case RTAX_GATEWAY: case RTAX_IFP: sa = rtm_scrub(type, i, NULL, sa, &ssbuf, - sizeof (ssbuf), credp, rtm_hint_flags); + sizeof (ssbuf), credp); break; default: @@ -1298,7 +1319,7 @@ rt_newaddrmsg(int cmd, struct ifaddr *ifa, int error, struct rtentry *rt) struct ifnet *ifp = ifa->ifa_ifp; struct sockproto route_proto = { PF_ROUTE, 0 }; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK_ASSERT_HELD(rt); if (route_cb.any_count == 0) @@ -1493,7 +1514,6 @@ sysctl_dumpentry(struct radix_node *rn, void *vw) int error = 0, size; struct rt_addrinfo info; kauth_cred_t cred; - uint32_t rtm_hint_flags = 0; cred = kauth_cred_proc_ref(current_proc()); @@ -1506,11 +1526,8 @@ sysctl_dumpentry(struct radix_node *rn, void *vw) info.rti_info[RTAX_NETMASK] = rt_mask(rt); info.rti_info[RTAX_GENMASK] = rt->rt_genmask; - if (rt->rt_ifp == lo_ifp) - rtm_hint_flags |= RTMF_HIDE_LLADDR; - if (w->w_op != NET_RT_DUMP2) { - size = rt_msg2(RTM_GET, &info, NULL, w, &cred, rtm_hint_flags); + size = rt_msg2(RTM_GET, &info, NULL, w, &cred); if (w->w_req != NULL && w->w_tmem != NULL) { struct rt_msghdr *rtm = (struct rt_msghdr *)(void *)w->w_tmem; @@ -1526,7 +1543,7 @@ sysctl_dumpentry(struct radix_node *rn, void *vw) error = SYSCTL_OUT(w->w_req, (caddr_t)rtm, size); } } else { - size = rt_msg2(RTM_GET2, &info, NULL, w, &cred, rtm_hint_flags); + size = rt_msg2(RTM_GET2, &info, NULL, w, &cred); if (w->w_req != NULL && w->w_tmem != NULL) { struct rt_msghdr2 *rtm = (struct rt_msghdr2 *)(void *)w->w_tmem; @@ -1563,7 +1580,6 @@ sysctl_dumpentry_ext(struct radix_node *rn, void *vw) int error = 0, size; struct rt_addrinfo info; kauth_cred_t cred; - uint32_t rtm_hint_flags = 0; cred = kauth_cred_proc_ref(current_proc()); @@ -1576,10 +1592,7 @@ sysctl_dumpentry_ext(struct radix_node *rn, void *vw) info.rti_info[RTAX_NETMASK] = rt_mask(rt); info.rti_info[RTAX_GENMASK] = rt->rt_genmask; - if (rt->rt_ifp == lo_ifp) - rtm_hint_flags |= RTMF_HIDE_LLADDR; - - size = rt_msg2(RTM_GET_EXT, &info, NULL, w, &cred, rtm_hint_flags); + size = rt_msg2(RTM_GET_EXT, &info, NULL, w, &cred); if (w->w_req != NULL && w->w_tmem != NULL) { struct rt_msghdr_ext *ertm = (struct rt_msghdr_ext *)(void *)w->w_tmem; @@ -1628,7 +1641,7 @@ sysctl_iflist(int af, struct walkarg *w) struct ifnet *ifp; struct ifaddr *ifa; struct rt_addrinfo info; - int len, error = 0; + int len = 0, error = 0; int pass = 0; int total_len = 0, current_len = 0; char *total_buffer = NULL, *cp = NULL; @@ -1655,7 +1668,7 @@ sysctl_iflist(int af, struct walkarg *w) */ ifa = ifp->if_lladdr; info.rti_info[RTAX_IFP] = ifa->ifa_addr; - len = rt_msg2(RTM_IFINFO, &info, NULL, NULL, &cred, RTMF_HIDE_LLADDR); + len = rt_msg2(RTM_IFINFO, &info, NULL, NULL, &cred); if (pass == 0) { total_len += len; } else { @@ -1668,7 +1681,7 @@ sysctl_iflist(int af, struct walkarg *w) } info.rti_info[RTAX_IFP] = ifa->ifa_addr; len = rt_msg2(RTM_IFINFO, &info, - (caddr_t)cp, NULL, &cred, RTMF_HIDE_LLADDR); + (caddr_t)cp, NULL, &cred); info.rti_info[RTAX_IFP] = NULL; ifm = (struct if_msghdr *)(void *)cp; @@ -1677,6 +1690,14 @@ sysctl_iflist(int af, struct walkarg *w) if_data_internal_to_if_data(ifp, &ifp->if_data, &ifm->ifm_data); ifm->ifm_addrs = info.rti_addrs; + /* + * <rdar://problem/32940901> + * Round bytes only for non-platform + */ + if (!csproc_get_platform_binary(w->w_req->p)) { + ALIGN_BYTES(ifm->ifm_data.ifi_ibytes); + ALIGN_BYTES(ifm->ifm_data.ifi_obytes); + } cp += len; VERIFY(IS_P2ALIGNED(cp, sizeof (u_int32_t))); @@ -1692,7 +1713,7 @@ sysctl_iflist(int af, struct walkarg *w) info.rti_info[RTAX_NETMASK] = ifa->ifa_netmask; info.rti_info[RTAX_BRD] = ifa->ifa_dstaddr; len = rt_msg2(RTM_NEWADDR, &info, NULL, NULL, - &cred, RTMF_HIDE_LLADDR); + &cred); if (pass == 0) { total_len += len; } else { @@ -1704,7 +1725,7 @@ sysctl_iflist(int af, struct walkarg *w) break; } len = rt_msg2(RTM_NEWADDR, &info, - (caddr_t)cp, NULL, &cred, RTMF_HIDE_LLADDR); + (caddr_t)cp, NULL, &cred); ifam = (struct ifa_msghdr *)(void *)cp; ifam->ifam_index = @@ -1770,7 +1791,7 @@ sysctl_iflist2(int af, struct walkarg *w) struct ifnet *ifp; struct ifaddr *ifa; struct rt_addrinfo info; - int len, error = 0; + int len = 0, error = 0; int pass = 0; int total_len = 0, current_len = 0; char *total_buffer = NULL, *cp = NULL; @@ -1799,7 +1820,7 @@ sysctl_iflist2(int af, struct walkarg *w) */ ifa = ifp->if_lladdr; info.rti_info[RTAX_IFP] = ifa->ifa_addr; - len = rt_msg2(RTM_IFINFO2, &info, NULL, NULL, &cred, RTMF_HIDE_LLADDR); + len = rt_msg2(RTM_IFINFO2, &info, NULL, NULL, &cred); if (pass == 0) { total_len += len; } else { @@ -1812,7 +1833,7 @@ sysctl_iflist2(int af, struct walkarg *w) } info.rti_info[RTAX_IFP] = ifa->ifa_addr; len = rt_msg2(RTM_IFINFO2, &info, - (caddr_t)cp, NULL, &cred, RTMF_HIDE_LLADDR); + (caddr_t)cp, NULL, &cred); info.rti_info[RTAX_IFP] = NULL; ifm = (struct if_msghdr2 *)(void *)cp; @@ -1826,6 +1847,14 @@ sysctl_iflist2(int af, struct walkarg *w) ifm->ifm_timer = ifp->if_timer; if_data_internal_to_if_data64(ifp, &ifp->if_data, &ifm->ifm_data); + /* + * <rdar://problem/32940901> + * Round bytes only for non-platform + */ + if (!csproc_get_platform_binary(w->w_req->p)) { + ALIGN_BYTES(ifm->ifm_data.ifi_ibytes); + ALIGN_BYTES(ifm->ifm_data.ifi_obytes); + } cp += len; VERIFY(IS_P2ALIGNED(cp, sizeof (u_int32_t))); @@ -1841,7 +1870,7 @@ sysctl_iflist2(int af, struct walkarg *w) info.rti_info[RTAX_NETMASK] = ifa->ifa_netmask; info.rti_info[RTAX_BRD] = ifa->ifa_dstaddr; len = rt_msg2(RTM_NEWADDR, &info, NULL, NULL, - &cred, RTMF_HIDE_LLADDR); + &cred); if (pass == 0) { total_len += len; } else { @@ -1853,7 +1882,7 @@ sysctl_iflist2(int af, struct walkarg *w) break; } len = rt_msg2(RTM_NEWADDR, &info, - (caddr_t)cp, NULL, &cred, RTMF_HIDE_LLADDR); + (caddr_t)cp, NULL, &cred); ifam = (struct ifa_msghdr *)(void *)cp; ifam->ifam_index = @@ -1897,7 +1926,7 @@ sysctl_iflist2(int af, struct walkarg *w) info.rti_info[RTAX_GATEWAY] = ifma->ifma_ll->ifma_addr; len = rt_msg2(RTM_NEWMADDR2, &info, NULL, NULL, - &cred, RTMF_HIDE_LLADDR); + &cred); if (pass == 0) { total_len += len; } else { @@ -1909,7 +1938,7 @@ sysctl_iflist2(int af, struct walkarg *w) break; } len = rt_msg2(RTM_NEWMADDR2, &info, - (caddr_t)cp, NULL, &cred, RTMF_HIDE_LLADDR); + (caddr_t)cp, NULL, &cred); ifmam = (struct ifma_msghdr2 *)(void *)cp; diff --git a/bsd/net/skmem_sysctl.c b/bsd/net/skmem_sysctl.c new file mode 100644 index 000000000..3fe18dc63 --- /dev/null +++ b/bsd/net/skmem_sysctl.c @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include <sys/sysctl.h> + diff --git a/bsd/netinet/Makefile b/bsd/netinet/Makefile index ea575bddc..6eddebfe3 100644 --- a/bsd/netinet/Makefile +++ b/bsd/netinet/Makefile @@ -35,7 +35,8 @@ PRIVATE_DATAFILES = \ tcp_debug.h \ tcp_var.h \ tcp_cache.h \ - udp.h + udp.h \ + in_stat.h PRIVATE_KERNELFILES = ${KERNELFILES} \ ip_ecn.h ip_encap.h diff --git a/bsd/netinet/cpu_in_cksum.c b/bsd/netinet/cpu_in_cksum_gen.c similarity index 71% rename from bsd/netinet/cpu_in_cksum.c rename to bsd/netinet/cpu_in_cksum_gen.c index a579371a4..3d88e15e4 100644 --- a/bsd/netinet/cpu_in_cksum.c +++ b/bsd/netinet/cpu_in_cksum_gen.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -55,18 +55,124 @@ * SUCH DAMAGE. */ +#ifdef KERNEL #include <sys/param.h> -#include <mach/boolean.h> #include <machine/endian.h> #include <sys/mcache.h> #include <sys/mbuf.h> #include <kern/debug.h> -#include <netinet/in.h> #include <libkern/libkern.h> +#include <mach/boolean.h> +#include <pexpert/pexpert.h> +#define CKSUM_ERR(fmt, args...) kprintf(fmt, ## args) +#else /* !KERNEL */ +#ifndef LIBSYSCALL_INTERFACE +#error "LIBSYSCALL_INTERFACE not defined" +#endif /* !LIBSYSCALL_INTERFACE */ +#include <stdlib.h> +#include <stddef.h> +#include <stdint.h> +#include <unistd.h> +#include <strings.h> +#include <mach/boolean.h> +#endif /* !KERNEL */ + +/* compile time assert */ +#ifndef _CASSERT +#define _CASSERT(x) _Static_assert(x, "compile-time assertion failed") +#endif /* !_CASSERT */ + +#ifndef VERIFY +#define VERIFY(EX) ((void)0) +#endif /* !VERIFY */ + +#ifndef CKSUM_ERR +#define CKSUM_ERR(fmt, args...) ((void)0) +#endif /* !CKSUM_ERR */ -int cpu_in_cksum(struct mbuf *, int, int, uint32_t); +#define PREDICT_TRUE(x) __builtin_expect(!!((long)(x)), 1L) +#define PREDICT_FALSE(x) __builtin_expect(!!((long)(x)), 0L) -#define PREDICT_FALSE(_exp) __builtin_expect((_exp), 0) +/* fake mbuf struct used only for calling os_cpu_in_cksum_mbuf() */ +struct _mbuf { + struct _mbuf *_m_next; + void *_m_pad; + uint8_t *_m_data; + int32_t _m_len; +}; + +extern uint32_t os_cpu_in_cksum(const void *, uint32_t, uint32_t); +extern uint32_t os_cpu_in_cksum_mbuf(struct _mbuf *, int, int, uint32_t); + +uint32_t +os_cpu_in_cksum(const void *data, uint32_t len, uint32_t initial_sum) +{ + /* + * If data is 4-bytes aligned, length is multiple of 4-bytes, + * and the amount to checksum is small, this would be quicker; + * this is suitable for IPv4 header. + */ + if (IS_P2ALIGNED(data, sizeof (uint32_t)) && + len <= 64 && (len & 3) == 0) { + uint8_t *p = __DECONST(uint8_t *, data); + uint64_t sum = initial_sum; + + if (PREDICT_TRUE(len == 20)) { /* simple IPv4 header */ + sum += *(uint32_t *)(void *)p; + sum += *(uint32_t *)(void *)(p + 4); + sum += *(uint32_t *)(void *)(p + 8); + sum += *(uint32_t *)(void *)(p + 12); + sum += *(uint32_t *)(void *)(p + 16); + } else { + while (len) { + sum += *(uint32_t *)(void *)p; + p += 4; + len -= 4; + } + } + + /* fold 64-bit to 16-bit (deferred carries) */ + sum = (sum >> 32) + (sum & 0xffffffff); /* 33-bit */ + sum = (sum >> 16) + (sum & 0xffff); /* 17-bit + carry */ + sum = (sum >> 16) + (sum & 0xffff); /* 16-bit + carry */ + sum = (sum >> 16) + (sum & 0xffff); /* final carry */ + + return (sum & 0xffff); + } + + /* + * Otherwise, let os_cpu_in_cksum_mbuf() handle it; it only looks + * at 3 fields: {next,data,len}, and since it doesn't care about + * the authenticity of the mbuf, we use a fake one here. Make + * sure the offsets are as expected. + */ +#if defined(__LP64__) + _CASSERT(offsetof(struct _mbuf, _m_next) == 0); + _CASSERT(offsetof(struct _mbuf, _m_data) == 16); + _CASSERT(offsetof(struct _mbuf, _m_len) == 24); +#else /* !__LP64__ */ + _CASSERT(offsetof(struct _mbuf, _m_next) == 0); + _CASSERT(offsetof(struct _mbuf, _m_data) == 8); + _CASSERT(offsetof(struct _mbuf, _m_len) == 12); +#endif /* !__LP64__ */ +#ifdef KERNEL + _CASSERT(offsetof(struct _mbuf, _m_next) == + offsetof(struct mbuf, m_next)); + _CASSERT(offsetof(struct _mbuf, _m_data) == + offsetof(struct mbuf, m_data)); + _CASSERT(offsetof(struct _mbuf, _m_len) == + offsetof(struct mbuf, m_len)); +#endif /* KERNEL */ + struct _mbuf m = { + ._m_next = NULL, + ._m_data = __DECONST(uint8_t *, data), + ._m_len = len, + }; + + return (os_cpu_in_cksum_mbuf(&m, len, 0, initial_sum)); +} + +#if defined(__i386__) || defined(__x86_64__) /* * Checksum routine for Internet Protocol family headers (Portable Version). @@ -88,10 +194,10 @@ int cpu_in_cksum(struct mbuf *, int, int, uint32_t); * reduction is done to avoid carry in long packets. */ -#if ULONG_MAX == 0xffffffffUL +#if !defined(__LP64__) /* 32-bit version */ -int -cpu_in_cksum(struct mbuf *m, int len, int off, uint32_t initial_sum) +uint32_t +os_cpu_in_cksum_mbuf(struct _mbuf *m, int len, int off, uint32_t initial_sum) { int mlen; uint32_t sum, partial; @@ -108,28 +214,28 @@ cpu_in_cksum(struct mbuf *m, int len, int off, uint32_t initial_sum) for (;;) { if (PREDICT_FALSE(m == NULL)) { - printf("%s: out of data\n", __func__); - return (-1); + CKSUM_ERR("%s: out of data\n", __func__); + return ((uint32_t)-1); } - mlen = m->m_len; + mlen = m->_m_len; if (mlen > off) { mlen -= off; - data = mtod(m, uint8_t *) + off; + data = m->_m_data + off; goto post_initial_offset; } off -= mlen; if (len == 0) break; - m = m->m_next; + m = m->_m_next; } - for (; len > 0; m = m->m_next) { + for (; len > 0; m = m->_m_next) { if (PREDICT_FALSE(m == NULL)) { - printf("%s: out of data\n", __func__); - return (-1); + CKSUM_ERR("%s: out of data\n", __func__); + return ((uint32_t)-1); } - mlen = m->m_len; - data = mtod(m, uint8_t *); + mlen = m->_m_len; + data = m->_m_data; post_initial_offset: if (mlen == 0) continue; @@ -231,13 +337,13 @@ post_initial_offset: } final_acc = ((sum >> 16) & 0xffff) + (sum & 0xffff); final_acc = (final_acc >> 16) + (final_acc & 0xffff); - return (~final_acc & 0xffff); + return (final_acc & 0xffff); } -#else +#else /* __LP64__ */ /* 64-bit version */ -int -cpu_in_cksum(struct mbuf *m, int len, int off, uint32_t initial_sum) +uint32_t +os_cpu_in_cksum_mbuf(struct _mbuf *m, int len, int off, uint32_t initial_sum) { int mlen; uint64_t sum, partial; @@ -254,28 +360,28 @@ cpu_in_cksum(struct mbuf *m, int len, int off, uint32_t initial_sum) for (;;) { if (PREDICT_FALSE(m == NULL)) { - printf("%s: out of data\n", __func__); - return (-1); + CKSUM_ERR("%s: out of data\n", __func__); + return ((uint32_t)-1); } - mlen = m->m_len; + mlen = m->_m_len; if (mlen > off) { mlen -= off; - data = mtod(m, uint8_t *) + off; + data = m->_m_data + off; goto post_initial_offset; } off -= mlen; if (len == 0) break; - m = m->m_next; + m = m->_m_next; } - for (; len > 0; m = m->m_next) { + for (; len > 0; m = m->_m_next) { if (PREDICT_FALSE(m == NULL)) { - printf("%s: out of data\n", __func__); - return (-1); + CKSUM_ERR("%s: out of data\n", __func__); + return ((uint32_t)-1); } - mlen = m->m_len; - data = mtod(m, uint8_t *); + mlen = m->_m_len; + data = m->_m_data; post_initial_offset: if (mlen == 0) continue; @@ -391,6 +497,8 @@ trailing_bytes: ((sum >> 16) & 0xffff) + (sum & 0xffff); final_acc = (final_acc >> 16) + (final_acc & 0xffff); final_acc = (final_acc >> 16) + (final_acc & 0xffff); - return (~final_acc & 0xffff); + return (final_acc & 0xffff); } -#endif /* ULONG_MAX != 0xffffffffUL */ +#endif /* __LP64 */ + +#endif /* __i386__ || __x86_64__ */ diff --git a/bsd/netinet/flow_divert.c b/bsd/netinet/flow_divert.c index 1e46e42c4..c5e974aab 100644 --- a/bsd/netinet/flow_divert.c +++ b/bsd/netinet/flow_divert.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -601,6 +601,7 @@ flow_divert_add_data_statistics(struct flow_divert_pcb *fd_cb, int data_len, Boo INP_ADD_STAT(inp, cell, wifi, wired, rxpackets, 1); INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, data_len); } + inp_set_activity_bitmap(inp); } static errno_t @@ -1037,7 +1038,11 @@ flow_divert_create_connect_packet(struct flow_divert_pcb *fd_cb, struct sockaddr if (signing_id != NULL) { uint16_t result = NULL_TRIE_IDX; lck_rw_lock_shared(&fd_cb->group->lck); - result = flow_divert_trie_search(&fd_cb->group->signing_id_trie, (uint8_t *)signing_id); + if (fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP) { + result = 1; + } else { + result = flow_divert_trie_search(&fd_cb->group->signing_id_trie, (uint8_t *)signing_id); + } lck_rw_done(&fd_cb->group->lck); if (result != NULL_TRIE_IDX) { error = 0; @@ -1360,9 +1365,11 @@ flow_divert_send_data_packet(struct flow_divert_pcb *fd_cb, mbuf_t data, size_t } } - last = m_last(packet); - mbuf_setnext(last, data); - mbuf_pkthdr_adjustlen(packet, data_len); + if (data_len > 0 && data != NULL) { + last = m_last(packet); + mbuf_setnext(last, data); + mbuf_pkthdr_adjustlen(packet, data_len); + } error = flow_divert_send_packet(fd_cb, packet, force); if (error) { @@ -1447,11 +1454,15 @@ flow_divert_send_buffered_data(struct flow_divert_pcb *fd_cb, Boolean force) } } data_len = mbuf_pkthdr_len(m); - FDLOG(LOG_DEBUG, fd_cb, "mbuf_copym() data_len = %lu", data_len); - error = mbuf_copym(m, 0, data_len, MBUF_DONTWAIT, &data); - if (error) { - FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error); - break; + if (data_len > 0) { + FDLOG(LOG_DEBUG, fd_cb, "mbuf_copym() data_len = %lu", data_len); + error = mbuf_copym(m, 0, data_len, MBUF_DONTWAIT, &data); + if (error) { + FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error); + break; + } + } else { + data = NULL; } error = flow_divert_send_data_packet(fd_cb, data, data_len, toaddr, force); if (error) { @@ -1551,7 +1562,7 @@ flow_divert_send_app_data(struct flow_divert_pcb *fd_cb, mbuf_t data, struct soc } } } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) { - if (to_send) { + if (to_send || mbuf_pkthdr_len(data) == 0) { error = flow_divert_send_data_packet(fd_cb, data, to_send, toaddr, FALSE); if (error) { FDLOG(LOG_ERR, fd_cb, "flow_divert_send_data_packet failed. send data size = %lu", to_send); @@ -2041,6 +2052,7 @@ flow_divert_handle_group_init(struct flow_divert_group *group, mbuf_t packet, in int error = 0; uint32_t key_size = 0; int log_level; + uint32_t flags = 0; error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, 0, NULL, &key_size); if (error) { @@ -2072,6 +2084,11 @@ flow_divert_handle_group_init(struct flow_divert_group *group, mbuf_t packet, in group->token_key_size = key_size; + error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_FLAGS, sizeof(flags), &flags, NULL); + if (!error) { + group->flags = flags; + } + lck_rw_done(&group->lck); } @@ -3133,7 +3150,7 @@ flow_divert_preconnect(struct socket *so) fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED; } - so->so_flags1 &= ~SOF1_PRECONNECT_DATA; + soclearfastopen(so); return error; } diff --git a/bsd/netinet/flow_divert.h b/bsd/netinet/flow_divert.h index 47abceaa3..c430a3935 100644 --- a/bsd/netinet/flow_divert.h +++ b/bsd/netinet/flow_divert.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2015 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -84,6 +84,7 @@ struct flow_divert_group { MBUFQ_HEAD(send_queue_head) send_queue; uint8_t *token_key; size_t token_key_size; + uint32_t flags; struct flow_divert_trie signing_id_trie; }; diff --git a/bsd/netinet/flow_divert_proto.h b/bsd/netinet/flow_divert_proto.h index 934746d01..675444b77 100644 --- a/bsd/netinet/flow_divert_proto.h +++ b/bsd/netinet/flow_divert_proto.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2015 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -81,6 +81,8 @@ #define FLOW_DIVERT_TOKEN_FLAG_TFO 0x0000002 #define FLOW_DIVERT_TOKEN_FLAG_MPTCP 0x0000004 +#define FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP 0x0000001 + struct flow_divert_packet_header { uint8_t packet_type; uint32_t conn_id; diff --git a/bsd/netinet/icmp6.h b/bsd/netinet/icmp6.h index 52228f43c..2176a0299 100644 --- a/bsd/netinet/icmp6.h +++ b/bsd/netinet/icmp6.h @@ -329,7 +329,7 @@ struct nd_opt_hdr { /* Neighbor discovery option header */ #define ND_OPT_REDIRECTED_HEADER 4 #define ND_OPT_MTU 5 #define ND_OPT_NONCE 14 /* RFC 3971 */ -#define ND_OPT_RDNSS 25 /* RFC 5006 */ +#define ND_OPT_RDNSS 25 /* RFC 6106 */ #define ND_OPT_DNSSL 31 /* RFC 6106 */ #define ND_OPT_ROUTE_INFO 200 /* draft-ietf-ipngwg-router-preference, not officially assigned yet */ diff --git a/bsd/netinet/igmp.c b/bsd/netinet/igmp.c index f2cd0966e..b96b869fa 100644 --- a/bsd/netinet/igmp.c +++ b/bsd/netinet/igmp.c @@ -190,9 +190,9 @@ static int current_state_timers_running; /* IGMPv1/v2 host #define IGMP_LOCK() \ lck_mtx_lock(&igmp_mtx) #define IGMP_LOCK_ASSERT_HELD() \ - lck_mtx_assert(&igmp_mtx, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&igmp_mtx, LCK_MTX_ASSERT_OWNED) #define IGMP_LOCK_ASSERT_NOTHELD() \ - lck_mtx_assert(&igmp_mtx, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&igmp_mtx, LCK_MTX_ASSERT_NOTOWNED) #define IGMP_UNLOCK() \ lck_mtx_unlock(&igmp_mtx) @@ -555,7 +555,7 @@ igmp_ra_alloc(void) MGET(m, M_WAITOK, MT_DATA); p = mtod(m, struct ipoption *); p->ipopt_dst.s_addr = INADDR_ANY; - p->ipopt_list[0] = IPOPT_RA; /* Router Alert Option */ + p->ipopt_list[0] = (char)IPOPT_RA; /* Router Alert Option */ p->ipopt_list[1] = 0x04; /* 4 bytes long */ p->ipopt_list[2] = IPOPT_EOL; /* End of IP option list */ p->ipopt_list[3] = 0x00; /* pad byte */ diff --git a/bsd/netinet/igmp_var.h b/bsd/netinet/igmp_var.h index 0d299ef24..22181c9e3 100644 --- a/bsd/netinet/igmp_var.h +++ b/bsd/netinet/igmp_var.h @@ -273,10 +273,10 @@ struct igmp_ifinfo { }; #define IGI_LOCK_ASSERT_HELD(_igi) \ - lck_mtx_assert(&(_igi)->igi_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_igi)->igi_lock, LCK_MTX_ASSERT_OWNED) #define IGI_LOCK_ASSERT_NOTHELD(_igi) \ - lck_mtx_assert(&(_igi)->igi_lock, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_igi)->igi_lock, LCK_MTX_ASSERT_NOTOWNED) #define IGI_LOCK(_igi) \ lck_mtx_lock(&(_igi)->igi_lock) diff --git a/bsd/netinet/in.c b/bsd/netinet/in.c index 0645f9670..cbb7f8cb6 100644 --- a/bsd/netinet/in.c +++ b/bsd/netinet/in.c @@ -83,6 +83,8 @@ #include <net/route.h> #include <net/kpi_protocol.h> #include <net/dlil.h> +#include <net/if_llatbl.h> +#include <net/if_arp.h> #if PF #include <net/pfvar.h> #endif /* PF */ @@ -95,6 +97,7 @@ #include <netinet/tcp.h> #include <netinet/tcp_timer.h> #include <netinet/tcp_var.h> +#include <netinet/if_ether.h> static int inctl_associd(struct socket *, u_long, caddr_t); static int inctl_connid(struct socket *, u_long, caddr_t); @@ -135,9 +138,24 @@ static void in_ifaddr_trace(struct ifaddr *, int); static int in_getassocids(struct socket *, uint32_t *, user_addr_t); static int in_getconnids(struct socket *, sae_associd_t, uint32_t *, user_addr_t); -static int in_getconninfo(struct socket *, sae_connid_t, uint32_t *, - uint32_t *, int32_t *, user_addr_t, socklen_t *, user_addr_t, socklen_t *, - uint32_t *, user_addr_t, uint32_t *); + +/* IPv4 Layer 2 neighbor cache management routines */ +static void in_lltable_destroy_lle_unlocked(struct llentry *lle); +static void in_lltable_destroy_lle(struct llentry *lle); +static struct llentry *in_lltable_new(struct in_addr addr4, u_int flags); +static int in_lltable_match_prefix(const struct sockaddr *saddr, + const struct sockaddr *smask, u_int flags, struct llentry *lle); +static void in_lltable_free_entry(struct lltable *llt, struct llentry *lle); +static int in_lltable_rtcheck(struct ifnet *ifp, u_int flags, const struct sockaddr *l3addr); +static inline uint32_t in_lltable_hash_dst(const struct in_addr dst, uint32_t hsize); +static uint32_t in_lltable_hash(const struct llentry *lle, uint32_t hsize); +static void in_lltable_fill_sa_entry(const struct llentry *lle, struct sockaddr *sa); +static inline struct llentry * in_lltable_find_dst(struct lltable *llt, struct in_addr dst); +static void in_lltable_delete_entry(struct lltable *llt, struct llentry *lle); +static struct llentry * in_lltable_alloc(struct lltable *llt, u_int flags, const struct sockaddr *l3addr); +static struct llentry * in_lltable_lookup(struct lltable *llt, u_int flags, const struct sockaddr *l3addr); +static int in_lltable_dump_entry(struct lltable *llt, struct llentry *lle, struct sysctl_req *wr); +static struct lltable * in_lltattach(struct ifnet *ifp); static int subnetsarelocal = 0; SYSCTL_INT(_net_inet_ip, OID_AUTO, subnets_are_local, @@ -353,6 +371,7 @@ in_domifattach(struct ifnet *ifp) pbuf = (void **)((intptr_t)base - sizeof (void *)); *pbuf = ext; ifp->if_inetdata = base; + IN_IFEXTRA(ifp)->ii_llt = in_lltattach(ifp); VERIFY(IS_P2ALIGNED(ifp->if_inetdata, sizeof (uint64_t))); } done: @@ -1529,7 +1548,7 @@ in_ifscrub(struct ifnet *ifp, struct in_ifaddr *ia, int locked) static void in_iahash_remove(struct in_ifaddr *ia) { - lck_rw_assert(in_ifaddr_rwlock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(in_ifaddr_rwlock, LCK_RW_ASSERT_EXCLUSIVE); IFA_LOCK_ASSERT_HELD(&ia->ia_ifa); if (!IA_IS_HASHED(ia)) { @@ -1551,7 +1570,7 @@ in_iahash_remove(struct in_ifaddr *ia) static void in_iahash_insert(struct in_ifaddr *ia) { - lck_rw_assert(in_ifaddr_rwlock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(in_ifaddr_rwlock, LCK_RW_ASSERT_EXCLUSIVE); IFA_LOCK_ASSERT_HELD(&ia->ia_ifa); if (ia->ia_addr.sin_family != AF_INET) { @@ -1581,7 +1600,7 @@ in_iahash_insert_ptp(struct in_ifaddr *ia) struct in_ifaddr *tmp_ifa; struct ifnet *tmp_ifp; - lck_rw_assert(in_ifaddr_rwlock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(in_ifaddr_rwlock, LCK_RW_ASSERT_EXCLUSIVE); IFA_LOCK_ASSERT_HELD(&ia->ia_ifa); if (ia->ia_addr.sin_family != AF_INET) { @@ -2135,13 +2154,12 @@ in_getconnids(struct socket *so, sae_associd_t aid, uint32_t *cnt, /* * Handle SIOCGCONNINFO ioctl for PF_INET domain. */ -static int +int in_getconninfo(struct socket *so, sae_connid_t cid, uint32_t *flags, uint32_t *ifindex, int32_t *soerror, user_addr_t src, socklen_t *src_len, user_addr_t dst, socklen_t *dst_len, uint32_t *aux_type, user_addr_t aux_data, uint32_t *aux_len) { -#pragma unused(aux_data) struct inpcb *inp = sotoinpcb(so); struct sockaddr_in sin; struct ifnet *ifp = NULL; @@ -2209,8 +2227,6 @@ in_getconninfo(struct socket *so, sae_connid_t cid, uint32_t *flags, } } - *aux_type = 0; - *aux_len = 0; if (SOCK_PROTO(so) == IPPROTO_TCP) { struct conninfo_tcp tcp_ci; @@ -2228,8 +2244,362 @@ in_getconninfo(struct socket *so, sae_connid_t cid, uint32_t *flags, *aux_len = copy_len; } } + } else { + *aux_type = 0; + *aux_len = 0; } out: return (error); } + +struct in_llentry { + struct llentry base; +}; + +#define IN_LLTBL_DEFAULT_HSIZE 32 +#define IN_LLTBL_HASH(k, h) \ + ((((((((k) >> 8) ^ (k)) >> 8) ^ (k)) >> 8) ^ (k)) & ((h) - 1)) + +/* + * Do actual deallocation of @lle. + */ +static void +in_lltable_destroy_lle_unlocked(struct llentry *lle) +{ + LLE_LOCK_DESTROY(lle); + LLE_REQ_DESTROY(lle); + FREE(lle, M_LLTABLE); +} + +/* + * Called by LLE_FREE_LOCKED when number of references + * drops to zero. + */ +static void +in_lltable_destroy_lle(struct llentry *lle) +{ + LLE_WUNLOCK(lle); + in_lltable_destroy_lle_unlocked(lle); +} + +static struct llentry * +in_lltable_new(struct in_addr addr4, u_int flags) +{ +#pragma unused(flags) + struct in_llentry *lle; + + MALLOC(lle, struct in_llentry *, sizeof(struct in_llentry), M_LLTABLE, M_NOWAIT | M_ZERO); + if (lle == NULL) /* NB: caller generates msg */ + return NULL; + + /* + * For IPv4 this will trigger "arpresolve" to generate + * an ARP request. + */ + lle->base.la_expire = net_uptime(); /* mark expired */ + lle->base.r_l3addr.addr4 = addr4; + lle->base.lle_refcnt = 1; + lle->base.lle_free = in_lltable_destroy_lle; + + LLE_LOCK_INIT(&lle->base); + LLE_REQ_INIT(&lle->base); + //callout_init(&lle->base.lle_timer, 1); + + return (&lle->base); +} + +#define IN_ARE_MASKED_ADDR_EQUAL(d, a, m) ( \ + ((((d).s_addr ^ (a).s_addr) & (m).s_addr)) == 0 ) + +static int +in_lltable_match_prefix(const struct sockaddr *saddr, + const struct sockaddr *smask, u_int flags, struct llentry *lle) +{ + struct in_addr addr, mask, lle_addr; + + addr = ((const struct sockaddr_in *)(const void *)saddr)->sin_addr; + mask = ((const struct sockaddr_in *)(const void *)smask)->sin_addr; + lle_addr.s_addr = ntohl(lle->r_l3addr.addr4.s_addr); + + if (IN_ARE_MASKED_ADDR_EQUAL(lle_addr, addr, mask) == 0) + return (0); + + if (lle->la_flags & LLE_IFADDR) { + /* + * Delete LLE_IFADDR records IFF address & flag matches. + * Note that addr is the interface address within prefix + * being matched. + * Note also we should handle 'ifdown' cases without removing + * ifaddr macs. + */ + if (addr.s_addr == lle_addr.s_addr && (flags & LLE_STATIC) != 0) + return (1); + return (0); + } + + /* flags & LLE_STATIC means deleting both dynamic and static entries */ + if ((flags & LLE_STATIC) || !(lle->la_flags & LLE_STATIC)) + return (1); + + return (0); +} + +static void +in_lltable_free_entry(struct lltable *llt, struct llentry *lle) +{ + struct ifnet *ifp; + size_t pkts_dropped; + + LLE_WLOCK_ASSERT(lle); + KASSERT(llt != NULL, ("lltable is NULL")); + + /* Unlink entry from table if not already */ + if ((lle->la_flags & LLE_LINKED) != 0) { + ifp = llt->llt_ifp; + IF_AFDATA_WLOCK_ASSERT(ifp, llt->llt_af); + lltable_unlink_entry(llt, lle); + } + +#if 0 + /* cancel timer */ + if (callout_stop(&lle->lle_timer) > 0) + LLE_REMREF(lle); +#endif + /* Drop hold queue */ + pkts_dropped = llentry_free(lle); + arpstat.dropped += pkts_dropped; +} + + +static int +in_lltable_rtcheck(struct ifnet *ifp, u_int flags, const struct sockaddr *l3addr) +{ +#pragma unused(flags) + struct rtentry *rt; + + KASSERT(l3addr->sa_family == AF_INET, + ("sin_family %d", l3addr->sa_family)); + + /* XXX rtalloc1 should take a const param */ + rt = rtalloc1(__DECONST(struct sockaddr *, l3addr), 0, 0); + if (rt == NULL || (rt->rt_flags & RTF_GATEWAY) || rt->rt_ifp != ifp) { + log(LOG_INFO, "IPv4 address: \"%s\" is not on the network\n", + inet_ntoa(((const struct sockaddr_in *)(const void *)l3addr)->sin_addr)); + if (rt != NULL) + rtfree_locked(rt); + return (EINVAL); + } + rtfree_locked(rt); + return 0; +} + +static inline uint32_t +in_lltable_hash_dst(const struct in_addr dst, uint32_t hsize) +{ + return (IN_LLTBL_HASH(dst.s_addr, hsize)); +} + +static uint32_t +in_lltable_hash(const struct llentry *lle, uint32_t hsize) +{ + return (in_lltable_hash_dst(lle->r_l3addr.addr4, hsize)); +} + + +static void +in_lltable_fill_sa_entry(const struct llentry *lle, struct sockaddr *sa) +{ + struct sockaddr_in *sin; + + sin = (struct sockaddr_in *)(void *)sa; + bzero(sin, sizeof(*sin)); + sin->sin_family = AF_INET; + sin->sin_len = sizeof(*sin); + sin->sin_addr = lle->r_l3addr.addr4; +} + +static inline struct llentry * +in_lltable_find_dst(struct lltable *llt, struct in_addr dst) +{ + struct llentry *lle; + struct llentries *lleh; + u_int hashidx; + + hashidx = in_lltable_hash_dst(dst, llt->llt_hsize); + lleh = &llt->lle_head[hashidx]; + LIST_FOREACH(lle, lleh, lle_next) { + if (lle->la_flags & LLE_DELETED) + continue; + if (lle->r_l3addr.addr4.s_addr == dst.s_addr) + break; + } + + return (lle); +} + +static void +in_lltable_delete_entry(struct lltable *llt, struct llentry *lle) +{ +#pragma unused(llt) + lle->la_flags |= LLE_DELETED; + //EVENTHANDLER_INVOKE(lle_event, lle, LLENTRY_DELETED); +#ifdef DIAGNOSTIC + log(LOG_INFO, "ifaddr cache = %p is deleted\n", lle); +#endif + llentry_free(lle); +} + +static struct llentry * +in_lltable_alloc(struct lltable *llt, u_int flags, const struct sockaddr *l3addr) +{ + const struct sockaddr_in *sin = (const struct sockaddr_in *) (const void *)l3addr; + struct ifnet *ifp = llt->llt_ifp; + struct llentry *lle; + + KASSERT(l3addr->sa_family == AF_INET, + ("sin_family %d", l3addr->sa_family)); + + /* + * A route that covers the given address must have + * been installed 1st because we are doing a resolution, + * verify this. + */ + if (!(flags & LLE_IFADDR) && + in_lltable_rtcheck(ifp, flags, l3addr) != 0) + return (NULL); + + lle = in_lltable_new(sin->sin_addr, flags); + if (lle == NULL) { + log(LOG_INFO, "lla_lookup: new lle malloc failed\n"); + return (NULL); + } + lle->la_flags = flags & ~LLE_CREATE; + if (flags & LLE_STATIC) + lle->r_flags |= RLLE_VALID; + if ((flags & LLE_IFADDR) == LLE_IFADDR) { + lltable_set_entry_addr(ifp, lle, LLADDR(SDL(ifp->if_lladdr->ifa_addr))); + lle->la_flags |= LLE_STATIC; + lle->r_flags |= (RLLE_VALID | RLLE_IFADDR); + } + return (lle); +} + +/* + * Return NULL if not found or marked for deletion. + * If found return lle read locked. + */ +static struct llentry * +in_lltable_lookup(struct lltable *llt, u_int flags, const struct sockaddr *l3addr) +{ + const struct sockaddr_in *sin = (const struct sockaddr_in *)(const void *)l3addr; + struct llentry *lle; + + IF_AFDATA_WLOCK_ASSERT(llt->llt_ifp, llt->llt_af); + + KASSERT(l3addr->sa_family == AF_INET, + ("sin_family %d", l3addr->sa_family)); + lle = in_lltable_find_dst(llt, sin->sin_addr); + + if (lle == NULL) + return (NULL); + + KASSERT((flags & (LLE_UNLOCKED|LLE_EXCLUSIVE)) != + (LLE_UNLOCKED|LLE_EXCLUSIVE),("wrong lle request flags: 0x%X", + flags)); + + if (flags & LLE_UNLOCKED) + return (lle); + + if (flags & LLE_EXCLUSIVE) + LLE_WLOCK(lle); + else + LLE_RLOCK(lle); + + return (lle); +} + +static int +in_lltable_dump_entry(struct lltable *llt, struct llentry *lle, + struct sysctl_req *wr) +{ + struct ifnet *ifp = llt->llt_ifp; + /* XXX stack use */ + struct { + struct rt_msghdr rtm; + struct sockaddr_in sin; + struct sockaddr_dl sdl; + } arpc; + struct sockaddr_dl *sdl; + int error; + + bzero(&arpc, sizeof(arpc)); + /* skip deleted entries */ + if ((lle->la_flags & LLE_DELETED) == LLE_DELETED) + return (0); + /* Skip if jailed and not a valid IP of the prison. */ + lltable_fill_sa_entry(lle,(struct sockaddr *)&arpc.sin); + /* + * produce a msg made of: + * struct rt_msghdr; + * struct sockaddr_in; (IPv4) + * struct sockaddr_dl; + */ + arpc.rtm.rtm_msglen = sizeof(arpc); + arpc.rtm.rtm_version = RTM_VERSION; + arpc.rtm.rtm_type = RTM_GET; + arpc.rtm.rtm_flags = RTF_UP; + arpc.rtm.rtm_addrs = RTA_DST | RTA_GATEWAY; + + /* publish */ + if (lle->la_flags & LLE_PUB) + arpc.rtm.rtm_flags |= RTF_ANNOUNCE; + + sdl = &arpc.sdl; + sdl->sdl_family = AF_LINK; + sdl->sdl_len = sizeof(*sdl); + sdl->sdl_index = ifp->if_index; + sdl->sdl_type = ifp->if_type; + if ((lle->la_flags & LLE_VALID) == LLE_VALID) { + sdl->sdl_alen = ifp->if_addrlen; + bcopy(&lle->ll_addr, LLADDR(sdl), ifp->if_addrlen); + } else { + sdl->sdl_alen = 0; + bzero(LLADDR(sdl), ifp->if_addrlen); + } + + arpc.rtm.rtm_rmx.rmx_expire = + lle->la_flags & LLE_STATIC ? 0 : lle->la_expire; + arpc.rtm.rtm_flags |= (RTF_HOST | RTF_LLDATA); + if (lle->la_flags & LLE_STATIC) + arpc.rtm.rtm_flags |= RTF_STATIC; + if (lle->la_flags & LLE_IFADDR) + arpc.rtm.rtm_flags |= RTF_PINNED; + arpc.rtm.rtm_flags |= RTF_PINNED; + arpc.rtm.rtm_index = ifp->if_index; + error = SYSCTL_OUT(wr, &arpc, sizeof(arpc)); + + return (error); +} + +static struct lltable * +in_lltattach(struct ifnet *ifp) +{ + struct lltable *llt; + + llt = lltable_allocate_htbl(IN_LLTBL_DEFAULT_HSIZE); + llt->llt_af = AF_INET; + llt->llt_ifp = ifp; + + llt->llt_lookup = in_lltable_lookup; + llt->llt_alloc_entry = in_lltable_alloc; + llt->llt_delete_entry = in_lltable_delete_entry; + llt->llt_dump_entry = in_lltable_dump_entry; + llt->llt_hash = in_lltable_hash; + llt->llt_fill_sa_entry = in_lltable_fill_sa_entry; + llt->llt_free_entry = in_lltable_free_entry; + llt->llt_match_prefix = in_lltable_match_prefix; + lltable_link(llt); + + return (llt); +} diff --git a/bsd/netinet/in.h b/bsd/netinet/in.h index 74918ba34..07732679a 100644 --- a/bsd/netinet/in.h +++ b/bsd/netinet/in.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2013 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -162,9 +162,9 @@ #define IPPROTO_SWIPE 53 /* IP with encryption */ #define IPPROTO_NHRP 54 /* Next Hop Resolution */ /* 55-57: Unassigned */ -#define IPPROTO_ICMPV6 58 /* ICMP6 */ -#define IPPROTO_NONE 59 /* IP6 no next header */ -#define IPPROTO_DSTOPTS 60 /* IP6 destination option */ +#define IPPROTO_ICMPV6 58 /* ICMP6 */ +#define IPPROTO_NONE 59 /* IP6 no next header */ +#define IPPROTO_DSTOPTS 60 /* IP6 destination option */ #define IPPROTO_AHIP 61 /* any host internal protocol */ #define IPPROTO_CFTP 62 /* CFTP */ #define IPPROTO_HELLO 63 /* "hello" routing protocol */ @@ -364,10 +364,19 @@ struct in_addr { (((u_int32_t)(i) & 0xfff00000) == 0xac100000) || \ (((u_int32_t)(i) & 0xffff0000) == 0xc0a80000)) +#ifdef PRIVATE +#define IN_SHARED_ADDRESS_SPACE(i) ((((u_int32_t)(i)) & (u_int32_t)0xffc00000) \ + == (u_int32_t)0x64400000) + +#define IN_DS_LITE(i) ((((u_int32_t)(i)) & (u_int32_t)0xfffffff8) == (u_int32_t)0xc0000000) + +#define IN_6TO4_RELAY_ANYCAST(i) ((((u_int32_t)(i)) & (u_int32_t)IN_CLASSC_NET) == (u_int32_t)0xc0586300) +#endif + #define IN_LOCAL_GROUP(i) (((u_int32_t)(i) & 0xffffff00) == 0xe0000000) - + #define IN_ANY_LOCAL(i) (IN_LINKLOCAL(i) || IN_LOCAL_GROUP(i)) -#endif +#endif /* __APPLE__ */ #define IN_LOOPBACKNET 127 /* official! */ #endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */ @@ -851,17 +860,31 @@ extern boolean_t in_broadcast(struct in_addr, struct ifnet *); extern boolean_t in_canforward(struct in_addr); extern u_int32_t in_netof(struct in_addr); +extern uint32_t os_cpu_in_cksum_mbuf(struct mbuf *m, int len, int off, + uint32_t initial_sum); + extern uint16_t inet_cksum(struct mbuf *, uint32_t, uint32_t, uint32_t); +extern uint16_t inet_cksum_buffer(const void *, uint32_t, uint32_t, uint32_t); extern uint16_t in_addword(uint16_t, uint16_t); extern uint16_t in_pseudo(uint32_t, uint32_t, uint32_t); extern uint16_t in_pseudo64(uint64_t, uint64_t, uint64_t); extern uint16_t in_cksum_hdr_opt(const struct ip *); extern uint16_t ip_cksum_hdr_dir(struct mbuf *, uint32_t, int); +extern uint16_t ip_cksum_hdr_dir_buffer(const void *, uint32_t, uint32_t, int); extern uint32_t in_finalize_cksum(struct mbuf *, uint32_t, uint32_t); extern uint16_t b_sum16(const void *buf, int len); +#if DEBUG || DEVELOPMENT +extern uint32_t in_cksum_mbuf_ref(struct mbuf *, int, int, uint32_t); +#endif /* DEBUG || DEVELOPMENT */ + +extern int in_getconninfo(struct socket *, sae_connid_t, uint32_t *, + uint32_t *, int32_t *, user_addr_t, socklen_t *, user_addr_t, socklen_t *, + uint32_t *, user_addr_t, uint32_t *); #define in_cksum(_m, _l) \ inet_cksum(_m, 0, 0, _l) +#define in_cksum_buffer(_b, _l) \ + inet_cksum_buffer(_b, 0, 0, _l) #define ip_cksum_hdr_in(_m, _l) \ ip_cksum_hdr_dir(_m, _l, 0) #define ip_cksum_hdr_out(_m, _l) \ diff --git a/bsd/netinet/in_arp.c b/bsd/netinet/in_arp.c index e7eafd51d..674da52bf 100644 --- a/bsd/netinet/in_arp.c +++ b/bsd/netinet/in_arp.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2016 Apple Inc. All rights reserved. + * Copyright (c) 2004-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -79,6 +79,7 @@ #include <net/if_types.h> #include <net/if_llreach.h> #include <net/route.h> +#include <net/nwk_wq.h> #include <netinet/if_ether.h> #include <netinet/in_var.h> @@ -121,20 +122,22 @@ struct llinfo_arp { * The following are protected by rnh_lock */ LIST_ENTRY(llinfo_arp) la_le; - struct rtentry *la_rt; + struct rtentry *la_rt; /* * The following are protected by rt_lock */ - class_queue_t la_holdq; /* packets awaiting resolution */ - struct if_llreach *la_llreach; /* link-layer reachability record */ - u_int64_t la_lastused; /* last used timestamp */ - u_int32_t la_asked; /* # of requests sent */ - u_int32_t la_maxtries; /* retry limit */ - u_int64_t la_probeexp; /* probe deadline timestamp */ + class_queue_t la_holdq; /* packets awaiting resolution */ + struct if_llreach *la_llreach; /* link-layer reachability record */ + u_int64_t la_lastused; /* last used timestamp */ + u_int32_t la_asked; /* # of requests sent */ + u_int32_t la_maxtries; /* retry limit */ + u_int64_t la_probeexp; /* probe deadline timestamp */ + u_int32_t la_prbreq_cnt; /* probe request count */ u_int32_t la_flags; -#define LLINFO_RTRFAIL_EVTSENT 0x1 /* sent an ARP event */ -#define LLINFO_PROBING 0x2 /* waiting for an ARP reply */ +#define LLINFO_RTRFAIL_EVTSENT 0x1 /* sent an ARP event */ +#define LLINFO_PROBING 0x2 /* waiting for an ARP reply */ }; + static LIST_HEAD(, llinfo_arp) llinfo_arp; static thread_call_t arp_timeout_tcall; @@ -163,7 +166,7 @@ static void arp_llinfo_refresh(struct rtentry *); static __inline void arp_llreach_use(struct llinfo_arp *); static __inline int arp_llreach_reachable(struct llinfo_arp *); static void arp_llreach_alloc(struct rtentry *, struct ifnet *, void *, - unsigned int, boolean_t); + unsigned int, boolean_t, uint32_t *); extern int tvtohz(struct timeval *); @@ -290,7 +293,7 @@ arp_llinfo_alloc(int how) * a head drop, details in arp_llinfo_addq(). */ _qinit(&la->la_holdq, Q_DROPHEAD, (arp_maxhold == 0) ? - (uint32_t)-1 : arp_maxhold); + (uint32_t)-1 : arp_maxhold, QP_MBUF); } return (la); @@ -349,11 +352,13 @@ arp_llinfo_flushq(struct llinfo_arp *la) { uint32_t held = qlen(&la->la_holdq); - atomic_add_32(&arpstat.purged, held); - atomic_add_32(&arpstat.held, -held); - _flushq(&la->la_holdq); + if (held != 0) { + atomic_add_32(&arpstat.purged, held); + atomic_add_32(&arpstat.held, -held); + _flushq(&la->la_holdq); + } + la->la_prbreq_cnt = 0; VERIFY(qempty(&la->la_holdq)); - return (held); } @@ -523,7 +528,7 @@ arp_llreach_reachable(struct llinfo_arp *la) */ static void arp_llreach_alloc(struct rtentry *rt, struct ifnet *ifp, void *addr, - unsigned int alen, boolean_t solicited) + unsigned int alen, boolean_t solicited, uint32_t *p_rt_event_code) { VERIFY(rt->rt_expire == 0 || rt->rt_rmx.rmx_expire != 0); VERIFY(rt->rt_expire != 0 || rt->rt_rmx.rmx_expire == 0); @@ -554,7 +559,15 @@ arp_llreach_alloc(struct rtentry *rt, struct ifnet *ifp, void *addr, lr = NULL; why = " for different target HW address; " "using new llreach record"; + *p_rt_event_code = ROUTE_LLENTRY_CHANGED; } else { + /* + * If we were doing unicast probing, we need to + * deliver an event for neighbor cache resolution + */ + if (lr->lr_probes != 0) + *p_rt_event_code = ROUTE_LLENTRY_RESOLVED; + lr->lr_probes = 0; /* reset probe count */ IFLR_UNLOCK(lr); if (solicited) { @@ -572,6 +585,7 @@ arp_llreach_alloc(struct rtentry *rt, struct ifnet *ifp, void *addr, if (why == NULL) why = "creating new llreach record"; } + *p_rt_event_code = ROUTE_LLENTRY_RESOLVED; } if (arp_verbose > 1 && lr != NULL && why != NULL) { @@ -605,7 +619,7 @@ arptfree(struct llinfo_arp *la, void *arg) struct rtentry *rt = la->la_rt; uint64_t timenow; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); /* rnh_lock acquired by caller protects rt from going away */ RT_LOCK(rt); @@ -623,9 +637,22 @@ arptfree(struct llinfo_arp *la, void *arg) if (sdl != NULL) sdl->sdl_alen = 0; (void) arp_llinfo_flushq(la); + /* + * Enqueue work item to invoke callback for this route entry + */ + route_event_enqueue_nwk_wq_entry(rt, NULL, + ROUTE_LLENTRY_UNREACH, NULL, TRUE); } + /* + * The following is mostly being used to arm the timer + * again and for logging. + * qlen is used to re-arm the timer. Therefore, pure probe + * requests can be considered as 0 length packets + * contributing only to length but not to the size. + */ ap->qlen += qlen(&la->la_holdq); + ap->qlen += la->la_prbreq_cnt; ap->qsize += qsize(&la->la_holdq); if (rt->rt_expire == 0 || (rt->rt_flags & RTF_STATIC)) { @@ -742,7 +769,7 @@ arp_timeout(thread_call_param_t arg0, thread_call_param_t arg1) static void arp_sched_timeout(struct timeval *atv) { - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); if (!arp_timeout_run) { struct timeval tv; @@ -811,7 +838,7 @@ arp_probe(thread_call_param_t arg0, thread_call_param_t arg1) static void arp_sched_probe(struct timeval *atv) { - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); if (!arp_probe_run) { struct timeval tv; @@ -856,7 +883,7 @@ arp_rtrequest(int req, struct rtentry *rt, struct sockaddr *sa) char buf[MAX_IPv4_STR_LEN]; VERIFY(arpinit_done); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK_ASSERT_HELD(rt); if (rt->rt_flags & RTF_GATEWAY) @@ -1160,6 +1187,19 @@ arp_lookup_route(const struct in_addr *addr, int create, int proxy, return (0); } +boolean_t +arp_is_entry_probing (route_t p_route) +{ + struct llinfo_arp *llinfo = p_route->rt_llinfo; + + if (llinfo != NULL && + llinfo->la_llreach != NULL && + llinfo->la_llreach->lr_probes != 0) + return (TRUE); + + return (FALSE); +} + /* * This is the ARP pre-output routine; care must be taken to ensure that * the "hint" route never gets freed via rtfree(), since the caller may @@ -1182,6 +1222,7 @@ arp_lookup_ip(ifnet_t ifp, const struct sockaddr_in *net_dest, struct sockaddr *sa; uint32_t rtflags; struct sockaddr_dl sdl; + boolean_t send_probe_notif = FALSE; if (ifp == NULL || net_dest == NULL) return (EINVAL); @@ -1316,6 +1357,13 @@ arp_lookup_ip(ifnet_t ifp, const struct sockaddr_in *net_dest, if (lr->lr_probes == 0) { llinfo->la_probeexp = (timenow + arpt_probe); llinfo->la_flags |= LLINFO_PROBING; + /* + * Provide notification that ARP unicast + * probing has started. + * We only do it for the first unicast probe + * attempt. + */ + send_probe_notif = TRUE; } /* @@ -1371,7 +1419,8 @@ arp_lookup_ip(ifnet_t ifp, const struct sockaddr_in *net_dest, */ if (packet != NULL) arp_llinfo_addq(llinfo, packet); - + else + llinfo->la_prbreq_cnt++; /* * Regardless of permanent vs. expirable entry, we need to * avoid having packets sit in la_holdq forever; thus mark the @@ -1380,10 +1429,11 @@ arp_lookup_ip(ifnet_t ifp, const struct sockaddr_in *net_dest, * moment we get an ARP reply. */ probing = TRUE; - if (qlen(&llinfo->la_holdq) == 1) { + if ((qlen(&llinfo->la_holdq) + llinfo->la_prbreq_cnt) == 1) { llinfo->la_probeexp = (timenow + arpt_probe); llinfo->la_flags |= LLINFO_PROBING; } + if (route->rt_expire) { route->rt_flags &= ~RTF_REJECT; if (llinfo->la_asked == 0 || route->rt_expire != timenow) { @@ -1464,6 +1514,12 @@ arp_lookup_ip(ifnet_t ifp, const struct sockaddr_in *net_dest, VERIFY(_m == packet); } result = EHOSTUNREACH; + + /* + * Enqueue work item to invoke callback for this route entry + */ + route_event_enqueue_nwk_wq_entry(route, NULL, + ROUTE_LLENTRY_UNREACH, NULL, TRUE); goto release; } } @@ -1477,6 +1533,30 @@ release: atomic_add_32(&arpstat.dropped, 1); if (route != NULL) { + if (send_probe_notif) { + route_event_enqueue_nwk_wq_entry(route, NULL, + ROUTE_LLENTRY_PROBED, NULL, TRUE); + + if (route->rt_flags & RTF_ROUTER) { + struct radix_node_head *rnh = NULL; + struct route_event rt_ev; + route_event_init(&rt_ev, route, NULL, ROUTE_LLENTRY_PROBED); + /* + * We already have a reference on rt. The function + * frees it before returning. + */ + RT_UNLOCK(route); + lck_mtx_lock(rnh_lock); + rnh = rt_tables[AF_INET]; + + if (rnh != NULL) + (void) rnh->rnh_walktree(rnh, + route_event_walktree, (void *)&rt_ev); + lck_mtx_unlock(rnh_lock); + RT_LOCK(route); + } + } + if (route == hint) { RT_REMREF_LOCKED(route); RT_UNLOCK(route); @@ -1512,6 +1592,7 @@ arp_ip_handle_input(ifnet_t ifp, u_short arpop, errno_t error; int created_announcement = 0; int bridged = 0, is_bridge = 0; + uint32_t rt_evcode = 0; /* * Here and other places within this routine where we don't hold @@ -1927,7 +2008,7 @@ match: /* cache the gateway (sender HW) address */ arp_llreach_alloc(route, ifp, LLADDR(gateway), gateway->sdl_alen, - (arpop == ARPOP_REPLY)); + (arpop == ARPOP_REPLY), &rt_evcode); llinfo = route->rt_llinfo; /* send a notification that the route is back up */ @@ -1956,6 +2037,34 @@ match: /* Update the llinfo, send out all queued packets at once */ llinfo->la_asked = 0; llinfo->la_flags &= ~LLINFO_PROBING; + llinfo->la_prbreq_cnt = 0; + + if (rt_evcode) { + /* + * Enqueue work item to invoke callback for this route entry + */ + route_event_enqueue_nwk_wq_entry(route, NULL, rt_evcode, NULL, TRUE); + + if (route->rt_flags & RTF_ROUTER) { + struct radix_node_head *rnh = NULL; + struct route_event rt_ev; + route_event_init(&rt_ev, route, NULL, rt_evcode); + /* + * We already have a reference on rt. The function + * frees it before returning. + */ + RT_UNLOCK(route); + lck_mtx_lock(rnh_lock); + rnh = rt_tables[AF_INET]; + + if (rnh != NULL) + (void) rnh->rnh_walktree(rnh, route_event_walktree, + (void *)&rt_ev); + lck_mtx_unlock(rnh_lock); + RT_LOCK(route); + } + } + if (!qempty(&llinfo->la_holdq)) { uint32_t held; struct mbuf *m0 = @@ -1973,7 +2082,6 @@ match: route = NULL; } - respond: if (route != NULL) { /* Mark use timestamp if we're going to send a reply */ diff --git a/bsd/netinet/in_arp.h b/bsd/netinet/in_arp.h index fdbc9e2d1..51310486a 100644 --- a/bsd/netinet/in_arp.h +++ b/bsd/netinet/in_arp.h @@ -61,6 +61,7 @@ struct sockaddr_in; * the packet. */ #ifdef BSD_KERNEL_PRIVATE +extern boolean_t arp_is_entry_probing (route_t p_route); extern errno_t arp_lookup_ip(ifnet_t interface, const struct sockaddr_in *ip_dest, struct sockaddr_dl *ll_dest, size_t ll_dest_len, route_t hint, mbuf_t packet); diff --git a/bsd/netinet/in_cksum.c b/bsd/netinet/in_cksum.c index bc302ae30..29594c123 100644 --- a/bsd/netinet/in_cksum.c +++ b/bsd/netinet/in_cksum.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2012 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -94,258 +94,7 @@ union q_util { uint64_t q; }; -#define PREDICT_FALSE(_exp) __builtin_expect((_exp), 0) - -static uint16_t in_cksumdata(const void *buf, int len); - -/* - * Portable version of 16-bit 1's complement sum function that works - * on a contiguous buffer. This is used mainly for instances where - * the caller is certain about the buffer requirements, e.g. for IP - * header checksum calculation, though it is capable of being used - * on any arbitrary data span. The platform-specific cpu_in_cksum() - * routine might be better-optmized, so use that instead for large - * data span. - * - * The logic is borrowed from <bsd/netinet/cpu_in_cksum.c> - */ - -#if ULONG_MAX == 0xffffffffUL -/* 32-bit version */ -static uint16_t -in_cksumdata(const void *buf, int mlen) -{ - uint32_t sum, partial; - unsigned int final_acc; - const uint8_t *data = (const uint8_t *)buf; - boolean_t needs_swap, started_on_odd; - - VERIFY(mlen >= 0); - - needs_swap = FALSE; - started_on_odd = FALSE; - - sum = 0; - partial = 0; - - if ((uintptr_t)data & 1) { - /* Align on word boundary */ - started_on_odd = !started_on_odd; -#if BYTE_ORDER == LITTLE_ENDIAN - partial = *data << 8; -#else - partial = *data; -#endif - ++data; - --mlen; - } - needs_swap = started_on_odd; - while (mlen >= 32) { - __builtin_prefetch(data + 32); - partial += *(const uint16_t *)(const void *)data; - partial += *(const uint16_t *)(const void *)(data + 2); - partial += *(const uint16_t *)(const void *)(data + 4); - partial += *(const uint16_t *)(const void *)(data + 6); - partial += *(const uint16_t *)(const void *)(data + 8); - partial += *(const uint16_t *)(const void *)(data + 10); - partial += *(const uint16_t *)(const void *)(data + 12); - partial += *(const uint16_t *)(const void *)(data + 14); - partial += *(const uint16_t *)(const void *)(data + 16); - partial += *(const uint16_t *)(const void *)(data + 18); - partial += *(const uint16_t *)(const void *)(data + 20); - partial += *(const uint16_t *)(const void *)(data + 22); - partial += *(const uint16_t *)(const void *)(data + 24); - partial += *(const uint16_t *)(const void *)(data + 26); - partial += *(const uint16_t *)(const void *)(data + 28); - partial += *(const uint16_t *)(const void *)(data + 30); - data += 32; - mlen -= 32; - if (PREDICT_FALSE(partial & 0xc0000000)) { - if (needs_swap) - partial = (partial << 8) + - (partial >> 24); - sum += (partial >> 16); - sum += (partial & 0xffff); - partial = 0; - } - } - if (mlen & 16) { - partial += *(const uint16_t *)(const void *)data; - partial += *(const uint16_t *)(const void *)(data + 2); - partial += *(const uint16_t *)(const void *)(data + 4); - partial += *(const uint16_t *)(const void *)(data + 6); - partial += *(const uint16_t *)(const void *)(data + 8); - partial += *(const uint16_t *)(const void *)(data + 10); - partial += *(const uint16_t *)(const void *)(data + 12); - partial += *(const uint16_t *)(const void *)(data + 14); - data += 16; - mlen -= 16; - } - /* - * mlen is not updated below as the remaining tests - * are using bit masks, which are not affected. - */ - if (mlen & 8) { - partial += *(const uint16_t *)(const void *)data; - partial += *(const uint16_t *)(const void *)(data + 2); - partial += *(const uint16_t *)(const void *)(data + 4); - partial += *(const uint16_t *)(const void *)(data + 6); - data += 8; - } - if (mlen & 4) { - partial += *(const uint16_t *)(const void *)data; - partial += *(const uint16_t *)(const void *)(data + 2); - data += 4; - } - if (mlen & 2) { - partial += *(const uint16_t *)(const void *)data; - data += 2; - } - if (mlen & 1) { -#if BYTE_ORDER == LITTLE_ENDIAN - partial += *data; -#else - partial += *data << 8; -#endif - started_on_odd = !started_on_odd; - } - - if (needs_swap) - partial = (partial << 8) + (partial >> 24); - sum += (partial >> 16) + (partial & 0xffff); - sum = (sum >> 16) + (sum & 0xffff); - - final_acc = ((sum >> 16) & 0xffff) + (sum & 0xffff); - final_acc = (final_acc >> 16) + (final_acc & 0xffff); - - return (final_acc); -} - -#else -/* 64-bit version */ -static uint16_t -in_cksumdata(const void *buf, int mlen) -{ - uint64_t sum, partial; - unsigned int final_acc; - const uint8_t *data = (const uint8_t *)buf; - boolean_t needs_swap, started_on_odd; - - VERIFY(mlen >= 0); - - needs_swap = FALSE; - started_on_odd = FALSE; - - sum = 0; - partial = 0; - - if ((uintptr_t)data & 1) { - /* Align on word boundary */ - started_on_odd = !started_on_odd; -#if BYTE_ORDER == LITTLE_ENDIAN - partial = *data << 8; -#else - partial = *data; -#endif - ++data; - --mlen; - } - needs_swap = started_on_odd; - if ((uintptr_t)data & 2) { - if (mlen < 2) - goto trailing_bytes; - partial += *(const uint16_t *)(const void *)data; - data += 2; - mlen -= 2; - } - while (mlen >= 64) { - __builtin_prefetch(data + 32); - __builtin_prefetch(data + 64); - partial += *(const uint32_t *)(const void *)data; - partial += *(const uint32_t *)(const void *)(data + 4); - partial += *(const uint32_t *)(const void *)(data + 8); - partial += *(const uint32_t *)(const void *)(data + 12); - partial += *(const uint32_t *)(const void *)(data + 16); - partial += *(const uint32_t *)(const void *)(data + 20); - partial += *(const uint32_t *)(const void *)(data + 24); - partial += *(const uint32_t *)(const void *)(data + 28); - partial += *(const uint32_t *)(const void *)(data + 32); - partial += *(const uint32_t *)(const void *)(data + 36); - partial += *(const uint32_t *)(const void *)(data + 40); - partial += *(const uint32_t *)(const void *)(data + 44); - partial += *(const uint32_t *)(const void *)(data + 48); - partial += *(const uint32_t *)(const void *)(data + 52); - partial += *(const uint32_t *)(const void *)(data + 56); - partial += *(const uint32_t *)(const void *)(data + 60); - data += 64; - mlen -= 64; - if (PREDICT_FALSE(partial & (3ULL << 62))) { - if (needs_swap) - partial = (partial << 8) + - (partial >> 56); - sum += (partial >> 32); - sum += (partial & 0xffffffff); - partial = 0; - } - } - /* - * mlen is not updated below as the remaining tests - * are using bit masks, which are not affected. - */ - if (mlen & 32) { - partial += *(const uint32_t *)(const void *)data; - partial += *(const uint32_t *)(const void *)(data + 4); - partial += *(const uint32_t *)(const void *)(data + 8); - partial += *(const uint32_t *)(const void *)(data + 12); - partial += *(const uint32_t *)(const void *)(data + 16); - partial += *(const uint32_t *)(const void *)(data + 20); - partial += *(const uint32_t *)(const void *)(data + 24); - partial += *(const uint32_t *)(const void *)(data + 28); - data += 32; - } - if (mlen & 16) { - partial += *(const uint32_t *)(const void *)data; - partial += *(const uint32_t *)(const void *)(data + 4); - partial += *(const uint32_t *)(const void *)(data + 8); - partial += *(const uint32_t *)(const void *)(data + 12); - data += 16; - } - if (mlen & 8) { - partial += *(const uint32_t *)(const void *)data; - partial += *(const uint32_t *)(const void *)(data + 4); - data += 8; - } - if (mlen & 4) { - partial += *(const uint32_t *)(const void *)data; - data += 4; - } - if (mlen & 2) { - partial += *(const uint16_t *)(const void *)data; - data += 2; - } -trailing_bytes: - if (mlen & 1) { -#if BYTE_ORDER == LITTLE_ENDIAN - partial += *data; -#else - partial += *data << 8; -#endif - started_on_odd = !started_on_odd; - } - - if (needs_swap) - partial = (partial << 8) + (partial >> 56); - sum += (partial >> 32) + (partial & 0xffffffff); - sum = (sum >> 32) + (sum & 0xffffffff); - - final_acc = (sum >> 48) + ((sum >> 32) & 0xffff) + - ((sum >> 16) & 0xffff) + (sum & 0xffff); - final_acc = (final_acc >> 16) + (final_acc & 0xffff); - final_acc = (final_acc >> 16) + (final_acc & 0xffff); - - return (final_acc); -} -#endif /* ULONG_MAX != 0xffffffffUL */ +extern uint32_t os_cpu_in_cksum(const void *, uint32_t, uint32_t); /* * Perform 16-bit 1's complement sum on a contiguous span. @@ -353,7 +102,7 @@ trailing_bytes: uint16_t b_sum16(const void *buf, int len) { - return (in_cksumdata(buf, len)); + return (os_cpu_in_cksum(buf, len, 0)); } uint16_t inet_cksum_simple(struct mbuf *, int); @@ -433,6 +182,27 @@ ip_cksum_hdr_dir(struct mbuf *m, uint32_t hlen, int out) return (inet_cksum(m, 0, 0, hlen)); } +uint16_t +ip_cksum_hdr_dir_buffer(const void *buffer, uint32_t hlen, uint32_t len, + int out) +{ + const struct ip *ip = buffer; + + if (out) { + ipstat.ips_snd_swcsum++; + ipstat.ips_snd_swcsum_bytes += hlen; + } else { + ipstat.ips_rcv_swcsum++; + ipstat.ips_rcv_swcsum_bytes += hlen; + } + + if (hlen == sizeof (*ip) && + len >= sizeof (*ip) && IP_HDR_ALIGNED_P(ip)) + return (in_cksum_hdr(ip)); + + return (inet_cksum_buffer(buffer, 0, 0, hlen)); +} + /* * m MUST contain at least an IP header, if nxt is specified; * nxt is the upper layer protocol number; @@ -488,3 +258,369 @@ inet_cksum(struct mbuf *m, uint32_t nxt, uint32_t off, uint32_t len) return (~sum & 0xffff); } + +/* + * buffer MUST contain at least an IP header, if nxt is specified; + * nxt is the upper layer protocol number; + * off is an offset where TCP/UDP/ICMP header starts; + * len is a total length of a transport segment (e.g. TCP header + TCP payload) + */ +uint16_t +inet_cksum_buffer(const void *buffer, uint32_t nxt, uint32_t off, + uint32_t len) +{ + uint32_t sum; + + if (off >= len) + panic("%s: off (%d) >= len (%d)", __func__, off, len); + + sum = b_sum16(&((const uint8_t *)buffer)[off], len); + + /* include pseudo header checksum? */ + if (nxt != 0) { + const struct ip *ip; + unsigned char buf[sizeof ((*ip))] __attribute__((aligned(8))); + + /* + * In case the IP header is not contiguous, or not 32-bit + * aligned, copy it to a local buffer. Note here that we + * expect the data pointer to point to the IP header. + */ + if (!IP_HDR_ALIGNED_P(buffer)) { + memcpy(buf, buffer, sizeof (*ip)); + ip = (const struct ip *)(const void *)buf; + } else { + ip = (const struct ip *)buffer; + } + + /* add pseudo header checksum */ + sum += in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr, + htonl(len + nxt)); + + /* fold in carry bits */ + ADDCARRY(sum); + } + + return (~sum & 0xffff); +} + +#if DEBUG || DEVELOPMENT +#include <mach/branch_predicates.h> +#include <pexpert/pexpert.h> + +#define CKSUM_ERR kprintf + +/* + * The following routines implement the portable, reference implementation + * of os_cpu_in_cksum_mbuf(). This is currently used only for validating + * the correctness of the platform-specific implementation, at boot time + * in dlil_verify_sum16(). It returns the 32-bit accumulator without doing + * a 1's complement on it. + */ +#if !defined(__LP64__) +/* 32-bit version */ +uint32_t +in_cksum_mbuf_ref(struct mbuf *m, int len, int off, uint32_t initial_sum) +{ + int mlen; + uint32_t sum, partial; + unsigned int final_acc; + uint8_t *data; + boolean_t needs_swap, started_on_odd; + + VERIFY(len >= 0); + VERIFY(off >= 0); + + needs_swap = FALSE; + started_on_odd = FALSE; + sum = (initial_sum >> 16) + (initial_sum & 0xffff); + + for (;;) { + if (__improbable(m == NULL)) { + CKSUM_ERR("%s: out of data\n", __func__); + return ((uint32_t)-1); + } + mlen = m->m_len; + if (mlen > off) { + mlen -= off; + data = mtod(m, uint8_t *) + off; + goto post_initial_offset; + } + off -= mlen; + if (len == 0) + break; + m = m->m_next; + } + + for (; len > 0; m = m->m_next) { + if (__improbable(m == NULL)) { + CKSUM_ERR("%s: out of data\n", __func__); + return ((uint32_t)-1); + } + mlen = m->m_len; + data = mtod(m, uint8_t *); +post_initial_offset: + if (mlen == 0) + continue; + if (mlen > len) + mlen = len; + len -= mlen; + + partial = 0; + if ((uintptr_t)data & 1) { + /* Align on word boundary */ + started_on_odd = !started_on_odd; +#if BYTE_ORDER == LITTLE_ENDIAN + partial = *data << 8; +#else /* BYTE_ORDER != LITTLE_ENDIAN */ + partial = *data; +#endif /* BYTE_ORDER != LITTLE_ENDIAN */ + ++data; + --mlen; + } + needs_swap = started_on_odd; + while (mlen >= 32) { + __builtin_prefetch(data + 32); + partial += *(uint16_t *)(void *)data; + partial += *(uint16_t *)(void *)(data + 2); + partial += *(uint16_t *)(void *)(data + 4); + partial += *(uint16_t *)(void *)(data + 6); + partial += *(uint16_t *)(void *)(data + 8); + partial += *(uint16_t *)(void *)(data + 10); + partial += *(uint16_t *)(void *)(data + 12); + partial += *(uint16_t *)(void *)(data + 14); + partial += *(uint16_t *)(void *)(data + 16); + partial += *(uint16_t *)(void *)(data + 18); + partial += *(uint16_t *)(void *)(data + 20); + partial += *(uint16_t *)(void *)(data + 22); + partial += *(uint16_t *)(void *)(data + 24); + partial += *(uint16_t *)(void *)(data + 26); + partial += *(uint16_t *)(void *)(data + 28); + partial += *(uint16_t *)(void *)(data + 30); + data += 32; + mlen -= 32; + if (__improbable(partial & 0xc0000000)) { + if (needs_swap) + partial = (partial << 8) + + (partial >> 24); + sum += (partial >> 16); + sum += (partial & 0xffff); + partial = 0; + } + } + if (mlen & 16) { + partial += *(uint16_t *)(void *)data; + partial += *(uint16_t *)(void *)(data + 2); + partial += *(uint16_t *)(void *)(data + 4); + partial += *(uint16_t *)(void *)(data + 6); + partial += *(uint16_t *)(void *)(data + 8); + partial += *(uint16_t *)(void *)(data + 10); + partial += *(uint16_t *)(void *)(data + 12); + partial += *(uint16_t *)(void *)(data + 14); + data += 16; + mlen -= 16; + } + /* + * mlen is not updated below as the remaining tests + * are using bit masks, which are not affected. + */ + if (mlen & 8) { + partial += *(uint16_t *)(void *)data; + partial += *(uint16_t *)(void *)(data + 2); + partial += *(uint16_t *)(void *)(data + 4); + partial += *(uint16_t *)(void *)(data + 6); + data += 8; + } + if (mlen & 4) { + partial += *(uint16_t *)(void *)data; + partial += *(uint16_t *)(void *)(data + 2); + data += 4; + } + if (mlen & 2) { + partial += *(uint16_t *)(void *)data; + data += 2; + } + if (mlen & 1) { +#if BYTE_ORDER == LITTLE_ENDIAN + partial += *data; +#else /* BYTE_ORDER != LITTLE_ENDIAN */ + partial += *data << 8; +#endif /* BYTE_ORDER != LITTLE_ENDIAN */ + started_on_odd = !started_on_odd; + } + + if (needs_swap) + partial = (partial << 8) + (partial >> 24); + sum += (partial >> 16) + (partial & 0xffff); + /* + * Reduce sum to allow potential byte swap + * in the next iteration without carry. + */ + sum = (sum >> 16) + (sum & 0xffff); + } + final_acc = ((sum >> 16) & 0xffff) + (sum & 0xffff); + final_acc = (final_acc >> 16) + (final_acc & 0xffff); + return (final_acc & 0xffff); +} + +#else /* __LP64__ */ +/* 64-bit version */ +uint32_t +in_cksum_mbuf_ref(struct mbuf *m, int len, int off, uint32_t initial_sum) +{ + int mlen; + uint64_t sum, partial; + unsigned int final_acc; + uint8_t *data; + boolean_t needs_swap, started_on_odd; + + VERIFY(len >= 0); + VERIFY(off >= 0); + + needs_swap = FALSE; + started_on_odd = FALSE; + sum = initial_sum; + + for (;;) { + if (__improbable(m == NULL)) { + CKSUM_ERR("%s: out of data\n", __func__); + return ((uint32_t)-1); + } + mlen = m->m_len; + if (mlen > off) { + mlen -= off; + data = mtod(m, uint8_t *) + off; + goto post_initial_offset; + } + off -= mlen; + if (len == 0) + break; + m = m->m_next; + } + + for (; len > 0; m = m->m_next) { + if (__improbable(m == NULL)) { + CKSUM_ERR("%s: out of data\n", __func__); + return ((uint32_t)-1); + } + mlen = m->m_len; + data = mtod(m, uint8_t *); +post_initial_offset: + if (mlen == 0) + continue; + if (mlen > len) + mlen = len; + len -= mlen; + + partial = 0; + if ((uintptr_t)data & 1) { + /* Align on word boundary */ + started_on_odd = !started_on_odd; +#if BYTE_ORDER == LITTLE_ENDIAN + partial = *data << 8; +#else /* BYTE_ORDER != LITTLE_ENDIAN */ + partial = *data; +#endif /* BYTE_ORDER != LITTLE_ENDIAN */ + ++data; + --mlen; + } + needs_swap = started_on_odd; + if ((uintptr_t)data & 2) { + if (mlen < 2) + goto trailing_bytes; + partial += *(uint16_t *)(void *)data; + data += 2; + mlen -= 2; + } + while (mlen >= 64) { + __builtin_prefetch(data + 32); + __builtin_prefetch(data + 64); + partial += *(uint32_t *)(void *)data; + partial += *(uint32_t *)(void *)(data + 4); + partial += *(uint32_t *)(void *)(data + 8); + partial += *(uint32_t *)(void *)(data + 12); + partial += *(uint32_t *)(void *)(data + 16); + partial += *(uint32_t *)(void *)(data + 20); + partial += *(uint32_t *)(void *)(data + 24); + partial += *(uint32_t *)(void *)(data + 28); + partial += *(uint32_t *)(void *)(data + 32); + partial += *(uint32_t *)(void *)(data + 36); + partial += *(uint32_t *)(void *)(data + 40); + partial += *(uint32_t *)(void *)(data + 44); + partial += *(uint32_t *)(void *)(data + 48); + partial += *(uint32_t *)(void *)(data + 52); + partial += *(uint32_t *)(void *)(data + 56); + partial += *(uint32_t *)(void *)(data + 60); + data += 64; + mlen -= 64; + if (__improbable(partial & (3ULL << 62))) { + if (needs_swap) + partial = (partial << 8) + + (partial >> 56); + sum += (partial >> 32); + sum += (partial & 0xffffffff); + partial = 0; + } + } + /* + * mlen is not updated below as the remaining tests + * are using bit masks, which are not affected. + */ + if (mlen & 32) { + partial += *(uint32_t *)(void *)data; + partial += *(uint32_t *)(void *)(data + 4); + partial += *(uint32_t *)(void *)(data + 8); + partial += *(uint32_t *)(void *)(data + 12); + partial += *(uint32_t *)(void *)(data + 16); + partial += *(uint32_t *)(void *)(data + 20); + partial += *(uint32_t *)(void *)(data + 24); + partial += *(uint32_t *)(void *)(data + 28); + data += 32; + } + if (mlen & 16) { + partial += *(uint32_t *)(void *)data; + partial += *(uint32_t *)(void *)(data + 4); + partial += *(uint32_t *)(void *)(data + 8); + partial += *(uint32_t *)(void *)(data + 12); + data += 16; + } + if (mlen & 8) { + partial += *(uint32_t *)(void *)data; + partial += *(uint32_t *)(void *)(data + 4); + data += 8; + } + if (mlen & 4) { + partial += *(uint32_t *)(void *)data; + data += 4; + } + if (mlen & 2) { + partial += *(uint16_t *)(void *)data; + data += 2; + } +trailing_bytes: + if (mlen & 1) { +#if BYTE_ORDER == LITTLE_ENDIAN + partial += *data; +#else /* BYTE_ORDER != LITTLE_ENDIAN */ + partial += *data << 8; +#endif /* BYTE_ORDER != LITTLE_ENDIAN */ + started_on_odd = !started_on_odd; + } + + if (needs_swap) + partial = (partial << 8) + (partial >> 56); + sum += (partial >> 32) + (partial & 0xffffffff); + /* + * Reduce sum to allow potential byte swap + * in the next iteration without carry. + */ + sum = (sum >> 32) + (sum & 0xffffffff); + } + final_acc = (sum >> 48) + ((sum >> 32) & 0xffff) + + ((sum >> 16) & 0xffff) + (sum & 0xffff); + final_acc = (final_acc >> 16) + (final_acc & 0xffff); + final_acc = (final_acc >> 16) + (final_acc & 0xffff); + return (final_acc & 0xffff); +} +#endif /* __LP64 */ +#endif /* DEBUG || DEVELOPMENT */ diff --git a/bsd/netinet/in_mcast.c b/bsd/netinet/in_mcast.c index 1d1b56563..3d8ca9699 100644 --- a/bsd/netinet/in_mcast.c +++ b/bsd/netinet/in_mcast.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2016 Apple Inc. All rights reserved. + * Copyright (c) 2010-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -80,6 +80,7 @@ #include <net/if.h> #include <net/if_dl.h> +#include <net/net_api_stats.h> #include <net/route.h> #include <netinet/in.h> @@ -89,17 +90,6 @@ #include <netinet/ip_var.h> #include <netinet/igmp_var.h> -#ifndef __SOCKUNION_DECLARED -union sockunion { - struct sockaddr_storage ss; - struct sockaddr sa; - struct sockaddr_dl sdl; - struct sockaddr_in sin; -}; -typedef union sockunion sockunion_t; -#define __SOCKUNION_DECLARED -#endif /* __SOCKUNION_DECLARED */ - /* * Functions with non-static linkage defined in this file should be * declared in in_var.h: @@ -128,10 +118,10 @@ static void imf_rollback(struct in_mfilter *); static void imf_reap(struct in_mfilter *); static int imo_grow(struct ip_moptions *, size_t); static size_t imo_match_group(const struct ip_moptions *, - const struct ifnet *, const struct sockaddr *); + const struct ifnet *, const struct sockaddr_in *); static struct in_msource * imo_match_source(const struct ip_moptions *, const size_t, - const struct sockaddr *); + const struct sockaddr_in *); static void ims_merge(struct ip_msource *ims, const struct in_msource *lims, const int rollback); static int in_getmulti(struct ifnet *, const struct in_addr *, @@ -335,16 +325,14 @@ imo_grow(struct ip_moptions *imo, size_t newmax) */ static size_t imo_match_group(const struct ip_moptions *imo, const struct ifnet *ifp, - const struct sockaddr *group) + const struct sockaddr_in *group) { - const struct sockaddr_in *gsin; struct in_multi *pinm; int idx; int nmships; IMO_LOCK_ASSERT_HELD(__DECONST(struct ip_moptions *, imo)); - gsin = (struct sockaddr_in *)(uintptr_t)(size_t)group; /* The imo_membership array may be lazy allocated. */ if (imo->imo_membership == NULL || imo->imo_num_memberships == 0) @@ -357,7 +345,7 @@ imo_match_group(const struct ip_moptions *imo, const struct ifnet *ifp, continue; INM_LOCK(pinm); if ((ifp == NULL || (pinm->inm_ifp == ifp)) && - in_hosteq(pinm->inm_addr, gsin->sin_addr)) { + in_hosteq(pinm->inm_addr, group->sin_addr)) { INM_UNLOCK(pinm); break; } @@ -378,16 +366,15 @@ imo_match_group(const struct ip_moptions *imo, const struct ifnet *ifp, */ static struct in_msource * imo_match_source(const struct ip_moptions *imo, const size_t gidx, - const struct sockaddr *src) + const struct sockaddr_in *src) { struct ip_msource find; struct in_mfilter *imf; struct ip_msource *ims; - const sockunion_t *psa; IMO_LOCK_ASSERT_HELD(__DECONST(struct ip_moptions *, imo)); - VERIFY(src->sa_family == AF_INET); + VERIFY(src->sin_family == AF_INET); VERIFY(gidx != (size_t)-1 && gidx < imo->imo_num_memberships); /* The imo_mfilters array may be lazy allocated. */ @@ -396,8 +383,7 @@ imo_match_source(const struct ip_moptions *imo, const size_t gidx, imf = &imo->imo_mfilters[gidx]; /* Source trees are keyed in host byte order. */ - psa = (sockunion_t *)(uintptr_t)(size_t)src; - find.ims_haddr = ntohl(psa->sin.sin_addr.s_addr); + find.ims_haddr = ntohl(src->sin_addr.s_addr); ims = RB_FIND(ip_msource_tree, &imf->imf_sources, &find); return ((struct in_msource *)ims); @@ -411,7 +397,7 @@ imo_match_source(const struct ip_moptions *imo, const size_t gidx, */ int imo_multi_filter(const struct ip_moptions *imo, const struct ifnet *ifp, - const struct sockaddr *group, const struct sockaddr *src) + const struct sockaddr_in *group, const struct sockaddr_in *src) { size_t gidx; struct in_msource *ims; @@ -1077,7 +1063,7 @@ ims_merge(struct ip_msource *ims, const struct in_msource *lims, static int inm_merge(struct in_multi *inm, /*const*/ struct in_mfilter *imf) { - struct ip_msource *ims, *nims; + struct ip_msource *ims, *nims = NULL; struct in_msource *lims; int schanged, error; int nsrc0, nsrc1; @@ -1455,7 +1441,7 @@ static int inp_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) { struct group_source_req gsr; - sockunion_t *gsa, *ssa; + struct sockaddr_in *gsa, *ssa; struct ifnet *ifp; struct in_mfilter *imf; struct ip_moptions *imo; @@ -1473,8 +1459,8 @@ inp_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) doblock = 0; memset(&gsr, 0, sizeof(struct group_source_req)); - gsa = (sockunion_t *)&gsr.gsr_group; - ssa = (sockunion_t *)&gsr.gsr_source; + gsa = (struct sockaddr_in *)&gsr.gsr_group; + ssa = (struct sockaddr_in *)&gsr.gsr_source; switch (sopt->sopt_name) { case IP_BLOCK_SOURCE: @@ -1487,13 +1473,13 @@ inp_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) if (error) return (error); - gsa->sin.sin_family = AF_INET; - gsa->sin.sin_len = sizeof(struct sockaddr_in); - gsa->sin.sin_addr = mreqs.imr_multiaddr; + gsa->sin_family = AF_INET; + gsa->sin_len = sizeof(struct sockaddr_in); + gsa->sin_addr = mreqs.imr_multiaddr; - ssa->sin.sin_family = AF_INET; - ssa->sin.sin_len = sizeof(struct sockaddr_in); - ssa->sin.sin_addr = mreqs.imr_sourceaddr; + ssa->sin_family = AF_INET; + ssa->sin_len = sizeof(struct sockaddr_in); + ssa->sin_addr = mreqs.imr_sourceaddr; if (!in_nullhost(mreqs.imr_interface)) ifp = ip_multicast_if(&mreqs.imr_interface, &ifindex); @@ -1515,12 +1501,12 @@ inp_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) if (error) return (error); - if (gsa->sin.sin_family != AF_INET || - gsa->sin.sin_len != sizeof(struct sockaddr_in)) + if (gsa->sin_family != AF_INET || + gsa->sin_len != sizeof(struct sockaddr_in)) return (EINVAL); - if (ssa->sin.sin_family != AF_INET || - ssa->sin.sin_len != sizeof(struct sockaddr_in)) + if (ssa->sin_family != AF_INET || + ssa->sin_len != sizeof(struct sockaddr_in)) return (EINVAL); ifnet_head_lock_shared(); @@ -1546,7 +1532,7 @@ inp_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) return (EOPNOTSUPP); } - if (!IN_MULTICAST(ntohl(gsa->sin.sin_addr.s_addr))) + if (!IN_MULTICAST(ntohl(gsa->sin_addr.s_addr))) return (EINVAL); /* @@ -1557,7 +1543,7 @@ inp_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) return (ENOMEM); IMO_LOCK(imo); - idx = imo_match_group(imo, ifp, &gsa->sa); + idx = imo_match_group(imo, ifp, gsa); if (idx == (size_t)-1 || imo->imo_mfilters == NULL) { error = EADDRNOTAVAIL; goto out_imo_locked; @@ -1583,9 +1569,9 @@ inp_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) * Asked to unblock, but nothing to unblock. * If adding a new block entry, allocate it. */ - ims = imo_match_source(imo, idx, &ssa->sa); + ims = imo_match_source(imo, idx, ssa); if ((ims != NULL && doblock) || (ims == NULL && !doblock)) { - IGMP_INET_PRINTF(ssa->sin.sin_addr, + IGMP_INET_PRINTF(ssa->sin_addr, ("%s: source %s %spresent\n", __func__, _igmp_inet_buf, doblock ? "" : "not ")); error = EADDRNOTAVAIL; @@ -1597,12 +1583,12 @@ inp_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) */ if (doblock) { IGMP_PRINTF(("%s: %s source\n", __func__, "block")); - ims = imf_graft(imf, fmode, &ssa->sin); + ims = imf_graft(imf, fmode, ssa); if (ims == NULL) error = ENOMEM; } else { IGMP_PRINTF(("%s: %s source\n", __func__, "allow")); - error = imf_prune(imf, &ssa->sin); + error = imf_prune(imf, ssa); } if (error) { @@ -1713,7 +1699,7 @@ inp_get_source_filters(struct inpcb *inp, struct sockopt *sopt) { struct __msfilterreq64 msfr, msfr64; struct __msfilterreq32 msfr32; - sockunion_t *gsa; + struct sockaddr_in *gsa; struct ifnet *ifp; struct ip_moptions *imo; struct in_mfilter *imf; @@ -1770,8 +1756,9 @@ inp_get_source_filters(struct inpcb *inp, struct sockopt *sopt) /* * Lookup group on the socket. */ - gsa = (sockunion_t *)&msfr.msfr_group; - idx = imo_match_group(imo, ifp, &gsa->sa); + gsa = (struct sockaddr_in *)&msfr.msfr_group; + + idx = imo_match_group(imo, ifp, gsa); if (idx == (size_t)-1 || imo->imo_mfilters == NULL) { IMO_UNLOCK(imo); return (EADDRNOTAVAIL); @@ -2063,7 +2050,7 @@ int inp_join_group(struct inpcb *inp, struct sockopt *sopt) { struct group_source_req gsr; - sockunion_t *gsa, *ssa; + struct sockaddr_in *gsa, *ssa; struct ifnet *ifp; struct in_mfilter *imf; struct ip_moptions *imo; @@ -2080,10 +2067,10 @@ inp_join_group(struct inpcb *inp, struct sockopt *sopt) is_new = 0; memset(&gsr, 0, sizeof(struct group_source_req)); - gsa = (sockunion_t *)&gsr.gsr_group; - gsa->ss.ss_family = AF_UNSPEC; - ssa = (sockunion_t *)&gsr.gsr_source; - ssa->ss.ss_family = AF_UNSPEC; + gsa = (struct sockaddr_in *)&gsr.gsr_group; + gsa->sin_family = AF_UNSPEC; + ssa = (struct sockaddr_in *)&gsr.gsr_source; + ssa->sin_family = AF_UNSPEC; switch (sopt->sopt_name) { case IP_ADD_MEMBERSHIP: @@ -2112,21 +2099,20 @@ inp_join_group(struct inpcb *inp, struct sockopt *sopt) return (error); } - gsa->sin.sin_family = AF_INET; - gsa->sin.sin_len = sizeof(struct sockaddr_in); - gsa->sin.sin_addr = mreqs.imr_multiaddr; + gsa->sin_family = AF_INET; + gsa->sin_len = sizeof(struct sockaddr_in); + gsa->sin_addr = mreqs.imr_multiaddr; if (sopt->sopt_name == IP_ADD_SOURCE_MEMBERSHIP) { - ssa->sin.sin_family = AF_INET; - ssa->sin.sin_len = sizeof(struct sockaddr_in); - ssa->sin.sin_addr = mreqs.imr_sourceaddr; + ssa->sin_family = AF_INET; + ssa->sin_len = sizeof(struct sockaddr_in); + ssa->sin_addr = mreqs.imr_sourceaddr; } - if (!IN_MULTICAST(ntohl(gsa->sin.sin_addr.s_addr))) + if (!IN_MULTICAST(ntohl(gsa->sin_addr.s_addr))) return (EINVAL); - ifp = inp_lookup_mcast_ifp(inp, &gsa->sin, - mreqs.imr_interface); + ifp = inp_lookup_mcast_ifp(inp, gsa, mreqs.imr_interface); IGMP_INET_PRINTF(mreqs.imr_interface, ("%s: imr_interface = %s, ifp = 0x%llx\n", __func__, _igmp_inet_buf, (uint64_t)VM_KERNEL_ADDRPERM(ifp))); @@ -2147,23 +2133,23 @@ inp_join_group(struct inpcb *inp, struct sockopt *sopt) if (error) return (error); - if (gsa->sin.sin_family != AF_INET || - gsa->sin.sin_len != sizeof(struct sockaddr_in)) + if (gsa->sin_family != AF_INET || + gsa->sin_len != sizeof(struct sockaddr_in)) return (EINVAL); /* * Overwrite the port field if present, as the sockaddr * being copied in may be matched with a binary comparison. */ - gsa->sin.sin_port = 0; + gsa->sin_port = 0; if (sopt->sopt_name == MCAST_JOIN_SOURCE_GROUP) { - if (ssa->sin.sin_family != AF_INET || - ssa->sin.sin_len != sizeof(struct sockaddr_in)) + if (ssa->sin_family != AF_INET || + ssa->sin_len != sizeof(struct sockaddr_in)) return (EINVAL); - ssa->sin.sin_port = 0; + ssa->sin_port = 0; } - if (!IN_MULTICAST(ntohl(gsa->sin.sin_addr.s_addr))) + if (!IN_MULTICAST(ntohl(gsa->sin_addr.s_addr))) return (EINVAL); ifnet_head_lock_shared(); @@ -2186,18 +2172,26 @@ inp_join_group(struct inpcb *inp, struct sockopt *sopt) if (ifp == NULL || (ifp->if_flags & IFF_MULTICAST) == 0) return (EADDRNOTAVAIL); + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_mcast_join_total); + /* + * TBD: revisit the criteria for non-OS initiated joins + */ + if (inp->inp_lport == htons(5353)) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_mcast_join_os_total); + } + imo = inp_findmoptions(inp); if (imo == NULL) return (ENOMEM); IMO_LOCK(imo); - idx = imo_match_group(imo, ifp, &gsa->sa); + idx = imo_match_group(imo, ifp, gsa); if (idx == (size_t)-1) { is_new = 1; } else { inm = imo->imo_membership[idx]; imf = &imo->imo_mfilters[idx]; - if (ssa->ss.ss_family != AF_UNSPEC) { + if (ssa->sin_family != AF_UNSPEC) { /* * MCAST_JOIN_SOURCE_GROUP on an exclusive membership * is an error. On an existing inclusive membership, @@ -2223,7 +2217,7 @@ inp_join_group(struct inpcb *inp, struct sockopt *sopt) * full-state SSM API with the delta-based API, * which is discouraged in the relevant RFCs. */ - lims = imo_match_source(imo, idx, &ssa->sa); + lims = imo_match_source(imo, idx, ssa); if (lims != NULL /*&& lims->imsl_st[1] == MCAST_INCLUDE*/) { error = EADDRNOTAVAIL; @@ -2281,7 +2275,7 @@ inp_join_group(struct inpcb *inp, struct sockopt *sopt) * been allocated yet if this is a new membership, however, * the in_mfilter slot will be allocated and must be initialized. */ - if (ssa->ss.ss_family != AF_UNSPEC) { + if (ssa->sin_family != AF_UNSPEC) { /* Membership starts in IN mode */ if (is_new) { IGMP_PRINTF(("%s: new join w/source\n", __func__)); @@ -2289,7 +2283,7 @@ inp_join_group(struct inpcb *inp, struct sockopt *sopt) } else { IGMP_PRINTF(("%s: %s source\n", __func__, "allow")); } - lims = imf_graft(imf, MCAST_INCLUDE, &ssa->sin); + lims = imf_graft(imf, MCAST_INCLUDE, ssa); if (lims == NULL) { IGMP_PRINTF(("%s: merge imf state failed\n", __func__)); @@ -2319,7 +2313,7 @@ inp_join_group(struct inpcb *inp, struct sockopt *sopt) socket_unlock(inp->inp_socket, 0); VERIFY(inm == NULL); - error = in_joingroup(ifp, &gsa->sin.sin_addr, imf, &inm); + error = in_joingroup(ifp, &gsa->sin_addr, imf, &inm); socket_lock(inp->inp_socket, 0); IMO_REMREF(imo); @@ -2388,7 +2382,7 @@ inp_leave_group(struct inpcb *inp, struct sockopt *sopt) { struct group_source_req gsr; struct ip_mreq_source mreqs; - sockunion_t *gsa, *ssa; + struct sockaddr_in *gsa, *ssa; struct ifnet *ifp; struct in_mfilter *imf; struct ip_moptions *imo; @@ -2405,10 +2399,8 @@ inp_leave_group(struct inpcb *inp, struct sockopt *sopt) is_final = 1; memset(&gsr, 0, sizeof(struct group_source_req)); - gsa = (sockunion_t *)&gsr.gsr_group; - gsa->ss.ss_family = AF_UNSPEC; - ssa = (sockunion_t *)&gsr.gsr_source; - ssa->ss.ss_family = AF_UNSPEC; + gsa = (struct sockaddr_in *)&gsr.gsr_group; + ssa = (struct sockaddr_in *)&gsr.gsr_source; switch (sopt->sopt_name) { case IP_DROP_MEMBERSHIP: @@ -2432,14 +2424,14 @@ inp_leave_group(struct inpcb *inp, struct sockopt *sopt) if (error) return (error); - gsa->sin.sin_family = AF_INET; - gsa->sin.sin_len = sizeof(struct sockaddr_in); - gsa->sin.sin_addr = mreqs.imr_multiaddr; + gsa->sin_family = AF_INET; + gsa->sin_len = sizeof(struct sockaddr_in); + gsa->sin_addr = mreqs.imr_multiaddr; if (sopt->sopt_name == IP_DROP_SOURCE_MEMBERSHIP) { - ssa->sin.sin_family = AF_INET; - ssa->sin.sin_len = sizeof(struct sockaddr_in); - ssa->sin.sin_addr = mreqs.imr_sourceaddr; + ssa->sin_family = AF_INET; + ssa->sin_len = sizeof(struct sockaddr_in); + ssa->sin_addr = mreqs.imr_sourceaddr; } /* * Attempt to look up hinted ifp from interface address. @@ -2471,13 +2463,13 @@ inp_leave_group(struct inpcb *inp, struct sockopt *sopt) if (error) return (error); - if (gsa->sin.sin_family != AF_INET || - gsa->sin.sin_len != sizeof(struct sockaddr_in)) + if (gsa->sin_family != AF_INET || + gsa->sin_len != sizeof(struct sockaddr_in)) return (EINVAL); if (sopt->sopt_name == MCAST_LEAVE_SOURCE_GROUP) { - if (ssa->sin.sin_family != AF_INET || - ssa->sin.sin_len != sizeof(struct sockaddr_in)) + if (ssa->sin_family != AF_INET || + ssa->sin_len != sizeof(struct sockaddr_in)) return (EINVAL); } @@ -2498,7 +2490,7 @@ inp_leave_group(struct inpcb *inp, struct sockopt *sopt) return (EOPNOTSUPP); } - if (!IN_MULTICAST(ntohl(gsa->sin.sin_addr.s_addr))) + if (!IN_MULTICAST(ntohl(gsa->sin_addr.s_addr))) return (EINVAL); /* @@ -2509,7 +2501,7 @@ inp_leave_group(struct inpcb *inp, struct sockopt *sopt) return (ENOMEM); IMO_LOCK(imo); - idx = imo_match_group(imo, ifp, &gsa->sa); + idx = imo_match_group(imo, ifp, gsa); if (idx == (size_t)-1) { error = EADDRNOTAVAIL; goto out_locked; @@ -2517,7 +2509,7 @@ inp_leave_group(struct inpcb *inp, struct sockopt *sopt) inm = imo->imo_membership[idx]; imf = &imo->imo_mfilters[idx]; - if (ssa->ss.ss_family != AF_UNSPEC) { + if (ssa->sin_family != AF_UNSPEC) { IGMP_PRINTF(("%s: opt=%d is_final=0\n", __func__, sopt->sopt_name)); is_final = 0; @@ -2538,16 +2530,16 @@ inp_leave_group(struct inpcb *inp, struct sockopt *sopt) error = EADDRNOTAVAIL; goto out_locked; } - ims = imo_match_source(imo, idx, &ssa->sa); + ims = imo_match_source(imo, idx, ssa); if (ims == NULL) { - IGMP_INET_PRINTF(ssa->sin.sin_addr, + IGMP_INET_PRINTF(ssa->sin_addr, ("%s: source %s %spresent\n", __func__, _igmp_inet_buf, "not ")); error = EADDRNOTAVAIL; goto out_locked; } IGMP_PRINTF(("%s: %s source\n", __func__, "block")); - error = imf_prune(imf, &ssa->sin); + error = imf_prune(imf, ssa); if (error) { IGMP_PRINTF(("%s: merge imf state failed\n", __func__)); @@ -2647,6 +2639,7 @@ inp_set_multicast_if(struct inpcb *inp, struct sockopt *sopt) int error = 0 ; unsigned int ifindex = 0; + bzero(&addr, sizeof(addr)); if (sopt->sopt_valsize == sizeof(struct ip_mreqn)) { /* * An interface index was specified using the @@ -2728,7 +2721,7 @@ inp_set_source_filters(struct inpcb *inp, struct sockopt *sopt) { struct __msfilterreq64 msfr, msfr64; struct __msfilterreq32 msfr32; - sockunion_t *gsa; + struct sockaddr_in *gsa; struct ifnet *ifp; struct in_mfilter *imf; struct ip_moptions *imo; @@ -2773,11 +2766,11 @@ inp_set_source_filters(struct inpcb *inp, struct sockopt *sopt) msfr.msfr_group.ss_len != sizeof(struct sockaddr_in)) return (EINVAL); - gsa = (sockunion_t *)&msfr.msfr_group; - if (!IN_MULTICAST(ntohl(gsa->sin.sin_addr.s_addr))) + gsa = (struct sockaddr_in *)&msfr.msfr_group; + if (!IN_MULTICAST(ntohl(gsa->sin_addr.s_addr))) return (EINVAL); - gsa->sin.sin_port = 0; /* ignore port */ + gsa->sin_port = 0; /* ignore port */ ifnet_head_lock_shared(); if (msfr.msfr_ifindex == 0 || (u_int)if_index < msfr.msfr_ifindex) { @@ -2798,7 +2791,7 @@ inp_set_source_filters(struct inpcb *inp, struct sockopt *sopt) return (ENOMEM); IMO_LOCK(imo); - idx = imo_match_group(imo, ifp, &gsa->sa); + idx = imo_match_group(imo, ifp, gsa); if (idx == (size_t)-1 || imo->imo_mfilters == NULL) { error = EADDRNOTAVAIL; goto out_imo_locked; @@ -3571,7 +3564,10 @@ in_multihead_lock_shared(void) void in_multihead_lock_assert(int what) { - lck_rw_assert(&in_multihead_lock, what); +#if !MACH_ASSERT +#pragma unused(what) +#endif + LCK_RW_ASSERT(&in_multihead_lock, what); } void diff --git a/bsd/netinet/in_pcb.c b/bsd/netinet/in_pcb.c index c20961c22..3371c71b3 100644 --- a/bsd/netinet/in_pcb.c +++ b/bsd/netinet/in_pcb.c @@ -135,12 +135,6 @@ static boolean_t intcoproc_unrestricted = FALSE; extern char *proc_best_name(proc_t); -/* - * If the total number of gc reqs is above a threshold, schedule - * garbage collect timer sooner - */ -static boolean_t inpcb_toomany_gcreq = FALSE; - #define INPCB_GCREQ_THRESHOLD 50000 static thread_call_t inpcb_thread_call, inpcb_fast_thread_call; @@ -221,7 +215,16 @@ SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast, static uint32_t apn_fallbk_debug = 0; #define apn_fallbk_log(x) do { if (apn_fallbk_debug >= 1) log x; } while (0) +#if CONFIG_EMBEDDED +static boolean_t apn_fallbk_enabled = TRUE; + +SYSCTL_DECL(_net_inet); +SYSCTL_NODE(_net_inet, OID_AUTO, apn_fallback, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "APN Fallback"); +SYSCTL_UINT(_net_inet_apn_fallback, OID_AUTO, debug, CTLFLAG_RW | CTLFLAG_LOCKED, + &apn_fallbk_debug, 0, "APN fallback debug enable"); +#else static boolean_t apn_fallbk_enabled = FALSE; +#endif extern int udp_use_randomport; extern int tcp_use_randomport; @@ -315,16 +318,10 @@ in_pcbinit(void) static void inpcb_timeout(void *arg0, void *arg1) { -#pragma unused(arg0) +#pragma unused(arg0, arg1) struct inpcbinfo *ipi; boolean_t t, gc; struct intimercount gccnt, tmcnt; - boolean_t toomany_gc = FALSE; - - if (arg1 != NULL) { - VERIFY(arg1 == &inpcb_toomany_gcreq); - toomany_gc = *(boolean_t *)arg1; - } /* * Update coarse-grained networking timestamp (in sec.); the idea @@ -368,7 +365,7 @@ inpcb_timeout(void *arg0, void *arg1) ipi->ipi_timer(ipi); tmcnt.intimer_lazy += ipi->ipi_timer_req.intimer_lazy; - tmcnt.intimer_lazy += + tmcnt.intimer_fast += ipi->ipi_timer_req.intimer_fast; tmcnt.intimer_nodelay += ipi->ipi_timer_req.intimer_nodelay; @@ -386,12 +383,8 @@ inpcb_timeout(void *arg0, void *arg1) inpcb_ticking = INPCB_HAVE_TIMER_REQ(tmcnt); /* re-arm the timer if there's work to do */ - if (toomany_gc) { - inpcb_toomany_gcreq = FALSE; - } else { - inpcb_timeout_run--; - VERIFY(inpcb_timeout_run >= 0 && inpcb_timeout_run < 2); - } + inpcb_timeout_run--; + VERIFY(inpcb_timeout_run >= 0 && inpcb_timeout_run < 2); if (gccnt.intimer_nodelay > 0 || tmcnt.intimer_nodelay > 0) inpcb_sched_timeout(); @@ -422,7 +415,7 @@ _inpcb_sched_timeout(unsigned int offset) uint64_t deadline, leeway; clock_interval_to_deadline(1, NSEC_PER_SEC, &deadline); - lck_mtx_assert(&inpcb_timeout_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&inpcb_timeout_lock, LCK_MTX_ASSERT_OWNED); if (inpcb_timeout_run == 0 && (inpcb_garbage_collecting || inpcb_ticking)) { lck_mtx_convert_spin(&inpcb_timeout_lock); @@ -457,25 +450,14 @@ void inpcb_gc_sched(struct inpcbinfo *ipi, u_int32_t type) { u_int32_t gccnt; - uint64_t deadline; lck_mtx_lock_spin(&inpcb_timeout_lock); inpcb_garbage_collecting = TRUE; gccnt = ipi->ipi_gc_req.intimer_nodelay + ipi->ipi_gc_req.intimer_fast; - if (gccnt > INPCB_GCREQ_THRESHOLD && !inpcb_toomany_gcreq) { - inpcb_toomany_gcreq = TRUE; - - /* - * There are toomany pcbs waiting to be garbage collected, - * schedule a much faster timeout in addition to - * the caller's request - */ - lck_mtx_convert_spin(&inpcb_timeout_lock); - clock_interval_to_deadline(100, NSEC_PER_MSEC, &deadline); - thread_call_enter1_delayed(inpcb_thread_call, - &inpcb_toomany_gcreq, deadline); + if (gccnt > INPCB_GCREQ_THRESHOLD) { + type = INPCB_TIMER_FAST; } switch (type) { @@ -681,7 +663,7 @@ in_pcblookup_local_and_cleanup(struct inpcbinfo *pcbinfo, struct in_addr laddr, if (inp != NULL && inp->inp_wantcnt == WNT_STOPUSING) { struct socket *so = inp->inp_socket; - lck_mtx_lock(&inp->inpcb_mtx); + socket_lock(so, 0); if (so->so_usecount == 0) { if (inp->inp_state != INPCB_STATE_DEAD) @@ -689,7 +671,7 @@ in_pcblookup_local_and_cleanup(struct inpcbinfo *pcbinfo, struct in_addr laddr, in_pcbdispose(inp); /* will unlock & destroy */ inp = NULL; } else { - lck_mtx_unlock(&inp->inpcb_mtx); + socket_unlock(so, 0); } } @@ -824,6 +806,7 @@ in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct proc *p) struct inpcb *t; uid_t u; +#if !CONFIG_EMBEDDED if (ntohs(lport) < IPPORT_RESERVED) { cred = kauth_cred_proc_ref(p); error = priv_check_cred(cred, @@ -835,6 +818,7 @@ in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct proc *p) return (EACCES); } } +#endif /* !CONFIG_EMBEDDED */ if (!IN_MULTICAST(ntohl(SIN(nam)->sin_addr.s_addr)) && (u = kauth_cred_getuid(so->so_cred)) != 0 && (t = in_pcblookup_local_and_cleanup( @@ -891,6 +875,7 @@ in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct proc *p) if (lport == 0) { u_short first, last; int count; + bool found; randomport = (so->so_flags & SOF_BINDRANDOMPORT) || (so->so_type == SOCK_STREAM ? tcp_use_randomport : @@ -935,16 +920,22 @@ in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct proc *p) * is not being tested on each round of the loop. */ if (first > last) { + struct in_addr lookup_addr; + /* * counting down */ if (randomport) { - read_random(&rand_port, sizeof (rand_port)); + read_frandom(&rand_port, sizeof (rand_port)); *lastport = first - (rand_port % (first - last)); } count = first - last; + lookup_addr = (laddr.s_addr != INADDR_ANY) ? laddr : + inp->inp_laddr; + + found = false; do { if (count-- < 0) { /* completely used? */ lck_rw_done(pcbinfo->ipi_lock); @@ -955,20 +946,27 @@ in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct proc *p) if (*lastport > first || *lastport < last) *lastport = first; lport = htons(*lastport); - } while (in_pcblookup_local_and_cleanup(pcbinfo, - ((laddr.s_addr != INADDR_ANY) ? laddr : - inp->inp_laddr), lport, wild)); + + found = in_pcblookup_local_and_cleanup(pcbinfo, + lookup_addr, lport, wild) == NULL; + } while (!found); } else { + struct in_addr lookup_addr; + /* * counting up */ if (randomport) { - read_random(&rand_port, sizeof (rand_port)); + read_frandom(&rand_port, sizeof (rand_port)); *lastport = first + (rand_port % (first - last)); } count = last - first; + lookup_addr = (laddr.s_addr != INADDR_ANY) ? laddr : + inp->inp_laddr; + + found = false; do { if (count-- < 0) { /* completely used? */ lck_rw_done(pcbinfo->ipi_lock); @@ -979,9 +977,10 @@ in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct proc *p) if (*lastport < first || *lastport > last) *lastport = first; lport = htons(*lastport); - } while (in_pcblookup_local_and_cleanup(pcbinfo, - ((laddr.s_addr != INADDR_ANY) ? laddr : - inp->inp_laddr), lport, wild)); + + found = in_pcblookup_local_and_cleanup(pcbinfo, + lookup_addr, lport, wild) == NULL; + } while (!found); } } socket_lock(so, 0); @@ -1586,6 +1585,12 @@ in_pcbdetach(struct inpcb *inp) } #endif /* IPSEC */ + if (inp->inp_stat != NULL && SOCK_PROTO(so) == IPPROTO_UDP) { + if (inp->inp_stat->rxpackets == 0 && inp->inp_stat->txpackets == 0) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_inet_dgram_no_data); + } + } + /* * Let NetworkStatistics know this PCB is going away * before we detach it. @@ -1664,7 +1669,7 @@ in_pcbdispose(struct inpcb *inp) } } - lck_rw_assert(ipi->ipi_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(ipi->ipi_lock, LCK_RW_ASSERT_EXCLUSIVE); inp->inp_gencnt = ++ipi->ipi_gencnt; /* access ipi in in_pcbremlists */ @@ -1687,6 +1692,11 @@ in_pcbdispose(struct inpcb *inp) /* NOTREACHED */ } lck_mtx_unlock(&inp->inpcb_mtx); + +#if NECP + necp_inpcb_remove_cb(inp); +#endif /* NECP */ + lck_mtx_destroy(&inp->inpcb_mtx, ipi->ipi_lock_grp); } /* makes sure we're not called twice from so_close */ @@ -1747,9 +1757,9 @@ in_getsockaddr(struct socket *so, struct sockaddr **nam) } int -in_getsockaddr_s(struct socket *so, struct sockaddr_storage *ss) +in_getsockaddr_s(struct socket *so, struct sockaddr_in *ss) { - struct sockaddr_in *sin = SIN(ss); + struct sockaddr_in *sin = ss; struct inpcb *inp; VERIFY(ss != NULL); @@ -1758,12 +1768,8 @@ in_getsockaddr_s(struct socket *so, struct sockaddr_storage *ss) sin->sin_family = AF_INET; sin->sin_len = sizeof (*sin); - if ((inp = sotoinpcb(so)) == NULL -#if NECP - || (necp_socket_should_use_flow_divert(inp)) -#endif /* NECP */ - ) - return (inp == NULL ? EINVAL : EPROTOTYPE); + if ((inp = sotoinpcb(so)) == NULL) + return (EINVAL); sin->sin_port = inp->inp_lport; sin->sin_addr = inp->inp_laddr; @@ -1797,31 +1803,6 @@ in_getpeeraddr(struct socket *so, struct sockaddr **nam) return (0); } -int -in_getpeeraddr_s(struct socket *so, struct sockaddr_storage *ss) -{ - struct sockaddr_in *sin = SIN(ss); - struct inpcb *inp; - - VERIFY(ss != NULL); - bzero(ss, sizeof (*ss)); - - sin->sin_family = AF_INET; - sin->sin_len = sizeof (*sin); - - if ((inp = sotoinpcb(so)) == NULL -#if NECP - || (necp_socket_should_use_flow_divert(inp)) -#endif /* NECP */ - ) { - return (inp == NULL ? EINVAL : EPROTOTYPE); - } - - sin->sin_port = inp->inp_fport; - sin->sin_addr = inp->inp_faddr; - return (0); -} - void in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int errno, void (*notify)(struct inpcb *, int)) @@ -2346,6 +2327,8 @@ in_pcbinshash(struct inpcb *inp, int locked) } VERIFY(!(inp->inp_flags2 & INP2_INHASHLIST)); + + inp->inp_phd = phd; LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist); LIST_INSERT_HEAD(pcbhash, inp, inp_hash); @@ -2594,6 +2577,7 @@ inpcb_to_compat(struct inpcb *inp, struct inpcb_compat *inp_compat) inp_compat->inp_depend6.inp6_hops = inp->inp_depend6.inp6_hops; } +#if !CONFIG_EMBEDDED void inpcb_to_xinpcb64(struct inpcb *inp, struct xinpcb64 *xinp) { @@ -2613,6 +2597,7 @@ inpcb_to_xinpcb64(struct inpcb *inp, struct xinpcb64 *xinp) xinp->inp_depend6.inp6_ifindex = 0; xinp->inp_depend6.inp6_hops = inp->inp_depend6.inp6_hops; } +#endif /* !CONFIG_EMBEDDED */ /* * The following routines implement this scheme: @@ -2645,7 +2630,7 @@ inp_route_copyout(struct inpcb *inp, struct route *dst) { struct route *src = &inp->inp_route; - lck_mtx_assert(&inp->inpcb_mtx, LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(inp->inp_socket); /* * If the route in the PCB is stale or not for IPv4, blow it away; @@ -2662,7 +2647,7 @@ inp_route_copyin(struct inpcb *inp, struct route *src) { struct route *dst = &inp->inp_route; - lck_mtx_assert(&inp->inpcb_mtx, LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(inp->inp_socket); /* Minor sanity check */ if (src->ro_rt != NULL && rt_key(src->ro_rt)->sa_family != AF_INET) @@ -3487,3 +3472,16 @@ inp_decr_sndbytes_allunsent(struct socket *so, u_int32_t th_ack) len = inp_get_sndbytes_allunsent(so, th_ack); inp_decr_sndbytes_unsent(so, len); } + + +inline void +inp_set_activity_bitmap(struct inpcb *inp) +{ + in_stat_set_activity_bitmap(&inp->inp_nw_activity, net_uptime()); +} + +inline void +inp_get_activity_bitmap(struct inpcb *inp, activity_bitmap_t *ab) +{ + bcopy(&inp->inp_nw_activity, ab, sizeof (*ab)); +} diff --git a/bsd/netinet/in_pcb.h b/bsd/netinet/in_pcb.h index 11ad54fe2..588a4d054 100644 --- a/bsd/netinet/in_pcb.h +++ b/bsd/netinet/in_pcb.h @@ -78,16 +78,17 @@ #include <sys/tree.h> #include <kern/locks.h> #include <kern/zalloc.h> +#include <netinet/in_stat.h> #endif /* BSD_KERNEL_PRIVATE */ +#if IPSEC #include <netinet6/ipsec.h> /* for IPSEC */ +#endif /* IPSEC */ + #if NECP #include <net/necp.h> #endif -#if IPSEC -#include <netinet6/ipsec.h> /* for IPSEC */ -#endif #ifdef BSD_KERNEL_PRIVATE /* @@ -219,6 +220,8 @@ struct inpcb { char *inp_account; } inp_necp_attributes; struct necp_inpcb_result inp_policyresult; + uuid_t necp_client_uuid; + void (*necp_cb)(void *, int, struct necp_client_flow *); #endif u_char *inp_keepalive_data; /* for keepalive offload */ u_int8_t inp_keepalive_datalen; /* keepalive data length */ @@ -233,6 +236,8 @@ struct inpcb { u_int8_t inp_cstat_store[sizeof (struct inp_stat) + sizeof (u_int64_t)]; u_int8_t inp_wstat_store[sizeof (struct inp_stat) + sizeof (u_int64_t)]; u_int8_t inp_Wstat_store[sizeof (struct inp_stat) + sizeof (u_int64_t)]; + activity_bitmap_t inp_nw_activity; + u_int64_t inp_start_timestamp; }; #define INP_ADD_STAT(_inp, _cnt_cellular, _cnt_wifi, _cnt_wired, _a, _n)\ @@ -367,6 +372,7 @@ struct xinpcb { u_quad_t xi_alignment_hack; }; +#if !CONFIG_EMBEDDED struct inpcb64_list_entry { u_int64_t le_next; u_int64_t le_prev; @@ -408,6 +414,7 @@ struct xinpcb64 { struct xsocket64 xi_socket; u_quad_t xi_alignment_hack; }; +#endif /* !CONFIG_EMBEDDED */ #ifdef PRIVATE struct xinpcb_list_entry { @@ -596,6 +603,7 @@ struct inpcbinfo { lck_grp_attr_t *ipi_lock_grp_attr; #define INPCBINFO_UPDATE_MSS 0x1 +#define INPCBINFO_HANDLE_LQM_ABORT 0x2 u_int32_t ipi_flags; }; @@ -699,6 +707,7 @@ struct inpcbinfo { #define INP2_AWDL_UNRESTRICTED 0x00000020 /* AWDL restricted mode allowed */ #define INP2_KEEPALIVE_OFFLOAD 0x00000040 /* Enable UDP or TCP keepalive offload */ #define INP2_INTCOPROC_ALLOWED 0x00000080 /* Allow communication via internal co-processor interfaces */ +#define INP2_CONNECT_IN_PROGRESS 0x00000100 /* A connect call is in progress, so binds are intermediate steps */ /* * Flags passed to in_pcblookup*() functions. @@ -768,13 +777,14 @@ extern void in_pcbnotifyall(struct inpcbinfo *, struct in_addr, int, void (*)(struct inpcb *, int)); extern void in_pcbrehash(struct inpcb *); extern int in_getpeeraddr(struct socket *, struct sockaddr **); -extern int in_getpeeraddr_s(struct socket *, struct sockaddr_storage *); extern int in_getsockaddr(struct socket *, struct sockaddr **); -extern int in_getsockaddr_s(struct socket *, struct sockaddr_storage *); +extern int in_getsockaddr_s(struct socket *, struct sockaddr_in *); extern int in_pcb_checkstate(struct inpcb *, int, int); extern void in_pcbremlists(struct inpcb *); extern void inpcb_to_compat(struct inpcb *, struct inpcb_compat *); +#if !CONFIG_EMBEDDED extern void inpcb_to_xinpcb64(struct inpcb *, struct xinpcb64 *); +#endif extern int get_pcblist_n(short, struct sysctl_req *, struct inpcbinfo *); #define INPCB_GET_PORTS_USED_WILDCARDOK 0x01 @@ -825,6 +835,8 @@ extern void inp_incr_sndbytes_unsent(struct socket *, int32_t); extern void inp_decr_sndbytes_unsent(struct socket *, int32_t); extern int32_t inp_get_sndbytes_allunsent(struct socket *, u_int32_t); extern void inp_decr_sndbytes_allunsent(struct socket *, u_int32_t); +extern void inp_set_activity_bitmap(struct inpcb *inp); +extern void inp_get_activity_bitmap(struct inpcb *inp, activity_bitmap_t *b); #endif /* BSD_KERNEL_PRIVATE */ #ifdef KERNEL_PRIVATE /* exported for PPP */ diff --git a/bsd/netinet/in_pcblist.c b/bsd/netinet/in_pcblist.c index 5e667775b..c67d76635 100644 --- a/bsd/netinet/in_pcblist.c +++ b/bsd/netinet/in_pcblist.c @@ -71,6 +71,7 @@ #include <net/route.h> #include <net/if_var.h> +#include <net/ntstat.h> #include <netinet/in.h> #include <netinet/in_pcb.h> @@ -269,22 +270,22 @@ get_pcblist_n(short proto, struct sysctl_req *req, struct inpcbinfo *pcbinfo) if (proto == IPPROTO_TCP) item_size += ROUNDUP64(sizeof (struct xtcpcb_n)); - /* - * The process of preparing the PCB list is too time-consuming and - * resource-intensive to repeat twice on every request. - */ - lck_rw_lock_exclusive(pcbinfo->ipi_lock); if (req->oldptr == USER_ADDR_NULL) { n = pcbinfo->ipi_count; - req->oldidx = 2 * (sizeof (xig)) + (n + n/8) * item_size; - goto done; + req->oldidx = 2 * (sizeof (xig)) + (n + n/8 + 1) * item_size; + return 0; } if (req->newptr != USER_ADDR_NULL) { - error = EPERM; - goto done; + return EPERM; } + + /* + * The process of preparing the PCB list is too time-consuming and + * resource-intensive to repeat twice on every request. + */ + lck_rw_lock_exclusive(pcbinfo->ipi_lock); /* * OK, now we're committed to doing something. */ @@ -303,7 +304,7 @@ get_pcblist_n(short proto, struct sysctl_req *req, struct inpcbinfo *pcbinfo) /* * We are done if there is no pcb */ - if (n == 0) { + if (xig.xig_count == 0) { goto done; } @@ -375,8 +376,12 @@ get_pcblist_n(short proto, struct sysctl_req *req, struct inpcbinfo *pcbinfo) inp->inp_ppcb, xt); } error = SYSCTL_OUT(req, buf, item_size); + if (error) { + break; + } } } + if (!error) { /* * Give the user an updated idea of our state. @@ -394,6 +399,7 @@ get_pcblist_n(short proto, struct sysctl_req *req, struct inpcbinfo *pcbinfo) } done: lck_rw_done(pcbinfo->ipi_lock); + if (inp_list != NULL) FREE(inp_list, M_TEMP); if (buf != NULL) @@ -517,7 +523,7 @@ inpcb_get_ports_used(uint32_t ifindex, int protocol, uint32_t flags, port = ntohs(inp->inp_lport); if (port == 0) continue; - bit_set(bitfield, port); + bitstr_set(bitfield, port); } lck_rw_done(pcbinfo->ipi_lock); } diff --git a/bsd/netinet/in_rmx.c b/bsd/netinet/in_rmx.c index fa4b580dc..1c0fc3696 100644 --- a/bsd/netinet/in_rmx.c +++ b/bsd/netinet/in_rmx.c @@ -103,8 +103,6 @@ static int in_rtqkill(struct radix_node *, void *); static int in_ifadownkill(struct radix_node *, void *); -#define RTPRF_OURS RTF_PROTO3 /* set on routes we manage */ - /* * Do what we need to do when inserting a route. */ @@ -119,7 +117,7 @@ in_addroute(void *v_arg, void *n_arg, struct radix_node_head *head, uint32_t flags = rt->rt_flags; boolean_t verbose = (rt_verbose > 1); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK_ASSERT_HELD(rt); if (verbose) @@ -252,7 +250,7 @@ in_deleteroute(void *v_arg, void *netmask_arg, struct radix_node_head *head) { struct radix_node *rn; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); rn = rn_delete(v_arg, netmask_arg, head); if (rt_verbose > 1 && rn != NULL) { @@ -362,11 +360,11 @@ in_clsroute(struct radix_node *rn, struct radix_node_head *head) struct rtentry *rt = (struct rtentry *)rn; boolean_t verbose = (rt_verbose > 1); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK_ASSERT_HELD(rt); if (!(rt->rt_flags & RTF_UP)) - return; /* prophylactic measures */ + return; /* prophylactic measures */ if ((rt->rt_flags & (RTF_LLINFO | RTF_HOST)) != RTF_HOST) return; @@ -464,7 +462,7 @@ in_rtqkill(struct radix_node *rn, void *rock) int err; timenow = net_uptime(); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK(rt); if (rt->rt_flags & RTPRF_OURS) { @@ -483,6 +481,7 @@ in_rtqkill(struct radix_node *rn, void *rock) rt, rt->rt_refcnt); /* NOTREACHED */ } + if (verbose) { log(LOG_DEBUG, "%s: deleting route to " "%s->%s->%s, flags=%b, draining=%d\n", @@ -617,7 +616,7 @@ in_rtqtimo(void *targ) static void in_sched_rtqtimo(struct timeval *atv) { - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); if (!in_rtqtimo_run) { struct timeval tv; @@ -712,7 +711,7 @@ in_ifadownkill(struct radix_node *rn, void *xap) boolean_t verbose = (rt_verbose != 0); int err; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK(rt); if (rt->rt_ifa == ap->ifa && @@ -765,7 +764,7 @@ in_ifadown(struct ifaddr *ifa, int delete) struct in_ifadown_arg arg; struct radix_node_head *rnh; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); /* * Holding rnh_lock here prevents the possibility of diff --git a/bsd/netinet/in_stat.c b/bsd/netinet/in_stat.c new file mode 100644 index 000000000..c7f36defb --- /dev/null +++ b/bsd/netinet/in_stat.c @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include <netinet/in_stat.h> + +#define IN_STAT_ACTIVITY_GRANULARITY 8 /* 8 sec granularity */ +#define IN_STAT_ACTIVITY_TIME_SEC_SHIFT 3 /* 8 sec per bit */ +#define IN_STAT_ACTIVITY_BITMAP_TOTAL_SIZE ((uint64_t) 128) +#define IN_STAT_ACTIVITY_BITMAP_FIELD_SIZE ((uint64_t) 64) +#define IN_STAT_ACTIVITY_TOTAL_TIME ((uint64_t) (8 * 128)) +#define IN_STAT_SET_MOST_SIGNIFICANT_BIT ((u_int64_t )0x8000000000000000) + +void +in_stat_set_activity_bitmap(activity_bitmap_t *activity, uint64_t now) +{ + uint64_t elapsed_time, slot; + uint64_t *bitmap; + if (activity->start == 0) + activity->start = now; + elapsed_time = now - activity->start; + + slot = elapsed_time >> IN_STAT_ACTIVITY_TIME_SEC_SHIFT; + if (slot < IN_STAT_ACTIVITY_BITMAP_TOTAL_SIZE) { + if (slot < IN_STAT_ACTIVITY_BITMAP_FIELD_SIZE) { + bitmap = &activity->bitmap[0]; + } else { + bitmap = &activity->bitmap[1]; + slot -= IN_STAT_ACTIVITY_BITMAP_FIELD_SIZE; + } + *bitmap |= (((u_int64_t) 1) << slot); + } else { + if (slot >= (IN_STAT_ACTIVITY_BITMAP_TOTAL_SIZE * 2)) { + activity->start = now - IN_STAT_ACTIVITY_TOTAL_TIME; + activity->bitmap[0] = activity->bitmap[1] = 0; + } else { + uint64_t shift = + slot - (IN_STAT_ACTIVITY_BITMAP_TOTAL_SIZE - 1); + /* + * Move the start time and bitmap forward to + * cover the lost time + */ + activity->start += + (shift << IN_STAT_ACTIVITY_TIME_SEC_SHIFT); + if (shift > IN_STAT_ACTIVITY_BITMAP_FIELD_SIZE) { + activity->bitmap[0] = activity->bitmap[1]; + activity->bitmap[1] = 0; + shift -= IN_STAT_ACTIVITY_BITMAP_FIELD_SIZE; + if (shift == IN_STAT_ACTIVITY_BITMAP_FIELD_SIZE) + activity->bitmap[0] = 0; + else + activity->bitmap[0] >>= shift; + } else { + uint64_t mask_lower, tmp; + uint64_t b1_low, b0_high; + + /* + * generate a mask with all of lower + * 'shift' bits set + */ + tmp = (((uint64_t)1) << (shift - 1)); + mask_lower = ((tmp - 1) ^ tmp); + activity->bitmap[0] >>= shift; + b1_low = (activity->bitmap[1] & mask_lower); + + b0_high = (b1_low << + (IN_STAT_ACTIVITY_BITMAP_FIELD_SIZE - + shift)); + activity->bitmap[0] |= b0_high; + activity->bitmap[1] >>= shift; + } + } + activity->bitmap[1] |= IN_STAT_SET_MOST_SIGNIFICANT_BIT; + } +} diff --git a/bsd/netinet/in_stat.h b/bsd/netinet/in_stat.h new file mode 100644 index 000000000..0c31148c7 --- /dev/null +++ b/bsd/netinet/in_stat.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef __NETINET_IN_STAT_H__ +#define __NETINET_IN_STAT_H__ + +#ifdef PRIVATE + +#include <stdint.h> + +typedef struct activity_bitmap { + uint64_t start; /* Start timestamp using uptime */ + uint64_t bitmap[2]; /* 128 bit map, each bit == 8 sec */ +} activity_bitmap_t; + +#endif /* PRIVATE */ + +#ifdef BSD_KERNEL_PRIVATE + +extern void in_stat_set_activity_bitmap(activity_bitmap_t *activity, uint64_t now); + +#endif /* BSD_KERNEL_PRIVATE */ + +#endif /* __NETINET_IN_STAT_H__ */ diff --git a/bsd/netinet/in_tclass.c b/bsd/netinet/in_tclass.c index 3fe179ce9..5d627e551 100644 --- a/bsd/netinet/in_tclass.c +++ b/bsd/netinet/in_tclass.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009-2016 Apple Inc. All rights reserved. + * Copyright (c) 2009-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -40,6 +40,7 @@ #include <sys/mbuf.h> #include <sys/queue.h> #include <sys/sysctl.h> +#include <sys/sysproto.h> #include <net/if.h> #include <net/if_var.h> @@ -1093,10 +1094,8 @@ set_tcp_stream_priority(struct socket *so) * happen when there is no indication of foreground * activity. */ - if (soissrcbackground(so) && - ((outifp->if_fg_sendts > 0 && - (int)(uptime - outifp->if_fg_sendts) <= - TCP_BG_SWITCH_TIME) || net_io_policy_throttled)) + if (soissrcbackground(so) && outifp->if_fg_sendts > 0 && + (int)(uptime - outifp->if_fg_sendts) <= TCP_BG_SWITCH_TIME) fg_active = true; /* @@ -1818,3 +1817,83 @@ sysctl_reset_dscp_to_wifi_ac_map SYSCTL_HANDLER_ARGS return (0); } + +/* + * Returns whether a large upload or download transfer should be marked as + * BK service type for network activity. This is a system level + * hint/suggestion to classify application traffic based on statistics + * collected from the current network attachment + * + * Returns 1 for BK and 0 for default + */ + +int +net_qos_guideline(struct proc *p, struct net_qos_guideline_args *arg, + int *retval) +{ +#pragma unused(p) +#define RETURN_USE_BK 1 +#define RETURN_USE_DEFAULT 0 + struct net_qos_param qos_arg; + struct ifnet *ipv4_primary, *ipv6_primary; + int err = 0; + + if (arg->param == USER_ADDR_NULL || retval == NULL || + arg->param_len != sizeof (qos_arg)) { + return (EINVAL); + } + err = copyin(arg->param, (caddr_t) &qos_arg, sizeof (qos_arg)); + if (err != 0) + return (err); + + *retval = RETURN_USE_DEFAULT; + ipv4_primary = ifindex2ifnet[get_primary_ifscope(AF_INET)]; + ipv6_primary = ifindex2ifnet[get_primary_ifscope(AF_INET6)]; + + /* + * If either of the interfaces is in Low Internet mode, enable + * background delay based algorithms on this transfer + */ + if (qos_arg.nq_uplink) { + if ((ipv4_primary != NULL && + (ipv4_primary->if_xflags & IFXF_LOW_INTERNET_UL)) || + (ipv6_primary != NULL && + (ipv6_primary->if_xflags & IFXF_LOW_INTERNET_UL))) { + *retval = RETURN_USE_BK; + return (0); + } + } else { + if ((ipv4_primary != NULL && + (ipv4_primary->if_xflags & IFXF_LOW_INTERNET_DL)) || + (ipv6_primary != NULL && + (ipv6_primary->if_xflags & IFXF_LOW_INTERNET_DL))) { + *retval = RETURN_USE_BK; + return (0); + } + } + + /* + * Some times IPv4 and IPv6 primary interfaces can be different. + * In this case, if either of them is non-cellular, we should mark + * the transfer as BK as it can potentially get used based on + * the host name resolution + */ + if (ipv4_primary != NULL && IFNET_IS_EXPENSIVE(ipv4_primary) && + ipv6_primary != NULL && IFNET_IS_EXPENSIVE(ipv6_primary)) { + if (qos_arg.nq_use_expensive) { + return (0); + } else { + *retval = RETURN_USE_BK; + return (0); + } + } + if (qos_arg.nq_transfer_size >= 5 * 1024 * 1024) { + *retval = RETURN_USE_BK; + return (0); + } + + +#undef RETURN_USE_BK +#undef RETURN_USE_DEFAULT + return (0); +} diff --git a/bsd/netinet/in_tclass.h b/bsd/netinet/in_tclass.h index 430de9f27..d42713162 100644 --- a/bsd/netinet/in_tclass.h +++ b/bsd/netinet/in_tclass.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016 Apple Inc. All rights reserved. + * Copyright (c) 2015-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -64,6 +64,29 @@ struct so_tcdbg { #define NET_QOS_MARKING_POLICY_ENABLE QOS_MODE_MARKING_POLICY_ENABLE /* obsolete, to be removed */ #define NET_QOS_MARKING_POLICY_DISABLE QOS_MODE_MARKING_POLICY_DISABLE /* obsolete, to be removed */ +struct net_qos_param { + u_int64_t nq_transfer_size; /* transfer size in bytes */ + u_int32_t nq_use_expensive:1, /* allowed = 1 otherwise 0 */ + nq_uplink:1; /* uplink = 1 otherwise 0 */ + u_int32_t nq_unused; /* for future expansion */ +}; + +#ifndef KERNEL + +/* + * Returns whether a large upload or download transfer should be marked as + * BK service type for network activity. This is a system level + * hint/suggestion to classify application traffic based on statistics + * collected from the current network attachment + * + * @param param transfer parameters + * @param param_len parameter length + * @return returns 1 for BK and 0 for default + */ +extern int net_qos_guideline(struct net_qos_param *param, size_t param_len); + +#endif /* !KERNEL */ + #ifdef BSD_KERNEL_PRIVATE extern int net_qos_policy_restricted; diff --git a/bsd/netinet/in_var.h b/bsd/netinet/in_var.h index 7368ed28d..5b0506120 100644 --- a/bsd/netinet/in_var.h +++ b/bsd/netinet/in_var.h @@ -147,6 +147,7 @@ struct kev_in_portinuse { #ifdef BSD_KERNEL_PRIVATE #include <net/if.h> #include <net/if_var.h> +#include <net/if_llatbl.h> #include <kern/locks.h> #include <sys/tree.h> /* @@ -335,10 +336,10 @@ struct in_multi { }; #define INM_LOCK_ASSERT_HELD(_inm) \ - lck_mtx_assert(&(_inm)->inm_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_inm)->inm_lock, LCK_MTX_ASSERT_OWNED) #define INM_LOCK_ASSERT_NOTHELD(_inm) \ - lck_mtx_assert(&(_inm)->inm_lock, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_inm)->inm_lock, LCK_MTX_ASSERT_NOTOWNED) #define INM_LOCK(_inm) \ lck_mtx_lock(&(_inm)->inm_lock) @@ -462,14 +463,17 @@ struct inpcb; struct in_ifextra { uint32_t netsig_len; u_int8_t netsig[IFNET_SIGNATURELEN]; + struct lltable *ii_llt; /* ARP state */ }; #define IN_IFEXTRA(_ifp) ((struct in_ifextra *)(_ifp->if_inetdata)) +#define LLTABLE(ifp) ((IN_IFEXTRA(ifp) == NULL) ? NULL : IN_IFEXTRA(ifp)->ii_llt) extern u_int32_t ipv4_ll_arp_aware; extern void in_ifaddr_init(void); -extern int imo_multi_filter(const struct ip_moptions *, const struct ifnet *, - const struct sockaddr *, const struct sockaddr *); +extern int imo_multi_filter(const struct ip_moptions *, + const struct ifnet *, const struct sockaddr_in *, + const struct sockaddr_in *); extern int imo_clone(struct inpcb *, struct inpcb *); extern void inm_commit(struct in_multi *); extern void inm_clear_recorded(struct in_multi *); diff --git a/bsd/netinet/ip_compat.h b/bsd/netinet/ip_compat.h index 70d0fadf4..5fb846683 100644 --- a/bsd/netinet/ip_compat.h +++ b/bsd/netinet/ip_compat.h @@ -334,9 +334,9 @@ extern vm_map_t kmem_map; # define SPL_X(x) (void) splx(x) # else # if !SOLARIS && !defined(linux) -# define SPL_IMP(x) x = splimp() -# define SPL_NET(x) x = splnet() -# define SPL_X(x) (void) splx(x) +# define SPL_IMP(x) ; +# define SPL_NET(x) ; +# define SPL_X(x) ; # endif # endif /* NetBSD && NetBSD <= 1991011 && NetBSD >= 199407 */ # define PANIC(x,y) if (x) panic y diff --git a/bsd/netinet/ip_divert.c b/bsd/netinet/ip_divert.c index 083b1484b..4981eb25e 100644 --- a/bsd/netinet/ip_divert.c +++ b/bsd/netinet/ip_divert.c @@ -757,7 +757,7 @@ div_unlock(struct socket *so, int refcount, void *lr) lck_rw_done(divcbinfo.ipi_lock); return (0); } - lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED); so->unlock_lr[so->next_unlock_lr] = lr_saved; so->next_unlock_lr = (so->next_unlock_lr+1) % SO_LCKDBG_MAX; lck_mtx_unlock(mutex_held); @@ -765,17 +765,17 @@ div_unlock(struct socket *so, int refcount, void *lr) } __private_extern__ lck_mtx_t * -div_getlock(struct socket *so, __unused int locktype) +div_getlock(struct socket *so, __unused int flags) { struct inpcb *inpcb = (struct inpcb *)so->so_pcb; - + if (so->so_pcb) { if (so->so_usecount < 0) - panic("div_getlock: so=%p usecount=%x lrh= %s\n", + panic("div_getlock: so=%p usecount=%x lrh= %s\n", so, so->so_usecount, solockhistory_nr(so)); return(&inpcb->inpcb_mtx); } else { - panic("div_getlock: so=%p NULL NO PCB lrh= %s\n", + panic("div_getlock: so=%p NULL NO PCB lrh= %s\n", so, solockhistory_nr(so)); return (so->so_proto->pr_domain->dom_mtx); } diff --git a/bsd/netinet/ip_dummynet.c b/bsd/netinet/ip_dummynet.c index b40ca4612..122698fa4 100644 --- a/bsd/netinet/ip_dummynet.c +++ b/bsd/netinet/ip_dummynet.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2013 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -85,6 +85,7 @@ #include <sys/mbuf.h> #include <sys/queue.h> /* XXX */ #include <sys/kernel.h> +#include <sys/random.h> #include <sys/socket.h> #include <sys/socketvar.h> #include <sys/time.h> @@ -95,6 +96,8 @@ #if DUMMYNET #include <net/kpi_protocol.h> #endif /* DUMMYNET */ +#include <net/nwk_wq.h> +#include <net/pfvar.h> #include <netinet/in.h> #include <netinet/in_systm.h> #include <netinet/in_var.h> @@ -167,7 +170,6 @@ static void dummynet_send(struct mbuf *m); static struct dn_pipe_head pipehash[HASHSIZE]; /* all pipes */ static struct dn_flow_set_head flowsethash[HASHSIZE]; /* all flowsets */ - #ifdef SYSCTL_NODE SYSCTL_NODE(_net_inet_ip, OID_AUTO, dummynet, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "Dummynet"); @@ -207,13 +209,6 @@ SYSCTL_INT(_net_inet_ip_dummynet, OID_AUTO, debug, CTLFLAG_RW | CTLFLAG_LOCKED, #define DPRINTF(X) #endif -/* contrary to the comment above random(), it does not actually - * return a value [0, 2^31 - 1], which breaks plr amongst other - * things. Masking it should work even if the behavior of - * the function is fixed. - */ -#define MY_RANDOM (random() & 0x7FFFFFFF) - /* dummynet lock */ static lck_grp_t *dn_mutex_grp; static lck_grp_attr_t *dn_mutex_grp_attr; @@ -229,8 +224,6 @@ static void dummynet_flush(void); void dummynet_drain(void); static ip_dn_io_t dummynet_io; -int if_tx_rdy(struct ifnet *ifp); - static void cp_flow_set_to_64_user(struct dn_flow_set *set, struct dn_flow_set_64 *fs_bp); static void cp_queue_to_64_user( struct dn_flow_queue *q, struct dn_flow_queue_64 *qp); static char *cp_pipe_to_64_user(struct dn_pipe *p, struct dn_pipe_64 *pipe_bp); @@ -243,6 +236,16 @@ static char *cp_pipe_to_32_user(struct dn_pipe *p, struct dn_pipe_32 *pipe_bp); static char* dn_copy_set_32(struct dn_flow_set *set, char *bp); static int cp_pipe_from_user_32( struct sockopt *sopt, struct dn_pipe *p ); +struct eventhandler_lists_ctxt dummynet_evhdlr_ctxt; + +uint32_t my_random(void) +{ + uint32_t val; + read_frandom(&val, sizeof(val)); + val &= 0x7FFFFFFF; + + return (val); +} /* * Heap management functions. @@ -730,7 +733,7 @@ transmit_event(struct dn_pipe *pipe, struct mbuf **head, struct mbuf **tail) struct dn_pkt_tag *pkt = NULL; u_int64_t schedule_time; - lck_mtx_assert(dn_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(dn_mutex, LCK_MTX_ASSERT_OWNED); ASSERT(serialize >= 0); if (serialize == 0) { while ((m = pipe->head) != NULL) { @@ -817,7 +820,7 @@ ready_event(struct dn_flow_queue *q, struct mbuf **head, struct mbuf **tail) struct dn_pipe *p = q->fs->pipe ; int p_was_empty ; - lck_mtx_assert(dn_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(dn_mutex, LCK_MTX_ASSERT_OWNED); if (p == NULL) { printf("dummynet: ready_event pipe is gone\n"); @@ -884,7 +887,7 @@ ready_event_wfq(struct dn_pipe *p, struct mbuf **head, struct mbuf **tail) struct dn_heap *neh = &(p->not_eligible_heap) ; int64_t p_numbytes = p->numbytes; - lck_mtx_assert(dn_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(dn_mutex, LCK_MTX_ASSERT_OWNED); if (p->if_name[0] == 0) /* tx clock is simulated */ p_numbytes += ( curr_time - p->sched_time ) * p->bandwidth; @@ -1145,58 +1148,6 @@ dummynet_send(struct mbuf *m) } } - - -/* - * called by an interface when tx_rdy occurs. - */ -int -if_tx_rdy(struct ifnet *ifp) -{ - struct dn_pipe *p; - struct mbuf *head = NULL, *tail = NULL; - int i; - - lck_mtx_lock(dn_mutex); - - for (i = 0; i < HASHSIZE; i++) - SLIST_FOREACH(p, &pipehash[i], next) - if (p->ifp == ifp) - break ; - if (p == NULL) { - char buf[32]; - snprintf(buf, sizeof(buf), "%s", if_name(ifp)); - for (i = 0; i < HASHSIZE; i++) - SLIST_FOREACH(p, &pipehash[i], next) - if (!strcmp(p->if_name, buf) ) { - p->ifp = ifp ; - DPRINTF(("dummynet: ++ tx rdy from %s (now found)\n", buf)); - break ; - } - } - if (p != NULL) { - DPRINTF(("dummynet: ++ tx rdy from %s - qlen %d\n", if_name(ifp), - IFCQ_LEN(&ifp->if_snd))); - p->numbytes = 0 ; /* mark ready for I/O */ - ready_event_wfq(p, &head, &tail); - } - - if (head != NULL) { - serialize++; - } - - lck_mtx_unlock(dn_mutex); - - /* Send out the de-queued list of ready-to-send packets */ - if (head != NULL) { - dummynet_send(head); - lck_mtx_lock(dn_mutex); - serialize--; - lck_mtx_unlock(dn_mutex); - } - return 0; -} - /* * Unconditionally expire empty queues in case of shortage. * Returns the number of queues freed. @@ -1460,7 +1411,7 @@ red_drops(struct dn_flow_set *fs, struct dn_flow_queue *q, int len) if (fs->flags_fs & DN_QSIZE_IS_BYTES) p_b = (p_b * len) / fs->max_pkt_size; if (++q->count == 0) - q->random = MY_RANDOM & 0xffff; + q->random = (my_random() & 0xffff); else { /* * q->count counts packets arrived since last drop, so a greater @@ -1470,7 +1421,7 @@ red_drops(struct dn_flow_set *fs, struct dn_flow_queue *q, int len) q->count = 0; DPRINTF(("dummynet: - red drop")); /* after a drop we calculate a new random value */ - q->random = MY_RANDOM & 0xffff; + q->random = (my_random() & 0xffff); return 1; /* drop */ } } @@ -1602,7 +1553,7 @@ dummynet_io(struct mbuf *m, int pipe_nr, int dir, struct ip_fw_args *fwa, int cl */ q->tot_bytes += len ; q->tot_pkts++ ; - if ( fs->plr && (MY_RANDOM < fs->plr) ) + if ( fs->plr && (my_random() < fs->plr)) goto dropit ; /* random pkt drop */ if ( fs->flags_fs & DN_QSIZE_IS_BYTES) { if (q->len_bytes > fs->qsize) @@ -1815,7 +1766,7 @@ purge_flow_set(struct dn_flow_set *fs, int all) struct dn_flow_queue *q, *qn ; int i ; - lck_mtx_assert(dn_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(dn_mutex, LCK_MTX_ASSERT_OWNED); for (i = 0 ; i <= fs->rq_size ; i++ ) { for (q = fs->rq[i] ; q ; q = qn ) { @@ -2065,7 +2016,6 @@ set_fs_parms(struct dn_flow_set *x, struct dn_flow_set *src) /* * setup pipe or queue parameters. */ - static int config_pipe(struct dn_pipe *p) { @@ -2087,7 +2037,7 @@ config_pipe(struct dn_pipe *p) return EINVAL ; if (p->pipe_nr != 0) { /* this is a pipe */ struct dn_pipe *x, *b; - + struct dummynet_event dn_event; lck_mtx_lock(dn_mutex); /* locate pipe */ @@ -2133,6 +2083,14 @@ config_pipe(struct dn_pipe *p) x, next); } lck_mtx_unlock(dn_mutex); + + bzero(&dn_event, sizeof(dn_event)); + dn_event.dn_event_code = DUMMYNET_PIPE_CONFIG; + dn_event.dn_event_pipe_config.bandwidth = p->bandwidth; + dn_event.dn_event_pipe_config.delay = p->delay; + dn_event.dn_event_pipe_config.plr = pfs->plr; + + dummynet_event_enqueue_nwk_wq_entry(&dn_event); } else { /* config queue */ struct dn_flow_set *x, *b ; @@ -2232,7 +2190,7 @@ dummynet_drain(void) struct mbuf *m, *mnext; int i; - lck_mtx_assert(dn_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(dn_mutex, LCK_MTX_ASSERT_OWNED); heap_free(&ready_heap); heap_free(&wfq_ready_heap); @@ -2348,7 +2306,7 @@ char* dn_copy_set_32(struct dn_flow_set *set, char *bp) struct dn_flow_queue *q; struct dn_flow_queue_32 *qp = (struct dn_flow_queue_32 *)bp; - lck_mtx_assert(dn_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(dn_mutex, LCK_MTX_ASSERT_OWNED); for (i = 0 ; i <= set->rq_size ; i++) for (q = set->rq[i] ; q ; q = q->next, qp++ ) { @@ -2380,7 +2338,7 @@ char* dn_copy_set_64(struct dn_flow_set *set, char *bp) struct dn_flow_queue *q; struct dn_flow_queue_64 *qp = (struct dn_flow_queue_64 *)bp; - lck_mtx_assert(dn_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(dn_mutex, LCK_MTX_ASSERT_OWNED); for (i = 0 ; i <= set->rq_size ; i++) for (q = set->rq[i] ; q ; q = q->next, qp++ ) { @@ -2417,7 +2375,7 @@ dn_calc_size(int is64user) size_t setsize; int i; - lck_mtx_assert(dn_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(dn_mutex, LCK_MTX_ASSERT_OWNED); if ( is64user ){ pipesize = sizeof(struct dn_pipe_64); queuesize = sizeof(struct dn_flow_queue_64); @@ -2445,70 +2403,74 @@ dn_calc_size(int is64user) static int dummynet_get(struct sockopt *sopt) { - char *buf, *bp=NULL; /* bp is the "copy-pointer" */ - size_t size ; - struct dn_flow_set *set ; - struct dn_pipe *p ; - int error=0, i ; - int is64user = 0; - - /* XXX lock held too long */ - lck_mtx_lock(dn_mutex); - /* - * XXX: Ugly, but we need to allocate memory with M_WAITOK flag and we - * cannot use this flag while holding a mutex. - */ + char *buf = NULL, *bp = NULL; /* bp is the "copy-pointer" */ + size_t size = 0; + struct dn_flow_set *set; + struct dn_pipe *p; + int error = 0, i; + int is64user = 0; + + /* XXX lock held too long */ + lck_mtx_lock(dn_mutex); + /* + * XXX: Ugly, but we need to allocate memory with M_WAITOK flag + * and we cannot use this flag while holding a mutex. + */ if (proc_is64bit(sopt->sopt_p)) is64user = 1; - for (i = 0; i < 10; i++) { + for (i = 0; i < 10; i++) { size = dn_calc_size(is64user); lck_mtx_unlock(dn_mutex); buf = _MALLOC(size, M_TEMP, M_WAITOK); if (buf == NULL) - return ENOBUFS; + return(ENOBUFS); lck_mtx_lock(dn_mutex); if (size == dn_calc_size(is64user)) break; FREE(buf, M_TEMP); buf = NULL; - } - if (buf == NULL) { + } + if (buf == NULL) { lck_mtx_unlock(dn_mutex); - return ENOBUFS ; - } - + return(ENOBUFS); + } - bp = buf; - for (i = 0; i < HASHSIZE; i++) - SLIST_FOREACH(p, &pipehash[i], next) { - /* - * copy pipe descriptor into *bp, convert delay back to ms, - * then copy the flow_set descriptor(s) one at a time. - * After each flow_set, copy the queue descriptor it owns. - */ - if ( is64user ){ - bp = cp_pipe_to_64_user(p, (struct dn_pipe_64 *)bp); + bp = buf; + for (i = 0; i < HASHSIZE; i++) { + SLIST_FOREACH(p, &pipehash[i], next) { + /* + * copy pipe descriptor into *bp, convert delay + * back to ms, then copy the flow_set descriptor(s) + * one at a time. After each flow_set, copy the + * queue descriptor it owns. + */ + if ( is64user ) { + bp = cp_pipe_to_64_user(p, + (struct dn_pipe_64 *)bp); + } else { + bp = cp_pipe_to_32_user(p, + (struct dn_pipe_32 *)bp); + } } - else{ - bp = cp_pipe_to_32_user(p, (struct dn_pipe_32 *)bp); + } + for (i = 0; i < HASHSIZE; i++) { + SLIST_FOREACH(set, &flowsethash[i], next) { + struct dn_flow_set_64 *fs_bp = + (struct dn_flow_set_64 *)bp ; + cp_flow_set_to_64_user(set, fs_bp); + /* XXX same hack as above */ + fs_bp->next = CAST_DOWN(user64_addr_t, + DN_IS_QUEUE); + fs_bp->pipe = USER_ADDR_NULL; + fs_bp->rq = USER_ADDR_NULL ; + bp += sizeof(struct dn_flow_set_64); + bp = dn_copy_set_64( set, bp ); } - } - for (i = 0; i < HASHSIZE; i++) - SLIST_FOREACH(set, &flowsethash[i], next) { - struct dn_flow_set_64 *fs_bp = (struct dn_flow_set_64 *)bp ; - cp_flow_set_to_64_user(set, fs_bp); - /* XXX same hack as above */ - fs_bp->next = CAST_DOWN(user64_addr_t, DN_IS_QUEUE); - fs_bp->pipe = USER_ADDR_NULL; - fs_bp->rq = USER_ADDR_NULL ; - bp += sizeof(struct dn_flow_set_64); - bp = dn_copy_set_64( set, bp ); - } - lck_mtx_unlock(dn_mutex); - - error = sooptcopyout(sopt, buf, size); - FREE(buf, M_TEMP); - return error ; + } + lck_mtx_unlock(dn_mutex); + error = sooptcopyout(sopt, buf, size); + FREE(buf, M_TEMP); + return(error); } /* @@ -2564,6 +2526,12 @@ ip_dn_ctl(struct sockopt *sopt) return error ; } +void +dummynet_init(void) +{ + eventhandler_lists_ctxt_init(&dummynet_evhdlr_ctxt); +} + void ip_dn_init(void) { @@ -2594,8 +2562,41 @@ ip_dn_init(void) default_rule.cmd[0].len = 1; default_rule.cmd[0].opcode = #ifdef IPFIREWALL_DEFAULT_TO_ACCEPT - (1) ? O_ACCEPT : + (1) ? O_ACCEPT : #endif - O_DENY; + O_DENY; #endif } + +struct dn_event_nwk_wq_entry +{ + struct nwk_wq_entry nwk_wqe; + struct dummynet_event dn_ev_arg; +}; + +static void +dummynet_event_callback(void *arg) +{ + struct dummynet_event *p_dn_ev = (struct dummynet_event *)arg; + + EVENTHANDLER_INVOKE(&dummynet_evhdlr_ctxt, dummynet_event, p_dn_ev); + return; +} + +void +dummynet_event_enqueue_nwk_wq_entry(struct dummynet_event *p_dn_event) +{ + struct dn_event_nwk_wq_entry *p_dn_ev = NULL; + + MALLOC(p_dn_ev, struct dn_event_nwk_wq_entry *, + sizeof(struct dn_event_nwk_wq_entry), + M_NWKWQ, M_WAITOK | M_ZERO); + + p_dn_ev->nwk_wqe.func = dummynet_event_callback; + p_dn_ev->nwk_wqe.is_arg_managed = TRUE; + p_dn_ev->nwk_wqe.arg = &p_dn_ev->dn_ev_arg; + + bcopy(p_dn_event, &(p_dn_ev->dn_ev_arg), + sizeof(struct dummynet_event)); + nwk_wq_enqueue((struct nwk_wq_entry*)p_dn_ev); +} diff --git a/bsd/netinet/ip_dummynet.h b/bsd/netinet/ip_dummynet.h index dde647d0b..7472e958c 100644 --- a/bsd/netinet/ip_dummynet.h +++ b/bsd/netinet/ip_dummynet.h @@ -60,7 +60,6 @@ #include <sys/appleapiopts.h> #ifdef PRIVATE - #include <netinet/ip_flowid.h> /* Apply ipv6 mask on ipv6 addr */ @@ -156,8 +155,11 @@ struct dn_heap { * processing requirements. */ #ifdef KERNEL +#include <net/if_var.h> +#include <net/route.h> #include <netinet/ip_var.h> /* for ip_out_args */ #include <netinet/ip6.h> /* for ip6_out_args */ +#include <netinet/in.h> #include <netinet6/ip6_var.h> /* for ip6_out_args */ struct dn_pkt_tag { @@ -418,7 +420,7 @@ struct dn_pipe { /* a pipe */ SLIST_HEAD(dn_pipe_head, dn_pipe); #ifdef BSD_KERNEL_PRIVATE - +extern uint32_t my_random(void); void ip_dn_init(void); /* called from raw_ip.c:load_ipfw() */ typedef int ip_dn_ctl_t(struct sockopt *); /* raw_ip.c */ @@ -678,8 +680,6 @@ struct dn_pipe_64 { /* a pipe */ struct dn_flow_set_64 fs ; /* used with fixed-rate flows */ }; - - /* * Return the IPFW rule associated with the dummynet tag; if any. * Make sure that the dummynet tag is not reused by lower layers. @@ -695,6 +695,64 @@ ip_dn_claim_rule(struct mbuf *m) } else return (NULL); } + +#include <sys/eventhandler.h> +/* Dummynet event handling declarations */ +extern struct eventhandler_lists_ctxt dummynet_evhdlr_ctxt; +extern void dummynet_init(void); + +struct dn_pipe_mini_config { + uint32_t bandwidth; + uint32_t delay; + uint32_t plr; +}; + +struct dn_rule_mini_config { + uint32_t dir; + uint32_t af; + uint32_t proto; + /* + * XXX PF rules actually define ranges of ports and + * along with range goes an opcode ((not) equal to, less than + * greater than, etc. + * For now the following works assuming there's no port range + * and the rule is for specific port. + * Also the operation is assumed as equal to. + */ + uint32_t src_port; + uint32_t dst_port; + char ifname[IFXNAMSIZ]; +}; + +struct dummynet_event { + uint32_t dn_event_code; + union { + struct dn_pipe_mini_config _dnev_pipe_config; + struct dn_rule_mini_config _dnev_rule_config; + } dn_event; +}; + +#define dn_event_pipe_config dn_event._dnev_pipe_config +#define dn_event_rule_config dn_event._dnev_rule_config + +extern void dummynet_event_enqueue_nwk_wq_entry(struct dummynet_event *); + +enum { + DUMMYNET_RULE_CONFIG, + DUMMYNET_RULE_DELETE, + DUMMYNET_PIPE_CONFIG, + DUMMYNET_PIPE_DELETE, + DUMMYNET_NLC_DISABLED, +}; + +enum { DN_INOUT, DN_IN, DN_OUT }; +/* + * The signature for the callback is: + * eventhandler_entry_arg __unused + * dummynet_event pointer to dummynet event object + */ +typedef void (*dummynet_event_fn) (struct eventhandler_entry_arg, struct dummynet_event *); +EVENTHANDLER_DECLARE(dummynet_event, dummynet_event_fn); #endif /* BSD_KERNEL_PRIVATE */ #endif /* PRIVATE */ #endif /* _IP_DUMMYNET_H */ diff --git a/bsd/netinet/ip_fw2.c b/bsd/netinet/ip_fw2.c index 63543fd6c..6694975b7 100644 --- a/bsd/netinet/ip_fw2.c +++ b/bsd/netinet/ip_fw2.c @@ -65,7 +65,6 @@ #endif /* INET */ #if IPFW2 -#include <machine/spl.h> #include <sys/param.h> #include <sys/systm.h> diff --git a/bsd/netinet/ip_icmp.c b/bsd/netinet/ip_icmp.c index 1e2877525..3d7ecafe3 100644 --- a/bsd/netinet/ip_icmp.c +++ b/bsd/netinet/ip_icmp.c @@ -154,7 +154,11 @@ const static int icmp_datalen = 8; /* Default values in case CONFIG_ICMP_BANDLIM is not defined in the MASTER file */ #ifndef CONFIG_ICMP_BANDLIM +#if !CONFIG_EMBEDDED #define CONFIG_ICMP_BANDLIM 250 +#else /* CONFIG_EMBEDDED */ +#define CONFIG_ICMP_BANDLIM 50 +#endif /* CONFIG_EMBEDDED */ #endif /* CONFIG_ICMP_BANDLIM */ /* @@ -379,7 +383,7 @@ icmp_input(struct mbuf *m, int hlen) int icmplen; int i; struct in_ifaddr *ia; - void (*ctlfunc)(int, struct sockaddr *, void *); + void (*ctlfunc)(int, struct sockaddr *, void *, struct ifnet *); int code; /* Expect 32-bit aligned data pointer on strict-align platforms */ @@ -536,7 +540,7 @@ icmp_input(struct mbuf *m, int hlen) ctlfunc = ip_protox[icp->icmp_ip.ip_p]->pr_ctlinput; if (ctlfunc) (*ctlfunc)(code, (struct sockaddr *)&icmpsrc, - (void *)&icp->icmp_ip); + (void *)&icp->icmp_ip, m->m_pkthdr.rcvif); break; badcode: diff --git a/bsd/netinet/ip_id.c b/bsd/netinet/ip_id.c index 4feff3e9c..94c791ea0 100644 --- a/bsd/netinet/ip_id.c +++ b/bsd/netinet/ip_id.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002-2013 Apple Inc. All rights reserved. + * Copyright (c) 2002-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -195,14 +195,14 @@ ip_randomid(void) if (random_id_statistics && new_id != 0) random_id_collisions++; read_random(&new_id, sizeof (new_id)); - } while (bit_test(id_bits, new_id) || new_id == 0); + } while (bitstr_test(id_bits, new_id) || new_id == 0); /* * These require serialization to maintain correctness. */ lck_mtx_lock_spin(&ipid_lock); - bit_clear(id_bits, id_array[array_ptr]); - bit_set(id_bits, new_id); + bitstr_clear(id_bits, id_array[array_ptr]); + bitstr_set(id_bits, new_id); id_array[array_ptr] = new_id; if (++array_ptr == ARRAY_SIZE) array_ptr = 0; diff --git a/bsd/netinet/ip_input.c b/bsd/netinet/ip_input.c index 3b32f0dde..4e4c27040 100644 --- a/bsd/netinet/ip_input.c +++ b/bsd/netinet/ip_input.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -240,6 +240,11 @@ SYSCTL_UINT(_net_inet_ip, OID_AUTO, adj_clear_hwcksum, CTLFLAG_RW | CTLFLAG_LOCKED, &ip_adj_clear_hwcksum, 0, "Invalidate hwcksum info when adjusting length"); +static uint32_t ip_adj_partial_sum = 1; +SYSCTL_UINT(_net_inet_ip, OID_AUTO, adj_partial_sum, + CTLFLAG_RW | CTLFLAG_LOCKED, &ip_adj_partial_sum, 0, + "Perform partial sum adjustment of trailing bytes at IP layer"); + /* * XXX - Setting ip_checkinterface mostly implements the receive side of * the Strong ES model described in RFC 1122, but since the routing table @@ -844,6 +849,76 @@ ip_input_setdst_chain(struct mbuf *m, uint32_t ifindex, struct in_ifaddr *ia) } } +static void +ip_input_adjust(struct mbuf *m, struct ip *ip, struct ifnet *inifp) +{ + boolean_t adjust = TRUE; + + ASSERT(m_pktlen(m) > ip->ip_len); + + /* + * Invalidate hardware checksum info if ip_adj_clear_hwcksum + * is set; useful to handle buggy drivers. Note that this + * should not be enabled by default, as we may get here due + * to link-layer padding. + */ + if (ip_adj_clear_hwcksum && + (m->m_pkthdr.csum_flags & CSUM_DATA_VALID) && + !(inifp->if_flags & IFF_LOOPBACK) && + !(m->m_pkthdr.pkt_flags & PKTF_LOOP)) { + m->m_pkthdr.csum_flags &= ~CSUM_DATA_VALID; + m->m_pkthdr.csum_data = 0; + ipstat.ips_adj_hwcsum_clr++; + } + + /* + * If partial checksum information is available, subtract + * out the partial sum of postpended extraneous bytes, and + * update the checksum metadata accordingly. By doing it + * here, the upper layer transport only needs to adjust any + * prepended extraneous bytes (else it will do both.) + */ + if (ip_adj_partial_sum && + (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID|CSUM_PARTIAL)) == + (CSUM_DATA_VALID|CSUM_PARTIAL)) { + m->m_pkthdr.csum_rx_val = m_adj_sum16(m, + m->m_pkthdr.csum_rx_start, m->m_pkthdr.csum_rx_start, + (ip->ip_len - m->m_pkthdr.csum_rx_start), + m->m_pkthdr.csum_rx_val); + } else if ((m->m_pkthdr.csum_flags & + (CSUM_DATA_VALID|CSUM_PARTIAL)) == + (CSUM_DATA_VALID|CSUM_PARTIAL)) { + /* + * If packet has partial checksum info and we decided not + * to subtract the partial sum of postpended extraneous + * bytes here (not the default case), leave that work to + * be handled by the other layers. For now, only TCP, UDP + * layers are capable of dealing with this. For all other + * protocols (including fragments), trim and ditch the + * partial sum as those layers might not implement partial + * checksumming (or adjustment) at all. + */ + if ((ip->ip_off & (IP_MF | IP_OFFMASK)) == 0 && + (ip->ip_p == IPPROTO_TCP || ip->ip_p == IPPROTO_UDP)) { + adjust = FALSE; + } else { + m->m_pkthdr.csum_flags &= ~CSUM_DATA_VALID; + m->m_pkthdr.csum_data = 0; + ipstat.ips_adj_hwcsum_clr++; + } + } + + if (adjust) { + ipstat.ips_adj++; + if (m->m_len == m->m_pkthdr.len) { + m->m_len = ip->ip_len; + m->m_pkthdr.len = ip->ip_len; + } else { + m_adj(m, ip->ip_len - m->m_pkthdr.len); + } + } +} + /* * First pass does all essential packet validation and places on a per flow * queue for doing operations that have same outcome for all packets of a flow. @@ -1123,27 +1198,7 @@ ipfw_tags_done: } if (m->m_pkthdr.len > ip->ip_len) { - /* - * Invalidate hardware checksum info if ip_adj_clear_hwcksum - * is set; useful to handle buggy drivers. Note that this - * should not be enabled by default, as we may get here due - * to link-layer padding. - */ - if (ip_adj_clear_hwcksum && - (m->m_pkthdr.csum_flags & CSUM_DATA_VALID) && - !(inifp->if_flags & IFF_LOOPBACK) && - !(m->m_pkthdr.pkt_flags & PKTF_LOOP)) { - m->m_pkthdr.csum_flags &= ~CSUM_DATA_VALID; - m->m_pkthdr.csum_data = 0; - ipstat.ips_adj_hwcsum_clr++; - } - - ipstat.ips_adj++; - if (m->m_len == m->m_pkthdr.len) { - m->m_len = ip->ip_len; - m->m_pkthdr.len = ip->ip_len; - } else - m_adj(m, ip->ip_len - m->m_pkthdr.len); + ip_input_adjust(m, ip, inifp); } /* for consistency */ @@ -1161,6 +1216,8 @@ check_with_pf: if (PF_IS_ENABLED) { int error; ip_input_cpout_args(args, &args1, &init); + ip = mtod(m, struct ip *); + src_ip = ip->ip_src; #if DUMMYNET error = pf_af_hook(inifp, NULL, &m, AF_INET, TRUE, &args1); @@ -2014,27 +2071,7 @@ tooshort: goto bad; } if (m->m_pkthdr.len > ip->ip_len) { - /* - * Invalidate hardware checksum info if ip_adj_clear_hwcksum - * is set; useful to handle buggy drivers. Note that this - * should not be enabled by default, as we may get here due - * to link-layer padding. - */ - if (ip_adj_clear_hwcksum && - (m->m_pkthdr.csum_flags & CSUM_DATA_VALID) && - !(inifp->if_flags & IFF_LOOPBACK) && - !(m->m_pkthdr.pkt_flags & PKTF_LOOP)) { - m->m_pkthdr.csum_flags &= ~CSUM_DATA_VALID; - m->m_pkthdr.csum_data = 0; - ipstat.ips_adj_hwcsum_clr++; - } - - ipstat.ips_adj++; - if (m->m_len == m->m_pkthdr.len) { - m->m_len = ip->ip_len; - m->m_pkthdr.len = ip->ip_len; - } else - m_adj(m, ip->ip_len - m->m_pkthdr.len); + ip_input_adjust(m, ip, inifp); } /* for consistency */ @@ -2446,7 +2483,7 @@ bad: static void ipq_updateparams(void) { - lck_mtx_assert(&ipqlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ipqlock, LCK_MTX_ASSERT_OWNED); /* * -1 for unlimited allocation. */ @@ -2519,7 +2556,7 @@ done: * When IPDIVERT enabled, keep additional state with each packet that * tells us if we need to divert or tee the packet we're building. * - * The IP header is *NOT* adjusted out of iplen. + * The IP header is *NOT* adjusted out of iplen (but in host byte order). */ static struct mbuf * #if IPDIVERT @@ -2617,35 +2654,49 @@ found: * * Perform 1's complement adjustment of octets that got included/ * excluded in the hardware-calculated checksum value. Ignore cases - * where the value includes or excludes the IP header span, as the - * sum for those octets would already be 0xffff and thus no-op. + * where the value includes the entire IPv4 header span, as the sum + * for those octets would already be 0 by the time we get here; IP + * has already performed its header checksum validation. Also take + * care of any trailing bytes and subtract out their partial sum. */ if (ip->ip_p == IPPROTO_UDP && hlen == sizeof (struct ip) && (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PARTIAL | CSUM_PSEUDO_HDR)) == (CSUM_DATA_VALID | CSUM_PARTIAL)) { - uint32_t start; + uint32_t start = m->m_pkthdr.csum_rx_start; + int32_t trailer = (m_pktlen(m) - ip->ip_len); + uint32_t swbytes = (uint32_t)trailer; - start = m->m_pkthdr.csum_rx_start; csum = m->m_pkthdr.csum_rx_val; - if (start != 0 && start != hlen) { + ASSERT(trailer >= 0); + if ((start != 0 && start != hlen) || trailer != 0) { #if BYTE_ORDER != BIG_ENDIAN if (start < hlen) { HTONS(ip->ip_len); HTONS(ip->ip_off); } -#endif +#endif /* BYTE_ORDER != BIG_ENDIAN */ /* callee folds in sum */ - csum = m_adj_sum16(m, start, hlen, csum); + csum = m_adj_sum16(m, start, hlen, + (ip->ip_len - hlen), csum); + if (hlen > start) + swbytes += (hlen - start); + else + swbytes += (start - hlen); #if BYTE_ORDER != BIG_ENDIAN if (start < hlen) { NTOHS(ip->ip_off); NTOHS(ip->ip_len); } -#endif +#endif /* BYTE_ORDER != BIG_ENDIAN */ } csum_flags = m->m_pkthdr.csum_flags; + + if (swbytes != 0) + udp_in_cksum_stats(swbytes); + if (trailer != 0) + m_adj(m, -trailer); } else { csum = 0; csum_flags = 0; @@ -3019,7 +3070,7 @@ dropfrag: static void frag_freef(struct ipqhead *fhp, struct ipq *fp) { - lck_mtx_assert(&ipqlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ipqlock, LCK_MTX_ASSERT_OWNED); fp->ipq_nfrags = 0; if (fp->ipq_frags != NULL) { @@ -3085,7 +3136,7 @@ frag_timeout(void *arg) static void frag_sched_timeout(void) { - lck_mtx_assert(&ipqlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ipqlock, LCK_MTX_ASSERT_OWNED); if (!frag_timeout_run && nipq > 0) { frag_timeout_run = 1; @@ -3632,16 +3683,11 @@ ip_srcroute(void) } /* - * Strip out IP options, at higher - * level protocol in the kernel. - * Second argument is buffer to which options - * will be moved, and return value is their length. - * XXX should be deleted; last arg currently ignored. + * Strip out IP options, at higher level protocol in the kernel. */ void -ip_stripoptions(struct mbuf *m, struct mbuf *mopt) +ip_stripoptions(struct mbuf *m) { -#pragma unused(mopt) int i; struct ip *ip = mtod(m, struct ip *); caddr_t opts; @@ -3650,6 +3696,7 @@ ip_stripoptions(struct mbuf *m, struct mbuf *mopt) /* Expect 32-bit aligned data pointer on strict-align platforms */ MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(m); + /* use bcopy() since it supports overlapping range */ olen = (IP_VHL_HL(ip->ip_vhl) << 2) - sizeof (struct ip); opts = (caddr_t)(ip + 1); i = m->m_len - (sizeof (struct ip) + olen); @@ -3658,6 +3705,27 @@ ip_stripoptions(struct mbuf *m, struct mbuf *mopt) if (m->m_flags & M_PKTHDR) m->m_pkthdr.len -= olen; ip->ip_vhl = IP_MAKE_VHL(IPVERSION, sizeof (struct ip) >> 2); + + /* + * We expect ip_{off,len} to be in host order by now, and + * that the original IP header length has been subtracted + * out from ip_len. Temporarily adjust ip_len for checksum + * recalculation, and restore it afterwards. + */ + ip->ip_len += sizeof (struct ip); + + /* recompute checksum now that IP header is smaller */ +#if BYTE_ORDER != BIG_ENDIAN + HTONS(ip->ip_len); + HTONS(ip->ip_off); +#endif /* BYTE_ORDER != BIG_ENDIAN */ + ip->ip_sum = in_cksum_hdr(ip); +#if BYTE_ORDER != BIG_ENDIAN + NTOHS(ip->ip_off); + NTOHS(ip->ip_len); +#endif /* BYTE_ORDER != BIG_ENDIAN */ + + ip->ip_len -= sizeof (struct ip); } u_char inetctlerrmap[PRC_NCMDS] = { diff --git a/bsd/netinet/ip_output.c b/bsd/netinet/ip_output.c index 9069bb8ba..16aff3c65 100644 --- a/bsd/netinet/ip_output.c +++ b/bsd/netinet/ip_output.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -207,6 +207,10 @@ SYSCTL_PROC(_net_inet_ip, OID_AUTO, output_perf_data, 0, 0, sysctl_ip_output_getperf, "S,net_perf", "IP output performance data (struct net_perf, net/net_perf.h)"); +__private_extern__ int rfc6864 = 1; +SYSCTL_INT(_net_inet_ip, OID_AUTO, rfc6864, CTLFLAG_RW | CTLFLAG_LOCKED, + &rfc6864, 0, "updated ip id field behavior"); + #define IMO_TRACE_HIST_SIZE 32 /* size of trace history */ /* For gdb */ @@ -586,7 +590,12 @@ loopit: if (!(flags & (IP_FORWARDING|IP_RAWOUTPUT))) { ip->ip_vhl = IP_MAKE_VHL(IPVERSION, hlen >> 2); ip->ip_off &= IP_DF; - ip->ip_id = ip_randomid(); + if (rfc6864 && IP_OFF_IS_ATOMIC(ip->ip_off)) { + // Per RFC6864, value of ip_id is undefined for atomic ip packets + ip->ip_id = 0; + } else { + ip->ip_id = ip_randomid(); + } OSAddAtomic(1, &ipstat.ips_localout); } else { hlen = IP_VHL_HL(ip->ip_vhl) << 2; @@ -901,7 +910,7 @@ loopit: if (IN_MULTICAST(ntohl(pkt_dst.s_addr))) { struct ifnet *srcifp = NULL; struct in_multi *inm; - u_int32_t vif; + u_int32_t vif = 0; u_int8_t ttl = IP_DEFAULT_MULTICAST_TTL; u_int8_t loop = IP_DEFAULT_MULTICAST_LOOP; @@ -1209,6 +1218,7 @@ sendit: /* Check if the interface is allowed */ if (!necp_packet_is_allowed_over_interface(m, ifp)) { error = EHOSTUNREACH; + OSAddAtomic(1, &ipstat.ips_necp_policy_drop); goto bad; } goto skip_ipsec; @@ -1216,6 +1226,7 @@ sendit: case NECP_KERNEL_POLICY_RESULT_SOCKET_DIVERT: /* Flow divert packets should be blocked at the IP layer */ error = EHOSTUNREACH; + OSAddAtomic(1, &ipstat.ips_necp_policy_drop); goto bad; case NECP_KERNEL_POLICY_RESULT_IP_TUNNEL: { /* Verify that the packet is being routed to the tunnel */ @@ -1224,6 +1235,7 @@ sendit: /* Check if the interface is allowed */ if (!necp_packet_is_allowed_over_interface(m, ifp)) { error = EHOSTUNREACH; + OSAddAtomic(1, &ipstat.ips_necp_policy_drop); goto bad; } goto skip_ipsec; @@ -1232,6 +1244,7 @@ sendit: /* Check if the interface is allowed */ if (!necp_packet_is_allowed_over_interface(m, policy_ifp)) { error = EHOSTUNREACH; + OSAddAtomic(1, &ipstat.ips_necp_policy_drop); goto bad; } @@ -1241,6 +1254,7 @@ sendit: goto skip_ipsec; } else { error = ENETUNREACH; + OSAddAtomic(1, &ipstat.ips_necp_policy_drop); goto bad; } } @@ -1252,6 +1266,7 @@ sendit: /* Catch-all to check if the interface is allowed */ if (!necp_packet_is_allowed_over_interface(m, ifp)) { error = EHOSTUNREACH; + OSAddAtomic(1, &ipstat.ips_necp_policy_drop); goto bad; } #endif /* NECP */ @@ -2256,7 +2271,8 @@ in_finalize_cksum(struct mbuf *m, uint32_t hoff, uint32_t csum_flags) ip_out_cksum_stats(ip->ip_p, len); /* RFC1122 4.1.3.4 */ - if (csum == 0 && (m->m_pkthdr.csum_flags & CSUM_UDP)) + if (csum == 0 && + (m->m_pkthdr.csum_flags & (CSUM_UDP|CSUM_ZERO_INVERT))) csum = 0xffff; /* Insert the checksum in the ULP csum field */ @@ -2268,8 +2284,8 @@ in_finalize_cksum(struct mbuf *m, uint32_t hoff, uint32_t csum_flags) } else { bcopy(&csum, (mtod(m, char *) + offset), sizeof (csum)); } - m->m_pkthdr.csum_flags &= - ~(CSUM_DELAY_DATA | CSUM_DATA_VALID | CSUM_PARTIAL); + m->m_pkthdr.csum_flags &= ~(CSUM_DELAY_DATA | CSUM_DATA_VALID | + CSUM_PARTIAL | CSUM_ZERO_INVERT); } if (sw_csum & CSUM_DELAY_IP) { @@ -2449,8 +2465,10 @@ ip_ctloutput(struct socket *so, struct sockopt *sopt) m->m_len = sopt->sopt_valsize; error = sooptcopyin(sopt, mtod(m, char *), m->m_len, m->m_len); - if (error) + if (error) { + m_freem(m); break; + } return (ip_pcbopts(sopt->sopt_name, &inp->inp_options, m)); @@ -3089,7 +3107,7 @@ ip_mloopback(struct ifnet *srcifp, struct ifnet *origifp, struct mbuf *m, * interface itself is lo0, this will be overridden by if_loop. */ if (hwcksum_rx) { - copym->m_pkthdr.csum_flags &= ~CSUM_PARTIAL; + copym->m_pkthdr.csum_flags &= ~(CSUM_PARTIAL|CSUM_ZERO_INVERT); copym->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; copym->m_pkthdr.csum_data = 0xffff; @@ -3443,10 +3461,13 @@ ip_output_checksum(struct ifnet *ifp, struct mbuf *m, int hlen, int ip_len, /* * Partial checksum offload, if non-IP fragment, and TCP only * (no UDP support, as the hardware may not be able to convert - * +0 to -0 (0xffff) per RFC1122 4.1.3.4.) + * +0 to -0 (0xffff) per RFC1122 4.1.3.4. unless the interface + * supports "invert zero" capability.) */ if (hwcksum_tx && !tso && - (m->m_pkthdr.csum_flags & CSUM_TCP) && + ((m->m_pkthdr.csum_flags & CSUM_TCP) || + ((hwcap & CSUM_ZERO_INVERT) && + (m->m_pkthdr.csum_flags & CSUM_ZERO_INVERT))) && ip_len <= ifp->if_mtu) { uint16_t start = sizeof (struct ip); uint16_t ulpoff = m->m_pkthdr.csum_data & 0xffff; diff --git a/bsd/netinet/ip_var.h b/bsd/netinet/ip_var.h index cc54d85d8..87985cc1c 100644 --- a/bsd/netinet/ip_var.h +++ b/bsd/netinet/ip_var.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -150,10 +150,10 @@ struct ip_moptions { }; #define IMO_LOCK_ASSERT_HELD(_imo) \ - lck_mtx_assert(&(_imo)->imo_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_imo)->imo_lock, LCK_MTX_ASSERT_OWNED) #define IMO_LOCK_ASSERT_NOTHELD(_imo) \ - lck_mtx_assert(&(_imo)->imo_lock, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_imo)->imo_lock, LCK_MTX_ASSERT_NOTOWNED) #define IMO_LOCK(_imo) \ lck_mtx_lock(&(_imo)->imo_lock) @@ -228,7 +228,7 @@ struct ipstat { u_int32_t ips_rxc_chainsz_gt4; /* rx chain size greater than 4 */ u_int32_t ips_rxc_notlist; /* count of pkts through ip_input */ u_int32_t ips_raw_sappend_fail; /* sock append failed */ - + u_int32_t ips_necp_policy_drop; /* NECP policy related drop */ }; struct ip_linklocal_stat { @@ -254,6 +254,7 @@ struct ip_moptions; #define IP_OUTARGS 0x100 /* has ancillary output info */ #define IP_HDR_ALIGNED_P(_ip) ((((uintptr_t)(_ip)) & ((uintptr_t)3)) == 0) +#define IP_OFF_IS_ATOMIC(_ip_off) ((_ip_off & (IP_DF | IP_MF | IP_OFFMASK)) == IP_DF) /* * On platforms which require strict alignment (currently for anything but @@ -307,6 +308,7 @@ extern int ip_use_randomid; extern u_short ip_id; /* ip packet ctr, for ids */ extern int ip_defttl; /* default IP ttl */ extern int ipforwarding; /* ip forwarding */ +extern int rfc6864; extern struct protosw *ip_protox[]; extern struct pr_usrreqs rip_usrreqs; @@ -334,7 +336,7 @@ extern struct in_ifaddr *ip_rtaddr(struct in_addr); extern int ip_savecontrol(struct inpcb *, struct mbuf **, struct ip *, struct mbuf *); extern struct mbuf *ip_srcroute(void); -extern void ip_stripoptions(struct mbuf *, struct mbuf *); +extern void ip_stripoptions(struct mbuf *); extern void ip_initid(void); extern u_int16_t ip_randomid(void); extern void ip_proto_dispatch_in_wrapper(struct mbuf *, int, u_int8_t); @@ -346,7 +348,7 @@ extern int ip_getsrcifaddr_info(struct mbuf *, uint32_t *, uint32_t *); extern int ip_getdstifaddr_info(struct mbuf *, uint32_t *, uint32_t *); extern int rip_ctloutput(struct socket *, struct sockopt *); -extern void rip_ctlinput(int, struct sockaddr *, void *); +extern void rip_ctlinput(int, struct sockaddr *, void *, struct ifnet *); extern void rip_init(struct protosw *, struct domain *); extern void rip_input(struct mbuf *, int); extern int rip_output(struct mbuf *, struct socket *, u_int32_t, struct mbuf *); diff --git a/bsd/netinet/kpi_ipfilter.c b/bsd/netinet/kpi_ipfilter.c index a63d4a583..2186c63ba 100644 --- a/bsd/netinet/kpi_ipfilter.c +++ b/bsd/netinet/kpi_ipfilter.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2004-2016 Apple Inc. All rights reserved. + * Copyright (c) 2004-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ @@ -36,10 +36,11 @@ #include <machine/endian.h> -#define _IP_VHL +#define _IP_VHL #include <net/if_var.h> #include <net/route.h> #include <net/kpi_protocol.h> +#include <net/net_api_stats.h> #include <netinet/in_systm.h> #include <netinet/in.h> @@ -51,12 +52,13 @@ #include <netinet6/ip6_var.h> #include <netinet/kpi_ipfilter_var.h> +#include <stdbool.h> /* * kipf_lock and kipf_ref protect the linkage of the list of IP filters * An IP filter can be removed only when kipf_ref is zero - * If an IP filter cannot be removed because kipf_ref is not null, then - * the IP filter is marjed and kipf_delayed_remove is set so that when + * If an IP filter cannot be removed because kipf_ref is not null, then + * the IP filter is marjed and kipf_delayed_remove is set so that when * kipf_ref eventually goes down to zero, the IP filter is removed */ decl_lck_mtx_data(static, kipf_lock_data); @@ -69,11 +71,21 @@ __private_extern__ struct ipfilter_list ipv4_filters = TAILQ_HEAD_INITIALIZER(ip __private_extern__ struct ipfilter_list ipv6_filters = TAILQ_HEAD_INITIALIZER(ipv6_filters); __private_extern__ struct ipfilter_list tbr_filters = TAILQ_HEAD_INITIALIZER(tbr_filters); +#undef ipf_addv4 +#undef ipf_addv6 +extern errno_t ipf_addv4(const struct ipf_filter *filter, + ipfilter_t *filter_ref); +extern errno_t ipf_addv6(const struct ipf_filter *filter, + ipfilter_t *filter_ref); + +static errno_t ipf_add(const struct ipf_filter *filter, + ipfilter_t *filter_ref, struct ipfilter_list *head, bool is_internal); + __private_extern__ void ipf_ref(void) { lck_mtx_lock(kipf_lock); - kipf_ref++; + kipf_ref++; lck_mtx_unlock(kipf_lock); } @@ -82,17 +94,19 @@ ipf_unref(void) { lck_mtx_lock(kipf_lock); - if (kipf_ref == 0) - panic("ipf_unref: kipf_ref == 0\n"); - - kipf_ref--; - if (kipf_ref == 0 && kipf_delayed_remove != 0) { - struct ipfilter *filter; + if (kipf_ref == 0) + panic("ipf_unref: kipf_ref == 0\n"); + + kipf_ref--; + if (kipf_ref == 0 && kipf_delayed_remove != 0) { + struct ipfilter *filter; while ((filter = TAILQ_FIRST(&tbr_filters))) { + VERIFY(OSDecrementAtomic64(&net_api_stats.nas_ipf_add_count) > 0); + ipf_detach_func ipf_detach = filter->ipf_filter.ipf_detach; void* cookie = filter->ipf_filter.cookie; - + TAILQ_REMOVE(filter->ipf_head, filter, ipf_link); TAILQ_REMOVE(&tbr_filters, filter, ipf_tbr); kipf_delayed_remove--; @@ -104,57 +118,80 @@ ipf_unref(void) /* In case some filter got to run while we released the lock */ if (kipf_ref != 0) break; - } + } } - } + } lck_mtx_unlock(kipf_lock); } static errno_t ipf_add( - const struct ipf_filter* filter, + const struct ipf_filter *filter, ipfilter_t *filter_ref, - struct ipfilter_list *head) + struct ipfilter_list *head, + bool is_internal) { struct ipfilter *new_filter; if (filter->name == NULL || (filter->ipf_input == NULL && filter->ipf_output == NULL)) - return EINVAL; - - MALLOC(new_filter, struct ipfilter*, sizeof(*new_filter), M_IFADDR, M_WAITOK); + return (EINVAL); + + MALLOC(new_filter, struct ipfilter *, sizeof(*new_filter), M_IFADDR, M_WAITOK); if (new_filter == NULL) - return ENOMEM; - + return (ENOMEM); + lck_mtx_lock(kipf_lock); new_filter->ipf_filter = *filter; new_filter->ipf_head = head; - + TAILQ_INSERT_HEAD(head, new_filter, ipf_link); - + + OSIncrementAtomic64(&net_api_stats.nas_ipf_add_count); + INC_ATOMIC_INT64_LIM(net_api_stats.nas_ipf_add_total); + if (is_internal) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_ipf_add_os_total); + } + lck_mtx_unlock(kipf_lock); - + *filter_ref = (ipfilter_t)new_filter; /* This will force TCP to re-evaluate its use of TSO */ OSAddAtomic(1, &kipf_count); routegenid_update(); - return 0; + return (0); +} + +errno_t +ipf_addv4_internal( + const struct ipf_filter *filter, + ipfilter_t *filter_ref) +{ + return (ipf_add(filter, filter_ref, &ipv4_filters, true)); } errno_t ipf_addv4( - const struct ipf_filter* filter, + const struct ipf_filter *filter, + ipfilter_t *filter_ref) +{ + return (ipf_add(filter, filter_ref, &ipv4_filters, false)); +} + +errno_t +ipf_addv6_internal( + const struct ipf_filter *filter, ipfilter_t *filter_ref) { - return ipf_add(filter, filter_ref, &ipv4_filters); + return (ipf_add(filter, filter_ref, &ipv6_filters, true)); } errno_t ipf_addv6( - const struct ipf_filter* filter, + const struct ipf_filter *filter, ipfilter_t *filter_ref) { - return ipf_add(filter, filter_ref, &ipv6_filters); + return (ipf_add(filter, filter_ref, &ipv6_filters, false)); } static errno_t @@ -185,20 +222,20 @@ errno_t ipf_remove( ipfilter_t filter_ref) { - struct ipfilter *match = (struct ipfilter*)filter_ref; + struct ipfilter *match = (struct ipfilter *)filter_ref; struct ipfilter_list *head; - + if (match == 0 || (match->ipf_head != &ipv4_filters && match->ipf_head != &ipv6_filters)) - return EINVAL; - + return (EINVAL); + head = match->ipf_head; - + lck_mtx_lock(kipf_lock); TAILQ_FOREACH(match, head, ipf_link) { - if (match == (struct ipfilter*)filter_ref) { + if (match == (struct ipfilter *)filter_ref) { ipf_detach_func ipf_detach = match->ipf_filter.ipf_detach; void* cookie = match->ipf_filter.cookie; - + /* * Cannot detach when they are filters running */ @@ -209,8 +246,11 @@ ipf_remove( match->ipf_filter.ipf_output = ipf_output_detached; lck_mtx_unlock(kipf_lock); } else { + VERIFY(OSDecrementAtomic64(&net_api_stats.nas_ipf_add_count) > 0); + TAILQ_REMOVE(head, match, ipf_link); lck_mtx_unlock(kipf_lock); + if (ipf_detach) ipf_detach(cookie); FREE(match, M_IFADDR); @@ -220,12 +260,12 @@ ipf_remove( routegenid_update(); } - return 0; + return (0); } } lck_mtx_unlock(kipf_lock); - - return ENOENT; + + return (ENOENT); } int log_for_en1 = 0; @@ -235,7 +275,7 @@ ipf_inject_input( mbuf_t data, ipfilter_t filter_ref) { - struct mbuf *m = (struct mbuf*)data; + struct mbuf *m = (struct mbuf *)data; struct m_tag *mtag = 0; struct ip *ip = mtod(m, struct ip *); u_int8_t vers; @@ -244,7 +284,7 @@ ipf_inject_input( protocol_family_t proto; vers = IP_VHL_V(ip->ip_vhl); - + switch (vers) { case 4: proto = PF_INET; @@ -256,7 +296,7 @@ ipf_inject_input( error = ENOTSUP; goto done; } - + if (filter_ref == 0 && m->m_pkthdr.rcvif == 0) { m->m_pkthdr.rcvif = lo_ifp; m->m_pkthdr.csum_data = 0; @@ -269,27 +309,27 @@ ipf_inject_input( } if (filter_ref != 0) { mtag = m_tag_create(KERNEL_MODULE_TAG_ID, KERNEL_TAG_TYPE_IPFILT, - sizeof (ipfilter_t), M_NOWAIT, m); + sizeof (ipfilter_t), M_NOWAIT, m); if (mtag == NULL) { error = ENOMEM; goto done; - } - *(ipfilter_t*)(mtag+1) = filter_ref; + } + *(ipfilter_t *)(mtag+1) = filter_ref; m_tag_prepend(m, mtag); } - + error = proto_inject(proto, data); done: - return error; + return (error); } static errno_t ipf_injectv4_out(mbuf_t data, ipfilter_t filter_ref, ipf_pktopts_t options) { struct route ro; - struct ip *ip; - struct mbuf *m = (struct mbuf*)data; + struct ip *ip; + struct mbuf *m = (struct mbuf *)data; errno_t error = 0; struct m_tag *mtag = NULL; struct ip_moptions *imo = NULL; @@ -365,8 +405,8 @@ static errno_t ipf_injectv6_out(mbuf_t data, ipfilter_t filter_ref, ipf_pktopts_t options) { struct route_in6 ro; - struct ip6_hdr *ip6; - struct mbuf *m = (struct mbuf*)data; + struct ip6_hdr *ip6; + struct mbuf *m = (struct mbuf *)data; errno_t error = 0; struct m_tag *mtag = NULL; struct ip6_moptions *im6o = NULL; @@ -379,7 +419,7 @@ ipf_injectv6_out(mbuf_t data, ipfilter_t filter_ref, ipf_pktopts_t options) if (m == NULL) return (ENOMEM); } - ip6 = (struct ip6_hdr*)m_mtod(m); + ip6 = (struct ip6_hdr *)m_mtod(m); if (filter_ref != 0) { mtag = m_tag_create(KERNEL_MODULE_TAG_ID, @@ -419,7 +459,7 @@ ipf_injectv6_out(mbuf_t data, ipfilter_t filter_ref, ipf_pktopts_t options) /* * Send mbuf and ifscope information. Check for correctness - * of ifscope information is done while searching for a route in + * of ifscope information is done while searching for a route in * ip6_output. */ error = ip6_output(m, NULL, &ro, IPV6_OUTARGS, im6o, NULL, &ip6oa); @@ -440,20 +480,19 @@ ipf_inject_output( ipfilter_t filter_ref, ipf_pktopts_t options) { - struct mbuf *m = (struct mbuf*)data; + struct mbuf *m = (struct mbuf *)data; u_int8_t vers; errno_t error = 0; /* Make one byte of the header contiguous in the mbuf */ if (m->m_len < 1) { m = m_pullup(m, 1); - if (m == NULL) + if (m == NULL) goto done; } - - vers = (*(u_int8_t*)m_mtod(m)) >> 4; - switch (vers) - { + + vers = (*(u_int8_t *)m_mtod(m)) >> 4; + switch (vers) { case 4: error = ipf_injectv4_out(data, filter_ref, options); break; @@ -468,8 +507,8 @@ ipf_inject_output( break; } -done: - return error; +done: + return (error); } __private_extern__ ipfilter_t @@ -477,14 +516,14 @@ ipf_get_inject_filter(struct mbuf *m) { ipfilter_t filter_ref = 0; struct m_tag *mtag; - + mtag = m_tag_locate(m, KERNEL_MODULE_TAG_ID, KERNEL_TAG_TYPE_IPFILT, NULL); if (mtag) { filter_ref = *(ipfilter_t *)(mtag+1); - + m_tag_delete(m, mtag); } - return filter_ref; + return (filter_ref); } __private_extern__ int @@ -494,28 +533,28 @@ ipf_init(void) lck_grp_attr_t *grp_attributes = 0; lck_attr_t *lck_attributes = 0; lck_grp_t *lck_grp = 0; - + grp_attributes = lck_grp_attr_alloc_init(); if (grp_attributes == 0) { printf("ipf_init: lck_grp_attr_alloc_init failed\n"); error = ENOMEM; goto done; } - + lck_grp = lck_grp_alloc_init("IP Filter", grp_attributes); if (lck_grp == 0) { printf("ipf_init: lck_grp_alloc_init failed\n"); error = ENOMEM; goto done; } - + lck_attributes = lck_attr_alloc_init(); if (lck_attributes == 0) { printf("ipf_init: lck_attr_alloc_init failed\n"); error = ENOMEM; goto done; } - + lck_mtx_init(kipf_lock, lck_grp, lck_attributes); done: @@ -531,6 +570,6 @@ ipf_init(void) lck_attr_free(lck_attributes); lck_attributes = 0; } - - return error; + + return (error); } diff --git a/bsd/netinet/kpi_ipfilter.h b/bsd/netinet/kpi_ipfilter.h index 210f99f97..392d0650b 100644 --- a/bsd/netinet/kpi_ipfilter.h +++ b/bsd/netinet/kpi_ipfilter.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2014 Apple Inc. All rights reserved. + * Copyright (c) 2008-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -156,8 +156,16 @@ typedef struct opaque_ipfilter *ipfilter_t; @param filter_ref A reference to the filter used to detach it. @result 0 on success otherwise the errno error. */ +#ifdef KERNEL_PRIVATE +extern errno_t ipf_addv4_internal(const struct ipf_filter *filter, + ipfilter_t *filter_ref); + +#define ipf_addv4(filter, filter_ref) \ + ipf_addv4_internal((filter), (filter_ref)) +#else extern errno_t ipf_addv4(const struct ipf_filter *filter, ipfilter_t *filter_ref); +#endif /* KERNEL_PRIVATE */ /*! @function ipf_addv6 @@ -166,8 +174,16 @@ extern errno_t ipf_addv4(const struct ipf_filter *filter, @param filter_ref A reference to the filter used to detach it. @result 0 on success otherwise the errno error. */ +#ifdef KERNEL_PRIVATE +extern errno_t ipf_addv6_internal(const struct ipf_filter *filter, + ipfilter_t *filter_ref); + +#define ipf_addv6(filter, filter_ref) \ + ipf_addv6_internal((filter), (filter_ref)) +#else extern errno_t ipf_addv6(const struct ipf_filter *filter, ipfilter_t *filter_ref); +#endif /* KERNEL_PRIVATE */ /*! @function ipf_remove diff --git a/bsd/netinet/mp_pcb.c b/bsd/netinet/mp_pcb.c index 31ea83fd0..288f29c71 100644 --- a/bsd/netinet/mp_pcb.c +++ b/bsd/netinet/mp_pcb.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -43,6 +43,7 @@ #include <netinet/mp_pcb.h> #include <netinet/mptcp_var.h> +#include <netinet6/in6_pcb.h> static lck_grp_t *mp_lock_grp; static lck_attr_t *mp_lock_attr; @@ -131,7 +132,7 @@ mp_timeout(void *arg) static void mp_sched_timeout(void) { - lck_mtx_assert(&mp_timeout_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&mp_timeout_lock, LCK_MTX_ASSERT_OWNED); if (!mp_timeout_run && (mp_garbage_collecting || mp_ticking)) { lck_mtx_convert_spin(&mp_timeout_lock); @@ -199,27 +200,9 @@ int mp_pcballoc(struct socket *so, struct mppcbinfo *mppi) { struct mppcb *mpp = NULL; + int error; - VERIFY(sotomppcb(so) == NULL); - - lck_mtx_lock(&mppi->mppi_lock); - if (mppi->mppi_count >= mptcp_socket_limit) { - lck_mtx_unlock(&mppi->mppi_lock); - mptcplog((LOG_ERR, "MPTCP Socket: Reached MPTCP socket limit."), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); - /* - * This limit may be reached either because of - * a leak or a transient condition where - * MPTCP connections are not released fast - * enough. - * We return EAFNOSUPPORT here to have user - * space library fallback to TCP. - * XXX We need to revist this when we get rid - * of the current low limit imposed on MPTCP. - */ - return (EAFNOSUPPORT); - } - lck_mtx_unlock(&mppi->mppi_lock); + VERIFY(mpsotomppcb(so) == NULL); mpp = zalloc(mppi->mppi_zone); if (mpp == NULL) { @@ -233,10 +216,11 @@ mp_pcballoc(struct socket *so, struct mppcbinfo *mppi) mpp->mpp_socket = so; so->so_pcb = mpp; - if (NULL == mppi->mppi_pcbe_create(so, mpp)) { + error = mptcp_sescreate(mpp); + if (error) { lck_mtx_destroy(&mpp->mpp_lock, mppi->mppi_lock_grp); zfree(mppi->mppi_zone, mpp); - return (ENOBUFS); + return (error); } lck_mtx_lock(&mppi->mppi_lock); @@ -249,15 +233,13 @@ mp_pcballoc(struct socket *so, struct mppcbinfo *mppi) } void -mp_pcbdetach(struct mppcb *mpp) +mp_pcbdetach(struct socket *mp_so) { - struct socket *so = mpp->mpp_socket; - - VERIFY(so->so_pcb == mpp); + struct mppcb *mpp = mpsotomppcb(mp_so); mpp->mpp_state = MPPCB_STATE_DEAD; - if (!(so->so_flags & SOF_PCBCLEARING)) - so->so_flags |= SOF_PCBCLEARING; + if (!(mp_so->so_flags & SOF_PCBCLEARING)) + mp_so->so_flags |= SOF_PCBCLEARING; mp_gc_sched(); } @@ -269,23 +251,105 @@ mp_pcbdispose(struct mppcb *mpp) VERIFY(mppi != NULL); - lck_mtx_assert(&mppi->mppi_lock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(&mpp->mpp_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&mppi->mppi_lock, LCK_MTX_ASSERT_OWNED); + mpp_lock_assert_held(mpp); VERIFY(mpp->mpp_state == MPPCB_STATE_DEAD); - VERIFY(mpp->mpp_flags & MPP_ATTACHED); + mpp->mpp_flags &= ~MPP_ATTACHED; TAILQ_REMOVE(&mppi->mppi_pcbs, mpp, mpp_entry); VERIFY(mppi->mppi_count != 0); mppi->mppi_count--; + mpp_unlock(mpp); + +#if NECP + necp_mppcb_dispose(mpp); +#endif /* NECP */ + + lck_mtx_destroy(&mpp->mpp_lock, mppi->mppi_lock_grp); + VERIFY(mpp->mpp_socket != NULL); VERIFY(mpp->mpp_socket->so_usecount == 0); mpp->mpp_socket->so_pcb = NULL; mpp->mpp_socket = NULL; - lck_mtx_unlock(&mpp->mpp_lock); - lck_mtx_destroy(&mpp->mpp_lock, mppi->mppi_lock_grp); zfree(mppi->mppi_zone, mpp); } + +static int +mp_getaddr_v4(struct socket *mp_so, struct sockaddr **nam, boolean_t peer) +{ + struct mptses *mpte = mpsotompte(mp_so); + struct sockaddr_in *sin; + + /* + * Do the malloc first in case it blocks. + */ + MALLOC(sin, struct sockaddr_in *, sizeof (*sin), M_SONAME, M_WAITOK); + if (sin == NULL) + return (ENOBUFS); + bzero(sin, sizeof (*sin)); + sin->sin_family = AF_INET; + sin->sin_len = sizeof (*sin); + + if (!peer) { + sin->sin_port = mpte->__mpte_src_v4.sin_port; + sin->sin_addr = mpte->__mpte_src_v4.sin_addr; + } else { + sin->sin_port = mpte->__mpte_dst_v4.sin_port; + sin->sin_addr = mpte->__mpte_dst_v4.sin_addr; + } + + *nam = (struct sockaddr *)sin; + return (0); +} + +static int +mp_getaddr_v6(struct socket *mp_so, struct sockaddr **nam, boolean_t peer) +{ + struct mptses *mpte = mpsotompte(mp_so); + struct in6_addr addr; + in_port_t port; + + if (!peer) { + port = mpte->__mpte_src_v6.sin6_port; + addr = mpte->__mpte_src_v6.sin6_addr; + } else { + port = mpte->__mpte_dst_v6.sin6_port; + addr = mpte->__mpte_dst_v6.sin6_addr; + } + + *nam = in6_sockaddr(port, &addr); + if (*nam == NULL) + return (ENOBUFS); + + return (0); +} + +int +mp_getsockaddr(struct socket *mp_so, struct sockaddr **nam) +{ + struct mptses *mpte = mpsotompte(mp_so); + + if (mpte->mpte_src.sa_family == AF_INET || mpte->mpte_src.sa_family == 0) + return mp_getaddr_v4(mp_so, nam, false); + else if (mpte->mpte_src.sa_family == AF_INET) + return mp_getaddr_v6(mp_so, nam, false); + else + return (EINVAL); +} + +int +mp_getpeeraddr(struct socket *mp_so, struct sockaddr **nam) +{ + struct mptses *mpte = mpsotompte(mp_so); + + if (mpte->mpte_src.sa_family == AF_INET || mpte->mpte_src.sa_family == 0) + return mp_getaddr_v4(mp_so, nam, true); + else if (mpte->mpte_src.sa_family == AF_INET) + return mp_getaddr_v6(mp_so, nam, true); + else + return (EINVAL); +} diff --git a/bsd/netinet/mp_pcb.h b/bsd/netinet/mp_pcb.h index eba202b85..f8fb188c0 100644 --- a/bsd/netinet/mp_pcb.h +++ b/bsd/netinet/mp_pcb.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2013 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -30,6 +30,9 @@ #define _NETINET_MP_PCB_H_ #ifdef BSD_KERNEL_PRIVATE +#include <sys/domain.h> +#include <sys/protosw.h> +#include <sys/socketvar.h> #include <sys/types.h> #include <sys/queue.h> #include <kern/locks.h> @@ -40,6 +43,10 @@ typedef enum mppcb_state { MPPCB_STATE_DEAD = 2, } mppcb_state_t; + +/* net/necp.h already includes mp_pcb.h - so we have to forward-declare */ +struct necp_client_flow; + /* * Multipath Protocol Control Block */ @@ -47,17 +54,42 @@ struct mppcb { TAILQ_ENTRY(mppcb) mpp_entry; /* glue to all PCBs */ decl_lck_mtx_data(, mpp_lock); /* per PCB lock */ struct mppcbinfo *mpp_pcbinfo; /* PCB info */ - void *mpp_pcbe; /* ptr to per-protocol ext */ + struct mptses *mpp_pcbe; /* ptr to MPTCP-session */ struct socket *mpp_socket; /* back pointer to socket */ uint32_t mpp_flags; /* PCB flags */ mppcb_state_t mpp_state; /* PCB state */ + +#if NECP + uuid_t necp_client_uuid; + void (*necp_cb)(void *, int, struct necp_client_flow *); +#endif }; -#define sotomppcb(so) ((struct mppcb *)((so)->so_pcb)) +static inline struct mppcb * +mpsotomppcb(struct socket *mp_so) +{ + VERIFY(SOCK_DOM(mp_so) == PF_MULTIPATH); + return ((struct mppcb *)mp_so->so_pcb); +} /* valid values for mpp_flags */ -#define MPP_ATTACHED 0x1 -#define MPP_DEFUNCT 0x2 +#define MPP_ATTACHED 0x001 +#define MPP_INSIDE_OUTPUT 0x002 /* MPTCP-stack is inside mptcp_subflow_output */ +#define MPP_INSIDE_INPUT 0x004 /* MPTCP-stack is inside mptcp_subflow_input */ +#define MPP_RUPCALL 0x008 /* MPTCP-stack is handling a read upcall */ +#define MPP_WUPCALL 0x010 /* MPTCP-stack is handling a read upcall */ +#define MPP_SHOULD_WORKLOOP 0x020 /* MPTCP-stack should call the workloop function */ +#define MPP_SHOULD_RWAKEUP 0x040 /* MPTCP-stack should call sorwakeup */ +#define MPP_SHOULD_WWAKEUP 0x080 /* MPTCP-stack should call sowwakeup */ +#define MPP_CREATE_SUBFLOWS 0x100 /* This connection needs to create subflows */ +#define MPP_SET_CELLICON 0x200 /* Set the cellicon (deferred) */ +#define MPP_UNSET_CELLICON 0x400 /* Unset the cellicon (deferred) */ + +static inline boolean_t +mptcp_should_defer_upcall(struct mppcb *mpp) +{ + return !!(mpp->mpp_flags & (MPP_INSIDE_OUTPUT | MPP_INSIDE_INPUT | MPP_RUPCALL | MPP_WUPCALL)); +} /* * Multipath PCB Information @@ -74,8 +106,6 @@ struct mppcbinfo { decl_lck_mtx_data(, mppi_lock); /* global PCB lock */ uint32_t (*mppi_gc)(struct mppcbinfo *); /* garbage collector func */ uint32_t (*mppi_timer)(struct mppcbinfo *); /* timer func */ - /* Extended pcb create func */ - void *(*mppi_pcbe_create) (struct socket *mp_so, struct mppcb *mpp); }; __BEGIN_DECLS @@ -83,10 +113,13 @@ extern void mp_pcbinit(void); extern void mp_pcbinfo_attach(struct mppcbinfo *); extern int mp_pcbinfo_detach(struct mppcbinfo *); extern int mp_pcballoc(struct socket *, struct mppcbinfo *); -extern void mp_pcbdetach(struct mppcb *); +extern void mp_pcbdetach(struct socket *); extern void mp_pcbdispose(struct mppcb *); extern void mp_gc_sched(void); extern void mptcp_timer_sched(void); +extern void mptcp_handle_deferred_upcalls(struct mppcb *mpp, uint32_t flag); +extern int mp_getsockaddr(struct socket *mp_so, struct sockaddr **nam); +extern int mp_getpeeraddr(struct socket *mp_so, struct sockaddr **nam); __END_DECLS #endif /* BSD_KERNEL_PRIVATE */ diff --git a/bsd/netinet/mp_proto.c b/bsd/netinet/mp_proto.c index 65eafb0be..1093fcc9b 100644 --- a/bsd/netinet/mp_proto.c +++ b/bsd/netinet/mp_proto.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -36,19 +36,13 @@ #include <kern/locks.h> #include <netinet/in.h> -#if MPTCP #include <netinet/mptcp_var.h> -#endif /* MPTCP */ extern struct domain mpdomain_s; -static struct domain *mpdomain = NULL; static void mp_dinit(struct domain *); -lck_mtx_t *mp_domain_mutex; -static struct protosw mpsw[] = { -#if MPTCP -{ +static struct protosw mpsw = { .pr_type = SOCK_STREAM, .pr_protocol = IPPROTO_TCP, .pr_flags = PR_CONNREQUIRED|PR_MULTICONN|PR_EVCONNINFO| @@ -60,12 +54,8 @@ static struct protosw mpsw[] = { .pr_lock = mptcp_lock, .pr_unlock = mptcp_unlock, .pr_getlock = mptcp_getlock, -}, -#endif /* MPTCP */ }; -static int mp_proto_count = (sizeof (mpsw) / sizeof (struct protosw)); - struct domain mpdomain_s = { .dom_family = PF_MULTIPATH, .dom_flags = DOM_REENTRANT, @@ -77,16 +67,7 @@ struct domain mpdomain_s = { void mp_dinit(struct domain *dp) { - struct protosw *pr; - int i; - VERIFY(!(dp->dom_flags & DOM_INITIALIZED)); - VERIFY(mpdomain == NULL); - - mpdomain = dp; - - for (i = 0, pr = &mpsw[0]; i < mp_proto_count; i++, pr++) - net_add_proto(pr, dp, 1); - mp_domain_mutex = dp->dom_mtx; + net_add_proto(&mpsw, dp, 1); } diff --git a/bsd/netinet/mptcp.c b/bsd/netinet/mptcp.c index 264c9d7c1..5d901a9da 100644 --- a/bsd/netinet/mptcp.c +++ b/bsd/netinet/mptcp.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2015 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -26,6 +26,59 @@ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ +/* + * A note on the MPTCP/NECP-interactions: + * + * MPTCP uses NECP-callbacks to get notified of interface/policy events. + * MPTCP registers to these events at the MPTCP-layer for interface-events + * through a call to necp_client_register_multipath_cb. + * To get per-flow events (aka per TCP-subflow), we register to it with + * necp_client_register_socket_flow. Both registrations happen by using the + * necp-client-uuid that comes from the app. + * + * The locking is rather tricky. In general, we expect the lock-ordering to + * happen from necp-fd -> necp->client -> mpp_lock. + * + * There are however some subtleties. + * + * 1. When registering the multipath_cb, we are holding the mpp_lock. This is + * safe, because it is the very first time this MPTCP-connection goes into NECP. + * As we go into NECP we take the NECP-locks and thus are guaranteed that no + * NECP-locks will deadlock us. Because these NECP-events will also first take + * the NECP-locks. Either they win the race and thus won't find our + * MPTCP-connection. Or, MPTCP wins the race and thus it will safely install + * the callbacks while holding the NECP lock. + * + * 2. When registering the subflow-callbacks we must unlock the mpp_lock. This, + * because we have already registered callbacks and we might race against an + * NECP-event that will match on our socket. So, we have to unlock to be safe. + * + * 3. When removing the multipath_cb, we do it in mp_pcbdispose(). The + * so_usecount has reached 0. We must be careful to not remove the mpp_socket + * pointers before we unregistered the callback. Because, again we might be + * racing against an NECP-event. Unregistering must happen with an unlocked + * mpp_lock, because of the lock-ordering constraint. It could be that + * before we had a chance to unregister an NECP-event triggers. That's why + * we need to check for the so_usecount in mptcp_session_necp_cb. If we get + * there while the socket is being garbage-collected, the use-count will go + * down to 0 and we exit. Removal of the multipath_cb again happens by taking + * the NECP-locks so any running NECP-events will finish first and exit cleanly. + * + * 4. When removing the subflow-callback, we do it in in_pcbdispose(). Again, + * the socket-lock must be unlocked for lock-ordering constraints. This gets a + * bit tricky here, as in tcp_garbage_collect we hold the mp_so and so lock. + * So, we drop the mp_so-lock as soon as the subflow is unlinked with + * mptcp_subflow_del. Then, in in_pcbdispose we drop the subflow-lock. + * If an NECP-event was waiting on the lock in mptcp_subflow_necp_cb, when it + * gets it, it will realize that the subflow became non-MPTCP and retry (see + * tcp_lock). Then it waits again on the subflow-lock. When we drop this lock + * in in_pcbdispose, and enter necp_inpcb_dispose, this one will have to wait + * for the NECP-lock (held by the other thread that is taking care of the NECP- + * event). So, the event now finally gets the subflow-lock and then hits an + * so_usecount that is 0 and exits. Eventually, we can remove the subflow from + * the NECP callback. + */ + #include <sys/param.h> #include <sys/systm.h> #include <sys/kernel.h> @@ -39,11 +92,8 @@ #include <kern/zalloc.h> #include <kern/locks.h> -#include <mach/thread_act.h> #include <mach/sdt.h> -#include <dev/random/randomdev.h> - #include <net/if.h> #include <netinet/in.h> #include <netinet/in_var.h> @@ -93,47 +143,6 @@ int mptcp_subflow_keeptime = 60*14; SYSCTL_INT(_net_inet_mptcp, OID_AUTO, keepalive, CTLFLAG_RW | CTLFLAG_LOCKED, &mptcp_subflow_keeptime, 0, "Keepalive in seconds"); -/* - * MP_PRIO option. - */ -int mptcp_mpprio_enable = 1; -SYSCTL_INT(_net_inet_mptcp, OID_AUTO, mpprio, CTLFLAG_RW | CTLFLAG_LOCKED, - &mptcp_mpprio_enable, 0, "Enable MP_PRIO option"); - -/* - * REMOVE_ADDR option. - */ -int mptcp_remaddr_enable = 1; -SYSCTL_INT(_net_inet_mptcp, OID_AUTO, remaddr, CTLFLAG_RW | CTLFLAG_LOCKED, - &mptcp_remaddr_enable, 0, "Enable REMOVE_ADDR option"); - -/* - * FastJoin Option - */ -int mptcp_fastjoin = 1; -SYSCTL_INT(_net_inet_mptcp, OID_AUTO, fastjoin, CTLFLAG_RW | CTLFLAG_LOCKED, - &mptcp_fastjoin, 0, "Enable FastJoin Option"); - -int mptcp_zerortt_fastjoin = 0; -SYSCTL_INT(_net_inet_mptcp, OID_AUTO, zerortt_fastjoin, CTLFLAG_RW | - CTLFLAG_LOCKED, &mptcp_zerortt_fastjoin, 0, - "Enable Zero RTT Fast Join"); - -/* - * R/W Notification on resume - */ -int mptcp_rwnotify = 0; -SYSCTL_INT(_net_inet_mptcp, OID_AUTO, rwnotify, CTLFLAG_RW | CTLFLAG_LOCKED, - &mptcp_rwnotify, 0, "Enable RW notify on resume"); - -/* - * Using RTT history for sending new data - */ -int mptcp_use_rtthist = 1; -SYSCTL_INT(_net_inet_mptcp, OID_AUTO, rtthist, CTLFLAG_RW | CTLFLAG_LOCKED, - &mptcp_use_rtthist, 0, "Disable RTT History"); - -#define MPTCP_RTTHIST_MINTHRESH 500 int mptcp_rtthist_rtthresh = 600; SYSCTL_INT(_net_inet_mptcp, OID_AUTO, rtthist_thresh, CTLFLAG_RW | CTLFLAG_LOCKED, &mptcp_rtthist_rtthresh, 0, "Rtt threshold"); @@ -145,32 +154,17 @@ int mptcp_use_rto = 1; SYSCTL_INT(_net_inet_mptcp, OID_AUTO, userto, CTLFLAG_RW | CTLFLAG_LOCKED, &mptcp_use_rto, 0, "Disable RTO for subflow selection"); -#define MPTCP_RTO_MINTHRESH 1000 int mptcp_rtothresh = 1500; SYSCTL_INT(_net_inet_mptcp, OID_AUTO, rto_thresh, CTLFLAG_RW | CTLFLAG_LOCKED, &mptcp_rtothresh, 0, "RTO threshold"); -/* - * Use server's chosen path for sending new data - */ -int mptcp_peerswitch = 1; -SYSCTL_INT(_net_inet_mptcp, OID_AUTO, use_peer, CTLFLAG_RW | CTLFLAG_LOCKED, - &mptcp_peerswitch, 0, "Use peer"); - -#define MPTCP_PEERSWITCH_CNTMIN 3 -uint32_t mptcp_peerswitch_cnt = 3; -SYSCTL_UINT(_net_inet_mptcp, OID_AUTO, peerswitchno, CTLFLAG_RW | CTLFLAG_LOCKED, - &mptcp_peerswitch_cnt, 0, "Set threshold based on peer's data arrival"); - /* * Probe the preferred path, when it is not in use */ -#define MPTCP_PROBETO_MIN 500 uint32_t mptcp_probeto = 1000; SYSCTL_UINT(_net_inet_mptcp, OID_AUTO, probeto, CTLFLAG_RW | CTLFLAG_LOCKED, &mptcp_probeto, 0, "Disable probing by setting to 0"); -#define MPTCP_PROBE_MX 15 uint32_t mptcp_probecnt = 5; SYSCTL_UINT(_net_inet_mptcp, OID_AUTO, probecnt, CTLFLAG_RW | CTLFLAG_LOCKED, &mptcp_probecnt, 0, "Number of probe writes"); @@ -178,8 +172,174 @@ SYSCTL_UINT(_net_inet_mptcp, OID_AUTO, probecnt, CTLFLAG_RW | CTLFLAG_LOCKED, /* * Static declarations */ -static int mptcp_validate_csum(struct tcpcb *, struct mbuf *, int); -static uint16_t mptcp_input_csum(struct tcpcb *, struct mbuf *, int); +static uint16_t mptcp_input_csum(struct tcpcb *, struct mbuf *, uint64_t, + uint32_t, uint16_t, uint16_t); + +static int +mptcp_reass_present(struct socket *mp_so) +{ + struct mptcb *mp_tp = mpsotomppcb(mp_so)->mpp_pcbe->mpte_mptcb; + struct tseg_qent *q; + int dowakeup = 0; + + /* + * Present data to user, advancing rcv_nxt through + * completed sequence space. + */ + if (mp_tp->mpt_state < MPTCPS_ESTABLISHED) + return (0); + q = LIST_FIRST(&mp_tp->mpt_segq); + if (!q || q->tqe_m->m_pkthdr.mp_dsn != mp_tp->mpt_rcvnxt) + return (0); + + /* + * If there is already another thread doing reassembly for this + * connection, it is better to let it finish the job -- + * (radar 16316196) + */ + if (mp_tp->mpt_flags & MPTCPF_REASS_INPROG) + return (0); + + mp_tp->mpt_flags |= MPTCPF_REASS_INPROG; + + do { + mp_tp->mpt_rcvnxt += q->tqe_len; + LIST_REMOVE(q, tqe_q); + if (mp_so->so_state & SS_CANTRCVMORE) { + m_freem(q->tqe_m); + } else { + if (sbappendstream(&mp_so->so_rcv, q->tqe_m)) + dowakeup = 1; + } + zfree(tcp_reass_zone, q); + mp_tp->mpt_reassqlen--; + q = LIST_FIRST(&mp_tp->mpt_segq); + } while (q && q->tqe_m->m_pkthdr.mp_dsn == mp_tp->mpt_rcvnxt); + mp_tp->mpt_flags &= ~MPTCPF_REASS_INPROG; + + if (dowakeup) + sorwakeup(mp_so); /* done with socket lock held */ + return (0); + +} + +static int +mptcp_reass(struct socket *mp_so, struct pkthdr *phdr, int *tlenp, struct mbuf *m) +{ + struct mptcb *mp_tp = mpsotomppcb(mp_so)->mpp_pcbe->mpte_mptcb; + u_int64_t mb_dsn = phdr->mp_dsn; + struct tseg_qent *q; + struct tseg_qent *p = NULL; + struct tseg_qent *nq; + struct tseg_qent *te = NULL; + u_int16_t qlimit; + + /* + * Limit the number of segments in the reassembly queue to prevent + * holding on to too many segments (and thus running out of mbufs). + * Make sure to let the missing segment through which caused this + * queue. Always keep one global queue entry spare to be able to + * process the missing segment. + */ + qlimit = min(max(100, mp_so->so_rcv.sb_hiwat >> 10), + (tcp_autorcvbuf_max >> 10)); + if (mb_dsn != mp_tp->mpt_rcvnxt && + (mp_tp->mpt_reassqlen + 1) >= qlimit) { + tcpstat.tcps_mptcp_rcvmemdrop++; + m_freem(m); + *tlenp = 0; + return (0); + } + + /* Allocate a new queue entry. If we can't, just drop the pkt. XXX */ + te = (struct tseg_qent *) zalloc(tcp_reass_zone); + if (te == NULL) { + tcpstat.tcps_mptcp_rcvmemdrop++; + m_freem(m); + return (0); + } + + mp_tp->mpt_reassqlen++; + + /* + * Find a segment which begins after this one does. + */ + LIST_FOREACH(q, &mp_tp->mpt_segq, tqe_q) { + if (MPTCP_SEQ_GT(q->tqe_m->m_pkthdr.mp_dsn, mb_dsn)) + break; + p = q; + } + + /* + * If there is a preceding segment, it may provide some of + * our data already. If so, drop the data from the incoming + * segment. If it provides all of our data, drop us. + */ + if (p != NULL) { + int64_t i; + /* conversion to int (in i) handles seq wraparound */ + i = p->tqe_m->m_pkthdr.mp_dsn + p->tqe_len - mb_dsn; + if (i > 0) { + if (i >= *tlenp) { + tcpstat.tcps_mptcp_rcvduppack++; + m_freem(m); + zfree(tcp_reass_zone, te); + te = NULL; + mp_tp->mpt_reassqlen--; + /* + * Try to present any queued data + * at the left window edge to the user. + * This is needed after the 3-WHS + * completes. + */ + goto out; + } + m_adj(m, i); + *tlenp -= i; + phdr->mp_dsn += i; + } + } + + tcpstat.tcps_mp_oodata++; + + /* + * While we overlap succeeding segments trim them or, + * if they are completely covered, dequeue them. + */ + while (q) { + int64_t i = (mb_dsn + *tlenp) - q->tqe_m->m_pkthdr.mp_dsn; + if (i <= 0) + break; + + if (i < q->tqe_len) { + q->tqe_m->m_pkthdr.mp_dsn += i; + q->tqe_len -= i; + m_adj(q->tqe_m, i); + break; + } + + nq = LIST_NEXT(q, tqe_q); + LIST_REMOVE(q, tqe_q); + m_freem(q->tqe_m); + zfree(tcp_reass_zone, q); + mp_tp->mpt_reassqlen--; + q = nq; + } + + /* Insert the new segment queue entry into place. */ + te->tqe_m = m; + te->tqe_th = NULL; + te->tqe_len = *tlenp; + + if (p == NULL) { + LIST_INSERT_HEAD(&mp_tp->mpt_segq, te, tqe_q); + } else { + LIST_INSERT_AFTER(p, te, tqe_q); + } + +out: + return (mptcp_reass_present(mp_so)); +} /* * MPTCP input, called when data has been read from a subflow socket. @@ -189,20 +349,21 @@ mptcp_input(struct mptses *mpte, struct mbuf *m) { struct socket *mp_so; struct mptcb *mp_tp = NULL; - u_int64_t mb_dsn; - u_int32_t mb_datalen; - int count = 0; + int count = 0, wakeup = 0; struct mbuf *save = NULL, *prev = NULL; struct mbuf *freelist = NULL, *tail = NULL; - boolean_t in_fallback = FALSE; VERIFY(m->m_flags & M_PKTHDR); - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - mp_so = mpte->mpte_mppcb->mpp_socket; + mpte_lock_assert_held(mpte); /* same as MP socket lock */ + + mp_so = mptetoso(mpte); + mp_tp = mpte->mpte_mptcb; DTRACE_MPTCP(input); + mp_tp->mpt_rcvwnd = mptcp_sbspace(mp_tp); + /* * Each mbuf contains MPTCP Data Sequence Map * Process the data for reassembly, delivery to MPTCP socket @@ -211,18 +372,13 @@ mptcp_input(struct mptses *mpte, struct mbuf *m) */ count = mp_so->so_rcv.sb_cc; - VERIFY(m != NULL); - mp_tp = mpte->mpte_mptcb; - VERIFY(mp_tp != NULL); - - /* Ok to check for this flag without lock as its set in this thread */ - in_fallback = (mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP); - /* * In the degraded fallback case, data is accepted without DSS map */ - if (in_fallback) { + if (mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) { fallback: + mptcp_sbrcv_grow(mp_tp); + /* * assume degraded flow as this may be the first packet * without DSS, and the subflow state is not updated yet. @@ -235,18 +391,19 @@ fallback: struct sockbuf *, &mp_so->so_snd, struct mptses *, mpte); count = mp_so->so_rcv.sb_cc - count; - mptcplog((LOG_DEBUG, "MPTCP Receiver: Fallback read %d bytes\n", + mptcplog((LOG_DEBUG, "%s: Fallback read %d bytes\n", __func__, count), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE); return; } - MPT_LOCK(mp_tp); do { + u_int64_t mb_dsn; + int32_t mb_datalen; + int64_t todrop; + /* If fallback occurs, mbufs will not have PKTF_MPTCP set */ - if (!(m->m_pkthdr.pkt_flags & PKTF_MPTCP)) { - MPT_UNLOCK(mp_tp); + if (!(m->m_pkthdr.pkt_flags & PKTF_MPTCP)) goto fallback; - } save = m->m_next; /* @@ -271,26 +428,40 @@ fallback: mb_dsn = m->m_pkthdr.mp_dsn; mb_datalen = m->m_pkthdr.mp_rlen; - if (MPTCP_SEQ_GT(mb_dsn, mp_tp->mpt_rcvatmark)) { - tcpstat.tcps_mp_oodata++; - MPT_UNLOCK(mp_tp); - m_freem(m); - return; - /* - * Reassembly queue support here in future. Per spec, - * senders must implement retransmission timer to - * retransmit unacked data. Dropping out of order - * gives a slight hit on performance but allows us to - * deploy MPTCP and protects us against in-window DoS - * attacks that attempt to use up memory by sending - * out of order data. When doing load sharing across - * subflows, out of order support is a must. - */ + todrop = (mb_dsn + mb_datalen) - (mp_tp->mpt_rcvnxt + mp_tp->mpt_rcvwnd); + if (todrop > 0) { + tcpstat.tcps_mptcp_rcvpackafterwin++; + + if (todrop >= mb_datalen) { + if (freelist == NULL) + freelist = m; + else + tail->m_next = m; + + if (prev != NULL) + tail = prev; + else + tail = m; + + m = save; + prev = save = NULL; + continue; + } else { + m_adj(m, -todrop); + mb_datalen -= todrop; + } } - if (MPTCP_SEQ_LT(mb_dsn, mp_tp->mpt_rcvatmark)) { + if (MPTCP_SEQ_GT(mb_dsn, mp_tp->mpt_rcvnxt) || + !LIST_EMPTY(&mp_tp->mpt_segq)) { + mptcp_reass(mp_so, &m->m_pkthdr, &mb_datalen, m); + + goto next; + } + + if (MPTCP_SEQ_LT(mb_dsn, mp_tp->mpt_rcvnxt)) { if (MPTCP_SEQ_LEQ((mb_dsn + mb_datalen), - mp_tp->mpt_rcvatmark)) { + mp_tp->mpt_rcvnxt)) { if (freelist == NULL) freelist = m; else @@ -305,44 +476,77 @@ fallback: prev = save = NULL; continue; } else { - m_adj(m, (mp_tp->mpt_rcvatmark - mb_dsn)); + m_adj(m, (mp_tp->mpt_rcvnxt - mb_dsn)); } - mptcplog((LOG_INFO, "MPTCP Receiver: Left Edge %llu\n", - mp_tp->mpt_rcvatmark), + mptcplog((LOG_INFO, "%s: Left Edge %llu\n", __func__, + mp_tp->mpt_rcvnxt), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE); } - MPT_UNLOCK(mp_tp); - if (sbappendstream(&mp_so->so_rcv, m)) { - sorwakeup(mp_so); - } + mptcp_sbrcv_grow(mp_tp); + + if (sbappendstream(&mp_so->so_rcv, m)) + wakeup = 1; + DTRACE_MPTCP6(receive, struct mbuf *, m, struct socket *, mp_so, struct sockbuf *, &mp_so->so_rcv, struct sockbuf *, &mp_so->so_snd, struct mptses *, mpte, struct mptcb *, mp_tp); - MPT_LOCK(mp_tp); count = mp_so->so_rcv.sb_cc - count; tcpstat.tcps_mp_rcvtotal++; tcpstat.tcps_mp_rcvbytes += count; - mptcplog((LOG_DEBUG, "MPTCP Receiver: Read %d bytes\n", count), + mptcplog((LOG_DEBUG, "%s: Read %d bytes\n", __func__, count), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE); - /* - * The data received at the MPTCP layer will never exceed the - * receive window because anything to the right of the - * receive window will be trimmed at the subflow level. - */ - mp_tp->mpt_rcvwnd = mptcp_sbspace(mp_tp); - mp_tp->mpt_rcvatmark += count; + mp_tp->mpt_rcvnxt += count; + +next: m = save; prev = save = NULL; count = mp_so->so_rcv.sb_cc; } while (m); - MPT_UNLOCK(mp_tp); if (freelist) m_freem(freelist); + + if (wakeup) + sorwakeup(mp_so); +} + +static boolean_t +mptcp_can_send_more(struct mptcb *mp_tp) +{ + struct socket *mp_so = mptetoso(mp_tp->mpt_mpte); + + /* + * Always send if there is data in the reinject-queue. + */ + if (mp_tp->mpt_mpte->mpte_reinjectq) + return (TRUE); + + /* + * Don't send, if: + * + * 1. snd_nxt >= snd_max : Means, basically everything has been sent. + * Except when using TFO, we might be doing a 0-byte write. + * 2. snd_una + snd_wnd <= snd_nxt: No space in the receiver's window + * 3. snd_nxt + 1 == snd_max and we are closing: A DATA_FIN is scheduled. + */ + + if (!(mp_so->so_flags1 & SOF1_PRECONNECT_DATA) && MPTCP_SEQ_GEQ(mp_tp->mpt_sndnxt, mp_tp->mpt_sndmax)) + return (FALSE); + + if (MPTCP_SEQ_LEQ(mp_tp->mpt_snduna + mp_tp->mpt_sndwnd, mp_tp->mpt_sndnxt)) + return (FALSE); + + if (mp_tp->mpt_sndnxt + 1 == mp_tp->mpt_sndmax && mp_tp->mpt_state > MPTCPS_CLOSE_WAIT) + return (FALSE); + + if (mp_tp->mpt_state >= MPTCPS_FIN_WAIT_2) + return (FALSE); + + return (TRUE); } /* @@ -351,295 +555,357 @@ fallback: int mptcp_output(struct mptses *mpte) { + struct mptcb *mp_tp; struct mptsub *mpts; struct mptsub *mpts_tried = NULL; struct socket *mp_so; struct mptsub *preferred_mpts = NULL; + uint64_t old_snd_nxt; int error = 0; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - mp_so = mpte->mpte_mppcb->mpp_socket; - if (mp_so->so_state & SS_CANTSENDMORE) { - mptcplog((LOG_DEBUG, "MPTCP Sender: cantsendmore\n"), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); - return (EPIPE); - } + mpte_lock_assert_held(mpte); + mp_so = mptetoso(mpte); + mp_tp = mpte->mpte_mptcb; -try_again: - /* get the "best" subflow to be used for transmission */ - mpts = mptcp_get_subflow(mpte, NULL, &preferred_mpts); - if (mpts == NULL) { - mptcplog((LOG_ERR, "MPTCP Sender: mp_so 0x%llx no subflow\n", - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so)), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); - goto out; - } + VERIFY(!(mpte->mpte_mppcb->mpp_flags & MPP_WUPCALL)); + mpte->mpte_mppcb->mpp_flags |= MPP_WUPCALL; + + mptcplog((LOG_DEBUG, "%s: snxt %u sndmax %u suna %u swnd %u reinjectq %u state %u\n", + __func__, (uint32_t)mp_tp->mpt_sndnxt, (uint32_t)mp_tp->mpt_sndmax, + (uint32_t)mp_tp->mpt_snduna, mp_tp->mpt_sndwnd, + mpte->mpte_reinjectq ? 1 : 0, + mp_tp->mpt_state), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); + + old_snd_nxt = mp_tp->mpt_sndnxt; + while (mptcp_can_send_more(mp_tp)) { + /* get the "best" subflow to be used for transmission */ + mpts = mptcp_get_subflow(mpte, NULL, &preferred_mpts); + if (mpts == NULL) { + mptcplog((LOG_INFO, "%s: no subflow\n", __func__), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); + break; + } - mptcplog((LOG_DEBUG, "MPTCP Sender: mp_so 0x%llx using cid %d \n", - (uint64_t)VM_KERNEL_ADDRPERM(mp_so), mpts->mpts_connid), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); - - /* In case there's just one flow, we reattempt later */ - MPTS_LOCK(mpts); - if ((mpts_tried != NULL) && ((mpts == mpts_tried) || - (mpts->mpts_flags & MPTSF_FAILINGOVER))) { - MPTS_UNLOCK(mpts); - MPTS_LOCK(mpts_tried); - mpts_tried->mpts_flags &= ~MPTSF_FAILINGOVER; - mpts_tried->mpts_flags |= MPTSF_ACTIVE; - MPTS_UNLOCK(mpts_tried); - mptcp_start_timer(mpte, MPTT_REXMT); - mptcplog((LOG_DEBUG, "MPTCP Sender: mp_so 0x%llx retry later\n", - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so)), + mptcplog((LOG_DEBUG, "%s: using id %u\n", __func__, mpts->mpts_connid), MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); - goto out; - } - DTRACE_MPTCP3(output, struct mptses *, mpte, struct mptsub *, mpts, - struct socket *, mp_so); - error = mptcp_subflow_output(mpte, mpts); - if (error && error != EWOULDBLOCK) { - /* can be a temporary loss of source address or other error */ - mpts->mpts_flags |= MPTSF_FAILINGOVER; - mpts->mpts_flags &= ~MPTSF_ACTIVE; - mpts_tried = mpts; - MPTS_UNLOCK(mpts); - mptcplog((LOG_INFO, "MPTCP Sender: %s Error = %d \n", - __func__, error), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); - goto try_again; - } - /* The model is to have only one active flow at a time */ - mpts->mpts_flags |= MPTSF_ACTIVE; - mpts->mpts_probesoon = mpts->mpts_probecnt = 0; - MPTS_UNLOCK(mpts); - - /* Allows us to update the smoothed rtt */ - if ((mptcp_probeto) && (mptcp_probeto >= MPTCP_PROBETO_MIN) && - (mpts != preferred_mpts) && (preferred_mpts != NULL)) { - MPTS_LOCK(preferred_mpts); - if (preferred_mpts->mpts_probesoon) { - if ((tcp_now - preferred_mpts->mpts_probesoon) > - mptcp_probeto) { - (void) mptcp_subflow_output(mpte, preferred_mpts); - if (preferred_mpts->mpts_probecnt >= - MIN(mptcp_probecnt, MPTCP_PROBE_MX)) { - preferred_mpts->mpts_probesoon = 0; - preferred_mpts->mpts_probecnt = 0; + /* In case there's just one flow, we reattempt later */ + if (mpts_tried != NULL && + (mpts == mpts_tried || (mpts->mpts_flags & MPTSF_FAILINGOVER))) { + mpts_tried->mpts_flags &= ~MPTSF_FAILINGOVER; + mpts_tried->mpts_flags |= MPTSF_ACTIVE; + mptcp_start_timer(mpte, MPTT_REXMT); + mptcplog((LOG_DEBUG, "%s: retry later\n", __func__), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); + break; + } + + /* + * Automatic sizing of send socket buffer. Increase the send + * socket buffer size if all of the following criteria are met + * 1. the receiver has enough buffer space for this data + * 2. send buffer is filled to 7/8th with data (so we actually + * have data to make use of it); + */ + if (tcp_do_autosendbuf == 1 && + (mp_so->so_snd.sb_flags & (SB_AUTOSIZE | SB_TRIM)) == SB_AUTOSIZE && + tcp_cansbgrow(&mp_so->so_snd)) { + if ((mp_tp->mpt_sndwnd / 4 * 5) >= mp_so->so_snd.sb_hiwat && + mp_so->so_snd.sb_cc >= (mp_so->so_snd.sb_hiwat / 8 * 7)) { + if (sbreserve(&mp_so->so_snd, + min(mp_so->so_snd.sb_hiwat + tcp_autosndbuf_inc, + tcp_autosndbuf_max)) == 1) { + mp_so->so_snd.sb_idealsize = mp_so->so_snd.sb_hiwat; + + mptcplog((LOG_DEBUG, "%s: increased snd hiwat to %u lowat %u\n", + __func__, mp_so->so_snd.sb_hiwat, + mp_so->so_snd.sb_lowat), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); } } - } else { - preferred_mpts->mpts_probesoon = tcp_now; - preferred_mpts->mpts_probecnt = 0; } - MPTS_UNLOCK(preferred_mpts); - } - if (mpte->mpte_active_sub == NULL) { - mpte->mpte_active_sub = mpts; - } else if (mpte->mpte_active_sub != mpts) { - mptcplog((LOG_DEBUG, "MPTCP Sender: switch [cid %d, srtt %d]" - "to [cid %d, srtt %d]\n", - mpte->mpte_active_sub->mpts_connid, - mpte->mpte_active_sub->mpts_srtt >> 5, - mpts->mpts_connid, - mpts->mpts_srtt >> 5), - MPTCP_SENDER_DBG | MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); - - MPTS_LOCK(mpte->mpte_active_sub); - mpte->mpte_active_sub->mpts_flags &= ~MPTSF_ACTIVE; - mpts->mpts_peerswitch = 0; - MPTS_UNLOCK(mpte->mpte_active_sub); - mpte->mpte_active_sub = mpts; - tcpstat.tcps_mp_switches++; + DTRACE_MPTCP3(output, struct mptses *, mpte, struct mptsub *, mpts, + struct socket *, mp_so); + error = mptcp_subflow_output(mpte, mpts, 0); + if (error) { + /* can be a temporary loss of source address or other error */ + mpts->mpts_flags |= MPTSF_FAILINGOVER; + mpts->mpts_flags &= ~MPTSF_ACTIVE; + mpts_tried = mpts; + mptcplog((LOG_ERR, "%s: Error = %d mpts_flags %#x\n", __func__, + error, mpts->mpts_flags), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR); + break; + } + /* The model is to have only one active flow at a time */ + mpts->mpts_flags |= MPTSF_ACTIVE; + mpts->mpts_probesoon = mpts->mpts_probecnt = 0; + + /* Allows us to update the smoothed rtt */ + if (mptcp_probeto && mpts != preferred_mpts && preferred_mpts != NULL) { + if (preferred_mpts->mpts_probesoon) { + if ((tcp_now - preferred_mpts->mpts_probesoon) > mptcp_probeto) { + mptcp_subflow_output(mpte, preferred_mpts, MPTCP_SUBOUT_PROBING); + if (preferred_mpts->mpts_probecnt >= mptcp_probecnt) { + preferred_mpts->mpts_probesoon = 0; + preferred_mpts->mpts_probecnt = 0; + } + } + } else { + preferred_mpts->mpts_probesoon = tcp_now; + preferred_mpts->mpts_probecnt = 0; + } + } + + if (mpte->mpte_active_sub == NULL) { + mpte->mpte_active_sub = mpts; + } else if (mpte->mpte_active_sub != mpts) { + struct tcpcb *tp = sototcpcb(mpts->mpts_socket); + struct tcpcb *acttp = sototcpcb(mpte->mpte_active_sub->mpts_socket); + + mptcplog((LOG_DEBUG, "%s: switch [%u, srtt %d] to [%u, srtt %d]\n", __func__, + mpte->mpte_active_sub->mpts_connid, acttp->t_srtt >> TCP_RTT_SHIFT, + mpts->mpts_connid, tp->t_srtt >> TCP_RTT_SHIFT), + (MPTCP_SENDER_DBG | MPTCP_SOCKET_DBG), MPTCP_LOGLVL_LOG); + + mpte->mpte_active_sub->mpts_flags &= ~MPTSF_ACTIVE; + mpte->mpte_active_sub = mpts; + + mptcpstats_inc_switch(mpte, mpts); + } } -out: + + mptcp_handle_deferred_upcalls(mpte->mpte_mppcb, MPP_WUPCALL); + /* subflow errors should not be percolated back up */ return (0); } + +static struct mptsub * +mptcp_choose_subflow(struct mptsub *mpts, struct mptsub *curbest, int *currtt) +{ + struct tcpcb *tp = sototcpcb(mpts->mpts_socket); + + /* + * Lower RTT? Take it, if it's our first one, or + * it doesn't has any loss, or the current one has + * loss as well. + */ + if (tp->t_srtt && *currtt > tp->t_srtt && + (curbest == NULL || tp->t_rxtshift == 0 || + sototcpcb(curbest->mpts_socket)->t_rxtshift)) { + *currtt = tp->t_srtt; + return (mpts); + } + + /* + * If we find a subflow without loss, take it always! + */ + if (curbest && + sototcpcb(curbest->mpts_socket)->t_rxtshift && + tp->t_rxtshift == 0) { + *currtt = tp->t_srtt; + return (mpts); + } + + return (curbest != NULL ? curbest : mpts); +} + +static struct mptsub * +mptcp_return_subflow(struct mptsub *mpts) +{ + if (mpts && mptcp_subflow_cwnd_space(mpts->mpts_socket) <= 0) + return (NULL); + + return (mpts); +} + /* * Return the most eligible subflow to be used for sending data. - * This function also serves to check if any alternate subflow is available - * or not. best and second_best flows are chosen by their priority. third_best - * could be best or second_best but is under loss at the time of evaluation. */ struct mptsub * mptcp_get_subflow(struct mptses *mpte, struct mptsub *ignore, struct mptsub **preferred) { + struct tcpcb *besttp, *secondtp; + struct inpcb *bestinp, *secondinp; struct mptsub *mpts; struct mptsub *best = NULL; struct mptsub *second_best = NULL; - struct mptsub *third_best = NULL; - struct mptsub *symptoms_best = NULL; - struct socket *so = NULL; + int exp_rtt = INT_MAX, cheap_rtt = INT_MAX; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + /* + * First Step: + * Choose the best subflow for cellular and non-cellular interfaces. + */ TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { - MPTS_LOCK(mpts); + struct socket *so = mpts->mpts_socket; + struct tcpcb *tp = sototcpcb(so); + struct inpcb *inp = sotoinpcb(so); + + mptcplog((LOG_DEBUG, "%s mpts %u ignore %d, mpts_flags %#x, suspended %u sostate %#x tpstate %u cellular %d rtt %u rxtshift %u cheap %u exp %u cwnd %d\n", + __func__, mpts->mpts_connid, ignore ? ignore->mpts_connid : -1, mpts->mpts_flags, + INP_WAIT_FOR_IF_FEEDBACK(inp), so->so_state, tp->t_state, + inp->inp_last_outifp ? IFNET_IS_CELLULAR(inp->inp_last_outifp) : -1, + tp->t_srtt, tp->t_rxtshift, cheap_rtt, exp_rtt, + mptcp_subflow_cwnd_space(so)), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); - if ((ignore) && (mpts == ignore)) { - MPTS_UNLOCK(mpts); + /* + * First, the hard conditions to reject subflows + * (e.g., not connected,...) + */ + if (mpts == ignore || inp->inp_last_outifp == NULL) + continue; + + if (INP_WAIT_FOR_IF_FEEDBACK(inp)) continue; - } /* There can only be one subflow in degraded state */ if (mpts->mpts_flags & MPTSF_MP_DEGRADED) { - MPTS_UNLOCK(mpts); best = mpts; break; } /* - * Subflows with TFO or Fastjoin allow data to be written before - * the subflow is mp capable. + * If this subflow is waiting to finally send, do it! */ - if (!(mpts->mpts_flags & MPTSF_MP_CAPABLE) && - !(mpts->mpts_flags & MPTSF_FASTJ_REQD) && - !(mpts->mpts_flags & MPTSF_TFO_REQD)) { - MPTS_UNLOCK(mpts); - continue; - } + if (so->so_flags1 & SOF1_PRECONNECT_DATA) + return (mptcp_return_subflow(mpts)); - if (mpts->mpts_flags & MPTSF_SUSPENDED) { - MPTS_UNLOCK(mpts); + /* + * Only send if the subflow is MP_CAPABLE. The exceptions to + * this rule (degraded or TFO) have been taken care of above. + */ + if (!(mpts->mpts_flags & MPTSF_MP_CAPABLE)) continue; - } - if ((mpts->mpts_flags & MPTSF_DISCONNECTED) || - (mpts->mpts_flags & MPTSF_DISCONNECTING)) { - MPTS_UNLOCK(mpts); + if ((so->so_state & SS_ISDISCONNECTED) || + !(so->so_state & SS_ISCONNECTED) || + !TCPS_HAVEESTABLISHED(tp->t_state) || + tp->t_state > TCPS_CLOSE_WAIT) continue; - } - - if (mpts->mpts_flags & MPTSF_FAILINGOVER) { - so = mpts->mpts_socket; - if ((so) && (!(so->so_flags & SOF_PCBCLEARING))) { - socket_lock(so, 1); - if ((so->so_snd.sb_cc == 0) && - (mptcp_no_rto_spike(so))) { - mpts->mpts_flags &= ~MPTSF_FAILINGOVER; - so->so_flags &= ~SOF_MP_TRYFAILOVER; - socket_unlock(so, 1); - } else { - third_best = mpts; - mptcplog((LOG_DEBUG, "MPTCP Sender: " - "%s cid %d in failover\n", - __func__, third_best->mpts_connid), - MPTCP_SENDER_DBG, - MPTCP_LOGLVL_VERBOSE); - socket_unlock(so, 1); - MPTS_UNLOCK(mpts); - continue; - } - } else { - MPTS_UNLOCK(mpts); - continue; - } - } - /* When there are no preferred flows, use first one in list */ - if ((!second_best) && !(mpts->mpts_flags & MPTSF_PREFERRED)) - second_best = mpts; - - if (mpts->mpts_flags & MPTSF_PREFERRED) { - best = mpts; - } - - MPTS_UNLOCK(mpts); + /* + * Second, the soft conditions to find the subflow with best + * conditions for each set (aka cellular vs non-cellular) + */ + if (IFNET_IS_CELLULAR(inp->inp_last_outifp)) + second_best = mptcp_choose_subflow(mpts, second_best, + &exp_rtt); + else + best = mptcp_choose_subflow(mpts, best, &cheap_rtt); } /* * If there is no preferred or backup subflow, and there is no active * subflow use the last usable subflow. */ - if (best == NULL) { - return (second_best ? second_best : third_best); - } + if (best == NULL) + return (mptcp_return_subflow(second_best)); - if (second_best == NULL) { - return (best ? best : third_best); - } + if (second_best == NULL) + return (mptcp_return_subflow(best)); + + besttp = sototcpcb(best->mpts_socket); + bestinp = sotoinpcb(best->mpts_socket); + secondtp = sototcpcb(second_best->mpts_socket); + secondinp = sotoinpcb(second_best->mpts_socket); if (preferred != NULL) - *preferred = best; - - /* Use a hint from symptomsd if it exists */ - symptoms_best = mptcp_use_symptoms_hints(best, second_best); - if (symptoms_best != NULL) - return (symptoms_best); - - /* Compare RTTs, select second_best if best's rtt exceeds rttthresh */ - if ((mptcp_use_rtthist) && - (best->mpts_srtt) && (second_best->mpts_srtt) && - (best->mpts_srtt > second_best->mpts_srtt) && - (best->mpts_srtt >= MAX((MPTCP_RTTHIST_MINTHRESH << 5), - (mptcp_rtthist_rtthresh << 5)))) { - tcpstat.tcps_mp_sel_rtt++; - mptcplog((LOG_DEBUG, "MPTCP Sender: %s best cid %d" - " at rtt %d, second cid %d at rtt %d\n", __func__, - best->mpts_connid, best->mpts_srtt >> 5, - second_best->mpts_connid, - second_best->mpts_srtt >> 5), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); - return (second_best); - } + *preferred = mptcp_return_subflow(best); - /* Compare RTOs, select second_best if best's rto exceeds rtothresh */ - if ((mptcp_use_rto) && - (best->mpts_rxtcur) && (second_best->mpts_rxtcur) && - (best->mpts_rxtcur > second_best->mpts_rxtcur) && - (best->mpts_rxtcur >= - MAX(MPTCP_RTO_MINTHRESH, mptcp_rtothresh))) { - tcpstat.tcps_mp_sel_rto++; - mptcplog((LOG_DEBUG, "MPTCP Sender: %s best cid %d" - " at rto %d, second cid %d at rto %d\n", __func__, - best->mpts_connid, best->mpts_rxtcur, - second_best->mpts_connid, second_best->mpts_rxtcur), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); - - return (second_best); - } + /* + * Second Step: Among best and second_best. Choose the one that is + * most appropriate for this particular service-type. + */ + if (mpte->mpte_svctype == MPTCP_SVCTYPE_HANDOVER) { + /* + * Only handover if Symptoms tells us to do so. + */ + if (IFNET_IS_WIFI(bestinp->inp_last_outifp) && + mptcp_is_wifi_unusable() && + besttp->t_rxtshift >= mptcp_fail_thresh) + return (mptcp_return_subflow(second_best)); + + return (mptcp_return_subflow(best)); + } else if (mpte->mpte_svctype == MPTCP_SVCTYPE_INTERACTIVE) { + int rtt_thresh = mptcp_rtthist_rtthresh << TCP_RTT_SHIFT; + int rto_thresh = mptcp_rtothresh; + + /* Adjust with symptoms information */ + if (IFNET_IS_WIFI(bestinp->inp_last_outifp) && + mptcp_is_wifi_unusable()) { + rtt_thresh /= 2; + rto_thresh /= 2; + } - /* If second_best received data, use second_best */ - if (mptcp_peerswitch && - (second_best->mpts_peerswitch > - MAX(MPTCP_PEERSWITCH_CNTMIN, mptcp_peerswitch_cnt))) { - tcpstat.tcps_mp_sel_peer++; - mptcplog((LOG_DEBUG, "MPTCP Sender: %s: best cid %d" - " but using cid %d after receiving %d segments\n", - __func__, best->mpts_connid, second_best->mpts_connid, - second_best->mpts_peerswitch), MPTCP_SENDER_DBG, - MPTCP_LOGLVL_LOG); - return (second_best); - } - return (best); -} + if (besttp->t_srtt && secondtp->t_srtt && + besttp->t_srtt >= rtt_thresh && + secondtp->t_srtt < rtt_thresh) { + tcpstat.tcps_mp_sel_rtt++; + mptcplog((LOG_DEBUG, "%s: best cid %d at rtt %d, second cid %d at rtt %d\n", __func__, + best->mpts_connid, besttp->t_srtt >> TCP_RTT_SHIFT, + second_best->mpts_connid, + secondtp->t_srtt >> TCP_RTT_SHIFT), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); + return (mptcp_return_subflow(second_best)); + } -struct mptsub * -mptcp_get_pending_subflow(struct mptses *mpte, struct mptsub *ignore) -{ - struct mptsub *mpts = NULL; + if (besttp->t_rxtshift >= mptcp_fail_thresh && + secondtp->t_rxtshift == 0) { + return (mptcp_return_subflow(second_best)); + } - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + /* Compare RTOs, select second_best if best's rto exceeds rtothresh */ + if (besttp->t_rxtcur && secondtp->t_rxtcur && + besttp->t_rxtcur >= rto_thresh && + secondtp->t_rxtcur < rto_thresh) { + tcpstat.tcps_mp_sel_rto++; + mptcplog((LOG_DEBUG, "%s: best cid %d at rto %d, second cid %d at rto %d\n", __func__, + best->mpts_connid, besttp->t_rxtcur, + second_best->mpts_connid, secondtp->t_rxtcur), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); + + return (mptcp_return_subflow(second_best)); + } - TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { - MPTS_LOCK(mpts); + /* + * None of the above conditions for sending on the secondary + * were true. So, let's schedule on the best one, if he still + * has some space in the congestion-window. + */ + return (mptcp_return_subflow(best)); + } else if (mpte->mpte_svctype == MPTCP_SVCTYPE_AGGREGATE) { + struct mptsub *tmp; - if ((ignore) && (mpts == ignore)) { - MPTS_UNLOCK(mpts); - continue; + /* + * We only care about RTT when aggregating + */ + if (besttp->t_srtt > secondtp->t_srtt) { + tmp = best; + best = second_best; + besttp = secondtp; + bestinp = secondinp; + + second_best = tmp; + secondtp = sototcpcb(second_best->mpts_socket); + secondinp = sotoinpcb(second_best->mpts_socket); } - if (mpts->mpts_flags & MPTSF_CONNECT_PENDING) { - MPTS_UNLOCK(mpts); - break; - } + /* Is there still space in the congestion window? */ + if (mptcp_subflow_cwnd_space(bestinp->inp_socket) <= 0) + return (mptcp_return_subflow(second_best)); - MPTS_UNLOCK(mpts); + return (mptcp_return_subflow(best)); + } else { + panic("Unknown service-type configured for MPTCP"); } - return (mpts); + + return (NULL); } static const char * @@ -702,7 +968,7 @@ mptcp_state_to_str(mptcp_state_t state) void mptcp_close_fsm(struct mptcb *mp_tp, uint32_t event) { - MPT_LOCK_ASSERT_HELD(mp_tp); + mpte_lock_assert_held(mp_tp->mpt_mpte); mptcp_state_t old_state = mp_tp->mpt_state; DTRACE_MPTCP2(state__change, struct mptcb *, mp_tp, @@ -747,7 +1013,7 @@ mptcp_close_fsm(struct mptcb *mp_tp, uint32_t event) case MPTCPS_LAST_ACK: if (event == MPCE_RECV_DATA_ACK) - mp_tp->mpt_state = MPTCPS_TERMINATE; + mptcp_close(mp_tp->mpt_mpte, mp_tp); break; case MPTCPS_FIN_WAIT_2: @@ -758,54 +1024,22 @@ mptcp_close_fsm(struct mptcb *mp_tp, uint32_t event) break; case MPTCPS_TIME_WAIT: - break; - case MPTCPS_TERMINATE: break; + default: VERIFY(0); /* NOTREACHED */ } DTRACE_MPTCP2(state__change, struct mptcb *, mp_tp, uint32_t, event); - mptcplog((LOG_INFO, "MPTCP State: %s to %s on event %s\n", + mptcplog((LOG_INFO, "%s: %s to %s on event %s\n", __func__, mptcp_state_to_str(old_state), mptcp_state_to_str(mp_tp->mpt_state), mptcp_event_to_str(event)), MPTCP_STATE_DBG, MPTCP_LOGLVL_LOG); } -/* - * Update the mptcb send state variables, but the actual sbdrop occurs - * in MPTCP layer - */ -void -mptcp_data_ack_rcvd(struct mptcb *mp_tp, struct tcpcb *tp, u_int64_t full_dack) -{ - u_int64_t acked = 0; - - acked = full_dack - mp_tp->mpt_snduna; - - if (acked) { - mp_tp->mpt_snduna += acked; - /* In degraded mode, we may get some Data ACKs */ - if ((tp->t_mpflags & TMPF_TCP_FALLBACK) && - !(mp_tp->mpt_flags & MPTCPF_POST_FALLBACK_SYNC) && - MPTCP_SEQ_GT(mp_tp->mpt_sndnxt, mp_tp->mpt_snduna)) { - /* bring back sndnxt to retransmit MPTCP data */ - mp_tp->mpt_sndnxt = mp_tp->mpt_dsn_at_csum_fail; - mp_tp->mpt_flags |= MPTCPF_POST_FALLBACK_SYNC; - tp->t_inpcb->inp_socket->so_flags1 |= - SOF1_POST_FALLBACK_SYNC; - } - } - if ((full_dack == mp_tp->mpt_sndmax) && - (mp_tp->mpt_state >= MPTCPS_FIN_WAIT_1)) { - mptcp_close_fsm(mp_tp, MPCE_RECV_DATA_ACK); - tp->t_mpflags &= ~TMPF_SEND_DFIN; - } -} - /* If you change this function, match up mptcp_update_rcv_state_f */ void mptcp_update_dss_rcv_state(struct mptcp_dsn_opt *dss_info, struct tcpcb *tp, @@ -819,9 +1053,7 @@ mptcp_update_dss_rcv_state(struct mptcp_dsn_opt *dss_info, struct tcpcb *tp, NTOHS(dss_info->mdss_data_len); /* XXX for autosndbuf grow sb here */ - MPT_LOCK(mp_tp); MPTCP_EXTEND_DSN(mp_tp->mpt_rcvnxt, dss_info->mdss_dsn, full_dsn); - MPT_UNLOCK(mp_tp); mptcp_update_rcv_state_meat(mp_tp, tp, full_dsn, dss_info->mdss_subflow_seqn, dss_info->mdss_data_len, csum); @@ -834,32 +1066,28 @@ mptcp_update_rcv_state_meat(struct mptcb *mp_tp, struct tcpcb *tp, uint16_t csum) { if (mdss_data_len == 0) { - mptcplog((LOG_INFO, "MPTCP Receiver: Infinite Mapping.\n"), + mptcplog((LOG_INFO, "%s: Infinite Mapping.\n", __func__), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_LOG); if ((mp_tp->mpt_flags & MPTCPF_CHECKSUM) && (csum != 0)) { - mptcplog((LOG_ERR, "MPTCP Receiver: Bad checksum %x \n", + mptcplog((LOG_ERR, "%s: Bad checksum %x \n", __func__, csum), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_ERR); } mptcp_notify_mpfail(tp->t_inpcb->inp_socket); return; } - MPT_LOCK(mp_tp); mptcplog((LOG_DEBUG, - "MPTCP Receiver: seqn = %x len = %x full = %llx " - "rcvnxt = %llu \n", + "%s: seqn = %x len = %x full = %llx rcvnxt = %llu \n", __func__, seqn, mdss_data_len, full_dsn, mp_tp->mpt_rcvnxt), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE); /* Process a Data FIN packet , handled in mptcp_do_fin_opt */ if ((seqn == 0) && (mdss_data_len == 1)) { - mptcplog((LOG_INFO, "MPTCP Receiver: Data FIN in %s state \n", + mptcplog((LOG_INFO, "%s: Data FIN in %s state \n", __func__, mptcp_state_to_str(mp_tp->mpt_state)), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_LOG); - MPT_UNLOCK(mp_tp); return; } - MPT_UNLOCK(mp_tp); mptcp_notify_mpready(tp->t_inpcb->inp_socket); tp->t_rcv_map.mpt_dsn = full_dsn; tp->t_rcv_map.mpt_sseq = seqn; @@ -869,78 +1097,20 @@ mptcp_update_rcv_state_meat(struct mptcb *mp_tp, struct tcpcb *tp, } -void -mptcp_update_rcv_state_f(struct mptcp_dss_ack_opt *dss_info, struct tcpcb *tp, - uint16_t csum) -{ - u_int64_t full_dsn = 0; - struct mptcb *mp_tp = tptomptp(tp); - - /* - * May happen, because the caller of this function does an soevent. - * Review after rdar://problem/24083886 - */ - if (!mp_tp) - return; - - NTOHL(dss_info->mdss_dsn); - NTOHL(dss_info->mdss_subflow_seqn); - NTOHS(dss_info->mdss_data_len); - MPT_LOCK(mp_tp); - MPTCP_EXTEND_DSN(mp_tp->mpt_rcvnxt, dss_info->mdss_dsn, full_dsn); - MPT_UNLOCK(mp_tp); - mptcp_update_rcv_state_meat(mp_tp, tp, - full_dsn, - dss_info->mdss_subflow_seqn, - dss_info->mdss_data_len, - csum); -} - -void -mptcp_update_rcv_state_g(struct mptcp_dss64_ack32_opt *dss_info, - struct tcpcb *tp, uint16_t csum) -{ - u_int64_t dsn = mptcp_ntoh64(dss_info->mdss_dsn); - struct mptcb *mp_tp = tptomptp(tp); - - /* - * May happen, because the caller of this function does an soevent. - * Review after rdar://problem/24083886 - */ - if (!mp_tp) - return; - - NTOHL(dss_info->mdss_subflow_seqn); - NTOHS(dss_info->mdss_data_len); - mptcp_update_rcv_state_meat(mp_tp, tp, - dsn, - dss_info->mdss_subflow_seqn, - dss_info->mdss_data_len, - csum); -} - static int mptcp_validate_dss_map(struct socket *so, struct tcpcb *tp, struct mbuf *m, int hdrlen) { - u_int32_t sseq, datalen; + u_int32_t datalen; if (!(m->m_pkthdr.pkt_flags & PKTF_MPTCP)) return 0; - sseq = m->m_pkthdr.mp_rseq + tp->irs; datalen = m->m_pkthdr.mp_rlen; -#if 0 - /* enable this to test TCP fallback post connection establishment */ - if (SEQ_GT(sseq, (tp->irs+1))) - datalen = m->m_pkthdr.len - hdrlen - 1; -#endif - /* unacceptable DSS option, fallback to TCP */ if (m->m_pkthdr.len > ((int) datalen + hdrlen)) { - mptcplog((LOG_ERR, "MPTCP Receiver: " - "%s: mbuf len %d, MPTCP expected %d", + mptcplog((LOG_ERR, "%s: mbuf len %d, MPTCP expected %d", __func__, m->m_pkthdr.len, datalen), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_LOG); } else { @@ -955,9 +1125,6 @@ mptcp_validate_dss_map(struct socket *so, struct tcpcb *tp, struct mbuf *m, int mptcp_input_preproc(struct tcpcb *tp, struct mbuf *m, int drop_hdrlen) { - if (mptcp_validate_csum(tp, m, drop_hdrlen) != 0) - return -1; - mptcp_insert_rmap(tp, m); if (mptcp_validate_dss_map(tp->t_inpcb->inp_socket, tp, m, drop_hdrlen) != 0) @@ -973,31 +1140,29 @@ mptcp_input_preproc(struct tcpcb *tp, struct mbuf *m, int drop_hdrlen) * DSS option. */ -static int -mptcp_validate_csum(struct tcpcb *tp, struct mbuf *m, int drop_hdrlen) +int +mptcp_validate_csum(struct tcpcb *tp, struct mbuf *m, uint64_t dsn, + uint32_t sseq, uint16_t dlen, uint16_t csum) { - uint16_t mptcp_csum = 0; - mptcp_csum = mptcp_input_csum(tp, m, drop_hdrlen); + uint16_t mptcp_csum; + + mptcp_csum = mptcp_input_csum(tp, m, dsn, sseq, dlen, csum); if (mptcp_csum) { tp->t_mpflags |= TMPF_SND_MPFAIL; - tp->t_mpflags &= ~TMPF_EMBED_DSN; mptcp_notify_mpfail(tp->t_inpcb->inp_socket); m_freem(m); tcpstat.tcps_mp_badcsum++; - return -1; + return (-1); } - return 0; + return (0); } static uint16_t -mptcp_input_csum(struct tcpcb *tp, struct mbuf *m, int off) +mptcp_input_csum(struct tcpcb *tp, struct mbuf *m, uint64_t dsn, uint32_t sseq, + uint16_t dlen, uint16_t csum) { struct mptcb *mp_tp = tptomptp(tp); uint32_t sum = 0; - uint64_t dsn; - uint32_t sseq; - uint16_t len; - uint16_t csum; if (mp_tp == NULL) return (0); @@ -1005,9 +1170,6 @@ mptcp_input_csum(struct tcpcb *tp, struct mbuf *m, int off) if (!(mp_tp->mpt_flags & MPTCPF_CHECKSUM)) return (0); - if (!(tp->t_mpflags & TMPF_EMBED_DSN)) - return (0); - if (tp->t_mpflags & TMPF_TCP_FALLBACK) return (0); @@ -1015,62 +1177,42 @@ mptcp_input_csum(struct tcpcb *tp, struct mbuf *m, int off) * The remote side may send a packet with fewer bytes than the * claimed DSS checksum length. */ - if ((int)m_length2(m, NULL) < (off + tp->t_rcv_map.mpt_len)) + if ((int)m_length2(m, NULL) < dlen) return (0xffff); - if (tp->t_rcv_map.mpt_len != 0) - sum = m_sum16(m, off, tp->t_rcv_map.mpt_len); + if (dlen != 0) + sum = m_sum16(m, 0, dlen); - dsn = mptcp_hton64(tp->t_rcv_map.mpt_dsn); - sseq = htonl(tp->t_rcv_map.mpt_sseq); - len = htons(tp->t_rcv_map.mpt_len); - csum = tp->t_rcv_map.mpt_csum; - sum += in_pseudo64(dsn, sseq, (len + csum)); + sum += in_pseudo64(htonll(dsn), htonl(sseq), htons(dlen) + csum); ADDCARRY(sum); DTRACE_MPTCP3(checksum__result, struct tcpcb *, tp, struct mbuf *, m, uint32_t, sum); - mptcplog((LOG_DEBUG, "MPTCP Receiver: sum = %x \n", sum), + + mptcplog((LOG_DEBUG, "%s: sum = %x \n", __func__, sum), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE); return (~sum & 0xffff); } -void -mptcp_output_csum(struct tcpcb *tp, struct mbuf *m, int32_t len, - unsigned hdrlen, u_int64_t dss_val, u_int32_t *sseqp) +uint32_t +mptcp_output_csum(struct mbuf *m, uint64_t dss_val, uint32_t sseq, uint16_t dlen) { - struct mptcb *mp_tp = tptomptp(tp); u_int32_t sum = 0; - uint32_t sseq; - uint16_t dss_len; - uint16_t csum = 0; - uint16_t *csump = NULL; - - if (mp_tp == NULL) - return; - - if (!(mp_tp->mpt_flags & MPTCPF_CHECKSUM)) - return; - if (sseqp == NULL) - return; - - if (len) - sum = m_sum16(m, hdrlen, len); + if (dlen) + sum = m_sum16(m, 0, dlen); dss_val = mptcp_hton64(dss_val); - sseq = *sseqp; - dss_len = *(uint16_t *)(void *)((u_char*)sseqp + sizeof (u_int32_t)); - sum += in_pseudo64(dss_val, sseq, (dss_len + csum)); + sseq = htonl(sseq); + dlen = htons(dlen); + sum += in_pseudo64(dss_val, sseq, dlen); ADDCARRY(sum); sum = ~sum & 0xffff; - csump = (uint16_t *)(void *)((u_char*)sseqp + sizeof (u_int32_t) + - sizeof (uint16_t)); - DTRACE_MPTCP3(checksum__result, struct tcpcb *, tp, struct mbuf *, m, - uint32_t, sum); - *csump = sum; - mptcplog((LOG_DEBUG, "MPTCP Sender: sum = %x \n", sum), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); + DTRACE_MPTCP2(checksum__result, struct mbuf *, m, uint32_t, sum); + mptcplog((LOG_DEBUG, "%s: sum = %x \n", __func__, sum), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); + + return sum; } /* @@ -1084,11 +1226,11 @@ mptcp_no_rto_spike(struct socket *so) struct tcpcb *tp = intotcpcb(sotoinpcb(so)); int32_t spike = 0; - if (tp->t_rxtcur > MAX(mptcp_rtothresh, MPTCP_RTO_MINTHRESH)) { + if (tp->t_rxtcur > mptcp_rtothresh) { spike = tp->t_rxtcur - mptcp_rtothresh; - mptcplog((LOG_DEBUG, "MPTCP Socket: %s: spike = %d rto = %d" - "best = %d cur = %d\n", __func__, spike, + mptcplog((LOG_DEBUG, "%s: spike = %d rto = %d best = %d cur = %d\n", + __func__, spike, tp->t_rxtcur, tp->t_rttbest >> TCP_RTT_SHIFT, tp->t_rttcur), (MPTCP_SOCKET_DBG|MPTCP_SENDER_DBG), MPTCP_LOGLVL_LOG); @@ -1101,3 +1243,229 @@ mptcp_no_rto_spike(struct socket *so) return (TRUE); } } + +void +mptcp_handle_deferred_upcalls(struct mppcb *mpp, uint32_t flag) +{ + VERIFY(mpp->mpp_flags & flag); + mpp->mpp_flags &= ~flag; + + if (mptcp_should_defer_upcall(mpp)) + return; + + if (mpp->mpp_flags & MPP_SHOULD_WORKLOOP) { + mpp->mpp_flags &= ~MPP_SHOULD_WORKLOOP; + + mptcp_subflow_workloop(mpp->mpp_pcbe); + } + + if (mpp->mpp_flags & MPP_SHOULD_RWAKEUP) { + mpp->mpp_flags &= ~MPP_SHOULD_RWAKEUP; + + sorwakeup(mpp->mpp_socket); + } + + if (mpp->mpp_flags & MPP_SHOULD_WWAKEUP) { + mpp->mpp_flags &= ~MPP_SHOULD_WWAKEUP; + + sowwakeup(mpp->mpp_socket); + } + + if (mpp->mpp_flags & MPP_SET_CELLICON) { + mpp->mpp_flags &= ~MPP_SET_CELLICON; + + mptcp_set_cellicon(mpp->mpp_pcbe); + } + + if (mpp->mpp_flags & MPP_UNSET_CELLICON) { + mpp->mpp_flags &= ~MPP_UNSET_CELLICON; + + mptcp_unset_cellicon(); + } +} + +static void +mptcp_ask_for_nat64(struct ifnet *ifp) +{ + in6_post_msg(ifp, KEV_INET6_REQUEST_NAT64_PREFIX, NULL, NULL); + + mptcplog((LOG_DEBUG, "%s: asked for NAT64-prefix on %s\n", + __func__, ifp->if_name), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); +} + +static void +mptcp_reset_itfinfo(struct mpt_itf_info *info) +{ + info->ifindex = 0; + info->has_v4_conn = 0; + info->has_v6_conn = 0; +} + +void +mptcp_session_necp_cb(void *handle, int action, struct necp_client_flow *flow) +{ + struct mppcb *mp = (struct mppcb *)handle; + struct mptses *mpte = mptompte(mp); + struct socket *mp_so; + struct mptcb *mp_tp; + int locked = 0; + uint32_t i, ifindex; + + ifindex = flow->interface_index; + VERIFY(ifindex != IFSCOPE_NONE); + + /* ToDo - remove after rdar://problem/32007628 */ + if (!IF_INDEX_IN_RANGE(ifindex)) + printf("%s 1 ifindex %u not in range of flow %p action %d\n", + __func__, ifindex, flow, action); + + /* About to be garbage-collected (see note about MPTCP/NECP interactions) */ + if (mp->mpp_socket->so_usecount == 0) + return; + + if (action != NECP_CLIENT_CBACTION_INITIAL) { + mpte_lock(mpte); + locked = 1; + + /* Check again, because it might have changed while waiting */ + if (mp->mpp_socket->so_usecount == 0) + goto out; + } + + mp_tp = mpte->mpte_mptcb; + mp_so = mptetoso(mpte); + + mptcplog((LOG_DEBUG, "%s, action: %u ifindex %u usecount %u mpt_flags %#x state %u\n", + __func__, action, ifindex, mp->mpp_socket->so_usecount, mp_tp->mpt_flags, mp_tp->mpt_state), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + + /* No need on fallen back sockets */ + if (mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) + goto out; + + if (action == NECP_CLIENT_CBACTION_NONVIABLE) { + for (i = 0; i < mpte->mpte_itfinfo_size; i++) { + if (mpte->mpte_itfinfo[i].ifindex == ifindex) + mptcp_reset_itfinfo(&mpte->mpte_itfinfo[i]); + } + + mptcp_sched_create_subflows(mpte); + } else if (action == NECP_CLIENT_CBACTION_VIABLE || + action == NECP_CLIENT_CBACTION_INITIAL) { + int found_empty = 0, empty_index = -1; + struct ifnet *ifp; + + /* ToDo - remove after rdar://problem/32007628 */ + if (!IF_INDEX_IN_RANGE(ifindex)) + printf("%s 2 ifindex %u not in range of flow %p action %d\n", + __func__, ifindex, flow, action); + + ifnet_head_lock_shared(); + ifp = ifindex2ifnet[ifindex]; + ifnet_head_done(); + + /* ToDo - remove after rdar://problem/32007628 */ + if (!IF_INDEX_IN_RANGE(ifindex)) + printf("%s 3 ifindex %u not in range of flow %p action %d\n", + __func__, ifindex, flow, action); + + if (ifp == NULL) + goto out; + + if (IFNET_IS_EXPENSIVE(ifp) && + (mp_so->so_restrictions & SO_RESTRICT_DENY_EXPENSIVE)) + goto out; + + if (IFNET_IS_CELLULAR(ifp) && + (mp_so->so_restrictions & SO_RESTRICT_DENY_CELLULAR)) + goto out; + + for (i = 0; i < mpte->mpte_itfinfo_size; i++) { + if (mpte->mpte_itfinfo[i].ifindex == 0) { + found_empty = 1; + empty_index = i; + } + + if (mpte->mpte_itfinfo[i].ifindex == ifindex) { + /* Ok, it's already there */ + goto out; + } + } + + if ((mpte->mpte_dst.sa_family == AF_INET || mpte->mpte_dst.sa_family == 0) && + !(flow->necp_flow_flags & NECP_CLIENT_RESULT_FLAG_HAS_IPV4) && + ifnet_get_nat64prefix(ifp, NULL) == ENOENT) { + mptcp_ask_for_nat64(ifp); + goto out; + } + + if (found_empty == 0) { + int new_size = mpte->mpte_itfinfo_size * 2; + struct mpt_itf_info *info = _MALLOC(sizeof(*info) * new_size, M_TEMP, M_ZERO); + + if (info == NULL) { + mptcplog((LOG_ERR, "%s malloc failed for %u\n", __func__, new_size), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + goto out; + } + + memcpy(info, mpte->mpte_itfinfo, mpte->mpte_itfinfo_size * sizeof(*info)); + + if (mpte->mpte_itfinfo_size > MPTE_ITFINFO_SIZE) + _FREE(mpte->mpte_itfinfo, M_TEMP); + + /* We allocated a new one, thus the first must be empty */ + empty_index = mpte->mpte_itfinfo_size; + + mpte->mpte_itfinfo = info; + mpte->mpte_itfinfo_size = new_size; + + mptcplog((LOG_DEBUG, "%s Needed to realloc to %u\n", __func__, new_size), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + } + + VERIFY(empty_index >= 0 && empty_index < (int)mpte->mpte_itfinfo_size); + mpte->mpte_itfinfo[empty_index].ifindex = ifindex; + mpte->mpte_itfinfo[empty_index].has_v4_conn = !!(flow->necp_flow_flags & NECP_CLIENT_RESULT_FLAG_HAS_IPV4); + mpte->mpte_itfinfo[empty_index].has_v6_conn = !!(flow->necp_flow_flags & NECP_CLIENT_RESULT_FLAG_HAS_IPV6); + + mptcp_sched_create_subflows(mpte); + } + +out: + if (locked) + mpte_unlock(mpte); +} + +void +mptcp_set_restrictions(struct socket *mp_so) +{ + struct mptses *mpte = mpsotompte(mp_so); + uint32_t i; + + mpte_lock_assert_held(mpte); + + ifnet_head_lock_shared(); + + for (i = 0; i < mpte->mpte_itfinfo_size; i++) { + struct mpt_itf_info *info = &mpte->mpte_itfinfo[i]; + uint32_t ifindex = info->ifindex; + struct ifnet *ifp; + + if (ifindex == IFSCOPE_NONE) + continue; + + ifp = ifindex2ifnet[ifindex]; + + if (IFNET_IS_EXPENSIVE(ifp) && + (mp_so->so_restrictions & SO_RESTRICT_DENY_EXPENSIVE)) + info->ifindex = IFSCOPE_NONE; + + if (IFNET_IS_CELLULAR(ifp) && + (mp_so->so_restrictions & SO_RESTRICT_DENY_CELLULAR)) + info->ifindex = IFSCOPE_NONE; + } + + ifnet_head_done(); +} + diff --git a/bsd/netinet/mptcp.h b/bsd/netinet/mptcp.h index 3ea265ebc..d0b77e6b6 100644 --- a/bsd/netinet/mptcp.h +++ b/bsd/netinet/mptcp.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2014 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -33,6 +33,8 @@ #include <machine/endian.h> +#include <libkern/crypto/sha1.h> + #if BYTE_ORDER == BIG_ENDIAN #define mptcp_hton64(x) (x) #define mptcp_ntoh64(x) (x) @@ -135,7 +137,7 @@ struct mptcp_mpjoin_opt_rsp2 { mmjo_reserved1:4; #endif u_int8_t mmjo_reserved2; - u_int8_t mmjo_mac[20]; /* This is 160 bits HMAC SHA-1 per RFC */ + u_int8_t mmjo_mac[SHA1_RESULTLEN]; /* This is 160 bits HMAC SHA-1 per RFC */ } __attribute__((__packed__)); diff --git a/bsd/netinet/mptcp_opt.c b/bsd/netinet/mptcp_opt.c index d2aec1750..8aa8a9a4e 100644 --- a/bsd/netinet/mptcp_opt.c +++ b/bsd/netinet/mptcp_opt.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -53,29 +53,22 @@ #include <mach/sdt.h> -/* - * SYSCTL for enforcing 64 bit dsn - */ -int32_t force_64bit_dsn = 0; -SYSCTL_INT(_net_inet_mptcp, OID_AUTO, force_64bit_dsn, - CTLFLAG_RW|CTLFLAG_LOCKED, &force_64bit_dsn, 0, - "Force MPTCP 64bit dsn"); - - static int mptcp_validate_join_hmac(struct tcpcb *, u_char*, int); static int mptcp_snd_mpprio(struct tcpcb *tp, u_char *cp, int optlen); +static void mptcp_send_remaddr_opt(struct tcpcb *, struct mptcp_remaddr_opt *); /* * MPTCP Options Output Processing */ static unsigned -mptcp_setup_first_subflow_syn_opts(struct socket *so, int flags, u_char *opt, - unsigned optlen) +mptcp_setup_first_subflow_syn_opts(struct socket *so, u_char *opt, unsigned optlen) { + struct mptcp_mpcapable_opt_common mptcp_opt; struct tcpcb *tp = sototcpcb(so); - struct mptcb *mp_tp = NULL; - mp_tp = tptomptp(tp); + struct mptcb *mp_tp = tptomptp(tp); + + mpte_lock_assert_held(mp_tp->mpt_mpte); /* * Avoid retransmitting the MP_CAPABLE option. @@ -93,72 +86,32 @@ mptcp_setup_first_subflow_syn_opts(struct socket *so, int flags, u_char *opt, return (optlen); } - if ((flags & (TH_SYN | TH_ACK)) == (TH_SYN | TH_ACK)) { - struct mptcp_mpcapable_opt_rsp mptcp_opt; - mptcp_key_t mp_localkey = 0; + bzero(&mptcp_opt, sizeof (struct mptcp_mpcapable_opt_common)); - mp_localkey = mptcp_get_localkey(mp_tp); - if (mp_localkey == 0) { - /* an embryonic connection was closed from above */ - return (optlen); - } - bzero(&mptcp_opt, - sizeof (struct mptcp_mpcapable_opt_rsp)); - mptcp_opt.mmc_common.mmco_kind = TCPOPT_MULTIPATH; - mptcp_opt.mmc_common.mmco_len = - sizeof (struct mptcp_mpcapable_opt_rsp); - mptcp_opt.mmc_common.mmco_subtype = MPO_CAPABLE; - MPT_LOCK_SPIN(mp_tp); - mptcp_opt.mmc_common.mmco_version = mp_tp->mpt_version; - mptcp_opt.mmc_common.mmco_flags |= MPCAP_PROPOSAL_SBIT; - if (mp_tp->mpt_flags & MPTCPF_CHECKSUM) - mptcp_opt.mmc_common.mmco_flags |= - MPCAP_CHECKSUM_CBIT; - MPT_UNLOCK(mp_tp); - mptcp_opt.mmc_localkey = mp_localkey; - memcpy(opt + optlen, &mptcp_opt, - mptcp_opt.mmc_common.mmco_len); - optlen += mptcp_opt.mmc_common.mmco_len; - } else { - /* Only the SYN flag is set */ - struct mptcp_mpcapable_opt_common mptcp_opt; - mptcp_key_t mp_localkey = 0; - mp_localkey = mptcp_get_localkey(mp_tp); - so->so_flags |= SOF_MPTCP_CLIENT; - if (mp_localkey == 0) { - /* an embryonic connection was closed */ - return (optlen); - } - bzero(&mptcp_opt, - sizeof (struct mptcp_mpcapable_opt_common)); - mptcp_opt.mmco_kind = TCPOPT_MULTIPATH; - mptcp_opt.mmco_len = - sizeof (struct mptcp_mpcapable_opt_common) + - sizeof (mptcp_key_t); - mptcp_opt.mmco_subtype = MPO_CAPABLE; - MPT_LOCK_SPIN(mp_tp); - mptcp_opt.mmco_version = mp_tp->mpt_version; - mptcp_opt.mmco_flags |= MPCAP_PROPOSAL_SBIT; - if (mp_tp->mpt_flags & MPTCPF_CHECKSUM) - mptcp_opt.mmco_flags |= MPCAP_CHECKSUM_CBIT; - MPT_UNLOCK(mp_tp); - (void) memcpy(opt + optlen, &mptcp_opt, - sizeof (struct mptcp_mpcapable_opt_common)); - optlen += sizeof (struct mptcp_mpcapable_opt_common); - (void) memcpy(opt + optlen, &mp_localkey, - sizeof (mptcp_key_t)); - optlen += sizeof (mptcp_key_t); - } + mptcp_opt.mmco_kind = TCPOPT_MULTIPATH; + mptcp_opt.mmco_len = + sizeof (struct mptcp_mpcapable_opt_common) + + sizeof (mptcp_key_t); + mptcp_opt.mmco_subtype = MPO_CAPABLE; + mptcp_opt.mmco_version = mp_tp->mpt_version; + mptcp_opt.mmco_flags |= MPCAP_PROPOSAL_SBIT; + if (mp_tp->mpt_flags & MPTCPF_CHECKSUM) + mptcp_opt.mmco_flags |= MPCAP_CHECKSUM_CBIT; + memcpy(opt + optlen, &mptcp_opt, sizeof (struct mptcp_mpcapable_opt_common)); + optlen += sizeof (struct mptcp_mpcapable_opt_common); + memcpy(opt + optlen, &mp_tp->mpt_localkey, sizeof (mptcp_key_t)); + optlen += sizeof (mptcp_key_t); return (optlen); } static unsigned -mptcp_setup_join_subflow_syn_opts(struct socket *so, int flags, u_char *opt, - unsigned optlen) +mptcp_setup_join_subflow_syn_opts(struct socket *so, u_char *opt, unsigned optlen) { + struct mptcp_mpjoin_opt_req mpjoin_req; struct inpcb *inp = sotoinpcb(so); struct tcpcb *tp = NULL; + struct mptsub *mpts; if (!inp) return (optlen); @@ -167,61 +120,37 @@ mptcp_setup_join_subflow_syn_opts(struct socket *so, int flags, u_char *opt, if (!tp) return (optlen); - if (!tp->t_mptcb) - return (optlen); + mpts = tp->t_mpsub; - if ((flags & (TH_SYN | TH_ACK)) == (TH_SYN | TH_ACK)) { - struct mptcp_mpjoin_opt_rsp mpjoin_rsp; - struct mptcb *mp_tp = tptomptp(tp); + VERIFY(tptomptp(tp)); + mpte_lock_assert_held(tptomptp(tp)->mpt_mpte); - if (mp_tp == NULL) - return (optlen); + bzero(&mpjoin_req, sizeof (mpjoin_req)); + mpjoin_req.mmjo_kind = TCPOPT_MULTIPATH; + mpjoin_req.mmjo_len = sizeof (mpjoin_req); + mpjoin_req.mmjo_subtype_bkp = MPO_JOIN << 4; - MPT_LOCK(mp_tp); - if (mptcp_get_localkey(mp_tp) == 0) { - MPT_UNLOCK(mp_tp); - return (optlen); - } - MPT_UNLOCK(mp_tp); - bzero(&mpjoin_rsp, sizeof (mpjoin_rsp)); - mpjoin_rsp.mmjo_kind = TCPOPT_MULTIPATH; - mpjoin_rsp.mmjo_len = sizeof (mpjoin_rsp); - mpjoin_rsp.mmjo_subtype_bkp = MPO_JOIN << 4; - if (tp->t_mpflags & TMPF_BACKUP_PATH) - mpjoin_rsp.mmjo_subtype_bkp |= MPTCP_BACKUP; - mpjoin_rsp.mmjo_addr_id = tp->t_local_aid; - mptcp_get_rands(tp->t_local_aid, tptomptp(tp), - &mpjoin_rsp.mmjo_rand, NULL); - mpjoin_rsp.mmjo_mac = mptcp_get_trunced_hmac(tp->t_local_aid, - mp_tp); - memcpy(opt + optlen, &mpjoin_rsp, mpjoin_rsp.mmjo_len); - optlen += mpjoin_rsp.mmjo_len; + if (tp->t_mpflags & TMPF_BACKUP_PATH) { + mpjoin_req.mmjo_subtype_bkp |= MPTCP_BACKUP; + } else if (inp->inp_boundifp && IFNET_IS_CELLULAR(inp->inp_boundifp) && + mpts->mpts_mpte->mpte_svctype != MPTCP_SVCTYPE_AGGREGATE) { + mpjoin_req.mmjo_subtype_bkp |= MPTCP_BACKUP; + tp->t_mpflags |= TMPF_BACKUP_PATH; } else { - struct mptcp_mpjoin_opt_req mpjoin_req; - - bzero(&mpjoin_req, sizeof (mpjoin_req)); - mpjoin_req.mmjo_kind = TCPOPT_MULTIPATH; - mpjoin_req.mmjo_len = sizeof (mpjoin_req); - mpjoin_req.mmjo_subtype_bkp = MPO_JOIN << 4; - if (tp->t_mpflags & TMPF_BACKUP_PATH) - mpjoin_req.mmjo_subtype_bkp |= MPTCP_BACKUP; - mpjoin_req.mmjo_addr_id = tp->t_local_aid; - mpjoin_req.mmjo_peer_token = mptcp_get_remotetoken(tp->t_mptcb); - if (mpjoin_req.mmjo_peer_token == 0) { - mptcplog((LOG_DEBUG, "MPTCP Socket: %s: peer token 0", - __func__), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); - } - mptcp_get_rands(tp->t_local_aid, tptomptp(tp), - &mpjoin_req.mmjo_rand, NULL); - memcpy(opt + optlen, &mpjoin_req, mpjoin_req.mmjo_len); - optlen += mpjoin_req.mmjo_len; - /* send an event up, if Fast Join is requested */ - if (mptcp_zerortt_fastjoin && - (so->so_flags & SOF_MPTCP_FASTJOIN)) { - soevent(so, (SO_FILT_HINT_LOCKED | SO_FILT_HINT_MPFASTJ)); - } + mpts->mpts_flags |= MPTSF_PREFERRED; + } + + mpjoin_req.mmjo_addr_id = tp->t_local_aid; + mpjoin_req.mmjo_peer_token = tptomptp(tp)->mpt_remotetoken; + if (mpjoin_req.mmjo_peer_token == 0) { + mptcplog((LOG_DEBUG, "%s: peer token 0", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); } + mptcp_get_rands(tp->t_local_aid, tptomptp(tp), + &mpjoin_req.mmjo_rand, NULL); + memcpy(opt + optlen, &mpjoin_req, mpjoin_req.mmjo_len); + optlen += mpjoin_req.mmjo_len; + return (optlen); } @@ -241,30 +170,22 @@ mptcp_setup_join_ack_opts(struct tcpcb *tp, u_char *opt, unsigned optlen) join_rsp2.mmjo_len = sizeof (struct mptcp_mpjoin_opt_rsp2); join_rsp2.mmjo_subtype = MPO_JOIN; mptcp_get_hmac(tp->t_local_aid, tptomptp(tp), - (u_char*)&join_rsp2.mmjo_mac, - sizeof (join_rsp2.mmjo_mac)); + (u_char*)&join_rsp2.mmjo_mac); memcpy(opt + optlen, &join_rsp2, join_rsp2.mmjo_len); new_optlen = optlen + join_rsp2.mmjo_len; - tp->t_mpflags |= TMPF_FASTJOINBY2_SEND; return (new_optlen); } unsigned -mptcp_setup_syn_opts(struct socket *so, int flags, u_char *opt, unsigned optlen) +mptcp_setup_syn_opts(struct socket *so, u_char *opt, unsigned optlen) { unsigned new_optlen; - if (!(so->so_flags & SOF_MP_SEC_SUBFLOW)) { - new_optlen = mptcp_setup_first_subflow_syn_opts(so, flags, opt, - optlen); - } else { - /* - * To simulate SYN_ACK with no join opt, comment this line on - * OS X server side. This serves as a testing hook. - */ - new_optlen = mptcp_setup_join_subflow_syn_opts(so, flags, opt, - optlen); - } + if (!(so->so_flags & SOF_MP_SEC_SUBFLOW)) + new_optlen = mptcp_setup_first_subflow_syn_opts(so, opt, optlen); + else + new_optlen = mptcp_setup_join_subflow_syn_opts(so, opt, optlen); + return (new_optlen); } @@ -284,15 +205,15 @@ mptcp_send_mpfail(struct tcpcb *tp, u_char *opt, unsigned int optlen) return (optlen); } + mpte_lock_assert_held(mp_tp->mpt_mpte); + /* if option space low give up */ if ((MAX_TCPOPTLEN - optlen) < sizeof (struct mptcp_mpfail_opt)) { tp->t_mpflags &= ~TMPF_SND_MPFAIL; return (optlen); } - MPT_LOCK(mp_tp); dsn = mp_tp->mpt_rcvnxt; - MPT_UNLOCK(mp_tp); bzero(&fail_opt, sizeof (fail_opt)); fail_opt.mfail_kind = TCPOPT_MULTIPATH; @@ -302,7 +223,7 @@ mptcp_send_mpfail(struct tcpcb *tp, u_char *opt, unsigned int optlen) memcpy(opt + optlen, &fail_opt, len); optlen += len; tp->t_mpflags &= ~TMPF_SND_MPFAIL; - mptcplog((LOG_DEBUG, "MPTCP Socket: %s: %d \n", __func__, + mptcplog((LOG_DEBUG, "%s: %d \n", __func__, tp->t_local_aid), (MPTCP_SOCKET_DBG | MPTCP_SENDER_DBG), MPTCP_LOGLVL_LOG); return (optlen); @@ -315,7 +236,6 @@ mptcp_send_infinite_mapping(struct tcpcb *tp, u_char *opt, unsigned int optlen) struct mptcb *mp_tp = NULL; size_t len = sizeof (struct mptcp_dsn_opt); struct socket *so = tp->t_inpcb->inp_socket; - int error = 0; int csum_len = 0; if (!so) @@ -325,15 +245,15 @@ mptcp_send_infinite_mapping(struct tcpcb *tp, u_char *opt, unsigned int optlen) if (mp_tp == NULL) return (optlen); - MPT_LOCK(mp_tp); + mpte_lock_assert_held(mp_tp->mpt_mpte); + if (mp_tp->mpt_flags & MPTCPF_CHECKSUM) csum_len = 2; /* try later */ - if ((MAX_TCPOPTLEN - optlen) < (len + csum_len)) { - MPT_UNLOCK(mp_tp); + if ((MAX_TCPOPTLEN - optlen) < (len + csum_len)) return (optlen); - } + bzero(&infin_opt, sizeof (infin_opt)); infin_opt.mdss_copt.mdss_kind = TCPOPT_MULTIPATH; infin_opt.mdss_copt.mdss_len = len + csum_len; @@ -352,20 +272,18 @@ mptcp_send_infinite_mapping(struct tcpcb *tp, u_char *opt, unsigned int optlen) ((mp_tp->mpt_local_idsn + 1) == mp_tp->mpt_snduna)) { infin_opt.mdss_subflow_seqn = 1; - mptcplog((LOG_DEBUG, "MPTCP Socket: %s: idsn %llu" - "snduna %llu \n", __func__, mp_tp->mpt_local_idsn, + mptcplog((LOG_DEBUG, "%s: idsn %llu snduna %llu \n", + __func__, mp_tp->mpt_local_idsn, mp_tp->mpt_snduna), (MPTCP_SOCKET_DBG | MPTCP_SENDER_DBG), MPTCP_LOGLVL_LOG); } else { - infin_opt.mdss_subflow_seqn = tp->snd_una - tp->iss; + infin_opt.mdss_subflow_seqn = tp->snd_una - tp->t_mpsub->mpts_iss; } infin_opt.mdss_dsn = (u_int32_t) MPTCP_DATASEQ_LOW32(mp_tp->mpt_snduna); } - MPT_UNLOCK(mp_tp); - if (error != 0) - return (optlen); + if ((infin_opt.mdss_dsn == 0) || (infin_opt.mdss_subflow_seqn == 0)) { return (optlen); } @@ -382,15 +300,13 @@ mptcp_send_infinite_mapping(struct tcpcb *tp, u_char *opt, unsigned int optlen) optlen += csum_len; } - mptcplog((LOG_DEBUG, "MPTCP Socket: %s: dsn = %x, seq = %x len = %x\n", - __func__, + mptcplog((LOG_DEBUG, "%s: dsn = %x, seq = %x len = %x\n", __func__, ntohl(infin_opt.mdss_dsn), ntohl(infin_opt.mdss_subflow_seqn), ntohs(infin_opt.mdss_data_len)), (MPTCP_SOCKET_DBG | MPTCP_SENDER_DBG), MPTCP_LOGLVL_LOG); - /* so->so_flags &= ~SOF_MPTCP_CLIENT; */ tp->t_mpflags |= TMPF_INFIN_SENT; tcpstat.tcps_estab_fallback++; return (optlen); @@ -400,24 +316,21 @@ mptcp_send_infinite_mapping(struct tcpcb *tp, u_char *opt, unsigned int optlen) static int mptcp_ok_to_fin(struct tcpcb *tp, u_int64_t dsn, u_int32_t datalen) { - struct mptcb *mp_tp = NULL; - mp_tp = tptomptp(tp); + struct mptcb *mp_tp = tptomptp(tp); + + mpte_lock_assert_held(mp_tp->mpt_mpte); - MPT_LOCK(mp_tp); dsn = (mp_tp->mpt_sndmax & MPTCP_DATASEQ_LOW32_MASK) | dsn; - if ((dsn + datalen) == mp_tp->mpt_sndmax) { - MPT_UNLOCK(mp_tp); + if ((dsn + datalen) == mp_tp->mpt_sndmax) return (1); - } - MPT_UNLOCK(mp_tp); + return (0); } unsigned int mptcp_setup_opts(struct tcpcb *tp, int32_t off, u_char *opt, - unsigned int optlen, int flags, int datalen, - unsigned int **dss_lenp, u_int8_t **finp, u_int64_t *dss_valp, - u_int32_t **sseqp, boolean_t *p_mptcp_acknow) + unsigned int optlen, int flags, int len, + boolean_t *p_mptcp_acknow) { struct inpcb *inp = (struct inpcb *)tp->t_inpcb; struct socket *so = inp->inp_socket; @@ -425,17 +338,15 @@ mptcp_setup_opts(struct tcpcb *tp, int32_t off, u_char *opt, boolean_t do_csum = FALSE; boolean_t send_64bit_dsn = FALSE; boolean_t send_64bit_ack = FALSE; - u_int32_t old_mpt_flags = tp->t_mpflags & - (TMPF_SND_MPPRIO | TMPF_SND_REM_ADDR | TMPF_SND_MPFAIL); + u_int32_t old_mpt_flags = tp->t_mpflags & TMPF_MPTCP_SIGNALS; - if ((mptcp_enable == 0) || - (mp_tp == NULL) || - (mp_tp->mpt_flags & MPTCPF_PEEL_OFF) || - (tp->t_state == TCPS_CLOSED)) { + if (mptcp_enable == 0 || mp_tp == NULL || tp->t_state == TCPS_CLOSED) { /* do nothing */ goto ret_optlen; } + mpte_lock_assert_held(mp_tp->mpt_mpte); + if (mp_tp->mpt_flags & MPTCPF_CHECKSUM) { do_csum = TRUE; } @@ -447,11 +358,9 @@ mptcp_setup_opts(struct tcpcb *tp, int32_t off, u_char *opt, if ((MAX_TCPOPTLEN - optlen) < sizeof (struct mptcp_mpcapable_opt_common)) { - mptcplog((LOG_ERR, "MPTCP Socket: " - "%s: no space left %d flags %x " - "tp->t_mpflags %x " - "len %d\n", __func__, optlen, flags, tp->t_mpflags, - datalen), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + mptcplog((LOG_ERR, "%s: no space left %d flags %x tp->t_mpflags %x len %d\n", + __func__, optlen, flags, tp->t_mpflags, len), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); goto ret_optlen; } @@ -463,17 +372,7 @@ mptcp_setup_opts(struct tcpcb *tp, int32_t off, u_char *opt, goto ret_optlen; } - if (((tp->t_mpflags & TMPF_FASTJOINBY2_SEND) || - (tp->t_mpflags & TMPF_FASTJOIN_SEND )) && - (datalen > 0)) { - tp->t_mpflags &= ~TMPF_FASTJOINBY2_SEND; - tp->t_mpflags &= ~TMPF_FASTJOIN_SEND; - goto fastjoin_send; - } - - if (((tp->t_mpflags & TMPF_PREESTABLISHED) && - (!(tp->t_mpflags & TMPF_SENT_KEYS)) && - (!(tp->t_mpflags & TMPF_JOINED_FLOW)))) { + if (tp->t_mpflags & TMPF_SND_KEYS) { struct mptcp_mpcapable_opt_rsp1 mptcp_opt; if ((MAX_TCPOPTLEN - optlen) < sizeof (struct mptcp_mpcapable_opt_rsp1)) @@ -486,17 +385,13 @@ mptcp_setup_opts(struct tcpcb *tp, int32_t off, u_char *opt, mptcp_opt.mmc_common.mmco_version = mp_tp->mpt_version; /* HMAC-SHA1 is the proposal */ mptcp_opt.mmc_common.mmco_flags |= MPCAP_PROPOSAL_SBIT; - MPT_LOCK(mp_tp); if (mp_tp->mpt_flags & MPTCPF_CHECKSUM) mptcp_opt.mmc_common.mmco_flags |= MPCAP_CHECKSUM_CBIT; - mptcp_opt.mmc_localkey = mptcp_get_localkey(mp_tp); - mptcp_opt.mmc_remotekey = mptcp_get_remotekey(mp_tp); - MPT_UNLOCK(mp_tp); + mptcp_opt.mmc_localkey = mp_tp->mpt_localkey; + mptcp_opt.mmc_remotekey = mp_tp->mpt_remotekey; memcpy(opt + optlen, &mptcp_opt, mptcp_opt.mmc_common.mmco_len); optlen += mptcp_opt.mmc_common.mmco_len; - tp->t_mpflags |= TMPF_SENT_KEYS | TMPF_MPTCP_TRUE; - so->so_flags |= SOF_MPTCP_TRUE; - tp->t_mpflags &= ~TMPF_PREESTABLISHED; + tp->t_mpflags &= ~TMPF_SND_KEYS; if (!tp->t_mpuna) { tp->t_mpuna = tp->snd_una; @@ -506,17 +401,7 @@ mptcp_setup_opts(struct tcpcb *tp, int32_t off, u_char *opt, goto ret_optlen; } - if ((tp->t_mpflags & TMPF_JOINED_FLOW) && - (tp->t_mpflags & TMPF_PREESTABLISHED) && - (!(tp->t_mpflags & TMPF_RECVD_JOIN)) && - (tp->t_mpflags & TMPF_SENT_JOIN) && - (!(tp->t_mpflags & TMPF_MPTCP_TRUE))) { - MPT_LOCK(mp_tp); - if (mptcp_get_localkey(mp_tp) == 0) { - MPT_UNLOCK(mp_tp); - goto ret_optlen; - } - MPT_UNLOCK(mp_tp); + if (tp->t_mpflags & TMPF_SND_JACK) { /* Do the ACK part */ optlen = mptcp_setup_join_ack_opts(tp, opt, optlen); if (!tp->t_mpuna) { @@ -525,12 +410,13 @@ mptcp_setup_opts(struct tcpcb *tp, int32_t off, u_char *opt, /* Start a timer to retransmit the ACK */ tp->t_timer[TCPT_JACK_RXMT] = OFFSET_FROM_START(tp, tcp_jack_rxmt); + + tp->t_mpflags &= ~TMPF_SND_JACK; goto ret_optlen; } if (!(tp->t_mpflags & TMPF_MPTCP_TRUE)) goto ret_optlen; -fastjoin_send: /* * From here on, all options are sent only if MPTCP_TRUE * or when data is sent early on as in Fast Join @@ -552,45 +438,39 @@ fastjoin_send: optlen = mptcp_snd_mpprio(tp, opt, optlen); } - MPT_LOCK(mp_tp); - if ((mp_tp->mpt_flags & MPTCPF_SND_64BITDSN) || force_64bit_dsn) { + if (mp_tp->mpt_flags & MPTCPF_SND_64BITDSN) { send_64bit_dsn = TRUE; } if (mp_tp->mpt_flags & MPTCPF_SND_64BITACK) send_64bit_ack = TRUE; - MPT_UNLOCK(mp_tp); - -#define CHECK_OPTLEN { \ - if ((MAX_TCPOPTLEN - optlen) < len) { \ - mptcplog((LOG_ERR, "MPTCP Socket: " \ - "%s: len %d optlen %d \n", __func__, len, optlen), \ - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); \ - goto ret_optlen; \ - } \ +#define CHECK_OPTLEN { \ + if ((MAX_TCPOPTLEN - optlen) < dssoptlen) { \ + mptcplog((LOG_ERR, "%s: dssoptlen %d optlen %d \n", __func__, \ + dssoptlen, optlen), \ + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); \ + goto ret_optlen; \ + } \ } #define DO_FIN(dsn_opt) { \ int sndfin = 0; \ - sndfin = mptcp_ok_to_fin(tp, dsn_opt.mdss_dsn, datalen); \ + sndfin = mptcp_ok_to_fin(tp, dsn_opt.mdss_dsn, len); \ if (sndfin) { \ dsn_opt.mdss_copt.mdss_flags |= MDSS_F; \ - *finp = opt + optlen + offsetof(struct mptcp_dss_copt, \ - mdss_flags); \ dsn_opt.mdss_data_len += 1; \ } \ } #define CHECK_DATALEN { \ /* MPTCP socket does not support IP options */ \ - if ((datalen + optlen + len) > tp->t_maxopd) { \ - mptcplog((LOG_ERR, "MPTCP Socket: " \ - "%s: nosp %d len %d opt %d %d %d\n", \ - __func__, datalen, len, optlen, \ + if ((len + optlen + dssoptlen) > tp->t_maxopd) { \ + mptcplog((LOG_ERR, "%s: nosp %d len %d opt %d %d %d\n", \ + __func__, len, dssoptlen, optlen, \ tp->t_maxseg, tp->t_maxopd), \ MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); \ /* remove option length from payload len */ \ - datalen = tp->t_maxopd - optlen - len; \ + len = tp->t_maxopd - optlen - dssoptlen; \ } \ } @@ -606,10 +486,11 @@ fastjoin_send: * XXX If this delay causes issue, remove the 2-byte padding. */ struct mptcp_dss64_ack32_opt dsn_ack_opt; - unsigned int len = sizeof (dsn_ack_opt); + unsigned int dssoptlen = sizeof (dsn_ack_opt); + uint16_t dss_csum; if (do_csum) { - len += 2; + dssoptlen += 2; } CHECK_OPTLEN; @@ -617,18 +498,17 @@ fastjoin_send: bzero(&dsn_ack_opt, sizeof (dsn_ack_opt)); dsn_ack_opt.mdss_copt.mdss_kind = TCPOPT_MULTIPATH; dsn_ack_opt.mdss_copt.mdss_subtype = MPO_DSS; - dsn_ack_opt.mdss_copt.mdss_len = len; + dsn_ack_opt.mdss_copt.mdss_len = dssoptlen; dsn_ack_opt.mdss_copt.mdss_flags |= MDSS_M | MDSS_m | MDSS_A; CHECK_DATALEN; - mptcp_output_getm_dsnmap64(so, off, (u_int32_t)datalen, - &dsn_ack_opt.mdss_dsn, - &dsn_ack_opt.mdss_subflow_seqn, - &dsn_ack_opt.mdss_data_len); - - *dss_valp = dsn_ack_opt.mdss_dsn; + mptcp_output_getm_dsnmap64(so, off, + &dsn_ack_opt.mdss_dsn, + &dsn_ack_opt.mdss_subflow_seqn, + &dsn_ack_opt.mdss_data_len, + &dss_csum); if ((dsn_ack_opt.mdss_data_len == 0) || (dsn_ack_opt.mdss_dsn == 0)) { @@ -639,30 +519,21 @@ fastjoin_send: DO_FIN(dsn_ack_opt); } - MPT_LOCK(mp_tp); dsn_ack_opt.mdss_ack = htonl(MPTCP_DATAACK_LOW32(mp_tp->mpt_rcvnxt)); - MPT_UNLOCK(mp_tp); dsn_ack_opt.mdss_dsn = mptcp_hton64(dsn_ack_opt.mdss_dsn); dsn_ack_opt.mdss_subflow_seqn = htonl( dsn_ack_opt.mdss_subflow_seqn); dsn_ack_opt.mdss_data_len = htons( dsn_ack_opt.mdss_data_len); - *dss_lenp = (unsigned int *)(void *)(opt + optlen + - offsetof(struct mptcp_dss64_ack32_opt, mdss_data_len)); memcpy(opt + optlen, &dsn_ack_opt, sizeof (dsn_ack_opt)); + if (do_csum) + *((uint16_t *)(void *)(opt + optlen + sizeof (dsn_ack_opt))) = dss_csum; - if (do_csum) { - *sseqp = (u_int32_t *)(void *)(opt + optlen + - offsetof(struct mptcp_dss64_ack32_opt, - mdss_subflow_seqn)); - } - optlen += len; - mptcplog((LOG_DEBUG,"MPTCP Socket: " - "%s: long DSS = %llx ACK = %llx \n", - __func__, + optlen += dssoptlen; + mptcplog((LOG_DEBUG,"%s: long DSS = %llx ACK = %llx \n", __func__, mptcp_ntoh64(dsn_ack_opt.mdss_dsn), mptcp_ntoh64(dsn_ack_opt.mdss_ack)), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); @@ -675,10 +546,11 @@ fastjoin_send: (!send_64bit_dsn) && !(tp->t_mpflags & TMPF_MPTCP_ACKNOW)) { struct mptcp_dsn_opt dsn_opt; - unsigned int len = sizeof (struct mptcp_dsn_opt); + unsigned int dssoptlen = sizeof (struct mptcp_dsn_opt); + uint16_t dss_csum; if (do_csum) { - len += 2; + dssoptlen += 2; } CHECK_OPTLEN; @@ -686,15 +558,15 @@ fastjoin_send: bzero(&dsn_opt, sizeof (dsn_opt)); dsn_opt.mdss_copt.mdss_kind = TCPOPT_MULTIPATH; dsn_opt.mdss_copt.mdss_subtype = MPO_DSS; - dsn_opt.mdss_copt.mdss_len = len; + dsn_opt.mdss_copt.mdss_len = dssoptlen; dsn_opt.mdss_copt.mdss_flags |= MDSS_M; CHECK_DATALEN; - mptcp_output_getm_dsnmap32(so, off, (u_int32_t)datalen, - &dsn_opt.mdss_dsn, - &dsn_opt.mdss_subflow_seqn, &dsn_opt.mdss_data_len, - dss_valp); + mptcp_output_getm_dsnmap32(so, off, &dsn_opt.mdss_dsn, + &dsn_opt.mdss_subflow_seqn, + &dsn_opt.mdss_data_len, + &dss_csum); if ((dsn_opt.mdss_data_len == 0) || (dsn_opt.mdss_dsn == 0)) { @@ -708,14 +580,11 @@ fastjoin_send: dsn_opt.mdss_dsn = htonl(dsn_opt.mdss_dsn); dsn_opt.mdss_subflow_seqn = htonl(dsn_opt.mdss_subflow_seqn); dsn_opt.mdss_data_len = htons(dsn_opt.mdss_data_len); - *dss_lenp = (unsigned int *)(void *)(opt + optlen + - offsetof(struct mptcp_dsn_opt, mdss_data_len)); memcpy(opt + optlen, &dsn_opt, sizeof (dsn_opt)); - if (do_csum) { - *sseqp = (u_int32_t *)(void *)(opt + optlen + - offsetof(struct mptcp_dsn_opt, mdss_subflow_seqn)); - } - optlen += len; + if (do_csum) + *((uint16_t *)(void *)(opt + optlen + sizeof (dsn_opt))) = dss_csum; + + optlen += dssoptlen; tp->t_mpflags &= ~TMPF_MPTCP_ACKNOW; goto ret_optlen; } @@ -727,23 +596,21 @@ fastjoin_send: !(tp->t_mpflags & TMPF_SEND_DFIN)) { struct mptcp_data_ack_opt dack_opt; - unsigned int len = 0; + unsigned int dssoptlen = 0; do_ack32_only: - len = sizeof (dack_opt); + dssoptlen = sizeof (dack_opt); CHECK_OPTLEN; - bzero(&dack_opt, len); + bzero(&dack_opt, dssoptlen); dack_opt.mdss_copt.mdss_kind = TCPOPT_MULTIPATH; - dack_opt.mdss_copt.mdss_len = len; + dack_opt.mdss_copt.mdss_len = dssoptlen; dack_opt.mdss_copt.mdss_subtype = MPO_DSS; dack_opt.mdss_copt.mdss_flags |= MDSS_A; - MPT_LOCK_SPIN(mp_tp); dack_opt.mdss_ack = htonl(MPTCP_DATAACK_LOW32(mp_tp->mpt_rcvnxt)); - MPT_UNLOCK(mp_tp); - memcpy(opt + optlen, &dack_opt, len); - optlen += len; + memcpy(opt + optlen, &dack_opt, dssoptlen); + optlen += dssoptlen; VERIFY(optlen <= MAX_TCPOPTLEN); tp->t_mpflags &= ~TMPF_MPTCP_ACKNOW; goto ret_optlen; @@ -755,27 +622,25 @@ do_ack32_only: !(tp->t_mpflags & TMPF_SEND_DSN) && !(tp->t_mpflags & TMPF_SEND_DFIN)) { struct mptcp_data_ack64_opt dack_opt; - unsigned int len = 0; + unsigned int dssoptlen = 0; do_ack64_only: - len = sizeof (dack_opt); + dssoptlen = sizeof (dack_opt); CHECK_OPTLEN; - bzero(&dack_opt, len); + bzero(&dack_opt, dssoptlen); dack_opt.mdss_copt.mdss_kind = TCPOPT_MULTIPATH; - dack_opt.mdss_copt.mdss_len = len; + dack_opt.mdss_copt.mdss_len = dssoptlen; dack_opt.mdss_copt.mdss_subtype = MPO_DSS; dack_opt.mdss_copt.mdss_flags |= (MDSS_A | MDSS_a); - MPT_LOCK_SPIN(mp_tp); dack_opt.mdss_ack = mptcp_hton64(mp_tp->mpt_rcvnxt); /* * The other end should retransmit 64-bit DSN until it * receives a 64-bit ACK. */ mp_tp->mpt_flags &= ~MPTCPF_SND_64BITACK; - MPT_UNLOCK(mp_tp); - memcpy(opt + optlen, &dack_opt, len); - optlen += len; + memcpy(opt + optlen, &dack_opt, dssoptlen); + optlen += dssoptlen; VERIFY(optlen <= MAX_TCPOPTLEN); tp->t_mpflags &= ~TMPF_MPTCP_ACKNOW; goto ret_optlen; @@ -787,30 +652,28 @@ do_ack64_only: (!send_64bit_ack) && (tp->t_mpflags & TMPF_MPTCP_ACKNOW)) { struct mptcp_dss_ack_opt dss_ack_opt; - unsigned int len = sizeof (dss_ack_opt); + unsigned int dssoptlen = sizeof (dss_ack_opt); + uint16_t dss_csum; if (do_csum) - len += 2; + dssoptlen += 2; CHECK_OPTLEN; bzero(&dss_ack_opt, sizeof (dss_ack_opt)); dss_ack_opt.mdss_copt.mdss_kind = TCPOPT_MULTIPATH; - dss_ack_opt.mdss_copt.mdss_len = len; + dss_ack_opt.mdss_copt.mdss_len = dssoptlen; dss_ack_opt.mdss_copt.mdss_subtype = MPO_DSS; dss_ack_opt.mdss_copt.mdss_flags |= MDSS_A | MDSS_M; - MPT_LOCK_SPIN(mp_tp); dss_ack_opt.mdss_ack = htonl(MPTCP_DATAACK_LOW32(mp_tp->mpt_rcvnxt)); - MPT_UNLOCK(mp_tp); CHECK_DATALEN; - mptcp_output_getm_dsnmap32(so, off, (u_int32_t)datalen, - &dss_ack_opt.mdss_dsn, - &dss_ack_opt.mdss_subflow_seqn, - &dss_ack_opt.mdss_data_len, - dss_valp); + mptcp_output_getm_dsnmap32(so, off, &dss_ack_opt.mdss_dsn, + &dss_ack_opt.mdss_subflow_seqn, + &dss_ack_opt.mdss_data_len, + &dss_csum); if ((dss_ack_opt.mdss_data_len == 0) || (dss_ack_opt.mdss_dsn == 0)) { @@ -825,16 +688,11 @@ do_ack64_only: dss_ack_opt.mdss_subflow_seqn = htonl(dss_ack_opt.mdss_subflow_seqn); dss_ack_opt.mdss_data_len = htons(dss_ack_opt.mdss_data_len); - *dss_lenp = (unsigned int *)(void *)(opt + optlen + - offsetof(struct mptcp_dss_ack_opt, mdss_data_len)); memcpy(opt + optlen, &dss_ack_opt, sizeof (dss_ack_opt)); - if (do_csum) { - *sseqp = (u_int32_t *)(void *)(opt + optlen + - offsetof(struct mptcp_dss_ack_opt, - mdss_subflow_seqn)); - } + if (do_csum) + *((uint16_t *)(void *)(opt + optlen + sizeof (dss_ack_opt))) = dss_csum; - optlen += len; + optlen += dssoptlen; if (optlen > MAX_TCPOPTLEN) panic("optlen too large"); @@ -848,28 +706,28 @@ do_ack64_only: (send_64bit_ack) && (tp->t_mpflags & TMPF_MPTCP_ACKNOW)) { struct mptcp_dss32_ack64_opt dss_ack_opt; - unsigned int len = sizeof (dss_ack_opt); + unsigned int dssoptlen = sizeof (dss_ack_opt); + uint16_t dss_csum; if (do_csum) - len += 2; + dssoptlen += 2; CHECK_OPTLEN; bzero(&dss_ack_opt, sizeof (dss_ack_opt)); dss_ack_opt.mdss_copt.mdss_kind = TCPOPT_MULTIPATH; - dss_ack_opt.mdss_copt.mdss_len = len; + dss_ack_opt.mdss_copt.mdss_len = dssoptlen; dss_ack_opt.mdss_copt.mdss_subtype = MPO_DSS; dss_ack_opt.mdss_copt.mdss_flags |= MDSS_M | MDSS_A | MDSS_a; - MPT_LOCK_SPIN(mp_tp); dss_ack_opt.mdss_ack = mptcp_hton64(mp_tp->mpt_rcvnxt); - MPT_UNLOCK(mp_tp); CHECK_DATALEN; - mptcp_output_getm_dsnmap32(so, off, (u_int32_t)datalen, - &dss_ack_opt.mdss_dsn, &dss_ack_opt.mdss_subflow_seqn, - &dss_ack_opt.mdss_data_len, dss_valp); + mptcp_output_getm_dsnmap32(so, off, &dss_ack_opt.mdss_dsn, + &dss_ack_opt.mdss_subflow_seqn, + &dss_ack_opt.mdss_data_len, + &dss_csum); if ((dss_ack_opt.mdss_data_len == 0) || (dss_ack_opt.mdss_dsn == 0)) { @@ -884,16 +742,11 @@ do_ack64_only: dss_ack_opt.mdss_subflow_seqn = htonl(dss_ack_opt.mdss_subflow_seqn); dss_ack_opt.mdss_data_len = htons(dss_ack_opt.mdss_data_len); - *dss_lenp = (unsigned int *)(void *)(opt + optlen + - offsetof(struct mptcp_dss32_ack64_opt, mdss_data_len)); memcpy(opt + optlen, &dss_ack_opt, sizeof (dss_ack_opt)); - if (do_csum) { - *sseqp = (u_int32_t *)(void *)(opt + optlen + - offsetof(struct mptcp_dss32_ack64_opt, - mdss_subflow_seqn)); - } + if (do_csum) + *((uint16_t *)(void *)(opt + optlen + sizeof (dss_ack_opt))) = dss_csum; - optlen += len; + optlen += dssoptlen; if (optlen > MAX_TCPOPTLEN) panic("optlen too large"); @@ -903,58 +756,48 @@ do_ack64_only: if (tp->t_mpflags & TMPF_SEND_DFIN) { struct mptcp_dss_ack_opt dss_ack_opt; - unsigned int len = sizeof (struct mptcp_dss_ack_opt); + unsigned int dssoptlen = sizeof (struct mptcp_dss_ack_opt); if (do_csum) - len += 2; + dssoptlen += 2; CHECK_OPTLEN; bzero(&dss_ack_opt, sizeof (dss_ack_opt)); - MPT_LOCK(mp_tp); /* * Data FIN occupies one sequence space. * Don't send it if it has been Acked. */ if (((mp_tp->mpt_sndnxt + 1) != mp_tp->mpt_sndmax) || - (mp_tp->mpt_snduna == mp_tp->mpt_sndmax)) { - MPT_UNLOCK(mp_tp); + (mp_tp->mpt_snduna == mp_tp->mpt_sndmax)) goto ret_optlen; - } dss_ack_opt.mdss_copt.mdss_kind = TCPOPT_MULTIPATH; - dss_ack_opt.mdss_copt.mdss_len = len; + dss_ack_opt.mdss_copt.mdss_len = dssoptlen; dss_ack_opt.mdss_copt.mdss_subtype = MPO_DSS; dss_ack_opt.mdss_copt.mdss_flags |= MDSS_A | MDSS_M | MDSS_F; dss_ack_opt.mdss_ack = htonl(MPTCP_DATAACK_LOW32(mp_tp->mpt_rcvnxt)); dss_ack_opt.mdss_dsn = htonl(MPTCP_DATASEQ_LOW32(mp_tp->mpt_sndnxt)); - MPT_UNLOCK(mp_tp); dss_ack_opt.mdss_subflow_seqn = 0; dss_ack_opt.mdss_data_len = 1; dss_ack_opt.mdss_data_len = htons(dss_ack_opt.mdss_data_len); memcpy(opt + optlen, &dss_ack_opt, sizeof (dss_ack_opt)); - if (do_csum) { - *dss_valp = mp_tp->mpt_sndnxt; - *sseqp = (u_int32_t *)(void *)(opt + optlen + - offsetof(struct mptcp_dss_ack_opt, - mdss_subflow_seqn)); - } - optlen += len; + optlen += dssoptlen; } ret_optlen: if (TRUE == *p_mptcp_acknow ) { VERIFY(old_mpt_flags != 0); - u_int32_t new_mpt_flags = tp->t_mpflags & - (TMPF_SND_MPPRIO | TMPF_SND_REM_ADDR | TMPF_SND_MPFAIL); + u_int32_t new_mpt_flags = tp->t_mpflags & TMPF_MPTCP_SIGNALS; /* * If none of the above mpflags were acted on by * this routine, reset these flags and set p_mptcp_acknow * to false. + * * XXX The reset value of p_mptcp_acknow can be used * to communicate tcp_output to NOT send a pure ack without any * MPTCP options as it will be treated as a dup ack. @@ -965,16 +808,14 @@ ret_optlen: * we haven't modified the logic in tcp_output to avoid * that. */ - if ((old_mpt_flags == new_mpt_flags) || (new_mpt_flags == 0)) { - tp->t_mpflags &= ~(TMPF_SND_MPPRIO - | TMPF_SND_REM_ADDR | TMPF_SND_MPFAIL); + if (old_mpt_flags == new_mpt_flags) { + tp->t_mpflags &= ~TMPF_MPTCP_SIGNALS; *p_mptcp_acknow = FALSE; - mptcplog((LOG_DEBUG, "MPTCP Sender: %s: no action \n", - __func__), MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); + mptcplog((LOG_DEBUG, "%s: no action \n", __func__), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); } else { - mptcplog((LOG_DEBUG, "MPTCP Sender: acknow set, " - "old flags %x new flags %x \n", - old_mpt_flags, new_mpt_flags), + mptcplog((LOG_DEBUG, "%s: acknow set, old flags %x new flags %x \n", + __func__, old_mpt_flags, new_mpt_flags), MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); } } @@ -993,8 +834,8 @@ mptcp_sanitize_option(struct tcpcb *tp, int mptcp_subtype) int ret = 1; if (mp_tp == NULL) { - mptcplog((LOG_ERR, "MPTCP Socket: %s: NULL mpsocket \n", - __func__), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + mptcplog((LOG_ERR, "%s: NULL mpsocket \n", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); return (0); } @@ -1013,8 +854,7 @@ mptcp_sanitize_option(struct tcpcb *tp, int mptcp_subtype) break; default: ret = 0; - mptcplog((LOG_ERR, "MPTCP Socket: " - "%s: type = %d \n", __func__, + mptcplog((LOG_ERR, "%s: type = %d \n", __func__, mptcp_subtype), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); break; @@ -1048,6 +888,8 @@ mptcp_do_mpcapable_opt(struct tcpcb *tp, u_char *cp, struct tcphdr *th, struct mptcp_mpcapable_opt_rsp *rsp = NULL; struct mptcb *mp_tp = tptomptp(tp); + mpte_lock_assert_held(mp_tp->mpt_mpte); + /* Only valid on SYN/ACK */ if ((th->th_flags & (TH_SYN | TH_ACK)) != (TH_SYN | TH_ACK)) return; @@ -1065,8 +907,7 @@ mptcp_do_mpcapable_opt(struct tcpcb *tp, u_char *cp, struct tcphdr *th, /* A SYN/ACK contains peer's key and flags */ if (optlen != sizeof (struct mptcp_mpcapable_opt_rsp)) { /* complain */ - mptcplog((LOG_ERR, "MPTCP Socket: " - "%s: SYN_ACK optlen = %d, sizeof mp opt = %lu \n", + mptcplog((LOG_ERR, "%s: SYN_ACK optlen = %d, sizeof mp opt = %lu \n", __func__, optlen, sizeof (struct mptcp_mpcapable_opt_rsp)), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); @@ -1083,7 +924,6 @@ mptcp_do_mpcapable_opt(struct tcpcb *tp, u_char *cp, struct tcphdr *th, mp_tp->mpt_flags |= MPTCPF_CHECKSUM; rsp = (struct mptcp_mpcapable_opt_rsp *)cp; - MPT_LOCK(mp_tp); mp_tp->mpt_remotekey = rsp->mmc_localkey; /* For now just downgrade to the peer's version */ mp_tp->mpt_peer_version = rsp->mmc_common.mmco_version; @@ -1093,12 +933,11 @@ mptcp_do_mpcapable_opt(struct tcpcb *tp, u_char *cp, struct tcphdr *th, } if (mptcp_init_remote_parms(mp_tp) != 0) { tcpstat.tcps_invalid_mpcap++; - MPT_UNLOCK(mp_tp); return; } - MPT_UNLOCK(mp_tp); tcp_heuristic_mptcp_success(tp); - tp->t_mpflags |= TMPF_PREESTABLISHED; + tp->t_mpflags |= (TMPF_SND_KEYS | TMPF_MPTCP_TRUE); + tp->t_inpcb->inp_socket->so_flags |= SOF_MPTCP_TRUE; } @@ -1122,9 +961,8 @@ mptcp_do_mpjoin_opt(struct tcpcb *tp, u_char *cp, struct tcphdr *th, int optlen) return; if (optlen != sizeof (struct mptcp_mpjoin_opt_rsp)) { - mptcplog((LOG_ERR, "MPTCP Socket: " - "SYN_ACK: unexpected optlen = %d mp " - "option = %lu\n", optlen, + mptcplog((LOG_ERR, "%s: SYN_ACK: unexpected optlen = %d mp " + "option = %lu\n", __func__, optlen, sizeof (struct mptcp_mpjoin_opt_rsp)), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); tp->t_mpflags &= ~TMPF_PREESTABLISHED; @@ -1138,77 +976,146 @@ mptcp_do_mpjoin_opt(struct tcpcb *tp, u_char *cp, struct tcphdr *th, int optlen) error = mptcp_validate_join_hmac(tp, (u_char*)&join_rsp->mmjo_mac, SHA1_TRUNCATED); if (error) { - mptcplog((LOG_ERR, "MPTCP Socket: %s: " - "SYN_ACK error = %d \n", __func__, error), + mptcplog((LOG_ERR, "%s: SYN_ACK error = %d \n", __func__, error), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); tp->t_mpflags &= ~TMPF_PREESTABLISHED; /* send RST and close */ MPTCP_JOPT_ERROR_PATH(tp); return; } - tp->t_mpflags |= TMPF_SENT_JOIN; + tp->t_mpflags |= (TMPF_SENT_JOIN | TMPF_SND_JACK); } static int mptcp_validate_join_hmac(struct tcpcb *tp, u_char* hmac, int mac_len) { u_char digest[SHA1_RESULTLEN] = {0}; - struct mptcb *mp_tp = NULL; - mptcp_key_t rem_key, loc_key; + struct mptcb *mp_tp = tptomptp(tp); u_int32_t rem_rand, loc_rand; - mp_tp = tp->t_mptcb; + mpte_lock_assert_held(mp_tp->mpt_mpte); rem_rand = loc_rand = 0; - MPT_LOCK(mp_tp); - rem_key = mp_tp->mpt_remotekey; - - /* - * Can happen if the MPTCP-connection is about to be closed and we - * receive an MP_JOIN in-between the events are being handled by the - * worker thread. - */ - if (mp_tp->mpt_localkey == NULL) { - MPT_UNLOCK(mp_tp); - return (-1); - } - - loc_key = *mp_tp->mpt_localkey; - MPT_UNLOCK(mp_tp); - mptcp_get_rands(tp->t_local_aid, mp_tp, &loc_rand, &rem_rand); if ((rem_rand == 0) || (loc_rand == 0)) return (-1); - mptcp_hmac_sha1(rem_key, loc_key, rem_rand, loc_rand, - digest, sizeof (digest)); + mptcp_hmac_sha1(mp_tp->mpt_remotekey, mp_tp->mpt_localkey, rem_rand, loc_rand, + digest); if (bcmp(digest, hmac, mac_len) == 0) return (0); /* matches */ else { printf("%s: remote key %llx local key %llx remote rand %x " - "local rand %x \n", __func__, rem_key, loc_key, + "local rand %x \n", __func__, mp_tp->mpt_remotekey, mp_tp->mpt_localkey, rem_rand, loc_rand); return (-1); } } +/* + * Update the mptcb send state variables, but the actual sbdrop occurs + * in MPTCP layer + */ +void +mptcp_data_ack_rcvd(struct mptcb *mp_tp, struct tcpcb *tp, u_int64_t full_dack) +{ + u_int64_t acked = 0; + + acked = full_dack - mp_tp->mpt_snduna; + + if (acked) { + struct socket *mp_so = mptetoso(mp_tp->mpt_mpte); + + if (acked > mp_so->so_snd.sb_cc) { + if (acked > mp_so->so_snd.sb_cc + 1 || + mp_tp->mpt_state < MPTCPS_FIN_WAIT_1) + mptcplog((LOG_ERR, "%s: acked %u, sb_cc %u full %u suna %u state %u\n", + __func__, (uint32_t)acked, mp_so->so_snd.sb_cc, + (uint32_t)full_dack, (uint32_t)mp_tp->mpt_snduna, + mp_tp->mpt_state), + MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_ERR); + + sbdrop(&mp_so->so_snd, (int)mp_so->so_snd.sb_cc); + } else { + sbdrop(&mp_so->so_snd, acked); + } + + mp_tp->mpt_snduna += acked; + /* In degraded mode, we may get some Data ACKs */ + if ((tp->t_mpflags & TMPF_TCP_FALLBACK) && + !(mp_tp->mpt_flags & MPTCPF_POST_FALLBACK_SYNC) && + MPTCP_SEQ_GT(mp_tp->mpt_sndnxt, mp_tp->mpt_snduna)) { + /* bring back sndnxt to retransmit MPTCP data */ + mp_tp->mpt_sndnxt = mp_tp->mpt_dsn_at_csum_fail; + mp_tp->mpt_flags |= MPTCPF_POST_FALLBACK_SYNC; + tp->t_inpcb->inp_socket->so_flags1 |= + SOF1_POST_FALLBACK_SYNC; + } + + mptcp_clean_reinjectq(mp_tp->mpt_mpte); + + sowwakeup(mp_so); + } + if (full_dack == mp_tp->mpt_sndmax && + mp_tp->mpt_state >= MPTCPS_FIN_WAIT_1) { + mptcp_close_fsm(mp_tp, MPCE_RECV_DATA_ACK); + tp->t_mpflags &= ~TMPF_SEND_DFIN; + } +} + +void +mptcp_update_window_fallback(struct tcpcb *tp) +{ + struct mptcb *mp_tp = tptomptp(tp); + + mpte_lock_assert_held(mp_tp->mpt_mpte); + + if (!(mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP)) + return; + + mptcplog((LOG_DEBUG, "%s: update window to %u\n", __func__, tp->snd_wnd), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + + mp_tp->mpt_sndwnd = tp->snd_wnd; + mp_tp->mpt_sndwl1 = mp_tp->mpt_rcvnxt; + mp_tp->mpt_sndwl2 = mp_tp->mpt_snduna; + + sowwakeup(tp->t_inpcb->inp_socket); +} + +static void +mptcp_update_window(struct mptcb *mp_tp, u_int64_t ack, u_int64_t seq, + u_int32_t tiwin) +{ + /* Don't look at the window if there is no ACK flag */ + if ((SEQ_LT(mp_tp->mpt_sndwl1, seq) || + (mp_tp->mpt_sndwl1 == seq && (SEQ_LT(mp_tp->mpt_sndwl2, ack) || + (mp_tp->mpt_sndwl2 == ack && tiwin > mp_tp->mpt_sndwnd))))) { + mp_tp->mpt_sndwnd = tiwin; + mp_tp->mpt_sndwl1 = seq; + mp_tp->mpt_sndwl2 = ack; + + mptcplog((LOG_DEBUG, "%s: Updating window to %u\n", __func__, + mp_tp->mpt_sndwnd), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE); + } +} + static void -mptcp_do_dss_opt_ack_meat(u_int64_t full_dack, struct tcpcb *tp) +mptcp_do_dss_opt_ack_meat(u_int64_t full_dack, u_int64_t full_dsn, + struct tcpcb *tp, u_int32_t tiwin) { struct mptcb *mp_tp = tptomptp(tp); int close_notify = 0; tp->t_mpflags |= TMPF_RCVD_DACK; - MPT_LOCK(mp_tp); if (MPTCP_SEQ_LEQ(full_dack, mp_tp->mpt_sndmax) && MPTCP_SEQ_GEQ(full_dack, mp_tp->mpt_snduna)) { mptcp_data_ack_rcvd(mp_tp, tp, full_dack); if (mp_tp->mpt_state > MPTCPS_FIN_WAIT_2) close_notify = 1; - MPT_UNLOCK(mp_tp); if (mp_tp->mpt_flags & MPTCPF_RCVD_64BITACK) { mp_tp->mpt_flags &= ~MPTCPF_RCVD_64BITACK; mp_tp->mpt_flags &= ~MPTCPF_SND_64BITDSN; @@ -1217,42 +1124,35 @@ mptcp_do_dss_opt_ack_meat(u_int64_t full_dack, struct tcpcb *tp) if (close_notify) mptcp_notify_close(tp->t_inpcb->inp_socket); } else { - MPT_UNLOCK(mp_tp); - mptcplog((LOG_ERR,"MPTCP Socket: " - "%s: unexpected dack %llx snduna %llx " - "sndmax %llx\n", __func__, full_dack, - mp_tp->mpt_snduna, mp_tp->mpt_sndmax), + mptcplog((LOG_ERR,"%s: unexpected dack %u snduna %u sndmax %u\n", __func__, + (u_int32_t)full_dack, (u_int32_t)mp_tp->mpt_snduna, + (u_int32_t)mp_tp->mpt_sndmax), (MPTCP_SOCKET_DBG|MPTCP_RECEIVER_DBG), MPTCP_LOGLVL_LOG); } + + mptcp_update_window(mp_tp, full_dack, full_dsn, tiwin); } static void -mptcp_do_dss_opt_meat(u_char *cp, struct tcpcb *tp) +mptcp_do_dss_opt_meat(u_char *cp, struct tcpcb *tp, struct tcphdr *th) { struct mptcp_dss_copt *dss_rsp = (struct mptcp_dss_copt *)cp; u_int64_t full_dack = 0; + u_int32_t tiwin = th->th_win << tp->snd_scale; struct mptcb *mp_tp = tptomptp(tp); int csum_len = 0; -#define MPTCP_DSS_OPT_SZ_CHK(len, expected_len) { \ - if (len != expected_len) { \ - mptcplog((LOG_ERR, "MPTCP Socket: " \ - "%s: bad len = %d dss: %x \n", __func__, \ - len, dss_rsp->mdss_flags), \ - (MPTCP_SOCKET_DBG|MPTCP_RECEIVER_DBG), \ - MPTCP_LOGLVL_LOG); \ - return; \ - } \ +#define MPTCP_DSS_OPT_SZ_CHK(len, expected_len) { \ + if (len != expected_len) { \ + mptcplog((LOG_ERR, "%s: bad len = %d dss: %x \n", __func__, \ + len, dss_rsp->mdss_flags), \ + (MPTCP_SOCKET_DBG|MPTCP_RECEIVER_DBG), \ + MPTCP_LOGLVL_LOG); \ + return; \ + } \ } - /* - * mp_tp might become NULL after the call to mptcp_do_fin_opt(). - * Review after rdar://problem/24083886 - */ - if (!mp_tp) - return; - if (mp_tp->mpt_flags & MPTCPF_CHECKSUM) csum_len = 2; @@ -1285,10 +1185,8 @@ mptcp_do_dss_opt_meat(u_char *cp, struct tcpcb *tp) u_int32_t dack = dack_opt->mdss_ack; NTOHL(dack); - MPT_LOCK_SPIN(mp_tp); MPTCP_EXTEND_DSN(mp_tp->mpt_snduna, dack, full_dack); - MPT_UNLOCK(mp_tp); - mptcp_do_dss_opt_ack_meat(full_dack, tp); + mptcp_do_dss_opt_ack_meat(full_dack, mp_tp->mpt_sndwl1, tp, tiwin); break; } case (MDSS_M | MDSS_A): @@ -1296,23 +1194,31 @@ mptcp_do_dss_opt_meat(u_char *cp, struct tcpcb *tp) /* 32-bit Data ACK + 32-bit DSS */ struct mptcp_dss_ack_opt *dss_ack_rsp; dss_ack_rsp = (struct mptcp_dss_ack_opt *)cp; + u_int64_t full_dsn; + uint16_t csum = 0; MPTCP_DSS_OPT_SZ_CHK(dss_ack_rsp->mdss_copt.mdss_len, sizeof (struct mptcp_dss_ack_opt) + csum_len); u_int32_t dack = dss_ack_rsp->mdss_ack; NTOHL(dack); - MPT_LOCK_SPIN(mp_tp); MPTCP_EXTEND_DSN(mp_tp->mpt_snduna, dack, full_dack); - MPT_UNLOCK(mp_tp); - mptcp_do_dss_opt_ack_meat(full_dack, tp); - if (csum_len == 0) - mptcp_update_rcv_state_f(dss_ack_rsp, tp, 0); - else - mptcp_update_rcv_state_f(dss_ack_rsp, tp, - *(uint16_t *)(void *)(cp + - (dss_ack_rsp->mdss_copt.mdss_len - - csum_len))); + + NTOHL(dss_ack_rsp->mdss_dsn); + NTOHL(dss_ack_rsp->mdss_subflow_seqn); + NTOHS(dss_ack_rsp->mdss_data_len); + MPTCP_EXTEND_DSN(mp_tp->mpt_rcvnxt, dss_ack_rsp->mdss_dsn, full_dsn); + + mptcp_do_dss_opt_ack_meat(full_dack, full_dsn, tp, tiwin); + + if (csum_len != 0) + csum = *(uint16_t *)(void *)(cp + (dss_ack_rsp->mdss_copt.mdss_len - csum_len)); + + mptcp_update_rcv_state_meat(mp_tp, tp, + full_dsn, + dss_ack_rsp->mdss_subflow_seqn, + dss_ack_rsp->mdss_data_len, + csum); break; } case (MDSS_M | MDSS_m): @@ -1321,33 +1227,24 @@ mptcp_do_dss_opt_meat(u_char *cp, struct tcpcb *tp) struct mptcp_dsn64_opt *dsn64; dsn64 = (struct mptcp_dsn64_opt *)cp; u_int64_t full_dsn; + uint16_t csum = 0; MPTCP_DSS_OPT_SZ_CHK(dsn64->mdss_copt.mdss_len, sizeof (struct mptcp_dsn64_opt) + csum_len); - mptcplog((LOG_DEBUG,"MPTCP Socket: " - "%s: 64-bit M present.\n", __func__), - (MPTCP_SOCKET_DBG|MPTCP_RECEIVER_DBG), - MPTCP_LOGLVL_LOG); - - MPT_LOCK_SPIN(mp_tp); mp_tp->mpt_flags |= MPTCPF_SND_64BITACK; - MPT_UNLOCK(mp_tp); full_dsn = mptcp_ntoh64(dsn64->mdss_dsn); NTOHL(dsn64->mdss_subflow_seqn); NTOHS(dsn64->mdss_data_len); - if (csum_len == 0) - mptcp_update_rcv_state_meat(mp_tp, tp, full_dsn, - dsn64->mdss_subflow_seqn, - dsn64->mdss_data_len, - 0); - else - mptcp_update_rcv_state_meat(mp_tp, tp, full_dsn, - dsn64->mdss_subflow_seqn, - dsn64->mdss_data_len, - *(uint16_t *)(void *)(cp + - dsn64->mdss_copt.mdss_len - csum_len)); + + if (csum_len != 0) + csum = *(uint16_t *)(void *)(cp + dsn64->mdss_copt.mdss_len - csum_len); + + mptcp_update_rcv_state_meat(mp_tp, tp, full_dsn, + dsn64->mdss_subflow_seqn, + dsn64->mdss_data_len, + csum); break; } case (MDSS_A | MDSS_a): @@ -1359,17 +1256,10 @@ mptcp_do_dss_opt_meat(u_char *cp, struct tcpcb *tp) MPTCP_DSS_OPT_SZ_CHK(dack64->mdss_copt.mdss_len, sizeof (struct mptcp_data_ack64_opt)); - mptcplog((LOG_DEBUG,"MPTCP Socket: " - "%s: 64-bit A present. \n", __func__), - (MPTCP_SOCKET_DBG|MPTCP_RECEIVER_DBG), - MPTCP_LOGLVL_LOG); - - MPT_LOCK_SPIN(mp_tp); mp_tp->mpt_flags |= MPTCPF_RCVD_64BITACK; - MPT_UNLOCK(mp_tp); full_dack = mptcp_ntoh64(dack64->mdss_ack); - mptcp_do_dss_opt_ack_meat(full_dack, tp); + mptcp_do_dss_opt_ack_meat(full_dack, mp_tp->mpt_sndwl1, tp, tiwin); break; } case (MDSS_M | MDSS_m | MDSS_A): @@ -1377,29 +1267,31 @@ mptcp_do_dss_opt_meat(u_char *cp, struct tcpcb *tp) /* 64-bit DSS + 32-bit Data ACK */ struct mptcp_dss64_ack32_opt *dss_ack_rsp; dss_ack_rsp = (struct mptcp_dss64_ack32_opt *)cp; + u_int64_t full_dsn; + uint16_t csum = 0; MPTCP_DSS_OPT_SZ_CHK(dss_ack_rsp->mdss_copt.mdss_len, sizeof (struct mptcp_dss64_ack32_opt) + csum_len); - mptcplog((LOG_DEBUG,"MPTCP Socket: " - "%s: 64-bit M and 32-bit A present.\n", __func__), - (MPTCP_SOCKET_DBG|MPTCP_RECEIVER_DBG), - MPTCP_LOGLVL_LOG); - u_int32_t dack = dss_ack_rsp->mdss_ack; NTOHL(dack); - MPT_LOCK_SPIN(mp_tp); mp_tp->mpt_flags |= MPTCPF_SND_64BITACK; MPTCP_EXTEND_DSN(mp_tp->mpt_snduna, dack, full_dack); - MPT_UNLOCK(mp_tp); - mptcp_do_dss_opt_ack_meat(full_dack, tp); - if (csum_len == 0) - mptcp_update_rcv_state_g(dss_ack_rsp, tp, 0); - else - mptcp_update_rcv_state_g(dss_ack_rsp, tp, - *(uint16_t *)(void *)(cp + - dss_ack_rsp->mdss_copt.mdss_len - - csum_len)); + + full_dsn = mptcp_ntoh64(dss_ack_rsp->mdss_dsn); + NTOHL(dss_ack_rsp->mdss_subflow_seqn); + NTOHS(dss_ack_rsp->mdss_data_len); + + mptcp_do_dss_opt_ack_meat(full_dack, full_dsn, tp, tiwin); + + if (csum_len != 0) + csum = *(uint16_t *)(void *)(cp + dss_ack_rsp->mdss_copt.mdss_len - csum_len); + + mptcp_update_rcv_state_meat(mp_tp, tp, full_dsn, + dss_ack_rsp->mdss_subflow_seqn, + dss_ack_rsp->mdss_data_len, + csum); + break; } case (MDSS_M | MDSS_A | MDSS_a): @@ -1413,21 +1305,15 @@ mptcp_do_dss_opt_meat(u_char *cp, struct tcpcb *tp) dss32_ack64_opt->mdss_copt.mdss_len, sizeof (struct mptcp_dss32_ack64_opt) + csum_len); - mptcplog((LOG_DEBUG,"MPTCP Socket: " - "%s: 32-bit M and 64-bit A present.\n", __func__), - (MPTCP_SOCKET_DBG|MPTCP_RECEIVER_DBG), - MPTCP_LOGLVL_LOG); - full_dack = mptcp_ntoh64(dss32_ack64_opt->mdss_ack); - mptcp_do_dss_opt_ack_meat(full_dack, tp); NTOHL(dss32_ack64_opt->mdss_dsn); - MPT_LOCK_SPIN(mp_tp); mp_tp->mpt_flags |= MPTCPF_RCVD_64BITACK; MPTCP_EXTEND_DSN(mp_tp->mpt_rcvnxt, dss32_ack64_opt->mdss_dsn, full_dsn); - MPT_UNLOCK(mp_tp); NTOHL(dss32_ack64_opt->mdss_subflow_seqn); NTOHS(dss32_ack64_opt->mdss_data_len); + + mptcp_do_dss_opt_ack_meat(full_dack, full_dsn, tp, tiwin); if (csum_len == 0) mptcp_update_rcv_state_meat(mp_tp, tp, full_dsn, dss32_ack64_opt->mdss_subflow_seqn, @@ -1451,18 +1337,11 @@ mptcp_do_dss_opt_meat(u_char *cp, struct tcpcb *tp) MPTCP_DSS_OPT_SZ_CHK(dss64_ack64->mdss_copt.mdss_len, sizeof (struct mptcp_dss64_ack64_opt) + csum_len); - mptcplog((LOG_DEBUG,"MPTCP Socket: " - "%s: 64-bit M and 64-bit A present.\n", __func__), - (MPTCP_SOCKET_DBG|MPTCP_RECEIVER_DBG), - MPTCP_LOGLVL_LOG); - - MPT_LOCK_SPIN(mp_tp); mp_tp->mpt_flags |= MPTCPF_RCVD_64BITACK; mp_tp->mpt_flags |= MPTCPF_SND_64BITACK; - MPT_UNLOCK(mp_tp); full_dsn = mptcp_ntoh64(dss64_ack64->mdss_dsn); full_dack = mptcp_ntoh64(dss64_ack64->mdss_dsn); - mptcp_do_dss_opt_ack_meat(full_dack, tp); + mptcp_do_dss_opt_ack_meat(full_dack, full_dsn, tp, tiwin); NTOHL(dss64_ack64->mdss_subflow_seqn); NTOHS(dss64_ack64->mdss_data_len); if (csum_len == 0) @@ -1479,30 +1358,22 @@ mptcp_do_dss_opt_meat(u_char *cp, struct tcpcb *tp) break; } default: - mptcplog((LOG_DEBUG,"MPTCP Socket: " - "%s: File bug, DSS flags = %x\n", __func__, - dss_rsp->mdss_flags), + mptcplog((LOG_DEBUG,"%s: File bug, DSS flags = %x\n", + __func__, dss_rsp->mdss_flags), (MPTCP_SOCKET_DBG|MPTCP_RECEIVER_DBG), MPTCP_LOGLVL_LOG); break; } } - static void mptcp_do_fin_opt(struct tcpcb *tp) { - struct mptcb *mp_tp = (struct mptcb *)tp->t_mptcb; - - mptcplog((LOG_DEBUG,"MPTCP Socket: %s \n", __func__), - (MPTCP_SOCKET_DBG|MPTCP_RECEIVER_DBG), - MPTCP_LOGLVL_LOG); + struct mptcb *mp_tp = tptomptp(tp); if (!(tp->t_mpflags & TMPF_RECV_DFIN)) { if (mp_tp != NULL) { - MPT_LOCK(mp_tp); mptcp_close_fsm(mp_tp, MPCE_RECV_DATA_FIN); - MPT_UNLOCK(mp_tp); if (tp->t_inpcb->inp_socket != NULL) { soevent(tp->t_inpcb->inp_socket, @@ -1525,8 +1396,8 @@ mptcp_do_fin_opt(struct tcpcb *tp) static void mptcp_do_dss_opt(struct tcpcb *tp, u_char *cp, struct tcphdr *th, int optlen) { -#pragma unused(th, optlen) - struct mptcb *mp_tp = (struct mptcb *)tp->t_mptcb; +#pragma unused(optlen) + struct mptcb *mp_tp = tptomptp(tp); if (!mp_tp) return; @@ -1541,7 +1412,7 @@ mptcp_do_dss_opt(struct tcpcb *tp, u_char *cp, struct tcphdr *th, int optlen) mptcp_do_fin_opt(tp); } - mptcp_do_dss_opt_meat(cp, tp); + mptcp_do_dss_opt_meat(cp, tp, th); } } } @@ -1555,20 +1426,16 @@ mptcp_do_fastclose_opt(struct tcpcb *tp, u_char *cp, struct tcphdr *th) if (th->th_flags != TH_ACK) return; - mptcplog((LOG_DEBUG,"MPTCP Socket: %s: \n", __func__), - (MPTCP_SOCKET_DBG|MPTCP_RECEIVER_DBG), - MPTCP_LOGLVL_LOG); - if (fc_opt->mfast_len != sizeof (struct mptcp_fastclose_opt)) { tcpstat.tcps_invalid_opt++; return; } - mp_tp = (struct mptcb *)tp->t_mptcb; + mp_tp = tptomptp(tp); if (!mp_tp) return; - if (fc_opt->mfast_key != mptcp_get_localkey(mp_tp)) { + if (fc_opt->mfast_key != mp_tp->mpt_localkey) { tcpstat.tcps_invalid_opt++; return; } @@ -1613,18 +1480,14 @@ mptcp_do_mpfail_opt(struct tcpcb *tp, u_char *cp, struct tcphdr *th) if ((th->th_flags != TH_ACK) && (th->th_flags != TH_RST)) return; - mptcplog((LOG_DEBUG, "MPTCP Socket: %s: \n", __func__), - (MPTCP_SOCKET_DBG|MPTCP_RECEIVER_DBG), MPTCP_LOGLVL_LOG); - if (fail_opt->mfail_len != sizeof (struct mptcp_mpfail_opt)) return; - mp_tp = (struct mptcb *)tp->t_mptcb; - MPT_LOCK(mp_tp); + mp_tp = tptomptp(tp); + mp_tp->mpt_flags |= MPTCPF_RECVD_MPFAIL; mp_tp->mpt_dsn_at_csum_fail = mptcp_hton64(fail_opt->mfail_dsn); - MPT_UNLOCK(mp_tp); - error = mptcp_get_map_for_dsn(tp->t_inpcb->inp_socket, + error = mptcp_get_map_for_dsn(tp->t_inpcb->inp_socket, mp_tp->mpt_dsn_at_csum_fail, &mdss_subflow_seqn); if (error == 0) { mp_tp->mpt_ssn_at_csum_fail = mdss_subflow_seqn; @@ -1638,6 +1501,12 @@ tcp_do_mptcp_options(struct tcpcb *tp, u_char *cp, struct tcphdr *th, struct tcpopt *to, int optlen) { int mptcp_subtype; + struct mptcb *mp_tp = tptomptp(tp); + + if (mp_tp == NULL) + return; + + mpte_lock_assert_held(mp_tp->mpt_mpte); /* All MPTCP options have atleast 4 bytes */ if (optlen < 4) @@ -1675,49 +1544,11 @@ tcp_do_mptcp_options(struct tcpcb *tp, u_char *cp, struct tcphdr *th, return; } -/* - * MPTCP ADD_ADDR and REMOVE_ADDR options - */ - -/* - * ADD_ADDR is only placeholder code - not sent on wire - * The ADD_ADDR option is not sent on wire because of security issues - * around connection hijacking. - */ -void -mptcp_send_addaddr_opt(struct tcpcb *tp, struct mptcp_addaddr_opt *opt) -{ - - opt->ma_kind = TCPOPT_MULTIPATH; - opt->ma_len = sizeof (struct mptcp_addaddr_opt); - opt->ma_subtype = MPO_ADD_ADDR; - opt->ma_addr_id = tp->t_local_aid; -#ifdef MPTCP_NOTYET - struct inpcb *inp = tp->t_inpcb; - if (inp->inp_vflag == AF_INET) { - opt->ma_ipver = MA_IPVer_V4; - bcopy((char *)&sin->sin_addr.s_addr, (char *)opt + opt->ma_len, - sizeof (in_addr_t)); - opt->ma_len += sizeof (in_addr_t); - } else if (inp->inp_vflag == AF_INET6) { - opt->ma_ipver = MA_IPVer_V6; - bcopy((char *)&sin6->sin6_addr, (char *)opt + opt->ma_len, - sizeof (struct in6_addr)); - opt->ma_len += sizeof (struct in6_addr); - } -#if 0 - if (tp->t_mp_port) { - /* add ports XXX */ - } -#endif -#endif -} - /* REMOVE_ADDR option is sent when a source address goes away */ -void +static void mptcp_send_remaddr_opt(struct tcpcb *tp, struct mptcp_remaddr_opt *opt) { - mptcplog((LOG_DEBUG,"MPTCP Socket: %s: local id %d remove id %d \n", + mptcplog((LOG_DEBUG,"%s: local id %d remove id %d \n", __func__, tp->t_local_aid, tp->t_rem_aid), (MPTCP_SOCKET_DBG|MPTCP_SENDER_DBG), MPTCP_LOGLVL_LOG); @@ -1729,33 +1560,6 @@ mptcp_send_remaddr_opt(struct tcpcb *tp, struct mptcp_remaddr_opt *opt) tp->t_mpflags &= ~TMPF_SND_REM_ADDR; } -/* - * MPTCP MP_PRIO option - */ - -#if 0 -/* - * Current implementation drops incoming MP_PRIO option and this code is - * just a placeholder. The option is dropped because only the mobile client can - * decide which of the subflows is preferred (usually wifi is preferred - * over Cellular). - */ -void -mptcp_do_mpprio_opt(struct tcpcb *tp, u_char *cp, struct tcphdr *th, - int optlen) -{ - int bkp = 0; - struct mptcp_mpprio_opt *mpprio = (struct mptcp_mpprio_opt *)cp; - - if ((tp == NULL) || !(tp->t_mpflags & TMPF_MPTCP_TRUE)) - return; - - if ((mpprio->mpprio_len != sizeof (struct mptcp_mpprio_addr_opt)) && - (mpprio->mpprio_len != sizeof (struct mptcp_mpprio_opt))) - return; -} -#endif - /* We send MP_PRIO option based on the values set by the SIOCSCONNORDER ioctl */ static int mptcp_snd_mpprio(struct tcpcb *tp, u_char *cp, int optlen) @@ -1767,11 +1571,6 @@ mptcp_snd_mpprio(struct tcpcb *tp, u_char *cp, int optlen) return (optlen); } - if (mptcp_mpprio_enable != 1) { - tp->t_mpflags &= ~TMPF_SND_MPPRIO; - return (optlen); - } - if ((MAX_TCPOPTLEN - optlen) < (int)sizeof (mpprio)) return (optlen); @@ -1786,8 +1585,9 @@ mptcp_snd_mpprio(struct tcpcb *tp, u_char *cp, int optlen) memcpy(cp + optlen, &mpprio, sizeof (mpprio)); optlen += sizeof (mpprio); tp->t_mpflags &= ~TMPF_SND_MPPRIO; - mptcplog((LOG_DEBUG, "MPTCP Socket: %s: aid = %d \n", __func__, + mptcplog((LOG_DEBUG, "%s: aid = %d \n", __func__, tp->t_local_aid), (MPTCP_SOCKET_DBG|MPTCP_SENDER_DBG), MPTCP_LOGLVL_LOG); return (optlen); } + diff --git a/bsd/netinet/mptcp_opt.h b/bsd/netinet/mptcp_opt.h index a9450dee9..785e1a998 100644 --- a/bsd/netinet/mptcp_opt.h +++ b/bsd/netinet/mptcp_opt.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2013 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -41,25 +41,19 @@ #define MPTCP_CAPABLE_RETRIES (2) __BEGIN_DECLS +extern void mptcp_data_ack_rcvd(struct mptcb *mp_tp, struct tcpcb *tp, u_int64_t full_dack); +extern void mptcp_update_window_fallback(struct tcpcb *tp); extern void tcp_do_mptcp_options(struct tcpcb *, u_char *, struct tcphdr *, struct tcpopt *, int); -extern unsigned mptcp_setup_syn_opts(struct socket *, int, u_char*, unsigned); +extern unsigned mptcp_setup_syn_opts(struct socket *, u_char*, unsigned); extern unsigned mptcp_setup_join_ack_opts(struct tcpcb *, u_char*, unsigned); -extern void mptcp_update_dss_send_state(struct mptcb *, u_int64_t); -extern void mptcp_send_addaddr_opt(struct tcpcb *, struct mptcp_addaddr_opt *); -extern void mptcp_send_remaddr_opt(struct tcpcb *, struct mptcp_remaddr_opt *); -extern unsigned int mptcp_setup_opts(struct tcpcb *, int, u_char *, - unsigned int, int, int, unsigned int **, u_int8_t **, u_int64_t *, - u_int32_t **, boolean_t *); +extern unsigned int mptcp_setup_opts(struct tcpcb *tp, int32_t off, u_char *opt, + unsigned int optlen, int flags, int len, + boolean_t *p_mptcp_acknow); extern void mptcp_update_dss_rcv_state(struct mptcp_dsn_opt *, struct tcpcb *, uint16_t); -extern void mptcp_update_rcv_state_f(struct mptcp_dss_ack_opt *, - struct tcpcb *, uint16_t); -extern void mptcp_update_rcv_state_g(struct mptcp_dss64_ack32_opt *, - struct tcpcb *, uint16_t); extern void mptcp_update_rcv_state_meat(struct mptcb *, struct tcpcb *, u_int64_t, u_int32_t, u_int16_t, uint16_t); -extern void mptcp_data_ack_rcvd(struct mptcb *, struct tcpcb *, u_int64_t); __END_DECLS #endif /* BSD_KERNEL_PRIVATE */ diff --git a/bsd/netinet/mptcp_seq.h b/bsd/netinet/mptcp_seq.h index a444baa08..c79232ee6 100644 --- a/bsd/netinet/mptcp_seq.h +++ b/bsd/netinet/mptcp_seq.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * diff --git a/bsd/netinet/mptcp_subr.c b/bsd/netinet/mptcp_subr.c index 90ff36152..19e128687 100644 --- a/bsd/netinet/mptcp_subr.c +++ b/bsd/netinet/mptcp_subr.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -26,26 +26,29 @@ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ -#include <sys/param.h> -#include <sys/proc.h> -#include <sys/systm.h> +#include <kern/locks.h> +#include <kern/policy_internal.h> +#include <kern/zalloc.h> + +#include <mach/sdt.h> + +#include <sys/domain.h> +#include <sys/kdebug.h> +#include <sys/kern_control.h> #include <sys/kernel.h> #include <sys/mbuf.h> #include <sys/mcache.h> +#include <sys/param.h> +#include <sys/proc.h> +#include <sys/protosw.h> #include <sys/resourcevar.h> #include <sys/socket.h> #include <sys/socketvar.h> -#include <sys/syslog.h> -#include <sys/domain.h> -#include <sys/protosw.h> #include <sys/sysctl.h> +#include <sys/syslog.h> +#include <sys/systm.h> -#include <kern/zalloc.h> -#include <kern/locks.h> - -#include <mach/thread_act.h> -#include <mach/sdt.h> - +#include <net/content_filter.h> #include <net/if.h> #include <net/if_var.h> #include <netinet/in.h> @@ -57,6 +60,7 @@ #include <netinet/tcp_var.h> #include <netinet/mptcp_var.h> #include <netinet/mptcp.h> +#include <netinet/mptcp_opt.h> #include <netinet/mptcp_seq.h> #include <netinet/mptcp_timer.h> #include <libkern/crypto/sha1.h> @@ -66,8 +70,6 @@ #endif /* INET6 */ #include <dev/random/randomdev.h> -extern char *proc_best_name(proc_t); - /* * Notes on MPTCP implementation. * @@ -86,67 +88,45 @@ extern char *proc_best_name(proc_t); * PCB (mppcb) as well as the MPTCP Session (mptses). * * The MPTCP Session is an MPTCP-specific extension to the Multipath PCB; - * in particular, the list of subflows as well as the MPTCP thread. * * A functioning MPTCP Session consists of one or more subflow sockets. Each * subflow socket is essentially a regular PF_INET/PF_INET6 TCP socket, and is * represented by the mptsub structure. Because each subflow requires access * to the MPTCP Session, the MPTCP socket's so_usecount is bumped up for each - * subflow. This gets decremented prior to the subflow's destruction. The - * subflow lock (mpts_lock) is used to protect accesses to the subflow. - * - * To handle events (read, write, control) from the subflows, an MPTCP thread - * is created; currently, there is one thread per MPTCP Session. In order to - * prevent the MPTCP socket from being destroyed while being accessed by the - * MPTCP thread, we bump up the MPTCP socket's so_usecount for the thread, - * which will be decremented prior to the thread's termination. The thread - * lock (mpte_thread_lock) is used to synchronize its signalling. + * subflow. This gets decremented prior to the subflow's destruction. * - * Lock ordering is defined as follows: + * To handle events (read, write, control) from the subflows, we do direct + * upcalls into the specific function. * - * mtcbinfo (mppi_lock) - * mp_so (mpp_lock) - * mpts (mpts_lock) - * so (inpcb_mtx) - * mptcb (mpt_lock) - * - * It is not a requirement that all of the above locks need to be acquired - * in succession, but the correct lock ordering must be followed when there - * are more than one locks that need to be held. The MPTCP thread lock is - * is not constrained by this arrangement, because none of the other locks - * is ever acquired while holding mpte_thread_lock; therefore it may be called - * at any moment to signal the thread. + * The whole MPTCP connection is protected by a single lock, the MPTCP socket's + * lock. Incoming data on a subflow also ends up taking this single lock. To + * achieve the latter, tcp_lock/unlock has been changed to rather use the lock + * of the MPTCP-socket. * * An MPTCP socket will be destroyed when its so_usecount drops to zero; this * work is done by the MPTCP garbage collector which is invoked on demand by * the PF_MULTIPATH garbage collector. This process will take place once all - * of the subflows have been destroyed, and the MPTCP thread be instructed to - * self-terminate. + * of the subflows have been destroyed. */ -static void mptcp_sesdestroy(struct mptses *); -static void mptcp_thread_signal_locked(struct mptses *); -static void mptcp_thread_terminate_signal(struct mptses *); -static void mptcp_thread_dowork(struct mptses *); -static void mptcp_thread_func(void *, wait_result_t); -static void mptcp_thread_destroy(struct mptses *); -static void mptcp_key_pool_init(void); static void mptcp_attach_to_subf(struct socket *, struct mptcb *, uint8_t); static void mptcp_detach_mptcb_from_subf(struct mptcb *, struct socket *); static uint32_t mptcp_gc(struct mppcbinfo *); -static int mptcp_subflow_soclose(struct mptsub *, struct socket *); -static int mptcp_subflow_soconnectx(struct mptses *, struct mptsub *); static int mptcp_subflow_soreceive(struct socket *, struct sockaddr **, struct uio *, struct mbuf **, struct mbuf **, int *); +static int mptcp_subflow_sosend(struct socket *, struct sockaddr *, + struct uio *, struct mbuf *, struct mbuf *, int); static void mptcp_subflow_rupcall(struct socket *, void *, int); static void mptcp_subflow_input(struct mptses *, struct mptsub *); static void mptcp_subflow_wupcall(struct socket *, void *, int); -static void mptcp_subflow_eupcall(struct socket *, void *, uint32_t); -static void mptcp_update_last_owner(struct mptsub *, struct socket *); -static void mptcp_output_needed(struct mptses *mpte, struct mptsub *to_mpts); -static void mptcp_get_rtt_measurement(struct mptsub *, struct mptses *); -static void mptcp_drop_tfo_data(struct mptses *, struct mptsub *, int *); +static void mptcp_subflow_eupcall1(struct socket *, void *, uint32_t); +static void mptcp_update_last_owner(struct socket *so, struct socket *mp_so); +static void mptcp_drop_tfo_data(struct mptses *, struct mptsub *); + +static void mptcp_subflow_abort(struct mptsub *, int); + +static void mptcp_send_dfin(struct socket *so); /* * Possible return values for subflow event handlers. Note that success @@ -163,28 +143,22 @@ typedef enum { } ev_ret_t; static ev_ret_t mptcp_subflow_events(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_subflow_connreset_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_subflow_cantrcvmore_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_subflow_cantsendmore_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_subflow_timeout_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_subflow_nosrcaddr_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_subflow_failover_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_subflow_ifdenied_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_subflow_suspend_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_subflow_resume_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_subflow_connected_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_subflow_disconnected_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_subflow_mpstatus_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_subflow_mustrst_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_fastjoin_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_deleteok_ev(struct mptses *, struct mptsub *, uint64_t *); -static ev_ret_t mptcp_subflow_mpcantrcvmore_ev(struct mptses *, struct mptsub *, uint64_t *); +static ev_ret_t mptcp_subflow_propagate_ev(struct mptses *, struct mptsub *, uint64_t *, uint64_t); +static ev_ret_t mptcp_subflow_nosrcaddr_ev(struct mptses *, struct mptsub *, uint64_t *, uint64_t); +static ev_ret_t mptcp_subflow_failover_ev(struct mptses *, struct mptsub *, uint64_t *, uint64_t); +static ev_ret_t mptcp_subflow_ifdenied_ev(struct mptses *, struct mptsub *, uint64_t *, uint64_t); +static ev_ret_t mptcp_subflow_connected_ev(struct mptses *, struct mptsub *, uint64_t *, uint64_t); +static ev_ret_t mptcp_subflow_disconnected_ev(struct mptses *, struct mptsub *, uint64_t *, uint64_t); +static ev_ret_t mptcp_subflow_mpstatus_ev(struct mptses *, struct mptsub *, uint64_t *, uint64_t); +static ev_ret_t mptcp_subflow_mustrst_ev(struct mptses *, struct mptsub *, uint64_t *, uint64_t); +static ev_ret_t mptcp_subflow_mpcantrcvmore_ev(struct mptses *, struct mptsub *, uint64_t *, uint64_t); +static ev_ret_t mptcp_subflow_adaptive_rtimo_ev(struct mptses *, struct mptsub *, uint64_t *, uint64_t); +static ev_ret_t mptcp_subflow_adaptive_wtimo_ev(struct mptses *, struct mptsub *, uint64_t *, uint64_t); static const char *mptcp_evret2str(ev_ret_t); -static mptcp_key_t *mptcp_reserve_key(void); -static int mptcp_do_sha1(mptcp_key_t *, char *, int); -static void mptcp_init_local_parms(struct mptcb *); +static void mptcp_do_sha1(mptcp_key_t *, char *); +static void mptcp_init_local_parms(struct mptses *); static unsigned int mptsub_zone_size; /* size of mptsub */ static struct zone *mptsub_zone; /* zone for mptsub */ @@ -197,8 +171,6 @@ static struct zone *mpt_subauth_zone; /* zone of subf auth entry */ struct mppcbinfo mtcbinfo; -static struct mptcp_keys_pool_head mptcp_keys_pool; - #define MPTCP_SUBFLOW_WRITELEN (8 * 1024) /* bytes to write each time */ #define MPTCP_SUBFLOW_READLEN (8 * 1024) /* bytes to read each time */ @@ -206,40 +178,17 @@ SYSCTL_DECL(_net_inet); SYSCTL_NODE(_net_inet, OID_AUTO, mptcp, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "MPTCP"); -uint32_t mptcp_dbg_area = 0; /* more noise if greater than 1 */ +uint32_t mptcp_dbg_area = 31; /* more noise if greater than 1 */ SYSCTL_UINT(_net_inet_mptcp, OID_AUTO, dbg_area, CTLFLAG_RW|CTLFLAG_LOCKED, &mptcp_dbg_area, 0, "MPTCP debug area"); -uint32_t mptcp_dbg_level = 0; +uint32_t mptcp_dbg_level = 1; SYSCTL_INT(_net_inet_mptcp, OID_AUTO, dbg_level, CTLFLAG_RW | CTLFLAG_LOCKED, &mptcp_dbg_level, 0, "MPTCP debug level"); - SYSCTL_UINT(_net_inet_mptcp, OID_AUTO, pcbcount, CTLFLAG_RD|CTLFLAG_LOCKED, &mtcbinfo.mppi_count, 0, "Number of active PCBs"); -/* - * Since there is one kernel thread per mptcp socket, imposing an artificial - * limit on number of allowed mptcp sockets. - */ -uint32_t mptcp_socket_limit = MPPCB_LIMIT; -SYSCTL_UINT(_net_inet_mptcp, OID_AUTO, sk_lim, CTLFLAG_RW|CTLFLAG_LOCKED, - &mptcp_socket_limit, 0, "MPTCP socket limit"); - -/* - * SYSCTL to turn on delayed cellular subflow start. - */ -uint32_t mptcp_delayed_subf_start = 0; -SYSCTL_UINT(_net_inet_mptcp, OID_AUTO, delayed, CTLFLAG_RW|CTLFLAG_LOCKED, - &mptcp_delayed_subf_start, 0, "MPTCP Delayed Subflow start"); - -/* - * sysctl to use network status hints from symptomsd - */ -uint32_t mptcp_use_symptomsd = 1; -SYSCTL_UINT(_net_inet_mptcp, OID_AUTO, usesymptoms, CTLFLAG_RW|CTLFLAG_LOCKED, - &mptcp_use_symptomsd, 0, "MPTCP Use SymptomsD"); - static struct protosw mptcp_subflow_protosw; static struct pr_usrreqs mptcp_subflow_usrreqs; #if INET6 @@ -247,26 +196,26 @@ static struct ip6protosw mptcp_subflow_protosw6; static struct pr_usrreqs mptcp_subflow_usrreqs6; #endif /* INET6 */ +static uint8_t mptcp_create_subflows_scheduled; + typedef struct mptcp_subflow_event_entry { uint64_t sofilt_hint_mask; ev_ret_t (*sofilt_hint_ev_hdlr)( struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint); + uint64_t *p_mpsofilt_hint, + uint64_t event); } mptsub_ev_entry_t; +static uint8_t mptcp_cellicon_is_set; +static uint32_t mptcp_last_cellicon_set; +#define MPTCP_CELLICON_TOGGLE_RATE (5 * TCP_RETRANSHZ) /* Only toggle every 5 seconds */ + /* * XXX The order of the event handlers below is really - * really important. - * SO_FILT_HINT_DELETEOK event has to be handled first, - * else we may end up missing on this event. - * Please read radar://24043716 for more details. + * really important. Think twice before changing it. */ static mptsub_ev_entry_t mpsub_ev_entry_tbl [] = { - { - .sofilt_hint_mask = SO_FILT_HINT_DELETEOK, - .sofilt_hint_ev_hdlr = mptcp_deleteok_ev, - }, { .sofilt_hint_mask = SO_FILT_HINT_MPCANTRCVMORE, .sofilt_hint_ev_hdlr = mptcp_subflow_mpcantrcvmore_ev, @@ -277,7 +226,7 @@ static mptsub_ev_entry_t mpsub_ev_entry_tbl [] = { }, { .sofilt_hint_mask = SO_FILT_HINT_CONNRESET, - .sofilt_hint_ev_hdlr = mptcp_subflow_connreset_ev, + .sofilt_hint_ev_hdlr = mptcp_subflow_propagate_ev, }, { .sofilt_hint_mask = SO_FILT_HINT_MUSTRST, @@ -285,14 +234,11 @@ static mptsub_ev_entry_t mpsub_ev_entry_tbl [] = { }, { .sofilt_hint_mask = SO_FILT_HINT_CANTRCVMORE, - .sofilt_hint_ev_hdlr = mptcp_subflow_cantrcvmore_ev, - }, - { .sofilt_hint_mask = SO_FILT_HINT_CANTSENDMORE, - .sofilt_hint_ev_hdlr = mptcp_subflow_cantsendmore_ev, + .sofilt_hint_ev_hdlr = mptcp_subflow_propagate_ev, }, { .sofilt_hint_mask = SO_FILT_HINT_TIMEOUT, - .sofilt_hint_ev_hdlr = mptcp_subflow_timeout_ev, + .sofilt_hint_ev_hdlr = mptcp_subflow_propagate_ev, }, { .sofilt_hint_mask = SO_FILT_HINT_NOSRCADDR, @@ -302,14 +248,6 @@ static mptsub_ev_entry_t mpsub_ev_entry_tbl [] = { .sofilt_hint_mask = SO_FILT_HINT_IFDENIED, .sofilt_hint_ev_hdlr = mptcp_subflow_ifdenied_ev, }, - { - .sofilt_hint_mask = SO_FILT_HINT_SUSPEND, - .sofilt_hint_ev_hdlr = mptcp_subflow_suspend_ev, - }, - { - .sofilt_hint_mask = SO_FILT_HINT_RESUME, - .sofilt_hint_ev_hdlr = mptcp_subflow_resume_ev, - }, { .sofilt_hint_mask = SO_FILT_HINT_CONNECTED, .sofilt_hint_ev_hdlr = mptcp_subflow_connected_ev, @@ -323,9 +261,13 @@ static mptsub_ev_entry_t mpsub_ev_entry_tbl [] = { .sofilt_hint_ev_hdlr = mptcp_subflow_disconnected_ev, }, { - .sofilt_hint_mask = SO_FILT_HINT_MPFASTJ, - .sofilt_hint_ev_hdlr = mptcp_fastjoin_ev, - } + .sofilt_hint_mask = SO_FILT_HINT_ADAPTIVE_RTIMO, + .sofilt_hint_ev_hdlr = mptcp_subflow_adaptive_rtimo_ev, + }, + { + .sofilt_hint_mask = SO_FILT_HINT_ADAPTIVE_WTIMO, + .sofilt_hint_ev_hdlr = mptcp_subflow_adaptive_wtimo_ev, + }, }; /* @@ -361,6 +303,7 @@ mptcp_init(struct protosw *pp, struct domain *dp) mptcp_subflow_protosw.pr_entry.tqe_prev = NULL; mptcp_subflow_protosw.pr_usrreqs = &mptcp_subflow_usrreqs; mptcp_subflow_usrreqs.pru_soreceive = mptcp_subflow_soreceive; + mptcp_subflow_usrreqs.pru_sosend = mptcp_subflow_sosend; mptcp_subflow_usrreqs.pru_rcvoob = pru_rcvoob_notsupp; /* * Socket filters shouldn't attach/detach to/from this protosw @@ -383,6 +326,7 @@ mptcp_init(struct protosw *pp, struct domain *dp) mptcp_subflow_protosw6.pr_entry.tqe_prev = NULL; mptcp_subflow_protosw6.pr_usrreqs = &mptcp_subflow_usrreqs6; mptcp_subflow_usrreqs6.pru_soreceive = mptcp_subflow_soreceive; + mptcp_subflow_usrreqs6.pru_sosend = mptcp_subflow_sosend; mptcp_subflow_usrreqs6.pru_rcvoob = pru_rcvoob_notsupp; /* * Socket filters shouldn't attach/detach to/from this protosw @@ -415,7 +359,6 @@ mptcp_init(struct protosw *pp, struct domain *dp) mtcbinfo.mppi_gc = mptcp_gc; mtcbinfo.mppi_timer = mptcp_timer; - mtcbinfo.mppi_pcbe_create = mptcp_sescreate; /* attach to MP domain for garbage collection to take place */ mp_pcbinfo_attach(&mtcbinfo); @@ -448,20 +391,82 @@ mptcp_init(struct protosw *pp, struct domain *dp) zone_change(mpt_subauth_zone, Z_CALLERACCT, FALSE); zone_change(mpt_subauth_zone, Z_EXPAND, TRUE); - /* Set up a list of unique keys */ - mptcp_key_pool_init(); + mptcp_last_cellicon_set = tcp_now; +} + +int +mptcp_get_statsindex(struct mptcp_itf_stats *stats, const struct mptsub *mpts) +{ + const struct ifnet *ifp = sotoinpcb(mpts->mpts_socket)->inp_last_outifp; + + int i, index = -1; + + if (ifp == NULL) { + mptcplog((LOG_ERR, "%s: no ifp on subflow\n", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + return (-1); + } + + for (i = 0; i < MPTCP_ITFSTATS_SIZE; i++) { + if (stats[i].ifindex == IFSCOPE_NONE) { + if (index < 0) + index = i; + continue; + } + + if (stats[i].ifindex == ifp->if_index) { + index = i; + return (index); + } + } + + if (index != -1) { + stats[index].ifindex = ifp->if_index; + if (stats[index].is_expensive == 0) + stats[index].is_expensive = IFNET_IS_CELLULAR(ifp); + } + + return (index); +} + +void +mptcpstats_inc_switch(struct mptses *mpte, const struct mptsub *mpts) +{ + int index; + + tcpstat.tcps_mp_switches++; + mpte->mpte_subflow_switches++; + + index = mptcp_get_statsindex(mpte->mpte_itfstats, mpts); + + if (index != -1) + mpte->mpte_itfstats[index].switches++; +} + +/* + * Flushes all recorded socket options from an MP socket. + */ +static void +mptcp_flush_sopts(struct mptses *mpte) +{ + struct mptopt *mpo, *tmpo; + + TAILQ_FOREACH_SAFE(mpo, &mpte->mpte_sopts, mpo_entry, tmpo) { + mptcp_sopt_remove(mpte, mpo); + mptcp_sopt_free(mpo); + } + VERIFY(TAILQ_EMPTY(&mpte->mpte_sopts)); } /* * Create an MPTCP session, called as a result of opening a MPTCP socket. */ -void * -mptcp_sescreate(struct socket *mp_so, struct mppcb *mpp) +int +mptcp_sescreate(struct mppcb *mpp) { struct mppcbinfo *mppi; struct mptses *mpte; struct mptcb *mp_tp; - int error = 0; VERIFY(mpp != NULL); mppi = mpp->mpp_pcbinfo; @@ -482,178 +487,646 @@ mptcp_sescreate(struct socket *mp_so, struct mppcb *mpp) mpte->mpte_associd = SAE_ASSOCID_ANY; mpte->mpte_connid_last = SAE_CONNID_ANY; - lck_mtx_init(&mpte->mpte_thread_lock, mppi->mppi_lock_grp, - mppi->mppi_lock_attr); - - /* - * XXX: adi@apple.com - * - * This can be rather expensive if we have lots of MPTCP sockets, - * but we need a kernel thread for this model to work. Perhaps we - * could amortize the costs by having one worker thread per a group - * of MPTCP sockets. - */ - if (kernel_thread_start(mptcp_thread_func, mpte, - &mpte->mpte_thread) != KERN_SUCCESS) { - error = ENOBUFS; - goto out; - } - mp_so->so_usecount++; /* for thread */ + mpte->mpte_itfinfo = &mpte->_mpte_itfinfo[0]; + mpte->mpte_itfinfo_size = MPTE_ITFINFO_SIZE; /* MPTCP Protocol Control Block */ bzero(mp_tp, sizeof (*mp_tp)); - lck_mtx_init(&mp_tp->mpt_lock, mppi->mppi_lock_grp, - mppi->mppi_lock_attr); mp_tp->mpt_mpte = mpte; mp_tp->mpt_state = MPTCPS_CLOSED; -out: - if (error != 0) - lck_mtx_destroy(&mpte->mpte_thread_lock, mppi->mppi_lock_grp); - DTRACE_MPTCP5(session__create, struct socket *, mp_so, - struct sockbuf *, &mp_so->so_rcv, - struct sockbuf *, &mp_so->so_snd, - struct mppcb *, mpp, int, error); - return ((error != 0) ? NULL : mpte); + DTRACE_MPTCP1(session__create, struct mppcb *, mpp); + + return (0); +} + +static void +mptcpstats_get_bytes(struct mptses *mpte, boolean_t initial_cell, + uint64_t *cellbytes, uint64_t *allbytes) +{ + int64_t mycellbytes = 0; + uint64_t myallbytes = 0; + int i; + + for (i = 0; i < MPTCP_ITFSTATS_SIZE; i++) { + if (mpte->mpte_itfstats[i].is_expensive) { + mycellbytes += mpte->mpte_itfstats[i].mpis_txbytes; + mycellbytes += mpte->mpte_itfstats[i].mpis_rxbytes; + } + + myallbytes += mpte->mpte_itfstats[i].mpis_txbytes; + myallbytes += mpte->mpte_itfstats[i].mpis_rxbytes; + } + + if (initial_cell) { + mycellbytes -= mpte->mpte_init_txbytes; + mycellbytes -= mpte->mpte_init_txbytes; + } + + if (mycellbytes < 0) { + mptcplog((LOG_ERR, "%s cellbytes is %d\n", __func__, mycellbytes), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + *cellbytes = 0; + *allbytes = 0; + } else { + *cellbytes = mycellbytes; + *allbytes = myallbytes; + } +} + +static void +mptcpstats_session_wrapup(struct mptses *mpte) +{ + boolean_t cell = mpte->mpte_initial_cell; + + switch (mpte->mpte_svctype) { + case MPTCP_SVCTYPE_HANDOVER: + if (mpte->mpte_flags & MPTE_FIRSTPARTY) { + tcpstat.tcps_mptcp_fp_handover_attempt++; + + if (cell && mpte->mpte_handshake_success) { + tcpstat.tcps_mptcp_fp_handover_success_cell++; + + if (mpte->mpte_used_wifi) + tcpstat.tcps_mptcp_handover_wifi_from_cell++; + } else if (mpte->mpte_handshake_success) { + tcpstat.tcps_mptcp_fp_handover_success_wifi++; + + if (mpte->mpte_used_cell) + tcpstat.tcps_mptcp_handover_cell_from_wifi++; + } + } else { + tcpstat.tcps_mptcp_handover_attempt++; + + if (cell && mpte->mpte_handshake_success) { + tcpstat.tcps_mptcp_handover_success_cell++; + + if (mpte->mpte_used_wifi) + tcpstat.tcps_mptcp_handover_wifi_from_cell++; + } else if (mpte->mpte_handshake_success) { + tcpstat.tcps_mptcp_handover_success_wifi++; + + if (mpte->mpte_used_cell) + tcpstat.tcps_mptcp_handover_cell_from_wifi++; + } + } + + if (mpte->mpte_handshake_success) { + uint64_t cellbytes; + uint64_t allbytes; + + mptcpstats_get_bytes(mpte, cell, &cellbytes, &allbytes); + + tcpstat.tcps_mptcp_handover_cell_bytes += cellbytes; + tcpstat.tcps_mptcp_handover_all_bytes += allbytes; + } + break; + case MPTCP_SVCTYPE_INTERACTIVE: + if (mpte->mpte_flags & MPTE_FIRSTPARTY) { + tcpstat.tcps_mptcp_fp_interactive_attempt++; + + if (mpte->mpte_handshake_success) { + tcpstat.tcps_mptcp_fp_interactive_success++; + + if (!cell && mpte->mpte_used_cell) + tcpstat.tcps_mptcp_interactive_cell_from_wifi++; + } + } else { + tcpstat.tcps_mptcp_interactive_attempt++; + + if (mpte->mpte_handshake_success) { + tcpstat.tcps_mptcp_interactive_success++; + + if (!cell && mpte->mpte_used_cell) + tcpstat.tcps_mptcp_interactive_cell_from_wifi++; + } + } + + if (mpte->mpte_handshake_success) { + uint64_t cellbytes; + uint64_t allbytes; + + mptcpstats_get_bytes(mpte, cell, &cellbytes, &allbytes); + + tcpstat.tcps_mptcp_interactive_cell_bytes += cellbytes; + tcpstat.tcps_mptcp_interactive_all_bytes += allbytes; + } + break; + case MPTCP_SVCTYPE_AGGREGATE: + if (mpte->mpte_flags & MPTE_FIRSTPARTY) { + tcpstat.tcps_mptcp_fp_aggregate_attempt++; + + if (mpte->mpte_handshake_success) + tcpstat.tcps_mptcp_fp_aggregate_success++; + } else { + tcpstat.tcps_mptcp_aggregate_attempt++; + + if (mpte->mpte_handshake_success) { + tcpstat.tcps_mptcp_aggregate_success++; + } + } + + if (mpte->mpte_handshake_success) { + uint64_t cellbytes; + uint64_t allbytes; + + mptcpstats_get_bytes(mpte, cell, &cellbytes, &allbytes); + + tcpstat.tcps_mptcp_aggregate_cell_bytes += cellbytes; + tcpstat.tcps_mptcp_aggregate_all_bytes += allbytes; + } + break; + } + + if (cell && mpte->mpte_handshake_success && mpte->mpte_used_wifi) + tcpstat.tcps_mptcp_back_to_wifi++; } /* * Destroy an MPTCP session. */ static void -mptcp_sesdestroy(struct mptses *mpte) +mptcp_session_destroy(struct mptses *mpte) { struct mptcb *mp_tp; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + mpte_lock_assert_held(mpte); /* same as MP socket lock */ mp_tp = mpte->mpte_mptcb; VERIFY(mp_tp != NULL); + mptcpstats_session_wrapup(mpte); + + mptcp_unset_cellicon(); + /* * MPTCP Multipath PCB Extension section */ mptcp_flush_sopts(mpte); VERIFY(TAILQ_EMPTY(&mpte->mpte_subflows) && mpte->mpte_numflows == 0); - lck_mtx_destroy(&mpte->mpte_thread_lock, - mpte->mpte_mppcb->mpp_pcbinfo->mppi_lock_grp); + if (mpte->mpte_itfinfo_size > MPTE_ITFINFO_SIZE) + _FREE(mpte->mpte_itfinfo, M_TEMP); + + mpte->mpte_itfinfo = NULL; + + m_freem_list(mpte->mpte_reinjectq); /* * MPTCP Protocol Control Block section */ - lck_mtx_destroy(&mp_tp->mpt_lock, - mpte->mpte_mppcb->mpp_pcbinfo->mppi_lock_grp); - DTRACE_MPTCP2(session__destroy, struct mptses *, mpte, struct mptcb *, mp_tp); } -/* - * Allocate an MPTCP socket option structure. - */ -struct mptopt * -mptcp_sopt_alloc(int how) +static boolean_t +mptcp_ok_to_create_subflows(struct mptcb *mp_tp) { - struct mptopt *mpo; + return (mp_tp->mpt_state >= MPTCPS_ESTABLISHED && + mp_tp->mpt_state < MPTCPS_TIME_WAIT && + !(mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP)); +} - mpo = (how == M_WAITOK) ? zalloc(mptopt_zone) : - zalloc_noblock(mptopt_zone); - if (mpo != NULL) { - bzero(mpo, mptopt_zone_size); +static int +mptcp_synthesize_nat64(struct in6_addr *addr, uint32_t len, struct in_addr *addrv4) +{ + static const struct in6_addr well_known_prefix = { + .__u6_addr.__u6_addr8 = {0x00, 0x64, 0xff, 0x9b, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}, + }; + char buf[MAX_IPv6_STR_LEN]; + char *ptrv4 = (char *)addrv4; + char *ptr = (char *)addr; + + if (IN_ZERONET(addrv4->s_addr) || // 0.0.0.0/8 Source hosts on local network + IN_LOOPBACK(addrv4->s_addr) || // 127.0.0.0/8 Loopback + IN_LINKLOCAL(addrv4->s_addr) || // 169.254.0.0/16 Link Local + IN_DS_LITE(addrv4->s_addr) || // 192.0.0.0/29 DS-Lite + IN_6TO4_RELAY_ANYCAST(addrv4->s_addr) || // 192.88.99.0/24 6to4 Relay Anycast + IN_MULTICAST(addrv4->s_addr) || // 224.0.0.0/4 Multicast + INADDR_BROADCAST == addrv4->s_addr) { // 255.255.255.255/32 Limited Broadcast + return (-1); } - return (mpo); -} + /* Check for the well-known prefix */ + if (len == NAT64_PREFIX_LEN_96 && + IN6_ARE_ADDR_EQUAL(addr, &well_known_prefix)) { + if (IN_PRIVATE(addrv4->s_addr) || // 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16 Private-Use + IN_SHARED_ADDRESS_SPACE(addrv4->s_addr)) // 100.64.0.0/10 Shared Address Space + return (-1); + } -/* - * Free an MPTCP socket option structure. - */ -void -mptcp_sopt_free(struct mptopt *mpo) -{ - VERIFY(!(mpo->mpo_flags & MPOF_ATTACHED)); + switch (len) { + case NAT64_PREFIX_LEN_96: + memcpy(ptr + 12, ptrv4, 4); + break; + case NAT64_PREFIX_LEN_64: + memcpy(ptr + 9, ptrv4, 4); + break; + case NAT64_PREFIX_LEN_56: + memcpy(ptr + 7, ptrv4, 1); + memcpy(ptr + 9, ptrv4 + 1, 3); + break; + case NAT64_PREFIX_LEN_48: + memcpy(ptr + 6, ptrv4, 2); + memcpy(ptr + 9, ptrv4 + 2, 2); + break; + case NAT64_PREFIX_LEN_40: + memcpy(ptr + 5, ptrv4, 3); + memcpy(ptr + 9, ptrv4 + 3, 1); + break; + case NAT64_PREFIX_LEN_32: + memcpy(ptr + 4, ptrv4, 4); + break; + default: + panic("NAT64-prefix len is wrong: %u\n", len); + } - zfree(mptopt_zone, mpo); -} + mptcplog((LOG_DEBUG, "%s: nat64prefix-len %u synthesized %s\n", __func__, + len, inet_ntop(AF_INET6, (void *)addr, buf, sizeof(buf))), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); -/* - * Add a socket option to the MPTCP socket option list. - */ -void -mptcp_sopt_insert(struct mptses *mpte, struct mptopt *mpo) -{ - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - VERIFY(!(mpo->mpo_flags & MPOF_ATTACHED)); - mpo->mpo_flags |= MPOF_ATTACHED; - TAILQ_INSERT_TAIL(&mpte->mpte_sopts, mpo, mpo_entry); + return (0); } -/* - * Remove a socket option from the MPTCP socket option list. - */ void -mptcp_sopt_remove(struct mptses *mpte, struct mptopt *mpo) +mptcp_check_subflows_and_add(struct mptses *mpte) { - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - VERIFY(mpo->mpo_flags & MPOF_ATTACHED); - mpo->mpo_flags &= ~MPOF_ATTACHED; - TAILQ_REMOVE(&mpte->mpte_sopts, mpo, mpo_entry); -} + struct mptcb *mp_tp = mpte->mpte_mptcb; + uint32_t i; -/* - * Search for an existing <sopt_level,sopt_name> socket option. - */ -struct mptopt * -mptcp_sopt_find(struct mptses *mpte, struct sockopt *sopt) -{ - struct mptopt *mpo; + if (!mptcp_ok_to_create_subflows(mp_tp)) + return; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + for (i = 0; i < mpte->mpte_itfinfo_size; i++) { + struct mpt_itf_info *info; + struct mptsub *mpts; + uint32_t ifindex; + int found = 0; - TAILQ_FOREACH(mpo, &mpte->mpte_sopts, mpo_entry) { - if (mpo->mpo_level == sopt->sopt_level && - mpo->mpo_name == sopt->sopt_name) - break; - } - VERIFY(mpo == NULL || sopt->sopt_valsize == sizeof (int)); + info = &mpte->mpte_itfinfo[i]; - return (mpo); -} + if (info->no_mptcp_support) + continue; -/* - * Flushes all recorded socket options from an MP socket. - */ -void -mptcp_flush_sopts(struct mptses *mpte) -{ - struct mptopt *mpo, *tmpo; + ifindex = info->ifindex; + if (ifindex == IFSCOPE_NONE) + continue; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { + const struct ifnet *ifp = sotoinpcb(mpts->mpts_socket)->inp_last_outifp; - TAILQ_FOREACH_SAFE(mpo, &mpte->mpte_sopts, mpo_entry, tmpo) { - mptcp_sopt_remove(mpte, mpo); - mptcp_sopt_free(mpo); - } - VERIFY(TAILQ_EMPTY(&mpte->mpte_sopts)); -} + if (ifp == NULL) + continue; -/* - * Allocate a MPTCP subflow structure. - */ -struct mptsub * -mptcp_subflow_alloc(int how) -{ - struct mptsub *mpts; + if (ifp->if_index == ifindex && + !(mpts->mpts_socket->so_state & SS_ISDISCONNECTED)) { + /* + * We found a subflow on this interface. + * No need to create a new one. + */ + found = 1; + break; + } + + /* + * In Handover mode, only create cell subflow if + * 1. Wi-Fi Assist is active + * 2. Symptoms marked WiFi as weak + * 3. We are experiencing RTOs or we are not sending data. + * + * This covers the scenario, where: + * 1. We send and get retransmission timeouts (thus, + * we confirmed that WiFi is indeed bad). + * 2. We are not sending and the server tries to send. + * Establshing a cell-subflow gives the server a + * chance to send us some data over cell if WiFi + * is dead. We establish the subflow with the + * backup-bit set, so the server is not allowed to + * send on this subflow as long as WiFi is providing + * good performance. + */ + if (mpte->mpte_svctype == MPTCP_SVCTYPE_HANDOVER && + !IFNET_IS_CELLULAR(ifp) && + !(mpts->mpts_flags & (MPTSF_DISCONNECTING | MPTSF_DISCONNECTED | MPTSF_CLOSE_REQD)) && + (!mptcp_is_wifi_unusable() || + (sototcpcb(mpts->mpts_socket)->t_rxtshift < mptcp_fail_thresh && + mptetoso(mpte)->so_snd.sb_cc))) { + mptcplog((LOG_DEBUG, "%s handover, wifi state %u rxt %u ifindex %u this %u\n", + __func__, mptcp_is_wifi_unusable(), sototcpcb(mpts->mpts_socket)->t_rxtshift, ifindex, + ifp->if_index), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + found = 1; + break; + } + } + + if (!found && !(mpte->mpte_flags & MPTE_FIRSTPARTY) && + !(mpte->mpte_flags & MPTE_ACCESS_GRANTED) && + mptcp_developer_mode == 0) { + mptcp_ask_symptoms(mpte); + return; + } + + if (!found) { + struct sockaddr *dst = &mpte->mpte_dst; + struct sockaddr_in6 nat64pre; + + if (mpte->mpte_dst.sa_family == AF_INET && + !info->has_v4_conn && info->has_v6_conn) { + struct ipv6_prefix nat64prefixes[NAT64_MAX_NUM_PREFIXES]; + struct ifnet *ifp; + int error, j; + + bzero(&nat64pre, sizeof(struct sockaddr_in6)); + + ifnet_head_lock_shared(); + ifp = ifindex2ifnet[ifindex]; + ifnet_head_done(); + + error = ifnet_get_nat64prefix(ifp, nat64prefixes); + if (error) { + mptcplog((LOG_ERR, "%s: no NAT64-prefix on itf %s, error %d\n", + __func__, ifp->if_name, error), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + continue; + } + + for (j = 0; j < NAT64_MAX_NUM_PREFIXES; j++) { + if (nat64prefixes[j].prefix_len != 0) + break; + } + + VERIFY(j < NAT64_MAX_NUM_PREFIXES); + + error = mptcp_synthesize_nat64(&nat64prefixes[j].ipv6_prefix, + nat64prefixes[j].prefix_len, + &mpte->__mpte_dst_v4.sin_addr); + if (error != 0) { + mptcplog((LOG_INFO, "%s: cannot synthesize this addr\n", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); + continue; + } + + memcpy(&nat64pre.sin6_addr, + &nat64prefixes[j].ipv6_prefix, + sizeof(nat64pre.sin6_addr)); + nat64pre.sin6_len = sizeof(struct sockaddr_in6); + nat64pre.sin6_family = AF_INET6; + nat64pre.sin6_port = mpte->__mpte_dst_v6.sin6_port; + nat64pre.sin6_flowinfo = 0; + nat64pre.sin6_scope_id = 0; + + dst = (struct sockaddr *)&nat64pre; + } + + mptcp_subflow_add(mpte, NULL, dst, ifindex, NULL); + } + } +} + +/* + * Based on the MPTCP Service-type and the state of the subflows, we + * will destroy subflows here. + */ +static void +mptcp_check_subflows_and_remove(struct mptses *mpte) +{ + struct mptsub *mpts, *tmpts; + int found_working_subflow = 0, removed_some = 0; + int wifi_unusable = mptcp_is_wifi_unusable(); + + if (mpte->mpte_svctype != MPTCP_SVCTYPE_HANDOVER) + return; + + /* + * Look for a subflow that is on a non-cellular interface + * and actually works (aka, no retransmission timeout). + */ + TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { + const struct ifnet *ifp = sotoinpcb(mpts->mpts_socket)->inp_last_outifp; + struct socket *so; + struct tcpcb *tp; + + if (ifp == NULL || IFNET_IS_CELLULAR(ifp)) + continue; + + so = mpts->mpts_socket; + tp = sototcpcb(so); + + if (!(mpts->mpts_flags & MPTSF_CONNECTED) || + tp->t_state != TCPS_ESTABLISHED) + continue; + + /* Either this subflow is in good condition while we try to send */ + if (tp->t_rxtshift == 0 && mptetoso(mpte)->so_snd.sb_cc) + found_working_subflow = 1; - mpts = (how == M_WAITOK) ? zalloc(mptsub_zone) : - zalloc_noblock(mptsub_zone); - if (mpts != NULL) { - bzero(mpts, mptsub_zone_size); - lck_mtx_init(&mpts->mpts_lock, mtcbinfo.mppi_lock_grp, - mtcbinfo.mppi_lock_attr); + /* Or WiFi is fine */ + if (!wifi_unusable) + found_working_subflow = 1; } + /* + * Couldn't find a working subflow, let's not remove those on a cellular + * interface. + */ + if (!found_working_subflow) + return; + + TAILQ_FOREACH_SAFE(mpts, &mpte->mpte_subflows, mpts_entry, tmpts) { + const struct ifnet *ifp = sotoinpcb(mpts->mpts_socket)->inp_last_outifp; + + /* Only remove cellular subflows */ + if (ifp == NULL || !IFNET_IS_CELLULAR(ifp)) + continue; + + soevent(mpts->mpts_socket, SO_FILT_HINT_LOCKED | SO_FILT_HINT_MUSTRST); + removed_some = 1; + } + + if (removed_some) + mptcp_unset_cellicon(); +} + +static void +mptcp_remove_subflows(struct mptses *mpte) +{ + struct mptsub *mpts, *tmpts; + + TAILQ_FOREACH_SAFE(mpts, &mpte->mpte_subflows, mpts_entry, tmpts) { + if (mpts->mpts_flags & MPTSF_CLOSE_REQD) { + mpts->mpts_flags &= ~MPTSF_CLOSE_REQD; + + soevent(mpts->mpts_socket, + SO_FILT_HINT_LOCKED | SO_FILT_HINT_NOSRCADDR); + } + } +} + +static void +mptcp_create_subflows(__unused void *arg) +{ + struct mppcb *mpp; + + /* + * Start with clearing, because we might be processing connections + * while a new event comes in. + */ + if (OSTestAndClear(0x01, &mptcp_create_subflows_scheduled)) + mptcplog((LOG_ERR, "%s: bit was already cleared!\n", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + + /* Iterate over all MPTCP connections */ + + lck_mtx_lock(&mtcbinfo.mppi_lock); + + TAILQ_FOREACH(mpp, &mtcbinfo.mppi_pcbs, mpp_entry) { + struct mptses *mpte; + struct socket *mp_so; + + if (!(mpp->mpp_flags & MPP_CREATE_SUBFLOWS)) + continue; + + mpp_lock(mpp); + + mpp->mpp_flags &= ~MPP_CREATE_SUBFLOWS; + + mpte = mpp->mpp_pcbe; + mp_so = mpp->mpp_socket; + + VERIFY(mp_so->so_usecount > 0); + + mptcp_check_subflows_and_add(mpte); + mptcp_remove_subflows(mpte); + + mp_so->so_usecount--; /* See mptcp_sched_create_subflows */ + mpp_unlock(mpp); + } + + lck_mtx_unlock(&mtcbinfo.mppi_lock); +} + +/* + * We need this because we are coming from an NECP-event. This event gets posted + * while holding NECP-locks. The creation of the subflow however leads us back + * into NECP (e.g., to add the necp_cb and also from tcp_connect). + * So, we would deadlock there as we already hold the NECP-lock. + * + * So, let's schedule this separately. It also gives NECP the chance to make + * progress, without having to wait for MPTCP to finish its subflow creation. + */ +void +mptcp_sched_create_subflows(struct mptses *mpte) +{ + struct mppcb *mpp = mpte->mpte_mppcb; + struct mptcb *mp_tp = mpte->mpte_mptcb; + struct socket *mp_so = mpp->mpp_socket; + + if (!mptcp_ok_to_create_subflows(mp_tp)) { + mptcplog((LOG_DEBUG, "%s: not a good time for subflows, state %u flags %#x", + __func__, mp_tp->mpt_state, mp_tp->mpt_flags), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + return; + } + + if (!(mpp->mpp_flags & MPP_CREATE_SUBFLOWS)) { + mp_so->so_usecount++; /* To prevent it from being free'd in-between */ + mpp->mpp_flags |= MPP_CREATE_SUBFLOWS; + } + + if (OSTestAndSet(0x01, &mptcp_create_subflows_scheduled)) + return; + + /* Do the call in 100ms to allow NECP to schedule it on all sockets */ + timeout(mptcp_create_subflows, NULL, hz/10); +} + +/* + * Allocate an MPTCP socket option structure. + */ +struct mptopt * +mptcp_sopt_alloc(int how) +{ + struct mptopt *mpo; + + mpo = (how == M_WAITOK) ? zalloc(mptopt_zone) : + zalloc_noblock(mptopt_zone); + if (mpo != NULL) { + bzero(mpo, mptopt_zone_size); + } + + return (mpo); +} + +/* + * Free an MPTCP socket option structure. + */ +void +mptcp_sopt_free(struct mptopt *mpo) +{ + VERIFY(!(mpo->mpo_flags & MPOF_ATTACHED)); + + zfree(mptopt_zone, mpo); +} + +/* + * Add a socket option to the MPTCP socket option list. + */ +void +mptcp_sopt_insert(struct mptses *mpte, struct mptopt *mpo) +{ + mpte_lock_assert_held(mpte); /* same as MP socket lock */ + VERIFY(!(mpo->mpo_flags & MPOF_ATTACHED)); + mpo->mpo_flags |= MPOF_ATTACHED; + TAILQ_INSERT_TAIL(&mpte->mpte_sopts, mpo, mpo_entry); +} + +/* + * Remove a socket option from the MPTCP socket option list. + */ +void +mptcp_sopt_remove(struct mptses *mpte, struct mptopt *mpo) +{ + mpte_lock_assert_held(mpte); /* same as MP socket lock */ + VERIFY(mpo->mpo_flags & MPOF_ATTACHED); + mpo->mpo_flags &= ~MPOF_ATTACHED; + TAILQ_REMOVE(&mpte->mpte_sopts, mpo, mpo_entry); +} + +/* + * Search for an existing <sopt_level,sopt_name> socket option. + */ +struct mptopt * +mptcp_sopt_find(struct mptses *mpte, struct sockopt *sopt) +{ + struct mptopt *mpo; + + mpte_lock_assert_held(mpte); /* same as MP socket lock */ + + TAILQ_FOREACH(mpo, &mpte->mpte_sopts, mpo_entry) { + if (mpo->mpo_level == sopt->sopt_level && + mpo->mpo_name == sopt->sopt_name) + break; + } + VERIFY(mpo == NULL || sopt->sopt_valsize == sizeof (int)); + + return (mpo); +} + +/* + * Allocate a MPTCP subflow structure. + */ +static struct mptsub * +mptcp_subflow_alloc(void) +{ + struct mptsub *mpts = zalloc(mptsub_zone); + + if (mpts == NULL) + return (NULL); + + bzero(mpts, mptsub_zone_size); return (mpts); } @@ -661,11 +1134,9 @@ mptcp_subflow_alloc(int how) * Deallocate a subflow structure, called when all of the references held * on it have been released. This implies that the subflow has been deleted. */ -void +static void mptcp_subflow_free(struct mptsub *mpts) { - MPTS_LOCK_ASSERT_HELD(mpts); - VERIFY(mpts->mpts_refcnt == 0); VERIFY(!(mpts->mpts_flags & MPTSF_ATTACHED)); VERIFY(mpts->mpts_mpte == NULL); @@ -675,30 +1146,128 @@ mptcp_subflow_free(struct mptsub *mpts) FREE(mpts->mpts_src, M_SONAME); mpts->mpts_src = NULL; } - if (mpts->mpts_dst != NULL) { - FREE(mpts->mpts_dst, M_SONAME); - mpts->mpts_dst = NULL; - } - MPTS_UNLOCK(mpts); - lck_mtx_destroy(&mpts->mpts_lock, mtcbinfo.mppi_lock_grp); zfree(mptsub_zone, mpts); } +static void +mptcp_subflow_addref(struct mptsub *mpts) +{ + if (++mpts->mpts_refcnt == 0) + panic("%s: mpts %p wraparound refcnt\n", __func__, mpts); + /* NOTREACHED */ +} + +static void +mptcp_subflow_remref(struct mptsub *mpts) +{ + if (mpts->mpts_refcnt == 0) { + panic("%s: mpts %p negative refcnt\n", __func__, mpts); + /* NOTREACHED */ + } + if (--mpts->mpts_refcnt > 0) + return; + + /* callee will unlock and destroy lock */ + mptcp_subflow_free(mpts); +} + +static void +mptcp_subflow_attach(struct mptses *mpte, struct mptsub *mpts, struct socket *so) +{ + struct socket *mp_so = mpte->mpte_mppcb->mpp_socket; + struct tcpcb *tp = sototcpcb(so); + + /* + * From this moment on, the subflow is linked to the MPTCP-connection. + * Locking,... happens now at the MPTCP-layer + */ + tp->t_mptcb = mpte->mpte_mptcb; + so->so_flags |= SOF_MP_SUBFLOW; + mp_so->so_usecount++; + + /* + * Insert the subflow into the list, and associate the MPTCP PCB + * as well as the the subflow socket. From this point on, removing + * the subflow needs to be done via mptcp_subflow_del(). + */ + TAILQ_INSERT_TAIL(&mpte->mpte_subflows, mpts, mpts_entry); + mpte->mpte_numflows++; + + atomic_bitset_32(&mpts->mpts_flags, MPTSF_ATTACHED); + mpts->mpts_mpte = mpte; + mpts->mpts_socket = so; + tp->t_mpsub = mpts; + mptcp_subflow_addref(mpts); /* for being in MPTCP subflow list */ + mptcp_subflow_addref(mpts); /* for subflow socket */ +} + +static void +mptcp_subflow_necp_cb(void *handle, __unused int action, + __unused struct necp_client_flow *flow) +{ + struct inpcb *inp = (struct inpcb *)handle; + struct socket *so = inp->inp_socket; + struct mptsub *mpts; + struct mptses *mpte; + + if (action != NECP_CLIENT_CBACTION_NONVIABLE) + return; + + /* + * The socket is being garbage-collected. There is nothing to be done + * here. + */ + if (so->so_usecount == 0) + return; + + socket_lock(so, 1); + + /* Check again after we acquired the lock. */ + if (so->so_usecount == 0) + goto out; + + mpte = tptomptp(sototcpcb(so))->mpt_mpte; + mpts = sototcpcb(so)->t_mpsub; + + mptcplog((LOG_DEBUG, "%s: Subflow became non-viable", __func__), + MPTCP_EVENTS_DBG, MPTCP_LOGLVL_VERBOSE); + + mpts->mpts_flags |= MPTSF_CLOSE_REQD; + + mptcp_sched_create_subflows(mpte); + + if (mpte->mpte_svctype == MPTCP_SVCTYPE_HANDOVER) + flow->viable = 1; + +out: + socket_unlock(so, 1); +} + /* * Create an MPTCP subflow socket. */ static int mptcp_subflow_socreate(struct mptses *mpte, struct mptsub *mpts, int dom, - struct proc *p, struct socket **so) + struct socket **so) { + lck_mtx_t *subflow_mtx; struct mptopt smpo, *mpo, *tmpo; + struct proc *p; struct socket *mp_so; int error; *so = NULL; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - mp_so = mpte->mpte_mppcb->mpp_socket; + mpte_lock_assert_held(mpte); /* same as MP socket lock */ + mp_so = mptetoso(mpte); + + p = proc_find(mp_so->last_pid); + if (p == PROC_NULL) { + mptcplog((LOG_ERR, "%s: Couldn't find proc for pid %u\n", __func__, mp_so->last_pid), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + + return (ESRCH); + } /* * Create the subflow socket (multipath subflow, non-blocking.) @@ -708,19 +1277,49 @@ mptcp_subflow_socreate(struct mptses *mpte, struct mptsub *mpts, int dom, * It also indicates to the underlying TCP to handle MPTCP options. * A multipath subflow socket implies SS_NOFDREF state. */ - if ((error = socreate_internal(dom, so, SOCK_STREAM, - IPPROTO_TCP, p, SOCF_ASYNC | SOCF_MP_SUBFLOW, PROC_NULL)) != 0) { - mptcplog((LOG_ERR, "MPTCP Socket: subflow socreate mp_so 0x%llx" - " unable to create subflow socket error %d\n", - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), error), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + + /* + * Unlock, because tcp_usr_attach ends up in in_pcballoc, which takes + * the ipi-lock. We cannot hold the socket-lock at that point. + */ + mpte_unlock(mpte); + error = socreate_internal(dom, so, SOCK_STREAM, IPPROTO_TCP, p, + SOCF_ASYNC, PROC_NULL); + mpte_lock(mpte); + if (error) { + mptcplog((LOG_ERR, "%s: subflow socreate mp_so 0x%llx unable to create subflow socket error %d\n", + (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), error), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + + proc_rele(p); + + mptcp_subflow_free(mpts); return (error); } - socket_lock(*so, 0); - VERIFY((*so)->so_flags & SOF_MP_SUBFLOW); - VERIFY(((*so)->so_state & (SS_NBIO|SS_NOFDREF)) == - (SS_NBIO|SS_NOFDREF)); + /* + * We need to protect the setting of SOF_MP_SUBFLOW with a lock, because + * this marks the moment of lock-switch from the TCP-lock to the MPTCP-lock. + * Which is why we also need to get the lock with pr_getlock, as after + * setting the flag, socket_unlock will work on the MPTCP-level lock. + */ + subflow_mtx = ((*so)->so_proto->pr_getlock)(*so, 0); + lck_mtx_lock(subflow_mtx); + + /* + * Must be the first thing we do, to make sure all pointers for this + * subflow are set. + */ + mptcp_subflow_attach(mpte, mpts, *so); + + /* + * A multipath subflow socket is used internally in the kernel, + * therefore it does not have a file desciptor associated by + * default. + */ + (*so)->so_state |= SS_NOFDREF; + + lck_mtx_unlock(subflow_mtx); /* prevent the socket buffers from being compressed */ (*so)->so_rcv.sb_flags |= SB_NOCOMPRESS; @@ -729,10 +1328,42 @@ mptcp_subflow_socreate(struct mptses *mpte, struct mptsub *mpts, int dom, /* Inherit preconnect and TFO data flags */ if (mp_so->so_flags1 & SOF1_PRECONNECT_DATA) (*so)->so_flags1 |= SOF1_PRECONNECT_DATA; - if (mp_so->so_flags1 & SOF1_DATA_IDEMPOTENT) (*so)->so_flags1 |= SOF1_DATA_IDEMPOTENT; + /* Inherit uuid and create the related flow. */ + if (!uuid_is_null(mpsotomppcb(mp_so)->necp_client_uuid)) { + struct mptcb *mp_tp = mpte->mpte_mptcb; + + sotoinpcb(*so)->necp_cb = mptcp_subflow_necp_cb; + + /* + * A note on the unlock: With MPTCP, we do multiple times a + * necp_client_register_socket_flow. This is problematic, + * because now the lock-ordering guarantee (first necp-locks, + * then socket-locks) is no more respected. So, we need to + * unlock here. + */ + mpte_unlock(mpte); + error = necp_client_register_socket_flow(mp_so->last_pid, + mpsotomppcb(mp_so)->necp_client_uuid, sotoinpcb(*so)); + mpte_lock(mpte); + + if (error) + goto out_err; + + /* Possible state-change during the unlock above */ + if (mp_tp->mpt_state >= MPTCPS_TIME_WAIT || + (mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP)) + goto out_err; + + uuid_copy(sotoinpcb(*so)->necp_client_uuid, mpsotomppcb(mp_so)->necp_client_uuid); + } else { + mptcplog((LOG_NOTICE, "%s: uuid is not set!\n"), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); + } + + /* inherit the other socket options */ bzero(&smpo, sizeof (smpo)); smpo.mpo_flags |= MPOF_SUBFLOW_OK; smpo.mpo_level = SOL_SOCKET; @@ -740,42 +1371,36 @@ mptcp_subflow_socreate(struct mptses *mpte, struct mptsub *mpts, int dom, /* disable SIGPIPE */ smpo.mpo_name = SO_NOSIGPIPE; - if ((error = mptcp_subflow_sosetopt(mpte, *so, &smpo)) != 0) - goto out; + if ((error = mptcp_subflow_sosetopt(mpte, mpts, &smpo)) != 0) + goto out_err; /* find out if the subflow's source address goes away */ smpo.mpo_name = SO_NOADDRERR; - if ((error = mptcp_subflow_sosetopt(mpte, *so, &smpo)) != 0) - goto out; + if ((error = mptcp_subflow_sosetopt(mpte, mpts, &smpo)) != 0) + goto out_err; /* enable keepalive */ smpo.mpo_name = SO_KEEPALIVE; - if ((error = mptcp_subflow_sosetopt(mpte, *so, &smpo)) != 0) - goto out; - - /* - * Limit the receive socket buffer size to 64k. - * - * We need to take into consideration the window scale option - * which could be negotiated in one subflow but disabled in - * another subflow. - * XXX This can be improved in the future. - */ - smpo.mpo_name = SO_RCVBUF; - smpo.mpo_intval = MPTCP_RWIN_MAX; - if ((error = mptcp_subflow_sosetopt(mpte, *so, &smpo)) != 0) - goto out; - - /* N.B.: set by sosetopt */ - VERIFY(!((*so)->so_rcv.sb_flags & SB_AUTOSIZE)); - /* Prevent automatic socket buffer sizing. */ - (*so)->so_snd.sb_flags &= ~SB_AUTOSIZE; + if ((error = mptcp_subflow_sosetopt(mpte, mpts, &smpo)) != 0) + goto out_err; smpo.mpo_level = IPPROTO_TCP; smpo.mpo_intval = mptcp_subflow_keeptime; smpo.mpo_name = TCP_KEEPALIVE; - if ((error = mptcp_subflow_sosetopt(mpte, *so, &smpo)) != 0) - goto out; + if ((error = mptcp_subflow_sosetopt(mpte, mpts, &smpo)) != 0) + goto out_err; + + if (mpte->mpte_mptcb->mpt_state >= MPTCPS_ESTABLISHED) { + /* + * On secondary subflows we might need to set the cell-fallback + * flag (see conditions in mptcp_subflow_sosetopt). + */ + smpo.mpo_level = SOL_SOCKET; + smpo.mpo_name = SO_MARK_CELLFALLBACK; + smpo.mpo_intval = 1; + if ((error = mptcp_subflow_sosetopt(mpte, mpts, &smpo)) != 0) + goto out_err; + } /* replay setsockopt(2) on the subflow sockets for eligible options */ TAILQ_FOREACH_SAFE(mpo, &mpte->mpte_sopts, mpo_entry, tmpo) { @@ -796,14 +1421,12 @@ mptcp_subflow_socreate(struct mptses *mpte, struct mptsub *mpts, int dom, continue; interim = (mpo->mpo_flags & MPOF_INTERIM); - if (mptcp_subflow_sosetopt(mpte, *so, mpo) != 0 && interim) { - char buf[32]; - mptcplog((LOG_ERR, "MPTCP Socket: subflow socreate" - " mp_so 0x%llx" - " sopt %s val %d interim record removed\n", + if (mptcp_subflow_sosetopt(mpte, mpts, mpo) != 0 && interim) { + mptcplog((LOG_ERR, "%s: subflow socreate mp_so 0x%llx" + " sopt %s val %d interim record removed\n", __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), - mptcp_sopt2str(mpo->mpo_level, mpo->mpo_name, - buf, sizeof (buf)), mpo->mpo_intval), + mptcp_sopt2str(mpo->mpo_level, mpo->mpo_name), + mpo->mpo_intval), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); mptcp_sopt_remove(mpte, mpo); mptcp_sopt_free(mpo); @@ -816,7 +1439,6 @@ mptcp_subflow_socreate(struct mptses *mpte, struct mptsub *mpts, int dom, * so use a customized socket receive function. We will undo * this when the socket is peeled off or closed. */ - mpts->mpts_oprotosw = (*so)->so_proto; switch (dom) { case PF_INET: (*so)->so_proto = &mptcp_subflow_protosw; @@ -831,11 +1453,20 @@ mptcp_subflow_socreate(struct mptses *mpte, struct mptsub *mpts, int dom, /* NOTREACHED */ } -out: - socket_unlock(*so, 0); + proc_rele(p); + + DTRACE_MPTCP3(subflow__create, struct mptses *, mpte, + int, dom, int, error); + + return (0); - DTRACE_MPTCP4(subflow__create, struct mptses *, mpte, - struct mptsub *, mpts, int, dom, int, error); +out_err: + mptcp_subflow_abort(mpts, error); + + proc_rele(p); + + mptcplog((LOG_ERR, "%s: subflow socreate failed with error %d\n", + __func__, error), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); return (error); } @@ -846,95 +1477,117 @@ out: * Note that this may be called on an embryonic subflow, and the only * thing that is guaranteed valid is the protocol-user request. */ -static int -mptcp_subflow_soclose(struct mptsub *mpts, struct socket *so) +static void +mptcp_subflow_soclose(struct mptsub *mpts) { - MPTS_LOCK_ASSERT_HELD(mpts); + struct socket *so = mpts->mpts_socket; + + if (mpts->mpts_flags & MPTSF_CLOSED) + return; - socket_lock(so, 0); + VERIFY(so != NULL); VERIFY(so->so_flags & SOF_MP_SUBFLOW); VERIFY((so->so_state & (SS_NBIO|SS_NOFDREF)) == (SS_NBIO|SS_NOFDREF)); - /* restore protocol-user requests */ - VERIFY(mpts->mpts_oprotosw != NULL); - so->so_proto = mpts->mpts_oprotosw; - socket_unlock(so, 0); - - mpts->mpts_socket = NULL; /* may already be NULL */ - DTRACE_MPTCP5(subflow__close, struct mptsub *, mpts, struct socket *, so, struct sockbuf *, &so->so_rcv, struct sockbuf *, &so->so_snd, struct mptses *, mpts->mpts_mpte); - return (soclose(so)); + mpts->mpts_flags |= MPTSF_CLOSED; + + if (so->so_retaincnt == 0) { + soclose_locked(so); + + return; + } else { + VERIFY(so->so_usecount > 0); + so->so_usecount--; + } + + return; } /* * Connect an MPTCP subflow socket. * - * This may be called inline as part of adding a subflow, or asynchronously - * by the thread (upon progressing to MPTCPF_JOIN_READY). Note that in the - * pending connect case, the subflow socket may have been bound to an interface - * and/or a source IP address which may no longer be around by the time this - * routine is called; in that case the connect attempt will most likely fail. + * Note that in the pending connect case, the subflow socket may have been + * bound to an interface and/or a source IP address which may no longer be + * around by the time this routine is called; in that case the connect attempt + * will most likely fail. */ static int mptcp_subflow_soconnectx(struct mptses *mpte, struct mptsub *mpts) { - struct socket *so; + char dbuf[MAX_IPv6_STR_LEN]; + struct socket *mp_so, *so; + struct mptcb *mp_tp; + struct sockaddr *dst; + struct proc *p; int af, error; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); - - VERIFY((mpts->mpts_flags & (MPTSF_CONNECTING|MPTSF_CONNECTED)) == - MPTSF_CONNECTING); - VERIFY(mpts->mpts_socket != NULL); - so = mpts->mpts_socket; - af = mpts->mpts_family; + mpte_lock_assert_held(mpte); /* same as MP socket lock */ - if (af == AF_INET || af == AF_INET6) { - struct sockaddr *dst; - char dbuf[MAX_IPv6_STR_LEN]; + mp_so = mptetoso(mpte); + mp_tp = mpte->mpte_mptcb; - dst = mpts->mpts_dst; + p = proc_find(mp_so->last_pid); + if (p == PROC_NULL) { + mptcplog((LOG_ERR, "%s: Couldn't find proc for pid %u\n", __func__, mp_so->last_pid), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); - mptcplog((LOG_DEBUG, "MPTCP Socket: connectx mp_so 0x%llx " - "dst %s[%d] cid %d [pended %s]\n", - (u_int64_t)VM_KERNEL_ADDRPERM(mpte->mpte_mppcb->mpp_socket), - inet_ntop(af, ((af == AF_INET) ? - (void *)&SIN(dst)->sin_addr.s_addr : - (void *)&SIN6(dst)->sin6_addr), - dbuf, sizeof (dbuf)), ((af == AF_INET) ? - ntohs(SIN(dst)->sin_port) : - ntohs(SIN6(dst)->sin6_port)), - mpts->mpts_connid, - ((mpts->mpts_flags & MPTSF_CONNECT_PENDING) ? - "YES" : "NO")), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); + return (ESRCH); } + so = mpts->mpts_socket; + af = mpts->mpts_dst.sa_family; + + VERIFY((mpts->mpts_flags & (MPTSF_CONNECTING|MPTSF_CONNECTED)) == MPTSF_CONNECTING); + VERIFY(mpts->mpts_socket != NULL); + VERIFY(af == AF_INET || af == AF_INET6); + + dst = &mpts->mpts_dst; + mptcplog((LOG_DEBUG, "%s: connectx mp_so 0x%llx dst %s[%d] cid %d [pended %s]\n", + __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), + inet_ntop(af, ((af == AF_INET) ? (void *)&SIN(dst)->sin_addr.s_addr : + (void *)&SIN6(dst)->sin6_addr), + dbuf, sizeof (dbuf)), + ((af == AF_INET) ? ntohs(SIN(dst)->sin_port) : ntohs(SIN6(dst)->sin6_port)), + mpts->mpts_connid, + ((mpts->mpts_flags & MPTSF_CONNECT_PENDING) ? "YES" : "NO")), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + mpts->mpts_flags &= ~MPTSF_CONNECT_PENDING; - socket_lock(so, 0); mptcp_attach_to_subf(so, mpte->mpte_mptcb, mpte->mpte_addrid_last); /* connect the subflow socket */ - error = soconnectxlocked(so, mpts->mpts_src, mpts->mpts_dst, - mpts->mpts_mpcr.mpcr_proc, mpts->mpts_mpcr.mpcr_ifscope, - mpte->mpte_associd, NULL, CONNREQF_MPTCP, - &mpts->mpts_mpcr, sizeof (mpts->mpts_mpcr), NULL, NULL); - socket_unlock(so, 0); + error = soconnectxlocked(so, mpts->mpts_src, &mpts->mpts_dst, + p, mpts->mpts_ifscope, + mpte->mpte_associd, NULL, 0, NULL, 0, NULL, NULL); + + mpts->mpts_iss = sototcpcb(so)->iss; + + /* See tcp_connect_complete */ + if (mp_tp->mpt_state < MPTCPS_ESTABLISHED && + (mp_so->so_flags1 & SOF1_PRECONNECT_DATA)) { + mp_tp->mpt_sndwnd = sototcpcb(so)->snd_wnd; + } /* Allocate a unique address id per subflow */ mpte->mpte_addrid_last++; if (mpte->mpte_addrid_last == 0) mpte->mpte_addrid_last++; + proc_rele(p); + DTRACE_MPTCP3(subflow__connect, struct mptses *, mpte, struct mptsub *, mpts, int, error); + if (error) + mptcplog((LOG_ERR, "%s: connectx failed with error %d ifscope %u\n", + __func__, error, mpts->mpts_ifscope), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); return (error); } @@ -947,12 +1600,13 @@ mptcp_subflow_soreceive(struct socket *so, struct sockaddr **psa, struct uio *uio, struct mbuf **mp0, struct mbuf **controlp, int *flagsp) { #pragma unused(uio) + struct socket *mp_so = mptetoso(tptomptp(sototcpcb(so))->mpt_mpte); int flags, error = 0; struct proc *p = current_proc(); struct mbuf *m, **mp = mp0; - struct mbuf *nextrecord; + boolean_t proc_held = FALSE; - socket_lock(so, 1); + mpte_lock_assert_held(tptomptp(sototcpcb(so))->mpt_mpte); VERIFY(so->so_proto->pr_flags & PR_CONNREQUIRED); #ifdef MORE_LOCKING_DEBUG @@ -966,10 +1620,9 @@ mptcp_subflow_soreceive(struct socket *so, struct sockaddr **psa, * to the MPTCP layer, so we require that the caller passes in the * expected parameters. */ - if (mp == NULL || controlp != NULL) { - socket_unlock(so, 1); + if (mp == NULL || controlp != NULL) return (EINVAL); - } + *mp = NULL; if (psa != NULL) *psa = NULL; @@ -978,10 +1631,9 @@ mptcp_subflow_soreceive(struct socket *so, struct sockaddr **psa, else flags = 0; - if (flags & (MSG_PEEK|MSG_OOB|MSG_NEEDSA|MSG_WAITALL|MSG_WAITSTREAM)) { - socket_unlock(so, 1); + if (flags & (MSG_PEEK|MSG_OOB|MSG_NEEDSA|MSG_WAITALL|MSG_WAITSTREAM)) return (EOPNOTSUPP); - } + flags |= (MSG_DONTWAIT|MSG_NBIO); /* @@ -993,10 +1645,6 @@ mptcp_subflow_soreceive(struct socket *so, struct sockaddr **psa, struct sockbuf *sb = &so->so_rcv; error = ENOTCONN; - SODEFUNCTLOG("%s[%d, %s]: defunct so 0x%llx [%d,%d] (%d)\n", - __func__, proc_pid(p), proc_best_name(p), - (uint64_t)VM_KERNEL_ADDRPERM(so), - SOCK_DOM(so), SOCK_TYPE(so), error); /* * This socket should have been disconnected and flushed * prior to being returned from sodefunct(); there should @@ -1004,7 +1652,6 @@ mptcp_subflow_soreceive(struct socket *so, struct sockaddr **psa, */ if (so->so_state & SS_DEFUNCT) sb_empty_assert(sb, __func__); - socket_unlock(so, 1); return (error); } @@ -1025,20 +1672,16 @@ mptcp_subflow_soreceive(struct socket *so, struct sockaddr **psa, * socket is closed for real, SOF_MP_SUBFLOW would be cleared. */ if ((so->so_state & (SS_NOFDREF | SS_CANTRCVMORE)) == - (SS_NOFDREF | SS_CANTRCVMORE) && !(so->so_flags & SOF_MP_SUBFLOW)) { - socket_unlock(so, 1); + (SS_NOFDREF | SS_CANTRCVMORE) && !(so->so_flags & SOF_MP_SUBFLOW)) return (0); - } /* * For consistency with soreceive() semantics, we need to obey * SB_LOCK in case some other code path has locked the buffer. */ error = sblock(&so->so_rcv, 0); - if (error != 0) { - socket_unlock(so, 1); + if (error != 0) return (error); - } m = so->so_rcv.sb_mb; if (m == NULL) { @@ -1057,158 +1700,223 @@ mptcp_subflow_soreceive(struct socket *so, struct sockaddr **psa, goto release; } - if (so->so_state & SS_CANTRCVMORE) { - goto release; + if (so->so_state & SS_CANTRCVMORE) { + goto release; + } + + if (!(so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING))) { + error = ENOTCONN; + goto release; + } + + /* + * MSG_DONTWAIT is implicitly defined and this routine will + * never block, so return EWOULDBLOCK when there is nothing. + */ + error = EWOULDBLOCK; + goto release; + } + + mptcp_update_last_owner(so, mp_so); + + if (mp_so->last_pid != proc_pid(p)) { + p = proc_find(mp_so->last_pid); + if (p == PROC_NULL) { + p = current_proc(); + } else { + proc_held = TRUE; + } + } + + OSIncrementAtomicLong(&p->p_stats->p_ru.ru_msgrcv); + SBLASTRECORDCHK(&so->so_rcv, "mptcp_subflow_soreceive 1"); + SBLASTMBUFCHK(&so->so_rcv, "mptcp_subflow_soreceive 1"); + + while (m != NULL) { + int dlen = 0; + struct mbuf *start = m; + uint64_t dsn; + uint32_t sseq; + uint16_t orig_dlen; + uint16_t csum; + + VERIFY(m->m_nextpkt == NULL); + + if ((m->m_flags & M_PKTHDR) && (m->m_pkthdr.pkt_flags & PKTF_MPTCP)) { + orig_dlen = dlen = m->m_pkthdr.mp_rlen; + dsn = m->m_pkthdr.mp_dsn; + sseq = m->m_pkthdr.mp_rseq; + csum = m->m_pkthdr.mp_csum; + } else { + /* We did fallback */ + mptcp_adj_rmap(so, m, 0); + + sbfree(&so->so_rcv, m); + + if (mp != NULL) { + *mp = m; + mp = &m->m_next; + so->so_rcv.sb_mb = m = m->m_next; + *mp = NULL; + + } + + if (m != NULL) { + so->so_rcv.sb_lastrecord = m; + } else { + SB_EMPTY_FIXUP(&so->so_rcv); + } + + continue; } - if (!(so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING))) { - error = ENOTCONN; + /* + * Check if the full mapping is now present + */ + if ((int)so->so_rcv.sb_cc < dlen) { + mptcplog((LOG_INFO, "%s not enough data (%u) need %u\n", + __func__, so->so_rcv.sb_cc, dlen), + MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_LOG); + + if (*mp0 == NULL) + error = EWOULDBLOCK; goto release; } - /* - * MSG_DONTWAIT is implicitly defined and this routine will - * never block, so return EWOULDBLOCK when there is nothing. - */ - error = EWOULDBLOCK; - goto release; - } + /* Now, get the full mapping */ + while (dlen > 0) { + mptcp_adj_rmap(so, m, orig_dlen - dlen); - OSIncrementAtomicLong(&p->p_stats->p_ru.ru_msgrcv); - SBLASTRECORDCHK(&so->so_rcv, "mptcp_subflow_soreceive 1"); - SBLASTMBUFCHK(&so->so_rcv, "mptcp_subflow_soreceive 1"); + dlen -= m->m_len; + sbfree(&so->so_rcv, m); - while (m != NULL) { - nextrecord = m->m_nextpkt; - sbfree(&so->so_rcv, m); - - if (mp != NULL) { - *mp = m; - mp = &m->m_next; - so->so_rcv.sb_mb = m = m->m_next; - *mp = NULL; + if (mp != NULL) { + *mp = m; + mp = &m->m_next; + so->so_rcv.sb_mb = m = m->m_next; + *mp = NULL; + } + + VERIFY(dlen <= 0 || m); } + VERIFY(dlen == 0); + if (m != NULL) { - m->m_nextpkt = nextrecord; - if (nextrecord == NULL) - so->so_rcv.sb_lastrecord = m; + so->so_rcv.sb_lastrecord = m; } else { - m = so->so_rcv.sb_mb = nextrecord; SB_EMPTY_FIXUP(&so->so_rcv); } + + if (mptcp_validate_csum(sototcpcb(so), start, dsn, sseq, orig_dlen, csum)) { + error = EIO; + *mp0 = NULL; + goto release; + } + SBLASTRECORDCHK(&so->so_rcv, "mptcp_subflow_soreceive 2"); SBLASTMBUFCHK(&so->so_rcv, "mptcp_subflow_soreceive 2"); } DTRACE_MPTCP3(subflow__receive, struct socket *, so, struct sockbuf *, &so->so_rcv, struct sockbuf *, &so->so_snd); - /* notify protocol that we drained all the data */ - if ((so->so_proto->pr_flags & PR_WANTRCVD) && so->so_pcb != NULL) - (*so->so_proto->pr_usrreqs->pru_rcvd)(so, flags); if (flagsp != NULL) *flagsp |= flags; release: - sbunlock(&so->so_rcv, FALSE); /* will unlock socket */ + sbunlock(&so->so_rcv, TRUE); + + if (proc_held) + proc_rele(p); + return (error); } - /* - * Prepare an MPTCP subflow socket for peeloff(2); basically undo - * the work done earlier when the subflow socket was created. + * MPTCP subflow socket send routine, derived from sosend(). */ -void -mptcp_subflow_sopeeloff(struct mptses *mpte, struct mptsub *mpts, - struct socket *so) +static int +mptcp_subflow_sosend(struct socket *so, struct sockaddr *addr, struct uio *uio, + struct mbuf *top, struct mbuf *control, int flags) { - struct mptopt smpo; - struct socket *mp_so; - int p, c; - - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - mp_so = mpte->mpte_mppcb->mpp_socket; - MPTS_LOCK_ASSERT_HELD(mpts); + struct socket *mp_so = mptetoso(tptomptp(sototcpcb(so))->mpt_mpte); + struct proc *p = current_proc(); + boolean_t en_tracing = FALSE, proc_held = FALSE; + int en_tracing_val; + int sblocked = 1; /* Pretend as if it is already locked, so we won't relock it */ + int error; - socket_lock(so, 0); - VERIFY(so->so_flags & SOF_MP_SUBFLOW); - VERIFY((so->so_state & (SS_NBIO|SS_NOFDREF)) == (SS_NBIO|SS_NOFDREF)); + VERIFY(control == NULL); + VERIFY(addr == NULL); + VERIFY(uio == NULL); + VERIFY(flags == 0); + VERIFY((so->so_flags & SOF_CONTENT_FILTER) == 0); - /* inherit MPTCP socket states */ - if (!(mp_so->so_state & SS_NBIO)) - so->so_state &= ~SS_NBIO; + VERIFY(top->m_pkthdr.len > 0 && top->m_pkthdr.len <= UINT16_MAX); + VERIFY(top->m_pkthdr.pkt_flags & PKTF_MPTCP); /* - * At this point, the socket is not yet closed, as there is at least - * one outstanding usecount previously held by mpts_socket from - * socreate(). Atomically clear SOF_MP_SUBFLOW and SS_NOFDREF here. + * trace if tracing & network (vs. unix) sockets & and + * non-loopback */ - so->so_flags &= ~SOF_MP_SUBFLOW; - so->so_state &= ~SS_NOFDREF; - so->so_flags &= ~SOF_MPTCP_TRUE; + if (ENTR_SHOULDTRACE && + (SOCK_CHECK_DOM(so, AF_INET) || SOCK_CHECK_DOM(so, AF_INET6))) { + struct inpcb *inp = sotoinpcb(so); + if (inp->inp_last_outifp != NULL && + !(inp->inp_last_outifp->if_flags & IFF_LOOPBACK)) { + en_tracing = TRUE; + en_tracing_val = top->m_pkthdr.len; + KERNEL_ENERGYTRACE(kEnTrActKernSockWrite, DBG_FUNC_START, + VM_KERNEL_ADDRPERM(so), + ((so->so_state & SS_NBIO) ? kEnTrFlagNonBlocking : 0), + (int64_t)en_tracing_val); + } + } - /* allow socket buffers to be compressed */ - so->so_rcv.sb_flags &= ~SB_NOCOMPRESS; - so->so_snd.sb_flags &= ~SB_NOCOMPRESS; + mptcp_update_last_owner(so, mp_so); - /* - * Allow socket buffer auto sizing. - * - * This will increase the current 64k buffer size to whatever is best. - */ - if (!(so->so_rcv.sb_flags & SB_USRSIZE)) - so->so_rcv.sb_flags |= SB_AUTOSIZE; - if (!(so->so_snd.sb_flags & SB_USRSIZE)) - so->so_snd.sb_flags |= SB_AUTOSIZE; + if (mp_so->last_pid != proc_pid(p)) { + p = proc_find(mp_so->last_pid); + if (p == PROC_NULL) { + p = current_proc(); + } else { + proc_held = TRUE; + } + } - /* restore protocol-user requests */ - VERIFY(mpts->mpts_oprotosw != NULL); - so->so_proto = mpts->mpts_oprotosw; +#if NECP + inp_update_necp_policy(sotoinpcb(so), NULL, NULL, 0); +#endif /* NECP */ - bzero(&smpo, sizeof (smpo)); - smpo.mpo_flags |= MPOF_SUBFLOW_OK; - smpo.mpo_level = SOL_SOCKET; + OSIncrementAtomicLong(&p->p_stats->p_ru.ru_msgsnd); - /* inherit SOF_NOSIGPIPE from parent MP socket */ - p = (mp_so->so_flags & SOF_NOSIGPIPE); - c = (so->so_flags & SOF_NOSIGPIPE); - smpo.mpo_intval = ((p - c) > 0) ? 1 : 0; - smpo.mpo_name = SO_NOSIGPIPE; - if ((p - c) != 0) - (void) mptcp_subflow_sosetopt(mpte, so, &smpo); + error = sosendcheck(so, NULL, top->m_pkthdr.len, 0, 1, 0, &sblocked, NULL); + if (error) + goto out; - /* inherit SOF_NOADDRAVAIL from parent MP socket */ - p = (mp_so->so_flags & SOF_NOADDRAVAIL); - c = (so->so_flags & SOF_NOADDRAVAIL); - smpo.mpo_intval = ((p - c) > 0) ? 1 : 0; - smpo.mpo_name = SO_NOADDRERR; - if ((p - c) != 0) - (void) mptcp_subflow_sosetopt(mpte, so, &smpo); + error = (*so->so_proto->pr_usrreqs->pru_send)(so, 0, top, NULL, NULL, p); + top = NULL; - /* inherit SO_KEEPALIVE from parent MP socket */ - p = (mp_so->so_options & SO_KEEPALIVE); - c = (so->so_options & SO_KEEPALIVE); - smpo.mpo_intval = ((p - c) > 0) ? 1 : 0; - smpo.mpo_name = SO_KEEPALIVE; - if ((p - c) != 0) - (void) mptcp_subflow_sosetopt(mpte, so, &smpo); +out: + if (top != NULL) + m_freem(top); - /* unset TCP level default keepalive option */ - p = (intotcpcb(sotoinpcb(mp_so)))->t_keepidle; - c = (intotcpcb(sotoinpcb(so)))->t_keepidle; - smpo.mpo_level = IPPROTO_TCP; - smpo.mpo_intval = 0; - smpo.mpo_name = TCP_KEEPALIVE; - if ((p - c) != 0) - (void) mptcp_subflow_sosetopt(mpte, so, &smpo); - socket_unlock(so, 0); + if (proc_held) + proc_rele(p); + + soclearfastopen(so); + + if (en_tracing) { + KERNEL_ENERGYTRACE(kEnTrActKernSockWrite, DBG_FUNC_END, + VM_KERNEL_ADDRPERM(so), + ((error == EWOULDBLOCK) ? kEnTrFlagNoWork : 0), + (int64_t)en_tracing_val); + } + + return (error); - DTRACE_MPTCP5(subflow__peeloff, struct mptses *, mpte, - struct mptsub *, mpts, struct socket *, so, - struct sockbuf *, &so->so_rcv, struct sockbuf *, &so->so_snd); } /* @@ -1216,59 +1924,70 @@ mptcp_subflow_sopeeloff(struct mptses *mpte, struct mptsub *mpts, * connected), or add a subflow to an existing MPTCP connection. */ int -mptcp_subflow_add(struct mptses *mpte, struct mptsub *mpts, - struct proc *p, uint32_t ifscope) +mptcp_subflow_add(struct mptses *mpte, struct sockaddr *src, + struct sockaddr *dst, uint32_t ifscope, sae_connid_t *pcid) { struct socket *mp_so, *so = NULL; - struct mptsub_connreq mpcr; struct mptcb *mp_tp; + struct mptsub *mpts = NULL; int af, error = 0; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - mp_so = mpte->mpte_mppcb->mpp_socket; + mpte_lock_assert_held(mpte); /* same as MP socket lock */ + mp_so = mptetoso(mpte); mp_tp = mpte->mpte_mptcb; - MPT_LOCK(mp_tp); if (mp_tp->mpt_state >= MPTCPS_CLOSE_WAIT) { /* If the remote end sends Data FIN, refuse subflow adds */ + mptcplog((LOG_ERR, "%s state %u\n", __func__, mp_tp->mpt_state), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); error = ENOTCONN; - MPT_UNLOCK(mp_tp); - return (error); + goto out_err; } - MPT_UNLOCK(mp_tp); - - MPTS_LOCK(mpts); - VERIFY(!(mpts->mpts_flags & (MPTSF_CONNECTING|MPTSF_CONNECTED))); - VERIFY(mpts->mpts_mpte == NULL); - VERIFY(mpts->mpts_socket == NULL); - VERIFY(mpts->mpts_dst != NULL); - VERIFY(mpts->mpts_connid == SAE_CONNID_ANY); - af = mpts->mpts_family = mpts->mpts_dst->sa_family; + mpts = mptcp_subflow_alloc(); + if (mpts == NULL) { + mptcplog((LOG_ERR, "%s malloc subflow failed\n", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + error = ENOMEM; + goto out_err; + } - /* - * If the source address is not specified, allocate a storage for - * it, so that later on we can fill it in with the actual source - * IP address chosen by the underlying layer for the subflow after - * it is connected. - */ - if (mpts->mpts_src == NULL) { - int len = mpts->mpts_dst->sa_len; + if (src != NULL) { + int len = src->sa_len; MALLOC(mpts->mpts_src, struct sockaddr *, len, M_SONAME, M_WAITOK | M_ZERO); if (mpts->mpts_src == NULL) { - error = ENOBUFS; - goto out; + mptcplog((LOG_ERR, "%s malloc mpts_src failed", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + error = ENOMEM; + goto out_err; } - bzero(mpts->mpts_src, len); - mpts->mpts_src->sa_len = len; - mpts->mpts_src->sa_family = mpts->mpts_dst->sa_family; + bcopy(src, mpts->mpts_src, len); } + memcpy(&mpts->mpts_dst, dst, dst->sa_len); + + af = mpts->mpts_dst.sa_family; + + mpts->mpts_ifscope = ifscope; + /* create the subflow socket */ - if ((error = mptcp_subflow_socreate(mpte, mpts, af, p, &so)) != 0) - goto out; + if ((error = mptcp_subflow_socreate(mpte, mpts, af, &so)) != 0) + /* + * Returning (error) and not cleaning up, because up to here + * all we did is creating mpts. + * + * And the contract is that the call to mptcp_subflow_socreate, + * moves ownership of mpts to mptcp_subflow_socreate. + */ + return (error); + + /* + * We may be called from within the kernel. Still need to account this + * one to the real app. + */ + mptcp_update_last_owner(mpts->mpts_socket, mp_so); /* * Increment the counter, while avoiding 0 (SAE_CONNID_ANY) and @@ -1280,8 +1999,6 @@ mptcp_subflow_add(struct mptses *mpte, struct mptsub *mpts, mpte->mpte_connid_last++; mpts->mpts_connid = mpte->mpte_connid_last; - VERIFY(mpts->mpts_connid != SAE_CONNID_ANY && - mpts->mpts_connid != SAE_CONNID_ALL); mpts->mpts_rel_seq = 1; @@ -1290,169 +2007,45 @@ mptcp_subflow_add(struct mptses *mpte, struct mptsub *mpts, if (mpte->mpte_addrid_last == 0) mpte->mpte_addrid_last++; - /* bind subflow socket to the specified interface */ - if (ifscope != IFSCOPE_NONE) { - socket_lock(so, 0); - error = inp_bindif(sotoinpcb(so), ifscope, &mpts->mpts_outif); - if (error != 0) { - socket_unlock(so, 0); - (void) mptcp_subflow_soclose(mpts, so); - goto out; - } - VERIFY(mpts->mpts_outif != NULL); - mpts->mpts_flags |= MPTSF_BOUND_IF; - - if (IFNET_IS_EXPENSIVE(mpts->mpts_outif)) { - sototcpcb(so)->t_mpflags |= TMPF_BACKUP_PATH; - } else { - mpts->mpts_flags |= MPTSF_PREFERRED; - } - - mptcplog((LOG_DEBUG, "MPTCP Socket: subflow_add mp_so 0x%llx " - "bindif %s[%d] cid %d expensive %d\n", - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), - mpts->mpts_outif->if_xname, - ifscope, mpts->mpts_connid, - IFNET_IS_EXPENSIVE(mpts->mpts_outif)), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); - socket_unlock(so, 0); - } - - /* if source address and/or port is specified, bind to it */ - if (mpts->mpts_src != NULL) { - struct sockaddr *sa = mpts->mpts_src; - uint32_t mpts_flags = 0; - in_port_t lport; - - switch (af) { - case AF_INET: - if (SIN(sa)->sin_addr.s_addr != INADDR_ANY) - mpts_flags |= MPTSF_BOUND_IP; - if ((lport = SIN(sa)->sin_port) != 0) - mpts_flags |= MPTSF_BOUND_PORT; - break; -#if INET6 - case AF_INET6: - VERIFY(af == AF_INET6); - if (!IN6_IS_ADDR_UNSPECIFIED(&SIN6(sa)->sin6_addr)) - mpts_flags |= MPTSF_BOUND_IP; - if ((lport = SIN6(sa)->sin6_port) != 0) - mpts_flags |= MPTSF_BOUND_PORT; - break; -#endif /* INET6 */ - } - - error = sobindlock(so, sa, 1); /* will lock/unlock socket */ - if (error != 0) { - (void) mptcp_subflow_soclose(mpts, so); - goto out; - } - mpts->mpts_flags |= mpts_flags; - - if (af == AF_INET || af == AF_INET6) { - char sbuf[MAX_IPv6_STR_LEN]; - - mptcplog((LOG_DEBUG, "MPTCP Socket: subflow_add " - "mp_so 0x%llx bindip %s[%d] cid %d\n", - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), - inet_ntop(af, ((af == AF_INET) ? - (void *)&SIN(sa)->sin_addr.s_addr : - (void *)&SIN6(sa)->sin6_addr), sbuf, sizeof (sbuf)), - ntohs(lport), mpts->mpts_connid), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); - } - } - - /* - * Insert the subflow into the list, and associate the MPTCP PCB - * as well as the the subflow socket. From this point on, removing - * the subflow needs to be done via mptcp_subflow_del(). - */ - TAILQ_INSERT_TAIL(&mpte->mpte_subflows, mpts, mpts_entry); - mpte->mpte_numflows++; - - atomic_bitset_32(&mpts->mpts_flags, MPTSF_ATTACHED); - mpts->mpts_mpte = mpte; - mpts->mpts_socket = so; - MPTS_ADDREF_LOCKED(mpts); /* for being in MPTCP subflow list */ - MPTS_ADDREF_LOCKED(mpts); /* for subflow socket */ - mp_so->so_usecount++; /* for subflow socket */ - /* register for subflow socket read/write events */ - (void) sock_setupcalls(so, mptcp_subflow_rupcall, mpts, - mptcp_subflow_wupcall, mpts); + sock_setupcalls_locked(so, mptcp_subflow_rupcall, mpts, mptcp_subflow_wupcall, mpts, 1); - /* - * Register for subflow socket control events; ignore - * SO_FILT_HINT_CONNINFO_UPDATED from below since we - * will generate it here. - */ - (void) sock_catchevents(so, mptcp_subflow_eupcall, mpts, + /* Register for subflow socket control events */ + sock_catchevents_locked(so, mptcp_subflow_eupcall1, mpts, SO_FILT_HINT_CONNRESET | SO_FILT_HINT_CANTRCVMORE | - SO_FILT_HINT_CANTSENDMORE | SO_FILT_HINT_TIMEOUT | - SO_FILT_HINT_NOSRCADDR | SO_FILT_HINT_IFDENIED | - SO_FILT_HINT_SUSPEND | SO_FILT_HINT_RESUME | - SO_FILT_HINT_CONNECTED | SO_FILT_HINT_DISCONNECTED | - SO_FILT_HINT_MPFAILOVER | SO_FILT_HINT_MPSTATUS | - SO_FILT_HINT_MUSTRST | SO_FILT_HINT_MPFASTJ | - SO_FILT_HINT_DELETEOK | SO_FILT_HINT_MPCANTRCVMORE); + SO_FILT_HINT_TIMEOUT | SO_FILT_HINT_NOSRCADDR | + SO_FILT_HINT_IFDENIED | SO_FILT_HINT_CONNECTED | + SO_FILT_HINT_DISCONNECTED | SO_FILT_HINT_MPFAILOVER | + SO_FILT_HINT_MPSTATUS | SO_FILT_HINT_MUSTRST | + SO_FILT_HINT_MPCANTRCVMORE | SO_FILT_HINT_ADAPTIVE_RTIMO | + SO_FILT_HINT_ADAPTIVE_WTIMO); /* sanity check */ VERIFY(!(mpts->mpts_flags & (MPTSF_CONNECTING|MPTSF_CONNECTED|MPTSF_CONNECT_PENDING))); - bzero(&mpcr, sizeof (mpcr)); - mpcr.mpcr_proc = p; - mpcr.mpcr_ifscope = ifscope; /* * Indicate to the TCP subflow whether or not it should establish * the initial MPTCP connection, or join an existing one. Fill * in the connection request structure with additional info needed * by the underlying TCP (to be used in the TCP options, etc.) */ - MPT_LOCK(mp_tp); if (mp_tp->mpt_state < MPTCPS_ESTABLISHED && mpte->mpte_numflows == 1) { + mpts->mpts_flags |= MPTSF_INITIAL_SUB; + if (mp_tp->mpt_state == MPTCPS_CLOSED) { - mptcp_init_local_parms(mp_tp); + mptcp_init_local_parms(mpte); } - MPT_UNLOCK(mp_tp); soisconnecting(mp_so); - mpcr.mpcr_type = MPTSUB_CONNREQ_MP_ENABLE; + + /* If fastopen is requested, set state in mpts */ + if (so->so_flags1 & SOF1_PRECONNECT_DATA) + mpts->mpts_flags |= MPTSF_TFO_REQD; } else { if (!(mp_tp->mpt_flags & MPTCPF_JOIN_READY)) mpts->mpts_flags |= MPTSF_CONNECT_PENDING; - - /* avoid starting up cellular subflow unless required */ - if ((mptcp_delayed_subf_start) && - (IFNET_IS_CELLULAR(mpts->mpts_outif))) { - mpts->mpts_flags |= MPTSF_CONNECT_PENDING; - } - MPT_UNLOCK(mp_tp); - mpcr.mpcr_type = MPTSUB_CONNREQ_MP_ADD; - } - - /* If fastjoin or fastopen is requested, set state in mpts */ - if (mpte->mpte_nummpcapflows == 0) { - if (so->so_flags1 & SOF1_PRECONNECT_DATA) { - MPT_LOCK(mp_tp); - if (mp_tp->mpt_state < MPTCPS_ESTABLISHED) { - mpts->mpts_flags |= MPTSF_TFO_REQD; - mpts->mpts_sndnxt = mp_tp->mpt_snduna; - } - MPT_UNLOCK(mp_tp); - } - - if (so->so_flags & SOF_MPTCP_FASTJOIN) { - MPT_LOCK(mp_tp); - if (mp_tp->mpt_state == MPTCPS_ESTABLISHED) { - mpts->mpts_flags |= MPTSF_FASTJ_REQD; - mpts->mpts_sndnxt = mp_tp->mpt_snduna; - } - MPT_UNLOCK(mp_tp); - } } - mpts->mpts_mpcr = mpcr; mpts->mpts_flags |= MPTSF_CONNECTING; if (af == AF_INET || af == AF_INET6) { @@ -1463,76 +2056,84 @@ mptcp_subflow_add(struct mptses *mpte, struct mptsub *mpts, "[pending %s]\n", __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), inet_ntop(af, ((af == AF_INET) ? - (void *)&SIN(mpts->mpts_dst)->sin_addr.s_addr : - (void *)&SIN6(mpts->mpts_dst)->sin6_addr), + (void *)&SIN(&mpts->mpts_dst)->sin_addr.s_addr : + (void *)&SIN6(&mpts->mpts_dst)->sin6_addr), dbuf, sizeof (dbuf)), ((af == AF_INET) ? - ntohs(SIN(mpts->mpts_dst)->sin_port) : - ntohs(SIN6(mpts->mpts_dst)->sin6_port)), + ntohs(SIN(&mpts->mpts_dst)->sin_port) : + ntohs(SIN6(&mpts->mpts_dst)->sin6_port)), mpts->mpts_connid, ((mpts->mpts_flags & MPTSF_CONNECT_PENDING) ? "YES" : "NO")), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); } /* connect right away if first attempt, or if join can be done now */ if (!(mpts->mpts_flags & MPTSF_CONNECT_PENDING)) error = mptcp_subflow_soconnectx(mpte, mpts); -out: - MPTS_UNLOCK(mpts); - if (error == 0) { - soevent(mp_so, SO_FILT_HINT_LOCKED | - SO_FILT_HINT_CONNINFO_UPDATED); - } + if (error) + goto out_err_close; + + if (pcid) + *pcid = mpts->mpts_connid; + + return (0); + +out_err_close: + mptcp_subflow_abort(mpts, error); + + return (error); + +out_err: + if (mpts) + mptcp_subflow_free(mpts); + return (error); } +void +mptcpstats_update(struct mptcp_itf_stats *stats, struct mptsub *mpts) +{ + int index = mptcp_get_statsindex(stats, mpts); + + if (index != -1) { + struct inpcb *inp = sotoinpcb(mpts->mpts_socket); + + stats[index].mpis_txbytes += inp->inp_stat->txbytes; + stats[index].mpis_rxbytes += inp->inp_stat->rxbytes; + } +} + /* * Delete/remove a subflow from an MPTCP. The underlying subflow socket * will no longer be accessible after a subflow is deleted, thus this * should occur only after the subflow socket has been disconnected. - * If peeloff(2) is called, leave the socket open. */ void -mptcp_subflow_del(struct mptses *mpte, struct mptsub *mpts, boolean_t close) +mptcp_subflow_del(struct mptses *mpte, struct mptsub *mpts) { - struct socket *mp_so, *so; - - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - mp_so = mpte->mpte_mppcb->mpp_socket; - - MPTS_LOCK(mpts); - so = mpts->mpts_socket; - VERIFY(so != NULL); + struct socket *mp_so = mptetoso(mpte); + struct socket *so = mpts->mpts_socket; + struct tcpcb *tp = sototcpcb(so); - if (close && !((mpts->mpts_flags & MPTSF_DELETEOK) && - (mpts->mpts_flags & MPTSF_USER_DISCONNECT))) { - MPTS_UNLOCK(mpts); - mptcplog((LOG_DEBUG, "MPTCP Socket: subflow_del returning" - " mp_so 0x%llx flags %x\n", - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), mpts->mpts_flags), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); - return; - } + mpte_lock_assert_held(mpte); /* same as MP socket lock */ + VERIFY(mpts->mpts_mpte == mpte); + VERIFY(mpts->mpts_flags & MPTSF_ATTACHED); + VERIFY(mpte->mpte_numflows != 0); + VERIFY(mp_so->so_usecount > 0); - mptcplog((LOG_DEBUG, "MPTCP Socket: subflow_del mp_so 0x%llx " - "[u=%d,r=%d] cid %d [close %s] %d %x error %d\n", - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), - mp_so->so_usecount, - mp_so->so_retaincnt, mpts->mpts_connid, - (close ? "YES" : "NO"), mpts->mpts_soerror, - mpts->mpts_flags, - mp_so->so_error), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); + mptcplog((LOG_DEBUG, "%s: mp_so 0x%llx [u=%d,r=%d] cid %d %x error %d\n", + __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), + mp_so->so_usecount, mp_so->so_retaincnt, mpts->mpts_connid, + mpts->mpts_flags, mp_so->so_error), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); - VERIFY(mpts->mpts_mpte == mpte); - VERIFY(mpts->mpts_connid != SAE_CONNID_ANY && - mpts->mpts_connid != SAE_CONNID_ALL); + mptcpstats_update(mpte->mpte_itfstats, mpts); + mpte->mpte_init_rxbytes = sotoinpcb(so)->inp_stat->rxbytes; + mpte->mpte_init_txbytes = sotoinpcb(so)->inp_stat->txbytes; - VERIFY(mpts->mpts_flags & MPTSF_ATTACHED); atomic_bitclear_32(&mpts->mpts_flags, MPTSF_ATTACHED); TAILQ_REMOVE(&mpte->mpte_subflows, mpts, mpts_entry); - VERIFY(mpte->mpte_numflows != 0); mpte->mpte_numflows--; if (mpte->mpte_active_sub == mpts) mpte->mpte_active_sub = NULL; @@ -1541,73 +2142,94 @@ mptcp_subflow_del(struct mptses *mpte, struct mptsub *mpts, boolean_t close) * Drop references held by this subflow socket; there * will be no further upcalls made from this point. */ - (void) sock_setupcalls(so, NULL, NULL, NULL, NULL); - (void) sock_catchevents(so, NULL, NULL, 0); + sock_setupcalls_locked(so, NULL, NULL, NULL, NULL, 0); + sock_catchevents_locked(so, NULL, NULL, 0); mptcp_detach_mptcb_from_subf(mpte->mpte_mptcb, so); - if (close) - (void) mptcp_subflow_soclose(mpts, so); - - VERIFY(mp_so->so_usecount > 0); mp_so->so_usecount--; /* for subflow socket */ mpts->mpts_mpte = NULL; mpts->mpts_socket = NULL; - MPTS_UNLOCK(mpts); - MPTS_REMREF(mpts); /* for MPTCP subflow list */ - MPTS_REMREF(mpts); /* for subflow socket */ + mptcp_subflow_remref(mpts); /* for MPTCP subflow list */ + mptcp_subflow_remref(mpts); /* for subflow socket */ + + so->so_flags &= ~SOF_MP_SUBFLOW; + tp->t_mptcb = NULL; + tp->t_mpsub = NULL; +} + +void +mptcp_subflow_shutdown(struct mptses *mpte, struct mptsub *mpts) +{ + struct socket *so = mpts->mpts_socket; + struct mptcb *mp_tp = mpte->mpte_mptcb; + int send_dfin = 0; + + if (mp_tp->mpt_state > MPTCPS_CLOSE_WAIT) + send_dfin = 1; + + if (!(so->so_state & (SS_ISDISCONNECTING | SS_ISDISCONNECTED)) && + (so->so_state & SS_ISCONNECTED)) { + mptcplog((LOG_DEBUG, "MPTCP subflow shutdown %s: cid %d fin %d\n", + __func__, mpts->mpts_connid, send_dfin), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + + if (send_dfin) + mptcp_send_dfin(so); + soshutdownlock(so, SHUT_WR); + } + +} + +static void +mptcp_subflow_abort(struct mptsub *mpts, int error) +{ + struct socket *so = mpts->mpts_socket; + struct tcpcb *tp = sototcpcb(so); + + if (mpts->mpts_flags & MPTSF_DISCONNECTED) + return; + + mptcplog((LOG_DEBUG, "%s aborting connection state %u\n", __func__, tp->t_state), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); - soevent(mp_so, SO_FILT_HINT_LOCKED | SO_FILT_HINT_CONNINFO_UPDATED); + if (tp->t_state != TCPS_CLOSED) + tcp_drop(tp, error); + + mptcp_subflow_eupcall1(so, mpts, SO_FILT_HINT_DISCONNECTED); } /* * Disconnect a subflow socket. */ void -mptcp_subflow_disconnect(struct mptses *mpte, struct mptsub *mpts, - boolean_t deleteok) +mptcp_subflow_disconnect(struct mptses *mpte, struct mptsub *mpts) { struct socket *so; struct mptcb *mp_tp; int send_dfin = 0; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); + mpte_lock_assert_held(mpte); /* same as MP socket lock */ VERIFY(mpts->mpts_mpte == mpte); VERIFY(mpts->mpts_socket != NULL); - VERIFY(mpts->mpts_connid != SAE_CONNID_ANY && - mpts->mpts_connid != SAE_CONNID_ALL); if (mpts->mpts_flags & (MPTSF_DISCONNECTING|MPTSF_DISCONNECTED)) return; mpts->mpts_flags |= MPTSF_DISCONNECTING; - /* - * If this is coming from disconnectx(2) or issued as part of - * closing the MPTCP socket, the subflow shouldn't stick around. - * Otherwise let it linger around in case the upper layers need - * to retrieve its conninfo. - */ - if (deleteok) - mpts->mpts_flags |= MPTSF_DELETEOK; - so = mpts->mpts_socket; mp_tp = mpte->mpte_mptcb; - MPT_LOCK(mp_tp); - if (mp_tp->mpt_state > MPTCPS_ESTABLISHED) + if (mp_tp->mpt_state > MPTCPS_CLOSE_WAIT) send_dfin = 1; - MPT_UNLOCK(mp_tp); - socket_lock(so, 0); if (!(so->so_state & (SS_ISDISCONNECTING | SS_ISDISCONNECTED)) && (so->so_state & SS_ISCONNECTED)) { - mptcplog((LOG_DEBUG, "MPTCP Socket %s: cid %d fin %d " - "[linger %s]\n", __func__, mpts->mpts_connid, send_dfin, - (deleteok ? "NO" : "YES")), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); + mptcplog((LOG_DEBUG, "MPTCP Socket %s: cid %d fin %d\n", + __func__, mpts->mpts_connid, send_dfin), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); if (send_dfin) mptcp_send_dfin(so); @@ -1615,154 +2237,134 @@ mptcp_subflow_disconnect(struct mptses *mpte, struct mptsub *mpts, (void) soshutdownlock(so, SHUT_WR); (void) sodisconnectlocked(so); } - socket_unlock(so, 0); /* * Generate a disconnect event for this subflow socket, in case * the lower layer doesn't do it; this is needed because the - * subflow socket deletion relies on it. This will also end up - * generating SO_FILT_HINT_CONNINFO_UPDATED on the MPTCP socket; - * we cannot do that here because subflow lock is currently held. + * subflow socket deletion relies on it. */ - mptcp_subflow_eupcall(so, mpts, SO_FILT_HINT_DISCONNECTED); + mptcp_subflow_eupcall1(so, mpts, SO_FILT_HINT_DISCONNECTED); } /* - * Subflow socket read upcall. - * - * Called when the associated subflow socket posted a read event. The subflow - * socket lock has been released prior to invoking the callback. Note that the - * upcall may occur synchronously as a result of MPTCP performing an action on - * it, or asynchronously as a result of an event happening at the subflow layer. - * Therefore, to maintain lock ordering, the only lock that can be acquired - * here is the thread lock, for signalling purposes. + * Called when the associated subflow socket posted a read event. */ static void mptcp_subflow_rupcall(struct socket *so, void *arg, int waitf) { #pragma unused(so, waitf) - struct mptsub *mpts = arg; + struct mptsub *mpts = arg, *tmpts; struct mptses *mpte = mpts->mpts_mpte; - /* - * mpte should never be NULL, except in a race with - * mptcp_subflow_del - */ - if (mpte == NULL) + VERIFY(mpte != NULL); + + if (mptcp_should_defer_upcall(mpte->mpte_mppcb)) { + if (!(mpte->mpte_mppcb->mpp_flags & MPP_RUPCALL)) + mpte->mpte_mppcb->mpp_flags |= MPP_SHOULD_RWAKEUP; return; + } + + mpte->mpte_mppcb->mpp_flags |= MPP_RUPCALL; + TAILQ_FOREACH_SAFE(mpts, &mpte->mpte_subflows, mpts_entry, tmpts) { + if (mpts->mpts_socket->so_usecount == 0) { + /* Will be removed soon by tcp_garbage_collect */ + continue; + } + + mptcp_subflow_addref(mpts); + mpts->mpts_socket->so_usecount++; - lck_mtx_lock(&mpte->mpte_thread_lock); - mptcp_thread_signal_locked(mpte); - lck_mtx_unlock(&mpte->mpte_thread_lock); + mptcp_subflow_input(mpte, mpts); + + mptcp_subflow_remref(mpts); /* ours */ + + VERIFY(mpts->mpts_socket->so_usecount != 0); + mpts->mpts_socket->so_usecount--; + } + + mptcp_handle_deferred_upcalls(mpte->mpte_mppcb, MPP_RUPCALL); } /* * Subflow socket input. - * - * Called in the context of the MPTCP thread, for reading data from the - * underlying subflow socket and delivering it to MPTCP. */ static void mptcp_subflow_input(struct mptses *mpte, struct mptsub *mpts) { + struct socket *mp_so = mptetoso(mpte); struct mbuf *m = NULL; struct socket *so; - int error; - struct mptsub *mpts_alt = NULL; + int error, wakeup = 0; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); + VERIFY(!(mpte->mpte_mppcb->mpp_flags & MPP_INSIDE_INPUT)); + mpte->mpte_mppcb->mpp_flags |= MPP_INSIDE_INPUT; DTRACE_MPTCP2(subflow__input, struct mptses *, mpte, struct mptsub *, mpts); if (!(mpts->mpts_flags & MPTSF_CONNECTED)) - return; + goto out; so = mpts->mpts_socket; error = sock_receive_internal(so, NULL, &m, 0, NULL); if (error != 0 && error != EWOULDBLOCK) { - mptcplog((LOG_ERR, "MPTCP Receiver: %s cid %d error %d\n", + mptcplog((LOG_ERR, "%s: cid %d error %d\n", __func__, mpts->mpts_connid, error), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_ERR); - MPTS_UNLOCK(mpts); - mpts_alt = mptcp_get_subflow(mpte, mpts, NULL); - if (mpts_alt == NULL) { - if (mptcp_delayed_subf_start) { - mpts_alt = mptcp_get_pending_subflow(mpte, - mpts); - if (mpts_alt) { - mptcplog((LOG_DEBUG,"MPTCP Receiver:" - " %s: pending %d\n", - __func__, mpts_alt->mpts_connid), - MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_ERR); - } else { - mptcplog((LOG_ERR, "MPTCP Receiver:" - " %s: no pending flow for cid %d", - __func__, mpts->mpts_connid), - MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_ERR); - } - } else { - mptcplog((LOG_ERR, "MPTCP Receiver: %s: no alt" - " path for cid %d\n", __func__, - mpts->mpts_connid), - MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_ERR); - } - if (error == ENODATA) { - /* - * Don't ignore ENODATA so as to discover - * nasty middleboxes. - */ - struct socket *mp_so = - mpte->mpte_mppcb->mpp_socket; - mp_so->so_error = ENODATA; - sorwakeup(mp_so); - } + if (error == ENODATA) { + /* + * Don't ignore ENODATA so as to discover + * nasty middleboxes. + */ + mp_so->so_error = ENODATA; + + wakeup = 1; + goto out; } - MPTS_LOCK(mpts); } else if (error == 0) { - mptcplog((LOG_DEBUG, "MPTCP Receiver: %s: cid %d \n", - __func__, mpts->mpts_connid), + mptcplog((LOG_DEBUG, "%s: cid %d \n", __func__, mpts->mpts_connid), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE); } /* In fallback, make sure to accept data on all but one subflow */ - if ((mpts->mpts_flags & MPTSF_MP_DEGRADED) && - (!(mpts->mpts_flags & MPTSF_ACTIVE))) { + if (m && (mpts->mpts_flags & MPTSF_MP_DEGRADED) && + !(mpts->mpts_flags & MPTSF_ACTIVE)) { + mptcplog((LOG_DEBUG, "%s: degraded and got data on non-active flow\n", + __func__), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE); m_freem(m); - return; + goto out; } if (m != NULL) { + if (IFNET_IS_CELLULAR(sotoinpcb(so)->inp_last_outifp)) { + mpte->mpte_mppcb->mpp_flags |= MPP_SET_CELLICON; - /* Did we receive data on the backup subflow? */ - if (!(mpts->mpts_flags & MPTSF_ACTIVE)) - mpts->mpts_peerswitch++; - else - mpts->mpts_peerswitch = 0; + mpte->mpte_used_cell = 1; + } else { + mpte->mpte_mppcb->mpp_flags |= MPP_UNSET_CELLICON; + + mpte->mpte_used_wifi = 1; + } - /* - * Release subflow lock since this may trigger MPTCP to send, - * possibly on a different subflow. An extra reference has - * been held on the subflow by the MPTCP thread before coming - * here, so we can be sure that it won't go away, in the event - * the MP socket lock gets released. - */ - MPTS_UNLOCK(mpts); mptcp_input(mpte, m); - MPTS_LOCK(mpts); } + + /* notify protocol that we drained all the data */ + if (error == 0 && m != NULL && + (so->so_proto->pr_flags & PR_WANTRCVD) && so->so_pcb != NULL) + (*so->so_proto->pr_usrreqs->pru_rcvd)(so, 0); + +out: + if (wakeup) + mpte->mpte_mppcb->mpp_flags |= MPP_SHOULD_RWAKEUP; + + mptcp_handle_deferred_upcalls(mpte->mpte_mppcb, MPP_INSIDE_INPUT); } /* * Subflow socket write upcall. * - * Called when the associated subflow socket posted a read event. The subflow - * socket lock has been released prior to invoking the callback. Note that the - * upcall may occur synchronously as a result of MPTCP performing an action on - * it, or asynchronously as a result of an event happening at the subflow layer. - * Therefore, to maintain lock ordering, the only lock that can be acquired - * here is the thread lock, for signalling purposes. + * Called when the associated subflow socket posted a read event. */ static void mptcp_subflow_wupcall(struct socket *so, void *arg, int waitf) @@ -1771,18 +2373,15 @@ mptcp_subflow_wupcall(struct socket *so, void *arg, int waitf) struct mptsub *mpts = arg; struct mptses *mpte = mpts->mpts_mpte; - /* - * mpte should never be NULL except in a race with - * mptcp_subflow_del which doesn't hold socket lock across critical - * section. This upcall is made after releasing the socket lock. - * Interleaving of socket operations becomes possible therefore. - */ - if (mpte == NULL) + VERIFY(mpte != NULL); + + if (mptcp_should_defer_upcall(mpte->mpte_mppcb)) { + if (!(mpte->mpte_mppcb->mpp_flags & MPP_WUPCALL)) + mpte->mpte_mppcb->mpp_flags |= MPP_SHOULD_WWAKEUP; return; + } - lck_mtx_lock(&mpte->mpte_thread_lock); - mptcp_thread_signal_locked(mpte); - lck_mtx_unlock(&mpte->mpte_thread_lock); + mptcp_output(mpte); } /* @@ -1791,61 +2390,47 @@ mptcp_subflow_wupcall(struct socket *so, void *arg, int waitf) * Called for sending data from MPTCP to the underlying subflow socket. */ int -mptcp_subflow_output(struct mptses *mpte, struct mptsub *mpts) +mptcp_subflow_output(struct mptses *mpte, struct mptsub *mpts, int flags) { - struct socket *mp_so, *so; - size_t sb_cc = 0, tot_sent = 0; - struct mbuf *sb_mb; - int error = 0, wakeup = 0; - u_int64_t mpt_dsn = 0; struct mptcb *mp_tp = mpte->mpte_mptcb; - struct mbuf *mpt_mbuf = NULL; - u_int64_t off = 0; - struct mbuf *head, *tail; - int tcp_zero_len_write = 0; - - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); - mp_so = mpte->mpte_mppcb->mpp_socket; + struct mbuf *sb_mb, *m, *mpt_mbuf = NULL, *head, *tail; + struct socket *mp_so, *so; + struct tcpcb *tp; + uint64_t mpt_dsn = 0, off = 0; + int sb_cc = 0, error = 0, wakeup = 0; + uint32_t dss_csum; + uint16_t tot_sent = 0; + boolean_t reinjected = FALSE; + + mpte_lock_assert_held(mpte); + + mp_so = mptetoso(mpte); so = mpts->mpts_socket; + tp = sototcpcb(so); - DTRACE_MPTCP2(subflow__output, struct mptses *, mpte, - struct mptsub *, mpts); + VERIFY(!(mpte->mpte_mppcb->mpp_flags & MPP_INSIDE_OUTPUT)); + mpte->mpte_mppcb->mpp_flags |= MPP_INSIDE_OUTPUT; - /* subflow socket is suspended? */ - if (mpts->mpts_flags & MPTSF_SUSPENDED) { - mptcplog((LOG_ERR, "MPTCP Sender: %s mp_so 0x%llx cid %d is " - "flow controlled\n", __func__, - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), mpts->mpts_connid), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR); - goto out; - } + VERIFY(!INP_WAIT_FOR_IF_FEEDBACK(sotoinpcb(so))); + VERIFY((mpts->mpts_flags & MPTSF_MP_CAPABLE) || + (mpts->mpts_flags & MPTSF_MP_DEGRADED) || + (mpts->mpts_flags & MPTSF_TFO_REQD)); + VERIFY(mptcp_subflow_cwnd_space(mpts->mpts_socket) > 0); - /* subflow socket is not MPTCP capable? */ - if (!(mpts->mpts_flags & MPTSF_MP_CAPABLE) && - !(mpts->mpts_flags & MPTSF_MP_DEGRADED) && - !(mpts->mpts_flags & MPTSF_FASTJ_SEND) && - !(mpts->mpts_flags & MPTSF_TFO_REQD)) { - mptcplog((LOG_ERR, "MPTCP Sender: %s mp_so 0x%llx cid %d not " - "MPTCP capable\n", __func__, - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), mpts->mpts_connid), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR); - goto out; - } + mptcplog((LOG_DEBUG, "%s mpts_flags %#x, mpte_flags %#x cwnd_space %u\n", + __func__, mpts->mpts_flags, mpte->mpte_flags, + mptcp_subflow_cwnd_space(so)), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); + DTRACE_MPTCP2(subflow__output, struct mptses *, mpte, + struct mptsub *, mpts); /* Remove Addr Option is not sent reliably as per I-D */ if (mpte->mpte_flags & MPTE_SND_REM_ADDR) { - struct tcpcb *tp = intotcpcb(sotoinpcb(so)); tp->t_rem_aid = mpte->mpte_lost_aid; - if (mptcp_remaddr_enable) - tp->t_mpflags |= TMPF_SND_REM_ADDR; + tp->t_mpflags |= TMPF_SND_REM_ADDR; mpte->mpte_flags &= ~MPTE_SND_REM_ADDR; } - if (mpts->mpts_flags & MPTSF_TFO_REQD) { - mptcp_drop_tfo_data(mpte, mpts, &wakeup); - } - /* * The mbuf chains containing the metadata (as well as pointing to * the user data sitting at the MPTCP output queue) would then be @@ -1860,154 +2445,195 @@ mptcp_subflow_output(struct mptses *mpte, struct mptsub *mpts) * pkt_flags marked with the PKTF_MPTCP flag. */ - /* First, drop acknowledged data */ - sb_mb = mp_so->so_snd.sb_mb; + if (mpte->mpte_reinjectq) + sb_mb = mpte->mpte_reinjectq; + else + sb_mb = mp_so->so_snd.sb_mb; + if (sb_mb == NULL) { + mptcplog((LOG_ERR, "%s: No data in MPTCP-sendbuffer! smax %u snxt %u suna %u\n", + __func__, (uint32_t)mp_tp->mpt_sndmax, (uint32_t)mp_tp->mpt_sndnxt, (uint32_t)mp_tp->mpt_snduna), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR); goto out; } VERIFY(sb_mb->m_pkthdr.pkt_flags & PKTF_MPTCP); - mpt_mbuf = sb_mb; - while (mpt_mbuf && mpt_mbuf->m_pkthdr.mp_rlen == 0) { - if (((so->so_state & SS_ISCONNECTED) == 0) && - (mpt_mbuf->m_next == NULL) && - (so->so_flags1 & SOF1_PRECONNECT_DATA)) { - /* - * If TFO, allow connection establishment with zero - * length write. - */ - tcp_zero_len_write = 1; - goto zero_len_write; - } - mpt_mbuf = mpt_mbuf->m_next; - } - if (mpt_mbuf && (mpt_mbuf->m_pkthdr.pkt_flags & PKTF_MPTCP)) { - mpt_dsn = mpt_mbuf->m_pkthdr.mp_dsn; - } else { - goto out; + if (sb_mb->m_pkthdr.mp_rlen == 0 && + !(so->so_state & SS_ISCONNECTED) && + (so->so_flags1 & SOF1_PRECONNECT_DATA)) { + tp->t_mpflags |= TMPF_TFO_REQUEST; + goto zero_len_write; } - MPT_LOCK(mp_tp); + mpt_dsn = sb_mb->m_pkthdr.mp_dsn; + + /* First, drop acknowledged data */ if (MPTCP_SEQ_LT(mpt_dsn, mp_tp->mpt_snduna)) { - u_int64_t len = 0; - len = mp_tp->mpt_snduna - mpt_dsn; - MPT_UNLOCK(mp_tp); - sbdrop(&mp_so->so_snd, (int)len); - wakeup = 1; - MPT_LOCK(mp_tp); + mptcplog((LOG_ERR, "%s: dropping data, should have been done earlier " + "dsn %u suna %u reinject? %u\n", + __func__, (uint32_t)mpt_dsn, + (uint32_t)mp_tp->mpt_snduna, !!mpte->mpte_reinjectq), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR); + if (mpte->mpte_reinjectq) { + mptcp_clean_reinjectq(mpte); + } else { + uint64_t len = 0; + len = mp_tp->mpt_snduna - mpt_dsn; + sbdrop(&mp_so->so_snd, (int)len); + wakeup = 1; + } + } + + /* Check again because of above sbdrop */ + if (mp_so->so_snd.sb_mb == NULL && mpte->mpte_reinjectq == NULL) { + mptcplog((LOG_ERR, "%s send-buffer is empty\n", __func__), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR); + goto out; } /* * In degraded mode, we don't receive data acks, so force free * mbufs less than snd_nxt */ - if (mp_so->so_snd.sb_mb == NULL) { - MPT_UNLOCK(mp_tp); - goto out; - } - - mpt_dsn = mp_so->so_snd.sb_mb->m_pkthdr.mp_dsn; if ((mpts->mpts_flags & MPTSF_MP_DEGRADED) && (mp_tp->mpt_flags & MPTCPF_POST_FALLBACK_SYNC) && - MPTCP_SEQ_LT(mpt_dsn, mp_tp->mpt_sndnxt)) { - u_int64_t len = 0; - len = mp_tp->mpt_sndnxt - mpt_dsn; - sbdrop(&mp_so->so_snd, (int)len); - wakeup = 1; - mp_tp->mpt_snduna = mp_tp->mpt_sndnxt; + mp_so->so_snd.sb_mb) { + mpt_dsn = mp_so->so_snd.sb_mb->m_pkthdr.mp_dsn; + if (MPTCP_SEQ_LT(mpt_dsn, mp_tp->mpt_snduna)) { + uint64_t len = 0; + len = mp_tp->mpt_snduna - mpt_dsn; + sbdrop(&mp_so->so_snd, (int)len); + wakeup = 1; + + mptcplog((LOG_ERR, "%s: dropping data in degraded mode, should have been done earlier dsn %u sndnxt %u suna %u\n", + __func__, (uint32_t)mpt_dsn, (uint32_t)mp_tp->mpt_sndnxt, (uint32_t)mp_tp->mpt_snduna), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR); + } } if ((mpts->mpts_flags & MPTSF_MP_DEGRADED) && !(mp_tp->mpt_flags & MPTCPF_POST_FALLBACK_SYNC)) { mp_tp->mpt_flags |= MPTCPF_POST_FALLBACK_SYNC; so->so_flags1 |= SOF1_POST_FALLBACK_SYNC; - if (mp_tp->mpt_flags & MPTCPF_RECVD_MPFAIL) - mpts->mpts_sndnxt = mp_tp->mpt_dsn_at_csum_fail; - } - - /* - * Adjust the subflow's notion of next byte to send based on - * the last unacknowledged byte - */ - if (MPTCP_SEQ_LT(mpts->mpts_sndnxt, mp_tp->mpt_snduna)) { - mpts->mpts_sndnxt = mp_tp->mpt_snduna; } /* * Adjust the top level notion of next byte used for retransmissions * and sending FINs. */ - if (MPTCP_SEQ_LT(mp_tp->mpt_sndnxt, mp_tp->mpt_snduna)) { + if (MPTCP_SEQ_LT(mp_tp->mpt_sndnxt, mp_tp->mpt_snduna)) mp_tp->mpt_sndnxt = mp_tp->mpt_snduna; - } - /* Now determine the offset from which to start transmitting data */ - sb_mb = mp_so->so_snd.sb_mb; - sb_cc = mp_so->so_snd.sb_cc; + if (mpte->mpte_reinjectq) + sb_mb = mpte->mpte_reinjectq; + else + sb_mb = mp_so->so_snd.sb_mb; if (sb_mb == NULL) { - MPT_UNLOCK(mp_tp); + mptcplog((LOG_ERR, "%s send-buffer is still empty\n", __func__), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR); goto out; } - if (MPTCP_SEQ_LT(mpts->mpts_sndnxt, mp_tp->mpt_sndmax)) { - off = mpts->mpts_sndnxt - mp_tp->mpt_snduna; - sb_cc -= (size_t)off; + + if (mpte->mpte_reinjectq) { + sb_cc = sb_mb->m_pkthdr.mp_rlen; + } else if (flags & MPTCP_SUBOUT_PROBING) { + sb_cc = sb_mb->m_pkthdr.mp_rlen; + off = 0; } else { - MPT_UNLOCK(mp_tp); - goto out; + sb_cc = min(mp_so->so_snd.sb_cc, mp_tp->mpt_sndwnd); + + /* + * With TFO, there might be no data at all, thus still go into this + * code-path here. + */ + if ((mp_so->so_flags1 & SOF1_PRECONNECT_DATA) || + MPTCP_SEQ_LT(mp_tp->mpt_sndnxt, mp_tp->mpt_sndmax)) { + off = mp_tp->mpt_sndnxt - mp_tp->mpt_snduna; + sb_cc -= off; + } else { + mptcplog((LOG_ERR, "%s this should not happen: sndnxt %u sndmax %u\n", + __func__, (uint32_t)mp_tp->mpt_sndnxt, + (uint32_t)mp_tp->mpt_sndmax), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR); + + goto out; + } } - MPT_UNLOCK(mp_tp); - mpt_mbuf = sb_mb; + sb_cc = min(sb_cc, mptcp_subflow_cwnd_space(so)); + if (sb_cc <= 0) { + mptcplog((LOG_ERR, "%s sb_cc is %d, mp_so->sb_cc %u, sndwnd %u,sndnxt %u sndmax %u cwnd %u\n", + __func__, sb_cc, mp_so->so_snd.sb_cc, mp_tp->mpt_sndwnd, + (uint32_t)mp_tp->mpt_sndnxt, (uint32_t)mp_tp->mpt_sndmax, + mptcp_subflow_cwnd_space(so)), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR); + } + + sb_cc = min(sb_cc, UINT16_MAX); + + /* + * Create a DSN mapping for the data we are about to send. It all + * has the same mapping. + */ + if (mpte->mpte_reinjectq) + mpt_dsn = sb_mb->m_pkthdr.mp_dsn; + else + mpt_dsn = mp_tp->mpt_snduna + off; - while (mpt_mbuf && ((mpt_mbuf->m_pkthdr.mp_rlen == 0) || - (mpt_mbuf->m_pkthdr.mp_rlen <= (u_int32_t)off))) { + mpt_mbuf = sb_mb; + while (mpt_mbuf && mpte->mpte_reinjectq == NULL && + (mpt_mbuf->m_pkthdr.mp_rlen == 0 || + mpt_mbuf->m_pkthdr.mp_rlen <= (uint32_t)off)) { off -= mpt_mbuf->m_pkthdr.mp_rlen; mpt_mbuf = mpt_mbuf->m_next; } if (mpts->mpts_flags & MPTSF_MP_DEGRADED) - mptcplog((LOG_DEBUG, "MPTCP Sender: %s cid = %d " - "snduna = %llu sndnxt = %llu probe %d\n", - __func__, mpts->mpts_connid, - mp_tp->mpt_snduna, mpts->mpts_sndnxt, + mptcplog((LOG_DEBUG, "%s: %u snduna = %u sndnxt = %u probe %d\n", + __func__, mpts->mpts_connid, (uint32_t)mp_tp->mpt_snduna, (uint32_t)mp_tp->mpt_sndnxt, mpts->mpts_probecnt), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); + MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); VERIFY((mpt_mbuf == NULL) || (mpt_mbuf->m_pkthdr.pkt_flags & PKTF_MPTCP)); head = tail = NULL; while (tot_sent < sb_cc) { - struct mbuf *m; - size_t mlen; + ssize_t mlen; - mlen = mpt_mbuf->m_pkthdr.mp_rlen; + mlen = mpt_mbuf->m_len; mlen -= off; - if (mlen == 0) - goto out; + mlen = min(mlen, sb_cc - tot_sent); - if (mlen > sb_cc) { - panic("%s: unexpected %lu %lu \n", __func__, - mlen, sb_cc); + if (mlen < 0) { + mptcplog((LOG_ERR, "%s mlen %d mp_rlen %u off %u sb_cc %u tot_sent %u\n", + __func__, (int)mlen, mpt_mbuf->m_pkthdr.mp_rlen, + (uint32_t)off, sb_cc, tot_sent), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR); + goto out; } + if (mlen == 0) + goto next; + m = m_copym_mode(mpt_mbuf, (int)off, mlen, M_DONTWAIT, M_COPYM_MUST_COPY_HDR); if (m == NULL) { + mptcplog((LOG_ERR, "%s m_copym_mode failed\n", __func__), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR); error = ENOBUFS; break; } /* Create a DSN mapping for the data (m_copym does it) */ - mpt_dsn = mpt_mbuf->m_pkthdr.mp_dsn; VERIFY(m->m_flags & M_PKTHDR); + VERIFY(m->m_next == NULL); + m->m_pkthdr.pkt_flags |= PKTF_MPTCP; m->m_pkthdr.pkt_flags &= ~PKTF_MPSO; - m->m_pkthdr.mp_dsn = mpt_dsn + off; + m->m_pkthdr.mp_dsn = mpt_dsn; m->m_pkthdr.mp_rseq = mpts->mpts_rel_seq; - m->m_pkthdr.mp_rlen = mlen; - mpts->mpts_rel_seq += mlen; m->m_pkthdr.len = mlen; if (head == NULL) { @@ -2019,352 +2645,500 @@ mptcp_subflow_output(struct mptses *mpte, struct mptsub *mpts) tot_sent += mlen; off = 0; +next: mpt_mbuf = mpt_mbuf->m_next; } - if (head != NULL) { - struct tcpcb *tp = intotcpcb(sotoinpcb(so)); + if (mpte->mpte_reinjectq) { + reinjected = TRUE; + if (sb_cc < sb_mb->m_pkthdr.mp_rlen) { + struct mbuf *n = sb_mb; + + while (n) { + n->m_pkthdr.mp_dsn += sb_cc; + n->m_pkthdr.mp_rlen -= sb_cc; + n = n->m_next; + } + m_adj(sb_mb, sb_cc); + } else { + mpte->mpte_reinjectq = sb_mb->m_nextpkt; + m_freem(sb_mb); + } + } + + mptcplog((LOG_DEBUG, "%s: Queued dsn %u ssn %u len %u on sub %u\n", + __func__, (uint32_t)mpt_dsn, mpts->mpts_rel_seq, + tot_sent, mpts->mpts_connid), MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); + + if (head && (mp_tp->mpt_flags & MPTCPF_CHECKSUM)) { + dss_csum = mptcp_output_csum(head, mpt_dsn, mpts->mpts_rel_seq, + tot_sent); + } + + /* Now, let's update rel-seq and the data-level length */ + mpts->mpts_rel_seq += tot_sent; + m = head; + while (m) { + if (mp_tp->mpt_flags & MPTCPF_CHECKSUM) + m->m_pkthdr.mp_csum = dss_csum; + m->m_pkthdr.mp_rlen = tot_sent; + m = m->m_next; + } + + if (head != NULL) { if ((mpts->mpts_flags & MPTSF_TFO_REQD) && - (tp->t_tfo_stats == 0)) { + (tp->t_tfo_stats == 0)) tp->t_mpflags |= TMPF_TFO_REQUEST; - } else if (mpts->mpts_flags & MPTSF_FASTJ_SEND) { - tp->t_mpflags |= TMPF_FASTJOIN_SEND; - } error = sock_sendmbuf(so, NULL, head, 0, NULL); - DTRACE_MPTCP7(send, struct mbuf *, head, struct socket *, so, + DTRACE_MPTCP7(send, struct mbuf *, m, struct socket *, so, struct sockbuf *, &so->so_rcv, struct sockbuf *, &so->so_snd, struct mptses *, mpte, struct mptsub *, mpts, size_t, tot_sent); - } else if (tcp_zero_len_write == 1) { -zero_len_write: - socket_lock(so, 1); - /* Opting to call pru_send as no mbuf at subflow level */ - error = (*so->so_proto->pr_usrreqs->pru_send) - (so, 0, NULL, NULL, NULL, current_proc()); - socket_unlock(so, 1); } - if ((error == 0) || (error == EWOULDBLOCK)) { - mpts->mpts_sndnxt += tot_sent; +done_sending: + if (error == 0 || + (error == EWOULDBLOCK && (tp->t_mpflags & TMPF_TFO_REQUEST))) { + uint64_t new_sndnxt = mp_tp->mpt_sndnxt + tot_sent; if (mpts->mpts_probesoon && mpts->mpts_maxseg && tot_sent) { tcpstat.tcps_mp_num_probes++; - if (tot_sent < mpts->mpts_maxseg) + if ((uint32_t)tot_sent < mpts->mpts_maxseg) mpts->mpts_probecnt += 1; else mpts->mpts_probecnt += tot_sent/mpts->mpts_maxseg; } - MPT_LOCK(mp_tp); - - if (MPTCP_SEQ_LT(mp_tp->mpt_sndnxt, mpts->mpts_sndnxt)) { - if (MPTCP_DATASEQ_HIGH32(mpts->mpts_sndnxt) > + if (!reinjected && !(flags & MPTCP_SUBOUT_PROBING)) { + if (MPTCP_DATASEQ_HIGH32(new_sndnxt) > MPTCP_DATASEQ_HIGH32(mp_tp->mpt_sndnxt)) mp_tp->mpt_flags |= MPTCPF_SND_64BITDSN; - mp_tp->mpt_sndnxt = mpts->mpts_sndnxt; + mp_tp->mpt_sndnxt = new_sndnxt; } - mptcp_cancel_timer(mp_tp, MPTT_REXMT); - MPT_UNLOCK(mp_tp); - if (so->so_flags1 & SOF1_PRECONNECT_DATA) - so->so_flags1 &= ~SOF1_PRECONNECT_DATA; + mptcp_cancel_timer(mp_tp, MPTT_REXMT); - /* Send once in SYN_SENT state to avoid sending SYN spam */ - if (mpts->mpts_flags & MPTSF_FASTJ_SEND) { - so->so_flags &= ~SOF_MPTCP_FASTJOIN; - mpts->mpts_flags &= ~MPTSF_FASTJ_SEND; - } + /* Must be here as mptcp_can_send_more() checks for this */ + soclearfastopen(mp_so); if ((mpts->mpts_flags & MPTSF_MP_DEGRADED) || (mpts->mpts_probesoon != 0)) - mptcplog((LOG_DEBUG, "MPTCP Sender: %s cid %d " - "wrote %d %d probe %d probedelta %d\n", - __func__, mpts->mpts_connid, (int)tot_sent, - (int) sb_cc, mpts->mpts_probecnt, + mptcplog((LOG_DEBUG, "%s %u degraded %u wrote %d %d probe %d probedelta %d\n", + __func__, mpts->mpts_connid, + !!(mpts->mpts_flags & MPTSF_MP_DEGRADED), + tot_sent, (int) sb_cc, mpts->mpts_probecnt, (tcp_now - mpts->mpts_probesoon)), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); + MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); + + if (IFNET_IS_CELLULAR(sotoinpcb(so)->inp_last_outifp)) { + mpte->mpte_mppcb->mpp_flags |= MPP_SET_CELLICON; + + mpte->mpte_used_cell = 1; + } else { + mpte->mpte_mppcb->mpp_flags |= MPP_UNSET_CELLICON; + + mpte->mpte_used_wifi = 1; + } + + /* + * Don't propagate EWOULDBLOCK - it's already taken care of + * in mptcp_usr_send for TFO. + */ + error = 0; } else { - mptcplog((LOG_ERR, "MPTCP Sender: %s cid %d error %d len %zd\n", - __func__, mpts->mpts_connid, error, tot_sent), + mptcplog((LOG_ERR, "%s: %u error %d len %d subflags %#x sostate %#x soerror %u hiwat %u lowat %u\n", + __func__, mpts->mpts_connid, error, tot_sent, so->so_flags, so->so_state, so->so_error, so->so_snd.sb_hiwat, so->so_snd.sb_lowat), MPTCP_SENDER_DBG, MPTCP_LOGLVL_ERR); } out: + if (wakeup) - sowwakeup(mp_so); + mpte->mpte_mppcb->mpp_flags |= MPP_SHOULD_WWAKEUP; + mptcp_handle_deferred_upcalls(mpte->mpte_mppcb, MPP_INSIDE_OUTPUT); return (error); + +zero_len_write: + /* Opting to call pru_send as no mbuf at subflow level */ + error = (*so->so_proto->pr_usrreqs->pru_send)(so, 0, NULL, NULL, + NULL, current_proc()); + + goto done_sending; } -/* - * Subflow socket control event upcall. - * - * Called when the associated subflow socket posted one or more control events. - * The subflow socket lock has been released prior to invoking the callback. - * Note that the upcall may occur synchronously as a result of MPTCP performing - * an action on it, or asynchronously as a result of an event happening at the - * subflow layer. Therefore, to maintain lock ordering, the only lock that can - * be acquired here is the thread lock, for signalling purposes. - */ static void -mptcp_subflow_eupcall(struct socket *so, void *arg, uint32_t events) +mptcp_add_reinjectq(struct mptses *mpte, struct mbuf *m) { -#pragma unused(so) - struct mptsub *mpts = arg; - struct mptses *mpte = mpts->mpts_mpte; + struct mbuf *n, *prev = NULL; - VERIFY(mpte != NULL); + mptcplog((LOG_DEBUG, "%s reinjecting dsn %u dlen %u rseq %u\n", + __func__, (uint32_t)m->m_pkthdr.mp_dsn, m->m_pkthdr.mp_rlen, + m->m_pkthdr.mp_rseq), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + + n = mpte->mpte_reinjectq; + + /* First, look for an mbuf n, whose data-sequence-number is bigger or + * equal than m's sequence number. + */ + while (n) { + if (MPTCP_SEQ_GEQ(n->m_pkthdr.mp_dsn, m->m_pkthdr.mp_dsn)) + break; + + prev = n; + + n = n->m_nextpkt; + } + + if (n) { + /* m is already fully covered by the next mbuf in the queue */ + if (n->m_pkthdr.mp_dsn == m->m_pkthdr.mp_dsn && + n->m_pkthdr.mp_rlen >= m->m_pkthdr.mp_rlen) { + mptcplog((LOG_DEBUG, "%s fully covered with len %u\n", + __func__, n->m_pkthdr.mp_rlen), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + goto dont_queue; + } + + /* m is covering the next mbuf entirely, thus we remove this guy */ + if (m->m_pkthdr.mp_dsn + m->m_pkthdr.mp_rlen >= n->m_pkthdr.mp_dsn + n->m_pkthdr.mp_rlen) { + struct mbuf *tmp = n->m_nextpkt; + + mptcplog((LOG_DEBUG, "%s m is covering that guy dsn %u len %u dsn %u len %u\n", + __func__, m->m_pkthdr.mp_dsn, m->m_pkthdr.mp_rlen, + n->m_pkthdr.mp_dsn, n->m_pkthdr.mp_rlen), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + + m->m_nextpkt = NULL; + if (prev == NULL) + mpte->mpte_reinjectq = tmp; + else + prev->m_nextpkt = tmp; + + m_freem(n); + n = tmp; + } + + } + + if (prev) { + /* m is already fully covered by the previous mbuf in the queue */ + if (prev->m_pkthdr.mp_dsn + prev->m_pkthdr.mp_rlen >= m->m_pkthdr.mp_dsn + m->m_pkthdr.len) { + mptcplog((LOG_DEBUG, "%s prev covers us from %u with len %u\n", + __func__, prev->m_pkthdr.mp_dsn, prev->m_pkthdr.mp_rlen), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + goto dont_queue; + } + } + + if (prev == NULL) + mpte->mpte_reinjectq = m; + else + prev->m_nextpkt = m; - lck_mtx_lock(&mpte->mpte_thread_lock); - atomic_bitset_32(&mpts->mpts_evctl, events); - mptcp_thread_signal_locked(mpte); - lck_mtx_unlock(&mpte->mpte_thread_lock); + m->m_nextpkt = n; + + return; + +dont_queue: + m_freem(m); + return; } -/* - * Subflow socket control events. - * - * Called for handling events related to the underlying subflow socket. - */ -static ev_ret_t -mptcp_subflow_events(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) +static struct mbuf * +mptcp_lookup_dsn(struct mptses *mpte, uint64_t dsn) { - uint32_t events, save_events; - ev_ret_t ret = MPTS_EVRET_OK; - int i = 0; - int mpsub_ev_entry_count = sizeof(mpsub_ev_entry_tbl)/ - sizeof(mpsub_ev_entry_tbl[0]); - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); + struct socket *mp_so = mptetoso(mpte); + struct mbuf *m; - /* bail if there's nothing to process */ - if ((events = mpts->mpts_evctl) == 0) - return (ret); + m = mp_so->so_snd.sb_mb; - if (events & (SO_FILT_HINT_CONNRESET|SO_FILT_HINT_MUSTRST| - SO_FILT_HINT_CANTRCVMORE|SO_FILT_HINT_CANTSENDMORE| - SO_FILT_HINT_TIMEOUT|SO_FILT_HINT_NOSRCADDR| - SO_FILT_HINT_IFDENIED|SO_FILT_HINT_SUSPEND| - SO_FILT_HINT_DISCONNECTED)) { - events |= SO_FILT_HINT_MPFAILOVER; + while (m) { + /* If this segment covers what we are looking for, return it. */ + if (MPTCP_SEQ_LEQ(m->m_pkthdr.mp_dsn, dsn) && + MPTCP_SEQ_GT(m->m_pkthdr.mp_dsn + m->m_pkthdr.mp_rlen, dsn)) + break; + + + /* Segment is no more in the queue */ + if (MPTCP_SEQ_GT(m->m_pkthdr.mp_dsn, dsn)) + return NULL; + + m = m->m_next; } - save_events = events; + return m; +} - DTRACE_MPTCP3(subflow__events, struct mptses *, mpte, - struct mptsub *, mpts, uint32_t, events); +static struct mbuf * +mptcp_copy_mbuf_list(struct mbuf *m, int len) +{ + struct mbuf *top = NULL, *tail = NULL; + uint64_t dsn; + uint32_t dlen, rseq; - mptcplog((LOG_DEBUG, "MPTCP Events: %s cid %d events=%b\n", __func__, - mpts->mpts_connid, events, SO_FILT_HINT_BITS), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_VERBOSE); + dsn = m->m_pkthdr.mp_dsn; + dlen = m->m_pkthdr.mp_rlen; + rseq = m->m_pkthdr.mp_rseq; - /* - * Process all the socket filter hints and reset the hint - * once it is handled - */ - for (i = 0; (i < mpsub_ev_entry_count) && events; i++) { - /* - * Always execute the DISCONNECTED event, because it will wakeup - * the app. - */ - if ((events & mpsub_ev_entry_tbl[i].sofilt_hint_mask) && - (ret >= MPTS_EVRET_OK || - mpsub_ev_entry_tbl[i].sofilt_hint_mask == SO_FILT_HINT_DISCONNECTED)) { - ev_ret_t error = - mpsub_ev_entry_tbl[i].sofilt_hint_ev_hdlr(mpte, mpts, p_mpsofilt_hint); - events &= ~mpsub_ev_entry_tbl[i].sofilt_hint_mask; - ret = ((error >= MPTS_EVRET_OK) ? MAX(error, ret) : error); + while (len > 0) { + struct mbuf *n; + + VERIFY((m->m_flags & M_PKTHDR) && (m->m_pkthdr.pkt_flags & PKTF_MPTCP)); + + n = m_copym_mode(m, 0, m->m_len, M_DONTWAIT, M_COPYM_MUST_COPY_HDR); + if (n == NULL) { + mptcplog((LOG_ERR, "%s m_copym_mode returned NULL\n", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + goto err; } - } - /* - * We should be getting only events specified via sock_catchevents(), - * so loudly complain if we have any unprocessed one(s). - */ - if (events != 0 || ret < MPTS_EVRET_OK) { - mptcplog((LOG_ERR, "MPTCP Events %s%s: cid %d evret %s (%d)" - " unhandled events=%b\n", - (events != 0) && (ret == MPTS_EVRET_OK) ? "MPTCP_ERROR " : "", - __func__, mpts->mpts_connid, - mptcp_evret2str(ret), ret, events, SO_FILT_HINT_BITS), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_ERR); + VERIFY(n->m_flags & M_PKTHDR); + VERIFY(n->m_next == NULL); + VERIFY(n->m_pkthdr.mp_dsn == dsn); + VERIFY(n->m_pkthdr.mp_rlen == dlen); + VERIFY(n->m_pkthdr.mp_rseq == rseq); + VERIFY(n->m_len == m->m_len); + + n->m_pkthdr.pkt_flags |= (PKTF_MPSO | PKTF_MPTCP); + + if (top == NULL) + top = n; + + if (tail != NULL) + tail->m_next = n; + + tail = n; + + len -= m->m_len; + m = m->m_next; } - /* clear the ones we've processed */ - atomic_bitclear_32(&mpts->mpts_evctl, save_events); - return (ret); + return top; + +err: + if (top) + m_freem(top); + + return NULL; } -/* - * Handle SO_FILT_HINT_CONNRESET subflow socket event. - */ -static ev_ret_t -mptcp_subflow_connreset_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) +static void +mptcp_reinject_mbufs(struct socket *so) { - struct socket *mp_so, *so; - struct mptcb *mp_tp; - boolean_t linger; + struct tcpcb *tp = sototcpcb(so); + struct mptsub *mpts = tp->t_mpsub; + struct mptcb *mp_tp = tptomptp(tp); + struct mptses *mpte = mp_tp->mpt_mpte;; + struct sockbuf *sb = &so->so_snd; + struct mbuf *m; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); - VERIFY(mpte->mpte_mppcb != NULL); - mp_so = mpte->mpte_mppcb->mpp_socket; - mp_tp = mpte->mpte_mptcb; - so = mpts->mpts_socket; + m = sb->sb_mb; + while (m) { + struct mbuf *n = m->m_next, *orig = m; - linger = (!(mpts->mpts_flags & MPTSF_DELETEOK) && - !(mp_so->so_flags & SOF_PCBCLEARING)); + mptcplog((LOG_DEBUG, "%s working on suna %u relseq %u iss %u len %u pktflags %#x\n", + __func__, tp->snd_una, m->m_pkthdr.mp_rseq, mpts->mpts_iss, + m->m_pkthdr.mp_rlen, m->m_pkthdr.pkt_flags), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); - mptcplog((LOG_DEBUG, "MPTCP Events: " - "%s: cid %d [linger %s]\n", __func__, - mpts->mpts_connid, (linger ? "YES" : "NO")), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); + VERIFY((m->m_flags & M_PKTHDR) && (m->m_pkthdr.pkt_flags & PKTF_MPTCP)); - /* - * We got a TCP RST for this subflow connection. - * - * Right now, we simply propagate ECONNREFUSED to the MPTCP socket - * client if the MPTCP connection has not been established or - * if the connection has only one subflow and is a connection being - * resumed. Otherwise we close the socket. - */ - mptcp_subflow_disconnect(mpte, mpts, !linger); + if (m->m_pkthdr.pkt_flags & PKTF_MPTCP_REINJ) + goto next; - MPT_LOCK(mp_tp); - if (mp_tp->mpt_state < MPTCPS_ESTABLISHED) { - mpts->mpts_soerror = mp_so->so_error = ECONNREFUSED; - } else if (mpte->mpte_nummpcapflows < 1 || - ((mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) && - (mpts->mpts_flags & MPTSF_ACTIVE))) { - mpts->mpts_soerror = mp_so->so_error = ECONNRESET; - *p_mpsofilt_hint |= SO_FILT_HINT_LOCKED | SO_FILT_HINT_CONNRESET; + /* Has it all already been acknowledged at the data-level? */ + if (MPTCP_SEQ_GEQ(mp_tp->mpt_snduna, m->m_pkthdr.mp_dsn + m->m_pkthdr.mp_rlen)) + goto next; + + /* Part of this has already been acknowledged - lookup in the + * MPTCP-socket for the segment. + */ + if (SEQ_GT(tp->snd_una - mpts->mpts_iss, m->m_pkthdr.mp_rseq)) { + m = mptcp_lookup_dsn(mpte, m->m_pkthdr.mp_dsn); + if (m == NULL) + goto next; + } + + /* Copy the mbuf with headers (aka, DSN-numbers) */ + m = mptcp_copy_mbuf_list(m, m->m_pkthdr.mp_rlen); + if (m == NULL) + break; + + VERIFY(m->m_nextpkt == NULL); + + /* Now, add to the reinject-queue, eliminating overlapping + * segments + */ + mptcp_add_reinjectq(mpte, m); + + orig->m_pkthdr.pkt_flags |= PKTF_MPTCP_REINJ; + +next: + /* mp_rlen can cover multiple mbufs, so advance to the end of it. */ + while (n) { + VERIFY((n->m_flags & M_PKTHDR) && (n->m_pkthdr.pkt_flags & PKTF_MPTCP)); + + if (n->m_pkthdr.mp_dsn != orig->m_pkthdr.mp_dsn) + break; + + n->m_pkthdr.pkt_flags |= PKTF_MPTCP_REINJ; + n = n->m_next; + } + + m = n; } - MPT_UNLOCK(mp_tp); +} - /* - * Keep the subflow socket around, unless the MPTCP socket has - * been detached or the subflow has been disconnected explicitly, - * in which case it should be deleted right away. - */ - return (linger ? MPTS_EVRET_OK : MPTS_EVRET_DELETE); +void +mptcp_clean_reinjectq(struct mptses *mpte) +{ + struct mptcb *mp_tp = mpte->mpte_mptcb; + + mpte_lock_assert_held(mpte); + + while (mpte->mpte_reinjectq) { + struct mbuf *m = mpte->mpte_reinjectq; + + if (MPTCP_SEQ_GEQ(m->m_pkthdr.mp_dsn, mp_tp->mpt_snduna) || + MPTCP_SEQ_GEQ(m->m_pkthdr.mp_dsn + m->m_pkthdr.mp_rlen, mp_tp->mpt_snduna)) + break; + + mpte->mpte_reinjectq = m->m_nextpkt; + m->m_nextpkt = NULL; + m_freem(m); + } } /* - * Handle SO_FILT_HINT_CANTRCVMORE subflow socket event. + * Subflow socket control event upcall. */ -static ev_ret_t -mptcp_subflow_cantrcvmore_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) +static void +mptcp_subflow_eupcall1(struct socket *so, void *arg, uint32_t events) { - struct mptcb *mp_tp; - struct socket *so; - - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); +#pragma unused(so) + struct mptsub *mpts = arg; + struct mptses *mpte = mpts->mpts_mpte; - mp_tp = mpte->mpte_mptcb; - so = mpts->mpts_socket; + VERIFY(mpte != NULL); + mpte_lock_assert_held(mpte); - mptcplog((LOG_DEBUG, "MPTCP Events: " - "%s: cid %d\n", __func__, mpts->mpts_connid), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); + if ((mpts->mpts_evctl & events) == events) + return; - /* - * A FIN on a fallen back MPTCP-connection should be treated like a - * DATA_FIN. - */ - MPT_LOCK(mp_tp); - if ((mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) && - (mpts->mpts_flags & MPTSF_ACTIVE)) { - mptcp_close_fsm(mp_tp, MPCE_RECV_DATA_FIN); - if (mp_tp->mpt_state == MPTCPS_CLOSE_WAIT) { - *p_mpsofilt_hint |= SO_FILT_HINT_LOCKED | SO_FILT_HINT_CANTRCVMORE; - } + mpts->mpts_evctl |= events; + + if (mptcp_should_defer_upcall(mpte->mpte_mppcb)) { + mpte->mpte_mppcb->mpp_flags |= MPP_SHOULD_WORKLOOP; + return; } - MPT_UNLOCK(mp_tp); - return (MPTS_EVRET_OK); /* keep the subflow socket around */ + mptcp_subflow_workloop(mpte); } /* - * Handle SO_FILT_HINT_CANTSENDMORE subflow socket event. + * Subflow socket control events. + * + * Called for handling events related to the underlying subflow socket. */ static ev_ret_t -mptcp_subflow_cantsendmore_ev(struct mptses *mpte, struct mptsub *mpts, +mptcp_subflow_events(struct mptses *mpte, struct mptsub *mpts, uint64_t *p_mpsofilt_hint) { -#pragma unused(p_mpsofilt_hint) - struct socket *so; + ev_ret_t ret = MPTS_EVRET_OK; + int i, mpsub_ev_entry_count = sizeof(mpsub_ev_entry_tbl) / + sizeof(mpsub_ev_entry_tbl[0]); - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); + mpte_lock_assert_held(mpte); /* same as MP socket lock */ - so = mpts->mpts_socket; + /* bail if there's nothing to process */ + if (!mpts->mpts_evctl) + return (ret); - mptcplog((LOG_DEBUG, "MPTCP Events: " - "%s: cid %d\n", __func__, mpts->mpts_connid), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); + if (mpts->mpts_evctl & (SO_FILT_HINT_CONNRESET|SO_FILT_HINT_MUSTRST| + SO_FILT_HINT_CANTSENDMORE|SO_FILT_HINT_TIMEOUT| + SO_FILT_HINT_NOSRCADDR|SO_FILT_HINT_IFDENIED| + SO_FILT_HINT_DISCONNECTED)) { + mpts->mpts_evctl |= SO_FILT_HINT_MPFAILOVER; + } - return (MPTS_EVRET_OK); /* keep the subflow socket around */ + DTRACE_MPTCP3(subflow__events, struct mptses *, mpte, + struct mptsub *, mpts, uint32_t, mpts->mpts_evctl); + + mptcplog((LOG_DEBUG, "%s cid %d events=%b\n", __func__, + mpts->mpts_connid, mpts->mpts_evctl, SO_FILT_HINT_BITS), + MPTCP_EVENTS_DBG, MPTCP_LOGLVL_VERBOSE); + + /* + * Process all the socket filter hints and reset the hint + * once it is handled + */ + for (i = 0; i < mpsub_ev_entry_count && mpts->mpts_evctl; i++) { + /* + * Always execute the DISCONNECTED event, because it will wakeup + * the app. + */ + if ((mpts->mpts_evctl & mpsub_ev_entry_tbl[i].sofilt_hint_mask) && + (ret >= MPTS_EVRET_OK || + mpsub_ev_entry_tbl[i].sofilt_hint_mask == SO_FILT_HINT_DISCONNECTED)) { + mpts->mpts_evctl &= ~mpsub_ev_entry_tbl[i].sofilt_hint_mask; + ev_ret_t error = + mpsub_ev_entry_tbl[i].sofilt_hint_ev_hdlr(mpte, mpts, p_mpsofilt_hint, mpsub_ev_entry_tbl[i].sofilt_hint_mask); + ret = ((error >= MPTS_EVRET_OK) ? MAX(error, ret) : error); + } + } + + /* + * We should be getting only events specified via sock_catchevents(), + * so loudly complain if we have any unprocessed one(s). + */ + if (mpts->mpts_evctl || ret < MPTS_EVRET_OK) + mptcplog((LOG_WARNING, "%s%s: cid %d evret %s (%d) unhandled events=%b\n", __func__, + (mpts->mpts_evctl && ret == MPTS_EVRET_OK) ? "MPTCP_ERROR " : "", + mpts->mpts_connid, + mptcp_evret2str(ret), ret, mpts->mpts_evctl, SO_FILT_HINT_BITS), + MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); + else + mptcplog((LOG_DEBUG, "%s: Done, events %b\n", __func__, + mpts->mpts_evctl, SO_FILT_HINT_BITS), + MPTCP_EVENTS_DBG, MPTCP_LOGLVL_VERBOSE); + + return (ret); } -/* - * Handle SO_FILT_HINT_TIMEOUT subflow socket event. - */ static ev_ret_t -mptcp_subflow_timeout_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) +mptcp_subflow_propagate_ev(struct mptses *mpte, struct mptsub *mpts, + uint64_t *p_mpsofilt_hint, uint64_t event) { -#pragma unused(p_mpsofilt_hint) struct socket *mp_so, *so; struct mptcb *mp_tp; - boolean_t linger; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); + mpte_lock_assert_held(mpte); /* same as MP socket lock */ VERIFY(mpte->mpte_mppcb != NULL); - mp_so = mpte->mpte_mppcb->mpp_socket; + mp_so = mptetoso(mpte); mp_tp = mpte->mpte_mptcb; so = mpts->mpts_socket; - linger = (!(mpts->mpts_flags & MPTSF_DELETEOK) && - !(mp_so->so_flags & SOF_PCBCLEARING)); - - mptcplog((LOG_NOTICE, "MPTCP Events: " - "%s: cid %d [linger %s]\n", __func__, - mpts->mpts_connid, (linger ? "YES" : "NO")), + mptcplog((LOG_DEBUG, "%s: cid %d event %d\n", __func__, + mpts->mpts_connid, event), MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); - if (mpts->mpts_soerror == 0) - mpts->mpts_soerror = ETIMEDOUT; - /* - * The subflow connection has timed out. - * - * Right now, we simply propagate ETIMEDOUT to the MPTCP socket - * client if the MPTCP connection has not been established. Otherwise - * drop it. + * We got an event for this subflow that might need to be propagated, + * based on the state of the MPTCP connection. */ - mptcp_subflow_disconnect(mpte, mpts, !linger); - - MPT_LOCK(mp_tp); - if (mp_tp->mpt_state < MPTCPS_ESTABLISHED) { - mp_so->so_error = ETIMEDOUT; + if (mp_tp->mpt_state < MPTCPS_ESTABLISHED || + ((mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) && (mpts->mpts_flags & MPTSF_ACTIVE))) { + mp_so->so_error = so->so_error; + *p_mpsofilt_hint |= event; } - MPT_UNLOCK(mp_tp); - /* - * Keep the subflow socket around, unless the MPTCP socket has - * been detached or the subflow has been disconnected explicitly, - * in which case it should be deleted right away. - */ - return (linger ? MPTS_EVRET_OK : MPTS_EVRET_DELETE); + return (MPTS_EVRET_OK); } /* @@ -2372,24 +3146,18 @@ mptcp_subflow_timeout_ev(struct mptses *mpte, struct mptsub *mpts, */ static ev_ret_t mptcp_subflow_nosrcaddr_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) + uint64_t *p_mpsofilt_hint, uint64_t event) { -#pragma unused(p_mpsofilt_hint) - struct socket *mp_so, *so; - struct mptcb *mp_tp; - boolean_t linger; - struct tcpcb *tp = NULL; +#pragma unused(p_mpsofilt_hint, event) + struct socket *mp_so; + struct tcpcb *tp; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); + mpte_lock_assert_held(mpte); /* same as MP socket lock */ VERIFY(mpte->mpte_mppcb != NULL); - mp_so = mpte->mpte_mppcb->mpp_socket; - mp_tp = mpte->mpte_mptcb; - so = mpts->mpts_socket; + mp_so = mptetoso(mpte); + tp = intotcpcb(sotoinpcb(mpts->mpts_socket)); - /* Not grabbing socket lock as t_local_aid is write once only */ - tp = intotcpcb(sotoinpcb(so)); /* * This overwrites any previous mpte_lost_aid to avoid storing * too much state when the typical case has only two subflows. @@ -2397,42 +3165,18 @@ mptcp_subflow_nosrcaddr_ev(struct mptses *mpte, struct mptsub *mpts, mpte->mpte_flags |= MPTE_SND_REM_ADDR; mpte->mpte_lost_aid = tp->t_local_aid; - linger = (!(mpts->mpts_flags & MPTSF_DELETEOK) && - !(mp_so->so_flags & SOF_PCBCLEARING)); - - mptcplog((LOG_DEBUG, "MPTCP Events: " - "%s cid %d [linger %s]\n", __func__, - mpts->mpts_connid, (linger ? "YES" : "NO")), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); - - if (mpts->mpts_soerror == 0) - mpts->mpts_soerror = EADDRNOTAVAIL; + mptcplog((LOG_DEBUG, "%s cid %d\n", __func__, mpts->mpts_connid), + MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); /* * The subflow connection has lost its source address. - * - * Right now, we simply propagate EADDRNOTAVAIL to the MPTCP socket - * client if the MPTCP connection has not been established. If it - * has been established with one subflow , we keep the MPTCP - * connection valid without any subflows till closed by application. - * This lets tcp connection manager decide whether to close this or - * not as it reacts to reachability changes too. */ - mptcp_subflow_disconnect(mpte, mpts, !linger); + mptcp_subflow_abort(mpts, EADDRNOTAVAIL); - MPT_LOCK(mp_tp); - if ((mp_tp->mpt_state < MPTCPS_ESTABLISHED) && - (mp_so->so_flags & SOF_NOADDRAVAIL)) { - mp_so->so_error = EADDRNOTAVAIL; - } - MPT_UNLOCK(mp_tp); + if (mp_so->so_flags & SOF_NOADDRAVAIL) + mptcp_subflow_propagate_ev(mpte, mpts, p_mpsofilt_hint, event); - /* - * Keep the subflow socket around, unless the MPTCP socket has - * been detached or the subflow has been disconnected explicitly, - * in which case it should be deleted right away. - */ - return (linger ? MPTS_EVRET_OK : MPTS_EVRET_DELETE); + return (MPTS_EVRET_DELETE); } /* @@ -2441,19 +3185,15 @@ mptcp_subflow_nosrcaddr_ev(struct mptses *mpte, struct mptsub *mpts, */ static ev_ret_t mptcp_subflow_mpcantrcvmore_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) + uint64_t *p_mpsofilt_hint, uint64_t event) { - struct socket *so, *mp_so; +#pragma unused(event) struct mptcb *mp_tp; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); - mp_so = mpte->mpte_mppcb->mpp_socket; - so = mpts->mpts_socket; + mpte_lock_assert_held(mpte); /* same as MP socket lock */ mp_tp = mpte->mpte_mptcb; - mptcplog((LOG_DEBUG, "MPTCP Events: " - "%s: cid %d\n", __func__, mpts->mpts_connid), + mptcplog((LOG_DEBUG, "%s: cid %d\n", __func__, mpts->mpts_connid), MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); /* @@ -2462,11 +3202,9 @@ mptcp_subflow_mpcantrcvmore_ev(struct mptses *mpte, struct mptsub *mpts, * mptcp socket and the user is notified so that it may close * the socket if needed. */ - MPT_LOCK(mp_tp); if (mp_tp->mpt_state == MPTCPS_CLOSE_WAIT) - *p_mpsofilt_hint |= SO_FILT_HINT_LOCKED | SO_FILT_HINT_CANTRCVMORE; + *p_mpsofilt_hint |= SO_FILT_HINT_CANTRCVMORE; - MPT_UNLOCK(mp_tp); return (MPTS_EVRET_OK); /* keep the subflow socket around */ } @@ -2475,106 +3213,68 @@ mptcp_subflow_mpcantrcvmore_ev(struct mptses *mpte, struct mptsub *mpts, */ static ev_ret_t mptcp_subflow_failover_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) + uint64_t *p_mpsofilt_hint, uint64_t event) { +#pragma unused(event, p_mpsofilt_hint) struct mptsub *mpts_alt = NULL; - struct socket *so = NULL; + struct socket *alt_so = NULL; struct socket *mp_so; int altpath_exists = 0; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); - mp_so = mpte->mpte_mppcb->mpp_socket; - mptcplog((LOG_NOTICE, "MPTCP Events: " - "%s: mp_so 0x%llx\n", __func__, - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so)), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); + mpte_lock_assert_held(mpte); + mp_so = mptetoso(mpte); + mptcplog((LOG_NOTICE, "%s: mp_so 0x%llx\n", __func__, + (u_int64_t)VM_KERNEL_ADDRPERM(mp_so)), + MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); - MPTS_UNLOCK(mpts); - mpts_alt = mptcp_get_subflow(mpte, mpts, NULL); + mptcp_reinject_mbufs(mpts->mpts_socket); + mpts_alt = mptcp_get_subflow(mpte, mpts, NULL); /* * If there is no alternate eligible subflow, ignore the * failover hint. */ if (mpts_alt == NULL) { - mptcplog((LOG_WARNING, "MPTCP Events: " - "%s: no alternate path\n", __func__), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_ERR); - - if (mptcp_delayed_subf_start) { - mpts_alt = mptcp_get_pending_subflow(mpte, mpts); - if (mpts_alt != NULL) { - MPTS_LOCK(mpts_alt); - (void) mptcp_subflow_soconnectx(mpte, - mpts_alt); - MPTS_UNLOCK(mpts_alt); - } - } - MPTS_LOCK(mpts); + mptcplog((LOG_WARNING, "%s: no alternate path\n", __func__), + MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); + goto done; } - MPTS_LOCK(mpts_alt); + altpath_exists = 1; - so = mpts_alt->mpts_socket; + alt_so = mpts_alt->mpts_socket; if (mpts_alt->mpts_flags & MPTSF_FAILINGOVER) { - socket_lock(so, 1); /* All data acknowledged and no RTT spike */ - if ((so->so_snd.sb_cc == 0) && - (mptcp_no_rto_spike(so))) { - so->so_flags &= ~SOF_MP_TRYFAILOVER; + if (alt_so->so_snd.sb_cc == 0 && mptcp_no_rto_spike(alt_so)) { mpts_alt->mpts_flags &= ~MPTSF_FAILINGOVER; } else { /* no alternate path available */ altpath_exists = 0; } - socket_unlock(so, 1); - } - if (altpath_exists) { - mptcplog((LOG_INFO, "MPTCP Events: " - "%s: cid = %d\n", - __func__, mpts_alt->mpts_connid), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); - mpts_alt->mpts_flags |= MPTSF_ACTIVE; - mpts_alt->mpts_peerswitch = 0; - struct mptcb *mp_tp = mpte->mpte_mptcb; - /* Bring the subflow's notion of snd_nxt into the send window */ - MPT_LOCK(mp_tp); - mpts_alt->mpts_sndnxt = mp_tp->mpt_snduna; - MPT_UNLOCK(mp_tp); - mpte->mpte_active_sub = mpts_alt; - socket_lock(so, 1); - sowwakeup(so); - socket_unlock(so, 1); } - MPTS_UNLOCK(mpts_alt); if (altpath_exists) { - *p_mpsofilt_hint |= SO_FILT_HINT_LOCKED | SO_FILT_HINT_CONNINFO_UPDATED; - mptcplog((LOG_NOTICE, "MPTCP Events: " - "%s: mp_so 0x%llx switched from " - "%d to %d\n", __func__, - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), - mpts->mpts_connid, mpts_alt->mpts_connid), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); - tcpstat.tcps_mp_switches++; - } + mpts_alt->mpts_flags |= MPTSF_ACTIVE; - MPTS_LOCK(mpts); - if (altpath_exists) { + mpte->mpte_active_sub = mpts_alt; mpts->mpts_flags |= MPTSF_FAILINGOVER; mpts->mpts_flags &= ~MPTSF_ACTIVE; + + mptcplog((LOG_NOTICE, "%s: switched from %d to %d\n", + __func__, mpts->mpts_connid, mpts_alt->mpts_connid), + MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); + + mptcpstats_inc_switch(mpte, mpts); + + sowwakeup(alt_so); } else { - mptcplog((LOG_DEBUG, "MPTCP Events %s: no alt cid = %d\n", - __func__, mpts->mpts_connid), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); + mptcplog((LOG_DEBUG, "%s: no alt cid = %d\n", __func__, + mpts->mpts_connid), + MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); done: - so = mpts->mpts_socket; - socket_lock(so, 1); - so->so_flags &= ~SOF_MP_TRYFAILOVER; - socket_unlock(so, 1); + mpts->mpts_socket->so_flags &= ~SOF_MP_TRYFAILOVER; } - MPTS_LOCK_ASSERT_HELD(mpts); + return (MPTS_EVRET_OK); } @@ -2583,102 +3283,23 @@ done: */ static ev_ret_t mptcp_subflow_ifdenied_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) + uint64_t *p_mpsofilt_hint, uint64_t event) { - struct socket *mp_so, *so; - struct mptcb *mp_tp; - boolean_t linger; - - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); + mpte_lock_assert_held(mpte); /* same as MP socket lock */ VERIFY(mpte->mpte_mppcb != NULL); - mp_so = mpte->mpte_mppcb->mpp_socket; - mp_tp = mpte->mpte_mptcb; - so = mpts->mpts_socket; - - linger = (!(mpts->mpts_flags & MPTSF_DELETEOK) && - !(mp_so->so_flags & SOF_PCBCLEARING)); - - mptcplog((LOG_DEBUG, "MPTCP Events: " - "%s: cid %d [linger %s]\n", __func__, - mpts->mpts_connid, (linger ? "YES" : "NO")), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); - - if (mpts->mpts_soerror == 0) - mpts->mpts_soerror = EHOSTUNREACH; - /* - * The subflow connection cannot use the outgoing interface. - * - * Right now, we simply propagate EHOSTUNREACH to the MPTCP socket - * client if the MPTCP connection has not been established. If it - * has been established, let the upper layer call disconnectx. - */ - mptcp_subflow_disconnect(mpte, mpts, !linger); - *p_mpsofilt_hint |= SO_FILT_HINT_LOCKED | SO_FILT_HINT_IFDENIED; - - MPT_LOCK(mp_tp); - if (mp_tp->mpt_state < MPTCPS_ESTABLISHED) { - mp_so->so_error = EHOSTUNREACH; - } - MPT_UNLOCK(mp_tp); + mptcplog((LOG_DEBUG, "%s: cid %d\n", __func__, + mpts->mpts_connid), MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); /* - * Keep the subflow socket around, unless the MPTCP socket has - * been detached or the subflow has been disconnected explicitly, - * in which case it should be deleted right away. + * The subflow connection cannot use the outgoing interface, let's + * close this subflow. */ - return (linger ? MPTS_EVRET_OK : MPTS_EVRET_DELETE); -} - -/* - * Handle SO_FILT_HINT_SUSPEND subflow socket event. - */ -static ev_ret_t -mptcp_subflow_suspend_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) -{ -#pragma unused(p_mpsofilt_hint) - struct socket *so; - - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); + mptcp_subflow_abort(mpts, EPERM); - so = mpts->mpts_socket; - - /* the subflow connection is being flow controlled */ - mpts->mpts_flags |= MPTSF_SUSPENDED; - - mptcplog((LOG_DEBUG, "MPTCP Events: " - "%s: cid %d\n", __func__, - mpts->mpts_connid), MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); - - return (MPTS_EVRET_OK); /* keep the subflow socket around */ -} - -/* - * Handle SO_FILT_HINT_RESUME subflow socket event. - */ -static ev_ret_t -mptcp_subflow_resume_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) -{ -#pragma unused(p_mpsofilt_hint) - struct socket *so; - - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); - - so = mpts->mpts_socket; - - /* the subflow connection is no longer flow controlled */ - mpts->mpts_flags &= ~MPTSF_SUSPENDED; - - mptcplog((LOG_DEBUG, "MPTCP Events: " - "%s: cid %d\n", __func__, mpts->mpts_connid), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); + mptcp_subflow_propagate_ev(mpte, mpts, p_mpsofilt_hint, event); - return (MPTS_EVRET_OK); /* keep the subflow socket around */ + return (MPTS_EVRET_DELETE); } /* @@ -2686,45 +3307,39 @@ mptcp_subflow_resume_ev(struct mptses *mpte, struct mptsub *mpts, */ static ev_ret_t mptcp_subflow_connected_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) + uint64_t *p_mpsofilt_hint, uint64_t event) { - char buf0[MAX_IPv6_STR_LEN], buf1[MAX_IPv6_STR_LEN]; - struct sockaddr_storage src; +#pragma unused(event, p_mpsofilt_hint) struct socket *mp_so, *so; + struct inpcb *inp; + struct tcpcb *tp; struct mptcb *mp_tp; - struct ifnet *outifp; - int af, error = 0; + int af; boolean_t mpok = FALSE; - boolean_t cell = FALSE; - boolean_t wifi = FALSE; - boolean_t wired = FALSE; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + mpte_lock_assert_held(mpte); /* same as MP socket lock */ VERIFY(mpte->mpte_mppcb != NULL); - mp_so = mpte->mpte_mppcb->mpp_socket; - mp_tp = mpte->mpte_mptcb; - MPTS_LOCK_ASSERT_HELD(mpts); + mp_so = mptetoso(mpte); + mp_tp = mpte->mpte_mptcb; so = mpts->mpts_socket; - af = mpts->mpts_family; + tp = sototcpcb(so); + af = mpts->mpts_dst.sa_family; if (mpts->mpts_flags & MPTSF_CONNECTED) return (MPTS_EVRET_OK); if ((mpts->mpts_flags & MPTSF_DISCONNECTED) || (mpts->mpts_flags & MPTSF_DISCONNECTING)) { - socket_lock(so, 0); if (!(so->so_state & (SS_ISDISCONNECTING | SS_ISDISCONNECTED)) && (so->so_state & SS_ISCONNECTED)) { - mptcplog((LOG_DEBUG, "MPTCP Events: " - "%s: cid %d disconnect before tcp connect\n", + mptcplog((LOG_DEBUG, "%s: cid %d disconnect before tcp connect\n", __func__, mpts->mpts_connid), MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); (void) soshutdownlock(so, SHUT_RD); (void) soshutdownlock(so, SHUT_WR); (void) sodisconnectlocked(so); } - socket_unlock(so, 0); return (MPTS_EVRET_OK); } @@ -2734,19 +3349,15 @@ mptcp_subflow_connected_ev(struct mptses *mpte, struct mptsub *mpts, * * a. If MPTCP connection is not yet established, then this must be * the first subflow connection. If MPTCP failed to negotiate, - * indicate to the MPTCP socket client via EPROTO, that the - * underlying TCP connection may be peeled off via peeloff(2). - * Otherwise, mark the MPTCP socket as connected. + * fallback to regular TCP by degrading this subflow. * * b. If MPTCP connection has been established, then this must be * one of the subsequent subflow connections. If MPTCP failed - * to negotiate, disconnect the connection since peeloff(2) - * is no longer possible. + * to negotiate, disconnect the connection. * * Right now, we simply unblock any waiters at the MPTCP socket layer * if the MPTCP connection has not been established. */ - socket_lock(so, 0); if (so->so_state & SS_ISDISCONNECTED) { /* @@ -2756,203 +3367,80 @@ mptcp_subflow_connected_ev(struct mptses *mpte, struct mptsub *mpts, * where the subflow could get disconnected before the * connected event is processed. */ - socket_unlock(so, 0); return (MPTS_EVRET_OK); } - mpts->mpts_soerror = 0; - mpts->mpts_flags &= ~MPTSF_CONNECTING; - mpts->mpts_flags |= MPTSF_CONNECTED; + if (mpts->mpts_flags & MPTSF_TFO_REQD) + mptcp_drop_tfo_data(mpte, mpts); - if (!(so->so_flags1 & SOF1_DATA_IDEMPOTENT)) - mpts->mpts_flags &= ~MPTSF_TFO_REQD; + mpts->mpts_flags &= ~(MPTSF_CONNECTING | MPTSF_TFO_REQD); + mpts->mpts_flags |= MPTSF_CONNECTED; - struct tcpcb *tp = sototcpcb(so); if (tp->t_mpflags & TMPF_MPTCP_TRUE) mpts->mpts_flags |= MPTSF_MP_CAPABLE; tp->t_mpflags &= ~TMPF_TFO_REQUEST; - VERIFY(mpts->mpts_dst != NULL); - - VERIFY(mpts->mpts_src != NULL); - - /* get/check source IP address */ - switch (af) { - case AF_INET: { - error = in_getsockaddr_s(so, &src); - if (error == 0) { - struct sockaddr_in *ms = SIN(mpts->mpts_src); - struct sockaddr_in *s = SIN(&src); - - VERIFY(s->sin_len == ms->sin_len); - VERIFY(ms->sin_family == AF_INET); - - if ((mpts->mpts_flags & MPTSF_BOUND_IP) && - bcmp(&ms->sin_addr, &s->sin_addr, - sizeof (ms->sin_addr)) != 0) { - mptcplog((LOG_ERR, "MPTCP Events: " - "%s: cid %d local " - "address %s (expected %s)\n", __func__, - mpts->mpts_connid, inet_ntop(AF_INET, - (void *)&s->sin_addr.s_addr, buf0, - sizeof (buf0)), inet_ntop(AF_INET, - (void *)&ms->sin_addr.s_addr, buf1, - sizeof (buf1))), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_ERR); - } - bcopy(s, ms, sizeof (*s)); - } - break; - } -#if INET6 - case AF_INET6: { - error = in6_getsockaddr_s(so, &src); - if (error == 0) { - struct sockaddr_in6 *ms = SIN6(mpts->mpts_src); - struct sockaddr_in6 *s = SIN6(&src); - - VERIFY(s->sin6_len == ms->sin6_len); - VERIFY(ms->sin6_family == AF_INET6); - - if ((mpts->mpts_flags & MPTSF_BOUND_IP) && - bcmp(&ms->sin6_addr, &s->sin6_addr, - sizeof (ms->sin6_addr)) != 0) { - mptcplog((LOG_ERR, "MPTCP Events: " - "%s: cid %d local " - "address %s (expected %s)\n", __func__, - mpts->mpts_connid, inet_ntop(AF_INET6, - (void *)&s->sin6_addr, buf0, - sizeof (buf0)), inet_ntop(AF_INET6, - (void *)&ms->sin6_addr, buf1, - sizeof (buf1))), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_ERR); - } - bcopy(s, ms, sizeof (*s)); - } - break; - } -#endif /* INET6 */ - default: - VERIFY(0); - /* NOTREACHED */ - } - - if (error != 0) { - mptcplog((LOG_ERR, "MPTCP Events " - "%s: cid %d getsockaddr failed (%d)\n", - __func__, mpts->mpts_connid, error), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_ERR); - } - /* get/verify the outbound interface */ - outifp = sotoinpcb(so)->inp_last_outifp; /* could be NULL */ - if (mpts->mpts_flags & MPTSF_BOUND_IF) { - VERIFY(mpts->mpts_outif != NULL); - if (mpts->mpts_outif != outifp) { - mptcplog((LOG_ERR, "MPTCP Events: %s: cid %d outif %s " - "(expected %s)\n", __func__, mpts->mpts_connid, - ((outifp != NULL) ? outifp->if_xname : "NULL"), - mpts->mpts_outif->if_xname), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_ERR); - - if (outifp == NULL) - outifp = mpts->mpts_outif; - } - } else { - mpts->mpts_outif = outifp; - } - - mpts->mpts_srtt = (intotcpcb(sotoinpcb(so)))->t_srtt; - mpts->mpts_rxtcur = (intotcpcb(sotoinpcb(so)))->t_rxtcur; - mpts->mpts_maxseg = (intotcpcb(sotoinpcb(so)))->t_maxseg; - - cell = IFNET_IS_CELLULAR(mpts->mpts_outif); - wifi = (!cell && IFNET_IS_WIFI(mpts->mpts_outif)); - wired = (!wifi && IFNET_IS_WIRED(mpts->mpts_outif)); - - if (cell) - mpts->mpts_linktype |= MPTSL_CELL; - else if (wifi) - mpts->mpts_linktype |= MPTSL_WIFI; - else if (wired) - mpts->mpts_linktype |= MPTSL_WIRED; - - socket_unlock(so, 0); - - mptcplog((LOG_DEBUG, "MPTCP Sender: %s: cid %d " - "establishment srtt %d \n", __func__, - mpts->mpts_connid, (mpts->mpts_srtt >> 5)), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); + inp = sotoinpcb(so); + mpts->mpts_maxseg = tp->t_maxseg; - mptcplog((LOG_DEBUG, "MPTCP Socket: " - "%s: cid %d outif %s %s[%d] -> %s[%d] " - "is %s\n", __func__, mpts->mpts_connid, ((outifp != NULL) ? - outifp->if_xname : "NULL"), inet_ntop(af, (af == AF_INET) ? - (void *)&SIN(mpts->mpts_src)->sin_addr.s_addr : - (void *)&SIN6(mpts->mpts_src)->sin6_addr, buf0, sizeof (buf0)), - ((af == AF_INET) ? ntohs(SIN(mpts->mpts_src)->sin_port) : - ntohs(SIN6(mpts->mpts_src)->sin6_port)), - inet_ntop(af, ((af == AF_INET) ? - (void *)&SIN(mpts->mpts_dst)->sin_addr.s_addr : - (void *)&SIN6(mpts->mpts_dst)->sin6_addr), buf1, sizeof (buf1)), - ((af == AF_INET) ? ntohs(SIN(mpts->mpts_dst)->sin_port) : - ntohs(SIN6(mpts->mpts_dst)->sin6_port)), - ((mpts->mpts_flags & MPTSF_MP_CAPABLE) ? - "MPTCP capable" : "a regular TCP")), + mptcplog((LOG_DEBUG, "%s: cid %d outif %s is %s\n", __func__, mpts->mpts_connid, + ((inp->inp_last_outifp != NULL) ? inp->inp_last_outifp->if_xname : "NULL"), + ((mpts->mpts_flags & MPTSF_MP_CAPABLE) ? "MPTCP capable" : "a regular TCP")), (MPTCP_SOCKET_DBG | MPTCP_EVENTS_DBG), MPTCP_LOGLVL_LOG); mpok = (mpts->mpts_flags & MPTSF_MP_CAPABLE); - MPTS_UNLOCK(mpts); - - *p_mpsofilt_hint |= SO_FILT_HINT_LOCKED | SO_FILT_HINT_CONNINFO_UPDATED; - MPT_LOCK(mp_tp); if (mp_tp->mpt_state < MPTCPS_ESTABLISHED) { + mp_tp->mpt_state = MPTCPS_ESTABLISHED; + mpte->mpte_associd = mpts->mpts_connid; + DTRACE_MPTCP2(state__change, + struct mptcb *, mp_tp, + uint32_t, 0 /* event */); + + if (SOCK_DOM(so) == AF_INET) { + in_getsockaddr_s(so, &mpte->__mpte_src_v4); + } else { + in6_getsockaddr_s(so, &mpte->__mpte_src_v6); + } + /* case (a) above */ if (!mpok) { - mp_tp->mpt_flags |= MPTCPF_PEEL_OFF; - (void) mptcp_drop(mpte, mp_tp, EPROTO); - MPT_UNLOCK(mp_tp); + tcpstat.tcps_mpcap_fallback++; + + tp->t_mpflags |= TMPF_INFIN_SENT; + mptcp_notify_mpfail(so); } else { - MPT_UNLOCK(mp_tp); - mptcplog((LOG_DEBUG, "MPTCP State: " - "MPTCPS_ESTABLISHED for mp_so 0x%llx \n", - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so)), - MPTCP_STATE_DBG, MPTCP_LOGLVL_LOG); - mp_tp->mpt_state = MPTCPS_ESTABLISHED; - mpte->mpte_associd = mpts->mpts_connid; - DTRACE_MPTCP2(state__change, - struct mptcb *, mp_tp, - uint32_t, 0 /* event */); - - if (mpts->mpts_outif && - IFNET_IS_EXPENSIVE(mpts->mpts_outif)) { - sototcpcb(so)->t_mpflags |= (TMPF_BACKUP_PATH | TMPF_SND_MPPRIO); + if (IFNET_IS_CELLULAR(inp->inp_last_outifp) && + mpte->mpte_svctype != MPTCP_SVCTYPE_AGGREGATE) { + tp->t_mpflags |= (TMPF_BACKUP_PATH | TMPF_SND_MPPRIO); } else { mpts->mpts_flags |= MPTSF_PREFERRED; } mpts->mpts_flags |= MPTSF_ACTIVE; - soisconnected(mp_so); - } - MPTS_LOCK(mpts); - if (mpok) { + mpts->mpts_flags |= MPTSF_MPCAP_CTRSET; mpte->mpte_nummpcapflows++; - MPT_LOCK_SPIN(mp_tp); - /* With TFO, sndnxt may be initialized earlier */ - if (mpts->mpts_sndnxt == 0) - mpts->mpts_sndnxt = mp_tp->mpt_snduna; - MPT_UNLOCK(mp_tp); + + mptcp_check_subflows_and_add(mpte); + + if (IFNET_IS_CELLULAR(inp->inp_last_outifp)) + mpte->mpte_initial_cell = 1; + + mpte->mpte_handshake_success = 1; } + + mp_tp->mpt_sndwnd = tp->snd_wnd; + mp_tp->mpt_sndwl1 = mp_tp->mpt_rcvnxt; + mp_tp->mpt_sndwl2 = mp_tp->mpt_snduna; + soisconnected(mp_so); + + mptcplog((LOG_DEBUG, "%s: MPTCPS_ESTABLISHED for mp_so 0x%llx mpok %u\n", + __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), mpok), + MPTCP_STATE_DBG, MPTCP_LOGLVL_LOG); } else if (mpok) { - MPT_UNLOCK(mp_tp); - if (mptcp_rwnotify && (mpte->mpte_nummpcapflows == 0)) { - /* Experimental code, disabled by default. */ - sorwakeup(mp_so); - sowwakeup(mp_so); - } /* * case (b) above * In case of additional flows, the MPTCP socket is not @@ -2960,24 +3448,49 @@ mptcp_subflow_connected_ev(struct mptses *mpte, struct mptsub *mpts, * for 3-way handshake. TCP would have guaranteed that this * is an MPTCP subflow. */ - MPTS_LOCK(mpts); + if (IFNET_IS_CELLULAR(inp->inp_last_outifp) && + !(tp->t_mpflags & TMPF_BACKUP_PATH) && + mpte->mpte_svctype != MPTCP_SVCTYPE_AGGREGATE) { + tp->t_mpflags |= (TMPF_BACKUP_PATH | TMPF_SND_MPPRIO); + mpts->mpts_flags &= ~MPTSF_PREFERRED; + } else { + mpts->mpts_flags |= MPTSF_PREFERRED; + } + mpts->mpts_flags |= MPTSF_MPCAP_CTRSET; - mpts->mpts_flags &= ~MPTSF_FASTJ_REQD; mpte->mpte_nummpcapflows++; - MPT_LOCK_SPIN(mp_tp); - /* With Fastjoin, sndnxt is updated before connected_ev */ - if (mpts->mpts_sndnxt == 0) { - mpts->mpts_sndnxt = mp_tp->mpt_snduna; - mpts->mpts_rel_seq = 1; - } - MPT_UNLOCK(mp_tp); - mptcp_output_needed(mpte, mpts); + + mpts->mpts_rel_seq = 1; + + mptcp_check_subflows_and_remove(mpte); } else { - MPT_UNLOCK(mp_tp); - MPTS_LOCK(mpts); + unsigned int i; + + /* Mark this interface as non-MPTCP */ + for (i = 0; i < mpte->mpte_itfinfo_size; i++) { + struct mpt_itf_info *info = &mpte->mpte_itfinfo[i]; + + if (inp->inp_last_outifp->if_index == info->ifindex) { + info->no_mptcp_support = 1; + break; + } + } + + tcpstat.tcps_join_fallback++; + if (IFNET_IS_CELLULAR(inp->inp_last_outifp)) + tcpstat.tcps_mptcp_cell_proxy++; + else + tcpstat.tcps_mptcp_wifi_proxy++; + + soevent(mpts->mpts_socket, SO_FILT_HINT_LOCKED | SO_FILT_HINT_MUSTRST); + + return (MPTS_EVRET_OK); } - MPTS_LOCK_ASSERT_HELD(mpts); + /* This call, just to "book" an entry in the stats-table for this ifindex */ + mptcp_get_statsindex(mpte->mpte_itfstats, mpts); + + mptcp_output(mpte); return (MPTS_EVRET_OK); /* keep the subflow socket around */ } @@ -2987,77 +3500,56 @@ mptcp_subflow_connected_ev(struct mptses *mpte, struct mptsub *mpts, */ static ev_ret_t mptcp_subflow_disconnected_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) + uint64_t *p_mpsofilt_hint, uint64_t event) { +#pragma unused(event, p_mpsofilt_hint) struct socket *mp_so, *so; struct mptcb *mp_tp; - boolean_t linger; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); + mpte_lock_assert_held(mpte); /* same as MP socket lock */ VERIFY(mpte->mpte_mppcb != NULL); - mp_so = mpte->mpte_mppcb->mpp_socket; + mp_so = mptetoso(mpte); mp_tp = mpte->mpte_mptcb; so = mpts->mpts_socket; - linger = (!(mpts->mpts_flags & MPTSF_DELETEOK) && - !(mp_so->so_flags & SOF_PCBCLEARING)); - - mptcplog((LOG_DEBUG, "MPTCP Events: " - "%s: cid %d [linger %s]\n", __func__, - mpts->mpts_connid, (linger ? "YES" : "NO")), + mptcplog((LOG_DEBUG, "%s: cid %d, so_err %d, mpt_state %u fallback %u active %u flags %#x\n", + __func__, mpts->mpts_connid, so->so_error, mp_tp->mpt_state, + !!(mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP), + !!(mpts->mpts_flags & MPTSF_ACTIVE), sototcpcb(so)->t_mpflags), MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); if (mpts->mpts_flags & MPTSF_DISCONNECTED) - return (linger ? MPTS_EVRET_OK : MPTS_EVRET_DELETE); + return (MPTS_EVRET_DELETE); - /* - * Clear flags that are used by getconninfo to return state. - * Retain like MPTSF_DELETEOK for internal purposes. - */ - mpts->mpts_flags &= ~(MPTSF_CONNECTING|MPTSF_CONNECT_PENDING| - MPTSF_CONNECTED|MPTSF_DISCONNECTING|MPTSF_PREFERRED| - MPTSF_MP_CAPABLE|MPTSF_MP_READY|MPTSF_MP_DEGRADED| - MPTSF_SUSPENDED|MPTSF_ACTIVE); mpts->mpts_flags |= MPTSF_DISCONNECTED; - /* - * The subflow connection has been disconnected. - * - * Right now, we simply unblock any waiters at the MPTCP socket layer - * if the MPTCP connection has not been established. - */ - *p_mpsofilt_hint |= SO_FILT_HINT_LOCKED | SO_FILT_HINT_CONNINFO_UPDATED; + /* The subflow connection has been disconnected. */ if (mpts->mpts_flags & MPTSF_MPCAP_CTRSET) { mpte->mpte_nummpcapflows--; if (mpte->mpte_active_sub == mpts) { mpte->mpte_active_sub = NULL; - mptcplog((LOG_DEBUG, "MPTCP Events: " - "%s: resetting active subflow \n", + mptcplog((LOG_DEBUG, "%s: resetting active subflow \n", __func__), MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); } mpts->mpts_flags &= ~MPTSF_MPCAP_CTRSET; } - MPT_LOCK(mp_tp); - if (mp_tp->mpt_state < MPTCPS_ESTABLISHED) { - MPT_UNLOCK(mp_tp); - MPTS_UNLOCK(mpts); - soisdisconnected(mp_so); - MPTS_LOCK(mpts); - } else { - MPT_UNLOCK(mp_tp); + if (mp_tp->mpt_state < MPTCPS_ESTABLISHED || + ((mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) && (mpts->mpts_flags & MPTSF_ACTIVE)) || + (sototcpcb(so)->t_mpflags & TMPF_FASTCLOSERCV)) { + mptcp_drop(mpte, mp_tp, so->so_error); } /* - * The underlying subflow socket has been disconnected; - * it is no longer useful to us. Keep the subflow socket - * around, unless the MPTCP socket has been detached or - * the subflow has been disconnected explicitly, in which - * case it should be deleted right away. + * Clear flags that are used by getconninfo to return state. + * Retain like MPTSF_DELETEOK for internal purposes. */ - return (linger ? MPTS_EVRET_OK : MPTS_EVRET_DELETE); + mpts->mpts_flags &= ~(MPTSF_CONNECTING|MPTSF_CONNECT_PENDING| + MPTSF_CONNECTED|MPTSF_DISCONNECTING|MPTSF_PREFERRED| + MPTSF_MP_CAPABLE|MPTSF_MP_READY|MPTSF_MP_DEGRADED|MPTSF_ACTIVE); + + return (MPTS_EVRET_DELETE); } /* @@ -3065,23 +3557,19 @@ mptcp_subflow_disconnected_ev(struct mptses *mpte, struct mptsub *mpts, */ static ev_ret_t mptcp_subflow_mpstatus_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) + uint64_t *p_mpsofilt_hint, uint64_t event) { +#pragma unused(event, p_mpsofilt_hint) struct socket *mp_so, *so; struct mptcb *mp_tp; ev_ret_t ret = MPTS_EVRET_OK; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + mpte_lock_assert_held(mpte); /* same as MP socket lock */ VERIFY(mpte->mpte_mppcb != NULL); - mp_so = mpte->mpte_mppcb->mpp_socket; + mp_so = mptetoso(mpte); mp_tp = mpte->mpte_mptcb; - - MPTS_LOCK_ASSERT_HELD(mpts); so = mpts->mpts_socket; - socket_lock(so, 0); - MPT_LOCK(mp_tp); - if (sototcpcb(so)->t_mpflags & TMPF_MPTCP_TRUE) mpts->mpts_flags |= MPTSF_MP_CAPABLE; else @@ -3108,27 +3596,18 @@ mptcp_subflow_mpstatus_ev(struct mptses *mpte, struct mptsub *mpts, if (mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) { VERIFY(!(mp_tp->mpt_flags & MPTCPF_JOIN_READY)); ret = MPTS_EVRET_DISCONNECT_FALLBACK; - *p_mpsofilt_hint |= SO_FILT_HINT_LOCKED | - SO_FILT_HINT_CONNINFO_UPDATED; } else if (mpts->mpts_flags & MPTSF_MP_READY) { mp_tp->mpt_flags |= MPTCPF_JOIN_READY; ret = MPTS_EVRET_CONNECT_PENDING; - } else { - *p_mpsofilt_hint |= SO_FILT_HINT_LOCKED | - SO_FILT_HINT_CONNINFO_UPDATED; } - mptcplog((LOG_DEBUG, "MPTCP Events: " - "%s: mp_so 0x%llx mpt_flags=%b cid %d " - "mptsf=%b\n", __func__, - (u_int64_t)VM_KERNEL_ADDRPERM(mpte->mpte_mppcb->mpp_socket), - mp_tp->mpt_flags, MPTCPF_BITS, mpts->mpts_connid, - mpts->mpts_flags, MPTSF_BITS), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); + mptcplog((LOG_DEBUG, "%s: mp_so 0x%llx mpt_flags=%b cid %d mptsf=%b\n", + __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), + mp_tp->mpt_flags, MPTCPF_BITS, mpts->mpts_connid, + mpts->mpts_flags, MPTSF_BITS), + MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); done: - MPT_UNLOCK(mp_tp); - socket_unlock(so, 0); return (ret); } @@ -3137,28 +3616,20 @@ done: */ static ev_ret_t mptcp_subflow_mustrst_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) + uint64_t *p_mpsofilt_hint, uint64_t event) { +#pragma unused(event) struct socket *mp_so, *so; struct mptcb *mp_tp; - boolean_t linger, is_fastclose; + boolean_t is_fastclose; - - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); + mpte_lock_assert_held(mpte); /* same as MP socket lock */ VERIFY(mpte->mpte_mppcb != NULL); - mp_so = mpte->mpte_mppcb->mpp_socket; + mp_so = mptetoso(mpte); mp_tp = mpte->mpte_mptcb; so = mpts->mpts_socket; - linger = (!(mpts->mpts_flags & MPTSF_DELETEOK) && - !(mp_so->so_flags & SOF_PCBCLEARING)); - - if (mpts->mpts_soerror == 0) - mpts->mpts_soerror = ECONNABORTED; - /* We got an invalid option or a fast close */ - socket_lock(so, 0); struct tcptemp *t_template; struct inpcb *inp = sotoinpcb(so); struct tcpcb *tp = NULL; @@ -3189,12 +3660,7 @@ mptcp_subflow_mustrst_ev(struct mptses *mpte, struct mptsub *mpts, so, mpts->mpts_connid), MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); } - socket_unlock(so, 0); - mptcp_subflow_disconnect(mpte, mpts, !linger); - - *p_mpsofilt_hint |= (SO_FILT_HINT_LOCKED | SO_FILT_HINT_CONNINFO_UPDATED); - - MPT_LOCK(mp_tp); + mptcp_subflow_abort(mpts, ECONNABORTED); if (!(mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) && is_fastclose) { *p_mpsofilt_hint |= SO_FILT_HINT_CONNRESET; @@ -3212,65 +3678,64 @@ mptcp_subflow_mustrst_ev(struct mptses *mpte, struct mptsub *mpts, if (mp_tp->mpt_gc_ticks == MPT_GC_TICKS) mp_tp->mpt_gc_ticks = MPT_GC_TICKS_FAST; - MPT_UNLOCK(mp_tp); - /* - * Keep the subflow socket around unless the subflow has been - * disconnected explicitly. - */ - return (linger ? MPTS_EVRET_OK : MPTS_EVRET_DELETE); + return (MPTS_EVRET_DELETE); } static ev_ret_t -mptcp_fastjoin_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) +mptcp_subflow_adaptive_rtimo_ev(struct mptses *mpte, struct mptsub *mpts, + uint64_t *p_mpsofilt_hint, uint64_t event) { -#pragma unused(p_mpsofilt_hint) - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPTS_LOCK_ASSERT_HELD(mpts); - VERIFY(mpte->mpte_mppcb != NULL); +#pragma unused(event) + bool found_active = false; + + mpts->mpts_flags |= MPTSF_READ_STALL; - if (mpte->mpte_nummpcapflows == 0) { - struct mptcb *mp_tp = mpte->mpte_mptcb; - mptcplog((LOG_DEBUG,"MPTCP Events: %s: %llx %llx \n", - __func__, mp_tp->mpt_snduna, mpts->mpts_sndnxt), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); + TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { + struct tcpcb *tp = sototcpcb(mpts->mpts_socket); - mpte->mpte_active_sub = mpts; - mpts->mpts_flags |= (MPTSF_FASTJ_SEND | MPTSF_ACTIVE); - MPT_LOCK(mp_tp); - /* - * If mptcp_subflow_output is called before fastjoin_ev - * then mpts->mpts_sndnxt is initialized to mp_tp->mpt_snduna - * and further mpts->mpts_sndnxt is incremented by len copied. - */ - if (mpts->mpts_sndnxt == 0) { - mpts->mpts_sndnxt = mp_tp->mpt_snduna; + if (!TCPS_HAVEESTABLISHED(tp->t_state) || + TCPS_HAVERCVDFIN2(tp->t_state)) + continue; + + if (!(mpts->mpts_flags & MPTSF_READ_STALL)) { + found_active = true; + break; } - MPT_UNLOCK(mp_tp); } + if (!found_active) + *p_mpsofilt_hint |= SO_FILT_HINT_ADAPTIVE_RTIMO; + return (MPTS_EVRET_OK); } static ev_ret_t -mptcp_deleteok_ev(struct mptses *mpte, struct mptsub *mpts, - uint64_t *p_mpsofilt_hint) +mptcp_subflow_adaptive_wtimo_ev(struct mptses *mpte, struct mptsub *mpts, + uint64_t *p_mpsofilt_hint, uint64_t event) { -#pragma unused(p_mpsofilt_hint) - MPTE_LOCK_ASSERT_HELD(mpte); - MPTS_LOCK_ASSERT_HELD(mpts); - VERIFY(mpte->mpte_mppcb != NULL); +#pragma unused(event) + bool found_active = false; - mptcplog((LOG_DEBUG, "MPTCP Events: " - "%s cid %d\n", __func__, mpts->mpts_connid), - MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); + mpts->mpts_flags |= MPTSF_WRITE_STALL; - mpts->mpts_flags |= MPTSF_DELETEOK; - if (mpts->mpts_flags & MPTSF_DISCONNECTED) - return (MPTS_EVRET_DELETE); - else - return (MPTS_EVRET_OK); + TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { + struct tcpcb *tp = sototcpcb(mpts->mpts_socket); + + if (!TCPS_HAVEESTABLISHED(tp->t_state) || + tp->t_state > TCPS_CLOSE_WAIT) + continue; + + if (!(mpts->mpts_flags & MPTSF_WRITE_STALL)) { + found_active = true; + break; + } + } + + if (!found_active) + *p_mpsofilt_hint |= SO_FILT_HINT_ADAPTIVE_WTIMO; + + return (MPTS_EVRET_OK); } static const char * @@ -3297,63 +3762,58 @@ mptcp_evret2str(ev_ret_t ret) return (c); } -/* - * Add a reference to a subflow structure; used by MPTS_ADDREF(). - */ -void -mptcp_subflow_addref(struct mptsub *mpts, int locked) -{ - if (!locked) - MPTS_LOCK(mpts); - else - MPTS_LOCK_ASSERT_HELD(mpts); - - if (++mpts->mpts_refcnt == 0) { - panic("%s: mpts %p wraparound refcnt\n", __func__, mpts); - /* NOTREACHED */ - } - if (!locked) - MPTS_UNLOCK(mpts); -} - -/* - * Remove a reference held on a subflow structure; used by MPTS_REMREF(); - */ -void -mptcp_subflow_remref(struct mptsub *mpts) -{ - MPTS_LOCK(mpts); - if (mpts->mpts_refcnt == 0) { - panic("%s: mpts %p negative refcnt\n", __func__, mpts); - /* NOTREACHED */ - } - if (--mpts->mpts_refcnt > 0) { - MPTS_UNLOCK(mpts); - return; - } - /* callee will unlock and destroy lock */ - mptcp_subflow_free(mpts); -} - /* * Issues SOPT_SET on an MPTCP subflow socket; socket must already be locked, * caller must ensure that the option can be issued on subflow sockets, via * MPOF_SUBFLOW_OK flag. */ int -mptcp_subflow_sosetopt(struct mptses *mpte, struct socket *so, - struct mptopt *mpo) +mptcp_subflow_sosetopt(struct mptses *mpte, struct mptsub *mpts, struct mptopt *mpo) { - struct socket *mp_so; + struct socket *mp_so, *so; struct sockopt sopt; - char buf[32]; int error; VERIFY(mpo->mpo_flags & MPOF_SUBFLOW_OK); - mpo->mpo_flags &= ~MPOF_INTERIM; + mpte_lock_assert_held(mpte); + + mp_so = mptetoso(mpte); + so = mpts->mpts_socket; + + if (mpte->mpte_mptcb->mpt_state >= MPTCPS_ESTABLISHED && + mpo->mpo_level == SOL_SOCKET && + mpo->mpo_name == SO_MARK_CELLFALLBACK) { + mptcplog((LOG_DEBUG, "%s Setting CELL_FALLBACK, mpte_flags %#x, svctype %u wifi unusable %u lastcell? %d boundcell? %d\n", + __func__, mpte->mpte_flags, mpte->mpte_svctype, mptcp_is_wifi_unusable(), + sotoinpcb(so)->inp_last_outifp ? IFNET_IS_CELLULAR(sotoinpcb(so)->inp_last_outifp) : -1, + mpts->mpts_ifscope != IFSCOPE_NONE ? IFNET_IS_CELLULAR(ifindex2ifnet[mpts->mpts_ifscope]) : -1), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + + /* + * When we open a new subflow, mark it as cell fallback, if + * this subflow goes over cell. + * + * (except for first-party apps) + */ + + if (mpte->mpte_flags & MPTE_FIRSTPARTY) + return (0); - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - mp_so = mpte->mpte_mppcb->mpp_socket; + if (sotoinpcb(so)->inp_last_outifp && + !IFNET_IS_CELLULAR(sotoinpcb(so)->inp_last_outifp)) + return (0); + + /* + * This here is an OR, because if the app is not binding to the + * interface, then it definitely is not a cell-fallback + * connection. + */ + if (mpts->mpts_ifscope == IFSCOPE_NONE || + !IFNET_IS_CELLULAR(ifindex2ifnet[mpts->mpts_ifscope])) + return (0); + } + + mpo->mpo_flags &= ~MPOF_INTERIM; bzero(&sopt, sizeof (sopt)); sopt.sopt_dir = SOPT_SET; @@ -3363,23 +3823,21 @@ mptcp_subflow_sosetopt(struct mptses *mpte, struct socket *so, sopt.sopt_valsize = sizeof (int); sopt.sopt_p = kernproc; - error = sosetoptlock(so, &sopt, 0); /* already locked */ + error = sosetoptlock(so, &sopt, 0); if (error == 0) { - mptcplog((LOG_DEBUG, "MPTCP Socket: " - "%s: mp_so 0x%llx sopt %s " + mptcplog((LOG_INFO, "%s: mp_so 0x%llx sopt %s " "val %d set successful\n", __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), - mptcp_sopt2str(mpo->mpo_level, mpo->mpo_name, - buf, sizeof (buf)), mpo->mpo_intval), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + mptcp_sopt2str(mpo->mpo_level, mpo->mpo_name), + mpo->mpo_intval), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); } else { - mptcplog((LOG_ERR, "MPTCP Socket: " - "%s: mp_so 0x%llx sopt %s " + mptcplog((LOG_ERR, "%s:mp_so 0x%llx sopt %s " "val %d set error %d\n", __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), - mptcp_sopt2str(mpo->mpo_level, mpo->mpo_name, - buf, sizeof (buf)), mpo->mpo_intval, error), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + mptcp_sopt2str(mpo->mpo_level, mpo->mpo_name), + mpo->mpo_intval, error), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); } return (error); } @@ -3395,12 +3853,11 @@ mptcp_subflow_sogetopt(struct mptses *mpte, struct socket *so, { struct socket *mp_so; struct sockopt sopt; - char buf[32]; int error; VERIFY(mpo->mpo_flags & MPOF_SUBFLOW_OK); - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - mp_so = mpte->mpte_mppcb->mpp_socket; + mpte_lock_assert_held(mpte); /* same as MP socket lock */ + mp_so = mptetoso(mpte); bzero(&sopt, sizeof (sopt)); sopt.sopt_dir = SOPT_GET; @@ -3416,15 +3873,14 @@ mptcp_subflow_sogetopt(struct mptses *mpte, struct socket *so, "%s: mp_so 0x%llx sopt %s " "val %d get successful\n", __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), - mptcp_sopt2str(mpo->mpo_level, mpo->mpo_name, - buf, sizeof (buf)), mpo->mpo_intval), + mptcp_sopt2str(mpo->mpo_level, mpo->mpo_name), + mpo->mpo_intval), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); } else { mptcplog((LOG_ERR, "MPTCP Socket: " "%s: mp_so 0x%llx sopt %s get error %d\n", __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), - mptcp_sopt2str(mpo->mpo_level, - mpo->mpo_name, buf, sizeof (buf)), error), + mptcp_sopt2str(mpo->mpo_level, mpo->mpo_name), error), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); } return (error); @@ -3444,7 +3900,7 @@ mptcp_gc(struct mppcbinfo *mppi) struct mppcb *mpp, *tmpp; uint32_t active = 0; - lck_mtx_assert(&mppi->mppi_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&mppi->mppi_lock, LCK_MTX_ASSERT_OWNED); TAILQ_FOREACH_SAFE(mpp, &mppi->mppi_pcbs, mpp_entry, tmpp) { struct socket *mp_so; @@ -3466,9 +3922,9 @@ mptcp_gc(struct mppcbinfo *mppi) mp_so->so_retaincnt, mpp->mpp_state), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); - if (!lck_mtx_try_lock(&mpp->mpp_lock)) { + if (!mpte_try_lock(mpte)) { mptcplog((LOG_DEBUG, "MPTCP Socket: " - "%s: mp_so 0x%llx skipped " + "%s: mp_so 0x%llx skipped lock " "(u=%d,r=%d)\n", __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), mp_so->so_usecount, mp_so->so_retaincnt), @@ -3478,12 +3934,12 @@ mptcp_gc(struct mppcbinfo *mppi) } /* check again under the lock */ - if (mp_so->so_usecount > 1) { + if (mp_so->so_usecount > 0) { boolean_t wakeup = FALSE; struct mptsub *mpts, *tmpts; mptcplog((LOG_DEBUG, "MPTCP Socket: " - "%s: mp_so 0x%llx skipped " + "%s: mp_so 0x%llx skipped usecount " "[u=%d,r=%d] %d %d\n", __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), mp_so->so_usecount, mp_so->so_retaincnt, @@ -3491,70 +3947,37 @@ mptcp_gc(struct mppcbinfo *mppi) mp_tp->mpt_state), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); - MPT_LOCK(mp_tp); if (mp_tp->mpt_state >= MPTCPS_FIN_WAIT_1) { if (mp_tp->mpt_gc_ticks > 0) mp_tp->mpt_gc_ticks--; if (mp_tp->mpt_gc_ticks == 0) { wakeup = TRUE; - if (mp_tp->mpt_localkey != NULL) { - mptcp_free_key( - mp_tp->mpt_localkey); - mp_tp->mpt_localkey = NULL; - } } } - MPT_UNLOCK(mp_tp); if (wakeup) { TAILQ_FOREACH_SAFE(mpts, &mpte->mpte_subflows, mpts_entry, tmpts) { - MPTS_LOCK(mpts); - mpts->mpts_flags |= MPTSF_DELETEOK; - if (mpts->mpts_soerror == 0) - mpts->mpts_soerror = ETIMEDOUT; - mptcp_subflow_eupcall(mpts->mpts_socket, + mptcp_subflow_eupcall1(mpts->mpts_socket, mpts, SO_FILT_HINT_DISCONNECTED); - MPTS_UNLOCK(mpts); } } - lck_mtx_unlock(&mpp->mpp_lock); + mpte_unlock(mpte); active++; continue; } if (mpp->mpp_state != MPPCB_STATE_DEAD) { - mptcplog((LOG_DEBUG, "MPTCP Socket: " - "%s: mp_so 0x%llx skipped " - "[u=%d,r=%d,s=%d]\n", __func__, - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), - mp_so->so_usecount, mp_so->so_retaincnt, - mpp->mpp_state), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); - lck_mtx_unlock(&mpp->mpp_lock); - active++; - continue; + panic("MPTCP Socket: %s: mp_so 0x%llx skipped state " + "[u=%d,r=%d,s=%d]\n", __func__, + (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), + mp_so->so_usecount, mp_so->so_retaincnt, + mpp->mpp_state); } - /* - * The PCB has been detached, and there is exactly 1 refnct - * held by the MPTCP thread. Signal that thread to terminate, - * after which the last refcnt will be released. That will - * allow it to be destroyed below during the next round. - */ - if (mp_so->so_usecount == 1) { - mptcplog((LOG_DEBUG, "MPTCP Socket: " - "%s: mp_so 0x%llx scheduled for " - "termination [u=%d,r=%d]\n", __func__, - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), - mp_so->so_usecount, mp_so->so_retaincnt), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + if (mp_tp->mpt_state == MPTCPS_TIME_WAIT) + mptcp_close(mpte, mp_tp); - /* signal MPTCP thread to terminate */ - mptcp_thread_terminate_signal(mpte); - lck_mtx_unlock(&mpp->mpp_lock); - active++; - continue; - } + mptcp_session_destroy(mpte); mptcplog((LOG_DEBUG, "MPTCP Socket: " "%s: mp_so 0x%llx destroyed [u=%d,r=%d]\n", @@ -3582,12 +4005,10 @@ mptcp_drop(struct mptses *mpte, struct mptcb *mp_tp, int errno) { struct socket *mp_so; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPT_LOCK_ASSERT_HELD(mp_tp); + mpte_lock_assert_held(mpte); /* same as MP socket lock */ VERIFY(mpte->mpte_mptcb == mp_tp); - mp_so = mpte->mpte_mppcb->mpp_socket; + mp_so = mptetoso(mpte); - mp_tp->mpt_state = MPTCPS_TERMINATE; DTRACE_MPTCP2(state__change, struct mptcb *, mp_tp, uint32_t, 0 /* event */); @@ -3607,33 +4028,20 @@ mptcp_close(struct mptses *mpte, struct mptcb *mp_tp) struct socket *mp_so = NULL; struct mptsub *mpts = NULL, *tmpts = NULL; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - MPT_LOCK_ASSERT_HELD(mp_tp); + mpte_lock_assert_held(mpte); /* same as MP socket lock */ VERIFY(mpte->mpte_mptcb == mp_tp); - mp_so = mpte->mpte_mppcb->mpp_socket; - if (mp_tp->mpt_localkey != NULL) { - mptcp_free_key(mp_tp->mpt_localkey); - mp_tp->mpt_localkey = NULL; - } + mp_so = mptetoso(mpte); - MPT_UNLOCK(mp_tp); - soisdisconnected(mp_so); + mp_tp->mpt_state = MPTCPS_TERMINATE; - MPT_LOCK(mp_tp); - if (mp_tp->mpt_flags & MPTCPF_PEEL_OFF) { - return (NULL); - } - MPT_UNLOCK(mp_tp); + mptcp_freeq(mp_tp); + + soisdisconnected(mp_so); /* Clean up all subflows */ TAILQ_FOREACH_SAFE(mpts, &mpte->mpte_subflows, mpts_entry, tmpts) { - MPTS_LOCK(mpts); - mpts->mpts_flags |= MPTSF_USER_DISCONNECT; - mptcp_subflow_disconnect(mpte, mpts, TRUE); - MPTS_UNLOCK(mpts); - mptcp_subflow_del(mpte, mpts, TRUE); + mptcp_subflow_disconnect(mpte, mpts); } - MPT_LOCK(mp_tp); return (NULL); } @@ -3645,84 +4053,34 @@ mptcp_notify_close(struct socket *so) } /* - * Signal MPTCP thread to wake up. + * MPTCP workloop. */ void -mptcp_thread_signal(struct mptses *mpte) -{ - lck_mtx_lock(&mpte->mpte_thread_lock); - mptcp_thread_signal_locked(mpte); - lck_mtx_unlock(&mpte->mpte_thread_lock); -} - -/* - * Signal MPTCP thread to wake up (locked version) - */ -static void -mptcp_thread_signal_locked(struct mptses *mpte) -{ - lck_mtx_assert(&mpte->mpte_thread_lock, LCK_MTX_ASSERT_OWNED); - - mpte->mpte_thread_reqs++; - if (!mpte->mpte_thread_active && mpte->mpte_thread != THREAD_NULL) - wakeup_one((caddr_t)&mpte->mpte_thread); -} - -/* - * Signal MPTCP thread to terminate. - */ -static void -mptcp_thread_terminate_signal(struct mptses *mpte) -{ - lck_mtx_lock(&mpte->mpte_thread_lock); - if (mpte->mpte_thread != THREAD_NULL) { - mpte->mpte_thread = THREAD_NULL; - mpte->mpte_thread_reqs++; - if (!mpte->mpte_thread_active) - wakeup_one((caddr_t)&mpte->mpte_thread); - } - lck_mtx_unlock(&mpte->mpte_thread_lock); -} - -/* - * MPTCP thread workloop. - */ -static void -mptcp_thread_dowork(struct mptses *mpte) +mptcp_subflow_workloop(struct mptses *mpte) { struct socket *mp_so; struct mptsub *mpts, *tmpts; boolean_t connect_pending = FALSE, disconnect_fallback = FALSE; - uint64_t mpsofilt_hint_mask = 0; + uint64_t mpsofilt_hint_mask = SO_FILT_HINT_LOCKED; - MPTE_LOCK(mpte); /* same as MP socket lock */ + mpte_lock_assert_held(mpte); VERIFY(mpte->mpte_mppcb != NULL); - mp_so = mpte->mpte_mppcb->mpp_socket; + mp_so = mptetoso(mpte); VERIFY(mp_so != NULL); TAILQ_FOREACH_SAFE(mpts, &mpte->mpte_subflows, mpts_entry, tmpts) { ev_ret_t ret; - MPTS_LOCK(mpts); - MPTS_ADDREF_LOCKED(mpts); /* for us */ - - /* Update process ownership based on parent mptcp socket */ - mptcp_update_last_owner(mpts, mp_so); - - mptcp_subflow_input(mpte, mpts); + if (mpts->mpts_socket->so_usecount == 0) { + /* Will be removed soon by tcp_garbage_collect */ + continue; + } - mptcp_get_rtt_measurement(mpts, mpte); + mptcp_subflow_addref(mpts); + mpts->mpts_socket->so_usecount++; ret = mptcp_subflow_events(mpte, mpts, &mpsofilt_hint_mask); - if (mpts->mpts_flags & MPTSF_ACTIVE) { - mptcplog((LOG_DEBUG, "MPTCP Socket: " - "%s: cid %d \n", __func__, - mpts->mpts_connid), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); - (void) mptcp_subflow_output(mpte, mpts); - } - /* * If MPTCP socket is closed, disconnect all subflows. * This will generate a disconnect event which will @@ -3730,15 +4088,14 @@ mptcp_thread_dowork(struct mptses *mpte) * non-zero error to be returned above. */ if (mp_so->so_flags & SOF_PCBCLEARING) - mptcp_subflow_disconnect(mpte, mpts, FALSE); - MPTS_UNLOCK(mpts); + mptcp_subflow_disconnect(mpte, mpts); switch (ret) { case MPTS_EVRET_OK: /* nothing to do */ break; case MPTS_EVRET_DELETE: - mptcp_subflow_del(mpte, mpts, TRUE); + mptcp_subflow_soclose(mpts); break; case MPTS_EVRET_CONNECT_PENDING: connect_pending = TRUE; @@ -3754,53 +4111,43 @@ mptcp_thread_dowork(struct mptses *mpte) MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); break; } - MPTS_REMREF(mpts); /* ours */ + mptcp_subflow_remref(mpts); /* ours */ + + VERIFY(mpts->mpts_socket->so_usecount != 0); + mpts->mpts_socket->so_usecount--; } - if (mpsofilt_hint_mask) { + if (mpsofilt_hint_mask != SO_FILT_HINT_LOCKED) { + struct mptcb *mp_tp = mpte->mpte_mptcb; + + VERIFY(mpsofilt_hint_mask & SO_FILT_HINT_LOCKED); + if (mpsofilt_hint_mask & SO_FILT_HINT_CANTRCVMORE) { + mptcp_close_fsm(mp_tp, MPCE_RECV_DATA_FIN); socantrcvmore(mp_so); mpsofilt_hint_mask &= ~SO_FILT_HINT_CANTRCVMORE; } - if (mpsofilt_hint_mask & SO_FILT_HINT_CONNRESET) { - struct mptcb *mp_tp = mpte->mpte_mptcb; - - MPT_LOCK(mp_tp); - mptcp_drop(mpte, mp_tp, ECONNRESET); - MPT_UNLOCK(mp_tp); - } - soevent(mp_so, mpsofilt_hint_mask); } - if (!connect_pending && !disconnect_fallback) { - MPTE_UNLOCK(mpte); + if (!connect_pending && !disconnect_fallback) return; - } TAILQ_FOREACH_SAFE(mpts, &mpte->mpte_subflows, mpts_entry, tmpts) { - MPTS_LOCK(mpts); if (disconnect_fallback) { struct socket *so = NULL; struct inpcb *inp = NULL; struct tcpcb *tp = NULL; - if (mpts->mpts_flags & MPTSF_MP_DEGRADED) { - MPTS_UNLOCK(mpts); + if (mpts->mpts_flags & MPTSF_MP_DEGRADED) continue; - } mpts->mpts_flags |= MPTSF_MP_DEGRADED; if (mpts->mpts_flags & (MPTSF_DISCONNECTING| - MPTSF_DISCONNECTED|MPTSF_CONNECT_PENDING)) { - MPTS_UNLOCK(mpts); + MPTSF_DISCONNECTED|MPTSF_CONNECT_PENDING)) continue; - } - - if (mpts->mpts_flags & MPTSF_TFO_REQD) - mptcp_drop_tfo_data(mpte, mpts, NULL); so = mpts->mpts_socket; @@ -3813,7 +4160,6 @@ mptcp_thread_dowork(struct mptses *mpte) * ACTIVE one. */ - socket_lock(so, 1); inp = sotoinpcb(so); tp = intotcpcb(inp); tp->t_mpflags &= @@ -3821,26 +4167,11 @@ mptcp_thread_dowork(struct mptses *mpte) tp->t_mpflags |= TMPF_TCP_FALLBACK; if (mpts->mpts_flags & MPTSF_ACTIVE) { - socket_unlock(so, 1); - MPTS_UNLOCK(mpts); continue; } tp->t_mpflags |= TMPF_RESET; - soevent(so, SO_FILT_HINT_LOCKED | SO_FILT_HINT_MUSTRST); - socket_unlock(so, 1); - + soevent(so, SO_FILT_HINT_MUSTRST); } else if (connect_pending) { - /* - * If delayed subflow start is set and cellular, - * delay the connect till a retransmission timeout - */ - - if ((mptcp_delayed_subf_start) && - (IFNET_IS_CELLULAR(mpts->mpts_outif))) { - MPTS_UNLOCK(mpts); - continue; - } - /* * The MPTCP connection has progressed to a state * where it supports full multipath semantics; allow @@ -3848,102 +4179,22 @@ mptcp_thread_dowork(struct mptses *mpte) * that are in the PENDING state. */ if (mpts->mpts_flags & MPTSF_CONNECT_PENDING) { - (void) mptcp_subflow_soconnectx(mpte, mpts); - } - } - MPTS_UNLOCK(mpts); - } - - MPTE_UNLOCK(mpte); -} - -/* - * MPTCP thread. - */ -static void -mptcp_thread_func(void *v, wait_result_t w) -{ -#pragma unused(w) - struct mptses *mpte = v; - struct timespec *ts = NULL; + int error = mptcp_subflow_soconnectx(mpte, mpts); - VERIFY(mpte != NULL); - - lck_mtx_lock_spin(&mpte->mpte_thread_lock); - - for (;;) { - lck_mtx_assert(&mpte->mpte_thread_lock, LCK_MTX_ASSERT_OWNED); - - if (mpte->mpte_thread != THREAD_NULL) { - (void) msleep(&mpte->mpte_thread, - &mpte->mpte_thread_lock, (PZERO - 1) | PSPIN, - __func__, ts); - } - - /* MPTCP socket is closed? */ - if (mpte->mpte_thread == THREAD_NULL) { - lck_mtx_unlock(&mpte->mpte_thread_lock); - /* callee will destroy thread lock */ - mptcp_thread_destroy(mpte); - /* NOTREACHED */ - return; - } - - mpte->mpte_thread_active = 1; - for (;;) { - uint32_t reqs = mpte->mpte_thread_reqs; - - lck_mtx_unlock(&mpte->mpte_thread_lock); - mptcp_thread_dowork(mpte); - lck_mtx_lock_spin(&mpte->mpte_thread_lock); - - /* if there's no pending request, we're done */ - if (reqs == mpte->mpte_thread_reqs || - mpte->mpte_thread == THREAD_NULL) - break; + if (error) + mptcp_subflow_abort(mpts, error); + } } - mpte->mpte_thread_reqs = 0; - mpte->mpte_thread_active = 0; } } -/* - * Destroy a MTCP thread, to be called in the MPTCP thread context - * upon receiving an indication to self-terminate. This routine - * will not return, as the current thread is terminated at the end. - */ -static void -mptcp_thread_destroy(struct mptses *mpte) -{ - struct socket *mp_so; - - MPTE_LOCK(mpte); /* same as MP socket lock */ - VERIFY(mpte->mpte_thread == THREAD_NULL); - VERIFY(mpte->mpte_mppcb != NULL); - - mptcp_sesdestroy(mpte); - - mp_so = mpte->mpte_mppcb->mpp_socket; - VERIFY(mp_so != NULL); - VERIFY(mp_so->so_usecount > 0); - mp_so->so_usecount--; /* for thread */ - mpte->mpte_mppcb->mpp_flags |= MPP_DEFUNCT; - MPTE_UNLOCK(mpte); - - /* for the extra refcnt from kernel_thread_start() */ - thread_deallocate(current_thread()); - /* this is the end */ - thread_terminate(current_thread()); - /* NOTREACHED */ -} - /* * Protocol pr_lock callback. */ int mptcp_lock(struct socket *mp_so, int refcount, void *lr) { - struct mppcb *mpp = sotomppcb(mp_so); + struct mppcb *mpp = mpsotomppcb(mp_so); void *lr_saved; if (lr == NULL) @@ -3956,7 +4207,7 @@ mptcp_lock(struct socket *mp_so, int refcount, void *lr) mp_so, lr_saved, solockhistory_nr(mp_so)); /* NOTREACHED */ } - lck_mtx_lock(&mpp->mpp_lock); + mpp_lock(mpp); if (mp_so->so_usecount < 0) { panic("%s: so=%p so_pcb=%p lr=%p ref=%x lrh= %s\n", __func__, @@ -3976,207 +4227,58 @@ mptcp_lock(struct socket *mp_so, int refcount, void *lr) * Protocol pr_unlock callback. */ int -mptcp_unlock(struct socket *mp_so, int refcount, void *lr) -{ - struct mppcb *mpp = sotomppcb(mp_so); - void *lr_saved; - - if (lr == NULL) - lr_saved = __builtin_return_address(0); - else - lr_saved = lr; - - if (mpp == NULL) { - panic("%s: so=%p NO PCB usecount=%x lr=%p lrh= %s\n", __func__, - mp_so, mp_so->so_usecount, lr_saved, - solockhistory_nr(mp_so)); - /* NOTREACHED */ - } - lck_mtx_assert(&mpp->mpp_lock, LCK_MTX_ASSERT_OWNED); - - if (refcount != 0) - mp_so->so_usecount--; - - if (mp_so->so_usecount < 0) { - panic("%s: so=%p usecount=%x lrh= %s\n", __func__, - mp_so, mp_so->so_usecount, solockhistory_nr(mp_so)); - /* NOTREACHED */ - } - mp_so->unlock_lr[mp_so->next_unlock_lr] = lr_saved; - mp_so->next_unlock_lr = (mp_so->next_unlock_lr + 1) % SO_LCKDBG_MAX; - lck_mtx_unlock(&mpp->mpp_lock); - - return (0); -} - -/* - * Protocol pr_getlock callback. - */ -lck_mtx_t * -mptcp_getlock(struct socket *mp_so, int locktype) -{ -#pragma unused(locktype) - struct mppcb *mpp = sotomppcb(mp_so); - - if (mpp == NULL) { - panic("%s: so=%p NULL so_pcb %s\n", __func__, mp_so, - solockhistory_nr(mp_so)); - /* NOTREACHED */ - } - if (mp_so->so_usecount < 0) { - panic("%s: so=%p usecount=%x lrh= %s\n", __func__, - mp_so, mp_so->so_usecount, solockhistory_nr(mp_so)); - /* NOTREACHED */ - } - return (&mpp->mpp_lock); -} - -/* - * Key generation functions - */ -static void -mptcp_generate_unique_key(struct mptcp_key_entry *key_entry) -{ - struct mptcp_key_entry *key_elm; -try_again: - read_random(&key_entry->mkey_value, sizeof (key_entry->mkey_value)); - if (key_entry->mkey_value == 0) - goto try_again; - mptcp_do_sha1(&key_entry->mkey_value, key_entry->mkey_digest, - sizeof (key_entry->mkey_digest)); - - LIST_FOREACH(key_elm, &mptcp_keys_pool, mkey_next) { - if (key_elm->mkey_value == key_entry->mkey_value) { - goto try_again; - } - if (bcmp(key_elm->mkey_digest, key_entry->mkey_digest, 4) == - 0) { - goto try_again; - } - } -} - -static mptcp_key_t * -mptcp_reserve_key(void) -{ - struct mptcp_key_entry *key_elm; - struct mptcp_key_entry *found_elm = NULL; - - lck_mtx_lock(&mptcp_keys_pool.mkph_lock); - LIST_FOREACH(key_elm, &mptcp_keys_pool, mkey_next) { - if (key_elm->mkey_flags == MKEYF_FREE) { - key_elm->mkey_flags = MKEYF_INUSE; - found_elm = key_elm; - break; - } - } - lck_mtx_unlock(&mptcp_keys_pool.mkph_lock); - - if (found_elm) { - return (&found_elm->mkey_value); - } - - key_elm = (struct mptcp_key_entry *) - zalloc(mptcp_keys_pool.mkph_key_entry_zone); - key_elm->mkey_flags = MKEYF_INUSE; - - lck_mtx_lock(&mptcp_keys_pool.mkph_lock); - mptcp_generate_unique_key(key_elm); - LIST_INSERT_HEAD(&mptcp_keys_pool, key_elm, mkey_next); - mptcp_keys_pool.mkph_count += 1; - lck_mtx_unlock(&mptcp_keys_pool.mkph_lock); - return (&key_elm->mkey_value); -} - -static caddr_t -mptcp_get_stored_digest(mptcp_key_t *key) +mptcp_unlock(struct socket *mp_so, int refcount, void *lr) { - struct mptcp_key_entry *key_holder; - caddr_t digest = NULL; - - lck_mtx_lock(&mptcp_keys_pool.mkph_lock); - key_holder = (struct mptcp_key_entry *)(void *)((caddr_t)key - - offsetof(struct mptcp_key_entry, mkey_value)); - if (key_holder->mkey_flags != MKEYF_INUSE) - panic_plain("%s", __func__); - digest = &key_holder->mkey_digest[0]; - lck_mtx_unlock(&mptcp_keys_pool.mkph_lock); - return (digest); -} + struct mppcb *mpp = mpsotomppcb(mp_so); + void *lr_saved; -void -mptcp_free_key(mptcp_key_t *key) -{ - struct mptcp_key_entry *key_holder; - struct mptcp_key_entry *key_elm; - int pt = RandomULong(); + if (lr == NULL) + lr_saved = __builtin_return_address(0); + else + lr_saved = lr; - lck_mtx_lock(&mptcp_keys_pool.mkph_lock); - key_holder = (struct mptcp_key_entry *)(void*)((caddr_t)key - - offsetof(struct mptcp_key_entry, mkey_value)); - key_holder->mkey_flags = MKEYF_FREE; + if (mpp == NULL) { + panic("%s: so=%p NO PCB usecount=%x lr=%p lrh= %s\n", __func__, + mp_so, mp_so->so_usecount, lr_saved, + solockhistory_nr(mp_so)); + /* NOTREACHED */ + } + mpp_lock_assert_held(mpp); - LIST_REMOVE(key_holder, mkey_next); - mptcp_keys_pool.mkph_count -= 1; + if (refcount != 0) + mp_so->so_usecount--; - /* Free half the time */ - if (pt & 0x01) { - zfree(mptcp_keys_pool.mkph_key_entry_zone, key_holder); - } else { - /* Insert it at random point to avoid early reuse */ - int i = 0; - if (mptcp_keys_pool.mkph_count > 1) { - pt = pt % (mptcp_keys_pool.mkph_count - 1); - LIST_FOREACH(key_elm, &mptcp_keys_pool, mkey_next) { - if (++i >= pt) { - LIST_INSERT_AFTER(key_elm, key_holder, - mkey_next); - break; - } - } - if (i < pt) - panic("missed insertion"); - } else { - LIST_INSERT_HEAD(&mptcp_keys_pool, key_holder, - mkey_next); - } - mptcp_keys_pool.mkph_count += 1; + if (mp_so->so_usecount < 0) { + panic("%s: so=%p usecount=%x lrh= %s\n", __func__, + mp_so, mp_so->so_usecount, solockhistory_nr(mp_so)); + /* NOTREACHED */ } - lck_mtx_unlock(&mptcp_keys_pool.mkph_lock); + mp_so->unlock_lr[mp_so->next_unlock_lr] = lr_saved; + mp_so->next_unlock_lr = (mp_so->next_unlock_lr + 1) % SO_LCKDBG_MAX; + mpp_unlock(mpp); + + return (0); } -static void -mptcp_key_pool_init(void) +/* + * Protocol pr_getlock callback. + */ +lck_mtx_t * +mptcp_getlock(struct socket *mp_so, int flags) { - int i; - struct mptcp_key_entry *key_entry; - - LIST_INIT(&mptcp_keys_pool); - mptcp_keys_pool.mkph_count = 0; - - mptcp_keys_pool.mkph_key_elm_sz = (vm_size_t) - (sizeof (struct mptcp_key_entry)); - mptcp_keys_pool.mkph_key_entry_zone = zinit( - mptcp_keys_pool.mkph_key_elm_sz, - MPTCP_MX_KEY_ALLOCS * mptcp_keys_pool.mkph_key_elm_sz, - MPTCP_MX_PREALLOC_ZONE_SZ, "mptkeys"); - if (mptcp_keys_pool.mkph_key_entry_zone == NULL) { - panic("%s: unable to allocate MPTCP keys zone \n", __func__); + struct mppcb *mpp = mpsotomppcb(mp_so); + + if (mpp == NULL) { + panic("%s: so=%p NULL so_pcb %s\n", __func__, mp_so, + solockhistory_nr(mp_so)); /* NOTREACHED */ } - zone_change(mptcp_keys_pool.mkph_key_entry_zone, Z_CALLERACCT, FALSE); - zone_change(mptcp_keys_pool.mkph_key_entry_zone, Z_EXPAND, TRUE); - - for (i = 0; i < MPTCP_KEY_PREALLOCS_MX; i++) { - key_entry = (struct mptcp_key_entry *) - zalloc(mptcp_keys_pool.mkph_key_entry_zone); - key_entry->mkey_flags = MKEYF_FREE; - mptcp_generate_unique_key(key_entry); - LIST_INSERT_HEAD(&mptcp_keys_pool, key_entry, mkey_next); - mptcp_keys_pool.mkph_count += 1; + if (mp_so->so_usecount < 0) { + panic("%s: so=%p usecount=%x lrh= %s\n", __func__, + mp_so, mp_so->so_usecount, solockhistory_nr(mp_so)); + /* NOTREACHED */ } - lck_mtx_init(&mptcp_keys_pool.mkph_lock, mtcbinfo.mppi_lock_grp, - mtcbinfo.mppi_lock_attr); + return (mpp_getlock(mpp, flags)); } /* @@ -4189,10 +4291,8 @@ mptcp_attach_to_subf(struct socket *so, struct mptcb *mp_tp, { struct tcpcb *tp = sototcpcb(so); struct mptcp_subf_auth_entry *sauth_entry; - MPT_LOCK_ASSERT_NOTHELD(mp_tp); + mpte_lock_assert_held(mp_tp->mpt_mpte); - MPT_LOCK_SPIN(mp_tp); - tp->t_mptcb = mp_tp; /* * The address ID of the first flow is implicitly 0. */ @@ -4203,7 +4303,6 @@ mptcp_attach_to_subf(struct socket *so, struct mptcb *mp_tp, tp->t_mpflags |= (TMPF_PREESTABLISHED | TMPF_JOINED_FLOW); so->so_flags |= SOF_MP_SEC_SUBFLOW; } - MPT_UNLOCK(mp_tp); sauth_entry = zalloc(mpt_subauth_zone); sauth_entry->msae_laddr_id = tp->t_local_aid; sauth_entry->msae_raddr_id = 0; @@ -4212,9 +4311,7 @@ try_again: sauth_entry->msae_laddr_rand = RandomULong(); if (sauth_entry->msae_laddr_rand == 0) goto try_again; - MPT_LOCK_SPIN(mp_tp); LIST_INSERT_HEAD(&mp_tp->mpt_subauth_list, sauth_entry, msae_next); - MPT_UNLOCK(mp_tp); } static void @@ -4224,14 +4321,10 @@ mptcp_detach_mptcb_from_subf(struct mptcb *mp_tp, struct socket *so) struct tcpcb *tp = NULL; int found = 0; - socket_lock(so, 0); tp = sototcpcb(so); - if (tp == NULL) { - socket_unlock(so, 0); + if (tp == NULL) return; - } - MPT_LOCK(mp_tp); LIST_FOREACH(sauth_entry, &mp_tp->mpt_subauth_list, msae_next) { if (sauth_entry->msae_laddr_id == tp->t_local_aid) { found = 1; @@ -4241,13 +4334,9 @@ mptcp_detach_mptcb_from_subf(struct mptcb *mp_tp, struct socket *so) if (found) { LIST_REMOVE(sauth_entry, msae_next); } - MPT_UNLOCK(mp_tp); if (found) zfree(mpt_subauth_zone, sauth_entry); - - tp->t_mptcb = NULL; - socket_unlock(so, 0); } void @@ -4255,9 +4344,8 @@ mptcp_get_rands(mptcp_addr_id addr_id, struct mptcb *mp_tp, u_int32_t *lrand, u_int32_t *rrand) { struct mptcp_subf_auth_entry *sauth_entry; - MPT_LOCK_ASSERT_NOTHELD(mp_tp); + mpte_lock_assert_held(mp_tp->mpt_mpte); - MPT_LOCK(mp_tp); LIST_FOREACH(sauth_entry, &mp_tp->mpt_subauth_list, msae_next) { if (sauth_entry->msae_laddr_id == addr_id) { if (lrand) @@ -4267,7 +4355,6 @@ mptcp_get_rands(mptcp_addr_id addr_id, struct mptcb *mp_tp, u_int32_t *lrand, break; } } - MPT_UNLOCK(mp_tp); } void @@ -4275,9 +4362,8 @@ mptcp_set_raddr_rand(mptcp_addr_id laddr_id, struct mptcb *mp_tp, mptcp_addr_id raddr_id, u_int32_t raddr_rand) { struct mptcp_subf_auth_entry *sauth_entry; - MPT_LOCK_ASSERT_NOTHELD(mp_tp); + mpte_lock_assert_held(mp_tp->mpt_mpte); - MPT_LOCK(mp_tp); LIST_FOREACH(sauth_entry, &mp_tp->mpt_subauth_list, msae_next) { if (sauth_entry->msae_laddr_id == laddr_id) { if ((sauth_entry->msae_raddr_id != 0) && @@ -4286,7 +4372,6 @@ mptcp_set_raddr_rand(mptcp_addr_id laddr_id, struct mptcb *mp_tp, " address ids %d %d \n", __func__, raddr_id, sauth_entry->msae_raddr_id), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); - MPT_UNLOCK(mp_tp); return; } sauth_entry->msae_raddr_id = raddr_id; @@ -4297,42 +4382,34 @@ mptcp_set_raddr_rand(mptcp_addr_id laddr_id, struct mptcb *mp_tp, __func__, raddr_rand, sauth_entry->msae_raddr_rand), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); - MPT_UNLOCK(mp_tp); return; } sauth_entry->msae_raddr_rand = raddr_rand; - MPT_UNLOCK(mp_tp); return; } } - MPT_UNLOCK(mp_tp); } /* * SHA1 support for MPTCP */ -static int -mptcp_do_sha1(mptcp_key_t *key, char *sha_digest, int digest_len) +static void +mptcp_do_sha1(mptcp_key_t *key, char *sha_digest) { SHA1_CTX sha1ctxt; const unsigned char *sha1_base; int sha1_size; - if (digest_len != SHA1_RESULTLEN) { - return (FALSE); - } - sha1_base = (const unsigned char *) key; sha1_size = sizeof (mptcp_key_t); SHA1Init(&sha1ctxt); SHA1Update(&sha1ctxt, sha1_base, sha1_size); SHA1Final(sha_digest, &sha1ctxt); - return (TRUE); } void mptcp_hmac_sha1(mptcp_key_t key1, mptcp_key_t key2, - u_int32_t rand1, u_int32_t rand2, u_char *digest, int digest_len) + u_int32_t rand1, u_int32_t rand2, u_char *digest) { SHA1_CTX sha1ctxt; mptcp_key_t key_ipad[8] = {0}; /* key XOR'd with inner pad */ @@ -4340,7 +4417,7 @@ mptcp_hmac_sha1(mptcp_key_t key1, mptcp_key_t key2, u_int32_t data[2]; int i; - bzero(digest, digest_len); + bzero(digest, SHA1_RESULTLEN); /* Set up the Key for HMAC */ key_ipad[0] = key1; @@ -4380,41 +4457,22 @@ mptcp_hmac_sha1(mptcp_key_t key1, mptcp_key_t key2, * corresponds to MAC-A = MAC (Key=(Key-A+Key-B), Msg=(R-A+R-B)) */ void -mptcp_get_hmac(mptcp_addr_id aid, struct mptcb *mp_tp, u_char *digest, - int digest_len) +mptcp_get_hmac(mptcp_addr_id aid, struct mptcb *mp_tp, u_char *digest) { uint32_t lrand, rrand; - mptcp_key_t localkey, remotekey; - MPT_LOCK_ASSERT_NOTHELD(mp_tp); - if (digest_len != SHA1_RESULTLEN) - return; + mpte_lock_assert_held(mp_tp->mpt_mpte); lrand = rrand = 0; mptcp_get_rands(aid, mp_tp, &lrand, &rrand); - MPT_LOCK_SPIN(mp_tp); - localkey = *mp_tp->mpt_localkey; - remotekey = mp_tp->mpt_remotekey; - MPT_UNLOCK(mp_tp); - mptcp_hmac_sha1(localkey, remotekey, lrand, rrand, digest, - digest_len); -} - -u_int64_t -mptcp_get_trunced_hmac(mptcp_addr_id aid, struct mptcb *mp_tp) -{ - u_char digest[SHA1_RESULTLEN]; - u_int64_t trunced_digest; - - mptcp_get_hmac(aid, mp_tp, &digest[0], sizeof (digest)); - bcopy(digest, &trunced_digest, 8); - return (trunced_digest); + mptcp_hmac_sha1(mp_tp->mpt_localkey, mp_tp->mpt_remotekey, lrand, rrand, + digest); } /* * Authentication data generation */ -void +static void mptcp_generate_token(char *sha_digest, int sha_digest_len, caddr_t token, int token_len) { @@ -4426,7 +4484,7 @@ mptcp_generate_token(char *sha_digest, int sha_digest_len, caddr_t token, return; } -void +static void mptcp_generate_idsn(char *sha_digest, int sha_digest_len, caddr_t idsn, int idsn_len) { @@ -4466,15 +4524,17 @@ mptcp_conn_properties(struct mptcb *mp_tp) } static void -mptcp_init_local_parms(struct mptcb *mp_tp) +mptcp_init_local_parms(struct mptses *mpte) { - caddr_t local_digest = NULL; + struct mptcb *mp_tp = mpte->mpte_mptcb; + char key_digest[SHA1_RESULTLEN]; - mp_tp->mpt_localkey = mptcp_reserve_key(); - local_digest = mptcp_get_stored_digest(mp_tp->mpt_localkey); - mptcp_generate_token(local_digest, SHA1_RESULTLEN, + read_frandom(&mp_tp->mpt_localkey, sizeof(mp_tp->mpt_localkey)); + mptcp_do_sha1(&mp_tp->mpt_localkey, key_digest); + + mptcp_generate_token(key_digest, SHA1_RESULTLEN, (caddr_t)&mp_tp->mpt_localtoken, sizeof (mp_tp->mpt_localtoken)); - mptcp_generate_idsn(local_digest, SHA1_RESULTLEN, + mptcp_generate_idsn(key_digest, SHA1_RESULTLEN, (caddr_t)&mp_tp->mpt_local_idsn, sizeof (u_int64_t)); /* The subflow SYN is also first MPTCP byte */ @@ -4487,65 +4547,25 @@ mptcp_init_local_parms(struct mptcb *mp_tp) int mptcp_init_remote_parms(struct mptcb *mp_tp) { - char remote_digest[MPTCP_SHA1_RESULTLEN]; - MPT_LOCK_ASSERT_HELD(mp_tp); + char remote_digest[SHA1_RESULTLEN]; + mpte_lock_assert_held(mp_tp->mpt_mpte); /* Only Version 0 is supported for auth purposes */ if (mp_tp->mpt_version != MPTCP_STD_VERSION_0) return (-1); /* Setup local and remote tokens and Initial DSNs */ - - if (!mptcp_do_sha1(&mp_tp->mpt_remotekey, remote_digest, - SHA1_RESULTLEN)) { - mptcplog((LOG_ERR, "MPTCP Socket: %s: unexpected failure", - __func__), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); - return (-1); - } + mptcp_do_sha1(&mp_tp->mpt_remotekey, remote_digest); mptcp_generate_token(remote_digest, SHA1_RESULTLEN, (caddr_t)&mp_tp->mpt_remotetoken, sizeof (mp_tp->mpt_remotetoken)); mptcp_generate_idsn(remote_digest, SHA1_RESULTLEN, (caddr_t)&mp_tp->mpt_remote_idsn, sizeof (u_int64_t)); - mp_tp->mpt_rcvatmark = mp_tp->mpt_rcvnxt = mp_tp->mpt_remote_idsn + 1; + mp_tp->mpt_rcvnxt = mp_tp->mpt_remote_idsn + 1; return (0); } -/* - * Helper Functions - */ -mptcp_token_t -mptcp_get_localtoken(void* mptcb_arg) -{ - struct mptcb *mp_tp = (struct mptcb *)mptcb_arg; - return (mp_tp->mpt_localtoken); -} - -mptcp_token_t -mptcp_get_remotetoken(void* mptcb_arg) -{ - struct mptcb *mp_tp = (struct mptcb *)mptcb_arg; - return (mp_tp->mpt_remotetoken); -} - -u_int64_t -mptcp_get_localkey(void* mptcb_arg) -{ - struct mptcb *mp_tp = (struct mptcb *)mptcb_arg; - if (mp_tp->mpt_localkey != NULL) - return (*mp_tp->mpt_localkey); - else - return (0); -} - -u_int64_t -mptcp_get_remotekey(void* mptcb_arg) -{ - struct mptcb *mp_tp = (struct mptcb *)mptcb_arg; - return (mp_tp->mpt_remotekey); -} - -void +static void mptcp_send_dfin(struct socket *so) { struct tcpcb *tp = NULL; @@ -4575,7 +4595,8 @@ mptcp_insert_dsn(struct mppcb *mpp, struct mbuf *m) return; __IGNORE_WCASTALIGN(mp_tp = &((struct mpp_mtp *)mpp)->mtcb); - MPT_LOCK(mp_tp); + mpte_lock_assert_held(mp_tp->mpt_mpte); + while (m) { VERIFY(m->m_flags & M_PKTHDR); m->m_pkthdr.pkt_flags |= (PKTF_MPTCP | PKTF_MPSO); @@ -4584,56 +4605,97 @@ mptcp_insert_dsn(struct mppcb *mpp, struct mbuf *m) mp_tp->mpt_sndmax += m_pktlen(m); m = m->m_next; } - MPT_UNLOCK(mp_tp); +} + +void +mptcp_fallback_sbdrop(struct socket *so, struct mbuf *m, int len) +{ + struct mptcb *mp_tp = tptomptp(sototcpcb(so)); + uint64_t data_ack; + uint64_t dsn; + + if (!m || len == 0) + return; + + while (m && len > 0) { + VERIFY(m->m_flags & M_PKTHDR); + VERIFY(m->m_pkthdr.pkt_flags & PKTF_MPTCP); + + data_ack = m->m_pkthdr.mp_dsn + m->m_pkthdr.mp_rlen; + dsn = m->m_pkthdr.mp_dsn; + + len -= m->m_len; + m = m->m_next; + } + + if (m && len == 0) { + /* + * If there is one more mbuf in the chain, it automatically means + * that up to m->mp_dsn has been ack'ed. + * + * This means, we actually correct data_ack back down (compared + * to what we set inside the loop - dsn + data_len). Because in + * the loop we are "optimistic" and assume that the full mapping + * will be acked. If that's not the case and we get out of the + * loop with m != NULL, it means only up to m->mp_dsn has been + * really acked. + */ + data_ack = m->m_pkthdr.mp_dsn; + } + + if (len < 0) { + /* + * If len is negative, meaning we acked in the middle of an mbuf, + * only up to this mbuf's data-sequence number has been acked + * at the MPTCP-level. + */ + data_ack = dsn; + } + + mptcplog((LOG_DEBUG, "%s inferred ack up to %u\n", __func__, (uint32_t)data_ack), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + mptcp_data_ack_rcvd(mp_tp, sototcpcb(so), data_ack); } void mptcp_preproc_sbdrop(struct socket *so, struct mbuf *m, unsigned int len) { - u_int32_t sub_len = 0; int rewinding = 0; - if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) { - /* TFO makes things complicated. */ - if (so->so_flags1 & SOF1_TFO_REWIND) { - rewinding = 1; - so->so_flags1 &= ~SOF1_TFO_REWIND; - } + /* TFO makes things complicated. */ + if (so->so_flags1 & SOF1_TFO_REWIND) { + rewinding = 1; + so->so_flags1 &= ~SOF1_TFO_REWIND; } - while (m) { + while (m && (!(so->so_flags & SOF_MP_SUBFLOW) || rewinding)) { + u_int32_t sub_len; VERIFY(m->m_flags & M_PKTHDR); + VERIFY(m->m_pkthdr.pkt_flags & PKTF_MPTCP); - if (m->m_pkthdr.pkt_flags & PKTF_MPTCP) { - sub_len = m->m_pkthdr.mp_rlen; + sub_len = m->m_pkthdr.mp_rlen; - if (sub_len < len) { - m->m_pkthdr.mp_dsn += sub_len; - if (!(m->m_pkthdr.pkt_flags & PKTF_MPSO)) { - m->m_pkthdr.mp_rseq += sub_len; - } - m->m_pkthdr.mp_rlen = 0; - len -= sub_len; - } else { - /* sub_len >= len */ - if (rewinding == 0) - m->m_pkthdr.mp_dsn += len; - if (!(m->m_pkthdr.pkt_flags & PKTF_MPSO)) { - if (rewinding == 0) - m->m_pkthdr.mp_rseq += len; - } - mptcplog((LOG_DEBUG, "MPTCP Sender: " - "%s: dsn 0x%llx ssn %u len %d %d\n", - __func__, - m->m_pkthdr.mp_dsn, m->m_pkthdr.mp_rseq, - m->m_pkthdr.mp_rlen, len), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); - m->m_pkthdr.mp_rlen -= len; - break; + if (sub_len < len) { + m->m_pkthdr.mp_dsn += sub_len; + if (!(m->m_pkthdr.pkt_flags & PKTF_MPSO)) { + m->m_pkthdr.mp_rseq += sub_len; } + m->m_pkthdr.mp_rlen = 0; + len -= sub_len; } else { - panic("%s: MPTCP tag not set", __func__); - /* NOTREACHED */ + /* sub_len >= len */ + if (rewinding == 0) + m->m_pkthdr.mp_dsn += len; + if (!(m->m_pkthdr.pkt_flags & PKTF_MPSO)) { + if (rewinding == 0) + m->m_pkthdr.mp_rseq += len; + } + mptcplog((LOG_DEBUG, "%s: dsn %u ssn %u len %d %d\n", + __func__, (u_int32_t)m->m_pkthdr.mp_dsn, + m->m_pkthdr.mp_rseq, m->m_pkthdr.mp_rlen, len), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); + m->m_pkthdr.mp_rlen -= len; + break; } m = m->m_next; } @@ -4645,37 +4707,32 @@ mptcp_preproc_sbdrop(struct socket *so, struct mbuf *m, unsigned int len) * Received an ack without receiving a DATA_ACK. * Need to fallback to regular TCP (or destroy this subflow). */ + sototcpcb(so)->t_mpflags |= TMPF_INFIN_SENT; mptcp_notify_mpfail(so); } } /* Obtain the DSN mapping stored in the mbuf */ void -mptcp_output_getm_dsnmap32(struct socket *so, int off, uint32_t datalen, - u_int32_t *dsn, u_int32_t *relseq, u_int16_t *data_len, u_int64_t *dsn64p) +mptcp_output_getm_dsnmap32(struct socket *so, int off, + uint32_t *dsn, uint32_t *relseq, uint16_t *data_len, uint16_t *dss_csum) { u_int64_t dsn64; - mptcp_output_getm_dsnmap64(so, off, datalen, &dsn64, relseq, data_len); + mptcp_output_getm_dsnmap64(so, off, &dsn64, relseq, data_len, dss_csum); *dsn = (u_int32_t)MPTCP_DATASEQ_LOW32(dsn64); - *dsn64p = dsn64; } void -mptcp_output_getm_dsnmap64(struct socket *so, int off, uint32_t datalen, - u_int64_t *dsn, u_int32_t *relseq, u_int16_t *data_len) +mptcp_output_getm_dsnmap64(struct socket *so, int off, uint64_t *dsn, + uint32_t *relseq, uint16_t *data_len, + uint16_t *dss_csum) { struct mbuf *m = so->so_snd.sb_mb; - struct mbuf *mnext = NULL; - uint32_t runlen = 0; - u_int64_t dsn64; - uint32_t contig_len = 0; + int off_orig = off; - if (m == NULL) - return; + VERIFY(off >= 0); - if (off < 0) - return; /* * In the subflow socket, the DSN sequencing can be discontiguous, * but the subflow sequence mapping is contiguous. Use the subflow @@ -4684,97 +4741,29 @@ mptcp_output_getm_dsnmap64(struct socket *so, int off, uint32_t datalen, */ while (m) { - VERIFY(m->m_pkthdr.pkt_flags & PKTF_MPTCP); VERIFY(m->m_flags & M_PKTHDR); + VERIFY(m->m_pkthdr.pkt_flags & PKTF_MPTCP); - if ((unsigned int)off >= m->m_pkthdr.mp_rlen) { - off -= m->m_pkthdr.mp_rlen; + if (off >= m->m_len) { + off -= m->m_len; m = m->m_next; } else { break; } } - if (m == NULL) { - panic("%s: bad offset", __func__); - /* NOTREACHED */ - } - - dsn64 = m->m_pkthdr.mp_dsn + off; - *dsn = dsn64; - *relseq = m->m_pkthdr.mp_rseq + off; - - /* - * Now find the last contiguous byte and its length from - * start. - */ - runlen = m->m_pkthdr.mp_rlen - off; - contig_len = runlen; - - /* If datalen does not span multiple mbufs, return */ - if (datalen <= runlen) { - *data_len = min(datalen, UINT16_MAX); - return; - } - - mnext = m->m_next; - while (datalen > runlen) { - if (mnext == NULL) { - panic("%s: bad datalen = %d, %d %d", __func__, datalen, - runlen, off); - /* NOTREACHED */ - } - VERIFY(mnext->m_flags & M_PKTHDR); - VERIFY(mnext->m_pkthdr.pkt_flags & PKTF_MPTCP); - - /* - * case A. contiguous DSN stream - * case B. discontiguous DSN stream - */ - if (mnext->m_pkthdr.mp_dsn == (dsn64 + runlen)) { - /* case A */ - runlen += mnext->m_pkthdr.mp_rlen; - contig_len += mnext->m_pkthdr.mp_rlen; - mptcplog((LOG_DEBUG, "MPTCP Sender: %s: contig \n", - __func__), MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); - } else { - /* case B */ - mptcplog((LOG_DEBUG, "MPTCP Sender: " - "%s: discontig datalen %d contig_len %d cc %d \n", - __func__, datalen, contig_len, so->so_snd.sb_cc), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); - break; - } - mnext = mnext->m_next; - } - datalen = min(datalen, UINT16_MAX); - *data_len = min(datalen, contig_len); - mptcplog((LOG_DEBUG, "MPTCP Sender: " - "%s: %llu %u %d %d \n", __func__, - *dsn, *relseq, *data_len, off), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); -} + VERIFY(m); + VERIFY(off >= 0); + VERIFY(m->m_pkthdr.mp_rlen <= UINT16_MAX); -/* - * MPTCP's notion of the next insequence Data Sequence number is adjusted - * here. It must be called from mptcp_adj_rmap() which is called only after - * reassembly of out of order data. The rcvnxt variable must - * be updated only when atleast some insequence new data is received. - */ -static void -mptcp_adj_rcvnxt(struct tcpcb *tp, struct mbuf *m) -{ - struct mptcb *mp_tp = tptomptp(tp); + *dsn = m->m_pkthdr.mp_dsn; + *relseq = m->m_pkthdr.mp_rseq; + *data_len = m->m_pkthdr.mp_rlen; + *dss_csum = m->m_pkthdr.mp_csum; - if (mp_tp == NULL) - return; - MPT_LOCK(mp_tp); - if ((MPTCP_SEQ_GEQ(mp_tp->mpt_rcvnxt, m->m_pkthdr.mp_dsn)) && - (MPTCP_SEQ_LEQ(mp_tp->mpt_rcvnxt, (m->m_pkthdr.mp_dsn + - m->m_pkthdr.mp_rlen)))) { - mp_tp->mpt_rcvnxt = m->m_pkthdr.mp_dsn + m->m_pkthdr.mp_rlen; - } - MPT_UNLOCK(mp_tp); + mptcplog((LOG_DEBUG, "%s: dsn %u ssn %u data_len %d off %d off_orig %d\n", + __func__, (u_int32_t)(*dsn), *relseq, *data_len, off, off_orig), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); } /* @@ -4798,61 +4787,37 @@ mptcp_insert_rmap(struct tcpcb *tp, struct mbuf *m) m->m_pkthdr.mp_dsn = tp->t_rcv_map.mpt_dsn; m->m_pkthdr.mp_rseq = tp->t_rcv_map.mpt_sseq; m->m_pkthdr.mp_rlen = tp->t_rcv_map.mpt_len; + m->m_pkthdr.mp_csum = tp->t_rcv_map.mpt_csum; m->m_pkthdr.pkt_flags |= PKTF_MPTCP; tp->t_mpflags &= ~TMPF_EMBED_DSN; tp->t_mpflags |= TMPF_MPTCP_ACKNOW; } } -int -mptcp_adj_rmap(struct socket *so, struct mbuf *m) +void +mptcp_adj_rmap(struct socket *so, struct mbuf *m, int off) { - u_int64_t dsn; - u_int32_t sseq, datalen; - struct tcpcb *tp = intotcpcb(sotoinpcb(so)); - u_int32_t old_rcvnxt = 0; + struct mptsub *mpts = sototcpcb(so)->t_mpsub; if (m_pktlen(m) == 0) - return 0; - - if (m->m_pkthdr.pkt_flags & PKTF_MPTCP) { - VERIFY(m->m_flags & M_PKTHDR); - - dsn = m->m_pkthdr.mp_dsn; - sseq = m->m_pkthdr.mp_rseq + tp->irs; - datalen = m->m_pkthdr.mp_rlen; - } else { - /* data arrived without an DSS option mapping */ - - /* initial subflow can fallback right after SYN handshake */ - mptcp_notify_mpfail(so); - return 0; - } - - /* In the common case, data is in window and in sequence */ - if (m->m_pkthdr.len == (int)datalen) { - mptcp_adj_rcvnxt(tp, m); - return 0; - } + return; - old_rcvnxt = tp->rcv_nxt - m->m_pkthdr.len; - if (SEQ_GT(old_rcvnxt, sseq)) { - /* data trimmed from the left */ - int off = old_rcvnxt - sseq; + if ((m->m_flags & M_PKTHDR) && (m->m_pkthdr.pkt_flags & PKTF_MPTCP)) { m->m_pkthdr.mp_dsn += off; m->m_pkthdr.mp_rseq += off; m->m_pkthdr.mp_rlen = m->m_pkthdr.len; - } else if (old_rcvnxt == sseq) { - /* - * data was trimmed from the right - */ - m->m_pkthdr.mp_rlen = m->m_pkthdr.len; } else { - mptcp_notify_mpfail(so); - return (-1); + if (!(mpts->mpts_flags & MPTSF_CONFIRMED)) { + /* data arrived without an DSS option mapping */ + + /* initial subflow can fallback right after SYN handshake */ + mptcp_notify_mpfail(so); + } } - mptcp_adj_rcvnxt(tp, m); - return 0; + + mpts->mpts_flags |= MPTSF_CONFIRMED; + + return; } /* @@ -4872,9 +4837,8 @@ mptcp_act_on_txfail(struct socket *so) if (tp == NULL) return; - if (so->so_flags & SOF_MP_TRYFAILOVER) { + if (so->so_flags & SOF_MP_TRYFAILOVER) return; - } so->so_flags |= SOF_MP_TRYFAILOVER; soevent(so, (SO_FILT_HINT_LOCKED | SO_FILT_HINT_MPFAILOVER)); @@ -4903,9 +4867,8 @@ mptcp_get_map_for_dsn(struct socket *so, u_int64_t dsn_fail, u_int32_t *tcp_seq) (MPTCP_SEQ_GEQ(dsn + datalen, dsn_fail))) { off = dsn_fail - dsn; *tcp_seq = m->m_pkthdr.mp_rseq + off; - mptcplog((LOG_DEBUG, "MPTCP Sender: %s: %llu %llu \n", - __func__, dsn, dsn_fail), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); + mptcplog((LOG_DEBUG, "%s: %llu %llu \n", __func__, dsn, + dsn_fail), MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); return (0); } @@ -4917,72 +4880,234 @@ mptcp_get_map_for_dsn(struct socket *so, u_int64_t dsn_fail, u_int32_t *tcp_seq) * not much else to do. */ - mptcplog((LOG_ERR, "MPTCP Sender: " - "%s: %llu not found \n", __func__, dsn_fail), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); - return (-1); + mptcplog((LOG_ERR, "MPTCP Sender: " + "%s: %llu not found \n", __func__, dsn_fail), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); + return (-1); +} + +/* + * Support for sending contiguous MPTCP bytes in subflow + * Also for preventing sending data with ACK in 3-way handshake + */ +int32_t +mptcp_adj_sendlen(struct socket *so, int32_t off) +{ + struct tcpcb *tp = sototcpcb(so); + struct mptsub *mpts = tp->t_mpsub; + uint64_t mdss_dsn; + uint32_t mdss_subflow_seq; + int mdss_subflow_off; + uint16_t mdss_data_len; + uint16_t dss_csum; + + mptcp_output_getm_dsnmap64(so, off, &mdss_dsn, &mdss_subflow_seq, + &mdss_data_len, &dss_csum); + + /* + * We need to compute how much of the mapping still remains. + * So, we compute the offset in the send-buffer of the dss-sub-seq. + */ + mdss_subflow_off = (mdss_subflow_seq + mpts->mpts_iss) - tp->snd_una; + + /* + * When TFO is used, we are sending the mpts->mpts_iss although the relative + * seq has been set to 1 (while it should be 0). + */ + if (tp->t_mpflags & TMPF_TFO_REQUEST) + mdss_subflow_off--; + + if (off < mdss_subflow_off) + printf("%s off %d mdss_subflow_off %d mdss_subflow_seq %u iss %u suna %u\n", __func__, + off, mdss_subflow_off, mdss_subflow_seq, mpts->mpts_iss, tp->snd_una); + VERIFY(off >= mdss_subflow_off); + + mptcplog((LOG_DEBUG, "%s dlen %u off %d sub_off %d sub_seq %u iss %u suna %u\n", + __func__, mdss_data_len, off, mdss_subflow_off, mdss_subflow_seq, + mpts->mpts_iss, tp->snd_una), MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); + return (mdss_data_len - (off - mdss_subflow_off)); +} + +static uint32_t +mptcp_get_maxseg(struct mptses *mpte) +{ + struct mptsub *mpts; + uint32_t maxseg = 0; + + TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { + struct tcpcb *tp = sototcpcb(mpts->mpts_socket); + + if (!TCPS_HAVEESTABLISHED(tp->t_state) || + TCPS_HAVERCVDFIN2(tp->t_state)) + continue; + + if (tp->t_maxseg > maxseg) + maxseg = tp->t_maxseg; + } + + return (maxseg); +} + +static uint8_t +mptcp_get_rcvscale(struct mptses *mpte) +{ + struct mptsub *mpts; + uint8_t rcvscale = UINT8_MAX; + + TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { + struct tcpcb *tp = sototcpcb(mpts->mpts_socket); + + if (!TCPS_HAVEESTABLISHED(tp->t_state) || + TCPS_HAVERCVDFIN2(tp->t_state)) + continue; + + if (tp->rcv_scale < rcvscale) + rcvscale = tp->rcv_scale; + } + + return (rcvscale); +} + +/* Similar to tcp_sbrcv_reserve */ +static void +mptcp_sbrcv_reserve(struct mptcb *mp_tp, struct sockbuf *sbrcv, + u_int32_t newsize, u_int32_t idealsize) +{ + uint8_t rcvscale = mptcp_get_rcvscale(mp_tp->mpt_mpte); + + /* newsize should not exceed max */ + newsize = min(newsize, tcp_autorcvbuf_max); + + /* The receive window scale negotiated at the + * beginning of the connection will also set a + * limit on the socket buffer size + */ + newsize = min(newsize, TCP_MAXWIN << rcvscale); + + /* Set new socket buffer size */ + if (newsize > sbrcv->sb_hiwat && + (sbreserve(sbrcv, newsize) == 1)) { + sbrcv->sb_idealsize = min(max(sbrcv->sb_idealsize, + (idealsize != 0) ? idealsize : newsize), tcp_autorcvbuf_max); + + /* Again check the limit set by the advertised + * window scale + */ + sbrcv->sb_idealsize = min(sbrcv->sb_idealsize, + TCP_MAXWIN << rcvscale); + } +} + +void +mptcp_sbrcv_grow(struct mptcb *mp_tp) +{ + struct mptses *mpte = mp_tp->mpt_mpte; + struct socket *mp_so = mpte->mpte_mppcb->mpp_socket; + struct sockbuf *sbrcv = &mp_so->so_rcv; + uint32_t hiwat_sum = 0; + uint32_t ideal_sum = 0; + struct mptsub *mpts; + + /* + * Do not grow the receive socket buffer if + * - auto resizing is disabled, globally or on this socket + * - the high water mark already reached the maximum + * - the stream is in background and receive side is being + * throttled + * - if there are segments in reassembly queue indicating loss, + * do not need to increase recv window during recovery as more + * data is not going to be sent. A duplicate ack sent during + * recovery should not change the receive window + */ + if (tcp_do_autorcvbuf == 0 || + (sbrcv->sb_flags & SB_AUTOSIZE) == 0 || + tcp_cansbgrow(sbrcv) == 0 || + sbrcv->sb_hiwat >= tcp_autorcvbuf_max || + (mp_so->so_flags1 & SOF1_EXTEND_BK_IDLE_WANTED) || + !LIST_EMPTY(&mp_tp->mpt_segq)) { + /* Can not resize the socket buffer, just return */ + return; + } + + /* + * Ideally, we want the rbuf to be (sum_i {bw_i} * rtt_max * 2) + * + * But, for this we first need accurate receiver-RTT estimations, which + * we currently don't have. + * + * Let's use a dummy algorithm for now, just taking the sum of all + * subflow's receive-buffers. It's too low, but that's all we can get + * for now. + */ + + TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { + hiwat_sum += mpts->mpts_socket->so_rcv.sb_hiwat; + ideal_sum += mpts->mpts_socket->so_rcv.sb_idealsize; + } + + mptcp_sbrcv_reserve(mp_tp, sbrcv, hiwat_sum, ideal_sum); } /* - * Support for sending contiguous MPTCP bytes in subflow - * Also for preventing sending data with ACK in 3-way handshake + * Determine if we can grow the recieve socket buffer to avoid sending + * a zero window update to the peer. We allow even socket buffers that + * have fixed size (set by the application) to grow if the resource + * constraints are met. They will also be trimmed after the application + * reads data. + * + * Similar to tcp_sbrcv_grow_rwin */ -int32_t -mptcp_adj_sendlen(struct socket *so, int32_t off, int32_t len) +static void +mptcp_sbrcv_grow_rwin(struct mptcb *mp_tp, struct sockbuf *sb) { - u_int64_t mdss_dsn = 0; - u_int32_t mdss_subflow_seq = 0; - u_int16_t mdss_data_len = 0; + struct socket *mp_so = mp_tp->mpt_mpte->mpte_mppcb->mpp_socket; + u_int32_t rcvbufinc = mptcp_get_maxseg(mp_tp->mpt_mpte) << 4; + u_int32_t rcvbuf = sb->sb_hiwat; - if (len == 0) - return (len); - - mptcp_output_getm_dsnmap64(so, off, (u_int32_t)len, - &mdss_dsn, &mdss_subflow_seq, &mdss_data_len); + if (tcp_recv_bg == 1 || IS_TCP_RECV_BG(mp_so)) + return; - /* - * Special case handling for Fast Join. We want to send data right - * after ACK of the 3-way handshake, but not piggyback the data - * with the 3rd ACK of the 3WHS. TMPF_FASTJOINBY2_SEND and - * mdss_data_len control this. - */ - struct tcpcb *tp = NULL; - tp = intotcpcb(sotoinpcb(so)); - if ((tp->t_mpflags & TMPF_JOINED_FLOW) && - (tp->t_mpflags & TMPF_PREESTABLISHED) && - (!(tp->t_mpflags & TMPF_RECVD_JOIN)) && - (tp->t_mpflags & TMPF_SENT_JOIN) && - (!(tp->t_mpflags & TMPF_MPTCP_TRUE)) && - (!(tp->t_mpflags & TMPF_FASTJOINBY2_SEND))) { - mdss_data_len = 0; - tp->t_mpflags |= TMPF_FASTJOINBY2_SEND; - } - - if ((tp->t_state > TCPS_SYN_SENT) && - (tp->t_mpflags & TMPF_TFO_REQUEST)) { - mdss_data_len = 0; - tp->t_mpflags &= ~TMPF_TFO_REQUEST; + if (tcp_do_autorcvbuf == 1 && + tcp_cansbgrow(sb) && + /* Diff to tcp_sbrcv_grow_rwin */ + (mp_so->so_flags1 & SOF1_EXTEND_BK_IDLE_WANTED) == 0 && + (rcvbuf - sb->sb_cc) < rcvbufinc && + rcvbuf < tcp_autorcvbuf_max && + (sb->sb_idealsize > 0 && + sb->sb_hiwat <= (sb->sb_idealsize + rcvbufinc))) { + sbreserve(sb, min((sb->sb_hiwat + rcvbufinc), tcp_autorcvbuf_max)); } - return (mdss_data_len); } +/* Similar to tcp_sbspace */ int32_t -mptcp_sbspace(struct mptcb *mpt) +mptcp_sbspace(struct mptcb *mp_tp) { - struct sockbuf *sb; + struct sockbuf *sb = &mp_tp->mpt_mpte->mpte_mppcb->mpp_socket->so_rcv; uint32_t rcvbuf; int32_t space; + int32_t pending = 0; + + mpte_lock_assert_held(mp_tp->mpt_mpte); - MPT_LOCK_ASSERT_HELD(mpt); - MPTE_LOCK_ASSERT_HELD(mpt->mpt_mpte); + mptcp_sbrcv_grow_rwin(mp_tp, sb); - sb = &mpt->mpt_mpte->mpte_mppcb->mpp_socket->so_rcv; + /* hiwat might have changed */ rcvbuf = sb->sb_hiwat; - space = ((int32_t)imin((rcvbuf - sb->sb_cc), - (sb->sb_mbmax - sb->sb_mbcnt))); + + space = ((int32_t) imin((rcvbuf - sb->sb_cc), + (sb->sb_mbmax - sb->sb_mbcnt))); if (space < 0) space = 0; - /* XXX check if it's too small? */ + +#if CONTENT_FILTER + /* Compensate for data being processed by content filters */ + pending = cfil_sock_data_space(sb); +#endif /* CONTENT_FILTER */ + if (pending > space) + space = 0; + else + space -= pending; return (space); } @@ -5052,12 +5177,11 @@ boolean_t mptcp_ok_to_keepalive(struct mptcb *mp_tp) { boolean_t ret = 1; - VERIFY(mp_tp != NULL); - MPT_LOCK(mp_tp); + mpte_lock_assert_held(mp_tp->mpt_mpte); + if (mp_tp->mpt_state >= MPTCPS_CLOSE_WAIT) { ret = 0; } - MPT_UNLOCK(mp_tp); return (ret); } @@ -5072,34 +5196,36 @@ mptcp_adj_mss(struct tcpcb *tp, boolean_t mtudisc) #define MPTCP_COMPUTE_LEN { \ mss_lower = sizeof (struct mptcp_dss_ack_opt); \ - MPT_LOCK(mp_tp); \ if (mp_tp->mpt_flags & MPTCPF_CHECKSUM) \ mss_lower += 2; \ else \ /* adjust to 32-bit boundary + EOL */ \ mss_lower += 2; \ - MPT_UNLOCK(mp_tp); \ } if (mp_tp == NULL) return (0); + mpte_lock_assert_held(mp_tp->mpt_mpte); + /* * For the first subflow and subsequent subflows, adjust mss for * most common MPTCP option size, for case where tcp_mss is called * during option processing and MTU discovery. */ - if ((tp->t_mpflags & TMPF_PREESTABLISHED) && - (!(tp->t_mpflags & TMPF_JOINED_FLOW))) { - MPTCP_COMPUTE_LEN; - } - - if ((tp->t_mpflags & TMPF_PREESTABLISHED) && - (tp->t_mpflags & TMPF_SENT_JOIN)) { - MPTCP_COMPUTE_LEN; - } + if (!mtudisc) { + if (tp->t_mpflags & TMPF_MPTCP_TRUE && + !(tp->t_mpflags & TMPF_JOINED_FLOW)) { + MPTCP_COMPUTE_LEN; + } - if ((mtudisc) && (tp->t_mpflags & TMPF_MPTCP_TRUE)) { - MPTCP_COMPUTE_LEN; + if (tp->t_mpflags & TMPF_PREESTABLISHED && + tp->t_mpflags & TMPF_SENT_JOIN) { + MPTCP_COMPUTE_LEN; + } + } else { + if (tp->t_mpflags & TMPF_MPTCP_TRUE) { + MPTCP_COMPUTE_LEN; + } } return (mss_lower); @@ -5109,21 +5235,15 @@ mptcp_adj_mss(struct tcpcb *tp, boolean_t mtudisc) * Update the pid, upid, uuid of the subflow so, based on parent so */ void -mptcp_update_last_owner(struct mptsub *mpts, struct socket *parent_mpso) +mptcp_update_last_owner(struct socket *so, struct socket *mp_so) { - struct socket *subflow_so = mpts->mpts_socket; - - MPTS_LOCK_ASSERT_HELD(mpts); - - socket_lock(subflow_so, 0); - if ((subflow_so->last_pid != parent_mpso->last_pid) || - (subflow_so->last_upid != parent_mpso->last_upid)) { - subflow_so->last_upid = parent_mpso->last_upid; - subflow_so->last_pid = parent_mpso->last_pid; - uuid_copy(subflow_so->last_uuid, parent_mpso->last_uuid); + if (so->last_pid != mp_so->last_pid || + so->last_upid != mp_so->last_upid) { + so->last_upid = mp_so->last_upid; + so->last_pid = mp_so->last_pid; + uuid_copy(so->last_uuid, mp_so->last_uuid); } - so_update_policy(subflow_so); - socket_unlock(subflow_so, 0); + so_update_policy(so); } static void @@ -5159,11 +5279,9 @@ fill_mptcp_subflow(struct socket *so, mptcp_flow_t *flow, struct mptsub *mpts) flow->flow_tcpci_offset = offsetof(mptcp_flow_t, flow_ci); flow->flow_flags = mpts->mpts_flags; flow->flow_cid = mpts->mpts_connid; - flow->flow_sndnxt = mpts->mpts_sndnxt; flow->flow_relseq = mpts->mpts_rel_seq; - flow->flow_soerror = mpts->mpts_soerror; + flow->flow_soerror = mpts->mpts_socket->so_error; flow->flow_probecnt = mpts->mpts_probecnt; - flow->flow_peerswitch = mpts->mpts_peerswitch; } static int @@ -5171,7 +5289,7 @@ mptcp_pcblist SYSCTL_HANDLER_ARGS { #pragma unused(oidp, arg1, arg2) int error = 0, f; - size_t n, len; + size_t len; struct mppcb *mpp; struct mptses *mpte; struct mptcb *mp_tp; @@ -5184,8 +5302,8 @@ mptcp_pcblist SYSCTL_HANDLER_ARGS return (EPERM); lck_mtx_lock(&mtcbinfo.mppi_lock); - n = mtcbinfo.mppi_count; if (req->oldptr == USER_ADDR_NULL) { + size_t n = mtcbinfo.mppi_count; lck_mtx_unlock(&mtcbinfo.mppi_lock); req->oldidx = (n + n/8) * sizeof(conninfo_mptcp_t) + 4 * (n + n/8) * sizeof(mptcp_flow_t); @@ -5193,19 +5311,15 @@ mptcp_pcblist SYSCTL_HANDLER_ARGS } TAILQ_FOREACH(mpp, &mtcbinfo.mppi_pcbs, mpp_entry) { flows = NULL; - lck_mtx_lock(&mpp->mpp_lock); + mpp_lock(mpp); VERIFY(mpp->mpp_flags & MPP_ATTACHED); - if (mpp->mpp_flags & MPP_DEFUNCT) { - lck_mtx_unlock(&mpp->mpp_lock); - continue; - } mpte = mptompte(mpp); VERIFY(mpte != NULL); + mpte_lock_assert_held(mpte); mp_tp = mpte->mpte_mptcb; VERIFY(mp_tp != NULL); bzero(&mptcpci, sizeof(mptcpci)); - MPT_LOCK(mp_tp); mptcpci.mptcpci_state = mp_tp->mpt_state; mptcpci.mptcpci_flags = mp_tp->mpt_flags; mptcpci.mptcpci_ltoken = mp_tp->mpt_localtoken; @@ -5217,10 +5331,9 @@ mptcp_pcblist SYSCTL_HANDLER_ARGS mptcpci.mptcpci_lidsn = mp_tp->mpt_local_idsn; mptcpci.mptcpci_sndwnd = mp_tp->mpt_sndwnd; mptcpci.mptcpci_rcvnxt = mp_tp->mpt_rcvnxt; - mptcpci.mptcpci_rcvatmark = mp_tp->mpt_rcvatmark; + mptcpci.mptcpci_rcvatmark = mp_tp->mpt_rcvnxt; mptcpci.mptcpci_ridsn = mp_tp->mpt_remote_idsn; mptcpci.mptcpci_rcvwnd = mp_tp->mpt_rcvwnd; - MPT_UNLOCK(mp_tp); mptcpci.mptcpci_nflows = mpte->mpte_numflows; mptcpci.mptcpci_mpte_flags = mpte->mpte_flags; @@ -5232,7 +5345,7 @@ mptcp_pcblist SYSCTL_HANDLER_ARGS if (mpte->mpte_numflows != 0) { flows = _MALLOC(len, M_TEMP, M_WAITOK | M_ZERO); if (flows == NULL) { - lck_mtx_unlock(&mpp->mpp_lock); + mpp_unlock(mpp); break; } mptcpci.mptcpci_len = sizeof(mptcpci) + @@ -5244,21 +5357,17 @@ mptcp_pcblist SYSCTL_HANDLER_ARGS error = SYSCTL_OUT(req, &mptcpci, sizeof(mptcpci)); } if (error) { - lck_mtx_unlock(&mpp->mpp_lock); + mpp_unlock(mpp); FREE(flows, M_TEMP); break; } f = 0; TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { - MPTS_LOCK(mpts); so = mpts->mpts_socket; - socket_lock(so, 0); fill_mptcp_subflow(so, &flows[f], mpts); - socket_unlock(so, 0); - MPTS_UNLOCK(mpts); f++; } - lck_mtx_unlock(&mpp->mpp_lock); + mpp_unlock(mpp); if (flows) { error = SYSCTL_OUT(req, flows, len); FREE(flows, M_TEMP); @@ -5275,42 +5384,6 @@ SYSCTL_PROC(_net_inet_mptcp, OID_AUTO, pcblist, CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0, mptcp_pcblist, "S,conninfo_mptcp_t", "List of active MPTCP connections"); -/* - * Check the health of the other subflows and do an mptcp_output if - * there is no other active or functional subflow at the time of - * call of this function. - */ -static void -mptcp_output_needed(struct mptses *mpte, struct mptsub *to_mpts) -{ - struct mptsub *from_mpts = NULL; - - MPTE_LOCK_ASSERT_HELD(mpte); - - MPTS_UNLOCK(to_mpts); - - from_mpts = mpte->mpte_active_sub; - - if (from_mpts == NULL) - goto output_needed; - - MPTS_LOCK(from_mpts); - - if ((from_mpts->mpts_flags & MPTSF_DISCONNECTED) || - (from_mpts->mpts_flags & MPTSF_DISCONNECTING)) { - MPTS_UNLOCK(from_mpts); - goto output_needed; - } - - MPTS_UNLOCK(from_mpts); - MPTS_LOCK(to_mpts); - return; - -output_needed: - mptcp_output(mpte); - MPTS_LOCK(to_mpts); -} - /* * Set notsent lowat mark on the MPTCB */ @@ -5328,7 +5401,7 @@ mptcp_set_notsent_lowat(struct mptses *mpte, int optval) else error = EINVAL; - return error; + return (error); } u_int32_t @@ -5340,13 +5413,14 @@ mptcp_get_notsent_lowat(struct mptses *mpte) mp_tp = mpte->mpte_mptcb; if (mp_tp) - return mp_tp->mpt_notsent_lowat; + return (mp_tp->mpt_notsent_lowat); else - return 0; + return (0); } int -mptcp_notsent_lowat_check(struct socket *so) { +mptcp_notsent_lowat_check(struct socket *so) +{ struct mptses *mpte; struct mppcb *mpp; struct mptcb *mp_tp; @@ -5354,15 +5428,15 @@ mptcp_notsent_lowat_check(struct socket *so) { int notsent = 0; - mpp = sotomppcb(so); + mpp = mpsotomppcb(so); if (mpp == NULL || mpp->mpp_state == MPPCB_STATE_DEAD) { return (0); } mpte = mptompte(mpp); + mpte_lock_assert_held(mpte); mp_tp = mpte->mpte_mptcb; - MPT_LOCK(mp_tp); notsent = so->so_snd.sb_cc; if ((notsent == 0) || @@ -5373,10 +5447,8 @@ mptcp_notsent_lowat_check(struct socket *so) { mp_tp->mpt_notsent_lowat, notsent, notsent - (mp_tp->mpt_sndnxt - mp_tp->mpt_snduna)), MPTCP_SENDER_DBG , MPTCP_LOGLVL_VERBOSE); - MPT_UNLOCK(mp_tp); return (1); } - MPT_UNLOCK(mp_tp); /* When Nagle's algorithm is not disabled, it is better * to wakeup the client even before there is atleast one @@ -5384,10 +5456,8 @@ mptcp_notsent_lowat_check(struct socket *so) { */ TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { int retval = 0; - MPTS_LOCK(mpts); if (mpts->mpts_flags & MPTSF_ACTIVE) { struct socket *subf_so = mpts->mpts_socket; - socket_lock(subf_so, 0); struct tcpcb *tp = intotcpcb(sotoinpcb(subf_so)); notsent = so->so_snd.sb_cc - @@ -5401,31 +5471,16 @@ mptcp_notsent_lowat_check(struct socket *so) { " nodelay false \n", mp_tp->mpt_notsent_lowat, notsent), MPTCP_SENDER_DBG , MPTCP_LOGLVL_VERBOSE); - socket_unlock(subf_so, 0); - MPTS_UNLOCK(mpts); return (retval); } - MPTS_UNLOCK(mpts); } return (0); } -static void -mptcp_get_rtt_measurement(struct mptsub *mpts, struct mptses *mpte) -{ - MPTE_LOCK_ASSERT_HELD(mpte); - MPTS_LOCK_ASSERT_HELD(mpts); - - struct socket *subflow_so = mpts->mpts_socket; - socket_lock(subflow_so, 0); - mpts->mpts_srtt = (intotcpcb(sotoinpcb(subflow_so)))->t_srtt; - mpts->mpts_rxtcur = (intotcpcb(sotoinpcb(subflow_so)))->t_rxtcur; - socket_unlock(subflow_so, 0); -} - /* Using Symptoms Advisory to detect poor WiFi or poor Cell */ static kern_ctl_ref mptcp_kern_ctrl_ref = NULL; static uint32_t mptcp_kern_skt_inuse = 0; +static uint32_t mptcp_kern_skt_unit; symptoms_advisory_t mptcp_advisory; static errno_t @@ -5433,14 +5488,141 @@ mptcp_symptoms_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac, void **unitinfo) { #pragma unused(kctlref, sac, unitinfo) - /* - * We don't need to do anything here. But we can atleast ensure - * only one user opens the MPTCP_KERN_CTL_NAME control socket. - */ - if (OSCompareAndSwap(0, 1, &mptcp_kern_skt_inuse)) - return (0); + + if (OSIncrementAtomic(&mptcp_kern_skt_inuse) > 0) + mptcplog((LOG_ERR, "%s MPTCP kernel-control socket already open!", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + + mptcp_kern_skt_unit = sac->sc_unit; + + return (0); +} + +static void +mptcp_allow_uuid(uuid_t uuid) +{ + struct mppcb *mpp; + + /* Iterate over all MPTCP connections */ + + lck_mtx_lock(&mtcbinfo.mppi_lock); + + TAILQ_FOREACH(mpp, &mtcbinfo.mppi_pcbs, mpp_entry) { + struct mptses *mpte; + struct socket *mp_so; + + mpp_lock(mpp); + + mpte = mpp->mpp_pcbe; + mp_so = mpp->mpp_socket; + + if (mp_so->so_flags & SOF_DELEGATED && + uuid_compare(uuid, mp_so->e_uuid)) + goto next; + else if (!(mp_so->so_flags & SOF_DELEGATED) && + uuid_compare(uuid, mp_so->last_uuid)) + goto next; + + mpte->mpte_flags |= MPTE_ACCESS_GRANTED; + + mptcp_check_subflows_and_add(mpte); + mptcp_remove_subflows(mpte); + + mpte->mpte_flags &= ~MPTE_ACCESS_GRANTED; + +next: + mpp_unlock(mpp); + } + + lck_mtx_unlock(&mtcbinfo.mppi_lock); +} + +static void +mptcp_wifi_status_changed(void) +{ + struct mppcb *mpp; + + /* Iterate over all MPTCP connections */ + + lck_mtx_lock(&mtcbinfo.mppi_lock); + + TAILQ_FOREACH(mpp, &mtcbinfo.mppi_pcbs, mpp_entry) { + struct mptses *mpte; + struct socket *mp_so; + + mpp_lock(mpp); + + mpte = mpp->mpp_pcbe; + mp_so = mpp->mpp_socket; + + /* Only handover-mode is purely driven by Symptom's Wi-Fi status */ + if (mpte->mpte_svctype != MPTCP_SVCTYPE_HANDOVER) + goto next; + + mptcp_check_subflows_and_add(mpte); + mptcp_check_subflows_and_remove(mpte); + +next: + mpp_unlock(mpp); + } + + lck_mtx_unlock(&mtcbinfo.mppi_lock); +} + +void +mptcp_ask_symptoms(struct mptses *mpte) +{ + struct mptcp_symptoms_ask_uuid ask; + struct socket *mp_so; + struct proc *p; + int pid, prio, err; + + if (mptcp_kern_skt_unit == 0) { + mptcplog((LOG_ERR, "%s skt_unit is still 0\n", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + return; + } + + mp_so = mptetoso(mpte); + + if (mp_so->so_flags & SOF_DELEGATED) + pid = mp_so->e_pid; + else + pid = mp_so->last_pid; + + p = proc_find(pid); + if (p == PROC_NULL) { + mptcplog((LOG_ERR, "%s Couldn't find proc for pid %u\n", __func__, + pid), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + return; + } + + ask.cmd = MPTCP_SYMPTOMS_ASK_UUID; + + if (mp_so->so_flags & SOF_DELEGATED) + uuid_copy(ask.uuid, mp_so->e_uuid); + else + uuid_copy(ask.uuid, mp_so->last_uuid); + + prio = proc_get_effective_task_policy(proc_task(p), TASK_POLICY_ROLE); + + if (prio == TASK_BACKGROUND_APPLICATION) + ask.priority = MPTCP_SYMPTOMS_BACKGROUND; + else if (prio == TASK_FOREGROUND_APPLICATION) + ask.priority = MPTCP_SYMPTOMS_FOREGROUND; else - return (EALREADY); + ask.priority = MPTCP_SYMPTOMS_UNKNOWN; + + mptcplog((LOG_DEBUG, "%s ask symptoms about pid %u, prio %u\n", __func__, + pid, ask.priority), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + + err = ctl_enqueuedata(mptcp_kern_ctrl_ref, mptcp_kern_skt_unit, + &ask, sizeof(ask), CTL_DATA_EOR); + if (err) + mptcplog((LOG_ERR, "%s ctl_enqueuedata failed %d\n", __func__, err), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + + proc_rele(p); } static errno_t @@ -5448,23 +5630,24 @@ mptcp_symptoms_ctl_disconnect(kern_ctl_ref kctlref, u_int32_t kcunit, void *unitinfo) { #pragma unused(kctlref, kcunit, unitinfo) - if (OSCompareAndSwap(1, 0, &mptcp_kern_skt_inuse)) { - /* TBD needs to be locked if the size grows more than an int */ - bzero(&mptcp_advisory, sizeof(mptcp_advisory)); - return (0); - } - else { - return (EINVAL); - } + + OSDecrementAtomic(&mptcp_kern_skt_inuse); + + return (0); } static errno_t mptcp_symptoms_ctl_send(kern_ctl_ref kctlref, u_int32_t kcunit, void *unitinfo, mbuf_t m, int flags) { -#pragma unused(kctlref, kcunit, unitinfo, flags) +#pragma unused(kctlref, unitinfo, flags) symptoms_advisory_t *sa = NULL; + if (kcunit != mptcp_kern_skt_unit) + mptcplog((LOG_ERR, "%s kcunit %u is different from expected one %u\n", + __func__, kcunit, mptcp_kern_skt_unit), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + if (mbuf_pkthdr_len(m) < sizeof(*sa)) { mbuf_freem(m); return (EINVAL); @@ -5475,41 +5658,36 @@ mptcp_symptoms_ctl_send(kern_ctl_ref kctlref, u_int32_t kcunit, void *unitinfo, else return (EINVAL); - if (mptcp_advisory.sa_nwk_status_int != sa->sa_nwk_status_int) { - /* - * we could use this notification to notify all mptcp pcbs - * of the change in network status. But its difficult to - * define if sending REMOVE_ADDR or MP_PRIO is appropriate - * given that these are only soft indicators of the network - * state. Leaving this as TBD for now. - */ - } + if (sa->sa_nwk_status != SYMPTOMS_ADVISORY_NOCOMMENT && + sa->sa_nwk_status != SYMPTOMS_ADVISORY_USEAPP) { + uint8_t old_wifi_status = mptcp_advisory.sa_wifi_status; - if (sa->sa_nwk_status != SYMPTOMS_ADVISORY_NOCOMMENT) { - mptcplog((LOG_DEBUG, "MPTCP Events: %s wifi %d,%d cell %d,%d\n", - __func__, sa->sa_wifi_status, mptcp_advisory.sa_wifi_status, - sa->sa_cell_status, mptcp_advisory.sa_cell_status), - MPTCP_SOCKET_DBG | MPTCP_EVENTS_DBG, - MPTCP_LOGLVL_LOG); + mptcplog((LOG_DEBUG, "%s: wifi %d,%d\n", + __func__, sa->sa_wifi_status, mptcp_advisory.sa_wifi_status), + MPTCP_EVENTS_DBG, MPTCP_LOGLVL_VERBOSE); if ((sa->sa_wifi_status & (SYMPTOMS_ADVISORY_WIFI_BAD | SYMPTOMS_ADVISORY_WIFI_OK)) != - (SYMPTOMS_ADVISORY_WIFI_BAD | SYMPTOMS_ADVISORY_WIFI_OK)) { + (SYMPTOMS_ADVISORY_WIFI_BAD | SYMPTOMS_ADVISORY_WIFI_OK)) mptcp_advisory.sa_wifi_status = sa->sa_wifi_status; - } - if ((sa->sa_cell_status & - (SYMPTOMS_ADVISORY_CELL_BAD | SYMPTOMS_ADVISORY_CELL_OK)) != - (SYMPTOMS_ADVISORY_CELL_BAD | SYMPTOMS_ADVISORY_CELL_OK)) { - mptcp_advisory.sa_cell_status = sa->sa_cell_status; - } - } else { - mptcplog((LOG_DEBUG, "MPTCP Events: %s NOCOMMENT " - "wifi %d cell %d\n", __func__, - mptcp_advisory.sa_wifi_status, - mptcp_advisory.sa_cell_status), - MPTCP_SOCKET_DBG | MPTCP_EVENTS_DBG, MPTCP_LOGLVL_LOG); + if (old_wifi_status != mptcp_advisory.sa_wifi_status) + mptcp_wifi_status_changed(); + } else if (sa->sa_nwk_status == SYMPTOMS_ADVISORY_NOCOMMENT) { + mptcplog((LOG_DEBUG, "%s: NOCOMMENT wifi %d\n", __func__, + mptcp_advisory.sa_wifi_status), + MPTCP_EVENTS_DBG, MPTCP_LOGLVL_VERBOSE); + } else if (sa->sa_nwk_status == SYMPTOMS_ADVISORY_USEAPP) { + uuid_t uuid; + + mptcplog((LOG_DEBUG, "%s Got response about useApp\n", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + + uuid_copy(uuid, (unsigned char *)(sa + 1)); + + mptcp_allow_uuid(uuid); } + return (0); } @@ -5537,141 +5715,172 @@ mptcp_is_wifi_unusable(void) return (mptcp_advisory.sa_wifi_status & SYMPTOMS_ADVISORY_WIFI_BAD); } -int -mptcp_is_cell_unusable(void) -{ - /* a false return val indicates there is no info or cell is ok */ - return (mptcp_advisory.sa_cell_status & SYMPTOMS_ADVISORY_CELL_BAD); -} - -struct mptsub* -mptcp_use_symptoms_hints(struct mptsub* best, struct mptsub *second_best) -{ - struct mptsub *cellsub = NULL; - struct mptsub *wifisub = NULL; - struct mptsub *wiredsub = NULL; - - VERIFY ((best != NULL) && (second_best != NULL)); - - if (!mptcp_use_symptomsd) - return (NULL); - - if (!mptcp_kern_skt_inuse) - return (NULL); - - /* - * There could be devices with more than one wifi interface or - * more than one wired or cell interfaces. - * TBD: SymptomsD is unavailable on such platforms as of now. - * Try to prefer best when possible in general. - * Also, SymptomsD sends notifications about wifi only when it - * is primary. - */ - if (best->mpts_linktype & MPTSL_WIFI) - wifisub = best; - else if (best->mpts_linktype & MPTSL_CELL) - cellsub = best; - else if (best->mpts_linktype & MPTSL_WIRED) - wiredsub = best; - - /* - * On platforms with wired paths, don't use hints about wifi or cell. - * Currently, SymptomsD is not available on platforms with wired paths. - */ - if (wiredsub) - return (NULL); - - if ((wifisub == NULL) && (second_best->mpts_linktype & MPTSL_WIFI)) - wifisub = second_best; - - if ((cellsub == NULL) && (second_best->mpts_linktype & MPTSL_CELL)) - cellsub = second_best; - - if ((wiredsub == NULL) && (second_best->mpts_linktype & MPTSL_WIRED)) - wiredsub = second_best; - - if ((wifisub == best) && mptcp_is_wifi_unusable()) { - tcpstat.tcps_mp_sel_symtomsd++; - if (mptcp_is_cell_unusable()) { - mptcplog((LOG_DEBUG, "MPTCP Sender: SymptomsD hint" - " suggests both Wifi and Cell are bad. Wired %s.", - (wiredsub == NULL) ? "none" : "present"), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); - return (wiredsub); - } else { - mptcplog((LOG_DEBUG, "MPTCP Sender: SymptomsD hint" - " suggests Wifi bad, Cell good. Wired %s.", - (wiredsub == NULL) ? "none" : "present"), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); - return ((wiredsub != NULL) ? wiredsub : cellsub); - } - } - - if ((cellsub == best) && (mptcp_is_cell_unusable())) { - tcpstat.tcps_mp_sel_symtomsd++; - if (mptcp_is_wifi_unusable()) { - mptcplog((LOG_DEBUG, "MPTCP Sender: SymptomsD hint" - " suggests both Cell and Wifi are bad. Wired %s.", - (wiredsub == NULL) ? "none" : "present"), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); - return (wiredsub); - } else { - mptcplog((LOG_DEBUG, "MPTCP Sender: SymptomsD hint" - " suggests Cell bad, Wifi good. Wired %s.", - (wiredsub == NULL) ? "none" : "present"), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); - return ((wiredsub != NULL) ? wiredsub : wifisub); - } - } - - /* little is known about the state of the network or wifi is good */ - return (NULL); -} - /* If TFO data is succesfully acked, it must be dropped from the mptcp so */ static void -mptcp_drop_tfo_data(struct mptses *mpte, struct mptsub *mpts, int *wakeup) +mptcp_drop_tfo_data(struct mptses *mpte, struct mptsub *mpts) { - struct socket *mp_so = mpte->mpte_mppcb->mpp_socket; + struct socket *mp_so = mptetoso(mpte); struct socket *so = mpts->mpts_socket; struct tcpcb *tp = intotcpcb(sotoinpcb(so)); struct mptcb *mp_tp = mpte->mpte_mptcb; /* If data was sent with SYN, rewind state */ if (tp->t_tfo_stats & TFO_S_SYN_DATA_ACKED) { - mpts->mpts_flags &= ~MPTSF_TFO_REQD; - tp->t_mpflags &= ~TMPF_TFO_REQUEST; - MPT_LOCK(mp_tp); - u_int64_t mp_droplen = mpts->mpts_sndnxt - mp_tp->mpt_snduna; + u_int64_t mp_droplen = mp_tp->mpt_sndnxt - mp_tp->mpt_snduna; unsigned int tcp_droplen = tp->snd_una - tp->iss - 1; + VERIFY(mp_droplen <= (UINT_MAX)); VERIFY(mp_droplen >= tcp_droplen); + mpts->mpts_flags &= ~MPTSF_TFO_REQD; + mpts->mpts_iss += tcp_droplen; + tp->t_mpflags &= ~TMPF_TFO_REQUEST; + if (mp_droplen > tcp_droplen) { /* handle partial TCP ack */ mp_so->so_flags1 |= SOF1_TFO_REWIND; mp_tp->mpt_sndnxt = mp_tp->mpt_snduna + (mp_droplen - tcp_droplen); - mpts->mpts_sndnxt = mp_tp->mpt_sndnxt; mp_droplen = tcp_droplen; } else { /* all data on SYN was acked */ mpts->mpts_rel_seq = 1; mp_tp->mpt_sndnxt = mp_tp->mpt_snduna; - mpts->mpts_sndnxt = mp_tp->mpt_snduna; } mp_tp->mpt_sndmax -= tcp_droplen; - MPT_UNLOCK(mp_tp); if (mp_droplen != 0) { VERIFY(mp_so->so_snd.sb_mb != NULL); sbdrop(&mp_so->so_snd, (int)mp_droplen); - if (wakeup) - *wakeup = 1; } - mptcplog((LOG_ERR, "MPTCP Sender: %s mp_so 0x%llx cid %d " - "TFO tcp len %d mptcp len %d\n", __func__, - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), mpts->mpts_connid, - tcp_droplen, mp_droplen), - MPTCP_SENDER_DBG, MPTCP_LOGLVL_LOG); + mptcplog((LOG_DEBUG, "%s: mp_so 0x%llx cid %d TFO tcp len %d mptcp len %d\n", + __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), + mpts->mpts_connid, tcp_droplen, mp_droplen), + MPTCP_SENDER_DBG, MPTCP_LOGLVL_VERBOSE); + } +} + +int +mptcp_freeq(struct mptcb *mp_tp) +{ + struct tseg_qent *q; + int rv = 0; + + while ((q = LIST_FIRST(&mp_tp->mpt_segq)) != NULL) { + LIST_REMOVE(q, tqe_q); + m_freem(q->tqe_m); + zfree(tcp_reass_zone, q); + rv = 1; + } + mp_tp->mpt_reassqlen = 0; + return (rv); +} + +static int +mptcp_post_event(u_int32_t event_code, int value) +{ + struct kev_mptcp_data event_data; + struct kev_msg ev_msg; + + memset(&ev_msg, 0, sizeof(ev_msg)); + + ev_msg.vendor_code = KEV_VENDOR_APPLE; + ev_msg.kev_class = KEV_NETWORK_CLASS; + ev_msg.kev_subclass = KEV_MPTCP_SUBCLASS; + ev_msg.event_code = event_code; + + event_data.value = value; + + ev_msg.dv[0].data_ptr = &event_data; + ev_msg.dv[0].data_length = sizeof(event_data); + + return kev_post_msg(&ev_msg); +} + +void +mptcp_set_cellicon(struct mptses *mpte) +{ + int error; + + /* First-party apps (Siri) don't flip the cellicon */ + if (mpte->mpte_flags & MPTE_FIRSTPARTY) + return; + + /* Remember the last time we set the cellicon (see mptcp_unset_cellicon) */ + mptcp_last_cellicon_set = tcp_now; + + /* If cellicon is already set, get out of here! */ + if (OSTestAndSet(7, &mptcp_cellicon_is_set)) + return; + + error = mptcp_post_event(KEV_MPTCP_CELLUSE, 1); + + if (error) + mptcplog((LOG_ERR, "%s: Setting cellicon failed with %d\n", + __func__, error), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + else + mptcplog((LOG_DEBUG, "%s successfully set the cellicon\n", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); +} + +void +mptcp_unset_cellicon(void) +{ + int error; + + /* If cellicon is already unset, get out of here! */ + if (OSTestAndClear(7, &mptcp_cellicon_is_set)) + return; + + /* + * If during the past MPTCP_CELLICON_TOGGLE_RATE seconds we didn't + * explicitly set the cellicon (see mptcp_set_cellicon()), then we unset + * it again. + */ + if (TSTMP_GT(mptcp_last_cellicon_set + MPTCP_CELLICON_TOGGLE_RATE, + tcp_now)) { + OSTestAndSet(7, &mptcp_cellicon_is_set); + return; } + + error = mptcp_post_event(KEV_MPTCP_CELLUSE, 0); + + if (error) + mptcplog((LOG_ERR, "%s: Unsetting cellicon failed with %d\n", + __func__, error), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + else + mptcplog((LOG_DEBUG, "%s successfully unset the cellicon\n", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); +} + +void +mptcp_reset_rexmit_state(struct tcpcb *tp) +{ + struct mptsub *mpts; + struct inpcb *inp; + struct socket *so; + + inp = tp->t_inpcb; + if (inp == NULL) + return; + + so = inp->inp_socket; + if (so == NULL) + return; + + if (!(so->so_flags & SOF_MP_SUBFLOW)) + return; + + mpts = tp->t_mpsub; + + mpts->mpts_flags &= ~MPTSF_WRITE_STALL; + so->so_flags &= ~SOF_MP_TRYFAILOVER; +} + +void +mptcp_reset_keepalive(struct tcpcb *tp) +{ + struct mptsub *mpts = tp->t_mpsub; + + mpts->mpts_flags &= ~MPTSF_READ_STALL; } + diff --git a/bsd/netinet/mptcp_timer.c b/bsd/netinet/mptcp_timer.c index 7ac605b74..0f427e184 100644 --- a/bsd/netinet/mptcp_timer.c +++ b/bsd/netinet/mptcp_timer.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2013 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -75,8 +75,7 @@ mptcp_timer_demux(struct mptses *mpte, uint32_t now_msecs) DTRACE_MPTCP2(timer, struct mptses *, mpte, struct mptcb *, mp_tp); - MPTE_LOCK_ASSERT_HELD(mpte); - MPT_LOCK(mp_tp); + mpte_lock_assert_held(mpte); switch (mp_tp->mpt_timer_vals) { case MPTT_REXMT: if (mp_tp->mpt_rxtstart == 0) @@ -95,13 +94,11 @@ mptcp_timer_demux(struct mptses *mpte, uint32_t now_msecs) DTRACE_MPTCP1(error, struct mptcb *, mp_tp); } else { mp_tp->mpt_sndnxt = mp_tp->mpt_rtseq; - MPT_UNLOCK(mp_tp); mptcplog((LOG_DEBUG, "MPTCP Socket: " "%s: REXMT %d times.\n", __func__, mp_tp->mpt_rxtshift), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); mptcp_output(mpte); - MPT_LOCK(mp_tp); } } else { resched_timer = 1; @@ -125,7 +122,6 @@ mptcp_timer_demux(struct mptses *mpte, uint32_t now_msecs) default: break; } - MPT_UNLOCK(mp_tp); return (resched_timer); } @@ -138,7 +134,7 @@ mptcp_timer(struct mppcbinfo *mppi) u_int32_t now_msecs; uint32_t resched_timer = 0; - lck_mtx_assert(&mppi->mppi_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&mppi->mppi_lock, LCK_MTX_ASSERT_OWNED); microuptime(&now); now_msecs = TIMEVAL_TO_HZ(now); @@ -150,17 +146,12 @@ mptcp_timer(struct mppcbinfo *mppi) VERIFY(mp_so != NULL); mpte = mptompte(mpp); VERIFY(mpte != NULL); - MPTE_LOCK(mpte); + mpte_lock(mpte); VERIFY(mpp->mpp_flags & MPP_ATTACHED); - if (mpp->mpp_flags & MPP_DEFUNCT) { - MPTE_UNLOCK(mpte); - continue; - } - if (mptcp_timer_demux(mpte, now_msecs)) resched_timer = 1; - MPTE_UNLOCK(mpte); + mpte_unlock(mpte); } return (resched_timer); @@ -178,21 +169,19 @@ mptcp_start_timer(struct mptses *mpte, int timer_type) mptcplog((LOG_DEBUG, "MPTCP Socket: %s: %d\n", __func__, timer_type), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + mpte_lock_assert_held(mpte); + switch (timer_type) { case MPTT_REXMT: - MPT_LOCK(mp_tp); mp_tp->mpt_timer_vals |= MPTT_REXMT; mp_tp->mpt_rxtstart = TIMEVAL_TO_HZ(now); mp_tp->mpt_rxtshift = 0; mp_tp->mpt_rtseq = mp_tp->mpt_sndnxt; - MPT_UNLOCK(mp_tp); break; case MPTT_TW: /* XXX: Not implemented yet */ - MPT_LOCK(mp_tp); mp_tp->mpt_timer_vals |= MPTT_TW; mp_tp->mpt_timewait = TIMEVAL_TO_HZ(now); - MPT_UNLOCK(mp_tp); break; case MPTT_FASTCLOSE: /* NO-OP */ @@ -207,7 +196,7 @@ mptcp_start_timer(struct mptses *mpte, int timer_type) void mptcp_cancel_timer(struct mptcb *mp_tp, int timer_type) { - MPT_LOCK_ASSERT_HELD(mp_tp); + mpte_lock_assert_held(mp_tp->mpt_mpte); DTRACE_MPTCP2(cancel__timer, struct mptcb *, mp_tp, int, timer_type); switch (timer_type) { diff --git a/bsd/netinet/mptcp_timer.h b/bsd/netinet/mptcp_timer.h index 94da71dc5..35ee57c48 100644 --- a/bsd/netinet/mptcp_timer.h +++ b/bsd/netinet/mptcp_timer.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * diff --git a/bsd/netinet/mptcp_usrreq.c b/bsd/netinet/mptcp_usrreq.c index cc84cb40e..1a5589901 100644 --- a/bsd/netinet/mptcp_usrreq.c +++ b/bsd/netinet/mptcp_usrreq.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2015 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -37,6 +37,8 @@ #include <sys/proc.h> #include <sys/proc_internal.h> #include <sys/resourcevar.h> +#include <sys/kauth.h> +#include <sys/priv.h> #include <net/if.h> #include <netinet/in.h> @@ -54,10 +56,6 @@ static int mptcp_usr_attach(struct socket *, int, struct proc *); static int mptcp_usr_detach(struct socket *); static int mptcp_attach(struct socket *, struct proc *); -static int mptcp_detach(struct socket *, struct mppcb *); -static int mptcp_connectx(struct mptses *, struct sockaddr *, - struct sockaddr *, struct proc *, uint32_t, sae_associd_t, - sae_connid_t *, uint32_t, void *, uint32_t); static int mptcp_usr_connectx(struct socket *, struct sockaddr *, struct sockaddr *, struct proc *, uint32_t, sae_associd_t, sae_connid_t *, uint32_t, void *, uint32_t, struct uio *, user_ssize_t *); @@ -69,25 +67,20 @@ static int mptcp_getconninfo(struct mptses *, sae_connid_t *, uint32_t *, uint32_t *, user_addr_t, uint32_t *); static int mptcp_usr_control(struct socket *, u_long, caddr_t, struct ifnet *, struct proc *); -static int mptcp_disconnectx(struct mptses *, sae_associd_t, sae_connid_t); +static int mptcp_disconnect(struct mptses *); static int mptcp_usr_disconnect(struct socket *); static int mptcp_usr_disconnectx(struct socket *, sae_associd_t, sae_connid_t); static struct mptses *mptcp_usrclosed(struct mptses *); -static int mptcp_usr_peeloff(struct socket *, sae_associd_t, struct socket **); -static int mptcp_peeloff(struct mptses *, sae_associd_t, struct socket **); static int mptcp_usr_rcvd(struct socket *, int); static int mptcp_usr_send(struct socket *, int, struct mbuf *, struct sockaddr *, struct mbuf *, struct proc *); static int mptcp_usr_shutdown(struct socket *); -static int mptcp_uiotombuf(struct uio *, int, int, uint32_t, struct mbuf **); static int mptcp_usr_sosend(struct socket *, struct sockaddr *, struct uio *, struct mbuf *, struct mbuf *, int); static int mptcp_usr_socheckopt(struct socket *, struct sockopt *); -static int mptcp_setopt_apply(struct mptses *, struct mptopt *); static int mptcp_setopt(struct mptses *, struct sockopt *); static int mptcp_getopt(struct mptses *, struct sockopt *); static int mptcp_default_tcp_optval(struct mptses *, struct sockopt *, int *); -static void mptcp_connorder_helper(struct mptsub *mpts); static int mptcp_usr_preconnect(struct socket *so); struct pr_usrreqs mptcp_usrreqs = { @@ -97,16 +90,29 @@ struct pr_usrreqs mptcp_usrreqs = { .pru_detach = mptcp_usr_detach, .pru_disconnect = mptcp_usr_disconnect, .pru_disconnectx = mptcp_usr_disconnectx, - .pru_peeloff = mptcp_usr_peeloff, + .pru_peeraddr = mp_getpeeraddr, .pru_rcvd = mptcp_usr_rcvd, .pru_send = mptcp_usr_send, .pru_shutdown = mptcp_usr_shutdown, + .pru_sockaddr = mp_getsockaddr, .pru_sosend = mptcp_usr_sosend, .pru_soreceive = soreceive, .pru_socheckopt = mptcp_usr_socheckopt, .pru_preconnect = mptcp_usr_preconnect, }; + +#if (DEVELOPMENT || DEBUG) +static int mptcp_disable_entitlements = 0; +SYSCTL_INT(_net_inet_mptcp, OID_AUTO, disable_entitlements, CTLFLAG_RW | CTLFLAG_LOCKED, + &mptcp_disable_entitlements, 0, "Disable Multipath TCP Entitlement Checking"); +#endif + +int mptcp_developer_mode = 0; +SYSCTL_INT(_net_inet_mptcp, OID_AUTO, allow_aggregate, CTLFLAG_RW | CTLFLAG_LOCKED, + &mptcp_developer_mode, 0, "Allow the Multipath aggregation mode"); + + /* * Attaches an MPTCP control block to a socket. */ @@ -116,7 +122,7 @@ mptcp_usr_attach(struct socket *mp_so, int proto, struct proc *p) #pragma unused(proto) int error; - VERIFY(sotomppcb(mp_so) == NULL); + VERIFY(mpsotomppcb(mp_so) == NULL); error = mptcp_attach(mp_so, p); if (error != 0) @@ -138,14 +144,27 @@ out: static int mptcp_usr_detach(struct socket *mp_so) { - struct mppcb *mpp = sotomppcb(mp_so); - int error = 0; + struct mptses *mpte = mpsotompte(mp_so); + struct mppcb *mpp = mpsotomppcb(mp_so); - VERIFY(mpp != NULL); - VERIFY(mpp->mpp_socket != NULL); + if (mpp == NULL || mpp->mpp_state == MPPCB_STATE_DEAD) { + mptcplog((LOG_ERR, "%s state: %d\n", __func__, + mpp ? mpp->mpp_state : -1), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + return (EINVAL); + } - error = mptcp_detach(mp_so, mpp); - return (error); + /* + * We are done with this MPTCP socket (it has been closed); + * trigger all subflows to be disconnected, if not already, + * by initiating the PCB detach sequence (SOF_PCBCLEARING + * will be set.) + */ + mp_pcbdetach(mp_so); + + mptcp_disconnect(mpte); + + return (0); } /* @@ -162,7 +181,7 @@ mptcp_attach(struct socket *mp_so, struct proc *p) int error = 0; if (mp_so->so_snd.sb_hiwat == 0 || mp_so->so_rcv.sb_hiwat == 0) { - error = soreserve(mp_so, tcp_sendspace, MPTCP_RWIN_MAX); + error = soreserve(mp_so, tcp_sendspace, tcp_recvspace); if (error != 0) goto out; } @@ -171,6 +190,11 @@ mptcp_attach(struct socket *mp_so, struct proc *p) soreserve_preconnect(mp_so, 2048); } + if ((mp_so->so_rcv.sb_flags & SB_USRSIZE) == 0) + mp_so->so_rcv.sb_flags |= SB_AUTOSIZE; + if ((mp_so->so_snd.sb_flags & SB_USRSIZE) == 0) + mp_so->so_snd.sb_flags |= SB_AUTOSIZE; + /* * MPTCP socket buffers cannot be compressed, due to the * fact that each mbuf chained via m_next is a M_PKTHDR @@ -179,15 +203,11 @@ mptcp_attach(struct socket *mp_so, struct proc *p) mp_so->so_snd.sb_flags |= SB_NOCOMPRESS; mp_so->so_rcv.sb_flags |= SB_NOCOMPRESS; - /* Disable socket buffer auto-tuning. */ - mp_so->so_rcv.sb_flags &= ~SB_AUTOSIZE; - mp_so->so_snd.sb_flags &= ~SB_AUTOSIZE; - if ((error = mp_pcballoc(mp_so, &mtcbinfo)) != 0) { goto out; } - mpp = sotomppcb(mp_so); + mpp = mpsotomppcb(mp_so); VERIFY(mpp != NULL); mpte = (struct mptses *)mpp->mpp_pcbe; VERIFY(mpte != NULL); @@ -197,43 +217,41 @@ out: return (error); } -/* - * Called when the socket layer loses its final reference to the socket; - * at this point, there is only one case in which we will keep things - * around: time wait. - */ static int -mptcp_detach(struct socket *mp_so, struct mppcb *mpp) +mptcp_entitlement_check(struct socket *mp_so) { - struct mptses *mpte; - struct mppcbinfo *mppi; + struct mptses *mpte = mpsotompte(mp_so); - VERIFY(mp_so->so_pcb == mpp); - VERIFY(mpp->mpp_socket == mp_so); - - mppi = mpp->mpp_pcbinfo; - VERIFY(mppi != NULL); - - __IGNORE_WCASTALIGN(mpte = &((struct mpp_mtp *)mpp)->mpp_ses); - VERIFY(mpte->mpte_mppcb == mpp); + if (soopt_cred_check(mp_so, PRIV_NET_RESTRICTED_MULTIPATH_EXTENDED, TRUE) == 0) { + /* + * This means the app has the extended entitlement. Thus, + * it's a first party app and can run without restrictions. + */ + mpte->mpte_flags |= MPTE_FIRSTPARTY; + goto grant; + } - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ +#if (DEVELOPMENT || DEBUG) + if (mptcp_disable_entitlements) + goto grant; +#endif - /* - * We are done with this MPTCP socket (it has been closed); - * trigger all subflows to be disconnected, if not already, - * by initiating the PCB detach sequence (SOF_PCBCLEARING - * will be set.) - */ - mp_pcbdetach(mpp); + if (soopt_cred_check(mp_so, PRIV_NET_PRIVILEGED_MULTIPATH, TRUE)) { + mptcplog((LOG_NOTICE, "%s Multipath Capability needed\n", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); + return (-1); + } - (void) mptcp_disconnectx(mpte, SAE_ASSOCID_ALL, SAE_CONNID_ALL); + if (mpte->mpte_svctype > MPTCP_SVCTYPE_INTERACTIVE && + mptcp_developer_mode == 0) { + mptcplog((LOG_NOTICE, "%s need to set allow_aggregate sysctl\n", + __func__), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); + return (-1); + } - /* - * XXX: adi@apple.com - * - * Here, we would want to handle time wait state. - */ +grant: + mptcplog((LOG_NOTICE, "%s entitlement granted for %u\n", __func__, mpte->mpte_svctype), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); return (0); } @@ -246,74 +264,20 @@ mptcp_detach(struct socket *mp_so, struct mppcb *mpp) */ static int mptcp_connectx(struct mptses *mpte, struct sockaddr *src, - struct sockaddr *dst, struct proc *p, uint32_t ifscope, - sae_associd_t aid, sae_connid_t *pcid, uint32_t flags, void *arg, - uint32_t arglen) + struct sockaddr *dst, uint32_t ifscope, sae_connid_t *pcid) { -#pragma unused(p, aid, flags, arg, arglen) - struct mptsub *mpts; - struct socket *mp_so; + struct socket *mp_so = mptetoso(mpte); int error = 0; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - mp_so = mpte->mpte_mppcb->mpp_socket; - VERIFY(dst != NULL); VERIFY(pcid != NULL); - mptcplog((LOG_DEBUG, "MPTCP Socket: " - "%s: mp_so 0x%llx\n", __func__, + mptcplog((LOG_DEBUG, "%s: mp_so 0x%llx\n", __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so)), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); + DTRACE_MPTCP2(connectx, struct mptses *, mpte, struct socket *, mp_so); - DTRACE_MPTCP3(connectx, struct mptses *, mpte, sae_associd_t, aid, - struct socket *, mp_so); - - mpts = mptcp_subflow_alloc(M_WAITOK); - if (mpts == NULL) { - error = ENOBUFS; - goto out; - } - MPTS_ADDREF(mpts); /* for this routine */ - - if (src != NULL) { - int len = src->sa_len; - - MALLOC(mpts->mpts_src, struct sockaddr *, len, M_SONAME, - M_WAITOK | M_ZERO); - if (mpts->mpts_src == NULL) { - error = ENOBUFS; - goto out; - } - bcopy(src, mpts->mpts_src, len); - } - - MALLOC(mpts->mpts_dst, struct sockaddr *, dst->sa_len, M_SONAME, - M_WAITOK | M_ZERO); - if (mpts->mpts_dst == NULL) { - error = ENOBUFS; - goto out; - } - bcopy(dst, mpts->mpts_dst, dst->sa_len); - - error = mptcp_subflow_add(mpte, mpts, p, ifscope); - if (error == 0 && pcid != NULL) - *pcid = mpts->mpts_connid; - -out: - if (mpts != NULL) { - if ((error != 0) && (error != EWOULDBLOCK)) { - MPTS_LOCK(mpts); - if (mpts->mpts_flags & MPTSF_ATTACHED) { - MPTS_UNLOCK(mpts); - MPTS_REMREF(mpts); - mptcp_subflow_del(mpte, mpts, TRUE); - return (error); - } - MPTS_UNLOCK(mpts); - } - MPTS_REMREF(mpts); - } + error = mptcp_subflow_add(mpte, src, dst, ifscope, pcid); return (error); } @@ -327,30 +291,90 @@ mptcp_usr_connectx(struct socket *mp_so, struct sockaddr *src, sae_associd_t aid, sae_connid_t *pcid, uint32_t flags, void *arg, uint32_t arglen, struct uio *auio, user_ssize_t *bytes_written) { - struct mppcb *mpp = sotomppcb(mp_so); +#pragma unused(p, aid, flags, arg, arglen) + struct mppcb *mpp = mpsotomppcb(mp_so); struct mptses *mpte = NULL; struct mptcb *mp_tp = NULL; user_ssize_t datalen; - int error = 0; if (mpp == NULL || mpp->mpp_state == MPPCB_STATE_DEAD) { + mptcplog((LOG_ERR, "%s state %d\n", __func__, + mpp ? mpp->mpp_state : -1), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); error = EINVAL; goto out; } mpte = mptompte(mpp); VERIFY(mpte != NULL); + mpte_lock_assert_held(mpte); mp_tp = mpte->mpte_mptcb; VERIFY(mp_tp != NULL); if (mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) { + mptcplog((LOG_ERR, "%s fell back to TCP\n", __func__), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); error = EINVAL; goto out; } - error = mptcp_connectx(mpte, src, dst, p, ifscope, - aid, pcid, flags, arg, arglen); + if (dst->sa_family == AF_INET && + dst->sa_len != sizeof(mpte->__mpte_dst_v4)) { + mptcplog((LOG_ERR, "%s IPv4 dst len %u\n", __func__, + dst->sa_len), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + error = EINVAL; + goto out; + } + + if (dst->sa_family == AF_INET6 && + dst->sa_len != sizeof(mpte->__mpte_dst_v6)) { + mptcplog((LOG_ERR, "%s IPv6 dst len %u\n", __func__, + dst->sa_len), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + error = EINVAL; + goto out; + } + + if (!(mpte->mpte_flags & MPTE_SVCTYPE_CHECKED)) { + if (mptcp_entitlement_check(mp_so) < 0) { + error = EPERM; + goto out; + } + + mpte->mpte_flags |= MPTE_SVCTYPE_CHECKED; + } + + if ((mp_so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING)) == 0) { + memcpy(&mpte->mpte_dst, dst, dst->sa_len); + } + + if (src) { + if (src->sa_family == AF_INET && + src->sa_len != sizeof(mpte->__mpte_src_v4)) { + mptcplog((LOG_ERR, "%s IPv4 src len %u\n", __func__, + src->sa_len), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + error = EINVAL; + goto out; + } + + if (src->sa_family == AF_INET6 && + src->sa_len != sizeof(mpte->__mpte_src_v6)) { + mptcplog((LOG_ERR, "%s IPv6 src len %u\n", __func__, + src->sa_len), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + error = EINVAL; + goto out; + } + + if ((mp_so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING)) == 0) { + memcpy(&mpte->mpte_src, src, src->sa_len); + } + } + + error = mptcp_connectx(mpte, src, dst, ifscope, pcid); /* If there is data, copy it */ if (auio != NULL) { @@ -358,7 +382,7 @@ mptcp_usr_connectx(struct socket *mp_so, struct sockaddr *src, socket_unlock(mp_so, 0); error = mp_so->so_proto->pr_usrreqs->pru_sosend(mp_so, NULL, (uio_t) auio, NULL, NULL, 0); - /* check if this can be supported with fast Join also. XXX */ + if (error == 0 || error == EWOULDBLOCK) *bytes_written = datalen - uio_resid(auio); @@ -366,16 +390,6 @@ mptcp_usr_connectx(struct socket *mp_so, struct sockaddr *src, error = EINPROGRESS; socket_lock(mp_so, 0); - MPT_LOCK(mp_tp); - if (mp_tp->mpt_flags & MPTCPF_PEEL_OFF) { - *bytes_written = datalen - uio_resid(auio); - /* - * Override errors like EPIPE that occur as - * a result of doing TFO during TCP fallback. - */ - error = EPROTO; - } - MPT_UNLOCK(mp_tp); } out: @@ -388,7 +402,7 @@ out: static int mptcp_getassocids(struct mptses *mpte, uint32_t *cnt, user_addr_t aidp) { - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + mpte_lock_assert_held(mpte); /* same as MP socket lock */ /* MPTCP has at most 1 association */ *cnt = (mpte->mpte_associd != SAE_ASSOCID_ANY) ? 1 : 0; @@ -411,7 +425,7 @@ mptcp_getconnids(struct mptses *mpte, sae_associd_t aid, uint32_t *cnt, struct mptsub *mpts; int error = 0; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + mpte_lock_assert_held(mpte); /* same as MP socket lock */ if (aid != SAE_ASSOCID_ANY && aid != SAE_ASSOCID_ALL && aid != mpte->mpte_associd) @@ -443,237 +457,132 @@ mptcp_getconninfo(struct mptses *mpte, sae_connid_t *cid, uint32_t *flags, user_addr_t dst, socklen_t *dst_len, uint32_t *aux_type, user_addr_t aux_data, uint32_t *aux_len) { -#pragma unused(aux_data) - struct ifnet *ifp = NULL; + struct socket *so; + struct inpcb *inp; struct mptsub *mpts; int error = 0; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - - if (*cid == SAE_CONNID_ALL) - return (EINVAL); - - TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { - if (mpts->mpts_connid == *cid || *cid == SAE_CONNID_ANY) - break; - } - if (mpts == NULL) - return ((*cid == SAE_CONNID_ANY) ? ENXIO : EINVAL); - - MPTS_LOCK(mpts); - ifp = mpts->mpts_outif; - *cid = mpts->mpts_connid; - *ifindex = ((ifp != NULL) ? ifp->if_index : 0); - *soerror = mpts->mpts_soerror; *flags = 0; - if (mpts->mpts_flags & MPTSF_CONNECTING) - *flags |= CIF_CONNECTING; - if (mpts->mpts_flags & MPTSF_CONNECTED) - *flags |= CIF_CONNECTED; - if (mpts->mpts_flags & MPTSF_DISCONNECTING) - *flags |= CIF_DISCONNECTING; - if (mpts->mpts_flags & MPTSF_DISCONNECTED) - *flags |= CIF_DISCONNECTED; - if (mpts->mpts_flags & MPTSF_BOUND_IF) - *flags |= CIF_BOUND_IF; - if (mpts->mpts_flags & MPTSF_BOUND_IP) - *flags |= CIF_BOUND_IP; - if (mpts->mpts_flags & MPTSF_BOUND_PORT) - *flags |= CIF_BOUND_PORT; - if (mpts->mpts_flags & MPTSF_PREFERRED) - *flags |= CIF_PREFERRED; - if (mpts->mpts_flags & MPTSF_MP_CAPABLE) - *flags |= CIF_MP_CAPABLE; - if (mpts->mpts_flags & MPTSF_MP_DEGRADED) - *flags |= CIF_MP_DEGRADED; - if (mpts->mpts_flags & MPTSF_MP_READY) - *flags |= CIF_MP_READY; - if (mpts->mpts_flags & MPTSF_ACTIVE) - *flags |= CIF_MP_ACTIVE; + *aux_type = 0; + *ifindex = 0; + *soerror = 0; + + if (*cid == SAE_CONNID_ALL) { + struct socket *mp_so = mptetoso(mpte); + struct mptcb *mp_tp = mpte->mpte_mptcb; + struct conninfo_multipathtcp mptcp_ci; + + if (*aux_len != 0 && *aux_len != sizeof(mptcp_ci)) + return (EINVAL); + + if (mp_so->so_state & SS_ISCONNECTING) + *flags |= CIF_CONNECTING; + if (mp_so->so_state & SS_ISCONNECTED) + *flags |= CIF_CONNECTED; + if (mp_so->so_state & SS_ISDISCONNECTING) + *flags |= CIF_DISCONNECTING; + if (mp_so->so_state & SS_ISDISCONNECTED) + *flags |= CIF_DISCONNECTED; + if (!(mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP)) + *flags |= CIF_MP_CAPABLE; + if (mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) + *flags |= CIF_MP_DEGRADED; + + *src_len = 0; + *dst_len = 0; + + *aux_type = CIAUX_MPTCP; + *aux_len = sizeof(mptcp_ci); - VERIFY(mpts->mpts_src != NULL); - *src_len = mpts->mpts_src->sa_len; - if (src != USER_ADDR_NULL) { - error = copyout(mpts->mpts_src, src, mpts->mpts_src->sa_len); - if (error != 0) - goto out; - } + if (aux_data != USER_ADDR_NULL) { + unsigned long i = 0; + int initial_info_set = 0; - VERIFY(mpts->mpts_dst != NULL); - *dst_len = mpts->mpts_dst->sa_len; - if (dst != USER_ADDR_NULL) { - error = copyout(mpts->mpts_dst, dst, mpts->mpts_dst->sa_len); - if (error != 0) - goto out; - } + bzero(&mptcp_ci, sizeof (mptcp_ci)); + mptcp_ci.mptcpci_subflow_count = mpte->mpte_numflows; + mptcp_ci.mptcpci_switch_count = mpte->mpte_subflow_switches; - *aux_type = 0; - *aux_len = 0; - if (mpts->mpts_socket != NULL) { - struct conninfo_tcp tcp_ci; + VERIFY(sizeof(mptcp_ci.mptcpci_itfstats) == sizeof(mpte->mpte_itfstats)); + memcpy(mptcp_ci.mptcpci_itfstats, mpte->mpte_itfstats, sizeof(mptcp_ci.mptcpci_itfstats)); - *aux_type = CIAUX_TCP; - *aux_len = sizeof (tcp_ci); + TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { + if (i >= sizeof(mptcp_ci.mptcpci_subflow_connids) / sizeof(sae_connid_t)) + break; + mptcp_ci.mptcpci_subflow_connids[i] = mpts->mpts_connid; - if (aux_data != USER_ADDR_NULL) { - struct socket *so = mpts->mpts_socket; - - VERIFY(SOCK_PROTO(so) == IPPROTO_TCP); - bzero(&tcp_ci, sizeof (tcp_ci)); - socket_lock(so, 0); - tcp_getconninfo(so, &tcp_ci); - socket_unlock(so, 0); - error = copyout(&tcp_ci, aux_data, sizeof (tcp_ci)); - if (error != 0) - goto out; - } - } - mptcplog((LOG_DEBUG, "MPTCP Socket: " - "%s: cid %d flags %x \n", - __func__, mpts->mpts_connid, mpts->mpts_flags), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + if (mpts->mpts_flags & MPTSF_INITIAL_SUB) { + inp = sotoinpcb(mpts->mpts_socket); -out: - MPTS_UNLOCK(mpts); - return (error); -} + mptcp_ci.mptcpci_init_rxbytes = inp->inp_stat->rxbytes; + mptcp_ci.mptcpci_init_txbytes = inp->inp_stat->txbytes; + initial_info_set = 1; + } -/* - * Handle SIOCSCONNORDER - */ -int -mptcp_setconnorder(struct mptses *mpte, sae_connid_t cid, uint32_t rank) -{ - struct mptsub *mpts, *mpts1; - int error = 0; + mptcpstats_update(mptcp_ci.mptcpci_itfstats, mpts); - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - mptcplog((LOG_DEBUG, "MPTCP Socket: " - "%s: cid %d rank %d \n", __func__, cid, rank), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + i++; + } - if (cid == SAE_CONNID_ANY || cid == SAE_CONNID_ALL) { - error = EINVAL; - goto out; - } + if (initial_info_set == 0) { + mptcp_ci.mptcpci_init_rxbytes = mpte->mpte_init_rxbytes; + mptcp_ci.mptcpci_init_txbytes = mpte->mpte_init_txbytes; + } - TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { - if (mpts->mpts_connid == cid) - break; - } - if (mpts == NULL) { - error = ENXIO; - goto out; - } + if (mpte->mpte_flags & MPTE_FIRSTPARTY) + mptcp_ci.mptcpci_flags |= MPTCPCI_FIRSTPARTY; - if (rank == 0 || rank > 1) { - /* - * If rank is 0, determine whether this should be the - * primary or backup subflow, depending on what we have. - * - * Otherwise, if greater than 0, make it a backup flow. - */ - TAILQ_FOREACH(mpts1, &mpte->mpte_subflows, mpts_entry) { - MPTS_LOCK(mpts1); - if (mpts1->mpts_flags & MPTSF_PREFERRED) { - MPTS_UNLOCK(mpts1); - break; + error = copyout(&mptcp_ci, aux_data, sizeof(mptcp_ci)); + if (error != 0) { + mptcplog((LOG_ERR, "%s copyout failed: %d\n", + __func__, error), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + return (error); } - MPTS_UNLOCK(mpts1); } - MPTS_LOCK(mpts); - mpts->mpts_flags &= ~MPTSF_PREFERRED; - mpts->mpts_rank = rank; - if (mpts1 != NULL && mpts != mpts1) { - /* preferred subflow found; set rank as necessary */ - if (rank == 0) - mpts->mpts_rank = (mpts1->mpts_rank + 1); - } else if (rank == 0) { - /* no preferred one found; promote this */ - rank = 1; - } - MPTS_UNLOCK(mpts); + return (0); } - if (rank == 1) { - /* - * If rank is 1, promote this subflow to be preferred. - */ - TAILQ_FOREACH(mpts1, &mpte->mpte_subflows, mpts_entry) { - MPTS_LOCK(mpts1); - if (mpts1 != mpts && - (mpts1->mpts_flags & MPTSF_PREFERRED)) { - mpts1->mpts_flags &= ~MPTSF_PREFERRED; - if (mpte->mpte_nummpcapflows > 1) - mptcp_connorder_helper(mpts1); - } else if (mpts1 == mpts) { - mpts1->mpts_rank = 1; - if (mpts1->mpts_flags & MPTSF_MP_CAPABLE) { - mpts1->mpts_flags |= MPTSF_PREFERRED; - if (mpte->mpte_nummpcapflows > 1) - mptcp_connorder_helper(mpts1); - } - } - MPTS_UNLOCK(mpts1); - } + TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { + if (mpts->mpts_connid == *cid || *cid == SAE_CONNID_ANY) + break; } + if (mpts == NULL) + return ((*cid == SAE_CONNID_ANY) ? ENXIO : EINVAL); -out: - return (error); -} - -static void -mptcp_connorder_helper(struct mptsub *mpts) -{ - struct socket *so = mpts->mpts_socket; - struct tcpcb *tp = NULL; - - socket_lock(so, 0); + so = mpts->mpts_socket; + inp = sotoinpcb(so); - tp = intotcpcb(sotoinpcb(so)); - tp->t_mpflags |= TMPF_SND_MPPRIO; - if (mpts->mpts_flags & MPTSF_PREFERRED) - tp->t_mpflags &= ~TMPF_BACKUP_PATH; + if (inp->inp_vflag & INP_IPV4) + error = in_getconninfo(so, SAE_CONNID_ANY, flags, ifindex, + soerror, src, src_len, dst, dst_len, + aux_type, aux_data, aux_len); else - tp->t_mpflags |= TMPF_BACKUP_PATH; - - socket_unlock(so, 0); - -} - -/* - * Handle SIOCSGONNORDER - */ -int -mptcp_getconnorder(struct mptses *mpte, sae_connid_t cid, uint32_t *rank) -{ - struct mptsub *mpts; - int error = 0; - - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - VERIFY(rank != NULL); - *rank = 0; + error = in6_getconninfo(so, SAE_CONNID_ANY, flags, ifindex, + soerror, src, src_len, dst, dst_len, + aux_type, aux_data, aux_len); - if (cid == SAE_CONNID_ANY || cid == SAE_CONNID_ALL) { - error = EINVAL; - goto out; + if (error != 0) { + mptcplog((LOG_ERR, "%s error from in_getconninfo %d\n", + __func__, error), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + return (error); } - TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { - if (mpts->mpts_connid == cid) - break; - } - if (mpts == NULL) { - error = ENXIO; - goto out; - } + if (mpts->mpts_flags & MPTSF_MP_CAPABLE) + *flags |= CIF_MP_CAPABLE; + if (mpts->mpts_flags & MPTSF_MP_DEGRADED) + *flags |= CIF_MP_DEGRADED; + if (mpts->mpts_flags & MPTSF_MP_READY) + *flags |= CIF_MP_READY; + if (mpts->mpts_flags & MPTSF_ACTIVE) + *flags |= CIF_MP_ACTIVE; - MPTS_LOCK(mpts); - *rank = mpts->mpts_rank; - MPTS_UNLOCK(mpts); -out: - return (error); + mptcplog((LOG_DEBUG, "%s: cid %d flags %x \n", __func__, + mpts->mpts_connid, mpts->mpts_flags), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_VERBOSE); + + return (0); } /* @@ -684,7 +593,7 @@ mptcp_usr_control(struct socket *mp_so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p) { #pragma unused(ifp, p) - struct mppcb *mpp = sotomppcb(mp_so); + struct mppcb *mpp = mpsotomppcb(mp_so); struct mptses *mpte; int error = 0; @@ -695,7 +604,7 @@ mptcp_usr_control(struct socket *mp_so, u_long cmd, caddr_t data, mpte = mptompte(mpp); VERIFY(mpte != NULL); - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + mpte_lock_assert_held(mpte); /* same as MP socket lock */ switch (cmd) { case SIOCGASSOCIDS32: { /* struct so_aidreq32 */ @@ -764,24 +673,6 @@ mptcp_usr_control(struct socket *mp_so, u_long cmd, caddr_t data, break; } - case SIOCSCONNORDER: { /* struct so_cordreq */ - struct so_cordreq cor; - bcopy(data, &cor, sizeof (cor)); - error = mptcp_setconnorder(mpte, cor.sco_cid, cor.sco_rank); - if (error == 0) - bcopy(&cor, data, sizeof (cor)); - break; - } - - case SIOCGCONNORDER: { /* struct so_cordreq */ - struct so_cordreq cor; - bcopy(data, &cor, sizeof (cor)); - error = mptcp_getconnorder(mpte, cor.sco_cid, &cor.sco_rank); - if (error == 0) - bcopy(&cor, data, sizeof (cor)); - break; - } - default: error = EOPNOTSUPP; break; @@ -790,131 +681,66 @@ out: return (error); } -/* - * Initiate a disconnect. MPTCP-level disconnection is specified by - * CONNID_{ANY,ALL}. Otherwise, selectively disconnect a subflow - * connection while keeping the MPTCP-level connection (association). - */ static int -mptcp_disconnectx(struct mptses *mpte, sae_associd_t aid, sae_connid_t cid) +mptcp_disconnect(struct mptses *mpte) { - struct mptsub *mpts; struct socket *mp_so; struct mptcb *mp_tp; int error = 0; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + mpte_lock_assert_held(mpte); /* same as MP socket lock */ - mp_so = mpte->mpte_mppcb->mpp_socket; + mp_so = mptetoso(mpte); mp_tp = mpte->mpte_mptcb; mptcplog((LOG_DEBUG, "MPTCP Socket: " - "%s: mp_so 0x%llx aid %d cid %d %d\n", __func__, - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), aid, cid, mp_so->so_error), + "%s: mp_so 0x%llx %d\n", __func__, + (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), mp_so->so_error), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); - DTRACE_MPTCP5(disconnectx, struct mptses *, mpte, sae_associd_t, aid, - sae_connid_t, cid, struct socket *, mp_so, struct mptcb *, mp_tp); - - VERIFY(aid == SAE_ASSOCID_ANY || aid == SAE_ASSOCID_ALL || - aid == mpte->mpte_associd); - - /* terminate the association? */ - if (cid == SAE_CONNID_ANY || cid == SAE_CONNID_ALL) { - /* if we're not detached, go thru socket state checks */ - if (!(mp_so->so_flags & SOF_PCBCLEARING)) { - if (!(mp_so->so_state & (SS_ISCONNECTED| - SS_ISCONNECTING))) { - error = ENOTCONN; - goto out; - } - if (mp_so->so_state & SS_ISDISCONNECTING) { - error = EALREADY; - goto out; - } - } - MPT_LOCK(mp_tp); - mptcp_cancel_all_timers(mp_tp); - if (mp_tp->mpt_state < MPTCPS_ESTABLISHED) { - (void) mptcp_close(mpte, mp_tp); - MPT_UNLOCK(mp_tp); - } else if ((mp_so->so_options & SO_LINGER) && - mp_so->so_linger == 0) { - (void) mptcp_drop(mpte, mp_tp, 0); - MPT_UNLOCK(mp_tp); - } else { - MPT_UNLOCK(mp_tp); - soisdisconnecting(mp_so); - sbflush(&mp_so->so_rcv); - if (mptcp_usrclosed(mpte) != NULL) - (void) mptcp_output(mpte); - } - } else { - bool disconnect_embryonic_subflows = false; - struct socket *so = NULL; - - TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { - if (mpts->mpts_connid != cid) - continue; - - MPTS_LOCK(mpts); - /* - * Check if disconnected subflow is the one used - * to initiate MPTCP connection. - * If it is and the connection is not yet join ready - * disconnect all other subflows. - */ - so = mpts->mpts_socket; - if (!(mp_tp->mpt_flags & MPTCPF_JOIN_READY) && - so && !(so->so_flags & SOF_MP_SEC_SUBFLOW)) { - disconnect_embryonic_subflows = true; - } + DTRACE_MPTCP3(disconnectx, struct mptses *, mpte, + struct socket *, mp_so, struct mptcb *, mp_tp); - mpts->mpts_flags |= MPTSF_USER_DISCONNECT; - mptcp_subflow_disconnect(mpte, mpts, FALSE); - MPTS_UNLOCK(mpts); - break; - } - - if (mpts == NULL) { - error = EINVAL; + /* if we're not detached, go thru socket state checks */ + if (!(mp_so->so_flags & SOF_PCBCLEARING)) { + if (!(mp_so->so_state & (SS_ISCONNECTED| + SS_ISCONNECTING))) { + error = ENOTCONN; goto out; } - - if (disconnect_embryonic_subflows) { - TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { - if (mpts->mpts_connid == cid) - continue; - MPTS_LOCK(mpts); - mptcp_subflow_disconnect(mpte, mpts, TRUE); - MPTS_UNLOCK(mpts); - } + if (mp_so->so_state & SS_ISDISCONNECTING) { + error = EALREADY; + goto out; } } - if (error == 0) - mptcp_thread_signal(mpte); - - if ((mp_so->so_state & (SS_CANTRCVMORE | SS_CANTSENDMORE)) == - (SS_CANTRCVMORE | SS_CANTSENDMORE)) { - /* the socket has been shutdown, no more sockopt's */ - mptcp_flush_sopts(mpte); + mptcp_cancel_all_timers(mp_tp); + if (mp_tp->mpt_state < MPTCPS_ESTABLISHED) { + mptcp_close(mpte, mp_tp); + } else if ((mp_so->so_options & SO_LINGER) && + mp_so->so_linger == 0) { + mptcp_drop(mpte, mp_tp, 0); + } else { + soisdisconnecting(mp_so); + sbflush(&mp_so->so_rcv); + if (mptcp_usrclosed(mpte) != NULL) + mptcp_output(mpte); } + if (error == 0) + mptcp_subflow_workloop(mpte); + out: return (error); } /* - * Wrapper function to support disconnect on socket + * Wrapper function to support disconnect on socket */ static int mptcp_usr_disconnect(struct socket *mp_so) { - int error = 0; - - error = mptcp_usr_disconnectx(mp_so, SAE_ASSOCID_ALL, SAE_CONNID_ALL); - return (error); + return (mptcp_disconnect(mpsotompte(mp_so))); } /* @@ -923,27 +749,13 @@ mptcp_usr_disconnect(struct socket *mp_so) static int mptcp_usr_disconnectx(struct socket *mp_so, sae_associd_t aid, sae_connid_t cid) { - struct mppcb *mpp = sotomppcb(mp_so); - struct mptses *mpte; - int error = 0; - - if (mpp == NULL || mpp->mpp_state == MPPCB_STATE_DEAD) { - error = EINVAL; - goto out; - } - mpte = mptompte(mpp); - VERIFY(mpte != NULL); - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + if (aid != SAE_ASSOCID_ANY && aid != SAE_ASSOCID_ALL) + return (EINVAL); - if (aid != SAE_ASSOCID_ANY && aid != SAE_ASSOCID_ALL && - aid != mpte->mpte_associd) { - error = EINVAL; - goto out; - } + if (cid != SAE_CONNID_ANY && cid != SAE_CONNID_ALL) + return (EINVAL); - error = mptcp_disconnectx(mpte, aid, cid); -out: - return (error); + return (mptcp_usr_disconnect(mp_so)); } /* @@ -956,142 +768,29 @@ mptcp_usrclosed(struct mptses *mpte) struct mptcb *mp_tp; struct mptsub *mpts; - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - mp_so = mpte->mpte_mppcb->mpp_socket; + mpte_lock_assert_held(mpte); /* same as MP socket lock */ + mp_so = mptetoso(mpte); mp_tp = mpte->mpte_mptcb; - MPT_LOCK(mp_tp); mptcp_close_fsm(mp_tp, MPCE_CLOSE); if (mp_tp->mpt_state == MPTCPS_CLOSED) { mpte = mptcp_close(mpte, mp_tp); - MPT_UNLOCK(mp_tp); } else if (mp_tp->mpt_state >= MPTCPS_FIN_WAIT_2) { - MPT_UNLOCK(mp_tp); soisdisconnected(mp_so); - TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { - MPTS_LOCK(mpts); - mpts->mpts_flags |= MPTSF_USER_DISCONNECT; - MPTS_UNLOCK(mpts); - } } else { - MPT_UNLOCK(mp_tp); - TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { - MPTS_LOCK(mpts); - mpts->mpts_flags |= MPTSF_USER_DISCONNECT; - mptcp_subflow_disconnect(mpte, mpts, FALSE); - MPTS_UNLOCK(mpts); + if ((mp_so->so_state & (SS_CANTRCVMORE|SS_CANTSENDMORE)) == + (SS_CANTRCVMORE | SS_CANTSENDMORE)) + mptcp_subflow_disconnect(mpte, mpts); + else + mptcp_subflow_shutdown(mpte, mpts); } } return (mpte); } -/* - * User-protocol pru_peeloff callback. - */ -static int -mptcp_usr_peeloff(struct socket *mp_so, sae_associd_t aid, struct socket **psop) -{ - struct mppcb *mpp = sotomppcb(mp_so); - struct mptses *mpte; - int error = 0; - - VERIFY(psop != NULL); - - if (mpp == NULL || mpp->mpp_state == MPPCB_STATE_DEAD) { - error = EINVAL; - goto out; - } - mpte = mptompte(mpp); - VERIFY(mpte != NULL); - - error = mptcp_peeloff(mpte, aid, psop); -out: - return (error); -} - -/* - * Transform a previously connected TCP subflow connection which has - * failed to negotiate MPTCP to its own socket which can be externalized - * with a file descriptor. Valid only when the MPTCP socket is not - * yet associated (MPTCP-level connection has not been established.) - */ -static int -mptcp_peeloff(struct mptses *mpte, sae_associd_t aid, struct socket **psop) -{ - struct socket *so = NULL, *mp_so; - struct mptsub *mpts; - int error = 0; - - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - mp_so = mpte->mpte_mppcb->mpp_socket; - - VERIFY(psop != NULL); - *psop = NULL; - - DTRACE_MPTCP3(peeloff, struct mptses *, mpte, sae_associd_t, aid, - struct socket *, mp_so); - - /* peeloff cannot happen after an association is established */ - if (mpte->mpte_associd != SAE_ASSOCID_ANY) { - error = EINVAL; - goto out; - } - - if (aid != SAE_ASSOCID_ANY && aid != SAE_ASSOCID_ALL) { - error = EINVAL; - goto out; - } - - TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { - MPTS_LOCK(mpts); - if (mpts->mpts_flags & MPTSF_MP_CAPABLE) { - panic("%s: so %p is MPTCP capable but mp_so %p " - "aid is %d\n", __func__, so, mp_so, - mpte->mpte_associd); - /* NOTREACHED */ - } - MPTS_ADDREF_LOCKED(mpts); /* for us */ - so = mpts->mpts_socket; - VERIFY(so != NULL); - /* - * This subflow socket is about to be externalized; make it - * appear as if it has the same properties as the MPTCP socket, - * undo what's done earlier in mptcp_subflow_add(). - */ - mptcp_subflow_sopeeloff(mpte, mpts, so); - MPTS_UNLOCK(mpts); - - mptcp_subflow_del(mpte, mpts, FALSE); - MPTS_REMREF(mpts); /* ours */ - /* - * XXX adi@apple.com - * - * Here we need to make sure the subflow socket is not - * flow controlled; need to clear both INP_FLOW_CONTROLLED - * and INP_FLOW_SUSPENDED on the subflow socket, since - * we will no longer be monitoring its events. - */ - break; - } - - if (so == NULL) { - error = EINVAL; - goto out; - } - *psop = so; - - mptcplog((LOG_DEBUG, "MPTCP Socket: " - "%s: mp_so 0x%llx\n", __func__, - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so)), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); - -out: - return (error); -} - /* * After a receive, possible send some update to peer. */ @@ -1099,7 +798,7 @@ static int mptcp_usr_rcvd(struct socket *mp_so, int flags) { #pragma unused(flags) - struct mppcb *mpp = sotomppcb(mp_so); + struct mppcb *mpp = mpsotomppcb(mp_so); struct mptses *mpte; int error = 0; @@ -1123,7 +822,7 @@ mptcp_usr_send(struct socket *mp_so, int prus_flags, struct mbuf *m, struct sockaddr *nam, struct mbuf *control, struct proc *p) { #pragma unused(nam, p) - struct mppcb *mpp = sotomppcb(mp_so); + struct mppcb *mpp = mpsotomppcb(mp_so); struct mptses *mpte; int error = 0; @@ -1150,21 +849,16 @@ mptcp_usr_send(struct socket *mp_so, int prus_flags, struct mbuf *m, VERIFY(mpte != NULL); if (!(mp_so->so_state & SS_ISCONNECTED) && - (!(mp_so->so_flags1 & SOF1_PRECONNECT_DATA))) { + !(mp_so->so_flags1 & SOF1_PRECONNECT_DATA)) { error = ENOTCONN; goto out; } mptcp_insert_dsn(mpp, m); VERIFY(mp_so->so_snd.sb_flags & SB_NOCOMPRESS); - (void) sbappendstream(&mp_so->so_snd, m); + sbappendstream(&mp_so->so_snd, m); m = NULL; - /* - * XXX: adi@apple.com - * - * PRUS_MORETOCOME could be set, but we don't check it now. - */ error = mptcp_output(mpte); if (error != 0) goto out; @@ -1192,7 +886,7 @@ out: static int mptcp_usr_shutdown(struct socket *mp_so) { - struct mppcb *mpp = sotomppcb(mp_so); + struct mppcb *mpp = mpsotomppcb(mp_so); struct mptses *mpte; int error = 0; @@ -1426,9 +1120,7 @@ out: if (control != NULL) m_freem(control); - /* clear SOF1_PRECONNECT_DATA after one write */ - if (mp_so->so_flags1 & SOF1_PRECONNECT_DATA) - mp_so->so_flags1 &= ~SOF1_PRECONNECT_DATA; + soclearfastopen(mp_so); return (error); } @@ -1457,13 +1149,7 @@ mptcp_usr_socheckopt(struct socket *mp_so, struct sockopt *sopt) * * Need to consider the following cases: * - * a. In the event peeloff(2) occurs on the subflow socket, - * we may want to issue those options which are now - * handled at the MP socket. In that case, we will need - * to record them in mptcp_setopt() so that they can - * be replayed during peeloff. - * - * b. Certain socket options don't have a clear definition + * a. Certain socket options don't have a clear definition * on the expected behavior post connect(2). At the time * those options are issued on the MP socket, there may * be existing subflow sockets that are already connected. @@ -1489,6 +1175,12 @@ mptcp_usr_socheckopt(struct socket *mp_so, struct sockopt *sopt) case SO_DEFUNCTOK: /* MP */ case SO_ISDEFUNCT: /* MP */ case SO_TRAFFIC_CLASS_DBG: /* MP */ + case SO_DELEGATED: /* MP */ + case SO_DELEGATED_UUID: /* MP */ +#if NECP + case SO_NECP_ATTRIBUTES: + case SO_NECP_CLIENTUUID: +#endif /* NECP */ /* * Tell the caller that these options are to be processed. */ @@ -1504,9 +1196,9 @@ mptcp_usr_socheckopt(struct socket *mp_so, struct sockopt *sopt) case SO_RECV_ANYIF: /* MP + subflow */ case SO_RESTRICTIONS: /* MP + subflow */ case SO_FLUSH: /* MP + subflow */ - case SO_MPTCP_FASTJOIN: /* MP + subflow */ case SO_NOWAKEFROMSLEEP: case SO_NOAPNFALLBK: + case SO_MARK_CELLFALLBACK: /* * Tell the caller that these options are to be processed; * these will also be recorded later by mptcp_setopt(). @@ -1556,8 +1248,8 @@ mptcp_setopt_apply(struct mptses *mpte, struct mptopt *mpo) goto out; } - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - mp_so = mpte->mpte_mppcb->mpp_socket; + mpte_lock_assert_held(mpte); /* same as MP socket lock */ + mp_so = mptetoso(mpte); /* * Don't bother going further if there's no subflow; mark the option @@ -1581,33 +1273,25 @@ mptcp_setopt_apply(struct mptses *mpte, struct mptopt *mpo) TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { struct socket *so; - MPTS_LOCK(mpts); mpts->mpts_flags &= ~(MPTSF_SOPT_OLDVAL|MPTSF_SOPT_INPROG); mpts->mpts_oldintval = 0; smpo.mpo_intval = 0; VERIFY(mpts->mpts_socket != NULL); so = mpts->mpts_socket; - socket_lock(so, 0); if (mptcp_subflow_sogetopt(mpte, so, &smpo) == 0) { mpts->mpts_flags |= MPTSF_SOPT_OLDVAL; mpts->mpts_oldintval = smpo.mpo_intval; } - socket_unlock(so, 0); - MPTS_UNLOCK(mpts); } /* apply socket option */ TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { struct socket *so; - MPTS_LOCK(mpts); mpts->mpts_flags |= MPTSF_SOPT_INPROG; VERIFY(mpts->mpts_socket != NULL); so = mpts->mpts_socket; - socket_lock(so, 0); - error = mptcp_subflow_sosetopt(mpte, so, mpo); - socket_unlock(so, 0); - MPTS_UNLOCK(mpts); + error = mptcp_subflow_sosetopt(mpte, mpts, mpo); if (error != 0) break; } @@ -1616,32 +1300,26 @@ mptcp_setopt_apply(struct mptses *mpte, struct mptopt *mpo) TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) { struct socket *so; - MPTS_LOCK(mpts); if (!(mpts->mpts_flags & MPTSF_SOPT_INPROG)) { /* clear in case it's set */ mpts->mpts_flags &= ~MPTSF_SOPT_OLDVAL; mpts->mpts_oldintval = 0; - MPTS_UNLOCK(mpts); continue; } if (!(mpts->mpts_flags & MPTSF_SOPT_OLDVAL)) { mpts->mpts_flags &= ~MPTSF_SOPT_INPROG; VERIFY(mpts->mpts_oldintval == 0); - MPTS_UNLOCK(mpts); continue; } /* error during sosetopt, so roll it back */ if (error != 0) { VERIFY(mpts->mpts_socket != NULL); so = mpts->mpts_socket; - socket_lock(so, 0); smpo.mpo_intval = mpts->mpts_oldintval; - (void) mptcp_subflow_sosetopt(mpte, so, &smpo); - socket_unlock(so, 0); + mptcp_subflow_sosetopt(mpte, mpts, &smpo); } mpts->mpts_oldintval = 0; mpts->mpts_flags &= ~(MPTSF_SOPT_OLDVAL|MPTSF_SOPT_INPROG); - MPTS_UNLOCK(mpts); } out: @@ -1654,18 +1332,17 @@ out: static int mptcp_setopt(struct mptses *mpte, struct sockopt *sopt) { - int error = 0, optval, level, optname, rec = 1; + int error = 0, optval = 0, level, optname, rec = 1; struct mptopt smpo, *mpo = NULL; struct socket *mp_so; - char buf[32]; level = sopt->sopt_level; optname = sopt->sopt_name; VERIFY(sopt->sopt_dir == SOPT_SET); VERIFY(level == SOL_SOCKET || level == IPPROTO_TCP); - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ - mp_so = mpte->mpte_mppcb->mpp_socket; + mpte_lock_assert_held(mpte); /* same as MP socket lock */ + mp_so = mptetoso(mpte); /* * Record socket options which are applicable to subflow sockets so @@ -1684,16 +1361,46 @@ mptcp_setopt(struct mptses *mpte, struct sockopt *sopt) case SO_RECV_ANYIF: case SO_RESTRICTIONS: case SO_NOWAKEFROMSLEEP: - case SO_MPTCP_FASTJOIN: case SO_NOAPNFALLBK: + case SO_MARK_CELLFALLBACK: /* record it */ break; case SO_FLUSH: /* don't record it */ rec = 0; break; + + /* Next ones, record at MPTCP-level */ +#if NECP + case SO_NECP_CLIENTUUID: + if (!uuid_is_null(mpsotomppcb(mp_so)->necp_client_uuid)) { + error = EINVAL; + goto out; + } + + error = sooptcopyin(sopt, &mpsotomppcb(mp_so)->necp_client_uuid, + sizeof(uuid_t), sizeof(uuid_t)); + if (error != 0) { + goto out; + } + + mpsotomppcb(mp_so)->necp_cb = mptcp_session_necp_cb; + error = necp_client_register_multipath_cb(mp_so->last_pid, + mpsotomppcb(mp_so)->necp_client_uuid, + mpsotomppcb(mp_so)); + if (error) + goto out; + + if (uuid_is_null(mpsotomppcb(mp_so)->necp_client_uuid)) { + error = EINVAL; + goto out; + } + + goto out; + case SO_NECP_ATTRIBUTES: +#endif /* NECP */ default: - /* nothing to do; just return success */ + /* nothing to do; just return */ goto out; } } else { @@ -1706,6 +1413,8 @@ mptcp_setopt(struct mptses *mpte, struct sockopt *sopt) case TCP_CONNECTIONTIMEOUT: case TCP_RXT_CONNDROPTIME: case PERSIST_TIMEOUT: + case TCP_ADAPTIVE_READ_TIMEOUT: + case TCP_ADAPTIVE_WRITE_TIMEOUT: /* eligible; record it */ break; case TCP_NOTSENT_LOWAT: @@ -1727,6 +1436,27 @@ mptcp_setopt(struct mptses *mpte, struct sockopt *sopt) optval); } } + goto out; + case MPTCP_SERVICE_TYPE: + /* record at MPTCP level */ + error = sooptcopyin(sopt, &optval, sizeof(optval), + sizeof(optval)); + if (error) + goto out; + if (optval < 0 || optval >= MPTCP_SVCTYPE_MAX) { + error = EINVAL; + goto out; + } + + mpte->mpte_svctype = optval; + + if (mptcp_entitlement_check(mp_so) < 0) { + error = EACCES; + goto out; + } + + mpte->mpte_flags |= MPTE_SVCTYPE_CHECKED; + goto out; default: /* not eligible */ @@ -1747,12 +1477,9 @@ mptcp_setopt(struct mptses *mpte, struct sockopt *sopt) if (mpo == NULL) { error = ENOBUFS; } else { - mptcplog((LOG_DEBUG, "MPTCP Socket: " - "%s: mp_so 0x%llx sopt %s " - "val %d %s\n", __func__, - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), - mptcp_sopt2str(level, optname, buf, - sizeof (buf)), optval, + mptcplog((LOG_DEBUG, "%s: mp_so 0x%llx sopt %s val %d %s\n", + __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), + mptcp_sopt2str(level, optname), optval, (mpo->mpo_flags & MPOF_ATTACHED) ? "updated" : "recorded"), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); @@ -1791,19 +1518,16 @@ mptcp_setopt(struct mptses *mpte, struct sockopt *sopt) } out: if (error == 0 && mpo != NULL) { - mptcplog((LOG_ERR, "MPTCP Socket: " - "%s: mp_so 0x%llx sopt %s val %d set %s\n", + mptcplog((LOG_INFO, "%s: mp_so 0x%llx sopt %s val %d set %s\n", __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), - mptcp_sopt2str(level, optname, buf, - sizeof (buf)), optval, (mpo->mpo_flags & MPOF_INTERIM) ? + mptcp_sopt2str(level, optname), optval, + (mpo->mpo_flags & MPOF_INTERIM) ? "pending" : "successful"), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); } else if (error != 0) { - mptcplog((LOG_ERR, "MPTCP Socket: " - "%s: mp_so 0x%llx sopt %s can't be issued " - "error %d\n", __func__, - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), mptcp_sopt2str(level, - optname, buf, sizeof (buf)), error), + mptcplog((LOG_ERR, "%s: mp_so 0x%llx sopt %s (%d, %d) val %d can't be issued error %d\n", + __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), + mptcp_sopt2str(level, optname), level, optname, optval, error), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); } return (error); @@ -1818,7 +1542,7 @@ mptcp_getopt(struct mptses *mpte, struct sockopt *sopt) int error = 0, optval; VERIFY(sopt->sopt_dir == SOPT_GET); - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + mpte_lock_assert_held(mpte); /* same as MP socket lock */ /* * We only handle SOPT_GET for TCP level socket options; we should @@ -1839,7 +1563,10 @@ mptcp_getopt(struct mptses *mpte, struct sockopt *sopt) case TCP_CONNECTIONTIMEOUT: case TCP_RXT_CONNDROPTIME: case PERSIST_TIMEOUT: + case TCP_ADAPTIVE_READ_TIMEOUT: + case TCP_ADAPTIVE_WRITE_TIMEOUT: case TCP_NOTSENT_LOWAT: + case MPTCP_SERVICE_TYPE: /* eligible; get the default value just in case */ error = mptcp_default_tcp_optval(mpte, sopt, &optval); break; @@ -1851,11 +1578,14 @@ mptcp_getopt(struct mptses *mpte, struct sockopt *sopt) switch (sopt->sopt_name) { case TCP_NOTSENT_LOWAT: - if (mpte->mpte_mppcb->mpp_socket->so_flags & SOF_NOTSENT_LOWAT) + if (mptetoso(mpte)->so_flags & SOF_NOTSENT_LOWAT) optval = mptcp_get_notsent_lowat(mpte); else optval = 0; goto out; + case MPTCP_SERVICE_TYPE: + optval = mpte->mpte_svctype; + goto out; } /* @@ -1890,7 +1620,7 @@ mptcp_default_tcp_optval(struct mptses *mpte, struct sockopt *sopt, int *optval) VERIFY(sopt->sopt_level == IPPROTO_TCP); VERIFY(sopt->sopt_dir == SOPT_GET); - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + mpte_lock_assert_held(mpte); /* same as MP socket lock */ /* try to do what tcp_newtcpcb() does */ switch (sopt->sopt_name) { @@ -1901,6 +1631,9 @@ mptcp_default_tcp_optval(struct mptses *mpte, struct sockopt *sopt, int *optval) case TCP_CONNECTIONTIMEOUT: case TCP_RXT_CONNDROPTIME: case TCP_NOTSENT_LOWAT: + case TCP_ADAPTIVE_READ_TIMEOUT: + case TCP_ADAPTIVE_WRITE_TIMEOUT: + case MPTCP_SERVICE_TYPE: *optval = 0; break; @@ -1927,7 +1660,7 @@ mptcp_default_tcp_optval(struct mptses *mpte, struct sockopt *sopt, int *optval) int mptcp_ctloutput(struct socket *mp_so, struct sockopt *sopt) { - struct mppcb *mpp = sotomppcb(mp_so); + struct mppcb *mpp = mpsotomppcb(mp_so); struct mptses *mpte; int error = 0; @@ -1936,16 +1669,14 @@ mptcp_ctloutput(struct socket *mp_so, struct sockopt *sopt) goto out; } mpte = mptompte(mpp); - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + mpte_lock_assert_held(mpte); /* same as MP socket lock */ /* we only handle socket and TCP-level socket options for MPTCP */ if (sopt->sopt_level != SOL_SOCKET && sopt->sopt_level != IPPROTO_TCP) { - char buf[32]; mptcplog((LOG_DEBUG, "MPTCP Socket: " "%s: mp_so 0x%llx sopt %s level not " "handled\n", __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so), - mptcp_sopt2str(sopt->sopt_level, - sopt->sopt_name, buf, sizeof (buf))), + mptcp_sopt2str(sopt->sopt_level, sopt->sopt_name)), MPTCP_SOCKET_DBG, MPTCP_LOGLVL_LOG); error = EINVAL; goto out; @@ -1964,185 +1695,153 @@ out: return (error); } -/* - * Return a string representation of <sopt_level,sopt_name> - */ const char * -mptcp_sopt2str(int level, int optname, char *dst, int size) +mptcp_sopt2str(int level, int optname) { - char lbuf[32], obuf[32]; - const char *l = lbuf, *o = obuf; - - (void) snprintf(lbuf, sizeof (lbuf), "0x%x", level); - (void) snprintf(obuf, sizeof (obuf), "0x%x", optname); - switch (level) { case SOL_SOCKET: - l = "SOL_SOCKET"; switch (optname) { case SO_LINGER: - o = "SO_LINGER"; - break; + return ("SO_LINGER"); case SO_LINGER_SEC: - o = "SO_LINGER_SEC"; - break; + return ("SO_LINGER_SEC"); case SO_DEBUG: - o = "SO_DEBUG"; - break; + return ("SO_DEBUG"); case SO_KEEPALIVE: - o = "SO_KEEPALIVE"; - break; + return ("SO_KEEPALIVE"); case SO_USELOOPBACK: - o = "SO_USELOOPBACK"; - break; + return ("SO_USELOOPBACK"); case SO_TYPE: - o = "SO_TYPE"; - break; + return ("SO_TYPE"); case SO_NREAD: - o = "SO_NREAD"; - break; + return ("SO_NREAD"); case SO_NWRITE: - o = "SO_NWRITE"; - break; + return ("SO_NWRITE"); case SO_ERROR: - o = "SO_ERROR"; - break; + return ("SO_ERROR"); case SO_SNDBUF: - o = "SO_SNDBUF"; - break; + return ("SO_SNDBUF"); case SO_RCVBUF: - o = "SO_RCVBUF"; - break; + return ("SO_RCVBUF"); case SO_SNDLOWAT: - o = "SO_SNDLOWAT"; - break; + return ("SO_SNDLOWAT"); case SO_RCVLOWAT: - o = "SO_RCVLOWAT"; - break; + return ("SO_RCVLOWAT"); case SO_SNDTIMEO: - o = "SO_SNDTIMEO"; - break; + return ("SO_SNDTIMEO"); case SO_RCVTIMEO: - o = "SO_RCVTIMEO"; - break; + return ("SO_RCVTIMEO"); case SO_NKE: - o = "SO_NKE"; - break; + return ("SO_NKE"); case SO_NOSIGPIPE: - o = "SO_NOSIGPIPE"; - break; + return ("SO_NOSIGPIPE"); case SO_NOADDRERR: - o = "SO_NOADDRERR"; - break; + return ("SO_NOADDRERR"); case SO_RESTRICTIONS: - o = "SO_RESTRICTIONS"; - break; + return ("SO_RESTRICTIONS"); case SO_LABEL: - o = "SO_LABEL"; - break; + return ("SO_LABEL"); case SO_PEERLABEL: - o = "SO_PEERLABEL"; - break; + return ("SO_PEERLABEL"); case SO_RANDOMPORT: - o = "SO_RANDOMPORT"; - break; + return ("SO_RANDOMPORT"); case SO_TRAFFIC_CLASS: - o = "SO_TRAFFIC_CLASS"; - break; + return ("SO_TRAFFIC_CLASS"); case SO_RECV_TRAFFIC_CLASS: - o = "SO_RECV_TRAFFIC_CLASS"; - break; + return ("SO_RECV_TRAFFIC_CLASS"); case SO_TRAFFIC_CLASS_DBG: - o = "SO_TRAFFIC_CLASS_DBG"; - break; + return ("SO_TRAFFIC_CLASS_DBG"); case SO_PRIVILEGED_TRAFFIC_CLASS: - o = "SO_PRIVILEGED_TRAFFIC_CLASS"; - break; + return ("SO_PRIVILEGED_TRAFFIC_CLASS"); case SO_DEFUNCTOK: - o = "SO_DEFUNCTOK"; - break; + return ("SO_DEFUNCTOK"); case SO_ISDEFUNCT: - o = "SO_ISDEFUNCT"; - break; + return ("SO_ISDEFUNCT"); case SO_OPPORTUNISTIC: - o = "SO_OPPORTUNISTIC"; - break; + return ("SO_OPPORTUNISTIC"); case SO_FLUSH: - o = "SO_FLUSH"; - break; + return ("SO_FLUSH"); case SO_RECV_ANYIF: - o = "SO_RECV_ANYIF"; - break; + return ("SO_RECV_ANYIF"); case SO_NOWAKEFROMSLEEP: - o = "SO_NOWAKEFROMSLEEP"; - break; - case SO_MPTCP_FASTJOIN: - o = "SO_MPTCP_FASTJOIN"; - break; + return ("SO_NOWAKEFROMSLEEP"); case SO_NOAPNFALLBK: - o = "SO_NOAPNFALLBK"; - break; + return ("SO_NOAPNFALLBK"); + case SO_MARK_CELLFALLBACK: + return ("SO_CELLFALLBACK"); + case SO_DELEGATED: + return ("SO_DELEGATED"); + case SO_DELEGATED_UUID: + return ("SO_DELEGATED_UUID"); +#if NECP + case SO_NECP_ATTRIBUTES: + return ("SO_NECP_ATTRIBUTES"); + case SO_NECP_CLIENTUUID: + return ("SO_NECP_CLIENTUUID"); +#endif /* NECP */ } + break; case IPPROTO_TCP: - l = "IPPROTO_TCP"; switch (optname) { + case TCP_NODELAY: + return ("TCP_NODELAY"); case TCP_KEEPALIVE: - o = "TCP_KEEPALIVE"; - break; + return ("TCP_KEEPALIVE"); case TCP_KEEPINTVL: - o = "TCP_KEEPINTVL"; - break; + return ("TCP_KEEPINTVL"); case TCP_KEEPCNT: - o = "TCP_KEEPCNT"; - break; + return ("TCP_KEEPCNT"); case TCP_CONNECTIONTIMEOUT: - o = "TCP_CONNECTIONTIMEOUT"; - break; + return ("TCP_CONNECTIONTIMEOUT"); case TCP_RXT_CONNDROPTIME: - o = "TCP_RXT_CONNDROPTIME"; - break; + return ("TCP_RXT_CONNDROPTIME"); case PERSIST_TIMEOUT: - o = "PERSIST_TIMEOUT"; - break; + return ("PERSIST_TIMEOUT"); + case TCP_NOTSENT_LOWAT: + return ("NOTSENT_LOWAT"); + case TCP_ADAPTIVE_READ_TIMEOUT: + return ("ADAPTIVE_READ_TIMEOUT"); + case TCP_ADAPTIVE_WRITE_TIMEOUT: + return ("ADAPTIVE_WRITE_TIMEOUT"); + case MPTCP_SERVICE_TYPE: + return ("MPTCP_SERVICE_TYPE"); } + break; } - (void) snprintf(dst, size, "<%s,%s>", l, o); - return (dst); + return ("unknown"); } static int mptcp_usr_preconnect(struct socket *mp_so) { struct mptsub *mpts = NULL; - struct mppcb *mpp = sotomppcb(mp_so); + struct mppcb *mpp = mpsotomppcb(mp_so); struct mptses *mpte; struct socket *so; struct tcpcb *tp = NULL; + int error; mpte = mptompte(mpp); VERIFY(mpte != NULL); - MPTE_LOCK_ASSERT_HELD(mpte); /* same as MP socket lock */ + mpte_lock_assert_held(mpte); /* same as MP socket lock */ mpts = mptcp_get_subflow(mpte, NULL, NULL); if (mpts == NULL) { - mptcplog((LOG_ERR, "MPTCP Socket: " - "%s: mp_so 0x%llx invalid preconnect ", __func__, - (u_int64_t)VM_KERNEL_ADDRPERM(mp_so)), - MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); + mptcplog((LOG_ERR, "%s: mp_so 0x%llx invalid preconnect ", + __func__, (u_int64_t)VM_KERNEL_ADDRPERM(mp_so)), + MPTCP_SOCKET_DBG, MPTCP_LOGLVL_ERR); return (EINVAL); } - MPTS_LOCK(mpts); mpts->mpts_flags &= ~MPTSF_TFO_REQD; so = mpts->mpts_socket; - socket_lock(so, 0); tp = intotcpcb(sotoinpcb(so)); tp->t_mpflags &= ~TMPF_TFO_REQUEST; - int error = tcp_output(sototcpcb(so)); - socket_unlock(so, 0); - MPTS_UNLOCK(mpts); - mp_so->so_flags1 &= ~SOF1_PRECONNECT_DATA; + error = tcp_output(sototcpcb(so)); + + soclearfastopen(mp_so); + return (error); } diff --git a/bsd/netinet/mptcp_var.h b/bsd/netinet/mptcp_var.h index 46e47dc96..2d1f99061 100644 --- a/bsd/netinet/mptcp_var.h +++ b/bsd/netinet/mptcp_var.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2015 Apple Inc. All rights reserved. + * Copyright (c) 2012-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -40,13 +40,20 @@ #include <kern/locks.h> #include <mach/boolean.h> #include <netinet/mp_pcb.h> +#include <netinet/tcp_var.h> + +struct mpt_itf_info { + uint32_t ifindex; + uint32_t has_v4_conn:1, + has_v6_conn:1, + no_mptcp_support:1; +}; /* * MPTCP Session * * This is an extension to the multipath PCB specific for MPTCP, protected by - * the per-PCB mpp_lock (also the socket's lock); MPTCP thread signalling uses - * its own mpte_thread_lock due to lock ordering constraints. + * the per-PCB mpp_lock (also the socket's lock); */ struct mptses { struct mppcb *mpte_mppcb; /* back ptr to multipath PCB */ @@ -57,45 +64,170 @@ struct mptses { uint16_t mpte_nummpcapflows; /* # of MP_CAP subflows */ sae_associd_t mpte_associd; /* MPTCP association ID */ sae_connid_t mpte_connid_last; /* last used connection ID */ - /* - * Threading (protected by mpte_thread_lock) - */ - decl_lck_mtx_data(, mpte_thread_lock); /* thread lock */ - struct thread *mpte_thread; /* worker thread */ - uint32_t mpte_thread_active; /* thread is running */ - uint32_t mpte_thread_reqs; /* # of requests for thread */ + + union { + /* Source address of initial subflow */ + struct sockaddr mpte_src; + struct sockaddr_in __mpte_src_v4; + struct sockaddr_in6 __mpte_src_v6; + }; + + union { + /* Destination address of initial subflow */ + struct sockaddr mpte_dst; + struct sockaddr_in __mpte_dst_v4; + struct sockaddr_in6 __mpte_dst_v6; + }; + struct mptsub *mpte_active_sub; /* ptr to last active subf */ uint8_t mpte_flags; /* per mptcp session flags */ +#define MPTE_SND_REM_ADDR 0x01 /* Send Remove_addr option */ +#define MPTE_SVCTYPE_CHECKED 0x02 /* Did entitlement-check for service-type */ +#define MPTE_FIRSTPARTY 0x04 /* First-party app used multipath_extended entitlement */ +#define MPTE_ACCESS_GRANTED 0x08 /* Access to cellular has been granted for this connection */ + uint8_t mpte_svctype; /* MPTCP Service type */ uint8_t mpte_lost_aid; /* storing lost address id */ uint8_t mpte_addrid_last; /* storing address id parm */ + +#define MPTE_ITFINFO_SIZE 4 + uint32_t mpte_itfinfo_size; + struct mpt_itf_info _mpte_itfinfo[MPTE_ITFINFO_SIZE]; + struct mpt_itf_info *mpte_itfinfo; + + struct mbuf *mpte_reinjectq; + + /* The below is used for stats */ + uint32_t mpte_subflow_switches; /* Number of subflow-switches in sending */ + uint32_t mpte_used_cell:1, + mpte_used_wifi:1, + mpte_initial_cell:1, + mpte_handshake_success:1; + + struct mptcp_itf_stats mpte_itfstats[MPTCP_ITFSTATS_SIZE]; + uint64_t mpte_init_txbytes __attribute__((aligned(8))); + uint64_t mpte_init_rxbytes __attribute__((aligned(8))); }; -/* - * Valid values for mpte_flags. - */ -#define MPTE_SND_REM_ADDR 0x01 /* Send Remove_addr option */ +static inline struct socket * +mptetoso(struct mptses *mpte) +{ + return (mpte->mpte_mppcb->mpp_socket); +} + +static inline struct mptses * +mptompte(struct mppcb *mp) +{ + return ((struct mptses *)mp->mpp_pcbe); +} -#define mptompte(mp) ((struct mptses *)(mp)->mpp_pcbe) +static inline struct mptses * +mpsotompte(struct socket *so) +{ + return (mptompte(mpsotomppcb(so))); +} -#define MPTE_LOCK_ASSERT_HELD(_mpte) \ - lck_mtx_assert(&(_mpte)->mpte_mppcb->mpp_lock, LCK_MTX_ASSERT_OWNED) +static inline void +mpp_lock_assert_held(struct mppcb *mp) +{ +#if !MACH_ASSERT +#pragma unused(mp) +#endif + LCK_MTX_ASSERT(&mp->mpp_lock, LCK_MTX_ASSERT_OWNED); +} -#define MPTE_LOCK_ASSERT_NOTHELD(_mpte) \ - lck_mtx_assert(&(_mpte)->mpte_mppcb->mpp_lock, LCK_MTX_ASSERT_NOTOWNED) +static inline void +mpp_lock_assert_notheld(struct mppcb *mp) +{ +#if !MACH_ASSERT +#pragma unused(mp) +#endif + LCK_MTX_ASSERT(&mp->mpp_lock, LCK_MTX_ASSERT_NOTOWNED); +} -#define MPTE_LOCK(_mpte) \ - lck_mtx_lock(&(_mpte)->mpte_mppcb->mpp_lock) +static inline boolean_t +mpp_try_lock(struct mppcb *mp) +{ + if (!lck_mtx_try_lock(&mp->mpp_lock)) + return false; -#define MPTE_LOCK_SPIN(_mpte) \ - lck_mtx_lock_spin(&(_mpte)->mpte_mppcb->mpp_lock) + VERIFY(!(mp->mpp_flags & MPP_INSIDE_OUTPUT)); + VERIFY(!(mp->mpp_flags & MPP_INSIDE_INPUT)); -#define MPTE_CONVERT_LOCK(_mpte) do { \ - MPTE_LOCK_ASSERT_HELD(_mpte); \ - lck_mtx_convert_spin(&(_mpte)->mpte_mppcb->mpp_lock); \ -} while (0) + return true; +} + +static inline void +mpp_lock(struct mppcb *mp) +{ + lck_mtx_lock(&mp->mpp_lock); + VERIFY(!(mp->mpp_flags & MPP_INSIDE_OUTPUT)); + VERIFY(!(mp->mpp_flags & MPP_INSIDE_INPUT)); +} + +static inline void +mpp_unlock(struct mppcb *mp) +{ + VERIFY(!(mp->mpp_flags & MPP_INSIDE_OUTPUT)); + VERIFY(!(mp->mpp_flags & MPP_INSIDE_INPUT)); + lck_mtx_unlock(&mp->mpp_lock); +} + +static inline lck_mtx_t * +mpp_getlock(struct mppcb *mp, int flags) +{ + if (flags & PR_F_WILLUNLOCK) { + VERIFY(!(mp->mpp_flags & MPP_INSIDE_OUTPUT)); + VERIFY(!(mp->mpp_flags & MPP_INSIDE_INPUT)); + } + + return (&mp->mpp_lock); +} + +static inline void +mpte_lock_assert_held(struct mptses *mpte) +{ + mpp_lock_assert_held(mpte->mpte_mppcb); +} + +static inline void +mpte_lock_assert_notheld(struct mptses *mpte) +{ + mpp_lock_assert_notheld(mpte->mpte_mppcb); +} + +static inline boolean_t +mpte_try_lock(struct mptses *mpte) +{ + return (mpp_try_lock(mpte->mpte_mppcb)); +} + +static inline void +mpte_lock(struct mptses *mpte) +{ + mpp_lock(mpte->mpte_mppcb); +} + +static inline void +mpte_unlock(struct mptses *mpte) +{ + mpp_unlock(mpte->mpte_mppcb); +} + +static inline lck_mtx_t * +mpte_getlock(struct mptses *mpte, int flags) +{ + return mpp_getlock(mpte->mpte_mppcb, flags); +} + +static inline int +mptcp_subflow_cwnd_space(struct socket *so) +{ + struct tcpcb *tp = sototcpcb(so); + int cwnd = min(tp->snd_wnd, tp->snd_cwnd) - (tp->snd_nxt - tp->snd_una); + + return (min(cwnd, sbspace(&so->so_snd))); +} -#define MPTE_UNLOCK(_mpte) \ - lck_mtx_unlock(&(_mpte)->mpte_mppcb->mpp_lock) /* * MPTCP socket options @@ -112,56 +244,35 @@ struct mptopt { #define MPOF_SUBFLOW_OK 0x2 /* can be issued on subflow socket */ #define MPOF_INTERIM 0x4 /* has not been issued on any subflow */ -/* - * Structure passed down to TCP during subflow connection establishment - * containing information pertaining to the MPTCP. - */ -struct mptsub_connreq { - uint32_t mpcr_type; /* see MPTSUB_CONNREQ_* below */ - uint32_t mpcr_ifscope; /* ifscope parameter to connectx(2) */ - struct proc *mpcr_proc; /* process issuing connectx(2) */ -}; - -/* valid values for mpcr_type */ -#define MPTSUB_CONNREQ_MP_ENABLE 1 /* enable MPTCP */ -#define MPTSUB_CONNREQ_MP_ADD 2 /* join an existing MPTCP */ - /* * MPTCP subflow * - * Protected by the the per-subflow mpts_lock. Note that mpts_flags - * and mpts_evctl are modified via atomic operations. + * Note that mpts_flags and mpts_evctl are modified via atomic operations. */ struct mptsub { - decl_lck_mtx_data(, mpts_lock); /* per-subflow lock */ TAILQ_ENTRY(mptsub) mpts_entry; /* glue to peer subflows */ uint32_t mpts_refcnt; /* reference count */ uint32_t mpts_flags; /* see flags below */ uint32_t mpts_evctl; /* subflow control events */ - uint32_t mpts_family; /* address family */ sae_connid_t mpts_connid; /* subflow connection ID */ int mpts_oldintval; /* sopt_val before sosetopt */ - uint32_t mpts_rank; /* subflow priority/rank */ - int32_t mpts_soerror; /* most recent subflow error */ struct mptses *mpts_mpte; /* back ptr to MPTCP session */ struct socket *mpts_socket; /* subflow socket */ struct sockaddr *mpts_src; /* source address */ - struct sockaddr *mpts_dst; /* destination address */ - struct ifnet *mpts_outif; /* outbound interface */ - u_int64_t mpts_sndnxt; /* next byte to send in mp so */ + + union { + /* destination address */ + struct sockaddr mpts_dst; + struct sockaddr_in __mpts_dst_v4; + struct sockaddr_in6 __mpts_dst_v6; + }; + u_int32_t mpts_rel_seq; /* running count of subflow # */ - struct protosw *mpts_oprotosw; /* original protosw */ - struct mptsub_connreq mpts_mpcr; /* connection request */ - int32_t mpts_srtt; /* tcp's rtt estimate */ - int32_t mpts_rxtcur; /* tcp's rto estimate */ + u_int32_t mpts_iss; /* Initial sequence number, taking TFO into account */ + u_int32_t mpts_ifscope; /* scoped to the interface */ uint32_t mpts_probesoon; /* send probe after probeto */ uint32_t mpts_probecnt; /* number of probes sent */ uint32_t mpts_maxseg; /* cached value of t_maxseg */ - uint32_t mpts_peerswitch;/* no of uses of backup so */ -#define MPTSL_WIRED 0x01 -#define MPTSL_WIFI 0x02 -#define MPTSL_CELL 0x04 - uint32_t mpts_linktype; /* wired, wifi, cell */ }; /* @@ -193,58 +304,36 @@ struct mptsub { * * Keep in sync with bsd/dev/dtrace/scripts/mptcp.d. */ -#define MPTSF_ATTACHED 0x1 /* attached to MPTCP PCB */ -#define MPTSF_CONNECTING 0x2 /* connection was attempted */ -#define MPTSF_CONNECT_PENDING 0x4 /* will connect when MPTCP is ready */ -#define MPTSF_CONNECTED 0x8 /* connection is established */ -#define MPTSF_DISCONNECTING 0x10 /* disconnection was attempted */ -#define MPTSF_DISCONNECTED 0x20 /* has been disconnected */ -#define MPTSF_MP_CAPABLE 0x40 /* connected as a MPTCP subflow */ -#define MPTSF_MP_READY 0x80 /* MPTCP has been confirmed */ -#define MPTSF_MP_DEGRADED 0x100 /* has lost its MPTCP capabilities */ -#define MPTSF_SUSPENDED 0x200 /* write-side is flow controlled */ -#define MPTSF_BOUND_IF 0x400 /* subflow bound to an interface */ -#define MPTSF_BOUND_IP 0x800 /* subflow bound to a src address */ -#define MPTSF_BOUND_PORT 0x1000 /* subflow bound to a src port */ -#define MPTSF_PREFERRED 0x2000 /* primary/preferred subflow */ -#define MPTSF_SOPT_OLDVAL 0x4000 /* old option value is valid */ -#define MPTSF_SOPT_INPROG 0x8000 /* sosetopt in progress */ -#define MPTSF_DELETEOK 0x10000 /* subflow can be deleted */ -#define MPTSF_FAILINGOVER 0x20000 /* subflow not used for output */ -#define MPTSF_ACTIVE 0x40000 /* subflow currently in use */ -#define MPTSF_MPCAP_CTRSET 0x80000 /* mpcap counter */ -#define MPTSF_FASTJ_SEND 0x100000 /* send data after SYN in MP_JOIN */ -#define MPTSF_FASTJ_REQD 0x200000 /* fastjoin required */ -#define MPTSF_USER_DISCONNECT 0x400000 /* User triggered disconnect */ -#define MPTSF_TFO_REQD 0x800000 /* TFO requested */ +#define MPTSF_ATTACHED 0x00000001 /* attached to MPTCP PCB */ +#define MPTSF_CONNECTING 0x00000002 /* connection was attempted */ +#define MPTSF_CONNECT_PENDING 0x00000004 /* will connect when MPTCP is ready */ +#define MPTSF_CONNECTED 0x00000008 /* connection is established */ +#define MPTSF_DISCONNECTING 0x00000010 /* disconnection was attempted */ +#define MPTSF_DISCONNECTED 0x00000020 /* has been disconnected */ +#define MPTSF_MP_CAPABLE 0x00000040 /* connected as a MPTCP subflow */ +#define MPTSF_MP_READY 0x00000080 /* MPTCP has been confirmed */ +#define MPTSF_MP_DEGRADED 0x00000100 /* has lost its MPTCP capabilities */ +#define MPTSF_PREFERRED 0x00000200 /* primary/preferred subflow */ +#define MPTSF_SOPT_OLDVAL 0x00000400 /* old option value is valid */ +#define MPTSF_SOPT_INPROG 0x00000800 /* sosetopt in progress */ +#define MPTSF_FAILINGOVER 0x00001000 /* subflow not used for output */ +#define MPTSF_ACTIVE 0x00002000 /* subflow currently in use */ +#define MPTSF_MPCAP_CTRSET 0x00004000 /* mpcap counter */ +#define MPTSF_CLOSED 0x00008000 /* soclose_locked has been called on this subflow */ +#define MPTSF_TFO_REQD 0x00010000 /* TFO requested */ +#define MPTSF_CLOSE_REQD 0x00020000 /* A close has been requested from NECP */ +#define MPTSF_INITIAL_SUB 0x00040000 /* This is the initial subflow */ +#define MPTSF_READ_STALL 0x00080000 /* A read-stall has been detected */ +#define MPTSF_WRITE_STALL 0x00100000 /* A write-stall has been detected */ +#define MPTSF_CONFIRMED 0x00200000 /* Subflow confirmed to be MPTCP-capable */ #define MPTSF_BITS \ "\020\1ATTACHED\2CONNECTING\3PENDING\4CONNECTED\5DISCONNECTING" \ - "\6DISCONNECTED\7MP_CAPABLE\10MP_READY\11MP_DEGRADED\12SUSPENDED" \ - "\13BOUND_IF\14BOUND_IP\15BOUND_PORT\16PREFERRED\17SOPT_OLDVAL" \ - "\20SOPT_INPROG\21NOLINGER\22FAILINGOVER\23ACTIVE\24MPCAP_CTRSET" \ - "\25FASTJ_SEND\26FASTJ_REQD\27USER_DISCONNECT" - -#define MPTS_LOCK_ASSERT_HELD(_mpts) \ - lck_mtx_assert(&(_mpts)->mpts_lock, LCK_MTX_ASSERT_OWNED) - -#define MPTS_LOCK_ASSERT_NOTHELD(_mpts) \ - lck_mtx_assert(&(_mpts)->mpts_lock, LCK_MTX_ASSERT_NOTOWNED) - -#define MPTS_LOCK(_mpts) \ - lck_mtx_lock(&(_mpts)->mpts_lock) - -#define MPTS_UNLOCK(_mpts) \ - lck_mtx_unlock(&(_mpts)->mpts_lock) - -#define MPTS_ADDREF(_mpts) \ - mptcp_subflow_addref(_mpts, 0) - -#define MPTS_ADDREF_LOCKED(_mpts) \ - mptcp_subflow_addref(_mpts, 1) - -#define MPTS_REMREF(_mpts) \ - mptcp_subflow_remref(_mpts) + "\6DISCONNECTED\7MP_CAPABLE\10MP_READY\11MP_DEGRADED" \ + "\12PREFERRED\13SOPT_OLDVAL" \ + "\14SOPT_INPROG\15FAILINGOVER\16ACTIVE\17MPCAP_CTRSET" \ + "\20CLOSED\21TFO_REQD\22CLOSEREQD\23INITIALSUB\24READ_STALL" \ + "\25WRITE_STALL\26CONFIRMED" /* * MPTCP states @@ -284,17 +373,15 @@ struct mptcp_subf_auth_entry { * Keep in sync with bsd/dev/dtrace/scripts/mptcp.d. */ struct mptcb { - decl_lck_mtx_data(, mpt_lock); /* per MPTCP PCB lock */ struct mptses *mpt_mpte; /* back ptr to MPTCP session */ mptcp_state_t mpt_state; /* MPTCP state */ u_int32_t mpt_flags; /* see flags below */ - u_int32_t mpt_refcnt; /* references held on mptcb */ u_int32_t mpt_version; /* MPTCP proto version */ int mpt_softerror; /* error not yet reported */ /* * Authentication and metadata invariants */ - mptcp_key_t *mpt_localkey; /* in network byte order */ + mptcp_key_t mpt_localkey; /* in network byte order */ mptcp_key_t mpt_remotekey; /* in network byte order */ mptcp_token_t mpt_localtoken; /* HMAC SHA1 of local key */ mptcp_token_t mpt_remotetoken; /* HMAC SHA1 of remote key */ @@ -316,11 +403,12 @@ struct mptcb { u_int64_t mpt_sndmax; /* DSN of max byte sent */ u_int64_t mpt_local_idsn; /* First byte's DSN */ u_int32_t mpt_sndwnd; + u_int64_t mpt_sndwl1; + u_int64_t mpt_sndwl2; /* * Receiving side */ u_int64_t mpt_rcvnxt; /* Next expected DSN */ - u_int64_t mpt_rcvatmark; /* mpsocket marker of rcvnxt */ u_int64_t mpt_remote_idsn; /* Peer's IDSN */ u_int32_t mpt_rcvwnd; LIST_HEAD(, mptcp_subf_auth_entry) mpt_subauth_list; /* address IDs */ @@ -338,51 +426,33 @@ struct mptcb { u_int32_t mpt_notsent_lowat; /* TCP_NOTSENT_LOWAT support */ u_int32_t mpt_peer_version; /* Version from peer */ + + struct tsegqe_head mpt_segq; + u_int16_t mpt_reassqlen; /* length of reassembly queue */ }; /* valid values for mpt_flags (see also notes on mpts_flags above) */ -#define MPTCPF_CHECKSUM 0x1 /* checksum DSS option */ -#define MPTCPF_FALLBACK_TO_TCP 0x2 /* Fallback to TCP */ -#define MPTCPF_JOIN_READY 0x4 /* Ready to start 2 or more subflows */ -#define MPTCPF_RECVD_MPFAIL 0x8 /* Received MP_FAIL option */ -#define MPTCPF_PEEL_OFF 0x10 /* Peel off this socket */ -#define MPTCPF_SND_64BITDSN 0x20 /* Send full 64-bit DSN */ -#define MPTCPF_SND_64BITACK 0x40 /* Send 64-bit ACK response */ -#define MPTCPF_RCVD_64BITACK 0x80 /* Received 64-bit Data ACK */ -#define MPTCPF_POST_FALLBACK_SYNC 0x100 /* Post fallback resend data */ -#define MPTCPF_FALLBACK_HEURISTIC 0x200 /* Send SYN without MP_CAPABLE due to heuristic */ -#define MPTCPF_HEURISTIC_TRAC 0x400 /* Tracked this connection in the heuristics as a failure */ +#define MPTCPF_CHECKSUM 0x001 /* checksum DSS option */ +#define MPTCPF_FALLBACK_TO_TCP 0x002 /* Fallback to TCP */ +#define MPTCPF_JOIN_READY 0x004 /* Ready to start 2 or more subflows */ +#define MPTCPF_RECVD_MPFAIL 0x008 /* Received MP_FAIL option */ +#define MPTCPF_SND_64BITDSN 0x010 /* Send full 64-bit DSN */ +#define MPTCPF_SND_64BITACK 0x020 /* Send 64-bit ACK response */ +#define MPTCPF_RCVD_64BITACK 0x040 /* Received 64-bit Data ACK */ +#define MPTCPF_POST_FALLBACK_SYNC 0x080 /* Post fallback resend data */ +#define MPTCPF_FALLBACK_HEURISTIC 0x100 /* Send SYN without MP_CAPABLE due to heuristic */ +#define MPTCPF_HEURISTIC_TRAC 0x200 /* Tracked this connection in the heuristics as a failure */ +#define MPTCPF_REASS_INPROG 0x400 /* Reassembly is in progress */ #define MPTCPF_BITS \ - "\020\1CHECKSUM\2FALLBACK_TO_TCP\3JOIN_READY\4RECVD_MPFAIL\5PEEL_OFF" \ - "\6SND_64BITDSN\7SND_64BITACK\10RCVD_64BITACK\11POST_FALLBACK_SYNC" \ - "\12FALLBACK_HEURISTIC\13HEURISTIC_TRAC" + "\020\1CHECKSUM\2FALLBACK_TO_TCP\3JOIN_READY\4RECVD_MPFAIL" \ + "\5SND_64BITDSN\6SND_64BITACK\7RCVD_64BITACK\10POST_FALLBACK_SYNC" \ + "\11FALLBACK_HEURISTIC\12HEURISTIC_TRAC\13REASS_INPROG" /* valid values for mpt_timer_vals */ #define MPTT_REXMT 0x01 /* Starting Retransmit Timer */ #define MPTT_TW 0x02 /* Starting Timewait Timer */ #define MPTT_FASTCLOSE 0x04 /* Starting Fastclose wait timer */ -//#define MPTT_PROBE_TIMER 0x08 /* Timer for probing preferred path */ - -#define MPT_LOCK_ASSERT_HELD(_mpt) \ - lck_mtx_assert(&(_mpt)->mpt_lock, LCK_MTX_ASSERT_OWNED) - -#define MPT_LOCK_ASSERT_NOTHELD(_mpt) \ - lck_mtx_assert(&(_mpt)->mpt_lock, LCK_MTX_ASSERT_NOTOWNED) - -#define MPT_LOCK(_mpt) \ - lck_mtx_lock(&(_mpt)->mpt_lock) - -#define MPT_LOCK_SPIN(_mpt) \ - lck_mtx_lock_spin(&(_mpt)->mpt_lock) - -#define MPT_CONVERT_LOCK(_mpt) do { \ - MPT_LOCK_ASSERT_HELD(_mpt); \ - lck_mtx_convert_spin(&(_mpt)->mpt_lock); \ -} while (0) - -#define MPT_UNLOCK(_mpt) \ - lck_mtx_unlock(&(_mpt)->mpt_lock) /* events for close FSM */ #define MPCE_CLOSE 0x1 @@ -390,7 +460,10 @@ struct mptcb { #define MPCE_RECV_DATA_FIN 0x4 /* mptcb manipulation */ -#define tptomptp(tp) ((struct mptcb *)((tp)->t_mptcb)) +static inline struct mptcb *tptomptp(struct tcpcb *tp) +{ + return (tp->t_mptcb); +} /* * MPTCP control block and state structures are allocated along with @@ -410,41 +483,13 @@ extern struct mppcbinfo mtcbinfo; extern struct pr_usrreqs mptcp_usrreqs; /* Encryption algorithm related definitions */ -#define MPTCP_SHA1_RESULTLEN 20 #define SHA1_TRUNCATED 8 -/* List of valid keys to use for MPTCP connections */ -#define MPTCP_KEY_DIGEST_LEN (MPTCP_SHA1_RESULTLEN) -#define MPTCP_MX_KEY_ALLOCS (256) -#define MPTCP_KEY_PREALLOCS_MX (16) -#define MPTCP_MX_PREALLOC_ZONE_SZ (8192) - -struct mptcp_key_entry { - LIST_ENTRY(mptcp_key_entry) mkey_next; - mptcp_key_t mkey_value; -#define MKEYF_FREE 0x0 -#define MKEYF_INUSE 0x1 - u_int32_t mkey_flags; - char mkey_digest[MPTCP_KEY_DIGEST_LEN]; -}; - -/* structure for managing unique key list */ -struct mptcp_keys_pool_head { - struct mptcp_key_entry *lh_first; /* list of keys */ - u_int32_t mkph_count; /* total keys in pool */ - vm_size_t mkph_key_elm_sz; /* size of key entry */ - struct zone *mkph_key_entry_zone; /* zone for key entry */ - decl_lck_mtx_data(, mkph_lock); /* lock for key list */ -}; - -/* MPTCP Receive Window */ -#define MPTCP_RWIN_MAX (1<<16) - /* MPTCP Debugging Levels */ #define MPTCP_LOGLVL_NONE 0x0 /* No debug logging */ #define MPTCP_LOGLVL_ERR 0x1 /* Errors in execution are logged */ #define MPTCP_LOGLVL_LOG 0x2 /* Important logs */ -#define MPTCP_LOGLVL_VERBOSE 0x3 /* Verbose logs */ +#define MPTCP_LOGLVL_VERBOSE 0x4 /* Verbose logs */ /* MPTCP sub-components for debug logging */ #define MPTCP_NO_DBG 0x00 /* No areas are logged */ @@ -453,8 +498,6 @@ struct mptcp_keys_pool_head { #define MPTCP_SENDER_DBG 0x04 /* Sender side logging */ #define MPTCP_RECEIVER_DBG 0x08 /* Receiver logging */ #define MPTCP_EVENTS_DBG 0x10 /* Subflow events logging */ -#define MPTCP_ALL_DBG (MPTCP_STATE_DBG | MPTCP_SOCKET_DBG | \ - MPTCP_SENDER_DBG | MPTCP_RECEIVER_DBG | MPTCP_EVENTS_DBG) /* Mask to obtain 32-bit portion of data sequence number */ #define MPTCP_DATASEQ_LOW32_MASK (0xffffffff) @@ -510,8 +553,7 @@ struct mptcp_keys_pool_head { } #define mptcplog(x, y, z) do { \ - if ((mptcp_dbg_area & y) && \ - (mptcp_dbg_level >= z)) \ + if ((mptcp_dbg_area & y) && (mptcp_dbg_level & z)) \ log x; \ } while (0) @@ -521,54 +563,47 @@ extern int mptcp_join_retries; /* Multipath TCP Join retries */ extern int mptcp_dss_csum; /* Multipath DSS Option checksum */ extern int mptcp_fail_thresh; /* Multipath failover thresh of retransmits */ extern int mptcp_subflow_keeptime; /* Multipath subflow TCP_KEEPALIVE opt */ -extern int mptcp_mpprio_enable; /* MP_PRIO option enable/disable */ -extern int mptcp_remaddr_enable;/* REMOVE_ADDR option enable/disable */ -extern int mptcp_fastjoin; /* Enable FastJoin */ -extern int mptcp_zerortt_fastjoin; /* Enable Data after SYN Fast Join */ -extern int mptcp_rwnotify; /* Enable RW notification on resume */ extern uint32_t mptcp_dbg_level; /* Multipath TCP debugging level */ extern uint32_t mptcp_dbg_area; /* Multipath TCP debugging area */ +extern int mptcp_developer_mode; /* Allow aggregation mode */ -#define MPPCB_LIMIT 32 -extern uint32_t mptcp_socket_limit; /* max number of mptcp sockets allowed */ -extern uint32_t mptcp_delayed_subf_start; /* delayed cellular subflow start */ extern int tcp_jack_rxmt; /* Join ACK retransmission value in msecs */ __BEGIN_DECLS extern void mptcp_init(struct protosw *, struct domain *); extern int mptcp_ctloutput(struct socket *, struct sockopt *); -extern void *mptcp_sescreate(struct socket *, struct mppcb *); -extern void mptcp_drain(void); +extern int mptcp_sescreate(struct mppcb *); +extern void mptcp_check_subflows_and_add(struct mptses *); +extern int mptcp_get_statsindex(struct mptcp_itf_stats *stats, + const struct mptsub *mpts); +extern void mptcpstats_inc_switch(struct mptses *, const struct mptsub *); extern struct mptses *mptcp_drop(struct mptses *, struct mptcb *, int); extern struct mptses *mptcp_close(struct mptses *, struct mptcb *); extern int mptcp_lock(struct socket *, int, void *); extern int mptcp_unlock(struct socket *, int, void *); extern lck_mtx_t *mptcp_getlock(struct socket *, int); -extern void mptcp_thread_signal(struct mptses *); -extern void mptcp_flush_sopts(struct mptses *); -extern int mptcp_setconnorder(struct mptses *, sae_connid_t, uint32_t); -extern int mptcp_getconnorder(struct mptses *, sae_connid_t, uint32_t *); +extern void mptcp_subflow_workloop(struct mptses *); + +extern void mptcp_sched_create_subflows(struct mptses *); extern struct mptopt *mptcp_sopt_alloc(int); -extern const char *mptcp_sopt2str(int, int, char *, int); +extern const char *mptcp_sopt2str(int, int); extern void mptcp_sopt_free(struct mptopt *); extern void mptcp_sopt_insert(struct mptses *, struct mptopt *); extern void mptcp_sopt_remove(struct mptses *, struct mptopt *); extern struct mptopt *mptcp_sopt_find(struct mptses *, struct sockopt *); -extern struct mptsub *mptcp_subflow_alloc(int); -extern void mptcp_subflow_free(struct mptsub *); -extern void mptcp_subflow_addref(struct mptsub *, int); -extern int mptcp_subflow_add(struct mptses *, struct mptsub *, - struct proc *, uint32_t); -extern void mptcp_subflow_del(struct mptses *, struct mptsub *, boolean_t); -extern void mptcp_subflow_remref(struct mptsub *); -extern int mptcp_subflow_output(struct mptses *, struct mptsub *); -extern void mptcp_subflow_disconnect(struct mptses *, struct mptsub *, - boolean_t); -extern void mptcp_subflow_sopeeloff(struct mptses *, struct mptsub *, - struct socket *); -extern int mptcp_subflow_sosetopt(struct mptses *, struct socket *, +extern int mptcp_subflow_add(struct mptses *, struct sockaddr *, + struct sockaddr *, uint32_t, sae_connid_t *); +extern void mptcpstats_update(struct mptcp_itf_stats *stats, struct mptsub *mpts); +extern void mptcp_subflow_del(struct mptses *, struct mptsub *); + +#define MPTCP_SUBOUT_PROBING 0x01 +extern int mptcp_subflow_output(struct mptses *mpte, struct mptsub *mpts, int flags); +extern void mptcp_clean_reinjectq(struct mptses *mpte); +extern void mptcp_subflow_shutdown(struct mptses *, struct mptsub *); +extern void mptcp_subflow_disconnect(struct mptses *, struct mptsub *); +extern int mptcp_subflow_sosetopt(struct mptses *, struct mptsub *, struct mptopt *); extern int mptcp_subflow_sogetopt(struct mptses *, struct socket *, struct mptopt *); @@ -577,40 +612,28 @@ extern void mptcp_input(struct mptses *, struct mbuf *); extern int mptcp_output(struct mptses *); extern void mptcp_close_fsm(struct mptcb *, uint32_t); -extern mptcp_token_t mptcp_get_localtoken(void *); -extern mptcp_token_t mptcp_get_remotetoken(void *); - -extern u_int64_t mptcp_get_localkey(void *); -extern u_int64_t mptcp_get_remotekey(void *); - -extern void mptcp_free_key(mptcp_key_t *key); extern void mptcp_hmac_sha1(mptcp_key_t, mptcp_key_t, u_int32_t, u_int32_t, - u_char*, int); -extern void mptcp_get_hmac(mptcp_addr_id, struct mptcb *, u_char *, int); + u_char*); +extern void mptcp_get_hmac(mptcp_addr_id, struct mptcb *, u_char *); extern void mptcp_get_rands(mptcp_addr_id, struct mptcb *, u_int32_t *, u_int32_t *); extern void mptcp_set_raddr_rand(mptcp_addr_id, struct mptcb *, mptcp_addr_id, u_int32_t); -extern u_int64_t mptcp_get_trunced_hmac(mptcp_addr_id, struct mptcb *mp_tp); -extern void mptcp_generate_token(char *, int, caddr_t, int); -extern void mptcp_generate_idsn(char *, int, caddr_t, int); extern int mptcp_init_remote_parms(struct mptcb *); extern boolean_t mptcp_ok_to_keepalive(struct mptcb *); extern void mptcp_insert_dsn(struct mppcb *, struct mbuf *); -extern void mptcp_output_getm_dsnmap32(struct socket *, int, uint32_t, - u_int32_t *, u_int32_t *, u_int16_t *, u_int64_t *); -extern void mptcp_output_getm_dsnmap64(struct socket *, int, uint32_t, - u_int64_t *, u_int32_t *, u_int16_t *); -extern void mptcp_send_dfin(struct socket *); +extern void mptcp_output_getm_dsnmap32(struct socket *so, int off, + uint32_t *dsn, uint32_t *relseq, + uint16_t *data_len, uint16_t *dss_csum); +extern void mptcp_output_getm_dsnmap64(struct socket *so, int off, + uint64_t *dsn, uint32_t *relseq, + uint16_t *data_len, uint16_t *dss_csum); extern void mptcp_act_on_txfail(struct socket *); extern struct mptsub *mptcp_get_subflow(struct mptses *, struct mptsub *, struct mptsub **); -extern struct mptsub *mptcp_get_pending_subflow(struct mptses *, - struct mptsub *); -extern struct mptsub* mptcp_use_symptoms_hints(struct mptsub*, - struct mptsub *); extern int mptcp_get_map_for_dsn(struct socket *, u_int64_t, u_int32_t *); -extern int32_t mptcp_adj_sendlen(struct socket *so, int32_t off, int32_t len); +extern int32_t mptcp_adj_sendlen(struct socket *so, int32_t off); +extern void mptcp_sbrcv_grow(struct mptcb *mp_tp); extern int32_t mptcp_sbspace(struct mptcb *); extern void mptcp_notify_mpready(struct socket *); extern void mptcp_notify_mpfail(struct socket *); @@ -619,9 +642,18 @@ extern boolean_t mptcp_no_rto_spike(struct socket*); extern int mptcp_set_notsent_lowat(struct mptses *mpte, int optval); extern u_int32_t mptcp_get_notsent_lowat(struct mptses *mpte); extern int mptcp_notsent_lowat_check(struct socket *so); +extern void mptcp_ask_symptoms(struct mptses *mpte); extern void mptcp_control_register(void); extern int mptcp_is_wifi_unusable(void); -extern int mptcp_is_cell_unusable(void); +extern void mptcp_session_necp_cb(void *, int, struct necp_client_flow *); +extern void mptcp_set_restrictions(struct socket *mp_so); +extern int mptcp_freeq(struct mptcb *); +extern void mptcp_set_cellicon(struct mptses *mpte); +extern void mptcp_unset_cellicon(void); +extern void mptcp_reset_rexmit_state(struct tcpcb *tp); +extern void mptcp_reset_keepalive(struct tcpcb *tp); +extern int mptcp_validate_csum(struct tcpcb *tp, struct mbuf *m, uint64_t dsn, + uint32_t sseq, uint16_t dlen, uint16_t csum); __END_DECLS #endif /* BSD_KERNEL_PRIVATE */ @@ -634,11 +666,9 @@ typedef struct mptcp_flow { sae_connid_t flow_cid; struct sockaddr_storage flow_src; struct sockaddr_storage flow_dst; - uint64_t flow_sndnxt; /* subflow's sndnxt snapshot */ uint32_t flow_relseq; /* last subflow rel seq# */ int32_t flow_soerror; /* subflow level error */ uint32_t flow_probecnt; /* number of probes sent */ - uint32_t flow_peerswitch;/* did peer switch */ conninfo_tcp_t flow_ci; /* must be the last field */ } mptcp_flow_t; @@ -678,7 +708,8 @@ typedef struct symptoms_advisory { uint32_t sa_nwk_status_int; struct { union { -#define SYMPTOMS_ADVISORY_NOCOMMENT 0x00 +#define SYMPTOMS_ADVISORY_NOCOMMENT 0x0000 +#define SYMPTOMS_ADVISORY_USEAPP 0xFFFF /* Very ugly workaround to avoid breaking backwards compatibility - ToDo: Fix it in +1 */ uint16_t sa_nwk_status; struct { #define SYMPTOMS_ADVISORY_WIFI_BAD 0x01 @@ -694,6 +725,19 @@ typedef struct symptoms_advisory { }; } symptoms_advisory_t; +struct mptcp_symptoms_ask_uuid { + uint32_t cmd; +#define MPTCP_SYMPTOMS_ASK_UUID 1 + uuid_t uuid; + uint32_t priority; +#define MPTCP_SYMPTOMS_UNKNOWN 0 +#define MPTCP_SYMPTOMS_BACKGROUND 1 +#define MPTCP_SYMPTOMS_FOREGROUND 2 +}; + +struct kev_mptcp_data { + int value; +}; #endif /* PRIVATE */ #endif /* _NETINET_MPTCP_VAR_H_ */ diff --git a/bsd/netinet/raw_ip.c b/bsd/netinet/raw_ip.c index 1f7ccb227..cb4e31a22 100644 --- a/bsd/netinet/raw_ip.c +++ b/bsd/netinet/raw_ip.c @@ -84,6 +84,7 @@ #include <pexpert/pexpert.h> #include <net/if.h> +#include <net/net_api_stats.h> #include <net/route.h> #define _IP_VHL @@ -436,7 +437,7 @@ rip_output( m_freem(m); return EINVAL; } - if (ip->ip_id == 0) + if (ip->ip_id == 0 && !(rfc6864 && IP_OFF_IS_ATOMIC(ntohs(ip->ip_off)))) ip->ip_id = ip_randomid(); /* XXX prevent ip_output from overwriting header fields */ flags |= IP_RAWOUTPUT; @@ -499,13 +500,6 @@ rip_output( if (inp->inp_route.ro_rt != NULL) rt_ifp = inp->inp_route.ro_rt->rt_ifp; - printf("%s inp %p last_pid %u inp_boundifp %d inp_last_outifp %d rt_ifp %d route_rule_id %u\n", - __func__, inp, - inp->inp_socket != NULL ? inp->inp_socket->last_pid : -1, - inp->inp_boundifp != NULL ? inp->inp_boundifp->if_index : -1, - inp->inp_last_outifp != NULL ? inp->inp_last_outifp->if_index : -1, - rt_ifp != NULL ? rt_ifp->if_index : -1, - route_rule_id); necp_socket_update_qos_marking(inp, inp->inp_route.ro_rt, NULL, route_rule_id); } @@ -571,8 +565,10 @@ rip_output( * route is unicast, update outif with that of the * route interface used by IP. */ - if (rt != NULL && (outif = rt->rt_ifp) != inp->inp_last_outifp) + if (rt != NULL && + (outif = rt->rt_ifp) != inp->inp_last_outifp) { inp->inp_last_outifp = outif; + } } else { ROUTE_RELEASE(&inp->inp_route); } @@ -752,11 +748,12 @@ void rip_ctlinput( int cmd, struct sockaddr *sa, - __unused void *vip) + __unused void *vip, + __unused struct ifnet *ifp) { - struct in_ifaddr *ia; - struct ifnet *ifp; - int err; + struct in_ifaddr *ia = NULL; + struct ifnet *iaifp = NULL; + int err = 0; int flags, done = 0; switch (cmd) { @@ -816,10 +813,10 @@ rip_ctlinput( lck_rw_done(in_ifaddr_rwlock); flags = RTF_UP; - ifp = ia->ia_ifa.ifa_ifp; + iaifp = ia->ia_ifa.ifa_ifp; - if ((ifp->if_flags & IFF_LOOPBACK) - || (ifp->if_flags & IFF_POINTOPOINT)) + if ((iaifp->if_flags & IFF_LOOPBACK) + || (iaifp->if_flags & IFF_POINTOPOINT)) flags |= RTF_HOST; err = rtinit(&ia->ia_ifa, RTM_ADD, flags); @@ -940,6 +937,7 @@ rip_bind(struct socket *so, struct sockaddr *nam, struct proc *p) } inp->inp_laddr = sin.sin_addr; inp->inp_last_outifp = outif; + return (0); } @@ -962,6 +960,12 @@ rip_connect(struct socket *so, struct sockaddr *nam, __unused struct proc *p) if ((addr->sin_family != AF_INET) && (addr->sin_family != AF_IMPLINK)) return EAFNOSUPPORT; + + if (!(so->so_flags1 & SOF1_CONNECT_COUNTED)) { + so->so_flags1 |= SOF1_CONNECT_COUNTED; + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_inet_dgram_connected); + } + inp->inp_faddr = addr->sin_addr; soisconnected(so); @@ -1169,6 +1173,7 @@ SYSCTL_PROC(_net_inet_raw, OID_AUTO/*XXX*/, pcblist, CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0, rip_pcblist, "S,xinpcb", "List of active raw IP sockets"); +#if !CONFIG_EMBEDDED static int rip_pcblist64 SYSCTL_HANDLER_ARGS @@ -1272,6 +1277,7 @@ SYSCTL_PROC(_net_inet_raw, OID_AUTO, pcblist64, CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0, rip_pcblist64, "S,xinpcb64", "List of active raw IP sockets"); +#endif /* !CONFIG_EMBEDDED */ static int diff --git a/bsd/netinet/tcp.h b/bsd/netinet/tcp.h index 35d85c160..d960c1f77 100644 --- a/bsd/netinet/tcp.h +++ b/bsd/netinet/tcp.h @@ -66,6 +66,7 @@ #include <sys/types.h> #include <sys/appleapiopts.h> #include <machine/endian.h> +#include <machine/types.h> /* __uint32_t */ #if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE) typedef __uint32_t tcp_seq; @@ -224,8 +225,8 @@ struct tcphdr { #define TCP_CONNECTION_INFO 0x106 /* State of TCP connection */ #ifdef PRIVATE -#define TCP_INFO 0x200 /* retrieve tcp_info structure */ -#define TCP_MEASURE_SND_BW 0x202 /* Measure sender's bandwidth for this connection */ +#define TCP_INFO 0x200 /* retrieve tcp_info structure */ +#define TCP_MEASURE_SND_BW 0x202 /* Measure sender's bandwidth for this connection */ #endif /* PRIVATE */ @@ -280,6 +281,21 @@ struct tcp_notify_ack_complete { }; #define TCP_NOTIFY_ACKNOWLEDGEMENT 0x212 /* Notify when data is acknowledged */ +#define MPTCP_SERVICE_TYPE 0x213 /* MPTCP Service type */ +#define TCP_FASTOPEN_FORCE_HEURISTICS 0x214 /* Make sure TFO-heuristics never get disabled */ + +#define MPTCP_SVCTYPE_HANDOVER 0 /* Default 0 */ +#define MPTCP_SVCTYPE_INTERACTIVE 1 +#define MPTCP_SVCTYPE_AGGREGATE 2 +#define MPTCP_SVCTYPE_MAX 3 +/* + * Specify minimum time in seconds before which an established + * TCP connection will not be dropped when there is no response from the + * peer + */ +#define TCP_RXT_MINIMUM_TIMEOUT 0x215 + +#define TCP_RXT_MINIMUM_TIMEOUT_LIMIT (5 * 60) /* Limit is 5 minutes */ /* * The TCP_INFO socket option is a private API and is subject to change @@ -380,7 +396,8 @@ struct tcp_info { tcpi_tfo_no_cookie_rcv:1, /* We did not receive a cookie upon our request */ tcpi_tfo_heuristics_disable:1, /* TFO-heuristics disabled it */ tcpi_tfo_send_blackhole:1, /* A sending-blackhole got detected */ - tcpi_tfo_recv_blackhole:1; /* A receiver-blackhole got detected */ + tcpi_tfo_recv_blackhole:1, /* A receiver-blackhole got detected */ + tcpi_tfo_onebyte_proxy:1; /* A proxy acknowledges all but one byte of the SYN */ u_int16_t tcpi_ecn_client_setup:1, /* Attempted ECN setup from client side */ tcpi_ecn_server_setup:1, /* Attempted ECN setup from server side */ @@ -388,8 +405,13 @@ struct tcp_info { tcpi_ecn_lost_syn:1, /* Lost SYN with ECN setup */ tcpi_ecn_lost_synack:1, /* Lost SYN-ACK with ECN setup */ tcpi_local_peer:1, /* Local to the host or the subnet */ - tcpi_if_cell:1, /* Interface is cellular */ - tcpi_if_wifi:1; /* Interface is WiFi */ + tcpi_if_cell:1, /* Interface is cellular */ + tcpi_if_wifi:1, /* Interface is WiFi */ + tcpi_if_wired:1, /* Interface is wired - ethernet , thunderbolt etc,. */ + tcpi_if_wifi_infra:1, /* Interface is wifi infrastructure */ + tcpi_if_wifi_awdl:1, /* Interface is wifi AWDL */ + tcpi_snd_background:1, /* Using delay based algorithm on sender side */ + tcpi_rcv_background:1; /* Using delay based algorithm on receive side */ u_int32_t tcpi_ecn_recv_ce; /* Packets received with CE */ u_int32_t tcpi_ecn_recv_cwr; /* Packets received with CWR */ @@ -429,16 +451,16 @@ struct info_tuple { }; #define itpl_local_sa itpl_localaddr._itpl_sa -#define itpl_local_sin itpl_localaddr._itpl_sin +#define itpl_local_sin itpl_localaddr._itpl_sin #define itpl_local_sin6 itpl_localaddr._itpl_sin6 -#define itpl_remote_sa itpl_remoteaddr._itpl_sa +#define itpl_remote_sa itpl_remoteaddr._itpl_sa #define itpl_remote_sin itpl_remoteaddr._itpl_sin #define itpl_remote_sin6 itpl_remoteaddr._itpl_sin6 /* * TCP connection info auxiliary data (CIAUX_TCP) * - * Do not add new fields to this structure, just add them to tcp_info + * Do not add new fields to this structure, just add them to tcp_info * structure towards the end. This will preserve binary compatibility. */ typedef struct conninfo_tcp { @@ -448,6 +470,31 @@ typedef struct conninfo_tcp { #pragma pack() +struct mptcp_itf_stats { + uint16_t ifindex; + uint16_t switches; + uint32_t is_expensive:1; + uint64_t mpis_txbytes __attribute__((aligned(8))); + uint64_t mpis_rxbytes __attribute__((aligned(8))); +}; + +/* Version solely used to let libnetcore survive */ +#define CONNINFO_MPTCP_VERSION 3 +typedef struct conninfo_multipathtcp { + uint32_t mptcpci_subflow_count; + uint32_t mptcpci_switch_count; + sae_connid_t mptcpci_subflow_connids[4]; + + uint64_t mptcpci_init_rxbytes; + uint64_t mptcpci_init_txbytes; + +#define MPTCP_ITFSTATS_SIZE 4 + struct mptcp_itf_stats mptcpci_itfstats[MPTCP_ITFSTATS_SIZE]; + + uint32_t mptcpci_flags; +#define MPTCPCI_FIRSTPARTY 0x01 +} conninfo_multipathtcp_t; + #endif /* PRIVATE */ struct tcp_connection_info { @@ -488,7 +535,8 @@ struct tcp_connection_info { tcpi_tfo_heuristics_disable:1, /* TFO-heuristics disabled it */ tcpi_tfo_send_blackhole:1, /* A sending-blackhole got detected */ tcpi_tfo_recv_blackhole:1, /* A receiver-blackhole got detected */ - __pad2:18; + tcpi_tfo_onebyte_proxy:1, /* A proxy acknowledges all but one byte of the SYN */ + __pad2:17; u_int64_t tcpi_txpackets __attribute__((aligned(8))); u_int64_t tcpi_txbytes __attribute__((aligned(8))); u_int64_t tcpi_txretransmitbytes __attribute__((aligned(8))); diff --git a/bsd/netinet/tcp_cache.c b/bsd/netinet/tcp_cache.c index ecd3ad590..293a0fc0c 100644 --- a/bsd/netinet/tcp_cache.c +++ b/bsd/netinet/tcp_cache.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016 Apple Inc. All rights reserved. + * Copyright (c) 2015-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -30,7 +30,9 @@ #include <net/flowhash.h> #include <net/route.h> +#include <net/necp.h> #include <netinet/in_pcb.h> +#include <netinet/mptcp_var.h> #include <netinet/tcp_cache.h> #include <netinet/tcp_seq.h> #include <netinet/tcp_var.h> @@ -38,13 +40,15 @@ #include <sys/queue.h> #include <dev/random/randomdev.h> +typedef union { + struct in_addr addr; + struct in6_addr addr6; +} in_4_6_addr; + struct tcp_heuristic_key { union { uint8_t thk_net_signature[IFNET_SIGNATURELEN]; - union { - struct in_addr addr; - struct in6_addr addr6; - } thk_ip; + in_4_6_addr thk_ip; }; sa_family_t thk_family; }; @@ -52,27 +56,29 @@ struct tcp_heuristic_key { struct tcp_heuristic { SLIST_ENTRY(tcp_heuristic) list; - u_int32_t th_last_access; + uint32_t th_last_access; struct tcp_heuristic_key th_key; char th_val_start[0]; /* Marker for memsetting to 0 */ - u_int8_t th_tfo_cookie_loss; /* The number of times a SYN+cookie has been lost */ - u_int8_t th_mptcp_loss; /* The number of times a SYN+MP_CAPABLE has been lost */ - u_int8_t th_ecn_loss; /* The number of times a SYN+ecn has been lost */ - u_int8_t th_ecn_aggressive; /* The number of times we did an aggressive fallback */ - u_int8_t th_ecn_droprst; /* The number of times ECN connections received a RST after first data pkt */ - u_int8_t th_ecn_droprxmt; /* The number of times ECN connection is dropped after multiple retransmits */ - u_int32_t th_tfo_fallback_trials; /* Number of times we did not try out TFO due to SYN-loss */ - u_int32_t th_tfo_cookie_backoff; /* Time until when we should not try out TFO */ - u_int32_t th_mptcp_backoff; /* Time until when we should not try out MPTCP */ - u_int32_t th_ecn_backoff; /* Time until when we should not try out ECN */ - - u_int8_t th_tfo_in_backoff:1, /* Are we avoiding TFO due to the backoff timer? */ - th_tfo_aggressive_fallback:1, /* Aggressive fallback due to nasty middlebox */ - th_tfo_snd_middlebox_supp:1, /* We are sure that the network supports TFO in upstream direction */ - th_tfo_rcv_middlebox_supp:1, /* We are sure that the network supports TFO in downstream direction*/ + uint8_t th_tfo_data_loss; /* The number of times a SYN+data has been lost */ + uint8_t th_tfo_req_loss; /* The number of times a SYN+cookie-req has been lost */ + uint8_t th_tfo_data_rst; /* The number of times a SYN+data has received a RST */ + uint8_t th_tfo_req_rst; /* The number of times a SYN+cookie-req has received a RST */ + uint8_t th_mptcp_loss; /* The number of times a SYN+MP_CAPABLE has been lost */ + uint8_t th_ecn_loss; /* The number of times a SYN+ecn has been lost */ + uint8_t th_ecn_aggressive; /* The number of times we did an aggressive fallback */ + uint8_t th_ecn_droprst; /* The number of times ECN connections received a RST after first data pkt */ + uint8_t th_ecn_droprxmt; /* The number of times ECN connection is dropped after multiple retransmits */ + uint8_t th_ecn_synrst; /* number of times RST was received in response to an ECN enabled SYN */ + uint32_t th_tfo_enabled_time; /* The moment when we reenabled TFO after backing off */ + uint32_t th_tfo_backoff_until; /* Time until when we should not try out TFO */ + uint32_t th_tfo_backoff; /* Current backoff timer */ + uint32_t th_mptcp_backoff; /* Time until when we should not try out MPTCP */ + uint32_t th_ecn_backoff; /* Time until when we should not try out ECN */ + + uint8_t th_tfo_in_backoff:1, /* Are we avoiding TFO due to the backoff timer? */ th_mptcp_in_backoff:1; /* Are we avoiding MPTCP due to the backoff timer? */ char th_val_end[0]; /* Marker for memsetting to 0 */ @@ -89,10 +95,7 @@ struct tcp_cache_key { sa_family_t tck_family; struct tcp_heuristic_key tck_src; - union { - struct in_addr addr; - struct in6_addr addr6; - } tck_dst; + in_4_6_addr tck_dst; }; struct tcp_cache { @@ -113,6 +116,13 @@ struct tcp_cache_head { lck_mtx_t tch_mtx; }; +struct tcp_cache_key_src { + struct ifnet *ifp; + in_4_6_addr laddr; + in_4_6_addr faddr; + int af; +}; + static u_int32_t tcp_cache_hash_seed; size_t tcp_cache_size; @@ -139,13 +149,24 @@ static lck_attr_t *tcp_heuristic_mtx_attr; static lck_grp_t *tcp_heuristic_mtx_grp; static lck_grp_attr_t *tcp_heuristic_mtx_grp_attr; -static int tcp_ecn_timeout = 60; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, ecn_timeout, CTLFLAG_RW | CTLFLAG_LOCKED, - &tcp_ecn_timeout, 0, "Initial minutes to wait before re-trying ECN"); +static uint32_t tcp_backoff_maximum = 65536; + +SYSCTL_UINT(_net_inet_tcp, OID_AUTO, backoff_maximum, CTLFLAG_RW | CTLFLAG_LOCKED, + &tcp_backoff_maximum, 0, "Maximum time for which we won't try TFO"); + +SYSCTL_SKMEM_TCP_INT(OID_AUTO, ecn_timeout, CTLFLAG_RW | CTLFLAG_LOCKED, + static int, tcp_ecn_timeout, 60, "Initial minutes to wait before re-trying ECN"); + +SYSCTL_SKMEM_TCP_INT(OID_AUTO, disable_tcp_heuristics, CTLFLAG_RW | CTLFLAG_LOCKED, + static int, disable_tcp_heuristics, 0, "Set to 1, to disable all TCP heuristics (TFO, ECN, MPTCP)"); -static int disable_tcp_heuristics = 0; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, disable_tcp_heuristics, CTLFLAG_RW | CTLFLAG_LOCKED, - &disable_tcp_heuristics, 0, "Set to 1, to disable all TCP heuristics (TFO, ECN, MPTCP)"); +static uint32_t tcp_min_to_hz(uint32_t minutes) +{ + if (minutes > 65536) + return ((uint32_t)65536 * 60 * TCP_RETRANSHZ); + + return (minutes * 60 * TCP_RETRANSHZ); +} /* * This number is coupled with tcp_ecn_timeout, because we want to prevent @@ -158,19 +179,34 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, disable_tcp_heuristics, CTLFLAG_RW | CTLFLAG #define TFO_MAX_COOKIE_LOSS 2 #define ECN_MAX_SYN_LOSS 2 #define MPTCP_MAX_SYN_LOSS 2 -#define ECN_MAX_DROPRST 2 +#define ECN_MAX_DROPRST 1 #define ECN_MAX_DROPRXMT 4 - -/* Flags for setting/unsetting loss-heuristics, limited to 1 byte */ -#define TCPCACHE_F_TFO 0x01 -#define TCPCACHE_F_ECN 0x02 -#define TCPCACHE_F_MPTCP 0x04 -#define TCPCACHE_F_ECN_DROPRST 0x08 -#define TCPCACHE_F_ECN_DROPRXMT 0x10 +#define ECN_MAX_SYNRST 4 + +/* Flags for setting/unsetting loss-heuristics, limited to 4 bytes */ +#define TCPCACHE_F_TFO_REQ 0x01 +#define TCPCACHE_F_TFO_DATA 0x02 +#define TCPCACHE_F_ECN 0x04 +#define TCPCACHE_F_MPTCP 0x08 +#define TCPCACHE_F_ECN_DROPRST 0x10 +#define TCPCACHE_F_ECN_DROPRXMT 0x20 +#define TCPCACHE_F_TFO_REQ_RST 0x40 +#define TCPCACHE_F_TFO_DATA_RST 0x80 +#define TCPCACHE_F_ECN_SYNRST 0x100 /* Always retry ECN after backing off to this level for some heuristics */ #define ECN_RETRY_LIMIT 9 +#define TCP_CACHE_INC_IFNET_STAT(_ifp_, _af_, _stat_) { \ + if ((_ifp_) != NULL) { \ + if ((_af_) == AF_INET6) { \ + (_ifp_)->if_ipv6_stat->_stat_++;\ + } else { \ + (_ifp_)->if_ipv4_stat->_stat_++;\ + }\ + }\ +} + /* * Round up to next higher power-of 2. See "Bit Twiddling Hacks". * @@ -190,17 +226,17 @@ static u_int32_t tcp_cache_roundup2(u_int32_t a) return a; } -static void tcp_cache_hash_src(struct inpcb *inp, struct tcp_heuristic_key *key) +static void tcp_cache_hash_src(struct tcp_cache_key_src *tcks, struct tcp_heuristic_key *key) { - struct ifnet *ifn = inp->inp_last_outifp; + struct ifnet *ifp = tcks->ifp; uint8_t len = sizeof(key->thk_net_signature); uint16_t flags; - if (inp->inp_vflag & INP_IPV6) { + if (tcks->af == AF_INET6) { int ret; key->thk_family = AF_INET6; - ret = ifnet_get_netsignature(ifn, AF_INET6, &len, &flags, + ret = ifnet_get_netsignature(ifp, AF_INET6, &len, &flags, key->thk_net_signature); /* @@ -209,13 +245,13 @@ static void tcp_cache_hash_src(struct inpcb *inp, struct tcp_heuristic_key *key) * in this case we should take the connection's address. */ if (ret == ENOENT || ret == EINVAL) - memcpy(&key->thk_ip.addr6, &inp->in6p_laddr, sizeof(struct in6_addr)); + memcpy(&key->thk_ip.addr6, &tcks->laddr.addr6, sizeof(struct in6_addr)); } else { int ret; key->thk_family = AF_INET; - ret = ifnet_get_netsignature(ifn, AF_INET, &len, &flags, - key->thk_net_signature); + ret = ifnet_get_netsignature(ifp, AF_INET, &len, &flags, + key->thk_net_signature); /* * ifnet_get_netsignature only returns EINVAL if ifn is NULL @@ -223,25 +259,25 @@ static void tcp_cache_hash_src(struct inpcb *inp, struct tcp_heuristic_key *key) * in this case we should take the connection's address. */ if (ret == ENOENT || ret == EINVAL) - memcpy(&key->thk_ip.addr, &inp->inp_laddr, sizeof(struct in_addr)); + memcpy(&key->thk_ip.addr, &tcks->laddr.addr, sizeof(struct in_addr)); } } -static u_int16_t tcp_cache_hash(struct inpcb *inp, struct tcp_cache_key *key) +static u_int16_t tcp_cache_hash(struct tcp_cache_key_src *tcks, struct tcp_cache_key *key) { u_int32_t hash; bzero(key, sizeof(struct tcp_cache_key)); - tcp_cache_hash_src(inp, &key->tck_src); + tcp_cache_hash_src(tcks, &key->tck_src); - if (inp->inp_vflag & INP_IPV6) { + if (tcks->af == AF_INET6) { key->tck_family = AF_INET6; - memcpy(&key->tck_dst.addr6, &inp->in6p_faddr, + memcpy(&key->tck_dst.addr6, &tcks->faddr.addr6, sizeof(struct in6_addr)); } else { key->tck_family = AF_INET; - memcpy(&key->tck_dst.addr, &inp->inp_faddr, + memcpy(&key->tck_dst.addr, &tcks->faddr.addr, sizeof(struct in_addr)); } @@ -266,17 +302,16 @@ static void tcp_cache_unlock(struct tcp_cache_head *head) * That's why we provide the head as a "return"-pointer so that the caller * can give it back to use for tcp_cache_unlock(). */ -static struct tcp_cache *tcp_getcache_with_lock(struct tcpcb *tp, int create, - struct tcp_cache_head **headarg) +static struct tcp_cache *tcp_getcache_with_lock(struct tcp_cache_key_src *tcks, + int create, struct tcp_cache_head **headarg) { - struct inpcb *inp = tp->t_inpcb; struct tcp_cache *tpcache = NULL; struct tcp_cache_head *head; struct tcp_cache_key key; u_int16_t hash; int i = 0; - hash = tcp_cache_hash(inp, &key); + hash = tcp_cache_hash(tcks, &key); head = &tcp_cache[hash]; lck_mtx_lock(&head->tch_mtx); @@ -336,13 +371,33 @@ out_null: return (NULL); } -void tcp_cache_set_cookie(struct tcpcb *tp, u_char *cookie, u_int8_t len) +static void tcp_cache_key_src_create(struct tcpcb *tp, struct tcp_cache_key_src *tcks) +{ + struct inpcb *inp = tp->t_inpcb; + memset(tcks, 0, sizeof(*tcks)); + + tcks->ifp = inp->inp_last_outifp; + + if (inp->inp_vflag & INP_IPV6) { + memcpy(&tcks->laddr.addr6, &inp->in6p_laddr, sizeof(struct in6_addr)); + memcpy(&tcks->faddr.addr6, &inp->in6p_faddr, sizeof(struct in6_addr)); + tcks->af = AF_INET6; + } else { + memcpy(&tcks->laddr.addr, &inp->inp_laddr, sizeof(struct in_addr)); + memcpy(&tcks->faddr.addr, &inp->inp_faddr, sizeof(struct in_addr)); + tcks->af = AF_INET; + } + + return; +} + +static void tcp_cache_set_cookie_common(struct tcp_cache_key_src *tcks, u_char *cookie, u_int8_t len) { struct tcp_cache_head *head; struct tcp_cache *tpcache; /* Call lookup/create function */ - tpcache = tcp_getcache_with_lock(tp, 1, &head); + tpcache = tcp_getcache_with_lock(tcks, 1, &head); if (tpcache == NULL) return; @@ -352,23 +407,24 @@ void tcp_cache_set_cookie(struct tcpcb *tp, u_char *cookie, u_int8_t len) tcp_cache_unlock(head); } -/* - * Get the cookie related to 'tp', and copy it into 'cookie', provided that len - * is big enough (len designates the available memory. - * Upon return, 'len' is set to the cookie's length. - * - * Returns 0 if we should request a cookie. - * Returns 1 if the cookie has been found and written. - */ -int tcp_cache_get_cookie(struct tcpcb *tp, u_char *cookie, u_int8_t *len) +void tcp_cache_set_cookie(struct tcpcb *tp, u_char *cookie, u_int8_t len) +{ + struct tcp_cache_key_src tcks; + + tcp_cache_key_src_create(tp, &tcks); + tcp_cache_set_cookie_common(&tcks, cookie, len); +} + +static int tcp_cache_get_cookie_common(struct tcp_cache_key_src *tcks, u_char *cookie, u_int8_t *len) { struct tcp_cache_head *head; struct tcp_cache *tpcache; /* Call lookup/create function */ - tpcache = tcp_getcache_with_lock(tp, 1, &head); - if (tpcache == NULL) + tpcache = tcp_getcache_with_lock(tcks, 1, &head); + if (tpcache == NULL) { return (0); + } if (tpcache->tc_tfo_cookie_len == 0) { tcp_cache_unlock(head); @@ -389,14 +445,30 @@ int tcp_cache_get_cookie(struct tcpcb *tp, u_char *cookie, u_int8_t *len) return (1); } -unsigned int tcp_cache_get_cookie_len(struct tcpcb *tp) +/* + * Get the cookie related to 'tp', and copy it into 'cookie', provided that len + * is big enough (len designates the available memory. + * Upon return, 'len' is set to the cookie's length. + * + * Returns 0 if we should request a cookie. + * Returns 1 if the cookie has been found and written. + */ +int tcp_cache_get_cookie(struct tcpcb *tp, u_char *cookie, u_int8_t *len) +{ + struct tcp_cache_key_src tcks; + + tcp_cache_key_src_create(tp, &tcks); + return tcp_cache_get_cookie_common(&tcks, cookie, len); +} + +static unsigned int tcp_cache_get_cookie_len_common(struct tcp_cache_key_src *tcks) { struct tcp_cache_head *head; struct tcp_cache *tpcache; unsigned int cookie_len; /* Call lookup/create function */ - tpcache = tcp_getcache_with_lock(tp, 1, &head); + tpcache = tcp_getcache_with_lock(tcks, 1, &head); if (tpcache == NULL) return (0); @@ -407,14 +479,21 @@ unsigned int tcp_cache_get_cookie_len(struct tcpcb *tp) return cookie_len; } -static u_int16_t tcp_heuristics_hash(struct inpcb *inp, - struct tcp_heuristic_key *key) +unsigned int tcp_cache_get_cookie_len(struct tcpcb *tp) +{ + struct tcp_cache_key_src tcks; + + tcp_cache_key_src_create(tp, &tcks); + return tcp_cache_get_cookie_len_common(&tcks); +} + +static u_int16_t tcp_heuristics_hash(struct tcp_cache_key_src *tcks, struct tcp_heuristic_key *key) { u_int32_t hash; bzero(key, sizeof(struct tcp_heuristic_key)); - tcp_cache_hash_src(inp, key); + tcp_cache_hash_src(tcks, key); hash = net_flowhash(key, sizeof(struct tcp_heuristic_key), tcp_cache_hash_seed); @@ -441,17 +520,16 @@ static void tcp_heuristic_unlock(struct tcp_heuristics_head *head) * ToDo - way too much code-duplication. We should create an interface to handle * bucketized hashtables with recycling of the oldest element. */ -static struct tcp_heuristic *tcp_getheuristic_with_lock(struct tcpcb *tp, +static struct tcp_heuristic *tcp_getheuristic_with_lock(struct tcp_cache_key_src *tcks, int create, struct tcp_heuristics_head **headarg) { - struct inpcb *inp = tp->t_inpcb; struct tcp_heuristic *tpheur = NULL; struct tcp_heuristics_head *head; struct tcp_heuristic_key key; u_int16_t hash; int i = 0; - hash = tcp_heuristics_hash(inp, &key); + hash = tcp_heuristics_hash(tcks, &key); head = &tcp_heuristics[hash]; lck_mtx_lock(&head->thh_mtx); @@ -500,8 +578,9 @@ static struct tcp_heuristic *tcp_getheuristic_with_lock(struct tcpcb *tp, * near future. */ tpheur->th_ecn_backoff = tcp_now; - tpheur->th_tfo_cookie_backoff = tcp_now; + tpheur->th_tfo_backoff_until = tcp_now; tpheur->th_mptcp_backoff = tcp_now; + tpheur->th_tfo_backoff = tcp_min_to_hz(tcp_ecn_timeout); memcpy(&tpheur->th_key, &key, sizeof(key)); } @@ -520,7 +599,7 @@ out_null: return (NULL); } -static void tcp_heuristic_reset_loss(struct tcpcb *tp, u_int8_t flags) +static void tcp_heuristic_reset_counters(struct tcp_cache_key_src *tcks, u_int8_t flags) { struct tcp_heuristics_head *head; struct tcp_heuristic *tpheur; @@ -530,15 +609,30 @@ static void tcp_heuristic_reset_loss(struct tcpcb *tp, u_int8_t flags) * server does not support TFO. This reduces the lookup-cost on * our side. */ - tpheur = tcp_getheuristic_with_lock(tp, 0, &head); + tpheur = tcp_getheuristic_with_lock(tcks, 0, &head); if (tpheur == NULL) return; - if (flags & TCPCACHE_F_TFO) - tpheur->th_tfo_cookie_loss = 0; + if (flags & TCPCACHE_F_TFO_DATA) { + tpheur->th_tfo_data_loss = 0; + } + + if (flags & TCPCACHE_F_TFO_REQ) { + tpheur->th_tfo_req_loss = 0; + } + + if (flags & TCPCACHE_F_TFO_DATA_RST) { + tpheur->th_tfo_data_rst = 0; + } + + if (flags & TCPCACHE_F_TFO_REQ_RST) { + tpheur->th_tfo_req_rst = 0; + } - if (flags & TCPCACHE_F_ECN) + if (flags & TCPCACHE_F_ECN) { tpheur->th_ecn_loss = 0; + tpheur->th_ecn_synrst = 0; + } if (flags & TCPCACHE_F_MPTCP) tpheur->th_mptcp_loss = 0; @@ -548,69 +642,120 @@ static void tcp_heuristic_reset_loss(struct tcpcb *tp, u_int8_t flags) void tcp_heuristic_tfo_success(struct tcpcb *tp) { - tcp_heuristic_reset_loss(tp, TCPCACHE_F_TFO); + struct tcp_cache_key_src tcks; + uint8_t flag = 0; + + tcp_cache_key_src_create(tp, &tcks); + + if (tp->t_tfo_stats & TFO_S_SYN_DATA_SENT) + flag = (TCPCACHE_F_TFO_DATA | TCPCACHE_F_TFO_REQ | + TCPCACHE_F_TFO_DATA_RST | TCPCACHE_F_TFO_REQ_RST ); + if (tp->t_tfo_stats & TFO_S_COOKIE_REQ) + flag = (TCPCACHE_F_TFO_REQ | TCPCACHE_F_TFO_REQ_RST); + + tcp_heuristic_reset_counters(&tcks, flag); } void tcp_heuristic_mptcp_success(struct tcpcb *tp) { - tcp_heuristic_reset_loss(tp, TCPCACHE_F_MPTCP); + struct tcp_cache_key_src tcks; + + tcp_cache_key_src_create(tp, &tcks); + tcp_heuristic_reset_counters(&tcks, TCPCACHE_F_MPTCP); } void tcp_heuristic_ecn_success(struct tcpcb *tp) { - tcp_heuristic_reset_loss(tp, TCPCACHE_F_ECN); + struct tcp_cache_key_src tcks; + + tcp_cache_key_src_create(tp, &tcks); + tcp_heuristic_reset_counters(&tcks, TCPCACHE_F_ECN); } -void tcp_heuristic_tfo_rcv_good(struct tcpcb *tp) +static void __tcp_heuristic_tfo_middlebox_common(struct tcp_heuristic *tpheur) { - struct tcp_heuristics_head *head; - - struct tcp_heuristic *tpheur = tcp_getheuristic_with_lock(tp, 1, &head); - if (tpheur == NULL) + if (tpheur->th_tfo_in_backoff) return; - tpheur->th_tfo_rcv_middlebox_supp = 1; + tpheur->th_tfo_in_backoff = 1; - tcp_heuristic_unlock(head); + if (tpheur->th_tfo_enabled_time) { + uint32_t old_backoff = tpheur->th_tfo_backoff; + + tpheur->th_tfo_backoff -= (tcp_now - tpheur->th_tfo_enabled_time); + if (tpheur->th_tfo_backoff > old_backoff) + tpheur->th_tfo_backoff = tcp_min_to_hz(tcp_ecn_timeout); + } - tp->t_tfo_flags |= TFO_F_NO_RCVPROBING; + tpheur->th_tfo_backoff_until = tcp_now + tpheur->th_tfo_backoff; + + /* Then, increase the backoff time */ + tpheur->th_tfo_backoff *= 2; + + if (tpheur->th_tfo_backoff > tcp_min_to_hz(tcp_backoff_maximum)) + tpheur->th_tfo_backoff = tcp_min_to_hz(tcp_ecn_timeout); } -void tcp_heuristic_tfo_snd_good(struct tcpcb *tp) +static void tcp_heuristic_tfo_middlebox_common(struct tcp_cache_key_src *tcks) { struct tcp_heuristics_head *head; + struct tcp_heuristic *tpheur; - struct tcp_heuristic *tpheur = tcp_getheuristic_with_lock(tp, 1, &head); + tpheur = tcp_getheuristic_with_lock(tcks, 1, &head); if (tpheur == NULL) return; - tpheur->th_tfo_snd_middlebox_supp = 1; + __tcp_heuristic_tfo_middlebox_common(tpheur); tcp_heuristic_unlock(head); - - tp->t_tfo_flags |= TFO_F_NO_SNDPROBING; } -static void tcp_heuristic_inc_loss(struct tcpcb *tp, u_int8_t flags) +static void tcp_heuristic_inc_counters(struct tcp_cache_key_src *tcks, + u_int32_t flags) { struct tcp_heuristics_head *head; struct tcp_heuristic *tpheur; - tpheur = tcp_getheuristic_with_lock(tp, 1, &head); + tpheur = tcp_getheuristic_with_lock(tcks, 1, &head); if (tpheur == NULL) return; /* Limit to prevent integer-overflow during exponential backoff */ - if ((flags & TCPCACHE_F_TFO) && tpheur->th_tfo_cookie_loss < TCP_CACHE_OVERFLOW_PROTECT) - tpheur->th_tfo_cookie_loss++; + if ((flags & TCPCACHE_F_TFO_DATA) && tpheur->th_tfo_data_loss < TCP_CACHE_OVERFLOW_PROTECT) { + tpheur->th_tfo_data_loss++; + + if (tpheur->th_tfo_data_loss >= TFO_MAX_COOKIE_LOSS) + __tcp_heuristic_tfo_middlebox_common(tpheur); + } + + if ((flags & TCPCACHE_F_TFO_REQ) && tpheur->th_tfo_req_loss < TCP_CACHE_OVERFLOW_PROTECT) { + tpheur->th_tfo_req_loss++; + + if (tpheur->th_tfo_req_loss >= TFO_MAX_COOKIE_LOSS) + __tcp_heuristic_tfo_middlebox_common(tpheur); + } + + if ((flags & TCPCACHE_F_TFO_DATA_RST) && tpheur->th_tfo_data_rst < TCP_CACHE_OVERFLOW_PROTECT) { + tpheur->th_tfo_data_rst++; + + if (tpheur->th_tfo_data_rst >= TFO_MAX_COOKIE_LOSS) + __tcp_heuristic_tfo_middlebox_common(tpheur); + } + + if ((flags & TCPCACHE_F_TFO_REQ_RST) && tpheur->th_tfo_req_rst < TCP_CACHE_OVERFLOW_PROTECT) { + tpheur->th_tfo_req_rst++; + + if (tpheur->th_tfo_req_rst >= TFO_MAX_COOKIE_LOSS) + __tcp_heuristic_tfo_middlebox_common(tpheur); + } if ((flags & TCPCACHE_F_ECN) && tpheur->th_ecn_loss < TCP_CACHE_OVERFLOW_PROTECT) { tpheur->th_ecn_loss++; if (tpheur->th_ecn_loss >= ECN_MAX_SYN_LOSS) { tcpstat.tcps_ecn_fallback_synloss++; - INP_INC_IFNET_STAT(tp->t_inpcb, ecn_fallback_synloss); + TCP_CACHE_INC_IFNET_STAT(tcks->ifp, tcks->af, ecn_fallback_synloss); tpheur->th_ecn_backoff = tcp_now + - ((tcp_ecn_timeout * 60 * TCP_RETRANSHZ) << + (tcp_min_to_hz(tcp_ecn_timeout) << (tpheur->th_ecn_loss - ECN_MAX_SYN_LOSS)); } } @@ -624,7 +769,7 @@ static void tcp_heuristic_inc_loss(struct tcpcb *tp, u_int8_t flags) * another sysctl that is just used for testing. */ tpheur->th_mptcp_backoff = tcp_now + - ((tcp_ecn_timeout * 60 * TCP_RETRANSHZ) << + (tcp_min_to_hz(tcp_ecn_timeout) << (tpheur->th_mptcp_loss - MPTCP_MAX_SYN_LOSS)); } } @@ -634,79 +779,139 @@ static void tcp_heuristic_inc_loss(struct tcpcb *tp, u_int8_t flags) tpheur->th_ecn_droprst++; if (tpheur->th_ecn_droprst >= ECN_MAX_DROPRST) { tcpstat.tcps_ecn_fallback_droprst++; - INP_INC_IFNET_STAT(tp->t_inpcb, ecn_fallback_droprst); + TCP_CACHE_INC_IFNET_STAT(tcks->ifp, tcks->af, + ecn_fallback_droprst); tpheur->th_ecn_backoff = tcp_now + - ((tcp_ecn_timeout * 60 * TCP_RETRANSHZ) << + (tcp_min_to_hz(tcp_ecn_timeout) << (tpheur->th_ecn_droprst - ECN_MAX_DROPRST)); } } if ((flags & TCPCACHE_F_ECN_DROPRXMT) && - tpheur->th_ecn_droprst < TCP_CACHE_OVERFLOW_PROTECT) { + tpheur->th_ecn_droprxmt < TCP_CACHE_OVERFLOW_PROTECT) { tpheur->th_ecn_droprxmt++; if (tpheur->th_ecn_droprxmt >= ECN_MAX_DROPRXMT) { tcpstat.tcps_ecn_fallback_droprxmt++; - INP_INC_IFNET_STAT(tp->t_inpcb, ecn_fallback_droprxmt); + TCP_CACHE_INC_IFNET_STAT(tcks->ifp, tcks->af, + ecn_fallback_droprxmt); tpheur->th_ecn_backoff = tcp_now + - ((tcp_ecn_timeout * 60 * TCP_RETRANSHZ) << + (tcp_min_to_hz(tcp_ecn_timeout) << (tpheur->th_ecn_droprxmt - ECN_MAX_DROPRXMT)); } } + if ((flags & TCPCACHE_F_ECN_SYNRST) && + tpheur->th_ecn_synrst < TCP_CACHE_OVERFLOW_PROTECT) { + tpheur->th_ecn_synrst++; + if (tpheur->th_ecn_synrst >= ECN_MAX_SYNRST) { + tcpstat.tcps_ecn_fallback_synrst++; + TCP_CACHE_INC_IFNET_STAT(tcks->ifp, tcks->af, + ecn_fallback_synrst); + tpheur->th_ecn_backoff = tcp_now + + (tcp_min_to_hz(tcp_ecn_timeout) << + (tpheur->th_ecn_synrst - ECN_MAX_SYNRST)); + } + } tcp_heuristic_unlock(head); } void tcp_heuristic_tfo_loss(struct tcpcb *tp) { - tcp_heuristic_inc_loss(tp, TCPCACHE_F_TFO); + struct tcp_cache_key_src tcks; + uint32_t flag = 0; + + tcp_cache_key_src_create(tp, &tcks); + + if (tp->t_tfo_stats & TFO_S_SYN_DATA_SENT) + flag = (TCPCACHE_F_TFO_DATA | TCPCACHE_F_TFO_REQ); + if (tp->t_tfo_stats & TFO_S_COOKIE_REQ) + flag = TCPCACHE_F_TFO_REQ; + + tcp_heuristic_inc_counters(&tcks, flag); +} + +void tcp_heuristic_tfo_rst(struct tcpcb *tp) +{ + struct tcp_cache_key_src tcks; + uint32_t flag = 0; + + tcp_cache_key_src_create(tp, &tcks); + + if (tp->t_tfo_stats & TFO_S_SYN_DATA_SENT) + flag = (TCPCACHE_F_TFO_DATA_RST | TCPCACHE_F_TFO_REQ_RST); + if (tp->t_tfo_stats & TFO_S_COOKIE_REQ) + flag = TCPCACHE_F_TFO_REQ_RST; + + tcp_heuristic_inc_counters(&tcks, flag); } void tcp_heuristic_mptcp_loss(struct tcpcb *tp) { - tcp_heuristic_inc_loss(tp, TCPCACHE_F_MPTCP); + struct tcp_cache_key_src tcks; + + tcp_cache_key_src_create(tp, &tcks); + + tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_MPTCP); } void tcp_heuristic_ecn_loss(struct tcpcb *tp) { - tcp_heuristic_inc_loss(tp, TCPCACHE_F_ECN); + struct tcp_cache_key_src tcks; + + tcp_cache_key_src_create(tp, &tcks); + + tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_ECN); } void tcp_heuristic_ecn_droprst(struct tcpcb *tp) { - tcp_heuristic_inc_loss(tp, TCPCACHE_F_ECN_DROPRST); + struct tcp_cache_key_src tcks; + + tcp_cache_key_src_create(tp, &tcks); + + tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_ECN_DROPRST); } void tcp_heuristic_ecn_droprxmt(struct tcpcb *tp) { - tcp_heuristic_inc_loss(tp, TCPCACHE_F_ECN_DROPRXMT); + struct tcp_cache_key_src tcks; + + tcp_cache_key_src_create(tp, &tcks); + + tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_ECN_DROPRXMT); } -void tcp_heuristic_tfo_middlebox(struct tcpcb *tp) +void tcp_heuristic_ecn_synrst(struct tcpcb *tp) { - struct tcp_heuristics_head *head; - struct tcp_heuristic *tpheur; + struct tcp_cache_key_src tcks; - tpheur = tcp_getheuristic_with_lock(tp, 1, &head); - if (tpheur == NULL) - return; + tcp_cache_key_src_create(tp, &tcks); - tpheur->th_tfo_aggressive_fallback = 1; + tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_ECN_SYNRST); +} - tcp_heuristic_unlock(head); +void tcp_heuristic_tfo_middlebox(struct tcpcb *tp) +{ + struct tcp_cache_key_src tcks; + + tp->t_tfo_flags |= TFO_F_HEURISTIC_DONE; + + tcp_cache_key_src_create(tp, &tcks); + tcp_heuristic_tfo_middlebox_common(&tcks); } -void tcp_heuristic_ecn_aggressive(struct tcpcb *tp) +static void tcp_heuristic_ecn_aggressive_common(struct tcp_cache_key_src *tcks) { struct tcp_heuristics_head *head; struct tcp_heuristic *tpheur; - tpheur = tcp_getheuristic_with_lock(tp, 1, &head); + tpheur = tcp_getheuristic_with_lock(tcks, 1, &head); if (tpheur == NULL) return; /* Must be done before, otherwise we will start off with expo-backoff */ tpheur->th_ecn_backoff = tcp_now + - ((tcp_ecn_timeout * 60 * TCP_RETRANSHZ) << (tpheur->th_ecn_aggressive)); + (tcp_min_to_hz(tcp_ecn_timeout) << (tpheur->th_ecn_aggressive)); /* * Ugly way to prevent integer overflow... limit to prevent in @@ -718,7 +923,15 @@ void tcp_heuristic_ecn_aggressive(struct tcpcb *tp) tcp_heuristic_unlock(head); } -boolean_t tcp_heuristic_do_tfo(struct tcpcb *tp) +void tcp_heuristic_ecn_aggressive(struct tcpcb *tp) +{ + struct tcp_cache_key_src tcks; + + tcp_cache_key_src_create(tp, &tcks); + tcp_heuristic_ecn_aggressive_common(&tcks); +} + +static boolean_t tcp_heuristic_do_tfo_common(struct tcp_cache_key_src *tcks) { struct tcp_heuristics_head *head; struct tcp_heuristic *tpheur; @@ -727,85 +940,75 @@ boolean_t tcp_heuristic_do_tfo(struct tcpcb *tp) return (TRUE); /* Get the tcp-heuristic. */ - tpheur = tcp_getheuristic_with_lock(tp, 0, &head); + tpheur = tcp_getheuristic_with_lock(tcks, 0, &head); if (tpheur == NULL) return (TRUE); - if (tpheur->th_tfo_aggressive_fallback) { - /* Aggressive fallback - don't do TFO anymore... :'( */ - tcp_heuristic_unlock(head); - return (FALSE); - } + if (tpheur->th_tfo_in_backoff == 0) + goto tfo_ok; - if (tpheur->th_tfo_cookie_loss >= TFO_MAX_COOKIE_LOSS && - (tpheur->th_tfo_fallback_trials < tcp_tfo_fallback_min || - TSTMP_GT(tpheur->th_tfo_cookie_backoff, tcp_now))) { - /* - * So, when we are in SYN-loss mode we try to stop using TFO - * for the next 'tcp_tfo_fallback_min' connections. That way, - * we are sure that never more than 1 out of tcp_tfo_fallback_min - * connections will suffer from our nice little middelbox. - * - * After that we first wait for 2 minutes. If we fail again, - * we wait for yet another 60 minutes. - */ - tpheur->th_tfo_fallback_trials++; - if (tpheur->th_tfo_fallback_trials >= tcp_tfo_fallback_min && - !tpheur->th_tfo_in_backoff) { - if (tpheur->th_tfo_cookie_loss == TFO_MAX_COOKIE_LOSS) - /* Backoff for 2 minutes */ - tpheur->th_tfo_cookie_backoff = tcp_now + (60 * 2 * TCP_RETRANSHZ); - else - /* Backoff for 60 minutes */ - tpheur->th_tfo_cookie_backoff = tcp_now + (60 * 60 * TCP_RETRANSHZ); - - tpheur->th_tfo_in_backoff = 1; - } + if (TSTMP_GT(tcp_now, tpheur->th_tfo_backoff_until)) { + tpheur->th_tfo_in_backoff = 0; + tpheur->th_tfo_enabled_time = tcp_now; - tcp_heuristic_unlock(head); - return (FALSE); + goto tfo_ok; } - /* - * We give it a new shot, set trials back to 0. This allows to - * start counting again from zero in case we get yet another SYN-loss - */ - tpheur->th_tfo_fallback_trials = 0; - tpheur->th_tfo_in_backoff = 0; - - if (tpheur->th_tfo_rcv_middlebox_supp) - tp->t_tfo_flags |= TFO_F_NO_RCVPROBING; - if (tpheur->th_tfo_snd_middlebox_supp) - tp->t_tfo_flags |= TFO_F_NO_SNDPROBING; - tcp_heuristic_unlock(head); + return (FALSE); +tfo_ok: + tcp_heuristic_unlock(head); return (TRUE); } +boolean_t tcp_heuristic_do_tfo(struct tcpcb *tp) +{ + struct tcp_cache_key_src tcks; + + tcp_cache_key_src_create(tp, &tcks); + if (tcp_heuristic_do_tfo_common(&tcks)) + return (TRUE); + + return (FALSE); +} + boolean_t tcp_heuristic_do_mptcp(struct tcpcb *tp) { - struct tcp_heuristics_head *head; + struct tcp_cache_key_src tcks; + struct tcp_heuristics_head *head = NULL; struct tcp_heuristic *tpheur; - boolean_t ret = TRUE; if (disable_tcp_heuristics) return (TRUE); + tcp_cache_key_src_create(tp, &tcks); + /* Get the tcp-heuristic. */ - tpheur = tcp_getheuristic_with_lock(tp, 0, &head); + tpheur = tcp_getheuristic_with_lock(&tcks, 0, &head); if (tpheur == NULL) - return ret; + return (TRUE); if (TSTMP_GT(tpheur->th_mptcp_backoff, tcp_now)) - ret = FALSE; + goto fallback; tcp_heuristic_unlock(head); - return (ret); + return (TRUE); + +fallback: + if (head) + tcp_heuristic_unlock(head); + + if (tptomptp(tp)->mpt_mpte->mpte_flags & MPTE_FIRSTPARTY) + tcpstat.tcps_mptcp_fp_heuristic_fallback++; + else + tcpstat.tcps_mptcp_heuristic_fallback++; + + return (FALSE); } -boolean_t tcp_heuristic_do_ecn(struct tcpcb *tp) +static boolean_t tcp_heuristic_do_ecn_common(struct tcp_cache_key_src *tcks) { struct tcp_heuristics_head *head; struct tcp_heuristic *tpheur; @@ -815,7 +1018,7 @@ boolean_t tcp_heuristic_do_ecn(struct tcpcb *tp) return (TRUE); /* Get the tcp-heuristic. */ - tpheur = tcp_getheuristic_with_lock(tp, 0, &head); + tpheur = tcp_getheuristic_with_lock(tcks, 0, &head); if (tpheur == NULL) return ret; @@ -827,6 +1030,8 @@ boolean_t tcp_heuristic_do_ecn(struct tcpcb *tp) tpheur->th_ecn_droprst = 0; if (tpheur->th_ecn_droprxmt >= ECN_RETRY_LIMIT) tpheur->th_ecn_droprxmt = 0; + if (tpheur->th_ecn_synrst >= ECN_RETRY_LIMIT) + tpheur->th_ecn_synrst = 0; } tcp_heuristic_unlock(head); @@ -834,6 +1039,152 @@ boolean_t tcp_heuristic_do_ecn(struct tcpcb *tp) return (ret); } +boolean_t tcp_heuristic_do_ecn(struct tcpcb *tp) +{ + struct tcp_cache_key_src tcks; + + tcp_cache_key_src_create(tp, &tcks); + return tcp_heuristic_do_ecn_common(&tcks); +} + +boolean_t tcp_heuristic_do_ecn_with_address(struct ifnet *ifp, + union sockaddr_in_4_6 *local_address) +{ + struct tcp_cache_key_src tcks; + + memset(&tcks, 0, sizeof(tcks)); + tcks.ifp = ifp; + + calculate_tcp_clock(); + + if (local_address->sa.sa_family == AF_INET6) { + memcpy(&tcks.laddr.addr6, &local_address->sin6.sin6_addr, sizeof(struct in6_addr)); + tcks.af = AF_INET6; + } else if (local_address->sa.sa_family == AF_INET) { + memcpy(&tcks.laddr.addr, &local_address->sin.sin_addr, sizeof(struct in_addr)); + tcks.af = AF_INET; + } + + return tcp_heuristic_do_ecn_common(&tcks); +} + +void tcp_heuristics_ecn_update(struct necp_tcp_ecn_cache *necp_buffer, + struct ifnet *ifp, union sockaddr_in_4_6 *local_address) +{ + struct tcp_cache_key_src tcks; + + memset(&tcks, 0, sizeof(tcks)); + tcks.ifp = ifp; + + calculate_tcp_clock(); + + if (local_address->sa.sa_family == AF_INET6) { + memcpy(&tcks.laddr.addr6, &local_address->sin6.sin6_addr, sizeof(struct in6_addr)); + tcks.af = AF_INET6; + } else if (local_address->sa.sa_family == AF_INET) { + memcpy(&tcks.laddr.addr, &local_address->sin.sin_addr, sizeof(struct in_addr)); + tcks.af = AF_INET; + } + + if (necp_buffer->necp_tcp_ecn_heuristics_success) { + tcp_heuristic_reset_counters(&tcks, TCPCACHE_F_ECN); + } else if (necp_buffer->necp_tcp_ecn_heuristics_loss) { + tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_ECN); + } else if (necp_buffer->necp_tcp_ecn_heuristics_drop_rst) { + tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_ECN_DROPRST); + } else if (necp_buffer->necp_tcp_ecn_heuristics_drop_rxmt) { + tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_ECN_DROPRXMT); + } else if (necp_buffer->necp_tcp_ecn_heuristics_syn_rst) { + tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_ECN_SYNRST); + } else if (necp_buffer->necp_tcp_ecn_heuristics_aggressive) { + tcp_heuristic_ecn_aggressive_common(&tcks); + } + + return; +} + +boolean_t tcp_heuristic_do_tfo_with_address(struct ifnet *ifp, + union sockaddr_in_4_6 *local_address, union sockaddr_in_4_6 *remote_address, + u_int8_t *cookie, u_int8_t *cookie_len) +{ + struct tcp_cache_key_src tcks; + + memset(&tcks, 0, sizeof(tcks)); + tcks.ifp = ifp; + + calculate_tcp_clock(); + + if (remote_address->sa.sa_family == AF_INET6) { + memcpy(&tcks.laddr.addr6, &local_address->sin6.sin6_addr, sizeof(struct in6_addr)); + memcpy(&tcks.faddr.addr6, &remote_address->sin6.sin6_addr, sizeof(struct in6_addr)); + tcks.af = AF_INET6; + } else if (remote_address->sa.sa_family == AF_INET) { + memcpy(&tcks.laddr.addr, &local_address->sin.sin_addr, sizeof(struct in_addr)); + memcpy(&tcks.faddr.addr, &remote_address->sin.sin_addr, sizeof(struct in_addr)); + tcks.af = AF_INET; + } + + if (tcp_heuristic_do_tfo_common(&tcks)) { + if (!tcp_cache_get_cookie_common(&tcks, cookie, cookie_len)) { + *cookie_len = 0; + } + return TRUE; + } + + return FALSE; +} + +void tcp_heuristics_tfo_update(struct necp_tcp_tfo_cache *necp_buffer, + struct ifnet *ifp, union sockaddr_in_4_6 *local_address, + union sockaddr_in_4_6 *remote_address) +{ + struct tcp_cache_key_src tcks; + + memset(&tcks, 0, sizeof(tcks)); + tcks.ifp = ifp; + + calculate_tcp_clock(); + + if (remote_address->sa.sa_family == AF_INET6) { + memcpy(&tcks.laddr.addr6, &local_address->sin6.sin6_addr, sizeof(struct in6_addr)); + memcpy(&tcks.faddr.addr6, &remote_address->sin6.sin6_addr, sizeof(struct in6_addr)); + tcks.af = AF_INET6; + } else if (remote_address->sa.sa_family == AF_INET) { + memcpy(&tcks.laddr.addr, &local_address->sin.sin_addr, sizeof(struct in_addr)); + memcpy(&tcks.faddr.addr, &remote_address->sin.sin_addr, sizeof(struct in_addr)); + tcks.af = AF_INET; + } + + if (necp_buffer->necp_tcp_tfo_heuristics_success) + tcp_heuristic_reset_counters(&tcks, TCPCACHE_F_TFO_REQ | TCPCACHE_F_TFO_DATA | + TCPCACHE_F_TFO_REQ_RST | TCPCACHE_F_TFO_DATA_RST); + + if (necp_buffer->necp_tcp_tfo_heuristics_success_req) + tcp_heuristic_reset_counters(&tcks, TCPCACHE_F_TFO_REQ | TCPCACHE_F_TFO_REQ_RST); + + if (necp_buffer->necp_tcp_tfo_heuristics_loss) + tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_TFO_REQ | TCPCACHE_F_TFO_DATA); + + if (necp_buffer->necp_tcp_tfo_heuristics_loss_req) + tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_TFO_REQ); + + if (necp_buffer->necp_tcp_tfo_heuristics_rst_data) + tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_TFO_REQ_RST | TCPCACHE_F_TFO_DATA_RST); + + if (necp_buffer->necp_tcp_tfo_heuristics_rst_req) + tcp_heuristic_inc_counters(&tcks, TCPCACHE_F_TFO_REQ_RST); + + if (necp_buffer->necp_tcp_tfo_heuristics_middlebox) + tcp_heuristic_tfo_middlebox_common(&tcks); + + if (necp_buffer->necp_tcp_tfo_cookie_len != 0) { + tcp_cache_set_cookie_common(&tcks, + necp_buffer->necp_tcp_tfo_cookie, necp_buffer->necp_tcp_tfo_cookie_len); + } + + return; +} + static void sysctl_cleartfocache(void) { int i; diff --git a/bsd/netinet/tcp_cache.h b/bsd/netinet/tcp_cache.h index 4516d7578..bd7044ea3 100644 --- a/bsd/netinet/tcp_cache.h +++ b/bsd/netinet/tcp_cache.h @@ -42,10 +42,9 @@ extern int tcp_cache_get_cookie(struct tcpcb *tp, u_char *cookie, u_int8_t *len) extern unsigned int tcp_cache_get_cookie_len(struct tcpcb *tp); extern void tcp_heuristic_tfo_loss(struct tcpcb *tp); +extern void tcp_heuristic_tfo_rst(struct tcpcb *tp); extern void tcp_heuristic_mptcp_loss(struct tcpcb *tp); extern void tcp_heuristic_ecn_loss(struct tcpcb *tp); -extern void tcp_heuristic_tfo_snd_good(struct tcpcb *tp); -extern void tcp_heuristic_tfo_rcv_good(struct tcpcb *tp); extern void tcp_heuristic_tfo_middlebox(struct tcpcb *tp); extern void tcp_heuristic_ecn_aggressive(struct tcpcb *tp); extern void tcp_heuristic_tfo_success(struct tcpcb *tp); @@ -56,6 +55,18 @@ extern boolean_t tcp_heuristic_do_mptcp(struct tcpcb *tp); extern boolean_t tcp_heuristic_do_ecn(struct tcpcb *tp); extern void tcp_heuristic_ecn_droprst(struct tcpcb *tp); extern void tcp_heuristic_ecn_droprxmt(struct tcpcb *tp); +extern void tcp_heuristic_ecn_synrst(struct tcpcb *tp); + +extern boolean_t tcp_heuristic_do_ecn_with_address(struct ifnet *ifp, + union sockaddr_in_4_6 *local_address); +extern void tcp_heuristics_ecn_update(struct necp_tcp_ecn_cache *necp_buffer, + struct ifnet *ifp, union sockaddr_in_4_6 *local_address); +extern boolean_t tcp_heuristic_do_tfo_with_address(struct ifnet *ifp, + union sockaddr_in_4_6 *local_address, union sockaddr_in_4_6 *remote_address, + u_int8_t *cookie, u_int8_t *cookie_len); +extern void tcp_heuristics_tfo_update(struct necp_tcp_tfo_cache *necp_buffer, + struct ifnet *ifp, union sockaddr_in_4_6 *local_address, + union sockaddr_in_4_6 *remote_address); extern void tcp_cache_init(void); diff --git a/bsd/netinet/tcp_cc.c b/bsd/netinet/tcp_cc.c index a15fd6a07..1f634a174 100644 --- a/bsd/netinet/tcp_cc.c +++ b/bsd/netinet/tcp_cc.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2014 Apple Inc. All rights reserved. + * Copyright (c) 2013-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -76,9 +76,8 @@ struct tcp_cc_debug_state { } u; }; -int tcp_cc_debug = 0; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, cc_debug, CTLFLAG_RW | CTLFLAG_LOCKED, - &tcp_cc_debug, 0, "Enable debug data collection"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, cc_debug, CTLFLAG_RW | CTLFLAG_LOCKED, + int, tcp_cc_debug, 0, "Enable debug data collection"); extern struct tcp_cc_algo tcp_cc_newreno; SYSCTL_INT(_net_inet_tcp, OID_AUTO, newreno_sockets, @@ -95,9 +94,8 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, cubic_sockets, CTLFLAG_RD | CTLFLAG_LOCKED,&tcp_cc_cubic.num_sockets, 0, "Number of sockets using cubic"); -int tcp_use_newreno = 0; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, use_newreno, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_use_newreno, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, use_newreno, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_use_newreno, 0, "Use TCP NewReno by default"); static int tcp_check_cwnd_nonvalidated = 1; @@ -299,7 +297,7 @@ void tcp_bad_rexmt_fix_sndbuf(struct tcpcb *tp) sb = &tp->t_inpcb->inp_socket->so_snd; if ((sb->sb_flags & (SB_TRIM|SB_AUTOSIZE)) == (SB_TRIM|SB_AUTOSIZE)) { /* - * If there was a retransmission that was not necessary + * If there was a retransmission that was not necessary * then the size of socket buffer can be restored to * what it was before */ @@ -426,11 +424,19 @@ tcp_cc_after_idle_stretchack(struct tcpcb *tp) inline uint32_t tcp_cc_is_cwnd_nonvalidated(struct tcpcb *tp) { + struct socket *so = tp->t_inpcb->inp_socket; if (tp->t_pipeack == 0 || tcp_check_cwnd_nonvalidated == 0) { tp->t_flagsext &= ~TF_CWND_NONVALIDATED; return (0); } - if (tp->t_pipeack >= (tp->snd_cwnd) >> 1) + + /* + * The congestion window is validated if the number of bytes acked + * is more than half of the current window or if there is more + * data to send in the send socket buffer + */ + if (tp->t_pipeack >= (tp->snd_cwnd >> 1) || + (so != NULL && so->so_snd.sb_cc > tp->snd_cwnd)) tp->t_flagsext &= ~TF_CWND_NONVALIDATED; else tp->t_flagsext |= TF_CWND_NONVALIDATED; diff --git a/bsd/netinet/tcp_cc.h b/bsd/netinet/tcp_cc.h index e9df6b451..7c83900d1 100644 --- a/bsd/netinet/tcp_cc.h +++ b/bsd/netinet/tcp_cc.h @@ -74,6 +74,8 @@ #define TCP_CA_NAME_MAX 16 /* Maximum characters in the name of a CC algorithm */ +extern int tcp_recv_bg; + /* * Structure to hold definition various actions defined by a congestion * control algorithm for TCP. This can be used to change the congestion diff --git a/bsd/netinet/tcp_cubic.c b/bsd/netinet/tcp_cubic.c index 29a3aed78..c9ce59c04 100644 --- a/bsd/netinet/tcp_cubic.c +++ b/bsd/netinet/tcp_cubic.c @@ -89,20 +89,14 @@ const float tcp_cubic_backoff = 0.2; /* multiplicative decrease factor */ const float tcp_cubic_coeff = 0.4; const float tcp_cubic_fast_convergence_factor = 0.875; -static int tcp_cubic_tcp_friendliness = 0; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, cubic_tcp_friendliness, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_cubic_tcp_friendliness, 0, - "Enable TCP friendliness"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, cubic_tcp_friendliness, CTLFLAG_RW | CTLFLAG_LOCKED, + static int, tcp_cubic_tcp_friendliness, 0, "Enable TCP friendliness"); -static int tcp_cubic_fast_convergence = 0; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, cubic_fast_convergence, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_cubic_fast_convergence, 0, - "Enable fast convergence"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, cubic_fast_convergence, CTLFLAG_RW | CTLFLAG_LOCKED, + static int, tcp_cubic_fast_convergence, 0, "Enable fast convergence"); -static int tcp_cubic_use_minrtt = 0; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, cubic_use_minrtt, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_cubic_use_minrtt, 0, - "use a min of 5 sec rtt"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, cubic_use_minrtt, CTLFLAG_RW | CTLFLAG_LOCKED, + static int, tcp_cubic_use_minrtt, 0, "use a min of 5 sec rtt"); static int tcp_cubic_init(struct tcpcb *tp) { @@ -214,7 +208,7 @@ tcp_cubic_update(struct tcpcb *tp, u_int32_t rtt) var = (elapsed_time - tp->t_ccstate->cub_epoch_period) / TCP_RETRANSHZ; var = var * var * var * (tcp_cubic_coeff * tp->t_maxseg); - tp->t_ccstate->cub_target_win = tp->t_ccstate->cub_origin_point + var; + tp->t_ccstate->cub_target_win = (u_int32_t)(tp->t_ccstate->cub_origin_point + var); return (tp->t_ccstate->cub_target_win); } @@ -355,7 +349,7 @@ tcp_cubic_ack_rcvd(struct tcpcb *tp, struct tcphdr *th) static void tcp_cubic_pre_fr(struct tcpcb *tp) { - uint32_t win, avg; + u_int32_t win, avg; int32_t dev; tp->t_ccstate->cub_epoch_start = 0; tp->t_ccstate->cub_tcp_win = 0; @@ -382,8 +376,8 @@ tcp_cubic_pre_fr(struct tcpcb *tp) */ if (win < tp->t_ccstate->cub_last_max && tcp_cubic_fast_convergence == 1) - tp->t_ccstate->cub_last_max = win * - tcp_cubic_fast_convergence_factor; + tp->t_ccstate->cub_last_max = (u_int32_t)(win * + tcp_cubic_fast_convergence_factor); else tp->t_ccstate->cub_last_max = win; @@ -429,7 +423,7 @@ tcp_cubic_pre_fr(struct tcpcb *tp) } /* Backoff congestion window by tcp_cubic_backoff factor */ - win = win - (win * tcp_cubic_backoff); + win = (u_int32_t)(win - (win * tcp_cubic_backoff)); win = (win / tp->t_maxseg); if (win < 2) win = 2; diff --git a/bsd/netinet/tcp_fsm.h b/bsd/netinet/tcp_fsm.h index b963d865f..2a46e6ca5 100644 --- a/bsd/netinet/tcp_fsm.h +++ b/bsd/netinet/tcp_fsm.h @@ -104,6 +104,10 @@ #define TCPS_HAVERCVDSYN(s) ((s) >= TCPS_SYN_RECEIVED) #define TCPS_HAVEESTABLISHED(s) ((s) >= TCPS_ESTABLISHED) #define TCPS_HAVERCVDFIN(s) ((s) >= TCPS_TIME_WAIT) +#define TCPS_HAVERCVDFIN2(s) ((s) == TCPS_CLOSE_WAIT || \ + (s) == TCPS_CLOSING || \ + (s) == TCPS_LAST_ACK || \ + (s) == TCPS_TIME_WAIT) #ifdef KERNEL_PRIVATE #ifdef TCPOUTFLAGS diff --git a/bsd/netinet/tcp_input.c b/bsd/netinet/tcp_input.c index 7767910b8..741aa00d5 100644 --- a/bsd/netinet/tcp_input.c +++ b/bsd/netinet/tcp_input.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -79,7 +79,10 @@ #include <sys/socketvar.h> #include <sys/syslog.h> #include <sys/mcache.h> +#if !CONFIG_EMBEDDED #include <sys/kasl.h> +#endif +#include <sys/kauth.h> #include <kern/cpu_number.h> /* before tcp_seq.h, for tcp_random18() */ #include <machine/endian.h> @@ -156,8 +159,6 @@ struct tcphdr tcp_savetcp; #define TCP_RECV_THROTTLE_WIN (5 * TCP_RETRANSHZ) #define TCP_STRETCHACK_ENABLE_PKTCNT 2000 -tcp_cc tcp_ccgen; - struct tcpstat tcpstat; static int log_in_vain = 0; @@ -170,24 +171,20 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, blackhole, CTLFLAG_RW | CTLFLAG_LOCKED, &blackhole, 0, "Do not send RST when dropping refused connections"); -int tcp_delack_enabled = 3; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, delayed_ack, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_delack_enabled, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, delayed_ack, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_delack_enabled, 3, "Delay ACK to try and piggyback it onto a data packet"); -int tcp_lq_overflow = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcp_lq_overflow, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_lq_overflow, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, tcp_lq_overflow, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_lq_overflow, 1, "Listen Queue Overflow"); -int tcp_recv_bg = 0; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, recvbg, CTLFLAG_RW | CTLFLAG_LOCKED, - &tcp_recv_bg, 0, "Receive background"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, recvbg, CTLFLAG_RW | CTLFLAG_LOCKED, + int, tcp_recv_bg, 0, "Receive background"); #if TCP_DROP_SYNFIN -static int drop_synfin = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, drop_synfin, - CTLFLAG_RW | CTLFLAG_LOCKED, &drop_synfin, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, drop_synfin, + CTLFLAG_RW | CTLFLAG_LOCKED, static int, drop_synfin, 1, "Drop TCP packets with SYN+FIN set"); #endif @@ -200,32 +197,27 @@ SYSCTL_INT(_net_inet_tcp_reass, OID_AUTO, overflows, "Global number of TCP Segment Reassembly Queue Overflows"); -__private_extern__ int slowlink_wsize = 8192; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, slowlink_wsize, - CTLFLAG_RW | CTLFLAG_LOCKED, - &slowlink_wsize, 0, "Maximum advertised window size for slowlink"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, slowlink_wsize, CTLFLAG_RW | CTLFLAG_LOCKED, + __private_extern__ int, slowlink_wsize, 8192, + "Maximum advertised window size for slowlink"); -int maxseg_unacked = 8; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, maxseg_unacked, - CTLFLAG_RW | CTLFLAG_LOCKED, &maxseg_unacked, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, maxseg_unacked, + CTLFLAG_RW | CTLFLAG_LOCKED, int, maxseg_unacked, 8, "Maximum number of outstanding segments left unacked"); -int tcp_do_rfc3465 = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, rfc3465, CTLFLAG_RW | CTLFLAG_LOCKED, - &tcp_do_rfc3465, 0, ""); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, rfc3465, CTLFLAG_RW | CTLFLAG_LOCKED, + int, tcp_do_rfc3465, 1, ""); -int tcp_do_rfc3465_lim2 = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, rfc3465_lim2, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_do_rfc3465_lim2, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, rfc3465_lim2, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_do_rfc3465_lim2, 1, "Appropriate bytes counting w/ L=2*SMSS"); int rtt_samples_per_slot = 20; -int tcp_allowed_iaj = ALLOWED_IAJ; int tcp_acc_iaj_high_thresh = ACC_IAJ_HIGH_THRESH; u_int32_t tcp_autorcvbuf_inc_shift = 3; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, recv_allowed_iaj, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_allowed_iaj, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, recv_allowed_iaj, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_allowed_iaj, ALLOWED_IAJ, "Allowed inter-packet arrival jiter"); #if (DEVELOPMENT || DEBUG) SYSCTL_INT(_net_inet_tcp, OID_AUTO, acc_iaj_high_thresh, @@ -237,14 +229,12 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, autorcvbufincshift, "Shift for increment in receive socket buffer size"); #endif /* (DEVELOPMENT || DEBUG) */ -u_int32_t tcp_do_autorcvbuf = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, doautorcvbuf, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_do_autorcvbuf, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, doautorcvbuf, + CTLFLAG_RW | CTLFLAG_LOCKED, u_int32_t, tcp_do_autorcvbuf, 1, "Enable automatic socket buffer tuning"); -u_int32_t tcp_autorcvbuf_max = 512 * 1024; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, autorcvbufmax, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_autorcvbuf_max, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, autorcvbufmax, + CTLFLAG_RW | CTLFLAG_LOCKED, u_int32_t, tcp_autorcvbuf_max, 512 * 1024, "Maximum receive socket buffer size"); u_int32_t tcp_autorcvbuf_max_ca = 512 * 1024; @@ -254,7 +244,11 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, autorcvbufmaxca, "Maximum receive socket buffer size"); #endif /* (DEBUG || DEVELOPMENT) */ +#if CONFIG_EMBEDDED +int sw_lro = 1; +#else int sw_lro = 0; +#endif /* !CONFIG_EMBEDDED */ SYSCTL_INT(_net_inet_tcp, OID_AUTO, lro, CTLFLAG_RW | CTLFLAG_LOCKED, &sw_lro, 0, "Used to coalesce TCP packets"); @@ -268,8 +262,6 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, lro_startcnt, CTLFLAG_RW | CTLFLAG_LOCKED, &lro_start, 0, "Segments for starting LRO computed as power of 2"); -extern int tcp_do_autosendbuf; - int limited_txmt = 1; int early_rexmt = 1; int sack_ackadv = 1; @@ -291,7 +283,13 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, sack_ackadv, SYSCTL_INT(_net_inet_tcp, OID_AUTO, dsack_enable, CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_dsack_enable, 0, "use DSACK TCP option to report duplicate segments"); + #endif /* (DEVELOPMENT || DEBUG) */ +int tcp_disable_access_to_stats = 1; +SYSCTL_INT(_net_inet_tcp, OID_AUTO, disable_access_to_stats, + CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_disable_access_to_stats, 0, + "Disable access to tcpstat"); + extern int tcp_TCPTV_MIN; extern int tcp_acc_iaj_high; @@ -330,6 +328,8 @@ static void compute_iaj_meat(struct tcpcb *tp, uint32_t cur_iaj); static inline unsigned int tcp_maxmtu6(struct rtentry *); #endif +unsigned int get_maxmtu(struct rtentry *); + static void tcp_sbrcv_grow(struct tcpcb *tp, struct sockbuf *sb, struct tcpopt *to, u_int32_t tlen, u_int32_t rcvbuf_max); void tcp_sbrcv_trim(struct tcpcb *tp, struct sockbuf *sb); @@ -377,9 +377,8 @@ extern void ipfw_stealth_stats_incr_tcp(void); int tcp_rcvunackwin = TCPTV_UNACKWIN; int tcp_maxrcvidle = TCPTV_MAXRCVIDLE; -int tcp_rcvsspktcnt = TCP_RCV_SS_PKTCOUNT; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, rcvsspktcnt, CTLFLAG_RW | CTLFLAG_LOCKED, - &tcp_rcvsspktcnt, 0, "packets to be seen before receiver stretches acks"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, rcvsspktcnt, CTLFLAG_RW | CTLFLAG_LOCKED, + int, tcp_rcvsspktcnt, TCP_RCV_SS_PKTCOUNT, "packets to be seen before receiver stretches acks"); #define DELAY_ACK(tp, th) \ (CC_ALGO(tp)->delay_ack != NULL && CC_ALGO(tp)->delay_ack(tp, th)) @@ -560,8 +559,12 @@ tcp_bwmeas_check(struct tcpcb *tp) { int32_t bw_meas_bytes; uint32_t bw, bytes, elapsed_time; + + if (SEQ_LEQ(tp->snd_una, tp->t_bwmeas->bw_start)) + return; + bw_meas_bytes = tp->snd_una - tp->t_bwmeas->bw_start; - if ((tp->t_flagsext & TF_BWMEAS_INPROGRESS) != 0 && + if ((tp->t_flagsext & TF_BWMEAS_INPROGRESS) && bw_meas_bytes >= (int32_t)(tp->t_bwmeas->bw_size)) { bytes = bw_meas_bytes; elapsed_time = tcp_now - tp->t_bwmeas->bw_ts; @@ -570,10 +573,22 @@ tcp_bwmeas_check(struct tcpcb *tp) if ( bw > 0) { if (tp->t_bwmeas->bw_sndbw > 0) { tp->t_bwmeas->bw_sndbw = - (((tp->t_bwmeas->bw_sndbw << 3) - tp->t_bwmeas->bw_sndbw) + bw) >> 3; + (((tp->t_bwmeas->bw_sndbw << 3) + - tp->t_bwmeas->bw_sndbw) + + bw) >> 3; } else { tp->t_bwmeas->bw_sndbw = bw; } + + /* Store the maximum value */ + if (tp->t_bwmeas->bw_sndbw_max == 0) { + tp->t_bwmeas->bw_sndbw_max = + tp->t_bwmeas->bw_sndbw; + } else { + tp->t_bwmeas->bw_sndbw_max = + max(tp->t_bwmeas->bw_sndbw, + tp->t_bwmeas->bw_sndbw_max); + } } } tp->t_flagsext &= ~(TF_BWMEAS_INPROGRESS); @@ -692,6 +707,7 @@ tcp_reass(struct tcpcb *tp, struct tcphdr *th, int *tlenp, struct mbuf *m, INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, *tlenp); tp->t_stat.rxduplicatebytes += *tlenp; + inp_set_activity_bitmap(inp); } m_freem(m); zfree(tcp_reass_zone, te); @@ -719,6 +735,7 @@ tcp_reass(struct tcpcb *tp, struct tcphdr *th, int *tlenp, struct mbuf *m, INP_ADD_STAT(inp, cell, wifi, wired, rxpackets, 1); INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, *tlenp); tp->t_stat.rxoutoforderbytes += *tlenp; + inp_set_activity_bitmap(inp); } /* @@ -1171,6 +1188,19 @@ tcp_sbrcv_grow(struct tcpcb *tp, struct sockbuf *sbrcv, sbrcv->sb_hiwat + rcvbuf_inc, (tp->rfbuf_cnt * 2), rcvbuf_max); } + /* Measure instantaneous receive bandwidth */ + if (tp->t_bwmeas != NULL && tp->rfbuf_cnt > 0 && + TSTMP_GT(tcp_now, tp->rfbuf_ts)) { + u_int32_t rcv_bw; + rcv_bw = tp->rfbuf_cnt / + (int)(tcp_now - tp->rfbuf_ts); + if (tp->t_bwmeas->bw_rcvbw_max == 0) { + tp->t_bwmeas->bw_rcvbw_max = rcv_bw; + } else { + tp->t_bwmeas->bw_rcvbw_max = max( + tp->t_bwmeas->bw_rcvbw_max, rcv_bw); + } + } goto out; } else { tp->rfbuf_cnt += pktlen; @@ -1725,30 +1755,24 @@ tcp_tfo_synack(struct tcpcb *tp, struct tcpopt *to) static void tcp_tfo_rcv_probe(struct tcpcb *tp, int tlen) { - if (tlen == 0) { - tp->t_tfo_probe_state = TFO_PROBE_PROBING; + if (tlen != 0) + return; - /* - * We send the probe out rather quickly (after one RTO). It does not - * really hurt that much, it's only one additional segment on the wire. - */ - tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START(tp, (TCP_REXMTVAL(tp))); - } else { - /* If SYN/ACK+data, don't probe. We got the data! */ - tcp_heuristic_tfo_rcv_good(tp); - } + tp->t_tfo_probe_state = TFO_PROBE_PROBING; + + /* + * We send the probe out rather quickly (after one RTO). It does not + * really hurt that much, it's only one additional segment on the wire. + */ + tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START(tp, (TCP_REXMTVAL(tp))); } static void tcp_tfo_rcv_data(struct tcpcb *tp) { /* Transition from PROBING to NONE as data has been received */ - if (tp->t_tfo_probe_state >= TFO_PROBE_PROBING) { + if (tp->t_tfo_probe_state >= TFO_PROBE_PROBING) tp->t_tfo_probe_state = TFO_PROBE_NONE; - - /* Data has been received - we are good to go! */ - tcp_heuristic_tfo_rcv_good(tp); - } } static void @@ -1789,6 +1813,9 @@ tcp_update_window(struct tcpcb *tp, int thflags, struct tcphdr * th, tp->snd_wl2 = th->th_ack; if (tp->snd_wnd > tp->max_sndwnd) tp->max_sndwnd = tp->snd_wnd; + + if (tp->t_inpcb->inp_socket->so_flags & SOF_MP_SUBFLOW) + mptcp_update_window_fallback(tp); return (true); } return (false); @@ -1920,7 +1947,7 @@ tcp_input(struct mbuf *m, int off0) * Note: IP leaves IP header in first mbuf. */ if (off0 > sizeof (struct ip)) { - ip_stripoptions(m, (struct mbuf *)0); + ip_stripoptions(m); off0 = sizeof(struct ip); } if (m->m_len < sizeof (struct tcpiphdr)) { @@ -2209,30 +2236,39 @@ findpcb: goto dropnosock; } - tcp_lock(so, 1, 0); + socket_lock(so, 1); if (in_pcb_checkstate(inp, WNT_RELEASE, 1) == WNT_STOPUSING) { - tcp_unlock(so, 1, (void *)2); + socket_unlock(so, 1); inp = NULL; // pretend we didn't find it goto dropnosock; } #if NECP -#if INET6 - if (isipv6) { - if (!necp_socket_is_allowed_to_send_recv_v6(inp, - th->th_dport, th->th_sport, &ip6->ip6_dst, - &ip6->ip6_src, ifp, NULL, NULL)) { - IF_TCP_STATINC(ifp, badformatipsec); + if (so->so_state & SS_ISCONNECTED) { + // Connected TCP sockets have a fully-bound local and remote, + // so the policy check doesn't need to override addresses + if (!necp_socket_is_allowed_to_send_recv(inp, NULL, NULL)) { + IF_TCP_STATINC(ifp, badformat); goto drop; } - } else + } else { +#if INET6 + if (isipv6) { + if (!necp_socket_is_allowed_to_send_recv_v6(inp, + th->th_dport, th->th_sport, &ip6->ip6_dst, + &ip6->ip6_src, ifp, NULL, NULL)) { + IF_TCP_STATINC(ifp, badformat); + goto drop; + } + } else #endif - { - if (!necp_socket_is_allowed_to_send_recv_v4(inp, - th->th_dport, th->th_sport, &ip->ip_dst, &ip->ip_src, - ifp, NULL, NULL)) { - IF_TCP_STATINC(ifp, badformatipsec); - goto drop; + { + if (!necp_socket_is_allowed_to_send_recv_v4(inp, + th->th_dport, th->th_sport, &ip->ip_dst, &ip->ip_src, + ifp, NULL, NULL)) { + IF_TCP_STATINC(ifp, badformat); + goto drop; + } } } #endif /* NECP */ @@ -2433,10 +2469,10 @@ findpcb: tp = intotcpcb(inp); oso = so; - tcp_unlock(so, 0, 0); /* Unlock but keep a reference on listener for now */ + socket_unlock(so, 0); /* Unlock but keep a reference on listener for now */ so = so2; - tcp_lock(so, 1, 0); + socket_lock(so, 1); /* * Mark socket as temporary until we're * committed to keeping it. The code at @@ -2503,38 +2539,38 @@ findpcb: #endif /* INET6 */ inp->inp_laddr.s_addr = INADDR_ANY; inp->inp_lport = 0; - tcp_lock(oso, 0, 0); /* release ref on parent */ - tcp_unlock(oso, 1, 0); + socket_lock(oso, 0); /* release ref on parent */ + socket_unlock(oso, 1); goto drop; } #if INET6 if (isipv6) { - /* - * Inherit socket options from the listening - * socket. - * Note that in6p_inputopts are not (even - * should not be) copied, since it stores + /* + * Inherit socket options from the listening + * socket. + * Note that in6p_inputopts are not (even + * should not be) copied, since it stores * previously received options and is used to - * detect if each new option is different than - * the previous one and hence should be passed - * to a user. - * If we copied in6p_inputopts, a user would - * not be able to receive options just after - * calling the accept system call. - */ + * detect if each new option is different than + * the previous one and hence should be passed + * to a user. + * If we copied in6p_inputopts, a user would + * not be able to receive options just after + * calling the accept system call. + */ inp->inp_flags |= oinp->inp_flags & INP_CONTROLOPTS; - if (oinp->in6p_outputopts) - inp->in6p_outputopts = - ip6_copypktopts(oinp->in6p_outputopts, - M_NOWAIT); + if (oinp->in6p_outputopts) + inp->in6p_outputopts = + ip6_copypktopts(oinp->in6p_outputopts, + M_NOWAIT); } else #endif /* INET6 */ { inp->inp_options = ip_srcroute(); inp->inp_ip_tos = oinp->inp_ip_tos; } - tcp_lock(oso, 0, 0); + socket_lock(oso, 0); #if IPSEC /* copy old policy into new socket's */ if (sotoinpcb(oso)->inp_sp) @@ -2565,15 +2601,14 @@ findpcb: tp0->t_inpcb->inp_flags2 & INP2_KEEPALIVE_OFFLOAD; /* now drop the reference on the listener */ - tcp_unlock(oso, 1, 0); + socket_unlock(oso, 1); tcp_set_max_rwinscale(tp, so, TCP_AUTORCVBUF_MAX(ifp)); KERNEL_DEBUG(DBG_FNC_TCP_NEWCONN | DBG_FUNC_END,0,0,0,0,0); } } - lck_mtx_assert(&((struct inpcb *)so->so_pcb)->inpcb_mtx, - LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(so); if (tp->t_state == TCPS_ESTABLISHED && tlen > 0) { /* @@ -2693,9 +2728,13 @@ findpcb: * Segment received on connection. * Reset idle time and keep-alive timer. */ - if (TCPS_HAVEESTABLISHED(tp->t_state)) + if (TCPS_HAVEESTABLISHED(tp->t_state)) { tcp_keepalive_reset(tp); + if (tp->t_mpsub) + mptcp_reset_keepalive(tp); + } + /* * Process options if not in LISTEN state, * else do it below (after getting remote address). @@ -2707,7 +2746,7 @@ findpcb: tp->t_flags |= TF_ACKNOW; (void) tcp_output(tp); tcp_check_timer_state(tp); - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); KERNEL_DEBUG(DBG_FNC_TCP_INPUT | DBG_FUNC_END,0,0,0,0,0); return; @@ -2928,7 +2967,7 @@ findpcb: tcp_tfo_rcv_ack(tp, th); tcp_check_timer_state(tp); - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); KERNEL_DEBUG(DBG_FNC_TCP_INPUT | DBG_FUNC_END,0,0,0,0,0); return; } @@ -2942,7 +2981,7 @@ findpcb: */ /* - * If this is a connection in steady state, start + * If this is a connection in steady state, start * coalescing packets belonging to this flow. */ if (turnoff_lro) { @@ -2992,6 +3031,7 @@ findpcb: } INP_ADD_STAT(inp, cell, wifi, wired,rxbytes, tlen); + inp_set_activity_bitmap(inp); } /* @@ -3050,7 +3090,7 @@ findpcb: tcp_tfo_rcv_data(tp); tcp_check_timer_state(tp); - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); KERNEL_DEBUG(DBG_FNC_TCP_INPUT | DBG_FUNC_END,0,0,0,0,0); return; } @@ -3062,8 +3102,7 @@ findpcb: * Receive window is amount of space in rcv queue, * but not less than advertised window. */ - lck_mtx_assert(&((struct inpcb *)so->so_pcb)->inpcb_mtx, - LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(so); win = tcp_sbspace(tp); if (win < 0) win = 0; @@ -3079,12 +3118,11 @@ findpcb: */ if ((tp->t_mpflags & TMPF_MPTCP_TRUE) && (mp_tp = tptomptp(tp))) { - MPT_LOCK(mp_tp); + mpte_lock_assert_held(mp_tp->mpt_mpte); if (tp->rcv_wnd > mp_tp->mpt_rcvwnd) { tp->rcv_wnd = imax(mp_tp->mpt_rcvwnd, (int)(tp->rcv_adv - tp->rcv_nxt)); tcpstat.tcps_mp_reducedwin++; } - MPT_UNLOCK(mp_tp); } #endif /* MPTCP */ @@ -3105,8 +3143,7 @@ findpcb: struct sockaddr_in6 *sin6; #endif - lck_mtx_assert(&((struct inpcb *)so->so_pcb)->inpcb_mtx, - LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(so); #if INET6 if (isipv6) { MALLOC(sin6, struct sockaddr_in6 *, sizeof *sin6, @@ -3131,11 +3168,9 @@ findpcb: } else #endif { - lck_mtx_assert( - &((struct inpcb *)so->so_pcb)->inpcb_mtx, - LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(so); MALLOC(sin, struct sockaddr_in *, sizeof *sin, M_SONAME, - M_NOWAIT); + M_NOWAIT); if (sin == NULL) goto drop; sin->sin_family = AF_INET; @@ -3255,14 +3290,21 @@ findpcb: } if (thflags & TH_RST) { if ((thflags & TH_ACK) != 0) { -#if MPTCP - if ((so->so_flags & SOF_MPTCP_FASTJOIN) && - SEQ_GT(th->th_ack, tp->iss+1)) { - so->so_flags &= ~SOF_MPTCP_FASTJOIN; - /* ignore the RST and retransmit SYN */ - goto drop; + if (tfo_enabled(tp)) + tcp_heuristic_tfo_rst(tp); + if ((tp->ecn_flags & (TE_SETUPSENT | TE_RCVD_SYN_RST)) == TE_SETUPSENT) { + /* + * On local connections, send + * non-ECN syn one time before + * dropping the connection + */ + if (tp->t_flags & TF_LOCAL) { + tp->ecn_flags |= TE_RCVD_SYN_RST; + goto drop; + } else { + tcp_heuristic_ecn_synrst(tp); + } } -#endif /* MPTCP */ soevent(so, (SO_FILT_HINT_LOCKED | SO_FILT_HINT_CONNRESET)); @@ -3323,8 +3365,24 @@ findpcb: * We should restart the sending from what the receiver * has acknowledged immediately. */ - if (SEQ_GT(tp->snd_nxt, th->th_ack)) + if (SEQ_GT(tp->snd_nxt, th->th_ack)) { + /* + * rdar://problem/33214601 + * There is a middlebox that acks all but one + * byte and still drops the data. + */ + if ((tp->t_tfo_stats & TFO_S_SYN_DATA_SENT) && + tp->snd_max == th->th_ack + 1 && + tp->snd_max > tp->snd_una + 1) { + tcp_heuristic_tfo_middlebox(tp); + + so->so_error = ENODATA; + + tp->t_tfo_stats |= TFO_S_ONE_BYTE_PROXY; + } + tp->snd_max = tp->snd_nxt = th->th_ack; + } /* * If there's data, delay ACK; if there's also a FIN @@ -3382,11 +3440,6 @@ findpcb: if ((!(tp->t_mpflags & TMPF_MPTCP_TRUE)) && (tp->t_mpflags & TMPF_SENT_JOIN)) { isconnected = FALSE; - /* Start data xmit if fastjoin */ - if (mptcp_fastjoin && (so->so_flags & SOF_MPTCP_FASTJOIN)) { - soevent(so, (SO_FILT_HINT_LOCKED | - SO_FILT_HINT_MPFASTJ)); - } } else #endif /* MPTCP */ isconnected = TRUE; @@ -3403,14 +3456,13 @@ findpcb: if (so->so_flags & SOF_MP_SUBFLOW) so->so_flags1 |= SOF1_TFO_REWIND; #endif - if (!(tp->t_tfo_flags & TFO_F_NO_RCVPROBING)) - tcp_tfo_rcv_probe(tp, tlen); + tcp_tfo_rcv_probe(tp, tlen); } } } else { /* * Received initial SYN in SYN-SENT[*] state => simul- - * taneous open. If segment contains CC option and there is + * taneous open. If segment contains CC option and there is * a cached CC, apply TAO test; if it succeeds, connection is * half-synchronized. Otherwise, do 3-way handshake: * SYN-SENT -> SYN-RECEIVED @@ -3654,6 +3706,7 @@ trimthenstep6: INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, tlen); tp->t_stat.rxduplicatebytes += tlen; + inp_set_activity_bitmap(inp); } if (tlen > 0) goto dropafterack; @@ -3730,6 +3783,7 @@ trimthenstep6: INP_ADD_STAT(inp, cell, wifi, wired, rxpackets, 1); INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, todrop); tp->t_stat.rxduplicatebytes += todrop; + inp_set_activity_bitmap(inp); } drop_hdrlen += todrop; /* drop from the top afterwards */ th->th_seq += todrop; @@ -3748,19 +3802,30 @@ trimthenstep6: * Send also a RST when we received a data segment after we've * sent our FIN when the socket is defunct. * Note that an MPTCP subflow socket would have SS_NOFDREF set - * by default so check to make sure that we test for SOF_MP_SUBFLOW - * socket flag (which would be cleared when the socket is closed.) + * by default. So, if it's an MPTCP-subflow we rather check the + * MPTCP-level's socket state for SS_NOFDREF. */ - if (!(so->so_flags & SOF_MP_SUBFLOW) && tlen && - (((so->so_state & SS_NOFDREF) && - tp->t_state > TCPS_CLOSE_WAIT) || - ((so->so_flags & SOF_DEFUNCT) && - tp->t_state > TCPS_FIN_WAIT_1))) { - tp = tcp_close(tp); - tcpstat.tcps_rcvafterclose++; - rstreason = BANDLIM_UNLIMITED; - IF_TCP_STATINC(ifp, cleanup); - goto dropwithreset; + if (tlen) { + boolean_t close_it = FALSE; + + if (!(so->so_flags & SOF_MP_SUBFLOW) && (so->so_state & SS_NOFDREF) && + tp->t_state > TCPS_CLOSE_WAIT) + close_it = TRUE; + + if ((so->so_flags & SOF_MP_SUBFLOW) && (mptetoso(tptomptp(tp)->mpt_mpte)->so_state & SS_NOFDREF) && + tp->t_state > TCPS_CLOSE_WAIT) + close_it = TRUE; + + if ((so->so_flags & SOF_DEFUNCT) && tp->t_state > TCPS_FIN_WAIT_1) + close_it = TRUE; + + if (close_it) { + tp = tcp_close(tp); + tcpstat.tcps_rcvafterclose++; + rstreason = BANDLIM_UNLIMITED; + IF_TCP_STATINC(ifp, cleanup); + goto dropwithreset; + } } /* @@ -3783,7 +3848,7 @@ trimthenstep6: SEQ_GT(th->th_seq, tp->rcv_nxt)) { iss = tcp_new_isn(tp); tp = tcp_close(tp); - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); goto findpcb; } /* @@ -4044,7 +4109,7 @@ trimthenstep6: tcp_sack_doack(tp, &to, th, &sack_bytes_acked); #if MPTCP - if ((tp->t_mpuna) && (SEQ_GEQ(th->th_ack, tp->t_mpuna))) { + if (tp->t_mpuna && SEQ_GEQ(th->th_ack, tp->t_mpuna)) { if (tp->t_mpflags & TMPF_PREESTABLISHED) { /* MP TCP establishment succeeded */ tp->t_mpuna = 0; @@ -4068,7 +4133,6 @@ trimthenstep6: } } else { isconnected = TRUE; - tp->t_mpflags &= ~TMPF_SENT_KEYS; } } } @@ -4482,7 +4546,7 @@ process_ACK: if ((tp->t_tfo_stats & TFO_S_SYN_DATA_SENT) && !(tp->t_tfo_flags & TFO_F_NO_SNDPROBING) && !(th->th_flags & TH_SYN)) - tcp_heuristic_tfo_snd_good(tp); + tp->t_tfo_flags |= TFO_F_NO_SNDPROBING; /* * If TH_ECE is received, make sure that ECN is enabled @@ -4832,6 +4896,7 @@ dodata: } INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, tlen); + inp_set_activity_bitmap(inp); } tcp_sbrcv_grow(tp, &so->so_rcv, &to, tlen, TCP_AUTORCVBUF_MAX(ifp)); @@ -4974,7 +5039,7 @@ dodata: tcp_check_timer_state(tp); - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); KERNEL_DEBUG(DBG_FNC_TCP_INPUT | DBG_FUNC_END,0,0,0,0,0); return; @@ -5011,7 +5076,7 @@ dropafterack: (void) tcp_output(tp); /* Don't need to check timer state as we should have done it during tcp_output */ - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); KERNEL_DEBUG(DBG_FNC_TCP_INPUT | DBG_FUNC_END,0,0,0,0,0); return; dropwithresetnosock: @@ -5069,9 +5134,9 @@ dropwithreset: /* destroy temporarily created socket */ if (dropsocket) { (void) soabort(so); - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); } else if ((inp != NULL) && (nosock == 0)) { - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); } KERNEL_DEBUG(DBG_FNC_TCP_INPUT | DBG_FUNC_END,0,0,0,0,0); return; @@ -5090,10 +5155,10 @@ drop: /* destroy temporarily created socket */ if (dropsocket) { (void) soabort(so); - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); } else if (nosock == 0) { - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); } KERNEL_DEBUG(DBG_FNC_TCP_INPUT | DBG_FUNC_END,0,0,0,0,0); return; @@ -5567,6 +5632,26 @@ tcp_maxmtu6(struct rtentry *rt) } #endif +unsigned int +get_maxmtu(struct rtentry *rt) +{ + unsigned int maxmtu = 0; + + RT_LOCK_ASSERT_NOTHELD(rt); + + RT_LOCK(rt); + + if (rt_key(rt)->sa_family == AF_INET6) { + maxmtu = tcp_maxmtu6(rt); + } else { + maxmtu = tcp_maxmtu(rt); + } + + RT_UNLOCK(rt); + + return (maxmtu); +} + /* * Determine a reasonable value for maxseg size. * If the route is known, check route for mtu. @@ -5944,7 +6029,7 @@ tcp_dropdropablreq(struct socket *head) return (0); so_acquire_accept_list(head, NULL); - socket_unlock(head, NULL); + socket_unlock(head, 0); /* * Check if there is any socket in the incomp queue @@ -5965,7 +6050,7 @@ tcp_dropdropablreq(struct socket *head) * issues because it is in the incomp queue and * is not visible to others. */ - if (lck_mtx_try_lock(&inp->inpcb_mtx)) { + if (socket_try_lock(so)) { so->so_usecount++; goto found_victim; } else { @@ -6000,8 +6085,7 @@ tcp_dropdropablreq(struct socket *head) sonext = TAILQ_NEXT(so, so_list); - if (in_pcb_checkstate(inp, WNT_ACQUIRE, 0) - != WNT_STOPUSING) { + if (in_pcb_checkstate(inp, WNT_ACQUIRE, 0) != WNT_STOPUSING) { /* * Avoid the issue of a socket being accepted * by one input thread and being dropped by @@ -6009,7 +6093,7 @@ tcp_dropdropablreq(struct socket *head) * on this mutex, then grab the next socket in * line. */ - if (lck_mtx_try_lock(&inp->inpcb_mtx)) { + if (socket_try_lock(so)) { so->so_usecount++; if ((so->so_usecount == 2) && (so->so_state & SS_INCOMP) && @@ -6021,7 +6105,7 @@ tcp_dropdropablreq(struct socket *head) * used in any other way */ in_pcb_checkstate(inp, WNT_RELEASE, 1); - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); } } else { /* @@ -6044,7 +6128,7 @@ tcp_dropdropablreq(struct socket *head) /* Makes sure socket is still in the right state to be discarded */ if (in_pcb_checkstate(inp, WNT_RELEASE, 1) == WNT_STOPUSING) { - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); socket_lock(head, 0); so_release_accept_list(head); return (0); @@ -6053,7 +6137,7 @@ tcp_dropdropablreq(struct socket *head) found_victim: if (so->so_usecount != 2 || !(so->so_state & SS_INCOMP)) { /* do not discard: that socket is being accepted */ - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); socket_lock(head, 0); so_release_accept_list(head); return (0); @@ -6067,9 +6151,9 @@ found_victim: so->so_flags |= SOF_OVERFLOW; so->so_head = NULL; so_release_accept_list(head); - tcp_unlock(head, 0, 0); + socket_unlock(head, 0); - lck_mtx_assert(&inp->inpcb_mtx, LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(so); tp = sototcpcb(so); tcp_close(tp); @@ -6083,18 +6167,18 @@ found_victim: */ VERIFY(so->so_usecount > 0); so->so_usecount--; - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); } else { /* * Unlock this socket and leave the reference on. * We need to acquire the pcbinfo lock in order to * fully dispose it off */ - tcp_unlock(so, 0, 0); + socket_unlock(so, 0); lck_rw_lock_exclusive(tcbinfo.ipi_lock); - tcp_lock(so, 0, 0); + socket_lock(so, 0); /* Release the reference held for so_incomp queue */ VERIFY(so->so_usecount > 0); so->so_usecount--; @@ -6108,7 +6192,7 @@ found_victim: * was unlocked. This socket will have to be * garbage collected later */ - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); } else { /* Drop the reference held for this function */ VERIFY(so->so_usecount > 0); @@ -6120,7 +6204,7 @@ found_victim: } tcpstat.tcps_drops++; - tcp_lock(head, 0, 0); + socket_lock(head, 0); return(1); } @@ -6227,12 +6311,14 @@ tcp_getstat SYSCTL_HANDLER_ARGS #pragma unused(oidp, arg1, arg2) int error; - + struct tcpstat *stat; + stat = &tcpstat; +#if !CONFIG_EMBEDDED proc_t caller = PROC_NULL; proc_t caller_parent = PROC_NULL; char command_name[MAXCOMLEN + 1] = ""; char parent_name[MAXCOMLEN + 1] = ""; - + struct tcpstat zero_stat; if ((caller = proc_self()) != PROC_NULL) { /* get process name */ strlcpy(command_name, caller->p_comm, sizeof(command_name)); @@ -6262,12 +6348,19 @@ tcp_getstat SYSCTL_HANDLER_ARGS } if (caller != PROC_NULL) proc_rele(caller); + if (tcp_disable_access_to_stats && + !kauth_cred_issuser(kauth_cred_get())) { + bzero(&zero_stat, sizeof(zero_stat)); + stat = &zero_stat; + } + +#endif /* !CONFIG_EMBEDDED */ if (req->oldptr == 0) { req->oldlen= (size_t)sizeof(struct tcpstat); } - error = SYSCTL_OUT(req, &tcpstat, MIN(sizeof (tcpstat), req->oldlen)); + error = SYSCTL_OUT(req, stat, MIN(sizeof (tcpstat), req->oldlen)); return (error); @@ -6289,39 +6382,65 @@ tcp_input_checksum(int af, struct mbuf *m, struct tcphdr *th, int off, int tlen) if (m->m_pkthdr.pkt_flags & PKTF_SW_LRO_DID_CSUM) return (0); + /* ip_stripoptions() must have been called before we get here */ + ASSERT((ip->ip_hl << 2) == sizeof (*ip)); + if ((hwcksum_rx || (ifp->if_flags & IFF_LOOPBACK) || (m->m_pkthdr.pkt_flags & PKTF_LOOP)) && (m->m_pkthdr.csum_flags & CSUM_DATA_VALID)) { if (m->m_pkthdr.csum_flags & CSUM_PSEUDO_HDR) { th->th_sum = m->m_pkthdr.csum_rx_val; } else { - uint16_t sum = m->m_pkthdr.csum_rx_val; - uint16_t start = m->m_pkthdr.csum_rx_start; + uint32_t sum = m->m_pkthdr.csum_rx_val; + uint32_t start = m->m_pkthdr.csum_rx_start; + int32_t trailer = (m_pktlen(m) - (off + tlen)); /* * Perform 1's complement adjustment of octets * that got included/excluded in the hardware- * calculated checksum value. Ignore cases - * where the value includes or excludes the IP - * header span, as the sum for those octets - * would already be 0xffff and thus no-op. + * where the value already includes the entire + * IP header span, as the sum for those octets + * would already be 0 by the time we get here; + * IP has already performed its header checksum + * checks. If we do need to adjust, restore + * the original fields in the IP header when + * computing the adjustment value. Also take + * care of any trailing bytes and subtract out + * their partial sum. */ + ASSERT(trailer >= 0); if ((m->m_pkthdr.csum_flags & CSUM_PARTIAL) && - start != 0 && (off - start) != off) { -#if BYTE_ORDER != BIG_ENDIAN + ((start != 0 && start != off) || trailer)) { + uint32_t swbytes = (uint32_t)trailer; + if (start < off) { + ip->ip_len += sizeof (*ip); +#if BYTE_ORDER != BIG_ENDIAN HTONS(ip->ip_len); HTONS(ip->ip_off); +#endif /* BYTE_ORDER != BIG_ENDIAN */ } -#endif /* callee folds in sum */ - sum = m_adj_sum16(m, start, off, sum); -#if BYTE_ORDER != BIG_ENDIAN + sum = m_adj_sum16(m, start, off, + tlen, sum); + if (off > start) + swbytes += (off - start); + else + swbytes += (start - off); + if (start < off) { +#if BYTE_ORDER != BIG_ENDIAN NTOHS(ip->ip_off); NTOHS(ip->ip_len); +#endif /* BYTE_ORDER != BIG_ENDIAN */ + ip->ip_len -= sizeof (*ip); } -#endif + + if (swbytes != 0) + tcp_in_cksum_stats(swbytes); + if (trailer != 0) + m_adj(m, -trailer); } /* callee folds in sum */ @@ -6364,17 +6483,22 @@ tcp_input_checksum(int af, struct mbuf *m, struct tcphdr *th, int off, int tlen) if (m->m_pkthdr.csum_flags & CSUM_PSEUDO_HDR) { th->th_sum = m->m_pkthdr.csum_rx_val; } else { - uint16_t sum = m->m_pkthdr.csum_rx_val; - uint16_t start = m->m_pkthdr.csum_rx_start; + uint32_t sum = m->m_pkthdr.csum_rx_val; + uint32_t start = m->m_pkthdr.csum_rx_start; + int32_t trailer = (m_pktlen(m) - (off + tlen)); /* * Perform 1's complement adjustment of octets * that got included/excluded in the hardware- - * calculated checksum value. + * calculated checksum value. Also take care + * of any trailing bytes and subtract out their + * partial sum. */ + ASSERT(trailer >= 0); if ((m->m_pkthdr.csum_flags & CSUM_PARTIAL) && - start != off) { - uint16_t s, d; + (start != off || trailer != 0)) { + uint16_t s = 0, d = 0; + uint32_t swbytes = (uint32_t)trailer; if (IN6_IS_SCOPE_EMBED(&ip6->ip6_src)) { s = ip6->ip6_src.s6_addr16[1]; @@ -6386,12 +6510,22 @@ tcp_input_checksum(int af, struct mbuf *m, struct tcphdr *th, int off, int tlen) } /* callee folds in sum */ - sum = m_adj_sum16(m, start, off, sum); + sum = m_adj_sum16(m, start, off, + tlen, sum); + if (off > start) + swbytes += (off - start); + else + swbytes += (start - off); if (IN6_IS_SCOPE_EMBED(&ip6->ip6_src)) ip6->ip6_src.s6_addr16[1] = s; if (IN6_IS_SCOPE_EMBED(&ip6->ip6_dst)) ip6->ip6_dst.s6_addr16[1] = d; + + if (swbytes != 0) + tcp_in6_cksum_stats(swbytes); + if (trailer != 0) + m_adj(m, -trailer); } th->th_sum = in6_pseudo( diff --git a/bsd/netinet/tcp_ledbat.c b/bsd/netinet/tcp_ledbat.c index 457233d8d..fd066b42a 100644 --- a/bsd/netinet/tcp_ledbat.c +++ b/bsd/netinet/tcp_ledbat.c @@ -91,9 +91,8 @@ struct tcp_cc_algo tcp_cc_ledbat = { * The LEDBAT draft says that target queue delay MUST be 100 ms for * inter-operability. */ -int target_qdelay = 100; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, bg_target_qdelay, CTLFLAG_RW | CTLFLAG_LOCKED, - &target_qdelay , 100, "Target queuing delay"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, bg_target_qdelay, CTLFLAG_RW | CTLFLAG_LOCKED, + int, target_qdelay, 100, "Target queuing delay"); /* Allowed increase and tether are used to place an upper bound on * congestion window based on the amount of data that is outstanding. @@ -112,23 +111,21 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, bg_target_qdelay, CTLFLAG_RW | CTLFLAG_LOCKE * 'Tether' is also set to 2. We do not want this to limit the growth of cwnd * during slow-start. */ -int allowed_increase = 8; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, bg_allowed_increase, CTLFLAG_RW | CTLFLAG_LOCKED, - &allowed_increase, 1, "Additive constant used to calculate max allowed congestion window"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, bg_allowed_increase, CTLFLAG_RW | CTLFLAG_LOCKED, + int, allowed_increase, 8, + "Additive constant used to calculate max allowed congestion window"); /* Left shift for cwnd to get tether value of 2 */ -int tether_shift = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, bg_tether_shift, CTLFLAG_RW | CTLFLAG_LOCKED, - &tether_shift, 1, "Tether shift for max allowed congestion window"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, bg_tether_shift, CTLFLAG_RW | CTLFLAG_LOCKED, + int, tether_shift, 1, "Tether shift for max allowed congestion window"); /* Start with an initial window of 2. This will help to get more accurate * minimum RTT measurement in the beginning. It will help to probe * the path slowly and will not add to the existing delay if the path is * already congested. Using 2 packets will reduce the delay induced by delayed-ack. */ -uint32_t bg_ss_fltsz = 2; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, bg_ss_fltsz, CTLFLAG_RW | CTLFLAG_LOCKED, - &bg_ss_fltsz, 2, "Initial congestion window for background transport"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, bg_ss_fltsz, CTLFLAG_RW | CTLFLAG_LOCKED, + uint32_t, bg_ss_fltsz, 2, "Initial congestion window for background transport"); extern int rtt_samples_per_slot; diff --git a/bsd/netinet/tcp_lro.c b/bsd/netinet/tcp_lro.c index 59ee6f445..bc1f21b6d 100644 --- a/bsd/netinet/tcp_lro.c +++ b/bsd/netinet/tcp_lro.c @@ -460,6 +460,7 @@ tcp_lro_process_pkt(struct mbuf *lro_mb, int drop_hdrlen) * quickly get the values now and not bother calling * tcp_dooptions(), etc. */ + bzero(&to, sizeof(to)); if ((optlen == TCPOLEN_TSTAMP_APPA || (optlen > TCPOLEN_TSTAMP_APPA && optp[TCPOLEN_TSTAMP_APPA] == TCPOPT_EOL)) && diff --git a/bsd/netinet/tcp_output.c b/bsd/netinet/tcp_output.c index f37afed3a..88ced34e9 100644 --- a/bsd/netinet/tcp_output.c +++ b/bsd/netinet/tcp_output.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -136,28 +136,24 @@ #define DBG_LAYER_END NETDBG_CODE(DBG_NETTCP, 3) #define DBG_FNC_TCP_OUTPUT NETDBG_CODE(DBG_NETTCP, (4 << 8) | 1) -int path_mtu_discovery = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, path_mtu_discovery, - CTLFLAG_RW | CTLFLAG_LOCKED, &path_mtu_discovery, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, path_mtu_discovery, + CTLFLAG_RW | CTLFLAG_LOCKED, int, path_mtu_discovery, 1, "Enable Path MTU Discovery"); -int ss_fltsz = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, slowstart_flightsize, - CTLFLAG_RW | CTLFLAG_LOCKED,&ss_fltsz, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, slowstart_flightsize, + CTLFLAG_RW | CTLFLAG_LOCKED, int, ss_fltsz, 1, "Slow start flight size"); -int ss_fltsz_local = 8; /* starts with eight segments max */ -SYSCTL_INT(_net_inet_tcp, OID_AUTO, local_slowstart_flightsize, - CTLFLAG_RW | CTLFLAG_LOCKED, &ss_fltsz_local, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, local_slowstart_flightsize, + CTLFLAG_RW | CTLFLAG_LOCKED, int, ss_fltsz_local, 8, "Slow start flight size for local networks"); int tcp_do_tso = 1; SYSCTL_INT(_net_inet_tcp, OID_AUTO, tso, CTLFLAG_RW | CTLFLAG_LOCKED, - &tcp_do_tso, 0, "Enable TCP Segmentation Offload"); + &tcp_do_tso, 0, "Enable TCP Segmentation Offload"); -int tcp_ecn_setup_percentage = 50; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, ecn_setup_percentage, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_ecn_setup_percentage, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, ecn_setup_percentage, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_ecn_setup_percentage, 100, "Max ECN setup percentage"); static int @@ -176,6 +172,7 @@ sysctl_change_ecn_setting SYSCTL_HANDLER_ARGS if ((tcp_ecn_outbound == 0 || tcp_ecn_outbound == 1) && (i == 0 || i == 1)) { tcp_ecn_outbound = i; + SYSCTL_SKMEM_UPDATE_FIELD(tcp.ecn_initiate_out, tcp_ecn_outbound); return(err); } if (tcp_ecn_outbound == 2 && (i == 0 || i == 1)) { @@ -211,10 +208,13 @@ sysctl_change_ecn_setting SYSCTL_HANDLER_ARGS ifnet_head_done(); } tcp_ecn_outbound = i; + SYSCTL_SKMEM_UPDATE_FIELD(tcp.ecn_initiate_out, tcp_ecn_outbound); } /* Change the other one too as the work is done */ - if (i == 2 || tcp_ecn_inbound == 2) + if (i == 2 || tcp_ecn_inbound == 2) { tcp_ecn_inbound = i; + SYSCTL_SKMEM_UPDATE_FIELD(tcp.ecn_negotiate_in, tcp_ecn_inbound); + } return (err); } @@ -230,65 +230,53 @@ SYSCTL_PROC(_net_inet_tcp, OID_AUTO, ecn_negotiate_in, sysctl_change_ecn_setting, "IU", "Initiate ECN for inbound connections"); -int tcp_packet_chaining = 50; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, packetchain, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_packet_chaining, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, packetchain, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_packet_chaining, 50, "Enable TCP output packet chaining"); -int tcp_output_unlocked = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, socket_unlocked_on_output, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_output_unlocked, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, socket_unlocked_on_output, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_output_unlocked, 1, "Unlock TCP when sending packets down to IP"); -int tcp_do_rfc3390 = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, rfc3390, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_do_rfc3390, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, rfc3390, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_do_rfc3390, 1, "Calculate intial slowstart cwnd depending on MSS"); -int tcp_min_iaj_win = MIN_IAJ_WIN; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, min_iaj_win, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_min_iaj_win, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, min_iaj_win, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_min_iaj_win, MIN_IAJ_WIN, "Minimum recv win based on inter-packet arrival jitter"); -int tcp_acc_iaj_react_limit = ACC_IAJ_REACT_LIMIT; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, acc_iaj_react_limit, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_acc_iaj_react_limit, 1, - "Accumulated IAJ when receiver starts to react"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, acc_iaj_react_limit, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_acc_iaj_react_limit, + ACC_IAJ_REACT_LIMIT, "Accumulated IAJ when receiver starts to react"); -uint32_t tcp_do_autosendbuf = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, doautosndbuf, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_do_autosendbuf, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, doautosndbuf, + CTLFLAG_RW | CTLFLAG_LOCKED, uint32_t, tcp_do_autosendbuf, 1, "Enable send socket buffer auto-tuning"); -uint32_t tcp_autosndbuf_inc = 8 * 1024; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, autosndbufinc, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_autosndbuf_inc, 1, - "Increment in send socket bufffer size"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, autosndbufinc, + CTLFLAG_RW | CTLFLAG_LOCKED, uint32_t, tcp_autosndbuf_inc, + 8 * 1024, "Increment in send socket bufffer size"); -uint32_t tcp_autosndbuf_max = 512 * 1024; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, autosndbufmax, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_autosndbuf_max, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, autosndbufmax, + CTLFLAG_RW | CTLFLAG_LOCKED, uint32_t, tcp_autosndbuf_max, 512 * 1024, "Maximum send socket buffer size"); -uint32_t tcp_prioritize_acks = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, ack_prioritize, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_prioritize_acks, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, ack_prioritize, + CTLFLAG_RW | CTLFLAG_LOCKED, uint32_t, tcp_prioritize_acks, 1, "Prioritize pure acks"); -uint32_t tcp_use_rtt_recvbg = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, rtt_recvbg, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_use_rtt_recvbg, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, rtt_recvbg, + CTLFLAG_RW | CTLFLAG_LOCKED, uint32_t, tcp_use_rtt_recvbg, 1, "Use RTT for bg recv algorithm"); -uint32_t tcp_recv_throttle_minwin = 16 * 1024; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, recv_throttle_minwin, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_recv_throttle_minwin, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, recv_throttle_minwin, + CTLFLAG_RW | CTLFLAG_LOCKED, uint32_t, tcp_recv_throttle_minwin, 16 * 1024, "Minimum recv win for throttling"); -int32_t tcp_enable_tlp = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, enable_tlp, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, enable_tlp, CTLFLAG_RW | CTLFLAG_LOCKED, - &tcp_enable_tlp, 1, "Enable Tail loss probe"); + int32_t, tcp_enable_tlp, 1, "Enable Tail loss probe"); static int32_t packchain_newlist = 0; static int32_t packchain_looped = 0; @@ -322,7 +310,8 @@ static int32_t tcp_tfo_check(struct tcpcb *tp, int32_t len) if (tp->t_flags & TF_NOOPT) goto fallback; - if (so->so_flags & SOF1_DATA_AUTHENTICATED) + if ((so->so_flags1 & SOF1_DATA_AUTHENTICATED) && + !(tp->t_flagsext & TF_FASTOPEN_HEUR)) return (len); if (!tcp_heuristic_do_tfo(tp)) { @@ -331,6 +320,9 @@ static int32_t tcp_tfo_check(struct tcpcb *tp, int32_t len) goto fallback; } + if (so->so_flags1 & SOF1_DATA_AUTHENTICATED) + return (len); + optlen += TCPOLEN_MAXSEG; if (tp->t_flags & TF_REQ_SCALE) @@ -412,7 +404,7 @@ tcp_tfo_write_cookie(struct tcpcb *tp, unsigned optlen, int32_t len, int res; u_char *bp; - if (so->so_flags & SOF1_DATA_AUTHENTICATED) { + if (so->so_flags1 & SOF1_DATA_AUTHENTICATED) { /* If there is some data, let's track it */ if (len > 0) { tp->t_tfo_stats |= TFO_S_SYN_DATA_SENT; @@ -597,12 +589,7 @@ tcp_output(struct tcpcb *tp) struct mbuf *mnext = NULL; int sackoptlen = 0; #if MPTCP - unsigned int *dlenp = NULL; - u_int8_t *finp = NULL; - u_int32_t *sseqp = NULL; - u_int64_t dss_val = 0; - boolean_t mptcp_acknow = FALSE; - boolean_t early_data_sent = FALSE; + boolean_t mptcp_acknow; #endif /* MPTCP */ boolean_t cell = FALSE; boolean_t wifi = FALSE; @@ -660,6 +647,10 @@ tcp_output(struct tcpcb *tp) #endif /* MPTCP */ again: +#if MPTCP + mptcp_acknow = FALSE; +#endif + KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_START, 0,0,0,0,0); #if INET6 @@ -982,22 +973,6 @@ after_sack_rexmit: } } -#if MPTCP - if ((tp->t_mpflags & TMPF_FASTJOIN_SEND) && - (tp->t_state == TCPS_SYN_SENT) && - (!(tp->t_flags & TF_CLOSING)) && - (so->so_snd.sb_cc != 0) && - (tp->t_rxtshift == 0)) { - flags &= ~TH_SYN; - flags |= TH_ACK; - off = 0; - len = min(so->so_snd.sb_cc, tp->t_maxseg); - early_data_sent = TRUE; - } else if (early_data_sent) { - /* for now, we allow only one data segment to be sent */ - return (0); - } -#endif /* MPTCP */ /* * Lop off SYN bit if it has already been sent. However, if this * is SYN-SENT state and if segment contains data and if we don't @@ -1019,7 +994,7 @@ after_sack_rexmit: error = tcp_ip_output(so, tp, packetlist, packchain_listadd, tp_inp_options, (so_options & SO_DONTROUTE), - (sack_rxmit | (sack_bytes_rxmt != 0)), + (sack_rxmit || (sack_bytes_rxmt != 0)), isipv6); } @@ -1170,14 +1145,21 @@ after_sack_rexmit: if ((so->so_flags & SOF_MP_SUBFLOW) && !(tp->t_mpflags & TMPF_TCP_FALLBACK)) { int newlen = len; - if ((tp->t_state >= TCPS_ESTABLISHED) && - ((tp->t_mpflags & TMPF_SND_MPPRIO) || - (tp->t_mpflags & TMPF_SND_REM_ADDR) || - (tp->t_mpflags & TMPF_SND_MPFAIL))) { + if (tp->t_state >= TCPS_ESTABLISHED && + (tp->t_mpflags & TMPF_SND_MPPRIO || + tp->t_mpflags & TMPF_SND_REM_ADDR || + tp->t_mpflags & TMPF_SND_MPFAIL || + tp->t_mpflags & TMPF_SND_KEYS || + tp->t_mpflags & TMPF_SND_JACK)) { if (len > 0) { len = 0; } - sendalot = 1; + /* + * On a new subflow, don't try to send again, because + * we are still waiting for the fourth ack. + */ + if (!(tp->t_mpflags & TMPF_PREESTABLISHED)) + sendalot = 1; mptcp_acknow = TRUE; } else { mptcp_acknow = FALSE; @@ -1189,7 +1171,7 @@ after_sack_rexmit: * the contiguous MPTCP level. Set sendalot to send remainder. */ if (len > 0) - newlen = mptcp_adj_sendlen(so, off, len); + newlen = mptcp_adj_sendlen(so, off); if (newlen < len) { len = newlen; sendalot = 1; @@ -1243,9 +1225,8 @@ after_sack_rexmit: struct mptcb *mp_tp = tptomptp(tp); if (mp_tp != NULL) { - MPT_LOCK(mp_tp); - recwin = imin(recwin, (int)mp_tp->mpt_rcvwnd); - MPT_UNLOCK(mp_tp); + mpte_lock_assert_held(mp_tp->mpt_mpte); + recwin = imin(recwin, mptcp_sbspace(mp_tp)); } } #endif @@ -1271,7 +1252,15 @@ after_sack_rexmit: if (recwin > (int32_t)(TCP_MAXWIN << tp->rcv_scale)) recwin = (int32_t)(TCP_MAXWIN << tp->rcv_scale); - if (recwin < (int32_t)(tp->rcv_adv - tp->rcv_nxt)) + + /* + * MPTCP needs to be able to announce a smaller window than previously, + * because the other subflow may have filled up the available window- + * space. So we have to be able to go backwards and announce a smaller + * window. + */ + if (!(so->so_flags & SOF_MPTCP_TRUE) && + recwin < (int32_t)(tp->rcv_adv - tp->rcv_nxt)) recwin = (int32_t)(tp->rcv_adv - tp->rcv_nxt); /* @@ -1302,12 +1291,20 @@ after_sack_rexmit: tp->t_state != TCPS_ESTABLISHED) { if (len >= tp->t_maxseg) goto send; + if (!(tp->t_flags & TF_MORETOCOME) && (idle || tp->t_flags & TF_NODELAY || (tp->t_flags & TF_MAXSEGSNT) || ALLOW_LIMITED_TRANSMIT(tp)) && (tp->t_flags & TF_NOPUSH) == 0 && - len + off >= so->so_snd.sb_cc) + (len + off >= so->so_snd.sb_cc || + /* + * MPTCP needs to respect the DSS-mappings. So, it + * may be sending data that *could* have been + * coalesced, but cannot because of + * mptcp_adj_sendlen(). + */ + so->so_flags & SOF_MP_SUBFLOW)) goto send; if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0) goto send; @@ -1441,7 +1438,7 @@ just_return: error = tcp_ip_output(so, tp, packetlist, packchain_listadd, tp_inp_options, (so_options & SO_DONTROUTE), - (sack_rxmit | (sack_bytes_rxmt != 0)), isipv6); + (sack_rxmit || (sack_bytes_rxmt != 0)), isipv6); } /* tcp was closed while we were in ip; resume close */ if (inp->inp_sndinprog_cnt == 0 && @@ -1503,8 +1500,7 @@ send: } #if MPTCP if (mptcp_enable && (so->so_flags & SOF_MP_SUBFLOW)) { - optlen = mptcp_setup_syn_opts(so, flags, opt, - optlen); + optlen = mptcp_setup_syn_opts(so, opt, optlen); } #endif /* MPTCP */ } @@ -1572,7 +1568,7 @@ send: tp->t_mpflags |= TMPF_MPTCP_ACKNOW; } optlen = mptcp_setup_opts(tp, off, &opt[0], optlen, flags, - len, &dlenp, &finp, &dss_val, &sseqp, &mptcp_acknow); + len, &mptcp_acknow); tp->t_mpflags &= ~TMPF_SEND_DSN; } #endif /* MPTCP */ @@ -1835,43 +1831,30 @@ send: sendalot = 1; } } -#if MPTCP - /* Adjust the length in the DSS option, if it is lesser than len */ - if (dlenp) { - /* - * To test this path without SACK, artificially - * decrement len with something like - * if (len > 10) - len -= 10; - */ - if (ntohs(*dlenp) > len) { - *dlenp = htons(len); - /* Unset the FIN flag, if len was adjusted */ - if (finp) { - *finp &= ~MDSS_F; - } - sendalot = 1; - } - } -#endif /* MPTCP */ if (max_linkhdr + hdrlen > MCLBYTES) panic("tcphdr too big"); /* Check if there is enough data in the send socket - * buffer to start measuring bw + * buffer to start measuring bandwidth */ if ((tp->t_flagsext & TF_MEASURESNDBW) != 0 && (tp->t_bwmeas != NULL) && - (tp->t_flagsext & TF_BWMEAS_INPROGRESS) == 0 && - (so->so_snd.sb_cc - (tp->snd_max - tp->snd_una)) >= - tp->t_bwmeas->bw_minsize) { - tp->t_bwmeas->bw_size = min( - (so->so_snd.sb_cc - (tp->snd_max - tp->snd_una)), - tp->t_bwmeas->bw_maxsize); - tp->t_flagsext |= TF_BWMEAS_INPROGRESS; - tp->t_bwmeas->bw_start = tp->snd_max; - tp->t_bwmeas->bw_ts = tcp_now; + (tp->t_flagsext & TF_BWMEAS_INPROGRESS) == 0) { + tp->t_bwmeas->bw_size = min(min( + (so->so_snd.sb_cc - (tp->snd_max - tp->snd_una)), + tp->snd_cwnd), tp->snd_wnd); + if (tp->t_bwmeas->bw_minsize > 0 && + tp->t_bwmeas->bw_size < tp->t_bwmeas->bw_minsize) + tp->t_bwmeas->bw_size = 0; + if (tp->t_bwmeas->bw_maxsize > 0) + tp->t_bwmeas->bw_size = min(tp->t_bwmeas->bw_size, + tp->t_bwmeas->bw_maxsize); + if (tp->t_bwmeas->bw_size > 0) { + tp->t_flagsext |= TF_BWMEAS_INPROGRESS; + tp->t_bwmeas->bw_start = tp->snd_max; + tp->t_bwmeas->bw_ts = tcp_now; + } } VERIFY(inp->inp_flowhash != 0); @@ -1909,6 +1892,7 @@ send: } inp_decr_sndbytes_unsent(so, len); } + inp_set_activity_bitmap(inp); #if MPTCP if (tp->t_mpflags & TMPF_MPTCP_TRUE) { tcpstat.tcps_mp_sndpacks++; @@ -2051,10 +2035,6 @@ send: m->m_len = hdrlen; } m->m_pkthdr.rcvif = 0; -#if MPTCP - /* Before opt is copied to the mbuf, set the csum field */ - mptcp_output_csum(tp, m, len, hdrlen, dss_val, sseqp); -#endif /* MPTCP */ #if CONFIG_MACF_NET mac_mbuf_label_associate_inpcb(inp, m); #endif @@ -2148,12 +2128,6 @@ send: } th->th_ack = htonl(tp->rcv_nxt); tp->last_ack_sent = tp->rcv_nxt; -#if MPTCP - /* Initialize the ACK field to a value as 0 ack fields are dropped */ - if (early_data_sent) { - th->th_ack = th->th_seq + 1; - } -#endif /* MPTCP */ if (optlen) { bcopy(opt, th + 1, optlen); th->th_off = (sizeof (struct tcphdr) + optlen) >> 2; @@ -2274,6 +2248,7 @@ send: } if (SEQ_GT(tp->snd_nxt, tp->snd_max)) { tp->snd_max = tp->snd_nxt; + tp->t_sndtime = tcp_now; /* * Time this transmission if not a retransmission and * not currently timing anything. @@ -2343,7 +2318,7 @@ timer: */ pto = tp->t_rxtcur; - /* Reset the next RTO to be after PTO. */ + /* Reset the next RTO to be after PTO. */ TCPT_RANGESET(new_rto, (pto + TCP_REXMTVAL(tp)), max(tp->t_rttmin, tp->t_rttcur + 2), @@ -2366,8 +2341,10 @@ timer: ++xlen; tp->t_flags |= TF_SENTFIN; } - if (SEQ_GT(tp->snd_nxt + xlen, tp->snd_max)) + if (SEQ_GT(tp->snd_nxt + xlen, tp->snd_max)) { tp->snd_max = tp->snd_nxt + len; + tp->t_sndtime = tcp_now; + } } #if TCPDEBUG @@ -2466,12 +2443,7 @@ timer: */ m->m_pkthdr.pkt_flowsrc = FLOWSRC_INPCB; m->m_pkthdr.pkt_flowid = inp->inp_flowhash; - m->m_pkthdr.pkt_flags |= PKTF_FLOW_ID | PKTF_FLOW_LOCALSRC; -#if MPTCP - /* Disable flow advisory when using MPTCP. */ - if (!(tp->t_mpflags & TMPF_MPTCP_TRUE)) -#endif /* MPTCP */ - m->m_pkthdr.pkt_flags |= PKTF_FLOW_ADV; + m->m_pkthdr.pkt_flags |= (PKTF_FLOW_ID | PKTF_FLOW_LOCALSRC | PKTF_FLOW_ADV); m->m_pkthdr.pkt_proto = IPPROTO_TCP; m->m_nextpkt = NULL; @@ -2553,7 +2525,7 @@ timer: if (sendalot == 0 || (tp->t_state != TCPS_ESTABLISHED) || (tp->snd_cwnd <= (tp->snd_wnd / 8)) || - (tp->t_flags & (TH_PUSH | TF_ACKNOW)) || + (tp->t_flags & TF_ACKNOW) || (tp->t_flagsext & TF_FORCE) || tp->t_lastchain >= tcp_packet_chaining) { error = 0; @@ -2568,7 +2540,7 @@ timer: error = tcp_ip_output(so, tp, packetlist, packchain_listadd, tp_inp_options, (so_options & SO_DONTROUTE), - (sack_rxmit | (sack_bytes_rxmt != 0)), isipv6); + (sack_rxmit || (sack_bytes_rxmt != 0)), isipv6); if (error) { /* * Take into account the rest of unsent @@ -2820,7 +2792,7 @@ tcp_ip_output(struct socket *so, struct tcpcb *tp, struct mbuf *pkt, */ if (tcp_output_unlocked && !so->so_upcallusecount && (tp->t_state == TCPS_ESTABLISHED) && (sack_in_progress == 0) && - !IN_FASTRECOVERY(tp)) { + !IN_FASTRECOVERY(tp) && !(so->so_flags & SOF_MP_SUBFLOW)) { unlocked = TRUE; socket_unlock(so, 0); @@ -2892,7 +2864,8 @@ tcp_ip_output(struct socket *so, struct tcpcb *tp, struct mbuf *pkt, /* * Enter flow controlled state if the connection is established - * and is not in recovery. + * and is not in recovery. Flow control is allowed only if there + * is outstanding data. * * A connection will enter suspended state even if it is in * recovery. @@ -2900,7 +2873,8 @@ tcp_ip_output(struct socket *so, struct tcpcb *tp, struct mbuf *pkt, if (((adv->code == FADV_FLOW_CONTROLLED && !IN_FASTRECOVERY(tp)) || adv->code == FADV_SUSPENDED) && !(tp->t_flags & TF_CLOSING) && - tp->t_state == TCPS_ESTABLISHED) { + tp->t_state == TCPS_ESTABLISHED && + SEQ_GT(tp->snd_max, tp->snd_una)) { int rc; rc = inp_set_fc_state(inp, adv->code); @@ -2940,6 +2914,7 @@ tcp_ip_output(struct socket *so, struct tcpcb *tp, struct mbuf *pkt, so->so_snd.sb_flags &= ~SB_SNDBYTE_CNT; } inp->inp_last_outifp = outif; + } if (error != 0 && ifdenied && @@ -2966,6 +2941,8 @@ tcp_ip_output(struct socket *so, struct tcpcb *tp, struct mbuf *pkt, return (error); } +int tcptv_persmin_val = TCPTV_PERSMIN; + void tcp_setpersist(struct tcpcb *tp) { @@ -2988,7 +2965,7 @@ tcp_setpersist(struct tcpcb *tp) */ TCPT_RANGESET(tp->t_timer[TCPT_PERSIST], t * tcp_backoff[tp->t_rxtshift], - TCPTV_PERSMIN, TCPTV_PERSMAX, 0); + tcptv_persmin_val, TCPTV_PERSMAX, 0); tp->t_timer[TCPT_PERSIST] = OFFSET_FROM_START(tp, tp->t_timer[TCPT_PERSIST]); if (tp->t_rxtshift < TCP_MAXRXTSHIFT) diff --git a/bsd/netinet/tcp_sack.c b/bsd/netinet/tcp_sack.c index 4a68325db..4b4c04dbe 100644 --- a/bsd/netinet/tcp_sack.c +++ b/bsd/netinet/tcp_sack.c @@ -105,17 +105,14 @@ #include <libkern/OSAtomic.h> -int tcp_do_sack = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, sack, CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_do_sack, 0, - "Enable/Disable TCP SACK support"); -static int tcp_sack_maxholes = 128; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, sack_maxholes, CTLFLAG_RW | CTLFLAG_LOCKED, - &tcp_sack_maxholes, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, sack, CTLFLAG_RW | CTLFLAG_LOCKED, + int, tcp_do_sack, 1, "Enable/Disable TCP SACK support"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, sack_maxholes, CTLFLAG_RW | CTLFLAG_LOCKED, + static int, tcp_sack_maxholes, 128, "Maximum number of TCP SACK holes allowed per connection"); -static int tcp_sack_globalmaxholes = 65536; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, sack_globalmaxholes, CTLFLAG_RW | CTLFLAG_LOCKED, - &tcp_sack_globalmaxholes, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, sack_globalmaxholes, + CTLFLAG_RW | CTLFLAG_LOCKED, static int, tcp_sack_globalmaxholes, 65536, "Global maximum number of TCP SACK holes"); static SInt32 tcp_sack_globalholes = 0; diff --git a/bsd/netinet/tcp_seq.h b/bsd/netinet/tcp_seq.h index 5eb5c3b93..772d33f36 100644 --- a/bsd/netinet/tcp_seq.h +++ b/bsd/netinet/tcp_seq.h @@ -107,11 +107,9 @@ #define tcp_sendseqinit(tp) \ (tp)->snd_una = (tp)->snd_nxt = (tp)->snd_max = (tp)->snd_up = \ - (tp)->snd_recover = (tp)->iss + (tp)->snd_recover = (tp)->iss #define TCP_PAWS_IDLE (24 * 24 * 60 * 60 * TCP_RETRANSHZ) - /* timestamp wrap-around time */ - -extern tcp_cc tcp_ccgen; /* global connection count */ +/* timestamp wrap-around time */ #endif /* KERNEL_PRIVATE */ #endif /* _NETINET_TCP_SEQ_H_ */ diff --git a/bsd/netinet/tcp_subr.c b/bsd/netinet/tcp_subr.c index b2e18eb05..1be5b6a80 100644 --- a/bsd/netinet/tcp_subr.c +++ b/bsd/netinet/tcp_subr.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -99,6 +99,7 @@ #include <netinet/ip_icmp.h> #if INET6 #include <netinet/ip6.h> +#include <netinet/icmp6.h> #endif #include <netinet/in_pcb.h> #if INET6 @@ -110,6 +111,7 @@ #if INET6 #include <netinet6/ip6_var.h> #endif +#include <netinet/mptcp_var.h> #include <netinet/tcp.h> #include <netinet/tcp_fsm.h> #include <netinet/tcp_seq.h> @@ -155,24 +157,21 @@ #define DBG_FNC_TCP_CLOSE NETDBG_CODE(DBG_NETTCP, ((5 << 8) | 2)) +static tcp_cc tcp_ccgen; extern int tcp_lq_overflow; extern struct tcptimerlist tcp_timer_list; extern struct tcptailq tcp_tw_tailq; -int tcp_mssdflt = TCP_MSS; -SYSCTL_INT(_net_inet_tcp, TCPCTL_MSSDFLT, mssdflt, CTLFLAG_RW | CTLFLAG_LOCKED, - &tcp_mssdflt, 0, "Default TCP Maximum Segment Size"); +SYSCTL_SKMEM_TCP_INT(TCPCTL_MSSDFLT, mssdflt, CTLFLAG_RW | CTLFLAG_LOCKED, + int, tcp_mssdflt, TCP_MSS, "Default TCP Maximum Segment Size"); #if INET6 -int tcp_v6mssdflt = TCP6_MSS; -SYSCTL_INT(_net_inet_tcp, TCPCTL_V6MSSDFLT, v6mssdflt, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_v6mssdflt, 0, +SYSCTL_SKMEM_TCP_INT(TCPCTL_V6MSSDFLT, v6mssdflt, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_v6mssdflt, TCP6_MSS, "Default TCP Maximum Segment Size for IPv6"); #endif -extern int tcp_do_autorcvbuf; - int tcp_sysctl_fastopenkey(struct sysctl_oid *, void *, int, struct sysctl_req *); SYSCTL_PROC(_net_inet_tcp, OID_AUTO, fastopen_key, CTLTYPE_STRING | CTLFLAG_WR, @@ -182,19 +181,19 @@ SYSCTL_PROC(_net_inet_tcp, OID_AUTO, fastopen_key, CTLTYPE_STRING | CTLFLAG_WR, int tcp_tfo_halfcnt = 0; /* Maximum of half-open TFO connection backlog */ -int tcp_tfo_backlog = 10; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, fastopen_backlog, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_tfo_backlog, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, fastopen_backlog, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_tfo_backlog, 10, "Backlog queue for half-open TFO connections"); -int tcp_fastopen = TCP_FASTOPEN_CLIENT | TCP_FASTOPEN_SERVER; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, fastopen, CTLFLAG_RW | CTLFLAG_LOCKED, - &tcp_fastopen, 0, "Enable TCP Fastopen (RFC 7413)"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, fastopen, CTLFLAG_RW | CTLFLAG_LOCKED, + int, tcp_fastopen, TCP_FASTOPEN_CLIENT | TCP_FASTOPEN_SERVER, + "Enable TCP Fastopen (RFC 7413)"); + +SYSCTL_SKMEM_TCP_INT(OID_AUTO, now_init, CTLFLAG_RD | CTLFLAG_LOCKED, + uint32_t, tcp_now_init, 0, "Initial tcp now value"); -int tcp_tfo_fallback_min = 10; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, fastopen_fallback_min, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_tfo_fallback_min, 0, - "Mininum number of trials without TFO when in fallback mode"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, microuptime_init, CTLFLAG_RD | CTLFLAG_LOCKED, + uint32_t, tcp_microuptime_init, 0, "Initial tcp uptime value in micro seconds"); /* * Minimum MSS we accept and use. This prevents DoS attacks where @@ -204,9 +203,8 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, fastopen_fallback_min, * with packet generation and sending. Set to zero to disable MINMSS * checking. This setting prevents us from sending too small packets. */ -int tcp_minmss = TCP_MINMSS; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, minmss, CTLFLAG_RW | CTLFLAG_LOCKED, - &tcp_minmss, 0, "Minmum TCP Maximum Segment Size"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, minmss, CTLFLAG_RW | CTLFLAG_LOCKED, + int, tcp_minmss, TCP_MINMSS, "Minmum TCP Maximum Segment Size"); int tcp_do_rfc1323 = 1; #if (DEVELOPMENT || DEBUG) SYSCTL_INT(_net_inet_tcp, TCPCTL_DO_RFC1323, rfc1323, @@ -220,9 +218,8 @@ SYSCTL_INT(_net_inet_tcp, TCPCTL_DO_RFC1644, rfc1644, CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_do_rfc1644, 0, "Enable rfc1644 (TTCP) extensions"); -static int do_tcpdrain = 0; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, do_tcpdrain, CTLFLAG_RW | CTLFLAG_LOCKED, - &do_tcpdrain, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, do_tcpdrain, CTLFLAG_RW | CTLFLAG_LOCKED, + static int, do_tcpdrain, 0, "Enable tcp_drain routine for extra help when low on mbufs"); SYSCTL_INT(_net_inet_tcp, OID_AUTO, pcbcount, CTLFLAG_RD | CTLFLAG_LOCKED, @@ -231,9 +228,8 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, pcbcount, CTLFLAG_RD | CTLFLAG_LOCKED, SYSCTL_INT(_net_inet_tcp, OID_AUTO, tw_pcbcount, CTLFLAG_RD | CTLFLAG_LOCKED, &tcbinfo.ipi_twcount, 0, "Number of pcbs in time-wait state"); -static int icmp_may_rst = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, icmp_may_rst, CTLFLAG_RW | CTLFLAG_LOCKED, - &icmp_may_rst, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, icmp_may_rst, CTLFLAG_RW | CTLFLAG_LOCKED, + static int, icmp_may_rst, 1, "Certain ICMP unreachable messages may abort connections in SYN_SENT"); static int tcp_strict_rfc1948 = 0; @@ -247,23 +243,18 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, isn_reseed_interval, &tcp_isn_reseed_interval, 0, "Seconds between reseeding of ISN secret"); #endif /* (DEVELOPMENT || DEBUG) */ -int tcp_TCPTV_MIN = 100; /* 100ms minimum RTT */ -SYSCTL_INT(_net_inet_tcp, OID_AUTO, rtt_min, CTLFLAG_RW | CTLFLAG_LOCKED, - &tcp_TCPTV_MIN, 0, "min rtt value allowed"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, rtt_min, CTLFLAG_RW | CTLFLAG_LOCKED, + int, tcp_TCPTV_MIN, 100, "min rtt value allowed"); -int tcp_rexmt_slop = TCPTV_REXMTSLOP; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, rexmt_slop, CTLFLAG_RW, - &tcp_rexmt_slop, 0, "Slop added to retransmit timeout"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, rexmt_slop, CTLFLAG_RW, + int, tcp_rexmt_slop, TCPTV_REXMTSLOP, "Slop added to retransmit timeout"); -__private_extern__ int tcp_use_randomport = 0; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, randomize_ports, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_use_randomport, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, randomize_ports, CTLFLAG_RW | CTLFLAG_LOCKED, + __private_extern__ int , tcp_use_randomport, 0, "Randomize TCP port numbers"); -__private_extern__ int tcp_win_scale = 3; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, win_scale_factor, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_win_scale, 0, - "Window scaling factor"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, win_scale_factor, CTLFLAG_RW | CTLFLAG_LOCKED, + __private_extern__ int, tcp_win_scale, 3, "Window scaling factor"); static void tcp_cleartaocache(void); static void tcp_notify(struct inpcb *, int); @@ -321,6 +312,10 @@ static lck_attr_t *tcp_uptime_mtx_attr = NULL; static lck_grp_t *tcp_uptime_mtx_grp = NULL; static lck_grp_attr_t *tcp_uptime_mtx_grp_attr = NULL; int tcp_notsent_lowat_check(struct socket *so); +static void tcp_flow_lim_stats(struct ifnet_stats_per_flow *ifs, + struct if_lim_perf_stat *stat); +static void tcp_flow_ecn_perf_stats(struct ifnet_stats_per_flow *ifs, + struct if_tcp_ecn_perf_stat *stat); static aes_encrypt_ctx tfo_ctx; /* Crypto-context for TFO */ @@ -452,7 +447,7 @@ tcp_tfo_init(void) { u_char key[TCP_FASTOPEN_KEYLEN]; - read_random(key, sizeof(key)); + read_frandom(key, sizeof(key)); aes_encrypt_key128(key, &tfo_ctx); } @@ -484,11 +479,17 @@ tcp_init(struct protosw *pp, struct domain *dp) tcp_msl = TCPTV_MSL; microuptime(&tcp_uptime); - read_random(&tcp_now, sizeof(tcp_now)); + read_frandom(&tcp_now, sizeof(tcp_now)); /* Starts tcp internal clock at a random value */ tcp_now = tcp_now & 0x3fffffff; + /* expose initial uptime/now via systcl for utcp to keep time sync */ + tcp_now_init = tcp_now; + tcp_microuptime_init = tcp_uptime.tv_sec * 1000 + tcp_uptime.tv_usec; + SYSCTL_SKMEM_UPDATE_FIELD(tcp.microuptime_init, tcp_microuptime_init); + SYSCTL_SKMEM_UPDATE_FIELD(tcp.now_init, tcp_now_init); + tcp_tfo_init(); LIST_INIT(&tcb); @@ -638,8 +639,15 @@ tcp_init(struct protosw *pp, struct domain *dp) * maximum allowed receive and send socket buffer size. */ if (nmbclusters > 30720) { - tcp_autorcvbuf_max = 1024 * 1024; - tcp_autosndbuf_max = 1024 * 1024; + #if CONFIG_EMBEDDED + tcp_autorcvbuf_max = 2 * 1024 * 1024; + tcp_autosndbuf_max = 2 * 1024 * 1024; + #else + tcp_autorcvbuf_max = 1024 * 1024; + tcp_autosndbuf_max = 1024 * 1024; + #endif /* CONFIG_EMBEDDED */ + SYSCTL_SKMEM_UPDATE_FIELD(tcp.autorcvbufmax, tcp_autorcvbuf_max); + SYSCTL_SKMEM_UPDATE_FIELD(tcp.autosndbufmax, tcp_autosndbuf_max); /* * Receive buffer max for cellular interfaces supporting @@ -941,12 +949,7 @@ tcp_respond(struct tcpcb *tp, void *ipgen, struct tcphdr *th, struct mbuf *m, /* Embed flowhash and flow control flags */ m->m_pkthdr.pkt_flowsrc = FLOWSRC_INPCB; m->m_pkthdr.pkt_flowid = tp->t_inpcb->inp_flowhash; - m->m_pkthdr.pkt_flags |= PKTF_FLOW_ID | PKTF_FLOW_LOCALSRC; -#if MPTCP - /* Disable flow advisory when using MPTCP. */ - if (!(tp->t_mpflags & TMPF_MPTCP_TRUE)) -#endif /* MPTCP */ - m->m_pkthdr.pkt_flags |= PKTF_FLOW_ADV; + m->m_pkthdr.pkt_flags |= (PKTF_FLOW_ID | PKTF_FLOW_LOCALSRC | PKTF_FLOW_ADV); m->m_pkthdr.pkt_proto = IPPROTO_TCP; } @@ -977,8 +980,9 @@ tcp_respond(struct tcpcb *tp, void *ipgen, struct tcphdr *th, struct mbuf *m, if (tp != NULL && ro6 != NULL && ro6->ro_rt != NULL && (outif = ro6->ro_rt->rt_ifp) != - tp->t_inpcb->in6p_last_outifp) + tp->t_inpcb->in6p_last_outifp) { tp->t_inpcb->in6p_last_outifp = outif; + } if (ro6 == &sro6) ROUTE_RELEASE(ro6); @@ -1014,9 +1018,10 @@ tcp_respond(struct tcpcb *tp, void *ipgen, struct tcphdr *th, struct mbuf *m, if (tp != NULL && sro.ro_rt != NULL && (outif = sro.ro_rt->rt_ifp) != - tp->t_inpcb->inp_last_outifp) + tp->t_inpcb->inp_last_outifp) { tp->t_inpcb->inp_last_outifp = outif; + } if (ro != &sro) { /* Synchronize cached PCB route */ inp_route_copyin(tp->t_inpcb, &sro); @@ -1100,6 +1105,14 @@ tcp_newtcpcb(struct inpcb *inp) tp->t_flagsext |= TF_RCVUNACK_WAITSS; tp->t_rexmtthresh = tcprexmtthresh; + /* Enable bandwidth measurement on this connection */ + tp->t_flagsext |= TF_MEASURESNDBW; + if (tp->t_bwmeas == NULL) { + tp->t_bwmeas = tcp_bwmeas_alloc(tp); + if (tp->t_bwmeas == NULL) + tp->t_flagsext &= ~TF_MEASURESNDBW; + } + /* Clear time wait tailq entry */ tp->t_twentry.tqe_next = NULL; tp->t_twentry.tqe_prev = NULL; @@ -1177,33 +1190,77 @@ tcp_getrt_rtt(struct tcpcb *tp, struct rtentry *rt) } static inline void -tcp_update_ecn_perf_stats(struct tcpcb *tp, +tcp_create_ifnet_stats_per_flow(struct tcpcb *tp, + struct ifnet_stats_per_flow *ifs) +{ + struct inpcb *inp; + struct socket *so; + if (tp == NULL || ifs == NULL) + return; + + bzero(ifs, sizeof(*ifs)); + inp = tp->t_inpcb; + so = inp->inp_socket; + + ifs->ipv4 = (inp->inp_vflag & INP_IPV6) ? 0 : 1; + ifs->local = (tp->t_flags & TF_LOCAL) ? 1 : 0; + ifs->connreset = (so->so_error == ECONNRESET) ? 1 : 0; + ifs->conntimeout = (so->so_error == ETIMEDOUT) ? 1 : 0; + ifs->ecn_flags = tp->ecn_flags; + ifs->txretransmitbytes = tp->t_stat.txretransmitbytes; + ifs->rxoutoforderbytes = tp->t_stat.rxoutoforderbytes; + ifs->rxmitpkts = tp->t_stat.rxmitpkts; + ifs->rcvoopack = tp->t_rcvoopack; + ifs->pawsdrop = tp->t_pawsdrop; + ifs->sack_recovery_episodes = tp->t_sack_recovery_episode; + ifs->reordered_pkts = tp->t_reordered_pkts; + ifs->dsack_sent = tp->t_dsack_sent; + ifs->dsack_recvd = tp->t_dsack_recvd; + ifs->srtt = tp->t_srtt; + ifs->rttupdated = tp->t_rttupdated; + ifs->rttvar = tp->t_rttvar; + ifs->rttmin = get_base_rtt(tp); + if (tp->t_bwmeas != NULL && tp->t_bwmeas->bw_sndbw_max > 0) { + ifs->bw_sndbw_max = tp->t_bwmeas->bw_sndbw_max; + } else { + ifs->bw_sndbw_max = 0; + } + if (tp->t_bwmeas!= NULL && tp->t_bwmeas->bw_rcvbw_max > 0) { + ifs->bw_rcvbw_max = tp->t_bwmeas->bw_rcvbw_max; + } else { + ifs->bw_rcvbw_max = 0; + } + ifs->bk_txpackets = so->so_tc_stats[MBUF_TC_BK].txpackets; + ifs->txpackets = inp->inp_stat->txpackets; + ifs->rxpackets = inp->inp_stat->rxpackets; +} + +static inline void +tcp_flow_ecn_perf_stats(struct ifnet_stats_per_flow *ifs, struct if_tcp_ecn_perf_stat *stat) { u_int64_t curval, oldval; - struct inpcb *inp = tp->t_inpcb; - stat->total_txpkts += inp->inp_stat->txpackets; - stat->total_rxpkts += inp->inp_stat->rxpackets; - stat->total_rxmitpkts += tp->t_stat.rxmitpkts; - stat->total_oopkts += tp->t_rcvoopack; - stat->total_reorderpkts += (tp->t_reordered_pkts + tp->t_pawsdrop + - tp->t_dsack_sent + tp->t_dsack_recvd); + stat->total_txpkts += ifs->txpackets; + stat->total_rxpkts += ifs->rxpackets; + stat->total_rxmitpkts += ifs->rxmitpkts; + stat->total_oopkts += ifs->rcvoopack; + stat->total_reorderpkts += (ifs->reordered_pkts + + ifs->pawsdrop + ifs->dsack_sent + ifs->dsack_recvd); /* Average RTT */ - curval = (tp->t_srtt >> TCP_RTT_SHIFT); - if (curval > 0 && tp->t_rttupdated >= 16) { + curval = ifs->srtt >> TCP_RTT_SHIFT; + if (curval > 0 && ifs->rttupdated >= 16) { if (stat->rtt_avg == 0) { stat->rtt_avg = curval; } else { oldval = stat->rtt_avg; - stat->rtt_avg = - ((oldval << 4) - oldval + curval) >> 4; + stat->rtt_avg = ((oldval << 4) - oldval + curval) >> 4; } } /* RTT variance */ - curval = tp->t_rttvar >> TCP_RTTVAR_SHIFT; - if (curval > 0 && tp->t_rttupdated >= 16) { + curval = ifs->rttvar >> TCP_RTTVAR_SHIFT; + if (curval > 0 && ifs->rttupdated >= 16) { if (stat->rtt_var == 0) { stat->rtt_var = curval; } else { @@ -1213,13 +1270,77 @@ tcp_update_ecn_perf_stats(struct tcpcb *tp, } } - /* Total number of SACK recovery episodes */ - stat->sack_episodes += tp->t_sack_recovery_episode; - - if (inp->inp_socket->so_error == ECONNRESET) + /* SACK episodes */ + stat->sack_episodes += ifs->sack_recovery_episodes; + if (ifs->connreset) stat->rst_drop++; } +static inline void +tcp_flow_lim_stats(struct ifnet_stats_per_flow *ifs, + struct if_lim_perf_stat *stat) +{ + u_int64_t curval, oldval; + + stat->lim_total_txpkts += ifs->txpackets; + stat->lim_total_rxpkts += ifs->rxpackets; + stat->lim_total_retxpkts += ifs->rxmitpkts; + stat->lim_total_oopkts += ifs->rcvoopack; + + if (ifs->bw_sndbw_max > 0) { + /* convert from bytes per ms to bits per second */ + ifs->bw_sndbw_max *= 8000; + stat->lim_ul_max_bandwidth = max(stat->lim_ul_max_bandwidth, + ifs->bw_sndbw_max); + } + + if (ifs->bw_rcvbw_max > 0) { + /* convert from bytes per ms to bits per second */ + ifs->bw_rcvbw_max *= 8000; + stat->lim_dl_max_bandwidth = max(stat->lim_dl_max_bandwidth, + ifs->bw_rcvbw_max); + } + + /* Average RTT */ + curval = ifs->srtt >> TCP_RTT_SHIFT; + if (curval > 0 && ifs->rttupdated >= 16) { + if (stat->lim_rtt_average == 0) { + stat->lim_rtt_average = curval; + } else { + oldval = stat->lim_rtt_average; + stat->lim_rtt_average = + ((oldval << 4) - oldval + curval) >> 4; + } + } + + /* RTT variance */ + curval = ifs->rttvar >> TCP_RTTVAR_SHIFT; + if (curval > 0 && ifs->rttupdated >= 16) { + if (stat->lim_rtt_variance == 0) { + stat->lim_rtt_variance = curval; + } else { + oldval = stat->lim_rtt_variance; + stat->lim_rtt_variance = + ((oldval << 4) - oldval + curval) >> 4; + } + } + + if (stat->lim_rtt_min == 0) { + stat->lim_rtt_min = ifs->rttmin; + } else { + stat->lim_rtt_min = min(stat->lim_rtt_min, ifs->rttmin); + } + + /* connection timeouts */ + stat->lim_conn_attempts++; + if (ifs->conntimeout) + stat->lim_conn_timeouts++; + + /* bytes sent using background delay-based algorithms */ + stat->lim_bk_txpkts += ifs->bk_txpackets; + +} + /* * Close a TCP control block: * discard all space held by the tcp @@ -1237,6 +1358,7 @@ tcp_close(struct tcpcb *tp) struct route *ro; struct rtentry *rt; int dosavessthresh; + struct ifnet_stats_per_flow ifs; /* tcp_close was called previously, bail */ if (inp->inp_ppcb == NULL) @@ -1398,90 +1520,9 @@ no_valid_rt: /* free the reassembly queue, if any */ (void) tcp_freeq(tp); - /* Collect ECN related statistics */ - if (tp->ecn_flags & TE_SETUPSENT) { - if (tp->ecn_flags & TE_CLIENT_SETUP) { - INP_INC_IFNET_STAT(inp, ecn_client_setup); - if (TCP_ECN_ENABLED(tp)) { - INP_INC_IFNET_STAT(inp, - ecn_client_success); - } else if (tp->ecn_flags & TE_LOST_SYN) { - INP_INC_IFNET_STAT(inp, ecn_syn_lost); - } else { - INP_INC_IFNET_STAT(inp, - ecn_peer_nosupport); - } - } else { - INP_INC_IFNET_STAT(inp, ecn_server_setup); - if (TCP_ECN_ENABLED(tp)) { - INP_INC_IFNET_STAT(inp, - ecn_server_success); - } else if (tp->ecn_flags & TE_LOST_SYNACK) { - INP_INC_IFNET_STAT(inp, - ecn_synack_lost); - } else { - INP_INC_IFNET_STAT(inp, - ecn_peer_nosupport); - } - } - } else { - INP_INC_IFNET_STAT(inp, ecn_off_conn); - } - if (TCP_ECN_ENABLED(tp)) { - if (tp->ecn_flags & TE_RECV_ECN_CE) { - tcpstat.tcps_ecn_conn_recv_ce++; - INP_INC_IFNET_STAT(inp, ecn_conn_recv_ce); - } - if (tp->ecn_flags & TE_RECV_ECN_ECE) { - tcpstat.tcps_ecn_conn_recv_ece++; - INP_INC_IFNET_STAT(inp, ecn_conn_recv_ece); - } - if (tp->ecn_flags & (TE_RECV_ECN_CE | TE_RECV_ECN_ECE)) { - if (tp->t_stat.txretransmitbytes > 0 || - tp->t_stat.rxoutoforderbytes > 0) { - tcpstat.tcps_ecn_conn_pl_ce++; - INP_INC_IFNET_STAT(inp, ecn_conn_plce); - } else { - tcpstat.tcps_ecn_conn_nopl_ce++; - INP_INC_IFNET_STAT(inp, ecn_conn_noplce); - } - } else { - if (tp->t_stat.txretransmitbytes > 0 || - tp->t_stat.rxoutoforderbytes > 0) { - tcpstat.tcps_ecn_conn_plnoce++; - INP_INC_IFNET_STAT(inp, ecn_conn_plnoce); - } - } - } - - /* Aggregate performance stats */ - if (inp->inp_last_outifp != NULL && !(tp->t_flags & TF_LOCAL)) { - struct ifnet *ifp = inp->inp_last_outifp; - ifnet_lock_shared(ifp); - if ((ifp->if_refflags & (IFRF_ATTACHED | IFRF_DETACHING)) == - IFRF_ATTACHED) { - if (inp->inp_vflag & INP_IPV6) { - ifp->if_ipv6_stat->timestamp = net_uptime(); - if (TCP_ECN_ENABLED(tp)) { - tcp_update_ecn_perf_stats(tp, - &ifp->if_ipv6_stat->ecn_on); - } else { - tcp_update_ecn_perf_stats(tp, - &ifp->if_ipv6_stat->ecn_off); - } - } else { - ifp->if_ipv4_stat->timestamp = net_uptime(); - if (TCP_ECN_ENABLED(tp)) { - tcp_update_ecn_perf_stats(tp, - &ifp->if_ipv4_stat->ecn_on); - } else { - tcp_update_ecn_perf_stats(tp, - &ifp->if_ipv4_stat->ecn_off); - } - } - } - ifnet_lock_done(ifp); - } + /* performance stats per interface */ + tcp_create_ifnet_stats_per_flow(tp, &ifs); + tcp_update_stats_per_flow(&ifs, inp->inp_last_outifp); tcp_free_sackholes(tp); tcp_notify_ack_free(tp); @@ -1497,16 +1538,6 @@ no_valid_rt: m_freem_list(tp->t_pktlist_head); TCP_PKTLIST_CLEAR(tp); -#if MPTCP - /* Clear MPTCP state */ - if ((so->so_flags & SOF_MPTCP_TRUE) || - (so->so_flags & SOF_MP_SUBFLOW)) { - soevent(so, (SO_FILT_HINT_LOCKED | SO_FILT_HINT_DELETEOK)); - } - tp->t_mpflags = 0; - tp->t_mptcb = NULL; -#endif /* MPTCP */ - if (so->so_flags1 & SOF1_CACHED_IN_SOCK_LAYER) inp->inp_saved_ppcb = (caddr_t) tp; @@ -1609,11 +1640,11 @@ tcp_drain(void) LIST_FOREACH(inp, tcbinfo.ipi_listhead, inp_list) { if (in_pcb_checkstate(inp, WNT_ACQUIRE, 0) != WNT_STOPUSING) { - tcp_lock(inp->inp_socket, 1, 0); + socket_lock(inp->inp_socket, 1); if (in_pcb_checkstate(inp, WNT_RELEASE, 1) == WNT_STOPUSING) { /* lost a race, try the next one */ - tcp_unlock(inp->inp_socket, 1, 0); + socket_unlock(inp->inp_socket, 1); continue; } tp = intotcpcb(inp); @@ -1623,7 +1654,7 @@ tcp_drain(void) so_drain_extended_bk_idle(inp->inp_socket); - tcp_unlock(inp->inp_socket, 1, 0); + socket_unlock(inp->inp_socket, 1); } } lck_rw_done(tcbinfo.ipi_lock); @@ -1648,6 +1679,7 @@ tcp_notify(struct inpcb *inp, int error) tp = (struct tcpcb *)inp->inp_ppcb; + VERIFY(tp != NULL); /* * Ignore some errors if we are hooked up. * If connection hasn't completed, has retransmitted several times, @@ -1658,7 +1690,10 @@ tcp_notify(struct inpcb *inp, int error) if (tp->t_state == TCPS_ESTABLISHED && (error == EHOSTUNREACH || error == ENETUNREACH || error == EHOSTDOWN)) { - return; + if (inp->inp_route.ro_rt) { + rtfree(inp->inp_route.ro_rt); + inp->inp_route.ro_rt = (struct rtentry *)NULL; + } } else if (tp->t_state < TCPS_ESTABLISHED && tp->t_rxtshift > 3 && tp->t_softerror) tcp_drop(tp, error); @@ -1681,9 +1716,7 @@ tcp_bwmeas_alloc(struct tcpcb *tp) bzero(elm, bwmeas_elm_size); elm->bw_minsizepkts = TCP_BWMEAS_BURST_MINSIZE; - elm->bw_maxsizepkts = TCP_BWMEAS_BURST_MAXSIZE; elm->bw_minsize = elm->bw_minsizepkts * tp->t_maxseg; - elm->bw_maxsize = elm->bw_maxsizepkts * tp->t_maxseg; return (elm); } @@ -1854,13 +1887,13 @@ tcp_pcblist SYSCTL_HANDLER_ARGS if (in_pcb_checkstate(inp, WNT_ACQUIRE, 0) == WNT_STOPUSING) continue; - tcp_lock(inp->inp_socket, 1, NULL); + socket_lock(inp->inp_socket, 1); if (in_pcb_checkstate(inp, WNT_RELEASE, 1) == WNT_STOPUSING) { - tcp_unlock(inp->inp_socket, 1, NULL); + socket_unlock(inp->inp_socket, 1); continue; } if (inp->inp_gencnt > gencnt) { - tcp_unlock(inp->inp_socket, 1, NULL); + socket_unlock(inp->inp_socket, 1); continue; } @@ -1878,7 +1911,7 @@ tcp_pcblist SYSCTL_HANDLER_ARGS if (inp->inp_socket) sotoxsocket(inp->inp_socket, &xt.xt_socket); - tcp_unlock(inp->inp_socket, 1, NULL); + socket_unlock(inp->inp_socket, 1); error = SYSCTL_OUT(req, &xt, sizeof(xt)); } @@ -1906,6 +1939,7 @@ SYSCTL_PROC(_net_inet_tcp, TCPCTL_PCBLIST, pcblist, CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0, tcp_pcblist, "S,xtcpcb", "List of active TCP connections"); +#if !CONFIG_EMBEDDED static void tcpcb_to_xtcpcb64(struct tcpcb *tp, struct xtcpcb64 *otp) @@ -2029,18 +2063,18 @@ tcp_pcblist64 SYSCTL_HANDLER_ARGS for (i = 0; i < n; i++) { struct xtcpcb64 xt; struct inpcb *inp; - + inp = inp_list[i]; if (in_pcb_checkstate(inp, WNT_ACQUIRE, 0) == WNT_STOPUSING) continue; - tcp_lock(inp->inp_socket, 1, NULL); + socket_lock(inp->inp_socket, 1); if (in_pcb_checkstate(inp, WNT_RELEASE, 1) == WNT_STOPUSING) { - tcp_unlock(inp->inp_socket, 1, NULL); + socket_unlock(inp->inp_socket, 1); continue; } if (inp->inp_gencnt > gencnt) { - tcp_unlock(inp->inp_socket, 1, NULL); + socket_unlock(inp->inp_socket, 1); continue; } @@ -2056,7 +2090,7 @@ tcp_pcblist64 SYSCTL_HANDLER_ARGS sotoxsocket64(inp->inp_socket, &xt.xt_inpcb.xi_socket); - tcp_unlock(inp->inp_socket, 1, NULL); + socket_unlock(inp->inp_socket, 1); error = SYSCTL_OUT(req, &xt, sizeof(xt)); } @@ -2084,6 +2118,7 @@ SYSCTL_PROC(_net_inet_tcp, OID_AUTO, pcblist64, CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0, tcp_pcblist64, "S,xtcpcb64", "List of active TCP connections"); +#endif /* !CONFIG_EMBEDDED */ static int tcp_pcblist_n SYSCTL_HANDLER_ARGS @@ -2106,8 +2141,8 @@ __private_extern__ void tcp_get_ports_used(uint32_t ifindex, int protocol, uint32_t flags, bitstr_t *bitfield) { - inpcb_get_ports_used(ifindex, protocol, flags, - bitfield, &tcbinfo); + inpcb_get_ports_used(ifindex, protocol, flags, bitfield, + &tcbinfo); } __private_extern__ uint32_t @@ -2119,7 +2154,7 @@ tcp_count_opportunistic(unsigned int ifindex, u_int32_t flags) __private_extern__ uint32_t tcp_find_anypcb_byaddr(struct ifaddr *ifa) { - return (inpcb_find_anypcb_byaddr(ifa, &tcbinfo)); + return (inpcb_find_anypcb_byaddr(ifa, &tcbinfo)); } static void @@ -2219,14 +2254,15 @@ tcp_handle_msgsize(struct ip *ip, struct inpcb *inp) } void -tcp_ctlinput(int cmd, struct sockaddr *sa, void *vip) +tcp_ctlinput(int cmd, struct sockaddr *sa, void *vip, __unused struct ifnet *ifp) { tcp_seq icmp_tcp_seq; struct ip *ip = vip; struct in_addr faddr; struct inpcb *inp; struct tcpcb *tp; - + struct tcphdr *th; + struct icmp *icp; void (*notify)(struct inpcb *, int) = tcp_notify; faddr = ((struct sockaddr_in *)(void *)sa)->sin_addr; @@ -2236,121 +2272,189 @@ tcp_ctlinput(int cmd, struct sockaddr *sa, void *vip) if ((unsigned)cmd >= PRC_NCMDS) return; + /* Source quench is deprecated */ + if (cmd == PRC_QUENCH) + return; + if (cmd == PRC_MSGSIZE) notify = tcp_mtudisc; else if (icmp_may_rst && (cmd == PRC_UNREACH_ADMIN_PROHIB || - cmd == PRC_UNREACH_PORT) && ip) + cmd == PRC_UNREACH_PORT || cmd == PRC_UNREACH_PROTOCOL || + cmd == PRC_TIMXCEED_INTRANS) && ip) notify = tcp_drop_syn_sent; - else if (PRC_IS_REDIRECT(cmd)) { - ip = 0; - notify = in_rtchange; - } else if (cmd == PRC_HOSTDEAD) - ip = 0; - /* Source quench is deprecated */ - else if (cmd == PRC_QUENCH) + /* + * Hostdead is ugly because it goes linearly through all PCBs. + * XXX: We never get this from ICMP, otherwise it makes an + * excellent DoS attack on machines with many connections. + */ + else if (cmd == PRC_HOSTDEAD) + ip = NULL; + else if (inetctlerrmap[cmd] == 0 && !PRC_IS_REDIRECT(cmd)) return; - else if (inetctlerrmap[cmd] == 0) + + + if (ip == NULL) { + in_pcbnotifyall(&tcbinfo, faddr, inetctlerrmap[cmd], notify); return; - if (ip) { - struct tcphdr th; - struct icmp *icp; + } - icp = (struct icmp *)(void *) - ((caddr_t)ip - offsetof(struct icmp, icmp_ip)); - /* - * Only the first 8 bytes of TCP header will be returned. - */ - bzero(&th, sizeof(th)); - bcopy(((caddr_t)ip + (IP_VHL_HL(ip->ip_vhl) << 2)), &th, 8); - inp = in_pcblookup_hash(&tcbinfo, faddr, th.th_dport, - ip->ip_src, th.th_sport, 0, NULL); - if (inp != NULL && inp->inp_socket != NULL) { - tcp_lock(inp->inp_socket, 1, 0); - if (in_pcb_checkstate(inp, WNT_RELEASE, 1) == - WNT_STOPUSING) { - tcp_unlock(inp->inp_socket, 1, 0); - return; - } - icmp_tcp_seq = htonl(th.th_seq); - tp = intotcpcb(inp); - if (SEQ_GEQ(icmp_tcp_seq, tp->snd_una) && - SEQ_LT(icmp_tcp_seq, tp->snd_max)) { - if (cmd == PRC_MSGSIZE) - tcp_handle_msgsize(ip, inp); + icp = (struct icmp *)(void *) + ((caddr_t)ip - offsetof(struct icmp, icmp_ip)); + th = (struct tcphdr *)(void *)((caddr_t)ip + (IP_VHL_HL(ip->ip_vhl) << 2)); + icmp_tcp_seq = ntohl(th->th_seq); - (*notify)(inp, inetctlerrmap[cmd]); - } - tcp_unlock(inp->inp_socket, 1, 0); + inp = in_pcblookup_hash(&tcbinfo, faddr, th->th_dport, + ip->ip_src, th->th_sport, 0, NULL); + + if (inp == NULL || + inp->inp_socket == NULL) { + return; + } + + socket_lock(inp->inp_socket, 1); + if (in_pcb_checkstate(inp, WNT_RELEASE, 1) == + WNT_STOPUSING) { + socket_unlock(inp->inp_socket, 1); + return; + } + + if (PRC_IS_REDIRECT(cmd)) { + /* signal EHOSTDOWN, as it flushes the cached route */ + (*notify)(inp, EHOSTDOWN); + } else { + tp = intotcpcb(inp); + if (SEQ_GEQ(icmp_tcp_seq, tp->snd_una) && + SEQ_LT(icmp_tcp_seq, tp->snd_max)) { + if (cmd == PRC_MSGSIZE) + tcp_handle_msgsize(ip, inp); + + (*notify)(inp, inetctlerrmap[cmd]); } - } else - in_pcbnotifyall(&tcbinfo, faddr, inetctlerrmap[cmd], notify); + } + socket_unlock(inp->inp_socket, 1); } #if INET6 void -tcp6_ctlinput(int cmd, struct sockaddr *sa, void *d) +tcp6_ctlinput(int cmd, struct sockaddr *sa, void *d, __unused struct ifnet *ifp) { - struct tcphdr th; + tcp_seq icmp_tcp_seq; + struct in6_addr *dst; + struct tcphdr *th; void (*notify)(struct inpcb *, int) = tcp_notify; struct ip6_hdr *ip6; struct mbuf *m; + struct inpcb *inp; + struct tcpcb *tp; + struct icmp6_hdr *icmp6; struct ip6ctlparam *ip6cp = NULL; const struct sockaddr_in6 *sa6_src = NULL; - int off; - struct tcp_portonly { - u_int16_t th_sport; - u_int16_t th_dport; - } *thp; + unsigned int mtu; + unsigned int off; if (sa->sa_family != AF_INET6 || sa->sa_len != sizeof(struct sockaddr_in6)) return; - if ((unsigned)cmd >= PRC_NCMDS) + /* Source quench is deprecated */ + if (cmd == PRC_QUENCH) return; - if (cmd == PRC_MSGSIZE) - notify = tcp_mtudisc; - else if (!PRC_IS_REDIRECT(cmd) && (inet6ctlerrmap[cmd] == 0)) - return; - /* Source quench is deprecated */ - else if (cmd == PRC_QUENCH) + if ((unsigned)cmd >= PRC_NCMDS) return; /* if the parameter is from icmp6, decode it. */ if (d != NULL) { ip6cp = (struct ip6ctlparam *)d; + icmp6 = ip6cp->ip6c_icmp6; m = ip6cp->ip6c_m; ip6 = ip6cp->ip6c_ip6; off = ip6cp->ip6c_off; sa6_src = ip6cp->ip6c_src; + dst = ip6cp->ip6c_finaldst; } else { m = NULL; ip6 = NULL; off = 0; /* fool gcc */ sa6_src = &sa6_any; + dst = NULL; } - if (ip6) { + if (cmd == PRC_MSGSIZE) + notify = tcp_mtudisc; + else if (icmp_may_rst && (cmd == PRC_UNREACH_ADMIN_PROHIB || + cmd == PRC_UNREACH_PORT || cmd == PRC_TIMXCEED_INTRANS) && + ip6 != NULL) + notify = tcp_drop_syn_sent; + /* + * Hostdead is ugly because it goes linearly through all PCBs. + * XXX: We never get this from ICMP, otherwise it makes an + * excellent DoS attack on machines with many connections. + */ + else if (cmd == PRC_HOSTDEAD) + ip6 = NULL; + else if (inet6ctlerrmap[cmd] == 0 && !PRC_IS_REDIRECT(cmd)) + return; + + + if (ip6 == NULL) { + in6_pcbnotify(&tcbinfo, sa, 0, (struct sockaddr *)(size_t)sa6_src, + 0, cmd, NULL, notify); + return; + } + + if (m == NULL || + (m->m_pkthdr.len < (int32_t) (off + offsetof(struct tcphdr, th_seq)))) + return; + + th = (struct tcphdr *)(void *)mtodo(m, off); + icmp_tcp_seq = ntohl(th->th_seq); + + if (cmd == PRC_MSGSIZE) { + mtu = ntohl(icmp6->icmp6_mtu); /* - * XXX: We assume that when IPV6 is non NULL, - * M and OFF are valid. + * If no alternative MTU was proposed, or the proposed + * MTU was too small, set to the min. */ + if (mtu < IPV6_MMTU) + mtu = IPV6_MMTU - 8; + } - /* check if we can safely examine src and dst ports */ - if (m->m_pkthdr.len < off + sizeof(*thp)) - return; + inp = in6_pcblookup_hash(&tcbinfo, &ip6->ip6_dst, th->th_dport, + &ip6->ip6_src, th->th_sport, 0, NULL); - bzero(&th, sizeof(th)); - m_copydata(m, off, sizeof(*thp), (caddr_t)&th); + if (inp == NULL || + inp->inp_socket == NULL) { + return; + } - in6_pcbnotify(&tcbinfo, sa, th.th_dport, - (struct sockaddr *)ip6cp->ip6c_src, - th.th_sport, cmd, NULL, notify); + socket_lock(inp->inp_socket, 1); + if (in_pcb_checkstate(inp, WNT_RELEASE, 1) == + WNT_STOPUSING) { + socket_unlock(inp->inp_socket, 1); + return; + } + + if (PRC_IS_REDIRECT(cmd)) { + /* signal EHOSTDOWN, as it flushes the cached route */ + (*notify)(inp, EHOSTDOWN); } else { - in6_pcbnotify(&tcbinfo, sa, 0, - (struct sockaddr *)(size_t)sa6_src, 0, cmd, NULL, notify); + tp = intotcpcb(inp); + if (SEQ_GEQ(icmp_tcp_seq, tp->snd_una) && + SEQ_LT(icmp_tcp_seq, tp->snd_max)) { + if (cmd == PRC_MSGSIZE) { + /* + * Only process the offered MTU if it + * is smaller than the current one. + */ + if (mtu < tp->t_maxseg + + (sizeof (*th) + sizeof (*ip6))) + (*notify)(inp, inetctlerrmap[cmd]); + } else + (*notify)(inp, inetctlerrmap[cmd]); + } } + socket_unlock(inp->inp_socket, 1); } #endif /* INET6 */ @@ -2426,7 +2530,7 @@ tcp_new_isn(struct tcpcb *tp) (((u_int)isn_last_reseed + (u_int)tcp_isn_reseed_interval*hz) < (u_int)timenow.tv_sec))) { #ifdef __APPLE__ - read_random(&isn_secret, sizeof(isn_secret)); + read_frandom(&isn_secret, sizeof(isn_secret)); #else read_random_unlimited(&isn_secret, sizeof(isn_secret)); #endif @@ -2597,7 +2701,7 @@ tcp_rtlookup(struct inpcb *inp, unsigned int input_ifscope) struct rtentry *rt; struct tcpcb *tp; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); ro = &inp->inp_route; if ((rt = ro->ro_rt) != NULL) @@ -2659,8 +2763,10 @@ tcp_rtlookup(struct inpcb *inp, unsigned int input_ifscope) soif2kcl(inp->inp_socket, (rt->rt_ifp->if_eflags & IFEF_2KCL)); tcp_set_ecn(tp, rt->rt_ifp); - if (inp->inp_last_outifp == NULL) + if (inp->inp_last_outifp == NULL) { inp->inp_last_outifp = rt->rt_ifp; + + } } /* Note if the peer is local */ @@ -2685,7 +2791,7 @@ tcp_rtlookup6(struct inpcb *inp, unsigned int input_ifscope) struct rtentry *rt; struct tcpcb *tp; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); ro6 = &inp->in6p_route; if ((rt = ro6->ro_rt) != NULL) @@ -2757,8 +2863,9 @@ tcp_rtlookup6(struct inpcb *inp, unsigned int input_ifscope) soif2kcl(inp->inp_socket, (rt->rt_ifp->if_eflags & IFEF_2KCL)); tcp_set_ecn(tp, rt->rt_ifp); - if (inp->inp_last_outifp == NULL) + if (inp->inp_last_outifp == NULL) { inp->inp_last_outifp = rt->rt_ifp; + } } /* Note if the peer is local */ @@ -2875,8 +2982,36 @@ tcp_lock(struct socket *so, int refcount, void *lr) else lr_saved = lr; +retry: if (so->so_pcb != NULL) { - lck_mtx_lock(&((struct inpcb *)so->so_pcb)->inpcb_mtx); + if (so->so_flags & SOF_MP_SUBFLOW) { + struct mptcb *mp_tp = tptomptp(sototcpcb(so)); + VERIFY(mp_tp); + + mpte_lock_assert_notheld(mp_tp->mpt_mpte); + + mpte_lock(mp_tp->mpt_mpte); + + /* + * Check if we became non-MPTCP while waiting for the lock. + * If yes, we have to retry to grab the right lock. + */ + if (!(so->so_flags & SOF_MP_SUBFLOW)) { + mpte_unlock(mp_tp->mpt_mpte); + goto retry; + } + } else { + lck_mtx_lock(&((struct inpcb *)so->so_pcb)->inpcb_mtx); + + if (so->so_flags & SOF_MP_SUBFLOW) { + /* + * While waiting for the lock, we might have + * become MPTCP-enabled (see mptcp_subflow_socreate). + */ + lck_mtx_unlock(&((struct inpcb *)so->so_pcb)->inpcb_mtx); + goto retry; + } + } } else { panic("tcp_lock: so=%p NO PCB! lr=%p lrh= %s\n", so, lr_saved, solockhistory_nr(so)); @@ -2926,17 +3061,27 @@ tcp_unlock(struct socket *so, int refcount, void *lr) so, so->so_usecount, lr_saved, solockhistory_nr(so)); /* NOTREACHED */ } else { - lck_mtx_assert(&((struct inpcb *)so->so_pcb)->inpcb_mtx, - LCK_MTX_ASSERT_OWNED); so->unlock_lr[so->next_unlock_lr] = lr_saved; so->next_unlock_lr = (so->next_unlock_lr+1) % SO_LCKDBG_MAX; - lck_mtx_unlock(&((struct inpcb *)so->so_pcb)->inpcb_mtx); + + if (so->so_flags & SOF_MP_SUBFLOW) { + struct mptcb *mp_tp = tptomptp(sototcpcb(so)); + + VERIFY(mp_tp); + mpte_lock_assert_held(mp_tp->mpt_mpte); + + mpte_unlock(mp_tp->mpt_mpte); + } else { + LCK_MTX_ASSERT(&((struct inpcb *)so->so_pcb)->inpcb_mtx, + LCK_MTX_ASSERT_OWNED); + lck_mtx_unlock(&((struct inpcb *)so->so_pcb)->inpcb_mtx); + } } return (0); } lck_mtx_t * -tcp_getlock(struct socket *so, __unused int locktype) +tcp_getlock(struct socket *so, int flags) { struct inpcb *inp = sotoinpcb(so); @@ -2944,7 +3089,14 @@ tcp_getlock(struct socket *so, __unused int locktype) if (so->so_usecount < 0) panic("tcp_getlock: so=%p usecount=%x lrh= %s\n", so, so->so_usecount, solockhistory_nr(so)); - return (&inp->inpcb_mtx); + + if (so->so_flags & SOF_MP_SUBFLOW) { + struct mptcb *mp_tp = tptomptp(sototcpcb(so)); + + return (mpte_getlock(mp_tp->mpt_mpte, flags)); + } else { + return (&inp->inpcb_mtx); + } } else { panic("tcp_getlock: so=%p NULL so_pcb %s\n", so, solockhistory_nr(so)); @@ -3118,7 +3270,7 @@ calculate_tcp_clock(void) * is to update the counter returnable via net_uptime() when * we read time. */ - net_update_uptime_secs(now.tv_sec); + net_update_uptime_with_time(&now); timevaladd(&tv, &interval); if (timevalcmp(&now, &tv, >)) { @@ -3447,7 +3599,8 @@ tcp_make_keepalive_frame(struct tcpcb *tp, struct ifnet *ifp, ip = (__typeof__(ip))(void *)data; - ip->ip_id = ip_randomid(); + ip->ip_id = rfc6864 ? 0 : ip_randomid(); + ip->ip_off = htons(IP_DF); ip->ip_len = htons(sizeof(struct ip) + sizeof(struct tcphdr)); ip->ip_ttl = inp->inp_ip_ttl; ip->ip_tos |= (inp->inp_ip_tos & ~IPTOS_ECN_MASK); @@ -3547,43 +3700,43 @@ tcp_fill_keepalive_offload_frames(ifnet_t ifp, if (inp->inp_ppcb == NULL || in_pcb_checkstate(inp, WNT_ACQUIRE, 0) == WNT_STOPUSING) continue; - tcp_lock(so, 1, 0); + socket_lock(so, 1); /* Release the want count */ if (inp->inp_ppcb == NULL || (in_pcb_checkstate(inp, WNT_RELEASE, 1) == WNT_STOPUSING)) { - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); continue; } if ((inp->inp_vflag & INP_IPV4) && (inp->inp_laddr.s_addr == INADDR_ANY || inp->inp_faddr.s_addr == INADDR_ANY)) { - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); continue; } if ((inp->inp_vflag & INP_IPV6) && (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr) || IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr))) { - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); continue; } if (inp->inp_lport == 0 || inp->inp_fport == 0) { - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); continue; } if (inp->inp_last_outifp == NULL || inp->inp_last_outifp->if_index != ifp->if_index) { - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); continue; } if ((inp->inp_vflag & INP_IPV4) && frame_data_offset + sizeof(struct ip) + sizeof(struct tcphdr) > IFNET_KEEPALIVE_OFFLOAD_FRAME_DATA_SIZE) { - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); continue; } else if (!(inp->inp_vflag & INP_IPV4) && frame_data_offset + sizeof(struct ip6_hdr) + sizeof(struct tcphdr) > IFNET_KEEPALIVE_OFFLOAD_FRAME_DATA_SIZE) { - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); continue; } /* @@ -3592,7 +3745,7 @@ tcp_fill_keepalive_offload_frames(ifnet_t ifp, * for processes that will sent and receive data */ if (tp->t_state != TCPS_ESTABLISHED) { - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); continue; } /* @@ -3646,7 +3799,7 @@ tcp_fill_keepalive_offload_frames(ifnet_t ifp, */ m = tcp_make_keepalive_frame(tp, ifp, TRUE); if (m == NULL) { - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); continue; } bcopy(m->m_data, frame->data + frame_data_offset, @@ -3658,7 +3811,7 @@ tcp_fill_keepalive_offload_frames(ifnet_t ifp, */ m = tcp_make_keepalive_frame(tp, ifp, FALSE); if (m == NULL) { - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); continue; } bcopy(m->m_data, frame->reply_data + frame_data_offset, @@ -3666,7 +3819,7 @@ tcp_fill_keepalive_offload_frames(ifnet_t ifp, m_freem(m); frame_index++; - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); } lck_rw_done(tcbinfo.ipi_lock); *used_frames_count = frame_index; @@ -3820,3 +3973,125 @@ inp_get_sndbytes_allunsent(struct socket *so, u_int32_t th_ack) } return (0); } + +#define IFP_PER_FLOW_STAT(_ipv4_, _stat_) { \ + if (_ipv4_) { \ + ifp->if_ipv4_stat->_stat_++; \ + } else { \ + ifp->if_ipv6_stat->_stat_++; \ + } \ +} + +#define FLOW_ECN_ENABLED(_flags_) \ + ((_flags_ & (TE_ECN_ON)) == (TE_ECN_ON)) + +void tcp_update_stats_per_flow(struct ifnet_stats_per_flow *ifs, + struct ifnet *ifp) +{ + if (ifp == NULL || !IF_FULLY_ATTACHED(ifp)) + return; + + ifnet_lock_shared(ifp); + if (ifs->ecn_flags & TE_SETUPSENT) { + if (ifs->ecn_flags & TE_CLIENT_SETUP) { + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_client_setup); + if (FLOW_ECN_ENABLED(ifs->ecn_flags)) { + IFP_PER_FLOW_STAT(ifs->ipv4, + ecn_client_success); + } else if (ifs->ecn_flags & TE_LOST_SYN) { + IFP_PER_FLOW_STAT(ifs->ipv4, + ecn_syn_lost); + } else { + IFP_PER_FLOW_STAT(ifs->ipv4, + ecn_peer_nosupport); + } + } else { + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_server_setup); + if (FLOW_ECN_ENABLED(ifs->ecn_flags)) { + IFP_PER_FLOW_STAT(ifs->ipv4, + ecn_server_success); + } else if (ifs->ecn_flags & TE_LOST_SYN) { + IFP_PER_FLOW_STAT(ifs->ipv4, + ecn_synack_lost); + } else { + IFP_PER_FLOW_STAT(ifs->ipv4, + ecn_peer_nosupport); + } + } + } else { + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_off_conn); + } + if (FLOW_ECN_ENABLED(ifs->ecn_flags)) { + if (ifs->ecn_flags & TE_RECV_ECN_CE) { + tcpstat.tcps_ecn_conn_recv_ce++; + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_conn_recv_ce); + } + if (ifs->ecn_flags & TE_RECV_ECN_ECE) { + tcpstat.tcps_ecn_conn_recv_ece++; + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_conn_recv_ece); + } + if (ifs->ecn_flags & (TE_RECV_ECN_CE | TE_RECV_ECN_ECE)) { + if (ifs->txretransmitbytes > 0 || + ifs->rxoutoforderbytes > 0) { + tcpstat.tcps_ecn_conn_pl_ce++; + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_conn_plce); + } else { + tcpstat.tcps_ecn_conn_nopl_ce++; + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_conn_noplce); + } + } else { + if (ifs->txretransmitbytes > 0 || + ifs->rxoutoforderbytes > 0) { + tcpstat.tcps_ecn_conn_plnoce++; + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_conn_plnoce); + } + } + } + + /* Other stats are interesting for non-local connections only */ + if (ifs->local) { + ifnet_lock_done(ifp); + return; + } + + if (ifs->ipv4) { + ifp->if_ipv4_stat->timestamp = net_uptime(); + if (FLOW_ECN_ENABLED(ifs->ecn_flags)) { + tcp_flow_ecn_perf_stats(ifs, &ifp->if_ipv4_stat->ecn_on); + } else { + tcp_flow_ecn_perf_stats(ifs, &ifp->if_ipv4_stat->ecn_off); + } + } else { + ifp->if_ipv6_stat->timestamp = net_uptime(); + if (FLOW_ECN_ENABLED(ifs->ecn_flags)) { + tcp_flow_ecn_perf_stats(ifs, &ifp->if_ipv6_stat->ecn_on); + } else { + tcp_flow_ecn_perf_stats(ifs, &ifp->if_ipv6_stat->ecn_off); + } + } + + if (ifs->rxmit_drop) { + if (FLOW_ECN_ENABLED(ifs->ecn_flags)) { + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_on.rxmit_drop); + } else { + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_off.rxmit_drop); + } + } + if (ifs->ecn_fallback_synloss) + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_fallback_synloss); + if (ifs->ecn_fallback_droprst) + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_fallback_droprst); + if (ifs->ecn_fallback_droprxmt) + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_fallback_droprxmt); + if (ifs->ecn_fallback_ce) + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_fallback_ce); + if (ifs->ecn_fallback_reorder) + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_fallback_reorder); + if (ifs->ecn_recv_ce > 0) + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_recv_ce); + if (ifs->ecn_recv_ece > 0) + IFP_PER_FLOW_STAT(ifs->ipv4, ecn_recv_ece); + + tcp_flow_lim_stats(ifs, &ifp->if_lim_stat); + ifnet_lock_done(ifp); +} diff --git a/bsd/netinet/tcp_timer.c b/bsd/netinet/tcp_timer.c index 5c87a2d18..f410382ed 100644 --- a/bsd/netinet/tcp_timer.c +++ b/bsd/netinet/tcp_timer.c @@ -124,10 +124,10 @@ struct tcptailq tcp_tw_tailq; static int sysctl_msec_to_ticks SYSCTL_HANDLER_ARGS { -#pragma unused(arg1, arg2) +#pragma unused(arg2) int error, s, tt; - tt = *(int *)oidp->oid_arg1; + tt = *(int *)arg1; s = tt * 1000 / TCP_RETRANSHZ;; error = sysctl_handle_int(oidp, &s, 0, req); @@ -138,34 +138,65 @@ sysctl_msec_to_ticks SYSCTL_HANDLER_ARGS if (tt < 1) return (EINVAL); - *(int *)oidp->oid_arg1 = tt; - return (0); + *(int *)arg1 = tt; + SYSCTL_SKMEM_UPDATE_AT_OFFSET(arg2, *(int*)arg1); + return (0); } -int tcp_keepinit; +#if SYSCTL_SKMEM +int tcp_keepinit = TCPTV_KEEP_INIT; +SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINIT, keepinit, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, + &tcp_keepinit, offsetof(skmem_sysctl, tcp.keepinit), + sysctl_msec_to_ticks, "I", ""); + +int tcp_keepidle = TCPTV_KEEP_IDLE; +SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPIDLE, keepidle, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, + &tcp_keepidle, offsetof(skmem_sysctl, tcp.keepidle), + sysctl_msec_to_ticks, "I", ""); + +int tcp_keepintvl = TCPTV_KEEPINTVL; +SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINTVL, keepintvl, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, + &tcp_keepintvl, offsetof(skmem_sysctl, tcp.keepintvl), + sysctl_msec_to_ticks, "I", ""); + +SYSCTL_SKMEM_TCP_INT(OID_AUTO, keepcnt, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, + int, tcp_keepcnt, TCPTV_KEEPCNT, "number of times to repeat keepalive"); + +int tcp_msl = TCPTV_MSL; +SYSCTL_PROC(_net_inet_tcp, OID_AUTO, msl, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, + &tcp_msl, offsetof(skmem_sysctl, tcp.msl), + sysctl_msec_to_ticks, "I", "Maximum segment lifetime"); +#else /* SYSCTL_SKMEM */ +int tcp_keepinit; SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINIT, keepinit, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_keepinit, 0, sysctl_msec_to_ticks, "I", ""); -int tcp_keepidle; +int tcp_keepidle; SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPIDLE, keepidle, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_keepidle, 0, sysctl_msec_to_ticks, "I", ""); -int tcp_keepintvl; +int tcp_keepintvl; SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINTVL, keepintvl, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_keepintvl, 0, sysctl_msec_to_ticks, "I", ""); -int tcp_keepcnt; +int tcp_keepcnt; SYSCTL_INT(_net_inet_tcp, OID_AUTO, keepcnt, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_keepcnt, 0, "number of times to repeat keepalive"); -int tcp_msl; +int tcp_msl; SYSCTL_PROC(_net_inet_tcp, OID_AUTO, msl, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_msl, 0, sysctl_msec_to_ticks, "I", "Maximum segment lifetime"); +#endif /* SYSCTL_SKMEM */ /* * Avoid DoS via TCP Robustness in Persist Condition @@ -176,26 +207,32 @@ SYSCTL_PROC(_net_inet_tcp, OID_AUTO, msl, * Expressed in milliseconds to be consistent without timeout related * values, the TCP socket option is in seconds. */ +#if SYSCTL_SKMEM +u_int32_t tcp_max_persist_timeout = 0; +SYSCTL_PROC(_net_inet_tcp, OID_AUTO, max_persist_timeout, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, + &tcp_max_persist_timeout, offsetof(skmem_sysctl, tcp.max_persist_timeout), + sysctl_msec_to_ticks, "I", "Maximum persistence timeout for ZWP"); +#else /* SYSCTL_SKMEM */ u_int32_t tcp_max_persist_timeout = 0; SYSCTL_PROC(_net_inet_tcp, OID_AUTO, max_persist_timeout, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_max_persist_timeout, 0, sysctl_msec_to_ticks, "I", "Maximum persistence timeout for ZWP"); +#endif /* SYSCTL_SKMEM */ -static int always_keepalive = 0; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, always_keepalive, - CTLFLAG_RW | CTLFLAG_LOCKED, - &always_keepalive , 0, "Assume SO_KEEPALIVE on all TCP connections"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, always_keepalive, + CTLFLAG_RW | CTLFLAG_LOCKED, static int, always_keepalive, 0, + "Assume SO_KEEPALIVE on all TCP connections"); /* * This parameter determines how long the timer list will stay in fast or * quick mode even though all connections are idle. In this state, the * timer will run more frequently anticipating new data. */ -int timer_fastmode_idlemax = TCP_FASTMODE_IDLERUN_MAX; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, timer_fastmode_idlemax, - CTLFLAG_RW | CTLFLAG_LOCKED, - &timer_fastmode_idlemax, 0, "Maximum idle generations in fast mode"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, timer_fastmode_idlemax, + CTLFLAG_RW | CTLFLAG_LOCKED, int, timer_fastmode_idlemax, + TCP_FASTMODE_IDLERUN_MAX, "Maximum idle generations in fast mode"); /* * See tcp_syn_backoff[] for interval values between SYN retransmits; @@ -204,10 +241,9 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, timer_fastmode_idlemax, * SYN retransmits. Setting it to 0 disables the dropping off of those * two options. */ -static int tcp_broken_peer_syn_rxmit_thres = 10; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, broken_peer_syn_rexmit_thres, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_broken_peer_syn_rxmit_thres, 0, - "Number of retransmitted SYNs before disabling RFC 1323 " +SYSCTL_SKMEM_TCP_INT(OID_AUTO, broken_peer_syn_rexmit_thres, + CTLFLAG_RW | CTLFLAG_LOCKED, static int, tcp_broken_peer_syn_rxmit_thres, + 10, "Number of retransmitted SYNs before disabling RFC 1323 " "options on local connections"); static int tcp_timer_advanced = 0; @@ -220,14 +256,12 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcp_resched_timerlist, CTLFLAG_RD | CTLFLAG_LOCKED, &tcp_resched_timerlist, 0, "Number of times timer list was rescheduled as part of processing a packet"); -int tcp_pmtud_black_hole_detect = 1 ; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, pmtud_blackhole_detection, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_pmtud_black_hole_detect, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, pmtud_blackhole_detection, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_pmtud_black_hole_detect, 1, "Path MTU Discovery Black Hole Detection"); -int tcp_pmtud_black_hole_mss = 1200 ; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, pmtud_blackhole_mss, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_pmtud_black_hole_mss, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, pmtud_blackhole_mss, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_pmtud_black_hole_mss, 1200, "Path MTU Discovery Black Hole Detection lowered MSS"); static u_int32_t tcp_mss_rec_medium = 1200; @@ -240,7 +274,7 @@ int tcp_report_stats_interval = TCP_REPORT_STATS_INTERVAL; static boolean_t tcp_gc_done = FALSE; /* max idle probes */ -int tcp_maxpersistidle; +int tcp_maxpersistidle = TCPTV_KEEP_IDLE; /* * TCP delack timer is set to 100 ms. Since the processing of timer list @@ -311,6 +345,36 @@ struct tcp_last_report_stats { u_int32_t tcps_tfo_no_cookie_rcv; u_int32_t tcps_tfo_heuristics_disable; u_int32_t tcps_tfo_sndblackhole; + + /* MPTCP-related statistics */ + u_int32_t tcps_mptcp_handover_attempt; + u_int32_t tcps_mptcp_interactive_attempt; + u_int32_t tcps_mptcp_aggregate_attempt; + u_int32_t tcps_mptcp_fp_handover_attempt; + u_int32_t tcps_mptcp_fp_interactive_attempt; + u_int32_t tcps_mptcp_fp_aggregate_attempt; + u_int32_t tcps_mptcp_heuristic_fallback; + u_int32_t tcps_mptcp_fp_heuristic_fallback; + u_int32_t tcps_mptcp_handover_success_wifi; + u_int32_t tcps_mptcp_handover_success_cell; + u_int32_t tcps_mptcp_interactive_success; + u_int32_t tcps_mptcp_aggregate_success; + u_int32_t tcps_mptcp_fp_handover_success_wifi; + u_int32_t tcps_mptcp_fp_handover_success_cell; + u_int32_t tcps_mptcp_fp_interactive_success; + u_int32_t tcps_mptcp_fp_aggregate_success; + u_int32_t tcps_mptcp_handover_cell_from_wifi; + u_int32_t tcps_mptcp_handover_wifi_from_cell; + u_int32_t tcps_mptcp_interactive_cell_from_wifi; + u_int64_t tcps_mptcp_handover_cell_bytes; + u_int64_t tcps_mptcp_interactive_cell_bytes; + u_int64_t tcps_mptcp_aggregate_cell_bytes; + u_int64_t tcps_mptcp_handover_all_bytes; + u_int64_t tcps_mptcp_interactive_all_bytes; + u_int64_t tcps_mptcp_aggregate_all_bytes; + u_int32_t tcps_mptcp_back_to_wifi; + u_int32_t tcps_mptcp_wifi_proxy; + u_int32_t tcps_mptcp_cell_proxy; }; @@ -422,7 +486,7 @@ add_to_time_wait_locked(struct tcpcb *tp, uint32_t delay) uint32_t timer; /* pcb list should be locked when we get here */ - lck_rw_assert(pcbinfo->ipi_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(pcbinfo->ipi_lock, LCK_RW_ASSERT_EXCLUSIVE); /* We may get here multiple times, so check */ if (!(inp->inp_flags2 & INP2_TIMEWAIT)) { @@ -458,9 +522,9 @@ add_to_time_wait(struct tcpcb *tp, uint32_t delay) nstat_pcb_detach(tp->t_inpcb); if (!lck_rw_try_lock_exclusive(pcbinfo->ipi_lock)) { - tcp_unlock(tp->t_inpcb->inp_socket, 0, 0); + socket_unlock(tp->t_inpcb->inp_socket, 0); lck_rw_lock_exclusive(pcbinfo->ipi_lock); - tcp_lock(tp->t_inpcb->inp_socket, 0, 0); + socket_lock(tp->t_inpcb->inp_socket, 0); } add_to_time_wait_locked(tp, delay); lck_rw_done(pcbinfo->ipi_lock); @@ -481,12 +545,22 @@ static boolean_t tcp_garbage_collect(struct inpcb *inp, int istimewait) { boolean_t active = FALSE; - struct socket *so; + struct socket *so, *mp_so = NULL; struct tcpcb *tp; so = inp->inp_socket; tp = intotcpcb(inp); + if (so->so_flags & SOF_MP_SUBFLOW) { + mp_so = mptetoso(tptomptp(tp)->mpt_mpte); + if (!socket_try_lock(mp_so)) { + mp_so = NULL; + active = TRUE; + goto out; + } + mp_so->so_usecount++; + } + /* * Skip if still in use or busy; it would have been more efficient * if we were to test so_usecount against 0, but this isn't possible @@ -494,20 +568,21 @@ tcp_garbage_collect(struct inpcb *inp, int istimewait) * overflow sockets that are eligible for garbage collection have * their usecounts set to 1. */ - if (!lck_mtx_try_lock_spin(&inp->inpcb_mtx)) - return (TRUE); + if (!lck_mtx_try_lock_spin(&inp->inpcb_mtx)) { + active = TRUE; + goto out; + } /* Check again under the lock */ if (so->so_usecount > 1) { if (inp->inp_wantcnt == WNT_STOPUSING) active = TRUE; lck_mtx_unlock(&inp->inpcb_mtx); - return (active); + goto out; } - if (istimewait && - TSTMP_GEQ(tcp_now, tp->t_timer[TCPT_2MSL]) && - tp->t_state != TCPS_CLOSED) { + if (istimewait && TSTMP_GEQ(tcp_now, tp->t_timer[TCPT_2MSL]) && + tp->t_state != TCPS_CLOSED) { /* Become a regular mutex */ lck_mtx_convert_spin(&inp->inpcb_mtx); tcp_close(tp); @@ -544,10 +619,11 @@ tcp_garbage_collect(struct inpcb *inp, int istimewait) if (inp->inp_wantcnt == WNT_STOPUSING) active = TRUE; lck_mtx_unlock(&inp->inpcb_mtx); - return (active); + goto out; } else if (inp->inp_wantcnt != WNT_STOPUSING) { lck_mtx_unlock(&inp->inpcb_mtx); - return (FALSE); + active = FALSE; + goto out; } /* @@ -583,12 +659,28 @@ tcp_garbage_collect(struct inpcb *inp, int istimewait) #endif /* INET6 */ in_pcbdetach(inp); } + + if (mp_so) { + mptcp_subflow_del(tptomptp(tp)->mpt_mpte, tp->t_mpsub); + + /* so is now unlinked from mp_so - let's drop the lock */ + socket_unlock(mp_so, 1); + mp_so = NULL; + } + in_pcbdispose(inp); - return (FALSE); + active = FALSE; + goto out; } lck_mtx_unlock(&inp->inpcb_mtx); - return (TRUE); + active = TRUE; + +out: + if (mp_so) + socket_unlock(mp_so, 1); + + return (active); } /* @@ -758,6 +850,9 @@ tcp_pmtud_revert_segment_size(struct tcpcb *tp) CC_ALGO(tp)->cwnd_init(tp); tp->t_pmtud_start_ts = 0; tcpstat.tcps_pmtudbh_reverted++; + + /* change MSS according to recommendation, if there was one */ + tcp_update_mss_locked(tp->t_inpcb->inp_socket, NULL); } /* @@ -833,6 +928,20 @@ tcp_timers(struct tcpcb *tp, int timer) ((tp->t_flagsext & TF_RXTFINDROP) != 0 && (tp->t_flags & TF_SENTFIN) != 0 && tp->t_rxtshift >= 4) || (tp->t_rxtshift > 4 && last_sleep_ms >= TCP_SLEEP_TOO_LONG)) { + if (tp->t_state == TCPS_ESTABLISHED && + tp->t_rxt_minimum_timeout > 0) { + /* + * Avoid dropping a connection if minimum + * timeout is set and that time did not + * pass. We will retry sending + * retransmissions at the maximum interval + */ + if (TSTMP_LT(tcp_now, (tp->t_rxtstart + + tp->t_rxt_minimum_timeout))) { + tp->t_rxtshift = TCP_MAXRXTSHIFT - 1; + goto retransmit_packet; + } + } if ((tp->t_flagsext & TF_RXTFINDROP) != 0) { tcpstat.tcps_rxtfindrop++; } else if (last_sleep_ms >= TCP_SLEEP_TOO_LONG) { @@ -863,7 +972,7 @@ tcp_timers(struct tcpcb *tp, int timer) break; } - +retransmit_packet: tcpstat.tcps_rexmttimeo++; tp->t_accsleep_ms = accsleep_ms; @@ -886,6 +995,12 @@ tcp_timers(struct tcpcb *tp, int timer) mptcp_act_on_txfail(so); } + + if (so->so_flags & SOF_MP_SUBFLOW) { + struct mptses *mpte = tptomptp(tp)->mpt_mpte; + + mptcp_check_subflows_and_add(mpte); + } #endif /* MPTCP */ if (tp->t_adaptive_wtimo > 0 && @@ -921,7 +1036,8 @@ tcp_timers(struct tcpcb *tp, int timer) if (tp->t_state == TCPS_SYN_RECEIVED) tcp_disable_tfo(tp); - if ((tp->t_tfo_stats & TFO_S_SYN_DATA_SENT) && + if (!(tp->t_tfo_flags & TFO_F_HEURISTIC_DONE) && + (tp->t_tfo_stats & TFO_S_SYN_DATA_SENT) && !(tp->t_tfo_flags & TFO_F_NO_SNDPROBING) && ((tp->t_state != TCPS_SYN_SENT && tp->t_rxtshift > 1) || tp->t_rxtshift > 2)) { @@ -944,12 +1060,26 @@ tcp_timers(struct tcpcb *tp, int timer) tcpstat.tcps_tfo_sndblackhole++; } + if (!(tp->t_tfo_flags & TFO_F_HEURISTIC_DONE) && + (tp->t_tfo_stats & TFO_S_SYN_DATA_ACKED) && + tp->t_rxtshift > 1) { + if (TSTMP_GT(tp->t_sndtime - 10 * TCP_RETRANSHZ, tp->t_rcvtime)) { + tcp_heuristic_tfo_middlebox(tp); + + so->so_error = ENODATA; + sorwakeup(so); + sowwakeup(so); + } + } + if (tp->t_state == TCPS_SYN_SENT) { rexmt = TCP_REXMTVAL(tp) * tcp_syn_backoff[tp->t_rxtshift]; tp->t_stat.synrxtshift = tp->t_rxtshift; /* When retransmitting, disable TFO */ - if (tfo_enabled(tp) && !(so->so_flags & SOF1_DATA_AUTHENTICATED)) { + if (tfo_enabled(tp) && + (!(so->so_flags1 & SOF1_DATA_AUTHENTICATED) || + (tp->t_flagsext & TF_FASTOPEN_HEUR))) { tp->t_flagsext &= ~TF_FASTOPEN; tp->t_tfo_flags |= TFO_F_SYN_LOSS; } @@ -1153,7 +1283,7 @@ fc_output: * Regular TCP connections do not send keepalives after closing * MPTCP must not also, after sending Data FINs. */ - struct mptcb *mp_tp = tp->t_mptcb; + struct mptcb *mp_tp = tptomptp(tp); if ((tp->t_mpflags & TMPF_MPTCP_TRUE) && (tp->t_state > TCPS_ESTABLISHED)) { goto dropit; @@ -1261,7 +1391,8 @@ fc_output: tp->t_timer[TCPT_KEEP] = min(OFFSET_FROM_START( tp, tcp_backoff[ind] * TCP_REXMTVAL(tp)), tp->t_timer[TCPT_KEEP]); - } else if (tp->t_tfo_probe_state == TFO_PROBE_WAIT_DATA) { + } else if (!(tp->t_tfo_flags & TFO_F_HEURISTIC_DONE) && + tp->t_tfo_probe_state == TFO_PROBE_WAIT_DATA) { /* Still no data! Let's assume a TFO-error and err out... */ tcp_heuristic_tfo_middlebox(tp); @@ -1328,13 +1459,14 @@ fc_output: tcpstat.tcps_timeoutdrop++; postevent(so, 0, EV_TIMEOUT); soevent(so, - (SO_FILT_HINT_LOCKED| + (SO_FILT_HINT_LOCKED| SO_FILT_HINT_TIMEOUT)); tp = tcp_drop(tp, tp->t_softerror ? - tp->t_softerror : ETIMEDOUT); + tp->t_softerror : ETIMEDOUT); break; } tcpstat.tcps_join_rxmts++; + tp->t_mpflags |= TMPF_SND_JACK; tp->t_flags |= TF_ACKNOW; /* @@ -1448,7 +1580,7 @@ tcp_remove_timer(struct tcpcb *tp) { struct tcptimerlist *listp = &tcp_timer_list; - lck_mtx_assert(&tp->t_inpcb->inpcb_mtx, LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(tp->t_inpcb->inp_socket); if (!(TIMER_IS_ON_LIST(tp))) { return; } @@ -1521,10 +1653,11 @@ tcp_sched_timerlist(uint32_t offset) uint64_t deadline = 0; struct tcptimerlist *listp = &tcp_timer_list; - lck_mtx_assert(listp->mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(listp->mtx, LCK_MTX_ASSERT_OWNED); offset = min(offset, TCP_TIMERLIST_MAX_OFFSET); listp->runtime = tcp_now + offset; + listp->schedtime = tcp_now; if (listp->runtime == 0) { listp->runtime++; offset++; @@ -1560,7 +1693,7 @@ tcp_run_conn_timer(struct tcpcb *tp, u_int16_t *te_mode, bzero(needtorun, sizeof(needtorun)); *te_mode = 0; - tcp_lock(tp->t_inpcb->inp_socket, 1, 0); + socket_lock(tp->t_inpcb->inp_socket, 1); so = tp->t_inpcb->inp_socket; /* Release the want count on inp */ @@ -1676,7 +1809,7 @@ done: offset = 0; } - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); return(offset); } @@ -1696,6 +1829,27 @@ tcp_run_timerlist(void * arg1, void * arg2) lck_mtx_lock(listp->mtx); + int32_t drift = tcp_now - listp->runtime; + if (drift <= 1) { + tcpstat.tcps_timer_drift_le_1_ms++; + } else if (drift <= 10) { + tcpstat.tcps_timer_drift_le_10_ms++; + } else if (drift <= 20) { + tcpstat.tcps_timer_drift_le_20_ms++; + } else if (drift <= 50) { + tcpstat.tcps_timer_drift_le_50_ms++; + } else if (drift <= 100) { + tcpstat.tcps_timer_drift_le_100_ms++; + } else if (drift <= 200) { + tcpstat.tcps_timer_drift_le_200_ms++; + } else if (drift <= 500) { + tcpstat.tcps_timer_drift_le_500_ms++; + } else if (drift <= 1000) { + tcpstat.tcps_timer_drift_le_1000_ms++; + } else { + tcpstat.tcps_timer_drift_gt_1000_ms++; + } + listp->running = TRUE; LIST_FOREACH_SAFE(te, &listp->lhead, le, next_te) { @@ -1975,7 +2129,7 @@ tcp_set_lotimer_index(struct tcpcb *tp) void tcp_check_timer_state(struct tcpcb *tp) { - lck_mtx_assert(&tp->t_inpcb->inpcb_mtx, LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(tp->t_inpcb->inp_socket); if (tp->t_inpcb->inp_flags2 & INP2_TIMEWAIT) return; @@ -1999,6 +2153,19 @@ tcp_cumulative_stat(u_int32_t cur, u_int32_t *prev, u_int32_t *dest) return; } +static inline void +tcp_cumulative_stat64(u_int64_t cur, u_int64_t *prev, u_int64_t *dest) +{ + /* handle wrap around */ + int64_t diff = (int64_t) (cur - *prev); + if (diff > 0) + *dest = diff; + else + *dest = 0; + *prev = cur; + return; +} + __private_extern__ void tcp_report_stats(void) { @@ -2164,6 +2331,62 @@ tcp_report_stats(void) &prev.tcps_tfo_sndblackhole, &stat.tfo_sndblackhole); + tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_attempt, + &prev.tcps_mptcp_handover_attempt , &stat.mptcp_handover_attempt); + tcp_cumulative_stat(tcpstat.tcps_mptcp_interactive_attempt, + &prev.tcps_mptcp_interactive_attempt , &stat.mptcp_interactive_attempt); + tcp_cumulative_stat(tcpstat.tcps_mptcp_aggregate_attempt, + &prev.tcps_mptcp_aggregate_attempt , &stat.mptcp_aggregate_attempt); + tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_handover_attempt, + &prev.tcps_mptcp_fp_handover_attempt , &stat.mptcp_fp_handover_attempt); + tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_interactive_attempt, + &prev.tcps_mptcp_fp_interactive_attempt , &stat.mptcp_fp_interactive_attempt); + tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_aggregate_attempt, + &prev.tcps_mptcp_fp_aggregate_attempt , &stat.mptcp_fp_aggregate_attempt); + tcp_cumulative_stat(tcpstat.tcps_mptcp_heuristic_fallback, + &prev.tcps_mptcp_heuristic_fallback , &stat.mptcp_heuristic_fallback); + tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_heuristic_fallback, + &prev.tcps_mptcp_fp_heuristic_fallback , &stat.mptcp_fp_heuristic_fallback); + tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_success_wifi, + &prev.tcps_mptcp_handover_success_wifi , &stat.mptcp_handover_success_wifi); + tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_success_cell, + &prev.tcps_mptcp_handover_success_cell , &stat.mptcp_handover_success_cell); + tcp_cumulative_stat(tcpstat.tcps_mptcp_interactive_success, + &prev.tcps_mptcp_interactive_success , &stat.mptcp_interactive_success); + tcp_cumulative_stat(tcpstat.tcps_mptcp_aggregate_success, + &prev.tcps_mptcp_aggregate_success , &stat.mptcp_aggregate_success); + tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_handover_success_wifi, + &prev.tcps_mptcp_fp_handover_success_wifi , &stat.mptcp_fp_handover_success_wifi); + tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_handover_success_cell, + &prev.tcps_mptcp_fp_handover_success_cell , &stat.mptcp_fp_handover_success_cell); + tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_interactive_success, + &prev.tcps_mptcp_fp_interactive_success , &stat.mptcp_fp_interactive_success); + tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_aggregate_success, + &prev.tcps_mptcp_fp_aggregate_success , &stat.mptcp_fp_aggregate_success); + tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_cell_from_wifi, + &prev.tcps_mptcp_handover_cell_from_wifi , &stat.mptcp_handover_cell_from_wifi); + tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_wifi_from_cell, + &prev.tcps_mptcp_handover_wifi_from_cell , &stat.mptcp_handover_wifi_from_cell); + tcp_cumulative_stat(tcpstat.tcps_mptcp_interactive_cell_from_wifi, + &prev.tcps_mptcp_interactive_cell_from_wifi , &stat.mptcp_interactive_cell_from_wifi); + tcp_cumulative_stat64(tcpstat.tcps_mptcp_handover_cell_bytes, + &prev.tcps_mptcp_handover_cell_bytes , &stat.mptcp_handover_cell_bytes); + tcp_cumulative_stat64(tcpstat.tcps_mptcp_interactive_cell_bytes, + &prev.tcps_mptcp_interactive_cell_bytes , &stat.mptcp_interactive_cell_bytes); + tcp_cumulative_stat64(tcpstat.tcps_mptcp_aggregate_cell_bytes, + &prev.tcps_mptcp_aggregate_cell_bytes , &stat.mptcp_aggregate_cell_bytes); + tcp_cumulative_stat64(tcpstat.tcps_mptcp_handover_all_bytes, + &prev.tcps_mptcp_handover_all_bytes , &stat.mptcp_handover_all_bytes); + tcp_cumulative_stat64(tcpstat.tcps_mptcp_interactive_all_bytes, + &prev.tcps_mptcp_interactive_all_bytes , &stat.mptcp_interactive_all_bytes); + tcp_cumulative_stat64(tcpstat.tcps_mptcp_aggregate_all_bytes, + &prev.tcps_mptcp_aggregate_all_bytes , &stat.mptcp_aggregate_all_bytes); + tcp_cumulative_stat(tcpstat.tcps_mptcp_back_to_wifi, + &prev.tcps_mptcp_back_to_wifi , &stat.mptcp_back_to_wifi); + tcp_cumulative_stat(tcpstat.tcps_mptcp_wifi_proxy, + &prev.tcps_mptcp_wifi_proxy , &stat.mptcp_wifi_proxy); + tcp_cumulative_stat(tcpstat.tcps_mptcp_cell_proxy, + &prev.tcps_mptcp_cell_proxy , &stat.mptcp_cell_proxy); nstat_sysinfo_send_data(&data); @@ -2263,6 +2486,9 @@ tcp_disable_read_probe(struct tcpcb *tp) ((tp->t_flagsext & TF_DETECT_READSTALL) || tp->t_rtimo_probes > 0)) { tcp_keepalive_reset(tp); + + if (tp->t_mpsub) + mptcp_reset_keepalive(tp); } } @@ -2297,12 +2523,12 @@ tcp_probe_connectivity(struct ifnet *ifp, u_int32_t enable) continue; /* Acquire lock to look at the state of the connection */ - tcp_lock(inp->inp_socket, 1, 0); + socket_lock(inp->inp_socket, 1); /* Release the want count */ if (inp->inp_ppcb == NULL || (in_pcb_checkstate(inp, WNT_RELEASE, 1) == WNT_STOPUSING)) { - tcp_unlock(inp->inp_socket, 1, 0); + socket_unlock(inp->inp_socket, 1); continue; } tp = intotcpcb(inp); @@ -2311,7 +2537,7 @@ tcp_probe_connectivity(struct ifnet *ifp, u_int32_t enable) else tcp_disable_read_probe(tp); - tcp_unlock(inp->inp_socket, 1, 0); + socket_unlock(inp->inp_socket, 1); } lck_rw_done(pcbinfo->ipi_lock); @@ -2384,12 +2610,9 @@ tcp_update_mss_locked(struct socket *so, struct ifnet *ifp) struct inpcb *inp = sotoinpcb(so); struct tcpcb *tp = intotcpcb(inp); - if (ifp == NULL && inp->inp_last_outifp == NULL) + if (ifp == NULL && (ifp = inp->inp_last_outifp) == NULL) return; - if (ifp == NULL) - ifp = inp->inp_last_outifp; - if (!IFNET_IS_CELLULAR(ifp)) { /* * This optimization is implemented for cellular @@ -2428,23 +2651,41 @@ tcp_itimer(struct inpcbinfo *ipi) LIST_FOREACH_SAFE(inp, &tcb, inp_list, nxt) { struct socket *so; + struct ifnet *ifp; if (inp->inp_ppcb == NULL || in_pcb_checkstate(inp, WNT_ACQUIRE, 0) == WNT_STOPUSING) continue; so = inp->inp_socket; - tcp_lock(so, 1, 0); + ifp = inp->inp_last_outifp; + socket_lock(so, 1); if (in_pcb_checkstate(inp, WNT_RELEASE, 1) == WNT_STOPUSING) { - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); continue; } so_check_extended_bk_idle_time(so); if (ipi->ipi_flags & INPCBINFO_UPDATE_MSS) { tcp_update_mss_locked(so, NULL); } - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); + + /* + * Defunct all system-initiated background sockets if the + * socket is using the cellular interface and the interface + * has its LQM set to abort. + */ + if ((ipi->ipi_flags & INPCBINFO_HANDLE_LQM_ABORT) && + IS_SO_TC_BACKGROUNDSYSTEM(so->so_traffic_class) && + ifp != NULL && IFNET_IS_CELLULAR(ifp) && + (ifp->if_interface_state.valid_bitmask & + IF_INTERFACE_STATE_LQM_STATE_VALID) && + ifp->if_interface_state.lqm_state == + IFNET_LQM_THRESH_ABORT) { + socket_defunct(current_proc(), so, + SHUTDOWN_SOCKET_LEVEL_DISCONNECT_ALL); + } } - ipi->ipi_flags &= ~INPCBINFO_UPDATE_MSS; + ipi->ipi_flags &= ~(INPCBINFO_UPDATE_MSS | INPCBINFO_HANDLE_LQM_ABORT); lck_rw_done(ipi->ipi_lock); } diff --git a/bsd/netinet/tcp_timer.h b/bsd/netinet/tcp_timer.h index 177cd162c..04a9eb5e4 100644 --- a/bsd/netinet/tcp_timer.h +++ b/bsd/netinet/tcp_timer.h @@ -177,6 +177,8 @@ #define TCPTV_PERSMIN ( 5*TCP_RETRANSHZ) /* retransmit persistence */ #define TCPTV_PERSMAX ( 60*TCP_RETRANSHZ) /* maximum persist interval */ +extern int tcptv_persmin_val; + #define TCPTV_KEEP_INIT ( 75*TCP_RETRANSHZ) /* connect keep alive */ #define TCPTV_KEEP_IDLE (120*60*TCP_RETRANSHZ) /* time before probing */ #define TCPTV_KEEPINTVL ( 75*TCP_RETRANSHZ) /* default probe interval */ @@ -248,6 +250,7 @@ struct tcptimerlist { lck_grp_attr_t *mtx_grp_attr; /* mutex group attributes */ thread_call_t call; /* call entry */ uint32_t runtime; /* time at which this list is going to run */ + uint32_t schedtime; /* time at which this list was scheduled */ uint32_t entries; /* Number of entries on the list */ uint32_t maxentries; /* Max number of entries at any time */ diff --git a/bsd/netinet/tcp_usrreq.c b/bsd/netinet/tcp_usrreq.c index a4ae7aaf4..0f0dc353e 100644 --- a/bsd/netinet/tcp_usrreq.c +++ b/bsd/netinet/tcp_usrreq.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -70,7 +70,9 @@ #if INET6 #include <sys/domain.h> #endif /* INET6 */ +#if !CONFIG_EMBEDDED #include <sys/kasl.h> +#endif #include <sys/socket.h> #include <sys/socketvar.h> #include <sys/protosw.h> @@ -101,6 +103,7 @@ #include <netinet/tcp_timer.h> #include <netinet/tcp_var.h> #include <netinet/tcpip.h> +#include <netinet/tcp_cc.h> #include <mach/sdt.h> #if TCPDEBUG #include <netinet/tcp_debug.h> @@ -117,7 +120,6 @@ #include <netinet/flow_divert.h> #endif /* FLOW_DIVERT */ -void tcp_fill_info(struct tcpcb *, struct tcp_info *); errno_t tcp_fill_info_for_info_tuple(struct info_tuple *, struct tcp_info *); int tcp_sysctl_info(struct sysctl_oid *, void *, int , struct sysctl_req *); @@ -209,7 +211,7 @@ tcp_usr_detach(struct socket *so) if (inp == 0 || (inp->inp_state == INPCB_STATE_DEAD)) { return EINVAL; /* XXX */ } - lck_mtx_assert(&((struct inpcb *)so->so_pcb)->inpcb_mtx, LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(so); tp = intotcpcb(inp); /* In case we got disconnected from the peer */ if (tp == NULL) @@ -291,6 +293,17 @@ tcp_usr_bind(struct socket *so, struct sockaddr *nam, struct proc *p) error = in_pcbbind(inp, nam, p); if (error) goto out; + +#if NECP + /* Update NECP client with bind result if not in middle of connect */ + if ((inp->inp_flags2 & INP2_CONNECT_IN_PROGRESS) && + !uuid_is_null(inp->necp_client_uuid)) { + socket_unlock(so, 0); + necp_client_assign_from_socket(so->last_pid, inp->necp_client_uuid, inp); + socket_lock(so, 0); + } +#endif /* NECP */ + COMMON_END(PRU_BIND); } @@ -396,8 +409,18 @@ static int tcp_connect_complete(struct socket *so) { struct tcpcb *tp = sototcpcb(so); + struct inpcb *inp = sotoinpcb(so); int error = 0; +#if NECP + /* Update NECP client with connected five-tuple */ + if (!uuid_is_null(inp->necp_client_uuid)) { + socket_unlock(so, 0); + necp_client_assign_from_socket(so->last_pid, inp->necp_client_uuid, inp); + socket_lock(so, 0); + } +#endif /* NECP */ + /* TFO delays the tcp_output until later, when the app calls write() */ if (so->so_flags1 & SOF1_PRECONNECT_DATA) { if (!necp_socket_is_allowed_to_send_recv(sotoinpcb(so), NULL, NULL)) @@ -495,12 +518,9 @@ tcp_usr_connectx_common(struct socket *so, int af, uint32_t flags, void *arg, uint32_t arglen, struct uio *auio, user_ssize_t *bytes_written) { -#pragma unused(aid) -#if !MPTCP -#pragma unused(flags, arg, arglen) -#endif /* !MPTCP */ +#pragma unused(aid, flags, arg, arglen) struct inpcb *inp = sotoinpcb(so); - int error; + int error = 0; user_ssize_t datalen = 0; if (inp == NULL) @@ -508,6 +528,9 @@ tcp_usr_connectx_common(struct socket *so, int af, VERIFY(dst != NULL); + ASSERT(!(inp->inp_flags2 & INP2_CONNECT_IN_PROGRESS)); + inp->inp_flags2 |= INP2_CONNECT_IN_PROGRESS; + #if NECP inp_update_necp_policy(inp, src, dst, ifscope); #endif /* NECP */ @@ -516,47 +539,17 @@ tcp_usr_connectx_common(struct socket *so, int af, (tcp_fastopen & TCP_FASTOPEN_CLIENT)) sototcpcb(so)->t_flagsext |= TF_FASTOPEN; - /* - * We get here for 2 cases: - * - * a. From MPTCP, to connect a subflow. There is no need to - * bind the socket to the source address and/or interface, - * since everything has been taken care of by MPTCP. We - * simply check whether or not this is for the initial - * MPTCP connection attempt, or to join an existing one. - * - * b. From the socket layer, to connect a TCP. Perform the - * bind to source address and/or interface as necessary. - */ -#if MPTCP - if (flags & CONNREQF_MPTCP) { - struct mptsub_connreq *mpcr = arg; - - /* Check to make sure this came down from MPTCP */ - if (arg == NULL || arglen != sizeof (*mpcr)) - return (EOPNOTSUPP); - - switch (mpcr->mpcr_type) { - case MPTSUB_CONNREQ_MP_ENABLE: - break; - case MPTSUB_CONNREQ_MP_ADD: - break; - default: - return (EOPNOTSUPP); - } - } else -#endif /* MPTCP */ - { - /* bind socket to the specified interface, if requested */ - if (ifscope != IFSCOPE_NONE && - (error = inp_bindif(inp, ifscope, NULL)) != 0) - return (error); + /* bind socket to the specified interface, if requested */ + if (ifscope != IFSCOPE_NONE && + (error = inp_bindif(inp, ifscope, NULL)) != 0) { + goto done; + } - /* if source address and/or port is specified, bind to it */ - if (src != NULL) { - error = sobindlock(so, src, 0); /* already locked */ - if (error != 0) - return (error); + /* if source address and/or port is specified, bind to it */ + if (src != NULL) { + error = sobindlock(so, src, 0); /* already locked */ + if (error != 0) { + goto done; } } @@ -574,8 +567,9 @@ tcp_usr_connectx_common(struct socket *so, int af, /* NOTREACHED */ } - if (error != 0) - return (error); + if (error != 0) { + goto done; + } /* if there is data, copy it */ if (auio != NULL) { @@ -606,6 +600,8 @@ tcp_usr_connectx_common(struct socket *so, int af, if (error == 0 && pcid != NULL) *pcid = 1; /* there is only one connection in regular TCP */ +done: + inp->inp_flags2 &= ~INP2_CONNECT_IN_PROGRESS; return (error); } @@ -735,8 +731,7 @@ tcp_usr_disconnect(struct socket *so) struct inpcb *inp = sotoinpcb(so); struct tcpcb *tp; - lck_mtx_assert(&((struct inpcb *)so->so_pcb)->inpcb_mtx, - LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(so); COMMON_START(); /* In case we got disconnected from the peer */ if (tp == NULL) @@ -1023,8 +1018,8 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m, m_freem(control); control = NULL; } else if (control->m_len) { - /* - * if not unordered, TCP should not have + /* + * if not unordered, TCP should not have * control mbufs */ m_freem(control); @@ -1089,8 +1084,8 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m, tp->t_flags &= ~TF_MORETOCOME; } } else { - if (sbspace(&so->so_snd) == 0) { - /* if no space is left in sockbuf, + if (sbspace(&so->so_snd) == 0) { + /* if no space is left in sockbuf, * do not try to squeeze in OOB traffic */ m_freem(m); error = ENOBUFS; @@ -1141,7 +1136,7 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m, error = sbwait(&so->so_snd); } - COMMON_END((flags & PRUS_OOB) ? PRU_SENDOOB : + COMMON_END((flags & PRUS_OOB) ? PRU_SENDOOB : ((flags & PRUS_EOF) ? PRU_SEND_EOF : PRU_SEND)); } @@ -1218,8 +1213,7 @@ tcp_usr_preconnect(struct socket *so) error = tcp_output(sototcpcb(so)); - /* One read has been done. This was enough. Get back to "normal" behavior. */ - so->so_flags1 &= ~SOF1_PRECONNECT_DATA; + soclearfastopen(so); COMMON_END(PRU_PRECONNECT); } @@ -1323,19 +1317,19 @@ tcp_connect(struct tcpcb *tp, struct sockaddr *nam, struct proc *p) if (error) goto done; - tcp_unlock(inp->inp_socket, 0, 0); + socket_unlock(inp->inp_socket, 0); oinp = in_pcblookup_hash(inp->inp_pcbinfo, sin->sin_addr, sin->sin_port, inp->inp_laddr.s_addr != INADDR_ANY ? inp->inp_laddr : laddr, inp->inp_lport, 0, NULL); - tcp_lock(inp->inp_socket, 0, 0); + socket_lock(inp->inp_socket, 0); if (oinp) { if (oinp != inp) /* 4143933: avoid deadlock if inp == oinp */ - tcp_lock(oinp->inp_socket, 1, 0); + socket_lock(oinp->inp_socket, 1); if (in_pcb_checkstate(oinp, WNT_RELEASE, 1) == WNT_STOPUSING) { if (oinp != inp) - tcp_unlock(oinp->inp_socket, 1, 0); + socket_unlock(oinp->inp_socket, 1); goto skip_oinp; } @@ -1348,12 +1342,12 @@ tcp_connect(struct tcpcb *tp, struct sockaddr *nam, struct proc *p) printf("tcp_connect: inp=0x%llx err=EADDRINUSE\n", (uint64_t)VM_KERNEL_ADDRPERM(inp)); if (oinp != inp) - tcp_unlock(oinp->inp_socket, 1, 0); + socket_unlock(oinp->inp_socket, 1); error = EADDRINUSE; goto done; } if (oinp != inp) - tcp_unlock(oinp->inp_socket, 1, 0); + socket_unlock(oinp->inp_socket, 1); } skip_oinp: if ((inp->inp_laddr.s_addr == INADDR_ANY ? laddr.s_addr : @@ -1372,6 +1366,7 @@ skip_oinp: inp->inp_laddr = laddr; /* no reference needed */ inp->inp_last_outifp = outif; + inp->inp_flags |= INP_INADDR_ANY; } inp->inp_faddr = sin->sin_addr; @@ -1430,14 +1425,14 @@ tcp6_connect(struct tcpcb *tp, struct sockaddr *nam, struct proc *p) error = in6_pcbladdr(inp, nam, &addr6, &outif); if (error) goto done; - tcp_unlock(inp->inp_socket, 0, 0); + socket_unlock(inp->inp_socket, 0); oinp = in6_pcblookup_hash(inp->inp_pcbinfo, &sin6->sin6_addr, sin6->sin6_port, IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr) ? &addr6 : &inp->in6p_laddr, inp->inp_lport, 0, NULL); - tcp_lock(inp->inp_socket, 0, 0); + socket_lock(inp->inp_socket, 0); if (oinp) { if (oinp != inp && (otp = intotcpcb(oinp)) != NULL && otp->t_state == TCPS_TIME_WAIT && @@ -1481,7 +1476,7 @@ tcp6_connect(struct tcpcb *tp, struct sockaddr *nam, struct proc *p) soisconnecting(so); tcpstat.tcps_connattempt++; tp->t_state = TCPS_SYN_SENT; - tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START(tp, + tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START(tp, TCP_CONN_KEEPINIT(tp)); tp->iss = tcp_new_isn(tp); tcp_sendseqinit(tp); @@ -1499,11 +1494,11 @@ done: /* * Export TCP internal state information via a struct tcp_info */ -__private_extern__ void +void tcp_fill_info(struct tcpcb *tp, struct tcp_info *ti) { struct inpcb *inp = tp->t_inpcb; - + bzero(ti, sizeof(*ti)); ti->tcpi_state = tp->t_state; @@ -1541,7 +1536,7 @@ tcp_fill_info(struct tcpcb *tp, struct tcp_info *ti) ti->tcpi_snd_ssthresh = tp->snd_ssthresh; ti->tcpi_snd_cwnd = tp->snd_cwnd; ti->tcpi_snd_sbbytes = inp->inp_socket->so_snd.sb_cc; - + ti->tcpi_rcv_space = tp->rcv_wnd; ti->tcpi_snd_wnd = tp->snd_wnd; @@ -1604,6 +1599,7 @@ tcp_fill_info(struct tcpcb *tp, struct tcp_info *ti) ti->tcpi_tfo_heuristics_disable = !!(tp->t_tfo_stats & TFO_S_HEURISTICS_DISABLE); ti->tcpi_tfo_send_blackhole = !!(tp->t_tfo_stats & TFO_S_SEND_BLACKHOLE); ti->tcpi_tfo_recv_blackhole = !!(tp->t_tfo_stats & TFO_S_RECV_BLACKHOLE); + ti->tcpi_tfo_onebyte_proxy = !!(tp->t_tfo_stats & TFO_S_ONE_BYTE_PROXY); ti->tcpi_ecn_client_setup = !!(tp->ecn_flags & TE_SETUPSENT); ti->tcpi_ecn_server_setup = !!(tp->ecn_flags & TE_SETUPRECEIVED); @@ -1616,9 +1612,20 @@ tcp_fill_info(struct tcpcb *tp, struct tcp_info *ti) if (tp->t_inpcb->inp_last_outifp != NULL) { if (IFNET_IS_CELLULAR(tp->t_inpcb->inp_last_outifp)) ti->tcpi_if_cell = 1; - else if (IFNET_IS_WIFI(tp->t_inpcb->inp_last_outifp)) + if (IFNET_IS_WIFI(tp->t_inpcb->inp_last_outifp)) ti->tcpi_if_wifi = 1; + if (IFNET_IS_WIRED(tp->t_inpcb->inp_last_outifp)) + ti->tcpi_if_wired = 1; + if (IFNET_IS_WIFI_INFRA(tp->t_inpcb->inp_last_outifp)) + ti->tcpi_if_wifi_infra = 1; + if (tp->t_inpcb->inp_last_outifp->if_eflags & IFEF_AWDL) + ti->tcpi_if_wifi_awdl = 1; } + if (tp->tcp_cc_index == TCP_CC_ALGO_BACKGROUND_INDEX) + ti->tcpi_snd_background = 1; + if (tcp_recv_bg == 1 || + IS_TCP_RECV_BG(tp->t_inpcb->inp_socket)) + ti->tcpi_rcv_background = 1; ti->tcpi_ecn_recv_ce = tp->t_ecn_recv_ce; ti->tcpi_ecn_recv_cwr = tp->t_ecn_recv_cwr; @@ -1639,15 +1646,15 @@ tcp_fill_info_for_info_tuple(struct info_tuple *itpl, struct tcp_info *ti) struct inpcb *inp = NULL; struct socket *so; struct tcpcb *tp; - + if (itpl->itpl_proto == IPPROTO_TCP) pcbinfo = &tcbinfo; else return EINVAL; - + if (itpl->itpl_local_sa.sa_family == AF_INET && itpl->itpl_remote_sa.sa_family == AF_INET) { - inp = in_pcblookup_hash(pcbinfo, + inp = in_pcblookup_hash(pcbinfo, itpl->itpl_remote_sin.sin_addr, itpl->itpl_remote_sin.sin_port, itpl->itpl_local_sin.sin_addr, @@ -1657,18 +1664,18 @@ tcp_fill_info_for_info_tuple(struct info_tuple *itpl, struct tcp_info *ti) itpl->itpl_remote_sa.sa_family == AF_INET6) { struct in6_addr ina6_local; struct in6_addr ina6_remote; - + ina6_local = itpl->itpl_local_sin6.sin6_addr; - if (IN6_IS_SCOPE_LINKLOCAL(&ina6_local) && + if (IN6_IS_SCOPE_LINKLOCAL(&ina6_local) && itpl->itpl_local_sin6.sin6_scope_id) ina6_local.s6_addr16[1] = htons(itpl->itpl_local_sin6.sin6_scope_id); ina6_remote = itpl->itpl_remote_sin6.sin6_addr; - if (IN6_IS_SCOPE_LINKLOCAL(&ina6_remote) && + if (IN6_IS_SCOPE_LINKLOCAL(&ina6_remote) && itpl->itpl_remote_sin6.sin6_scope_id) ina6_remote.s6_addr16[1] = htons(itpl->itpl_remote_sin6.sin6_scope_id); - - inp = in6_pcblookup_hash(pcbinfo, + + inp = in6_pcblookup_hash(pcbinfo, &ina6_remote, itpl->itpl_remote_sin6.sin6_port, &ina6_local, @@ -1749,16 +1756,18 @@ tcp_connection_fill_info(struct tcpcb *tp, struct tcp_connection_info *tci) tci->tcpi_tfo_heuristics_disable = !!(tp->t_tfo_stats & TFO_S_HEURISTICS_DISABLE); tci->tcpi_tfo_send_blackhole = !!(tp->t_tfo_stats & TFO_S_SEND_BLACKHOLE); tci->tcpi_tfo_recv_blackhole = !!(tp->t_tfo_stats & TFO_S_RECV_BLACKHOLE); + tci->tcpi_tfo_onebyte_proxy = !!(tp->t_tfo_stats & TFO_S_ONE_BYTE_PROXY); } } -__private_extern__ int +__private_extern__ int tcp_sysctl_info(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req) { int error; struct tcp_info ti; struct info_tuple itpl; +#if !CONFIG_EMBEDDED proc_t caller = PROC_NULL; proc_t caller_parent = PROC_NULL; char command_name[MAXCOMLEN + 1] = ""; @@ -1794,6 +1803,7 @@ tcp_sysctl_info(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused if (caller != PROC_NULL) proc_rele(caller); +#endif /* !CONFIG_EMBEDDED */ if (req->newptr == USER_ADDR_NULL) { return EINVAL; @@ -1813,7 +1823,7 @@ tcp_sysctl_info(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused if (error != 0) { return error; } - + return 0; } @@ -1823,12 +1833,12 @@ tcp_lookup_peer_pid_locked(struct socket *so, pid_t *out_pid) int error = EHOSTUNREACH; *out_pid = -1; if ((so->so_state & SS_ISCONNECTED) == 0) return ENOTCONN; - + struct inpcb *inp = (struct inpcb*)so->so_pcb; uint16_t lport = inp->inp_lport; uint16_t fport = inp->inp_fport; struct inpcb *finp = NULL; - + if (inp->inp_vflag & INP_IPV6) { struct in6_addr laddr6 = inp->in6p_laddr; struct in6_addr faddr6 = inp->in6p_faddr; @@ -1842,13 +1852,17 @@ tcp_lookup_peer_pid_locked(struct socket *so, pid_t *out_pid) finp = in_pcblookup_hash(&tcbinfo, laddr4, lport, faddr4, fport, 0, NULL); socket_lock(so, 0); } - + if (finp) { *out_pid = finp->inp_socket->last_pid; error = 0; - in_pcb_checkstate(finp, WNT_RELEASE, 0); + /* Avoid deadlock due to same inpcb for loopback socket */ + if (inp == finp) + in_pcb_checkstate(finp, WNT_RELEASE, 1); + else + in_pcb_checkstate(finp, WNT_RELEASE, 0); } - + return error; } @@ -1988,7 +2002,7 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) error = EINVAL; break; } - minpkts = (in.min_burst_size != 0) ? in.min_burst_size : + minpkts = (in.min_burst_size != 0) ? in.min_burst_size : tp->t_bwmeas->bw_minsizepkts; maxpkts = (in.max_burst_size != 0) ? in.max_burst_size : tp->t_bwmeas->bw_maxsizepkts; @@ -2025,7 +2039,7 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) } else { tp->t_keepidle = optval * TCP_RETRANSHZ; /* reset the timer to new value */ - tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START(tp, + tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START(tp, TCP_CONN_KEEPIDLE(tp)); tcp_check_timer_state(tp); } @@ -2093,7 +2107,7 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) if (optval < 0 || optval > INT32_MAX) { error = EINVAL; break; - } + } if (optval != 0) inp->inp_flags2 |= INP2_KEEPALIVE_OFFLOAD; else @@ -2107,7 +2121,7 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) break; if (optval < 0) error = EINVAL; - else + else tp->t_persist_timeout = optval * TCP_RETRANSHZ; break; case TCP_RXT_CONNDROPTIME: @@ -2132,7 +2146,7 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) if (optval == 0) { so->so_flags &= ~(SOF_NOTSENT_LOWAT); tp->t_notsent_lowat = 0; - } else { + } else { so->so_flags |= SOF_NOTSENT_LOWAT; tp->t_notsent_lowat = optval; } @@ -2143,13 +2157,16 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) sizeof(optval)); if (error) break; - if (optval < 0 || + if (optval < 0 || optval > TCP_ADAPTIVE_TIMEOUT_MAX) { error = EINVAL; break; } else if (optval == 0) { tp->t_adaptive_rtimo = 0; tcp_keepalive_reset(tp); + + if (tp->t_mpsub) + mptcp_reset_keepalive(tp); } else { tp->t_adaptive_rtimo = optval; } @@ -2159,7 +2176,7 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) sizeof (optval)); if (error) break; - if (optval < 0 || + if (optval < 0 || optval > TCP_ADAPTIVE_TIMEOUT_MAX) { error = EINVAL; break; @@ -2261,6 +2278,27 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) else tcp_disable_tfo(tp); break; + case TCP_FASTOPEN_FORCE_HEURISTICS: + error = sooptcopyin(sopt, &optval, sizeof(optval), + sizeof(optval)); + + if (error) + break; + if (optval < 0 || optval > 1) { + error = EINVAL; + break; + } + + if (tp->t_state != TCPS_CLOSED) { + error = EINVAL; + break; + } + if (optval) + tp->t_flagsext |= TF_FASTOPEN_HEUR; + else + tp->t_flagsext &= ~TF_FASTOPEN_HEUR; + + break; case TCP_ENABLE_ECN: error = sooptcopyin(sopt, &optval, sizeof optval, sizeof optval); @@ -2271,6 +2309,7 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) tp->ecn_flags &= ~TE_ECN_MODE_DISABLE; } else { tp->ecn_flags &= ~TE_ECN_MODE_ENABLE; + tp->ecn_flags |= TE_ECN_MODE_DISABLE; } break; case TCP_ECN_MODE: @@ -2336,7 +2375,24 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) TRAFFIC_MGT_SO_BACKGROUND); } break; - + case TCP_RXT_MINIMUM_TIMEOUT: + error = sooptcopyin(sopt, &optval, sizeof(optval), + sizeof(optval)); + if (error) + break; + if (optval < 0) { + error = EINVAL; + break; + } + if (optval == 0) { + tp->t_rxt_minimum_timeout = 0; + } else { + tp->t_rxt_minimum_timeout = min(optval, + TCP_RXT_MINIMUM_TIMEOUT_LIMIT); + /* convert to milliseconds */ + tp->t_rxt_minimum_timeout *= TCP_RETRANSHZ; + } + break; default: error = ENOPROTOOPT; break; @@ -2400,7 +2456,7 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) break; case TCP_RXT_FINDROP: optval = tp->t_flagsext & TF_RXTFINDROP; - break; + break; case TCP_NOTIMEWAIT: optval = (tp->t_flagsext & TF_NOTIMEWAIT) ? 1 : 0; break; @@ -2412,6 +2468,9 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) } optval = tfo_enabled(tp); break; + case TCP_FASTOPEN_FORCE_HEURISTICS: + optval = (tp->t_flagsext & TF_FASTOPEN_HEUR) ? 1 : 0; + break; case TCP_MEASURE_SND_BW: optval = tp->t_flagsext & TF_MEASURESNDBW; break; @@ -2501,6 +2560,9 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt) error = sooptcopyout(sopt, &retid, sizeof (retid)); goto done; } + case TCP_RXT_MINIMUM_TIMEOUT: + optval = tp->t_rxt_minimum_timeout / TCP_RETRANSHZ; + break; default: error = ENOPROTOOPT; break; @@ -2525,10 +2587,12 @@ u_int32_t tcp_recvspace = 1448*384; * sb_max in sbreserve. Disallow setting the tcp send and recv space * to be more than sb_max because that will cause tcp_attach to fail * (see radar 5713060) - */ + */ static int sysctl_tcp_sospace(struct sysctl_oid *oidp, __unused void *arg1, - __unused int arg2, struct sysctl_req *req) { + int arg2, struct sysctl_req *req) +{ +#pragma unused(arg2) u_int32_t new_value = 0, *space_p = NULL; int changed = 0, error = 0; u_quad_t sb_effective_max = (sb_max / (MSIZE+MCLBYTES)) * MCLBYTES; @@ -2548,6 +2612,7 @@ sysctl_tcp_sospace(struct sysctl_oid *oidp, __unused void *arg1, if (changed) { if (new_value > 0 && new_value <= sb_effective_max) { *space_p = new_value; + SYSCTL_SKMEM_UPDATE_AT_OFFSET(arg2, new_value); } else { error = ERANGE; } @@ -2555,10 +2620,21 @@ sysctl_tcp_sospace(struct sysctl_oid *oidp, __unused void *arg1, return error; } +#if SYSCTL_SKMEM +SYSCTL_PROC(_net_inet_tcp, TCPCTL_SENDSPACE, sendspace, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_sendspace, + offsetof(skmem_sysctl, tcp.sendspace), sysctl_tcp_sospace, + "IU", "Maximum outgoing TCP datagram size"); +SYSCTL_PROC(_net_inet_tcp, TCPCTL_RECVSPACE, recvspace, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_recvspace, + offsetof(skmem_sysctl, tcp.recvspace), sysctl_tcp_sospace, + "IU", "Maximum incoming TCP datagram size"); +#else /* SYSCTL_SKMEM */ SYSCTL_PROC(_net_inet_tcp, TCPCTL_SENDSPACE, sendspace, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_sendspace , 0, &sysctl_tcp_sospace, "IU", "Maximum outgoing TCP datagram size"); SYSCTL_PROC(_net_inet_tcp, TCPCTL_RECVSPACE, recvspace, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_recvspace , 0, &sysctl_tcp_sospace, "IU", "Maximum incoming TCP datagram size"); +#endif /* SYSCTL_SKMEM */ /* * Attach TCP protocol to socket, allocating @@ -2689,17 +2765,17 @@ tcp_usrclosed(struct tcpcb *tp) break; case TCPS_ESTABLISHED: - DTRACE_TCP4(state__change, void, NULL, + DTRACE_TCP4(state__change, void, NULL, struct inpcb *, tp->t_inpcb, - struct tcpcb *, tp, + struct tcpcb *, tp, int32_t, TCPS_FIN_WAIT_1); tp->t_state = TCPS_FIN_WAIT_1; break; case TCPS_CLOSE_WAIT: - DTRACE_TCP4(state__change, void, NULL, + DTRACE_TCP4(state__change, void, NULL, struct inpcb *, tp->t_inpcb, - struct tcpcb *, tp, + struct tcpcb *, tp, int32_t, TCPS_LAST_ACK); tp->t_state = TCPS_LAST_ACK; break; @@ -2708,7 +2784,7 @@ tcp_usrclosed(struct tcpcb *tp) soisdisconnected(tp->t_inpcb->inp_socket); /* To prevent the connection hanging in FIN_WAIT_2 forever. */ if (tp->t_state == TCPS_FIN_WAIT_2) - tp->t_timer[TCPT_2MSL] = OFFSET_FROM_START(tp, + tp->t_timer[TCPT_2MSL] = OFFSET_FROM_START(tp, TCP_CONN_MAXIDLE(tp)); } return (tp); @@ -2743,12 +2819,12 @@ tcp_out6_cksum_stats(u_int32_t len) tcpstat.tcps_snd6_swcsum_bytes += len; } -/* +/* * When messages are enabled on a TCP socket, the message priority * is sent as a control message. This function will extract it. */ int -tcp_get_msg_priority(struct mbuf *control, uint32_t *msgpri) +tcp_get_msg_priority(struct mbuf *control, uint32_t *msgpri) { struct cmsghdr *cm; if (control == NULL) @@ -2767,7 +2843,7 @@ tcp_get_msg_priority(struct mbuf *control, uint32_t *msgpri) } } - VERIFY(*msgpri >= MSG_PRI_MIN && *msgpri <= MSG_PRI_MAX); + VERIFY(*msgpri >= MSG_PRI_MIN && *msgpri <= MSG_PRI_MAX); return (0); } #endif /* INET6 */ diff --git a/bsd/netinet/tcp_var.h b/bsd/netinet/tcp_var.h index aeb7f3b56..4a5e1d3b4 100644 --- a/bsd/netinet/tcp_var.h +++ b/bsd/netinet/tcp_var.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -198,14 +198,16 @@ struct tcptemp { }; struct bwmeas { - tcp_seq bw_start; /* start of bw measurement */ + tcp_seq bw_start; /* start of bw measurement */ uint32_t bw_ts; /* timestamp when bw measurement started */ - uint32_t bw_size; /* burst size in bytes for this bw measurement */ - uint32_t bw_minsizepkts; /* Min burst size as segments */ - uint32_t bw_maxsizepkts; /* Max burst size as segments */ + uint32_t bw_size; /* burst size in bytes for this bw measurement */ + uint32_t bw_minsizepkts; /* Min burst size as segments */ + uint32_t bw_maxsizepkts; /* Max burst size as segments */ uint32_t bw_minsize; /* Min size in bytes */ uint32_t bw_maxsize; /* Max size in bytes */ - uint32_t bw_sndbw; /* Measured send bw */ + uint32_t bw_sndbw; /* Measured send bandwidth */ + uint32_t bw_sndbw_max; /* Max measured bandwidth */ + uint32_t bw_rcvbw_max; /* Max receive bandwidth measured */ }; /* MPTCP Data sequence map entry */ @@ -315,6 +317,7 @@ struct tcpcb { u_int32_t t_maxopd; /* mss plus options */ u_int32_t t_rcvtime; /* time at which a packet was received */ + u_int32_t t_sndtime; /* time at which we last sent new data */ u_int32_t t_starttime; /* time connection was established */ int t_rtttime; /* tcp clock when rtt calculation was started */ tcp_seq t_rtseq; /* sequence number being timed */ @@ -398,6 +401,7 @@ struct tcpcb { #define TE_ECN_ON (TE_SETUPSENT | TE_SETUPRECEIVED) /* Indicate ECN was successfully negotiated on a connection) */ #define TE_CEHEURI_SET 0x2000 /* We did our CE-probing at the beginning */ #define TE_CLIENT_SETUP 0x4000 /* setup from client side */ +#define TE_RCVD_SYN_RST 0x8000 /* Received RST to the first ECN enabled SYN */ u_int32_t t_ecn_recv_ce; /* Received CE from the network */ u_int32_t t_ecn_recv_cwr; /* Packets received with CWR */ @@ -478,10 +482,11 @@ struct tcpcb { #define TF_PROBING 0x200000 /* Trigger probe timeout */ #define TF_FASTOPEN 0x400000 /* TCP Fastopen is enabled */ #define TF_REASS_INPROG 0x800000 /* Reassembly is in progress */ +#define TF_FASTOPEN_HEUR 0x1000000 /* Make sure that heuristics get never skipped */ #if TRAFFIC_MGT /* Inter-arrival jitter related state */ - uint32_t iaj_rcv_ts; /* tcp clock when the first packet was received */ + uint32_t iaj_rcv_ts; /* tcp clock when the first packet was received */ uint16_t iaj_size; /* Size of packet for iaj measurement */ uint8_t iaj_small_pkt; /* Count of packets smaller than iaj_size */ uint8_t t_pipeack_ind; /* index for next pipeack sample */ @@ -514,8 +519,8 @@ struct tcpcb { #if MPTCP u_int32_t t_mpflags; /* flags for multipath TCP */ -#define TMPF_PREESTABLISHED 0x00000001 /* conn in pre-established state */ -#define TMPF_SENT_KEYS 0x00000002 /* indicates that keys were sent */ +#define TMPF_PREESTABLISHED 0x00000001 /* conn in pre-established state */ +#define TMPF_SND_KEYS 0x00000002 /* indicates that keys should be send */ #define TMPF_MPTCP_TRUE 0x00000004 /* negotiated MPTCP successfully */ #define TMPF_MPTCP_RCVD_KEY 0x00000008 /* state for 3-way handshake */ #define TMPF_SND_MPPRIO 0x00000010 /* send priority of subflow */ @@ -536,12 +541,14 @@ struct tcpcb { #define TMPF_MPTCP_READY 0x00080000 /* Can send DSS options on data */ #define TMPF_INFIN_SENT 0x00100000 /* Sent infinite mapping */ #define TMPF_SND_MPFAIL 0x00200000 /* Received mapping csum failure */ -#define TMPF_FASTJOIN_SEND 0x00400000 /* Fast join early data send */ -#define TMPF_FASTJOINBY2_SEND 0x00800000 /* Fast join send after 3 WHS */ -#define TMPF_TFO_REQUEST 0x02000000 /* TFO Requested */ +#define TMPF_SND_JACK 0x00400000 /* Send a Join-ACK */ +#define TMPF_TFO_REQUEST 0x00800000 /* TFO Requested */ + +#define TMPF_MPTCP_SIGNALS (TMPF_SND_MPPRIO | TMPF_SND_REM_ADDR | TMPF_SND_MPFAIL | TMPF_SND_KEYS | TMPF_SND_JACK) tcp_seq t_mpuna; /* unacknowledged sequence */ - void *t_mptcb; /* pointer to MPTCP TCB */ + struct mptcb *t_mptcb; /* pointer to MPTCP TCB */ + struct mptsub *t_mpsub; /* pointer to the MPTCP subflow */ struct mpt_dsn_map t_rcv_map; /* Receive mapping list */ u_int8_t t_local_aid; /* Addr Id for authentication */ u_int8_t t_rem_aid; /* Addr ID of another subflow */ @@ -553,8 +560,8 @@ struct tcpcb { #define TFO_F_COOKIE_REQ 0x04 /* Client requested a new cookie */ #define TFO_F_COOKIE_SENT 0x08 /* Client did send a cookie in the SYN */ #define TFO_F_SYN_LOSS 0x10 /* A SYN-loss triggered a fallback to regular TCP on the client-side */ -#define TFO_F_NO_RCVPROBING 0x20 /* This network is guaranteed to support TFO in the downstream direction */ -#define TFO_F_NO_SNDPROBING 0x40 /* This network is guaranteed to support TFO in the upstream direction */ +#define TFO_F_NO_SNDPROBING 0x20 /* This network is guaranteed to support TFO in the upstream direction */ +#define TFO_F_HEURISTIC_DONE 0x40 /* We have already marked this network as bad */ u_int8_t t_tfo_flags; #define TFO_S_SYNDATA_RCV 0x01 /* SYN+data has been received */ #define TFO_S_COOKIEREQ_RECV 0x02 /* TFO-cookie request received */ @@ -570,6 +577,7 @@ struct tcpcb { #define TFO_S_HEURISTICS_DISABLE 0x0800 /* TFO-heuristics disabled it for this connection */ #define TFO_S_SEND_BLACKHOLE 0x1000 /* TFO got blackholed in the send direction */ #define TFO_S_RECV_BLACKHOLE 0x2000 /* TFO got blackholed in the recv direction */ +#define TFO_S_ONE_BYTE_PROXY 0x4000 /* TFO failed because of a proxy acknowledging just one byte */ u_int16_t t_tfo_stats; u_int8_t t_tfo_probes; /* TFO-probes we did send */ @@ -610,6 +618,7 @@ struct tcpcb { u_int32_t t_dsack_recvd; /* Received a valid DSACK option */ SLIST_HEAD(,tcp_notify_ack_marker) t_notify_ack; /* state for notifying data acknowledgements */ u_int32_t t_recv_throttle_ts; /* TS for start of recv throttle */ + u_int32_t t_rxt_minimum_timeout; /* minimum retransmit timeout in ms */ }; #define IN_FASTRECOVERY(tp) (tp->t_flags & TF_FASTRECOVERY) @@ -705,6 +714,7 @@ extern int tcprexmtthresh; #define TCP_RESET_REXMT_STATE(_tp_) do { \ (_tp_)->t_rxtshift = 0; \ (_tp_)->t_rxtstart = 0; \ + mptcp_reset_rexmit_state((_tp_)); \ } while(0); #define TCP_AUTORCVBUF_MAX(_ifp_) (((_ifp_) != NULL && \ @@ -782,22 +792,6 @@ struct rmxp_tao { #define intotcpcb(ip) ((struct tcpcb *)(ip)->inp_ppcb) #define sototcpcb(so) (intotcpcb(sotoinpcb(so))) -/* - * The rtt measured is in milliseconds as the timestamp granularity is - * a millisecond. The smoothed round-trip time and estimated variance - * are stored as fixed point numbers scaled by the values below. - * For convenience, these scales are also used in smoothing the average - * (smoothed = (1/scale)sample + ((scale-1)/scale)smoothed). - * With these scales, srtt has 5 bits to the right of the binary point, - * and thus an "ALPHA" of 0.875. rttvar has 4 bits to the right of the - * binary point, and is smoothed with an ALPHA of 0.75. - */ -#define TCP_RTT_SCALE 32 /* multiplier for srtt; 3 bits frac. */ -#define TCP_RTT_SHIFT 5 /* shift for srtt; 5 bits frac. */ -#define TCP_RTTVAR_SCALE 16 /* multiplier for rttvar; 4 bits */ -#define TCP_RTTVAR_SHIFT 4 /* shift for rttvar; 4 bits */ -#define TCP_DELTA_SHIFT 2 /* see tcp_input.c */ - /* TFO-specific defines */ #define TFO_COOKIE_LEN_MIN 4 #define TFO_COOKIE_LEN_DEFAULT 8 @@ -1162,6 +1156,51 @@ struct tcpstat { u_int32_t tcps_mss_to_low; /* Change MSS to low using link status report */ u_int32_t tcps_ecn_fallback_droprst; /* ECN fallback caused by connection drop due to RST */ u_int32_t tcps_ecn_fallback_droprxmt; /* ECN fallback due to drop after multiple retransmits */ + u_int32_t tcps_ecn_fallback_synrst; /* ECN fallback due to rst after syn */ + + u_int32_t tcps_mptcp_rcvmemdrop; /* MPTCP packets dropped for lack of memory */ + u_int32_t tcps_mptcp_rcvduppack; /* MPTCP duplicate-only packets received */ + u_int32_t tcps_mptcp_rcvpackafterwin; /* MPTCP packets with data after window */ + + /* TCP timer statistics */ + u_int32_t tcps_timer_drift_le_1_ms; /* Timer drift less or equal to 1 ms */ + u_int32_t tcps_timer_drift_le_10_ms; /* Timer drift less or equal to 10 ms */ + u_int32_t tcps_timer_drift_le_20_ms; /* Timer drift less or equal to 20 ms */ + u_int32_t tcps_timer_drift_le_50_ms; /* Timer drift less or equal to 50 ms */ + u_int32_t tcps_timer_drift_le_100_ms; /* Timer drift less or equal to 100 ms */ + u_int32_t tcps_timer_drift_le_200_ms; /* Timer drift less or equal to 200 ms */ + u_int32_t tcps_timer_drift_le_500_ms; /* Timer drift less or equal to 500 ms */ + u_int32_t tcps_timer_drift_le_1000_ms; /* Timer drift less or equal to 1000 ms */ + u_int32_t tcps_timer_drift_gt_1000_ms; /* Timer drift greater than 1000 ms */ + + u_int32_t tcps_mptcp_handover_attempt; /* Total number of MPTCP-attempts using handover mode */ + u_int32_t tcps_mptcp_interactive_attempt; /* Total number of MPTCP-attempts using interactive mode */ + u_int32_t tcps_mptcp_aggregate_attempt; /* Total number of MPTCP-attempts using aggregate mode */ + u_int32_t tcps_mptcp_fp_handover_attempt; /* Same as previous three but only for first-party apps */ + u_int32_t tcps_mptcp_fp_interactive_attempt; + u_int32_t tcps_mptcp_fp_aggregate_attempt; + u_int32_t tcps_mptcp_heuristic_fallback; /* Total number of MPTCP-connections that fell back due to heuristics */ + u_int32_t tcps_mptcp_fp_heuristic_fallback; /* Same as previous but for first-party apps */ + u_int32_t tcps_mptcp_handover_success_wifi; /* Total number of successfull handover-mode connections that *started* on WiFi */ + u_int32_t tcps_mptcp_handover_success_cell; /* Total number of successfull handover-mode connections that *started* on Cell */ + u_int32_t tcps_mptcp_interactive_success; /* Total number of interactive-mode connections that negotiated MPTCP */ + u_int32_t tcps_mptcp_aggregate_success; /* Same as previous but for aggregate */ + u_int32_t tcps_mptcp_fp_handover_success_wifi; /* Same as previous four, but for first-party apps */ + u_int32_t tcps_mptcp_fp_handover_success_cell; + u_int32_t tcps_mptcp_fp_interactive_success; + u_int32_t tcps_mptcp_fp_aggregate_success; + u_int32_t tcps_mptcp_handover_cell_from_wifi; /* Total number of connections that use cell in handover-mode (coming from WiFi) */ + u_int32_t tcps_mptcp_handover_wifi_from_cell; /* Total number of connections that use WiFi in handover-mode (coming from cell) */ + u_int32_t tcps_mptcp_interactive_cell_from_wifi; /* Total number of connections that use cell in interactive mode (coming from WiFi) */ + u_int64_t tcps_mptcp_handover_cell_bytes; /* Total number of bytes sent on cell in handover-mode (on new subflows, ignoring initial one) */ + u_int64_t tcps_mptcp_interactive_cell_bytes; /* Same as previous but for interactive */ + u_int64_t tcps_mptcp_aggregate_cell_bytes; + u_int64_t tcps_mptcp_handover_all_bytes; /* Total number of bytes sent in handover */ + u_int64_t tcps_mptcp_interactive_all_bytes; + u_int64_t tcps_mptcp_aggregate_all_bytes; + u_int32_t tcps_mptcp_back_to_wifi; /* Total number of connections that succeed to move traffic away from cell (when starting on cell) */ + u_int32_t tcps_mptcp_wifi_proxy; /* Total number of new subflows that fell back to regular TCP on cell */ + u_int32_t tcps_mptcp_cell_proxy; /* Total number of new subflows that fell back to regular TCP on WiFi */ }; @@ -1206,6 +1245,7 @@ struct xtcpcb { u_quad_t xt_alignment_hack; }; +#if !CONFIG_EMBEDDED struct xtcpcb64 { u_int32_t xt_len; @@ -1286,6 +1326,7 @@ struct xtcpcb64 { u_quad_t xt_alignment_hack; }; +#endif /* !CONFIG_EMBEDDED */ #ifdef PRIVATE @@ -1365,6 +1406,22 @@ struct xtcpcb_n { u_int32_t snd_ssthresh_prev; /* ssthresh prior to retransmit */ }; + /* + * The rtt measured is in milliseconds as the timestamp granularity is + * a millisecond. The smoothed round-trip time and estimated variance + * are stored as fixed point numbers scaled by the values below. + * For convenience, these scales are also used in smoothing the average + * (smoothed = (1/scale)sample + ((scale-1)/scale)smoothed). + * With these scales, srtt has 5 bits to the right of the binary point, + * and thus an "ALPHA" of 0.875. rttvar has 4 bits to the right of the + * binary point, and is smoothed with an ALPHA of 0.75. + */ +#define TCP_RTT_SCALE 32 /* multiplier for srtt; 3 bits frac. */ +#define TCP_RTT_SHIFT 5 /* shift for srtt; 5 bits frac. */ +#define TCP_RTTVAR_SCALE 16 /* multiplier for rttvar; 4 bits */ +#define TCP_RTTVAR_SHIFT 4 /* shift for rttvar; 4 bits */ +#define TCP_DELTA_SHIFT 2 /* see tcp_input.c */ + #endif /* PRIVATE */ #pragma pack() @@ -1424,20 +1481,19 @@ extern int tcp_minmss; #define TCP_FASTOPEN_SERVER 0x01 #define TCP_FASTOPEN_CLIENT 0x02 -extern int tcp_tfo_halfcnt; -extern int tcp_tfo_backlog; -extern int tcp_fastopen; -extern int tcp_tfo_fallback_min; -extern int ss_fltsz; -extern int ss_fltsz_local; -extern int tcp_do_rfc3390; /* Calculate ss_fltsz according to RFC 3390 */ +extern int tcp_tfo_halfcnt; +extern int tcp_tfo_backlog; +extern int tcp_fastopen; +extern int ss_fltsz; +extern int ss_fltsz_local; +extern int tcp_do_rfc3390; /* Calculate ss_fltsz according to RFC 3390 */ extern int tcp_do_rfc1323; extern int target_qdelay; -extern u_int32_t tcp_now; /* for RFC 1323 timestamps */ +extern u_int32_t tcp_now; /* for RFC 1323 timestamps */ extern struct timeval tcp_uptime; extern lck_spin_t *tcp_uptime_lock; -extern int tcp_delack_enabled; -extern int tcp_do_sack; /* SACK enabled/disabled */ +extern int tcp_delack_enabled; +extern int tcp_do_sack; /* SACK enabled/disabled */ extern int tcp_do_rfc3465; extern int tcp_do_rfc3465_lim2; extern int maxseg_unacked; @@ -1446,6 +1502,7 @@ extern struct zone *tcp_reass_zone; extern struct zone *tcp_rxt_seg_zone; extern int tcp_ecn_outbound; extern int tcp_ecn_inbound; +extern u_int32_t tcp_do_autorcvbuf; extern u_int32_t tcp_autorcvbuf_max; extern u_int32_t tcp_autorcvbuf_max_ca; extern u_int32_t tcp_autorcvbuf_inc_shift; @@ -1465,7 +1522,7 @@ struct tcp_respond_args { void tcp_canceltimers(struct tcpcb *); struct tcpcb * tcp_close(struct tcpcb *); -void tcp_ctlinput(int, struct sockaddr *, void *); +void tcp_ctlinput(int, struct sockaddr *, void *, struct ifnet *); int tcp_ctloutput(struct socket *, struct sockopt *); struct tcpcb * tcp_drop(struct tcpcb *, int); @@ -1497,6 +1554,7 @@ void tcp_fillheaders(struct tcpcb *, void *, void *); struct tcpcb *tcp_timers(struct tcpcb *, int); void tcp_trace(int, int, struct tcpcb *, void *, struct tcphdr *, int); +void tcp_fill_info(struct tcpcb *, struct tcp_info *); void tcp_sack_doack(struct tcpcb *, struct tcpopt *, struct tcphdr *, u_int32_t *); extern boolean_t tcp_sack_process_dsack(struct tcpcb *, struct tcpopt *, @@ -1591,11 +1649,15 @@ extern bool tcp_notify_ack_active(struct socket *so); #if MPTCP extern int mptcp_input_preproc(struct tcpcb *, struct mbuf *, int); -extern void mptcp_output_csum(struct tcpcb *, struct mbuf *, int32_t, unsigned, - u_int64_t, u_int32_t *); +extern uint32_t mptcp_output_csum(struct mbuf *m, uint64_t dss_val, + uint32_t sseq, uint16_t dlen); extern int mptcp_adj_mss(struct tcpcb *, boolean_t); extern void mptcp_insert_rmap(struct tcpcb *, struct mbuf *); #endif + +__private_extern__ void tcp_update_stats_per_flow( + struct ifnet_stats_per_flow *, struct ifnet *); + #endif /* BSD_KERNEL_RPIVATE */ #endif /* _NETINET_TCP_VAR_H_ */ diff --git a/bsd/netinet/udp_usrreq.c b/bsd/netinet/udp_usrreq.c index 698aea8b6..044486ec5 100644 --- a/bsd/netinet/udp_usrreq.c +++ b/bsd/netinet/udp_usrreq.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -81,6 +81,7 @@ #include <net/if_types.h> #include <net/route.h> #include <net/dlil.h> +#include <net/net_api_stats.h> #include <netinet/in.h> #include <netinet/in_systm.h> @@ -201,17 +202,17 @@ struct udp_ip6 { u_char uip6_init_done : 1; }; -static int udp_abort(struct socket *); -static int udp_attach(struct socket *, int, struct proc *); -static int udp_bind(struct socket *, struct sockaddr *, struct proc *); -static int udp_connect(struct socket *, struct sockaddr *, struct proc *); -static int udp_connectx(struct socket *, struct sockaddr *, +int udp_abort(struct socket *); +int udp_attach(struct socket *, int, struct proc *); +int udp_bind(struct socket *, struct sockaddr *, struct proc *); +int udp_connect(struct socket *, struct sockaddr *, struct proc *); +int udp_connectx(struct socket *, struct sockaddr *, struct sockaddr *, struct proc *, uint32_t, sae_associd_t, sae_connid_t *, uint32_t, void *, uint32_t, struct uio *, user_ssize_t *); -static int udp_detach(struct socket *); -static int udp_disconnect(struct socket *); -static int udp_disconnectx(struct socket *, sae_associd_t, sae_connid_t); -static int udp_send(struct socket *, int, struct mbuf *, struct sockaddr *, +int udp_detach(struct socket *); +int udp_disconnect(struct socket *); +int udp_disconnectx(struct socket *, sae_associd_t, sae_connid_t); +int udp_send(struct socket *, int, struct mbuf *, struct sockaddr *, struct mbuf *, struct proc *); static void udp_append(struct inpcb *, struct ip *, struct mbuf *, int, struct sockaddr_in *, struct udp_in6 *, struct udp_ip6 *, struct ifnet *); @@ -220,7 +221,7 @@ static void udp_append(struct inpcb *, struct ip *, struct mbuf *, int, struct sockaddr_in *, struct ifnet *); #endif /* !INET6 */ static int udp_input_checksum(struct mbuf *, struct udphdr *, int, int); -static int udp_output(struct inpcb *, struct mbuf *, struct sockaddr *, +int udp_output(struct inpcb *, struct mbuf *, struct sockaddr *, struct mbuf *, struct proc *); static void ip_2_ip6_hdr(struct ip6_hdr *ip6, struct ip *ip); static void udp_gc(struct inpcbinfo *); @@ -331,7 +332,7 @@ udp_input(struct mbuf *m, int iphlen) * with options still present. */ if (iphlen > sizeof (struct ip)) { - ip_stripoptions(m, (struct mbuf *)0); + ip_stripoptions(m); iphlen = sizeof (struct ip); } @@ -493,8 +494,7 @@ udp_input(struct mbuf *m, int iphlen) group.sin_addr = ip->ip_dst; blocked = imo_multi_filter(imo, ifp, - (struct sockaddr *)&group, - (struct sockaddr *)&udp_in); + &group, &udp_in); if (blocked == MCAST_PASS) foundmembership = 1; @@ -743,6 +743,7 @@ udp_input(struct mbuf *m, int iphlen) if (nstat_collect) { INP_ADD_STAT(inp, cell, wifi, wired, rxpackets, 1); INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, m->m_pkthdr.len); + inp_set_activity_bitmap(inp); } so_recv_data_stat(inp->inp_socket, m, 0); if (sbappendaddr(&inp->inp_socket->so_rcv, append_sa, @@ -850,6 +851,7 @@ udp_append(struct inpcb *last, struct ip *ip, struct mbuf *n, int off, INP_ADD_STAT(last, cell, wifi, wired, rxpackets, 1); INP_ADD_STAT(last, cell, wifi, wired, rxbytes, n->m_pkthdr.len); + inp_set_activity_bitmap(last); } so_recv_data_stat(last->inp_socket, n, 0); m_adj(n, off); @@ -878,12 +880,12 @@ udp_notify(struct inpcb *inp, int errno) } void -udp_ctlinput(int cmd, struct sockaddr *sa, void *vip) +udp_ctlinput(int cmd, struct sockaddr *sa, void *vip, __unused struct ifnet * ifp) { struct ip *ip = vip; void (*notify)(struct inpcb *, int) = udp_notify; struct in_addr faddr; - struct inpcb *inp; + struct inpcb *inp = NULL; faddr = ((struct sockaddr_in *)(void *)sa)->sin_addr; if (sa->sa_family != AF_INET || faddr.s_addr == INADDR_ANY) @@ -1172,6 +1174,7 @@ SYSCTL_PROC(_net_inet_udp, UDPCTL_PCBLIST, pcblist, CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0, udp_pcblist, "S,xinpcb", "List of active UDP sockets"); +#if !CONFIG_EMBEDDED static int udp_pcblist64 SYSCTL_HANDLER_ARGS @@ -1290,6 +1293,7 @@ SYSCTL_PROC(_net_inet_udp, OID_AUTO, pcblist64, CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0, udp_pcblist64, "S,xinpcb64", "List of active UDP sockets"); +#endif /* !CONFIG_EMBEDDED */ static int udp_pcblist_n SYSCTL_HANDLER_ARGS @@ -1306,7 +1310,8 @@ __private_extern__ void udp_get_ports_used(uint32_t ifindex, int protocol, uint32_t flags, bitstr_t *bitfield) { - inpcb_get_ports_used(ifindex, protocol, flags, bitfield, &udbinfo); + inpcb_get_ports_used(ifindex, protocol, flags, bitfield, + &udbinfo); } __private_extern__ uint32_t @@ -1318,7 +1323,7 @@ udp_count_opportunistic(unsigned int ifindex, u_int32_t flags) __private_extern__ uint32_t udp_find_anypcb_byaddr(struct ifaddr *ifa) { - return (inpcb_find_anypcb_byaddr(ifa, &udbinfo)); + return (inpcb_find_anypcb_byaddr(ifa, &udbinfo)); } static int @@ -1395,7 +1400,7 @@ udp_check_pktinfo(struct mbuf *control, struct ifnet **outif, return (0); } -static int +int udp_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr, struct mbuf *control, struct proc *p) { @@ -1425,7 +1430,7 @@ udp_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr, KERNEL_DEBUG(DBG_FNC_UDP_OUTPUT | DBG_FUNC_START, 0, 0, 0, 0, 0); - lck_mtx_assert(&inp->inpcb_mtx, LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(so); if (control != NULL) { sotc = so_tc_from_control(control, &netsvctype); VERIFY(outif == NULL); @@ -1622,6 +1627,11 @@ udp_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr, if (inp->inp_flowhash == 0) inp->inp_flowhash = inp_calc_flowhash(inp); + if (fport == htons(53) && !(so->so_flags1 & SOF1_DNS_COUNTED)) { + so->so_flags1 |= SOF1_DNS_COUNTED; + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_inet_dgram_dns); + } + /* * Calculate data length and get a mbuf * for UDP and IP headers. @@ -1651,7 +1661,7 @@ udp_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr, if (udpcksum && !(inp->inp_flags & INP_UDP_NOCKSUM)) { ui->ui_sum = in_pseudo(ui->ui_src.s_addr, ui->ui_dst.s_addr, htons((u_short)len + sizeof (struct udphdr) + IPPROTO_UDP)); - m->m_pkthdr.csum_flags = CSUM_UDP; + m->m_pkthdr.csum_flags = (CSUM_UDP|CSUM_ZERO_INVERT); m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); } else { ui->ui_sum = 0; @@ -1735,6 +1745,7 @@ udp_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr, mopts->imo_multicast_ifp != NULL) { /* no reference needed */ inp->inp_last_outifp = mopts->imo_multicast_ifp; + } IMO_UNLOCK(mopts); } @@ -1777,6 +1788,7 @@ udp_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr, } INP_ADD_STAT(inp, cell, wifi, wired, txpackets, 1); INP_ADD_STAT(inp, cell, wifi, wired, txbytes, len); + inp_set_activity_bitmap(inp); } if (flowadv && (adv->code == FADV_FLOW_CONTROLLED || @@ -1804,6 +1816,7 @@ abort: inp->inp_laddr = origladdr; /* XXX rehash? */ /* no reference needed */ inp->inp_last_outifp = origoutifp; + } else if (inp->inp_route.ro_rt != NULL) { struct rtentry *rt = inp->inp_route.ro_rt; struct ifnet *outifp; @@ -1828,7 +1841,7 @@ abort: sizeof(struct udphdr) + sizeof(struct ip) + ifnet_hdrlen(outifp) + - ifnet_packetpreamblelen(outifp), + ifnet_mbuf_packetpreamblelen(outifp), sizeof(u_int32_t)); } } else { @@ -1904,7 +1917,7 @@ SYSCTL_PROC(_net_inet_udp, UDPCTL_MAXDGRAM, maxdgram, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &udp_sendspace, 0, &sysctl_udp_sospace, "IU", "Maximum outgoing UDP datagram size"); -static int +int udp_abort(struct socket *so) { struct inpcb *inp; @@ -1919,7 +1932,7 @@ udp_abort(struct socket *so) return (0); } -static int +int udp_attach(struct socket *so, int proto, struct proc *p) { #pragma unused(proto) @@ -1945,7 +1958,7 @@ udp_attach(struct socket *so, int proto, struct proc *p) return (0); } -static int +int udp_bind(struct socket *so, struct sockaddr *nam, struct proc *p) { struct inpcb *inp; @@ -1959,10 +1972,22 @@ udp_bind(struct socket *so, struct sockaddr *nam, struct proc *p) if (inp == NULL) return (EINVAL); error = in_pcbbind(inp, nam, p); + +#if NECP + /* Update NECP client with bind result if not in middle of connect */ + if (error == 0 && + (inp->inp_flags2 & INP2_CONNECT_IN_PROGRESS) && + !uuid_is_null(inp->necp_client_uuid)) { + socket_unlock(so, 0); + necp_client_assign_from_socket(so->last_pid, inp->necp_client_uuid, inp); + socket_lock(so, 0); + } +#endif /* NECP */ + return (error); } -static int +int udp_connect(struct socket *so, struct sockaddr *nam, struct proc *p) { struct inpcb *inp; @@ -1974,6 +1999,11 @@ udp_connect(struct socket *so, struct sockaddr *nam, struct proc *p) if (inp->inp_faddr.s_addr != INADDR_ANY) return (EISCONN); + if (!(so->so_flags1 & SOF1_CONNECT_COUNTED)) { + so->so_flags1 |= SOF1_CONNECT_COUNTED; + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_inet_dgram_connected); + } + #if NECP #if FLOW_DIVERT if (necp_socket_should_use_flow_divert(inp)) { @@ -1994,6 +2024,15 @@ udp_connect(struct socket *so, struct sockaddr *nam, struct proc *p) error = in_pcbconnect(inp, nam, p, IFSCOPE_NONE, NULL); if (error == 0) { +#if NECP + /* Update NECP client with connected five-tuple */ + if (!uuid_is_null(inp->necp_client_uuid)) { + socket_unlock(so, 0); + necp_client_assign_from_socket(so->last_pid, inp->necp_client_uuid, inp); + socket_lock(so, 0); + } +#endif /* NECP */ + soisconnected(so); if (inp->inp_flowhash == 0) inp->inp_flowhash = inp_calc_flowhash(inp); @@ -2009,7 +2048,7 @@ udp_connectx_common(struct socket *so, int af, struct sockaddr *src, struct sock { #pragma unused(aid, flags, arg, arglen) struct inpcb *inp = sotoinpcb(so); - int error; + int error = 0; user_ssize_t datalen = 0; if (inp == NULL) @@ -2017,20 +2056,25 @@ udp_connectx_common(struct socket *so, int af, struct sockaddr *src, struct sock VERIFY(dst != NULL); + ASSERT(!(inp->inp_flags2 & INP2_CONNECT_IN_PROGRESS)); + inp->inp_flags2 |= INP2_CONNECT_IN_PROGRESS; + #if NECP inp_update_necp_policy(inp, src, dst, ifscope); #endif /* NECP */ /* bind socket to the specified interface, if requested */ if (ifscope != IFSCOPE_NONE && - (error = inp_bindif(inp, ifscope, NULL)) != 0) - return (error); + (error = inp_bindif(inp, ifscope, NULL)) != 0) { + goto done; + } /* if source address and/or port is specified, bind to it */ if (src != NULL) { error = sobindlock(so, src, 0); /* already locked */ - if (error != 0) - return (error); + if (error != 0) { + goto done; + } } switch (af) { @@ -2047,8 +2091,9 @@ udp_connectx_common(struct socket *so, int af, struct sockaddr *src, struct sock /* NOTREACHED */ } - if (error != 0) - return (error); + if (error != 0) { + goto done; + } /* * If there is data, copy it. DATA_IDEMPOTENT is ignored. @@ -2081,10 +2126,12 @@ udp_connectx_common(struct socket *so, int af, struct sockaddr *src, struct sock if (error == 0 && pcid != NULL) *pcid = 1; /* there is only 1 connection for UDP */ +done: + inp->inp_flags2 &= ~INP2_CONNECT_IN_PROGRESS; return (error); } -static int +int udp_connectx(struct socket *so, struct sockaddr *src, struct sockaddr *dst, struct proc *p, uint32_t ifscope, sae_associd_t aid, sae_connid_t *pcid, uint32_t flags, void *arg, @@ -2094,7 +2141,7 @@ udp_connectx(struct socket *so, struct sockaddr *src, p, ifscope, aid, pcid, flags, arg, arglen, uio, bytes_written)); } -static int +int udp_detach(struct socket *so) { struct inpcb *inp; @@ -2119,7 +2166,7 @@ udp_detach(struct socket *so) return (0); } -static int +int udp_disconnect(struct socket *so) { struct inpcb *inp; @@ -2142,10 +2189,11 @@ udp_disconnect(struct socket *so) inp->inp_laddr.s_addr = INADDR_ANY; so->so_state &= ~SS_ISCONNECTED; /* XXX */ inp->inp_last_outifp = NULL; + return (0); } -static int +int udp_disconnectx(struct socket *so, sae_associd_t aid, sae_connid_t cid) { #pragma unused(cid) @@ -2155,7 +2203,7 @@ udp_disconnectx(struct socket *so, sae_associd_t aid, sae_connid_t cid) return (udp_disconnect(so)); } -static int +int udp_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *addr, struct mbuf *control, struct proc *p) { @@ -2209,7 +2257,7 @@ udp_lock(struct socket *so, int refcount, void *debug) lr_saved = debug; if (so->so_pcb != NULL) { - lck_mtx_assert(&((struct inpcb *)so->so_pcb)->inpcb_mtx, + LCK_MTX_ASSERT(&((struct inpcb *)so->so_pcb)->inpcb_mtx, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(&((struct inpcb *)so->so_pcb)->inpcb_mtx); } else { @@ -2244,7 +2292,7 @@ udp_unlock(struct socket *so, int refcount, void *debug) so, lr_saved, solockhistory_nr(so)); /* NOTREACHED */ } else { - lck_mtx_assert(&((struct inpcb *)so->so_pcb)->inpcb_mtx, + LCK_MTX_ASSERT(&((struct inpcb *)so->so_pcb)->inpcb_mtx, LCK_MTX_ASSERT_OWNED); so->unlock_lr[so->next_unlock_lr] = lr_saved; so->next_unlock_lr = (so->next_unlock_lr+1) % SO_LCKDBG_MAX; @@ -2254,9 +2302,9 @@ udp_unlock(struct socket *so, int refcount, void *debug) } lck_mtx_t * -udp_getlock(struct socket *so, int locktype) +udp_getlock(struct socket *so, int flags) { -#pragma unused(locktype) +#pragma unused(flags) struct inpcb *inp = sotoinpcb(so); if (so->so_pcb == NULL) { @@ -2306,7 +2354,7 @@ udp_gc(struct inpcbinfo *ipi) * Skip if busy, no hurry for cleanup. Keep gc active * and try the lock again during next round. */ - if (!lck_mtx_try_lock(&inp->inpcb_mtx)) { + if (!socket_try_lock(inp->inp_socket)) { atomic_add_32(&ipi->ipi_gc_req.intimer_fast, 1); continue; } @@ -2326,7 +2374,7 @@ udp_gc(struct inpcbinfo *ipi) } in_pcbdispose(inp); } else { - lck_mtx_unlock(&inp->inpcb_mtx); + socket_unlock(so, 0); atomic_add_32(&ipi->ipi_gc_req.intimer_fast, 1); } } @@ -2388,39 +2436,64 @@ udp_input_checksum(struct mbuf *m, struct udphdr *uh, int off, int ulen) return (0); } + /* ip_stripoptions() must have been called before we get here */ + ASSERT((ip->ip_hl << 2) == sizeof (*ip)); + if ((hwcksum_rx || (ifp->if_flags & IFF_LOOPBACK) || (m->m_pkthdr.pkt_flags & PKTF_LOOP)) && (m->m_pkthdr.csum_flags & CSUM_DATA_VALID)) { if (m->m_pkthdr.csum_flags & CSUM_PSEUDO_HDR) { uh->uh_sum = m->m_pkthdr.csum_rx_val; } else { - uint16_t sum = m->m_pkthdr.csum_rx_val; - uint16_t start = m->m_pkthdr.csum_rx_start; + uint32_t sum = m->m_pkthdr.csum_rx_val; + uint32_t start = m->m_pkthdr.csum_rx_start; + int32_t trailer = (m_pktlen(m) - (off + ulen)); /* * Perform 1's complement adjustment of octets * that got included/excluded in the hardware- * calculated checksum value. Ignore cases - * where the value includes or excludes the + * where the value already includes the entire * IP header span, as the sum for those octets - * would already be 0xffff and thus no-op. + * would already be 0 by the time we get here; + * IP has already performed its header checksum + * checks. If we do need to adjust, restore + * the original fields in the IP header when + * computing the adjustment value. Also take + * care of any trailing bytes and subtract out + * their partial sum. */ + ASSERT(trailer >= 0); if ((m->m_pkthdr.csum_flags & CSUM_PARTIAL) && - start != 0 && (off - start) != off) { -#if BYTE_ORDER != BIG_ENDIAN + ((start != 0 && start != off) || trailer != 0)) { + uint32_t swbytes = (uint32_t)trailer; + if (start < off) { + ip->ip_len += sizeof (*ip); +#if BYTE_ORDER != BIG_ENDIAN HTONS(ip->ip_len); HTONS(ip->ip_off); - } #endif /* BYTE_ORDER != BIG_ENDIAN */ + } /* callee folds in sum */ - sum = m_adj_sum16(m, start, off, sum); -#if BYTE_ORDER != BIG_ENDIAN + sum = m_adj_sum16(m, start, off, ulen, sum); + if (off > start) + swbytes += (off - start); + else + swbytes += (start - off); + if (start < off) { +#if BYTE_ORDER != BIG_ENDIAN NTOHS(ip->ip_off); NTOHS(ip->ip_len); - } #endif /* BYTE_ORDER != BIG_ENDIAN */ + ip->ip_len -= sizeof (*ip); + } + + if (swbytes != 0) + udp_in_cksum_stats(swbytes); + if (trailer != 0) + m_adj(m, -trailer); } /* callee folds in sum */ @@ -2611,7 +2684,8 @@ udp_fill_keepalive_offload_frames(ifnet_t ifp, htons(sizeof(struct udphdr) + (u_short)inp->inp_keepalive_datalen + IPPROTO_UDP)); - m->m_pkthdr.csum_flags = CSUM_UDP; + m->m_pkthdr.csum_flags = + (CSUM_UDP|CSUM_ZERO_INVERT); m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); } @@ -2682,7 +2756,8 @@ udp_fill_keepalive_offload_frames(ifnet_t ifp, htonl(sizeof(struct udphdr) + (u_short)inp->inp_keepalive_datalen + IPPROTO_UDP)); - m->m_pkthdr.csum_flags = CSUM_UDPIPV6; + m->m_pkthdr.csum_flags = + (CSUM_UDPIPV6|CSUM_ZERO_INVERT); m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); } diff --git a/bsd/netinet/udp_var.h b/bsd/netinet/udp_var.h index 2eb1e0806..f016b31de 100644 --- a/bsd/netinet/udp_var.h +++ b/bsd/netinet/udp_var.h @@ -163,7 +163,7 @@ extern struct udpstat udpstat; extern int udp_log_in_vain; __BEGIN_DECLS -extern void udp_ctlinput(int, struct sockaddr *, void *); +extern void udp_ctlinput(int, struct sockaddr *, void *, struct ifnet *); extern int udp_ctloutput(struct socket *, struct sockopt *); extern void udp_init(struct protosw *, struct domain *); extern void udp_input(struct mbuf *, int); diff --git a/bsd/netinet6/Makefile b/bsd/netinet6/Makefile index 039ad151d..9bccd060b 100644 --- a/bsd/netinet6/Makefile +++ b/bsd/netinet6/Makefile @@ -22,9 +22,9 @@ PRIVATE_DATAFILES = \ scope6_var.h PRIVATE_KERNELFILES = \ - ah6.h esp6.h esp_rijndael.h in6_gif.h in6_ifattach.h \ - ip6_ecn.h ip6protosw.h ipcomp6.h ipsec6.h \ - tcp6_var.h udp6_var.h + ah6.h esp6.h esp_rijndael.h esp_chachapoly.h \ + in6_gif.h in6_ifattach.h ip6_ecn.h ip6protosw.h \ + ipcomp6.h ipsec6.h tcp6_var.h udp6_var.h INSTALL_MI_LIST = ${DATAFILES} diff --git a/bsd/netinet6/ah_input.c b/bsd/netinet6/ah_input.c index a6054b601..fac8da28c 100644 --- a/bsd/netinet6/ah_input.c +++ b/bsd/netinet6/ah_input.c @@ -996,7 +996,7 @@ ah6_ctlinput(int cmd, struct sockaddr *sa, void *d) struct ip6_hdr *ip6; struct mbuf *m; struct ip6ctlparam *ip6cp = NULL; - int off; + int off = 0; struct sockaddr_in6 *sa6_src, *sa6_dst; if (sa->sa_family != AF_INET6 || diff --git a/bsd/netinet6/esp.h b/bsd/netinet6/esp.h index b2deaa2f3..94f6e26b7 100644 --- a/bsd/netinet6/esp.h +++ b/bsd/netinet6/esp.h @@ -129,6 +129,7 @@ extern int esp_max_ivlen(void); /* crypt routines */ extern int esp4_output(struct mbuf *, struct secasvar *); extern void esp4_input(struct mbuf *, int off); +extern struct mbuf *esp4_input_extended(struct mbuf *, int off, ifnet_t interface); extern size_t esp_hdrsiz(struct ipsecrequest *); extern int esp_schedule(const struct esp_algorithm *, struct secasvar *); diff --git a/bsd/netinet6/esp6.h b/bsd/netinet6/esp6.h index 767d0c387..384ec59dc 100644 --- a/bsd/netinet6/esp6.h +++ b/bsd/netinet6/esp6.h @@ -70,8 +70,9 @@ extern int esp6_output(struct mbuf *, u_char *, struct mbuf *, struct secasvar *); extern int esp6_input(struct mbuf **, int *, int); +extern int esp6_input_extended(struct mbuf **mp, int *offp, int proto, ifnet_t interface); -extern void esp6_ctlinput(int, struct sockaddr *, void *); +extern void esp6_ctlinput(int, struct sockaddr *, void *, struct ifnet *); #endif /* BSD_KERNEL_PRIVATE */ #endif /* _NETINET6_ESP6_H_ */ diff --git a/bsd/netinet6/esp_chachapoly.c b/bsd/netinet6/esp_chachapoly.c new file mode 100644 index 000000000..0970f6983 --- /dev/null +++ b/bsd/netinet6/esp_chachapoly.c @@ -0,0 +1,481 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/socket.h> +#include <sys/queue.h> +#include <sys/syslog.h> +#include <sys/errno.h> +#include <sys/mbuf.h> +#include <sys/mcache.h> +#include <mach/vm_param.h> +#include <kern/locks.h> +#include <string.h> +#include <net/if.h> +#include <net/route.h> +#include <net/net_osdep.h> +#include <netinet6/ipsec.h> +#include <netinet6/esp.h> +#include <netinet6/esp_chachapoly.h> +#include <netkey/key.h> +#include <netkey/keydb.h> +#include <corecrypto/cc.h> +#include <libkern/crypto/chacha20poly1305.h> + +#define ESP_CHACHAPOLY_SALT_LEN 4 +#define ESP_CHACHAPOLY_KEY_LEN 32 +#define ESP_CHACHAPOLY_NONCE_LEN 12 + +// The minimum alignment is documented in KALLOC_LOG2_MINALIGN +// which isn't accessible from here. Current minimum is 8. +_Static_assert(_Alignof(chacha20poly1305_ctx) <= 8, + "Alignment guarantee is broken"); + +#if ((( 8 * (ESP_CHACHAPOLY_KEY_LEN + ESP_CHACHAPOLY_SALT_LEN)) != ESP_CHACHAPOLY_KEYBITS_WITH_SALT) || \ + (ESP_CHACHAPOLY_KEY_LEN != CCCHACHA20_KEY_NBYTES) || \ + (ESP_CHACHAPOLY_NONCE_LEN != CCCHACHA20POLY1305_NONCE_NBYTES)) +#error "Invalid sizes" +#endif + +extern lck_mtx_t *sadb_mutex; + +typedef struct _esp_chachapoly_ctx { + chacha20poly1305_ctx ccp_ctx; + uint8_t ccp_salt[ESP_CHACHAPOLY_SALT_LEN]; + bool ccp_implicit_iv; +} esp_chachapoly_ctx_s, *esp_chachapoly_ctx_t; + + +#define ESP_ASSERT(_cond, _format, ...) \ + do { \ + if (!(_cond)) { \ + panic("%s:%d " _format, __FUNCTION__, __LINE__, ##__VA_ARGS__); \ + } \ + } while (0) + +#define ESP_CHECK_ARG(_arg) ESP_ASSERT(_arg != NULL, #_arg "is NULL") + +#define _esp_log(_level, _format, ...) \ + log(_level, "%s:%d " _format, __FUNCTION__, __LINE__, ##__VA_ARGS__) +#define esp_log_err(_format, ...) _esp_log(LOG_ERR, _format, ##__VA_ARGS__) + +#define _esp_packet_log(_level, _format, ...) \ + ipseclog((_level, "%s:%d " _format, __FUNCTION__, __LINE__, ##__VA_ARGS__)) +#define esp_packet_log_err(_format, ...) _esp_packet_log(LOG_ERR, _format, ##__VA_ARGS__) + +int +esp_chachapoly_mature(struct secasvar *sav) +{ + const struct esp_algorithm *algo; + + ESP_CHECK_ARG(sav); + + if ((sav->flags & SADB_X_EXT_OLD) != 0) { + esp_log_err("ChaChaPoly is incompatible with SADB_X_EXT_OLD"); + return 1; + } + if ((sav->flags & SADB_X_EXT_DERIV) != 0) { + esp_log_err("ChaChaPoly is incompatible with SADB_X_EXT_DERIV"); + return 1; + } + + if (sav->alg_enc != SADB_X_EALG_CHACHA20POLY1305) { + esp_log_err("ChaChaPoly unsupported algorithm %d", + sav->alg_enc); + return 1; + } + + if (sav->key_enc == NULL) { + esp_log_err("ChaChaPoly key is missing"); + return 1; + } + + algo = esp_algorithm_lookup(sav->alg_enc); + if (algo == NULL) { + esp_log_err("ChaChaPoly lookup failed for algorithm %d", + sav->alg_enc); + return 1; + } + + if (sav->key_enc->sadb_key_bits != ESP_CHACHAPOLY_KEYBITS_WITH_SALT) { + esp_log_err("ChaChaPoly invalid key length %d bits", + sav->key_enc->sadb_key_bits); + return 1; + } + + return 0; +} + +int +esp_chachapoly_schedlen(__unused const struct esp_algorithm *algo) +{ + return sizeof(esp_chachapoly_ctx_s); +} + +int +esp_chachapoly_schedule(__unused const struct esp_algorithm *algo, + struct secasvar *sav) +{ + esp_chachapoly_ctx_t esp_ccp_ctx; + int rc = 0; + + ESP_CHECK_ARG(sav); + if (sav->ivlen != ESP_CHACHAPOLY_IV_LEN) { + esp_log_err("Invalid ivlen %u", sav->ivlen); + return EINVAL; + } + if (_KEYLEN(sav->key_enc) != ESP_CHACHAPOLY_KEY_LEN + ESP_CHACHAPOLY_SALT_LEN) { + esp_log_err("Invalid key len %u", _KEYLEN(sav->key_enc)); + return EINVAL; + } + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); + + esp_ccp_ctx = (esp_chachapoly_ctx_t)sav->sched; + + rc = chacha20poly1305_init(&esp_ccp_ctx->ccp_ctx, + (const uint8_t *)_KEYBUF(sav->key_enc)); + if (rc != 0) { + esp_log_err("chacha20poly1305_init returned %d", rc); + return rc; + } + + memcpy(esp_ccp_ctx->ccp_salt, + (const uint8_t *)_KEYBUF(sav->key_enc) + ESP_CHACHAPOLY_KEY_LEN, + sizeof(esp_ccp_ctx->ccp_salt)); + + esp_ccp_ctx->ccp_implicit_iv = ((sav->flags & SADB_X_EXT_IIV) != 0); + + return 0; +} + +int +esp_chachapoly_encrypt_finalize(struct secasvar *sav, + unsigned char *tag, + unsigned int tag_bytes) +{ + esp_chachapoly_ctx_t esp_ccp_ctx; + int rc = 0; + + ESP_CHECK_ARG(sav); + ESP_CHECK_ARG(tag); + if (tag_bytes != ESP_CHACHAPOLY_ICV_LEN) { + esp_log_err("Invalid tag_bytes %u", tag_bytes); + return EINVAL; + } + + esp_ccp_ctx = (esp_chachapoly_ctx_t)sav->sched; + rc = chacha20poly1305_finalize(&esp_ccp_ctx->ccp_ctx, tag); + if (rc != 0) { + esp_log_err("chacha20poly1305_finalize returned %d", rc); + return rc; + } + return 0; +} + +int +esp_chachapoly_decrypt_finalize(struct secasvar *sav, + unsigned char *tag, + unsigned int tag_bytes) +{ + esp_chachapoly_ctx_t esp_ccp_ctx; + int rc = 0; + + ESP_CHECK_ARG(sav); + ESP_CHECK_ARG(tag); + if (tag_bytes != ESP_CHACHAPOLY_ICV_LEN) { + esp_log_err("Invalid tag_bytes %u", tag_bytes); + return EINVAL; + } + + esp_ccp_ctx = (esp_chachapoly_ctx_t)sav->sched; + rc = chacha20poly1305_verify(&esp_ccp_ctx->ccp_ctx, tag); + if (rc != 0) { + esp_log_err("chacha20poly1305_finalize returned %d", rc); + return rc; + } + return 0; +} + +int +esp_chachapoly_encrypt(struct mbuf *m, // head of mbuf chain + size_t off, // offset to ESP header + __unused size_t plen, + struct secasvar *sav, + __unused const struct esp_algorithm *algo, + int ivlen) +{ + struct mbuf *s = m; // this mbuf + int32_t soff = 0; // offset from the head of mbuf chain (m) to head of this mbuf (s) + int32_t sn = 0; // offset from the head of this mbuf (s) to the body + uint8_t *sp; // buffer of a given encryption round + size_t len; // length of a given encryption round + const int32_t ivoff = (int32_t)off + (int32_t)sizeof(struct newesp); // IV offset + int32_t bodyoff; // body offset + int rc = 0; // return code of corecrypto operations + struct newesp esp_hdr; // ESP header for AAD + _Static_assert(sizeof(esp_hdr) == 8, "Bad size"); + uint8_t nonce[ESP_CHACHAPOLY_NONCE_LEN]; + esp_chachapoly_ctx_t esp_ccp_ctx; + + ESP_CHECK_ARG(m); + ESP_CHECK_ARG(sav); + if (ivlen != ESP_CHACHAPOLY_IV_LEN) { + m_freem(m); + esp_log_err("Invalid ivlen %u", ivlen); + return EINVAL; + } + if (sav->ivlen != ESP_CHACHAPOLY_IV_LEN) { + m_freem(m); + esp_log_err("Invalid sav->ivlen %u", sav->ivlen); + return EINVAL; + } + + esp_ccp_ctx = (esp_chachapoly_ctx_t)sav->sched; + if (esp_ccp_ctx->ccp_implicit_iv) { + bodyoff = ivoff; + } else { + bodyoff = ivoff + ivlen; + } + // check if total packet length is enough to contain ESP + IV + if (m->m_pkthdr.len < bodyoff) { + esp_log_err("Packet too short %d < %zu", m->m_pkthdr.len, bodyoff); + m_freem(m); + return EINVAL; + } + + rc = chacha20poly1305_reset(&esp_ccp_ctx->ccp_ctx); + if (rc != 0) { + m_freem(m); + esp_log_err("chacha20poly1305_reset failed %d", rc); + return rc; + } + + // RFC 7634 dictates that the 12 byte nonce must be + // the 4 byte salt followed by the 8 byte IV. + // The IV MUST be non-repeating but does not need to be unpredictable, + // so we use 4 bytes of 0 followed by the 4 byte ESP sequence number. + // this allows us to use implicit IV -- draft-mglt-ipsecme-implicit-iv + memset(sav->iv, 0, 4); + memcpy(sav->iv + 4, &sav->seq, sizeof(sav->seq)); + _Static_assert(4 + sizeof(sav->seq) == ESP_CHACHAPOLY_IV_LEN, + "Bad IV length"); + memcpy(nonce, esp_ccp_ctx->ccp_salt, ESP_CHACHAPOLY_SALT_LEN); + memcpy(nonce + ESP_CHACHAPOLY_SALT_LEN, sav->iv, ESP_CHACHAPOLY_IV_LEN); + _Static_assert(ESP_CHACHAPOLY_SALT_LEN + ESP_CHACHAPOLY_IV_LEN == sizeof(nonce), + "Bad nonce length"); + + rc = chacha20poly1305_setnonce(&esp_ccp_ctx->ccp_ctx, nonce); + if (rc != 0) { + m_freem(m); + esp_log_err("chacha20poly1305_setnonce failed %d", rc); + return rc; + } + + if (!esp_ccp_ctx->ccp_implicit_iv) { + m_copyback(m, ivoff, ivlen, sav->iv); + } + cc_clear(sizeof(nonce), nonce); + + // Set Additional Authentication Data (AAD) + m_copydata(m, (int)off, sizeof(esp_hdr), (void *)&esp_hdr); + + rc = chacha20poly1305_aad(&esp_ccp_ctx->ccp_ctx, + sizeof(esp_hdr), + (void *)&esp_hdr); + if (rc != 0) { + m_freem(m); + esp_log_err("chacha20poly1305_aad failed %d", rc); + return rc; + } + + // skip headers/IV + while (s != NULL && soff < bodyoff) { + if (soff + s->m_len > bodyoff) { + sn = bodyoff - soff; + break; + } + + soff += s->m_len; + s = s->m_next; + } + + while (s != NULL && soff < m->m_pkthdr.len) { + len = (size_t)(s->m_len - sn); + if (len == 0) { + // skip empty mbufs + continue; + } + sp = mtod(s, uint8_t *) + sn; + + rc = chacha20poly1305_encrypt(&esp_ccp_ctx->ccp_ctx, + len, sp, sp); + if (rc != 0) { + m_freem(m); + esp_log_err("chacha20poly1305_encrypt failed %d", rc); + return rc; + } + + sn = 0; + soff += s->m_len; + s = s->m_next; + } + if (s == NULL && soff != m->m_pkthdr.len) { + m_freem(m); + esp_log_err("not enough mbufs %d %d", soff, m->m_pkthdr.len); + return EFBIG; + } + return 0; +} + +int +esp_chachapoly_decrypt(struct mbuf *m, // head of mbuf chain + size_t off, // offset to ESP header + struct secasvar *sav, + __unused const struct esp_algorithm *algo, + int ivlen) +{ + struct mbuf *s = m; // this mbuf + int32_t soff = 0; // offset from the head of mbuf chain (m) to head of this mbuf (s) + int32_t sn = 0; // offset from the head of this mbuf (s) to the body + uint8_t *sp; // buffer of a given encryption round + size_t len; // length of a given encryption round + const int32_t ivoff = (int32_t)off + (int32_t)sizeof(struct newesp); // IV offset + int32_t bodyoff; // body offset + int rc = 0; // return code of corecrypto operations + struct newesp esp_hdr; // ESP header for AAD + _Static_assert(sizeof(esp_hdr) == 8, "Bad size"); + uint8_t nonce[ESP_CHACHAPOLY_NONCE_LEN]; + esp_chachapoly_ctx_t esp_ccp_ctx; + + ESP_CHECK_ARG(m); + ESP_CHECK_ARG(sav); + if (ivlen != ESP_CHACHAPOLY_IV_LEN) { + m_freem(m); + esp_log_err("Invalid ivlen %u", ivlen); + return EINVAL; + } + if (sav->ivlen != ESP_CHACHAPOLY_IV_LEN) { + m_freem(m); + esp_log_err("Invalid sav->ivlen %u", sav->ivlen); + return EINVAL; + } + + esp_ccp_ctx = (esp_chachapoly_ctx_t)sav->sched; + if (esp_ccp_ctx->ccp_implicit_iv) { + bodyoff = ivoff; + } else { + bodyoff = ivoff + ivlen; + } + // check if total packet length is enough to contain ESP + IV + if (m->m_pkthdr.len < bodyoff) { + esp_packet_log_err("Packet too short %d < %zu", m->m_pkthdr.len, bodyoff); + m_freem(m); + return EINVAL; + } + + rc = chacha20poly1305_reset(&esp_ccp_ctx->ccp_ctx); + if (rc != 0) { + m_freem(m); + esp_log_err("chacha20poly1305_reset failed %d", rc); + return rc; + } + + m_copydata(m, (int)off, sizeof(esp_hdr), (void *)&esp_hdr); + + // RFC 7634 dictates that the 12 byte nonce must be + // the 4 byte salt followed by the 8 byte IV. + memcpy(nonce, esp_ccp_ctx->ccp_salt, ESP_CHACHAPOLY_SALT_LEN); + if (esp_ccp_ctx->ccp_implicit_iv) { + // IV is implicit (4 zero bytes followed by the ESP sequence number) + memset(nonce + ESP_CHACHAPOLY_SALT_LEN, 0, 4); + memcpy(nonce + ESP_CHACHAPOLY_SALT_LEN + 4, &esp_hdr.esp_seq, sizeof(esp_hdr.esp_seq)); + _Static_assert(4 + sizeof(esp_hdr.esp_seq) == ESP_CHACHAPOLY_IV_LEN, "Bad IV length"); + } else { + // copy IV from packet + m_copydata(m, ivoff, ESP_CHACHAPOLY_IV_LEN, nonce + ESP_CHACHAPOLY_SALT_LEN); + } + _Static_assert(ESP_CHACHAPOLY_SALT_LEN + ESP_CHACHAPOLY_IV_LEN == sizeof(nonce), + "Bad nonce length"); + + rc = chacha20poly1305_setnonce(&esp_ccp_ctx->ccp_ctx, nonce); + if (rc != 0) { + m_freem(m); + esp_log_err("chacha20poly1305_setnonce failed %d", rc); + return rc; + } + cc_clear(sizeof(nonce), nonce); + + // Set Additional Authentication Data (AAD) + rc = chacha20poly1305_aad(&esp_ccp_ctx->ccp_ctx, + sizeof(esp_hdr), + (void *)&esp_hdr); + if (rc != 0) { + m_freem(m); + esp_log_err("chacha20poly1305_aad failed %d", rc); + return rc; + } + + // skip headers/IV + while (s != NULL && soff < bodyoff) { + if (soff + s->m_len > bodyoff) { + sn = bodyoff - soff; + break; + } + + soff += s->m_len; + s = s->m_next; + } + + while (s != NULL && soff < m->m_pkthdr.len) { + len = (size_t)(s->m_len - sn); + if (len == 0) { + // skip empty mbufs + continue; + } + sp = mtod(s, uint8_t *) + sn; + + rc = chacha20poly1305_decrypt(&esp_ccp_ctx->ccp_ctx, + len, sp, sp); + if (rc != 0) { + m_freem(m); + esp_packet_log_err("chacha20poly1305_decrypt failed %d", rc); + return rc; + } + + sn = 0; + soff += s->m_len; + s = s->m_next; + } + if (s == NULL && soff != m->m_pkthdr.len) { + m_freem(m); + esp_packet_log_err("not enough mbufs %d %d", soff, m->m_pkthdr.len); + return EFBIG; + } + return 0; +} diff --git a/bsd/netinet6/esp_chachapoly.h b/bsd/netinet6/esp_chachapoly.h new file mode 100644 index 000000000..b98b77a40 --- /dev/null +++ b/bsd/netinet6/esp_chachapoly.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include <sys/appleapiopts.h> + +#ifdef BSD_KERNEL_PRIVATE + +#ifndef _ESP_CHACHA_POLY_H_ +#define _ESP_CHACHA_POLY_H_ + +#define ESP_CHACHAPOLY_PAD_BOUND 1 +#define ESP_CHACHAPOLY_IV_LEN 8 +#define ESP_CHACHAPOLY_ICV_LEN 16 +#define ESP_CHACHAPOLY_KEYBITS_WITH_SALT 288 /* 32 bytes key + 4 bytes salt */ + +int esp_chachapoly_schedlen(const struct esp_algorithm *); +int esp_chachapoly_schedule(const struct esp_algorithm *, + struct secasvar *); +int esp_chachapoly_encrypt(struct mbuf *, size_t, size_t, struct secasvar *, + const struct esp_algorithm *, int); +int esp_chachapoly_decrypt(struct mbuf *, size_t, struct secasvar *, + const struct esp_algorithm *, int); +int esp_chachapoly_encrypt_finalize(struct secasvar *, unsigned char *, unsigned int); +int esp_chachapoly_decrypt_finalize(struct secasvar *, unsigned char *, unsigned int); +int esp_chachapoly_mature(struct secasvar *); + +#endif /* _ESP_CHACHA_POLY_H_ */ +#endif /* BSD_KERNEL_PRIVATE */ diff --git a/bsd/netinet6/esp_core.c b/bsd/netinet6/esp_core.c index bc9a75ed6..a26873e45 100644 --- a/bsd/netinet6/esp_core.c +++ b/bsd/netinet6/esp_core.c @@ -98,6 +98,7 @@ #include <netinet6/esp6.h> #endif #include <netinet6/esp_rijndael.h> +#include <netinet6/esp_chachapoly.h> #include <net/pfkeyv2.h> #include <netkey/keydb.h> #include <netkey/key.h> @@ -183,6 +184,14 @@ static const struct esp_algorithm aes_gcm = esp_gcm_encrypt_aes, esp_gcm_schedule, 0, 0, 16, esp_gcm_decrypt_finalize, esp_gcm_encrypt_finalize}; +static const struct esp_algorithm chacha_poly = + { ESP_CHACHAPOLY_PAD_BOUND, ESP_CHACHAPOLY_IV_LEN, + esp_chachapoly_mature, ESP_CHACHAPOLY_KEYBITS_WITH_SALT, + ESP_CHACHAPOLY_KEYBITS_WITH_SALT, esp_chachapoly_schedlen, + "chacha-poly", esp_common_ivlen, esp_chachapoly_decrypt, + esp_chachapoly_encrypt, esp_chachapoly_schedule, + NULL, NULL, ESP_CHACHAPOLY_ICV_LEN, + esp_chachapoly_decrypt_finalize, esp_chachapoly_encrypt_finalize}; static const struct esp_algorithm *esp_algorithms[] = { &des_cbc, @@ -190,6 +199,7 @@ static const struct esp_algorithm *esp_algorithms[] = { &null_esp, &aes_cbc, &aes_gcm, + &chacha_poly, }; const struct esp_algorithm * @@ -206,6 +216,8 @@ esp_algorithm_lookup(int idx) return &aes_cbc; case SADB_X_EALG_AES_GCM: return &aes_gcm; + case SADB_X_EALG_CHACHA20POLY1305: + return &chacha_poly; default: return NULL; } @@ -248,6 +260,17 @@ esp_schedule(const struct esp_algorithm *algo, struct secasvar *sav) lck_mtx_unlock(sadb_mutex); return 0; } + + /* prevent disallowed implicit IV */ + if (((sav->flags & SADB_X_EXT_IIV) != 0) && + (sav->alg_enc != SADB_X_EALG_AES_GCM) && + (sav->alg_enc != SADB_X_EALG_CHACHA20POLY1305)) { + ipseclog((LOG_ERR, + "esp_schedule %s: implicit IV not allowed\n", + algo->name)); + return EINVAL; + } + /* no schedule necessary */ if (!algo->schedule || !algo->schedlen) { lck_mtx_unlock(sadb_mutex); @@ -384,7 +407,7 @@ esp_des_schedule( struct secasvar *sav) { - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); if (des_ecb_key_sched((des_cblock *)_KEYBUF(sav->key_enc), (des_ecb_key_schedule *)sav->sched)) return EINVAL; @@ -498,6 +521,11 @@ esp_gcm_mature(struct secasvar *sav) "esp_gcm_mature: algorithm incompatible with derived\n")); return 1; } + if (sav->flags & SADB_X_EXT_IIV) { + ipseclog((LOG_ERR, + "esp_gcm_mature: implicit IV not currently implemented\n")); + return 1; + } if (!sav->key_enc) { ipseclog((LOG_ERR, "esp_gcm_mature: no key is given.\n")); @@ -550,7 +578,7 @@ esp_3des_schedule( __unused const struct esp_algorithm *algo, struct secasvar *sav) { - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); if (des3_ecb_key_sched((des_cblock *)_KEYBUF(sav->key_enc), (des3_ecb_key_schedule *)sav->sched)) diff --git a/bsd/netinet6/esp_input.c b/bsd/netinet6/esp_input.c index 8056438bb..912dd3983 100644 --- a/bsd/netinet6/esp_input.c +++ b/bsd/netinet6/esp_input.c @@ -176,6 +176,12 @@ esp6_input_strip_udp_encap (struct mbuf *m, int ip6hlen) void esp4_input(struct mbuf *m, int off) +{ + (void)esp4_input_extended(m, off, NULL); +} + +struct mbuf * +esp4_input_extended(struct mbuf *m, int off, ifnet_t interface) { struct ip *ip; #if INET6 @@ -193,6 +199,7 @@ esp4_input(struct mbuf *m, int off) size_t hlen; size_t esplen; sa_family_t ifamily; + struct mbuf *out_m = NULL; KERNEL_DEBUG(DBG_FNC_ESPIN | DBG_FUNC_START, 0,0,0,0,0); /* sanity check for alignment. */ @@ -213,6 +220,8 @@ esp4_input(struct mbuf *m, int off) } } + m->m_pkthdr.csum_flags &= ~CSUM_RX_FLAGS; + /* Expect 32-bit aligned data pointer on strict-align platforms */ MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(m); @@ -235,9 +244,9 @@ esp4_input(struct mbuf *m, int off) /* find the sassoc. */ spi = esp->esp_spi; - if ((sav = key_allocsa(AF_INET, - (caddr_t)&ip->ip_src, (caddr_t)&ip->ip_dst, - IPPROTO_ESP, spi)) == 0) { + if ((sav = key_allocsa_extended(AF_INET, + (caddr_t)&ip->ip_src, (caddr_t)&ip->ip_dst, + IPPROTO_ESP, spi, interface)) == 0) { ipseclog((LOG_WARNING, "IPv4 ESP input: no key association found for spi %u\n", (u_int32_t)ntohl(spi))); @@ -337,7 +346,7 @@ esp4_input(struct mbuf *m, int off) } if (cc_cmp_safe(siz, sum0, sum)) { - ipseclog((LOG_WARNING, "auth fail in IPv4 ESP input: %s %s\n", + ipseclog((LOG_WARNING, "cc_cmp fail in IPv4 ESP input: %s %s\n", ipsec4_logpacketstr(ip, spi), ipsec_logsastr(sav))); IPSEC_STAT_INCREMENT(ipsecstat.in_espauthfail); goto bad; @@ -618,6 +627,13 @@ noreplaycheck: // Input via IPSec interface if (sav->sah->ipsec_if != NULL) { + // Return mbuf + if (interface != NULL && + interface == sav->sah->ipsec_if) { + out_m = m; + goto done; + } + if (ipsec_inject_inbound_packet(sav->sah->ipsec_if, m) == 0) { m = NULL; goto done; @@ -670,6 +686,7 @@ noreplaycheck: if (nxt == IPPROTO_TCP || nxt == IPPROTO_UDP) { m->m_pkthdr.csum_flags = CSUM_DATA_VALID | CSUM_PSEUDO_HDR; m->m_pkthdr.csum_data = 0xFFFF; + _CASSERT(offsetof(struct pkthdr, csum_data) == offsetof(struct pkthdr, csum_rx_val)); } if (nxt != IPPROTO_DONE) { @@ -718,12 +735,28 @@ noreplaycheck: struct ip *, ip, struct ifnet *, m->m_pkthdr.rcvif, struct ip *, ip, struct ip6_hdr *, NULL); - // Input via IPSec interface + // Input via IPsec interface legacy path if (sav->sah->ipsec_if != NULL) { + int mlen; + if ((mlen = m_length2(m, NULL)) < hlen) { + ipseclog((LOG_DEBUG, + "IPv4 ESP input: decrypted packet too short %d < %d\n", + mlen, hlen)); + IPSEC_STAT_INCREMENT(ipsecstat.in_inval); + goto bad; + } ip->ip_len = htons(ip->ip_len + hlen); ip->ip_off = htons(ip->ip_off); ip->ip_sum = 0; ip->ip_sum = ip_cksum_hdr_in(m, hlen); + + // Return mbuf + if (interface != NULL && + interface == sav->sah->ipsec_if) { + out_m = m; + goto done; + } + if (ipsec_inject_inbound_packet(sav->sah->ipsec_if, m) == 0) { m = NULL; goto done; @@ -733,8 +766,9 @@ noreplaycheck: } ip_proto_dispatch_in(m, off, nxt, 0); - } else + } else { m_freem(m); + } m = NULL; } @@ -746,8 +780,7 @@ done: key_freesav(sav, KEY_SADB_UNLOCKED); } IPSEC_STAT_INCREMENT(ipsecstat.in_success); - return; - + return out_m; bad: if (sav) { KEYDEBUG(KEYDEBUG_IPSEC_STAMP, @@ -755,16 +788,24 @@ bad: (uint64_t)VM_KERNEL_ADDRPERM(sav))); key_freesav(sav, KEY_SADB_UNLOCKED); } - if (m) + if (m) { m_freem(m); + } KERNEL_DEBUG(DBG_FNC_ESPIN | DBG_FUNC_END, 4,0,0,0,0); - return; + return out_m; } #endif /* INET */ #if INET6 + int esp6_input(struct mbuf **mp, int *offp, int proto) +{ + return esp6_input_extended(mp, offp, proto, NULL); +} + +int +esp6_input_extended(struct mbuf **mp, int *offp, int proto, ifnet_t interface) { #pragma unused(proto) struct mbuf *m = *mp; @@ -802,6 +843,8 @@ esp6_input(struct mbuf **mp, int *offp, int proto) return IPPROTO_DONE; } #endif + m->m_pkthdr.csum_flags &= ~CSUM_RX_FLAGS; + /* Expect 32-bit data aligned pointer on strict-align platforms */ MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(m); @@ -825,9 +868,9 @@ esp6_input(struct mbuf **mp, int *offp, int proto) /* find the sassoc. */ spi = esp->esp_spi; - if ((sav = key_allocsa(AF_INET6, - (caddr_t)&ip6->ip6_src, (caddr_t)&ip6->ip6_dst, - IPPROTO_ESP, spi)) == 0) { + if ((sav = key_allocsa_extended(AF_INET6, + (caddr_t)&ip6->ip6_src, (caddr_t)&ip6->ip6_dst, + IPPROTO_ESP, spi, interface)) == 0) { ipseclog((LOG_WARNING, "IPv6 ESP input: no key association found for spi %u\n", (u_int32_t)ntohl(spi))); @@ -1190,6 +1233,12 @@ noreplaycheck: // Input via IPSec interface if (sav->sah->ipsec_if != NULL) { + // Return mbuf + if (interface != NULL && + interface == sav->sah->ipsec_if) { + goto done; + } + if (ipsec_inject_inbound_packet(sav->sah->ipsec_if, m) == 0) { m = NULL; nxt = IPPROTO_DONE; @@ -1307,10 +1356,17 @@ noreplaycheck: if (nxt == IPPROTO_TCP || nxt == IPPROTO_UDP) { m->m_pkthdr.csum_flags = CSUM_DATA_VALID | CSUM_PSEUDO_HDR; m->m_pkthdr.csum_data = 0xFFFF; + _CASSERT(offsetof(struct pkthdr, csum_data) == offsetof(struct pkthdr, csum_rx_val)); } // Input via IPSec interface if (sav->sah->ipsec_if != NULL) { + // Return mbuf + if (interface != NULL && + interface == sav->sah->ipsec_if) { + goto done; + } + if (ipsec_inject_inbound_packet(sav->sah->ipsec_if, m) == 0) { m = NULL; nxt = IPPROTO_DONE; @@ -1341,13 +1397,17 @@ bad: (uint64_t)VM_KERNEL_ADDRPERM(sav))); key_freesav(sav, KEY_SADB_UNLOCKED); } - if (m) + if (m) { m_freem(m); + } + if (interface != NULL) { + *mp = NULL; + } return IPPROTO_DONE; } void -esp6_ctlinput(int cmd, struct sockaddr *sa, void *d) +esp6_ctlinput(int cmd, struct sockaddr *sa, void *d, __unused struct ifnet *ifp) { const struct newesp *espp; struct newesp esp; @@ -1355,7 +1415,7 @@ esp6_ctlinput(int cmd, struct sockaddr *sa, void *d) struct secasvar *sav; struct ip6_hdr *ip6; struct mbuf *m; - int off; + int off = 0; struct sockaddr_in6 *sa6_src, *sa6_dst; if (sa->sa_family != AF_INET6 || diff --git a/bsd/netinet6/esp_output.c b/bsd/netinet6/esp_output.c index 3b8b817d4..720846a19 100644 --- a/bsd/netinet6/esp_output.c +++ b/bsd/netinet6/esp_output.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Apple Inc. All rights reserved. + * Copyright (c) 2008-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -828,7 +828,7 @@ esp_output( ip6 = mtod(m, struct ip6_hdr *); udp->uh_ulen = htons(plen + siz + extendsiz + esphlen); udp->uh_sum = in6_pseudo(&ip6->ip6_src, &ip6->ip6_dst, htonl(ntohs(udp->uh_ulen) + IPPROTO_UDP)); - m->m_pkthdr.csum_flags = CSUM_UDPIPV6; + m->m_pkthdr.csum_flags = (CSUM_UDPIPV6|CSUM_ZERO_INVERT); m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); break; } diff --git a/bsd/netinet6/esp_rijndael.c b/bsd/netinet6/esp_rijndael.c index 56b560263..eba6c7fd3 100644 --- a/bsd/netinet6/esp_rijndael.c +++ b/bsd/netinet6/esp_rijndael.c @@ -109,7 +109,7 @@ esp_aes_schedule( struct secasvar *sav) { - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); aes_ctx *ctx = (aes_ctx*)sav->sched; aes_decrypt_key((const unsigned char *) _KEYBUF(sav->key_enc), _KEYLEN(sav->key_enc), &ctx->decrypt); @@ -287,12 +287,25 @@ esp_cbc_decrypt_aes( } else { sp_unaligned = sp; if (len > MAX_REALIGN_LEN) { + m_freem(m); + if (d0 != NULL) { + m_freem(d0); + } + if (sp_aligned != NULL) { + FREE(sp_aligned, M_SECA); + sp_aligned = NULL; + } return ENOBUFS; } if (sp_aligned == NULL) { sp_aligned = (u_int8_t *)_MALLOC(MAX_REALIGN_LEN, M_SECA, M_DONTWAIT); - if (sp_aligned == NULL) - return ENOMEM; + if (sp_aligned == NULL) { + m_freem(m); + if (d0 != NULL) { + m_freem(d0); + } + return ENOMEM; + } } sp = sp_aligned; memcpy(sp, sp_unaligned, len); @@ -482,12 +495,25 @@ esp_cbc_encrypt_aes( } else { sp_unaligned = sp; if (len > MAX_REALIGN_LEN) { + m_freem(m); + if (d0) { + m_freem(d0); + } + if (sp_aligned != NULL) { + FREE(sp_aligned, M_SECA); + sp_aligned = NULL; + } return ENOBUFS; } if (sp_aligned == NULL) { sp_aligned = (u_int8_t *)_MALLOC(MAX_REALIGN_LEN, M_SECA, M_DONTWAIT); - if (sp_aligned == NULL) + if (sp_aligned == NULL) { + m_freem(m); + if (d0) { + m_freem(d0); + } return ENOMEM; + } } sp = sp_aligned; memcpy(sp, sp_unaligned, len); @@ -556,7 +582,7 @@ int esp_gcm_schedule( __unused const struct esp_algorithm *algo, struct secasvar *sav) { - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); aes_gcm_ctx *ctx = (aes_gcm_ctx*)P2ROUNDUP(sav->sched, ESP_GCM_ALIGN); u_int ivlen = sav->ivlen; unsigned char nonce[ESP_GCM_SALT_LEN+ivlen]; @@ -764,12 +790,25 @@ esp_gcm_encrypt_aes( } else { sp_unaligned = sp; if (len > MAX_REALIGN_LEN) { + m_freem(m); + if (d0) { + m_freem(d0); + } + if (sp_aligned != NULL) { + FREE(sp_aligned, M_SECA); + sp_aligned = NULL; + } return ENOBUFS; } if (sp_aligned == NULL) { sp_aligned = (u_int8_t *)_MALLOC(MAX_REALIGN_LEN, M_SECA, M_DONTWAIT); - if (sp_aligned == NULL) + if (sp_aligned == NULL) { + m_freem(m); + if (d0) { + m_freem(d0); + } return ENOMEM; + } } sp = sp_aligned; memcpy(sp, sp_unaligned, len); @@ -960,12 +999,25 @@ esp_gcm_decrypt_aes( } else { sp_unaligned = sp; if (len > MAX_REALIGN_LEN) { + m_freem(m); + if (d0) { + m_freem(d0); + } + if (sp_aligned != NULL) { + FREE(sp_aligned, M_SECA); + sp_aligned = NULL; + } return ENOBUFS; } if (sp_aligned == NULL) { sp_aligned = (u_int8_t *)_MALLOC(MAX_REALIGN_LEN, M_SECA, M_DONTWAIT); - if (sp_aligned == NULL) + if (sp_aligned == NULL) { + m_freem(m); + if (d0) { + m_freem(d0); + } return ENOMEM; + } } sp = sp_aligned; memcpy(sp, sp_unaligned, len); diff --git a/bsd/netinet6/frag6.c b/bsd/netinet6/frag6.c index 8e5f416ea..5bdb1adf3 100644 --- a/bsd/netinet6/frag6.c +++ b/bsd/netinet6/frag6.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2015 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -79,6 +79,7 @@ #include <netinet/in.h> #include <netinet/in_var.h> #include <netinet/ip.h> +#include <netinet/ip_var.h> #include <netinet/ip6.h> #include <netinet6/ip6_var.h> #include <netinet/icmp6.h> @@ -204,7 +205,7 @@ frag6_restore_context(struct mbuf *m) static void frag6_icmp6_paramprob_error(struct fq6_head *diq6) { - lck_mtx_assert(&ip6qlock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&ip6qlock, LCK_MTX_ASSERT_NOTOWNED); if (!MBUFQ_EMPTY(diq6)) { struct mbuf *merr, *merr_tmp; @@ -227,7 +228,7 @@ frag6_icmp6_paramprob_error(struct fq6_head *diq6) static void frag6_icmp6_timeex_error(struct fq6_head *diq6) { - lck_mtx_assert(&ip6qlock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&ip6qlock, LCK_MTX_ASSERT_NOTOWNED); if (!MBUFQ_EMPTY(diq6)) { struct mbuf *m, *m_tmp; @@ -387,20 +388,24 @@ frag6_input(struct mbuf **mp, int *offp, int proto) * as that is the most common case. * * Perform 1's complement adjustment of octets that got included/ - * excluded in the hardware-calculated checksum value. + * excluded in the hardware-calculated checksum value. Also take + * care of any trailing bytes and subtract out their partial sum. */ if (ip6f->ip6f_nxt == IPPROTO_UDP && offset == (sizeof (*ip6) + sizeof (*ip6f)) && (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PARTIAL | CSUM_PSEUDO_HDR)) == (CSUM_DATA_VALID | CSUM_PARTIAL)) { - uint32_t start; + uint32_t start = m->m_pkthdr.csum_rx_start; + uint32_t ip_len = (sizeof (*ip6) + ntohs(ip6->ip6_plen)); + int32_t trailer = (m_pktlen(m) - ip_len); + uint32_t swbytes = (uint32_t)trailer; - start = m->m_pkthdr.csum_rx_start; csum = m->m_pkthdr.csum_rx_val; - if (start != offset) { - uint16_t s, d; + ASSERT(trailer >= 0); + if (start != offset || trailer != 0) { + uint16_t s = 0, d = 0; if (IN6_IS_SCOPE_EMBED(&ip6->ip6_src)) { s = ip6->ip6_src.s6_addr16[1]; @@ -412,7 +417,12 @@ frag6_input(struct mbuf **mp, int *offp, int proto) } /* callee folds in sum */ - csum = m_adj_sum16(m, start, offset, csum); + csum = m_adj_sum16(m, start, offset, + (ip_len - offset), csum); + if (offset > start) + swbytes += (offset - start); + else + swbytes += (start - offset); if (IN6_IS_SCOPE_EMBED(&ip6->ip6_src)) ip6->ip6_src.s6_addr16[1] = s; @@ -421,6 +431,11 @@ frag6_input(struct mbuf **mp, int *offp, int proto) } csum_flags = m->m_pkthdr.csum_flags; + + if (swbytes != 0) + udp_in6_cksum_stats(swbytes); + if (trailer != 0) + m_adj(m, -trailer); } else { csum = 0; csum_flags = 0; @@ -873,7 +888,7 @@ frag6_freef(struct ip6q *q6, struct fq6_head *dfq6, struct fq6_head *diq6) { struct ip6asfrag *af6, *down6; - lck_mtx_assert(&ip6qlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ip6qlock, LCK_MTX_ASSERT_OWNED); for (af6 = q6->ip6q_down; af6 != (struct ip6asfrag *)q6; af6 = down6) { @@ -916,7 +931,7 @@ frag6_freef(struct ip6q *q6, struct fq6_head *dfq6, struct fq6_head *diq6) void frag6_enq(struct ip6asfrag *af6, struct ip6asfrag *up6) { - lck_mtx_assert(&ip6qlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ip6qlock, LCK_MTX_ASSERT_OWNED); af6->ip6af_up = up6; af6->ip6af_down = up6->ip6af_down; @@ -930,7 +945,7 @@ frag6_enq(struct ip6asfrag *af6, struct ip6asfrag *up6) void frag6_deq(struct ip6asfrag *af6) { - lck_mtx_assert(&ip6qlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ip6qlock, LCK_MTX_ASSERT_OWNED); af6->ip6af_up->ip6af_down = af6->ip6af_down; af6->ip6af_down->ip6af_up = af6->ip6af_up; @@ -939,7 +954,7 @@ frag6_deq(struct ip6asfrag *af6) void frag6_insque(struct ip6q *new, struct ip6q *old) { - lck_mtx_assert(&ip6qlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ip6qlock, LCK_MTX_ASSERT_OWNED); new->ip6q_prev = old; new->ip6q_next = old->ip6q_next; @@ -950,7 +965,7 @@ frag6_insque(struct ip6q *new, struct ip6q *old) void frag6_remque(struct ip6q *p6) { - lck_mtx_assert(&ip6qlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ip6qlock, LCK_MTX_ASSERT_OWNED); p6->ip6q_prev->ip6q_next = p6->ip6q_next; p6->ip6q_next->ip6q_prev = p6->ip6q_prev; @@ -1021,7 +1036,7 @@ frag6_timeout(void *arg) static void frag6_sched_timeout(void) { - lck_mtx_assert(&ip6qlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ip6qlock, LCK_MTX_ASSERT_OWNED); if (!frag6_timeout_run && frag6_nfragpackets > 0) { frag6_timeout_run = 1; @@ -1125,7 +1140,7 @@ ip6af_free(struct ip6asfrag *af6) static void ip6q_updateparams(void) { - lck_mtx_assert(&ip6qlock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ip6qlock, LCK_MTX_ASSERT_OWNED); /* * -1 for unlimited allocation. */ diff --git a/bsd/netinet6/icmp6.c b/bsd/netinet6/icmp6.c index 23a9178d3..1580cae9d 100644 --- a/bsd/netinet6/icmp6.c +++ b/bsd/netinet6/icmp6.c @@ -574,15 +574,13 @@ icmp6_input(struct mbuf **mp, int *offp, int proto) icmp6_ifstat_inc(m->m_pkthdr.rcvif, ifs6_in_dstunreach); switch (code) { case ICMP6_DST_UNREACH_NOROUTE: + case ICMP6_DST_UNREACH_ADDR: /* PRC_HOSTDEAD is a DOS */ code = PRC_UNREACH_NET; break; case ICMP6_DST_UNREACH_ADMIN: icmp6_ifstat_inc(m->m_pkthdr.rcvif, ifs6_in_adminprohib); code = PRC_UNREACH_PROTOCOL; /* is this a good code? */ break; - case ICMP6_DST_UNREACH_ADDR: - code = PRC_HOSTDEAD; - break; case ICMP6_DST_UNREACH_BEYONDSCOPE: /* I mean "source address was incorrect." */ code = PRC_PARAMPROB; @@ -942,7 +940,7 @@ icmp6_notify_error(struct mbuf *m, int off, int icmp6len, int code) /* Detect the upper level protocol */ { - void (*ctlfunc)(int, struct sockaddr *, void *); + void (*ctlfunc)(int, struct sockaddr *, void *, struct ifnet *); u_int8_t nxt = eip6->ip6_nxt; int eoff = off + sizeof(struct icmp6_hdr) + sizeof(struct ip6_hdr); @@ -1133,11 +1131,10 @@ icmp6_notify_error(struct mbuf *m, int off, int icmp6len, int code) icmp6_mtudisc_update(&ip6cp, 1); /*XXX*/ } - ctlfunc = (void (*)(int, struct sockaddr *, void *)) - (ip6_protox[nxt]->pr_ctlinput); + ctlfunc = ip6_protox[nxt]->pr_ctlinput; if (ctlfunc) { (void) (*ctlfunc)(code, (struct sockaddr *)&icmp6dst, - &ip6cp); + &ip6cp, m->m_pkthdr.rcvif); } } return(0); @@ -1288,7 +1285,7 @@ ni6_input(struct mbuf *m, int off) !(icmp6_nodeinfo & ICMP6_NODEINFO_TMPADDROK)) { nd6log((LOG_DEBUG, "ni6_input: ignore node info to " "a temporary address in %s:%d", - __FILE__, __LINE__)); + __func__, __LINE__)); goto bad; } } @@ -2150,7 +2147,7 @@ icmp6_reflect(struct mbuf *m, size_t off) nd6log((LOG_DEBUG, "sanity fail: off=%lx, sizeof(ip6)=%lx in %s:%d\n", (u_int32_t)off, (u_int32_t)sizeof(struct ip6_hdr), - __FILE__, __LINE__)); + __func__, __LINE__)); goto bad; } @@ -2784,7 +2781,7 @@ nolladdropt:; * and truncates if not. */ if (m0->m_next || m0->m_pkthdr.len != m0->m_len) - panic("assumption failed in %s:%d\n", __FILE__, __LINE__); + panic("assumption failed in %s:%d\n", __func__, __LINE__); if (len - sizeof(*nd_opt_rh) < m0->m_pkthdr.len) { /* not enough room, truncate */ diff --git a/bsd/netinet6/in6.c b/bsd/netinet6/in6.c index 559815be5..a76b74157 100644 --- a/bsd/netinet6/in6.c +++ b/bsd/netinet6/in6.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003-2016 Apple Inc. All rights reserved. + * Copyright (c) 2003-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -105,9 +105,11 @@ #include <sys/kern_event.h> #include <sys/mcache.h> #include <sys/protosw.h> +#include <sys/sysctl.h> #include <kern/locks.h> #include <kern/zalloc.h> +#include <kern/clock.h> #include <libkern/OSAtomic.h> #include <machine/machine_routines.h> #include <mach/boolean.h> @@ -118,6 +120,7 @@ #include <net/route.h> #include <net/if_dl.h> #include <net/kpi_protocol.h> +#include <net/nwk_wq.h> #include <netinet/in.h> #include <netinet/in_var.h> @@ -142,6 +145,7 @@ #include <net/net_osdep.h> #include <net/dlil.h> +#include <net/if_llatbl.h> #if PF #include <net/pfvar.h> @@ -211,9 +215,6 @@ static int in6_to_kamescope(struct sockaddr_in6 *, struct ifnet *); static int in6_getassocids(struct socket *, uint32_t *, user_addr_t); static int in6_getconnids(struct socket *, sae_associd_t, uint32_t *, user_addr_t); -static int in6_getconninfo(struct socket *, sae_connid_t, uint32_t *, - uint32_t *, int32_t *, user_addr_t, socklen_t *, user_addr_t, socklen_t *, - uint32_t *, user_addr_t, uint32_t *); static void in6_if_up_dad_start(struct ifnet *); @@ -260,6 +261,8 @@ static struct zone *in6ifa_zone; /* zone for in6_ifaddr */ #define IN6IFA_ZONE_MAX 64 /* maximum elements in zone */ #define IN6IFA_ZONE_NAME "in6_ifaddr" /* zone name */ +struct eventhandler_lists_ctxt in6_evhdlr_ctxt; + /* * Subroutine for in6_ifaddloop() and in6_ifremloop(). * This routine does actual work. @@ -2377,7 +2380,7 @@ in6_purgeaddr(struct ifaddr *ifa) struct in6_ifaddr *ia = (struct in6_ifaddr *)ifa; struct in6_multi_mship *imm; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); /* stop DAD processing */ nd6_dad_stop(ifa); @@ -2435,7 +2438,7 @@ in6_unlink_ifa(struct in6_ifaddr *ia, struct ifnet *ifp) struct ifaddr *ifa; int unlinked; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); ifa = &ia->ia_ifa; IFA_ADDREF(ifa); @@ -2532,7 +2535,7 @@ in6_purgeif(struct ifnet *ifp) if (ifp == NULL) return; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); lck_rw_lock_exclusive(&in6_ifaddr_rwlock); ia = in6_ifaddrs; @@ -3466,6 +3469,15 @@ in6if_do_dad( (IFEF_IPV6_ND6ALT|IFEF_LOCALNET_PRIVATE|IFEF_DIRECTLINK)) return (0); + if (ifp->if_subfamily == IFNET_SUBFAMILY_IPSEC || + ifp->if_subfamily == IFNET_SUBFAMILY_UTUN) { + /* + * Ignore DAD for tunneling virtual interfaces, which get + * their IPv6 address explicitly assigned. + */ + return (0); + } + switch (ifp->if_type) { #if IFT_DUMMY case IFT_DUMMY: @@ -3680,21 +3692,23 @@ in6_post_msg(struct ifnet *ifp, u_int32_t event_code, struct in6_ifaddr *ifa, ev_msg.kev_subclass = KEV_INET6_SUBCLASS; ev_msg.event_code = event_code; - IFA_LOCK(&ifa->ia_ifa); - in6_event_data.ia_addr = ifa->ia_addr; - in6_event_data.ia_net = ifa->ia_net; - in6_event_data.ia_dstaddr = ifa->ia_dstaddr; - in6_event_data.ia_prefixmask = ifa->ia_prefixmask; - in6_event_data.ia_plen = ifa->ia_plen; - in6_event_data.ia6_flags = (u_int32_t)ifa->ia6_flags; - - /* retrieve time as calendar time (last arg is 1) */ - in6ifa_getlifetime(ifa, &ia6_lt, 1); - in6_event_data.ia_lifetime.ia6t_expire = ia6_lt.ia6t_expire; - in6_event_data.ia_lifetime.ia6t_preferred = ia6_lt.ia6t_preferred; - in6_event_data.ia_lifetime.ia6t_vltime = ia6_lt.ia6t_vltime; - in6_event_data.ia_lifetime.ia6t_pltime = ia6_lt.ia6t_pltime; - IFA_UNLOCK(&ifa->ia_ifa); + if (ifa) { + IFA_LOCK(&ifa->ia_ifa); + in6_event_data.ia_addr = ifa->ia_addr; + in6_event_data.ia_net = ifa->ia_net; + in6_event_data.ia_dstaddr = ifa->ia_dstaddr; + in6_event_data.ia_prefixmask = ifa->ia_prefixmask; + in6_event_data.ia_plen = ifa->ia_plen; + in6_event_data.ia6_flags = (u_int32_t)ifa->ia6_flags; + + /* retrieve time as calendar time (last arg is 1) */ + in6ifa_getlifetime(ifa, &ia6_lt, 1); + in6_event_data.ia_lifetime.ia6t_expire = ia6_lt.ia6t_expire; + in6_event_data.ia_lifetime.ia6t_preferred = ia6_lt.ia6t_preferred; + in6_event_data.ia_lifetime.ia6t_vltime = ia6_lt.ia6t_vltime; + in6_event_data.ia_lifetime.ia6t_pltime = ia6_lt.ia6t_pltime; + IFA_UNLOCK(&ifa->ia_ifa); + } if (ifp != NULL) { (void) strlcpy(&in6_event_data.link_data.if_name[0], @@ -3923,13 +3937,12 @@ in6_getconnids(struct socket *so, sae_associd_t aid, uint32_t *cnt, /* * Handle SIOCGCONNINFO ioctl for PF_INET6 domain. */ -static int +int in6_getconninfo(struct socket *so, sae_connid_t cid, uint32_t *flags, uint32_t *ifindex, int32_t *soerror, user_addr_t src, socklen_t *src_len, user_addr_t dst, socklen_t *dst_len, uint32_t *aux_type, user_addr_t aux_data, uint32_t *aux_len) { -#pragma unused(aux_data) struct in6pcb *in6p = sotoin6pcb(so); struct sockaddr_in6 sin6; struct ifnet *ifp = NULL; @@ -3997,8 +4010,6 @@ in6_getconninfo(struct socket *so, sae_connid_t cid, uint32_t *flags, } } - *aux_type = 0; - *aux_len = 0; if (SOCK_PROTO(so) == IPPROTO_TCP) { struct conninfo_tcp tcp_ci; @@ -4016,6 +4027,9 @@ in6_getconninfo(struct socket *so, sae_connid_t cid, uint32_t *flags, *aux_len = copy_len; } } + } else { + *aux_type = 0; + *aux_len = 0; } out: @@ -4053,3 +4067,528 @@ in6ioctl_cassert(void) ; } } + +struct in6_llentry { + struct llentry base; +}; + +#define IN6_LLTBL_DEFAULT_HSIZE 32 +#define IN6_LLTBL_HASH(k, h) \ + ((((((((k) >> 8) ^ (k)) >> 8) ^ (k)) >> 8) ^ (k)) & ((h) - 1)) + +/* + * Do actual deallocation of @lle. + */ +static void +in6_lltable_destroy_lle_unlocked(struct llentry *lle) +{ + LLE_LOCK_DESTROY(lle); + LLE_REQ_DESTROY(lle); + FREE(lle, M_LLTABLE); +} + +/* + * Called by LLE_FREE_LOCKED when number of references + * drops to zero. + */ +static void +in6_lltable_destroy_lle(struct llentry *lle) +{ + LLE_WUNLOCK(lle); + /* XXX TBD */ + //thread_call_free(lle->lle_timer); + in6_lltable_destroy_lle_unlocked(lle); +} + + +static struct llentry * +in6_lltable_new(const struct in6_addr *addr6, u_int flags) +{ +#pragma unused(flags) + struct in6_llentry *lle; + + MALLOC(lle, struct in6_llentry *, sizeof(struct in6_llentry), M_LLTABLE, M_NOWAIT | M_ZERO); + if (lle == NULL) /* NB: caller generates msg */ + return NULL; + + lle->base.r_l3addr.addr6 = *addr6; + lle->base.lle_refcnt = 1; + lle->base.lle_free = in6_lltable_destroy_lle; + LLE_LOCK_INIT(&lle->base); + LLE_REQ_INIT(&lle->base); +#if 0 + /* XXX TBD */ + lle->base.lle_timer = thread_call_allocate(nd6_llinfo_timer, lle); + + if (lle->base.lle_timer == NULL) { + printf("lle_timer thread call could not be allocated.\n"); + LLE_LOCK_DESTROY(&lle->base); + LLE_REQ_DESTROY(&lle->base); + FREE(lle, M_LLTABLE); + return NULL; + } +#endif + return (&lle->base); +} + +static int +in6_lltable_match_prefix(const struct sockaddr *saddr, + const struct sockaddr *smask, u_int flags, struct llentry *lle) +{ + const struct in6_addr *addr, *mask, *lle_addr; + + addr = &((const struct sockaddr_in6 *)(const void *)saddr)->sin6_addr; + mask = &((const struct sockaddr_in6 *)(const void *)smask)->sin6_addr; + lle_addr = &lle->r_l3addr.addr6; + + if (IN6_ARE_MASKED_ADDR_EQUAL(lle_addr, addr, mask) == 0) + return (0); + + if (lle->la_flags & LLE_IFADDR) { + /* + * Delete LLE_IFADDR records IFF address & flag matches. + * Note that addr is the interface address within prefix + * being matched. + */ + if (IN6_ARE_ADDR_EQUAL(addr, lle_addr) && + (flags & LLE_STATIC) != 0) + return (1); + return (0); + } + + /* flags & LLE_STATIC means deleting both dynamic and static entries */ + if ((flags & LLE_STATIC) || !(lle->la_flags & LLE_STATIC)) + return (1); + + return (0); +} + +static void +in6_lltable_free_entry(struct lltable *llt, struct llentry *lle) +{ + struct ifnet *ifp; + + LLE_WLOCK_ASSERT(lle); + KASSERT(llt != NULL, ("lltable is NULL")); + + /* Unlink entry from table */ + if ((lle->la_flags & LLE_LINKED) != 0) { + ifp = llt->llt_ifp; + if_afdata_wlock_assert(ifp, llt->llt_af); + lltable_unlink_entry(llt, lle); + } + +#if 0 + /* XXX TBD */ + if (thread_call_cancel(lle->lle_timer) == TRUE) + LLE_REMREF(lle); +#endif + llentry_free(lle); +} + +static int +in6_lltable_rtcheck(struct ifnet *ifp, + u_int flags, const struct sockaddr *l3addr) +{ +#pragma unused(flags) + struct rtentry *rt; + + KASSERT(l3addr->sa_family == AF_INET6, + ("sin_family %d", l3addr->sa_family)); + /* XXX rtalloc1 should take a const param */ + rt = rtalloc1(__DECONST(struct sockaddr *, l3addr), 0, 0); + if (rt == NULL || (rt->rt_flags & RTF_GATEWAY) || rt->rt_ifp != ifp) { + struct ifaddr *ifa; + /* + * Create an ND6 cache for an IPv6 neighbor + * that is not covered by our own prefix. + */ + /* XXX ifaof_ifpforaddr should take a const param */ + ifa = ifaof_ifpforaddr(__DECONST(struct sockaddr *, l3addr), ifp); + if (ifa != NULL) { + IFA_REMREF(ifa); + if (rt != NULL) + rtfree(rt); + return 0; + } + log(LOG_INFO, "IPv6 address: \"%s\" is not on the network\n", + ip6_sprintf(&((const struct sockaddr_in6 *)(const void *)l3addr)->sin6_addr)); + if (rt != NULL) + rtfree(rt); + return EINVAL; + } + rtfree(rt); + return 0; +} + +static inline uint32_t +in6_lltable_hash_dst(const struct in6_addr *dst, uint32_t hsize) +{ + return (IN6_LLTBL_HASH(dst->s6_addr32[3], hsize)); +} + +static uint32_t +in6_lltable_hash(const struct llentry *lle, uint32_t hsize) +{ + return (in6_lltable_hash_dst(&lle->r_l3addr.addr6, hsize)); +} + +static void +in6_lltable_fill_sa_entry(const struct llentry *lle, struct sockaddr *sa) +{ + struct sockaddr_in6 *sin6; + + sin6 = (struct sockaddr_in6 *)(void *)sa; + bzero(sin6, sizeof(*sin6)); + sin6->sin6_family = AF_INET6; + sin6->sin6_len = sizeof(*sin6); + sin6->sin6_addr = lle->r_l3addr.addr6; +} + +static inline struct llentry * +in6_lltable_find_dst(struct lltable *llt, const struct in6_addr *dst) +{ + struct llentry *lle; + struct llentries *lleh; + u_int hashidx; + + hashidx = in6_lltable_hash_dst(dst, llt->llt_hsize); + lleh = &llt->lle_head[hashidx]; + LIST_FOREACH(lle, lleh, lle_next) { + if (lle->la_flags & LLE_DELETED) + continue; + if (IN6_ARE_ADDR_EQUAL(&lle->r_l3addr.addr6, dst)) + break; + } + + return (lle); +} + +static void +in6_lltable_delete_entry(struct lltable *llt, struct llentry *lle) +{ +#pragma unused(llt) + lle->la_flags |= LLE_DELETED; + EVENTHANDLER_INVOKE(NULL, lle_event, lle, LLENTRY_DELETED); +#ifdef DIAGNOSTIC + log(LOG_INFO, "ifaddr cache = %p is deleted\n", lle); +#endif + llentry_free(lle); +} + +static struct llentry * +in6_lltable_alloc(struct lltable *llt, u_int flags, + const struct sockaddr *l3addr) +{ + const struct sockaddr_in6 *sin6 = (const struct sockaddr_in6 *)(const void *)l3addr; + struct ifnet *ifp = llt->llt_ifp; + struct llentry *lle; + + KASSERT(l3addr->sa_family == AF_INET6, + ("sin_family %d", l3addr->sa_family)); + + /* + * A route that covers the given address must have + * been installed 1st because we are doing a resolution, + * verify this. + */ + if (!(flags & LLE_IFADDR) && + in6_lltable_rtcheck(ifp, flags, l3addr) != 0) + return (NULL); + + lle = in6_lltable_new(&sin6->sin6_addr, flags); + if (lle == NULL) { + log(LOG_INFO, "lla_lookup: new lle malloc failed\n"); + return (NULL); + } + lle->la_flags = flags; + if ((flags & LLE_IFADDR) == LLE_IFADDR) { + lltable_set_entry_addr(ifp, lle, LLADDR(SDL(ifp->if_lladdr->ifa_addr))); + lle->la_flags |= LLE_STATIC; + } + + if ((lle->la_flags & LLE_STATIC) != 0) + lle->ln_state = ND6_LLINFO_REACHABLE; + + return (lle); +} + +static struct llentry * +in6_lltable_lookup(struct lltable *llt, u_int flags, + const struct sockaddr *l3addr) +{ + const struct sockaddr_in6 *sin6 = (const struct sockaddr_in6 *)(const void *)l3addr; + struct llentry *lle; + + IF_AFDATA_LOCK_ASSERT(llt->llt_ifp, llt->llt_af); + KASSERT(l3addr->sa_family == AF_INET6, + ("sin_family %d", l3addr->sa_family)); + + lle = in6_lltable_find_dst(llt, &sin6->sin6_addr); + + if (lle == NULL) + return (NULL); + + KASSERT((flags & (LLE_UNLOCKED|LLE_EXCLUSIVE)) != + (LLE_UNLOCKED|LLE_EXCLUSIVE),("wrong lle request flags: 0x%X", + flags)); + + if (flags & LLE_UNLOCKED) + return (lle); + + if (flags & LLE_EXCLUSIVE) + LLE_WLOCK(lle); + else + LLE_RLOCK(lle); + return (lle); +} + +static int +in6_lltable_dump_entry(struct lltable *llt, struct llentry *lle, + struct sysctl_req *wr) +{ + struct ifnet *ifp = llt->llt_ifp; + /* XXX stack use */ + struct { + struct rt_msghdr rtm; + struct sockaddr_in6 sin6; + /* + * ndp.c assumes that sdl is word aligned + */ +#ifdef __LP64__ + uint32_t pad; +#endif + struct sockaddr_dl sdl; + } ndpc; + struct sockaddr_dl *sdl; + int error; + + bzero(&ndpc, sizeof(ndpc)); + /* skip deleted entries */ + if ((lle->la_flags & LLE_DELETED) == LLE_DELETED) + return (0); + /* Skip if jailed and not a valid IP of the prison. */ + lltable_fill_sa_entry(lle, + (struct sockaddr *)&ndpc.sin6); + /* + * produce a msg made of: + * struct rt_msghdr; + * struct sockaddr_in6 (IPv6) + * struct sockaddr_dl; + */ + ndpc.rtm.rtm_msglen = sizeof(ndpc); + ndpc.rtm.rtm_version = RTM_VERSION; + ndpc.rtm.rtm_type = RTM_GET; + ndpc.rtm.rtm_flags = RTF_UP; + ndpc.rtm.rtm_addrs = RTA_DST | RTA_GATEWAY; + + /* publish */ + if (lle->la_flags & LLE_PUB) + ndpc.rtm.rtm_flags |= RTF_ANNOUNCE; + sdl = &ndpc.sdl; + sdl->sdl_family = AF_LINK; + sdl->sdl_len = sizeof(*sdl); + sdl->sdl_index = ifp->if_index; + sdl->sdl_type = ifp->if_type; + if ((lle->la_flags & LLE_VALID) == LLE_VALID) { + sdl->sdl_alen = ifp->if_addrlen; + bcopy(&lle->ll_addr, LLADDR(sdl), ifp->if_addrlen); + } else { + sdl->sdl_alen = 0; + bzero(LLADDR(sdl), ifp->if_addrlen); + } + if (lle->la_expire != 0) { + clock_sec_t secs; + clock_usec_t usecs; + + clock_get_calendar_microtime(&secs, &usecs); + ndpc.rtm.rtm_rmx.rmx_expire = lle->la_expire + + lle->lle_remtime / hz + + secs - net_uptime(); + } + ndpc.rtm.rtm_flags |= (RTF_HOST | RTF_LLDATA); + if (lle->la_flags & LLE_STATIC) + ndpc.rtm.rtm_flags |= RTF_STATIC; + if (lle->la_flags & LLE_IFADDR) + ndpc.rtm.rtm_flags |= RTF_PINNED; + if (lle->ln_router != 0) + ndpc.rtm.rtm_flags |= RTF_GATEWAY; + ndpc.rtm.rtm_rmx.rmx_pksent = lle->la_asked; + /* Store state in rmx_weight value */ + ndpc.rtm.rtm_rmx.rmx_state = lle->ln_state; + ndpc.rtm.rtm_index = ifp->if_index; + error = SYSCTL_OUT(wr, &ndpc, sizeof(ndpc)); + + return (error); +} + +struct lltable * +in6_lltattach(struct ifnet *ifp) +{ + struct lltable *llt; + + llt = lltable_allocate_htbl(IN6_LLTBL_DEFAULT_HSIZE); + llt->llt_af = AF_INET6; + llt->llt_ifp = ifp; + + llt->llt_lookup = in6_lltable_lookup; + llt->llt_alloc_entry = in6_lltable_alloc; + llt->llt_delete_entry = in6_lltable_delete_entry; + llt->llt_dump_entry = in6_lltable_dump_entry; + llt->llt_hash = in6_lltable_hash; + llt->llt_fill_sa_entry = in6_lltable_fill_sa_entry; + llt->llt_free_entry = in6_lltable_free_entry; + llt->llt_match_prefix = in6_lltable_match_prefix; + lltable_link(llt); + + return (llt); +} + +void +in6_ip6_to_sockaddr(const struct in6_addr *ip6, u_int16_t port, + struct sockaddr_in6 *sin6, u_int32_t maxlen) +{ + if (maxlen < sizeof(struct sockaddr_in6)) { + return; + } + + *sin6 = (struct sockaddr_in6) { + .sin6_family = AF_INET6, + .sin6_len = sizeof(*sin6), + .sin6_port = port, + .sin6_addr = *ip6, + }; + + if (IN6_IS_SCOPE_EMBED(&sin6->sin6_addr)) + { + sin6->sin6_scope_id = ntohs(sin6->sin6_addr.s6_addr16[1]); + sin6->sin6_addr.s6_addr16[1] = 0; + } +} + +/* IPv6 events */ +struct in6_event { + in6_evhdlr_code_t in6_event_code; + struct ifnet *in6_ifp; + struct in6_addr in6_address; + uint32_t val; +}; + +struct in6_event2kev in6_event2kev_array[IN6_EVENT_MAX] = { +{ + .in6_event_code = IN6_ADDR_MARKED_DUPLICATED, + .in6_event_kev_subclass = KEV_ND6_SUBCLASS, + .in6_event_kev_code = KEV_ND6_DAD_FAILURE, + .in6_event_str = "IN6_ADDR_MARKED_DUPLICATED", +}, +{ + .in6_event_code = IN6_ADDR_MARKED_DETACHED, + .in6_event_kev_subclass = KEV_ND6_SUBCLASS, + .in6_event_kev_code = KEV_ND6_ADDR_DETACHED, + .in6_event_str = "IN6_ADDR_MARKED_DETACHED", +}, +{ + .in6_event_code = IN6_ADDR_MARKED_DEPRECATED, + .in6_event_kev_subclass = KEV_ND6_SUBCLASS, + .in6_event_kev_code = KEV_ND6_ADDR_DEPRECATED, + .in6_event_str = "IN6_ADDR_MARKED_DEPRECATED", +}, +{ + .in6_event_code = IN6_NDP_RTR_EXPIRY, + .in6_event_kev_subclass = KEV_ND6_SUBCLASS, + .in6_event_kev_code = KEV_ND6_RTR_EXPIRED, + .in6_event_str = "IN6_NDP_RTR_EXPIRY", +}, +{ + .in6_event_code = IN6_NDP_PFX_EXPIRY, + .in6_event_kev_subclass = KEV_ND6_SUBCLASS, + .in6_event_kev_code = KEV_ND6_PFX_EXPIRED, + .in6_event_str = "IN6_NDP_PFX_EXPIRY", +}, +{ + .in6_event_code = IN6_NDP_ADDR_EXPIRY, + .in6_event_kev_subclass = KEV_ND6_SUBCLASS, + .in6_event_kev_code = KEV_ND6_ADDR_EXPIRED, + .in6_event_str = "IN6_NDP_ADDR_EXPIRY", +}, +}; + +void +in6_eventhdlr_callback(struct eventhandler_entry_arg arg0 __unused, + in6_evhdlr_code_t in6_ev_code, struct ifnet *ifp, + struct in6_addr *p_addr6, uint32_t val) +{ + struct kev_msg ev_msg; + struct kev_nd6_event nd6_event; + + bzero(&ev_msg, sizeof(ev_msg)); + bzero(&nd6_event, sizeof(nd6_event)); + + nd6log0((LOG_INFO, "%s Event %s received for %s\n", + __func__, in6_event2kev_array[in6_ev_code].in6_event_str, + ip6_sprintf(p_addr6))); + + ev_msg.vendor_code = KEV_VENDOR_APPLE; + ev_msg.kev_class = KEV_NETWORK_CLASS; + ev_msg.kev_subclass = + in6_event2kev_array[in6_ev_code].in6_event_kev_subclass; + ev_msg.event_code = + in6_event2kev_array[in6_ev_code].in6_event_kev_code; + + nd6_event.link_data.if_family = ifp->if_family; + nd6_event.link_data.if_unit = ifp->if_unit; + strlcpy(nd6_event.link_data.if_name, ifp->if_name, + sizeof(nd6_event.link_data.if_name)); + + VERIFY(p_addr6 != NULL); + bcopy(p_addr6, &nd6_event.in6_address, + sizeof(nd6_event.in6_address)); + nd6_event.val = val; + + ev_msg.dv[0].data_ptr = &nd6_event; + ev_msg.dv[0].data_length = sizeof(nd6_event); + + kev_post_msg(&ev_msg); +} + +static void +in6_event_callback(void *arg) +{ + struct in6_event *p_in6_ev = (struct in6_event *)arg; + + EVENTHANDLER_INVOKE(&in6_evhdlr_ctxt, in6_event, + p_in6_ev->in6_event_code, p_in6_ev->in6_ifp, + &p_in6_ev->in6_address, p_in6_ev->val); +} + +struct in6_event_nwk_wq_entry +{ + struct nwk_wq_entry nwk_wqe; + struct in6_event in6_ev_arg; +}; + +void +in6_event_enqueue_nwk_wq_entry(in6_evhdlr_code_t in6_event_code, + struct ifnet *ifp, struct in6_addr *p_addr6, + uint32_t val) +{ + struct in6_event_nwk_wq_entry *p_in6_ev = NULL; + + MALLOC(p_in6_ev, struct in6_event_nwk_wq_entry *, + sizeof(struct in6_event_nwk_wq_entry), + M_NWKWQ, M_WAITOK | M_ZERO); + + p_in6_ev->nwk_wqe.func = in6_event_callback; + p_in6_ev->nwk_wqe.is_arg_managed = TRUE; + p_in6_ev->nwk_wqe.arg = &p_in6_ev->in6_ev_arg; + + p_in6_ev->in6_ev_arg.in6_event_code = in6_event_code; + p_in6_ev->in6_ev_arg.in6_ifp = ifp; + if (p_addr6 != NULL) { + bcopy(p_addr6, &p_in6_ev->in6_ev_arg.in6_address, + sizeof(p_in6_ev->in6_ev_arg.in6_address)); + } + p_in6_ev->in6_ev_arg.val = val; + + nwk_wq_enqueue((struct nwk_wq_entry*)p_in6_ev); +} diff --git a/bsd/netinet6/in6.h b/bsd/netinet6/in6.h index b09c026b4..3fe1484e2 100644 --- a/bsd/netinet6/in6.h +++ b/bsd/netinet6/in6.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2015 Apple Inc. All rights reserved. + * Copyright (c) 2008-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -465,6 +465,9 @@ struct route_in6_old { #endif /* PRIVATE */ #ifdef BSD_KERNEL_PRIVATE +#include <net/if_llatbl.h> +#include <sys/eventhandler.h> + /* * IP6 route structure * @@ -479,6 +482,8 @@ struct route_in6 { * to a 'struct route *'. */ struct rtentry *ro_rt; + struct llentry *ro_lle; + struct ifaddr *ro_srcia; uint32_t ro_flags; /* route flags */ struct sockaddr_in6 ro_dst; @@ -846,13 +851,19 @@ struct cmsghdr; struct mbuf; struct ifnet; struct in6_aliasreq; +struct lltable; +extern struct lltable * in6_lltattach(struct ifnet *ifp); extern uint16_t in6_pseudo(const struct in6_addr *, const struct in6_addr *, uint32_t); extern u_int16_t inet6_cksum(struct mbuf *, uint32_t, uint32_t, uint32_t); +extern u_int16_t inet6_cksum_buffer(const uint8_t *, uint32_t, uint32_t, + uint32_t); -#define in6_cksum(_m, _n, _o, _l) \ +#define in6_cksum(_m, _n, _o, _l) \ inet6_cksum(_m, _n, _o, _l) +#define in6_cksum_buffer(_b, _n, _o, _l) \ + inet6_cksum_buffer(_b, _n, _o, _l) extern int in6_addrscope(struct in6_addr *); extern struct in6_ifaddr *in6_ifawithscope(struct ifnet *, struct in6_addr *); @@ -873,6 +884,54 @@ extern uint32_t in6_finalize_cksum(struct mbuf *, uint32_t, int32_t, ((void) in6_finalize_cksum(_m, 0, 0, -1, CSUM_DELAY_IPV6_DATA)) #define in6_delayed_cksum_offset(_m, _o, _s, _p) \ ((void) in6_finalize_cksum(_m, _o, _s, _p, CSUM_DELAY_IPV6_DATA)) + +/* IPv6 protocol events */ +extern struct eventhandler_lists_ctxt in6_evhdlr_ctxt; + +/* + * XXX Avoid reordering the enum values below. + * If the order is changed, please make sure + * in6_event2kev_array is also changed to reflect the + * change in order of the enums + */ +typedef enum { + /* Address events */ + /* + * XXX To avoid duplicacy and also for correctness + * only report these for link local and stable addresses + * NOTE: Link local address can never be marked detached + * or duplicated. + */ + IN6_ADDR_MARKED_DUPLICATED, + IN6_ADDR_MARKED_DETACHED, + IN6_ADDR_MARKED_DEPRECATED, + + /* Expiry events */ + IN6_NDP_RTR_EXPIRY, + IN6_NDP_PFX_EXPIRY, + IN6_NDP_ADDR_EXPIRY, + + /* XXX DNS expiry needs to be handled by user-space */ + /* MAX */ + IN6_EVENT_MAX, +} in6_evhdlr_code_t; + +struct in6_event2kev { + in6_evhdlr_code_t in6_event_code; + uint32_t in6_event_kev_subclass; + uint32_t in6_event_kev_code; + const char *in6_event_str; +}; +extern struct in6_event2kev in6_event2kev_array[]; + +extern void in6_eventhdlr_callback(struct eventhandler_entry_arg, in6_evhdlr_code_t, + struct ifnet *, struct in6_addr *, uint32_t); +extern void in6_event_enqueue_nwk_wq_entry(in6_evhdlr_code_t, + struct ifnet *, struct in6_addr *, uint32_t); + +typedef void (*in6_event_fn) (struct eventhandler_entry_arg, in6_evhdlr_code_t, + struct ifnet *, struct in6_addr *, uint32_t); +EVENTHANDLER_DECLARE(in6_event, in6_event_fn); #endif /* BSD_KERNEL_PRIVATE */ #ifdef KERNEL_PRIVATE diff --git a/bsd/netinet6/in6_cga.c b/bsd/netinet6/in6_cga.c index 5174dfdf0..981df0a18 100644 --- a/bsd/netinet6/in6_cga.c +++ b/bsd/netinet6/in6_cga.c @@ -81,8 +81,11 @@ static struct in6_cga_singleton in6_cga = { static void in6_cga_node_lock_assert(int owned) { +#if !MACH_ASSERT +#pragma unused(owned) +#endif VERIFY(in6_cga.cga_initialized); - lck_mtx_assert(&in6_cga.cga_mutex, owned); + LCK_MTX_ASSERT(&in6_cga.cga_mutex, owned); } static boolean_t diff --git a/bsd/netinet6/in6_cksum.c b/bsd/netinet6/in6_cksum.c index 2bea8cbcc..394ea35b2 100644 --- a/bsd/netinet6/in6_cksum.c +++ b/bsd/netinet6/in6_cksum.c @@ -232,3 +232,47 @@ inet6_cksum(struct mbuf *m, uint32_t nxt, uint32_t off, uint32_t len) return (~sum & 0xffff); } + +/* + * buffer MUST contain at least an IPv6 header, if nxt is specified; + * nxt is the upper layer protocol number; + * off is an offset where TCP/UDP/ICMP6 header starts; + * len is a total length of a transport segment (e.g. TCP header + TCP payload) + */ +u_int16_t +inet6_cksum_buffer(const uint8_t *buffer, uint32_t nxt, uint32_t off, + uint32_t len) +{ + uint32_t sum; + + if (off >= len) + panic("%s: off (%d) >= len (%d)", __func__, off, len); + + sum = b_sum16(&((const uint8_t *)buffer)[off], len); + + if (nxt != 0) { + const struct ip6_hdr *ip6; + unsigned char buf[sizeof (*ip6)] __attribute__((aligned(8))); + + /* + * In case the IPv6 header is not contiguous, or not 32-bit + * aligned, copy it to a local buffer. Note here that we + * expect the data pointer to point to the IPv6 header. + */ + if (!IP6_HDR_ALIGNED_P(buffer)) { + memcpy(buf, buffer, sizeof (*ip6)); + ip6 = (const struct ip6_hdr *)(const void *)buf; + } else { + ip6 = (const struct ip6_hdr *)buffer; + } + + /* add pseudo header checksum */ + sum += in6_pseudo(&ip6->ip6_src, &ip6->ip6_dst, + htonl(nxt + len)); + + /* fold in carry bits */ + ADDCARRY(sum); + } + + return (~sum & 0xffff); +} diff --git a/bsd/netinet6/in6_ifattach.c b/bsd/netinet6/in6_ifattach.c index 804a77d8e..86759c202 100644 --- a/bsd/netinet6/in6_ifattach.c +++ b/bsd/netinet6/in6_ifattach.c @@ -72,6 +72,7 @@ #include <net/if_types.h> #include <net/route.h> #include <net/kpi_protocol.h> +#include <net/if_llatbl.h> #include <netinet/in.h> #include <netinet/in_var.h> @@ -710,7 +711,7 @@ in6_ifattach_prelim(struct ifnet *ifp) * (previously, this was a silent error.) */ if ((ifp->if_flags & IFF_MULTICAST) == 0) { - nd6log((LOG_INFO, "in6_ifattach: ", + nd6log0((LOG_INFO, "in6_ifattach: ", "%s is not multicast capable, IPv6 not enabled\n", if_name(ifp))); return (EINVAL); @@ -732,6 +733,7 @@ skipmcast: pbuf = (void **)((intptr_t)base - sizeof(void *)); *pbuf = ext; ifp->if_inet6data = base; + IN6_IFEXTRA(ifp)->ii_llt = in6_lltattach(ifp); VERIFY(IS_P2ALIGNED(ifp->if_inet6data, sizeof(uint64_t))); } else { /* @@ -744,6 +746,11 @@ skipmcast: sizeof(IN6_IFEXTRA(ifp)->icmp6_ifstat)); bzero(&IN6_IFEXTRA(ifp)->in6_ifstat, sizeof(IN6_IFEXTRA(ifp)->in6_ifstat)); + IN6_IFEXTRA(ifp)->netsig_len = 0; + bzero(&IN6_IFEXTRA(ifp)->netsig, + sizeof(IN6_IFEXTRA(ifp)->netsig)); + bzero(IN6_IFEXTRA(ifp)->nat64_prefixes, sizeof(IN6_IFEXTRA(ifp)->nat64_prefixes)); + /* XXX TBD Purge the layer two table */ /* * XXX When recycling, nd_ifinfo gets initialized, other * than the lock, inside nd6_ifattach @@ -995,11 +1002,14 @@ in6_ifdetach(struct ifnet *ifp) struct in6_multi_mship *imm; int unlinked; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); /* remove neighbor management table */ nd6_purge(ifp); + if (LLTABLE6(ifp)) + lltable_free(LLTABLE6(ifp)); + /* nuke any of IPv6 addresses we have */ lck_rw_lock_exclusive(&in6_ifaddr_rwlock); ia = in6_ifaddrs; diff --git a/bsd/netinet6/in6_mcast.c b/bsd/netinet6/in6_mcast.c index 1fc7aca03..778bafa8d 100644 --- a/bsd/netinet6/in6_mcast.c +++ b/bsd/netinet6/in6_mcast.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2016 Apple Inc. All rights reserved. + * Copyright (c) 2010-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -80,6 +80,7 @@ #include <net/if.h> #include <net/if_dl.h> +#include <net/net_api_stats.h> #include <net/route.h> #include <netinet/in.h> @@ -96,17 +97,6 @@ #include <netinet6/mld6_var.h> #include <netinet6/scope6_var.h> -#ifndef __SOCKUNION_DECLARED -union sockunion { - struct sockaddr_storage ss; - struct sockaddr sa; - struct sockaddr_dl sdl; - struct sockaddr_in6 sin6; -}; -typedef union sockunion sockunion_t; -#define __SOCKUNION_DECLARED -#endif /* __SOCKUNION_DECLARED */ - static void im6f_commit(struct in6_mfilter *); static int im6f_get_source(struct in6_mfilter *imf, const struct sockaddr_in6 *psin, @@ -119,10 +109,10 @@ static void im6f_rollback(struct in6_mfilter *); static void im6f_reap(struct in6_mfilter *); static int im6o_grow(struct ip6_moptions *, size_t); static size_t im6o_match_group(const struct ip6_moptions *, - const struct ifnet *, const struct sockaddr *); + const struct ifnet *, const struct sockaddr_in6 *); static struct in6_msource * - im6o_match_source(const struct ip6_moptions *, const size_t, - const struct sockaddr *); + im6o_match_source(const struct ip6_moptions *, + const size_t, const struct sockaddr_in6 *); static void im6s_merge(struct ip6_msource *ims, const struct in6_msource *lims, const int rollback); static int in6_mc_get(struct ifnet *, const struct in6_addr *, @@ -339,7 +329,7 @@ im6o_grow(struct ip6_moptions *imo, size_t newmax) */ static size_t im6o_match_group(const struct ip6_moptions *imo, const struct ifnet *ifp, - const struct sockaddr *group) + const struct sockaddr_in6 *group) { const struct sockaddr_in6 *gsin6; struct in6_multi *pinm; @@ -348,7 +338,7 @@ im6o_match_group(const struct ip6_moptions *imo, const struct ifnet *ifp, IM6O_LOCK_ASSERT_HELD(__DECONST(struct ip6_moptions *, imo)); - gsin6 = (struct sockaddr_in6 *)(uintptr_t)(size_t)group; + gsin6 = group; /* The im6o_membership array may be lazy allocated. */ if (imo->im6o_membership == NULL || imo->im6o_num_memberships == 0) @@ -387,16 +377,16 @@ im6o_match_group(const struct ip6_moptions *imo, const struct ifnet *ifp, */ static struct in6_msource * im6o_match_source(const struct ip6_moptions *imo, const size_t gidx, - const struct sockaddr *src) + const struct sockaddr_in6 *src) { struct ip6_msource find; struct in6_mfilter *imf; struct ip6_msource *ims; - const sockunion_t *psa; + const struct sockaddr_in6 *psa; IM6O_LOCK_ASSERT_HELD(__DECONST(struct ip6_moptions *, imo)); - VERIFY(src->sa_family == AF_INET6); + VERIFY(src->sin6_family == AF_INET6); VERIFY(gidx != (size_t)-1 && gidx < imo->im6o_num_memberships); /* The im6o_mfilters array may be lazy allocated. */ @@ -404,8 +394,8 @@ im6o_match_source(const struct ip6_moptions *imo, const size_t gidx, return (NULL); imf = &imo->im6o_mfilters[gidx]; - psa = (sockunion_t *)(uintptr_t)(size_t)src; - find.im6s_addr = psa->sin6.sin6_addr; + psa = src; + find.im6s_addr = psa->sin6_addr; in6_clearscope(&find.im6s_addr); /* XXX */ ims = RB_FIND(ip6_msource_tree, &imf->im6f_sources, &find); @@ -420,7 +410,7 @@ im6o_match_source(const struct ip6_moptions *imo, const size_t gidx, */ int im6o_mc_filter(const struct ip6_moptions *imo, const struct ifnet *ifp, - const struct sockaddr *group, const struct sockaddr *src) + const struct sockaddr_in6 *group, const struct sockaddr_in6 *src) { size_t gidx; struct in6_msource *ims; @@ -995,7 +985,7 @@ im6s_merge(struct ip6_msource *ims, const struct in6_msource *lims, static int in6m_merge(struct in6_multi *inm, /*const*/ struct in6_mfilter *imf) { - struct ip6_msource *ims, *nims; + struct ip6_msource *ims, *nims = NULL; struct in6_msource *lims; int schanged, error; int nsrc0, nsrc1; @@ -1403,7 +1393,7 @@ static int in6p_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) { struct group_source_req gsr; - sockunion_t *gsa, *ssa; + struct sockaddr_in6 *gsa, *ssa; struct ifnet *ifp; struct in6_mfilter *imf; struct ip6_moptions *imo; @@ -1420,8 +1410,8 @@ in6p_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) doblock = 0; memset(&gsr, 0, sizeof(struct group_source_req)); - gsa = (sockunion_t *)&gsr.gsr_group; - ssa = (sockunion_t *)&gsr.gsr_source; + gsa = (struct sockaddr_in6 *)&gsr.gsr_group; + ssa = (struct sockaddr_in6 *)&gsr.gsr_source; switch (sopt->sopt_name) { case MCAST_BLOCK_SOURCE: @@ -1432,12 +1422,12 @@ in6p_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) if (error) return (error); - if (gsa->sin6.sin6_family != AF_INET6 || - gsa->sin6.sin6_len != sizeof(struct sockaddr_in6)) + if (gsa->sin6_family != AF_INET6 || + gsa->sin6_len != sizeof(struct sockaddr_in6)) return (EINVAL); - if (ssa->sin6.sin6_family != AF_INET6 || - ssa->sin6.sin6_len != sizeof(struct sockaddr_in6)) + if (ssa->sin6_family != AF_INET6 || + ssa->sin6_len != sizeof(struct sockaddr_in6)) return (EINVAL); ifnet_head_lock_shared(); @@ -1463,10 +1453,10 @@ in6p_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) return (EOPNOTSUPP); } - if (!IN6_IS_ADDR_MULTICAST(&gsa->sin6.sin6_addr)) + if (!IN6_IS_ADDR_MULTICAST(&gsa->sin6_addr)) return (EINVAL); - (void) in6_setscope(&gsa->sin6.sin6_addr, ifp, NULL); + (void) in6_setscope(&gsa->sin6_addr, ifp, NULL); /* * Check if we are actually a member of this group. @@ -1476,7 +1466,7 @@ in6p_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) return (ENOMEM); IM6O_LOCK(imo); - idx = im6o_match_group(imo, ifp, &gsa->sa); + idx = im6o_match_group(imo, ifp, gsa); if (idx == (size_t)-1 || imo->im6o_mfilters == NULL) { error = EADDRNOTAVAIL; goto out_imo_locked; @@ -1502,10 +1492,10 @@ in6p_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) * Asked to unblock, but nothing to unblock. * If adding a new block entry, allocate it. */ - ims = im6o_match_source(imo, idx, &ssa->sa); + ims = im6o_match_source(imo, idx, ssa); if ((ims != NULL && doblock) || (ims == NULL && !doblock)) { MLD_PRINTF(("%s: source %s %spresent\n", __func__, - ip6_sprintf(&ssa->sin6.sin6_addr), + ip6_sprintf(&ssa->sin6_addr), doblock ? "" : "not ")); error = EADDRNOTAVAIL; goto out_imo_locked; @@ -1516,12 +1506,12 @@ in6p_block_unblock_source(struct inpcb *inp, struct sockopt *sopt) */ if (doblock) { MLD_PRINTF(("%s: %s source\n", __func__, "block")); - ims = im6f_graft(imf, fmode, &ssa->sin6); + ims = im6f_graft(imf, fmode, ssa); if (ims == NULL) error = ENOMEM; } else { MLD_PRINTF(("%s: %s source\n", __func__, "allow")); - error = im6f_prune(imf, &ssa->sin6); + error = im6f_prune(imf, ssa); } if (error) { @@ -1631,7 +1621,7 @@ in6p_get_source_filters(struct inpcb *inp, struct sockopt *sopt) { struct __msfilterreq64 msfr, msfr64; struct __msfilterreq32 msfr32; - sockunion_t *gsa; + struct sockaddr_in6 *gsa; struct ifnet *ifp; struct ip6_moptions *imo; struct in6_mfilter *imf; @@ -1669,8 +1659,8 @@ in6p_get_source_filters(struct inpcb *inp, struct sockopt *sopt) msfr.msfr_group.ss_len != sizeof(struct sockaddr_in6)) return (EINVAL); - gsa = (sockunion_t *)&msfr.msfr_group; - if (!IN6_IS_ADDR_MULTICAST(&gsa->sin6.sin6_addr)) + gsa = (struct sockaddr_in6 *)&msfr.msfr_group; + if (!IN6_IS_ADDR_MULTICAST(&gsa->sin6_addr)) return (EINVAL); ifnet_head_lock_shared(); @@ -1691,13 +1681,13 @@ in6p_get_source_filters(struct inpcb *inp, struct sockopt *sopt) if (msfr.msfr_nsrcs > in6_mcast_maxsocksrc) msfr.msfr_nsrcs = in6_mcast_maxsocksrc; - (void)in6_setscope(&gsa->sin6.sin6_addr, ifp, NULL); + (void)in6_setscope(&gsa->sin6_addr, ifp, NULL); IM6O_LOCK(imo); /* * Lookup group on the socket. */ - idx = im6o_match_group(imo, ifp, &gsa->sa); + idx = im6o_match_group(imo, ifp, gsa); if (idx == (size_t)-1 || imo->im6o_mfilters == NULL) { IM6O_UNLOCK(imo); return (EADDRNOTAVAIL); @@ -1952,7 +1942,7 @@ static int in6p_join_group(struct inpcb *inp, struct sockopt *sopt) { struct group_source_req gsr; - sockunion_t *gsa, *ssa; + struct sockaddr_in6 *gsa, *ssa; struct ifnet *ifp; struct in6_mfilter *imf; struct ip6_moptions *imo; @@ -1970,10 +1960,8 @@ in6p_join_group(struct inpcb *inp, struct sockopt *sopt) is_new = 0; memset(&gsr, 0, sizeof(struct group_source_req)); - gsa = (sockunion_t *)&gsr.gsr_group; - gsa->ss.ss_family = AF_UNSPEC; - ssa = (sockunion_t *)&gsr.gsr_source; - ssa->ss.ss_family = AF_UNSPEC; + gsa = (struct sockaddr_in6 *)&gsr.gsr_group; + ssa = (struct sockaddr_in6 *)&gsr.gsr_source; /* * Chew everything into struct group_source_req. @@ -1984,7 +1972,6 @@ in6p_join_group(struct inpcb *inp, struct sockopt *sopt) switch (sopt->sopt_name) { case IPV6_JOIN_GROUP: { struct ipv6_mreq mreq; - struct sockaddr_in6 *gsin6; error = sooptcopyin(sopt, &mreq, sizeof(struct ipv6_mreq), sizeof(struct ipv6_mreq)); @@ -2011,19 +1998,17 @@ in6p_join_group(struct inpcb *inp, struct sockopt *sopt) return (inp_join_group(inp, &v4sopt)); } - gsa->sin6.sin6_family = AF_INET6; - gsa->sin6.sin6_len = sizeof(struct sockaddr_in6); - gsa->sin6.sin6_addr = mreq.ipv6mr_multiaddr; - - gsin6 = &gsa->sin6; + gsa->sin6_family = AF_INET6; + gsa->sin6_len = sizeof(struct sockaddr_in6); + gsa->sin6_addr = mreq.ipv6mr_multiaddr; /* Only allow IPv6 multicast addresses */ - if (IN6_IS_ADDR_MULTICAST(&gsin6->sin6_addr) == 0) { + if (IN6_IS_ADDR_MULTICAST(&gsa->sin6_addr) == 0) { return (EINVAL); } if (mreq.ipv6mr_interface == 0) { - ifp = in6p_lookup_mcast_ifp(inp, gsin6); + ifp = in6p_lookup_mcast_ifp(inp, gsa); } else { ifnet_head_lock_shared(); if ((u_int)if_index < mreq.ipv6mr_interface) { @@ -2053,24 +2038,24 @@ in6p_join_group(struct inpcb *inp, struct sockopt *sopt) if (error) return (error); - if (gsa->sin6.sin6_family != AF_INET6 || - gsa->sin6.sin6_len != sizeof(struct sockaddr_in6)) + if (gsa->sin6_family != AF_INET6 || + gsa->sin6_len != sizeof(struct sockaddr_in6)) return (EINVAL); if (sopt->sopt_name == MCAST_JOIN_SOURCE_GROUP) { - if (ssa->sin6.sin6_family != AF_INET6 || - ssa->sin6.sin6_len != sizeof(struct sockaddr_in6)) + if (ssa->sin6_family != AF_INET6 || + ssa->sin6_len != sizeof(struct sockaddr_in6)) return (EINVAL); - if (IN6_IS_ADDR_MULTICAST(&ssa->sin6.sin6_addr)) + if (IN6_IS_ADDR_MULTICAST(&ssa->sin6_addr)) return (EINVAL); /* * TODO: Validate embedded scope ID in source * list entry against passed-in ifp, if and only * if source list filter entry is iface or node local. */ - in6_clearscope(&ssa->sin6.sin6_addr); - ssa->sin6.sin6_port = 0; - ssa->sin6.sin6_scope_id = 0; + in6_clearscope(&ssa->sin6_addr); + ssa->sin6_port = 0; + ssa->sin6_scope_id = 0; } ifnet_head_lock_shared(); @@ -2089,28 +2074,36 @@ in6p_join_group(struct inpcb *inp, struct sockopt *sopt) return (EOPNOTSUPP); } - if (!IN6_IS_ADDR_MULTICAST(&gsa->sin6.sin6_addr)) + if (!IN6_IS_ADDR_MULTICAST(&gsa->sin6_addr)) return (EINVAL); if (ifp == NULL || (ifp->if_flags & IFF_MULTICAST) == 0) return (EADDRNOTAVAIL); - gsa->sin6.sin6_port = 0; - gsa->sin6.sin6_scope_id = 0; + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_mcast_join_total); + /* + * TBD: revisit the criteria for non-OS initiated joins + */ + if (inp->inp_lport == htons(5353)) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_mcast_join_os_total); + } + + gsa->sin6_port = 0; + gsa->sin6_scope_id = 0; /* * Always set the scope zone ID on memberships created from userland. * Use the passed-in ifp to do this. */ - (void)in6_setscope(&gsa->sin6.sin6_addr, ifp, &scopeid); + (void)in6_setscope(&gsa->sin6_addr, ifp, &scopeid); /* * Some addresses are not valid without an embedded scopeid. * This check must be present because otherwise we will later hit * a VERIFY() in in6_mc_join(). */ - if ((IN6_IS_ADDR_MC_LINKLOCAL(&gsa->sin6.sin6_addr) || - IN6_IS_ADDR_MC_INTFACELOCAL(&gsa->sin6.sin6_addr)) && - (scopeid == 0 || gsa->sin6.sin6_addr.s6_addr16[1] == 0)) + if ((IN6_IS_ADDR_MC_LINKLOCAL(&gsa->sin6_addr) || + IN6_IS_ADDR_MC_INTFACELOCAL(&gsa->sin6_addr)) && + (scopeid == 0 || gsa->sin6_addr.s6_addr16[1] == 0)) return (EINVAL); imo = in6p_findmoptions(inp); @@ -2118,13 +2111,13 @@ in6p_join_group(struct inpcb *inp, struct sockopt *sopt) return (ENOMEM); IM6O_LOCK(imo); - idx = im6o_match_group(imo, ifp, &gsa->sa); + idx = im6o_match_group(imo, ifp, gsa); if (idx == (size_t)-1) { is_new = 1; } else { inm = imo->im6o_membership[idx]; imf = &imo->im6o_mfilters[idx]; - if (ssa->ss.ss_family != AF_UNSPEC) { + if (ssa->sin6_family != AF_UNSPEC) { /* * MCAST_JOIN_SOURCE_GROUP on an exclusive membership * is an error. On an existing inclusive membership, @@ -2150,7 +2143,7 @@ in6p_join_group(struct inpcb *inp, struct sockopt *sopt) * full-state SSM API with the delta-based API, * which is discouraged in the relevant RFCs. */ - lims = im6o_match_source(imo, idx, &ssa->sa); + lims = im6o_match_source(imo, idx, ssa); if (lims != NULL /*&& lims->im6sl_st[1] == MCAST_INCLUDE*/) { error = EADDRNOTAVAIL; @@ -2213,7 +2206,7 @@ in6p_join_group(struct inpcb *inp, struct sockopt *sopt) * XXX: Should check for non-NULL lims (node exists but may * not be in-mode) for interop with full-state API. */ - if (ssa->ss.ss_family != AF_UNSPEC) { + if (ssa->sin6_family != AF_UNSPEC) { /* Membership starts in IN mode */ if (is_new) { MLD_PRINTF(("%s: new join w/source\n", __func__); @@ -2221,7 +2214,7 @@ in6p_join_group(struct inpcb *inp, struct sockopt *sopt) } else { MLD_PRINTF(("%s: %s source\n", __func__, "allow")); } - lims = im6f_graft(imf, MCAST_INCLUDE, &ssa->sin6); + lims = im6f_graft(imf, MCAST_INCLUDE, ssa); if (lims == NULL) { MLD_PRINTF(("%s: merge imf state failed\n", __func__)); @@ -2249,7 +2242,7 @@ in6p_join_group(struct inpcb *inp, struct sockopt *sopt) socket_unlock(inp->inp_socket, 0); VERIFY(inm == NULL); - error = in6_mc_join(ifp, &gsa->sin6.sin6_addr, imf, &inm, 0); + error = in6_mc_join(ifp, &gsa->sin6_addr, imf, &inm, 0); VERIFY(inm != NULL || error != 0); socket_lock(inp->inp_socket, 0); @@ -2315,7 +2308,7 @@ in6p_leave_group(struct inpcb *inp, struct sockopt *sopt) { struct ipv6_mreq mreq; struct group_source_req gsr; - sockunion_t *gsa, *ssa; + struct sockaddr_in6 *gsa, *ssa; struct ifnet *ifp; struct in6_mfilter *imf; struct ip6_moptions *imo; @@ -2332,10 +2325,8 @@ in6p_leave_group(struct inpcb *inp, struct sockopt *sopt) is_final = 1; memset(&gsr, 0, sizeof(struct group_source_req)); - gsa = (sockunion_t *)&gsr.gsr_group; - gsa->ss.ss_family = AF_UNSPEC; - ssa = (sockunion_t *)&gsr.gsr_source; - ssa->ss.ss_family = AF_UNSPEC; + gsa = (struct sockaddr_in6 *)&gsr.gsr_group; + ssa = (struct sockaddr_in6 *)&gsr.gsr_source; /* * Chew everything passed in up into a struct group_source_req @@ -2346,7 +2337,6 @@ in6p_leave_group(struct inpcb *inp, struct sockopt *sopt) */ switch (sopt->sopt_name) { case IPV6_LEAVE_GROUP: { - struct sockaddr_in6 *gsin6; error = sooptcopyin(sopt, &mreq, sizeof(struct ipv6_mreq), sizeof(struct ipv6_mreq)); @@ -2373,15 +2363,14 @@ in6p_leave_group(struct inpcb *inp, struct sockopt *sopt) return (inp_leave_group(inp, &v4sopt)); } - gsa->sin6.sin6_family = AF_INET6; - gsa->sin6.sin6_len = sizeof(struct sockaddr_in6); - gsa->sin6.sin6_addr = mreq.ipv6mr_multiaddr; - gsa->sin6.sin6_port = 0; - gsa->sin6.sin6_scope_id = 0; + gsa->sin6_family = AF_INET6; + gsa->sin6_len = sizeof(struct sockaddr_in6); + gsa->sin6_addr = mreq.ipv6mr_multiaddr; + gsa->sin6_port = 0; + gsa->sin6_scope_id = 0; ifindex = mreq.ipv6mr_interface; - gsin6 = &gsa->sin6; /* Only allow IPv6 multicast addresses */ - if (IN6_IS_ADDR_MULTICAST(&gsin6->sin6_addr) == 0) { + if (IN6_IS_ADDR_MULTICAST(&gsa->sin6_addr) == 0) { return (EINVAL); } break; @@ -2401,24 +2390,24 @@ in6p_leave_group(struct inpcb *inp, struct sockopt *sopt) if (error) return (error); - if (gsa->sin6.sin6_family != AF_INET6 || - gsa->sin6.sin6_len != sizeof(struct sockaddr_in6)) + if (gsa->sin6_family != AF_INET6 || + gsa->sin6_len != sizeof(struct sockaddr_in6)) return (EINVAL); if (sopt->sopt_name == MCAST_LEAVE_SOURCE_GROUP) { - if (ssa->sin6.sin6_family != AF_INET6 || - ssa->sin6.sin6_len != sizeof(struct sockaddr_in6)) + if (ssa->sin6_family != AF_INET6 || + ssa->sin6_len != sizeof(struct sockaddr_in6)) return (EINVAL); - if (IN6_IS_ADDR_MULTICAST(&ssa->sin6.sin6_addr)) + if (IN6_IS_ADDR_MULTICAST(&ssa->sin6_addr)) return (EINVAL); /* * TODO: Validate embedded scope ID in source * list entry against passed-in ifp, if and only * if source list filter entry is iface or node local. */ - in6_clearscope(&ssa->sin6.sin6_addr); + in6_clearscope(&ssa->sin6_addr); } - gsa->sin6.sin6_port = 0; - gsa->sin6.sin6_scope_id = 0; + gsa->sin6_port = 0; + gsa->sin6_scope_id = 0; ifindex = gsr.gsr_interface; break; @@ -2428,7 +2417,7 @@ in6p_leave_group(struct inpcb *inp, struct sockopt *sopt) return (EOPNOTSUPP); } - if (!IN6_IS_ADDR_MULTICAST(&gsa->sin6.sin6_addr)) + if (!IN6_IS_ADDR_MULTICAST(&gsa->sin6_addr)) return (EINVAL); /* @@ -2448,9 +2437,9 @@ in6p_leave_group(struct inpcb *inp, struct sockopt *sopt) ifnet_head_done(); if (ifp == NULL) return (EADDRNOTAVAIL); - (void) in6_setscope(&gsa->sin6.sin6_addr, ifp, NULL); + (void) in6_setscope(&gsa->sin6_addr, ifp, NULL); } else { - error = sa6_embedscope(&gsa->sin6, ip6_use_defzone); + error = sa6_embedscope(gsa, ip6_use_defzone); if (error) return (EADDRNOTAVAIL); /* @@ -2463,12 +2452,12 @@ in6p_leave_group(struct inpcb *inp, struct sockopt *sopt) * directly until such time as this implementation is * refactored, assuming the scope IDs are the way to go. */ - ifindex = ntohs(gsa->sin6.sin6_addr.s6_addr16[1]); + ifindex = ntohs(gsa->sin6_addr.s6_addr16[1]); if (ifindex == 0) { MLD_PRINTF(("%s: warning: no ifindex, looking up " "ifp for group %s.\n", __func__, - ip6_sprintf(&gsa->sin6.sin6_addr))); - ifp = in6p_lookup_mcast_ifp(inp, &gsa->sin6); + ip6_sprintf(&gsa->sin6_addr))); + ifp = in6p_lookup_mcast_ifp(inp, gsa); } else { if (!IF_INDEX_IN_RANGE(ifindex)) return (EADDRNOTAVAIL); @@ -2492,7 +2481,7 @@ in6p_leave_group(struct inpcb *inp, struct sockopt *sopt) return (ENOMEM); IM6O_LOCK(imo); - idx = im6o_match_group(imo, ifp, &gsa->sa); + idx = im6o_match_group(imo, ifp, gsa); if (idx == (size_t)-1) { error = EADDRNOTAVAIL; goto out_locked; @@ -2500,7 +2489,7 @@ in6p_leave_group(struct inpcb *inp, struct sockopt *sopt) inm = imo->im6o_membership[idx]; imf = &imo->im6o_mfilters[idx]; - if (ssa->ss.ss_family != AF_UNSPEC) + if (ssa->sin6_family != AF_UNSPEC) is_final = 0; /* @@ -2518,16 +2507,16 @@ in6p_leave_group(struct inpcb *inp, struct sockopt *sopt) error = EADDRNOTAVAIL; goto out_locked; } - ims = im6o_match_source(imo, idx, &ssa->sa); + ims = im6o_match_source(imo, idx, ssa); if (ims == NULL) { MLD_PRINTF(("%s: source %s %spresent\n", __func__, - ip6_sprintf(&ssa->sin6.sin6_addr), + ip6_sprintf(&ssa->sin6_addr), "not ")); error = EADDRNOTAVAIL; goto out_locked; } MLD_PRINTF(("%s: %s source\n", __func__, "block")); - error = im6f_prune(imf, &ssa->sin6); + error = im6f_prune(imf, ssa); if (error) { MLD_PRINTF(("%s: merge imf state failed\n", __func__)); @@ -2663,7 +2652,7 @@ in6p_set_source_filters(struct inpcb *inp, struct sockopt *sopt) { struct __msfilterreq64 msfr, msfr64; struct __msfilterreq32 msfr32; - sockunion_t *gsa; + struct sockaddr_in6 *gsa; struct ifnet *ifp; struct in6_mfilter *imf; struct ip6_moptions *imo; @@ -2708,11 +2697,11 @@ in6p_set_source_filters(struct inpcb *inp, struct sockopt *sopt) msfr.msfr_group.ss_len != sizeof(struct sockaddr_in6)) return (EINVAL); - gsa = (sockunion_t *)&msfr.msfr_group; - if (!IN6_IS_ADDR_MULTICAST(&gsa->sin6.sin6_addr)) + gsa = (struct sockaddr_in6 *)&msfr.msfr_group; + if (!IN6_IS_ADDR_MULTICAST(&gsa->sin6_addr)) return (EINVAL); - gsa->sin6.sin6_port = 0; /* ignore port */ + gsa->sin6_port = 0; /* ignore port */ ifnet_head_lock_shared(); if (msfr.msfr_ifindex == 0 || (u_int)if_index < msfr.msfr_ifindex) { @@ -2724,7 +2713,7 @@ in6p_set_source_filters(struct inpcb *inp, struct sockopt *sopt) if (ifp == NULL) return (EADDRNOTAVAIL); - (void)in6_setscope(&gsa->sin6.sin6_addr, ifp, NULL); + (void)in6_setscope(&gsa->sin6_addr, ifp, NULL); /* * Take the INP write lock. @@ -2735,7 +2724,7 @@ in6p_set_source_filters(struct inpcb *inp, struct sockopt *sopt) return (ENOMEM); IM6O_LOCK(imo); - idx = im6o_match_group(imo, ifp, &gsa->sa); + idx = im6o_match_group(imo, ifp, gsa); if (idx == (size_t)-1 || imo->im6o_mfilters == NULL) { error = EADDRNOTAVAIL; goto out_imo_locked; @@ -3430,7 +3419,10 @@ in6_multihead_lock_shared(void) void in6_multihead_lock_assert(int what) { - lck_rw_assert(&in6_multihead_lock, what); +#if !MACH_ASSERT +#pragma unused(what) +#endif + LCK_RW_ASSERT(&in6_multihead_lock, what); } void diff --git a/bsd/netinet6/in6_pcb.c b/bsd/netinet6/in6_pcb.c index 54b9555f6..29cbedd50 100644 --- a/bsd/netinet6/in6_pcb.c +++ b/bsd/netinet6/in6_pcb.c @@ -163,7 +163,7 @@ in6_pcblookup_local_and_cleanup(struct inpcbinfo *pcbinfo, if (inp != NULL && inp->inp_wantcnt == WNT_STOPUSING) { struct socket *so = inp->inp_socket; - lck_mtx_lock(&inp->inpcb_mtx); + socket_lock(so, 0); if (so->so_usecount == 0) { if (inp->inp_state != INPCB_STATE_DEAD) @@ -171,7 +171,7 @@ in6_pcblookup_local_and_cleanup(struct inpcbinfo *pcbinfo, in_pcbdispose(inp); /* will unlock & destroy */ inp = NULL; } else { - lck_mtx_unlock(&inp->inpcb_mtx); + socket_unlock(so, 0); } } @@ -191,8 +191,10 @@ in6_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct proc *p) int wild = 0, reuseport = (so->so_options & SO_REUSEPORT); struct ifnet *outif = NULL; struct sockaddr_in6 sin6; +#if !CONFIG_EMBEDDED int error; kauth_cred_t cred; +#endif /* !CONFIG_EMBEDDED */ if (!in6_ifaddrs) /* XXX broken! */ return (EADDRNOTAVAIL); @@ -290,6 +292,7 @@ in6_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct proc *p) uid_t u; /* GROSS */ +#if !CONFIG_EMBEDDED if (ntohs(lport) < IPV6PORT_RESERVED) { cred = kauth_cred_proc_ref(p); error = priv_check_cred(cred, @@ -301,6 +304,7 @@ in6_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct proc *p) return (EACCES); } } +#endif /* !CONFIG_EMBEDDED */ if (!IN6_IS_ADDR_MULTICAST(&sin6.sin6_addr) && (u = kauth_cred_getuid(so->so_cred)) != 0) { t = in6_pcblookup_local_and_cleanup(pcbinfo, @@ -525,6 +529,12 @@ in6_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct proc *p) struct ifnet *outif = NULL; struct socket *so = inp->inp_socket; + if (so->so_proto->pr_protocol == IPPROTO_UDP && + sin6->sin6_port == htons(53) && !(so->so_flags1 & SOF1_DNS_COUNTED)) { + so->so_flags1 |= SOF1_DNS_COUNTED; + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_inet_dgram_dns); + } + /* * Call inner routine, to assign local interface address. * in6_pcbladdr() may automatically fill in sin6_scope_id. @@ -618,13 +628,19 @@ in6_pcbdetach(struct inpcb *inp) inp, so, SOCK_PROTO(so)); /* NOTREACHED */ } - + #if IPSEC if (inp->in6p_sp != NULL) { (void) ipsec6_delete_pcbpolicy(inp); } #endif /* IPSEC */ + if (inp->inp_stat != NULL && SOCK_PROTO(so) == IPPROTO_UDP) { + if (inp->inp_stat->rxpackets == 0 && inp->inp_stat->txpackets == 0) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_inet6_dgram_no_data); + } + } + /* * Let NetworkStatistics know this PCB is going away * before we detach it. @@ -665,7 +681,7 @@ in6_pcbdetach(struct inpcb *inp) inp->inp_state = INPCB_STATE_DEAD; /* makes sure we're not called twice from so_close */ so->so_flags |= SOF_PCBCLEARING; - + inpcb_gc_sched(inp->inp_pcbinfo, INPCB_TIMER_FAST); /* @@ -752,7 +768,7 @@ in6_getsockaddr(struct socket *so, struct sockaddr **nam) } int -in6_getsockaddr_s(struct socket *so, struct sockaddr_storage *ss) +in6_getsockaddr_s(struct socket *so, struct sockaddr_in6 *ss) { struct inpcb *inp; struct in6_addr addr; @@ -761,17 +777,13 @@ in6_getsockaddr_s(struct socket *so, struct sockaddr_storage *ss) VERIFY(ss != NULL); bzero(ss, sizeof (*ss)); - if ((inp = sotoinpcb(so)) == NULL -#if NECP - || (necp_socket_should_use_flow_divert(inp)) -#endif /* NECP */ - ) - return (inp == NULL ? EINVAL : EPROTOTYPE); + if ((inp = sotoinpcb(so)) == NULL) + return (EINVAL); port = inp->inp_lport; addr = inp->in6p_laddr; - in6_sockaddr_s(port, &addr, SIN6(ss)); + in6_sockaddr_s(port, &addr, ss); return (0); } @@ -794,30 +806,6 @@ in6_getpeeraddr(struct socket *so, struct sockaddr **nam) return (0); } -int -in6_getpeeraddr_s(struct socket *so, struct sockaddr_storage *ss) -{ - struct inpcb *inp; - struct in6_addr addr; - in_port_t port; - - VERIFY(ss != NULL); - bzero(ss, sizeof (*ss)); - - if ((inp = sotoinpcb(so)) == NULL -#if NECP - || (necp_socket_should_use_flow_divert(inp)) -#endif /* NECP */ - ) - return (inp == NULL ? EINVAL : EPROTOTYPE); - - port = inp->inp_fport; - addr = inp->in6p_faddr; - - in6_sockaddr_s(port, &addr, SIN6(ss)); - return (0); -} - int in6_mapped_sockaddr(struct socket *so, struct sockaddr **nam) { @@ -924,13 +912,9 @@ in6_pcbnotify(struct inpcbinfo *pcbinfo, struct sockaddr *dst, u_int fport_arg, * sockets disconnected. * XXX: should we avoid to notify the value to TCP sockets? */ - if (cmd == PRC_MSGSIZE && (inp->inp_flags & IN6P_MTU) != 0 && - (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr) || - IN6_ARE_ADDR_EQUAL(&inp->in6p_faddr, - &sa6_dst->sin6_addr))) { + if (cmd == PRC_MSGSIZE) ip6_notify_pmtu(inp, (struct sockaddr_in6 *)(void *)dst, (u_int32_t *)cmdarg); - } /* * Detect if we should notify the error. If no source and @@ -1340,7 +1324,7 @@ in6p_route_copyout(struct inpcb *inp, struct route_in6 *dst) { struct route_in6 *src = &inp->in6p_route; - lck_mtx_assert(&inp->inpcb_mtx, LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(inp->inp_socket); /* Minor sanity check */ if (src->ro_rt != NULL && rt_key(src->ro_rt)->sa_family != AF_INET6) @@ -1354,7 +1338,7 @@ in6p_route_copyin(struct inpcb *inp, struct route_in6 *src) { struct route_in6 *dst = &inp->in6p_route; - lck_mtx_assert(&inp->inpcb_mtx, LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(inp->inp_socket); /* Minor sanity check */ if (src->ro_rt != NULL && rt_key(src->ro_rt)->sa_family != AF_INET6) diff --git a/bsd/netinet6/in6_pcb.h b/bsd/netinet6/in6_pcb.h index 093cef2f0..01973578a 100644 --- a/bsd/netinet6/in6_pcb.h +++ b/bsd/netinet6/in6_pcb.h @@ -114,9 +114,8 @@ extern void in6_rtchange(struct inpcb *, int); extern struct sockaddr *in6_sockaddr(in_port_t port, struct in6_addr *addr_p); extern void in6_sockaddr_s(in_port_t, struct in6_addr *, struct sockaddr_in6 *); extern int in6_getpeeraddr(struct socket *, struct sockaddr **); -extern int in6_getpeeraddr_s(struct socket *, struct sockaddr_storage *); extern int in6_getsockaddr(struct socket *, struct sockaddr **); -extern int in6_getsockaddr_s(struct socket *, struct sockaddr_storage *); +extern int in6_getsockaddr_s(struct socket *, struct sockaddr_in6 *); extern int in6_mapped_sockaddr(struct socket *so, struct sockaddr **nam); extern int in6_mapped_peeraddr(struct socket *so, struct sockaddr **nam); extern int in6_selecthlim(struct in6pcb *, struct ifnet *); diff --git a/bsd/netinet6/in6_proto.c b/bsd/netinet6/in6_proto.c index 3aedbea2f..34f8f754a 100644 --- a/bsd/netinet6/in6_proto.c +++ b/bsd/netinet6/in6_proto.c @@ -569,9 +569,16 @@ SYSCTL_INT(_net_inet6_ip6, IPV6CTL_DEFHLIM, SYSCTL_PROC(_net_inet6_ip6, IPV6CTL_STATS, stats, CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0, ip6_getstat, "S,ip6stat", ""); + +#if (DEVELOPMENT || DEBUG) +SYSCTL_INT(_net_inet6_ip6, IPV6CTL_ACCEPT_RTADV, + accept_rtadv, CTLFLAG_RW | CTLFLAG_LOCKED, + &ip6_accept_rtadv, 0, ""); +#else SYSCTL_INT(_net_inet6_ip6, IPV6CTL_ACCEPT_RTADV, accept_rtadv, CTLFLAG_RD | CTLFLAG_LOCKED, - &ip6_accept_rtadv, 0, ""); + &ip6_accept_rtadv, 0, ""); +#endif /* (DEVELOPMENT || DEBUG) */ SYSCTL_INT(_net_inet6_ip6, IPV6CTL_KEEPFAITH, keepfaith, CTLFLAG_RD | CTLFLAG_LOCKED, &ip6_keepfaith, 0, ""); SYSCTL_INT(_net_inet6_ip6, IPV6CTL_LOG_INTERVAL, diff --git a/bsd/netinet6/in6_rmx.c b/bsd/netinet6/in6_rmx.c index ff874090c..7647eaf0c 100644 --- a/bsd/netinet6/in6_rmx.c +++ b/bsd/netinet6/in6_rmx.c @@ -143,8 +143,6 @@ static struct radix_node *in6_matroute_args(void *, struct radix_node_head *, static void in6_clsroute(struct radix_node *, struct radix_node_head *); static int in6_rtqkill(struct radix_node *, void *); -#define RTPRF_OURS RTF_PROTO3 /* set on routes we manage */ - /* * Accessed by in6_addroute(), in6_deleteroute() and in6_rtqkill(), during * which the routing lock (rnh_lock) is held and thus protects the variable. @@ -165,7 +163,7 @@ in6_addroute(void *v_arg, void *n_arg, struct radix_node_head *head, uint32_t flags = rt->rt_flags; boolean_t verbose = (rt_verbose > 1); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK_ASSERT_HELD(rt); if (verbose) @@ -334,7 +332,7 @@ in6_deleteroute(void *v_arg, void *netmask_arg, struct radix_node_head *head) { struct radix_node *rn; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); rn = rn_delete(v_arg, netmask_arg, head); if (rn != NULL) { @@ -449,11 +447,11 @@ in6_clsroute(struct radix_node *rn, struct radix_node_head *head) struct rtentry *rt = (struct rtentry *)rn; boolean_t verbose = (rt_verbose > 1); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK_ASSERT_HELD(rt); if (!(rt->rt_flags & RTF_UP)) - return; /* prophylactic measures */ + return; /* prophylactic measures */ if ((rt->rt_flags & (RTF_LLINFO | RTF_HOST)) != RTF_HOST) return; @@ -517,9 +515,9 @@ in6_clsroute(struct radix_node *rn, struct radix_node_head *head) if (verbose) { log(LOG_DEBUG, "%s: route to %s->%s->%s invalidated, " - "flags=%b, expire=T+%u\n", __func__, dbuf, gbuf, - (rt->rt_ifp != NULL) ? rt->rt_ifp->if_xname : "", - rt->rt_flags, RTF_BITS, rt->rt_expire - timenow); + "flags=%b, expire=T+%u\n", __func__, dbuf, gbuf, + (rt->rt_ifp != NULL) ? rt->rt_ifp->if_xname : "", + rt->rt_flags, RTF_BITS, rt->rt_expire - timenow); } /* We have at least one entry; arm the timer if not already */ @@ -553,7 +551,7 @@ in6_rtqkill(struct radix_node *rn, void *rock) int err; timenow = net_uptime(); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK(rt); if (rt->rt_flags & RTPRF_OURS) { @@ -574,6 +572,7 @@ in6_rtqkill(struct radix_node *rn, void *rock) rt, rt->rt_refcnt); /* NOTREACHED */ } + if (verbose) { log(LOG_DEBUG, "%s: deleting route to " "%s->%s->%s, flags=%b, draining=%d\n", @@ -708,7 +707,7 @@ in6_rtqtimo(void *targ) static void in6_sched_rtqtimo(struct timeval *atv) { - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); if (!in6_rtqtimo_run) { struct timeval tv; diff --git a/bsd/netinet6/in6_src.c b/bsd/netinet6/in6_src.c index 28e831f86..a987f4250 100644 --- a/bsd/netinet6/in6_src.c +++ b/bsd/netinet6/in6_src.c @@ -143,6 +143,11 @@ SYSCTL_INT(_net_inet6_ip6, OID_AUTO, select_src_expensive_secondary_if, CTLFLAG_RW | CTLFLAG_LOCKED, &ip6_select_src_expensive_secondary_if, 0, "allow source interface selection to use expensive secondaries"); +static int ip6_select_src_strong_end = 1; +SYSCTL_INT(_net_inet6_ip6, OID_AUTO, select_src_strong_end, + CTLFLAG_RW | CTLFLAG_LOCKED, &ip6_select_src_strong_end, 0, + "limit source address selection to outgoing interface"); + #define ADDR_LABEL_NOTAPP (-1) struct in6_addrpolicy defaultaddrpolicy; @@ -181,7 +186,7 @@ void addrsel_policy_init(void); #define SASEL_LOG(fmt, ...) \ do { \ - if (SASEL_DO_DBG(inp)) \ + if (srcsel_debug) \ printf("%s:%d " fmt "\n",\ __FUNCTION__, __LINE__, ##__VA_ARGS__); \ } while (0); \ @@ -212,147 +217,40 @@ do { \ goto out; /* XXX: we can't use 'break' here */ \ } while (0) -/* - * Regardless of error, it will return an ifp with a reference held if the - * caller provides a non-NULL ifpp. The caller is responsible for checking - * if the returned ifp is valid and release its reference at all times. - */ struct in6_addr * -in6_selectsrc(struct sockaddr_in6 *dstsock, struct ip6_pktopts *opts, - struct inpcb *inp, struct route_in6 *ro, - struct ifnet **ifpp, struct in6_addr *src_storage, unsigned int ifscope, - int *errorp) +in6_selectsrc_core(struct sockaddr_in6 *dstsock, uint32_t hint_mask, + struct ifnet *ifp, int srcsel_debug, struct in6_addr *src_storage, + struct ifnet **sifp, int *errorp, struct ifaddr **ifapp) { + u_int32_t odstzone; + int bestrule = IP6S_SRCRULE_0; + struct in6_addrpolicy *dst_policy = NULL, *best_policy = NULL; struct in6_addr dst; - struct ifnet *ifp = NULL; struct in6_ifaddr *ia = NULL, *ia_best = NULL; - struct in6_pktinfo *pi = NULL; + char s_src[MAX_IPv6_STR_LEN] = {0}; + char s_dst[MAX_IPv6_STR_LEN] = {0}; + const struct in6_addr *tmp = NULL; int dst_scope = -1, best_scope = -1, best_matchlen = -1; - struct in6_addrpolicy *dst_policy = NULL, *best_policy = NULL; - u_int32_t odstzone; - int prefer_tempaddr; - struct ip6_moptions *mopts; - struct ip6_out_args ip6oa = { ifscope, { 0 }, IP6OAF_SELECT_SRCIF, 0, - SO_TC_UNSPEC, _NET_SERVICE_TYPE_UNSPEC }; - boolean_t islocal = FALSE; uint64_t secs = net_uptime(); - char s_src[MAX_IPv6_STR_LEN], s_dst[MAX_IPv6_STR_LEN]; - const struct in6_addr *tmp; - int bestrule = IP6S_SRCRULE_0; + VERIFY(dstsock != NULL); + VERIFY(src_storage != NULL); + VERIFY(ifp != NULL); - dst = dstsock->sin6_addr; /* make a copy for local operation */ - *errorp = 0; - if (ifpp != NULL) - *ifpp = NULL; + if (sifp != NULL) + *sifp = NULL; - if (inp != NULL) { - mopts = inp->in6p_moptions; - if (INP_NO_CELLULAR(inp)) - ip6oa.ip6oa_flags |= IP6OAF_NO_CELLULAR; - if (INP_NO_EXPENSIVE(inp)) - ip6oa.ip6oa_flags |= IP6OAF_NO_EXPENSIVE; - if (INP_AWDL_UNRESTRICTED(inp)) - ip6oa.ip6oa_flags |= IP6OAF_AWDL_UNRESTRICTED; - if (INP_INTCOPROC_ALLOWED(inp)) - ip6oa.ip6oa_flags |= IP6OAF_INTCOPROC_ALLOWED; - } else { - mopts = NULL; - /* Allow the kernel to retransmit packets. */ - ip6oa.ip6oa_flags |= IP6OAF_INTCOPROC_ALLOWED | - IP6OAF_AWDL_UNRESTRICTED; - } - - if (ip6oa.ip6oa_boundif != IFSCOPE_NONE) - ip6oa.ip6oa_flags |= IP6OAF_BOUND_IF; - - /* - * If the source address is explicitly specified by the caller, - * check if the requested source address is indeed a unicast address - * assigned to the node, and can be used as the packet's source - * address. If everything is okay, use the address as source. - */ - if (opts && (pi = opts->ip6po_pktinfo) && - !IN6_IS_ADDR_UNSPECIFIED(&pi->ipi6_addr)) { - struct sockaddr_in6 srcsock; - struct in6_ifaddr *ia6; + if (ifapp != NULL) + *ifapp = NULL; - /* get the outgoing interface */ - if ((*errorp = in6_selectif(dstsock, opts, mopts, ro, &ip6oa, - &ifp)) != 0) { - src_storage = NULL; - goto done; - } - - /* - * determine the appropriate zone id of the source based on - * the zone of the destination and the outgoing interface. - * If the specified address is ambiguous wrt the scope zone, - * the interface must be specified; otherwise, ifa_ifwithaddr() - * will fail matching the address. - */ - bzero(&srcsock, sizeof (srcsock)); - srcsock.sin6_family = AF_INET6; - srcsock.sin6_len = sizeof (srcsock); - srcsock.sin6_addr = pi->ipi6_addr; - if (ifp != NULL) { - *errorp = in6_setscope(&srcsock.sin6_addr, ifp, NULL); - if (*errorp != 0) { - src_storage = NULL; - goto done; - } - } - ia6 = (struct in6_ifaddr *)ifa_ifwithaddr((struct sockaddr *) - (&srcsock)); - if (ia6 == NULL) { - *errorp = EADDRNOTAVAIL; - src_storage = NULL; - goto done; - } - IFA_LOCK_SPIN(&ia6->ia_ifa); - if ((ia6->ia6_flags & (IN6_IFF_ANYCAST | IN6_IFF_NOTREADY)) || - (inp && inp_restricted_send(inp, ia6->ia_ifa.ifa_ifp))) { - IFA_UNLOCK(&ia6->ia_ifa); - IFA_REMREF(&ia6->ia_ifa); - *errorp = EHOSTUNREACH; - src_storage = NULL; - goto done; - } - - *src_storage = satosin6(&ia6->ia_addr)->sin6_addr; - IFA_UNLOCK(&ia6->ia_ifa); - IFA_REMREF(&ia6->ia_ifa); - goto done; - } - - /* - * Otherwise, if the socket has already bound the source, just use it. - */ - if (inp != NULL && !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) { - src_storage = &inp->in6p_laddr; - goto done; - } - - /* - * If the address is not specified, choose the best one based on - * the outgoing interface and the destination address. - */ - - /* get the outgoing interface */ - if ((*errorp = in6_selectif(dstsock, opts, mopts, ro, &ip6oa, - &ifp)) != 0) { - src_storage = NULL; - goto done; - } + dst = dstsock->sin6_addr; /* make a copy for local operation */ - if (SASEL_DO_DBG(inp)) { + if (srcsel_debug) { (void) inet_ntop(AF_INET6, &dst, s_dst, sizeof (s_src)); tmp = &in6addr_any; (void) inet_ntop(AF_INET6, tmp, s_src, sizeof (s_src)); - - printf("%s out src %s dst %s ifscope %d ifp %s\n", - __func__, s_src, s_dst, ifscope, - ifp ? ifp->if_xname : "NULL"); + printf("%s out src %s dst %s ifp %s\n", + __func__, s_src, s_dst, ifp->if_xname); } *errorp = in6_setscope(&dst, ifp, &odstzone); @@ -360,24 +258,37 @@ in6_selectsrc(struct sockaddr_in6 *dstsock, struct ip6_pktopts *opts, src_storage = NULL; goto done; } - lck_rw_lock_shared(&in6_ifaddr_rwlock); + lck_rw_lock_shared(&in6_ifaddr_rwlock); for (ia = in6_ifaddrs; ia; ia = ia->ia_next) { int new_scope = -1, new_matchlen = -1; struct in6_addrpolicy *new_policy = NULL; - u_int32_t srczone, osrczone, dstzone; + u_int32_t srczone = 0, osrczone, dstzone; struct in6_addr src; struct ifnet *ifp1 = ia->ia_ifp; int srcrule; - if (SASEL_DO_DBG(inp)) + if (srcsel_debug) (void) inet_ntop(AF_INET6, &ia->ia_addr.sin6_addr, - s_src, sizeof (s_src)); + s_src, sizeof (s_src)); IFA_LOCK(&ia->ia_ifa); + + /* + * XXX By default we are strong end system and will + * limit candidate set of source address to the ones + * configured on the outgoing interface. + */ + if (ip6_select_src_strong_end && + ifp1 != ifp) { + SASEL_LOG("NEXT ia %s ifp1 %s address is not on outgoing " + "interface \n", s_src, ifp1->if_xname); + goto next; + } + /* * We'll never take an address that breaks the scope zone - * of the destination. We also skip an address if its zone + * of the destination. We also skip an address if its zone * does not contain the outgoing interface. * XXX: we should probably use sin6_scope_id here. */ @@ -463,53 +374,17 @@ in6_selectsrc(struct sockaddr_in6 *dstsock, struct ip6_pktopts *opts, */ /* Rule 5: Prefer outgoing interface */ - if (ia_best->ia_ifp == ifp && ia->ia_ifp != ifp) - NEXTSRC(IP6S_SRCRULE_5); - if (ia_best->ia_ifp != ifp && ia->ia_ifp == ifp) - REPLACE(IP6S_SRCRULE_5); - - /* Rule 5.5: Prefer addresses in a prefix advertised by the next hop. */ - if (ro != NULL && ro->ro_rt != NULL && ia_best->ia6_ndpr != NULL && - ia->ia6_ndpr != NULL) { - struct rtentry *rta, *rtb; - int op; - - NDPR_LOCK(ia_best->ia6_ndpr); - rta = ia_best->ia6_ndpr->ndpr_rt; - if (rta != NULL) - RT_ADDREF(rta); - NDPR_UNLOCK(ia_best->ia6_ndpr); - - NDPR_LOCK(ia->ia6_ndpr); - rtb = ia->ia6_ndpr->ndpr_rt; - if (rtb != NULL) - RT_ADDREF(rtb); - NDPR_UNLOCK(ia->ia6_ndpr); - - if (rta == NULL || rtb == NULL) - op = 0; - else if (rta == ro->ro_rt && rtb != ro->ro_rt) - op = 1; - else if (rta != ro->ro_rt && rtb == ro->ro_rt) - op = 2; - else - op = 0; - - if (rta != NULL) - RT_REMREF(rta); - if (rtb != NULL) - RT_REMREF(rtb); - - switch (op) { - case 1: - NEXTSRC(IP6S_SRCRULE_5_5); - break; - case 2: - REPLACE(IP6S_SRCRULE_5_5); - break; - default: - break; - } + /* + * XXX By default we are strong end with source address + * selection. That means all address selection candidate + * addresses will be the ones hosted on the outgoing interface + * making the following check redundant. + */ + if (ip6_select_src_strong_end == 0) { + if (ia_best->ia_ifp == ifp && ia->ia_ifp != ifp) + NEXTSRC(IP6S_SRCRULE_5); + if (ia_best->ia_ifp != ifp && ia->ia_ifp == ifp) + REPLACE(IP6S_SRCRULE_5); } /* @@ -533,31 +408,17 @@ in6_selectsrc(struct sockaddr_in6 *dstsock, struct ip6_pktopts *opts, * We allow users to reverse the logic by configuring * a sysctl variable, so that transparency conscious users can * always prefer stable addresses. - * Don't use temporary addresses for local destinations or - * for multicast addresses unless we were passed in an option. */ - if (IN6_IS_ADDR_MULTICAST(&dst) || - in6_matchlen(&ia_best->ia_addr.sin6_addr, &dst) >= - ia_best->ia_plen) - islocal = TRUE; - if (opts == NULL || - opts->ip6po_prefer_tempaddr == IP6PO_TEMPADDR_SYSTEM) { - prefer_tempaddr = islocal ? 0 : ip6_prefer_tempaddr; - } else if (opts->ip6po_prefer_tempaddr == - IP6PO_TEMPADDR_NOTPREFER) { - prefer_tempaddr = 0; - } else - prefer_tempaddr = 1; if (!(ia_best->ia6_flags & IN6_IFF_TEMPORARY) && (ia->ia6_flags & IN6_IFF_TEMPORARY)) { - if (prefer_tempaddr) + if (hint_mask & IPV6_SRCSEL_HINT_PREFER_TMPADDR) REPLACE(IP6S_SRCRULE_7); else NEXTSRC(IP6S_SRCRULE_7); } if ((ia_best->ia6_flags & IN6_IFF_TEMPORARY) && !(ia->ia6_flags & IN6_IFF_TEMPORARY)) { - if (prefer_tempaddr) + if (hint_mask & IPV6_SRCSEL_HINT_PREFER_TMPADDR) NEXTSRC(IP6S_SRCRULE_7); else REPLACE(IP6S_SRCRULE_7); @@ -591,7 +452,7 @@ in6_selectsrc(struct sockaddr_in6 *dstsock, struct ip6_pktopts *opts, ip6_select_src_expensive_secondary_if == 0) { SASEL_LOG("NEXT ia %s ifp1 %s IFEF_EXPENSIVE\n", s_src, ifp1->if_xname); - ip6stat.ip6s_sources_skip_expensive_secondary_if++; + ip6stat.ip6s_sources_skip_expensive_secondary_if++; goto next; } SASEL_LOG("NEXT ia %s ifp1 %s last resort\n", @@ -608,7 +469,7 @@ replace: ip6_select_src_expensive_secondary_if == 0) { SASEL_LOG("NEXT ia %s ifp1 %s IFEF_EXPENSIVE\n", s_src, ifp1->if_xname); - ip6stat.ip6s_sources_skip_expensive_secondary_if++; + ip6stat.ip6s_sources_skip_expensive_secondary_if++; goto next; } bestrule = srcrule; @@ -620,7 +481,7 @@ replace: in6_matchlen(&ia->ia_addr.sin6_addr, &dst)); SASEL_LOG("NEXT ia %s ifp1 %s best_scope %d new_scope %d dst_scope %d\n", s_src, ifp1->if_xname, best_scope, new_scope, dst_scope); - IFA_ADDREF_LOCKED(&ia->ia_ifa); /* for ia_best */ + IFA_ADDREF_LOCKED(&ia->ia_ifa); /* for ia_best */ IFA_UNLOCK(&ia->ia_ifa); if (ia_best != NULL) IFA_REMREF(&ia_best->ia_ifa); @@ -632,7 +493,7 @@ next: continue; out: - IFA_ADDREF_LOCKED(&ia->ia_ifa); /* for ia_best */ + IFA_ADDREF_LOCKED(&ia->ia_ifa); /* for ia_best */ IFA_UNLOCK(&ia->ia_ifa); if (ia_best != NULL) IFA_REMREF(&ia_best->ia_ifa); @@ -642,13 +503,6 @@ out: lck_rw_done(&in6_ifaddr_rwlock); - if (ia_best != NULL && inp && - inp_restricted_send(inp, ia_best->ia_ifa.ifa_ifp)) { - IFA_REMREF(&ia_best->ia_ifa); - ia_best = NULL; - *errorp = EHOSTUNREACH; - } - if ((ia = ia_best) == NULL) { if (*errorp == 0) *errorp = EADDRNOTAVAIL; @@ -656,22 +510,191 @@ out: goto done; } + if (sifp != NULL) { + *sifp = ia->ia_ifa.ifa_ifp; + ifnet_reference(*sifp); + } + IFA_LOCK_SPIN(&ia->ia_ifa); if (bestrule < IP6S_SRCRULE_COUNT) ip6stat.ip6s_sources_rule[bestrule]++; *src_storage = satosin6(&ia->ia_addr)->sin6_addr; IFA_UNLOCK(&ia->ia_ifa); - IFA_REMREF(&ia->ia_ifa); + + if (ifapp != NULL) + *ifapp = &ia->ia_ifa; + else + IFA_REMREF(&ia->ia_ifa); + done: - if (SASEL_DO_DBG(inp)) { + if (srcsel_debug) { (void) inet_ntop(AF_INET6, &dst, s_dst, sizeof (s_src)); tmp = (src_storage != NULL) ? src_storage : &in6addr_any; (void) inet_ntop(AF_INET6, tmp, s_src, sizeof (s_src)); - printf("%s out src %s dst %s ifscope %d dst_scope %d best_scope %d\n", - __func__, s_src, s_dst, ifscope, dst_scope, best_scope); + printf("%s out src %s dst %s dst_scope %d best_scope %d\n", + __func__, s_src, s_dst, dst_scope, best_scope); } + + return (src_storage); +} + +/* + * Regardless of error, it will return an ifp with a reference held if the + * caller provides a non-NULL ifpp. The caller is responsible for checking + * if the returned ifp is valid and release its reference at all times. + */ +struct in6_addr * +in6_selectsrc(struct sockaddr_in6 *dstsock, struct ip6_pktopts *opts, + struct inpcb *inp, struct route_in6 *ro, + struct ifnet **ifpp, struct in6_addr *src_storage, unsigned int ifscope, + int *errorp) +{ + struct ifnet *ifp = NULL; + struct in6_pktinfo *pi = NULL; + struct ip6_moptions *mopts; + struct ip6_out_args ip6oa = { ifscope, { 0 }, IP6OAF_SELECT_SRCIF, 0, + SO_TC_UNSPEC, _NET_SERVICE_TYPE_UNSPEC }; + boolean_t inp_debug = FALSE; + uint32_t hint_mask = 0; + int prefer_tempaddr = 0; + struct ifnet *sifp = NULL; + + *errorp = 0; + if (ifpp != NULL) + *ifpp = NULL; + + if (inp != NULL) { + inp_debug = SASEL_DO_DBG(inp); + mopts = inp->in6p_moptions; + if (INP_NO_CELLULAR(inp)) + ip6oa.ip6oa_flags |= IP6OAF_NO_CELLULAR; + if (INP_NO_EXPENSIVE(inp)) + ip6oa.ip6oa_flags |= IP6OAF_NO_EXPENSIVE; + if (INP_AWDL_UNRESTRICTED(inp)) + ip6oa.ip6oa_flags |= IP6OAF_AWDL_UNRESTRICTED; + if (INP_INTCOPROC_ALLOWED(inp)) + ip6oa.ip6oa_flags |= IP6OAF_INTCOPROC_ALLOWED; + } else { + mopts = NULL; + /* Allow the kernel to retransmit packets. */ + ip6oa.ip6oa_flags |= IP6OAF_INTCOPROC_ALLOWED | + IP6OAF_AWDL_UNRESTRICTED; + } + + if (ip6oa.ip6oa_boundif != IFSCOPE_NONE) + ip6oa.ip6oa_flags |= IP6OAF_BOUND_IF; + + /* + * If the source address is explicitly specified by the caller, + * check if the requested source address is indeed a unicast address + * assigned to the node, and can be used as the packet's source + * address. If everything is okay, use the address as source. + */ + if (opts && (pi = opts->ip6po_pktinfo) && + !IN6_IS_ADDR_UNSPECIFIED(&pi->ipi6_addr)) { + struct sockaddr_in6 srcsock; + struct in6_ifaddr *ia6; + + /* get the outgoing interface */ + if ((*errorp = in6_selectif(dstsock, opts, mopts, ro, &ip6oa, + &ifp)) != 0) { + src_storage = NULL; + goto done; + } + + /* + * determine the appropriate zone id of the source based on + * the zone of the destination and the outgoing interface. + * If the specified address is ambiguous wrt the scope zone, + * the interface must be specified; otherwise, ifa_ifwithaddr() + * will fail matching the address. + */ + bzero(&srcsock, sizeof (srcsock)); + srcsock.sin6_family = AF_INET6; + srcsock.sin6_len = sizeof (srcsock); + srcsock.sin6_addr = pi->ipi6_addr; + if (ifp != NULL) { + *errorp = in6_setscope(&srcsock.sin6_addr, ifp, NULL); + if (*errorp != 0) { + src_storage = NULL; + goto done; + } + } + ia6 = (struct in6_ifaddr *)ifa_ifwithaddr((struct sockaddr *) + (&srcsock)); + if (ia6 == NULL) { + *errorp = EADDRNOTAVAIL; + src_storage = NULL; + goto done; + } + IFA_LOCK_SPIN(&ia6->ia_ifa); + if ((ia6->ia6_flags & (IN6_IFF_ANYCAST | IN6_IFF_NOTREADY)) || + (inp && inp_restricted_send(inp, ia6->ia_ifa.ifa_ifp))) { + IFA_UNLOCK(&ia6->ia_ifa); + IFA_REMREF(&ia6->ia_ifa); + *errorp = EHOSTUNREACH; + src_storage = NULL; + goto done; + } + + *src_storage = satosin6(&ia6->ia_addr)->sin6_addr; + IFA_UNLOCK(&ia6->ia_ifa); + IFA_REMREF(&ia6->ia_ifa); + goto done; + } + + /* + * Otherwise, if the socket has already bound the source, just use it. + */ + if (inp != NULL && !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) { + src_storage = &inp->in6p_laddr; + goto done; + } + + /* + * If the address is not specified, choose the best one based on + * the outgoing interface and the destination address. + */ + /* get the outgoing interface */ + if ((*errorp = in6_selectif(dstsock, opts, mopts, ro, &ip6oa, + &ifp)) != 0) { + src_storage = NULL; + goto done; + } + + VERIFY(ifp != NULL); + + if (opts == NULL || + opts->ip6po_prefer_tempaddr == IP6PO_TEMPADDR_SYSTEM) { + prefer_tempaddr = ip6_prefer_tempaddr; + } else if (opts->ip6po_prefer_tempaddr == IP6PO_TEMPADDR_NOTPREFER) { + prefer_tempaddr = 0; + } else + prefer_tempaddr = 1; + + if (prefer_tempaddr) + hint_mask |= IPV6_SRCSEL_HINT_PREFER_TMPADDR; + + if (in6_selectsrc_core(dstsock, hint_mask, ifp, inp_debug, src_storage, + &sifp, errorp, NULL) == NULL) { + src_storage = NULL; + goto done; + } + + VERIFY(sifp != NULL); + + if (inp && inp_restricted_send(inp, sifp)) { + src_storage = NULL; + *errorp = EHOSTUNREACH; + ifnet_release(sifp); + goto done; + } else { + ifnet_release(sifp); + } + +done: if (ifpp != NULL) { /* if ifp is non-NULL, refcnt held in in6_selectif() */ *ifpp = ifp; @@ -716,21 +739,6 @@ selectroute(struct sockaddr_in6 *srcsock, struct sockaddr_in6 *dstsock, unsigned int ifscope = ((ip6oa != NULL) ? ip6oa->ip6oa_boundif : IFSCOPE_NONE); -#if 0 - char ip6buf[INET6_ADDRSTRLEN]; - - if (dstsock->sin6_addr.s6_addr32[0] == 0 && - dstsock->sin6_addr.s6_addr32[1] == 0 && - !IN6_IS_ADDR_LOOPBACK(&dstsock->sin6_addr)) { - printf("in6_selectroute: strange destination %s\n", - ip6_sprintf(ip6buf, &dstsock->sin6_addr)); - } else { - printf("in6_selectroute: destination = %s%%%d\n", - ip6_sprintf(ip6buf, &dstsock->sin6_addr), - dstsock->sin6_scope_id); /* for debug */ - } -#endif - if (retifp != NULL) *retifp = NULL; @@ -1361,6 +1369,7 @@ in6_pcbsetport(struct in6_addr *laddr, struct inpcb *inp, struct proc *p, struct socket *so = inp->inp_socket; u_int16_t lport = 0, first, last, *lastport; int count, error = 0, wild = 0; + bool found; struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; kauth_cred_t cred; if (!locked) { /* Make sure we don't run into a deadlock: 4052373 */ @@ -1423,6 +1432,7 @@ in6_pcbsetport(struct in6_addr *laddr, struct inpcb *inp, struct proc *p, * counting down */ count = first - last; + found = false; do { if (count-- < 0) { /* completely used? */ @@ -1440,11 +1450,14 @@ in6_pcbsetport(struct in6_addr *laddr, struct inpcb *inp, struct proc *p, if (*lastport > first || *lastport < last) *lastport = first; lport = htons(*lastport); - } while (in6_pcblookup_local(pcbinfo, &inp->in6p_laddr, lport, - wild)); + + found = in6_pcblookup_local(pcbinfo, &inp->in6p_laddr, + lport, wild) == NULL; + } while (!found); } else { /* counting up */ count = last - first; + found = false; do { if (count-- < 0) { /* completely used? */ @@ -1462,8 +1475,10 @@ in6_pcbsetport(struct in6_addr *laddr, struct inpcb *inp, struct proc *p, if (*lastport < first || *lastport > last) *lastport = first; lport = htons(*lastport); - } while (in6_pcblookup_local(pcbinfo, &inp->in6p_laddr, lport, - wild)); + + found = in6_pcblookup_local(pcbinfo, &inp->in6p_laddr, + lport, wild) == NULL; + } while (!found); } inp->inp_lport = lport; diff --git a/bsd/netinet6/in6_var.h b/bsd/netinet6/in6_var.h index aee032c49..8a08baa85 100644 --- a/bsd/netinet6/in6_var.h +++ b/bsd/netinet6/in6_var.h @@ -227,6 +227,11 @@ struct in6_ifstat { /* NOTE: increment on final dst if */ u_quad_t ifs6_in_mcast; /* # of inbound multicast datagrams */ u_quad_t ifs6_out_mcast; /* # of outbound multicast datagrams */ + + u_quad_t ifs6_cantfoward_icmp6; /* # of ICMPv6 packets received for unreachable dest */ + u_quad_t ifs6_addr_expiry_cnt; /* # of address expiry events (excluding privacy addresses) */ + u_quad_t ifs6_pfx_expiry_cnt; /* # of prefix expiry events */ + u_quad_t ifs6_defrtr_expiry_cnt; /* # of default router expiry events */ }; /* @@ -865,9 +870,11 @@ struct in6_multi_mship { #ifdef BSD_KERNEL_PRIVATE #include <netinet6/nd6_var.h> +#include <net/if_llatbl.h> + /* - * * Per-interface IPv6 structures. - * */ + * Per-interface IPv6 structures. + */ struct in6_ifextra { struct scope6_id scope6_id; struct in6_ifstat in6_ifstat; @@ -875,8 +882,11 @@ struct in6_ifextra { struct nd_ifinfo nd_ifinfo; uint32_t netsig_len; u_int8_t netsig[IFNET_SIGNATURELEN]; + struct ipv6_prefix nat64_prefixes[NAT64_MAX_NUM_PREFIXES]; + struct lltable *ii_llt; /* NDP state */ }; #define IN6_IFEXTRA(_ifp) ((struct in6_ifextra *)(_ifp->if_inet6data)) +#define LLTABLE6(ifp) ((IN6_IFEXTRA(ifp) == NULL) ? NULL : IN6_IFEXTRA(ifp)->ii_llt) #endif /* BSD_KERNEL_PRIVATE */ struct mld_ifinfo; @@ -949,10 +959,10 @@ struct in6_multi { }; #define IN6M_LOCK_ASSERT_HELD(_in6m) \ - lck_mtx_assert(&(_in6m)->in6m_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_in6m)->in6m_lock, LCK_MTX_ASSERT_OWNED) #define IN6M_LOCK_ASSERT_NOTHELD(_in6m) \ - lck_mtx_assert(&(_in6m)->in6m_lock, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_in6m)->in6m_lock, LCK_MTX_ASSERT_NOTOWNED) #define IN6M_LOCK(_in6m) \ lck_mtx_lock(&(_in6m)->in6m_lock) @@ -1070,7 +1080,7 @@ struct ip6_pktopts; /* Multicast private KPIs. */ extern int im6o_mc_filter(const struct ip6_moptions *, const struct ifnet *, - const struct sockaddr *, const struct sockaddr *); + const struct sockaddr_in6 *, const struct sockaddr_in6 *); extern int in6_mc_join(struct ifnet *, const struct in6_addr *, struct in6_mfilter *, struct in6_multi **, int); extern int in6_mc_leave(struct in6_multi *, struct in6_mfilter *); @@ -1152,6 +1162,11 @@ extern ssize_t in6_cga_parameters_prepare(void *, size_t, const struct in6_addr *, u_int8_t, const struct in6_cga_modifier *); extern int in6_cga_generate(struct in6_cga_prepare *, u_int8_t, struct in6_addr *); +extern int in6_getconninfo(struct socket *, sae_connid_t, uint32_t *, + uint32_t *, int32_t *, user_addr_t, socklen_t *, + user_addr_t, socklen_t *, uint32_t *, user_addr_t, uint32_t *); +extern void in6_ip6_to_sockaddr(const struct in6_addr *ip6, u_int16_t port, + struct sockaddr_in6 *sin6, u_int32_t maxlen); #endif /* BSD_KERNEL_PRIVATE */ #endif /* _NETINET6_IN6_VAR_H_ */ diff --git a/bsd/netinet6/ip6_fw.c b/bsd/netinet6/ip6_fw.c index dfa12ff11..99d3e8c02 100644 --- a/bsd/netinet6/ip6_fw.c +++ b/bsd/netinet6/ip6_fw.c @@ -86,8 +86,6 @@ #endif #include <string.h> -#include <machine/spl.h> - #include <sys/param.h> #include <sys/systm.h> #include <sys/malloc.h> @@ -908,7 +906,6 @@ add_entry6(struct ip6_fw_head *chainptr, struct ip6_fw *frwl) struct ip6_fw *ftmp = 0; struct ip6_fw_chain *fwc = 0, *fcp, *fcpl = 0; u_short nbr = 0; - int s; fwc = _MALLOC(sizeof *fwc, M_IP6FW, M_WAITOK); ftmp = _MALLOC(sizeof *ftmp, M_IP6FW, M_WAITOK); @@ -925,16 +922,12 @@ add_entry6(struct ip6_fw_head *chainptr, struct ip6_fw *frwl) ftmp->fw_bcnt = 0L; fwc->rule = ftmp; - s = splnet(); - if (!chainptr->lh_first) { LIST_INSERT_HEAD(chainptr, fwc, chain); - splx(s); return(0); } else if (ftmp->fw_number == (u_short)-1) { if (fwc) FREE(fwc, M_IP6FW); if (ftmp) FREE(ftmp, M_IP6FW); - splx(s); dprintf(("%s bad rule number\n", err_prefix)); return (EINVAL); } @@ -967,7 +960,6 @@ add_entry6(struct ip6_fw_head *chainptr, struct ip6_fw *frwl) } bcopy(ftmp, frwl, sizeof(struct ip6_fw)); - splx(s); return (0); } @@ -975,16 +967,12 @@ static int del_entry6(struct ip6_fw_head *chainptr, u_short number) { struct ip6_fw_chain *fcp; - int s; - - s = splnet(); fcp = chainptr->lh_first; if (number != (u_short)-1) { for (; fcp; fcp = fcp->chain.le_next) { if (fcp->rule->fw_number == number) { LIST_REMOVE(fcp, chain); - splx(s); FREE(fcp->rule, M_IP6FW); FREE(fcp, M_IP6FW); return 0; @@ -992,7 +980,6 @@ del_entry6(struct ip6_fw_head *chainptr, u_short number) } } - splx(s); return (EINVAL); } @@ -1000,20 +987,17 @@ static int zero_entry6(struct ip6_fw *frwl) { struct ip6_fw_chain *fcp; - int s; /* * It's possible to insert multiple chain entries with the * same number, so we don't stop after finding the first * match if zeroing a specific entry. */ - s = splnet(); for (fcp = ip6_fw_chain.lh_first; fcp; fcp = fcp->chain.le_next) if (!frwl || frwl->fw_number == 0 || frwl->fw_number == fcp->rule->fw_number) { fcp->rule->fw_bcnt = fcp->rule->fw_pcnt = 0; fcp->rule->timestamp = 0; } - splx(s); if (fw6_verbose) { if (frwl) @@ -1282,7 +1266,6 @@ static int ip6_fw_ctl(struct sockopt *sopt) { int error = 0; - int spl; int valsize; struct ip6_fw rule; int is64user=0; @@ -1336,8 +1319,6 @@ ip6_fw_ctl(struct sockopt *sopt) size_t size = 0; size_t rulesize = 0; - spl = splnet(); - if ( is64user ) rulesize = sizeof(struct ip6_fw_64 ); else @@ -1369,7 +1350,6 @@ ip6_fw_ctl(struct sockopt *sopt) } } - splx(spl); if (buf) { sopt->sopt_valsize = valsize; @@ -1381,7 +1361,6 @@ ip6_fw_ctl(struct sockopt *sopt) } case IPV6_FW_FLUSH: - spl = splnet(); while (ip6_fw_chain.lh_first && ip6_fw_chain.lh_first->rule->fw_number != (u_short)-1) { @@ -1390,7 +1369,6 @@ ip6_fw_ctl(struct sockopt *sopt) FREE(fcp->rule, M_IP6FW); FREE(fcp, M_IP6FW); } - splx(spl); ip6fw_kev_post_msg(KEV_IP6FW_FLUSH); break; diff --git a/bsd/netinet6/ip6_input.c b/bsd/netinet6/ip6_input.c index ec32485cb..b3c2bcfcd 100644 --- a/bsd/netinet6/ip6_input.c +++ b/bsd/netinet6/ip6_input.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003-2016 Apple Inc. All rights reserved. + * Copyright (c) 2003-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -211,6 +211,11 @@ SYSCTL_UINT(_net_inet6_ip6, OID_AUTO, adj_clear_hwcksum, CTLFLAG_RW | CTLFLAG_LOCKED, &ip6_adj_clear_hwcksum, 0, "Invalidate hwcksum info when adjusting length"); +static uint32_t ip6_adj_partial_sum = 1; +SYSCTL_UINT(_net_inet6_ip6, OID_AUTO, adj_partial_sum, + CTLFLAG_RW | CTLFLAG_LOCKED, &ip6_adj_partial_sum, 0, + "Perform partial sum adjustment of trailing bytes at IP layer"); + static int ip6_input_measure = 0; SYSCTL_PROC(_net_inet6_ip6, OID_AUTO, input_perf, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, @@ -305,6 +310,14 @@ ip6_init(struct ip6protosw *pp, struct domain *dp) return; ip6_initialized = 1; + eventhandler_lists_ctxt_init(&in6_evhdlr_ctxt); + (void)EVENTHANDLER_REGISTER(&in6_evhdlr_ctxt, in6_event, + in6_eventhdlr_callback, eventhandler_entry_dummy_arg, + EVENTHANDLER_PRI_ANY); + + for (i = 0; i < IN6_EVENT_MAX; i++) + VERIFY(in6_event2kev_array[i].in6_event_code == i); + pr = pffindproto_locked(PF_INET6, IPPROTO_RAW, SOCK_RAW); if (pr == NULL) { panic("%s: Unable to find [PF_INET6,IPPROTO_RAW,SOCK_RAW]\n", @@ -453,6 +466,78 @@ ip6_init_delayed(void) #endif /* NSTF */ } +static void +ip6_input_adjust(struct mbuf *m, struct ip6_hdr *ip6, uint32_t plen, + struct ifnet *inifp) +{ + boolean_t adjust = TRUE; + uint32_t tot_len = sizeof (*ip6) + plen; + + ASSERT(m_pktlen(m) > tot_len); + + /* + * Invalidate hardware checksum info if ip6_adj_clear_hwcksum + * is set; useful to handle buggy drivers. Note that this + * should not be enabled by default, as we may get here due + * to link-layer padding. + */ + if (ip6_adj_clear_hwcksum && + (m->m_pkthdr.csum_flags & CSUM_DATA_VALID) && + !(inifp->if_flags & IFF_LOOPBACK) && + !(m->m_pkthdr.pkt_flags & PKTF_LOOP)) { + m->m_pkthdr.csum_flags &= ~CSUM_DATA_VALID; + m->m_pkthdr.csum_data = 0; + ip6stat.ip6s_adj_hwcsum_clr++; + } + + /* + * If partial checksum information is available, subtract + * out the partial sum of postpended extraneous bytes, and + * update the checksum metadata accordingly. By doing it + * here, the upper layer transport only needs to adjust any + * prepended extraneous bytes (else it will do both.) + */ + if (ip6_adj_partial_sum && + (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID|CSUM_PARTIAL)) == + (CSUM_DATA_VALID|CSUM_PARTIAL)) { + m->m_pkthdr.csum_rx_val = m_adj_sum16(m, + m->m_pkthdr.csum_rx_start, m->m_pkthdr.csum_rx_start, + (tot_len - m->m_pkthdr.csum_rx_start), + m->m_pkthdr.csum_rx_val); + } else if ((m->m_pkthdr.csum_flags & + (CSUM_DATA_VALID|CSUM_PARTIAL)) == + (CSUM_DATA_VALID|CSUM_PARTIAL)) { + /* + * If packet has partial checksum info and we decided not + * to subtract the partial sum of postpended extraneous + * bytes here (not the default case), leave that work to + * be handled by the other layers. For now, only TCP, UDP + * layers are capable of dealing with this. For all other + * protocols (including fragments), trim and ditch the + * partial sum as those layers might not implement partial + * checksumming (or adjustment) at all. + */ + if (ip6->ip6_nxt == IPPROTO_TCP || + ip6->ip6_nxt == IPPROTO_UDP) { + adjust = FALSE; + } else { + m->m_pkthdr.csum_flags &= ~CSUM_DATA_VALID; + m->m_pkthdr.csum_data = 0; + ip6stat.ip6s_adj_hwcsum_clr++; + } + } + + if (adjust) { + ip6stat.ip6s_adj++; + if (m->m_len == m->m_pkthdr.len) { + m->m_len = tot_len; + m->m_pkthdr.len = tot_len; + } else { + m_adj(m, tot_len - m->m_pkthdr.len); + } + } +} + void ip6_input(struct mbuf *m) { @@ -911,6 +996,14 @@ check_with_pf: if (!ip6_forwarding) { ip6stat.ip6s_cantforward++; in6_ifstat_inc(inifp, ifs6_in_discard); + /* + * Raise a kernel event if the packet received on cellular + * interface is not intended for local host. + * For now limit it to ICMPv6 packets. + */ + if (inifp->if_type == IFT_CELLULAR && + ip6->ip6_nxt == IPPROTO_ICMPV6) + in6_ifstat_inc(inifp, ifs6_cantfoward_icmp6); goto bad; } @@ -1004,29 +1097,7 @@ hbhcheck: goto bad; } if (m->m_pkthdr.len > sizeof (struct ip6_hdr) + plen) { - /* - * Invalidate hardware checksum info if ip6_adj_clear_hwcksum - * is set; useful to handle buggy drivers. Note that this - * should not be enabled by default, as we may get here due - * to link-layer padding. - */ - if (ip6_adj_clear_hwcksum && - (m->m_pkthdr.csum_flags & CSUM_DATA_VALID) && - !(inifp->if_flags & IFF_LOOPBACK) && - !(m->m_pkthdr.pkt_flags & PKTF_LOOP)) { - m->m_pkthdr.csum_flags &= ~CSUM_DATA_VALID; - m->m_pkthdr.csum_data = 0; - ip6stat.ip6s_adj_hwcsum_clr++; - } - - ip6stat.ip6s_adj++; - if (m->m_len == m->m_pkthdr.len) { - m->m_len = sizeof (struct ip6_hdr) + plen; - m->m_pkthdr.len = sizeof (struct ip6_hdr) + plen; - } else { - m_adj(m, sizeof (struct ip6_hdr) + plen - - m->m_pkthdr.len); - } + ip6_input_adjust(m, ip6, plen, inifp); } /* @@ -1741,6 +1812,9 @@ ip6_notify_pmtu(struct inpcb *in6p, struct sockaddr_in6 *dst, u_int32_t *mtu) so = in6p->inp_socket; + if ((in6p->inp_flags & IN6P_MTU) == 0) + return; + if (mtu == NULL) return; @@ -1751,6 +1825,14 @@ ip6_notify_pmtu(struct inpcb *in6p, struct sockaddr_in6 *dst, u_int32_t *mtu) } #endif + if (IN6_IS_ADDR_UNSPECIFIED(&in6p->in6p_faddr) && + (so->so_proto == NULL || so->so_proto->pr_protocol == IPPROTO_TCP)) + return; + + if (!IN6_IS_ADDR_UNSPECIFIED(&in6p->in6p_faddr) && + !IN6_ARE_ADDR_EQUAL(&in6p->in6p_faddr, &dst->sin6_addr)) + return; + bzero(&mtuctl, sizeof (mtuctl)); /* zero-clear for safety */ mtuctl.ip6m_mtu = *mtu; mtuctl.ip6m_addr = *dst; diff --git a/bsd/netinet6/ip6_output.c b/bsd/netinet6/ip6_output.c index 8096be24f..5e541f14c 100644 --- a/bsd/netinet6/ip6_output.c +++ b/bsd/netinet6/ip6_output.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -118,6 +118,7 @@ #include <net/if.h> #include <net/route.h> #include <net/dlil.h> +#include <net/net_api_stats.h> #include <net/net_osdep.h> #include <net/net_perf.h> @@ -181,10 +182,6 @@ static int ip6_splithdr(struct mbuf *, struct ip6_exthdrs *); static void ip6_output_checksum(struct ifnet *, uint32_t, struct mbuf *, int, uint32_t, uint32_t); extern int udp_ctloutput(struct socket *, struct sockopt *); -static int ip6_do_fragmentation(struct mbuf **morig, - uint32_t optlen, struct ifnet *ifp, uint32_t unfragpartlen, - struct ip6_hdr *ip6, struct ip6_exthdrs *exthdrsp, uint32_t mtu, - int nxt0); static int ip6_fragment_packet(struct mbuf **m, struct ip6_pktopts *opt, struct ip6_exthdrs *exthdrsp, struct ifnet *ifp, uint32_t mtu, boolean_t alwaysfrag, uint32_t unfragpartlen, @@ -280,13 +277,13 @@ ip6_output_list(struct mbuf *m0, int packetchain, struct ip6_pktopts *opt, struct mbuf *m, *mprev; struct mbuf *sendchain = NULL, *sendchain_last = NULL; struct mbuf *inputchain = NULL; - int nxt0; + int nxt0 = 0; struct route_in6 *ro_pmtu = NULL; struct rtentry *rt = NULL; - struct sockaddr_in6 *dst, src_sa, dst_sa; + struct sockaddr_in6 *dst = NULL, src_sa, dst_sa; int error = 0; struct in6_ifaddr *ia = NULL, *src_ia = NULL; - u_int32_t mtu; + u_int32_t mtu = 0; boolean_t alwaysfrag = FALSE; u_int32_t optlen = 0, plen = 0, unfragpartlen = 0; struct ip6_rthdr *rh; @@ -573,6 +570,7 @@ loopit: * layer. */ error = EHOSTUNREACH; + ip6stat.ip6s_necp_policy_drop++; goto freehdrs; case NECP_KERNEL_POLICY_RESULT_IP_TUNNEL: { /* @@ -612,6 +610,7 @@ loopit: goto skip_ipsec; } else { error = ENETUNREACH; + ip6stat.ip6s_necp_policy_drop++; goto freehdrs; } } @@ -1175,6 +1174,7 @@ skip_ipsec: /* Catch-all to check if the interface is allowed */ if (!necp_packet_is_allowed_over_interface(m, ifp)) { error = EHOSTUNREACH; + ip6stat.ip6s_necp_policy_drop++; goto bad; } #endif /* NECP */ @@ -1662,8 +1662,19 @@ ip6_fragment_packet(struct mbuf **mptr, struct ip6_pktopts *opt, size_t tlen = m->m_pkthdr.len; boolean_t dontfrag = (opt != NULL && (opt->ip6po_flags & IP6PO_DONTFRAG)); - if (m->m_pkthdr.pkt_flags & PKTF_FORWARDED) + if (m->m_pkthdr.pkt_flags & PKTF_FORWARDED) { dontfrag = TRUE; + /* + * Discard partial sum information if this packet originated + * from another interface; the packet would already have the + * final checksum and we shouldn't recompute it. + */ + if ((m->m_pkthdr.csum_flags & (CSUM_DATA_VALID|CSUM_PARTIAL)) == + (CSUM_DATA_VALID|CSUM_PARTIAL)) { + m->m_pkthdr.csum_flags &= ~CSUM_TX_FLAGS; + m->m_pkthdr.csum_data = 0; + } + } if (dontfrag && alwaysfrag) { /* case 4 */ /* conflicting request - can't transmit */ @@ -1720,7 +1731,7 @@ ip6_fragment_packet(struct mbuf **mptr, struct ip6_pktopts *opt, * of fragments is linked into the packet chain where morig existed. Otherwise, * an errno is returned. */ -static int +int ip6_do_fragmentation(struct mbuf **mptr, uint32_t optlen, struct ifnet *ifp, uint32_t unfragpartlen, struct ip6_hdr *ip6, struct ip6_exthdrs *exthdrsp, uint32_t mtu, int nxt0) @@ -2051,7 +2062,8 @@ in6_finalize_cksum(struct mbuf *m, uint32_t hoff, int32_t optlen, ip6_out_cksum_stats(nxt, plen - olen); /* RFC1122 4.1.3.4 */ - if (csum == 0 && (m->m_pkthdr.csum_flags & CSUM_UDPIPV6)) + if (csum == 0 && + (m->m_pkthdr.csum_flags & (CSUM_UDPIPV6|CSUM_ZERO_INVERT))) csum = 0xffff; /* Insert the checksum in the ULP csum field */ @@ -2063,8 +2075,8 @@ in6_finalize_cksum(struct mbuf *m, uint32_t hoff, int32_t optlen, } else { bcopy(&csum, (mtod(m, char *) + offset), sizeof (csum)); } - m->m_pkthdr.csum_flags &= - ~(CSUM_DELAY_IPV6_DATA | CSUM_DATA_VALID | CSUM_PARTIAL); + m->m_pkthdr.csum_flags &= ~(CSUM_DELAY_IPV6_DATA | CSUM_DATA_VALID | + CSUM_PARTIAL | CSUM_ZERO_INVERT); done: return (sw_csum); @@ -2217,6 +2229,10 @@ ip6_getpmtu(struct route_in6 *ro_pmtu, struct route_in6 *ro, u_int32_t mtu = 0; boolean_t alwaysfrag = FALSE; int error = 0; + boolean_t is_local = FALSE; + + if (IN6_IS_SCOPE_LINKLOCAL(dst)) + is_local = TRUE; if (ro_pmtu != ro) { /* The first hop and the final destination may differ. */ @@ -2287,7 +2303,7 @@ ip6_getpmtu(struct route_in6 *ro_pmtu, struct route_in6 *ro, } *mtup = mtu; - if (alwaysfragp != NULL) + if ((alwaysfragp != NULL) && !is_local) *alwaysfragp = alwaysfrag; return (error); } @@ -2319,6 +2335,7 @@ ip6_ctloutput(struct socket *so, struct sockopt *sopt) privileged = (proc_suser(p) == 0); if (level == IPPROTO_IPV6) { + boolean_t capture_exthdrstat_in = FALSE; switch (op) { case SOPT_SET: switch (optname) { @@ -2445,6 +2462,7 @@ ip6_ctloutput(struct socket *so, struct sockopt *sopt) break; } OPTSET(IN6P_HOPOPTS); + capture_exthdrstat_in = TRUE; break; case IPV6_RECVDSTOPTS: @@ -2454,6 +2472,7 @@ ip6_ctloutput(struct socket *so, struct sockopt *sopt) break; } OPTSET(IN6P_DSTOPTS); + capture_exthdrstat_in = TRUE; break; case IPV6_RECVRTHDRDSTOPTS: @@ -2463,6 +2482,7 @@ ip6_ctloutput(struct socket *so, struct sockopt *sopt) break; } OPTSET(IN6P_RTHDRDSTOPTS); + capture_exthdrstat_in = TRUE; break; case IPV6_RECVRTHDR: @@ -2472,6 +2492,7 @@ ip6_ctloutput(struct socket *so, struct sockopt *sopt) break; } OPTSET(IN6P_RTHDR); + capture_exthdrstat_in = TRUE; break; case IPV6_RECVPATHMTU: @@ -2566,15 +2587,18 @@ ip6_ctloutput(struct socket *so, struct sockopt *sopt) if (!privileged) return (EPERM); OPTSET2292(IN6P_HOPOPTS); + capture_exthdrstat_in = TRUE; break; case IPV6_2292DSTOPTS: if (!privileged) return (EPERM); OPTSET2292(IN6P_DSTOPTS| IN6P_RTHDRDSTOPTS); /* XXX */ + capture_exthdrstat_in = TRUE; break; case IPV6_2292RTHDR: OPTSET2292(IN6P_RTHDR); + capture_exthdrstat_in = TRUE; break; } break; @@ -2721,6 +2745,13 @@ ip6_ctloutput(struct socket *so, struct sockopt *sopt) error = ENOPROTOOPT; break; } + if (capture_exthdrstat_in) { + if (uproto == IPPROTO_TCP) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_sock_inet6_stream_exthdr_in); + } else if (uproto == IPPROTO_UDP) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_sock_inet6_dgram_exthdr_in); + } + } break; case SOPT_GET: @@ -3520,6 +3551,7 @@ ip6_setpktopt(int optname, u_char *buf, int len, struct ip6_pktopts *opt, { int minmtupolicy, preftemp; int error; + boolean_t capture_exthdrstat_out = FALSE; if (!sticky && !cmsg) { #ifdef DIAGNOSTIC @@ -3747,7 +3779,7 @@ ip6_setpktopt(int optname, u_char *buf, int len, struct ip6_pktopts *opt, if (opt->ip6po_hbh == NULL) return (ENOBUFS); bcopy(hbh, opt->ip6po_hbh, hbhlen); - + capture_exthdrstat_out = TRUE; break; } @@ -3811,6 +3843,7 @@ ip6_setpktopt(int optname, u_char *buf, int len, struct ip6_pktopts *opt, if (*newdest == NULL) return (ENOBUFS); bcopy(dest, *newdest, destlen); + capture_exthdrstat_out = TRUE; break; } @@ -3851,6 +3884,7 @@ ip6_setpktopt(int optname, u_char *buf, int len, struct ip6_pktopts *opt, if (opt->ip6po_rthdr == NULL) return (ENOBUFS); bcopy(rth, opt->ip6po_rthdr, rthlen); + capture_exthdrstat_out = TRUE; break; } @@ -3897,6 +3931,14 @@ ip6_setpktopt(int optname, u_char *buf, int len, struct ip6_pktopts *opt, return (ENOPROTOOPT); } /* end of switch */ + if (capture_exthdrstat_out) { + if (uproto == IPPROTO_TCP) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_sock_inet6_stream_exthdr_out); + } else if (uproto == IPPROTO_UDP) { + INC_ATOMIC_INT64_LIM(net_api_stats.nas_sock_inet6_dgram_exthdr_out); + } + } + return (0); } @@ -4030,13 +4072,15 @@ ip6_output_checksum(struct ifnet *ifp, uint32_t mtu, struct mbuf *m, } else if (!(sw_csum & CSUM_DELAY_IPV6_DATA) && (hwcap & CSUM_PARTIAL)) { /* - * Partial checksum offload, ere), if no extension - * headers, and TCP only (no UDP support, as the - * hardware may not be able to convert +0 to - * -0 (0xffff) per RFC1122 4.1.3.4.) + * Partial checksum offload, ere), if no extension headers, + * and TCP only (no UDP support, as the hardware may not be + * able to convert +0 to -0 (0xffff) per RFC1122 4.1.3.4. + * unless the interface supports "invert zero" capability.) */ if (hwcksum_tx && !tso && - (m->m_pkthdr.csum_flags & CSUM_TCPIPV6) && + ((m->m_pkthdr.csum_flags & CSUM_TCPIPV6) || + ((hwcap & CSUM_ZERO_INVERT) && + (m->m_pkthdr.csum_flags & CSUM_ZERO_INVERT))) && tlen <= mtu) { uint16_t start = sizeof (struct ip6_hdr); uint16_t ulpoff = diff --git a/bsd/netinet6/ip6_var.h b/bsd/netinet6/ip6_var.h index e04dee46a..153ad9dc4 100644 --- a/bsd/netinet6/ip6_var.h +++ b/bsd/netinet6/ip6_var.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -148,10 +148,10 @@ struct ip6_moptions { }; #define IM6O_LOCK_ASSERT_HELD(_im6o) \ - lck_mtx_assert(&(_im6o)->im6o_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_im6o)->im6o_lock, LCK_MTX_ASSERT_OWNED) #define IM6O_LOCK_ASSERT_NOTHELD(_im6o) \ - lck_mtx_assert(&(_im6o)->im6o_lock, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_im6o)->im6o_lock, LCK_MTX_ASSERT_NOTOWNED) #define IM6O_LOCK(_im6o) \ lck_mtx_lock(&(_im6o)->im6o_lock) @@ -336,11 +336,14 @@ struct ip6stat { /* DAD NS looped back */ u_quad_t ip6s_dad_loopcount; + + /* NECP policy related drop */ + u_quad_t ip6s_necp_policy_drop; }; enum ip6s_sources_rule_index { IP6S_SRCRULE_0, IP6S_SRCRULE_1, IP6S_SRCRULE_2, IP6S_SRCRULE_3, IP6S_SRCRULE_4, - IP6S_SRCRULE_5, IP6S_SRCRULE_5_5, IP6S_SRCRULE_6, IP6S_SRCRULE_7, + IP6S_SRCRULE_5, IP6S_SRCRULE_6, IP6S_SRCRULE_7, IP6S_SRCRULE_7x, IP6S_SRCRULE_8 }; @@ -471,6 +474,7 @@ extern struct pr_usrreqs icmp6_dgram_usrreqs; struct sockopt; struct inpcb; +struct ip6_hdr; struct in6_ifaddr; struct ip6protosw; struct domain; @@ -524,6 +528,8 @@ extern void ip6_clearpktopts(struct ip6_pktopts *, int); extern struct ip6_pktopts *ip6_copypktopts(struct ip6_pktopts *, int); extern int ip6_optlen(struct inpcb *); extern void ip6_drain(void); +extern int ip6_do_fragmentation(struct mbuf **, uint32_t, struct ifnet *, uint32_t, + struct ip6_hdr *, struct ip6_exthdrs *, uint32_t, int); extern int route6_input(struct mbuf **, int *, int); @@ -532,12 +538,20 @@ extern int frag6_input(struct mbuf **, int *, int); extern void frag6_drain(void); extern int rip6_input(struct mbuf **, int *, int); -extern void rip6_ctlinput(int, struct sockaddr *, void *); +extern void rip6_ctlinput(int, struct sockaddr *, void *, struct ifnet *); extern int rip6_ctloutput(struct socket *so, struct sockopt *sopt); extern int rip6_output(struct mbuf *, struct socket *, struct sockaddr_in6 *, struct mbuf *, int); extern int dest6_input(struct mbuf **, int *, int); +/* + * IPv6 source address selection hints + */ +#define IPV6_SRCSEL_HINT_PREFER_TMPADDR 0x00000001 + +extern struct in6_addr * in6_selectsrc_core(struct sockaddr_in6 *, + uint32_t, struct ifnet *, int, + struct in6_addr *, struct ifnet **, int *, struct ifaddr **); extern struct in6_addr *in6_selectsrc(struct sockaddr_in6 *, struct ip6_pktopts *, struct inpcb *, struct route_in6 *, struct ifnet **, struct in6_addr *, unsigned int, int *); diff --git a/bsd/netinet6/ip6protosw.h b/bsd/netinet6/ip6protosw.h index 12bec55b0..135b6f344 100644 --- a/bsd/netinet6/ip6protosw.h +++ b/bsd/netinet6/ip6protosw.h @@ -168,7 +168,7 @@ struct ip6protosw { (struct mbuf *m, struct socket *so, struct sockaddr_in6 *, struct mbuf *); void (*pr_ctlinput) /* control input (from below) */ - (int, struct sockaddr *, void *); + (int, struct sockaddr *, void *, struct ifnet *); int (*pr_ctloutput) /* control output (from above) */ (struct socket *, struct sockopt *); /* @@ -184,11 +184,11 @@ struct ip6protosw { /* for compat. with IPv4 protosw */ int (*pr_sysctl)(void); /* sysctl for protocol */ int (*pr_lock) /* lock function for protocol */ - (struct socket *so, int locktype, void *debug); + (struct socket *so, int refcnt, void *debug); int (*pr_unlock) /* unlock for protocol */ - (struct socket *so, int locktype, void *debug); + (struct socket *so, int refcnt, void *debug); lck_mtx_t *(*pr_getlock) /* retrieve protocol lock */ - (struct socket *so, int locktype); + (struct socket *so, int flags); /* * misc */ diff --git a/bsd/netinet6/ipsec.c b/bsd/netinet6/ipsec.c index 61302bab6..895d63e77 100644 --- a/bsd/netinet6/ipsec.c +++ b/bsd/netinet6/ipsec.c @@ -314,7 +314,7 @@ ipsec4_getpolicybysock(struct mbuf *m, struct secpolicy *currsp = NULL; /* policy on socket */ struct secpolicy *kernsp = NULL; /* policy on kernel */ - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (m == NULL || so == NULL || error == NULL) panic("ipsec4_getpolicybysock: NULL pointer was passed.\n"); @@ -518,7 +518,7 @@ ipsec4_getpolicybyaddr(struct mbuf *m, if (ipsec_bypass != 0) return 0; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (m == NULL || error == NULL) @@ -652,7 +652,7 @@ ipsec6_getpolicybysock(struct mbuf *m, struct secpolicy *currsp = NULL; /* policy on socket */ struct secpolicy *kernsp = NULL; /* policy on kernel */ - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (m == NULL || so == NULL || error == NULL) @@ -823,7 +823,7 @@ ipsec6_getpolicybyaddr(struct mbuf *m, { struct secpolicy *sp = NULL; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (m == NULL || error == NULL) @@ -1955,7 +1955,7 @@ ipsec4_in_reject_so(struct mbuf *m, struct socket *so) int error; int result; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (m == NULL) return 0; /* XXX should be panic ? */ @@ -1985,7 +1985,7 @@ ipsec4_in_reject_so(struct mbuf *m, struct socket *so) int ipsec4_in_reject(struct mbuf *m, struct inpcb *inp) { - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); if (inp == NULL) return ipsec4_in_reject_so(m, NULL); if (inp->inp_socket) @@ -2010,7 +2010,7 @@ ipsec6_in_reject_so(struct mbuf *m, struct socket *so) int error; int result; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (m == NULL) return 0; /* XXX should be panic ? */ @@ -2040,7 +2040,7 @@ int ipsec6_in_reject(struct mbuf *m, struct in6pcb *in6p) { - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); if (in6p == NULL) return ipsec6_in_reject_so(m, NULL); if (in6p->in6p_socket) @@ -2064,7 +2064,7 @@ ipsec_hdrsiz(struct secpolicy *sp) struct ipsecrequest *isr; size_t siz, clen; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); KEYDEBUG(KEYDEBUG_IPSEC_DATA, printf("ipsec_hdrsiz: using SP\n"); kdebug_secpolicy(sp)); @@ -2138,7 +2138,7 @@ ipsec4_hdrsiz(struct mbuf *m, u_int dir, struct inpcb *inp) int error; size_t size; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (m == NULL) return 0; /* XXX should be panic ? */ @@ -2179,7 +2179,7 @@ ipsec6_hdrsiz(struct mbuf *m, u_int dir, struct in6pcb *in6p) int error; size_t size; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (m == NULL) return 0; /* XXX shoud be panic ? */ @@ -2312,7 +2312,11 @@ ipsec4_encapsulate(struct mbuf *m, struct secasvar *sav) ipseclog((LOG_ERR, "IPv4 ipsec: size exceeds limit: " "leave ip_len as is (invalid packet)\n")); } - ip->ip_id = ip_randomid(); + if (rfc6864 && IP_OFF_IS_ATOMIC(ntohs(ip->ip_off))) { + ip->ip_id = 0; + } else { + ip->ip_id = ip_randomid(); + } bcopy(&((struct sockaddr_in *)&sav->sah->saidx.src)->sin_addr, &ip->ip_src, sizeof(ip->ip_src)); bcopy(&((struct sockaddr_in *)&sav->sah->saidx.dst)->sin_addr, @@ -3268,7 +3272,7 @@ ipsec4_interface_output(struct ipsec_output_state *state, ifnet_t interface) int error = 0; struct secasvar *sav = NULL; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); if (!state) panic("state == NULL in ipsec4_output"); @@ -3310,7 +3314,7 @@ ipsec4_output(struct ipsec_output_state *state, struct secpolicy *sp, __unused i int error = 0; struct sockaddr_in *sin; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); if (!state) panic("state == NULL in ipsec4_output"); @@ -3515,7 +3519,7 @@ ipsec6_output_trans( struct sockaddr_in6 *sin6; struct secasvar *sav = NULL; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); if (!state) panic("state == NULL in ipsec6_output_trans"); @@ -3914,7 +3918,7 @@ ipsec6_output_tunnel( struct secasvar *sav = NULL; int error = 0; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); if (!state) panic("state == NULL in ipsec6_output_tunnel"); @@ -4047,7 +4051,7 @@ ipsec6_interface_output(struct ipsec_output_state *state, ifnet_t interface, u_c int error = 0; struct secasvar *sav = NULL; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); if (!state) panic("state == NULL in ipsec6_output"); @@ -4185,7 +4189,7 @@ ipsec4_tunnel_validate( struct secpolicy *sp; struct ip *oip; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); #if DIAGNOSTIC if (m->m_len < sizeof(struct ip)) @@ -4303,7 +4307,7 @@ ipsec6_tunnel_validate( struct secpolicy *sp; struct ip6_hdr *oip6; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); #if DIAGNOSTIC if (m->m_len < sizeof(struct ip6_hdr)) @@ -4506,7 +4510,15 @@ struct ipsec_tag { struct socket *socket; u_int32_t history_count; struct ipsec_history history[]; +#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4) +/* For the newer ARMv7k ABI where 64-bit types are 64-bit aligned, but pointers + * are 32-bit: + * Aligning to 64-bit since we case to m_tag which is 64-bit aligned. + */ +} __attribute__ ((aligned(8))); +#else }; +#endif #define IPSEC_TAG_SIZE (MLEN - sizeof(struct m_tag)) #define IPSEC_TAG_HDR_SIZE (offsetof(struct ipsec_tag, history[0])) @@ -4674,7 +4686,7 @@ ipsec_send_natt_keepalive( struct route ro; int keepalive_interval = natt_keepalive_interval; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); if ((esp_udp_encap_port & 0xFFFF) == 0 || sav->remote_ike_port == 0) return FALSE; @@ -4804,7 +4816,11 @@ ipsec_fill_offload_frame(ifnet_t ifp, break; } ip->ip_len = htons(sizeof(struct udpiphdr) + 1); - ip->ip_id = ip_randomid(); + if (rfc6864 && IP_OFF_IS_ATOMIC(htons(ip->ip_off))) { + ip->ip_id = 0; + } else { + ip->ip_id = ip_randomid(); + } ip->ip_ttl = ip_defttl; ip->ip_p = IPPROTO_UDP; ip->ip_sum = 0; diff --git a/bsd/netinet6/ipsec.h b/bsd/netinet6/ipsec.h index 823bab27e..8d0e6dd51 100644 --- a/bsd/netinet6/ipsec.h +++ b/bsd/netinet6/ipsec.h @@ -290,8 +290,13 @@ struct ipsecstat { { "esp_randpad", CTLTYPE_INT }, \ } +#if defined(__ARM__) +#define IPSEC_IS_P2ALIGNED(p) IS_P2ALIGNED(p, sizeof (u_int32_t)) +#define IPSEC_GET_P2UNALIGNED_OFS(p) (sizeof(u_int32_t) - (((uintptr_t)(p)) & ((uintptr_t)(sizeof(u_int32_t)) - 1))) +#else #define IPSEC_IS_P2ALIGNED(p) 1 #define IPSEC_GET_P2UNALIGNED_OFS(p) 0 +#endif struct ipsec_output_state { int tunneled; diff --git a/bsd/netinet6/mld6.c b/bsd/netinet6/mld6.c index 6cbcaef13..ca0f1c7fd 100644 --- a/bsd/netinet6/mld6.c +++ b/bsd/netinet6/mld6.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -182,7 +182,7 @@ static const char * mld_rec_type_to_str(const int); #endif static uint32_t mld_set_version(struct mld_ifinfo *, const int); static void mld_flush_relq(struct mld_ifinfo *, struct mld_in6m_relhead *); -static void mld_dispatch_queue(struct mld_ifinfo *, struct ifqueue *, int); +static void mld_dispatch_queue_locked(struct mld_ifinfo *, struct ifqueue *, int); static int mld_v1_input_query(struct ifnet *, const struct ip6_hdr *, /*const*/ struct mld_hdr *); static int mld_v1_input_report(struct ifnet *, struct mbuf *, @@ -228,15 +228,16 @@ static int interface_timers_running6; static int state_change_timers_running6; static int current_state_timers_running6; +static unsigned int mld_mli_list_genid; /* * Subsystem lock macros. */ #define MLD_LOCK() \ lck_mtx_lock(&mld_mtx) #define MLD_LOCK_ASSERT_HELD() \ - lck_mtx_assert(&mld_mtx, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&mld_mtx, LCK_MTX_ASSERT_OWNED) #define MLD_LOCK_ASSERT_NOTHELD() \ - lck_mtx_assert(&mld_mtx, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&mld_mtx, LCK_MTX_ASSERT_NOTOWNED) #define MLD_UNLOCK() \ lck_mtx_unlock(&mld_mtx) @@ -487,12 +488,20 @@ out_locked: * Dispatch an entire queue of pending packet chains. * * Must not be called with in6m_lock held. + * XXX This routine unlocks MLD global lock and also mli locks. + * Make sure that the calling routine takes reference on the mli + * before calling this routine. + * Also if we are traversing mli_head, remember to check for + * mli list generation count and restart the loop if generation count + * has changed. */ static void -mld_dispatch_queue(struct mld_ifinfo *mli, struct ifqueue *ifq, int limit) +mld_dispatch_queue_locked(struct mld_ifinfo *mli, struct ifqueue *ifq, int limit) { struct mbuf *m; + MLD_LOCK_ASSERT_HELD(); + if (mli != NULL) MLI_LOCK_ASSERT_HELD(mli); @@ -503,11 +512,17 @@ mld_dispatch_queue(struct mld_ifinfo *mli, struct ifqueue *ifq, int limit) MLD_PRINTF(("%s: dispatch 0x%llx from 0x%llx\n", __func__, (uint64_t)VM_KERNEL_ADDRPERM(ifq), (uint64_t)VM_KERNEL_ADDRPERM(m))); + if (mli != NULL) MLI_UNLOCK(mli); + MLD_UNLOCK(); + mld_dispatch_packet(m); + + MLD_LOCK(); if (mli != NULL) MLI_LOCK(mli); + if (--limit == 0) break; } @@ -575,6 +590,7 @@ mld_domifattach(struct ifnet *ifp, int how) ifnet_lock_done(ifp); LIST_INSERT_HEAD(&mli_head, mli, mli_link); + mld_mli_list_genid++; MLD_UNLOCK(); @@ -608,6 +624,7 @@ mld_domifreattach(struct mld_ifinfo *mli) ifnet_lock_done(ifp); LIST_INSERT_HEAD(&mli_head, mli, mli_link); + mld_mli_list_genid++; MLD_UNLOCK(); @@ -664,6 +681,7 @@ mli_delete(const struct ifnet *ifp, struct mld_in6m_relhead *in6m_dthead) LIST_REMOVE(mli, mli_link); MLI_REMREF(mli); /* release mli_head reference */ + mld_mli_list_genid++; return; } MLI_UNLOCK(mli); @@ -1563,6 +1581,8 @@ mld_timeout(void *arg) struct mld_ifinfo *mli; struct in6_multi *inm; int uri_sec = 0; + unsigned int genid = mld_mli_list_genid; + SLIST_HEAD(, in6_multi) in6m_dthead; SLIST_INIT(&in6m_dthead); @@ -1600,12 +1620,30 @@ mld_timeout(void *arg) if (interface_timers_running6) { MLD_PRINTF(("%s: interface timers running\n", __func__)); interface_timers_running6 = 0; - LIST_FOREACH(mli, &mli_head, mli_link) { + mli = LIST_FIRST(&mli_head); + + while (mli != NULL) { + if (mli->mli_flags & MLIF_PROCESSED) { + mli = LIST_NEXT(mli, mli_link); + continue; + } + MLI_LOCK(mli); if (mli->mli_version != MLD_VERSION_2) { MLI_UNLOCK(mli); + mli = LIST_NEXT(mli, mli_link); continue; } + /* + * XXX The logic below ends up calling + * mld_dispatch_packet which can unlock mli + * and the global MLD lock. + * Therefore grab a reference on MLI and also + * check for generation count to see if we should + * iterate the list again. + */ + MLI_ADDREF_LOCKED(mli); + if (mli->mli_v2_timer == 0) { /* Do nothing. */ } else if (--mli->mli_v2_timer == 0) { @@ -1614,10 +1652,27 @@ mld_timeout(void *arg) } else { interface_timers_running6 = 1; } + mli->mli_flags |= MLIF_PROCESSED; MLI_UNLOCK(mli); + MLI_REMREF(mli); + + if (genid != mld_mli_list_genid) { + MLD_PRINTF(("%s: MLD information list changed " + "in the middle of iteration! Restart iteration.\n", + __func__)); + mli = LIST_FIRST(&mli_head); + genid = mld_mli_list_genid; + } else { + mli = LIST_NEXT(mli, mli_link); + } } + + LIST_FOREACH(mli, &mli_head, mli_link) + mli->mli_flags &= ~MLIF_PROCESSED; } + + if (!current_state_timers_running6 && !state_change_timers_running6) goto out_locked; @@ -1637,9 +1692,16 @@ mld_timeout(void *arg) * MLD host report and state-change timer processing. * Note: Processing a v2 group timer may remove a node. */ - LIST_FOREACH(mli, &mli_head, mli_link) { + mli = LIST_FIRST(&mli_head); + + while (mli != NULL) { struct in6_multistep step; + if (mli->mli_flags & MLIF_PROCESSED) { + mli = LIST_NEXT(mli, mli_link); + continue; + } + MLI_LOCK(mli); ifp = mli->mli_ifp; uri_sec = MLD_RANDOM_DELAY(mli->mli_uri); @@ -1670,13 +1732,22 @@ next: } in6_multihead_lock_done(); + /* + * XXX The logic below ends up calling + * mld_dispatch_packet which can unlock mli + * and the global MLD lock. + * Therefore grab a reference on MLI and also + * check for generation count to see if we should + * iterate the list again. + */ MLI_LOCK(mli); + MLI_ADDREF_LOCKED(mli); if (mli->mli_version == MLD_VERSION_1) { - mld_dispatch_queue(mli, &mli->mli_v1q, 0); + mld_dispatch_queue_locked(mli, &mli->mli_v1q, 0); } else if (mli->mli_version == MLD_VERSION_2) { MLI_UNLOCK(mli); - mld_dispatch_queue(NULL, &qrq, 0); - mld_dispatch_queue(NULL, &scq, 0); + mld_dispatch_queue_locked(NULL, &qrq, 0); + mld_dispatch_queue_locked(NULL, &scq, 0); VERIFY(qrq.ifq_len == 0); VERIFY(scq.ifq_len == 0); MLI_LOCK(mli); @@ -1694,12 +1765,27 @@ next: */ mld_flush_relq(mli, (struct mld_in6m_relhead *)&in6m_dthead); VERIFY(SLIST_EMPTY(&mli->mli_relinmhead)); + mli->mli_flags |= MLIF_PROCESSED; MLI_UNLOCK(mli); + MLI_REMREF(mli); IF_DRAIN(&qrq); IF_DRAIN(&scq); + + if (genid != mld_mli_list_genid) { + MLD_PRINTF(("%s: MLD information list changed " + "in the middle of iteration! Restart iteration.\n", + __func__)); + mli = LIST_FIRST(&mli_head); + genid = mld_mli_list_genid; + } else { + mli = LIST_NEXT(mli, mli_link); + } } + LIST_FOREACH(mli, &mli_head, mli_link) + mli->mli_flags &= ~MLIF_PROCESSED; + out_locked: /* re-arm the timer if there's work to do */ mld_timeout_run = 0; @@ -3407,7 +3493,7 @@ next: in6_multihead_lock_done(); MLI_LOCK(mli); - mld_dispatch_queue(mli, &mli->mli_gq, MLD_MAX_RESPONSE_BURST); + mld_dispatch_queue_locked(mli, &mli->mli_gq, MLD_MAX_RESPONSE_BURST); MLI_LOCK_ASSERT_HELD(mli); /* diff --git a/bsd/netinet6/mld6_var.h b/bsd/netinet6/mld6_var.h index 249506b85..c7623ea8d 100644 --- a/bsd/netinet6/mld6_var.h +++ b/bsd/netinet6/mld6_var.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2012 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -90,6 +90,7 @@ struct mld_ifinfo_u { #define MLIF_SILENT 0x00000001 /* Do not use MLD on this ifp */ #define MLIF_USEALLOW 0x00000002 /* Use ALLOW/BLOCK for joins/leaves */ +#define MLIF_PROCESSED 0x00000004 /* Entry has been processed and can be skipped */ /* * MLD version tag. @@ -193,10 +194,10 @@ struct mld_ifinfo { }; #define MLI_LOCK_ASSERT_HELD(_mli) \ - lck_mtx_assert(&(_mli)->mli_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_mli)->mli_lock, LCK_MTX_ASSERT_OWNED) #define MLI_LOCK_ASSERT_NOTHELD(_mli) \ - lck_mtx_assert(&(_mli)->mli_lock, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_mli)->mli_lock, LCK_MTX_ASSERT_NOTOWNED) #define MLI_LOCK(_mli) \ lck_mtx_lock(&(_mli)->mli_lock) diff --git a/bsd/netinet6/nd6.c b/bsd/netinet6/nd6.c index 2aef3803e..0a16cdb14 100644 --- a/bsd/netinet6/nd6.c +++ b/bsd/netinet6/nd6.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -91,6 +91,7 @@ #include <net/dlil.h> #include <net/ntstat.h> #include <net/net_osdep.h> +#include <net/nwk_wq.h> #include <netinet/in.h> #include <netinet/in_arp.h> @@ -226,7 +227,7 @@ static int nd6_sysctl_prlist SYSCTL_HANDLER_ARGS; * Insertion and removal from llinfo_nd6 must be done with rnh_lock held. */ #define LN_DEQUEUE(_ln) do { \ - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); \ + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); \ RT_LOCK_ASSERT_HELD((_ln)->ln_rt); \ (_ln)->ln_next->ln_prev = (_ln)->ln_prev; \ (_ln)->ln_prev->ln_next = (_ln)->ln_next; \ @@ -235,7 +236,7 @@ static int nd6_sysctl_prlist SYSCTL_HANDLER_ARGS; } while (0) #define LN_INSERTHEAD(_ln) do { \ - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); \ + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); \ RT_LOCK_ASSERT_HELD((_ln)->ln_rt); \ (_ln)->ln_next = llinfo_nd6.ln_next; \ llinfo_nd6.ln_next = (_ln); \ @@ -500,7 +501,7 @@ nd6_ifreset(struct ifnet *ifp) VERIFY(NULL != ndi); VERIFY(ndi->initialized); - lck_mtx_assert(&ndi->lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&ndi->lock, LCK_MTX_ASSERT_OWNED); ndi->linkmtu = ifp->if_mtu; ndi->chlim = IPV6_DEFHLIM; ndi->basereachable = REACHABLE_TIME; @@ -713,6 +714,7 @@ nd6_options(union nd_opts *ndopts) (struct nd_opt_prefix_info *)nd_opt; break; case ND_OPT_RDNSS: + case ND_OPT_DNSSL: /* ignore */ break; default: @@ -762,11 +764,12 @@ nd6_service(void *arg) struct ifnet *ifp = NULL; struct in6_ifaddr *ia6, *nia6; uint64_t timenow; - bool send_nc_failure_kev = false; + boolean_t send_nc_failure_kev = FALSE; struct nd_drhead nd_defrouter_tmp; struct nd_defrouter *ndr = NULL; + struct radix_node_head *rnh = rt_tables[AF_INET6]; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); /* * Since we may drop rnh_lock and nd6_mutex below, we want * to run this entire operation single threaded. @@ -778,7 +781,7 @@ nd6_service(void *arg) nd6_service_waiters++; (void) msleep(nd6_service_wc, rnh_lock, (PZERO-1), __func__, NULL); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); } /* We are busy now; tell everyone else to go away */ @@ -829,7 +832,7 @@ again: dlil_post_complete_msg(NULL, &ev_msg); } - send_nc_failure_kev = false; + send_nc_failure_kev = FALSE; ifp = NULL; /* * The global list llinfo_nd6 is modified by nd6_request() and is @@ -843,7 +846,7 @@ again: * pass thru the entries and clear the flag so they can be processed * during the next timeout. */ - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); ln = llinfo_nd6.ln_next; while (ln != NULL && ln != &llinfo_nd6) { @@ -852,6 +855,7 @@ again: struct llinfo_nd6 *next; u_int32_t retrans, flags; struct nd_ifinfo *ndi = NULL; + boolean_t is_router = FALSE; /* ln_next/prev/rt is protected by rnh_lock */ next = ln->ln_next; @@ -931,6 +935,7 @@ again: flags = ndi->flags; RT_LOCK_ASSERT_HELD(rt); + is_router = (rt->rt_flags & RTF_ROUTER) ? TRUE : FALSE; switch (ln->ln_state) { case ND6_LLINFO_INCOMPLETE: @@ -954,7 +959,7 @@ again: } else { struct mbuf *m = ln->ln_hold; ln->ln_hold = NULL; - send_nc_failure_kev = (rt->rt_flags & RTF_ROUTER) ? true : false; + send_nc_failure_kev = is_router; if (m != NULL) { RT_ADDREF_LOCKED(rt); RT_UNLOCK(rt); @@ -974,12 +979,34 @@ again: RT_UNLOCK(rt); lck_mtx_unlock(rnh_lock); } + + /* + * Enqueue work item to invoke callback for + * this route entry + */ + route_event_enqueue_nwk_wq_entry(rt, NULL, + ROUTE_LLENTRY_UNREACH, NULL, FALSE); nd6_free(rt); ap->killed++; lck_mtx_lock(rnh_lock); + /* + * nd6_free above would flush out the routing table of + * any cloned routes with same next-hop. + * Walk the tree anyways as there could be static routes + * left. + * + * We also already have a reference to rt that gets freed right + * after the block below executes. Don't need an extra reference + * on rt here. + */ + if (is_router) { + struct route_event rt_ev; + route_event_init(&rt_ev, rt, NULL, ROUTE_LLENTRY_UNREACH); + (void) rnh->rnh_walktree(rnh, route_event_walktree, (void *)&rt_ev); + } rtfree_locked(rt); } - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); goto again; case ND6_LLINFO_REACHABLE: @@ -987,8 +1014,24 @@ again: ND6_CACHE_STATE_TRANSITION(ln, ND6_LLINFO_STALE); ln_setexpire(ln, timenow + nd6_gctimer); ap->aging_lazy++; + /* + * Enqueue work item to invoke callback for + * this route entry + */ + route_event_enqueue_nwk_wq_entry(rt, NULL, + ROUTE_LLENTRY_STALE, NULL, TRUE); + + RT_ADDREF_LOCKED(rt); + RT_UNLOCK(rt); + if (is_router) { + struct route_event rt_ev; + route_event_init(&rt_ev, rt, NULL, ROUTE_LLENTRY_STALE); + (void) rnh->rnh_walktree(rnh, route_event_walktree, (void *)&rt_ev); + } + rtfree_locked(rt); + } else { + RT_UNLOCK(rt); } - RT_UNLOCK(rt); break; case ND6_LLINFO_STALE: @@ -1043,16 +1086,41 @@ again: ap->aging++; lck_mtx_lock(rnh_lock); } else { - send_nc_failure_kev = (rt->rt_flags & RTF_ROUTER) ? true : false; + is_router = (rt->rt_flags & RTF_ROUTER) ? TRUE : FALSE; + send_nc_failure_kev = is_router; RT_ADDREF_LOCKED(rt); RT_UNLOCK(rt); lck_mtx_unlock(rnh_lock); nd6_free(rt); ap->killed++; + + /* + * Enqueue work item to invoke callback for + * this route entry + */ + route_event_enqueue_nwk_wq_entry(rt, NULL, + ROUTE_LLENTRY_UNREACH, NULL, FALSE); + lck_mtx_lock(rnh_lock); + /* + * nd6_free above would flush out the routing table of + * any cloned routes with same next-hop. + * Walk the tree anyways as there could be static routes + * left. + * + * We also already have a reference to rt that gets freed right + * after the block below executes. Don't need an extra reference + * on rt here. + */ + if (is_router) { + struct route_event rt_ev; + route_event_init(&rt_ev, rt, NULL, ROUTE_LLENTRY_UNREACH); + (void) rnh->rnh_walktree(rnh, + route_event_walktree, (void *)&rt_ev); + } rtfree_locked(rt); } - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); goto again; default: @@ -1061,7 +1129,7 @@ again: } ln = next; } - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); /* Now clear the flag from all entries */ ln = llinfo_nd6.ln_next; @@ -1084,6 +1152,10 @@ again: TAILQ_FOREACH_SAFE(dr, &nd_defrouter, dr_entry, ndr) { ap->found++; if (dr->expire != 0 && dr->expire < timenow) { + VERIFY(dr->ifp != NULL); + in6_ifstat_inc(dr->ifp, ifs6_defrtr_expiry_cnt); + in6_event_enqueue_nwk_wq_entry(IN6_NDP_RTR_EXPIRY, dr->ifp, + &dr->rtaddr, dr->rtlifetime); if (dr->ifp != NULL && dr->ifp->if_type == IFT_CELLULAR) { /* @@ -1211,6 +1283,12 @@ addrloop: in6_purgeaddr(&ia6->ia_ifa); ap->killed++; + if ((ia6->ia6_flags & IN6_IFF_TEMPORARY) == 0) { + in6_ifstat_inc(ia6->ia_ifa.ifa_ifp, ifs6_addr_expiry_cnt); + in6_event_enqueue_nwk_wq_entry(IN6_NDP_ADDR_EXPIRY, + ia6->ia_ifa.ifa_ifp, &ia6->ia_addr.sin6_addr, + 0); + } /* Release extra reference taken above */ IFA_REMREF(&ia6->ia_ifa); goto addrloop; @@ -1228,9 +1306,17 @@ addrloop: IFA_LOCK_ASSERT_HELD(&ia6->ia_ifa); if (IFA6_IS_DEPRECATED(ia6, timenow)) { int oldflags = ia6->ia6_flags; - ia6->ia6_flags |= IN6_IFF_DEPRECATED; + /* + * Only enqueue the Deprecated event when the address just + * becomes deprecated + */ + if((oldflags & IN6_IFF_DEPRECATED) == 0) { + in6_event_enqueue_nwk_wq_entry(IN6_ADDR_MARKED_DEPRECATED, + ia6->ia_ifa.ifa_ifp, &ia6->ia_addr.sin6_addr, + 0); + } /* * If a temporary address has just become deprecated, * regenerate a new one if possible. @@ -1271,7 +1357,7 @@ addrloop: ia6->ia6_flags &= ~IN6_IFF_DEPRECATED; IFA_UNLOCK(&ia6->ia_ifa); } - lck_rw_assert(&in6_ifaddr_rwlock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(&in6_ifaddr_rwlock, LCK_RW_ASSERT_EXCLUSIVE); /* Release extra reference taken above */ IFA_REMREF(&ia6->ia_ifa); } @@ -1304,6 +1390,11 @@ addrloop: NDPR_ADDREF_LOCKED(pr); prelist_remove(pr); NDPR_UNLOCK(pr); + + in6_ifstat_inc(pr->ndpr_ifp, ifs6_pfx_expiry_cnt); + in6_event_enqueue_nwk_wq_entry(IN6_NDP_PFX_EXPIRY, + pr->ndpr_ifp, &pr->ndpr_prefix.sin6_addr, + 0); NDPR_REMREF(pr); pfxlist_onlink_check(); pr = nd_prefix.lh_first; @@ -1406,7 +1497,7 @@ nd6_sched_timeout(struct timeval *atv, struct timeval *ltv) { struct timeval tv; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); if (atv == NULL) { tv.tv_usec = 0; tv.tv_sec = MAX(nd6_prune, 1); @@ -1466,7 +1557,7 @@ nd6_sched_timeout(struct timeval *atv, struct timeval *ltv) */ void nd6_post_msg(u_int32_t code, struct nd_prefix_list *prefix_list, - u_int32_t list_length, u_int32_t mtu, char *dl_addr, u_int32_t dl_addr_len) + u_int32_t list_length, u_int32_t mtu) { struct kev_msg ev_msg; struct kev_nd6_ra_data nd6_ra_msg_data; @@ -1479,9 +1570,6 @@ nd6_post_msg(u_int32_t code, struct nd_prefix_list *prefix_list, ev_msg.event_code = code; bzero(&nd6_ra_msg_data, sizeof (nd6_ra_msg_data)); - nd6_ra_msg_data.lladdrlen = (dl_addr_len <= ND6_ROUTER_LL_SIZE) ? - dl_addr_len : ND6_ROUTER_LL_SIZE; - bcopy(dl_addr, &nd6_ra_msg_data.lladdr, nd6_ra_msg_data.lladdrlen); if (mtu > 0 && mtu >= IPV6_MMTU) { nd6_ra_msg_data.mtu = mtu; @@ -1750,7 +1838,7 @@ again: */ nd6_free(rt); RT_REMREF(rt); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); goto again; } else { RT_UNLOCK(rt); @@ -1781,7 +1869,7 @@ nd6_lookup(struct in6_addr *addr6, int create, struct ifnet *ifp, int rt_locked) ifscope = (ifp != NULL) ? ifp->if_index : IFSCOPE_NONE; if (rt_locked) { - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); rt = rtalloc1_scoped_locked(SA(&sin6), create, 0, ifscope); } else { rt = rtalloc1_scoped(SA(&sin6), create, 0, ifscope); @@ -1924,7 +2012,7 @@ nd6_is_new_addr_neighbor(struct sockaddr_in6 *addr, struct ifnet *ifp) struct nd_prefix *pr; struct ifaddr *dstaddr; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); /* * A link-local address is always a neighbor. @@ -2001,7 +2089,7 @@ nd6_is_addr_neighbor(struct sockaddr_in6 *addr, struct ifnet *ifp, { struct rtentry *rt; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(nd6_mutex); if (nd6_is_new_addr_neighbor(addr, ifp)) { lck_mtx_unlock(nd6_mutex); @@ -2036,7 +2124,7 @@ nd6_free(struct rtentry *rt) struct in6_addr in6; struct nd_defrouter *dr; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); RT_LOCK_ASSERT_NOTHELD(rt); lck_mtx_lock(nd6_mutex); @@ -2134,7 +2222,7 @@ nd6_rtrequest(int req, struct rtentry *rt, struct sockaddr *sa) VERIFY((NULL != ndi) && (TRUE == ndi->initialized)); VERIFY(nd6_init_done); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK_ASSERT_HELD(rt); /* @@ -2497,7 +2585,7 @@ nd6_siocgdrlst(void *data, int data_is_64) struct nd_defrouter *dr; int i = 0; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); dr = TAILQ_FIRST(&nd_defrouter); @@ -2580,7 +2668,7 @@ nd6_siocgprlst(void *data, int data_is_64) struct nd_prefix *pr; int i = 0; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); pr = nd_prefix.lh_first; @@ -3172,7 +3260,7 @@ fail: * 1 -- y -- (7) * STALE */ - if (lladdr) { /* (3-5) and (7) */ + if (lladdr != NULL) { /* (3-5) and (7) */ /* * Record source link-layer address * XXX is it dependent to ifp->if_type? @@ -3184,7 +3272,7 @@ fail: nd6_llreach_alloc(rt, ifp, LLADDR(sdl), sdl->sdl_alen, FALSE); } - if (!is_newentry) { + if (is_newentry == 0) { if ((!olladdr && lladdr != NULL) || /* (3) */ (olladdr && lladdr != NULL && llchange)) { /* (5) */ do_update = 1; @@ -3315,6 +3403,39 @@ fail: break; } + if (do_update) { + int route_ev_code = 0; + + if (llchange) + route_ev_code = ROUTE_LLENTRY_CHANGED; + else + route_ev_code = ROUTE_LLENTRY_RESOLVED; + + /* Enqueue work item to invoke callback for this route entry */ + route_event_enqueue_nwk_wq_entry(rt, NULL, route_ev_code, NULL, TRUE); + + if (ln->ln_router || (rt->rt_flags & RTF_ROUTER)) { + struct radix_node_head *rnh = NULL; + struct route_event rt_ev; + route_event_init(&rt_ev, rt, NULL, llchange ? ROUTE_LLENTRY_CHANGED : + ROUTE_LLENTRY_RESOLVED); + /* + * We already have a valid reference on rt. + * The function frees that before returning. + * We therefore don't need an extra reference here + */ + RT_UNLOCK(rt); + lck_mtx_lock(rnh_lock); + + rnh = rt_tables[AF_INET6]; + if (rnh != NULL) + (void) rnh->rnh_walktree(rnh, route_event_walktree, + (void *)&rt_ev); + lck_mtx_unlock(rnh_lock); + RT_LOCK(rt); + } + } + /* * When the link-layer address of a router changes, select the * best router again. In particular, when the neighbor entry is newly @@ -3748,8 +3869,7 @@ lookup: if (ln->ln_hold) m_freem_list(ln->ln_hold); ln->ln_hold = m0; - if (ln->ln_expire != 0 && ln->ln_asked < nd6_mmaxtries && - ln->ln_expire <= timenow) { + if (!ND6_LLINFO_PERMANENT(ln) && ln->ln_asked == 0) { ln->ln_asked++; ndi = ND_IFINFO(ifp); VERIFY(ndi != NULL && ndi->initialized); @@ -3767,9 +3887,6 @@ lookup: nd6_sched_timeout(NULL, NULL); lck_mtx_unlock(rnh_lock); } else { - if(ln->ln_state == ND6_LLINFO_INCOMPLETE) { - ln_setexpire(ln, timenow); - } RT_UNLOCK(rt); } /* diff --git a/bsd/netinet6/nd6.h b/bsd/netinet6/nd6.h index bd71a5a9b..13bb3e963 100644 --- a/bsd/netinet6/nd6.h +++ b/bsd/netinet6/nd6.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -70,6 +70,7 @@ #include <net/flowadv.h> #include <kern/locks.h> #include <sys/tree.h> +#include <sys/eventhandler.h> #include <netinet6/nd6_var.h> struct llinfo_nd6 { @@ -117,17 +118,18 @@ struct llinfo_nd6 { #ifdef BSD_KERNEL_PRIVATE #define ND6_CACHE_STATE_TRANSITION(ln, nstate) do {\ + struct rtentry *ln_rt = (ln)->ln_rt; \ if (nd6_debug >= 1) {\ nd6log((LOG_INFO,\ "[%s:%d]: NDP cache entry changed from %s -> %s",\ - __FILE__,\ + __func__,\ __LINE__,\ ndcache_state2str((ln)->ln_state),\ ndcache_state2str(nstate)));\ - if ((ln)->ln_rt)\ + if (ln_rt != NULL)\ nd6log((LOG_INFO,\ " for address: %s.\n",\ - ip6_sprintf(&SIN6(rt_key((ln)->ln_rt))->sin6_addr)));\ + ip6_sprintf(&SIN6(rt_key(ln_rt))->sin6_addr)));\ else\ nd6log((LOG_INFO, "\n"));\ }\ @@ -532,10 +534,10 @@ struct nd_defrouter { }; #define NDDR_LOCK_ASSERT_HELD(_nddr) \ - lck_mtx_assert(&(_nddr)->nddr_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_nddr)->nddr_lock, LCK_MTX_ASSERT_OWNED) #define NDDR_LOCK_ASSERT_NOTHELD(_nddr) \ - lck_mtx_assert(&(_nddr)->nddr_lock, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_nddr)->nddr_lock, LCK_MTX_ASSERT_NOTOWNED) #define NDDR_LOCK(_nddr) \ lck_mtx_lock(&(_nddr)->nddr_lock) @@ -610,10 +612,10 @@ struct nd_prefix { #define NDPR_KEEP_EXPIRED (1800 * 2) #define NDPR_LOCK_ASSERT_HELD(_ndpr) \ - lck_mtx_assert(&(_ndpr)->ndpr_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_ndpr)->ndpr_lock, LCK_MTX_ASSERT_OWNED) #define NDPR_LOCK_ASSERT_NOTHELD(_ndpr) \ - lck_mtx_assert(&(_ndpr)->ndpr_lock, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_ndpr)->ndpr_lock, LCK_MTX_ASSERT_NOTOWNED) #define NDPR_LOCK(_ndpr) \ lck_mtx_lock(&(_ndpr)->ndpr_lock) @@ -691,9 +693,6 @@ struct kev_nd6_ndalive { struct net_event_data link_data; }; -/* ND6 RA L2 source address length */ -#define ND6_ROUTER_LL_SIZE 64 - struct nd6_ra_prefix { struct sockaddr_in6 prefix; struct prf_ra raflags; @@ -713,8 +712,6 @@ struct nd6_ra_prefix { #define KEV_ND6_DATA_VALID_PREFIX (0x1 << 1) struct kev_nd6_ra_data { - u_int8_t lladdr[ND6_ROUTER_LL_SIZE]; - u_int32_t lladdrlen; u_int32_t mtu; u_int32_t list_index; u_int32_t list_length; @@ -723,6 +720,12 @@ struct kev_nd6_ra_data { u_int32_t pad; }; +struct kev_nd6_event { + struct net_event_data link_data; + struct in6_addr in6_address; + uint32_t val; +}; + struct nd6_lookup_ipv6_args { char ifname[IFNAMSIZ]; struct sockaddr_in6 ip6_dest; @@ -754,6 +757,7 @@ extern int nd6_debug; extern int nd6_onlink_ns_rfc4861; extern int nd6_optimistic_dad; +#define nd6log0(x) do { log x; } while (0) #define nd6log(x) do { if (nd6_debug >= 1) log x; } while (0) #define nd6log2(x) do { if (nd6_debug >= 2) log x; } while (0) @@ -839,7 +843,7 @@ extern int nd6_storelladdr(struct ifnet *, struct rtentry *, struct mbuf *, extern int nd6_need_cache(struct ifnet *); extern void nd6_drain(void *); extern void nd6_post_msg(u_int32_t, struct nd_prefix_list *, u_int32_t, - u_int32_t, char *, u_int32_t); + u_int32_t); extern int nd6_setifinfo(struct ifnet *, u_int32_t, u_int32_t); extern const char *ndcache_state2str(short); extern void ln_setexpire(struct llinfo_nd6 *, uint64_t); diff --git a/bsd/netinet6/nd6_nbr.c b/bsd/netinet6/nd6_nbr.c index 96eb73444..855b65e4c 100644 --- a/bsd/netinet6/nd6_nbr.c +++ b/bsd/netinet6/nd6_nbr.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -81,6 +81,7 @@ #include <net/if_llreach.h> #include <net/route.h> #include <net/dlil.h> +#include <net/nwk_wq.h> #include <netinet/in.h> #include <netinet/in_var.h> @@ -121,10 +122,10 @@ static struct zone *dad_zone; /* zone for dadq */ #define DAD_ZONE_NAME "nd6_dad" /* zone name */ #define DAD_LOCK_ASSERT_HELD(_dp) \ - lck_mtx_assert(&(_dp)->dad_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(_dp)->dad_lock, LCK_MTX_ASSERT_OWNED) #define DAD_LOCK_ASSERT_NOTHELD(_dp) \ - lck_mtx_assert(&(_dp)->dad_lock, LCK_MTX_ASSERT_NOTOWNED) + LCK_MTX_ASSERT(&(_dp)->dad_lock, LCK_MTX_ASSERT_NOTOWNED) #define DAD_LOCK(_dp) \ lck_mtx_lock(&(_dp)->dad_lock) @@ -272,11 +273,6 @@ nd6_ns_input( boolean_t is_dad_probe; int oflgclr = 0; - if ((ifp->if_eflags & IFEF_IPV6_ND6ALT) != 0) { - nd6log((LOG_INFO, "nd6_ns_input: on ND6ALT interface!\n")); - return; - } - /* Expect 32-bit aligned data pointer on strict-align platforms */ MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(m); @@ -1049,25 +1045,46 @@ nd6_na_input(struct mbuf *m, int off, int icmp6len) ND6_CACHE_STATE_TRANSITION(ln, ND6_LLINFO_STALE); ln_setexpire(ln, timenow + nd6_gctimer); } + + + /* + * Enqueue work item to invoke callback for this + * route entry + */ + route_event_enqueue_nwk_wq_entry(rt, NULL, + ROUTE_LLENTRY_RESOLVED, NULL, TRUE); + if ((ln->ln_router = is_router) != 0) { + struct radix_node_head *rnh = NULL; + struct route_event rt_ev; + route_event_init(&rt_ev, rt, NULL, ROUTE_LLENTRY_RESOLVED); /* * This means a router's state has changed from * non-reachable to probably reachable, and might * affect the status of associated prefixes.. + * We already have a reference on rt. Don't need to + * take one for the unlock/lock. */ RT_UNLOCK(rt); + lck_mtx_lock(rnh_lock); + rnh = rt_tables[AF_INET6]; + + if (rnh != NULL) + (void) rnh->rnh_walktree(rnh, route_event_walktree, + (void *)&rt_ev); + lck_mtx_unlock(rnh_lock); lck_mtx_lock(nd6_mutex); pfxlist_onlink_check(); lck_mtx_unlock(nd6_mutex); RT_LOCK(rt); } } else { - int llchange; + int llchange = 0; /* * Check if the link-layer address has changed or not. */ - if (!lladdr) + if (lladdr == NULL) llchange = 0; else { if (sdl->sdl_alen) { @@ -1080,7 +1097,7 @@ nd6_na_input(struct mbuf *m, int off, int icmp6len) } /* - * This is VERY complex. Look at it with care. + * This is VERY complex. Look at it with care. * * override solicit lladdr llchange action * (L: record lladdr) @@ -1149,6 +1166,44 @@ nd6_na_input(struct mbuf *m, int off, int icmp6len) ln_setexpire(ln, timenow + nd6_gctimer); } } + + /* + * XXX + * The above is somewhat convoluted, for now just + * issue a callback for LLENTRY changed. + */ + /* Enqueue work item to invoke callback for this route entry */ + route_event_enqueue_nwk_wq_entry(rt, NULL, + ROUTE_LLENTRY_CHANGED, NULL, TRUE); + + /* + * If the entry is no longer a router, the logic post this processing + * gets rid of all the route entries having the current entry as a next + * hop. + * So only walk the tree here when there's no such change. + */ + if (ln->ln_router && is_router) { + struct radix_node_head *rnh = NULL; + struct route_event rt_ev; + route_event_init(&rt_ev, rt, NULL, ROUTE_LLENTRY_CHANGED); + /* + * This means a router's state has changed from + * non-reachable to probably reachable, and might + * affect the status of associated prefixes.. + * + * We already have a valid rt reference here. + * We don't need to take another one for unlock/lock. + */ + RT_UNLOCK(rt); + lck_mtx_lock(rnh_lock); + rnh = rt_tables[AF_INET6]; + + if (rnh != NULL) + (void) rnh->rnh_walktree(rnh, route_event_walktree, + (void *)&rt_ev); + lck_mtx_unlock(rnh_lock); + RT_LOCK(rt); + } } if (ln->ln_router && !is_router) { @@ -1244,13 +1299,15 @@ nd6_na_input(struct mbuf *m, int off, int icmp6len) } RT_REMREF_LOCKED(rt); RT_UNLOCK(rt); + m_freem(m); + return; bad: icmp6stat.icp6s_badna++; /* fall through */ - freeit: m_freem(m); + return; } /* @@ -1614,11 +1671,11 @@ nd6_dad_start( */ IFA_LOCK(&ia->ia_ifa); if (!(ia->ia6_flags & IN6_IFF_DADPROGRESS)) { - log(LOG_DEBUG, + nd6log0((LOG_DEBUG, "nd6_dad_start: not a tentative or optimistic address " "%s(%s)\n", ip6_sprintf(&ia->ia_addr.sin6_addr), - ifa->ifa_ifp ? if_name(ifa->ifa_ifp) : "???"); + ifa->ifa_ifp ? if_name(ifa->ifa_ifp) : "???")); IFA_UNLOCK(&ia->ia_ifa); return; } @@ -1642,10 +1699,10 @@ nd6_dad_start( dp = zalloc(dad_zone); if (dp == NULL) { - log(LOG_ERR, "nd6_dad_start: memory allocation failed for " + nd6log0((LOG_ERR, "nd6_dad_start: memory allocation failed for " "%s(%s)\n", ip6_sprintf(&ia->ia_addr.sin6_addr), - ifa->ifa_ifp ? if_name(ifa->ifa_ifp) : "???"); + ifa->ifa_ifp ? if_name(ifa->ifa_ifp) : "???")); return; } bzero(dp, dad_size); @@ -1654,7 +1711,7 @@ nd6_dad_start( /* Callee adds one reference for us */ dp = nd6_dad_attach(dp, ifa); - nd6log((LOG_DEBUG, "%s: starting %sDAD %sfor %s\n", + nd6log0((LOG_DEBUG, "%s: starting %sDAD %sfor %s\n", if_name(ifa->ifa_ifp), (ia->ia6_flags & IN6_IFF_OPTIMISTIC) ? "optimistic " : "", (tick_delay == NULL) ? "immediately " : "", @@ -1793,7 +1850,7 @@ nd6_dad_timer(struct ifaddr *ifa) /* Sanity check */ if (ia == NULL) { - log(LOG_ERR, "nd6_dad_timer: called with null parameter\n"); + nd6log0((LOG_ERR, "nd6_dad_timer: called with null parameter\n")); goto done; } @@ -1805,23 +1862,23 @@ nd6_dad_timer(struct ifaddr *ifa) dp = nd6_dad_find(ifa, NULL); if (dp == NULL) { - log(LOG_ERR, "nd6_dad_timer: DAD structure not found\n"); + nd6log0((LOG_ERR, "nd6_dad_timer: DAD structure not found\n")); goto done; } IFA_LOCK(&ia->ia_ifa); if (ia->ia6_flags & IN6_IFF_DUPLICATED) { - log(LOG_ERR, "nd6_dad_timer: called with duplicated address " + nd6log0((LOG_ERR, "nd6_dad_timer: called with duplicated address " "%s(%s)\n", ip6_sprintf(&ia->ia_addr.sin6_addr), - ifa->ifa_ifp ? if_name(ifa->ifa_ifp) : "???"); + ifa->ifa_ifp ? if_name(ifa->ifa_ifp) : "???")); IFA_UNLOCK(&ia->ia_ifa); goto done; } if ((ia->ia6_flags & IN6_IFF_DADPROGRESS) == 0) { - log(LOG_ERR, "nd6_dad_timer: not a tentative or optimistic " + nd6log0((LOG_ERR, "nd6_dad_timer: not a tentative or optimistic " "address %s(%s)\n", ip6_sprintf(&ia->ia_addr.sin6_addr), - ifa->ifa_ifp ? if_name(ifa->ifa_ifp) : "???"); + ifa->ifa_ifp ? if_name(ifa->ifa_ifp) : "???")); IFA_UNLOCK(&ia->ia_ifa); goto done; } @@ -1831,7 +1888,7 @@ nd6_dad_timer(struct ifaddr *ifa) DAD_LOCK(dp); if (dp->dad_ns_tcount > dad_maxtry) { DAD_UNLOCK(dp); - nd6log((LOG_INFO, "%s: could not run DAD, driver problem?\n", + nd6log0((LOG_INFO, "%s: could not run DAD, driver problem?\n", if_name(ifa->ifa_ifp))); nd6_dad_detach(dp, ifa); @@ -1859,7 +1916,7 @@ nd6_dad_timer(struct ifaddr *ifa) if (dp->dad_na_icount > 0 || dp->dad_ns_icount) { /* We've seen NS or NA, means DAD has failed. */ DAD_UNLOCK(dp); - nd6log((LOG_INFO, + nd6log0((LOG_INFO, "%s: duplicate IPv6 address %s [timer]\n", __func__, ip6_sprintf(&ia->ia_addr.sin6_addr), if_name(ia->ia_ifp))); @@ -1883,7 +1940,7 @@ nd6_dad_timer(struct ifaddr *ifa) * additional probes until the loopback condition * becomes clear when a looped back probe is detected. */ - nd6log((LOG_INFO, + nd6log0((LOG_INFO, "%s: a looped back NS message is " "detected during DAD for %s. " "Another DAD probe is being sent on interface.\n", @@ -1917,14 +1974,14 @@ nd6_dad_timer(struct ifaddr *ifa) nd6_unsol_na_output(ifa); } - nd6log((LOG_DEBUG, + nd6log0((LOG_DEBUG, "%s: DAD complete for %s - no duplicates found%s\n", if_name(ifa->ifa_ifp), ip6_sprintf(&ia->ia_addr.sin6_addr), txunsolna ? ", tx unsolicited NA with O=1" : ".")); if (dp->dad_ns_lcount > 0) - nd6log((LOG_DEBUG, + nd6log0((LOG_DEBUG, "%s: DAD completed while " "a looped back NS message is detected " "during DAD for %s om interface %s\n", @@ -2029,6 +2086,9 @@ nd6_dad_duplicated(struct ifaddr *ifa) ia->ia6_flags &= ~IN6_IFF_DADPROGRESS; ia->ia6_flags |= IN6_IFF_DUPLICATED; + in6_event_enqueue_nwk_wq_entry(IN6_ADDR_MARKED_DUPLICATED, + ia->ia_ifa.ifa_ifp, &ia->ia_addr.sin6_addr, + 0); IFA_UNLOCK(&ia->ia_ifa); /* increment DAD collision counter */ @@ -2382,7 +2442,7 @@ nd6_alt_node_present(struct ifnet *ifp, struct sockaddr_in6 *sin6, { struct rtentry *rt; struct llinfo_nd6 *ln; - struct if_llreach *lr; + struct if_llreach *lr = NULL; const uint16_t temp_embedded_id = sin6->sin6_addr.s6_addr16[1]; if (IN6_IS_SCOPE_LINKLOCAL(&sin6->sin6_addr) && @@ -2392,7 +2452,7 @@ nd6_alt_node_present(struct ifnet *ifp, struct sockaddr_in6 *sin6, nd6_cache_lladdr(ifp, &sin6->sin6_addr, LLADDR(sdl), sdl->sdl_alen, ND_NEIGHBOR_ADVERT, 0); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(rnh_lock); rt = rtalloc1_scoped_locked((struct sockaddr *)sin6, 1, 0, @@ -2449,7 +2509,7 @@ nd6_alt_node_absent(struct ifnet *ifp, struct sockaddr_in6 *sin6) (temp_embedded_id == 0)) sin6->sin6_addr.s6_addr16[1] = htons(ifp->if_index); - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(rnh_lock); rt = rtalloc1_scoped_locked((struct sockaddr *)sin6, 0, 0, diff --git a/bsd/netinet6/nd6_prproxy.c b/bsd/netinet6/nd6_prproxy.c index c85830994..bb7f1448b 100644 --- a/bsd/netinet6/nd6_prproxy.c +++ b/bsd/netinet6/nd6_prproxy.c @@ -285,13 +285,13 @@ nd6_prproxy_prelist_setroute(boolean_t enable, struct nd6_prproxy_prelist *up, *down, *ndprl_tmp; struct nd_prefix *pr; - lck_mtx_assert(&proxy6_lock, LCK_MTX_ASSERT_OWNED); - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(&proxy6_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); SLIST_FOREACH_SAFE(up, up_head, ndprl_le, ndprl_tmp) { struct rtentry *rt; boolean_t prproxy, set_allmulti = FALSE; - int allmulti_sw; + int allmulti_sw = FALSE; struct ifnet *ifp = NULL; SLIST_REMOVE(up_head, up, nd6_prproxy_prelist, ndprl_le); @@ -347,7 +347,7 @@ nd6_prproxy_prelist_setroute(boolean_t enable, struct nd_prefix *pr_up; struct rtentry *rt; boolean_t prproxy, set_allmulti = FALSE; - int allmulti_sw; + int allmulti_sw = FALSE; struct ifnet *ifp = NULL; SLIST_REMOVE(down_head, down, nd6_prproxy_prelist, ndprl_le); @@ -744,7 +744,7 @@ nd6_prproxy_prelist_update(struct nd_prefix *pr_cur, struct nd_prefix *pr_up) SLIST_INIT(&down_head); VERIFY(pr_cur != NULL); - lck_mtx_assert(&proxy6_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(&proxy6_lock, LCK_MTX_ASSERT_OWNED); /* * Upstream prefix. If caller did not specify one, search for one @@ -857,7 +857,7 @@ nd6_prproxy_ifaddr(struct in6_ifaddr *ia) u_int32_t pr_len; boolean_t proxied = FALSE; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); IFA_LOCK(&ia->ia_ifa); bcopy(&ia->ia_addr.sin6_addr, &addr, sizeof (addr)); diff --git a/bsd/netinet6/nd6_rtr.c b/bsd/netinet6/nd6_rtr.c index be938f59e..8502334ea 100644 --- a/bsd/netinet6/nd6_rtr.c +++ b/bsd/netinet6/nd6_rtr.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003-2016 Apple Inc. All rights reserved. + * Copyright (c) 2003-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -382,7 +382,10 @@ nd6_ra_input( struct nd_defrouter dr0; u_int32_t advreachable; - +#if (DEVELOPMENT || DEBUG) + if (ip6_accept_rtadv == 0) + goto freeit; +#endif /* (DEVELOPMENT || DEBUG) */ /* Expect 32-bit aligned data pointer on strict-align platforms */ MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(m); @@ -685,7 +688,7 @@ skip: /* Post message */ nd6_post_msg(KEV_ND6_RA, nd_prefix_list_head, nd_prefix_list_length, - mtu, lladdr, lladdrlen); + mtu); /* * Installing a link-layer address might change the state of the @@ -751,7 +754,7 @@ defrouter_addreq(struct nd_defrouter *new, boolean_t scoped) int err; struct nd_ifinfo *ndi = ND_IFINFO(new->ifp); - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); NDDR_LOCK_ASSERT_NOTHELD(new); /* * We're free to lock and unlock NDDR because our callers @@ -854,7 +857,7 @@ defrouter_lookup( { struct nd_defrouter *dr; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); for (dr = TAILQ_FIRST(&nd_defrouter); dr; dr = TAILQ_NEXT(dr, dr_entry)) { @@ -883,7 +886,7 @@ defrouter_delreq(struct nd_defrouter *dr) unsigned int ifscope; int err; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); NDDR_LOCK_ASSERT_NOTHELD(dr); /* * We're free to lock and unlock NDDR because our callers @@ -963,7 +966,7 @@ defrouter_reset(void) { struct nd_defrouter *dr, drany; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); dr = TAILQ_FIRST(&nd_defrouter); while (dr) { @@ -1092,7 +1095,7 @@ defrtrlist_del(struct nd_defrouter *dr) struct nd_ifinfo *ndi = NULL; boolean_t resetmtu; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); #if (DEVELOPMENT || DEBUG) /* @@ -1292,7 +1295,7 @@ defrouter_select(struct ifnet *ifp) unsigned int genid = 0; boolean_t is_installed_reachable = FALSE; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); if (ifp == NULL) { nd6log2((LOG_INFO, @@ -1358,7 +1361,7 @@ defrouter_select(struct ifnet *ifp) nd_defrouter_waiters++; msleep(nd_defrouter_waitchan, nd6_mutex, (PZERO-1), __func__, NULL); - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); } nd_defrouter_busy = TRUE; @@ -1643,7 +1646,7 @@ out: NDDR_REMREF(selected_dr); if (installed_dr) NDDR_REMREF(installed_dr); - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); VERIFY(nd_defrouter_busy); nd_defrouter_busy = FALSE; if (nd_defrouter_waiters > 0) { @@ -1660,7 +1663,7 @@ defrtrlist_update_common(struct nd_defrouter *new, boolean_t scoped) struct nd_ifinfo *ndi = NULL; struct timeval caltime; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); if ((dr = defrouter_lookup(&new->rtaddr, ifp)) != NULL) { /* entry exists */ @@ -1808,7 +1811,7 @@ defrtrlist_update(struct nd_defrouter *new) { struct nd_defrouter *dr; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); dr = defrtrlist_update_common(new, (nd6_defifp != NULL && new->ifp != nd6_defifp)); @@ -1820,7 +1823,7 @@ pfxrtr_lookup(struct nd_prefix *pr, struct nd_defrouter *dr) { struct nd_pfxrouter *search; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); NDPR_LOCK_ASSERT_HELD(pr); for (search = pr->ndpr_advrtrs.lh_first; search; @@ -1837,7 +1840,7 @@ pfxrtr_add(struct nd_prefix *pr, struct nd_defrouter *dr) { struct nd_pfxrouter *new; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); NDPR_LOCK_ASSERT_NOTHELD(pr); new = zalloc(ndprtr_zone); @@ -1857,7 +1860,7 @@ pfxrtr_add(struct nd_prefix *pr, struct nd_defrouter *dr) static void pfxrtr_del(struct nd_pfxrouter *pfr, struct nd_prefix *pr) { - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); NDPR_LOCK_ASSERT_HELD(pr); pr->ndpr_genid++; LIST_REMOVE(pfr, pfr_entry); @@ -2006,7 +2009,7 @@ prelist_remove(struct nd_prefix *pr) int e; struct nd_ifinfo *ndi = NULL; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); NDPR_LOCK_ASSERT_HELD(pr); if (pr->ndpr_stateflags & NDPRF_DEFUNCT) @@ -2717,7 +2720,7 @@ find_pfxlist_reachable_router(struct nd_prefix *pr) struct in6_addr rtaddr; unsigned int genid; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); NDPR_LOCK_ASSERT_HELD(pr); genid = pr->ndpr_genid; @@ -2783,13 +2786,13 @@ pfxlist_onlink_check(void) struct ifaddr **ifap = NULL; struct nd_prefix *ndpr; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); while (nd_prefix_busy) { nd_prefix_waiters++; msleep(nd_prefix_waitchan, nd6_mutex, (PZERO-1), __func__, NULL); - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); } nd_prefix_busy = TRUE; @@ -3070,6 +3073,9 @@ pfxlist_onlink_check(void) NDPR_UNLOCK(ndpr); IFA_LOCK(&ifa->ia_ifa); ifa->ia6_flags |= IN6_IFF_DETACHED; + in6_event_enqueue_nwk_wq_entry(IN6_ADDR_MARKED_DETACHED, + ifa->ia_ifa.ifa_ifp, &ifa->ia_addr.sin6_addr, + 0); IFA_UNLOCK(&ifa->ia_ifa); } NDPR_REMREF(ndpr); @@ -3101,7 +3107,7 @@ nd6_prefix_equal_lookup(struct nd_prefix *pr, boolean_t primary_only) { struct nd_prefix *opr; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); for (opr = nd_prefix.lh_first; opr; opr = opr->ndpr_next) { if (opr == pr) @@ -3137,7 +3143,7 @@ nd6_prefix_sync(struct ifnet *ifp) struct nd_prefix *pr, *opr; int err = 0; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); if (ifp == NULL) return; @@ -3240,7 +3246,7 @@ nd6_prefix_onlink_common(struct nd_prefix *pr, boolean_t force_scoped, int error = 0, prproxy = 0; struct rtentry *rt = NULL; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_OWNED); /* sanity check */ NDPR_LOCK(pr); @@ -3453,7 +3459,7 @@ nd6_prefix_offlink(struct nd_prefix *pr) struct rtentry *rt = NULL, *ndpr_rt = NULL; unsigned int ifscope; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ NDPR_LOCK(pr); @@ -3963,7 +3969,7 @@ rt6_deleteroute( struct rtentry *rt = (struct rtentry *)rn; struct in6_addr *gate = (struct in6_addr *)arg; - lck_mtx_assert(rnh_lock, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(rnh_lock, LCK_MTX_ASSERT_OWNED); RT_LOCK(rt); if (rt->rt_gateway == NULL || rt->rt_gateway->sa_family != AF_INET6) { @@ -4010,7 +4016,7 @@ nd6_setdefaultiface( int error = 0; ifnet_t def_ifp = NULL; - lck_mtx_assert(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(nd6_mutex, LCK_MTX_ASSERT_NOTOWNED); ifnet_head_lock_shared(); if (ifindex < 0 || if_index < ifindex) { diff --git a/bsd/netinet6/nd6_send.c b/bsd/netinet6/nd6_send.c index 2b99a3bf4..cc7d35a5f 100644 --- a/bsd/netinet6/nd6_send.c +++ b/bsd/netinet6/nd6_send.c @@ -53,12 +53,12 @@ SYSCTL_NODE(_net_inet6, OID_AUTO, send, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "IPv6 Secure Neighbor Discovery"); static int nd6_send_opmode = ND6_SEND_OPMODE_CGA_QUIET; -SYSCTL_INT(_net_inet6_send, OID_AUTO, opstate, CTLFLAG_RD | CTLFLAG_LOCKED, - &nd6_send_opstate, 0, "current SEND operating state"); +SYSCTL_INT(_net_inet6_send, OID_AUTO, opmode, CTLFLAG_RW | CTLFLAG_LOCKED, + &nd6_send_opmode, 0, "configured SEND operating mode"); int nd6_send_opstate = ND6_SEND_OPMODE_DISABLED; -SYSCTL_INT(_net_inet6_send, OID_AUTO, opmode, CTLFLAG_RW | CTLFLAG_LOCKED, - &nd6_send_opmode, 0, "configured SEND operating mode"); +SYSCTL_INT(_net_inet6_send, OID_AUTO, opstate, CTLFLAG_RD | CTLFLAG_LOCKED, + &nd6_send_opstate, 0, "current SEND operating state"); static int sysctl_cga_parameters SYSCTL_HANDLER_ARGS; diff --git a/bsd/netinet6/raw_ip6.c b/bsd/netinet6/raw_ip6.c index 4fff23a3c..d7c6f689f 100644 --- a/bsd/netinet6/raw_ip6.c +++ b/bsd/netinet6/raw_ip6.c @@ -102,6 +102,7 @@ #include <sys/systm.h> #include <net/if.h> +#include <net/net_api_stats.h> #include <net/route.h> #include <net/if_types.h> @@ -282,7 +283,8 @@ void rip6_ctlinput( int cmd, struct sockaddr *sa, - void *d) + void *d, + __unused struct ifnet *ifp) { struct ip6_hdr *ip6; struct mbuf *m; @@ -667,8 +669,9 @@ rip6_output( * the route interface index used by IP. */ if (rt != NULL && - (outif = rt->rt_ifp) != in6p->in6p_last_outifp) + (outif = rt->rt_ifp) != in6p->in6p_last_outifp) { in6p->in6p_last_outifp = outif; + } } else { ROUTE_RELEASE(&in6p->in6p_route); } @@ -886,6 +889,7 @@ rip6_bind(struct socket *so, struct sockaddr *nam, struct proc *p) } inp->in6p_laddr = sin6.sin6_addr; inp->in6p_last_outifp = outif; + return (0); } @@ -915,6 +919,12 @@ rip6_connect(struct socket *so, struct sockaddr *nam, __unused struct proc *p) return EADDRNOTAVAIL; if (addr->sin6_family != AF_INET6) return EAFNOSUPPORT; + + if (!(so->so_flags1 & SOF1_CONNECT_COUNTED)) { + so->so_flags1 |= SOF1_CONNECT_COUNTED; + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_inet6_dgram_connected); + } + #if ENABLE_DEFAULT_SCOPE if (addr->sin6_scope_id == 0) { /* not change if specified */ /* avoid overwrites */ @@ -941,6 +951,7 @@ rip6_connect(struct socket *so, struct sockaddr *nam, __unused struct proc *p) if (inp->in6p_route.ro_rt != NULL) outif = inp->in6p_route.ro_rt->rt_ifp; inp->in6p_last_outifp = outif; + soisconnected(so); return 0; } diff --git a/bsd/netinet6/tcp6_var.h b/bsd/netinet6/tcp6_var.h index 8190f5269..d5b761e3a 100644 --- a/bsd/netinet6/tcp6_var.h +++ b/bsd/netinet6/tcp6_var.h @@ -105,7 +105,7 @@ SYSCTL_DECL(_net_inet6_tcp6); extern int tcp_v6mssdflt; /* XXX */ struct ip6_hdr; -void tcp6_ctlinput(int, struct sockaddr *, void *); +void tcp6_ctlinput(int, struct sockaddr *, void *, struct ifnet *); void tcp6_init(void); int tcp6_input(struct mbuf **, int *, int); struct rtentry *tcp_rtlookup6(struct inpcb *, unsigned int); diff --git a/bsd/netinet6/udp6_output.c b/bsd/netinet6/udp6_output.c index c0cd894e4..3db74ee52 100644 --- a/bsd/netinet6/udp6_output.c +++ b/bsd/netinet6/udp6_output.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -314,6 +314,11 @@ udp6_output(struct in6pcb *in6p, struct mbuf *m, struct sockaddr *addr6, if (af == AF_INET) hlen = sizeof (struct ip); + if (fport == htons(53) && !(so->so_flags1 & SOF1_DNS_COUNTED)) { + so->so_flags1 |= SOF1_DNS_COUNTED; + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_inet_dgram_dns); + } + /* * Calculate data length and get a mbuf * for UDP and IP6 headers. @@ -353,7 +358,7 @@ udp6_output(struct in6pcb *in6p, struct mbuf *m, struct sockaddr *addr6, udp6->uh_sum = in6_pseudo(laddr, faddr, htonl(plen + IPPROTO_UDP)); - m->m_pkthdr.csum_flags = CSUM_UDPIPV6; + m->m_pkthdr.csum_flags = (CSUM_UDPIPV6|CSUM_ZERO_INVERT); m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); if (!IN6_IS_ADDR_UNSPECIFIED(laddr)) @@ -477,6 +482,7 @@ udp6_output(struct in6pcb *in6p, struct mbuf *m, struct sockaddr *addr6, } INP_ADD_STAT(in6p, cell, wifi, wired, txpackets, 1); INP_ADD_STAT(in6p, cell, wifi, wired, txbytes, ulen); + inp_set_activity_bitmap(in6p); } if (flowadv && (adv->code == FADV_FLOW_CONTROLLED || @@ -500,7 +506,7 @@ udp6_output(struct in6pcb *in6p, struct mbuf *m, struct sockaddr *addr6, sizeof(struct udphdr) + hlen + ifnet_hdrlen(outif) + - ifnet_packetpreamblelen(outif), + ifnet_mbuf_packetpreamblelen(outif), sizeof(u_int32_t)); } @@ -533,7 +539,7 @@ udp6_output(struct in6pcb *in6p, struct mbuf *m, struct sockaddr *addr6, sizeof(struct udphdr) + hlen + ifnet_hdrlen(outif) + - ifnet_packetpreamblelen(outif), + ifnet_mbuf_packetpreamblelen(outif), sizeof(u_int32_t)); } } else { diff --git a/bsd/netinet6/udp6_usrreq.c b/bsd/netinet6/udp6_usrreq.c index b939c58bb..8680560fc 100644 --- a/bsd/netinet6/udp6_usrreq.c +++ b/bsd/netinet6/udp6_usrreq.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2015 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -92,11 +92,10 @@ * * @(#)udp_var.h 8.1 (Berkeley) 6/10/93 */ - -#include <sys/param.h> #include <sys/kernel.h> #include <sys/malloc.h> #include <sys/mbuf.h> +#include <sys/param.h> #include <sys/protosw.h> #include <sys/socket.h> #include <sys/socketvar.h> @@ -113,6 +112,7 @@ #include <net/if_types.h> #include <net/ntstat.h> #include <net/dlil.h> +#include <net/net_api_stats.h> #include <netinet/in.h> #include <netinet/in_systm.h> @@ -218,6 +218,7 @@ udp6_append(struct inpcb *last, struct ip6_hdr *ip6, if (nstat_collect) { INP_ADD_STAT(last, cell, wifi, wired, rxpackets, 1); INP_ADD_STAT(last, cell, wifi, wired, rxbytes, n->m_pkthdr.len); + inp_set_activity_bitmap(last); } so_recv_data_stat(last->in6p_socket, n, 0); if (sbappendaddr(&last->in6p_socket->so_rcv, @@ -374,8 +375,7 @@ udp6_input(struct mbuf **mp, int *offp, int proto) mcaddr.sin6_addr = ip6->ip6_dst; blocked = im6o_mc_filter(imo, ifp, - (struct sockaddr *)&mcaddr, - (struct sockaddr *)&fromsa); + &mcaddr, &fromsa); IM6O_UNLOCK(imo); if (blocked != MCAST_PASS) { udp_unlock(in6p->in6p_socket, 1, 0); @@ -582,6 +582,7 @@ udp6_input(struct mbuf **mp, int *offp, int proto) if (nstat_collect) { INP_ADD_STAT(in6p, cell, wifi, wired, rxpackets, 1); INP_ADD_STAT(in6p, cell, wifi, wired, rxbytes, m->m_pkthdr.len); + inp_set_activity_bitmap(in6p); } so_recv_data_stat(in6p->in6p_socket, m, 0); if (sbappendaddr(&in6p->in6p_socket->so_rcv, @@ -604,13 +605,14 @@ bad: } void -udp6_ctlinput(int cmd, struct sockaddr *sa, void *d) +udp6_ctlinput(int cmd, struct sockaddr *sa, void *d, __unused struct ifnet *ifp) { struct udphdr uh; struct ip6_hdr *ip6; struct mbuf *m; int off = 0; struct ip6ctlparam *ip6cp = NULL; + struct icmp6_hdr *icmp6 = NULL; const struct sockaddr_in6 *sa6_src = NULL; void (*notify)(struct inpcb *, int) = udp_notify; struct udp_portonly { @@ -635,6 +637,7 @@ udp6_ctlinput(int cmd, struct sockaddr *sa, void *d) /* if the parameter is from icmp6, decode it. */ if (d != NULL) { ip6cp = (struct ip6ctlparam *)d; + icmp6 = ip6cp->ip6c_icmp6; m = ip6cp->ip6c_m; ip6 = ip6cp->ip6c_ip6; off = ip6cp->ip6c_off; @@ -650,7 +653,6 @@ udp6_ctlinput(int cmd, struct sockaddr *sa, void *d) * XXX: We assume that when IPV6 is non NULL, * M and OFF are valid. */ - /* check if we can safely examine src and dst ports */ if (m->m_pkthdr.len < off + sizeof (*uhp)) return; @@ -780,6 +782,12 @@ udp6_connect(struct socket *so, struct sockaddr *nam, struct proc *p) if (inp->inp_faddr.s_addr != INADDR_ANY) return (EISCONN); + + if (!(so->so_flags1 & SOF1_CONNECT_COUNTED)) { + so->so_flags1 |= SOF1_CONNECT_COUNTED; + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_inet_dgram_connected); + } + in6_sin6_2_sin(&sin, sin6_p); #if defined(NECP) && defined(FLOW_DIVERT) if (should_use_flow_divert) { @@ -789,6 +797,14 @@ udp6_connect(struct socket *so, struct sockaddr *nam, struct proc *p) error = in_pcbconnect(inp, (struct sockaddr *)&sin, p, IFSCOPE_NONE, NULL); if (error == 0) { +#if NECP + /* Update NECP client with connected five-tuple */ + if (!uuid_is_null(inp->necp_client_uuid)) { + socket_unlock(so, 0); + necp_client_assign_from_socket(so->last_pid, inp->necp_client_uuid, inp); + socket_lock(so, 0); + } +#endif /* NECP */ inp->inp_vflag |= INP_IPV4; inp->inp_vflag &= ~INP_IPV6; soisconnected(so); @@ -800,6 +816,11 @@ udp6_connect(struct socket *so, struct sockaddr *nam, struct proc *p) if (!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr)) return (EISCONN); + if (!(so->so_flags1 & SOF1_CONNECT_COUNTED)) { + so->so_flags1 |= SOF1_CONNECT_COUNTED; + INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_inet6_dgram_connected); + } + #if defined(NECP) && defined(FLOW_DIVERT) do_flow_divert: if (should_use_flow_divert) { @@ -824,6 +845,14 @@ do_flow_divert: inp->inp_vflag &= ~INP_IPV4; inp->inp_vflag |= INP_IPV6; } +#if NECP + /* Update NECP client with connected five-tuple */ + if (!uuid_is_null(inp->necp_client_uuid)) { + socket_unlock(so, 0); + necp_client_assign_from_socket(so->last_pid, inp->necp_client_uuid, inp); + socket_lock(so, 0); + } +#endif /* NECP */ soisconnected(so); if (inp->inp_flowhash == 0) inp->inp_flowhash = inp_calc_flowhash(inp); @@ -890,6 +919,7 @@ udp6_disconnect(struct socket *so) inp->in6p_laddr = in6addr_any; inp->in6p_last_outifp = NULL; + so->so_state &= ~SS_ISCONNECTED; /* XXX */ return (0); } @@ -1014,17 +1044,22 @@ udp6_input_checksum(struct mbuf *m, struct udphdr *uh, int off, int ulen) if (m->m_pkthdr.csum_flags & CSUM_PSEUDO_HDR) { uh->uh_sum = m->m_pkthdr.csum_rx_val; } else { - uint16_t sum = m->m_pkthdr.csum_rx_val; - uint16_t start = m->m_pkthdr.csum_rx_start; + uint32_t sum = m->m_pkthdr.csum_rx_val; + uint32_t start = m->m_pkthdr.csum_rx_start; + int32_t trailer = (m_pktlen(m) - (off + ulen)); /* * Perform 1's complement adjustment of octets * that got included/excluded in the hardware- - * calculated checksum value. + * calculated checksum value. Also take care + * of any trailing bytes and subtract out + * their partial sum. */ + ASSERT(trailer >= 0); if ((m->m_pkthdr.csum_flags & CSUM_PARTIAL) && - start != off) { - uint16_t s, d; + (start != off || trailer != 0)) { + uint32_t swbytes = (uint32_t)trailer; + uint16_t s = 0, d = 0; if (IN6_IS_SCOPE_EMBED(&ip6->ip6_src)) { s = ip6->ip6_src.s6_addr16[1]; @@ -1036,12 +1071,21 @@ udp6_input_checksum(struct mbuf *m, struct udphdr *uh, int off, int ulen) } /* callee folds in sum */ - sum = m_adj_sum16(m, start, off, sum); + sum = m_adj_sum16(m, start, off, ulen, sum); + if (off > start) + swbytes += (off - start); + else + swbytes += (start - off); if (IN6_IS_SCOPE_EMBED(&ip6->ip6_src)) ip6->ip6_src.s6_addr16[1] = s; if (IN6_IS_SCOPE_EMBED(&ip6->ip6_dst)) ip6->ip6_dst.s6_addr16[1] = d; + + if (swbytes != 0) + udp_in_cksum_stats(swbytes); + if (trailer != 0) + m_adj(m, -trailer); } uh->uh_sum = in6_pseudo(&ip6->ip6_src, &ip6->ip6_dst, diff --git a/bsd/netinet6/udp6_var.h b/bsd/netinet6/udp6_var.h index 0758482f7..d21f257d3 100644 --- a/bsd/netinet6/udp6_var.h +++ b/bsd/netinet6/udp6_var.h @@ -98,7 +98,7 @@ SYSCTL_DECL(_net_inet6_udp6); extern struct pr_usrreqs udp6_usrreqs; -extern void udp6_ctlinput(int, struct sockaddr *, void *); +extern void udp6_ctlinput(int, struct sockaddr *, void *, struct ifnet *); extern int udp6_input(struct mbuf **, int *, int); extern int udp6_output(struct inpcb *, struct mbuf *, struct sockaddr *, struct mbuf *, struct proc *); diff --git a/bsd/netkey/key.c b/bsd/netkey/key.c index a82188ec9..d8e8a1b39 100644 --- a/bsd/netkey/key.c +++ b/bsd/netkey/key.c @@ -677,7 +677,7 @@ static void key_start_timehandler(void) { /* must be called while locked */ - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); if (key_timehandler_running == 0) { key_timehandler_running = 1; (void)timeout((void *)key_timehandler, (void *)0, hz); @@ -703,7 +703,7 @@ key_allocsp( struct secpolicy *sp; struct timeval tv; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (spidx == NULL) panic("key_allocsp: NULL pointer is passed.\n"); @@ -852,11 +852,14 @@ struct secasvar *key_alloc_outbound_sav_for_interface(ifnet_t interface, int fam if (interface == NULL) return NULL; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(sadb_mutex); LIST_FOREACH(sah, &sahtree, chain) { + if (sah->state == SADB_SASTATE_DEAD) { + continue; + } if (sah->ipsec_if == interface && (family == AF_INET6 || family == AF_INET) && sah->dir == IPSEC_DIR_OUTBOUND) { @@ -907,7 +910,7 @@ key_checkrequest( int error; struct sockaddr_in *sin; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); *sav = NULL; @@ -1101,7 +1104,7 @@ key_do_allocsa_policy( { struct secasvar *sav, *nextsav, *candidate, *natt_candidate, *no_natt_candidate, *d; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); /* initialize */ candidate = NULL; @@ -1230,6 +1233,17 @@ key_allocsa( caddr_t dst, u_int proto, u_int32_t spi) +{ + return key_allocsa_extended(family, src, dst, proto, spi, NULL); +} + +struct secasvar * +key_allocsa_extended(u_int family, + caddr_t src, + caddr_t dst, + u_int proto, + u_int32_t spi, + ifnet_t interface) { struct secasvar *sav, *match; u_int stateidx, state, tmpidx, matchidx; @@ -1238,7 +1252,7 @@ key_allocsa( const u_int *saorder_state_valid; int arraysize; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (src == NULL || dst == NULL) @@ -1272,6 +1286,10 @@ key_allocsa( LIST_FOREACH(sav, &spihash[SPIHASH(spi)], spihash) { if (sav->spi != spi) continue; + if (interface != NULL && + sav->sah->ipsec_if != interface) { + continue; + } if (proto != sav->sah->saidx.proto) continue; if (family != sav->sah->saidx.src.ss_family || @@ -1445,7 +1463,7 @@ key_do_get_translated_port( struct secasvar *currsav, *nextsav, *candidate; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); /* initilize */ candidate = NULL; @@ -1515,7 +1533,7 @@ key_freesp( if (!locked) lck_mtx_lock(sadb_mutex); else - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); sp->refcnt--; KEYDEBUG(KEYDEBUG_IPSEC_STAMP, printf("DP freesp cause refcnt--:%d SP:0x%llx\n", @@ -1546,7 +1564,7 @@ key_freesav( if (!locked) lck_mtx_lock(sadb_mutex); else - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); sav->refcnt--; KEYDEBUG(KEYDEBUG_IPSEC_STAMP, printf("DP freesav cause refcnt--:%d SA:0x%llx SPI %u\n", @@ -1573,7 +1591,7 @@ key_delsp( if (sp == NULL) panic("key_delsp: NULL pointer is passed.\n"); - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); sp->state = IPSEC_SPSTATE_DEAD; if (sp->refcnt > 0) @@ -1625,7 +1643,7 @@ key_getsp( { struct secpolicy *sp; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); /* sanity check */ if (spidx == NULL) @@ -1654,7 +1672,7 @@ key_getspbyid( { struct secpolicy *sp; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(sadb_mutex); sp = __key_getspbyid(id); @@ -1668,7 +1686,7 @@ __key_getspbyid(u_int32_t id) { struct secpolicy *sp; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); LIST_FOREACH(sp, &sptree[IPSEC_DIR_INBOUND], chain) { if (sp->state == IPSEC_SPSTATE_DEAD) @@ -1696,7 +1714,7 @@ key_newsp(void) { struct secpolicy *newsp = NULL; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); newsp = keydb_newsecpolicy(); if (!newsp) return newsp; @@ -1720,7 +1738,7 @@ key_msg2sp( { struct secpolicy *newsp; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (xpl0 == NULL) @@ -1770,6 +1788,13 @@ key_msg2sp( xisr = (struct sadb_x_ipsecrequest *)(xpl0 + 1); while (tlen > 0) { + if (tlen < sizeof(*xisr)) { + ipseclog((LOG_DEBUG, "key_msg2sp: " + "invalid ipsecrequest.\n")); + key_freesp(newsp, KEY_SADB_UNLOCKED); + *error = EINVAL; + return NULL; + } /* length check */ if (xisr->sadb_x_ipsecrequest_len < sizeof(*xisr)) { @@ -1873,8 +1898,25 @@ key_msg2sp( /* set IP addresses if there */ if (xisr->sadb_x_ipsecrequest_len > sizeof(*xisr)) { struct sockaddr *paddr; + + if (tlen < xisr->sadb_x_ipsecrequest_len) { + ipseclog((LOG_DEBUG, "key_msg2sp: invalid request " + "address length.\n")); + key_freesp(newsp, KEY_SADB_UNLOCKED); + *error = EINVAL; + return NULL; + } paddr = (struct sockaddr *)(xisr + 1); + uint8_t src_len = paddr->sa_len; + + if (xisr->sadb_x_ipsecrequest_len < src_len) { + ipseclog((LOG_DEBUG, "key_msg2sp: invalid request " + "invalid source address length.\n")); + key_freesp(newsp, KEY_SADB_UNLOCKED); + *error = EINVAL; + return NULL; + } /* validity check */ if (paddr->sa_len @@ -1885,11 +1927,20 @@ key_msg2sp( *error = EINVAL; return NULL; } + bcopy(paddr, &(*p_isr)->saidx.src, - paddr->sa_len); + MIN(paddr->sa_len, sizeof((*p_isr)->saidx.src))); - paddr = (struct sockaddr *)((caddr_t)paddr - + paddr->sa_len); + paddr = (struct sockaddr *)((caddr_t)paddr + paddr->sa_len); + uint8_t dst_len = paddr->sa_len; + + if (xisr->sadb_x_ipsecrequest_len < (src_len + dst_len)) { + ipseclog((LOG_DEBUG, "key_msg2sp: invalid request " + "invalid dest address length.\n")); + key_freesp(newsp, KEY_SADB_UNLOCKED); + *error = EINVAL; + return NULL; + } /* validity check */ if (paddr->sa_len @@ -1900,8 +1951,9 @@ key_msg2sp( *error = EINVAL; return NULL; } + bcopy(paddr, &(*p_isr)->saidx.dst, - paddr->sa_len); + MIN(paddr->sa_len, sizeof((*p_isr)->saidx.dst))); } (*p_isr)->sp = newsp; @@ -2134,7 +2186,7 @@ key_spdadd( struct mbuf *m, const struct sadb_msghdr *mhp) { - struct sadb_address *src0, *dst0, *src1, *dst1; + struct sadb_address *src0, *dst0, *src1 = NULL, *dst1 = NULL; struct sadb_x_policy *xpl0, *xpl; struct sadb_lifetime *lft = NULL; struct secpolicyindex spidx; @@ -2150,7 +2202,7 @@ key_spdadd( int init_disabled = 0; int address_family, address_len; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL) @@ -2536,7 +2588,7 @@ key_spddelete( struct mbuf *m, const struct sadb_msghdr *mhp) { - struct sadb_address *src0, *dst0, *src1, *dst1; + struct sadb_address *src0, *dst0, *src1 = NULL, *dst1 = NULL; struct sadb_x_policy *xpl0; struct secpolicyindex spidx; struct secpolicy *sp; @@ -2545,7 +2597,7 @@ key_spddelete( int use_src_range = 0; int use_dst_range = 0; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL) @@ -2690,7 +2742,7 @@ key_spddelete2( u_int32_t id; struct secpolicy *sp; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL) @@ -2780,7 +2832,7 @@ key_spdenable( u_int32_t id; struct secpolicy *sp; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL) @@ -2840,7 +2892,7 @@ key_spddisable( u_int32_t id; struct secpolicy *sp; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL) @@ -2913,7 +2965,7 @@ key_spdget( struct secpolicy *sp; struct mbuf *n; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL) @@ -2967,7 +3019,7 @@ key_spdacquire( struct secspacq *newspacq; int error; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (sp == NULL) @@ -3305,7 +3357,7 @@ key_spdexpire( int error = EINVAL; struct sadb_lifetime *lt; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (sp == NULL) @@ -3512,7 +3564,7 @@ key_delsah( u_int stateidx, state; int zombie = 0; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); /* sanity check */ if (sah == NULL) @@ -3598,7 +3650,7 @@ key_newsav( struct secasvar *newsav; const struct sadb_sa *xsa; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); /* sanity check */ if (m == NULL || mhp == NULL || mhp->msg == NULL || sah == NULL) @@ -3763,7 +3815,7 @@ key_newsav2(struct secashead *sah, { struct secasvar *newsav; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); /* sanity check */ if (sah == NULL) @@ -3861,7 +3913,7 @@ key_delsav( struct secasvar *sav) { - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); /* sanity check */ if (sav == NULL) @@ -3930,7 +3982,7 @@ key_getsah(struct secasindex *saidx) { struct secashead *sah; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); LIST_FOREACH(sah, &sahtree, chain) { if (sah->state == SADB_SASTATE_DEAD) @@ -3948,7 +4000,7 @@ key_newsah2 (struct secasindex *saidx, { struct secashead *sah; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); sah = key_getsah(saidx); if (!sah) { @@ -3972,7 +4024,7 @@ key_checkspidup( struct secasvar *sav; u_int stateidx, state; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); /* check address family */ if (saidx->src.ss_family != saidx->dst.ss_family) { @@ -4002,7 +4054,7 @@ key_setspi( struct secasvar *sav, u_int32_t spi) { - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); sav->spi = spi; if (sav->spihash.le_prev || sav->spihash.le_next) LIST_REMOVE(sav, spihash); @@ -4024,7 +4076,7 @@ key_getsavbyspi( struct secasvar *sav, *match; u_int stateidx, state, matchidx; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); match = NULL; matchidx = _ARRAYLEN(saorder_state_alive); LIST_FOREACH(sav, &spihash[SPIHASH(spi)], spihash) { @@ -4065,7 +4117,7 @@ key_setsaval( int error = 0; struct timeval tv; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); /* sanity check */ if (m == NULL || mhp == NULL || mhp->msg == NULL) @@ -4406,7 +4458,7 @@ key_setsaval2(struct secasvar *sav, int error = 0; struct timeval tv; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); /* initialization */ sav->replay = NULL; @@ -4611,7 +4663,7 @@ key_mature( mature = 0; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); /* check SPI value */ switch (sav->sah->saidx.proto) { @@ -5296,7 +5348,7 @@ key_newbuf( { caddr_t new; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); KMALLOC_NOWAIT(new, caddr_t, len); if (new == NULL) { lck_mtx_unlock(sadb_mutex); @@ -6456,7 +6508,7 @@ key_getspi( u_int32_t reqid; int error; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL) @@ -6658,7 +6710,7 @@ key_getspi2(struct sockaddr *src, u_int32_t spi; struct secasindex saidx; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* XXX boundary check against sa_len */ KEY_SETSECASIDX(proto, mode, reqid, src, dst, 0, &saidx); @@ -6719,7 +6771,7 @@ key_do_getnewspi( u_int32_t keymin, keymax; int count = key_spi_trycnt; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); /* set spi range to allocate */ if (spirange != NULL) { @@ -6812,7 +6864,7 @@ key_update( u_int16_t flags2; int error; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL) @@ -6993,7 +7045,7 @@ key_migrate(struct socket *so, struct secasvar *sav = NULL; u_int16_t proto; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL) @@ -7136,7 +7188,7 @@ key_getsavbyseq( struct secasvar *sav; u_int state; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); state = SADB_SASTATE_LARVAL; @@ -7191,7 +7243,7 @@ key_add( u_int32_t reqid; int error; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL) @@ -7335,7 +7387,7 @@ key_setident( const struct sadb_ident *idsrc, *iddst; int idsrclen, iddstlen; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); /* sanity check */ if (sah == NULL || m == NULL || mhp == NULL || mhp->msg == NULL) @@ -7473,7 +7525,7 @@ key_delete( struct secasvar *sav = NULL; u_int16_t proto; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL) @@ -7588,7 +7640,7 @@ key_delete_all( struct secasvar *sav, *nextsav; u_int stateidx, state; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); src0 = (struct sadb_address *)(mhp->ext[SADB_EXT_ADDRESS_SRC]); dst0 = (struct sadb_address *)(mhp->ext[SADB_EXT_ADDRESS_DST]); @@ -7680,7 +7732,7 @@ key_get( struct secasvar *sav = NULL; u_int16_t proto; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL) @@ -8121,7 +8173,7 @@ key_acquire( int error = -1; u_int32_t seq; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (saidx == NULL) @@ -8330,7 +8382,7 @@ key_getacq( { struct secacq *acq; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); LIST_FOREACH(acq, &acqtree, chain) { if (key_cmpsaidx(saidx, &acq->saidx, CMP_EXACTLY)) @@ -8346,7 +8398,7 @@ key_getacqbyseq( { struct secacq *acq; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); LIST_FOREACH(acq, &acqtree, chain) { if (acq->seq == seq) @@ -8392,7 +8444,7 @@ key_getspacq( { struct secspacq *acq; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); LIST_FOREACH(acq, &spacqtree, chain) { if (key_cmpspidx_exactly(spidx, &acq->spidx)) @@ -8796,7 +8848,7 @@ key_expire( int error = -1; struct sadb_lifetime *lt; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (sav == NULL) @@ -9020,7 +9072,7 @@ key_dump( struct mbuf *n; int error = 0; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL) @@ -9365,7 +9417,7 @@ key_parse( int target; Boolean keyAligned = FALSE; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); /* sanity check */ if (m == NULL || so == NULL) @@ -9580,7 +9632,7 @@ key_senderror( { struct sadb_msg *msg; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); if (m->m_len < sizeof(struct sadb_msg)) panic("invalid mbuf passed to key_senderror"); @@ -9721,7 +9773,7 @@ key_validate_ext( { struct sockaddr *sa; enum { NONE, ADDR } checktype = NONE; - int baselen; + int baselen = 0; const int sal = offsetof(struct sockaddr, sa_len) + sizeof(sa->sa_len); if (len != PFKEY_UNUNIT64(ext->sadb_ext_len)) @@ -9891,7 +9943,7 @@ key_sa_chgstate( if (sav->state == state) return; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); if (__LIST_CHAINED(sav)) LIST_REMOVE(sav, chain); @@ -10049,7 +10101,7 @@ key_getsastat (struct socket *so, return key_senderror(so, m, EINVAL); } - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); // exit early if there are no active SAs if (ipsec_sav_count <= 0) { @@ -10256,7 +10308,7 @@ key_delsp_for_ipsec_if (ifnet_t ipsec_if) if (ipsec_if == NULL) return; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(sadb_mutex); diff --git a/bsd/netkey/key.h b/bsd/netkey/key.h index aec0ae52d..c13c36947 100644 --- a/bsd/netkey/key.h +++ b/bsd/netkey/key.h @@ -62,6 +62,9 @@ extern int key_checkrequest(struct ipsecrequest *isr, struct secasindex *, struct secasvar **sav); extern struct secasvar *key_allocsa(u_int, caddr_t, caddr_t, u_int, u_int32_t); +struct secasvar * +key_allocsa_extended(u_int family, caddr_t src, caddr_t dst, + u_int proto, u_int32_t spi, ifnet_t interface); extern u_int16_t key_natt_get_translated_port(struct secasvar *); extern void key_freesp(struct secpolicy *, int); extern void key_freesav(struct secasvar *, int); diff --git a/bsd/netkey/keydb.c b/bsd/netkey/keydb.c index eba59f728..a5500fe9c 100644 --- a/bsd/netkey/keydb.c +++ b/bsd/netkey/keydb.c @@ -91,7 +91,7 @@ keydb_newsecpolicy(void) { struct secpolicy *p; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); return (struct secpolicy *)_MALLOC(sizeof(*p), M_SECA, M_WAITOK | M_ZERO); @@ -112,7 +112,7 @@ keydb_newsecashead(void) struct secashead *p; int i; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); p = (struct secashead *)_MALLOC(sizeof(*p), M_SECA, M_NOWAIT | M_ZERO); if (!p) { @@ -147,7 +147,7 @@ keydb_newsecasvar() { struct secasvar *p; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED); p = (struct secasvar *)_MALLOC(sizeof(*p), M_SECA, M_WAITOK); if (!p) @@ -162,7 +162,7 @@ keydb_refsecasvar(p) struct secasvar *p; { - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); p->refcnt++; } @@ -172,7 +172,7 @@ keydb_freesecasvar(p) struct secasvar *p; { - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); p->refcnt--; /* negative refcnt will cause panic intentionally */ @@ -200,7 +200,7 @@ keydb_newsecreplay(size_t wsize) { struct secreplay *p; - lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(sadb_mutex, LCK_MTX_ASSERT_OWNED); p = (struct secreplay *)_MALLOC(sizeof(*p), M_SECA, M_NOWAIT | M_ZERO); if (!p) { diff --git a/bsd/nfs/gss/gss_krb5_mech.c b/bsd/nfs/gss/gss_krb5_mech.c index 59df39f4a..0d592043a 100644 --- a/bsd/nfs/gss/gss_krb5_mech.c +++ b/bsd/nfs/gss/gss_krb5_mech.c @@ -82,7 +82,7 @@ typedef struct crypt_walker_ctx { typedef struct hmac_walker_ctx { const struct ccdigest_info *di; - cchmac_ctx_t hmac_ctx; + struct cchmac_ctx *hmac_ctx; } *hmac_walker_ctx_t; typedef size_t (*ccpad_func)(const struct ccmode_cbc *, cccbc_ctx *, cccbc_iv *, @@ -521,10 +521,10 @@ do_crypt(void *walker, uint8_t *data, uint32_t len) void do_hmac_init(hmac_walker_ctx_t wctx, crypto_ctx_t cctx, void *key) { - size_t alloc_size = cc_ctx_n(struct cchmac_ctx, cchmac_di_size(cctx->di)) * sizeof(struct cchmac_ctx); + size_t alloc_size = cchmac_di_size(cctx->di); wctx->di = cctx->di; - MALLOC(wctx->hmac_ctx.hdr, struct cchmac_ctx *, alloc_size, M_TEMP, M_WAITOK|M_ZERO); + MALLOC(wctx->hmac_ctx, struct cchmac_ctx *, alloc_size, M_TEMP, M_WAITOK|M_ZERO); cchmac_init(cctx->di, wctx->hmac_ctx, cctx->keylen, key); } @@ -622,7 +622,7 @@ krb5_mic_mbuf(crypto_ctx_t ctx, gss_buffer_t header, cchmac_update(ctx->di, wctx.hmac_ctx, trailer->length, trailer->value); cchmac_final(ctx->di, wctx.hmac_ctx, digest); - FREE(wctx.hmac_ctx.hdr, M_TEMP); + FREE(wctx.hmac_ctx, M_TEMP); if (verify) { *verify = (memcmp(mic, digest, ctx->digest_size) == 0); diff --git a/bsd/nfs/nfs.h b/bsd/nfs/nfs.h index 0232e77a1..9cacf96c5 100644 --- a/bsd/nfs/nfs.h +++ b/bsd/nfs/nfs.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2015 Apple Inc. All rights reserved. + * Copyright (c) 2000-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -200,7 +200,7 @@ extern int nfs_ticks; #define NFS_MFLAG_MUTEJUKEBOX 8 /* don't treat jukebox errors as unresponsive */ #define NFS_MFLAG_EPHEMERAL 9 /* ephemeral (mirror) mount */ #define NFS_MFLAG_NOCALLBACK 10 /* don't provide callback RPC service */ -#define NFS_MFLAG_NONAMEDATTR 11 /* don't use named attributes */ +#define NFS_MFLAG_NAMEDATTR 11 /* don't use named attributes */ #define NFS_MFLAG_NOACL 12 /* don't support ACLs */ #define NFS_MFLAG_ACLONLY 13 /* only support ACLs - not mode */ #define NFS_MFLAG_NFC 14 /* send NFC strings */ @@ -771,14 +771,19 @@ struct nfsstats { #include <sys/_types/_guid_t.h> /* for guid_t below */ #define MAXIDNAMELEN 1024 struct nfs_testmapid { - uint32_t ntm_name2id; /* lookup name 2 id or id 2 name */ + uint32_t ntm_lookup; /* lookup name 2 id or id 2 name */ uint32_t ntm_grpflag; /* Is this a group or user maping */ uint32_t ntm_id; /* id to map or return */ uint32_t pad; guid_t ntm_guid; /* intermidiate guid used in conversion */ char ntm_name[MAXIDNAMELEN]; /* name to map or return */ }; - + +#define NTM_ID2NAME 0 +#define NTM_NAME2ID 1 +#define NTM_NAME2GUID 2 +#define NTM_GUID2NAME 3 + /* * fs.nfs sysctl(3) identifiers */ @@ -998,9 +1003,6 @@ extern uint32_t nfs_debug_ctl; /* bits for nfs_idmap_ctrl: */ #define NFS_IDMAP_CTRL_USE_IDMAP_SERVICE 0x00000001 /* use the ID mapping service */ #define NFS_IDMAP_CTRL_FALLBACK_NO_COMMON_IDS 0x00000002 /* fallback should NOT handle common IDs like "root" and "nobody" */ -#define NFS_IDMAP_CTRL_FALLBACK_NO_WELLKNOWN_IDS 0x00000004 /* fallback should NOT handle the well known "XXX@" IDs */ -#define NFS_IDMAP_CTRL_UNKNOWN_IS_99 0x00000008 /* for unknown IDs use uid/gid 99 instead of -2/nobody */ -#define NFS_IDMAP_CTRL_COMPARE_RESULTS 0x00000010 /* compare results of ID mapping service and fallback */ #define NFS_IDMAP_CTRL_LOG_FAILED_MAPPINGS 0x00000020 /* log failed ID mapping attempts */ #define NFS_IDMAP_CTRL_LOG_SUCCESSFUL_MAPPINGS 0x00000040 /* log successful ID mapping attempts */ @@ -1150,7 +1152,7 @@ extern thread_call_t nfsrv_fmod_timer_call; #endif /* nfs 4 default domain for user mapping */ -extern char nfs4_domain[MAXPATHLEN]; +extern char nfs4_default_domain[MAXPATHLEN]; __BEGIN_DECLS @@ -1261,7 +1263,7 @@ uint32_t nfs4_ace_vfsflags_to_nfsflags(uint32_t); uint32_t nfs4_ace_nfsmask_to_vfsrights(uint32_t); uint32_t nfs4_ace_vfsrights_to_nfsmask(uint32_t); int nfs4_id2guid(char *, guid_t *, int); -int nfs4_guid2id(guid_t *, char *, int *, int); +int nfs4_guid2id(guid_t *, char *, size_t *, int); int nfs_parsefattr(struct nfsm_chain *, int, struct nfs_vattr *); int nfs4_parsefattr(struct nfsm_chain *, struct nfs_fsattr *, struct nfs_vattr *, fhandle_t *, struct dqblk *, struct nfs_fs_locations *); diff --git a/bsd/nfs/nfs4_subs.c b/bsd/nfs/nfs4_subs.c index 130b04f7a..d94f41582 100644 --- a/bsd/nfs/nfs4_subs.c +++ b/bsd/nfs/nfs4_subs.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2006-2011 Apple Inc. All rights reserved. + * Copyright (c) 2006-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -709,8 +709,8 @@ nfs4_default_attrs_for_referral_trigger( struct nfs_vattr *nvap, fhandle_t *fhp) { - struct timeval now; - microtime(&now); + struct timespec now; + nanotime(&now); int len; nvap->nva_flags = NFS_FFLAG_TRIGGER | NFS_FFLAG_TRIGGER_REFERRAL; @@ -752,17 +752,17 @@ nfs4_default_attrs_for_referral_trigger( if (!NFS_BITMAP_ISSET(nvap->nva_bitmap, NFS_FATTR_TIME_ACCESS)) { NFS_BITMAP_SET(nvap->nva_bitmap, NFS_FATTR_TIME_ACCESS); nvap->nva_timesec[NFSTIME_ACCESS] = now.tv_sec; - nvap->nva_timensec[NFSTIME_ACCESS] = now.tv_usec * 1000; + nvap->nva_timensec[NFSTIME_ACCESS] = now.tv_nsec; } if (!NFS_BITMAP_ISSET(nvap->nva_bitmap, NFS_FATTR_TIME_MODIFY)) { NFS_BITMAP_SET(nvap->nva_bitmap, NFS_FATTR_TIME_MODIFY); nvap->nva_timesec[NFSTIME_MODIFY] = now.tv_sec; - nvap->nva_timensec[NFSTIME_MODIFY] = now.tv_usec * 1000; + nvap->nva_timensec[NFSTIME_MODIFY] = now.tv_nsec; } if (!NFS_BITMAP_ISSET(nvap->nva_bitmap, NFS_FATTR_TIME_METADATA)) { NFS_BITMAP_SET(nvap->nva_bitmap, NFS_FATTR_TIME_METADATA); nvap->nva_timesec[NFSTIME_CHANGE] = now.tv_sec; - nvap->nva_timensec[NFSTIME_CHANGE] = now.tv_usec * 1000; + nvap->nva_timensec[NFSTIME_CHANGE] = now.tv_nsec; } if (!NFS_BITMAP_ISSET(nvap->nva_bitmap, NFS_FATTR_FILEID)) { NFS_BITMAP_SET(nvap->nva_bitmap, NFS_FATTR_FILEID); @@ -1012,6 +1012,158 @@ nfs4_ace_vfsrights_to_nfsmask(uint32_t vfsrights) return (nfsmask); } +/* + * nfs4_wkid2sidd:: + * mapid a wellknown identity to guid. + * Return 0 on success ENOENT if id does not map and EINVAL if the id is not a well known name. + */ +static int +nfs4_wkid2sid(const char *id, ntsid_t *sp) +{ + size_t len = strnlen(id, MAXIDNAMELEN); + + if (len == MAXIDNAMELEN || id[len-1] != '@') + return (EINVAL); + + bzero(sp, sizeof(ntsid_t)); + sp->sid_kind = 1; + sp->sid_authcount = 1; + if (!strcmp(id, "OWNER@")) { + // S-1-3-0 + sp->sid_authority[5] = 3; + sp->sid_authorities[0] = 0; + } else if (!strcmp(id, "GROUP@")) { + // S-1-3-1 + sp->sid_authority[5] = 3; + sp->sid_authorities[0] = 1; + } else if (!strcmp(id, "EVERYONE@")) { + // S-1-1-0 + sp->sid_authority[5] = 1; + sp->sid_authorities[0] = 0; + } else if (!strcmp(id, "INTERACTIVE@")) { + // S-1-5-4 + sp->sid_authority[5] = 5; + sp->sid_authorities[0] = 4; + } else if (!strcmp(id, "NETWORK@")) { + // S-1-5-2 + sp->sid_authority[5] = 5; + sp->sid_authorities[0] = 2; + } else if (!strcmp(id, "DIALUP@")) { + // S-1-5-1 + sp->sid_authority[5] = 5; + sp->sid_authorities[0] = 1; + } else if (!strcmp(id, "BATCH@")) { + // S-1-5-3 + sp->sid_authority[5] = 5; + sp->sid_authorities[0] = 3; + } else if (!strcmp(id, "ANONYMOUS@")) { + // S-1-5-7 + sp->sid_authority[5] = 5; + sp->sid_authorities[0] = 7; + } else if (!strcmp(id, "AUTHENTICATED@")) { + // S-1-5-11 + sp->sid_authority[5] = 5; + sp->sid_authorities[0] = 11; + } else if (!strcmp(id, "SERVICE@")) { + // S-1-5-6 + sp->sid_authority[5] = 5; + sp->sid_authorities[0] = 6; + } else { + // S-1-0-0 "NOBODY" + sp->sid_authority[5] = 0; + sp->sid_authorities[0] = 0; + } + return (0); +} + +static int +nfs4_fallback_name(const char *id, int have_at) +{ + if (have_at) { + /* must be user@domain */ + /* try to identify some well-known IDs */ + if (!strncmp(id, "root@", 5)) + return (0); + else if (!strncmp(id, "wheel@", 6)) + return (0); + else if (!strncmp(id, "nobody@", 7)) + return (-2); + else if (!strncmp(id, "nfsnobody@", 10)) + return (-2); + } + return (-2); +} + +static void +nfs4_mapid_log(int error, const char *idstr, int isgroup, guid_t *gp) +{ + if (error && (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_FAILED_MAPPINGS)) + printf("nfs4_id2guid: idmap failed for %s %s error %d\n", idstr, isgroup ? "G" : " ", error); + if (!error && (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_SUCCESSFUL_MAPPINGS)) + printf("nfs4_id2guid: idmap for %s %s got guid " + "%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x\n", + idstr, isgroup ? "G" : " ", + gp->g_guid[0], gp->g_guid[1], gp->g_guid[2], gp->g_guid[3], + gp->g_guid[4], gp->g_guid[5], gp->g_guid[6], gp->g_guid[7], + gp->g_guid[8], gp->g_guid[9], gp->g_guid[10], gp->g_guid[11], + gp->g_guid[12], gp->g_guid[13], gp->g_guid[14], gp->g_guid[15]); +} + +static char * +nfs4_map_domain(char *id, char **atp) +{ + char *at = *atp; + char *dsnode, *otw_nfs4domain; + char *new_id = NULL; + size_t otw_domain_len; + size_t otw_id_2_at_len; + int error; + + if (at == NULL) + at = strchr(id, '@'); + if (at == NULL || *at != '@') + return (NULL); + + otw_nfs4domain = at + 1; + otw_domain_len = strnlen(otw_nfs4domain, MAXPATHLEN); + otw_id_2_at_len = at - id + 1; + + MALLOC_ZONE(dsnode, char*, MAXPATHLEN, M_NAMEI, M_WAITOK); + /* first try to map nfs4 domain to dsnode for scoped lookups */ + error = kauth_cred_nfs4domain2dsnode(otw_nfs4domain, dsnode); + if (!error) { + /* Success! Make new id be id@dsnode */ + size_t dsnode_len = strnlen(dsnode, MAXPATHLEN); + size_t new_id_len = otw_id_2_at_len + dsnode_len + 1; + char tmp; + + MALLOC(new_id, char*, new_id_len, M_TEMP, M_WAITOK); + tmp = *otw_nfs4domain; + *otw_nfs4domain = '\0'; /* Chop of the old domain */ + strlcpy(new_id, id, MAXPATHLEN); + *otw_nfs4domain = tmp; /* Be nice and preserve callers original id */ + strlcat(new_id, dsnode, MAXPATHLEN); + at = strchr(new_id, '@'); + } else { + /* Bummer:-( See if default nfs4 set for unscoped lookup */ + size_t default_domain_len = strnlen(nfs4_default_domain, MAXPATHLEN); + + if ((otw_domain_len == default_domain_len) && + (strncmp(otw_nfs4domain, nfs4_default_domain, otw_domain_len) == 0)) { + /* Woohoo! We have matching domains, do unscoped lookups */ + *at = '\0'; + } + } + FREE_ZONE(dsnode, MAXPATHLEN, M_NAMEI); + + if (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_SUCCESSFUL_MAPPINGS) { + printf("nfs4_id2guid: after domain mapping id is %s\n", id); + } + + *atp = at; + return (new_id); +} + /* * Map an NFSv4 ID string to a VFS guid. * @@ -1020,16 +1172,12 @@ nfs4_ace_vfsrights_to_nfsmask(uint32_t vfsrights) int nfs4_id2guid(/*const*/ char *id, guid_t *guidp, int isgroup) { - int error1 = 0, error = 0, compare; - guid_t guid1, guid2, *gp; + int error = 0; ntsid_t sid; - long num, unknown; + long num; char *p, *at, *new_id = NULL; *guidp = kauth_null_guid; - compare = ((nfs_idmap_ctrl & NFS_IDMAP_CTRL_USE_IDMAP_SERVICE) && - (nfs_idmap_ctrl & NFS_IDMAP_CTRL_COMPARE_RESULTS)); - unknown = (nfs_idmap_ctrl & NFS_IDMAP_CTRL_UNKNOWN_IS_99) ? 99 : -2; /* * First check if it is just a simple numeric ID string or a special "XXX@" name. @@ -1046,58 +1194,31 @@ nfs4_id2guid(/*const*/ char *id, guid_t *guidp, int isgroup) at = p; p++; } - if (at && !at[1] && !isgroup) - isgroup = 1; /* special "XXX@" names should always be treated as groups */ + if (num) { /* must be numeric ID (or empty) */ - num = *id ? strtol(id, NULL, 10) : unknown; - gp = guidp; - /* Since we are not initilizing guid1 and guid2, skip compare */ - compare = 0; - goto gotnumid; + num = *id ? strtol(id, NULL, 10) : -2; + if (isgroup) + error = kauth_cred_gid2guid((gid_t)num, guidp); + else + error = kauth_cred_uid2guid((uid_t)num, guidp); + nfs4_mapid_log(error, id, isgroup, guidp); + return (error); + } + + /* See if this is a well known NFSv4 name */ + error = nfs4_wkid2sid(id, &sid); + if (!error) { + error = kauth_cred_ntsid2guid(&sid, guidp); + nfs4_mapid_log(error, id, 1, guidp); + return (error); } /* Handle nfs4 domain first */ if (at && at[1]) { - /* Try mapping nfs4 domain */ - char *dsnode, *nfs4domain = at + 1; - size_t otw_domain_len = strnlen(nfs4domain, MAXPATHLEN); - int otw_id_2_at_len = at - id + 1; - - MALLOC(dsnode, char*, MAXPATHLEN, M_NAMEI, M_WAITOK); - if (dsnode) { - /* first try to map nfs4 domain to dsnode for scoped lookups */ - memset(dsnode, 0, MAXPATHLEN); - error = kauth_cred_nfs4domain2dsnode(nfs4domain, dsnode); - if (!error) { - /* Success! Make new id be id@dsnode */ - int dsnode_len = strnlen(dsnode, MAXPATHLEN); - int new_id_len = otw_id_2_at_len + dsnode_len + 1; - - MALLOC(new_id, char*, new_id_len, M_NAMEI, M_WAITOK); - if (new_id) { - (void)strlcpy(new_id, id, otw_id_2_at_len + 1); - (void)strlcpy(new_id + otw_id_2_at_len, dsnode, dsnode_len + 1); - id = new_id; - at = id; - while (*at++ != '@'); - at--; - } - } else { - /* Bummer:-( See if default nfs4 set for unscoped lookup */ - size_t default_domain_len = strnlen(nfs4_domain, MAXPATHLEN); - - if ((otw_domain_len == default_domain_len) && (strncmp(nfs4domain, nfs4_domain, otw_domain_len) == 0)) { - /* Woohoo! We have matching domains, do unscoped lookups */ - *at = '\0'; - } - } - FREE(dsnode, M_NAMEI); - } - - if (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_SUCCESSFUL_MAPPINGS) { - printf("nfs4_id2guid: after domain mapping id is %s\n", id); - } + new_id = nfs4_map_domain(id, &at); + if (new_id) + id = new_id; } /* Now try to do actual id mapping */ @@ -1107,197 +1228,238 @@ nfs4_id2guid(/*const*/ char *id, guid_t *guidp, int isgroup) * * [sigh] this isn't a "pwnam/grnam" it's an NFS ID string! */ - gp = compare ? &guid1 : guidp; if (isgroup) - error = kauth_cred_grnam2guid(id, gp); + error = kauth_cred_grnam2guid(id, guidp); else - error = kauth_cred_pwnam2guid(id, gp); - if (error && (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_FAILED_MAPPINGS)) - printf("nfs4_id2guid: idmap failed for %s %s error %d\n", id, isgroup ? "G" : " ", error); - if (!error && (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_SUCCESSFUL_MAPPINGS)) - printf("nfs4_id2guid: idmap for %s %s got guid " - "%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x\n", - id, isgroup ? "G" : " ", - gp->g_guid[0], gp->g_guid[1], gp->g_guid[2], gp->g_guid[3], - gp->g_guid[4], gp->g_guid[5], gp->g_guid[6], gp->g_guid[7], - gp->g_guid[8], gp->g_guid[9], gp->g_guid[10], gp->g_guid[11], - gp->g_guid[12], gp->g_guid[13], gp->g_guid[14], gp->g_guid[15]); - error1 = error; - } - if (error || compare || !(nfs_idmap_ctrl & NFS_IDMAP_CTRL_USE_IDMAP_SERVICE)) { + error = kauth_cred_pwnam2guid(id, guidp); + nfs4_mapid_log(error, id, isgroup, guidp); + } else { + error = ENOTSUP; + } + + if (error) { /* * fallback path... see if we can come up with an answer ourselves. */ - gp = compare ? &guid2 : guidp; - - if (!(nfs_idmap_ctrl & NFS_IDMAP_CTRL_FALLBACK_NO_WELLKNOWN_IDS) && at && !at[1]) { - /* must be a special ACE "who" ID */ - bzero(&sid, sizeof(sid)); - sid.sid_kind = 1; - sid.sid_authcount = 1; - if (!strcmp(id, "OWNER@")) { - // S-1-3-0 - sid.sid_authority[5] = 3; - sid.sid_authorities[0] = 0; - } else if (!strcmp(id, "GROUP@")) { - // S-1-3-1 - sid.sid_authority[5] = 3; - sid.sid_authorities[0] = 1; - } else if (!strcmp(id, "EVERYONE@")) { - // S-1-1-0 - sid.sid_authority[5] = 1; - sid.sid_authorities[0] = 0; - } else if (!strcmp(id, "INTERACTIVE@")) { - // S-1-5-4 - sid.sid_authority[5] = 5; - sid.sid_authorities[0] = 4; - } else if (!strcmp(id, "NETWORK@")) { - // S-1-5-2 - sid.sid_authority[5] = 5; - sid.sid_authorities[0] = 2; - } else if (!strcmp(id, "DIALUP@")) { - // S-1-5-1 - sid.sid_authority[5] = 5; - sid.sid_authorities[0] = 1; - } else if (!strcmp(id, "BATCH@")) { - // S-1-5-3 - sid.sid_authority[5] = 5; - sid.sid_authorities[0] = 3; - } else if (!strcmp(id, "ANONYMOUS@")) { - // S-1-5-7 - sid.sid_authority[5] = 5; - sid.sid_authorities[0] = 7; - } else if (!strcmp(id, "AUTHENTICATED@")) { - // S-1-5-11 - sid.sid_authority[5] = 5; - sid.sid_authorities[0] = 11; - } else if (!strcmp(id, "SERVICE@")) { - // S-1-5-6 - sid.sid_authority[5] = 5; - sid.sid_authorities[0] = 6; - } else { - // S-1-0-0 "NOBODY" - sid.sid_authority[5] = 0; - sid.sid_authorities[0] = 0; - } - error = kauth_cred_ntsid2guid(&sid, gp); - } else { - if (!(nfs_idmap_ctrl & NFS_IDMAP_CTRL_FALLBACK_NO_COMMON_IDS) && at) { - /* must be user@domain */ - /* try to identify some well-known IDs */ - if (!strncmp(id, "root@", 5)) - num = 0; - else if (!strncmp(id, "wheel@", 6)) - num = 0; - else if (!strncmp(id, "nobody@", 7)) - num = -2; - else if (!strncmp(id, "nfsnobody@", 10)) - num = -2; - else - num = unknown; - } else if (!(nfs_idmap_ctrl & NFS_IDMAP_CTRL_FALLBACK_NO_COMMON_IDS) && !strcmp(id, "nobody")) { - num = -2; - } else { - num = unknown; - } -gotnumid: - if (isgroup) - error = kauth_cred_gid2guid((gid_t)num, gp); - else - error = kauth_cred_uid2guid((uid_t)num, gp); - } - if (error && (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_FAILED_MAPPINGS)) - printf("nfs4_id2guid: fallback map failed for %s %s error %d\n", id, isgroup ? "G" : " ", error); - if (!error && (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_SUCCESSFUL_MAPPINGS)) - printf("nfs4_id2guid: fallback map for %s %s got guid " - "%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x\n", - id, isgroup ? "G" : " ", - gp->g_guid[0], gp->g_guid[1], gp->g_guid[2], gp->g_guid[3], - gp->g_guid[4], gp->g_guid[5], gp->g_guid[6], gp->g_guid[7], - gp->g_guid[8], gp->g_guid[9], gp->g_guid[10], gp->g_guid[11], - gp->g_guid[12], gp->g_guid[13], gp->g_guid[14], gp->g_guid[15]); - } - - if (compare) { - /* compare the results, log if different */ - if (!error1 && !error) { - if (!kauth_guid_equal(&guid1, &guid2)) - printf("nfs4_id2guid: idmap/fallback results differ for %s %s - " - "idmap %02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x " - "fallback %02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x\n", - id, isgroup ? "G" : " ", - guid1.g_guid[0], guid1.g_guid[1], guid1.g_guid[2], guid1.g_guid[3], - guid1.g_guid[4], guid1.g_guid[5], guid1.g_guid[6], guid1.g_guid[7], - guid1.g_guid[8], guid1.g_guid[9], guid1.g_guid[10], guid1.g_guid[11], - guid1.g_guid[12], guid1.g_guid[13], guid1.g_guid[14], guid1.g_guid[15], - guid2.g_guid[0], guid2.g_guid[1], guid2.g_guid[2], guid2.g_guid[3], - guid2.g_guid[4], guid2.g_guid[5], guid2.g_guid[6], guid2.g_guid[7], - guid2.g_guid[8], guid2.g_guid[9], guid2.g_guid[10], guid2.g_guid[11], - guid2.g_guid[12], guid2.g_guid[13], guid2.g_guid[14], guid2.g_guid[15]); - /* copy idmap result to output guid */ - *guidp = guid1; - } else if (error1 && !error) { - printf("nfs4_id2guid: idmap/fallback results differ for %s %s - " - "idmap error %d " - "fallback %02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x\n", - id, isgroup ? "G" : " ", - error1, - guid2.g_guid[0], guid2.g_guid[1], guid2.g_guid[2], guid2.g_guid[3], - guid2.g_guid[4], guid2.g_guid[5], guid2.g_guid[6], guid2.g_guid[7], - guid2.g_guid[8], guid2.g_guid[9], guid2.g_guid[10], guid2.g_guid[11], - guid2.g_guid[12], guid2.g_guid[13], guid2.g_guid[14], guid2.g_guid[15]); - /* copy fallback result to output guid */ - *guidp = guid2; - } else if (!error1 && error) { - printf("nfs4_id2guid: idmap/fallback results differ for %s %s - " - "idmap %02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x " - "fallback error %d\n", - id, isgroup ? "G" : " ", - guid1.g_guid[0], guid1.g_guid[1], guid1.g_guid[2], guid1.g_guid[3], - guid1.g_guid[4], guid1.g_guid[5], guid1.g_guid[6], guid1.g_guid[7], - guid1.g_guid[8], guid1.g_guid[9], guid1.g_guid[10], guid1.g_guid[11], - guid1.g_guid[12], guid1.g_guid[13], guid1.g_guid[14], guid1.g_guid[15], - error); - /* copy idmap result to output guid */ - *guidp = guid1; - error = 0; - } else { - if (error1 != error) - printf("nfs4_id2guid: idmap/fallback results differ for %s %s - " - "idmap error %d fallback error %d\n", - id, isgroup ? "G" : " ", error1, error); - } + num = nfs4_fallback_name(id, at != NULL); + if (isgroup) + error = kauth_cred_gid2guid((gid_t)num, guidp); + else + error = kauth_cred_uid2guid((uid_t)num, guidp); + nfs4_mapid_log(error, id, isgroup, guidp); } + /* restore @ symbol in case we clobered for unscoped lookup */ if (at && *at == '\0') *at = '@'; /* free mapped domain id string */ - if (id == new_id) - FREE(id, M_NAMEI); + if (new_id) + FREE(new_id, M_TEMP); + + return (error); +} + +/* + * nfs4_sid2wkid: + * mapid a wellknown identity to guid. + * returns well known name for the sid or NULL if sid does not map. + */ +#define MAXWELLKNOWNID 18 + +static const char* +nfs4_sid2wkid(ntsid_t *sp) +{ + if ((sp->sid_kind == 1) && (sp->sid_authcount == 1)) { + /* check if it's one of our well-known ACE WHO names */ + if (sp->sid_authority[5] == 0) { + if (sp->sid_authorities[0] == 0) // S-1-0-0 + return ("nobody@localdomain"); + } else if (sp->sid_authority[5] == 1) { + if (sp->sid_authorities[0] == 0) // S-1-1-0 + return ("EVERYONE@"); + } else if (sp->sid_authority[5] == 3) { + if (sp->sid_authorities[0] == 0) // S-1-3-0 + return ("OWNER@"); + else if (sp->sid_authorities[0] == 1) // S-1-3-1 + return ("GROUP@"); + } else if (sp->sid_authority[5] == 5) { + if (sp->sid_authorities[0] == 1) // S-1-5-1 + return ("DIALUP@"); + else if (sp->sid_authorities[0] == 2) // S-1-5-2 + return ("NETWORK@"); + else if (sp->sid_authorities[0] == 3) // S-1-5-3 + return ("BATCH@"); + else if (sp->sid_authorities[0] == 4) // S-1-5-4 + return ("INTERACTIVE@"); + else if (sp->sid_authorities[0] == 6) // S-1-5-6 + return ("SERVICE@"); + else if (sp->sid_authorities[0] == 7) // S-1-5-7 + return ("ANONYMOUS@"); + else if (sp->sid_authorities[0] == 11) // S-1-5-11 + return ("AUTHENTICATED@"); + } + } + return (NULL); +} + +static void +nfs4_mapguid_log(int error, const char *where, guid_t *gp, int isgroup, const char *idstr) +{ + if (error && (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_FAILED_MAPPINGS)) + printf("nfs4_guid2id: %s idmap failed for " + "%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x %s " + "error %d\n", where, + gp->g_guid[0], gp->g_guid[1], gp->g_guid[2], gp->g_guid[3], + gp->g_guid[4], gp->g_guid[5], gp->g_guid[6], gp->g_guid[7], + gp->g_guid[8], gp->g_guid[9], gp->g_guid[10], gp->g_guid[11], + gp->g_guid[12], gp->g_guid[13], gp->g_guid[14], gp->g_guid[15], + isgroup ? "G" : " ", error); + if (!error && (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_SUCCESSFUL_MAPPINGS)) + printf("nfs4_guid2id: %s idmap for " + "%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x %s " + "got ID %s\n", where, + gp->g_guid[0], gp->g_guid[1], gp->g_guid[2], gp->g_guid[3], + gp->g_guid[4], gp->g_guid[5], gp->g_guid[6], gp->g_guid[7], + gp->g_guid[8], gp->g_guid[9], gp->g_guid[10], gp->g_guid[11], + gp->g_guid[12], gp->g_guid[13], gp->g_guid[14], gp->g_guid[15], + isgroup ? "G" : " ", idstr); +} + +static int +nfs4_addv4domain(char *id, size_t *idlen) +{ + char *at = NULL, *cp; + int have_domain; + int error = 0; + size_t idsize; + + + if (id == NULL || *id == '\0') + return (EINVAL); + + for (cp = id; *cp != '\0'; cp++) { + if (*cp == '@') { + at = cp; + break; + } + } + + have_domain = (at && at[1] != '\0'); + + if (have_domain) { + char *dsnode = at + 1; + char *nfs4domain; + size_t domain_len; + char *mapped_domain; + + MALLOC_ZONE(nfs4domain, char*, MAXPATHLEN, M_NAMEI, M_WAITOK); + error = kauth_cred_dsnode2nfs4domain(dsnode, nfs4domain); + if (!error) { + domain_len = strnlen(nfs4domain, MAXPATHLEN); + mapped_domain = nfs4domain; + } else { + error = 0; + domain_len = strnlen(nfs4_default_domain, MAXPATHLEN); + mapped_domain = nfs4_default_domain; + } + if (domain_len) { + /* chop off id after the '@' */ + at[1] = '\0'; + /* Add our mapped_domain */ + idsize = strlcat(id, mapped_domain, *idlen); + if (*idlen > idsize) + *idlen = idsize; + else + error = ENOSPC; + } + FREE_ZONE(nfs4domain, MAXPATHLEN, M_NAMEI); + } else if (at == NULL) { + /* + * If we didn't find an 'at' then cp points to the end of id passed in. + * and if we have a nfs4_default_domain set. Try to append the + * default domain if we have root or set ENOSPC. + */ + size_t default_domain_len = strnlen(nfs4_default_domain, MAXPATHLEN); + + if (default_domain_len) { + strlcat(id, "@", *idlen); + idsize = strlcat(id, nfs4_default_domain, *idlen); + if (*idlen > idsize) + *idlen = idsize; + else + error = ENOSPC; + } else { + ; /* Unscoped name otw */ + } + } + + if (!error && nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_SUCCESSFUL_MAPPINGS) + printf("nfs4_guid2id: id after nfs4 domain map: %s[%zd].\n", id, *idlen); return (error); } +static char * +nfs4_fallback_id(int numid, int isgrp, char *buf, size_t size) +{ + const char *idp = NULL; + + if (!(nfs_idmap_ctrl & NFS_IDMAP_CTRL_FALLBACK_NO_COMMON_IDS)) { + /* map well known uid's to strings */ + if (numid == 0) + idp = isgrp ? "wheel" : "root"; + else if (numid == -2) + idp = "nobody"; + } + if (!idp) { + /* or just use a decimal number string. */ + snprintf(buf, size-1, "%d", numid); + buf[size-1] = '\0'; + } else { + size_t idplen = strlcpy(buf, idp, size); + if (idplen >= size) + return (NULL); + } + + return (buf); +} + /* * Map a VFS guid to an NFSv4 ID string. * * Try to use the ID mapping service... but we may fallback to trying to do it ourselves. */ int -nfs4_guid2id(guid_t *guidp, char *id, int *idlen, int isgroup) +nfs4_guid2id(guid_t *guidp, char *id, size_t *idlen, int isgroup) { - int error1 = 0, error = 0, compare; - int id1len, id2len, len; - char *id1buf, *id1, *at; + int error = 0; + size_t id1len, len; + char *id1buf, *id1; char numbuf[32]; - const char *id2 = NULL; + ntsid_t sid; id1buf = id1 = NULL; - id1len = id2len = 0; - compare = ((nfs_idmap_ctrl & NFS_IDMAP_CTRL_USE_IDMAP_SERVICE) && - (nfs_idmap_ctrl & NFS_IDMAP_CTRL_COMPARE_RESULTS)); + id1len = 0; + + /* + * See if our guid maps to a well known NFSv4 name + */ + error = kauth_cred_guid2ntsid(guidp, &sid); + if (!error) { + const char *wkid = nfs4_sid2wkid(&sid); + if (wkid) { + len = strnlen(wkid, MAXWELLKNOWNID); + strlcpy(id, wkid, *idlen); + error = (len < *idlen) ? 0 : ENOSPC; + *idlen = len; + nfs4_mapguid_log(error, "kauth_cred_guid2ntsid", guidp, 1, id); + return (error); + } + } else { + nfs4_mapguid_log(error, "kauth_cred_guid2ntsid", guidp, isgroup, NULL); + } if (nfs_idmap_ctrl & NFS_IDMAP_CTRL_USE_IDMAP_SERVICE) { /* @@ -1311,10 +1473,9 @@ nfs4_guid2id(guid_t *guidp, char *id, int *idlen, int isgroup) * be at least MAXPATHLEN bytes long even though most if not all ID * strings will be much much shorter than that. */ - if (compare || (*idlen < MAXPATHLEN)) { + + if (*idlen < MAXPATHLEN) { MALLOC_ZONE(id1buf, char*, MAXPATHLEN, M_NAMEI, M_WAITOK); - if (!id1buf) - return (ENOMEM); id1 = id1buf; id1len = MAXPATHLEN; } else { @@ -1322,243 +1483,54 @@ nfs4_guid2id(guid_t *guidp, char *id, int *idlen, int isgroup) id1len = *idlen; } - memset(id1, 0, id1len); if (isgroup) error = kauth_cred_guid2grnam(guidp, id1); else error = kauth_cred_guid2pwnam(guidp, id1); - if (error && (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_FAILED_MAPPINGS)) - printf("nfs4_guid2id: idmap failed for " - "%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x %s " - "error %d\n", - guidp->g_guid[0], guidp->g_guid[1], guidp->g_guid[2], guidp->g_guid[3], - guidp->g_guid[4], guidp->g_guid[5], guidp->g_guid[6], guidp->g_guid[7], - guidp->g_guid[8], guidp->g_guid[9], guidp->g_guid[10], guidp->g_guid[11], - guidp->g_guid[12], guidp->g_guid[13], guidp->g_guid[14], guidp->g_guid[15], - isgroup ? "G" : " ", error); - if (!error && (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_SUCCESSFUL_MAPPINGS)) - printf("nfs4_guid2id: idmap for " - "%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x %s " - "got ID %s\n", - guidp->g_guid[0], guidp->g_guid[1], guidp->g_guid[2], guidp->g_guid[3], - guidp->g_guid[4], guidp->g_guid[5], guidp->g_guid[6], guidp->g_guid[7], - guidp->g_guid[8], guidp->g_guid[9], guidp->g_guid[10], guidp->g_guid[11], - guidp->g_guid[12], guidp->g_guid[13], guidp->g_guid[14], guidp->g_guid[15], - isgroup ? "G" : " ", id1); - error1 = error; - if (!error) { - if (compare) { - id1len = strnlen(id1, id1len); - } else if (id1 == id1buf) { - /* copy idmap result to output buffer */ - len = strlcpy(id, id1, *idlen); - if (len >= *idlen) - error = ENOSPC; - else - *idlen = len; - } - } + if (error) + nfs4_mapguid_log(error, "kauth_cred2[pw|gr]nam", guidp, isgroup, id1); + } else { + error = ENOTSUP; } - if (error || compare || !(nfs_idmap_ctrl & NFS_IDMAP_CTRL_USE_IDMAP_SERVICE)) { + + if (error) { /* * fallback path... see if we can come up with an answer ourselves. */ - ntsid_t sid; uid_t uid; - if (!(nfs_idmap_ctrl & NFS_IDMAP_CTRL_FALLBACK_NO_WELLKNOWN_IDS)) { - error = kauth_cred_guid2ntsid(guidp, &sid); - if (!error && (sid.sid_kind == 1) && (sid.sid_authcount == 1)) { - /* check if it's one of our well-known ACE WHO names */ - if (sid.sid_authority[5] == 0) { - if (sid.sid_authorities[0] == 0) // S-1-0-0 - id2 = "nobody@localdomain"; - } else if (sid.sid_authority[5] == 1) { - if (sid.sid_authorities[0] == 0) // S-1-1-0 - id2 = "EVERYONE@"; - } else if (sid.sid_authority[5] == 3) { - if (sid.sid_authorities[0] == 0) // S-1-3-0 - id2 = "OWNER@"; - else if (sid.sid_authorities[0] == 1) // S-1-3-1 - id2 = "GROUP@"; - } else if (sid.sid_authority[5] == 5) { - if (sid.sid_authorities[0] == ntohl(1)) // S-1-5-1 - id2 = "DIALUP@"; - else if (sid.sid_authorities[0] == ntohl(2)) // S-1-5-2 - id2 = "NETWORK@"; - else if (sid.sid_authorities[0] == ntohl(3)) // S-1-5-3 - id2 = "BATCH@"; - else if (sid.sid_authorities[0] == ntohl(4)) // S-1-5-4 - id2 = "INTERACTIVE@"; - else if (sid.sid_authorities[0] == ntohl(6)) // S-1-5-6 - id2 = "SERVICE@"; - else if (sid.sid_authorities[0] == ntohl(7)) // S-1-5-7 - id2 = "ANONYMOUS@"; - else if (sid.sid_authorities[0] == ntohl(11)) // S-1-5-11 - id2 = "AUTHENTICATED@"; - } - } - } - if (!id2) { - /* OK, let's just try mapping it to a UID/GID */ - if (isgroup) - error = kauth_cred_guid2gid(guidp, (gid_t*)&uid); + /* OK, let's just try mapping it to a UID/GID */ + if (isgroup) + error = kauth_cred_guid2gid(guidp, (gid_t*)&uid); + else + error = kauth_cred_guid2uid(guidp, &uid); + if (!error) { + char *fbidp = nfs4_fallback_id(uid, isgroup, numbuf, sizeof(numbuf)); + if (fbidp == NULL) + error = ENOSPC; else - error = kauth_cred_guid2uid(guidp, &uid); - if (!error) { - if (!(nfs_idmap_ctrl & NFS_IDMAP_CTRL_FALLBACK_NO_COMMON_IDS)) { - /* map well known uid's to strings */ - if (uid == 0) - id2 = isgroup ? "wheel@localdomain" : "root@localdomain"; - else if (uid == (uid_t)-2) - id2 = "nobody@localdomain"; - } - if (!id2) { - /* or just use a decimal number string. */ - snprintf(numbuf, sizeof(numbuf), "%d", uid); - id2 = numbuf; - } - } - } - if (error && (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_FAILED_MAPPINGS)) - printf("nfs4_guid2id: fallback map failed for " - "%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x %s " - "error %d\n", - guidp->g_guid[0], guidp->g_guid[1], guidp->g_guid[2], guidp->g_guid[3], - guidp->g_guid[4], guidp->g_guid[5], guidp->g_guid[6], guidp->g_guid[7], - guidp->g_guid[8], guidp->g_guid[9], guidp->g_guid[10], guidp->g_guid[11], - guidp->g_guid[12], guidp->g_guid[13], guidp->g_guid[14], guidp->g_guid[15], - isgroup ? "G" : " ", error); - if (!error && (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_SUCCESSFUL_MAPPINGS)) - printf("nfs4_guid2id: fallback map for " - "%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x %s " - "got ID %s\n", - guidp->g_guid[0], guidp->g_guid[1], guidp->g_guid[2], guidp->g_guid[3], - guidp->g_guid[4], guidp->g_guid[5], guidp->g_guid[6], guidp->g_guid[7], - guidp->g_guid[8], guidp->g_guid[9], guidp->g_guid[10], guidp->g_guid[11], - guidp->g_guid[12], guidp->g_guid[13], guidp->g_guid[14], guidp->g_guid[15], - isgroup ? "G" : " ", id2); - if (!error && id2) { - if (compare) { - id2len = strnlen(id2, MAXPATHLEN); - } else { - /* copy fallback result to output buffer */ - len = strlcpy(id, id2, *idlen); - if (len >= *idlen) - error = ENOSPC; - else - *idlen = len; - } + id1 = fbidp; } + } else { + error = nfs4_addv4domain(id1, &id1len); } - if (compare) { - /* compare the results, log if different */ - if (!error1 && !error) { - if ((id1len != id2len) || strncmp(id1, id2, id1len)) - printf("nfs4_guid2id: idmap/fallback results differ for " - "%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x %s " - "idmap %s fallback %s\n", - guidp->g_guid[0], guidp->g_guid[1], guidp->g_guid[2], guidp->g_guid[3], - guidp->g_guid[4], guidp->g_guid[5], guidp->g_guid[6], guidp->g_guid[7], - guidp->g_guid[8], guidp->g_guid[9], guidp->g_guid[10], guidp->g_guid[11], - guidp->g_guid[12], guidp->g_guid[13], guidp->g_guid[14], guidp->g_guid[15], - isgroup ? "G" : " ", id1, id2); - if (id1 == id1buf) { - /* copy idmap result to output buffer */ - len = strlcpy(id, id1, *idlen); - if (len >= *idlen) - error = ENOSPC; - else - *idlen = len; - } - } else if (error1 && !error) { - printf("nfs4_guid2id: idmap/fallback results differ for " - "%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x %s " - "idmap error %d fallback %s\n", - guidp->g_guid[0], guidp->g_guid[1], guidp->g_guid[2], guidp->g_guid[3], - guidp->g_guid[4], guidp->g_guid[5], guidp->g_guid[6], guidp->g_guid[7], - guidp->g_guid[8], guidp->g_guid[9], guidp->g_guid[10], guidp->g_guid[11], - guidp->g_guid[12], guidp->g_guid[13], guidp->g_guid[14], guidp->g_guid[15], - isgroup ? "G" : " ", error1, id2); - /* copy fallback result to output buffer */ - len = strlcpy(id, id2, *idlen); + if (!error) { + + if (id1 != id) { + /* copy idmap result to output buffer */ + len = strlcpy(id, id1, *idlen); if (len >= *idlen) error = ENOSPC; else *idlen = len; - } else if (!error1 && error) { - printf("nfs4_guid2id: idmap/fallback results differ for " - "%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x_%02x%02x%02x%02x %s " - "idmap %s fallback error %d\n", - guidp->g_guid[0], guidp->g_guid[1], guidp->g_guid[2], guidp->g_guid[3], - guidp->g_guid[4], guidp->g_guid[5], guidp->g_guid[6], guidp->g_guid[7], - guidp->g_guid[8], guidp->g_guid[9], guidp->g_guid[10], guidp->g_guid[11], - guidp->g_guid[12], guidp->g_guid[13], guidp->g_guid[14], guidp->g_guid[15], - isgroup ? "G" : " ", id1, error); - if (id1 == id1buf) { - /* copy idmap result to output buffer */ - len = strlcpy(id, id1, *idlen); - if (len >= *idlen) - error = ENOSPC; - else - *idlen = len; - } - error = 0; - } else { - if (error1 != error) - printf("nfs4_guid2id: idmap/fallback results differ for %s %s - " - "idmap error %d fallback error %d\n", - id, isgroup ? "G" : " ", error1, error); - } - } - - at = id; - while (at && at[0] != '@' && at[0] != '\0' && at++); - if (at && at[0] == '@' && at[1] != '\0') { - char *dsnode = at + 1; - int id_2_at_len = at - id + 1; - char *nfs4domain, *new_id; - MALLOC(nfs4domain, char*, MAXPATHLEN, M_NAMEI, M_WAITOK); - if (nfs4domain) { - int domain_len; - char *mapped_domain; - memset(nfs4domain, 0, MAXPATHLEN); - error = kauth_cred_dsnode2nfs4domain(dsnode, nfs4domain); - if (!error) { - domain_len = strnlen(nfs4domain, MAXPATHLEN); - mapped_domain = nfs4domain; - } else { - domain_len = strnlen(nfs4_domain, MAXPATHLEN); - mapped_domain = nfs4_domain; - } - if (domain_len) { - MALLOC(new_id, char*, MAXPATHLEN, M_NAMEI, M_WAITOK); - if (new_id) { - strlcpy(new_id, id, id_2_at_len + 1); - strlcpy(new_id + id_2_at_len, mapped_domain, domain_len + 1); - strlcpy(id, new_id, strnlen(new_id, MAXPATHLEN) + 1); - *idlen = strnlen(id, MAXPATHLEN); - FREE(new_id, M_NAMEI); - } - } - FREE(nfs4domain, M_NAMEI); - } - } else if (at && at[0] == '\0') { - int default_domain_len = strnlen(nfs4_domain, MAXPATHLEN); - - if (default_domain_len && MAXPATHLEN - *idlen > default_domain_len) { - at[0] = '@'; - strlcpy(at + 1, nfs4_domain, default_domain_len + 1); - *idlen = strnlen(id, MAXPATHLEN); } } - - if (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_SUCCESSFUL_MAPPINGS) - printf("nfs4_guid2id: id after nfs4 domain map: %s[%d].\n", id, *idlen); + nfs4_mapguid_log(error, "End of routine", guidp, isgroup, id1); if (id1buf) FREE_ZONE(id1buf, MAXPATHLEN, M_NAMEI); + return (error); } @@ -2119,7 +2091,7 @@ nfs4_parsefattr( error = kauth_cred_guid2uid(&nvap->nva_uuuid, &nvap->nva_uid); if (error) { /* unable to get either GUID or UID, set to default */ - nvap->nva_uid = (uid_t)((nfs_idmap_ctrl & NFS_IDMAP_CTRL_UNKNOWN_IS_99) ? 99 : -2); + nvap->nva_uid = (uid_t)(-2); if (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_FAILED_MAPPINGS) printf("nfs4_parsefattr: owner %s is no one, no %s?, error %d\n", s, kauth_guid_equal(&nvap->nva_uuuid, &kauth_null_guid) ? "guid" : "uid", @@ -2154,7 +2126,7 @@ nfs4_parsefattr( error = kauth_cred_guid2gid(&nvap->nva_guuid, &nvap->nva_gid); if (error) { /* unable to get either GUID or GID, set to default */ - nvap->nva_gid = (gid_t)((nfs_idmap_ctrl & NFS_IDMAP_CTRL_UNKNOWN_IS_99) ? 99 : -2); + nvap->nva_gid = (gid_t)(-2); if (nfs_idmap_ctrl & NFS_IDMAP_CTRL_LOG_FAILED_MAPPINGS) printf("nfs4_parsefattr: group %s is no one, no %s?, error %d\n", s, kauth_guid_equal(&nvap->nva_guuid, &kauth_null_guid) ? "guid" : "gid", @@ -2274,7 +2246,8 @@ nfsmout: int nfsm_chain_add_fattr4_f(struct nfsm_chain *nmc, struct vnode_attr *vap, struct nfsmount *nmp) { - int error = 0, attrbytes, slen, len, i, isgroup; + int error = 0, attrbytes, i, isgroup; + size_t slen, len; uint32_t *pattrbytes, val, acecount;; uint32_t bitmap[NFS_ATTR_BITMAP_LEN]; char sbuf[64], *s; @@ -2314,11 +2287,12 @@ nfsm_chain_add_fattr4_f(struct nfsm_chain *nmc, struct vnode_attr *vap, struct n val = nfs4_ace_vfstype_to_nfstype(val, &error); nfsm_chain_add_32(error, nmc, val); val = nfs4_ace_vfsflags_to_nfsflags(acl->acl_ace[i].ace_flags); + isgroup = (kauth_cred_guid2gid(&acl->acl_ace[i].ace_applicable, &gid) == 0); + val |= (isgroup) ? NFS_ACE_IDENTIFIER_GROUP : 0; nfsm_chain_add_32(error, nmc, val); val = nfs4_ace_vfsrights_to_nfsmask(acl->acl_ace[i].ace_rights); nfsm_chain_add_32(error, nmc, val); len = slen; - isgroup = (kauth_cred_guid2gid(&acl->acl_ace[i].ace_applicable, &gid) == 0); error = nfs4_guid2id(&acl->acl_ace[i].ace_applicable, s, &len, isgroup); if (error == ENOSPC) { if (s != sbuf) { diff --git a/bsd/nfs/nfs4_vnops.c b/bsd/nfs/nfs4_vnops.c index 13e4ad3ff..d0812cfaa 100644 --- a/bsd/nfs/nfs4_vnops.c +++ b/bsd/nfs/nfs4_vnops.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2006-2015 Apple Inc. All rights reserved. + * Copyright (c) 2006-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -1401,6 +1401,8 @@ nfs4_vnop_getattr( if (error) return (error); + vap->va_flags |= VA_64BITOBJIDS; + /* copy what we have in nva to *a_vap */ if (VATTR_IS_ACTIVE(vap, va_rdev) && NFS_BITMAP_ISSET(nva.nva_bitmap, NFS_FATTR_RAWDEV)) { dev_t rdev = makedev(nva.nva_rawdev.specdata1, nva.nva_rawdev.specdata2); diff --git a/bsd/nfs/nfs_bio.c b/bsd/nfs/nfs_bio.c index acaf26c24..d65d98a1a 100644 --- a/bsd/nfs/nfs_bio.c +++ b/bsd/nfs/nfs_bio.c @@ -409,8 +409,8 @@ nfs_buf_upl_setup(struct nfsbuf *bp) */ upl_flags |= UPL_WILL_MODIFY; } - kret = ubc_create_upl(NFSTOV(bp->nb_np), NBOFF(bp), bp->nb_bufsize, - &upl, NULL, upl_flags); + kret = ubc_create_upl_kernel(NFSTOV(bp->nb_np), NBOFF(bp), bp->nb_bufsize, + &upl, NULL, upl_flags, VM_KERN_MEMORY_FILE); if (kret == KERN_INVALID_ARGUMENT) { /* vm object probably doesn't exist any more */ bp->nb_pagelist = NULL; diff --git a/bsd/nfs/nfs_gss.c b/bsd/nfs/nfs_gss.c index 53f4a08f2..0c97dda9c 100644 --- a/bsd/nfs/nfs_gss.c +++ b/bsd/nfs/nfs_gss.c @@ -629,7 +629,7 @@ nfs_gss_clnt_mnt_rele(struct nfsmount *nmp) } } -int nfs_root_steals_ctx = 1; +int nfs_root_steals_ctx = 0; static int nfs_gss_clnt_ctx_find_principal(struct nfsreq *req, uint8_t *principal, uint32_t plen, uint32_t nt) @@ -1834,6 +1834,12 @@ nfs_gss_clnt_gssd_upcall(struct nfsreq *req, struct nfs_gss_clnt_ctx *cp, uint32 cp->gss_clnt_token = NULL; cp->gss_clnt_tokenlen = 0; cp->gss_clnt_proc = RPCSEC_GSS_INIT; + /* Server's handle isn't valid. Don't reuse */ + cp->gss_clnt_handle_len = 0; + if (cp->gss_clnt_handle != NULL) { + FREE(cp->gss_clnt_handle, M_TEMP); + cp->gss_clnt_handle = NULL; + } } NFS_GSS_DBG("Retrycnt = %d nm_etype.count = %d\n", retrycnt, nmp->nm_etype.count); @@ -1879,10 +1885,7 @@ nfs_gss_clnt_gssd_upcall(struct nfsreq *req, struct nfs_gss_clnt_ctx *cp, uint32 nt = cp->gss_clnt_prinnt; } else if (nmp->nm_principal && IS_VALID_CRED(nmp->nm_mcred) && req->r_cred == nmp->nm_mcred) { plen = (uint32_t)strlen(nmp->nm_principal); - MALLOC(principal, uint8_t *, plen, M_TEMP, M_WAITOK | M_ZERO); - if (principal == NULL) - return (ENOMEM); - bcopy(nmp->nm_principal, principal, plen); + principal = (uint8_t *)nmp->nm_principal; cp->gss_clnt_prinnt = nt = GSSD_USER; } else if (nmp->nm_realm) { @@ -1978,6 +1981,12 @@ skip: cp->gss_clnt_major != GSS_S_CONTINUE_NEEDED) { NFS_GSS_DBG("Up call returned error\n"); nfs_gss_clnt_log_error(req, cp, major, minor); + /* Server's handle isn't valid. Don't reuse */ + cp->gss_clnt_handle_len = 0; + if (cp->gss_clnt_handle != NULL) { + FREE(cp->gss_clnt_handle, M_TEMP); + cp->gss_clnt_handle = NULL; + } } if (lucidlen > 0) { @@ -2045,6 +2054,12 @@ out: FREE(cp->gss_clnt_token, M_TEMP); cp->gss_clnt_token = NULL; cp->gss_clnt_tokenlen = 0; + /* Server's handle isn't valid. Don't reuse */ + cp->gss_clnt_handle_len = 0; + if (cp->gss_clnt_handle != NULL) { + FREE(cp->gss_clnt_handle, M_TEMP); + cp->gss_clnt_handle = NULL; + } NFS_GSS_DBG("Up call returned NFSERR_EAUTH"); return (NFSERR_EAUTH); @@ -3728,18 +3743,18 @@ nfs_gss_mach_alloc_buffer(u_char *buf, uint32_t buflen, vm_map_copy_t *addr) tbuflen = vm_map_round_page(buflen, vm_map_page_mask(ipc_kernel_map)); - kr = vm_allocate(ipc_kernel_map, &kmem_buf, tbuflen, VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_FILE)); + kr = vm_allocate_kernel(ipc_kernel_map, &kmem_buf, tbuflen, VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_FILE); if (kr != 0) { printf("nfs_gss_mach_alloc_buffer: vm_allocate failed\n"); return; } - kr = vm_map_wire(ipc_kernel_map, + kr = vm_map_wire_kernel(ipc_kernel_map, vm_map_trunc_page(kmem_buf, vm_map_page_mask(ipc_kernel_map)), vm_map_round_page(kmem_buf + tbuflen, vm_map_page_mask(ipc_kernel_map)), - VM_PROT_READ|VM_PROT_WRITE|VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_FILE), FALSE); + VM_PROT_READ|VM_PROT_WRITE, VM_KERN_MEMORY_FILE, FALSE); if (kr != 0) { printf("nfs_gss_mach_alloc_buffer: vm_map_wire failed\n"); return; diff --git a/bsd/nfs/nfs_ioctl.h b/bsd/nfs/nfs_ioctl.h index ff4f3eaae..f140e3ea8 100644 --- a/bsd/nfs/nfs_ioctl.h +++ b/bsd/nfs/nfs_ioctl.h @@ -38,12 +38,10 @@ * fsctl (vnop_ioctl) to detroy the callers credentials associated with the vnode's mount */ #define NFS_IOC_DESTROY_CRED _IO('n', 1) -#define NFS_FSCTL_DESTROY_CRED IOCBASECMD(NFS_IOC_DESTROY_CRED) + /* - * fsclt (vnop_ioctl) to set the callers credentials associated with the vnode's mount + * fsctl (vnop_ioctl) to set the callers credentials associated with the vnode's mount */ - - struct nfs_gss_principal { uint32_t princlen; /* length of data */ @@ -62,7 +60,7 @@ struct user_nfs_gss_principal { uint32_t princlen; /* length of data */ uint32_t nametype; /* nametype of data */ - user_addr_t principal; /* principal data in userspace */ + user64_addr_t principal; /* principal data in userspace */ uint32_t flags; /* Returned flags */ }; #endif @@ -72,8 +70,14 @@ struct user_nfs_gss_principal #define NFS_IOC_INVALID_CRED_FLAG 2 /* Found a credential, but its not valid */ #define NFS_IOC_SET_CRED _IOW('n', 2, struct nfs_gss_principal) -#define NFS_FSCTL_SET_CRED IOCBASECMD(NFS_IOC_SET_CRED) #define NFS_IOC_GET_CRED _IOWR('n', 3, struct nfs_gss_principal) -#define NFS_FSCTL_GET_CRED IOCBASECMD(NFS_IOC_GET_CRED) + +#ifdef KERNEL + +#define NFS_IOC_SET_CRED64 _IOW('n', 2, struct user_nfs_gss_principal) + +#define NFS_IOC_GET_CRED64 _IOWR('n', 3, struct user_nfs_gss_principal) +#endif + #endif diff --git a/bsd/nfs/nfs_lock.c b/bsd/nfs/nfs_lock.c index aaf567271..9920f3c89 100644 --- a/bsd/nfs/nfs_lock.c +++ b/bsd/nfs/nfs_lock.c @@ -607,7 +607,7 @@ wait_for_granted: ((lastmsg + nmp->nm_tprintf_delay) < now.tv_sec)) { lck_mtx_unlock(&nmp->nm_lock); lastmsg = now.tv_sec; - nfs_down(nmp, thd, 0, NFSSTA_LOCKTIMEO, "lockd not responding", 0); + nfs_down(nmp, thd, 0, NFSSTA_LOCKTIMEO, "lockd not responding", 1); wentdown = 1; } else lck_mtx_unlock(&nmp->nm_lock); diff --git a/bsd/nfs/nfs_socket.c b/bsd/nfs/nfs_socket.c index 023163518..5ade6666b 100644 --- a/bsd/nfs/nfs_socket.c +++ b/bsd/nfs/nfs_socket.c @@ -3727,7 +3727,7 @@ nfs_request_destroy(struct nfsreq *req) if (!req || !(req->r_flags & R_INITTED)) return; - nmp = req->r_np ? NFSTONMP(req->r_np) : req->r_nmp; + nmp = req->r_nmp; req->r_flags &= ~R_INITTED; if (req->r_lflags & RL_QUEUED) nfs_reqdequeue(req); @@ -3851,7 +3851,7 @@ nfs_request_add_header(struct nfsreq *req) req->r_mhead = NULL; } - nmp = req->r_np ? NFSTONMP(req->r_np) : req->r_nmp; + nmp = req->r_nmp; if (nfs_mount_gone(nmp)) return (ENXIO); @@ -3860,7 +3860,7 @@ nfs_request_add_header(struct nfsreq *req) return (error); req->r_mreqlen = mbuf_pkthdr_len(req->r_mhead); - nmp = req->r_np ? NFSTONMP(req->r_np) : req->r_nmp; + nmp = req->r_nmp; if (nfs_mount_gone(nmp)) return (ENXIO); lck_mtx_lock(&nmp->nm_lock); @@ -3889,7 +3889,7 @@ nfs_request_send(struct nfsreq *req, int wait) lck_mtx_lock(nfs_request_mutex); - nmp = req->r_np ? NFSTONMP(req->r_np) : req->r_nmp; + nmp = req->r_nmp; if (nfs_mount_gone(nmp)) { lck_mtx_unlock(nfs_request_mutex); return (ENXIO); @@ -3963,7 +3963,7 @@ nfs_request_finish( mrep = req->r_nmrep.nmc_mhead; - nmp = req->r_np ? NFSTONMP(req->r_np) : req->r_nmp; + nmp = req->r_nmp; if ((req->r_flags & R_CWND) && nmp) { /* diff --git a/bsd/nfs/nfs_subs.c b/bsd/nfs/nfs_subs.c index d9d6e016b..266710149 100644 --- a/bsd/nfs/nfs_subs.c +++ b/bsd/nfs/nfs_subs.c @@ -3240,7 +3240,7 @@ nfsrv_check_exports_allow_address(mbuf_t nam) { struct nfs_exportfs *nxfs; struct nfs_export *nx; - struct nfs_export_options *nxo; + struct nfs_export_options *nxo = NULL; if (nam == NULL) return (EACCES); diff --git a/bsd/nfs/nfs_syscalls.c b/bsd/nfs/nfs_syscalls.c index d4dead825..2a20ee2ea 100644 --- a/bsd/nfs/nfs_syscalls.c +++ b/bsd/nfs/nfs_syscalls.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2014 Apple Inc. All rights reserved. + * Copyright (c) 2000-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -173,7 +173,7 @@ SYSCTL_INT(_vfs_generic_nfs_client, OID_AUTO, squishy_flags, CTLFLAG_RW | CTLFLA SYSCTL_UINT(_vfs_generic_nfs_client, OID_AUTO, debug_ctl, CTLFLAG_RW | CTLFLAG_LOCKED, &nfs_debug_ctl, 0, ""); SYSCTL_INT(_vfs_generic_nfs_client, OID_AUTO, readlink_nocache, CTLFLAG_RW | CTLFLAG_LOCKED, &nfs_readlink_nocache, 0, ""); SYSCTL_INT(_vfs_generic_nfs_client, OID_AUTO, root_steals_gss_context, CTLFLAG_RW | CTLFLAG_LOCKED, &nfs_root_steals_ctx, 0, ""); -SYSCTL_STRING(_vfs_generic_nfs_client, OID_AUTO, default_nfs4domain, CTLFLAG_RW | CTLFLAG_LOCKED, nfs4_domain, sizeof(nfs4_domain), ""); +SYSCTL_STRING(_vfs_generic_nfs_client, OID_AUTO, default_nfs4domain, CTLFLAG_RW | CTLFLAG_LOCKED, nfs4_default_domain, sizeof(nfs4_default_domain), ""); #endif /* NFSCLIENT */ #if NFSSERVER @@ -226,7 +226,7 @@ static int mapid2name(struct nfs_testmapid *map) { int error; - int len = sizeof(map->ntm_name); + size_t len = sizeof(map->ntm_name); if (map->ntm_grpflag) error = kauth_cred_gid2guid((gid_t)map->ntm_id, &map->ntm_guid); @@ -242,12 +242,12 @@ mapid2name(struct nfs_testmapid *map) } - static int nfsclnt_testidmap(proc_t p, user_addr_t argp) { struct nfs_testmapid mapid; int error, coerror; + size_t len = sizeof(mapid.ntm_name); /* Let root make this call. */ error = proc_suser(p); @@ -257,10 +257,22 @@ nfsclnt_testidmap(proc_t p, user_addr_t argp) error = copyin(argp, &mapid, sizeof(mapid)); if (error) return (error); - if (mapid.ntm_name2id) + switch (mapid.ntm_lookup) { + case NTM_NAME2ID: error = mapname2id(&mapid); - else + break; + case NTM_ID2NAME: error = mapid2name(&mapid); + break; + case NTM_NAME2GUID: + error = nfs4_id2guid(mapid.ntm_name, &mapid.ntm_guid, mapid.ntm_grpflag); + break; + case NTM_GUID2NAME: + error = nfs4_guid2id(&mapid.ntm_guid, mapid.ntm_name, &len, mapid.ntm_grpflag); + break; + default: + return (EINVAL); + } coerror = copyout(&mapid, argp, sizeof(mapid)); @@ -581,6 +593,11 @@ out: vnode_put(vp); if (error) return (error); + /* + * At first blush, this may appear to leak a kernel stack + * address, but the copyout() never reaches &nfh.nfh_fhp + * (sizeof(fhandle_t) < sizeof(nfh)). + */ error = copyout((caddr_t)&nfh, uap->fhp, sizeof(fhandle_t)); return (error); } diff --git a/bsd/nfs/nfs_vfsops.c b/bsd/nfs/nfs_vfsops.c index f6524f3bb..3c5f2831a 100644 --- a/bsd/nfs/nfs_vfsops.c +++ b/bsd/nfs/nfs_vfsops.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2014 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -150,7 +150,7 @@ uint32_t nfs_open_owner_seqnum = 0; uint32_t nfs_lock_owner_seqnum = 0; thread_call_t nfs4_callback_timer_call; int nfs4_callback_timer_on = 0; -char nfs4_domain[MAXPATHLEN]; +char nfs4_default_domain[MAXPATHLEN]; /* nfsiod */ lck_grp_t *nfsiod_lck_grp; @@ -491,6 +491,51 @@ nfsmout: return (error); } +/* + * Return an NFS volume name from the mntfrom name. + */ +static void +nfs_get_volname(struct mount *mp, char *volname, size_t len) +{ + const char *ptr, *cptr; + const char *mntfrom = mp->mnt_vfsstat.f_mntfromname; + size_t mflen = strnlen(mntfrom, MAXPATHLEN+1); + + if (mflen > MAXPATHLEN || mflen == 0) { + strlcpy(volname, "Bad volname", len); + return; + } + + /* Move back over trailing slashes */ + for (ptr = &mntfrom[mflen-1]; ptr != mntfrom && *ptr == '/'; ptr--) { + mflen--; + } + + /* Find first character after the last slash */ + cptr = ptr = NULL; + for(size_t i = 0; i < mflen; i++) { + if (mntfrom[i] == '/') + ptr = &mntfrom[i+1]; + /* And the first character after the first colon */ + else if (cptr == NULL && mntfrom[i] == ':') + cptr = &mntfrom[i+1]; + } + + /* + * No slash or nothing after the last slash + * use everything past the first colon + */ + if (ptr == NULL || *ptr == '\0') + ptr = cptr; + /* Otherwise use the mntfrom name */ + if (ptr == NULL) + ptr = mntfrom; + + mflen = &mntfrom[mflen] - ptr; + len = mflen+1 < len ? mflen+1 : len; + + strlcpy(volname, ptr, len); +} /* * The NFS VFS_GETATTR function: "statfs"-type information is retrieved @@ -569,6 +614,11 @@ nfs_vfs_getattr(mount_t mp, struct vfs_attr *fsap, vfs_context_t ctx) lck_mtx_unlock(&nmp->nm_lock); } + if (VFSATTR_IS_ACTIVE(fsap, f_vol_name)) { + /*%%% IF fail over support is implemented we may need to take nm_lock */ + nfs_get_volname(mp, fsap->f_vol_name, MAXPATHLEN); + VFSATTR_SET_SUPPORTED(fsap, f_vol_name); + } if (VFSATTR_IS_ACTIVE(fsap, f_capabilities)) { u_int32_t caps, valid; nfsnode_t np = nmp->nm_dnp; @@ -748,14 +798,14 @@ nfs_vfs_getattr(mount_t mp, struct vfs_attr *fsap, vfs_context_t ctx) if (VFSATTR_IS_ACTIVE(fsap, f_attributes)) { fsap->f_attributes.validattr.commonattr = 0; fsap->f_attributes.validattr.volattr = - ATTR_VOL_CAPABILITIES | ATTR_VOL_ATTRIBUTES; + ATTR_VOL_NAME | ATTR_VOL_CAPABILITIES | ATTR_VOL_ATTRIBUTES; fsap->f_attributes.validattr.dirattr = 0; fsap->f_attributes.validattr.fileattr = 0; fsap->f_attributes.validattr.forkattr = 0; fsap->f_attributes.nativeattr.commonattr = 0; fsap->f_attributes.nativeattr.volattr = - ATTR_VOL_CAPABILITIES | ATTR_VOL_ATTRIBUTES; + ATTR_VOL_NAME | ATTR_VOL_CAPABILITIES | ATTR_VOL_ATTRIBUTES; fsap->f_attributes.nativeattr.dirattr = 0; fsap->f_attributes.nativeattr.fileattr = 0; fsap->f_attributes.nativeattr.forkattr = 0; @@ -1297,7 +1347,7 @@ nfs_mount_diskless_private( mp->mnt_segreadcnt = mp->mnt_segwritecnt = 32; mp->mnt_ioflags = 0; mp->mnt_realrootvp = NULLVP; - mp->mnt_authcache_ttl = CACHED_LOOKUP_RIGHT_TTL; + mp->mnt_authcache_ttl = 0; /* Allways go to our lookup */ mount_lock_init(mp); TAILQ_INIT(&mp->mnt_vnodelist); @@ -2261,7 +2311,7 @@ nocomponents: NFS_CLEAR_ATTRIBUTES(bitmap); NFS4_DEFAULT_ATTRIBUTES(bitmap); /* if no namedattr support or component is ".zfs", clear NFS_FATTR_NAMED_ATTR */ - if (NMFLAG(nmp, NONAMEDATTR) || !strcmp(fspath.np_components[comp], ".zfs")) + if (!NMFLAG(nmp, NAMEDATTR) || !strcmp(fspath.np_components[comp], ".zfs")) NFS_BITMAP_CLR(bitmap, NFS_FATTR_NAMED_ATTR); nfsm_chain_add_bitmap(error, &nmreq, bitmap, NFS_ATTR_BITMAP_LEN); nfsm_chain_build_done(error, &nmreq); @@ -2399,7 +2449,7 @@ nocomponents: gotfh: /* get attrs for mount point root */ - numops = NMFLAG(nmp, NONAMEDATTR) ? 2 : 3; // PUTFH + GETATTR + OPENATTR + numops = NMFLAG(nmp, NAMEDATTR) ? 3 : 2; // PUTFH + GETATTR + OPENATTR nfsm_chain_build_alloc_init(error, &nmreq, 25 * NFSX_UNSIGNED); nfsm_chain_add_compound_header(error, &nmreq, "mount", nmp->nm_minor_vers, numops); numops--; @@ -2410,10 +2460,10 @@ gotfh: NFS_CLEAR_ATTRIBUTES(bitmap); NFS4_DEFAULT_ATTRIBUTES(bitmap); /* if no namedattr support or last component is ".zfs", clear NFS_FATTR_NAMED_ATTR */ - if (NMFLAG(nmp, NONAMEDATTR) || ((fspath.np_compcount > 0) && !strcmp(fspath.np_components[fspath.np_compcount-1], ".zfs"))) + if (!NMFLAG(nmp, NAMEDATTR) || ((fspath.np_compcount > 0) && !strcmp(fspath.np_components[fspath.np_compcount-1], ".zfs"))) NFS_BITMAP_CLR(bitmap, NFS_FATTR_NAMED_ATTR); nfsm_chain_add_bitmap(error, &nmreq, bitmap, NFS_ATTR_BITMAP_LEN); - if (!NMFLAG(nmp, NONAMEDATTR)) { + if (NMFLAG(nmp, NAMEDATTR)) { numops--; nfsm_chain_add_32(error, &nmreq, NFS_OP_OPENATTR); nfsm_chain_add_32(error, &nmreq, 0); @@ -2433,7 +2483,7 @@ gotfh: NFS_CLEAR_ATTRIBUTES(nmp->nm_fsattr.nfsa_bitmap); error = nfs4_parsefattr(&nmrep, &nmp->nm_fsattr, &nvattr, NULL, NULL, NULL); nfsmout_if(error); - if (!NMFLAG(nmp, NONAMEDATTR)) { + if (NMFLAG(nmp, NAMEDATTR)) { nfsm_chain_op_check(error, &nmrep, NFS_OP_OPENATTR); if (error == ENOENT) error = 0; @@ -2700,6 +2750,13 @@ mountnfs( vfs_getnewfsid(mp); nmp->nm_mountp = mp; vfs_setauthopaque(mp); + /* + * Disable cache_lookup_path for NFS. NFS lookup always needs + * to be called to check if the directory attribute cache is + * valid and possibly purge the directory before calling + * cache_lookup. + */ + vfs_setauthcache_ttl(mp, 0); nfs_nhinit_finish(); @@ -3217,7 +3274,7 @@ mountnfs( } else { /* ignore these if not v4 */ NFS_BITMAP_CLR(nmp->nm_flags, NFS_MFLAG_NOCALLBACK); - NFS_BITMAP_CLR(nmp->nm_flags, NFS_MFLAG_NONAMEDATTR); + NFS_BITMAP_CLR(nmp->nm_flags, NFS_MFLAG_NAMEDATTR); NFS_BITMAP_CLR(nmp->nm_flags, NFS_MFLAG_NOACL); NFS_BITMAP_CLR(nmp->nm_flags, NFS_MFLAG_ACLONLY); } @@ -3312,9 +3369,13 @@ mountnfs( * buffers into multiple requests if the buffer size is * larger than the I/O size. */ +#ifndef CONFIG_EMBEDDED iosize = max(nmp->nm_rsize, nmp->nm_wsize); if (iosize < PAGE_SIZE) iosize = PAGE_SIZE; +#else + iosize = PAGE_SIZE; +#endif nmp->nm_biosize = trunc_page_32(iosize); /* For NFSv3 and greater, there is a (relatively) reliable ACCESS call. */ @@ -4634,6 +4695,30 @@ nfs_vfs_quotactl( } #else +static int +nfs_sa_getport(struct sockaddr *sa, int *error) +{ + int port = 0; + + if (sa->sa_family == AF_INET6) + port = ntohs(((struct sockaddr_in6*)sa)->sin6_port); + else if (sa->sa_family == AF_INET) + port = ntohs(((struct sockaddr_in*)sa)->sin_port); + else if (error) + *error = EIO; + + return port; +} + +static void +nfs_sa_setport(struct sockaddr *sa, int port) +{ + if (sa->sa_family == AF_INET6) + ((struct sockaddr_in6*)sa)->sin6_port = htons(port); + else if (sa->sa_family == AF_INET) + ((struct sockaddr_in*)sa)->sin_port = htons(port); +} + int nfs3_getquota(struct nfsmount *nmp, vfs_context_t ctx, uid_t id, int type, struct dqblk *dqb) { @@ -4648,6 +4733,7 @@ nfs3_getquota(struct nfsmount *nmp, vfs_context_t ctx, uid_t id, int type, struc uint32_t val = 0, bsize = 0; struct sockaddr *rqsaddr; struct timeval now; + struct timespec ts = { 1, 0 }; if (!nmp->nm_saddr) return (ENXIO); @@ -4655,38 +4741,91 @@ nfs3_getquota(struct nfsmount *nmp, vfs_context_t ctx, uid_t id, int type, struc if (NMFLAG(nmp, NOQUOTA)) return (ENOTSUP); - if (!nmp->nm_rqsaddr) - MALLOC(nmp->nm_rqsaddr, struct sockaddr *, sizeof(struct sockaddr_storage), M_SONAME, M_WAITOK|M_ZERO); - if (!nmp->nm_rqsaddr) - return (ENOMEM); - rqsaddr = nmp->nm_rqsaddr; - if (rqsaddr->sa_family == AF_INET6) - rqport = ntohs(((struct sockaddr_in6*)rqsaddr)->sin6_port); - else if (rqsaddr->sa_family == AF_INET) - rqport = ntohs(((struct sockaddr_in*)rqsaddr)->sin_port); + /* + * Allocate an address for rquotad if needed + */ + if (!nmp->nm_rqsaddr) { + int need_free = 0; + + MALLOC(rqsaddr, struct sockaddr *, sizeof(struct sockaddr_storage), M_SONAME, M_WAITOK|M_ZERO); + bcopy(nmp->nm_saddr, rqsaddr, min(sizeof(struct sockaddr_storage), nmp->nm_saddr->sa_len)); + /* Set the port to zero, will call rpcbind to get the port below */ + nfs_sa_setport(rqsaddr, 0); + microuptime(&now); + + lck_mtx_lock(&nmp->nm_lock); + if (!nmp->nm_rqsaddr) { + nmp->nm_rqsaddr = rqsaddr; + nmp->nm_rqsaddrstamp = now.tv_sec; + } else { + need_free = 1; + } + lck_mtx_unlock(&nmp->nm_lock); + if (need_free) + FREE(rqsaddr, M_SONAME); + } timeo = NMFLAG(nmp, SOFT) ? 10 : 60; rqproto = IPPROTO_UDP; /* XXX should prefer TCP if mount is TCP */ /* check if we have a recently cached rquota port */ microuptime(&now); - if (!rqport || ((nmp->nm_rqsaddrstamp + 60) >= (uint32_t)now.tv_sec)) { + lck_mtx_lock(&nmp->nm_lock); + rqsaddr = nmp->nm_rqsaddr; + rqport = nfs_sa_getport(rqsaddr, &error); + while (!error && (!rqport || ((nmp->nm_rqsaddrstamp + 60) <= (uint32_t)now.tv_sec))) { + error = nfs_sigintr(nmp, NULL, thd, 1); + if (error) { + lck_mtx_unlock(&nmp->nm_lock); + return (error); + } + if (nmp->nm_state & NFSSTA_RQUOTAINPROG) { + nmp->nm_state |= NFSSTA_WANTRQUOTA; + msleep(&nmp->nm_rqsaddr, &nmp->nm_lock, PZERO-1, "nfswaitrquotaaddr", &ts); + rqport = nfs_sa_getport(rqsaddr, &error); + continue; + } + nmp->nm_state |= NFSSTA_RQUOTAINPROG; + lck_mtx_unlock(&nmp->nm_lock); + /* send portmap request to get rquota port */ - bcopy(nmp->nm_saddr, rqsaddr, min(sizeof(struct sockaddr_storage), nmp->nm_saddr->sa_len)); error = nfs_portmap_lookup(nmp, ctx, rqsaddr, NULL, RPCPROG_RQUOTA, rqvers, rqproto, timeo); if (error) - return (error); - if (rqsaddr->sa_family == AF_INET6) - rqport = ntohs(((struct sockaddr_in6*)rqsaddr)->sin6_port); - else if (rqsaddr->sa_family == AF_INET) - rqport = ntohs(((struct sockaddr_in*)rqsaddr)->sin_port); - else - return (EIO); - if (!rqport) - return (ENOTSUP); + goto out; + rqport = nfs_sa_getport(rqsaddr, &error); + if (error) + goto out; + + if (!rqport) { + /* + * We overload PMAPPORT for the port if rquotad is not + * currently registered or up at the server. In the + * while loop above, port will be set and we will defer + * for a bit. Perhaps the service isn't online yet. + * + * Note that precludes using indirect, but we're not doing + * that here. + */ + rqport = PMAPPORT; + nfs_sa_setport(rqsaddr, rqport); + } microuptime(&now); nmp->nm_rqsaddrstamp = now.tv_sec; + out: + lck_mtx_lock(&nmp->nm_lock); + nmp->nm_state &= ~NFSSTA_RQUOTAINPROG; + if (nmp->nm_state & NFSSTA_WANTRQUOTA) { + nmp->nm_state &= ~NFSSTA_WANTRQUOTA; + wakeup(&nmp->nm_rqsaddr); + } } + lck_mtx_unlock(&nmp->nm_lock); + if (error) + return (error); + + /* Using PMAPPORT for unavailabe rquota service */ + if (rqport == PMAPPORT) + return (ENOTSUP); /* rquota request */ nfsm_chain_null(&nmreq); @@ -5095,7 +5234,7 @@ nfs_mountinfo_assemble(struct nfsmount *nmp, struct xdrbuf *xb) if (nmp->nm_vers >= NFS_VER4) { NFS_BITMAP_SET(mflags_mask, NFS_MFLAG_EPHEMERAL); NFS_BITMAP_SET(mflags_mask, NFS_MFLAG_NOCALLBACK); - NFS_BITMAP_SET(mflags_mask, NFS_MFLAG_NONAMEDATTR); + NFS_BITMAP_SET(mflags_mask, NFS_MFLAG_NAMEDATTR); NFS_BITMAP_SET(mflags_mask, NFS_MFLAG_NOACL); NFS_BITMAP_SET(mflags_mask, NFS_MFLAG_ACLONLY); } @@ -5129,8 +5268,8 @@ nfs_mountinfo_assemble(struct nfsmount *nmp, struct xdrbuf *xb) NFS_BITMAP_SET(mflags, NFS_MFLAG_EPHEMERAL); if (NMFLAG(nmp, NOCALLBACK)) NFS_BITMAP_SET(mflags, NFS_MFLAG_NOCALLBACK); - if (NMFLAG(nmp, NONAMEDATTR)) - NFS_BITMAP_SET(mflags, NFS_MFLAG_NONAMEDATTR); + if (NMFLAG(nmp, NAMEDATTR)) + NFS_BITMAP_SET(mflags, NFS_MFLAG_NAMEDATTR); if (NMFLAG(nmp, NOACL)) NFS_BITMAP_SET(mflags, NFS_MFLAG_NOACL); if (NMFLAG(nmp, ACLONLY)) @@ -5306,7 +5445,9 @@ nfs_vfs_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp, user_addr_t newp, size_t newlen, vfs_context_t ctx) { int error = 0, val; +#ifndef CONFIG_EMBEDDED int softnobrowse; +#endif struct sysctl_req *req = NULL; union union_vfsidctl vc; mount_t mp; @@ -5346,7 +5487,9 @@ nfs_vfs_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp, case VFS_CTL_TIMEO: case VFS_CTL_NOLOCKS: case VFS_CTL_NSTATUS: +#ifndef CONFIG_EMBEDDED case VFS_CTL_QUERY: +#endif req = CAST_DOWN(struct sysctl_req *, oldp); if (req == NULL) { return EFAULT; @@ -5370,6 +5513,10 @@ nfs_vfs_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp, req->newlen = vc.vc32.vc_len; } break; +#if CONFIG_EMBEDDED + case VFS_CTL_QUERY: + return EPERM; +#endif } switch(name[0]) { @@ -5508,6 +5655,7 @@ nfs_vfs_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp, break; /* build exported filesystem path */ + memset(statrec.path, 0, sizeof(statrec.path)); snprintf(statrec.path, sizeof(statrec.path), "%s%s%s", nxfs->nxfs_path, ((nxfs->nxfs_path[1] && nx->nx_path[0]) ? "/" : ""), nx->nx_path); @@ -5562,6 +5710,7 @@ nfs_vfs_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp, LIST_FOREACH(nx, &nxfs->nxfs_exports, nx_next) { /* copy out path */ if (bytes_avail >= sizeof(struct nfs_user_stat_path_rec)) { + memset(upath_rec.path, 0, sizeof(upath_rec.path)); snprintf(upath_rec.path, sizeof(upath_rec.path), "%s%s%s", nxfs->nxfs_path, ((nxfs->nxfs_path[1] && nx->nx_path[0]) ? "/" : ""), nx->nx_path); @@ -5593,6 +5742,7 @@ nfs_vfs_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp, if (bytes_avail >= sizeof(struct nfs_user_stat_user_rec)) { /* prepare a user stat rec for copying out */ ustat_rec.uid = unode->uid; + memset(&ustat_rec.sock, 0, sizeof(ustat_rec.sock)); bcopy(&unode->sock, &ustat_rec.sock, unode->sock.ss_len); ustat_rec.ops = unode->ops; ustat_rec.bytes_read = unode->bytes_read; @@ -5694,6 +5844,7 @@ ustat_skip: lck_mtx_unlock(&nmp->nm_lock); } break; +#ifndef CONFIG_EMBEDDED case VFS_CTL_QUERY: lck_mtx_lock(&nmp->nm_lock); /* XXX don't allow users to know about/disconnect unresponsive, soft, nobrowse mounts */ @@ -5710,6 +5861,7 @@ ustat_skip: lck_mtx_unlock(&nmp->nm_lock); error = SYSCTL_OUT(req, &vq, sizeof(vq)); break; +#endif case VFS_CTL_TIMEO: if (req->oldptr != USER_ADDR_NULL) { lck_mtx_lock(&nmp->nm_lock); diff --git a/bsd/nfs/nfs_vnops.c b/bsd/nfs/nfs_vnops.c index 38653ed5a..4c492ac65 100644 --- a/bsd/nfs/nfs_vnops.c +++ b/bsd/nfs/nfs_vnops.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2015 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -1731,6 +1731,7 @@ nfs3_vnop_getattr( int error; struct nfs_vattr nva; struct vnode_attr *vap = ap->a_vap; + struct nfsmount *nmp; dev_t rdev; /* @@ -1751,6 +1752,8 @@ nfs3_vnop_getattr( return (error); /* copy nva to *a_vap */ + nmp = VTONMP(ap->a_vp); + vap->va_flags |= nmp ? (nmp->nm_vers > 2 ? VA_64BITOBJIDS : 0) : 0; VATTR_RETURN(vap, va_type, nva.nva_type); VATTR_RETURN(vap, va_mode, nva.nva_mode); rdev = makedev(nva.nva_rawdev.specdata1, nva.nva_rawdev.specdata2); @@ -6696,7 +6699,7 @@ nfsspec_vnop_read( } */ *ap) { nfsnode_t np = VTONFS(ap->a_vp); - struct timeval now; + struct timespec now; int error; /* @@ -6705,9 +6708,9 @@ nfsspec_vnop_read( if ((error = nfs_node_lock(np))) return (error); np->n_flag |= NACC; - microtime(&now); + nanotime(&now); np->n_atim.tv_sec = now.tv_sec; - np->n_atim.tv_nsec = now.tv_usec * 1000; + np->n_atim.tv_nsec = now.tv_nsec; nfs_node_unlock(np); return (VOCALL(spec_vnodeop_p, VOFFSET(vnop_read), ap)); } @@ -6726,7 +6729,7 @@ nfsspec_vnop_write( } */ *ap) { nfsnode_t np = VTONFS(ap->a_vp); - struct timeval now; + struct timespec now; int error; /* @@ -6735,9 +6738,9 @@ nfsspec_vnop_write( if ((error = nfs_node_lock(np))) return (error); np->n_flag |= NUPD; - microtime(&now); + nanotime(&now); np->n_mtim.tv_sec = now.tv_sec; - np->n_mtim.tv_nsec = now.tv_usec * 1000; + np->n_mtim.tv_nsec = now.tv_nsec; nfs_node_unlock(np); return (VOCALL(spec_vnodeop_p, VOFFSET(vnop_write), ap)); } @@ -6804,7 +6807,7 @@ nfsfifo_vnop_read( } */ *ap) { nfsnode_t np = VTONFS(ap->a_vp); - struct timeval now; + struct timespec now; int error; /* @@ -6813,9 +6816,9 @@ nfsfifo_vnop_read( if ((error = nfs_node_lock(np))) return (error); np->n_flag |= NACC; - microtime(&now); + nanotime(&now); np->n_atim.tv_sec = now.tv_sec; - np->n_atim.tv_nsec = now.tv_usec * 1000; + np->n_atim.tv_nsec = now.tv_nsec; nfs_node_unlock(np); return (VOCALL(fifo_vnodeop_p, VOFFSET(vnop_read), ap)); } @@ -6834,7 +6837,7 @@ nfsfifo_vnop_write( } */ *ap) { nfsnode_t np = VTONFS(ap->a_vp); - struct timeval now; + struct timespec now; int error; /* @@ -6843,9 +6846,9 @@ nfsfifo_vnop_write( if ((error = nfs_node_lock(np))) return (error); np->n_flag |= NUPD; - microtime(&now); + nanotime(&now); np->n_mtim.tv_sec = now.tv_sec; - np->n_mtim.tv_nsec = now.tv_usec * 1000; + np->n_mtim.tv_nsec = now.tv_nsec; nfs_node_unlock(np); return (VOCALL(fifo_vnodeop_p, VOFFSET(vnop_write), ap)); } @@ -6867,21 +6870,21 @@ nfsfifo_vnop_close( vnode_t vp = ap->a_vp; nfsnode_t np = VTONFS(vp); struct vnode_attr vattr; - struct timeval now; + struct timespec now; mount_t mp; int error; if ((error = nfs_node_lock(np))) return (error); if (np->n_flag & (NACC | NUPD)) { - microtime(&now); + nanotime(&now); if (np->n_flag & NACC) { np->n_atim.tv_sec = now.tv_sec; - np->n_atim.tv_nsec = now.tv_usec * 1000; + np->n_atim.tv_nsec = now.tv_nsec; } if (np->n_flag & NUPD) { np->n_mtim.tv_sec = now.tv_sec; - np->n_mtim.tv_nsec = now.tv_usec * 1000; + np->n_mtim.tv_nsec = now.tv_nsec; } np->n_flag |= NCHG; if (!vnode_isinuse(vp, 1) && (mp = vnode_mount(vp)) && !vfs_isrdonly(mp)) { @@ -6935,14 +6938,18 @@ nfs_vnop_ioctl( return (EROFS); error = nfs_flush(VTONFS(vp), MNT_WAIT, vfs_context_thread(ctx), 0); break; - case NFS_FSCTL_DESTROY_CRED: + case NFS_IOC_DESTROY_CRED: if (!auth_is_kerberized(mp->nm_auth)) return (ENOTSUP); error = nfs_gss_clnt_ctx_remove(mp, vfs_context_ucred(ctx)); break; - case NFS_FSCTL_SET_CRED: + case NFS_IOC_SET_CRED: + case NFS_IOC_SET_CRED64: if (!auth_is_kerberized(mp->nm_auth)) return (ENOTSUP); + if ((ap->a_command == NFS_IOC_SET_CRED && vfs_context_is64bit(ctx)) || + (ap->a_command == NFS_IOC_SET_CRED64 && !vfs_context_is64bit(ctx))) + return (EINVAL); if (vfs_context_is64bit(ctx)) { gprinc = *(struct user_nfs_gss_principal *)ap->a_data; } else { @@ -6971,9 +6978,13 @@ nfs_vnop_ioctl( NFS_DBG(NFS_FAC_GSS, 7, "Seting credential to principal %s returned %d\n", p, error); FREE(p, M_TEMP); break; - case NFS_FSCTL_GET_CRED: + case NFS_IOC_GET_CRED: + case NFS_IOC_GET_CRED64: if (!auth_is_kerberized(mp->nm_auth)) return (ENOTSUP); + if ((ap->a_command == NFS_IOC_GET_CRED && vfs_context_is64bit(ctx)) || + (ap->a_command == NFS_IOC_GET_CRED64 && !vfs_context_is64bit(ctx))) + return (EINVAL); error = nfs_gss_clnt_ctx_get_principal(mp, ctx, &gprinc); if (error) break; @@ -7906,6 +7917,8 @@ nfs_vnode_notify(nfsnode_t np, uint32_t events) if (!nfs_getattrcache(np, &nvattr, 0)) { vap = &vattr; VATTR_INIT(vap); + + vap->va_flags |= nmp->nm_vers > 2 ? VA_64BITOBJIDS : 0; VATTR_RETURN(vap, va_fsid, vfs_statfs(nmp->nm_mountp)->f_fsid.val[0]); VATTR_RETURN(vap, va_fileid, nvattr.nva_fileid); VATTR_RETURN(vap, va_mode, nvattr.nva_mode); diff --git a/bsd/nfs/nfsmount.h b/bsd/nfs/nfsmount.h index feb205951..78aa778ad 100644 --- a/bsd/nfs/nfsmount.h +++ b/bsd/nfs/nfsmount.h @@ -381,6 +381,8 @@ struct nfsmount { #define NFSSTA_HASWRITEVERF 0x00040000 /* Has write verifier for V3 */ #define NFSSTA_GOTPATHCONF 0x00080000 /* Got the V3 pathconf info */ #define NFSSTA_GOTFSINFO 0x00100000 /* Got the V3 fsinfo */ +#define NFSSTA_WANTRQUOTA 0x00200000 /* Want rquota address */ +#define NFSSTA_RQUOTAINPROG 0x00400000 /* Getting rquota address */ #define NFSSTA_SENDING 0x00800000 /* Sending on socket */ #define NFSSTA_SNDLOCK 0x01000000 /* Send socket lock */ #define NFSSTA_WANTSND 0x02000000 /* Want above */ diff --git a/bsd/pgo/profile_runtime.c b/bsd/pgo/profile_runtime.c index f519a2690..ec24bfffe 100644 --- a/bsd/pgo/profile_runtime.c +++ b/bsd/pgo/profile_runtime.c @@ -26,6 +26,7 @@ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ +#include <machine/machine_routines.h> #include <sys/sysproto.h> #include <sys/malloc.h> #include <sys/systm.h> @@ -98,9 +99,8 @@ static int write_buffer(int flags, char *buffer) int kdp_pgo_reset_counters = 0; /* called in debugger context */ -static kern_return_t do_pgo_reset_counters(void *context) +kern_return_t do_pgo_reset_counters() { -#pragma unused(context) #ifdef PROFILE memset(&__pgo_hib_CountersStart, 0, ((uintptr_t)(&__pgo_hib_CountersEnd)) - ((uintptr_t)(&__pgo_hib_CountersStart))); @@ -110,13 +110,27 @@ static kern_return_t do_pgo_reset_counters(void *context) return KERN_SUCCESS; } +static kern_return_t +kextpgo_trap() +{ + return DebuggerTrapWithState(DBOP_RESET_PGO_COUNTERS, NULL, NULL, NULL, 0, FALSE, 0); +} + static kern_return_t pgo_reset_counters() { kern_return_t r; + boolean_t istate; + OSKextResetPgoCountersLock(); + + istate = ml_set_interrupts_enabled(FALSE); + kdp_pgo_reset_counters = 1; - r = DebuggerWithCallback(do_pgo_reset_counters, NULL, FALSE); + r = kextpgo_trap(); + + ml_set_interrupts_enabled(istate); + OSKextResetPgoCountersUnlock(); return r; } @@ -244,7 +258,7 @@ int grab_pgo_data(struct proc *p, goto out; } - MALLOC(buffer, char *, size64, M_TEMP, M_WAITOK); + MALLOC(buffer, char *, size64, M_TEMP, M_WAITOK | M_ZERO); if (!buffer) { err = ENOMEM; goto out; @@ -302,7 +316,7 @@ int grab_pgo_data(struct proc *p, err = EINVAL; goto out; } else { - MALLOC(buffer, char *, size, M_TEMP, M_WAITOK); + MALLOC(buffer, char *, size, M_TEMP, M_WAITOK | M_ZERO); if (!buffer) { err = ENOMEM; goto out; diff --git a/bsd/security/audit/audit.h b/bsd/security/audit/audit.h index 79136bbb7..61e818f60 100644 --- a/bsd/security/audit/audit.h +++ b/bsd/security/audit/audit.h @@ -44,11 +44,6 @@ #if defined(_KERNEL) || defined(KERNEL) -#if CONFIG_MACF -#include <sys/queue.h> -#include <security/mac_framework.h> -#endif - #include <bsm/audit.h> #include <sys/sysctl.h> diff --git a/bsd/security/audit/audit_bsm.c b/bsd/security/audit/audit_bsm.c index da938d8a1..edebfd61b 100644 --- a/bsd/security/audit/audit_bsm.c +++ b/bsd/security/audit/audit_bsm.c @@ -1256,6 +1256,20 @@ kaudit_to_bsm(struct kaudit_record *kar, struct au_record **pau) case AUE_UNMOUNT: UPATH1_VNODE1_TOKENS; break; + case AUE_FMOUNT: + if (ARG_IS_VALID(kar, ARG_FD)) { + tok = au_to_arg32(2, "dir fd", ar->ar_arg_fd); + kau_write(rec, tok); + } + if (ARG_IS_VALID(kar, ARG_FFLAGS)) { + tok = au_to_arg32(3, "flags", ar->ar_arg_fflags); + kau_write(rec, tok); + } + if (ARG_IS_VALID(kar, ARG_TEXT)) { + tok = au_to_text(ar->ar_arg_text); + kau_write(rec, tok); + } + break; case AUE_MSGCTL: ar->ar_event = audit_msgctl_to_event(ar->ar_arg_svipc_cmd); @@ -1412,6 +1426,7 @@ kaudit_to_bsm(struct kaudit_record *kar, struct au_record **pau) case AUE_SYMLINKAT: case AUE_MKDIRAT: case AUE_GETATTRLISTAT: + case AUE_SETATTRLISTAT: if (ARG_IS_VALID(kar, ARG_FD)) { tok = au_to_arg32(1, "dir fd", ar->ar_arg_fd); kau_write(rec, tok); diff --git a/bsd/sys/Makefile b/bsd/sys/Makefile index 366a86188..88c3a51c4 100644 --- a/bsd/sys/Makefile +++ b/bsd/sys/Makefile @@ -19,21 +19,21 @@ DATAFILES = \ appleapiopts.h acct.h aio.h attr.h \ buf.h cdefs.h clonefile.h conf.h \ dir.h dirent.h disk.h dkstat.h dtrace.h dtrace_glue.h dtrace_impl.h \ - errno.h ev.h event.h fasttrap.h fasttrap_isa.h fcntl.h file.h filedesc.h filio.h gmon.h \ + errno.h ev.h event.h fasttrap.h fasttrap_isa.h fcntl.h file.h filedesc.h filio.h fsgetpath.h gmon.h \ ioccom.h ioctl.h \ ioctl_compat.h ipc.h kernel.h kernel_types.h kern_event.h lctx.h loadable_fs.h lock.h lockf.h \ kauth.h kdebug.h kdebug_signpost.h kern_control.h lockstat.h malloc.h \ mbuf.h mman.h mount.h msg.h msgbuf.h netport.h param.h paths.h pipe.h poll.h \ proc.h proc_info.h ptrace.h queue.h quota.h reboot.h resource.h resourcevar.h \ sbuf.h posix_sem.h posix_shm.h random.h sdt.h\ - select.h sem.h semaphore.h shm.h signal.h signalvar.h socket.h socketvar.h sockio.h stat.h stdio.h \ + select.h sem.h semaphore.h shm.h signal.h signalvar.h snapshot.h socket.h socketvar.h sockio.h stat.h stdio.h \ sysctl.h syslimits.h syslog.h sys_domain.h termios.h time.h \ timeb.h times.h tprintf.h trace.h tty.h ttychars.h ttycom.h \ ttydefaults.h ttydev.h types.h ubc.h ucontext.h ucred.h uio.h un.h unistd.h unpcb.h \ user.h utfconv.h utsname.h vadvise.h vcmd.h \ vm.h vmmeter.h vmparam.h vnioctl.h vnode.h vnode_if.h vstat.h wait.h xattr.h \ _select.h _structs.h _types.h _endian.h domain.h protosw.h \ - spawn.h + spawn.h timex.h commpage.h # Installs header file for Apple internal use in user level - # $(DSTROOT)/System/Library/Frameworks/System.framework/PrivateHeaders @@ -67,6 +67,7 @@ PRIVATE_DATAFILES = \ kern_overrides.h \ mbuf.h \ mman.h \ + monotonic.h \ persona.h \ priv.h \ proc.h \ @@ -94,7 +95,8 @@ PRIVATE_DATAFILES = \ proc_uuid_policy.h \ priv.h \ pgo.h \ - memory_maintenance.h + memory_maintenance.h \ + commpage.h # Installs header file for kernel extensions - # $(DSTROOT)/System/Library/Frameworks/Kernel.framework/Headers @@ -118,7 +120,7 @@ KERNELFILES = \ kpi_mbuf.h kpi_socket.h kpi_socketfilter.h \ ttycom.h termios.h msg.h \ wait.h \ - spawn.h + spawn.h timex.h commpage.h # The last line was added to export needed headers for the MAC calls # whose source is outside of the xnu/bsd tree. @@ -131,6 +133,7 @@ PRIVATE_KERNELFILES = \ csr.h \ decmpfs.h \ disktab.h \ + eventhandler.h \ fbt.h \ fileport.h \ fsctl.h \ @@ -140,12 +143,14 @@ PRIVATE_KERNELFILES = \ kpi_private.h \ ktrace.h \ mach_swapon.h \ + monotonic.h \ msgbuf.h \ eventvar.h \ persona.h \ proc_info.h \ pthread_shims.h \ quota.h \ + reboot.h \ sem_internal.h \ shm_internal.h \ signalvar.h \ @@ -161,7 +166,8 @@ PRIVATE_KERNELFILES = \ pgo.h \ memory_maintenance.h \ doc_tombstone.h \ - fsevents.h + fsevents.h \ + work_interval.h \ # /usr/include INSTALL_MI_LIST = ${DATAFILES} diff --git a/bsd/sys/_types/Makefile b/bsd/sys/_types/Makefile index 0cf91f657..c64ec4c8a 100644 --- a/bsd/sys/_types/Makefile +++ b/bsd/sys/_types/Makefile @@ -17,6 +17,7 @@ EXPINC_SUBDIRS = DATAFILES = \ _blkcnt_t.h \ _blksize_t.h \ + _caddr_t.h \ _clock_t.h \ _ct_rune_t.h \ _dev_t.h \ @@ -79,16 +80,23 @@ DATAFILES = \ _ucontext.h \ _ucontext64.h \ _uid_t.h \ + _u_char.h \ + _u_int.h \ _u_int16_t.h \ _u_int32_t.h \ _u_int64_t.h \ _u_int8_t.h \ + _u_short.h \ _uintptr_t.h \ _useconds_t.h \ _uuid_t.h \ _va_list.h \ _wchar_t.h \ _wint_t.h \ + _user32_timex.h \ + _user64_timex.h \ + _user32_ntptimeval.h \ + _user64_ntptimeval.h \ # Installs header file for Apple internal use in user level - # $(DSTROOT)/System/Library/Frameworks/System.framework/PrivateHeaders diff --git a/bsd/sys/_types/_blkcnt_t.h b/bsd/sys/_types/_blkcnt_t.h index c8a9d3963..30668f8d6 100644 --- a/bsd/sys/_types/_blkcnt_t.h +++ b/bsd/sys/_types/_blkcnt_t.h @@ -27,5 +27,6 @@ */ #ifndef _BLKCNT_T #define _BLKCNT_T +#include <sys/_types.h> /* __darwin_blkcnt_t */ typedef __darwin_blkcnt_t blkcnt_t; #endif /* _BLKCNT_T */ diff --git a/bsd/sys/_types/_blksize_t.h b/bsd/sys/_types/_blksize_t.h index de50f2ca3..a71c373ce 100644 --- a/bsd/sys/_types/_blksize_t.h +++ b/bsd/sys/_types/_blksize_t.h @@ -27,5 +27,6 @@ */ #ifndef _BLKSIZE_T #define _BLKSIZE_T +#include <sys/_types.h> /* __darwin_blksize_t */ typedef __darwin_blksize_t blksize_t; #endif /* _BLKSIZE_T */ diff --git a/bsd/sys/_types/_caddr_t.h b/bsd/sys/_types/_caddr_t.h new file mode 100644 index 000000000..ad1ad5f62 --- /dev/null +++ b/bsd/sys/_types/_caddr_t.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _CADDR_T +#define _CADDR_T +typedef char * caddr_t; +#endif /* _CADDR_T */ diff --git a/bsd/sys/_types/_clock_t.h b/bsd/sys/_types/_clock_t.h index d58801cde..6fcdf6ba9 100644 --- a/bsd/sys/_types/_clock_t.h +++ b/bsd/sys/_types/_clock_t.h @@ -27,5 +27,6 @@ */ #ifndef _CLOCK_T #define _CLOCK_T +#include <machine/types.h> /* __darwin_clock_t */ typedef __darwin_clock_t clock_t; #endif /* _CLOCK_T */ diff --git a/bsd/sys/_types/_ct_rune_t.h b/bsd/sys/_types/_ct_rune_t.h index 116174cad..ad66d423a 100644 --- a/bsd/sys/_types/_ct_rune_t.h +++ b/bsd/sys/_types/_ct_rune_t.h @@ -28,5 +28,6 @@ #ifndef _CT_RUNE_T #define _CT_RUNE_T +#include <machine/_types.h> /* __darwin_ct_rune_t */ typedef __darwin_ct_rune_t ct_rune_t; #endif /* _CT_RUNE_T */ diff --git a/bsd/sys/_types/_dev_t.h b/bsd/sys/_types/_dev_t.h index cf6d3ad22..8a1e4053f 100644 --- a/bsd/sys/_types/_dev_t.h +++ b/bsd/sys/_types/_dev_t.h @@ -27,5 +27,6 @@ */ #ifndef _DEV_T #define _DEV_T +#include <sys/_types.h> /* __darwin_dev_t */ typedef __darwin_dev_t dev_t; /* device number */ #endif /* _DEV_T */ diff --git a/bsd/sys/_types/_fd_def.h b/bsd/sys/_types/_fd_def.h index 158fb8f16..51c43746c 100644 --- a/bsd/sys/_types/_fd_def.h +++ b/bsd/sys/_types/_fd_def.h @@ -27,6 +27,9 @@ */ #ifndef _FD_SET #define _FD_SET + +#include <machine/types.h> /* __int32_t */ + /* * Select uses bit masks of file descriptors in longs. These macros * manipulate such bit fields (the filesystem macros use chars). The diff --git a/bsd/sys/_types/_fsblkcnt_t.h b/bsd/sys/_types/_fsblkcnt_t.h index 12e03a537..ac012b146 100644 --- a/bsd/sys/_types/_fsblkcnt_t.h +++ b/bsd/sys/_types/_fsblkcnt_t.h @@ -27,5 +27,6 @@ */ #ifndef _FSBLKCNT_T #define _FSBLKCNT_T +#include <sys/_types.h> /* __darwin_fsblkcnt_t */ typedef __darwin_fsblkcnt_t fsblkcnt_t; #endif /* _FSBLKCNT_T */ diff --git a/bsd/sys/_types/_fsfilcnt_t.h b/bsd/sys/_types/_fsfilcnt_t.h index 9a72eca63..80bfa76ae 100644 --- a/bsd/sys/_types/_fsfilcnt_t.h +++ b/bsd/sys/_types/_fsfilcnt_t.h @@ -27,5 +27,6 @@ */ #ifndef _FSFILCNT_T #define _FSFILCNT_T +#include <sys/_types.h> /* __darwin_fsfilcnt_t */ typedef __darwin_fsfilcnt_t fsfilcnt_t; #endif /* _FSFILCNT_T */ diff --git a/bsd/sys/_types/_fsid_t.h b/bsd/sys/_types/_fsid_t.h index 5532b5715..5806d16ca 100644 --- a/bsd/sys/_types/_fsid_t.h +++ b/bsd/sys/_types/_fsid_t.h @@ -27,5 +27,6 @@ */ #ifndef _FSID_T #define _FSID_T +#include <sys/_types/_int32_t.h> /* int32_t */ typedef struct fsid { int32_t val[2]; } fsid_t; /* file system id type */ #endif /* _FSID_T */ diff --git a/bsd/sys/_types/_fsobj_id_t.h b/bsd/sys/_types/_fsobj_id_t.h index 20e1bcff9..a396cdff9 100644 --- a/bsd/sys/_types/_fsobj_id_t.h +++ b/bsd/sys/_types/_fsobj_id_t.h @@ -28,6 +28,8 @@ #ifndef _FSOBJ_ID_T #define _FSOBJ_ID_T +#include <sys/_types/_u_int32_t.h> /* u_int32_t */ + typedef struct fsobj_id { u_int32_t fid_objno; u_int32_t fid_generation; diff --git a/bsd/sys/_types/_gid_t.h b/bsd/sys/_types/_gid_t.h index f64f56c62..402f5c219 100644 --- a/bsd/sys/_types/_gid_t.h +++ b/bsd/sys/_types/_gid_t.h @@ -27,5 +27,6 @@ */ #ifndef _GID_T #define _GID_T +#include <sys/_types.h> /* __darwin_gid_t */ typedef __darwin_gid_t gid_t; #endif diff --git a/bsd/sys/_types/_id_t.h b/bsd/sys/_types/_id_t.h index b5a8a2f2c..79cd778da 100644 --- a/bsd/sys/_types/_id_t.h +++ b/bsd/sys/_types/_id_t.h @@ -27,5 +27,6 @@ */ #ifndef _ID_T #define _ID_T +#include <sys/_types.h> /* __darwin_id_t */ typedef __darwin_id_t id_t; /* can hold pid_t, gid_t, or uid_t */ #endif /* _ID_T */ diff --git a/bsd/sys/_types/_in_addr_t.h b/bsd/sys/_types/_in_addr_t.h index a534517c6..aa4956a1c 100644 --- a/bsd/sys/_types/_in_addr_t.h +++ b/bsd/sys/_types/_in_addr_t.h @@ -27,5 +27,6 @@ */ #ifndef _IN_ADDR_T #define _IN_ADDR_T +#include <machine/types.h> /* __uint32_t */ typedef __uint32_t in_addr_t; /* base type for internet address */ #endif /* _IN_ADDR_T */ diff --git a/bsd/sys/_types/_in_port_t.h b/bsd/sys/_types/_in_port_t.h index cf3da0020..69e719e89 100644 --- a/bsd/sys/_types/_in_port_t.h +++ b/bsd/sys/_types/_in_port_t.h @@ -27,5 +27,6 @@ */ #ifndef _IN_PORT_T #define _IN_PORT_T +#include <machine/types.h> /* __uint16_t */ typedef __uint16_t in_port_t; #endif /* _IN_PORT_T */ diff --git a/bsd/sys/_types/_ino64_t.h b/bsd/sys/_types/_ino64_t.h index a7ca59e97..effe9f6e6 100644 --- a/bsd/sys/_types/_ino64_t.h +++ b/bsd/sys/_types/_ino64_t.h @@ -27,5 +27,6 @@ */ #ifndef _INO64_T #define _INO64_T +#include <sys/_types.h> /* __darwin_ino64_t */ typedef __darwin_ino64_t ino64_t; /* 64bit inode number */ #endif /* _INO64_T */ diff --git a/bsd/sys/_types/_ino_t.h b/bsd/sys/_types/_ino_t.h index 2bc666f92..721f8646e 100644 --- a/bsd/sys/_types/_ino_t.h +++ b/bsd/sys/_types/_ino_t.h @@ -27,5 +27,6 @@ */ #ifndef _INO_T #define _INO_T +#include <sys/_types.h> /* __darwin_ino_t */ typedef __darwin_ino_t ino_t; /* inode number */ #endif /* _INO_T */ diff --git a/bsd/sys/_types/_intptr_t.h b/bsd/sys/_types/_intptr_t.h index c01f906f5..0e050f7a0 100644 --- a/bsd/sys/_types/_intptr_t.h +++ b/bsd/sys/_types/_intptr_t.h @@ -27,5 +27,7 @@ */ #ifndef _INTPTR_T #define _INTPTR_T +#include <machine/types.h> /* __darwin_intptr_t */ + typedef __darwin_intptr_t intptr_t; #endif /* _INTPTR_T */ diff --git a/bsd/sys/_types/_iovec_t.h b/bsd/sys/_types/_iovec_t.h index 9aa311d29..6905450ec 100644 --- a/bsd/sys/_types/_iovec_t.h +++ b/bsd/sys/_types/_iovec_t.h @@ -27,6 +27,7 @@ */ #ifndef _STRUCT_IOVEC #define _STRUCT_IOVEC +#include <sys/_types/_size_t.h> /* size_t */ struct iovec { void * iov_base; /* [XSI] Base address of I/O memory region */ size_t iov_len; /* [XSI] Size of region iov_base points to */ diff --git a/bsd/sys/_types/_key_t.h b/bsd/sys/_types/_key_t.h index 2b5bdbd4b..1d4ca01ed 100644 --- a/bsd/sys/_types/_key_t.h +++ b/bsd/sys/_types/_key_t.h @@ -27,5 +27,6 @@ */ #ifndef _KEY_T #define _KEY_T +#include <machine/types.h> /* __int32_t */ typedef __int32_t key_t; /* IPC key (for Sys V IPC) */ #endif /* _KEY_T */ diff --git a/bsd/sys/_types/_mach_port_t.h b/bsd/sys/_types/_mach_port_t.h index d2bbae678..8920a37b2 100644 --- a/bsd/sys/_types/_mach_port_t.h +++ b/bsd/sys/_types/_mach_port_t.h @@ -46,5 +46,6 @@ #ifndef _MACH_PORT_T #define _MACH_PORT_T +#include <sys/_types.h> /* __darwin_mach_port_t */ typedef __darwin_mach_port_t mach_port_t; #endif /* _MACH_PORT_T */ diff --git a/bsd/sys/_types/_mbstate_t.h b/bsd/sys/_types/_mbstate_t.h index 790d112a2..0f51de45c 100644 --- a/bsd/sys/_types/_mbstate_t.h +++ b/bsd/sys/_types/_mbstate_t.h @@ -28,5 +28,6 @@ #ifndef _MBSTATE_T #define _MBSTATE_T +#include <machine/types.h> /* __darwin_mbstate_t */ typedef __darwin_mbstate_t mbstate_t; #endif /* _MBSTATE_T */ diff --git a/bsd/sys/_types/_mode_t.h b/bsd/sys/_types/_mode_t.h index a378b7dcc..c4de010c7 100644 --- a/bsd/sys/_types/_mode_t.h +++ b/bsd/sys/_types/_mode_t.h @@ -27,5 +27,6 @@ */ #ifndef _MODE_T #define _MODE_T +#include <sys/_types.h> /* __darwin_mode_t */ typedef __darwin_mode_t mode_t; #endif /* _MODE_T */ diff --git a/bsd/sys/_types/_nlink_t.h b/bsd/sys/_types/_nlink_t.h index 6b0e8cd32..7d066e178 100644 --- a/bsd/sys/_types/_nlink_t.h +++ b/bsd/sys/_types/_nlink_t.h @@ -27,5 +27,6 @@ */ #ifndef _NLINK_T #define _NLINK_T +#include <machine/types.h> /* __uint16_t */ typedef __uint16_t nlink_t; /* link count */ #endif /* _NLINK_T */ diff --git a/bsd/sys/_types/_null.h b/bsd/sys/_types/_null.h index 8a32fe0d5..537c10a3f 100644 --- a/bsd/sys/_types/_null.h +++ b/bsd/sys/_types/_null.h @@ -25,6 +25,7 @@ * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ -#ifndef NULL +#ifndef NULL +#include <sys/_types.h> /* __DARWIN_NULL */ #define NULL __DARWIN_NULL #endif /* NULL */ diff --git a/bsd/sys/_types/_off_t.h b/bsd/sys/_types/_off_t.h index fc6eacad4..205207ea3 100644 --- a/bsd/sys/_types/_off_t.h +++ b/bsd/sys/_types/_off_t.h @@ -27,5 +27,6 @@ */ #ifndef _OFF_T #define _OFF_T +#include <sys/_types.h> /* __darwin_off_t */ typedef __darwin_off_t off_t; #endif /* _OFF_T */ diff --git a/bsd/sys/_types/_pid_t.h b/bsd/sys/_types/_pid_t.h index ea369b218..5050d5278 100644 --- a/bsd/sys/_types/_pid_t.h +++ b/bsd/sys/_types/_pid_t.h @@ -27,5 +27,6 @@ */ #ifndef _PID_T #define _PID_T +#include <sys/_types.h> /* __darwin_pid_t */ typedef __darwin_pid_t pid_t; #endif /* _PID_T */ diff --git a/bsd/sys/_types/_ptrdiff_t.h b/bsd/sys/_types/_ptrdiff_t.h index 2f7344551..40cba6035 100644 --- a/bsd/sys/_types/_ptrdiff_t.h +++ b/bsd/sys/_types/_ptrdiff_t.h @@ -28,5 +28,6 @@ #ifndef _PTRDIFF_T #define _PTRDIFF_T +#include <machine/types.h> /* __darwin_ptrdiff_t */ typedef __darwin_ptrdiff_t ptrdiff_t; #endif /* _PTRDIFF_T */ diff --git a/bsd/sys/_types/_rsize_t.h b/bsd/sys/_types/_rsize_t.h index 68e18ef71..7150c6693 100644 --- a/bsd/sys/_types/_rsize_t.h +++ b/bsd/sys/_types/_rsize_t.h @@ -27,5 +27,6 @@ */ #ifndef _RSIZE_T #define _RSIZE_T +#include <machine/types.h> /* __darwin_size_t */ typedef __darwin_size_t rsize_t; #endif /* _RSIZE_T */ diff --git a/bsd/sys/_types/_rune_t.h b/bsd/sys/_types/_rune_t.h index 19a231899..aa9d0470d 100644 --- a/bsd/sys/_types/_rune_t.h +++ b/bsd/sys/_types/_rune_t.h @@ -27,5 +27,6 @@ */ #ifndef _RUNE_T #define _RUNE_T +#include <machine/_types.h> /* __darwin_rune_t */ typedef __darwin_rune_t rune_t; #endif /* _RUNE_T */ diff --git a/bsd/sys/_types/_sa_family_t.h b/bsd/sys/_types/_sa_family_t.h index 3460f2661..ccd168b2c 100644 --- a/bsd/sys/_types/_sa_family_t.h +++ b/bsd/sys/_types/_sa_family_t.h @@ -27,5 +27,6 @@ */ #ifndef _SA_FAMILY_T #define _SA_FAMILY_T +#include <machine/types.h> /* __uint8_t */ typedef __uint8_t sa_family_t; #endif /* _SA_FAMILY_T */ diff --git a/bsd/sys/_types/_seek_set.h b/bsd/sys/_types/_seek_set.h index e6302b386..6bcdec84e 100644 --- a/bsd/sys/_types/_seek_set.h +++ b/bsd/sys/_types/_seek_set.h @@ -26,6 +26,8 @@ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ +#include <sys/cdefs.h> + /* whence values for lseek(2) */ #ifndef SEEK_SET #define SEEK_SET 0 /* set file offset to offset */ @@ -33,6 +35,7 @@ #define SEEK_END 2 /* set file offset to EOF plus offset */ #endif /* !SEEK_SET */ +#if __DARWIN_C_LEVEL >= __DARWIN_C_FULL #ifndef SEEK_HOLE #define SEEK_HOLE 3 /* set file offset to the start of the next hole greater than or equal to the supplied offset */ #endif @@ -40,3 +43,4 @@ #ifndef SEEK_DATA #define SEEK_DATA 4 /* set file offset to the start of the next non-hole file region greater than or equal to the supplied offset */ #endif +#endif /* __DARWIN_C_LEVEL >= __DARWIN_C_FULL */ diff --git a/bsd/sys/_types/_sigaltstack.h b/bsd/sys/_types/_sigaltstack.h index 24d23fefb..353cd5b98 100644 --- a/bsd/sys/_types/_sigaltstack.h +++ b/bsd/sys/_types/_sigaltstack.h @@ -28,11 +28,17 @@ /* Structure used in sigaltstack call. */ #ifndef _STRUCT_SIGALTSTACK + +#include <sys/cdefs.h> /* __DARWIN_UNIX03 */ + #if __DARWIN_UNIX03 #define _STRUCT_SIGALTSTACK struct __darwin_sigaltstack #else /* !__DARWIN_UNIX03 */ #define _STRUCT_SIGALTSTACK struct sigaltstack #endif /* __DARWIN_UNIX03 */ + +#include <machine/types.h> /* __darwin_size_t */ + _STRUCT_SIGALTSTACK { void *ss_sp; /* signal stack base */ diff --git a/bsd/sys/_types/_sigset_t.h b/bsd/sys/_types/_sigset_t.h index d4e9b6538..6bf670407 100644 --- a/bsd/sys/_types/_sigset_t.h +++ b/bsd/sys/_types/_sigset_t.h @@ -27,5 +27,6 @@ */ #ifndef _SIGSET_T #define _SIGSET_T +#include <sys/_types.h> /* __darwin_sigset_t */ typedef __darwin_sigset_t sigset_t; #endif /* _SIGSET_T */ diff --git a/bsd/sys/_types/_size_t.h b/bsd/sys/_types/_size_t.h index 8346ba897..67786d594 100644 --- a/bsd/sys/_types/_size_t.h +++ b/bsd/sys/_types/_size_t.h @@ -27,5 +27,6 @@ */ #ifndef _SIZE_T #define _SIZE_T +#include <machine/_types.h> /* __darwin_size_t */ typedef __darwin_size_t size_t; #endif /* _SIZE_T */ diff --git a/bsd/sys/_types/_socklen_t.h b/bsd/sys/_types/_socklen_t.h index 110a3fa74..b9354fde5 100644 --- a/bsd/sys/_types/_socklen_t.h +++ b/bsd/sys/_types/_socklen_t.h @@ -27,6 +27,7 @@ */ #ifndef _SOCKLEN_T #define _SOCKLEN_T +#include <machine/types.h> /* __darwin_socklen_t */ typedef __darwin_socklen_t socklen_t; #endif diff --git a/bsd/sys/_types/_ssize_t.h b/bsd/sys/_types/_ssize_t.h index 636a850d4..fef63730f 100644 --- a/bsd/sys/_types/_ssize_t.h +++ b/bsd/sys/_types/_ssize_t.h @@ -27,5 +27,6 @@ */ #ifndef _SSIZE_T #define _SSIZE_T +#include <machine/types.h> /* __darwin_ssize_t */ typedef __darwin_ssize_t ssize_t; #endif /* _SSIZE_T */ diff --git a/bsd/sys/_types/_suseconds_t.h b/bsd/sys/_types/_suseconds_t.h index 883143a73..837c4cab1 100644 --- a/bsd/sys/_types/_suseconds_t.h +++ b/bsd/sys/_types/_suseconds_t.h @@ -27,5 +27,6 @@ */ #ifndef _SUSECONDS_T #define _SUSECONDS_T +#include <sys/_types.h> /* __darwin_suseconds_t */ typedef __darwin_suseconds_t suseconds_t; #endif /* _SUSECONDS_T */ diff --git a/bsd/sys/_types/_time_t.h b/bsd/sys/_types/_time_t.h index 19b5f5e1e..ae87acb6f 100644 --- a/bsd/sys/_types/_time_t.h +++ b/bsd/sys/_types/_time_t.h @@ -27,5 +27,6 @@ */ #ifndef _TIME_T #define _TIME_T +#include <machine/types.h> /* __darwin_time_t */ typedef __darwin_time_t time_t; #endif /* _TIME_T */ diff --git a/bsd/sys/_types/_timespec.h b/bsd/sys/_types/_timespec.h index 73525337d..6837be1ad 100644 --- a/bsd/sys/_types/_timespec.h +++ b/bsd/sys/_types/_timespec.h @@ -27,6 +27,9 @@ */ #ifndef _STRUCT_TIMESPEC #define _STRUCT_TIMESPEC struct timespec + +#include <machine/types.h> /* __darwin_time_t */ + _STRUCT_TIMESPEC { __darwin_time_t tv_sec; diff --git a/bsd/sys/_types/_timeval.h b/bsd/sys/_types/_timeval.h index 2f73808a4..2f854b9d9 100644 --- a/bsd/sys/_types/_timeval.h +++ b/bsd/sys/_types/_timeval.h @@ -27,6 +27,10 @@ */ #ifndef _STRUCT_TIMEVAL #define _STRUCT_TIMEVAL struct timeval + +#include <machine/types.h> /* __darwin_time_t */ +#include <sys/_types.h> /* __darwin_suseconds_t */ + _STRUCT_TIMEVAL { __darwin_time_t tv_sec; /* seconds */ diff --git a/bsd/sys/_types/_timeval32.h b/bsd/sys/_types/_timeval32.h index ae5d3fe8b..dbb66d36e 100644 --- a/bsd/sys/_types/_timeval32.h +++ b/bsd/sys/_types/_timeval32.h @@ -27,6 +27,9 @@ */ #ifndef _STRUCT_TIMEVAL32 #define _STRUCT_TIMEVAL32 struct timeval32 + +#include <machine/types.h> /* __int32_t */ + _STRUCT_TIMEVAL32 { __int32_t tv_sec; /* seconds */ diff --git a/bsd/sys/_types/_timeval64.h b/bsd/sys/_types/_timeval64.h index c14f8338f..58a3255f9 100644 --- a/bsd/sys/_types/_timeval64.h +++ b/bsd/sys/_types/_timeval64.h @@ -28,6 +28,9 @@ #ifndef _STRUCT_TIMEVAL64 #define _STRUCT_TIMEVAL64 + +#include <machine/types.h> /* __int64_t */ + struct timeval64 { __int64_t tv_sec; /* seconds */ diff --git a/bsd/sys/_types/_u_char.h b/bsd/sys/_types/_u_char.h new file mode 100644 index 000000000..2a8a5b47e --- /dev/null +++ b/bsd/sys/_types/_u_char.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _U_CHAR +#define _U_CHAR +typedef unsigned char u_char; +#endif /* _U_CHAR */ diff --git a/bsd/sys/_types/_u_int.h b/bsd/sys/_types/_u_int.h new file mode 100644 index 000000000..79c36d1b4 --- /dev/null +++ b/bsd/sys/_types/_u_int.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _U_INT +#define _U_INT +typedef unsigned int u_int; +#endif /* _U_INT */ diff --git a/bsd/sys/_types/_u_short.h b/bsd/sys/_types/_u_short.h new file mode 100644 index 000000000..c610d14c8 --- /dev/null +++ b/bsd/sys/_types/_u_short.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _U_SHORT +#define _U_SHORT +typedef unsigned short u_short; +#endif /* _U_SHORT */ diff --git a/bsd/sys/_types/_ucontext.h b/bsd/sys/_types/_ucontext.h index 159ff0a7e..56a520d7a 100644 --- a/bsd/sys/_types/_ucontext.h +++ b/bsd/sys/_types/_ucontext.h @@ -26,11 +26,19 @@ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #ifndef _STRUCT_UCONTEXT + +#include <sys/cdefs.h> /* __DARWIN_UNIX03 */ + #if __DARWIN_UNIX03 #define _STRUCT_UCONTEXT struct __darwin_ucontext #else /* !__DARWIN_UNIX03 */ #define _STRUCT_UCONTEXT struct ucontext #endif /* __DARWIN_UNIX03 */ + +#include <machine/types.h> /* __darwin_size_t */ +#include <machine/_mcontext.h> /* _STRUCT_MCONTEXT */ +#include <sys/_types.h> /* __darwin_sigset_t */ + _STRUCT_UCONTEXT { int uc_onstack; diff --git a/bsd/sys/_types/_ucontext64.h b/bsd/sys/_types/_ucontext64.h index f2a620a19..1befcc9b0 100644 --- a/bsd/sys/_types/_ucontext64.h +++ b/bsd/sys/_types/_ucontext64.h @@ -26,11 +26,19 @@ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #ifndef _STRUCT_UCONTEXT64 + +#include <sys/cdefs.h> /* __DARWIN_UNIX03 */ + #if __DARWIN_UNIX03 #define _STRUCT_UCONTEXT64 struct __darwin_ucontext64 #else /* !__DARWIN_UNIX03 */ #define _STRUCT_UCONTEXT64 struct ucontext64 #endif /* __DARWIN_UNIX03 */ + +#include <machine/types.h> /* __darwin_size_t */ +#include <machine/_mcontext.h> /* _STRUCT_MCONTEXT */ +#include <sys/_types.h> /* __darwin_sigset_t */ + _STRUCT_UCONTEXT64 { int uc_onstack; diff --git a/bsd/sys/_types/_uid_t.h b/bsd/sys/_types/_uid_t.h index 678f7db14..a4ca9cb6f 100644 --- a/bsd/sys/_types/_uid_t.h +++ b/bsd/sys/_types/_uid_t.h @@ -27,5 +27,6 @@ */ #ifndef _UID_T #define _UID_T +#include <sys/_types.h> /* __darwin_uid_t */ typedef __darwin_uid_t uid_t; #endif /* _UID_T */ diff --git a/bsd/sys/_types/_useconds_t.h b/bsd/sys/_types/_useconds_t.h index 780d2364d..751a3748d 100644 --- a/bsd/sys/_types/_useconds_t.h +++ b/bsd/sys/_types/_useconds_t.h @@ -27,5 +27,6 @@ */ #ifndef _USECONDS_T #define _USECONDS_T +#include <sys/_types.h> /* __darwin_useconds_t */ typedef __darwin_useconds_t useconds_t; #endif /* _USECONDS_T */ diff --git a/bsd/sys/_types/_user32_ntptimeval.h b/bsd/sys/_types/_user32_ntptimeval.h new file mode 100644 index 000000000..cb69d5949 --- /dev/null +++ b/bsd/sys/_types/_user32_ntptimeval.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifdef KERNEL +#ifndef _STRUCT_USER32_NTPTIMEVAL +#define _STRUCT_USER32_NTPTIMEVAL struct user32_ntptimeval +_STRUCT_USER32_NTPTIMEVAL +{ + struct user32_timespec time; + user32_long_t maxerror; + user32_long_t esterror; + user32_long_t tai; + __int32_t time_state; + +}; +#endif /* _STRUCT_USER32_NTPTIMEVAL */ +#endif /* KERNEL */ diff --git a/bsd/sys/_types/_user32_timex.h b/bsd/sys/_types/_user32_timex.h new file mode 100644 index 000000000..5627982d6 --- /dev/null +++ b/bsd/sys/_types/_user32_timex.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifdef KERNEL +#ifndef _STRUCT_USER32_TIMEX +#define _STRUCT_USER32_TIMEX struct user32_timex +_STRUCT_USER32_TIMEX +{ + u_int32_t modes; + user32_long_t offset; + user32_long_t freq; + user32_long_t maxerror; + user32_long_t esterror; + __int32_t status; + user32_long_t constant; + user32_long_t precision; + user32_long_t tolerance; + + user32_long_t ppsfreq; + user32_long_t jitter; + __int32_t shift; + user32_long_t stabil; + user32_long_t jitcnt; + user32_long_t calcnt; + user32_long_t errcnt; + user32_long_t stbcnt; + +}; +#endif /* _STRUCT_USER32_TIMEX */ +#endif /* KERNEL */ diff --git a/bsd/sys/_types/_user64_ntptimeval.h b/bsd/sys/_types/_user64_ntptimeval.h new file mode 100644 index 000000000..3c3c557fd --- /dev/null +++ b/bsd/sys/_types/_user64_ntptimeval.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifdef KERNEL +#ifndef _STRUCT_USER64_NTPTIMEVAL +#define _STRUCT_USER64_NTPTIMEVAL struct user64_ntptimeval +_STRUCT_USER64_NTPTIMEVAL +{ + struct user64_timespec time; + user64_long_t maxerror; + user64_long_t esterror; + user64_long_t tai; + __int64_t time_state; + +}; +#endif /* _STRUCT_USER64_NTPTIMEVAL */ +#endif /* KERNEL */ diff --git a/bsd/sys/_types/_user64_timex.h b/bsd/sys/_types/_user64_timex.h new file mode 100644 index 000000000..2547592f4 --- /dev/null +++ b/bsd/sys/_types/_user64_timex.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifdef KERNEL +#ifndef _STRUCT_USER64_TIMEX +#define _STRUCT_USER64_TIMEX struct user64_timex +_STRUCT_USER64_TIMEX +{ + u_int64_t modes; + user64_long_t offset; + user64_long_t freq; + user64_long_t maxerror; + user64_long_t esterror; + __int64_t status; + user64_long_t constant; + user64_long_t precision; + user64_long_t tolerance; + + user64_long_t ppsfreq; + user64_long_t jitter; + __int64_t shift; + user64_long_t stabil; + user64_long_t jitcnt; + user64_long_t calcnt; + user64_long_t errcnt; + user64_long_t stbcnt; + +}; +#endif /* _STRUCT_USER64_TIMEX */ +#endif /* KERNEL */ diff --git a/bsd/sys/_types/_uuid_t.h b/bsd/sys/_types/_uuid_t.h index b61023892..e459143cd 100644 --- a/bsd/sys/_types/_uuid_t.h +++ b/bsd/sys/_types/_uuid_t.h @@ -27,5 +27,6 @@ */ #ifndef _UUID_T #define _UUID_T +#include <sys/_types.h> /* __darwin_uuid_t */ typedef __darwin_uuid_t uuid_t; #endif /* _UUID_T */ diff --git a/bsd/sys/_types/_va_list.h b/bsd/sys/_types/_va_list.h index c36072a5f..48a2b9969 100644 --- a/bsd/sys/_types/_va_list.h +++ b/bsd/sys/_types/_va_list.h @@ -28,5 +28,6 @@ #ifndef _VA_LIST_T #define _VA_LIST_T +#include <machine/types.h> /* __darwin_va_list */ typedef __darwin_va_list va_list; #endif /* _VA_LIST_T */ diff --git a/bsd/sys/_types/_wchar_t.h b/bsd/sys/_types/_wchar_t.h index 5a5d56cb7..a452a5fac 100644 --- a/bsd/sys/_types/_wchar_t.h +++ b/bsd/sys/_types/_wchar_t.h @@ -30,6 +30,7 @@ #ifndef __cplusplus #ifndef _WCHAR_T #define _WCHAR_T +#include <machine/_types.h> /* __darwin_wchar_t */ typedef __darwin_wchar_t wchar_t; #endif /* _WCHAR_T */ #endif /* __cplusplus */ diff --git a/bsd/sys/_types/_wint_t.h b/bsd/sys/_types/_wint_t.h index d1bbbad87..66dd7c37f 100644 --- a/bsd/sys/_types/_wint_t.h +++ b/bsd/sys/_types/_wint_t.h @@ -28,5 +28,6 @@ #ifndef _WINT_T #define _WINT_T +#include <machine/_types.h> /* __darwin_wint_t */ typedef __darwin_wint_t wint_t; #endif /* _WINT_T */ diff --git a/bsd/sys/acct.h b/bsd/sys/acct.h index 050ccc3f9..8162eb91f 100644 --- a/bsd/sys/acct.h +++ b/bsd/sys/acct.h @@ -70,6 +70,10 @@ #include <sys/appleapiopts.h> #include <sys/cdefs.h> +#include <sys/_types/_u_int16_t.h> /* u_int16_t */ +#include <sys/_types/_u_int32_t.h> /* u_int32_t */ +#include <sys/_types/_uid_t.h> /* uid_t */ + /* * Accounting structures; these use a comp_t type which is a 3 bits base 8 * exponent, 13 bit fraction ``floating point'' number. Units are 1/AHZ diff --git a/bsd/sys/attr.h b/bsd/sys/attr.h index 3437f9cc7..45540a9aa 100644 --- a/bsd/sys/attr.h +++ b/bsd/sys/attr.h @@ -320,6 +320,8 @@ typedef struct vol_capabilities_attr { * * VOL_CAP_INT_CLONE: When set, the volume supports clones. * + * VOL_CAP_INT_SNAPSHOT: When set, the volume supports snapshots. + * * VOL_CAP_INT_RENAME_SWAP: When set, the volume supports swapping * file system objects. * @@ -347,10 +349,7 @@ typedef struct vol_capabilities_attr { #define VOL_CAP_INT_REMOTE_EVENT 0x00008000 #endif /* PRIVATE */ #define VOL_CAP_INT_CLONE 0x00010000 -#ifdef PRIVATE -/* Volume supports snapshots */ #define VOL_CAP_INT_SNAPSHOT 0x00020000 -#endif /* PRIVATE */ #define VOL_CAP_INT_RENAME_SWAP 0x00040000 #define VOL_CAP_INT_RENAME_EXCL 0x00080000 @@ -432,7 +431,7 @@ typedef struct vol_attributes_attr { * * ATTR_CMN_DATA_PROTECT_FLAGS */ -#define ATTR_CMN_SETMASK 0x41C7FF00 +#define ATTR_CMN_SETMASK 0x51C7FF00 #define ATTR_CMN_VOLSETMASK 0x00006700 #define ATTR_VOL_FSTYPE 0x00000001 @@ -506,9 +505,7 @@ typedef struct vol_attributes_attr { /* CMNEXT attributes extend the common attributes, but in the forkattr field */ #define ATTR_CMNEXT_RELPATH 0x00000004 #define ATTR_CMNEXT_PRIVATESIZE 0x00000008 -#ifdef PRIVATE #define ATTR_CMNEXT_LINKID 0x00000010 -#endif /* PRIVATE */ #define ATTR_CMNEXT_VALIDMASK 0x0000001c #define ATTR_CMNEXT_SETMASK 0x00000000 diff --git a/bsd/sys/bitstring.h b/bsd/sys/bitstring.h index f4bb7fa54..3da8f42db 100644 --- a/bsd/sys/bitstring.h +++ b/bsd/sys/bitstring.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013 Apple Inc. All rights reserved. + * Copyright (c) 2013-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -68,11 +68,11 @@ typedef uint8_t bitstr_t; /* internal macros */ /* byte of the bitstring bit is in */ -#define _bit_byte(bit) \ +#define _bitstr_byte(bit) \ ((bit) >> 3) /* mask for the bit within its byte */ -#define _bit_mask(bit) \ +#define _bitstr_mask(bit) \ (1 << ((bit) & 0x7)) /* external macros */ @@ -85,31 +85,31 @@ typedef uint8_t bitstr_t; ((name)[bitstr_size(nbits)]) /* is bit N of bitstring name set? */ -#define bit_test(name, bit) \ - ((name)[_bit_byte(bit)] & _bit_mask(bit)) +#define bitstr_test(name, bit) \ + ((name)[_bitstr_byte(bit)] & _bitstr_mask(bit)) /* set bit N of bitstring name */ -#define bit_set(name, bit) \ - ((name)[_bit_byte(bit)] |= _bit_mask(bit)) +#define bitstr_set(name, bit) \ + ((name)[_bitstr_byte(bit)] |= _bitstr_mask(bit)) /* set bit N of bitstring name (atomic) */ -#define bit_set_atomic(name, bit) \ - atomic_bitset_8(&((name)[_bit_byte(bit)]), _bit_mask(bit)) +#define bitstr_set_atomic(name, bit) \ + atomic_bitset_8(&((name)[_bitstr_byte(bit)]), _bitstr_mask(bit)) /* clear bit N of bitstring name */ -#define bit_clear(name, bit) \ - ((name)[_bit_byte(bit)] &= ~_bit_mask(bit)) +#define bitstr_clear(name, bit) \ + ((name)[_bitstr_byte(bit)] &= ~_bitstr_mask(bit)) /* clear bit N of bitstring name (atomic) */ -#define bit_clear_atomic(name, bit) \ - atomic_bitclear_8(&((name)[_bit_byte(bit)]), _bit_mask(bit)) +#define bitstr_clear_atomic(name, bit) \ + atomic_bitclear_8(&((name)[_bitstr_byte(bit)]), _bitstr_mask(bit)) /* clear bits start ... stop in bitstring */ -#define bit_nclear(name, start, stop) do { \ +#define bitstr_nclear(name, start, stop) do { \ bitstr_t *_name = (name); \ int _start = (start), _stop = (stop); \ - int _startbyte = _bit_byte(_start); \ - int _stopbyte = _bit_byte(_stop); \ + int _startbyte = _bitstr_byte(_start); \ + int _stopbyte = _bitstr_byte(_stop); \ if (_startbyte == _stopbyte) { \ _name[_startbyte] &= ((0xff >> (8 - (_start & 0x7))) | \ (0xff << ((_stop & 0x7) + 1))); \ @@ -122,11 +122,11 @@ typedef uint8_t bitstr_t; } while (0) /* set bits start ... stop in bitstring */ -#define bit_nset(name, start, stop) do { \ +#define bitstr_nset(name, start, stop) do { \ bitstr_t *_name = (name); \ int _start = (start), _stop = (stop); \ - int _startbyte = _bit_byte(_start); \ - int _stopbyte = _bit_byte(_stop); \ + int _startbyte = _bitstr_byte(_start); \ + int _stopbyte = _bitstr_byte(_stop); \ if (_startbyte == _stopbyte) { \ _name[_startbyte] |= ((0xff << (_start & 0x7)) & \ (0xff >> (7 - (_stop & 0x7)))); \ @@ -139,10 +139,10 @@ typedef uint8_t bitstr_t; } while (0) /* find first bit clear in name */ -#define bit_ffc(name, nbits, value) do { \ +#define bitstr_ffc(name, nbits, value) do { \ bitstr_t *_name = (name); \ int _byte, _nbits = (nbits); \ - int _stopbyte = _bit_byte(_nbits - 1), _value = -1; \ + int _stopbyte = _bitstr_byte(_nbits - 1), _value = -1; \ if (_nbits > 0) \ for (_byte = 0; _byte <= _stopbyte; ++_byte) \ if (_name[_byte] != 0xff) { \ @@ -158,10 +158,10 @@ typedef uint8_t bitstr_t; } while (0) /* find first bit set in name */ -#define bit_ffs(name, nbits, value) do { \ +#define bitstr_ffs(name, nbits, value) do { \ bitstr_t *_name = (name); \ int _byte, _nbits = (nbits); \ - int _stopbyte = _bit_byte(_nbits - 1), _value = -1; \ + int _stopbyte = _bitstr_byte(_nbits - 1), _value = -1; \ if (_nbits > 0) \ for (_byte = 0; _byte <= _stopbyte; ++_byte) \ if (_name[_byte]) { \ diff --git a/bsd/sys/buf_internal.h b/bsd/sys/buf_internal.h index 3b007d99e..23c9ecba6 100644 --- a/bsd/sys/buf_internal.h +++ b/bsd/sys/buf_internal.h @@ -104,6 +104,7 @@ struct buf { LIST_ENTRY(buf) b_vnbufs; /* Buffer's associated vnode. */ TAILQ_ENTRY(buf) b_freelist; /* Free list position if not active. */ int b_timestamp; /* timestamp for queuing operation */ + struct timeval b_timestamp_tv; /* microuptime for disk conditioner */ int b_whichq; /* the free list the buffer belongs to */ volatile uint32_t b_flags; /* B_* flags. */ volatile uint32_t b_lflags; /* BL_BUSY | BL_WANTED flags... protected by buf_mtx */ diff --git a/bsd/sys/cdefs.h b/bsd/sys/cdefs.h index 8137cb3d9..68b4d22d9 100644 --- a/bsd/sys/cdefs.h +++ b/bsd/sys/cdefs.h @@ -498,6 +498,12 @@ #define __DARWIN_ONLY_UNIX_CONFORMANCE 1 #define __DARWIN_ONLY_VERS_1050 1 #endif /* PLATFORM_WatchSimulator */ +#ifdef PLATFORM_BridgeOS +/* Platform: BridgeOS */ +#define __DARWIN_ONLY_64_BIT_INO_T 1 +#define __DARWIN_ONLY_UNIX_CONFORMANCE 1 +#define __DARWIN_ONLY_VERS_1050 1 +#endif /* PLATFORM_BridgeOS */ #ifdef PLATFORM_MacOSX /* Platform: MacOSX */ #define __DARWIN_ONLY_64_BIT_INO_T 0 @@ -849,6 +855,8 @@ */ #if !defined(__sys_cdefs_arch_unknown__) && defined(__i386__) #elif !defined(__sys_cdefs_arch_unknown__) && defined(__x86_64__) +#elif !defined(__sys_cdefs_arch_unknown__) && defined(__arm__) +#elif !defined(__sys_cdefs_arch_unknown__) && defined(__arm64__) #else #error Unsupported architecture #endif diff --git a/bsd/sys/clonefile.h b/bsd/sys/clonefile.h index 17773fd3a..45dfef2bf 100644 --- a/bsd/sys/clonefile.h +++ b/bsd/sys/clonefile.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016 Apple Inc. All rights reserved. + * Copyright (c) 2015-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -30,7 +30,8 @@ #define _SYS_CLONEFILE_H_ /* Options for clonefile calls */ -#define CLONE_NOFOLLOW 0x0001 /* Don't follow symbolic links */ +#define CLONE_NOFOLLOW 0x0001 /* Don't follow symbolic links */ +#define CLONE_NOOWNERCOPY 0x0002 /* Don't copy ownership information from source */ #ifndef KERNEL diff --git a/bsd/sys/coalition.h b/bsd/sys/coalition.h index cf2811c5b..d3ab93f32 100644 --- a/bsd/sys/coalition.h +++ b/bsd/sys/coalition.h @@ -47,6 +47,8 @@ int coalition_reap(uint64_t cid, uint32_t flags); /* Wrappers around __coalition_info syscall (with proper struct types) */ int coalition_info_resource_usage(uint64_t cid, struct coalition_resource_usage *cru, size_t sz); +int coalition_info_set_name(uint64_t cid, const char *name, size_t size); +int coalition_info_set_efficiency(uint64_t cid, uint64_t flags); #else /* KERNEL */ diff --git a/bsd/sys/codesign.h b/bsd/sys/codesign.h index ca225ce58..15a39365b 100644 --- a/bsd/sys/codesign.h +++ b/bsd/sys/codesign.h @@ -29,36 +29,11 @@ #ifndef _SYS_CODESIGN_H_ #define _SYS_CODESIGN_H_ -/* code signing attributes of a process */ -#define CS_VALID 0x0000001 /* dynamically valid */ -#define CS_ADHOC 0x0000002 /* ad hoc signed */ -#define CS_GET_TASK_ALLOW 0x0000004 /* has get-task-allow entitlement */ -#define CS_INSTALLER 0x0000008 /* has installer entitlement */ - -#define CS_HARD 0x0000100 /* don't load invalid pages */ -#define CS_KILL 0x0000200 /* kill process if it becomes invalid */ -#define CS_CHECK_EXPIRATION 0x0000400 /* force expiration checking */ -#define CS_RESTRICT 0x0000800 /* tell dyld to treat restricted */ -#define CS_ENFORCEMENT 0x0001000 /* require enforcement */ -#define CS_REQUIRE_LV 0x0002000 /* require library validation */ -#define CS_ENTITLEMENTS_VALIDATED 0x0004000 /* code signature permits restricted entitlements */ - -#define CS_ALLOWED_MACHO (CS_ADHOC | CS_HARD | CS_KILL | CS_CHECK_EXPIRATION | CS_RESTRICT | CS_ENFORCEMENT | CS_REQUIRE_LV) - -#define CS_EXEC_SET_HARD 0x0100000 /* set CS_HARD on any exec'ed process */ -#define CS_EXEC_SET_KILL 0x0200000 /* set CS_KILL on any exec'ed process */ -#define CS_EXEC_SET_ENFORCEMENT 0x0400000 /* set CS_ENFORCEMENT on any exec'ed process */ -#define CS_EXEC_SET_INSTALLER 0x0800000 /* set CS_INSTALLER on any exec'ed process */ - -#define CS_KILLED 0x1000000 /* was killed by kernel for invalidity */ -#define CS_DYLD_PLATFORM 0x2000000 /* dyld used to load this is a platform binary */ -#define CS_PLATFORM_BINARY 0x4000000 /* this is a platform binary */ -#define CS_PLATFORM_PATH 0x8000000 /* platform binary by the fact of path (osx only) */ -#define CS_DEBUGGED 0x10000000 /* process is currently or has previously been debugged and allowed to run with invalid pages */ -#define CS_SIGNED 0x20000000 /* process has a signature (may have gone invalid) */ -#define CS_DEV_CODE 0x40000000 /* code is dev signed, cannot be loaded into prod signed code (will go away with rdar://problem/28322552) */ - -#define CS_ENTITLEMENT_FLAGS (CS_GET_TASK_ALLOW | CS_INSTALLER) +#if KERNEL +#include <kern/cs_blobs.h> +#else +#include <System/kern/cs_blobs.h> +#endif /* MAC flags used by F_ADDFILESIGS_* */ #define MAC_VNODE_CHECK_DYLD_SIM 0x1 /* tells the MAC framework that dyld-sim is being loaded */ @@ -79,111 +54,9 @@ #define CS_OPS_BLOB 10 /* get codesign blob */ #define CS_OPS_IDENTITY 11 /* get codesign identity */ #define CS_OPS_CLEARINSTALLER 12 /* clear INSTALLER flag */ +#define CS_OPS_CLEARPLATFORM 13 /* clear platform binary status (DEVELOPMENT-only) */ -/* - * Magic numbers used by Code Signing - */ -enum { - CSMAGIC_REQUIREMENT = 0xfade0c00, /* single Requirement blob */ - CSMAGIC_REQUIREMENTS = 0xfade0c01, /* Requirements vector (internal requirements) */ - CSMAGIC_CODEDIRECTORY = 0xfade0c02, /* CodeDirectory blob */ - CSMAGIC_EMBEDDED_SIGNATURE = 0xfade0cc0, /* embedded form of signature data */ - CSMAGIC_EMBEDDED_SIGNATURE_OLD = 0xfade0b02, /* XXX */ - CSMAGIC_EMBEDDED_ENTITLEMENTS = 0xfade7171, /* embedded entitlements */ - CSMAGIC_DETACHED_SIGNATURE = 0xfade0cc1, /* multi-arch collection of embedded signatures */ - CSMAGIC_BLOBWRAPPER = 0xfade0b01, /* CMS Signature, among other things */ - - CS_SUPPORTSSCATTER = 0x20100, - CS_SUPPORTSTEAMID = 0x20200, - - CSSLOT_CODEDIRECTORY = 0, /* slot index for CodeDirectory */ - CSSLOT_INFOSLOT = 1, - CSSLOT_REQUIREMENTS = 2, - CSSLOT_RESOURCEDIR = 3, - CSSLOT_APPLICATION = 4, - CSSLOT_ENTITLEMENTS = 5, - - CSSLOT_ALTERNATE_CODEDIRECTORIES = 0x1000, /* first alternate CodeDirectory, if any */ - CSSLOT_ALTERNATE_CODEDIRECTORY_MAX = 5, /* max number of alternate CD slots */ - CSSLOT_ALTERNATE_CODEDIRECTORY_LIMIT = CSSLOT_ALTERNATE_CODEDIRECTORIES + CSSLOT_ALTERNATE_CODEDIRECTORY_MAX, /* one past the last */ - - CSSLOT_SIGNATURESLOT = 0x10000, /* CMS Signature */ - - CSTYPE_INDEX_REQUIREMENTS = 0x00000002, /* compat with amfi */ - CSTYPE_INDEX_ENTITLEMENTS = 0x00000005, /* compat with amfi */ - - CS_HASHTYPE_SHA1 = 1, - CS_HASHTYPE_SHA256 = 2, - CS_HASHTYPE_SHA256_TRUNCATED = 3, - CS_HASHTYPE_SHA384 = 4, - - CS_SHA1_LEN = 20, - CS_SHA256_TRUNCATED_LEN = 20, - - CS_CDHASH_LEN = 20, /* always - larger hashes are truncated */ - CS_HASH_MAX_SIZE = 48, /* max size of the hash we'll support */ -}; - - -#define KERNEL_HAVE_CS_CODEDIRECTORY 1 -#define KERNEL_CS_CODEDIRECTORY_HAVE_PLATFORM 1 - -/* - * C form of a CodeDirectory. - */ -typedef struct __CodeDirectory { - uint32_t magic; /* magic number (CSMAGIC_CODEDIRECTORY) */ - uint32_t length; /* total length of CodeDirectory blob */ - uint32_t version; /* compatibility version */ - uint32_t flags; /* setup and mode flags */ - uint32_t hashOffset; /* offset of hash slot element at index zero */ - uint32_t identOffset; /* offset of identifier string */ - uint32_t nSpecialSlots; /* number of special hash slots */ - uint32_t nCodeSlots; /* number of ordinary (code) hash slots */ - uint32_t codeLimit; /* limit to main image signature range */ - uint8_t hashSize; /* size of each hash in bytes */ - uint8_t hashType; /* type of hash (cdHashType* constants) */ - uint8_t platform; /* platform identifier; zero if not platform binary */ - uint8_t pageSize; /* log2(page size in bytes); 0 => infinite */ - uint32_t spare2; /* unused (must be zero) */ - /* Version 0x20100 */ - uint32_t scatterOffset; /* offset of optional scatter vector */ - /* Version 0x20200 */ - uint32_t teamOffset; /* offset of optional team identifier */ - /* followed by dynamic content as located by offset fields above */ -} CS_CodeDirectory; - -/* - * Structure of an embedded-signature SuperBlob - */ - -typedef struct __BlobIndex { - uint32_t type; /* type of entry */ - uint32_t offset; /* offset of entry */ -} CS_BlobIndex; - -typedef struct __SC_SuperBlob { - uint32_t magic; /* magic number */ - uint32_t length; /* total length of SuperBlob */ - uint32_t count; /* number of index entries following */ - CS_BlobIndex index[]; /* (count) entries */ - /* followed by Blobs in no particular order as indicated by offsets in index */ -} CS_SuperBlob; - -#define KERNEL_HAVE_CS_GENERICBLOB 1 -typedef struct __SC_GenericBlob { - uint32_t magic; /* magic number */ - uint32_t length; /* total length of blob */ - char data[]; -} CS_GenericBlob; - -typedef struct __SC_Scatter { - uint32_t count; // number of pages; zero for sentinel (only) - uint32_t base; // first page number - uint64_t targetOffset; // offset in target - uint64_t spare; // reserved -} SC_Scatter; - +#define CS_MAX_TEAMID_LEN 64 #ifndef KERNEL @@ -228,7 +101,12 @@ const char * csblob_get_teamid(struct cs_blob *); const char * csblob_get_identity(struct cs_blob *); const uint8_t * csblob_get_cdhash(struct cs_blob *); int csblob_get_platform_binary(struct cs_blob *); -unsigned int csblob_get_flags(struct cs_blob *blob); +unsigned int csblob_get_flags(struct cs_blob *); +uint8_t csblob_get_hashtype(struct cs_blob const *); +unsigned int csblob_get_signer_type(struct cs_blob *); +#if DEVELOPMENT || DEBUG +void csproc_clear_platform_binary(struct proc *); +#endif int csblob_get_entitlements(struct cs_blob *, void **, size_t *); @@ -252,6 +130,8 @@ int csfg_get_path(struct fileglob *, char *, int *); int csfg_get_platform_binary(struct fileglob *); uint8_t * csfg_get_cdhash(struct fileglob *, uint64_t, size_t *); int csfg_get_prod_signed(struct fileglob *); +unsigned int csfg_get_signer_type(struct fileglob *); +unsigned int csproc_get_signer_type(struct proc *); extern int cs_debug; diff --git a/bsd/sys/commpage.h b/bsd/sys/commpage.h new file mode 100644 index 000000000..42bfe61b2 --- /dev/null +++ b/bsd/sys/commpage.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _COMMPAGE_H +#define _COMMPAGE_H + +#ifdef PRIVATE +typedef volatile struct commpage_timeofday_data{ + uint64_t TimeStamp_tick; + uint64_t TimeStamp_sec; + uint64_t TimeStamp_frac; + uint64_t Ticks_scale; + uint64_t Ticks_per_sec; +} new_commpage_timeofday_data_t; +#endif + +#endif diff --git a/bsd/sys/conf.h b/bsd/sys/conf.h index 3b3f0e234..273b690ad 100644 --- a/bsd/sys/conf.h +++ b/bsd/sys/conf.h @@ -200,8 +200,10 @@ struct cdevsw { #ifdef BSD_KERNEL_PRIVATE extern uint64_t cdevsw_flags[]; -#define CDEVSW_SELECT_KQUEUE 0x01 -#define CDEVSW_USE_OFFSET 0x02 +#define CDEVSW_SELECT_KQUEUE 0x01 +#define CDEVSW_USE_OFFSET 0x02 +#define CDEVSW_IS_PTC 0x04 +#define CDEVSW_IS_PTS 0x08 struct thread; diff --git a/bsd/sys/csr.h b/bsd/sys/csr.h index 602b6061b..4c7f51ece 100644 --- a/bsd/sys/csr.h +++ b/bsd/sys/csr.h @@ -49,6 +49,7 @@ typedef uint32_t csr_op_t; #define CSR_ALLOW_UNRESTRICTED_NVRAM (1 << 6) #define CSR_ALLOW_DEVICE_CONFIGURATION (1 << 7) #define CSR_ALLOW_ANY_RECOVERY_OS (1 << 8) +#define CSR_ALLOW_UNAPPROVED_KEXTS (1 << 9) #define CSR_VALID_FLAGS (CSR_ALLOW_UNTRUSTED_KEXTS | \ CSR_ALLOW_UNRESTRICTED_FS | \ @@ -58,7 +59,8 @@ typedef uint32_t csr_op_t; CSR_ALLOW_UNRESTRICTED_DTRACE | \ CSR_ALLOW_UNRESTRICTED_NVRAM | \ CSR_ALLOW_DEVICE_CONFIGURATION | \ - CSR_ALLOW_ANY_RECOVERY_OS) + CSR_ALLOW_ANY_RECOVERY_OS | \ + CSR_ALLOW_UNAPPROVED_KEXTS) #define CSR_ALWAYS_ENFORCED_FLAGS (CSR_ALLOW_DEVICE_CONFIGURATION | CSR_ALLOW_ANY_RECOVERY_OS) diff --git a/bsd/sys/decmpfs.h b/bsd/sys/decmpfs.h index 3638f9fa0..f30a6decc 100644 --- a/bsd/sys/decmpfs.h +++ b/bsd/sys/decmpfs.h @@ -160,7 +160,7 @@ typedef struct { } decmpfs_registration; /* hooks for kexts to call */ -errno_t register_decmpfs_decompressor(uint32_t compression_type, decmpfs_registration *registration); +errno_t register_decmpfs_decompressor(uint32_t compression_type, const decmpfs_registration *registration); errno_t unregister_decmpfs_decompressor(uint32_t compression_type, decmpfs_registration *registration); #endif /* KERNEL */ diff --git a/bsd/sys/disk.h b/bsd/sys/disk.h index a365b5030..f046e7f76 100644 --- a/bsd/sys/disk.h +++ b/bsd/sys/disk.h @@ -331,10 +331,21 @@ typedef struct _dk_cs_unmap { #define _DKIOCCSMAP _IOWR('d', 202, _dk_cs_map_t) // No longer used: _DKIOCCSSETFSVNODE (203) & _DKIOCCSGETFREEBYTES (204) #define _DKIOCCSUNMAP _IOWR('d', 205, _dk_cs_unmap_t) + +typedef enum { + DK_APFS_ONE_DEVICE = 1, + DK_APFS_FUSION +} dk_apfs_flavour_t; + +#define DKIOCGETAPFSFLAVOUR _IOR('d', 91, dk_apfs_flavour_t) + #endif /* PRIVATE */ #endif /* KERNEL */ #ifdef PRIVATE +#if TARGET_OS_EMBEDDED +#define _DKIOCSETSTATIC _IO('d', 84) +#endif /* TARGET_OS_EMBEDDED */ #endif /* PRIVATE */ #endif /* _SYS_DISK_H_ */ diff --git a/bsd/sys/domain.h b/bsd/sys/domain.h index fa77f963b..6c4a1a0b6 100644 --- a/bsd/sys/domain.h +++ b/bsd/sys/domain.h @@ -124,6 +124,9 @@ struct domain { #pragma pack() #ifdef XNU_KERNEL_PRIVATE + +#include <sys/queue.h> + /* * Internal, private and extendable representation of domain. */ diff --git a/bsd/sys/dtrace.h b/bsd/sys/dtrace.h index b44140030..29540c2a3 100644 --- a/bsd/sys/dtrace.h +++ b/bsd/sys/dtrace.h @@ -342,6 +342,10 @@ typedef enum dtrace_probespec { #define DIF_VAR_DISPATCHQADDR 0x0201 /* Apple specific dispatch queue addr */ #define DIF_VAR_MACHTIMESTAMP 0x0202 /* mach_absolute_timestamp() */ #define DIF_VAR_CPU 0x0203 /* cpu number */ +#define DIF_VAR_CPUINSTRS 0x0204 /* cpu instructions */ +#define DIF_VAR_CPUCYCLES 0x0205 /* cpu cycles */ +#define DIF_VAR_VINSTRS 0x0206 /* virtual instructions */ +#define DIF_VAR_VCYCLES 0x0207 /* virtual cycles */ #endif /* __APPLE __ */ #define DIF_SUBR_RAND 0 @@ -2508,6 +2512,9 @@ extern int (*dtrace_return_probe_ptr)(struct regs *); #if defined (__i386__) || defined(__x86_64__) extern int (*dtrace_pid_probe_ptr)(x86_saved_state_t *regs); extern int (*dtrace_return_probe_ptr)(x86_saved_state_t* regs); +#elif defined (__arm__) || defined(__arm64__) +extern int (*dtrace_pid_probe_ptr)(arm_saved_state_t *regs); +extern int (*dtrace_return_probe_ptr)(arm_saved_state_t *regs); #else #error architecture not supported #endif @@ -2579,6 +2586,13 @@ extern void *dtrace_invop_callsite_pre; extern void *dtrace_invop_callsite_post; #endif +#if defined(__arm__) || defined(__arm64__) +extern int dtrace_instr_size(uint32_t instr, int thumb_mode); +extern void dtrace_invop_add(int (*)(uintptr_t, uintptr_t *, uintptr_t)); +extern void dtrace_invop_remove(int (*)(uintptr_t, uintptr_t *, uintptr_t)); +extern void *dtrace_invop_callsite_pre; +extern void *dtrace_invop_callsite_post; +#endif #undef proc_t #endif /* __APPLE__ */ @@ -2617,6 +2631,13 @@ extern void *dtrace_invop_callsite_post; #endif +#if defined(__arm__) || defined(__arm64__) + +#define DTRACE_INVOP_NOP 4 +#define DTRACE_INVOP_RET 5 +#define DTRACE_INVOP_B 6 + +#endif #endif /* __APPLE__ */ diff --git a/bsd/sys/dtrace_glue.h b/bsd/sys/dtrace_glue.h index c5a1ebf6d..d8a65de0c 100644 --- a/bsd/sys/dtrace_glue.h +++ b/bsd/sys/dtrace_glue.h @@ -33,6 +33,7 @@ #include <libkern/libkern.h> #include <kern/locks.h> +#include <kern/debug.h> #include <kern/thread_call.h> #include <kern/thread.h> #include <machine/machine_routines.h> @@ -231,6 +232,8 @@ typedef struct modctl { #define MODCTL_HAS_UUID 0x40 // Module has UUID #define MODCTL_FBT_PRIVATE_PROBES_PROVIDED 0x80 // fbt private probes have been provided #define MODCTL_FBT_PROVIDE_PRIVATE_PROBES 0x100 // fbt provider must provide private probes +#define MODCTL_FBT_PROVIDE_BLACKLISTED_PROBES 0x200 // fbt provider must provide blacklisted probes +#define MODCTL_FBT_BLACKLISTED_PROBES_PROVIDED 0x400 // fbt blacklisted probes have been provided /* Simple/singular mod_flags accessors */ #define MOD_IS_MACH_KERNEL(mod) (mod->mod_flags & MODCTL_IS_MACH_KERNEL) @@ -243,10 +246,13 @@ typedef struct modctl { #define MOD_HAS_UUID(mod) (mod->mod_flags & MODCTL_HAS_UUID) #define MOD_FBT_PRIVATE_PROBES_PROVIDED(mod) (mod->mod_flags & MODCTL_FBT_PRIVATE_PROBES_PROVIDED) #define MOD_FBT_PROVIDE_PRIVATE_PROBES(mod) (mod->mod_flags & MODCTL_FBT_PROVIDE_PRIVATE_PROBES) +#define MOD_FBT_BLACKLISTED_PROBES_PROVIDED(mod) (mod->mod_flags & MODCTL_FBT_BLACKLISTED_PROBES_PROVIDED) +#define MOD_FBT_PROVIDE_BLACKLISTED_PROBES(mod) (mod->mod_flags & MODCTL_FBT_PROVIDE_BLACKLISTED_PROBES) /* Compound accessors */ #define MOD_FBT_PRIVATE_PROBES_DONE(mod) (MOD_FBT_PRIVATE_PROBES_PROVIDED(mod) || !MOD_FBT_PROVIDE_PRIVATE_PROBES(mod)) -#define MOD_FBT_DONE(mod) ((MOD_FBT_PROBES_PROVIDED(mod) && MOD_FBT_PRIVATE_PROBES_DONE(mod)) || MOD_FBT_INVALID(mod)) +#define MOD_FBT_BLACKLISTED_PROBES_DONE(mod) (MOD_FBT_BLACKLISTED_PROBES_PROVIDED(mod) || !MOD_FBT_PROVIDE_BLACKLISTED_PROBES(mod)) +#define MOD_FBT_DONE(mod) ((MOD_FBT_PROBES_PROVIDED(mod) && MOD_FBT_PRIVATE_PROBES_DONE(mod) && MOD_FBT_BLACKLISTED_PROBES_DONE(mod)) || MOD_FBT_INVALID(mod)) #define MOD_SDT_DONE(mod) (MOD_SDT_PROBES_PROVIDED(mod) || MOD_SDT_INVALID(mod)) #define MOD_SYMBOLS_DONE(mod) (MOD_FBT_DONE(mod) && MOD_SDT_DONE(mod)) @@ -325,12 +331,6 @@ extern void cyclic_remove_omni(cyclic_id_list_t); extern cyclic_id_t cyclic_timer_add(cyc_handler_t *, cyc_time_t *); extern void cyclic_timer_remove(cyclic_id_t); -/* - * timeout / untimeout (converted to dtrace_timeout / dtrace_untimeout due to name collision) - */ - -thread_call_t dtrace_timeout(void (*func)(void *, void *), void* arg, uint64_t nanos); - /* * ddi */ @@ -505,6 +505,16 @@ extern void vmem_free(vmem_t *vmp, void *vaddr, size_t size); * Atomic */ +static inline uint8_t atomic_or_8(uint8_t *addr, uint8_t mask) +{ + return OSBitOrAtomic8(mask, addr); +} + +static inline uint32_t atomic_and_32( uint32_t *addr, int32_t mask) +{ + return OSBitAndAtomic(mask, addr); +} + static inline uint32_t atomic_add_32( uint32_t *theAddress, int32_t theAmount ) { return OSAddAtomic( theAmount, theAddress ); @@ -515,12 +525,22 @@ static inline void atomic_add_64( uint64_t *theAddress, int64_t theAmount ) { (void)OSAddAtomic64( theAmount, (SInt64 *)theAddress ); } -#endif - -static inline uint32_t atomic_and_32(uint32_t *addr, uint32_t mask) +#elif defined(__arm__) +static inline void atomic_add_64( uint64_t *theAddress, int64_t theAmount ) { - return OSBitAndAtomic(mask, addr); + // FIXME + // atomic_add_64() is at present only called from fasttrap.c to increment + // or decrement a 64bit counter. Narrow to 32bits since arm has + // no convenient 64bit atomic op. + + (void)OSAddAtomic( (int32_t)theAmount, &(((SInt32 *)theAddress)[0])); } +#elif defined (__arm64__) +static inline void atomic_add_64( uint64_t *theAddress, int64_t theAmount ) +{ + (void)OSAddAtomic64( theAmount, (SInt64 *)theAddress ); +} +#endif static inline uint32_t atomic_or_32(uint32_t *addr, uint32_t mask) { @@ -534,12 +554,14 @@ static inline uint32_t atomic_or_32(uint32_t *addr, uint32_t mask) typedef uintptr_t pc_t; typedef uintptr_t greg_t; /* For dtrace_impl.h prototype of dtrace_getfp() */ +#if defined(__arm__) || defined(__arm64__) +#define regs arm_saved_state +#endif extern struct regs *find_user_regs( thread_t thread); extern vm_offset_t dtrace_get_cpu_int_stack_top(void); extern vm_offset_t max_valid_stack_address(void); /* kern/thread.h */ -extern volatile int panicwait; /* kern/debug.c */ -#define panic_quiesce (panicwait) +#define panic_quiesce (panic_active()) #define IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0) @@ -548,6 +570,10 @@ extern int vuprintf(const char *, va_list); extern hrtime_t dtrace_abs_to_nano(uint64_t); __private_extern__ const char * strstr(const char *, const char *); +const void* bsearch(const void*, const void*, size_t, size_t, int (*compar)(const void *, const void *)); + +int dtrace_buffer_copyout(const void*, user_addr_t, vm_size_t); + #define DTRACE_NCLIENTS 32 diff --git a/bsd/sys/dtrace_impl.h b/bsd/sys/dtrace_impl.h index 9229998a3..125293dbf 100644 --- a/bsd/sys/dtrace_impl.h +++ b/bsd/sys/dtrace_impl.h @@ -1343,7 +1343,7 @@ typedef struct dtrace_toxrange { uintptr_t dtt_limit; /* limit of toxic range */ } dtrace_toxrange_t; -extern uint64_t dtrace_getarg(int, int); +extern uint64_t dtrace_getarg(int, int, dtrace_mstate_t*, dtrace_vstate_t*); extern int dtrace_getipl(void); extern uintptr_t dtrace_caller(int); extern uint32_t dtrace_cas32(uint32_t *, uint32_t, uint32_t); @@ -1353,6 +1353,9 @@ extern void dtrace_copyinstr(user_addr_t, uintptr_t, size_t, volatile uint16_t * extern void dtrace_copyout(uintptr_t, user_addr_t, size_t, volatile uint16_t *); extern void dtrace_copyoutstr(uintptr_t, user_addr_t, size_t, volatile uint16_t *); extern void dtrace_getpcstack(pc_t *, int, int, uint32_t *); +extern uint64_t dtrace_load64(uintptr_t); +extern int dtrace_canload(uint64_t, size_t, dtrace_mstate_t*, dtrace_vstate_t*); + extern uint64_t dtrace_getreg(struct regs *, uint_t); extern int dtrace_getstackdepth(int); extern void dtrace_getupcstack(uint64_t *, int); diff --git a/bsd/sys/dtrace_ptss.h b/bsd/sys/dtrace_ptss.h index e7f1825dd..e7d8d9b0c 100644 --- a/bsd/sys/dtrace_ptss.h +++ b/bsd/sys/dtrace_ptss.h @@ -79,6 +79,9 @@ extern "C" { struct dtrace_ptss_page_entry { struct dtrace_ptss_page_entry* next; user_addr_t addr; +#if CONFIG_EMBEDDED + user_addr_t write_addr; +#endif }; struct dtrace_ptss_page { diff --git a/bsd/sys/event.h b/bsd/sys/event.h index 14b23450b..f10e15777 100644 --- a/bsd/sys/event.h +++ b/bsd/sys/event.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003-2016 Apple Inc. All rights reserved. + * Copyright (c) 2003-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -60,6 +60,9 @@ #include <sys/cdefs.h> #include <stdint.h> +/* + * Filter types + */ #define EVFILT_READ (-1) #define EVFILT_WRITE (-2) #define EVFILT_AIO (-3) /* attached to aio requests */ @@ -78,8 +81,11 @@ #define EVFILT_MEMORYSTATUS (-14) /* Memorystatus events */ #endif /* PRIVATE */ #define EVFILT_EXCEPT (-15) /* Exception events */ +#ifdef PRIVATE +#define EVFILT_WORKLOOP (-17) /* Workloop events */ +#endif /* PRIVATE */ -#define EVFILT_SYSCOUNT 15 +#define EVFILT_SYSCOUNT 17 #define EVFILT_THREADMARKER EVFILT_SYSCOUNT /* Internal use only */ #pragma pack(4) @@ -125,7 +131,7 @@ struct kevent_internal_s { uint64_t ext[4]; /* filter-specific extensions */ }; -#endif +#endif /* KERNEL_PRIVATE */ #pragma pack() @@ -151,6 +157,12 @@ struct kevent_qos_s { int64_t data; /* filter-specific data */ uint64_t ext[4]; /* filter-specific extensions */ }; + +/* + * Type definition for names/ids of dynamically allocated kqueues. + */ +typedef uint64_t kqueue_id_t; + #endif /* PRIVATE */ #define EV_SET(kevp, a, b, c, d, e, f) do { \ @@ -177,9 +189,9 @@ struct kevent_qos_s { /* kevent system call flags */ -#define KEVENT_FLAG_NONE 0x000 /* no flag value */ -#define KEVENT_FLAG_IMMEDIATE 0x001 /* immediate timeout */ -#define KEVENT_FLAG_ERROR_EVENTS 0x002 /* output events only include change errors */ +#define KEVENT_FLAG_NONE 0x000 /* no flag value */ +#define KEVENT_FLAG_IMMEDIATE 0x001 /* immediate timeout */ +#define KEVENT_FLAG_ERROR_EVENTS 0x002 /* output events only include change errors */ #ifdef PRIVATE @@ -189,42 +201,58 @@ struct kevent_qos_s { * instead. */ -#define KEVENT_FLAG_STACK_EVENTS 0x004 /* output events treated as stack (grows down) */ -#define KEVENT_FLAG_STACK_DATA 0x008 /* output data allocated as stack (grows down) */ -#define KEVENT_FLAG_WORKQ 0x020 /* interact with the default workq kq */ -#define KEVENT_FLAG_WORKQ_MANAGER 0x200 /* current thread is the workq manager */ +#define KEVENT_FLAG_STACK_EVENTS 0x004 /* output events treated as stack (grows down) */ +#define KEVENT_FLAG_STACK_DATA 0x008 /* output data allocated as stack (grows down) */ +#define KEVENT_FLAG_UNBIND_CHECK_FLAGS 0x010 /* check the flags passed to kevent_qos_internal_unbind */ +#define KEVENT_FLAG_WORKQ 0x020 /* interact with the default workq kq */ +#define KEVENT_FLAG_WORKQ_MANAGER 0x200 /* current thread is the workq manager */ +#define KEVENT_FLAG_WORKLOOP 0x400 /* interact with the specified workloop kq */ +#define KEVENT_FLAG_SYNCHRONOUS_BIND 0x800 /* synchronous bind callback */ + +#define KEVENT_FLAG_WORKLOOP_SERVICER_ATTACH 0x8000 /* attach current thread to workloop */ +#define KEVENT_FLAG_WORKLOOP_SERVICER_DETACH 0x10000 /* unbind current thread from workloop */ +#define KEVENT_FLAG_DYNAMIC_KQ_MUST_EXIST 0x20000 /* kq lookup by id must exist */ +#define KEVENT_FLAG_DYNAMIC_KQ_MUST_NOT_EXIST 0x40000 /* kq lookup by id must not exist */ +#define KEVENT_FLAG_WORKLOOP_NO_WQ_THREAD 0x80000 /* do not create workqueue threads for this worloop */ #ifdef XNU_KERNEL_PRIVATE -#define KEVENT_FLAG_LEGACY32 0x040 /* event data in legacy 32-bit format */ -#define KEVENT_FLAG_LEGACY64 0x080 /* event data in legacy 64-bit format */ -#define KEVENT_FLAG_KERNEL 0x100 /* caller is in-kernel */ -#define KEVENT_FLAG_USER (KEVENT_FLAG_IMMEDIATE | KEVENT_FLAG_ERROR_EVENTS | \ - KEVENT_FLAG_STACK_EVENTS | KEVENT_FLAG_STACK_DATA | \ - KEVENT_FLAG_WORKQ) - -/* - * Since some filter ops are not part of the standard sysfilt_ops, we - * use kn_filtid starting from EVFILT_SYSCOUNT to identify these cases. - * This is to let kn_fops() get the correct fops for all cases. +#define KEVENT_FLAG_LEGACY32 0x040 /* event data in legacy 32-bit format */ +#define KEVENT_FLAG_LEGACY64 0x080 /* event data in legacy 64-bit format */ +#define KEVENT_FLAG_KERNEL 0x1000 /* caller is in-kernel */ +#define KEVENT_FLAG_DYNAMIC_KQUEUE 0x2000 /* kqueue is dynamically allocated */ +#define KEVENT_FLAG_WORKLOOP_CANCELED 0x4000 /* workloop bind was cancelled */ + +#define KEVENT_FLAG_USER (KEVENT_FLAG_IMMEDIATE | KEVENT_FLAG_ERROR_EVENTS | \ + KEVENT_FLAG_STACK_EVENTS | KEVENT_FLAG_STACK_DATA | \ + KEVENT_FLAG_WORKQ | KEVENT_FLAG_WORKLOOP | \ + KEVENT_FLAG_WORKLOOP_SERVICER_ATTACH | KEVENT_FLAG_WORKLOOP_SERVICER_DETACH | \ + KEVENT_FLAG_DYNAMIC_KQ_MUST_EXIST | KEVENT_FLAG_DYNAMIC_KQ_MUST_NOT_EXIST | \ + KEVENT_FLAG_WORKLOOP_NO_WQ_THREAD) + +/* + * Since some filter ops are not part of the standard sysfilt_ops, we use + * kn_filtid starting from EVFILT_SYSCOUNT to identify these cases. This is to + * let kn_fops() get the correct fops for all cases. */ -#define EVFILTID_KQREAD (EVFILT_SYSCOUNT) -#define EVFILTID_PIPE_R (EVFILT_SYSCOUNT + 1) -#define EVFILTID_PIPE_W (EVFILT_SYSCOUNT + 2) -#define EVFILTID_PTSD (EVFILT_SYSCOUNT + 3) -#define EVFILTID_SOREAD (EVFILT_SYSCOUNT + 4) -#define EVFILTID_SOWRITE (EVFILT_SYSCOUNT + 5) -#define EVFILTID_SCK (EVFILT_SYSCOUNT + 6) -#define EVFILTID_SOEXCEPT (EVFILT_SYSCOUNT + 7) -#define EVFILTID_SPEC (EVFILT_SYSCOUNT + 8) -#define EVFILTID_BPFREAD (EVFILT_SYSCOUNT + 9) -#define EVFILTID_NECP_FD (EVFILT_SYSCOUNT + 10) -#define EVFILTID_FSEVENT (EVFILT_SYSCOUNT + 13) -#define EVFILTID_VN (EVFILT_SYSCOUNT + 14) - -#define EVFILTID_MAX (EVFILT_SYSCOUNT + 15) - -#endif /* XNU_KERNEL_PRIVATE */ +#define EVFILTID_KQREAD (EVFILT_SYSCOUNT) +#define EVFILTID_PIPE_R (EVFILT_SYSCOUNT + 1) +#define EVFILTID_PIPE_W (EVFILT_SYSCOUNT + 2) +#define EVFILTID_PTSD (EVFILT_SYSCOUNT + 3) +#define EVFILTID_SOREAD (EVFILT_SYSCOUNT + 4) +#define EVFILTID_SOWRITE (EVFILT_SYSCOUNT + 5) +#define EVFILTID_SCK (EVFILT_SYSCOUNT + 6) +#define EVFILTID_SOEXCEPT (EVFILT_SYSCOUNT + 7) +#define EVFILTID_SPEC (EVFILT_SYSCOUNT + 8) +#define EVFILTID_BPFREAD (EVFILT_SYSCOUNT + 9) +#define EVFILTID_NECP_FD (EVFILT_SYSCOUNT + 10) +#define EVFILTID_FSEVENT (EVFILT_SYSCOUNT + 13) +#define EVFILTID_VN (EVFILT_SYSCOUNT + 14) +#define EVFILTID_TTY (EVFILT_SYSCOUNT + 16) + +#define EVFILTID_MAX (EVFILT_SYSCOUNT + 17) + +#endif /* defined(XNU_KERNEL_PRIVATE) */ #define EV_SET_QOS 0 @@ -315,6 +343,82 @@ struct kevent_qos_s { #define NOTE_FFCTRLMASK 0xc0000000 /* mask for operations */ #define NOTE_FFLAGSMASK 0x00ffffff +#ifdef PRIVATE +/* + * data/hint fflags for EVFILT_WORKLOOP, shared with userspace + * + * The ident for thread requests should be the dynamic ID of the workloop + * The ident for each sync waiter must be unique to that waiter [for this workloop] + * + * + * Commands: + * + * @const NOTE_WL_THREAD_REQUEST [in/out] + * The kevent represents asynchronous userspace work and its associated QoS. + * There can only be a single knote with this flag set per workloop. + * + * @const NOTE_WL_SYNC_WAIT [in/out] + * This bit is set when the caller is waiting to become the owner of a workloop. + * If the NOTE_WL_SYNC_WAKE bit is already set then the caller is not blocked, + * else it blocks until it is set. + * + * The QoS field of the knote is used to push on other owners or servicers. + * + * @const NOTE_WL_SYNC_WAKE [in/out] + * Marks the waiter knote as being eligible to become an owner + * This bit can only be set once, trying it again will fail with EALREADY. + * + * + * Flags/Modifiers: + * + * @const NOTE_WL_UPDATE_QOS [in] (only NOTE_WL_THREAD_REQUEST) + * For successful updates (EV_ADD only), learn the new userspace async QoS from + * the kevent qos field. + * + * @const NOTE_WL_END_OWNERSHIP [in] + * If the update is successful (including deletions) or returns ESTALE, and + * the caller thread or the "suspended" thread is currently owning the workloop, + * then ownership is forgotten. + * + * @const NOTE_WL_DISCOVER_OWNER [in] + * If the update is successful (including deletions), learn the owner identity + * from the loaded value during debounce. This requires an address to have been + * filled in the EV_EXTIDX_WL_ADDR ext field, but doesn't require a mask to have + * been set in the EV_EXTIDX_WL_MASK. + * + * @const NOTE_WL_IGNORE_ESTALE [in] + * If the operation would fail with ESTALE, mask the error and pretend the + * update was successful. However the operation itself didn't happen, meaning + * that: + * - attaching a new knote will not happen + * - dropping an existing knote will not happen + * - NOTE_WL_UPDATE_QOS or NOTE_WL_DISCOVER_OWNER will have no effect + * + * This modifier doesn't affect NOTE_WL_END_OWNERSHIP. + */ +#define NOTE_WL_THREAD_REQUEST 0x00000001 +#define NOTE_WL_SYNC_WAIT 0x00000004 +#define NOTE_WL_SYNC_WAKE 0x00000008 +#define NOTE_WL_COMMANDS_MASK 0x0000000f /* Mask of all the [in] commands above */ + +#define NOTE_WL_UPDATE_QOS 0x00000010 +#define NOTE_WL_END_OWNERSHIP 0x00000020 +#define NOTE_WL_UPDATE_OWNER 0 /* ... compatibility define ... */ +#define NOTE_WL_DISCOVER_OWNER 0x00000080 +#define NOTE_WL_IGNORE_ESTALE 0x00000100 +#define NOTE_WL_UPDATES_MASK 0x000001f0 /* Mask of all the [in] updates above */ + +/* + * EVFILT_WORKLOOP ext[] array indexes/meanings. + */ +#define EV_EXTIDX_WL_LANE 0 /* lane identifier [in: sync waiter] + [out: thread request] */ +#define EV_EXTIDX_WL_ADDR 1 /* debounce address [in: NULL==no debounce] */ +#define EV_EXTIDX_WL_MASK 2 /* debounce mask [in] */ +#define EV_EXTIDX_WL_VALUE 3 /* debounce value [in: not current->ESTALE] + [out: new/debounce value] */ +#endif /* PRIVATE */ + /* * data/hint fflags for EVFILT_{READ|WRITE}, shared with userspace * @@ -415,6 +519,7 @@ enum { #define NOTE_MEMORYSTATUS_LOW_SWAP 0x00000008 /* system is in a low-swap state */ #define NOTE_MEMORYSTATUS_PROC_LIMIT_WARN 0x00000010 /* process memory limit has hit a warning state */ #define NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL 0x00000020 /* process memory limit has hit a critical state - soft limit */ +#define NOTE_MEMORYSTATUS_MSL_STATUS 0xf0000000 /* bits used to request change to process MSL status */ #ifdef KERNEL_PRIVATE /* @@ -429,7 +534,8 @@ enum { * Use this mask to protect the kernel private flags. */ #define EVFILT_MEMORYSTATUS_ALL_MASK \ - (NOTE_MEMORYSTATUS_PRESSURE_NORMAL | NOTE_MEMORYSTATUS_PRESSURE_WARN | NOTE_MEMORYSTATUS_PRESSURE_CRITICAL | NOTE_MEMORYSTATUS_LOW_SWAP | NOTE_MEMORYSTATUS_PROC_LIMIT_WARN | NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) + (NOTE_MEMORYSTATUS_PRESSURE_NORMAL | NOTE_MEMORYSTATUS_PRESSURE_WARN | NOTE_MEMORYSTATUS_PRESSURE_CRITICAL | NOTE_MEMORYSTATUS_LOW_SWAP | \ + NOTE_MEMORYSTATUS_PROC_LIMIT_WARN | NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL | NOTE_MEMORYSTATUS_MSL_STATUS) #endif /* KERNEL_PRIVATE */ @@ -453,11 +559,21 @@ typedef enum vm_pressure_level { #define NOTE_USECONDS 0x00000002 /* data is microseconds */ #define NOTE_NSECONDS 0x00000004 /* data is nanoseconds */ #define NOTE_ABSOLUTE 0x00000008 /* absolute timeout */ - /* ... implicit EV_ONESHOT */ + /* ... implicit EV_ONESHOT, timeout uses the gettimeofday epoch */ #define NOTE_LEEWAY 0x00000010 /* ext[1] holds leeway for power aware timers */ #define NOTE_CRITICAL 0x00000020 /* system does minimal timer coalescing */ #define NOTE_BACKGROUND 0x00000040 /* system does maximum timer coalescing */ -#define NOTE_MACH_CONTINUOUS_TIME 0x00000080 /* use continuous time base */ +#define NOTE_MACH_CONTINUOUS_TIME 0x00000080 + /* + * NOTE_MACH_CONTINUOUS_TIME: + * with NOTE_ABSOLUTE: causes the timer to continue to tick across sleep, + * still uses gettimeofday epoch + * with NOTE_MACHTIME and NOTE_ABSOLUTE: uses mach continuous time epoch + * without NOTE_ABSOLUTE (interval timer mode): continues to tick across sleep + */ +#define NOTE_MACHTIME 0x00000100 /* data is mach absolute time units */ + /* timeout uses the mach absolute time epoch */ + #ifdef PRIVATE /* * data/hint fflags for EVFILT_SOCK, shared with userspace. @@ -530,6 +646,8 @@ typedef enum vm_pressure_level { #define NOTE_CHILD 0x00000004 /* am a child process */ +#ifdef PRIVATE +#endif /* PRIVATE */ #ifndef KERNEL /* Temporay solution for BootX to use inode.h till kqueue moves to vfs layer */ @@ -543,6 +661,8 @@ SLIST_HEAD(klist, knote); #ifdef XNU_KERNEL_PRIVATE #include <sys/queue.h> #include <kern/kern_types.h> +#include <sys/fcntl.h> /* FREAD, FWRITE */ +#include <kern/debug.h> /* panic */ #ifdef MALLOC_DECLARE MALLOC_DECLARE(M_KQUEUE); @@ -586,12 +706,16 @@ struct knote { struct proc *p_proc; /* proc pointer */ struct ipc_mqueue *p_mqueue; /* pset pointer */ } kn_ptr; - uint64_t kn_req_index:4, /* requested qos index */ - kn_qos_index:4, /* in-use qos index */ - kn_qos_override:4, /* qos override index */ - kn_reserved:4, /* reserved bits */ - kn_filtid:8, /* filter id to index filter ops */ - kn_kq_packed:KNOTE_KQ_BITSIZE; /* packed pointer for kq */ + uint64_t kn_req_index:3, /* requested qos index */ + kn_qos_index:3, /* in-use qos index */ + kn_qos_override:3, /* qos override index */ + kn_qos_sync_override:3, /* qos sync override index */ + kn_vnode_kqok:1, + kn_vnode_use_ofst:1, + kn_qos_override_is_sync:1, /* qos override index is a sync override */ + kn_reserved:1, /* reserved bits */ + kn_filtid:8, /* filter id to index filter ops */ + kn_kq_packed:KNOTE_KQ_BITSIZE; /* packed pointer for kq */ union { void *kn_hook; @@ -634,6 +758,26 @@ static inline void knote_set_kq(struct knote *kn, void *kq) } } +static inline int knote_get_seltype(struct knote *kn) +{ + switch (kn->kn_filter) { + case EVFILT_READ: + return FREAD; + case EVFILT_WRITE: + return FWRITE; + default: + panic("%s(%p): invalid filter %d\n", + __func__, kn, kn->kn_filter); + return 0; + } +} + +static inline void knote_set_error(struct knote *kn, int error) +{ + kn->kn_flags |= EV_ERROR; + kn->kn_data = error; +} + struct filt_process_s { int fp_fd; unsigned int fp_flags; @@ -660,8 +804,20 @@ typedef struct filt_process_s *filt_process_data_t; * * Otherwise the knote is hashed by the ident and has no auto-close behavior. * + * f_adjusts_qos - + * identifies if the filter can adjust its QoS during its lifetime. + * + * Currently, EVFILT_MAACHPORT is the only filter using this facility. + * + * f_needs_boost - + * [OPTIONAL] used by filters to communicate they need to hold a boost + * while holding a usecount on this knote. This is called with the kqlock + * held. + * + * This is only used by EVFILT_WORKLOOP currently. + * * f_attach - - * called to attach the knote to the underlying object that will be delivering events + * called to attach the knote to the underlying object that will be delivering events * through it when EV_ADD is supplied and no existing matching event is found * * provided a knote that is pre-attached to the fd or hashed (see above) but is @@ -679,9 +835,21 @@ typedef struct filt_process_s *filt_process_data_t; * The return value indicates if the knote should already be considered "activated" at * the time of attach (one or more of the interest events has already occured). * + * f_post_attach - + * [OPTIONAL] called after a successful attach, with the kqueue lock held, + * returns lock held, may drop and re-acquire + * + * If this function is non-null, then it indicates that the filter wants + * to perform an action after a successful ATTACH of a knote. + * + * Currently, EVFILT_WORKLOOP is the only filter using this facility. + * + * The return value indicates an error to report to userland. + * + * * f_detach - * called to disassociate the knote from the underlying object delivering events - * the filter should not attempt to deliver events through this knote after this + * the filter should not attempt to deliver events through this knote after this * operation returns control to the kq system. * * f_event - @@ -695,6 +863,22 @@ typedef struct filt_process_s *filt_process_data_t; * The return value indicates if the knote should already be considered "activated" at * the time of attach (one or more of the interest events has already occured). * + * f_drop_and_unlock - + * [OPTIONAL] called with the kqueue locked, and has to unlock + * + * If this function is non-null, then it indicates that the filter + * wants to handle EV_DELETE events. This is necessary if a particular + * filter needs to synchronize knote deletion with its own filter lock. + * Currently, EVFILT_WORKLOOP is the only filter using this facility. + * + * The return value indicates an error during the knote drop, i.e., the + * knote still exists and user space should re-drive the EV_DELETE. + * + * If the return value is ERESTART, kevent_register() is called from + * scratch again (useful to wait for usecounts to drop and then + * reevaluate the relevance of that drop) + * + * * f_process - * called when attempting to deliver triggered events to user-space. * @@ -733,17 +917,35 @@ typedef struct filt_process_s *filt_process_data_t; * explicit indication of just delivering a current event vs delivering * an event with more events still pending. * + * f_touch - + * called to update the knote with new state from the user during EVFILT_ADD/ENABLE/DISABLE + * on an already-attached knote. + * + * f_touch should copy relevant new data from the kevent into the knote. + * (if KN_UDATA_SPECIFIC is not set, you may need to update the udata too) + * + * operator must lock against concurrent f_event and f_process operations. + * + * A return value of 1 indicates that the knote should now be considered 'activated'. + * + * f_touch can set EV_ERROR with specific error in the data field to return an error to the client. + * You should return 1 to indicate that the kevent needs to be activated and processed. + * * f_peek - * For knotes marked KN_STAYACTIVE, indicate if the knote is truly active at * the moment (not used for event delivery, but for status checks). */ struct filterops { - int f_isfd; /* true if ident == filedescriptor */ - int (*f_attach)(struct knote *kn); - void (*f_detach)(struct knote *kn); - int (*f_event)(struct knote *kn, long hint); + bool f_isfd; /* true if ident == filedescriptor */ + bool f_adjusts_qos; /* true if the filter can override the knote */ + bool (*f_needs_boost)(struct kevent_internal_s *kev); + int (*f_attach)(struct knote *kn, struct kevent_internal_s *kev); + int (*f_post_attach)(struct knote *kn, struct kevent_internal_s *kev); + void (*f_detach)(struct knote *kn); + int (*f_event)(struct knote *kn, long hint); int (*f_touch)(struct knote *kn, struct kevent_internal_s *kev); + int (*f_drop_and_unlock)(struct knote *kn, struct kevent_internal_s *kev); int (*f_process)(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); unsigned (*f_peek)(struct knote *kn); }; @@ -759,7 +961,6 @@ extern void klist_init(struct klist *list); #define KNOTE_ATTACH(list, kn) knote_attach(list, kn) #define KNOTE_DETACH(list, kn) knote_detach(list, kn) - extern void knote(struct klist *list, long hint); extern int knote_attach(struct klist *list, struct knote *kn); extern int knote_detach(struct klist *list, struct knote *kn); @@ -769,8 +970,19 @@ extern int knote_unlink_waitq(struct knote *kn, struct waitq *wq); extern void knote_fdclose(struct proc *p, int fd, int force); extern void knote_markstayactive(struct knote *kn); extern void knote_clearstayactive(struct knote *kn); -extern void knote_adjust_qos(struct knote *kn, int qos, int override); -extern struct filterops *knote_fops(struct knote *kn); +extern void knote_adjust_qos(struct knote *kn, int qos, int override, kq_index_t sync_override_index); +extern void knote_adjust_sync_qos(struct knote *kn, kq_index_t sync_qos, boolean_t lock_kq); +extern const struct filterops *knote_fops(struct knote *kn); +extern void knote_set_error(struct knote *kn, int error); + +int kevent_exit_on_workloop_ownership_leak(thread_t thread); +int kevent_proc_copy_uptrs(void *proc, uint64_t *buf, int bufsize); +int kevent_copyout_proc_dynkqids(void *proc, user_addr_t ubuf, + uint32_t ubufsize, int32_t *nkqueues_out); +int kevent_copyout_dynkqinfo(void *proc, kqueue_id_t kq_id, user_addr_t ubuf, + uint32_t ubufsize, int32_t *size_out); +int kevent_copyout_dynkqextinfo(void *proc, kqueue_id_t kq_id, user_addr_t ubuf, + uint32_t ubufsize, int32_t *nknotes_out); #elif defined(KERNEL_PRIVATE) /* !XNU_KERNEL_PRIVATE: kexts still need a klist structure definition */ @@ -792,9 +1004,15 @@ extern int kevent_qos_internal(struct proc *p, int fd, unsigned int flags, int32_t *retval); extern int kevent_qos_internal_bind(struct proc *p, - int fd, thread_t thread, unsigned int flags); + int qos, thread_t thread, unsigned int flags); extern int kevent_qos_internal_unbind(struct proc *p, - int fd, thread_t thread, unsigned int flags); + int qos, thread_t thread, unsigned int flags); + +extern int kevent_id_internal(struct proc *p, kqueue_id_t *id, + user_addr_t changelist, int nchanges, + user_addr_t eventlist, int nevents, + user_addr_t data_out, user_size_t *data_available, + unsigned int flags, int32_t *retval); #endif /* PRIVATE */ #endif /* KERNEL_PRIVATE */ @@ -823,6 +1041,12 @@ int kevent_qos(int kq, struct kevent_qos_s *eventlist, int nevents, void *data_out, size_t *data_available, unsigned int flags); + +int kevent_id(kqueue_id_t id, + const struct kevent_qos_s *changelist, int nchanges, + struct kevent_qos_s *eventlist, int nevents, + void *data_out, size_t *data_available, + unsigned int flags); #endif /* PRIVATE */ __END_DECLS @@ -830,5 +1054,13 @@ __END_DECLS #endif /* KERNEL */ +#ifdef PRIVATE + +/* Flags for pending events notified by kernel via return-to-kernel ast */ +#define R2K_WORKLOOP_PENDING_EVENTS 0x1 +#define R2K_WORKQ_PENDING_EVENTS 0x2 + +#endif /* PRIVATE */ + #endif /* !_SYS_EVENT_H_ */ diff --git a/bsd/sys/eventhandler.h b/bsd/sys/eventhandler.h new file mode 100644 index 000000000..794aa9acd --- /dev/null +++ b/bsd/sys/eventhandler.h @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2016-2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/*- + * Copyright (c) 1999 Michael Smith <msmith@freebsd.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SYS_EVENTHANDLER_H_ +#define _SYS_EVENTHANDLER_H_ + +#include <kern/locks.h> +#include <sys/queue.h> +#include <sys/cdefs.h> +#include <sys/syslog.h> +#include <uuid/uuid.h> + +extern int evh_debug; +extern lck_grp_t *el_lock_grp; +extern lck_attr_t *el_lock_attr; +extern struct eventhandler_entry_arg eventhandler_entry_dummy_arg; + +struct eventhandler_lists_ctxt { + TAILQ_HEAD(, eventhandler_list) eventhandler_lists; + int eventhandler_lists_initted; + decl_lck_mtx_data(, eventhandler_mutex); +}; + +struct eventhandler_entry_arg { + union { + /* Generic cookie object reference */ + void *ee_voidptr; + /* Skywalk ids */ + struct { + pid_t ee_fe_pid; + uuid_t ee_fe_uuid; + uuid_t ee_nx_uuid; + } sk_ids; + /* Generic UUID */ + uuid_t ee_uuid; + } ee_arg; +}; + +#define ee_voidptr ee_arg.ee_voidptr +#define ee_fe_pid ee_arg.sk_ids.ee_fe_pid +#define ee_fe_uuid ee_arg.sk_ids.ee_fe_uuid +#define ee_nx_uuid ee_arg.sk_ids.ee_nx_uuid +#define ee_uuid ee_arg.ee_uuid + +struct eventhandler_entry { + TAILQ_ENTRY(eventhandler_entry) ee_link; + int ee_priority; +#define EHE_DEAD_PRIORITY (-1) + struct eventhandler_entry_arg ee_arg; +}; + +struct eventhandler_list { + char *el_name; + int el_flags; +#define EHL_INITTED (1<<0) + u_int el_runcount; + decl_lck_mtx_data(, el_lock); + TAILQ_ENTRY(eventhandler_list) el_link; + TAILQ_HEAD(,eventhandler_entry) el_entries; +}; + +typedef struct eventhandler_entry *eventhandler_tag; + +#define EHL_LOCK_INIT(p) lck_mtx_init(&(p)->el_lock, el_lock_grp, el_lock_attr) +#define EHL_LOCK(p) lck_mtx_lock(&(p)->el_lock) +#define EHL_UNLOCK(p) lck_mtx_unlock(&(p)->el_lock) +#define EHL_LOCK_ASSERT(p, x) LCK_MTX_ASSERT(&(p)->el_lock, x) +#define EHL_LOCK_DESTROY(p) lck_mtx_destroy(&(p)->el_lock, el_lock_grp) + +#define evhlog(x) do { if (evh_debug >= 1) log x; } while (0) + +/* + * Macro to invoke the handlers for a given event. + */ +#define _EVENTHANDLER_INVOKE(name, list, ...) do { \ + struct eventhandler_entry *_ep; \ + struct eventhandler_entry_ ## name *_t; \ + \ + VERIFY((list)->el_flags & EHL_INITTED); \ + EHL_LOCK_ASSERT((list), LCK_MTX_ASSERT_OWNED); \ + (list)->el_runcount++; \ + VERIFY((list)->el_runcount > 0); \ + evhlog((LOG_DEBUG, "eventhandler_invoke(\"" __STRING(name) "\")")); \ + TAILQ_FOREACH(_ep, &((list)->el_entries), ee_link) { \ + if (_ep->ee_priority != EHE_DEAD_PRIORITY) { \ + EHL_UNLOCK((list)); \ + _t = (struct eventhandler_entry_ ## name *)_ep; \ + evhlog((LOG_DEBUG, "eventhandler_invoke: executing %p", \ + VM_KERNEL_UNSLIDE((void *)_t->eh_func))); \ + _t->eh_func(_ep->ee_arg , ## __VA_ARGS__); \ + EHL_LOCK((list)); \ + } \ + } \ + KASSERT((list)->el_runcount > 0, \ + ("eventhandler_invoke: runcount underflow")); \ + (list)->el_runcount--; \ + if ((list)->el_runcount == 0) \ + eventhandler_prune_list(list); \ + EHL_UNLOCK((list)); \ +} while (0) + +/* + * Slow handlers are entirely dynamic; lists are created + * when entries are added to them, and thus have no concept of "owner", + * + * Slow handlers need to be declared, but do not need to be defined. The + * declaration must be in scope wherever the handler is to be invoked. + */ +#define EVENTHANDLER_DECLARE(name, type) \ +struct eventhandler_entry_ ## name \ +{ \ + struct eventhandler_entry ee; \ + type eh_func; \ +}; \ +struct __hack + +/* + * XXX EVENTHANDLER_DEFINE by itself doesn't do much on XNU + * All it does is that it declares the static eventhandler_tag + * and defines an init routine that still needs to called to put the + * event and callback on the list. + */ +#define EVENTHANDLER_DEFINE(evthdlr_ref, name, func, arg, priority) \ + static eventhandler_tag name ## _tag; \ + static void name ## _evh_init(void *ctx) \ + { \ + name ## _tag = EVENTHANDLER_REGISTER(evthdlr_ref, name, func, ctx, \ + priority); \ + } \ + SYSINIT(name ## _evh_init, SI_SUB_CONFIGURE, SI_ORDER_ANY, \ + name ## _evh_init, arg); \ + struct __hack + +#define EVENTHANDLER_INVOKE(evthdlr_ref, name, ...) \ +do { \ + struct eventhandler_list *_el; \ + \ + if ((_el = eventhandler_find_list(evthdlr_ref, #name)) != NULL) \ + _EVENTHANDLER_INVOKE(name, _el , ## __VA_ARGS__); \ +} while (0) + +#define EVENTHANDLER_REGISTER(evthdlr_ref, name, func, arg, priority) \ + eventhandler_register(evthdlr_ref, NULL, #name, func, arg, priority) + +#define EVENTHANDLER_DEREGISTER(evthdlr_ref, name, tag) \ +do { \ + struct eventhandler_list *_el; \ + \ + if ((_el = eventhandler_find_list(evthdlr_ref, #name)) != NULL) \ + eventhandler_deregister(_el, tag); \ +} while(0) + +void eventhandler_init(void); +eventhandler_tag eventhandler_register(struct eventhandler_lists_ctxt *evthdlr_lists_ctxt, + struct eventhandler_list *list, const char *name, void *func, struct eventhandler_entry_arg arg, int priority); +void eventhandler_deregister(struct eventhandler_list *list, + eventhandler_tag tag); +struct eventhandler_list *eventhandler_find_list( + struct eventhandler_lists_ctxt *evthdlr_lists_ctxt, const char *name); +void eventhandler_prune_list(struct eventhandler_list *list); +void eventhandler_lists_ctxt_init(struct eventhandler_lists_ctxt *evthdlr_lists_ctxt); +void eventhandler_lists_ctxt_destroy(struct eventhandler_lists_ctxt *evthdlr_lists_ctxt); + +/* Generic priority levels */ +#define EVENTHANDLER_PRI_FIRST 0 +#define EVENTHANDLER_PRI_ANY 10000 +#define EVENTHANDLER_PRI_LAST 20000 + +#endif /* _SYS_EVENTHANDLER_H_ */ diff --git a/bsd/sys/eventvar.h b/bsd/sys/eventvar.h index 8d47aad64..82323625f 100644 --- a/bsd/sys/eventvar.h +++ b/bsd/sys/eventvar.h @@ -63,9 +63,41 @@ #if defined(XNU_KERNEL_PRIVATE) +typedef int (*kevent_callback_t)(struct kqueue *, struct kevent_internal_s *, void *); +typedef void (*kqueue_continue_t)(struct kqueue *, void *, int); + +#include <stdint.h> #include <kern/locks.h> +#include <sys/pthread_shims.h> #include <mach/thread_policy.h> +/* + * Lock ordering: + * + * The kqueue locking order can follow a few different patterns: + * + * Standard file-based kqueues (from above): + * proc fd lock -> kq lock -> kq-waitq-set lock -> thread lock + * + * WorkQ/WorkLoop kqueues (from above): + * proc fd lock -> kq lock -> kq-request lock -> pthread kext locks -> thread lock + * + * Whenever kqueues interact with source locks, it drops all of its own + * locks in exchange for a use-reference on the knote used to synchronize + * with the source code. When those sources post events from below, they + * have the following lock hierarchy. + * + * Standard file-based kqueues (from below): + * XXX lock -> kq lock -> kq-waitq-set lock -> thread lock + * Standard file-based kqueues with non-kq-aware sources (from below): + * XXX lock -> kq-waitq-set lock -> thread lock + * + * WorkQ/WorkLoop kqueues (from below): + * XXX lock -> kq lock -> kq-request lock -> pthread kext locks -> thread lock + * WorkQ/WorkLoop kqueues with non-kq-aware sources (from below): + * XXX -> kq-waitq-set lock -> kq-request lock -> pthread kext locks -> thread lock + */ + #define KQEXTENT 256 /* linear growth by this amount */ /* @@ -85,17 +117,19 @@ struct kqueue { struct kqtailq kq_queue[1]; /* variable array of kqtailq structs */ }; -#define KQ_SEL 0x001 /* select was recorded for kq */ -#define KQ_SLEEP 0x002 /* thread is waiting for events */ -#define KQ_PROCWAIT 0x004 /* thread waiting for processing */ -#define KQ_KEV32 0x008 /* kq is used with 32-bit events */ -#define KQ_KEV64 0x010 /* kq is used with 64-bit events */ -#define KQ_KEV_QOS 0x020 /* kq events carry QoS info */ -#define KQ_WORKQ 0x040 /* KQ is bould to process workq */ -#define KQ_PROCESSING 0x080 /* KQ is being processed */ -#define KQ_DRAIN 0x100 /* kq is draining */ -#define KQ_WAKEUP 0x200 /* kq awakened while processing */ - +#define KQ_SEL 0x001 /* select was recorded for kq */ +#define KQ_SLEEP 0x002 /* thread is waiting for events */ +#define KQ_PROCWAIT 0x004 /* thread waiting for processing */ +#define KQ_KEV32 0x008 /* kq is used with 32-bit events */ +#define KQ_KEV64 0x010 /* kq is used with 64-bit events */ +#define KQ_KEV_QOS 0x020 /* kq events carry QoS info */ +#define KQ_WORKQ 0x040 /* KQ is bound to process workq */ +#define KQ_WORKLOOP 0x080 /* KQ is part of a workloop */ +#define KQ_PROCESSING 0x100 /* KQ is being processed */ +#define KQ_DRAIN 0x200 /* kq is draining */ +#define KQ_WAKEUP 0x400 /* kq awakened while processing */ +#define KQ_DYNAMIC 0x800 /* kqueue is dynamically managed */ +#define KQ_NO_WQ_THREAD 0x1000 /* kq will not have workqueue threads dynamically created */ /* * kqfile - definition of a typical kqueue opened as a file descriptor * via the kqueue() system call. @@ -119,6 +153,53 @@ struct kqfile { #define QOS_INDEX_KQFILE 0 /* number of qos levels in a file kq */ +struct kqr_bound { + struct kqtailq kqrb_suppressed; /* Per-QoS suppression queues */ + thread_t kqrb_thread; /* thread to satisfy request */ +}; + +/* + * kqrequest - per-QoS thread request status + */ +struct kqrequest { +#if 0 + union { + struct kqr_bound kqru_bound; /* used when thread is bound */ + struct workq_threadreq_s kqru_req; /* used when request oustanding */ + } kqr_u; +#define kqr_suppressed kqr_u.kqru_bound.kqrb_suppressed +#define kqr_thread kqr_u.kqru_bound.kqrb_thread +#define kqr_req kqr_u.kqru_req +#else + struct kqr_bound kqr_bound; /* used when thread is bound */ + struct workq_threadreq_s kqr_req; /* used when request oustanding */ +#define kqr_suppressed kqr_bound.kqrb_suppressed +#define kqr_thread kqr_bound.kqrb_thread +#endif + uint8_t kqr_state; /* KQ/workq interaction state */ + uint8_t kqr_wakeup_indexes; /* QoS/override levels that woke */ + uint16_t kqr_dsync_waiters:13, /* number of dispatch sync waiters */ + kqr_dsync_owner_qos:3; /* Qos override on dispatch sync owner */ + uint16_t kqr_sync_suppress_count; /* number of suppressed sync ipc knotes */ + kq_index_t kqr_stayactive_qos:3, /* max QoS of statyactive knotes */ + kqr_owner_override_is_sync:1, /* sync owner has sync ipc override */ + kqr_override_index:3, /* highest wakeup override index */ + kqr_has_sync_override:1; /* Qos/override at UI is sync ipc override */ + + /* set under both the kqlock and the filt_wllock */ + kq_index_t :0; /* prevent bitfields coalescing <rdar://problem/31854115> */ + kq_index_t kqr_qos_index:4, /* QoS for the thread request */ + kqr_dsync_waiters_qos:4; /* override from dispatch sync waiters */ +}; + + +#define KQR_PROCESSING 0x01 /* requested thread is running the q */ +#define KQR_THREQUESTED 0x02 /* thread has been requested from workq */ +#define KQR_WAKEUP 0x04 /* wakeup called during processing */ +#define KQR_BOUND 0x08 /* servicing thread is bound */ +#define KQR_THOVERCOMMIT 0x20 /* overcommit needed for thread requests */ +#define KQR_DRAIN 0x40 /* cancel initiated - drain fulfill */ +#define KQR_R2K_NOTIF_ARMED 0x80 /* ast notifications armed */ /* * WorkQ kqueues need to request threads to service the triggered * knotes in the queue. These threads are brought up on a @@ -136,17 +217,6 @@ struct kqfile { #define KQWQ_NQOS (KQWQ_QOS_MANAGER + 1) #endif - -/* - * kqrequest - per-QoS thread request status - */ -struct kqrequest { - struct kqtailq kqr_suppressed; /* Per-QoS suppression queues */ - thread_t kqr_thread; /* thread to satisfy request */ - uint8_t kqr_state; /* KQ/workq interaction state */ - uint8_t kqr_override_delta; /* current override delta */ -}; - /* * Workq thread start out a particular effective-requested-QoS, but * additional events processed by the filters may represent @@ -202,21 +272,85 @@ struct kqworkq { #define kqwq_p kqwq_kqueue.kq_p #define kqwq_queue kqwq_kqueue.kq_queue -#define kqwq_req_lock(kqwq) (lck_spin_lock(&kqwq->kqwq_reqlock)) -#define kqwq_req_unlock(kqwq) (lck_spin_unlock(&kqwq->kqwq_reqlock)) -#define kqwq_req_held(kqwq) (lck_spin_held(&kqwq->kqwq_reqlock)) +#define kqwq_req_lock(kqwq) lck_spin_lock(&kqwq->kqwq_reqlock) +#define kqwq_req_unlock(kqwq) lck_spin_unlock(&kqwq->kqwq_reqlock) +#define kqwq_req_held(kqwq) LCK_SPIN_ASSERT(&kqwq->kqwq_reqlock, LCK_ASSERT_OWNED) + +#define KQWQ_THMANAGER 0x10 /* expect manager thread to run the queue */ + +/* + * WorkLoop kqueues need to request a thread to service the triggered + * knotes in the queue. The thread is brought up on a + * effective-requested-QoS basis. Knotes are segregated based on + * that value. Once a request is made, it cannot be undone. If + * events with higher QoS arrive after, they are stored in their + * own queues and an override applied to the original request based + * on the delta between the two QoS values. + */ + +/* + * "Stay-active" knotes are held in a separate bucket that indicates + * special handling required. They are kept separate because the + * wakeups issued to them don't have context to tell us where to go + * to find and process them. All processing of them happens at the + * highest QoS. Unlike WorkQ kqueues, there is no special singular + * "manager thread" for a process. We simply request a servicing + * thread at the higest known QoS when these are woken (or override + * an existing request to that). + */ +#define KQWL_BUCKET_STAYACTIVE (THREAD_QOS_LAST) + +#if !defined(KQWL_NBUCKETS) +#define KQWL_NBUCKETS (KQWL_BUCKET_STAYACTIVE + 1) +#endif + +/* + * kqworkloop - definition of a private kqueue used to coordinate event + * handling for pthread workloops. + * + * Workloops vary from workqs in that only a single thread is ever + * requested to service a workloop at a time. But unlike workqs, + * workloops may be "owned" by user-space threads that are + * synchronously draining an event off the workloop. In those cases, + * any overrides have to be applied to the owner until it relinqueshes + * ownership. + * + * NOTE: "lane" support is TBD. + */ +struct kqworkloop { + struct kqueue kqwl_kqueue; /* queue of events */ + struct kqtailq kqwl_queuecont[KQWL_NBUCKETS-1]; /* continue array of queues */ + struct kqrequest kqwl_request; /* thread request state */ + lck_spin_t kqwl_reqlock; /* kqueue request lock */ + lck_mtx_t kqwl_statelock; /* state/debounce lock */ + thread_t kqwl_owner; /* current [sync] owner thread */ + uint32_t kqwl_retains; /* retain references */ + kqueue_id_t kqwl_dynamicid; /* dynamic identity */ + SLIST_ENTRY(kqworkloop) kqwl_hashlink; /* linkage for search list */ +}; -#define KQWQ_PROCESSING 0x01 /* running the kq in workq mode */ -#define KQWQ_THREQUESTED 0x02 /* thread requested from workq */ -#define KQWQ_THMANAGER 0x04 /* expect manager thread to run the queue */ -#define KQWQ_HOOKCALLED 0x10 /* hook called during processing */ -#define KQWQ_WAKEUP 0x20 /* wakeup called during processing */ +SLIST_HEAD(kqlist, kqworkloop); + +#define kqwl_wqs kqwl_kqueue.kq_wqs +#define kqwl_lock kqwl_kqueue.kq_lock +#define kqwl_state kqwl_kqueue.kq_state +#define kqwl_level kqwl_kqueue.kq_level +#define kqwl_count kqwl_kqueue.kq_count +#define kqwl_p kqwl_kqueue.kq_p +#define kqwl_queue kqwl_kqueue.kq_queue + +#define kqwl_req_lock(kqwl) lck_spin_lock(&kqwl->kqwl_reqlock) +#define kqwl_req_unlock(kqwl) lck_spin_unlock(&kqwl->kqwl_reqlock) +#define kqwl_req_held(kqwl) LCK_SPIN_ASSERT(&kqwl->kqwl_reqlock, LCK_ASSERT_OWNED) + +#define KQ_WORKLOOP_RETAINS_MAX UINT32_MAX + +extern int workloop_fulfill_threadreq(struct proc *p, workq_threadreq_t req, thread_t thread, int flags); extern struct kqueue *kqueue_alloc(struct proc *, unsigned int); extern void kqueue_dealloc(struct kqueue *); -typedef int (*kevent_callback_t)(struct kqueue *, struct kevent_internal_s *, void *); -typedef void (*kqueue_continue_t)(struct kqueue *, void *, int); +extern void knotes_dealloc(struct proc *); extern void kevent_register(struct kqueue *, struct kevent_internal_s *, struct proc *); extern int kqueue_scan(struct kqueue *, kevent_callback_t, kqueue_continue_t, diff --git a/bsd/sys/fasttrap.h b/bsd/sys/fasttrap.h index 3aa0db471..ec2ece46c 100644 --- a/bsd/sys/fasttrap.h +++ b/bsd/sys/fasttrap.h @@ -75,14 +75,21 @@ typedef struct fasttrap_probe_spec { #if defined(__APPLE__) fasttrap_provider_type_t ftps_provider_type; fasttrap_probe_type_t ftps_probe_type; +#if defined(__arm__) || defined(__arm64__) + uint32_t ftps_arch_subinfo; // For any additional per probe architecture specific data +#endif #endif char ftps_func[DTRACE_FUNCNAMELEN]; char ftps_mod[DTRACE_MODNAMELEN]; #if defined(__APPLE__) +#if defined(__arm__) || defined(__arm64__) + // We already have 'padding' from the ftps_arch_subinfo above +#else #if !defined(__LP64__) uint32_t pad; /* Explicit pad to keep ILP32 and LP64 lined up. */ #endif +#endif #endif uint64_t ftps_pc; uint64_t ftps_size; diff --git a/bsd/sys/fasttrap_impl.h b/bsd/sys/fasttrap_impl.h index 439d36864..1ca389cb6 100644 --- a/bsd/sys/fasttrap_impl.h +++ b/bsd/sys/fasttrap_impl.h @@ -97,6 +97,7 @@ typedef struct fasttrap_provider { uint_t ftp_retired; /* mark when retired */ lck_mtx_t ftp_mtx; /* provider lock */ lck_mtx_t ftp_cmtx; /* lock on creating probes */ + uint64_t ftp_pcount; /* probes in provider count */ uint64_t ftp_rcount; /* enabled probes ref count */ uint64_t ftp_ccount; /* consumers creating probes */ uint64_t ftp_mcount; /* meta provider count */ @@ -130,6 +131,7 @@ struct fasttrap_probe { uint8_t *ftp_argmap; /* native to translated args */ uint8_t ftp_nargs; /* translated argument count */ uint8_t ftp_enabled; /* is this probe enabled */ + uint8_t ftp_triggered; char *ftp_xtypes; /* translated types index */ char *ftp_ntypes; /* native types index */ fasttrap_id_tp_t ftp_tps[1]; /* flexible array */ @@ -188,6 +190,13 @@ extern fasttrap_hash_t fasttrap_tpoints; #define FASTTRAP_TPOINTS_INDEX(pid, pc) \ (((pc) / sizeof (fasttrap_instr_t) + (pid)) & fasttrap_tpoints.fth_mask) + +#ifdef CONFIG_EMBEDDED +#define FASTTRAP_ASYNC_REMOVE +#endif + +extern void fasttrap_tracepoint_retire(proc_t *p, fasttrap_tracepoint_t *tp); + /* * Must be implemented by fasttrap_isa.c */ @@ -199,6 +208,9 @@ extern int fasttrap_tracepoint_remove(proc_t *, fasttrap_tracepoint_t *); #if defined(__x86_64__) extern int fasttrap_pid_probe(x86_saved_state_t *regs); extern int fasttrap_return_probe(x86_saved_state_t* regs); +#elif defined(__arm__) || defined(__arm64__) +extern int fasttrap_pid_probe(arm_saved_state_t *rp); +extern int fasttrap_return_probe(arm_saved_state_t *regs); #else #error architecture not supported #endif @@ -206,6 +218,7 @@ extern int fasttrap_return_probe(x86_saved_state_t* regs); extern uint64_t fasttrap_pid_getarg(void *, dtrace_id_t, void *, int, int); extern uint64_t fasttrap_usdt_getarg(void *, dtrace_id_t, void *, int, int); + #ifdef __cplusplus } #endif diff --git a/bsd/sys/fbt.h b/bsd/sys/fbt.h index 4c67eef5c..fdda0b161 100644 --- a/bsd/sys/fbt.h +++ b/bsd/sys/fbt.h @@ -31,6 +31,10 @@ #if defined (__x86_64__) typedef uint8_t machine_inst_t; +#elif defined(__arm__) +typedef uint16_t machine_inst_t; +#elif defined(__arm64__) +typedef uint32_t machine_inst_t; #else #error Unknown Architecture #endif @@ -61,4 +65,11 @@ extern int dtrace_invop(uintptr_t, uintptr_t *, uintptr_t); extern int fbt_invop(uintptr_t, uintptr_t *, uintptr_t); extern void fbt_provide_module(void *, struct modctl *); extern int fbt_enable (void *arg, dtrace_id_t id, void *parg); + +extern int fbt_module_excluded(struct modctl*); +extern int fbt_excluded(const char *); + +extern void fbt_provide_probe(struct modctl *ctl, uintptr_t instr_low, uintptr_t instr_high, char *modname, char* symbol_name, machine_inst_t* symbol_start); + +extern void fbt_provide_module_kernel_syms(struct modctl *ctl); #endif /* _FBT_H */ diff --git a/bsd/sys/fcntl.h b/bsd/sys/fcntl.h index 0e24d78ce..02c868bba 100644 --- a/bsd/sys/fcntl.h +++ b/bsd/sys/fcntl.h @@ -361,7 +361,7 @@ #define F_GETCONFINED 96 /* is-fd-confined? */ #endif -#define F_ADDFILESIGS_RETURN 97 /* Add signature from same file, return end offset in structure on sucess */ +#define F_ADDFILESIGS_RETURN 97 /* Add signature from same file, return end offset in structure on success */ #define F_CHECK_LV 98 /* Check if Library Validation allows this Mach-O file to be mapped into the calling process */ #define F_PUNCHHOLE 99 /* Deallocate a range of the file */ diff --git a/bsd/sys/file_internal.h b/bsd/sys/file_internal.h index 60a47afa5..0b9ed96b2 100644 --- a/bsd/sys/file_internal.h +++ b/bsd/sys/file_internal.h @@ -80,6 +80,8 @@ struct proc; struct uio; struct knote; +struct kevent_internal_s; + #ifdef __APPLE_API_UNSTABLE struct file; @@ -183,7 +185,7 @@ struct fileglob { void *wql, vfs_context_t ctx); int (*fo_close) (struct fileglob *fg, vfs_context_t ctx); int (*fo_kqfilter) (struct fileproc *fp, struct knote *kn, - vfs_context_t ctx); + struct kevent_internal_s *kev, vfs_context_t ctx); int (*fo_drain) (struct fileproc *fp, vfs_context_t ctx); } *fg_ops; off_t fg_offset; @@ -213,7 +215,8 @@ int fo_write(struct fileproc *fp, struct uio *uio, int flags, int fo_ioctl(struct fileproc *fp, u_long com, caddr_t data, vfs_context_t ctx); int fo_select(struct fileproc *fp, int which, void *wql, vfs_context_t ctx); int fo_close(struct fileglob *fg, vfs_context_t ctx); -int fo_kqfilter(struct fileproc *fp, struct knote *kn, vfs_context_t ctx); +int fo_kqfilter(struct fileproc *fp, struct knote *kn, + struct kevent_internal_s *kev, vfs_context_t ctx); void fileproc_drain(proc_t, struct fileproc *); int fp_tryswap(proc_t, int fd, struct fileproc *nfp); int fp_drop(struct proc *p, int fd, struct fileproc *fp, int locked); diff --git a/bsd/sys/filedesc.h b/bsd/sys/filedesc.h index 41dc190b5..b79440b48 100644 --- a/bsd/sys/filedesc.h +++ b/bsd/sys/filedesc.h @@ -85,10 +85,17 @@ #ifdef BSD_KERNEL_PRIVATE +#include <kern/locks.h> + struct klist; +struct kqlist; struct filedesc { struct fileproc **fd_ofiles; /* file structures for open files */ + lck_mtx_t fd_kqhashlock; /* lock for dynamic kqueue hash */ + u_long fd_kqhashmask; /* size of dynamic kqueue hash */ + struct kqlist *fd_kqhash; /* hash table for dynamic kqueues */ + struct kqueue *fd_wqkqueue; /* the workq kqueue */ char *fd_ofileflags; /* per-process open file flags */ struct vnode *fd_cdir; /* current directory */ struct vnode *fd_rdir; /* root directory */ @@ -96,11 +103,12 @@ struct filedesc { int fd_lastfile; /* high-water mark of fd_ofiles */ int fd_freefile; /* approx. next free file */ u_short fd_cmask; /* mask for file creation */ + int fd_flags; int fd_knlistsize; /* size of knlist */ struct klist *fd_knlist; /* list of attached knotes */ u_long fd_knhashmask; /* size of knhash */ struct klist *fd_knhash; /* hash table for attached knotes */ - int fd_flags; + lck_mtx_t fd_knhashlock; /* lock for hash table for attached knotes */ }; /* @@ -165,7 +173,7 @@ extern int falloc_withalloc(proc_t p, struct fileproc **resultfp, extern struct filedesc *fdcopy(proc_t p, struct vnode *uth_cdir); extern void fdfree(proc_t p); -extern void fdexec(proc_t p, short flags); +extern void fdexec(proc_t p, short flags, int self_exec); #endif /* __APPLE_API_PRIVATE */ #endif /* KERNEL */ diff --git a/bsd/sys/fsctl.h b/bsd/sys/fsctl.h index 5d87fbf7f..8a3624d3b 100644 --- a/bsd/sys/fsctl.h +++ b/bsd/sys/fsctl.h @@ -244,6 +244,15 @@ typedef struct package_ext_info { uint32_t max_width; } package_ext_info; +/* Disk conditioner configuration */ +typedef struct disk_conditioner_info { + int enabled; + uint64_t access_time_usec; // maximum latency until transfer begins + uint64_t read_throughput_mbps; // maximum throughput for reads + uint64_t write_throughput_mbps; // maximum throughput for writes + int is_ssd; // behave like an SSD +} disk_conditioner_info; + #define FSCTL_SYNC_FULLSYNC (1<<0) /* Flush the data fully to disk, if supported by the filesystem */ #define FSCTL_SYNC_WAIT (1<<1) /* Wait for the sync to complete */ @@ -299,18 +308,16 @@ typedef struct package_ext_info { #define FSIOC_FIOSEEKDATA _IOWR('A', 17, off_t) #define FSCTL_FIOSEEKDATA IOCBASECMD(FSIOC_FIOSEEKDATA) -// -// IO commands 16 and 17 are currently unused -// +/* Disk conditioner */ +#define DISK_CONDITIONER_IOC_GET _IOR('A', 18, disk_conditioner_info) +#define DISK_CONDITIONER_FSCTL_GET IOCBASECMD(DISK_CONDITIONER_IOC_GET) +#define DISK_CONDITIONER_IOC_SET _IOW('A', 19, disk_conditioner_info) +#define DISK_CONDITIONER_FSCTL_SET IOCBASECMD(DISK_CONDITIONER_IOC_SET) // // Spotlight and fseventsd use these fsctl()'s to find out // the mount time of a volume and the last time it was -// unmounted. Both HFS and ZFS support these calls. -// -// User space code should pass the "_IOC_" macros while the -// kernel should test for the "_FSCTL_" variant of the macro -// in its vnop_ioctl function. +// unmounted. Both HFS and APFS support these calls. // // NOTE: the values for these defines should _not_ be changed // or else it will break binary compatibility with mds @@ -321,7 +328,6 @@ typedef struct package_ext_info { #define SPOTLIGHT_IOC_GET_LAST_MTIME _IOR('h', 19, u_int32_t) #define SPOTLIGHT_FSCTL_GET_LAST_MTIME IOCBASECMD(SPOTLIGHT_IOC_GET_LAST_MTIME) - #ifndef KERNEL #include <sys/cdefs.h> diff --git a/bsd/sys/fsevents.h b/bsd/sys/fsevents.h index a9c922557..bf338c6e0 100644 --- a/bsd/sys/fsevents.h +++ b/bsd/sys/fsevents.h @@ -44,8 +44,9 @@ #define FSE_DOCID_CREATED 11 #define FSE_DOCID_CHANGED 12 #define FSE_UNMOUNT_PENDING 13 // iOS-only: client must respond via FSEVENTS_UNMOUNT_PENDING_ACK +#define FSE_CLONE 14 -#define FSE_MAX_EVENTS 14 +#define FSE_MAX_EVENTS 15 #define FSE_ALL_EVENTS 998 #define FSE_EVENTS_DROPPED 999 @@ -99,6 +100,7 @@ #define FSE_MODE_LAST_HLINK (1 << 30) // link count == 0 on a hard-link delete #define FSE_REMOTE_DIR_EVENT (1 << 29) // this is a remotely generated directory-level granularity event #define FSE_TRUNCATED_PATH (1 << 28) // the path for this item had to be truncated +#define FSE_MODE_CLONE (1 << 27) // notification is for a clone // ioctl's on /dev/fsevents typedef struct fsevent_clone_args { diff --git a/bsd/sys/fsgetpath.h b/bsd/sys/fsgetpath.h index bad8b4e1b..da8e53173 100644 --- a/bsd/sys/fsgetpath.h +++ b/bsd/sys/fsgetpath.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008 Apple Inc. All rights reserved. + * Copyright (c) 2008-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -29,23 +29,39 @@ #ifndef _FSGETPATH_H_ #define _FSGETPATH_H_ +#ifndef KERNEL + +#include <machine/_types.h> +#include <sys/_types/_ssize_t.h> +#include <sys/_types/_size_t.h> +#include <sys/_types/_fsid_t.h> +#include <_types/_uint64_t.h> +#include <Availability.h> + +#ifdef PRIVATE +/* + * These are only included for compatibility with previous header + */ #include <sys/types.h> #include <sys/mount.h> - #ifdef __APPLE_API_PRIVATE - -#ifndef KERNEL -__BEGIN_DECLS - #include <sys/syscall.h> #include <unistd.h> +#endif /* __APPLE_API_PRIVATE */ +#endif /* PRIVATE */ + +__BEGIN_DECLS /* * Obtain the full pathname of a file system object by id. - * - * This is a private SPI used by the File Manager. */ -ssize_t fsgetpath(char * __restrict buf, size_t bufsize, fsid_t* fsid, uint64_t objid); +ssize_t fsgetpath(char *, size_t, fsid_t *, uint64_t) __OSX_AVAILABLE(10.13) __IOS_AVAILABLE(11.0) __TVOS_AVAILABLE(11.0) __WATCHOS_AVAILABLE(4.0); + +#ifdef PRIVATE +#include <sys/_types/_fsobj_id_t.h> + +#ifdef __APPLE_API_PRIVATE + /* * openbyid_np: open a file given a file system id and a file system object id @@ -65,9 +81,11 @@ ssize_t fsgetpath(char * __restrict buf, size_t bufsize, fsid_t* fsid, uint64_t */ int openbyid_np(fsid_t* fsid, fsobj_id_t* objid, int flags); +#endif /* __APPLE_API_PRIVATE */ +#endif /* PRIVATE */ + __END_DECLS -#endif /* KERNEL */ -#endif /* __APPLE_API_PRIVATE */ +#endif /* KERNEL */ #endif /* !_FSGETPATH_H_ */ diff --git a/bsd/sys/guarded.h b/bsd/sys/guarded.h index 6fa6a7752..ae1ec05f9 100644 --- a/bsd/sys/guarded.h +++ b/bsd/sys/guarded.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 Apple Inc. All rights reserved. + * Copyright (c) 2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -59,12 +59,10 @@ extern ssize_t guarded_pwrite_np(int fd, const guardid_t *guard, const void *buf extern ssize_t guarded_writev_np(int fd, const guardid_t *guard, const struct iovec *iovp, int iovcnt); #endif /* KERNEL */ -/* - * Guard types. - * - * GUARD_TYPE_FD: Guarded file descriptor. - */ -#define GUARD_TYPE_FD 0x2 +#ifndef GUARD_TYPE_FD +/* temporary source compat: use <kern/exc_guard.h> instead */ +#define GUARD_TYPE_FD 0x2 +#endif /* * File descriptor guard flavors. @@ -102,7 +100,7 @@ extern ssize_t guarded_writev_np(int fd, const guardid_t *guard, const struct io * Violating a guard results in an error (EPERM), and potentially * an exception with one or more of the following bits set. */ -enum guard_exception_codes { +enum guard_fd_exception_codes { kGUARD_EXC_CLOSE = 1u << 0, /* close of a guarded fd */ kGUARD_EXC_DUP = 1u << 1, /* dup of a guarded fd */ kGUARD_EXC_NOCLOEXEC = 1u << 2, /* clear close-on-exec */ @@ -112,6 +110,60 @@ enum guard_exception_codes { kGUARD_EXC_WRITE = 1u << 6 /* write on a guarded fd */ }; +/* + * Experimental guarded vnode support + */ +#define VNG_RENAME_TO (1u << 0) +#define VNG_RENAME_FROM (1u << 1) +#define VNG_UNLINK (1u << 2) +#define VNG_WRITE_OTHER (1u << 3) +#define VNG_TRUNC_OTHER (1u << 4) +#define VNG_LINK (1u << 5) +#define VNG_EXCHDATA (1u << 6) + +#define VNG_ALL \ + (VNG_RENAME_TO | VNG_RENAME_FROM | VNG_UNLINK | VNG_LINK | \ + VNG_WRITE_OTHER | VNG_TRUNC_OTHER | VNG_EXCHDATA) + +struct vnguard_set { + int vns_fd; + unsigned vns_attrs; + guardid_t vns_guard; +}; + +#define VNG_SYSC_PING 0 +#define VNG_SYSC_SET_GUARD 1 + +#define VNG_POLICY_NAME "vnguard" + +/* + * Violating a guard may result in an error (EPERM), and potentially + * an exception with one or more of the following bits set. + */ +enum guard_vn_exception_codes { + kGUARD_EXC_RENAME_TO = VNG_RENAME_TO, + kGUARD_EXC_RENAME_FROM = VNG_RENAME_FROM, + kGUARD_EXC_UNLINK = VNG_UNLINK, + kGUARD_EXC_WRITE_OTHER = VNG_WRITE_OTHER, + kGUARD_EXC_TRUNC_OTHER = VNG_TRUNC_OTHER, + kGUARD_EXC_LINK = VNG_LINK, + kGUARD_EXC_EXCHDATA = VNG_EXCHDATA, +}; + +#if defined(KERNEL) + +/* Guard violation behaviors: not all combinations make sense */ + +#define kVNG_POLICY_LOGMSG (1u << 0) +#define kVNG_POLICY_EPERM (1u << 1) +#define kVNG_POLICY_EXC (1u << 2) +#define kVNG_POLICY_EXC_CORPSE (1u << 3) +#define kVNG_POLICY_SIGKILL (1u << 4) + +extern int vnguard_exceptions_active(void); +extern void vnguard_policy_init(void); +#endif /* KERNEL */ + #endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */ __END_DECLS diff --git a/bsd/sys/imgact.h b/bsd/sys/imgact.h index fc23e70e9..c1fbde252 100644 --- a/bsd/sys/imgact.h +++ b/bsd/sys/imgact.h @@ -119,6 +119,9 @@ struct image_params { void *ip_px_smpx; /* MAC-specific spawn attrs. */ void *ip_px_persona; /* persona args */ void *ip_cs_error; /* codesigning error reason */ + + uint64_t ip_dyld_fsid; + uint64_t ip_dyld_fsobjid; }; /* @@ -134,5 +137,6 @@ struct image_params { #define IMGPF_ALLOW_DATA_EXEC 0x00000040 /* forcibly disallow data execution */ #define IMGPF_VFORK_EXEC 0x00000080 /* vfork followed by exec */ #define IMGPF_EXEC 0x00000100 /* exec */ +#define IMGPF_HIGH_BITS_ASLR 0x00000200 /* randomize high bits of ASLR slide */ #endif /* !_SYS_IMGACT */ diff --git a/bsd/sys/kauth.h b/bsd/sys/kauth.h index dd496f8da..6d46a5afb 100644 --- a/bsd/sys/kauth.h +++ b/bsd/sys/kauth.h @@ -138,6 +138,8 @@ struct kauth_cache_sizes { #define KAUTH_SET_CACHE_SIZES (1<<4) #define KAUTH_CLEAR_CACHES (1<<5) +#define IDENTITYSVC_ENTITLEMENT "com.apple.private.identitysvc" + #ifdef KERNEL /* diff --git a/bsd/sys/kdebug.h b/bsd/sys/kdebug.h index d29e0cd17..0e3988617 100644 --- a/bsd/sys/kdebug.h +++ b/bsd/sys/kdebug.h @@ -35,6 +35,7 @@ #include <sys/appleapiopts.h> #include <sys/cdefs.h> + __BEGIN_DECLS #ifdef __APPLE_API_UNSTABLE @@ -192,6 +193,7 @@ extern void kernel_debug_enter( #define DBG_WORKQUEUE 9 #define DBG_CORESTORAGE 10 #define DBG_CG 11 +#define DBG_MONOTONIC 12 #define DBG_MISC 20 #define DBG_SECURITY 30 #define DBG_DYLD 31 @@ -354,37 +356,45 @@ extern void kdebug_reset(void); #endif /* XNU_KERNEL_PRIVATE */ /* **** The Kernel Debug Sub Classes for Mach (DBG_MACH) **** */ -#define DBG_MACH_EXCP_KTRAP_x86 0x02 /* Kernel Traps on x86 */ -#define DBG_MACH_EXCP_DFLT 0x03 /* Data Translation Fault */ -#define DBG_MACH_EXCP_IFLT 0x04 /* Inst Translation Fault */ -#define DBG_MACH_EXCP_INTR 0x05 /* Interrupts */ -#define DBG_MACH_EXCP_ALNG 0x06 /* Alignment Exception */ -#define DBG_MACH_EXCP_UTRAP_x86 0x07 /* User Traps on x86 */ -#define DBG_MACH_EXCP_FP 0x08 /* FP Unavail */ -#define DBG_MACH_EXCP_DECI 0x09 /* Decrementer Interrupt */ -#define DBG_MACH_CHUD 0x0A /* deprecated name */ -#define DBG_MACH_SIGNPOST 0x0A /* kernel signposts */ -#define DBG_MACH_EXCP_SC 0x0C /* System Calls */ -#define DBG_MACH_EXCP_TRACE 0x0D /* Trace exception */ -#define DBG_MACH_EXCP_EMUL 0x0E /* Instruction emulated */ -#define DBG_MACH_IHDLR 0x10 /* Interrupt Handlers */ -#define DBG_MACH_IPC 0x20 /* Inter Process Comm */ -#define DBG_MACH_RESOURCE 0x25 /* tracing limits, etc */ -#define DBG_MACH_VM 0x30 /* Virtual Memory */ -#define DBG_MACH_LEAKS 0x31 /* alloc/free */ -#define DBG_MACH_WORKINGSET 0x32 /* private subclass for working set related debugging */ -#define DBG_MACH_SCHED 0x40 /* Scheduler */ -#define DBG_MACH_MSGID_INVALID 0x50 /* Messages - invalid */ -#define DBG_MACH_LOCKS 0x60 /* new lock APIs */ -#define DBG_MACH_PMAP 0x70 /* pmap */ -#define DBG_MACH_CLOCK 0x80 /* clock */ -#define DBG_MACH_MP 0x90 /* MP related */ -#define DBG_MACH_VM_PRESSURE 0xA0 /* Memory Pressure Events */ -#define DBG_MACH_STACKSHOT 0xA1 /* Stackshot/Microstackshot subsystem */ -#define DBG_MACH_SFI 0xA2 /* Selective Forced Idle (SFI) */ -#define DBG_MACH_ENERGY_PERF 0xA3 /* Energy/performance resource stats */ -#define DBG_MACH_SYSDIAGNOSE 0xA4 /* sysdiagnose keychord */ -#define DBG_MACH_ZALLOC 0xA5 /* Zone allocator */ +#define DBG_MACH_EXCP_KTRAP_x86 0x02 /* Kernel Traps on x86 */ +#define DBG_MACH_EXCP_DFLT 0x03 /* Data Translation Fault */ +#define DBG_MACH_EXCP_IFLT 0x04 /* Inst Translation Fault */ +#define DBG_MACH_EXCP_INTR 0x05 /* Interrupts */ +#define DBG_MACH_EXCP_ALNG 0x06 /* Alignment Exception */ +#define DBG_MACH_EXCP_UTRAP_x86 0x07 /* User Traps on x86 */ +#define DBG_MACH_EXCP_FP 0x08 /* FP Unavail */ +#define DBG_MACH_EXCP_DECI 0x09 /* Decrementer Interrupt */ +#define DBG_MACH_CHUD 0x0A /* deprecated name */ +#define DBG_MACH_SIGNPOST 0x0A /* kernel signposts */ +#define DBG_MACH_EXCP_SC 0x0C /* System Calls */ +#define DBG_MACH_EXCP_TRACE 0x0D /* Trace exception */ +#define DBG_MACH_EXCP_EMUL 0x0E /* Instruction emulated */ +#define DBG_MACH_IHDLR 0x10 /* Interrupt Handlers */ +#define DBG_MACH_IPC 0x20 /* Inter Process Comm */ +#define DBG_MACH_RESOURCE 0x25 /* tracing limits, etc */ +#define DBG_MACH_VM 0x30 /* Virtual Memory */ +#define DBG_MACH_LEAKS 0x31 /* alloc/free */ +#define DBG_MACH_WORKINGSET 0x32 /* private subclass for working set related debugging */ +#define DBG_MACH_SCHED 0x40 /* Scheduler */ +#define DBG_MACH_MSGID_INVALID 0x50 /* Messages - invalid */ +#define DBG_MACH_LOCKS 0x60 /* new lock APIs */ +#define DBG_MACH_PMAP 0x70 /* pmap */ +#define DBG_MACH_CLOCK 0x80 /* clock */ +#define DBG_MACH_MP 0x90 /* MP related */ +#define DBG_MACH_VM_PRESSURE 0xA0 /* Memory Pressure Events */ +#define DBG_MACH_STACKSHOT 0xA1 /* Stackshot/Microstackshot subsystem */ +#define DBG_MACH_SFI 0xA2 /* Selective Forced Idle (SFI) */ +#define DBG_MACH_ENERGY_PERF 0xA3 /* Energy/performance resource stats */ +#define DBG_MACH_SYSDIAGNOSE 0xA4 /* sysdiagnose */ +#define DBG_MACH_ZALLOC 0xA5 /* Zone allocator */ +#define DBG_MACH_THREAD_GROUP 0xA6 /* Thread groups */ +#define DBG_MACH_COALITION 0xA7 /* Coalitions */ + +/* Interrupt type bits for DBG_MACH_EXCP_INTR */ +#define DBG_INTR_TYPE_UNKNOWN 0x0 /* default/unknown interrupt */ +#define DBG_INTR_TYPE_IPI 0x1 /* interprocessor interrupt */ +#define DBG_INTR_TYPE_TIMER 0x2 /* timer interrupt */ +#define DBG_INTR_TYPE_OTHER 0x3 /* other (usually external) interrupt */ /* Codes for Scheduler (DBG_MACH_SCHED) */ #define MACH_SCHED 0x0 /* Scheduler */ @@ -399,8 +409,8 @@ extern void kdebug_reset(void); #define MACH_IDLE 0x9 /* processor idling */ #define MACH_STACK_DEPTH 0xa /* stack depth at switch */ #define MACH_MOVED 0xb /* did not use original scheduling decision */ -/* unused 0xc */ -/* unused 0xd */ +#define MACH_PSET_LOAD_AVERAGE 0xc +#define MACH_AMP_DEBUG 0xd #define MACH_FAILSAFE 0xe /* tripped fixed-pri/RT failsafe */ #define MACH_BLOCK 0xf /* thread block */ #define MACH_WAIT 0x10 /* thread wait assertion */ @@ -433,6 +443,10 @@ extern void kdebug_reset(void); #define MACH_SCHED_LOAD 0x2d /* load update */ #define MACH_REC_CORES_FAILSAFE 0x2e /* recommended processor failsafe kicked in */ #define MACH_SCHED_QUANTUM_EXPIRED 0x2f /* thread quantum expired */ +#define MACH_EXEC_PROMOTE 0x30 /* Thread promoted by exec boost */ +#define MACH_EXEC_DEMOTE 0x31 /* Thread demoted from exec boost */ +#define MACH_AMP_SIGNAL_SPILL 0x32 /* AMP spill signal sent to cpuid */ +#define MACH_AMP_STEAL 0x33 /* AMP thread stolen or spilled */ /* Variants for MACH_MULTIQ_DEQUEUE */ #define MACH_MULTIQ_BOUND 1 @@ -465,6 +479,21 @@ extern void kdebug_reset(void); #define MACH_IPC_KMSG_INFO 0xa /* Send/Receive info for a kmsg */ #define MACH_IPC_KMSG_LINK 0xb /* link a kernel kmsg pointer to user mach_msg_header_t */ +/* Codes for thread groups (DBG_MACH_THREAD_GROUP) */ +#define MACH_THREAD_GROUP_NEW 0x0 +#define MACH_THREAD_GROUP_FREE 0x1 +#define MACH_THREAD_GROUP_SET 0x2 +#define MACH_THREAD_GROUP_NAME 0x3 +#define MACH_THREAD_GROUP_NAME_FREE 0x4 +#define MACH_THREAD_GROUP_FLAGS 0x5 + +/* Codes for coalitions (DBG_MACH_COALITION) */ +#define MACH_COALITION_NEW 0x0 +#define MACH_COALITION_FREE 0x1 +#define MACH_COALITION_ADOPT 0x2 +#define MACH_COALITION_REMOVE 0x3 +#define MACH_COALITION_THREAD_GROUP_SET 0x4 + /* Codes for pmap (DBG_MACH_PMAP) */ #define PMAP__CREATE 0x0 #define PMAP__DESTROY 0x1 @@ -477,23 +506,26 @@ extern void kdebug_reset(void); #define PMAP__FLUSH_TLBS 0x8 #define PMAP__UPDATE_INTERRUPT 0x9 #define PMAP__ATTRIBUTE_CLEAR 0xa -#define PMAP__REUSABLE 0xb +#define PMAP__REUSABLE 0xb /* This appears to be unused */ #define PMAP__QUERY_RESIDENT 0xc #define PMAP__FLUSH_KERN_TLBS 0xd #define PMAP__FLUSH_DELAYED_TLBS 0xe #define PMAP__FLUSH_TLBS_TO 0xf #define PMAP__FLUSH_EPT 0x10 +#define PMAP__FAST_FAULT 0x11 /* Codes for clock (DBG_MACH_CLOCK) */ #define MACH_EPOCH_CHANGE 0x0 /* wake epoch change */ - /* Codes for Stackshot/Microstackshot (DBG_MACH_STACKSHOT) */ #define MICROSTACKSHOT_RECORD 0x0 #define MICROSTACKSHOT_GATHER 0x1 -/* Codes for sysdiagnose */ -#define SYSDIAGNOSE_NOTIFY_USER 0x0 +/* Codes for sysdiagnose (DBG_MACH_SYSDIAGNOSE) */ +#define SYSDIAGNOSE_NOTIFY_USER 0x0 +#define SYSDIAGNOSE_FULL 0x1 +#define SYSDIAGNOSE_STACKSHOT 0x2 +#define SYSDIAGNOSE_TAILSPIN 0x3 /* Codes for Selective Forced Idle (DBG_MACH_SFI) */ #define SFI_SET_WINDOW 0x0 @@ -597,35 +629,37 @@ extern void kdebug_reset(void); #define DBG_IOGRAPHICS 50 /* Graphics */ #define DBG_HIBERNATE 51 /* hibernation related events */ #define DBG_IOTHUNDERBOLT 52 /* Thunderbolt */ - +#define DBG_BOOTER 53 /* booter related events */ /* Backwards compatibility */ #define DBG_IOPOINTING DBG_IOHID /* OBSOLETE: Use DBG_IOHID instead */ #define DBG_IODISK DBG_IOSTORAGE /* OBSOLETE: Use DBG_IOSTORAGE instead */ /* **** The Kernel Debug Sub Classes for Device Drivers (DBG_DRIVERS) **** */ -#define DBG_DRVSTORAGE 1 /* Storage layers */ -#define DBG_DRVNETWORK 2 /* Network layers */ -#define DBG_DRVKEYBOARD 3 /* Keyboard */ -#define DBG_DRVHID 4 /* HID Devices */ -#define DBG_DRVAUDIO 5 /* Audio */ -#define DBG_DRVSERIAL 7 /* Serial */ -#define DBG_DRVSAM 8 /* SCSI Architecture Model layers */ -#define DBG_DRVPARALLELATA 9 /* Parallel ATA */ -#define DBG_DRVPARALLELSCSI 10 /* Parallel SCSI */ -#define DBG_DRVSATA 11 /* Serial ATA */ -#define DBG_DRVSAS 12 /* SAS */ -#define DBG_DRVFIBRECHANNEL 13 /* FiberChannel */ -#define DBG_DRVUSB 14 /* USB */ -#define DBG_DRVBLUETOOTH 15 /* Bluetooth */ -#define DBG_DRVFIREWIRE 16 /* FireWire */ -#define DBG_DRVINFINIBAND 17 /* Infiniband */ -#define DBG_DRVGRAPHICS 18 /* Graphics */ -#define DBG_DRVSD 19 /* Secure Digital */ -#define DBG_DRVNAND 20 /* NAND drivers and layers */ -#define DBG_SSD 21 /* SSD */ -#define DBG_DRVSPI 22 /* SPI */ -#define DBG_DRVWLAN_802_11 23 /* WLAN 802.11 */ +#define DBG_DRVSTORAGE 1 /* Storage layers */ +#define DBG_DRVNETWORK 2 /* Network layers */ +#define DBG_DRVKEYBOARD 3 /* Keyboard */ +#define DBG_DRVHID 4 /* HID Devices */ +#define DBG_DRVAUDIO 5 /* Audio */ +#define DBG_DRVSERIAL 7 /* Serial */ +#define DBG_DRVSAM 8 /* SCSI Architecture Model layers */ +#define DBG_DRVPARALLELATA 9 /* Parallel ATA */ +#define DBG_DRVPARALLELSCSI 10 /* Parallel SCSI */ +#define DBG_DRVSATA 11 /* Serial ATA */ +#define DBG_DRVSAS 12 /* SAS */ +#define DBG_DRVFIBRECHANNEL 13 /* FiberChannel */ +#define DBG_DRVUSB 14 /* USB */ +#define DBG_DRVBLUETOOTH 15 /* Bluetooth */ +#define DBG_DRVFIREWIRE 16 /* FireWire */ +#define DBG_DRVINFINIBAND 17 /* Infiniband */ +#define DBG_DRVGRAPHICS 18 /* Graphics */ +#define DBG_DRVSD 19 /* Secure Digital */ +#define DBG_DRVNAND 20 /* NAND drivers and layers */ +#define DBG_SSD 21 /* SSD */ +#define DBG_DRVSPI 22 /* SPI */ +#define DBG_DRVWLAN_802_11 23 /* WLAN 802.11 */ +#define DBG_DRVSSM 24 /* System State Manager(AppleSSM) */ +#define DBG_DRVSMC 25 /* System Management Controller */ /* Backwards compatibility */ #define DBG_DRVPOINTING DBG_DRVHID /* OBSOLETE: Use DBG_DRVHID instead */ @@ -638,6 +672,7 @@ extern void kdebug_reset(void); #define DBG_DLIL_PR_FLT 4 /* DLIL Protocol Filter */ #define DBG_DLIL_IF_FLT 5 /* DLIL Interface FIlter */ + /* The Kernel Debug Sub Classes for File System (DBG_FSYSTEM) */ #define DBG_FSRW 0x1 /* reads and writes to the filesystem */ #define DBG_DKRW 0x2 /* reads and writes to the disk */ @@ -670,13 +705,13 @@ extern void kdebug_reset(void); /* The Kernel Debug Sub Classes for BSD */ #define DBG_BSD_PROC 0x01 /* process/signals related */ #define DBG_BSD_MEMSTAT 0x02 /* memorystatus / jetsam operations */ +#define DBG_BSD_KEVENT 0x03 /* kqueue / kevent related */ #define DBG_BSD_EXCP_SC 0x0C /* System Calls */ #define DBG_BSD_AIO 0x0D /* aio (POSIX async IO) */ #define DBG_BSD_SC_EXTENDED_INFO 0x0E /* System Calls, extended info */ #define DBG_BSD_SC_EXTENDED_INFO2 0x0F /* System Calls, extended info */ #define DBG_BSD_KDEBUG_TEST 0xFF /* for testing kdebug */ - /* The Codes for BSD subcode class DBG_BSD_PROC */ #define BSD_PROC_EXIT 1 /* process exit */ #define BSD_PROC_FRCEXIT 2 /* Kernel force termination */ @@ -701,36 +736,66 @@ extern void kdebug_reset(void); #define BSD_MEMSTAT_DO_KILL 13 /* memorystatus kills */ #endif /* PRIVATE */ +/* Codes for BSD subcode class DBG_BSD_KEVENT */ +#define BSD_KEVENT_KQ_PROCESS_BEGIN 1 +#define BSD_KEVENT_KQ_PROCESS_END 2 +#define BSD_KEVENT_KQWQ_PROCESS_BEGIN 3 +#define BSD_KEVENT_KQWQ_PROCESS_END 4 +#define BSD_KEVENT_KQWQ_BIND 5 +#define BSD_KEVENT_KQWQ_UNBIND 6 +#define BSD_KEVENT_KQWQ_THREQUEST 7 +#define BSD_KEVENT_KQWL_PROCESS_BEGIN 8 +#define BSD_KEVENT_KQWL_PROCESS_END 9 +#define BSD_KEVENT_KQWL_THREQUEST 10 +#define BSD_KEVENT_KQWL_THADJUST 11 +#define BSD_KEVENT_KQ_REGISTER 12 +#define BSD_KEVENT_KQWQ_REGISTER 13 +#define BSD_KEVENT_KQWL_REGISTER 14 +#define BSD_KEVENT_KNOTE_ACTIVATE 15 +#define BSD_KEVENT_KQ_PROCESS 16 +#define BSD_KEVENT_KQWQ_PROCESS 17 +#define BSD_KEVENT_KQWL_PROCESS 18 +#define BSD_KEVENT_KQWL_BIND 19 +#define BSD_KEVENT_KQWL_UNBIND 20 +#define BSD_KEVENT_KNOTE_ENABLE 21 + /* The Kernel Debug Sub Classes for DBG_TRACE */ #define DBG_TRACE_DATA 0 #define DBG_TRACE_STRING 1 #define DBG_TRACE_INFO 2 /* The Kernel Debug events: */ -#define TRACE_DATA_NEWTHREAD (TRACEDBG_CODE(DBG_TRACE_DATA, 1)) -#define TRACE_DATA_EXEC (TRACEDBG_CODE(DBG_TRACE_DATA, 2)) -#define TRACE_DATA_THREAD_TERMINATE (TRACEDBG_CODE(DBG_TRACE_DATA, 3)) -#define TRACE_DATA_THREAD_TERMINATE_PID (TRACEDBG_CODE(DBG_TRACE_DATA, 4)) -#define TRACE_STRING_GLOBAL (TRACEDBG_CODE(DBG_TRACE_STRING, 0)) -#define TRACE_STRING_NEWTHREAD (TRACEDBG_CODE(DBG_TRACE_STRING, 1)) -#define TRACE_STRING_EXEC (TRACEDBG_CODE(DBG_TRACE_STRING, 2)) -#define TRACE_STRING_PROC_EXIT (TRACEDBG_CODE(DBG_TRACE_STRING, 3)) -#define TRACE_STRING_THREADNAME (TRACEDBG_CODE(DBG_TRACE_STRING, 4)) -#define TRACE_STRING_THREADNAME_PREV (TRACEDBG_CODE(DBG_TRACE_STRING, 5)) -#define TRACE_PANIC (TRACEDBG_CODE(DBG_TRACE_INFO, 0)) -#define TRACE_TIMESTAMPS (TRACEDBG_CODE(DBG_TRACE_INFO, 1)) -#define TRACE_LOST_EVENTS (TRACEDBG_CODE(DBG_TRACE_INFO, 2)) -#define TRACE_WRITING_EVENTS (TRACEDBG_CODE(DBG_TRACE_INFO, 3)) -#define TRACE_INFO_STRING (TRACEDBG_CODE(DBG_TRACE_INFO, 4)) +#define TRACE_DATA_NEWTHREAD (TRACEDBG_CODE(DBG_TRACE_DATA, 1)) +#define TRACE_DATA_EXEC (TRACEDBG_CODE(DBG_TRACE_DATA, 2)) +#define TRACE_DATA_THREAD_TERMINATE (TRACEDBG_CODE(DBG_TRACE_DATA, 3)) +#define TRACE_DATA_THREAD_TERMINATE_PID (TRACEDBG_CODE(DBG_TRACE_DATA, 4)) +#define TRACE_STRING_GLOBAL (TRACEDBG_CODE(DBG_TRACE_STRING, 0)) +#define TRACE_STRING_NEWTHREAD (TRACEDBG_CODE(DBG_TRACE_STRING, 1)) +#define TRACE_STRING_EXEC (TRACEDBG_CODE(DBG_TRACE_STRING, 2)) +#define TRACE_STRING_PROC_EXIT (TRACEDBG_CODE(DBG_TRACE_STRING, 3)) +#define TRACE_STRING_THREADNAME (TRACEDBG_CODE(DBG_TRACE_STRING, 4)) +#define TRACE_STRING_THREADNAME_PREV (TRACEDBG_CODE(DBG_TRACE_STRING, 5)) +#define TRACE_PANIC (TRACEDBG_CODE(DBG_TRACE_INFO, 0)) +#define TRACE_TIMESTAMPS (TRACEDBG_CODE(DBG_TRACE_INFO, 1)) +#define TRACE_LOST_EVENTS (TRACEDBG_CODE(DBG_TRACE_INFO, 2)) +#define TRACE_WRITING_EVENTS (TRACEDBG_CODE(DBG_TRACE_INFO, 3)) +#define TRACE_INFO_STRING (TRACEDBG_CODE(DBG_TRACE_INFO, 4)) +#define TRACE_RETROGRADE_EVENTS (TRACEDBG_CODE(DBG_TRACE_INFO, 5)) /* The Kernel Debug Sub Classes for DBG_CORESTORAGE */ #define DBG_CS_IO 0 /* The Kernel Debug Sub Classes for DBG_SECURITY */ -#define DBG_SEC_KERNEL 0 /* raw entropy collected by the kernel */ +#define DBG_SEC_KERNEL 0 /* raw entropy collected by the kernel */ +#define DBG_SEC_SANDBOX 1 /* Sub-class codes for CoreGraphics (DBG_CG) are defined in its component. */ +/* The Kernel Debug Sub Classes for DBG_MONOTONIC */ +#define DBG_MT_INSTRS_CYCLES 1 +#define DBG_MT_TMPTH 0xfe +#define DBG_MT_TMPCPU 0xff + /* The Kernel Debug Sub Classes for DBG_MISC */ #define DBG_EVENT 0x10 #define DBG_BUFFER 0x20 @@ -782,6 +847,7 @@ extern void kdebug_reset(void); #define OPEN_THROTTLE_WINDOW 0x1 #define PROCESS_THROTTLED 0x2 #define IO_THROTTLE_DISABLE 0x3 +#define IO_TIER_UPL_MISMATCH 0x4 /* Subclasses for MACH Importance Policies (DBG_IMPORTANCE) */ @@ -796,6 +862,7 @@ extern void kdebug_reset(void); #define IMP_USYNCH_QOS_OVERRIDE 0x1A /* Userspace synchronization applied QoS override to resource owning thread */ #define IMP_DONOR_CHANGE 0x1B /* The iit_donor bit changed */ #define IMP_MAIN_THREAD_QOS 0x1C /* The task's main thread QoS was set */ +#define IMP_SYNC_IPC_QOS 0x1D /* Sync IPC QOS override */ /* DBG_IMPORTANCE subclasses 0x20 - 0x3F reserved for task policy flavors */ /* Codes for IMP_ASSERTION */ @@ -822,6 +889,12 @@ extern void kdebug_reset(void); #define IMP_DONOR_UPDATE_LIVE_DONOR_STATE 0x0 #define IMP_DONOR_INIT_DONOR_STATE 0x1 +/* Code for IMP_SYNC_IPC_QOS */ +#define IMP_SYNC_IPC_QOS_APPLIED 0x0 +#define IMP_SYNC_IPC_QOS_REMOVED 0x1 +#define IMP_SYNC_IPC_QOS_OVERFLOW 0x2 +#define IMP_SYNC_IPC_QOS_UNDERFLOW 0x3 + /* Subclasses for MACH Bank Voucher Attribute Manager (DBG_BANK) */ #define BANK_ACCOUNT_INFO 0x10 /* Trace points related to bank account struct */ #define BANK_TASK_INFO 0x11 /* Trace points related to bank task struct */ @@ -834,6 +907,7 @@ extern void kdebug_reset(void); /* Codes for BANK_ACCOUNT_INFO */ #define BANK_SETTLE_CPU_TIME 0x1 /* Bank ledger(chit) rolled up to tasks. */ #define BANK_SECURE_ORIGINATOR_CHANGED 0x2 /* Secure Originator changed. */ +#define BANK_SETTLE_ENERGY 0x3 /* Bank ledger(energy field) rolled up to tasks. */ /* Codes for ATM_SUBAID_INFO */ #define ATM_MIN_CALLED 0x1 @@ -848,7 +922,8 @@ extern void kdebug_reset(void); #define ATM_VALUE_DIFF_MAILBOX 0x2 /* Kernel Debug Sub Classes for daemons (DBG_DAEMON) */ -#define DBG_DAEMON_COREDUET 0x1 +#define DBG_DAEMON_COREDUET 0x1 +#define DBG_DAEMON_POWERD 0x2 /* Subclasses for the user space allocator */ #define DBG_UMALLOC_EXTERNAL 0x1 @@ -893,6 +968,7 @@ extern void kdebug_reset(void); /* Kernel Debug Macros for specific daemons */ #define COREDUETDBG_CODE(code) DAEMONDBG_CODE(DBG_DAEMON_COREDUET, code) +#define POWERDDBG_CODE(code) DAEMONDBG_CODE(DBG_DAEMON_POWERD, code) /* * To use kdebug in the kernel: @@ -1295,18 +1371,33 @@ extern void kernel_debug_disable(void); struct proc; -extern boolean_t kdebug_debugid_enabled(uint32_t debugid); -extern uint32_t kdebug_commpage_state(void); -extern void kdebug_lookup_gen_events(long *dbg_parms, int dbg_namelen, void *dp, boolean_t lookup); -extern void kdbg_trace_data(struct proc *proc, long *arg_pid); +/* + * Returns false if the debugid is disabled by filters, and true if the + * debugid is allowed to be traced. A debugid may not be traced if the + * typefilter disables its class and subclass, it's outside a range + * check, or if it's not an allowed debugid in a value check. Trace + * system events bypass this check. + */ +boolean_t kdebug_debugid_enabled(uint32_t debugid); + +/* + * Returns true only if the debugid is explicitly enabled by filters. Returns + * false otherwise, including when no filters are active. + */ +boolean_t kdebug_debugid_explicitly_enabled(uint32_t debugid); -extern void kdbg_trace_string(struct proc *proc, long *arg1, long *arg2, long *arg3, long *arg4); +uint32_t kdebug_commpage_state(void); +void kdebug_lookup_gen_events(long *dbg_parms, int dbg_namelen, void *dp, boolean_t lookup); +void kdbg_trace_data(struct proc *proc, long *arg_pid, long *arg_uniqueid); -extern void kdbg_dump_trace_to_file(const char *); -void kdebug_boot_trace(unsigned int n_events, char *filterdesc); -void kdebug_trace_start(unsigned int n_events, const char *filterdesc, boolean_t need_map); +void kdbg_trace_string(struct proc *proc, long *arg1, long *arg2, long *arg3, long *arg4); + +void kdbg_dump_trace_to_file(const char *); +void kdebug_init(unsigned int n_events, char *filterdesc); +void kdebug_trace_start(unsigned int n_events, const char *filterdesc, boolean_t at_wake); +void kdebug_free_early_buf(void); struct task; -extern void kdbg_get_task_name(char*, int, struct task *task); +void kdbg_get_task_name(char*, int, struct task *task); boolean_t disable_wrap(uint32_t *old_slowcheck, uint32_t *old_flags); void enable_wrap(uint32_t old_slowcheck, boolean_t lostevents); void release_storage_unit(int cpu, uint32_t storage_unit); @@ -1626,9 +1717,11 @@ int kdbg_write_v3_chunk_to_fd(uint32_t tag, uint32_t sub_tag, uint64_t length, v #define VFS_LOOKUP (FSDBG_CODE(DBG_FSRW,36)) #define VFS_LOOKUP_DONE (FSDBG_CODE(DBG_FSRW,39)) +#if !CONFIG_EMBEDDED #if defined(XNU_KERNEL_PRIVATE) && (DEVELOPMENT || DEBUG) #define KDEBUG_MOJO_TRACE 1 #endif +#endif #endif /* __APPLE_API_PRIVATE */ #endif /* PRIVATE */ diff --git a/bsd/sys/kern_control.h b/bsd/sys/kern_control.h index ba5f6be37..8e1b514bd 100644 --- a/bsd/sys/kern_control.h +++ b/bsd/sys/kern_control.h @@ -36,6 +36,9 @@ #include <sys/appleapiopts.h> +#include <sys/_types/_u_char.h> +#include <sys/_types/_u_int16_t.h> +#include <sys/_types/_u_int32_t.h> /* * Define Controller event subclass, and associated events. diff --git a/bsd/sys/kern_memorystatus.h b/bsd/sys/kern_memorystatus.h index b5850a7b9..52bce789c 100644 --- a/bsd/sys/kern_memorystatus.h +++ b/bsd/sys/kern_memorystatus.h @@ -33,6 +33,7 @@ #include <sys/time.h> #include <sys/proc.h> #include <sys/param.h> +#include <mach_debug/zone_info.h> #define MEMORYSTATUS_ENTITLEMENT "com.apple.private.memorystatus" @@ -109,6 +110,10 @@ typedef struct memorystatus_kernel_stats { uint64_t compressions; uint64_t decompressions; uint64_t total_uncompressed_pages_in_compressor; + uint64_t zone_map_size; + uint64_t zone_map_capacity; + uint64_t largest_zone_size; + char largest_zone_name[MACH_ZONE_NAME_MAX_LEN]; } memorystatus_kernel_stats_t; /* @@ -172,20 +177,14 @@ typedef struct memorystatus_freeze_entry { #define kMemorystatusSupportsIdleExit 0x10 #define kMemorystatusDirty 0x20 -/* Cause */ -enum { - kMemorystatusKilled = 1, - kMemorystatusKilledHiwat, - kMemorystatusKilledVnodes, - kMemorystatusKilledVMPageShortage, - kMemorystatusKilledVMThrashing, - kMemorystatusKilledFCThrashing, - kMemorystatusKilledPerProcessLimit, - kMemorystatusKilledDiagnostic, - kMemorystatusKilledIdleExit -}; - -/* Jetsam exit reason definitions */ +/* + * Jetsam exit reason definitions - related to memorystatus + * + * When adding new exit reasons also update: + * JETSAM_REASON_MEMORYSTATUS_MAX + * kMemorystatusKilled... Cause enum + * memorystatus_kill_cause_name[] + */ #define JETSAM_REASON_INVALID 0 #define JETSAM_REASON_GENERIC 1 #define JETSAM_REASON_MEMORY_HIGHWATER 2 @@ -196,12 +195,28 @@ enum { #define JETSAM_REASON_MEMORY_PERPROCESSLIMIT 7 #define JETSAM_REASON_MEMORY_DIAGNOSTIC 8 #define JETSAM_REASON_MEMORY_IDLE_EXIT 9 -#define JETSAM_REASON_CPULIMIT 10 +#define JETSAM_REASON_ZONE_MAP_EXHAUSTION 10 + +#define JETSAM_REASON_MEMORYSTATUS_MAX JETSAM_REASON_ZONE_MAP_EXHAUSTION + +/* + * Jetsam exit reason definitions - not related to memorystatus + */ +#define JETSAM_REASON_CPULIMIT 100 -/* Temporary, to prevent the need for a linked submission of ReportCrash */ -/* Remove when <rdar://problem/13210532> has been integrated */ +/* Cause */ enum { - kMemorystatusKilledVM = kMemorystatusKilledVMPageShortage + kMemorystatusInvalid = JETSAM_REASON_INVALID, + kMemorystatusKilled = JETSAM_REASON_GENERIC, + kMemorystatusKilledHiwat = JETSAM_REASON_MEMORY_HIGHWATER, + kMemorystatusKilledVnodes = JETSAM_REASON_VNODE, + kMemorystatusKilledVMPageShortage = JETSAM_REASON_MEMORY_VMPAGESHORTAGE, + kMemorystatusKilledVMThrashing = JETSAM_REASON_MEMORY_VMTHRASHING, + kMemorystatusKilledFCThrashing = JETSAM_REASON_MEMORY_FCTHRASHING, + kMemorystatusKilledPerProcessLimit = JETSAM_REASON_MEMORY_PERPROCESSLIMIT, + kMemorystatusKilledDiagnostic = JETSAM_REASON_MEMORY_DIAGNOSTIC, + kMemorystatusKilledIdleExit = JETSAM_REASON_MEMORY_IDLE_EXIT, + kMemorystatusKilledZoneMapExhaustion = JETSAM_REASON_ZONE_MAP_EXHAUSTION }; /* Memorystatus control */ @@ -317,12 +332,6 @@ typedef struct memorystatus_memlimit_properties { * Non-fatal limit types are the * - high-water-mark limit * - * P_MEMSTAT_MEMLIMIT_BACKGROUND is translated in posix_spawn as - * the fatal system_wide task limit when active - * non-fatal inactive limit based on limit provided. - * This is necessary for backward compatibility until the - * the flag can be considered obsolete. - * * Processes that opt into dirty tracking are evaluated * based on clean vs dirty state. * dirty ==> active @@ -350,7 +359,7 @@ typedef struct memorystatus_memlimit_properties { #define P_MEMSTAT_FOREGROUND 0x00000100 #define P_MEMSTAT_DIAG_SUSPENDED 0x00000200 #define P_MEMSTAT_PRIOR_THAW 0x00000400 -#define P_MEMSTAT_MEMLIMIT_BACKGROUND 0x00000800 /* Task has a memory limit for when it's in the background. Used for a process' "high water mark".*/ +/* unused 0x00000800 */ #define P_MEMSTAT_INTERNAL 0x00001000 #define P_MEMSTAT_FATAL_MEMLIMIT 0x00002000 /* current fatal state of the process's memlimit */ #define P_MEMSTAT_MEMLIMIT_ACTIVE_FATAL 0x00004000 /* if set, exceeding limit is fatal when the process is active */ @@ -365,7 +374,7 @@ extern void memorystatus_init_at_boot_snapshot(void); extern int memorystatus_add(proc_t p, boolean_t locked); extern int memorystatus_update(proc_t p, int priority, uint64_t user_data, boolean_t effective, boolean_t update_memlimit, int32_t memlimit_active, boolean_t memlimit_active_is_fatal, - int32_t memlimit_inactive, boolean_t memlimit_inactive_is_fatal, boolean_t memlimit_background); + int32_t memlimit_inactive, boolean_t memlimit_inactive_is_fatal); extern int memorystatus_remove(proc_t p, boolean_t locked); @@ -399,10 +408,10 @@ void proc_memstat_terminated(proc_t p, boolean_t set); boolean_t memorystatus_proc_is_dirty_unsafe(void *v); #endif /* CONFIG_MEMORYSTATUS */ -#if CONFIG_JETSAM - int memorystatus_get_pressure_status_kdp(void); +#if CONFIG_JETSAM + typedef enum memorystatus_policy { kPolicyDefault = 0x0, kPolicyMoreFree = 0x1, @@ -415,19 +424,19 @@ extern int memorystatus_jetsam_wakeup; extern unsigned int memorystatus_jetsam_running; boolean_t memorystatus_kill_on_VM_page_shortage(boolean_t async); -boolean_t memorystatus_kill_on_VM_thrashing(boolean_t async); boolean_t memorystatus_kill_on_FC_thrashing(boolean_t async); boolean_t memorystatus_kill_on_vnode_limit(void); void jetsam_on_ledger_cpulimit_exceeded(void); -void memorystatus_pages_update(unsigned int pages_avail); +#endif /* CONFIG_JETSAM */ -#else /* CONFIG_JETSAM */ +boolean_t memorystatus_kill_on_zone_map_exhaustion(pid_t pid); +boolean_t memorystatus_kill_on_VM_thrashing(boolean_t async); +void memorystatus_pages_update(unsigned int pages_avail); boolean_t memorystatus_idle_exit_from_VM(void); -#endif /* !CONFIG_JETSAM */ #ifdef CONFIG_FREEZE diff --git a/bsd/sys/kpi_mbuf.h b/bsd/sys/kpi_mbuf.h index 6b5693c00..d877f0974 100644 --- a/bsd/sys/kpi_mbuf.h +++ b/bsd/sys/kpi_mbuf.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Apple Inc. All rights reserved. + * Copyright (c) 2008-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -166,6 +166,7 @@ enum { #ifdef KERNEL_PRIVATE MBUF_CSUM_PARTIAL = 0x1000, /* 16-bit 1's complement sum */ MBUF_CSUM_REQ_SUM16 = MBUF_CSUM_PARTIAL, + MBUF_CSUM_REQ_ZERO_INVERT = 0x2000, #endif /* KERNEL_PRIVATE */ MBUF_CSUM_REQ_IP = 0x0001, MBUF_CSUM_REQ_TCP = 0x0002, @@ -283,7 +284,7 @@ __BEGIN_DECLS than one mbuf. In addition, data that is virtually contiguous might not be represented by physically contiguous pages; see further comments in mbuf_data_to_physical. Use mbuf_len to - determine the lenght of data available in this mbuf. If a data + determine the length of data available in this mbuf. If a data structure you want to access stradles two mbufs in a chain, either use mbuf_pullup to get the data contiguous in one mbuf or copy the pieces of data from each mbuf in to a contiguous @@ -953,8 +954,7 @@ extern errno_t mbuf_copy_pkthdr(mbuf_t dest, const mbuf_t src); /*! @function mbuf_pkthdr_len @discussion Returns the length as reported by the packet header. - @param mbuf The mbuf containing the packet header with the length to - be changed. + @param mbuf The mbuf containing the packet header @result The length, in bytes, of the packet. */ extern size_t mbuf_pkthdr_len(const mbuf_t mbuf); diff --git a/bsd/sys/kpi_socket.h b/bsd/sys/kpi_socket.h index 6045af624..837611b5c 100644 --- a/bsd/sys/kpi_socket.h +++ b/bsd/sys/kpi_socket.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Apple Inc. All rights reserved. + * Copyright (c) 2008-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -105,8 +105,17 @@ typedef void (*sock_evupcall)(socket_t so, void *cookie, u_int32_t event); socket for tracking the connection. @result 0 on success otherwise the errno error. */ +#ifdef KERNEL_PRIVATE +extern errno_t sock_accept_internal(socket_t so, struct sockaddr *from, int fromlen, + int flags, sock_upcall callback, void *cookie, socket_t *new_so); + +#define sock_accept(so, from, fromlen, flags, callback, cookie, new_so) \ + sock_accept_internal((so), (from), (fromlen), (flags), (callback), \ + (cookie), (new_so)) +#else extern errno_t sock_accept(socket_t so, struct sockaddr *from, int fromlen, int flags, sock_upcall callback, void *cookie, socket_t *new_so); +#endif /* KERNEL_PRIVATE */ /*! @function sock_bind @@ -364,8 +373,17 @@ extern errno_t sock_shutdown(socket_t so, int how); @param new_so Upon success, a reference to the new socket. @result 0 on success otherwise the errno error. */ +#ifdef KERNEL_PRIVATE +extern errno_t sock_socket_internal(int domain, int type, int protocol, + sock_upcall callback, void *cookie, socket_t *new_so); + +#define sock_socket(domain, type, protocol, callback, cookie, new_so) \ + sock_socket_internal((domain), (type), (protocol), \ + (callback), (cookie), (new_so)) +#else extern errno_t sock_socket(int domain, int type, int protocol, sock_upcall callback, void *cookie, socket_t *new_so); +#endif /* KERNEL_PRIVATE */ /*! @function sock_close @@ -537,6 +555,17 @@ extern errno_t sock_setupcall(socket_t sock, sock_upcall callback, extern errno_t sock_setupcalls(socket_t sock, sock_upcall read_callback, void *read_context, sock_upcall write_callback, void *write_context); +/* + @function sock_setupcalls_locked + @discussion The locked version of sock_setupcalls + @param locked: When sets, indicates that the callbacks expect to be + on a locked socket. Thus, no unlock is done prior to + calling the callback. + */ +extern void sock_setupcalls_locked(socket_t sock, + sock_upcall rcallback, void *rcontext, + sock_upcall wcallback, void *wcontext, int locked); + /* @function sock_catchevents @discussion Set the notifier function to be called when an event @@ -551,6 +580,11 @@ extern errno_t sock_setupcalls(socket_t sock, sock_upcall read_callback, */ extern errno_t sock_catchevents(socket_t sock, sock_evupcall event_callback, void *event_context, u_int32_t event_mask); + +extern void sock_catchevents_locked(socket_t sock, sock_evupcall ecallback, + void *econtext, u_int32_t emask); + + /* @function sock_iskernel @discussion Returns true if the socket was created by the kernel or diff --git a/bsd/sys/kpi_socketfilter.h b/bsd/sys/kpi_socketfilter.h index bb82c5439..d3ac71b96 100644 --- a/bsd/sys/kpi_socketfilter.h +++ b/bsd/sys/kpi_socketfilter.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2008-2017 Apple Computer, Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -569,8 +569,16 @@ struct sflt_filter { @param protocol The protocol these filters will be attached to. @result 0 on success otherwise the errno error. */ +#ifdef KERNEL_PRIVATE +extern errno_t sflt_register_internal(const struct sflt_filter *filter, + int domain, int type, int protocol); + +#define sflt_register(filter, domain, type, protocol) \ + sflt_register_internal((filter), (domain), (type), (protocol)) +#else extern errno_t sflt_register(const struct sflt_filter *filter, int domain, int type, int protocol); +#endif /* KERNEL_PRIVATE */ /*! @function sflt_unregister diff --git a/bsd/sys/ktrace.h b/bsd/sys/ktrace.h index c67c9f6d1..735dc82c3 100644 --- a/bsd/sys/ktrace.h +++ b/bsd/sys/ktrace.h @@ -43,7 +43,11 @@ enum ktrace_state { KTRACE_STATE_BG }; -extern lck_mtx_t *ktrace_lock; +void ktrace_lock(void); +void ktrace_unlock(void); +void ktrace_assert_lock_held(void); +void ktrace_start_single_threaded(void); +void ktrace_end_single_threaded(void); /* * Subsystems that use ktrace to manage ownership. These values are passed as @@ -125,7 +129,7 @@ bool ktrace_background_active(void); * These functions exist for the transition for kperf to allow blessing other * processes. They should not be used by other clients. */ -extern boolean_t ktrace_keep_ownership_on_reset; +extern bool ktrace_keep_ownership_on_reset; extern int ktrace_root_set_owner_allowed; int ktrace_set_owning_pid(int pid); diff --git a/bsd/sys/lctx.h b/bsd/sys/lctx.h index 554176c4a..e5205c632 100644 --- a/bsd/sys/lctx.h +++ b/bsd/sys/lctx.h @@ -5,6 +5,8 @@ #define _SYS_LCTX_H_ #ifndef KERNEL +#include <sys/errno.h> /* errno, ENOSYS */ +#include <sys/_types/_pid_t.h> /* pid_t */ static __inline pid_t getlcid(pid_t pid) { diff --git a/bsd/sys/linker_set.h b/bsd/sys/linker_set.h index 1aea00848..8fd29dbb9 100644 --- a/bsd/sys/linker_set.h +++ b/bsd/sys/linker_set.h @@ -88,6 +88,12 @@ * * void const * __set_SET_sym_SYM __attribute__((section("__DATA,SET"))) = & SYM */ + +/* Wrap entries in a type that can be blacklisted from KASAN */ +struct linker_set_entry { + void *ptr; +} __attribute__((packed)); + #ifdef __LS_VA_STRINGIFY__ # undef __LS_VA_STRINGIFY__ #endif @@ -97,8 +103,8 @@ #define __LS_VA_STRINGIFY(_x...) #_x #define __LS_VA_STRCONCAT(_x,_y) __LS_VA_STRINGIFY(_x,_y) #define __LINKER_MAKE_SET(_set, _sym) \ - /*__unused*/ /*static*/ void const * /*const*/ __set_##_set##_sym_##_sym \ - __attribute__ ((section(__LS_VA_STRCONCAT(__DATA,_set)),used)) = (void *)&_sym + /*__unused*/ /*static*/ const struct linker_set_entry /*const*/ __set_##_set##_sym_##_sym \ + __attribute__ ((section(__LS_VA_STRCONCAT(__DATA,_set)),used)) = { (void *)&_sym } /* the line above is very fragile - if your compiler breaks linker sets, just play around with "static", "const", "used" etc. :-) */ diff --git a/bsd/sys/malloc.h b/bsd/sys/malloc.h index 27a8c2d04..0dd7117f5 100644 --- a/bsd/sys/malloc.h +++ b/bsd/sys/malloc.h @@ -217,8 +217,11 @@ #define M_FD_VN_DATA 122 /* Per fd vnode data */ #define M_FD_DIRBUF 123 /* Directory entries' buffer */ #define M_NETAGENT 124 /* Network Agents */ +#define M_EVENTHANDLER 125 /* Eventhandler */ +#define M_LLTABLE 126 /* Link layer table */ +#define M_NWKWQ 127 /* Network work queue */ -#define M_LAST 125 /* Must be last type + 1 */ +#define M_LAST 128 /* Must be last type + 1 */ #else /* BSD_KERNEL_PRIVATE */ @@ -263,22 +266,23 @@ extern struct kmemstats kmemstats[]; #include <mach/vm_types.h> -#define MALLOC(space, cast, size, type, flags) \ - ({ static vm_allocation_site_t site __attribute__((section("__DATA, __data"))); \ +#define MALLOC(space, cast, size, type, flags) \ + ({ VM_ALLOC_SITE_STATIC(0, 0); \ (space) = (cast)__MALLOC(size, type, flags, &site); }) -#define REALLOC(space, cast, addr, size, type, flags) \ - ({ static vm_allocation_site_t site __attribute__((section("__DATA, __data"))); \ + +#define REALLOC(space, cast, addr, size, type, flags) \ + ({ VM_ALLOC_SITE_STATIC(0, 0); \ (space) = (cast)__REALLOC(addr, size, type, flags, &site); }) -#define _MALLOC(size, type, flags) \ - ({ static vm_allocation_site_t site __attribute__((section("__DATA, __data"))); \ +#define _MALLOC(size, type, flags) \ + ({ VM_ALLOC_SITE_STATIC(0, 0); \ __MALLOC(size, type, flags, &site); }) -#define _REALLOC(addr, size, type, flags) \ - ({ static vm_allocation_site_t site __attribute__((section("__DATA, __data"))); \ +#define _REALLOC(addr, size, type, flags) \ + ({ VM_ALLOC_SITE_STATIC(0, 0); \ __REALLOC(addr, size, type, flags, &site); }) -#define _MALLOC_ZONE(size, type, flags) \ - ({ static vm_allocation_site_t site __attribute__((section("__DATA, __data"))); \ +#define _MALLOC_ZONE(size, type, flags) \ + ({ VM_ALLOC_SITE_STATIC(0, 0); \ __MALLOC_ZONE(size, type, flags, &site); }) #define FREE(addr, type) \ @@ -294,7 +298,7 @@ extern void *__MALLOC( size_t size, int type, int flags, - vm_allocation_site_t *site); + vm_allocation_site_t *site) __attribute__((alloc_size(1))); extern void _FREE( void *addr, @@ -305,7 +309,7 @@ extern void *__REALLOC( size_t size, int type, int flags, - vm_allocation_site_t *site); + vm_allocation_site_t *site) __attribute__((alloc_size(2))); extern void *__MALLOC_ZONE( size_t size, diff --git a/bsd/sys/mbuf.h b/bsd/sys/mbuf.h index 6b585f2b9..828f9f44d 100644 --- a/bsd/sys/mbuf.h +++ b/bsd/sys/mbuf.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999-2015 Apple Inc. All rights reserved. + * Copyright (c) 1999-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -81,6 +81,9 @@ #include <sys/cdefs.h> #include <sys/appleapiopts.h> +#include <sys/_types/_u_int32_t.h> /* u_int32_t */ +#include <sys/_types/_u_int64_t.h> /* u_int64_t */ +#include <sys/_types/_u_short.h> /* u_short */ #ifdef XNU_KERNEL_PRIVATE @@ -127,9 +130,11 @@ /* * Macros for type conversion * mtod(m,t) - convert mbuf pointer to data pointer of correct type + * mtodo(m, o) -- Same as above but with offset 'o' into data. * dtom(x) - convert data pointer within mbuf to mbuf pointer (XXX) */ #define mtod(m, t) ((t)m_mtod(m)) +#define mtodo(m, o) ((void *)(mtod(m, uint8_t *) + (o))) #define dtom(x) m_dtom(x) /* header at beginning of each mbuf: */ @@ -140,7 +145,18 @@ struct m_hdr { int32_t mh_len; /* amount of data in this mbuf */ u_int16_t mh_type; /* type of data in this mbuf */ u_int16_t mh_flags; /* flags; see below */ +#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4) +/* This is needed because of how _MLEN is defined and used. Ideally, _MLEN + * should be defined using the offsetof(struct mbuf, M_dat), since there is + * no guarantee that mbuf.M_dat will start where mbuf.m_hdr ends. The compiler + * may (and does in the armv7k case) insert padding between m_hdr and M_dat in + * mbuf. We cannot easily use offsetof, however, since _MLEN is referenced + * in the definition of mbuf. + */ +} __attribute__((aligned(8))); +#else }; +#endif /* * Packet tag structure (see below for details). @@ -199,9 +215,6 @@ struct pf_mtag { u_int16_t pftag_rtableid; /* alternate routing table id */ u_int16_t pftag_tag; u_int16_t pftag_routed; -#if PF_ALTQ - u_int32_t pftag_qid; -#endif /* PF_ALTQ */ #if PF_ECN void *pftag_hdr; /* saved hdr pos in mbuf, for ECN */ #endif /* PF_ECN */ @@ -241,10 +254,12 @@ struct tcp_pktinfo { struct mptcp_pktinfo { u_int64_t mtpi_dsn; /* MPTCP Data Sequence Number */ u_int32_t mtpi_rel_seq; /* Relative Seq Number */ - u_int32_t mtpi_length; /* Length of mapping */ + u_int16_t mtpi_length; /* Length of mapping */ + u_int16_t mtpi_csum; #define mp_dsn proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_dsn #define mp_rseq proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_rel_seq #define mp_rlen proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_length +#define mp_csum proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_csum }; /* @@ -390,9 +405,6 @@ struct pkthdr { #define bufstatus_if _pkt_bsr.if_data #define bufstatus_sndbuf _pkt_bsr.sndbuf_data }; -#if MEASURE_BW - u_int64_t pkt_bwseq; /* sequence # */ -#endif /* MEASURE_BW */ u_int64_t pkt_timestamp; /* enqueue time */ /* @@ -403,7 +415,8 @@ struct pkthdr { /* * Module private scratch space (32-bit aligned), currently 16-bytes * large. Anything stored here is not guaranteed to survive across - * modules. + * modules. The AQM layer (outbound) uses all 16-bytes for both + * packet scheduling and flow advisory information. */ struct { union { @@ -419,6 +432,11 @@ struct pkthdr { u_int64_t __mpriv64[2]; } __mpriv_u; } pkt_mpriv __attribute__((aligned(4))); +#define pkt_mpriv_hash pkt_mpriv.__mpriv_u.__mpriv32[0].__mpriv32_u.__val32 +#define pkt_mpriv_flags pkt_mpriv.__mpriv_u.__mpriv32[1].__mpriv32_u.__val32 +#define pkt_mpriv_srcid pkt_mpriv.__mpriv_u.__mpriv32[2].__mpriv32_u.__val32 +#define pkt_mpriv_fidx pkt_mpriv.__mpriv_u.__mpriv32[3].__mpriv32_u.__val32 + u_int32_t redzone; /* red zone */ u_int32_t pkt_compl_callbacks; /* Packet completion callbacks */ }; @@ -480,11 +498,12 @@ struct pkthdr { #define PKTF_TCP_REXMT 0x200000 /* packet is TCP retransmission */ #define PKTF_REASSEMBLED 0x400000 /* Packet was reassembled */ #define PKTF_TX_COMPL_TS_REQ 0x800000 /* tx completion timestamp requested */ -#define PKTF_DRV_TS_VALID 0x1000000 /* driver timestamp is valid */ +#define PKTF_TS_VALID 0x1000000 /* pkt timestamp is valid */ #define PKTF_DRIVER_MTAG 0x2000000 /* driver mbuf tags fields inited */ #define PKTF_NEW_FLOW 0x4000000 /* Data from a new flow */ #define PKTF_START_SEQ 0x8000000 /* valid start sequence */ #define PKTF_LAST_PKT 0x10000000 /* last packet in the flow */ +#define PKTF_MPTCP_REINJ 0x20000000 /* Packet has been reinjected for MPTCP */ /* flags related to flow control/advisory and identification */ #define PKTF_FLOW_MASK \ @@ -591,6 +610,7 @@ struct mbuf { #define CSUM_DATA_VALID 0x0400 /* csum_data field is valid */ #define CSUM_PSEUDO_HDR 0x0800 /* csum_data has pseudo hdr */ #define CSUM_PARTIAL 0x1000 /* simple Sum16 computation */ +#define CSUM_ZERO_INVERT 0x2000 /* invert 0 to -0 (0xffff) */ #define CSUM_DELAY_DATA (CSUM_TCP | CSUM_UDP) #define CSUM_DELAY_IP (CSUM_IP) /* IPv4 only: no IPv6 IP cksum */ @@ -599,7 +619,7 @@ struct mbuf { #define CSUM_TX_FLAGS \ (CSUM_DELAY_IP | CSUM_DELAY_DATA | CSUM_DELAY_IPV6_DATA | \ - CSUM_DATA_VALID | CSUM_PARTIAL) + CSUM_DATA_VALID | CSUM_PARTIAL | CSUM_ZERO_INVERT) #define CSUM_RX_FLAGS \ (CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_PSEUDO_HDR | \ @@ -1454,10 +1474,11 @@ __private_extern__ mbuf_traffic_class_t m_get_traffic_class(struct mbuf *); } while (0) __private_extern__ u_int16_t m_adj_sum16(struct mbuf *, u_int32_t, - u_int32_t, u_int32_t); + u_int32_t, u_int32_t, u_int32_t); __private_extern__ u_int16_t m_sum16(struct mbuf *, u_int32_t, u_int32_t); -__private_extern__ void m_set_ext(struct mbuf *, struct ext_ref *, m_ext_free_func_t, caddr_t); +__private_extern__ void m_set_ext(struct mbuf *, struct ext_ref *, + m_ext_free_func_t, caddr_t); __private_extern__ struct ext_ref *m_get_rfa(struct mbuf *); __private_extern__ m_ext_free_func_t m_get_ext_free(struct mbuf *); __private_extern__ caddr_t m_get_ext_arg(struct mbuf *); diff --git a/bsd/sys/mcache.h b/bsd/sys/mcache.h index 906363384..9bf6ed529 100644 --- a/bsd/sys/mcache.h +++ b/bsd/sys/mcache.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2006-2015 Apple Inc. All rights reserved. + * Copyright (c) 2006-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -50,11 +50,11 @@ extern "C" { #endif /* - * Unlike VERIFY(), ASSERT() is evaluated only in DEBUG build. + * Unlike VERIFY(), ASSERT() is evaluated only in DEBUG/DEVELOPMENT build. */ #define VERIFY(EX) \ ((void)(__probable((EX)) || assfail(#EX, __FILE__, __LINE__))) -#if DEBUG +#if (DEBUG || DEVELOPMENT) #define ASSERT(EX) VERIFY(EX) #else #define ASSERT(EX) ((void)0) @@ -147,6 +147,9 @@ extern "C" { #define atomic_bitset_32(a, n) \ atomic_or_32(a, n) +#define atomic_bitset_32_ov(a, n) \ + atomic_or_32_ov(a, n) + #define atomic_and_8_ov(a, n) \ ((u_int8_t) OSBitAndAtomic8(n, (volatile UInt8 *)a)) @@ -198,6 +201,11 @@ extern "C" { (((uintptr_t)(x)) & ~((uintptr_t)(align) - 1)) #endif /* P2ROUNDDOWN */ +#ifndef P2ALIGN +#define P2ALIGN(x, align) \ + ((uintptr_t)(x) & -((uintptr_t)(align))) +#endif /* P2ALIGN */ + #define MCACHE_FREE_PATTERN 0xdeadbeefdeadbeefULL #define MCACHE_UNINITIALIZED_PATTERN 0xbaddcafebaddcafeULL @@ -382,6 +390,7 @@ __private_extern__ unsigned int mcache_alloc_ext(mcache_t *, mcache_obj_t **, unsigned int, int); __private_extern__ void mcache_free_ext(mcache_t *, mcache_obj_t *); __private_extern__ void mcache_reap(void); +__private_extern__ void mcache_reap_now(mcache_t *, boolean_t); __private_extern__ boolean_t mcache_purge_cache(mcache_t *, boolean_t); __private_extern__ void mcache_waiter_inc(mcache_t *); __private_extern__ void mcache_waiter_dec(mcache_t *); diff --git a/bsd/sys/mman.h b/bsd/sys/mman.h index 06c76abf5..8aba6441f 100644 --- a/bsd/sys/mman.h +++ b/bsd/sys/mman.h @@ -202,6 +202,9 @@ #define MINCORE_MODIFIED 0x4 /* Page has been modified by us */ #define MINCORE_REFERENCED_OTHER 0x8 /* Page has been referenced */ #define MINCORE_MODIFIED_OTHER 0x10 /* Page has been modified */ +#define MINCORE_PAGED_OUT 0x20 /* Page has been paged out */ +#define MINCORE_COPIED 0x40 /* Page has been copied */ +#define MINCORE_ANONYMOUS 0x80 /* Page belongs to an anonymous object */ #endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */ diff --git a/bsd/sys/monotonic.h b/bsd/sys/monotonic.h new file mode 100644 index 000000000..883b6a0ad --- /dev/null +++ b/bsd/sys/monotonic.h @@ -0,0 +1,149 @@ +#ifndef SYS_MONOTONIC_H +#define SYS_MONOTONIC_H + +#include <stdbool.h> +#include <stdint.h> +#include <sys/cdefs.h> +#include <sys/ioccom.h> + +__BEGIN_DECLS + +/* + * XXX These declarations are subject to change at any time. + */ + +struct monotonic_config { + uint64_t event; + uint64_t allowed_ctr_mask; +}; + +union monotonic_ctl_add { + struct { + struct monotonic_config config; + } in; + + struct { + uint32_t ctr; + } out; +}; + +union monotonic_ctl_enable { + struct { + bool enable; + } in; +}; + +union monotonic_ctl_counts { + struct { + uint64_t ctr_mask; + } in; + + struct { + uint64_t counts[1]; + } out; +}; + +#define MT_IOC(x) _IO('m', (x)) + +/* + * FIXME + * + * - Consider a separate IOC for disable -- to avoid the copyin to determine which way to set it. + * + * - Maybe IOC_COUNTS should just return all the enable counters' counts. + */ +enum monotonic_ioc { + MT_IOC_RESET = MT_IOC(0), + MT_IOC_ADD = MT_IOC(1), + MT_IOC_ENABLE = MT_IOC(2), + MT_IOC_COUNTS = MT_IOC(3), +}; + +#undef MT_IOC + +#if XNU_KERNEL_PRIVATE + +#include <kern/monotonic.h> +#include <machine/monotonic.h> +#include <sys/kdebug.h> +#include <kern/locks.h> + +#ifdef MT_CORE_INSTRS +#define COUNTS_INSTRS __counts[MT_CORE_INSTRS] +#else /* defined(MT_CORE_INSTRS) */ +#define COUNTS_INSTRS 0 +#endif /* !defined(MT_CORE_INSTRS) */ + +/* + * MT_KDBG_TMP* macros are meant for temporary (i.e. not checked-in) + * performance investigations. + */ + +/* + * Record the current CPU counters. + * + * Preemption must be disabled. + */ +#define MT_KDBG_TMPCPU_EVT(CODE) \ + KDBG_EVENTID(DBG_MONOTONIC, DBG_MT_TMPCPU, CODE) + +#define MT_KDBG_TMPCPU_(CODE, FUNC) \ + do { \ + if (kdebug_enable && \ + kdebug_debugid_enabled(MT_KDBG_TMPCPU_EVT(CODE))) { \ + uint64_t __counts[MT_CORE_NFIXED]; \ + mt_fixed_counts(__counts); \ + KDBG(MT_KDBG_TMPCPU_EVT(CODE) | (FUNC), COUNTS_INSTRS, \ + __counts[MT_CORE_CYCLES]); \ + } \ + } while (0) + +#define MT_KDBG_TMPCPU(CODE) MT_KDBG_TMPCPU_(CODE, DBG_FUNC_NONE) +#define MT_KDBG_TMPCPU_START(CODE) MT_KDBG_TMPCPU_(CODE, DBG_FUNC_START) +#define MT_KDBG_TMPCPU_END(CODE) MT_KDBG_TMPCPU_(CODE, DBG_FUNC_END) + +/* + * Record the current thread counters. + * + * Interrupts must be disabled. + */ +#define MT_KDBG_TMPTH_EVT(CODE) \ + KDBG_EVENTID(DBG_MONOTONIC, DBG_MT_TMPTH, CODE) + +#define MT_KDBG_TMPTH_(CODE, FUNC) \ + do { \ + if (kdebug_enable && \ + kdebug_debugid_enabled(MT_KDBG_TMPTH_EVT(CODE))) { \ + uint64_t __counts[MT_CORE_NFIXED]; \ + mt_cur_thread_fixed_counts(__counts); \ + KDBG(MT_KDBG_TMPTH_EVT(CODE) | (FUNC), COUNTS_INSTRS, \ + __counts[MT_CORE_CYCLES]); \ + } \ + } while (0) + +#define MT_KDBG_TMPTH(CODE) MT_KDBG_TMPTH_(CODE, DBG_FUNC_NONE) +#define MT_KDBG_TMPTH_START(CODE) MT_KDBG_TMPTH_(CODE, DBG_FUNC_START) +#define MT_KDBG_TMPTH_END(CODE) MT_KDBG_TMPTH_(CODE, DBG_FUNC_END) + +/* maybe provider, bank, group, set, unit, pmu */ + +struct monotonic_dev { + const char *mtd_name; + int (*mtd_init)(void); + int (*mtd_add)(struct monotonic_config *config, uint32_t *ctr_out); + void (*mtd_reset)(void); + void (*mtd_enable)(bool enable); + int (*mtd_read)(uint64_t ctr_mask, uint64_t *counts_out); +}; + +extern const struct monotonic_dev monotonic_devs[]; + +extern lck_grp_t *mt_lock_grp; + +int mt_dev_init(void); + +#endif /* XNU_KERNEL_PRIVATE */ + +__END_DECLS + +#endif /* !defined(SYS_MONOTONIC_H) */ diff --git a/bsd/sys/mount.h b/bsd/sys/mount.h index 70db71af6..8ee2cc256 100644 --- a/bsd/sys/mount.h +++ b/bsd/sys/mount.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -320,7 +320,8 @@ struct vfs_attr { #define MNT_NOUSERXATTR 0x01000000 /* Don't allow user extended attributes */ #define MNT_DEFWRITE 0x02000000 /* filesystem should defer writes */ #define MNT_MULTILABEL 0x04000000 /* MAC support for individual labels */ -#define MNT_NOATIME 0x10000000 /* disable update of file access time */ +#define MNT_NOATIME 0x10000000 /* disable update of file access time */ +#define MNT_SNAPSHOT 0x40000000 /* The mount is a snapshot */ #ifdef BSD_KERNEL_PRIVATE /* #define MNT_IMGSRC_BY_INDEX 0x20000000 see sys/imgsrc.h */ #endif /* BSD_KERNEL_PRIVATE */ @@ -340,7 +341,7 @@ struct vfs_attr { MNT_ROOTFS | MNT_DOVOLFS | MNT_DONTBROWSE | \ MNT_IGNORE_OWNERSHIP | MNT_AUTOMOUNTED | MNT_JOURNALED | \ MNT_NOUSERXATTR | MNT_DEFWRITE | MNT_MULTILABEL | \ - MNT_NOATIME | MNT_CPROTECT) + MNT_NOATIME | MNT_SNAPSHOT | MNT_CPROTECT) /* * External filesystem command modifier flags. * Unmount can use the MNT_FORCE flag. @@ -760,19 +761,16 @@ struct fs_snapshot_mount_args { }; #define VFSIOC_MOUNT_SNAPSHOT _IOW('V', 1, struct fs_snapshot_mount_args) -#define VFSCTL_MOUNT_SNAPSHOT IOCBASECMD(VFSIOC_MOUNT_SNAPSHOT) struct fs_snapshot_revert_args { struct componentname *sr_cnp; }; #define VFSIOC_REVERT_SNAPSHOT _IOW('V', 2, struct fs_snapshot_revert_args) -#define VFSCTL_REVERT_SNAPSHOT IOCBASECMD(VFSIOC_REVERT_SNAPSHOT) struct fs_snapshot_root_args { struct componentname *sr_cnp; }; #define VFSIOC_ROOT_SNAPSHOT _IOW('V', 3, struct fs_snapshot_root_args) -#define VFSCTL_ROOT_SNAPSHOT IOCBASECMD(VFSIOC_ROOT_SNAPSHOT) #endif /* KERNEL */ @@ -1110,7 +1108,7 @@ void vfs_setfsprivate(mount_t mp, void *mntdata); @abstract Get information about filesystem status. @discussion Each filesystem has a struct vfsstatfs associated with it which is updated as events occur; this function returns a pointer to it. Note that the data in the structure will continue to change over time and also that it may - be quite stale of vfs_update_vfsstat has not been called recently. + be quite stale if vfs_update_vfsstat has not been called recently. @param mp Mount for which to get vfsstatfs pointer. @return Pointer to vfsstatfs. */ @@ -1262,6 +1260,13 @@ void vfs_event_signal(fsid_t *fsid, u_int32_t event, intptr_t data); */ void vfs_event_init(void); /* XXX We should not export this */ +/*! + @function vfs_set_root_unmount_cleanly + @abstract This function should be called by the root file system + when it is being mounted if the file system state is consistent. +*/ +void vfs_set_root_unmounted_cleanly(void); + #ifdef KERNEL_PRIVATE int vfs_getbyid(fsid_t *fsid, ino64_t ino, vnode_t *vpp, vfs_context_t ctx); int vfs_getattr(mount_t mp, struct vfs_attr *vfa, vfs_context_t ctx); @@ -1381,10 +1386,14 @@ int getfsstat(struct statfs *, int, int) __DARWIN_INODE64(getfsstat); int getfsstat64(struct statfs64 *, int, int) __OSX_AVAILABLE_BUT_DEPRECATED(__MAC_10_5,__MAC_10_6,__IPHONE_NA,__IPHONE_NA); #endif /* !__DARWIN_ONLY_64_BIT_INO_T */ int getmntinfo(struct statfs **, int) __DARWIN_INODE64(getmntinfo); +int getmntinfo_r_np(struct statfs **, int) __DARWIN_INODE64(getmntinfo_r_np) + __OSX_AVAILABLE(10.13) __IOS_AVAILABLE(11.0) + __TVOS_AVAILABLE(11.0) __WATCHOS_AVAILABLE(4.0); #if !__DARWIN_ONLY_64_BIT_INO_T int getmntinfo64(struct statfs64 **, int) __OSX_AVAILABLE_BUT_DEPRECATED(__MAC_10_5,__MAC_10_6,__IPHONE_NA,__IPHONE_NA); #endif /* !__DARWIN_ONLY_64_BIT_INO_T */ int mount(const char *, const char *, int, void *); +int fmount(const char *, int, int, void *) __OSX_AVAILABLE(10.13) __IOS_AVAILABLE(11.0) __TVOS_AVAILABLE(11.0) __WATCHOS_AVAILABLE(4.0); int statfs(const char *, struct statfs *) __DARWIN_INODE64(statfs); #if !__DARWIN_ONLY_64_BIT_INO_T int statfs64(const char *, struct statfs64 *) __OSX_AVAILABLE_BUT_DEPRECATED(__MAC_10_5,__MAC_10_6,__IPHONE_NA,__IPHONE_NA); diff --git a/bsd/sys/mount_internal.h b/bsd/sys/mount_internal.h index 6393fd712..10ed1f50b 100644 --- a/bsd/sys/mount_internal.h +++ b/bsd/sys/mount_internal.h @@ -196,6 +196,8 @@ struct mount { uint32_t mnt_iobufinuse; + void *mnt_disk_conditioner_info; + lck_mtx_t mnt_iter_lock; /* mutex that protects iteration of vnodes */ }; diff --git a/bsd/sys/munge.h b/bsd/sys/munge.h index 47f07923c..b7e55762e 100644 --- a/bsd/sys/munge.h +++ b/bsd/sys/munge.h @@ -71,6 +71,57 @@ * of uu_arg[] and work our way back to the beginning of the array. */ +#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4) +int munge_w(const void *regs, void *args); +int munge_ww(const void *regs, void *args); +int munge_www(const void *regs, void *args); +int munge_wwww(const void *regs, void *args); +int munge_wwwww(const void *regs, void *args); +int munge_wwwwww(const void *regs, void *args); +int munge_wwwwwww(const void *regs, void *args); +int munge_wwwwwwww(const void *regs, void *args); +int munge_wl(const void *regs, void *args); +int munge_wwl(const void *regs, void *args); +int munge_wwlw(const void *regs, void *args); +int munge_wwlll(const void *regs, void *args); +int munge_wwllww(const void *regs, void *args); +int munge_wlw(const void *regs, void *args); +int munge_wlww(const void *regs, void *args); +int munge_wlwwwl(const void *regs, void *args); +int munge_wlwwwll(const void *regs, void *args); +int munge_wlwwwllw(const void *regs, void *args); +int munge_wlwwlwlw(const void *regs, void *args); +int munge_wll(const void *regs, void *args); +int munge_wllww(const void *regs, void *args); +int munge_wlll(const void *regs, void *args); +int munge_wllll(const void *regs, void *args); +int munge_wllwwll(const void *regs, void *args); +int munge_wwwlw(const void *regs, void *args); +int munge_wwwlww(const void *regs, void *args); +int munge_wwwl(const void *regs, void *args); +int munge_wwwwlw(const void *regs, void *args); +int munge_wwwwl(const void *regs, void *args); +int munge_wwwwwl(const void *regs, void *args); +int munge_wwwwwlww(const void *regs, void *args); +int munge_wwwwwllw(const void *regs, void *args); +int munge_wwwwwlll(const void *regs, void *args); +int munge_wwwwwwl(const void *regs, void *args); +int munge_wwwwwwlw(const void *regs, void *args); +int munge_wwwwwwll(const void *regs, void *args); +int munge_wsw(const void *regs, void *args); +int munge_wws(const void *regs, void *args); +int munge_wwws(const void *regs, void *args); +int munge_wwwsw(const void *regs, void *args); +int munge_llllll(const void *regs, void *args); +int munge_l(const void *regs, void *args); +int munge_ll(const void *regs, void *args); +int munge_lw(const void *regs, void *args); +int munge_lwww(const void *regs, void *args); +int munge_lwwwwwww(const void *regs, void *args); +int munge_wwlww(const void *regs, void *args); +int munge_wwlwww(const void *regs, void *args); +int munge_wwlwwwl(const void *regs, void *args); +#else void munge_w(void *args); void munge_ww(void *args); void munge_www(void *args); @@ -116,7 +167,9 @@ void munge_l(void *args); void munge_ll(void *args); void munge_lw(void *args); void munge_lwww(void *args); +void munge_lwwwwwww(void *args); void munge_wwlww(void *args); void munge_wwlwww(void *args); void munge_wwlwwwl(void *args); +#endif /* __arm__ && (__BIGGEST_ALIGNMENT__ > 4) */ #endif /* __MUNGE_H__ */ diff --git a/bsd/sys/netport.h b/bsd/sys/netport.h index 74eba9efa..0095d9dda 100644 --- a/bsd/sys/netport.h +++ b/bsd/sys/netport.h @@ -32,6 +32,8 @@ #ifndef _SYS_NETPORT_H_ #define _SYS_NETPORT_H_ +#include <_types/_uint32_t.h> /* uint32_t */ + typedef uint32_t netaddr_t; /* diff --git a/bsd/sys/persona.h b/bsd/sys/persona.h index d0912055f..f1efd66d4 100644 --- a/bsd/sys/persona.h +++ b/bsd/sys/persona.h @@ -225,7 +225,7 @@ struct persona { #define persona_try_lock(persona) lck_mtx_try_lock(&(persona)->pna_lock) #define persona_lock_assert_held(persona) \ - lck_mtx_assert(&(persona)->pna_lock, LCK_MTX_ASSERT_OWNED) + LCK_MTX_ASSERT(&(persona)->pna_lock, LCK_MTX_ASSERT_OWNED) #ifdef PERSONA_DEBUG static inline const char *persona_desc(struct persona *persona, int locked) diff --git a/bsd/sys/pgo.h b/bsd/sys/pgo.h index 167b212fa..fcd669b51 100644 --- a/bsd/sys/pgo.h +++ b/bsd/sys/pgo.h @@ -91,4 +91,9 @@ ssize_t grab_pgo_data( #endif +#ifdef XNU_KERNEL_PRIVATE +kern_return_t do_pgo_reset_counters(void); +#endif + + #endif diff --git a/bsd/sys/pipe.h b/bsd/sys/pipe.h index 3437710b2..09b25dbce 100644 --- a/bsd/sys/pipe.h +++ b/bsd/sys/pipe.h @@ -63,6 +63,8 @@ #include <sys/queue.h> /* for TAILQ macros */ #include <sys/ev.h> #include <sys/cdefs.h> +#include <sys/_types/_caddr_t.h> +#include <sys/_types/_u_int.h> /* * Pipe buffer size, keep moderate in value, pipes take kva space. @@ -167,7 +169,7 @@ struct pipe { #define PIPE_LOCK(pipe) lck_mtx_lock(PIPE_MTX(pipe)) #define PIPE_UNLOCK(pipe) lck_mtx_unlock(PIPE_MTX(pipe)) -#define PIPE_LOCK_ASSERT(pipe, type) lck_mtx_assert(PIPE_MTX(pipe), (type)) +#define PIPE_LOCK_ASSERT(pipe, type) LCK_MTX_ASSERT(PIPE_MTX(pipe), (type)) __BEGIN_DECLS void pipeinit(void); diff --git a/bsd/sys/priv.h b/bsd/sys/priv.h index af9cd806c..58f568b4b 100644 --- a/bsd/sys/priv.h +++ b/bsd/sys/priv.h @@ -92,12 +92,13 @@ #define PRIV_SETPRIORITY_DARWIN_ROLE 1012 /* Allow setpriority(PRIO_DARWIN_ROLE) */ #define PRIV_PACKAGE_EXTENSIONS 1013 /* Push package extension list used by vn_path_package_check() */ #define PRIV_TRIM_ACTIVE_FILE 1014 /* Allow freeing space out from under an active file */ +#define PRIV_PROC_CPUMON_OVERRIDE 1015 /* Allow CPU usage monitor parameters less restrictive than default */ /* * Virtual memory privileges. */ #define PRIV_VM_PRESSURE 6000 /* Check VM pressure. */ -#define PRIV_VM_JETSAM 6001 /* Adjust jetsam configuration. */ +#define PRIV_VM_JETSAM 6001 /* Adjust jetsam configuration. */ #define PRIV_VM_FOOTPRINT_LIMIT 6002 /* Adjust physical footprint limit. */ /* @@ -112,6 +113,9 @@ #define PRIV_NET_PRIVILEGED_NECP_MATCH 10006 /* Privilege verified by Network Extension policies */ #define PRIV_NET_QOSMARKING_POLICY_OVERRIDE 10007 /* Privilege verified by Network Extension policies */ #define PRIV_NET_RESTRICTED_INTCOPROC 10008 /* Access to internal co-processor network interfaces */ + +#define PRIV_NET_PRIVILEGED_MULTIPATH 10009 /* Multipath usage */ +#define PRIV_NET_RESTRICTED_MULTIPATH_EXTENDED 10010 /* Extended multipath (more aggressive on cell) */ /* * IPv4 and IPv6 privileges. */ @@ -121,17 +125,28 @@ /* * VFS privileges */ -#define PRIV_VFS_OPEN_BY_ID 14000 /* Allow calling openbyid_np() */ +#define PRIV_VFS_OPEN_BY_ID 14000 /* Allow calling openbyid_np() */ #define PRIV_VFS_MOVE_DATA_EXTENTS 14001 /* Allow F_MOVEDATAEXTENTS fcntl */ #define PRIV_VFS_SNAPSHOT 14002 /* Allow create/rename/delete of snapshots */ #define PRIV_VFS_SNAPSHOT_REVERT 14003 /* Allow reverting filesystem to a previous snapshot */ +#define PRIV_APFS_EMBED_DRIVER 14100 /* Allow embedding an EFI driver into the APFS container */ +#define PRIV_APFS_FUSION_DEBUG 14101 /* Allow getting internal statistics and controlling the APFS fusion container */ #ifdef KERNEL /* * Privilege check interface. No flags are currently defined for the API. */ +#include <sys/cdefs.h> #include <sys/kauth.h> + +/* + * flags for priv_check_cred + */ +#define PRIVCHECK_DEFAULT_UNPRIVILEGED_FLAG (1) /* Don't grant root privilege by default */ + +__BEGIN_DECLS int priv_check_cred(kauth_cred_t cred, int priv, int flags); +__END_DECLS #endif #endif /* !_SYS_PRIV_H_ */ diff --git a/bsd/sys/proc.h b/bsd/sys/proc.h index 279e9670c..7878ff644 100644 --- a/bsd/sys/proc.h +++ b/bsd/sys/proc.h @@ -76,6 +76,7 @@ #include <sys/lock.h> #include <sys/param.h> #include <sys/event.h> +#include <sys/time.h> #ifdef KERNEL #include <sys/kernel_types.h> #include <uuid/uuid.h> @@ -303,7 +304,11 @@ pid_t proc_pgrpid(proc_t p); #ifdef KERNEL_PRIVATE // mark a process as being allowed to call vfs_markdependency() void bsd_set_dependency_capable(task_t task); +#ifdef __arm__ +static inline int IS_64BIT_PROCESS(__unused proc_t p) { return 0; } +#else extern int IS_64BIT_PROCESS(proc_t); +#endif /* __arm__ */ extern int tsleep(void *chan, int pri, const char *wmesg, int timo); extern int msleep1(void *chan, lck_mtx_t *mtx, int pri, const char *wmesg, u_int64_t timo); @@ -388,7 +393,11 @@ __BEGIN_DECLS int pid_suspend(int pid); int pid_resume(int pid); - +#if defined(__arm__) || defined(__arm64__) +int pid_hibernate(int pid); +#endif /* defined(__arm__) || defined(__arm64__) */ +int pid_shutdown_sockets(int pid, int level); +int pid_shutdown_networking(int pid, int level); __END_DECLS #endif /* !KERNEL */ diff --git a/bsd/sys/proc_info.h b/bsd/sys/proc_info.h index 8f22d8007..01c7d8b0a 100644 --- a/bsd/sys/proc_info.h +++ b/bsd/sys/proc_info.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2005-2016 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2005-2017 Apple Computer, Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -58,6 +58,7 @@ __BEGIN_DECLS #define PROC_UID_ONLY 4 #define PROC_RUID_ONLY 5 #define PROC_PPID_ONLY 6 +#define PROC_KDBG_ONLY 7 struct proc_bsdinfo { uint32_t pbi_flags; /* 64bit; emulated etc */ @@ -636,6 +637,19 @@ struct kqueue_info { uint32_t rfu_1; /* reserved */ }; +struct kqueue_dyninfo { + struct kqueue_info kqdi_info; + uint64_t kqdi_servicer; + uint64_t kqdi_owner; + uint32_t kqdi_sync_waiters; + uint8_t kqdi_sync_waiter_qos; + uint8_t kqdi_async_qos; + uint16_t kqdi_request_state; + uint8_t kqdi_events_qos; + uint8_t _kqdi_reserved0[7]; + uint64_t _kqdi_reserved1[4]; +}; + /* keep in sync with KQ_* in sys/eventvar.h */ #define PROC_KQUEUE_SELECT 0x01 #define PROC_KQUEUE_SLEEP 0x02 @@ -773,6 +787,12 @@ struct proc_fileportinfo { #define PROC_PIDEXITREASONBASICINFO 25 #define PROC_PIDEXITREASONBASICINFOSIZE (sizeof(struct proc_exitreasonbasicinfo)) +#define PROC_PIDLISTUPTRS 26 +#define PROC_PIDLISTUPTRS_SIZE (sizeof(uint64_t)) + +#define PROC_PIDLISTDYNKQUEUES 27 +#define PROC_PIDLISTDYNKQUEUES_SIZE (sizeof(kqueue_id_t)) + #endif /* Flavors for proc_pidfdinfo */ @@ -805,6 +825,7 @@ struct proc_fileportinfo { #define PROC_PIDFDKQUEUE_EXTINFO 9 #define PROC_PIDFDKQUEUE_EXTINFO_SIZE (sizeof(struct kevent_extinfo)) #define PROC_PIDFDKQUEUE_KNOTES_MAX (1024 * 128) +#define PROC_PIDDYNKQUEUES_MAX (1024 * 128) #endif /* PRIVATE */ @@ -883,19 +904,28 @@ struct proc_fileportinfo { #define PROC_FGHW_VOUCHER_ERROR 98 /* error in voucher / originator callout */ #define PROC_FGHW_ERROR 99 /* syscall parameter/permissions error */ +/* flavors for proc_piddynkqueueinfo */ +#ifdef PRIVATE +#define PROC_PIDDYNKQUEUE_INFO 0 +#define PROC_PIDDYNKQUEUE_INFO_SIZE (sizeof(struct kqueue_dyninfo)) +#define PROC_PIDDYNKQUEUE_EXTINFO 1 +#define PROC_PIDDYNKQUEUE_EXTINFO_SIZE (sizeof(struct kevent_extinfo)) +#endif + /* __proc_info() call numbers */ -#define PROC_INFO_CALL_LISTPIDS 0x1 -#define PROC_INFO_CALL_PIDINFO 0x2 -#define PROC_INFO_CALL_PIDFDINFO 0x3 -#define PROC_INFO_CALL_KERNMSGBUF 0x4 -#define PROC_INFO_CALL_SETCONTROL 0x5 -#define PROC_INFO_CALL_PIDFILEPORTINFO 0x6 -#define PROC_INFO_CALL_TERMINATE 0x7 -#define PROC_INFO_CALL_DIRTYCONTROL 0x8 -#define PROC_INFO_CALL_PIDRUSAGE 0x9 +#define PROC_INFO_CALL_LISTPIDS 0x1 +#define PROC_INFO_CALL_PIDINFO 0x2 +#define PROC_INFO_CALL_PIDFDINFO 0x3 +#define PROC_INFO_CALL_KERNMSGBUF 0x4 +#define PROC_INFO_CALL_SETCONTROL 0x5 +#define PROC_INFO_CALL_PIDFILEPORTINFO 0x6 +#define PROC_INFO_CALL_TERMINATE 0x7 +#define PROC_INFO_CALL_DIRTYCONTROL 0x8 +#define PROC_INFO_CALL_PIDRUSAGE 0x9 #define PROC_INFO_CALL_PIDORIGINATORINFO 0xa -#define PROC_INFO_CALL_LISTCOALITIONS 0xb -#define PROC_INFO_CALL_CANUSEFGHW 0xc +#define PROC_INFO_CALL_LISTCOALITIONS 0xb +#define PROC_INFO_CALL_CANUSEFGHW 0xc +#define PROC_INFO_CALL_PIDDYNKQUEUEINFO 0xd #endif /* PRIVATE */ @@ -921,6 +951,10 @@ extern int pid_kqueue_extinfo(proc_t, struct kqueue * kq, user_addr_t buffer, uint32_t buffersize, int32_t * retval); extern int pid_kqueue_udatainfo(proc_t p, struct kqueue *kq, uint64_t *buf, uint32_t bufsize); +extern int pid_kqueue_listdynamickqueues(proc_t p, user_addr_t ubuf, + uint32_t bufsize, int32_t *retval); +extern int pid_dynamickqueue_extinfo(proc_t p, kqueue_id_t kq_id, + user_addr_t ubuf, uint32_t bufsize, int32_t *retval); extern int fill_procworkqueue(proc_t, struct proc_workqueueinfo *); extern boolean_t workqueue_get_pwq_exceeded(void *v, boolean_t *exceeded_total, boolean_t *exceeded_constrained); diff --git a/bsd/sys/proc_internal.h b/bsd/sys/proc_internal.h index a6bfe6bc3..b6d40d6ee 100644 --- a/bsd/sys/proc_internal.h +++ b/bsd/sys/proc_internal.h @@ -98,7 +98,6 @@ __END_DECLS * PFDL = Process File Desc Lock * PUCL = Process User Credentials Lock * PSL = Process Spin Lock - * PPL = Parent Process Lock (planed for later usage) * LL = List Lock * SL = Session Lock */ @@ -255,6 +254,7 @@ struct proc { pid_t p_oppid; /* Save parent pid during ptrace. XXX */ u_int p_xstat; /* Exit status for wait; also stop signal. */ + uint8_t p_xhighbits; /* Stores the top byte of exit status to avoid truncation*/ #ifdef _PROC_HAS_SCHEDINFO_ /* may need cleanup, not used */ @@ -318,10 +318,6 @@ struct proc { char p_nice; /* Process "nice" value.(PL) */ u_char p_resv1; /* (NU) User-priority based on p_cpu and p_nice. */ -#if CONFIG_MACF - int p_mac_enforce; /* MAC policy enforcement control */ -#endif - // types currently in sys/param.h command_t p_comm; proc_name_t p_name; /* can be changed by the process */ @@ -364,7 +360,6 @@ struct proc { uint32_t p_pth_tsd_offset; /* offset from pthread_t to TSD for new threads */ user_addr_t p_stack_addr_hint; /* stack allocation hint for wq threads */ void * p_wqptr; /* workq ptr */ - struct kqueue * p_wqkqueue; /* private workq kqueue */ struct timeval p_start; /* starting time */ void * p_rcall; @@ -384,6 +379,8 @@ struct proc { #endif /* DIAGNOSTIC */ uint64_t p_dispatchqueue_offset; uint64_t p_dispatchqueue_serialno_offset; + uint64_t p_return_to_kernel_offset; + uint64_t p_mach_thread_self_offset; #if VM_PRESSURE_EVENTS struct timeval vm_pressure_last_notify_tstamp; #endif @@ -409,6 +406,7 @@ struct proc { /* cached proc-specific data required for corpse inspection */ pid_t p_responsible_pid; /* pid resonsible for this process */ + _Atomic uint32_t p_user_faults; /* count the number of user faults generated */ struct os_reason *p_exit_reason; }; @@ -515,7 +513,9 @@ struct proc { /* This packing breaks symmetry with userspace side (struct extern_proc * of proc.h) for the ARMV7K ABI where 64-bit types are 64-bit aligned */ +#if !(__arm__ && (__BIGGEST_ALIGNMENT__ > 4)) #pragma pack(4) +#endif struct user32_extern_proc { union { struct { @@ -655,9 +655,11 @@ extern LIST_HEAD(sesshashhead, session) *sesshashtbl; extern u_long sesshash; extern lck_grp_t * proc_lck_grp; +extern lck_grp_t * proc_fdmlock_grp; +extern lck_grp_t * proc_kqhashlock_grp; +extern lck_grp_t * proc_knhashlock_grp; #if CONFIG_FINE_LOCK_GROUPS extern lck_grp_t * proc_mlock_grp; -extern lck_grp_t * proc_fdmlock_grp; extern lck_grp_t * proc_ucred_mlock_grp; extern lck_grp_t * proc_slock_grp; #endif diff --git a/bsd/sys/process_policy.h b/bsd/sys/process_policy.h index 4aec9e13e..ca679bf2f 100644 --- a/bsd/sys/process_policy.h +++ b/bsd/sys/process_policy.h @@ -65,7 +65,11 @@ __BEGIN_DECLS #define PROC_POLICY_HARDWARE_ACCESS 2 /* access to various hardware */ #define PROC_POLICY_RESOURCE_STARVATION 3 /* behavior on resource starvation */ #define PROC_POLICY_RESOURCE_USAGE 4 /* behavior on resource consumption */ +#if CONFIG_EMBEDDED || TARGET_OS_EMBEDDED +#define PROC_POLICY_APP_LIFECYCLE 5 /* app life cycle management */ +#else /* CONFIG_EMBEDDED */ #define PROC_POLICY_RESERVED 5 /* behavior on resource consumption */ +#endif /* CONFIG_EMBEDDED */ #define PROC_POLICY_APPTYPE 6 /* behavior on resource consumption */ #define PROC_POLICY_BOOST 7 /* importance boost/drop */ @@ -75,7 +79,11 @@ __BEGIN_DECLS #define PROC_POLICY_BG_DISKTHROTTLE 2 /* disk accesses throttled */ #define PROC_POLICY_BG_NETTHROTTLE 4 /* network accesses throttled */ #define PROC_POLICY_BG_GPUDENY 8 /* no access to GPU */ +#if CONFIG_EMBEDDED || TARGET_OS_EMBEDDED +#define PROC_POLICY_BG_ALL 0x0F +#else /* CONFIG_EMBEDDED */ #define PROC_POLICY_BG_ALL 0x07 +#endif /* CONFIG_EMBEDDED */ #define PROC_POLICY_BG_DEFAULT PROC_POLICY_BG_ALL /* sub policies for hardware */ @@ -161,10 +169,20 @@ typedef struct proc_policy_cpuusage_attr { uint64_t ppattr_cpu_attr_deadline; /* 64bit deadline in nsecs */ } proc_policy_cpuusage_attr_t; +#if CONFIG_EMBEDDED || TARGET_OS_EMBEDDED +/* sub policies for app lifecycle management */ +#define PROC_POLICY_APPLIFE_NONE 0 /* does nothing.. */ +#define PROC_POLICY_APPLIFE_STATE 1 /* sets the app to various lifecycle states */ +#define PROC_POLICY_APPLIFE_DEVSTATUS 2 /* notes the device in inactive or short/long term */ +#define PROC_POLICY_APPLIFE_PIDBIND 3 /* a thread is to be bound to another processes app state */ +#endif /* CONFIG_EMBEDDED */ /* sub policies for PROC_POLICY_APPTYPE */ #define PROC_POLICY_APPTYPE_NONE 0 /* does nothing.. */ #define PROC_POLICY_APPTYPE_MODIFY 1 /* sets the app to various lifecycle states */ +#if CONFIG_EMBEDDED || TARGET_OS_EMBEDDED +#define PROC_POLICY_APPTYPE_THREADTHR 2 /* notes the device in inactive or short/long term */ +#endif /* CONFIG_EMBEDDED */ /* exported apptypes for PROC_POLICY_APPTYPE */ #define PROC_POLICY_OSX_APPTYPE_TAL 1 /* TAL-launched app */ diff --git a/bsd/sys/protosw.h b/bsd/sys/protosw.h index c636c19e6..9a9311ee6 100644 --- a/bsd/sys/protosw.h +++ b/bsd/sys/protosw.h @@ -72,6 +72,44 @@ /* XXX: this will go away */ #define PR_SLOWHZ 2 /* 2 slow timeouts per second */ +/* + * The arguments to the ctlinput routine are + * (*protosw[].pr_ctlinput)(cmd, sa, arg); + * where cmd is one of the commands below, sa is a pointer to a sockaddr, + * and arg is a `void *' argument used within a protocol family. + */ +#define PRC_IFDOWN 0 /* interface transition */ +#define PRC_ROUTEDEAD 1 /* select new route if possible ??? */ +#define PRC_IFUP 2 /* interface has come back up */ +#define PRC_QUENCH2 3 /* DEC congestion bit says slow down */ +#define PRC_QUENCH 4 /* some one said to slow down */ +#define PRC_MSGSIZE 5 /* message size forced drop */ +#define PRC_HOSTDEAD 6 /* host appears to be down */ +#define PRC_HOSTUNREACH 7 /* deprecated (use PRC_UNREACH_HOST) */ +#define PRC_UNREACH_NET 8 /* no route to network */ +#define PRC_UNREACH_HOST 9 /* no route to host */ +#define PRC_UNREACH_PROTOCOL 10 /* dst says bad protocol */ +#define PRC_UNREACH_PORT 11 /* bad port # */ +/* was PRC_UNREACH_NEEDFRAG 12 (use PRC_MSGSIZE) */ +#define PRC_UNREACH_SRCFAIL 13 /* source route failed */ +#define PRC_REDIRECT_NET 14 /* net routing redirect */ +#define PRC_REDIRECT_HOST 15 /* host routing redirect */ +#define PRC_REDIRECT_TOSNET 16 /* redirect for type of service & net */ +#define PRC_REDIRECT_TOSHOST 17 /* redirect for tos & host */ +#define PRC_TIMXCEED_INTRANS 18 /* packet lifetime expired in transit */ +#define PRC_TIMXCEED_REASS 19 /* lifetime expired on reass q */ +#define PRC_PARAMPROB 20 /* header incorrect */ +#define PRC_UNREACH_ADMIN_PROHIB 21 /* packet administrativly prohibited */ + +#define PRC_NCMDS 22 + +#define PRC_IS_REDIRECT(cmd) \ + ((cmd) >= PRC_REDIRECT_NET && (cmd) <= PRC_REDIRECT_TOSHOST) + +#ifdef BSD_KERNEL_PRIVATE +#include <sys/eventhandler.h> +#endif + #ifdef KERNEL_PRIVATE #include <sys/socket.h> #include <sys/socketvar.h> @@ -86,6 +124,7 @@ struct socket; struct sockopt; struct socket_filter; struct uio; +struct ifnet; #ifdef XNU_KERNEL_PRIVATE struct domain_old; #endif /* XNU_KERNEL_PRIVATE */ @@ -118,7 +157,7 @@ struct protosw { int (*pr_output) /* output to protocol (from above) */ (struct mbuf *m, struct socket *so); void (*pr_ctlinput) /* control input (from below) */ - (int, struct sockaddr *, void *); + (int, struct sockaddr *, void *, struct ifnet *); int (*pr_ctloutput) /* control output (from above) */ (struct socket *, struct sockopt *); /* @@ -140,11 +179,11 @@ struct protosw { struct pr_usrreqs *pr_usrreqs; /* supersedes pr_usrreq() */ #endif /* !XNU_KERNEL_PRIVATE */ int (*pr_lock) /* lock function for protocol */ - (struct socket *so, int locktype, void *debug); + (struct socket *so, int refcnt, void *debug); int (*pr_unlock) /* unlock for protocol */ - (struct socket *so, int locktype, void *debug); + (struct socket *so, int refcnt, void *debug); lck_mtx_t *(*pr_getlock) /* retrieve protocol lock */ - (struct socket *so, int locktype); + (struct socket *so, int flags); /* * Implant hooks */ @@ -204,7 +243,7 @@ struct protosw { int (*pr_output) /* output to protocol (from above) */ (struct mbuf *m, struct socket *so); void (*pr_ctlinput) /* control input (from below) */ - (int, struct sockaddr *, void *); + (int, struct sockaddr *, void *, struct ifnet *); int (*pr_ctloutput) /* control output (from above) */ (struct socket *, struct sockopt *); /* @@ -220,17 +259,23 @@ struct protosw { int (*pr_sysctl) /* sysctl for protocol */ (int *, u_int, void *, size_t *, void *, size_t); int (*pr_lock) /* lock function for protocol */ - (struct socket *so, int locktype, void *debug); + (struct socket *so, int refcnt, void *debug); int (*pr_unlock) /* unlock for protocol */ - (struct socket *so, int locktype, void *debug); + (struct socket *so, int refcnt, void *debug); lck_mtx_t *(*pr_getlock) /* retrieve protocol lock */ - (struct socket *so, int locktype); + (struct socket *so, int flags); /* * misc */ TAILQ_HEAD(, socket_filter) pr_filter_head; struct protosw_old *pr_old; }; + +/* + * Values for the flags argument of pr_getlock + */ +#define PR_F_WILLUNLOCK 0x01 /* Will unlock (e.g., msleep) after the pr_getlock call */ + #endif /* XNU_KERNEL_PRIVATE */ /* @@ -268,40 +313,6 @@ struct protosw { #endif /* BSD_KERNEL_PRIVATE */ #ifdef BSD_KERNEL_PRIVATE -/* - * The arguments to the ctlinput routine are - * (*protosw[].pr_ctlinput)(cmd, sa, arg); - * where cmd is one of the commands below, sa is a pointer to a sockaddr, - * and arg is a `void *' argument used within a protocol family. - */ -#define PRC_IFDOWN 0 /* interface transition */ -#define PRC_ROUTEDEAD 1 /* select new route if possible ??? */ -#define PRC_IFUP 2 /* interface has come back up */ -#define PRC_QUENCH2 3 /* DEC congestion bit says slow down */ -#define PRC_QUENCH 4 /* some one said to slow down */ -#define PRC_MSGSIZE 5 /* message size forced drop */ -#define PRC_HOSTDEAD 6 /* host appears to be down */ -#define PRC_HOSTUNREACH 7 /* deprecated (use PRC_UNREACH_HOST) */ -#define PRC_UNREACH_NET 8 /* no route to network */ -#define PRC_UNREACH_HOST 9 /* no route to host */ -#define PRC_UNREACH_PROTOCOL 10 /* dst says bad protocol */ -#define PRC_UNREACH_PORT 11 /* bad port # */ -/* was PRC_UNREACH_NEEDFRAG 12 (use PRC_MSGSIZE) */ -#define PRC_UNREACH_SRCFAIL 13 /* source route failed */ -#define PRC_REDIRECT_NET 14 /* net routing redirect */ -#define PRC_REDIRECT_HOST 15 /* host routing redirect */ -#define PRC_REDIRECT_TOSNET 16 /* redirect for type of service & net */ -#define PRC_REDIRECT_TOSHOST 17 /* redirect for tos & host */ -#define PRC_TIMXCEED_INTRANS 18 /* packet lifetime expired in transit */ -#define PRC_TIMXCEED_REASS 19 /* lifetime expired on reass q */ -#define PRC_PARAMPROB 20 /* header incorrect */ -#define PRC_UNREACH_ADMIN_PROHIB 21 /* packet administrativly prohibited */ - -#define PRC_NCMDS 22 - -#define PRC_IS_REDIRECT(cmd) \ - ((cmd) >= PRC_REDIRECT_NET && (cmd) <= PRC_REDIRECT_TOSHOST) - #ifdef PRCREQUESTS char *prcrequests[] = { "IFDOWN", "ROUTEDEAD", "IFUP", "DEC-BIT-QUENCH2", @@ -383,7 +394,6 @@ char *prurequests[] = { #endif /* PRUREQUESTS */ #endif /* BSD_KERNEL_PRIVATE */ -struct ifnet; struct stat; struct ucred; struct uio; @@ -463,8 +473,6 @@ struct pr_usrreqs { int (*pru_disconnectx)(struct socket *, sae_associd_t, sae_connid_t); int (*pru_listen)(struct socket *, struct proc *); - int (*pru_peeloff)(struct socket *, - sae_associd_t, struct socket **); int (*pru_peeraddr)(struct socket *, struct sockaddr **); int (*pru_rcvd)(struct socket *, int); int (*pru_rcvoob)(struct socket *, struct mbuf *, int); @@ -493,6 +501,12 @@ struct pr_usrreqs { /* Values for pru_flags */ #define PRUF_OLD 0x10000000 /* added via net_add_proto */ +#ifdef BSD_KERNEL_PRIVATE +/* + * For faster access than net_uptime(), bypassing the initialization. + */ +extern u_int64_t _net_uptime; +#endif /* BSD_KERNEL_PRIVATE */ #endif /* XNU_KERNEL_PRIVATE */ __BEGIN_DECLS @@ -511,7 +525,6 @@ extern int pru_connectx_notsupp(struct socket *, struct sockaddr *, extern int pru_disconnectx_notsupp(struct socket *, sae_associd_t, sae_connid_t); extern int pru_socheckopt_null(struct socket *, struct sockopt *); -extern int pru_peeloff_notsupp(struct socket *, sae_associd_t, struct socket **); #endif /* XNU_KERNEL_PRIVATE */ extern int pru_control_notsupp(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p); @@ -555,7 +568,7 @@ extern int net_del_proto(int, int, struct domain *); extern int net_add_proto_old(struct protosw_old *, struct domain_old *); extern int net_del_proto_old(int, int, struct domain_old *); extern void net_update_uptime(void); -extern void net_update_uptime_secs(uint64_t secs); +extern void net_update_uptime_with_time(const struct timeval *); extern u_int64_t net_uptime(void); extern void net_uptime2timeval(struct timeval *); #else diff --git a/bsd/sys/pthread_internal.h b/bsd/sys/pthread_internal.h index 634470f89..1dff9968f 100644 --- a/bsd/sys/pthread_internal.h +++ b/bsd/sys/pthread_internal.h @@ -45,6 +45,7 @@ void workqueue_exit(struct proc *); void pthread_init(void); int thread_qos_from_pthread_priority(unsigned long, unsigned long *); unsigned long pthread_priority_canonicalize(unsigned long priority, boolean_t propagation); +boolean_t workq_thread_has_been_unbound(thread_t th, int qos_class); #endif /* _SYS_PTHREAD_INTERNAL_H_ */ diff --git a/bsd/sys/pthread_shims.h b/bsd/sys/pthread_shims.h index bd3322d42..2256a4a01 100644 --- a/bsd/sys/pthread_shims.h +++ b/bsd/sys/pthread_shims.h @@ -37,21 +37,36 @@ #include <kern/kern_types.h> #include <kern/kcdata.h> #include <kern/locks.h> +#include <sys/user.h> #include <sys/_types.h> #include <sys/_types/_sigset_t.h> #include <sys/kernel_types.h> -#include <sys/proc_info.h> #ifndef PTHREAD_INTERNAL struct uthread; #define M_PROC 41 #endif -#ifdef NEEDS_SCHED_CALL_T +#if !defined(_SCHED_CALL_T_DEFINED) +#define _SCHED_CALL_T_DEFINED typedef void (*sched_call_t)(int type, thread_t thread); #endif typedef struct workq_reqthreads_req_s {unsigned long priority; int count;} *workq_reqthreads_req_t; +typedef struct workq_threadreq_s { void *opaqueptr[2]; uint32_t opaqueint[2];} *workq_threadreq_t; +enum workq_threadreq_type { + WORKQ_THREADREQ_KEVENT = 1, + WORKQ_THREADREQ_WORKLOOP = 2, + WORKQ_THREADREQ_WORKLOOP_NO_THREAD_CALL = 3, + WORKQ_THREADREQ_REDRIVE = 4, +}; +enum workq_threadreq_op { + WORKQ_THREADREQ_CHANGE_PRI = 1, + WORKQ_THREADREQ_CANCEL = 2, + WORKQ_THREADREQ_CHANGE_PRI_NO_THREAD_CALL = 3, +}; +#define WORKQ_THREADREQ_FLAG_NOEMERGENCY 0x1 + /* * Increment each time new reserved slots are used. When the pthread @@ -65,11 +80,10 @@ typedef const struct pthread_functions_s { /* internal calls, kernel core -> kext */ void (*pthread_init)(void); - int (*fill_procworkqueue)(proc_t p, struct proc_workqueueinfo * pwqinfo); + int (*fill_procworkqueue)(proc_t p, void* pwqinfo); - // UNUSED - TO BE DELETED - void (*workqueue_init_lock)(proc_t p); - void (*workqueue_destroy_lock)(proc_t p); + void (*__unused1)(void); + void (*__unused2)(void); void (*workqueue_exit)(struct proc *p); void (*workqueue_mark_exiting)(struct proc *p); @@ -115,14 +129,51 @@ typedef const struct pthread_functions_s { /* try to get wq flags in debugger context */ uint32_t (*get_pwq_state_kdp)(proc_t p); - unsigned long (*pthread_priority_canonicalize)(unsigned long pthread_priority); + void (*__unused3)(void); unsigned long (*pthread_priority_canonicalize2)(unsigned long pthread_priority, boolean_t propagation); + /* Returns true on success, false on mismatch */ + boolean_t (*workq_thread_has_been_unbound)(thread_t th, int qos_class); + void (*pthread_find_owner)(thread_t thread, struct stackshot_thread_waitinfo *waitinfo); void *(*pthread_get_thread_kwq)(thread_t thread); + /* + * Submits a threadreq to the workq system. + * + * If type is WORKQ_THREADREQ_KEVENT, the semantics are similar to a call + * to workq_reqthreads and the kevent bind function will be called to + * indicate the thread fulfilling the request. The req argument is ignored. + * + * If type is WORKQ_THREADREQ_WORKLOOP, The req argument should point to + * allocated memory of at least the sizeof(workq_threadreq_t). That memory + * is lent to the workq system until workloop_fulfill_threadreq is called + * and passed the pointer, at which point it may be freed. + * + * The properties of the request are passed in the (pthread) priority and flags arguments. + * + * Will return zero upon success or an error value on failure. An error of + * ENOTSUP means the type argument was not understood. + */ + int (*workq_threadreq)(struct proc *p, workq_threadreq_t req, + enum workq_threadreq_type, unsigned long priority, int flags); + + /* + * Modifies an already submitted thread request. + * + * If operation is WORKQ_THREADREQ_CHANGE_PRI, arg1 is the new priority and arg2 is unused. + * + * If operation is WORKQ_THREADREQ_CANCEL, arg1 and arg2 are unused. + * + * Will return zero upon success or an error value on failure. An error of + * ENOTSUP means the operation argument was not understood. + */ + int (*workq_threadreq_modify)(struct proc *t, workq_threadreq_t req, + enum workq_threadreq_op operation, + unsigned long arg1, unsigned long arg2); + /* padding for future */ - void * _pad[90]; + void * _pad[87]; } * pthread_functions_t; typedef const struct pthread_callbacks_s { @@ -142,8 +193,14 @@ typedef const struct pthread_callbacks_s { void (*proc_set_wqthread)(struct proc *t, user_addr_t addr); int (*proc_get_pthsize)(struct proc *t); void (*proc_set_pthsize)(struct proc *t, int size); +#if defined(__arm64__) + unsigned __int128 (*atomic_fetch_add_128_relaxed)(_Atomic unsigned __int128 *ptr, + unsigned __int128 value); + unsigned __int128 (*atomic_load_128_relaxed)(_Atomic unsigned __int128 *ptr); +#else void *unused_was_proc_get_targconc; void *unused_was_proc_set_targconc; +#endif uint64_t (*proc_get_dispatchqueue_offset)(struct proc *t); void (*proc_set_dispatchqueue_offset)(struct proc *t, uint64_t offset); void *unused_was_proc_get_wqlockptr; @@ -178,11 +235,13 @@ typedef const struct pthread_callbacks_s { /* wq functions */ kern_return_t (*thread_set_wq_state32)(thread_t thread, thread_state_t state); +#if !defined(__arm__) kern_return_t (*thread_set_wq_state64)(thread_t thread, thread_state_t state); +#endif /* sched_prim.h */ - void (*thread_exception_return)(); - void (*thread_bootstrap_return)(); + void (*thread_exception_return)(void); + void (*thread_bootstrap_return)(void); /* kern/clock.h */ void (*absolutetime_to_microtime)(uint64_t abstime, clock_sec_t *secs, clock_usec_t *microsecs); @@ -228,6 +287,9 @@ typedef const struct pthread_callbacks_s { /* osfmk/<arch>/machine_routines.h */ int (*ml_get_max_cpus)(void); + #if defined(__arm__) + uint32_t (*map_is_1gb)(vm_map_t); + #endif /* <rdar://problem/12809089> xnu: struct proc p_dispatchqueue_serialno_offset additions */ uint64_t (*proc_get_dispatchqueue_serialno_offset)(struct proc *p); @@ -243,8 +305,8 @@ typedef const struct pthread_callbacks_s { kern_return_t (*thread_set_tsd_base)(thread_t thread, mach_vm_offset_t tsd_base); int (*proc_usynch_get_requested_thread_qos)(struct uthread *); - void *unused_was_proc_usynch_thread_qos_add_override; - void *unused_was_proc_usynch_thread_qos_remove_override; + uint64_t (*proc_get_mach_thread_self_tsd_offset)(struct proc *p); + void (*proc_set_mach_thread_self_tsd_offset)(struct proc *p, uint64_t mach_thread_self_tsd_offset); kern_return_t (*thread_policy_get)(thread_t t, thread_policy_flavor_t flavor, thread_policy_t info, mach_msg_type_number_t *count, boolean_t *get_default); boolean_t (*qos_main_thread_active)(void); @@ -268,8 +330,24 @@ typedef const struct pthread_callbacks_s { user_addr_t (*proc_get_stack_addr_hint)(struct proc *p); void (*proc_set_stack_addr_hint)(struct proc *p, user_addr_t stack_addr_hint); + uint64_t (*proc_get_return_to_kernel_offset)(struct proc *t); + void (*proc_set_return_to_kernel_offset)(struct proc *t, uint64_t offset); + + /* indicates call is being made synchronously with workq_threadreq call */ +# define WORKLOOP_FULFILL_THREADREQ_SYNC 0x1 +# define WORKLOOP_FULFILL_THREADREQ_CANCEL 0x2 + int (*workloop_fulfill_threadreq)(struct proc *p, workq_threadreq_t req, thread_t thread, int flags); + void (*thread_will_park_or_terminate)(thread_t thread); + + /* For getting maximum parallelism for a given QoS */ + uint32_t (*qos_max_parallelism)(int qos, uint64_t options); + + /* proc_internal.h: struct proc user_stack accessor */ + user_addr_t (*proc_get_user_stack)(struct proc *p); + void (*proc_set_user_stack)(struct proc *p, user_addr_t user_stack); + /* padding for future */ - void* _pad[76]; + void* _pad[69]; } *pthread_callbacks_t; diff --git a/bsd/sys/quota.h b/bsd/sys/quota.h index 35cb6b70a..d0022c0d9 100644 --- a/bsd/sys/quota.h +++ b/bsd/sys/quota.h @@ -69,6 +69,7 @@ #include <sys/appleapiopts.h> #include <sys/cdefs.h> +#include <sys/types.h> /* u_int32_t */ #ifdef KERNEL_PRIVATE #include <kern/locks.h> #endif diff --git a/bsd/sys/reason.h b/bsd/sys/reason.h index 0952e5e96..2d81e3a41 100644 --- a/bsd/sys/reason.h +++ b/bsd/sys/reason.h @@ -97,22 +97,30 @@ void os_reason_free(os_reason_t cur_reason); #define OS_REASON_REPORTCRASH 12 #define OS_REASON_COREANIMATION 13 #define OS_REASON_AGGREGATED 14 +#define OS_REASON_ASSERTIOND 15 +#define OS_REASON_SKYWALK 16 +#define OS_REASON_SETTINGS 17 +#define OS_REASON_LIBSYSTEM 18 +#define OS_REASON_FOUNDATION 19 +#define OS_REASON_WATCHDOG 20 +#define OS_REASON_METAL 21 /* * Update whenever new OS_REASON namespaces are added. */ -#define OS_REASON_MAX_VALID_NAMESPACE OS_REASON_AGGREGATED +#define OS_REASON_MAX_VALID_NAMESPACE OS_REASON_METAL #define OS_REASON_BUFFER_MAX_SIZE 5120 -#define OS_REASON_FLAG_NO_CRASH_REPORT 0x1 /* Don't create a crash report */ -#define OS_REASON_FLAG_GENERATE_CRASH_REPORT 0x2 /* Create a crash report - the default for userspace requests */ -#define OS_REASON_FLAG_FROM_USERSPACE 0x4 /* Reason created from a userspace syscall */ -#define OS_REASON_FLAG_FAILED_DATA_COPYIN 0x8 /* We failed to copyin data from userspace */ -#define OS_REASON_FLAG_PAYLOAD_TRUNCATED 0x10 /* The payload was truncated because it was longer than allowed */ -#define OS_REASON_FLAG_BAD_PARAMS 0x20 /* Invalid parameters were passed involved with creating this reason */ -#define OS_REASON_FLAG_CONSISTENT_FAILURE 0x40 /* Whatever caused this reason to be created will happen again */ -#define OS_REASON_FLAG_ONE_TIME_FAILURE 0x80 /* Whatever caused this reason to be created was a one time issue */ +#define OS_REASON_FLAG_NO_CRASH_REPORT 0x1 /* Don't create a crash report */ +#define OS_REASON_FLAG_GENERATE_CRASH_REPORT 0x2 /* Create a crash report - the default for userspace requests */ +#define OS_REASON_FLAG_FROM_USERSPACE 0x4 /* Reason created from a userspace syscall */ +#define OS_REASON_FLAG_FAILED_DATA_COPYIN 0x8 /* We failed to copyin data from userspace */ +#define OS_REASON_FLAG_PAYLOAD_TRUNCATED 0x10 /* The payload was truncated because it was longer than allowed */ +#define OS_REASON_FLAG_BAD_PARAMS 0x20 /* Invalid parameters were passed involved with creating this reason */ +#define OS_REASON_FLAG_CONSISTENT_FAILURE 0x40 /* Whatever caused this reason to be created will happen again */ +#define OS_REASON_FLAG_ONE_TIME_FAILURE 0x80 /* Whatever caused this reason to be created was a one time issue */ +#define OS_REASON_FLAG_NO_CRASHED_TID 0x100 /* Don't include the TID that processed the exit in the crash report */ /* * Set of flags that are allowed to be passed from userspace diff --git a/bsd/sys/resource.h b/bsd/sys/resource.h index 1dfb214a5..2f0316c87 100644 --- a/bsd/sys/resource.h +++ b/bsd/sys/resource.h @@ -210,7 +210,8 @@ struct rusage { #define RUSAGE_INFO_V1 1 #define RUSAGE_INFO_V2 2 #define RUSAGE_INFO_V3 3 -#define RUSAGE_INFO_CURRENT RUSAGE_INFO_V3 +#define RUSAGE_INFO_V4 4 +#define RUSAGE_INFO_CURRENT RUSAGE_INFO_V4 typedef void *rusage_info_t; @@ -222,7 +223,7 @@ struct rusage_info_v0 { uint64_t ri_interrupt_wkups; uint64_t ri_pageins; uint64_t ri_wired_size; - uint64_t ri_resident_size; + uint64_t ri_resident_size; uint64_t ri_phys_footprint; uint64_t ri_proc_start_abstime; uint64_t ri_proc_exit_abstime; @@ -236,7 +237,7 @@ struct rusage_info_v1 { uint64_t ri_interrupt_wkups; uint64_t ri_pageins; uint64_t ri_wired_size; - uint64_t ri_resident_size; + uint64_t ri_resident_size; uint64_t ri_phys_footprint; uint64_t ri_proc_start_abstime; uint64_t ri_proc_exit_abstime; @@ -256,7 +257,7 @@ struct rusage_info_v2 { uint64_t ri_interrupt_wkups; uint64_t ri_pageins; uint64_t ri_wired_size; - uint64_t ri_resident_size; + uint64_t ri_resident_size; uint64_t ri_phys_footprint; uint64_t ri_proc_start_abstime; uint64_t ri_proc_exit_abstime; @@ -278,7 +279,7 @@ struct rusage_info_v3 { uint64_t ri_interrupt_wkups; uint64_t ri_pageins; uint64_t ri_wired_size; - uint64_t ri_resident_size; + uint64_t ri_resident_size; uint64_t ri_phys_footprint; uint64_t ri_proc_start_abstime; uint64_t ri_proc_exit_abstime; @@ -301,7 +302,46 @@ struct rusage_info_v3 { uint64_t ri_serviced_system_time; }; -typedef struct rusage_info_v3 rusage_info_current; +struct rusage_info_v4 { + uint8_t ri_uuid[16]; + uint64_t ri_user_time; + uint64_t ri_system_time; + uint64_t ri_pkg_idle_wkups; + uint64_t ri_interrupt_wkups; + uint64_t ri_pageins; + uint64_t ri_wired_size; + uint64_t ri_resident_size; + uint64_t ri_phys_footprint; + uint64_t ri_proc_start_abstime; + uint64_t ri_proc_exit_abstime; + uint64_t ri_child_user_time; + uint64_t ri_child_system_time; + uint64_t ri_child_pkg_idle_wkups; + uint64_t ri_child_interrupt_wkups; + uint64_t ri_child_pageins; + uint64_t ri_child_elapsed_abstime; + uint64_t ri_diskio_bytesread; + uint64_t ri_diskio_byteswritten; + uint64_t ri_cpu_time_qos_default; + uint64_t ri_cpu_time_qos_maintenance; + uint64_t ri_cpu_time_qos_background; + uint64_t ri_cpu_time_qos_utility; + uint64_t ri_cpu_time_qos_legacy; + uint64_t ri_cpu_time_qos_user_initiated; + uint64_t ri_cpu_time_qos_user_interactive; + uint64_t ri_billed_system_time; + uint64_t ri_serviced_system_time; + uint64_t ri_logical_writes; + uint64_t ri_lifetime_max_phys_footprint; + uint64_t ri_instructions; + uint64_t ri_cycles; + uint64_t ri_billed_energy; + uint64_t ri_serviced_energy; + // We're reserving 2 counters for future extension + uint64_t ri_unused[2]; +}; + +typedef struct rusage_info_v4 rusage_info_current; #endif /* __DARWIN_C_LEVEL >= __DARWIN_C_FULL */ diff --git a/bsd/sys/resourcevar.h b/bsd/sys/resourcevar.h index 57ae8eb7c..6a21d2b6d 100644 --- a/bsd/sys/resourcevar.h +++ b/bsd/sys/resourcevar.h @@ -65,6 +65,7 @@ #define _SYS_RESOURCEVAR_H_ #include <sys/appleapiopts.h> +#include <sys/resource.h> /* * Kernel per-process accounting / statistics diff --git a/bsd/sys/sdt_impl.h b/bsd/sys/sdt_impl.h index 7517dd627..a0675b2b1 100644 --- a/bsd/sys/sdt_impl.h +++ b/bsd/sys/sdt_impl.h @@ -77,6 +77,10 @@ extern int sdt_probetab_mask; #if defined(__x86_64__) typedef uint8_t sdt_instr_t; +#elif defined(__arm__) +typedef uint16_t sdt_instr_t; +#elif defined(__arm64__) +typedef uint32_t sdt_instr_t; #else #error Unknown implementation #endif diff --git a/bsd/sys/sem.h b/bsd/sys/sem.h index 67c6064aa..4c19918c5 100644 --- a/bsd/sys/sem.h +++ b/bsd/sys/sem.h @@ -40,6 +40,7 @@ #include <sys/cdefs.h> #include <sys/_types.h> +#include <machine/types.h> /* __int32_t */ /* * [XSI] All of the symbols from <sys/ipc.h> SHALL be defined diff --git a/bsd/sys/signal.h b/bsd/sys/signal.h index 2483e8db3..817454ab5 100644 --- a/bsd/sys/signal.h +++ b/bsd/sys/signal.h @@ -568,12 +568,12 @@ struct sigstack { * signals delivered on a per-thread basis. */ #define threadmask (sigmask(SIGILL)|sigmask(SIGTRAP)|\ - sigmask(SIGIOT)|sigmask(SIGEMT)|\ + sigmask(SIGABRT)|sigmask(SIGEMT)|\ sigmask(SIGFPE)|sigmask(SIGBUS)|\ sigmask(SIGSEGV)|sigmask(SIGSYS)|\ sigmask(SIGPIPE)|sigmask(SIGKILL)) -#define workq_threadmask (threadmask | sigcantmask | sigmask(SIGPROF)) +#define workq_threadmask ((threadmask | sigcantmask | sigmask(SIGPROF)) & ~sigmask(SIGABRT)) /* * Signals carried across exec. diff --git a/bsd/sys/snapshot.h b/bsd/sys/snapshot.h index 76f115727..3953eab3e 100644 --- a/bsd/sys/snapshot.h +++ b/bsd/sys/snapshot.h @@ -52,7 +52,9 @@ int fs_snapshot_mount(int, const char *, const char *, uint32_t) __OSX_AVAILABLE int fs_snapshot_revert(int, const char *, uint32_t) __OSX_AVAILABLE(10.12) __IOS_AVAILABLE(10.0) __TVOS_AVAILABLE(10.0) __WATCHOS_AVAILABLE(3.0); +#ifdef PRIVATE int fs_snapshot_root(int, const char *, uint32_t) __OSX_AVAILABLE(10.12.4) __IOS_AVAILABLE(10.3) __TVOS_AVAILABLE(10.3) __WATCHOS_AVAILABLE(3.3); +#endif __END_DECLS diff --git a/bsd/sys/socket.h b/bsd/sys/socket.h index edb1dfdd1..2bdccabf6 100644 --- a/bsd/sys/socket.h +++ b/bsd/sys/socket.h @@ -196,7 +196,7 @@ #endif #ifdef PRIVATE -#define SO_EXECPATH 0x1085 /* Application Firewall Socket option */ +#define SO_EXECPATH 0x1085 /* Application Firewall Socket option */ /* * Traffic service class definitions (lowest to highest): @@ -324,13 +324,11 @@ #define SO_DELEGATED_UUID 0x1108 /* set socket as delegate (uuid_t) */ #define SO_NECP_ATTRIBUTES 0x1109 /* NECP socket attributes (domain, account, etc.) */ #define SO_CFIL_SOCK_ID 0x1110 /* get content filter socket ID (cfil_sock_id_t) */ -#if MPTCP -#define SO_MPTCP_FASTJOIN 0x1111 /* fast join MPTCP */ -#endif /* MPTCP */ +#define SO_NECP_CLIENTUUID 0x1111 /* NECP Client uuid */ #endif /* PRIVATE */ #define SO_NUMRCVPKT 0x1112 /* number of datagrams in receive socket buffer */ #ifdef PRIVATE -#define SO_AWDL_UNRESTRICTED 0x1113 /* try to use AWDL in restricted mode */ +#define SO_AWDL_UNRESTRICTED 0x1113 /* try to use AWDL in restricted mode */ #define SO_EXTENDED_BK_IDLE 0x1114 /* extended time to keep socket idle after app is suspended (int) */ #define SO_MARK_CELLFALLBACK 0x1115 /* Mark as initiated by cell fallback */ #endif /* PRIVATE */ @@ -1269,6 +1267,7 @@ struct so_cinforeq64 { /* valid connection info auxiliary data types */ #define CIAUX_TCP 0x1 /* TCP auxiliary data (conninfo_tcp_t) */ +#define CIAUX_MPTCP 0x2 /* MPTCP auxiliary data (conninfo_mptcp_t) */ /* * Structure for SIOC{S,G}CONNORDER @@ -1425,8 +1424,12 @@ int sendfile(int, int, off_t, off_t *, struct sf_hdtr *, int); #if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE) void pfctlinput(int, struct sockaddr *); + +__API_AVAILABLE(macosx(10.11), ios(9.0), tvos(9.0), watchos(2.0)) int connectx(int, const sa_endpoints_t *, sae_associd_t, unsigned int, const struct iovec *, unsigned int, size_t *, sae_connid_t *); + +__API_AVAILABLE(macosx(10.11), ios(9.0), tvos(9.0), watchos(2.0)) int disconnectx(int, sae_associd_t, sae_connid_t); #endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */ __END_DECLS diff --git a/bsd/sys/socketvar.h b/bsd/sys/socketvar.h index adff7ecd7..a8faa3c64 100644 --- a/bsd/sys/socketvar.h +++ b/bsd/sys/socketvar.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -74,6 +74,7 @@ #include <sys/appleapiopts.h> #include <sys/cdefs.h> +#include <sys/types.h> /* u_quad_t */ #ifdef KERNEL_PRIVATE #include <sys/queue.h> /* for TAILQ macros */ #include <sys/select.h> /* for struct selinfo */ @@ -222,6 +223,7 @@ struct socket { #define SB_TRIM 0x800 /* Trim the socket buffer */ #define SB_NOCOMPRESS 0x1000 /* do not compress socket buffer */ #define SB_SNDBYTE_CNT 0x2000 /* keep track of snd bytes per interface */ +#define SB_UPCALL_LOCK 0x4000 /* Keep socket locked when doing the upcall */ caddr_t so_tpcb; /* Misc. protocol control block, used by some kexts */ @@ -274,12 +276,10 @@ struct socket { #define SOF_FLOW_DIVERT 0x00800000 /* Flow Divert is enabled */ #define SOF_MP_SUBFLOW 0x01000000 /* is a multipath subflow socket */ #define SOF_MPTCP_TRUE 0x02000000 /* Established e2e MPTCP connection */ -#define SOF_MPTCP_CLIENT 0x04000000 /* Only client starts addtnal flows */ -#define SOF_MP_SEC_SUBFLOW 0x08000000 /* Set up secondary flow */ -#define SOF_MP_TRYFAILOVER 0x10000000 /* Failing subflow */ -#define SOF_DELEGATED 0x20000000 /* on behalf of another process */ -#define SOF_MPTCP_FASTJOIN 0x40000000 /* fast join support */ -#define SOF_CONTENT_FILTER 0x80000000 /* Content filter enabled */ +#define SOF_MP_SEC_SUBFLOW 0x04000000 /* Set up secondary flow */ +#define SOF_MP_TRYFAILOVER 0x08000000 /* Failing subflow */ +#define SOF_DELEGATED 0x10000000 /* on behalf of another process */ +#define SOF_CONTENT_FILTER 0x20000000 /* Content filter enabled */ uint32_t so_upcallusecount; /* number of upcalls in progress */ int so_usecount; /* refcounting of socket use */; @@ -344,6 +344,12 @@ struct socket { #define SOF1_QOSMARKING_POLICY_OVERRIDE 0x00008000 /* Opt-out of QoS marking NECP policy */ #define SOF1_DATA_AUTHENTICATED 0x00010000 /* idempotent data is authenticated */ #define SOF1_ACCEPT_LIST_HELD 0x00020000 /* Another thread is accessing one of the accept lists */ +#define SOF1_CONTENT_FILTER_SKIP 0x00040000 /* Content filter should be skipped, socket is blessed */ +#define SOF1_HAS_NECP_CLIENT_UUID 0x00080000 /* NECP client UUID option set */ +#define SOF1_IN_KERNEL_SOCKET 0x00100000 /* Socket created in kernel via KPI */ +#define SOF1_CONNECT_COUNTED 0x00200000 /* connect() call was counted */ +#define SOF1_DNS_COUNTED 0x00400000 /* socket counted to send DNS queries */ + u_int64_t so_extended_bk_start; }; @@ -441,6 +447,7 @@ struct xsocket { uid_t so_uid; /* XXX */ }; +#if !CONFIG_EMBEDDED struct xsocket64 { u_int32_t xso_len; /* length of this structure */ u_int64_t xso_so; /* makes a convenient handle */ @@ -462,6 +469,7 @@ struct xsocket64 { struct xsockbuf so_snd; uid_t so_uid; /* XXX */ }; +#endif /* !CONFIG_EMBEDDED */ #ifdef PRIVATE #define XSO_SOCKET 0x001 @@ -580,35 +588,32 @@ struct kextcb { #define EXT_NULL 0x0 /* STATE: Not in use */ /* Hints for socket event processing */ -#define SO_FILT_HINT_LOCKED 0x00000001 /* socket is already locked */ -#define SO_FILT_HINT_CONNRESET 0x00000002 /* Reset is received */ +#define SO_FILT_HINT_LOCKED 0x00000001 /* socket is already locked */ +#define SO_FILT_HINT_CONNRESET 0x00000002 /* Reset is received */ #define SO_FILT_HINT_CANTRCVMORE 0x00000004 /* No more data to read */ #define SO_FILT_HINT_CANTSENDMORE 0x00000008 /* Can't write more data */ -#define SO_FILT_HINT_TIMEOUT 0x00000010 /* timeout */ -#define SO_FILT_HINT_NOSRCADDR 0x00000020 /* No src address available */ -#define SO_FILT_HINT_IFDENIED 0x00000040 /* interface denied access */ -#define SO_FILT_HINT_SUSPEND 0x00000080 /* output queue suspended */ -#define SO_FILT_HINT_RESUME 0x00000100 /* output queue resumed */ -#define SO_FILT_HINT_KEEPALIVE 0x00000200 /* TCP Keepalive received */ -#define SO_FILT_HINT_ADAPTIVE_WTIMO 0x00000400 /* TCP adaptive write timeout */ -#define SO_FILT_HINT_ADAPTIVE_RTIMO 0x00000800 /* TCP adaptive read timeout */ -#define SO_FILT_HINT_CONNECTED 0x00001000 /* socket is connected */ +#define SO_FILT_HINT_TIMEOUT 0x00000010 /* timeout */ +#define SO_FILT_HINT_NOSRCADDR 0x00000020 /* No src address available */ +#define SO_FILT_HINT_IFDENIED 0x00000040 /* interface denied access */ +#define SO_FILT_HINT_SUSPEND 0x00000080 /* output queue suspended */ +#define SO_FILT_HINT_RESUME 0x00000100 /* output queue resumed */ +#define SO_FILT_HINT_KEEPALIVE 0x00000200 /* TCP Keepalive received */ +#define SO_FILT_HINT_ADAPTIVE_WTIMO 0x00000400 /* TCP adaptive write timeout */ +#define SO_FILT_HINT_ADAPTIVE_RTIMO 0x00000800 /* TCP adaptive read timeout */ +#define SO_FILT_HINT_CONNECTED 0x00001000 /* socket is connected */ #define SO_FILT_HINT_DISCONNECTED 0x00002000 /* socket is disconnected */ -#define SO_FILT_HINT_CONNINFO_UPDATED 0x00004000 /* updated conninfo avail. */ -#define SO_FILT_HINT_MPFAILOVER 0x00008000 /* multipath failover */ -#define SO_FILT_HINT_MPSTATUS 0x00010000 /* multipath status */ -#define SO_FILT_HINT_MUSTRST 0x00020000 /* must send RST and close */ -#define SO_FILT_HINT_MPFASTJ 0x00040000 /* can do MPTCP fast join */ -#define SO_FILT_HINT_DELETEOK 0x00100000 /* Ok to delete socket */ -#define SO_FILT_HINT_MPCANTRCVMORE 0x00200000 /* MPTCP DFIN Received */ -#define SO_FILT_HINT_NOTIFY_ACK 0x00400000 /* Notify Acknowledgement */ +#define SO_FILT_HINT_CONNINFO_UPDATED 0x00004000 /* updated conninfo avail. */ +#define SO_FILT_HINT_MPFAILOVER 0x00008000 /* multipath failover */ +#define SO_FILT_HINT_MPSTATUS 0x00010000 /* multipath status */ +#define SO_FILT_HINT_MUSTRST 0x00020000 /* must send RST and close */ +#define SO_FILT_HINT_MPCANTRCVMORE 0x00040000 /* MPTCP DFIN Received */ +#define SO_FILT_HINT_NOTIFY_ACK 0x00080000 /* Notify Acknowledgement */ #define SO_FILT_HINT_BITS \ "\020\1LOCKED\2CONNRESET\3CANTRCVMORE\4CANTSENDMORE\5TIMEOUT" \ "\6NOSRCADDR\7IFDENIED\10SUSPEND\11RESUME\12KEEPALIVE\13AWTIMO" \ - "\14ARTIMO\15CONNECTED\16DISCONNECTED\17CONNINFO_UPDATED" \ - "\20MPFAILOVER\21MPSTATUS\22MUSTRST\23MPFASTJ\25DELETEOK" \ - "\26MPCANTRCVMORE\27NOTIFYACK" + "\14ARTIMO\15CONNECTED\16DISCONNECTED\17CONNINFO_UPDATED" \ + "\20MPFAILOVER\21MPSTATUS\22MUSTRST\23MPCANTRCVMORE\24NOTIFYACK" /* Mask for hints that have corresponding kqueue events */ #define SO_FILT_HINT_EV \ @@ -701,9 +706,10 @@ extern int sothrottlelog; extern int sorestrictrecv; extern int sorestrictsend; extern int somaxconn; +extern uint32_t tcp_do_autosendbuf; extern uint32_t tcp_autosndbuf_max; +extern uint32_t tcp_autosndbuf_inc; extern u_int32_t sotcdb; -extern u_int32_t net_io_policy_throttled; extern u_int32_t net_io_policy_log; extern u_int32_t net_io_policy_throttle_best_effort; #if CONFIG_PROC_UUID_POLICY @@ -763,6 +769,7 @@ extern int sopoll(struct socket *so, int events, struct ucred *cred, void *wql); extern int sooptcopyin(struct sockopt *sopt, void *data, size_t len, size_t minlen); extern int sooptcopyout(struct sockopt *sopt, void *data, size_t len); +extern int soopt_cred_check(struct socket *so, int priv, boolean_t allow_root); extern int soreceive(struct socket *so, struct sockaddr **paddr, struct uio *uio, struct mbuf **mp0, struct mbuf **controlp, int *flagsp); extern int soreserve(struct socket *so, u_int32_t sndcc, u_int32_t rcvcc); @@ -853,11 +860,9 @@ extern int soconnectxlocked(struct socket *so, struct sockaddr *src, sae_connid_t *, uint32_t, void *, u_int32_t, uio_t, user_ssize_t *); extern int sodisconnectx(struct socket *so, sae_associd_t, sae_connid_t); extern int sodisconnectxlocked(struct socket *so, sae_associd_t, sae_connid_t); -extern int sopeelofflocked(struct socket *so, sae_associd_t, struct socket **); extern void soevupcall(struct socket *, u_int32_t); /* flags for socreate_internal */ #define SOCF_ASYNC 0x1 /* non-blocking socket */ -#define SOCF_MP_SUBFLOW 0x2 /* multipath subflow socket */ extern int socreate_internal(int dom, struct socket **aso, int type, int proto, struct proc *, uint32_t, struct proc *); extern int socreate(int dom, struct socket **aso, int type, int proto); @@ -882,10 +887,13 @@ extern int soisprivilegedtraffic(struct socket *so); extern int soissrcbackground(struct socket *so); extern int soissrcrealtime(struct socket *so); extern int soissrcbesteffort(struct socket *so); +extern void soclearfastopen(struct socket *so); extern int solisten(struct socket *so, int backlog); extern struct socket *sodropablereq(struct socket *head); -extern int socket_lock(struct socket *so, int refcount); -extern int socket_unlock(struct socket *so, int refcount); +extern void socket_lock(struct socket *so, int refcount); +extern void socket_lock_assert_owned(struct socket *so); +extern int socket_try_lock(struct socket *so); +extern void socket_unlock(struct socket *so, int refcount); extern int sogetaddr_locked(struct socket *, struct sockaddr **, int); extern const char *solockhistory_nr(struct socket *); extern void soevent(struct socket *so, long hint); @@ -899,7 +907,9 @@ extern int soshutdown(struct socket *so, int how); extern int soshutdownlock(struct socket *so, int how); extern int soshutdownlock_final(struct socket *so, int how); extern void sotoxsocket(struct socket *so, struct xsocket *xso); +#if !CONFIG_EMBEDDED extern void sotoxsocket64(struct socket *so, struct xsocket64 *xso); +#endif /* !CONFIG_EMBEDDED */ extern int sosendallatonce(struct socket *so); extern int soreadable(struct socket *so); extern int sowriteable(struct socket *so); @@ -910,7 +920,8 @@ extern int sosendcheck(struct socket *, struct sockaddr *, user_ssize_t, extern int soo_ioctl(struct fileproc *, u_long, caddr_t, vfs_context_t); extern int soo_stat(struct socket *, void *, int); extern int soo_select(struct fileproc *, int, void *, vfs_context_t); -extern int soo_kqfilter(struct fileproc *, struct knote *, vfs_context_t); +extern int soo_kqfilter(struct fileproc *, struct knote *, + struct kevent_internal_s *kev, vfs_context_t); /* Service class flags used for setting service class on a packet */ #define PKT_SCF_IPV6 0x00000001 /* IPv6 packet */ @@ -965,10 +976,11 @@ extern int soopt_mcopyin(struct sockopt *sopt, struct mbuf *m); extern int soopt_mcopyout(struct sockopt *sopt, struct mbuf *m); extern boolean_t so_cache_timer(void); +extern void mptcp_fallback_sbdrop(struct socket *so, struct mbuf *m, int len); extern void mptcp_preproc_sbdrop(struct socket *, struct mbuf *, unsigned int); extern void mptcp_postproc_sbdrop(struct mbuf *, u_int64_t, u_int32_t, u_int32_t); -extern int mptcp_adj_rmap(struct socket *, struct mbuf *); +extern void mptcp_adj_rmap(struct socket *, struct mbuf *, int); extern void netpolicy_post_msg(uint32_t, struct netpolicy_event_data *, uint32_t); diff --git a/bsd/sys/sockio.h b/bsd/sys/sockio.h index f098e7a13..78f3b6a7f 100644 --- a/bsd/sys/sockio.h +++ b/bsd/sys/sockio.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */ @@ -151,14 +151,14 @@ #define SIOCGIFBOND _IOWR('i', 71, struct ifreq) /* get bond if config */ #ifdef PRIVATE -/* +/* * temporary control calls to attach/detach IP to/from an ethernet interface */ #define SIOCPROTOATTACH _IOWR('i', 80, struct ifreq) /* attach proto to interface */ #define SIOCPROTODETACH _IOWR('i', 81, struct ifreq) /* detach proto from interface */ #endif /* PRIVATE */ -#define SIOCSIFCAP _IOW('i', 90, struct ifreq) /* set IF features */ +#define SIOCSIFCAP _IOW('i', 90, struct ifreq) /* set IF features */ #define SIOCGIFCAP _IOWR('i', 91, struct ifreq) /* get IF features */ #define SIOCIFCREATE _IOWR('i', 120, struct ifreq) /* create clone if */ @@ -268,7 +268,9 @@ #define SIOCSIFPROBECONNECTIVITY _IOWR('i', 171, struct ifreq) /* Start/Stop probes to check connectivity */ #define SIOCGIFPROBECONNECTIVITY _IOWR('i', 172, struct ifreq) /* check if connectivity probes are enabled */ +#endif /* PRIVATE */ #define SIOCGIFFUNCTIONALTYPE _IOWR('i', 173, struct ifreq) /* get interface functional type */ +#ifdef PRIVATE #define SIOCSIFNETSIGNATURE _IOWR('i', 174, struct if_nsreq) #define SIOCGIFNETSIGNATURE _IOWR('i', 175, struct if_nsreq) @@ -293,11 +295,21 @@ #define SIOCSIFDISABLEOUTPUT _IOWR('i', 187, struct ifreq) #define SIOCGIFAGENTLIST _IOWR('i', 190, struct netagentlist_req) /* Get netagent dump */ + #ifdef BSD_KERNEL_PRIVATE #define SIOCGIFAGENTLIST32 _IOWR('i', 190, struct netagentlist_req32) #define SIOCGIFAGENTLIST64 _IOWR('i', 190, struct netagentlist_req64) #endif /* BSD_KERNEL_PRIVATE */ +#define SIOCSIFLOWINTERNET _IOWR('i', 191, struct ifreq) +#define SIOCGIFLOWINTERNET _IOWR('i', 192, struct ifreq) + +#if INET6 +#define SIOCGIFNAT64PREFIX _IOWR('i', 193, struct if_nat64req) +#define SIOCSIFNAT64PREFIX _IOWR('i', 194, struct if_nat64req) +#endif +#define SIOCGIFNEXUS _IOWR('i', 195, struct if_nexusreq) + #endif /* PRIVATE */ #endif /* !_SYS_SOCKIO_H_ */ diff --git a/bsd/sys/spawn.h b/bsd/sys/spawn.h index edd7020d6..bcf1d6367 100644 --- a/bsd/sys/spawn.h +++ b/bsd/sys/spawn.h @@ -67,6 +67,9 @@ #define _POSIX_SPAWN_ALLOW_DATA_EXEC 0x2000 #endif /* PRIVATE */ #define POSIX_SPAWN_CLOEXEC_DEFAULT 0x4000 +#ifdef PRIVATE +#define _POSIX_SPAWN_HIGH_BITS_ASLR 0x8000 +#endif /* PRIVATE */ /* * Possible values to be set for the process control actions on resource starvation. diff --git a/bsd/sys/stat.h b/bsd/sys/stat.h index 75b8d9322..183fdd207 100644 --- a/bsd/sys/stat.h +++ b/bsd/sys/stat.h @@ -368,7 +368,15 @@ struct user32_stat64 { __uint32_t st_gen; /* file generation number */ __uint32_t st_lspare; /* RESERVED: DO NOT USE! */ __int64_t st_qspare[2]; /* RESERVED: DO NOT USE! */ +#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4) +/* For the newer ARMv7k ABI where 64-bit types are 64-bit aligned, but pointers + * are 32-bit: + * Applying attributes here causes a mismatch with the user-space struct stat64 + */ +}; +#else } __attribute__((packed,aligned(4))); +#endif extern void munge_user64_stat64(struct stat64 *sbp, struct user64_stat64 *usbp); extern void munge_user32_stat64(struct stat64 *sbp, struct user32_stat64 *usbp); @@ -474,7 +482,10 @@ extern void munge_user32_stat64(struct stat64 *sbp, struct user32_stat64 *usbp); notifications for deletes or renames for files which have UF_TRACKED set. */ #define UF_TRACKED 0x00000040 -/* Bits 0x0080 through 0x4000 are currently undefined. */ +#define UF_DATAVAULT 0x00000080 /* entitlement required for reading */ + /* and writing */ + +/* Bits 0x0100 through 0x4000 are currently undefined. */ #define UF_HIDDEN 0x00008000 /* hint that this item should not be */ /* displayed in a GUI */ /* @@ -485,7 +496,7 @@ extern void munge_user32_stat64(struct stat64 *sbp, struct user32_stat64 *usbp); #define SF_ARCHIVED 0x00010000 /* file is archived */ #define SF_IMMUTABLE 0x00020000 /* file may not be changed */ #define SF_APPEND 0x00040000 /* writes to file may only append */ -#define SF_RESTRICTED 0x00080000 /* restricted access */ +#define SF_RESTRICTED 0x00080000 /* entitlement required for writing */ #define SF_NOUNLINK 0x00100000 /* Item may not be removed, renamed or mounted on */ /* @@ -523,6 +534,13 @@ mode_t umask(mode_t); int fchmodat(int, const char *, mode_t, int) __OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0); int fstatat(int, const char *, struct stat *, int) __DARWIN_INODE64(fstatat) __OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0); int mkdirat(int, const char *, mode_t) __OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0); + +#define UTIME_NOW -1 +#define UTIME_OMIT -2 + +int futimens(int __fd, const struct timespec __times[2]) __API_AVAILABLE(macosx(10.13), ios(11.0), tvos(11.0), watchos(4.0)); +int utimensat(int __fd, const char *__path, const struct timespec __times[2], + int __flag) __API_AVAILABLE(macosx(10.13), ios(11.0), tvos(11.0), watchos(4.0)); #endif #if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE) diff --git a/bsd/sys/subr_prf.h b/bsd/sys/subr_prf.h index c3408ef9f..966aedf5a 100644 --- a/bsd/sys/subr_prf.h +++ b/bsd/sys/subr_prf.h @@ -83,7 +83,7 @@ #define TOSTR 0x00000008 /* output to string */ #define TOLOGLOCKED 0x00000010 /* output to log (log lock held) */ -extern int prf(const char *fmt, va_list ap, int flags, struct tty *ttyp); +extern int prf(const char *fmt, va_list ap, int flags, struct tty *ttyp) __printflike(1,0); #endif /* __APPLE_API_PRIVATE */ #endif /* KERNEL_PRIVATE */ diff --git a/bsd/sys/sysctl.h b/bsd/sys/sysctl.h index 4ccc92c72..3acef8603 100644 --- a/bsd/sys/sysctl.h +++ b/bsd/sys/sysctl.h @@ -86,6 +86,7 @@ #ifndef XNU_KERNEL_PRIVATE #include <libkern/sysctl.h> #endif + #endif #include <sys/proc.h> #include <sys/vm.h> @@ -372,6 +373,12 @@ __END_DECLS ptr, 0, sysctl_handle_long, "L", descr); \ typedef char _sysctl_##parent##_##name##_size_check[(__builtin_constant_p(ptr) || sizeof(*(ptr)) == sizeof(long)) ? 0 : -1]; +/* Oid for a unsigned long. The pointer must be non NULL. */ +#define SYSCTL_ULONG(parent, nbr, name, access, ptr, descr) \ + SYSCTL_OID(parent, nbr, name, CTLTYPE_INT|access, \ + ptr, 0, sysctl_handle_long, "LU", descr); \ + typedef char _sysctl_##parent##_##name##_size_check[(__builtin_constant_p(ptr) || sizeof(*(ptr)) == sizeof(unsigned long)) ? 0 : -1]; + /* Oid for a quad. The pointer must be non NULL. */ #define SYSCTL_QUAD(parent, nbr, name, access, ptr, descr) \ SYSCTL_OID(parent, nbr, name, CTLTYPE_QUAD|access, \ @@ -414,6 +421,27 @@ SYSCTL_DECL(_user); SYSCTL_DECL(_hw_features); #endif + +#ifndef SYSCTL_SKMEM_UPDATE_FIELD + +#define SYSCTL_SKMEM 0 +#define SYSCTL_SKMEM_UPDATE_FIELD(field, value) +#define SYSCTL_SKMEM_UPDATE_AT_OFFSET(offset, value) +#define SYSCTL_SKMEM_INT(parent, oid, sysctl_name, access, ptr, offset, descr) \ + SYSCTL_INT(parent, oid, sysctl_name, access, ptr, 0, descr) + +#define SYSCTL_SKMEM_TCP_INT(oid, sysctl_name, access, variable_type, \ + variable_name, initial_value, descr) \ + variable_type variable_name = initial_value; \ + SYSCTL_SKMEM_INT(_net_inet_tcp, oid, sysctl_name, access, \ + &variable_name, 0, descr) + +#else /* SYSCTL_SKMEM_UPDATE_FIELD */ +#define SYSCTL_SKMEM 1 +#endif /* SYSCTL_SKMEM_UPDATE_FIELD */ + + + #endif /* KERNEL */ #ifdef XNU_KERNEL_PRIVATE diff --git a/bsd/sys/sysent.h b/bsd/sys/sysent.h index 8182994d3..6d529d6f5 100644 --- a/bsd/sys/sysent.h +++ b/bsd/sys/sysent.h @@ -38,6 +38,8 @@ typedef int32_t sy_call_t(struct proc *, void *, int *); #if CONFIG_REQUIRES_U32_MUNGING typedef void sy_munge_t(void *); +#elif __arm__ && (__BIGGEST_ALIGNMENT__ > 4) +typedef int sy_munge_t(const void *, void *); #endif struct sysent { /* system call table */ diff --git a/bsd/sys/syslog.h b/bsd/sys/syslog.h index 2449ad379..42ff28cc6 100644 --- a/bsd/sys/syslog.h +++ b/bsd/sys/syslog.h @@ -229,7 +229,11 @@ __BEGIN_DECLS void closelog(void); void openlog(const char *, int, int); int setlogmask(int); +#if defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && __DARWIN_C_LEVEL >= __DARWIN_C_FULL +void syslog(int, const char *, ...) __printflike(2, 3) __not_tail_called __DARWIN_ALIAS_STARTING(__MAC_10_13, __IPHONE_NA, __DARWIN_EXTSN(syslog)); +#else void syslog(int, const char *, ...) __printflike(2, 3) __not_tail_called; +#endif #if __DARWIN_C_LEVEL >= __DARWIN_C_FULL void vsyslog(int, const char *, __darwin_va_list) __printflike(2, 0) __not_tail_called; #endif @@ -323,7 +327,7 @@ __BEGIN_DECLS void log(int, const char *, ...); #ifdef XNU_KERNEL_PRIVATE void logpri(int); -int vaddlog(const char *, va_list); +int vaddlog(const char *, va_list) __printflike(1,0); void logtime(time_t); #endif /* XNU_KERNEL_PRIVATE */ diff --git a/bsd/sys/systm.h b/bsd/sys/systm.h index e346413c8..fb2badf23 100644 --- a/bsd/sys/systm.h +++ b/bsd/sys/systm.h @@ -124,6 +124,9 @@ extern const char copyright[]; /* system copyright */ extern int boothowto; /* reboot flags, from console subsystem */ extern int show_space; extern int minimalboot; +#if CONFIG_EMBEDDED +extern int darkboot; +#endif extern const int nblkdev; /* number of entries in bdevsw */ extern const int nchrdev; /* number of entries in cdevsw */ @@ -157,7 +160,7 @@ void realitexpire(struct proc *); int hzto(struct timeval *tv); void tablefull(const char *); int kvprintf(char const *, void (*)(int, void*), void *, int, - __darwin_va_list); + __darwin_va_list) __printflike(1,0); void uprintf(const char *, ...) __printflike(1,2); int copywithin(void *saddr, void *daddr, size_t len); int64_t fulong(user_addr_t addr); diff --git a/bsd/sys/systrace_args.h b/bsd/sys/systrace_args.h new file mode 100644 index 000000000..5c1ef577e --- /dev/null +++ b/bsd/sys/systrace_args.h @@ -0,0 +1,36 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2017 Apple, Inc. All rights reserved. + * Use is subject to license terms. + */ +#ifndef _SYS_SYSTRACE_ARGS_H +#define _SYS_SYSTRACE_ARGS_H + +#include <sys/syscall.h> +#include <sys/sysproto.h> + +void systrace_args(int sysnum, void *params, uint64_t *uarg); +void systrace_entry_setargdesc(int sysnum, int ndx, char *desc, size_t descsz); +void systrace_return_setargdesc(int sysnum, int ndx, char *desc, size_t descsz); + +#endif /* _SYS_SYSTRACE_ARGS_H */ diff --git a/bsd/sys/time.h b/bsd/sys/time.h index 85e7fe3df..97a536416 100644 --- a/bsd/sys/time.h +++ b/bsd/sys/time.h @@ -193,6 +193,10 @@ struct clockinfo { }; #endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */ +#ifdef XNU_KERNEL_PRIVATE +#define SETTIME_ENTITLEMENT "com.apple.private.settime" +#endif + #ifdef KERNEL #if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE) diff --git a/bsd/sys/timex.h b/bsd/sys/timex.h new file mode 100644 index 000000000..5e8a3bfdc --- /dev/null +++ b/bsd/sys/timex.h @@ -0,0 +1,212 @@ +/*- + *********************************************************************** + * * + * Copyright (c) David L. Mills 1993-2001 * + * Copyright (c) Poul-Henning Kamp 2000-2001 * + * * + * Permission to use, copy, modify, and distribute this software and * + * its documentation for any purpose and without fee is hereby * + * granted, provided that the above copyright notice appears in all * + * copies and that both the copyright notice and this permission * + * notice appear in supporting documentation, and that the name * + * University of Delaware not be used in advertising or publicity * + * pertaining to distribution of the software without specific, * + * written prior permission. The University of Delaware makes no * + * representations about the suitability this software for any * + * purpose. It is provided "as is" without express or implied * + * warranty. * + * * + *********************************************************************** + * + * $FreeBSD$ + * + * This header file defines the Network Time Protocol (NTP) interfaces + * for user and daemon application programs. + * + * This file was originally created 17 Sep 93 by David L. Mills, Professor + * of University of Delaware, building on work which had already been ongoing + * for a decade and a half at that point in time. + * + * In 2000 the APIs got a upgrade from microseconds to nanoseconds, + * a joint work between Poul-Henning Kamp and David L. Mills. + * + */ + +/* + * Copyright (c) 2017 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef _SYS_TIMEX_H_ +#define _SYS_TIMEX_H_ 1 + +#include <sys/time.h> + +#define NTP_API 4 /* NTP API version */ + +/* + * The following defines establish the performance envelope of the + * kernel discipline loop. Phase or frequency errors greater than + * NAXPHASE or MAXFREQ are clamped to these maxima. For update intervals + * less than MINSEC, the loop always operates in PLL mode; while, for + * update intervals greater than MAXSEC, the loop always operates in FLL + * mode. Between these two limits the operating mode is selected by the + * STA_FLL bit in the status word. + */ + +#define MAXPHASE 500000000L /* max phase error (ns) */ +#define MAXFREQ 500000L /* max freq error (ns/s) */ +#define MINSEC 256 /* min FLL update interval (s) */ +#define MAXSEC 2048 /* max PLL update interval (s) */ +#define NANOSECOND 1000000000L /* nanoseconds in one second */ +#define SCALE_PPM (65536 / 1000) /* crude ns/s to scaled PPM */ +#define MAXTC 10 /* max time constant */ + +/* Codes for PPS (pulse-per-second) signals or leap seconds are not used but kept + * unchanged and commented for future compatibility. + */ + +/* + * Control mode codes (timex.modes) + */ +#define MOD_OFFSET 0x0001 /* set time offset */ +#define MOD_FREQUENCY 0x0002 /* set frequency offset */ +#define MOD_MAXERROR 0x0004 /* set maximum time error */ +#define MOD_ESTERROR 0x0008 /* set estimated time error */ +#define MOD_STATUS 0x0010 /* set clock status bits */ +#define MOD_TIMECONST 0x0020 /* set PLL time constant */ +#define MOD_PPSMAX 0x0040 /* set PPS maximum averaging time */ +#define MOD_TAI 0x0080 /* set TAI offset */ +#define MOD_MICRO 0x1000 /* select microsecond resolution */ +#define MOD_NANO 0x2000 /* select nanosecond resolution */ +#define MOD_CLKB 0x4000 /* select clock B */ +#define MOD_CLKA 0x8000 /* select clock A */ + +/* + * Status codes (timex.status) + */ +#define STA_PLL 0x0001 /* enable PLL updates (rw) */ +#define STA_PPSFREQ 0x0002 /* enable PPS freq discipline (rw) */ +#define STA_PPSTIME 0x0004 /* enable PPS time discipline (rw) */ +#define STA_FLL 0x0008 /* enable FLL mode (rw) */ +#define STA_INS 0x0010 /* insert leap (rw) */ +#define STA_DEL 0x0020 /* delete leap (rw) */ +#define STA_UNSYNC 0x0040 /* clock unsynchronized (rw) */ +#define STA_FREQHOLD 0x0080 /* hold frequency (rw) */ +#define STA_PPSSIGNAL 0x0100 /* PPS signal present (ro) */ +#define STA_PPSJITTER 0x0200 /* PPS signal jitter exceeded (ro) */ +#define STA_PPSWANDER 0x0400 /* PPS signal wander exceeded (ro) */ +#define STA_PPSERROR 0x0800 /* PPS signal calibration error (ro) */ +#define STA_CLOCKERR 0x1000 /* clock hardware fault (ro) */ +#define STA_NANO 0x2000 /* resolution (0 = us, 1 = ns) (ro) */ +#define STA_MODE 0x4000 /* mode (0 = PLL, 1 = FLL) (ro) */ +#define STA_CLK 0x8000 /* clock source (0 = A, 1 = B) (ro) */ + +#define STA_RONLY (STA_PPSSIGNAL | STA_PPSJITTER | STA_PPSWANDER | \ + STA_PPSERROR | STA_CLOCKERR | STA_NANO | STA_MODE | STA_CLK) + +#define STA_SUPPORTED (STA_PLL | STA_FLL | STA_UNSYNC | STA_FREQHOLD | \ + STA_CLOCKERR | STA_NANO | STA_MODE | STA_CLK) + +/* + * Clock states (ntptimeval.time_state) + */ +#define TIME_OK 0 /* no leap second warning */ +#define TIME_INS 1 /* insert leap second warning */ +#define TIME_DEL 2 /* delete leap second warning */ +#define TIME_OOP 3 /* leap second in progress */ +#define TIME_WAIT 4 /* leap second has occurred */ +#define TIME_ERROR 5 /* error (see status word) */ + +/* + * NTP user interface -- ntp_gettime - used to read kernel clock values + */ +struct ntptimeval { + struct timespec time; /* current time (ns) (ro) */ + long maxerror; /* maximum error (us) (ro) */ + long esterror; /* estimated error (us) (ro) */ + long tai; /* TAI offset */ + int time_state; /* time status */ +}; + +/* + * NTP daemon interface -- ntp_adjtime -- used to discipline CPU clock + * oscillator and control/determine status. + * + * Note: The offset, precision and jitter members are in microseconds if + * STA_NANO is zero and nanoseconds if not. + */ +struct timex { + unsigned int modes; /* clock mode bits (wo) */ + long offset; /* time offset (ns/us) (rw) */ + long freq; /* frequency offset (scaled PPM) (rw) */ + long maxerror; /* maximum error (us) (rw) */ + long esterror; /* estimated error (us) (rw) */ + int status; /* clock status bits (rw) */ + long constant; /* poll interval (log2 s) (rw) */ + long precision; /* clock precision (ns/us) (ro) */ + long tolerance; /* clock frequency tolerance (scaled + * PPM) (ro) */ + /* + * The following read-only structure members are used by + * the PPS signal discipline that is currently not supported. + * They are included for compatibility. + */ + long ppsfreq; /* PPS frequency (scaled PPM) (ro) */ + long jitter; /* PPS jitter (ns/us) (ro) */ + int shift; /* interval duration (s) (shift) (ro) */ + long stabil; /* PPS stability (scaled PPM) (ro) */ + long jitcnt; /* jitter limit exceeded (ro) */ + long calcnt; /* calibration intervals (ro) */ + long errcnt; /* calibration errors (ro) */ + long stbcnt; /* stability limit exceeded (ro) */ +}; + +#ifdef KERNEL +#ifdef XNU_KERNEL_PRIVATE +#include <sys/_types/_user32_timex.h> +#include <sys/_types/_user64_timex.h> +#include <sys/_types/_user32_ntptimeval.h> +#include <sys/_types/_user64_ntptimeval.h> +#include <kern/clock.h> + +int64_t ntp_get_freq(void); +void ntp_update_second(int64_t *adjustment, clock_sec_t secs); +void ntp_init(void); +#endif +#else /* !_KERNEL */ +#include <sys/cdefs.h> + +__BEGIN_DECLS +#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE) +int ntp_adjtime(struct timex *); +int ntp_gettime(struct ntptimeval *); +#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */ +__END_DECLS +#endif /* KERNEL */ + + +#endif /* !_SYS_TIMEX_H_ */ diff --git a/bsd/sys/tprintf.h b/bsd/sys/tprintf.h index cf066876d..a9d704a4e 100644 --- a/bsd/sys/tprintf.h +++ b/bsd/sys/tprintf.h @@ -74,7 +74,7 @@ typedef struct session *tpr_t; __BEGIN_DECLS tpr_t tprintf_open(struct proc *); void tprintf_close(tpr_t); -void tprintf(tpr_t, const char *fmt, ...); +void tprintf(tpr_t, const char *fmt, ...) __printflike(2,3); __END_DECLS #endif /* __APPLE_API_UNSTABLE */ diff --git a/bsd/sys/tty.h b/bsd/sys/tty.h index 7fdcd02c7..fb9bcbfd7 100644 --- a/bsd/sys/tty.h +++ b/bsd/sys/tty.h @@ -103,7 +103,6 @@ struct clist { #define TTYCLSIZE 1024 #endif - /* * Per-tty structure. * @@ -331,9 +330,18 @@ void ttyfree(struct tty *); void ttysetpgrphup(struct tty *tp); void ttyclrpgrphup(struct tty *tp); +#ifdef XNU_KERNEL_PRIVATE +extern void ttyhold(struct tty *tp); + +#define TTY_LOCK_OWNED(tp) LCK_MTX_ASSERT(&tp->t_lock, LCK_MTX_ASSERT_OWNED) +#define TTY_LOCK_NOTOWNED(tp) LCK_MTX_ASSERT(&tp->t_lock, LCK_MTX_ASSERT_NOTOWNED) + +#define PTS_MAJOR 4 +#define PTC_MAJOR 5 +#endif /* defined(XNU_KERNEL_PRIVATE) */ + __END_DECLS #endif /* KERNEL */ - #endif /* !_SYS_TTY_H_ */ diff --git a/bsd/sys/types.h b/bsd/sys/types.h index a9fc63938..f11f23ff2 100644 --- a/bsd/sys/types.h +++ b/bsd/sys/types.h @@ -81,9 +81,9 @@ #include <machine/endian.h> #if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE) -typedef unsigned char u_char; -typedef unsigned short u_short; -typedef unsigned int u_int; +#include <sys/_types/_u_char.h> +#include <sys/_types/_u_short.h> +#include <sys/_types/_u_int.h> #ifndef _U_LONG typedef unsigned long u_long; #define _U_LONG @@ -96,7 +96,8 @@ typedef u_int64_t u_quad_t; /* quads */ typedef int64_t quad_t; typedef quad_t * qaddr_t; -typedef char * caddr_t; /* core address */ +#include <sys/_types/_caddr_t.h> /* core address */ + typedef int32_t daddr_t; /* disk address */ #include <sys/_types/_dev_t.h> /* device number */ diff --git a/bsd/sys/ubc.h b/bsd/sys/ubc.h index 278c84889..0699b5b03 100644 --- a/bsd/sys/ubc.h +++ b/bsd/sys/ubc.h @@ -129,7 +129,9 @@ cl_direct_read_lock_t *cluster_lock_direct_read(vnode_t vp, lck_rw_type_t exclus void cluster_unlock_direct_read(cl_direct_read_lock_t *lck); /* UPL routines */ +#ifndef XNU_KERNEL_PRIVATE int ubc_create_upl(vnode_t, off_t, int, upl_t *, upl_page_info_t **, int); +#endif /* XNU_KERNEL_PRIVATE */ int ubc_upl_map(upl_t, vm_offset_t *); int ubc_upl_unmap(upl_t); int ubc_upl_commit(upl_t); @@ -147,6 +149,11 @@ errno_t mach_to_bsd_errno(kern_return_t mach_err); #ifdef KERNEL_PRIVATE +int ubc_create_upl_external(vnode_t, off_t, int, upl_t *, upl_page_info_t **, int); +#ifdef XNU_KERNEL_PRIVATE +int ubc_create_upl_kernel(vnode_t, off_t, int, upl_t *, upl_page_info_t **, int, vm_tag_t); +#endif /* XNU_KERNEL_PRIVATE */ + __attribute__((pure)) boolean_t ubc_is_mapped(const struct vnode *, boolean_t *writable); __attribute__((pure)) boolean_t ubc_is_mapped_writable(const struct vnode *); diff --git a/bsd/sys/ubc_internal.h b/bsd/sys/ubc_internal.h index ba3d848c1..3724348ad 100644 --- a/bsd/sys/ubc_internal.h +++ b/bsd/sys/ubc_internal.h @@ -96,6 +96,8 @@ struct cl_writebehind { struct cs_hash; +uint8_t cs_hash_type(struct cs_hash const *); + struct cs_blob { struct cs_blob *csb_next; cpu_type_t csb_cpu_type; @@ -116,6 +118,9 @@ struct cs_blob { const char *csb_teamid; const CS_GenericBlob *csb_entitlements_blob; /* raw blob, subrange of csb_mem_kaddr */ void * csb_entitlements; /* The entitlements as an OSDictionary */ + unsigned int csb_signer_type; + + /* The following two will be replaced by the csb_signer_type. */ unsigned int csb_platform_binary:1; unsigned int csb_platform_path:1; }; diff --git a/bsd/sys/unistd.h b/bsd/sys/unistd.h index b27ab38f8..e373f2feb 100644 --- a/bsd/sys/unistd.h +++ b/bsd/sys/unistd.h @@ -180,11 +180,14 @@ struct accessx_descriptor { #include <machine/_types.h> #include <sys/_types/_size_t.h> #include <_types/_uint64_t.h> +#include <_types/_uint32_t.h> #include <Availability.h> __BEGIN_DECLS -int getattrlistbulk(int, void *, void *, size_t, uint64_t) __OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0); +int getattrlistbulk(int, void *, void *, size_t, uint64_t) __OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0); +int getattrlistat(int, const char *, void *, void *, size_t, unsigned long) __OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0); +int setattrlistat(int, const char *, void *, void *, size_t, uint32_t) __OSX_AVAILABLE(10.13) __IOS_AVAILABLE(11.0) __TVOS_AVAILABLE(11.0) __WATCHOS_AVAILABLE(4.0); __END_DECLS @@ -208,7 +211,6 @@ int linkat(int, const char *, int, const char *, int) __OSX_AVAILABLE_STARTING(_ ssize_t readlinkat(int, const char *, char *, size_t) __OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0); int symlinkat(const char *, int, const char *) __OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0); int unlinkat(int, const char *, int) __OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0); -int getattrlistat(int, const char *, void *, void *, size_t, unsigned long) __OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_8_0); __END_DECLS diff --git a/bsd/sys/unpcb.h b/bsd/sys/unpcb.h index 936e99801..9c66ca69b 100644 --- a/bsd/sys/unpcb.h +++ b/bsd/sys/unpcb.h @@ -67,6 +67,7 @@ #include <sys/queue.h> #include <sys/un.h> #include <sys/ucred.h> +#include <sys/socketvar.h> /* * Protocol control block for an active @@ -203,6 +204,7 @@ struct xunpcb { u_quad_t xu_alignment_hack; }; +#if !CONFIG_EMBEDDED struct xunpcb64_list_entry { u_int64_t le_next; @@ -236,6 +238,7 @@ struct xunpcb64 { struct xsocket64 xu_socket; }; +#endif /* !CONFIG_EMBEDDED */ #pragma pack() diff --git a/bsd/sys/user.h b/bsd/sys/user.h index 588a5e642..92b235bb9 100644 --- a/bsd/sys/user.h +++ b/bsd/sys/user.h @@ -75,7 +75,9 @@ struct waitq_set; #include <sys/uio.h> #endif #ifdef XNU_KERNEL_PRIVATE +#include <sys/resource.h> #include <sys/resourcevar.h> +#include <sys/signal.h> #include <sys/signalvar.h> #endif #include <sys/vm.h> /* XXX */ @@ -84,6 +86,7 @@ struct waitq_set; #ifdef KERNEL #ifdef BSD_KERNEL_PRIVATE #include <sys/pthread_internal.h> /* for uu_kwe entry */ +#include <sys/eventvar.h> #endif /* BSD_KERNEL_PRIVATE */ #ifdef __APPLE_API_PRIVATE #include <sys/eventvar.h> @@ -134,14 +137,13 @@ struct uthread { kevent_callback_t call; /* per-event callback */ kqueue_continue_t cont; /* whole call continuation */ filt_process_data_t process_data; /* needed for filter processing */ - uint8_t servicer_qos_index; /* requested qos index of servicer */ uint64_t deadline; /* computed deadline for operation */ void *data; /* caller's private data */ } ss_kqueue_scan; /* saved state for kevent_scan() */ struct _kevent { struct _kqueue_scan scan; /* space for the generic data */ struct fileproc *fp; /* fileproc we hold iocount on */ - int fd; /* filedescriptor for kq */ + int fd; /* fd for fileproc (if held) */ int eventcount; /* user-level event count */ int eventout; /* number of events output */ struct filt_process_s process_data; /* space for process data fed thru */ @@ -216,8 +218,10 @@ struct uthread { int uu_dupfd; /* fd in fdesc_open/dupfdopen */ int uu_defer_reclaims; - unsigned int uu_kqueue_bound; /* qos index we are bound to service */ - unsigned int uu_kqueue_flags; /* if so, the flags being using */ + struct kqueue *uu_kqueue_bound; /* kqueue we are bound to service */ + unsigned int uu_kqueue_qos_index; /* qos index we are bound to service */ + unsigned int uu_kqueue_flags; /* the flags we are using */ + boolean_t uu_kqueue_override_is_sync; /* sync qos override applied to servicer */ #ifdef JOE_DEBUG int uu_iocount; diff --git a/bsd/sys/vm.h b/bsd/sys/vm.h index a4e3df795..752ef89b5 100644 --- a/bsd/sys/vm.h +++ b/bsd/sys/vm.h @@ -136,6 +136,10 @@ struct user_vmspace { #include <kern/thread.h> #else /* BSD_KERNEL_PRIVATE */ + +#include <sys/_types/_caddr_t.h> /* caddr_t */ +#include <sys/_types/_int32_t.h> /* int32_t */ + /* just to keep kinfo_proc happy */ /* NOTE: Pointer fields are size variant for LP64 */ struct vmspace { diff --git a/bsd/sys/vnode.h b/bsd/sys/vnode.h index 0baabcbd7..74e0704e8 100644 --- a/bsd/sys/vnode.h +++ b/bsd/sys/vnode.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -117,7 +117,7 @@ enum vtagtype { */ #define VNODE_READ 0x01 #define VNODE_WRITE 0x02 - +#define VNODE_BLOCKMAP_NO_TRACK 0x04 // APFS Fusion: Do not track this request /* flags for VNOP_ALLOCATE */ @@ -551,6 +551,55 @@ struct vnode_trigger_param { #define VNODE_ATTR_va_private_size (1LL<<43) /* 80000000000 */ #define VNODE_ATTR_BIT(n) (VNODE_ATTR_ ## n) + +/* + * ALL of the attributes. + */ +#define VNODE_ATTR_ALL (VNODE_ATTR_BIT(va_rdev) | \ + VNODE_ATTR_BIT(va_nlink) | \ + VNODE_ATTR_BIT(va_total_size) | \ + VNODE_ATTR_BIT(va_total_alloc) | \ + VNODE_ATTR_BIT(va_data_size) | \ + VNODE_ATTR_BIT(va_data_alloc) | \ + VNODE_ATTR_BIT(va_iosize) | \ + VNODE_ATTR_BIT(va_uid) | \ + VNODE_ATTR_BIT(va_gid) | \ + VNODE_ATTR_BIT(va_mode) | \ + VNODE_ATTR_BIT(va_flags) | \ + VNODE_ATTR_BIT(va_acl) | \ + VNODE_ATTR_BIT(va_create_time) | \ + VNODE_ATTR_BIT(va_access_time) | \ + VNODE_ATTR_BIT(va_modify_time) | \ + VNODE_ATTR_BIT(va_change_time) | \ + VNODE_ATTR_BIT(va_backup_time) | \ + VNODE_ATTR_BIT(va_fileid) | \ + VNODE_ATTR_BIT(va_linkid) | \ + VNODE_ATTR_BIT(va_parentid) | \ + VNODE_ATTR_BIT(va_fsid) | \ + VNODE_ATTR_BIT(va_filerev) | \ + VNODE_ATTR_BIT(va_gen) | \ + VNODE_ATTR_BIT(va_encoding) | \ + VNODE_ATTR_BIT(va_type) | \ + VNODE_ATTR_BIT(va_name) | \ + VNODE_ATTR_BIT(va_uuuid) | \ + VNODE_ATTR_BIT(va_guuid) | \ + VNODE_ATTR_BIT(va_nchildren) | \ + VNODE_ATTR_BIT(va_dirlinkcount) | \ + VNODE_ATTR_BIT(va_addedtime) | \ + VNODE_ATTR_BIT(va_dataprotect_class) | \ + VNODE_ATTR_BIT(va_dataprotect_flags) | \ + VNODE_ATTR_BIT(va_document_id) | \ + VNODE_ATTR_BIT(va_devid) | \ + VNODE_ATTR_BIT(va_objtype) | \ + VNODE_ATTR_BIT(va_objtag) | \ + VNODE_ATTR_BIT(va_user_access) | \ + VNODE_ATTR_BIT(va_finderinfo) | \ + VNODE_ATTR_BIT(va_rsrc_length) | \ + VNODE_ATTR_BIT(va_rsrc_alloc) | \ + VNODE_ATTR_BIT(va_fsid64) | \ + VNODE_ATTR_BIT(va_write_gencount) | \ + VNODE_ATTR_BIT(va_private_size)) + /* * Read-only attributes. */ @@ -570,7 +619,6 @@ struct vnode_trigger_param { VNODE_ATTR_BIT(va_type) | \ VNODE_ATTR_BIT(va_nchildren) | \ VNODE_ATTR_BIT(va_dirlinkcount) | \ - VNODE_ATTR_BIT(va_addedtime) | \ VNODE_ATTR_BIT(va_devid) | \ VNODE_ATTR_BIT(va_objtype) | \ VNODE_ATTR_BIT(va_objtag) | \ @@ -754,7 +802,8 @@ extern int vttoif_tab[]; #define VNODE_READDIR_NAMEMAX 0x0008 /* For extended readdir, try to limit names to NAME_MAX bytes */ /* VNOP_CLONEFILE flags: */ -#define VNODE_CLONEFILE_DEFAULT 0x0000 +#define VNODE_CLONEFILE_DEFAULT 0x0000 +#define VNODE_CLONEFILE_NOOWNERCOPY 0x0001 /* Don't copy ownership information */ #define NULLVP ((struct vnode *)NULL) @@ -1045,7 +1094,6 @@ int vnode_ischr(vnode_t vp); */ int vnode_isswap(vnode_t vp); -#ifdef __APPLE_API_UNSTABLE /*! @function vnode_isnamedstream @abstract Determine if a vnode is a named stream. @@ -1053,7 +1101,6 @@ int vnode_isswap(vnode_t vp); @return Nonzero if the vnode is a named stream, 0 otherwise. */ int vnode_isnamedstream(vnode_t vp); -#endif /*! @function vnode_ismountedon @@ -1342,7 +1389,7 @@ int vfs_context_issignal(vfs_context_t ctx, sigset_t mask); @function vfs_context_suser @abstract Determine if a vfs_context_t corresponds to the superuser. @param ctx Context to examine. - @return Nonzero if context belongs to superuser, 0 otherwise. + @return 0 if context belongs to superuser, EPERM otherwise. */ int vfs_context_suser(vfs_context_t ctx); @@ -1588,6 +1635,21 @@ int vnode_isdyldsharedcache(vnode_t vp); */ int vn_getpath_fsenter(struct vnode *vp, char *pathbuf, int *len); +/*! + @function vn_getpath_fsenter_with_parent + @abstract Attempt to get a vnode's path by entering the file system if needed given a vnode and it's directory vnode. + @discussion Same as vn_getpath_fsenter but is given the directory vnode as well as the target vnode. Used +to get the path from the vnode while performing rename, rmdir, and unlink. This is done to avoid potential +dead lock if another thread is doing a forced unmount. + @param dvp Containing directory vnode. Must be holding an IO count. + @param vp Vnode whose path to get. Must be holding an IO count. + @param pathbuf Buffer in which to store path. + @param len Destination for length of resulting path string. Result will include NULL-terminator in count--that is, "len" + will be strlen(pathbuf) + 1. + @return 0 for success or an error. +*/ +int vn_getpath_fsenter_with_parent(struct vnode *dvp, struct vnode *vp, char *pathbuf, int *len); + #endif /* KERNEL_PRIVATE */ #define VNODE_UPDATE_PARENT 0x01 diff --git a/bsd/sys/vnode_if.h b/bsd/sys/vnode_if.h index 44dd90957..75f5cdd7e 100644 --- a/bsd/sys/vnode_if.h +++ b/bsd/sys/vnode_if.h @@ -1374,6 +1374,7 @@ struct vnop_clonefile_args { vnode_t sdvp, /* source directory vnode pointer (optional) */ mount_t mp, /* mount point of filesystem */ dir_clone_authorizer_op_t vattr_op, /* specific operation requested : setup, authorization or cleanup */ + uint32_t flags, /* needs to have the value passed to a_flags */ vfs_context_t ctx, /* As passed to VNOP */ void *reserved); /* Always NULL */ void *a_reserved; /* Currently unused */ @@ -1690,6 +1691,9 @@ errno_t VNOP_SETLABEL(vnode_t, struct label *, vfs_context_t); enum nsoperation { NS_OPEN, NS_CREATE, NS_DELETE }; +/* a_flags for vnop_getnamedstream_args: */ +#define NS_GETRAWENCRYPTED 0x00000001 + struct vnop_getnamedstream_args { struct vnodeop_desc *a_desc; vnode_t a_vp; @@ -1712,7 +1716,7 @@ struct vnop_getnamedstream_args { @param operation Operation to perform. In HFS and AFP, this parameter is only considered as follows: if the resource fork has not been opened and the operation is not NS_OPEN, fail with ENOATTR. Currently only passed as NS_OPEN by VFS. - @param flags Currently unused. + @param flags Flags used to control getnamedstream behavior. Currently only used for raw-encrypted-requests. @param ctx Context to authenticate for getting named stream. @return 0 for success, else an error code. */ diff --git a/bsd/sys/vnode_internal.h b/bsd/sys/vnode_internal.h index bcc58053b..d06102237 100644 --- a/bsd/sys/vnode_internal.h +++ b/bsd/sys/vnode_internal.h @@ -451,7 +451,7 @@ int vn_authorize_mkdir(vnode_t, struct componentname *, struct vnode_attr *, vfs int vn_authorize_null(vnode_t, struct componentname *, struct vnode_attr *, vfs_context_t, void*); int vnode_attr_authorize_dir_clone(struct vnode_attr *vap, kauth_action_t action, struct vnode_attr *dvap, vnode_t sdvp, mount_t mp, dir_clone_authorizer_op_t vattr_op, - vfs_context_t ctx, void *reserved); + uint32_t flags, vfs_context_t ctx, void *reserved); /* End of authorization subroutines */ #define VN_CREATE_NOAUTH (1<<0) @@ -602,6 +602,8 @@ void vnode_trigger_rearm(vnode_t, vfs_context_t); void vfs_nested_trigger_unmounts(mount_t, int, vfs_context_t); #endif /* CONFIG_TRIGGERS */ +int build_path_with_parent(vnode_t, vnode_t /* parent */, char *, int, int *, int, vfs_context_t); + #endif /* BSD_KERNEL_PRIVATE */ #endif /* !_SYS_VNODE_INTERNAL_H_ */ diff --git a/bsd/sys/work_interval.h b/bsd/sys/work_interval.h index cc9ba9fb7..f7e46ec08 100644 --- a/bsd/sys/work_interval.h +++ b/bsd/sys/work_interval.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015 Apple Inc. All rights reserved. + * Copyright (c) 2015-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -30,12 +30,17 @@ #define _SYS_WORK_INTERVAL_H #include <stdint.h> -#include <sys/types.h> #include <sys/cdefs.h> +#include <sys/_types/_size_t.h> + +#include <mach/port.h> __BEGIN_DECLS /* + * A work interval is a repeatable unit of work characterized by a + * start, finish, and deadline. + * * Trusted clients with deadline-sensitive work may report information * about the execution of their work using the work interval facility. * This is intended to be a higher-level semantic than realtime scheduling, @@ -97,45 +102,151 @@ __BEGIN_DECLS * Failure to do so will adversely impact system power and performance. * */ + +/* Flags to be passed with work_interval_create() */ + +/* If interval is joinable, create no longer implicitly joins, you must use work_interval_join */ +#define WORK_INTERVAL_FLAG_JOINABLE (0x1) +/* Only threads that join the group are measured together, otherwise the group is the creator's home group */ +#define WORK_INTERVAL_FLAG_GROUP (0x2) + +/* Flags to describe the interval flavor to the performance controller */ +#define WORK_INTERVAL_TYPE_MASK (0xF0000000) +#define WORK_INTERVAL_TYPE_DEFAULT (0x0 << 28) +#define WORK_INTERVAL_TYPE_COREAUDIO (0x1 << 28) +#define WORK_INTERVAL_TYPE_COREANIMATION (0x2 << 28) +#define WORK_INTERVAL_TYPE_LAST (0xF << 28) + #ifndef KERNEL typedef struct work_interval *work_interval_t; -/* Create a new work interval handle (currently for the current thread only). Flags is unused */ -int work_interval_create(work_interval_t *interval_handle, uint32_t flags); +/* + * Create a new work interval handle. + * + * May fail with EALREADY if the current group already has a work interval. + * + * With no flags: + * Auto-joins the work interval to the creating thread + * May only use interval_handle from creating thread + * Data provided affects native thread group + * + * With the JOINABLE flag + * interval_handle is usable by the process + * creating thread does not auto-join + * notifying thread must have joined when notifying + * + * With the GROUP flag + * creates a new thread group to isolate the joined threads from + * the rest of the process for performance controller analysis + * Threads which join the work interval become members of this new group + * + * TODO: Add a name parameter so that clients can name the work interval + * Can also take the thread name from the notifying thread + * + * Requires the 'com.apple.private.kernel.work-interval' entitlement (PRIV_WORK_INTERVAL) + * + * Note that joining a work interval supersedes automatic thread group management via vouchers + */ +int work_interval_create(work_interval_t *interval_handle, uint32_t flags); -/* Notify the power management subsystem that the work for a current interval has completed */ -int work_interval_notify(work_interval_t interval_handle, uint64_t start, uint64_t finish, uint64_t deadline, uint64_t next_start, uint32_t flags); +/* + * Notify the power management subsystem that the work for a current interval has completed + * + * Only the process which created the work interval may notify + */ +int work_interval_notify(work_interval_t interval_handle, + uint64_t start, uint64_t finish, + uint64_t deadline, uint64_t next_start, + uint32_t flags); -/* Notify, with "finish" implicitly set to the current time */ -int work_interval_notify_simple(work_interval_t interval_handle, uint64_t start, uint64_t deadline, uint64_t next_start); +/* + * Notify, with "finish" implicitly set to the current time + * + * Only the process which created the work interval may notify + */ +int work_interval_notify_simple(work_interval_t interval_handle, + uint64_t start, uint64_t deadline, + uint64_t next_start); -/* Deallocate work interval (currently for the current thread only) */ -int work_interval_destroy(work_interval_t interval_handle); +/* + * Deallocate work interval handle + * For non-JOINABLE, also removes thread from work interval + * For JOINABLE, does not remove thread (needs a leave as well) + */ +int work_interval_destroy(work_interval_t interval_handle); -#endif /* KERNEL */ +/* + * Join work interval via work interval handle + * Only allowed if interval is using the joinable and group flags + * + * Supersedes automatic thread group management via vouchers + */ +int work_interval_join(work_interval_t interval_handle); + +/* + * extract Mach send right representing work interval thread group + * Returns a +1 send right ref, which must be deallocated via mach_port_deallocate + * Only allowed if interval is joinable, otherwise returns ENOTSUP + * + * Supersedes automatic thread group management via vouchers + */ +int work_interval_copy_port(work_interval_t interval_handle, mach_port_t *port); + +/* + * Join work interval via Mach send right + * + * Does NOT consume Mach send right, must deallocate with mach_port_deallocate after using + * It's safe to deallocate the right after joining, the thread will stay joined + * + * Can be sent to clients via xpc_dictionary_copy_mach_send, and similar + * + * Supersedes automatic thread group management via vouchers + * + * If the underlying work interval object is terminated then this may return ENOENT + * <rdar://problem/31819320> + */ +int work_interval_join_port(mach_port_t port); + +/* + * Leave the current thread's work interval + */ +int work_interval_leave(void); + +/* TODO: complexity measure <rdar://problem/31586510> */ + +#endif /* !KERNEL */ #if PRIVATE /* Private interface between Libsyscall and xnu */ -#define WORK_INTERVAL_OPERATION_CREATE 0x00000001 /* arg is a uint64_t * that accepts work interval ID as an OUT param */ -#define WORK_INTERVAL_OPERATION_DESTROY 0x00000002 -#define WORK_INTERVAL_OPERATION_NOTIFY 0x00000003 /* arg is a work_interval_notification_t */ +#define WORK_INTERVAL_OPERATION_CREATE 0x00000001 /* deprecated */ +#define WORK_INTERVAL_OPERATION_DESTROY 0x00000002 /* arg is NULL */ +#define WORK_INTERVAL_OPERATION_NOTIFY 0x00000003 /* arg is a work_interval_notification_t */ +#define WORK_INTERVAL_OPERATION_CREATE2 0x00000004 /* arg is a work_interval_create_params */ +#define WORK_INTERVAL_OPERATION_JOIN 0x00000005 /* arg is a port_name */ struct work_interval_notification { uint64_t start; uint64_t finish; uint64_t deadline; uint64_t next_start; - uint32_t flags; - uint32_t unused1; + uint32_t notify_flags; + uint32_t create_flags; }; typedef struct work_interval_notification *work_interval_notification_t; -int __work_interval_ctl(uint32_t operation, uint64_t work_interval_id, void *arg, size_t len); +struct work_interval_create_params { + uint64_t wicp_id; /* out param */ + uint32_t wicp_port; /* out param */ + uint32_t wicp_create_flags; +}; + +int __work_interval_ctl(uint32_t operation, uint64_t work_interval_id, void *arg, size_t len); #endif /* PRIVATE */ __END_DECLS #endif /* _SYS_WORK_INTERVAL_H */ + diff --git a/bsd/vfs/Makefile b/bsd/vfs/Makefile index ee71a736c..d9223229e 100644 --- a/bsd/vfs/Makefile +++ b/bsd/vfs/Makefile @@ -13,7 +13,9 @@ INSTALL_MI_LIST = ${DATAFILES} INSTALL_MI_DIR = vfs -EXPORT_MI_LIST = ${DATAFILES} +INSTALL_KF_MI_LIST = ${DATAFILES} + +EXPORT_MI_LIST = ${DATAFILES} vfs_disk_conditioner.h EXPORT_MI_DIR = vfs diff --git a/bsd/vfs/kpi_vfs.c b/bsd/vfs/kpi_vfs.c index dd3560fed..060866928 100644 --- a/bsd/vfs/kpi_vfs.c +++ b/bsd/vfs/kpi_vfs.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -2168,6 +2168,11 @@ vnode_get_filesec(vnode_t vp, kauth_filesec_t *fsecp, vfs_context_t ctx) /* how many entries would fit? */ fsec_size = KAUTH_FILESEC_COUNT(xsize); + if (fsec_size > KAUTH_ACL_MAX_ENTRIES) { + KAUTH_DEBUG(" ERROR - Bogus (too large) kauth_fiilesec_t: %ld bytes", xsize); + error = 0; + goto out; + } /* get buffer and uio */ if (((fsec = kauth_filesec_alloc(fsec_size)) == NULL) || @@ -2314,6 +2319,7 @@ out: /* * Returns: 0 Success * ENOMEM Not enough space [only if has filesec] + * EINVAL Requested unknown attributes * VNOP_GETATTR: ??? * vnode_get_filesec: ??? * kauth_cred_guid2uid: ??? @@ -2329,6 +2335,12 @@ vnode_getattr(vnode_t vp, struct vnode_attr *vap, vfs_context_t ctx) uid_t nuid; gid_t ngid; + /* + * Reject attempts to fetch unknown attributes. + */ + if (vap->va_active & ~VNODE_ATTR_ALL) + return (EINVAL); + /* don't ask for extended security data if the filesystem doesn't support it */ if (!vfs_extendedsecurity(vnode_mount(vp))) { VATTR_CLEAR_ACTIVE(vap, va_acl); @@ -2555,7 +2567,18 @@ out: int vnode_setattr(vnode_t vp, struct vnode_attr *vap, vfs_context_t ctx) { - int error, is_perm_change=0; + int error; +#if CONFIG_FSE + uint64_t active; + int is_perm_change = 0; + int is_stat_change = 0; +#endif + + /* + * Reject attempts to set unknown attributes. + */ + if (vap->va_active & ~VNODE_ATTR_ALL) + return (EINVAL); /* * Make sure the filesystem is mounted R/W. @@ -2615,11 +2638,6 @@ vnode_setattr(vnode_t vp, struct vnode_attr *vap, vfs_context_t ctx) VATTR_CLEAR_ACTIVE(vap, va_gid); } - if ( VATTR_IS_ACTIVE(vap, va_uid) || VATTR_IS_ACTIVE(vap, va_gid) - || VATTR_IS_ACTIVE(vap, va_mode) || VATTR_IS_ACTIVE(vap, va_acl)) { - is_perm_change = 1; - } - /* * Make sure that extended security is enabled if we're going to try * to set any. @@ -2636,23 +2654,55 @@ vnode_setattr(vnode_t vp, struct vnode_attr *vap, vfs_context_t ctx) vap->va_flags &= (SF_SUPPORTED | UF_SETTABLE); } +#if CONFIG_FSE + /* + * Remember all of the active attributes that we're + * attempting to modify. + */ + active = vap->va_active & ~VNODE_ATTR_RDONLY; +#endif + error = VNOP_SETATTR(vp, vap, ctx); if ((error == 0) && !VATTR_ALL_SUPPORTED(vap)) error = vnode_setattr_fallback(vp, vap, ctx); #if CONFIG_FSE - // only send a stat_changed event if this is more than - // just an access or backup time update - if (error == 0 && (vap->va_active != VNODE_ATTR_BIT(va_access_time)) && (vap->va_active != VNODE_ATTR_BIT(va_backup_time))) { +#define PERMISSION_BITS (VNODE_ATTR_BIT(va_uid) | VNODE_ATTR_BIT(va_uuuid) | \ + VNODE_ATTR_BIT(va_gid) | VNODE_ATTR_BIT(va_guuid) | \ + VNODE_ATTR_BIT(va_mode) | VNODE_ATTR_BIT(va_acl)) + + /* + * Now that we've changed them, decide whether to send an + * FSevent. + */ + if ((active & PERMISSION_BITS) & vap->va_supported) { + is_perm_change = 1; + } else { + /* + * We've already checked the permission bits, and we + * also want to filter out access time / backup time + * changes. + */ + active &= ~(PERMISSION_BITS | + VNODE_ATTR_BIT(va_access_time) | + VNODE_ATTR_BIT(va_backup_time)); + + /* Anything left to notify about? */ + if (active & vap->va_supported) + is_stat_change = 1; + } + + if (error == 0) { if (is_perm_change) { if (need_fsevent(FSE_CHOWN, vp)) { add_fsevent(FSE_CHOWN, ctx, FSE_ARG_VNODE, vp, FSE_ARG_DONE); } - } else if(need_fsevent(FSE_STAT_CHANGED, vp)) { + } else if (is_stat_change && need_fsevent(FSE_STAT_CHANGED, vp)) { add_fsevent(FSE_STAT_CHANGED, ctx, FSE_ARG_VNODE, vp, FSE_ARG_DONE); } } +#undef PERMISSION_BITS #endif out: @@ -4013,14 +4063,17 @@ vn_rename(struct vnode *fdvp, struct vnode **fvpp, struct componentname *fcnp, s */ if (_err == 0) { _err = vnode_flags(tdvp, &tdfflags, ctx); - if (_err == 0 && (tdfflags & SF_RESTRICTED)) { - uint32_t fflags; - _err = vnode_flags(*fvpp, &fflags, ctx); - if (_err == 0 && !(fflags & SF_RESTRICTED)) { - struct vnode_attr va; - VATTR_INIT(&va); - VATTR_SET(&va, va_flags, fflags | SF_RESTRICTED); - _err = vnode_setattr(*fvpp, &va, ctx); + if (_err == 0) { + uint32_t inherit_flags = tdfflags & (UF_DATAVAULT | SF_RESTRICTED); + if (inherit_flags) { + uint32_t fflags; + _err = vnode_flags(*fvpp, &fflags, ctx); + if (_err == 0 && fflags != (fflags | inherit_flags)) { + struct vnode_attr va; + VATTR_INIT(&va); + VATTR_SET(&va, va_flags, fflags | inherit_flags); + _err = vnode_setattr(*fvpp, &va, ctx); + } } } } @@ -5383,6 +5436,7 @@ struct vnop_clonefile_args { vnode_t sdvp, /* source directory vnode pointer (optional) */ mount_t mp, /* mount point of filesystem */ dir_clone_authorizer_op_t vattr_op, /* specific operation requested : setup, authorization or cleanup */ + uint32_t flags; /* value passed in a_flags to the VNOP */ vfs_context_t ctx, /* As passed to VNOP */ void *reserved); /* Always NULL */ void *a_reserved; /* Currently unused */ diff --git a/bsd/vfs/vfs_attrlist.c b/bsd/vfs/vfs_attrlist.c index 51eb4756f..4aab3ac01 100644 --- a/bsd/vfs/vfs_attrlist.c +++ b/bsd/vfs/vfs_attrlist.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 1995-2016 Apple Inc. All rights reserved. + * Copyright (c) 1995-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -51,6 +51,7 @@ #include <sys/fsevents.h> #include <kern/kalloc.h> #include <miscfs/specfs/specdev.h> +#include <security/audit/audit.h> #if CONFIG_MACF #include <security/mac_framework.h> @@ -269,7 +270,7 @@ attrlist_pack_string(struct _attrlist_buf *ab, const char *source, ssize_t count #define ATTR_PACK8(AB, V) \ do { \ if ((AB.allocated - (AB.fixedcursor - AB.base)) >= 8) { \ - *(uint64_t *)AB.fixedcursor = *(uint64_t *)&V; \ + memcpy(AB.fixedcursor, &V, 8); \ AB.fixedcursor += 8; \ } \ } while (0) @@ -356,8 +357,8 @@ static struct getvolattrlist_attrtab getvolattrlist_vol_tab[] = { {ATTR_VOL_ENCODINGSUSED, 0, sizeof(uint64_t)}, {ATTR_VOL_CAPABILITIES, VFSATTR_BIT(f_capabilities), sizeof(vol_capabilities_attr_t)}, {ATTR_VOL_UUID, VFSATTR_BIT(f_uuid), sizeof(uuid_t)}, - {ATTR_VOL_QUOTA_SIZE, VFSATTR_BIT(f_quota), sizeof(off_t)}, - {ATTR_VOL_RESERVED_SIZE, VFSATTR_BIT(f_reserved), sizeof(off_t)}, + {ATTR_VOL_QUOTA_SIZE, VFSATTR_BIT(f_quota) | VFSATTR_BIT(f_bsize), sizeof(off_t)}, + {ATTR_VOL_RESERVED_SIZE, VFSATTR_BIT(f_reserved) | VFSATTR_BIT(f_bsize), sizeof(off_t)}, {ATTR_VOL_ATTRIBUTES, VFSATTR_BIT(f_attributes), sizeof(vol_attributes_attr_t)}, {ATTR_VOL_INFO, 0, 0}, {0, 0, 0} @@ -1394,6 +1395,21 @@ getvolattrlist(vfs_context_t ctx, vnode_t vp, struct attrlist *alp, vs.f_capabilities.capabilities[VOL_CAPABILITIES_INTERFACES] &= ~VOL_CAP_INT_EXTENDED_SECURITY; } vs.f_capabilities.valid[VOL_CAPABILITIES_INTERFACES] |= VOL_CAP_INT_EXTENDED_SECURITY; + + /* + * if the filesystem doesn't mark either VOL_CAP_FMT_NO_IMMUTABLE_FILES + * or VOL_CAP_FMT_NO_PERMISSIONS as valid, assume they're not supported + */ + if (!(vs.f_capabilities.valid[VOL_CAPABILITIES_FORMAT] & VOL_CAP_FMT_NO_IMMUTABLE_FILES)) { + vs.f_capabilities.capabilities[VOL_CAPABILITIES_FORMAT] &= ~VOL_CAP_FMT_NO_IMMUTABLE_FILES; + vs.f_capabilities.valid[VOL_CAPABILITIES_FORMAT] |= VOL_CAP_FMT_NO_IMMUTABLE_FILES; + } + + if (!(vs.f_capabilities.valid[VOL_CAPABILITIES_FORMAT] & VOL_CAP_FMT_NO_PERMISSIONS)) { + vs.f_capabilities.capabilities[VOL_CAPABILITIES_FORMAT] &= ~VOL_CAP_FMT_NO_PERMISSIONS; + vs.f_capabilities.valid[VOL_CAPABILITIES_FORMAT] |= VOL_CAP_FMT_NO_PERMISSIONS; + } + ATTR_PACK(&ab, vs.f_capabilities); ab.actual.volattr |= ATTR_VOL_CAPABILITIES; } @@ -2743,7 +2759,7 @@ out: static int getattrlist_internal(vfs_context_t ctx, vnode_t vp, struct attrlist *alp, user_addr_t attributeBuffer, size_t bufferSize, uint64_t options, - enum uio_seg segflg, char* alt_name, struct ucred *file_cred) + enum uio_seg segflg, char* authoritative_name, struct ucred *file_cred) { struct vnode_attr va; kauth_action_t action; @@ -2875,9 +2891,22 @@ getattrlist_internal(vfs_context_t ctx, vnode_t vp, struct attrlist *alp, VFS_DEBUG(ctx, vp, "ATTRLIST - ERROR: cannot allocate va_name buffer"); goto out; } + /* + * If we have an authoritative_name, prefer that name. + * + * N.B. Since authoritative_name implies this is coming from getattrlistbulk, + * we know the name is authoritative. For /dev/fd, we want to use the file + * descriptor as the name not the underlying name of the associate vnode in a + * particular file system. + */ + if (authoritative_name) { + /* Don't ask the file system */ + VATTR_CLEAR_ACTIVE(&va, va_name); + strlcpy(va_name, authoritative_name, MAXPATHLEN); + } } - va.va_name = va_name; + va.va_name = authoritative_name ? NULL : va_name; /* * Call the filesystem. @@ -2907,16 +2936,19 @@ getattrlist_internal(vfs_context_t ctx, vnode_t vp, struct attrlist *alp, #endif /* - * If ATTR_CMN_NAME is not supported by filesystem and the - * caller has provided a name, use that. + * It we ask for the name, i.e., vname is non null and + * we have an authoritative name, then reset va_name is + * active and if needed set va_name is supported. + * * A (buggy) filesystem may change fields which belong * to us. We try to deal with that here as well. */ va.va_active = va_active; - if (alt_name && va_name && - !(VATTR_IS_SUPPORTED(&va, va_name))) { - strlcpy(va_name, alt_name, MAXPATHLEN); - VATTR_SET_SUPPORTED(&va, va_name); + if (authoritative_name && va_name) { + VATTR_SET_ACTIVE(&va, va_name); + if (!(VATTR_IS_SUPPORTED(&va, va_name))) { + VATTR_SET_SUPPORTED(&va, va_name); + } } va.va_name = va_name; } @@ -3680,11 +3712,6 @@ getattrlistbulk(proc_t p, struct getattrlistbulk_args *uap, int32_t *retval) if (uap->options & FSOPT_LIST_SNAPSHOT) { vnode_t snapdvp; - if (!vfs_context_issuser(ctx)) { - error = EPERM; - goto out; - } - if (!vnode_isvroot(dvp)) { error = EINVAL; goto out; @@ -4124,6 +4151,10 @@ setattrlist_internal(vnode_t vp, struct setattrlist_args *uap, proc_t p, vfs_con ATTR_UNPACK(va.va_guuid); VATTR_SET_ACTIVE(&va, va_guuid); } + if (al.commonattr & ATTR_CMN_ADDEDTIME) { + ATTR_UNPACK_TIME(va.va_addedtime, proc_is64); + VATTR_SET_ACTIVE(&va, va_addedtime); + } /* Support setattrlist of data protection class */ if (al.commonattr & ATTR_CMN_DATA_PROTECT_FLAGS) { ATTR_UNPACK(va.va_dataprotect_class); @@ -4310,6 +4341,44 @@ out: return error; } +int +setattrlistat(proc_t p, struct setattrlistat_args *uap, __unused int32_t *retval) +{ + struct setattrlist_args ap; + struct vfs_context *ctx; + struct nameidata nd; + vnode_t vp = NULLVP; + uint32_t nameiflags; + int error; + + ctx = vfs_context_current(); + + AUDIT_ARG(fd, uap->fd); + /* + * Look up the file. + */ + nameiflags = AUDITVNPATH1; + if (!(uap->options & FSOPT_NOFOLLOW)) + nameiflags |= FOLLOW; + NDINIT(&nd, LOOKUP, OP_SETATTR, nameiflags, UIO_USERSPACE, uap->path, ctx); + if ((error = nameiat(&nd, uap->fd)) != 0) + goto out; + vp = nd.ni_vp; + nameidone(&nd); + + ap.path = 0; + ap.alist = uap->alist; + ap.attributeBuffer = uap->attributeBuffer; + ap.bufferSize = uap->bufferSize; + ap.options = uap->options; + + error = setattrlist_internal(vp, &ap, p, ctx); +out: + if (vp) + vnode_put(vp); + return (error); +} + int fsetattrlist(proc_t p, struct fsetattrlist_args *uap, __unused int32_t *retval) { diff --git a/bsd/vfs/vfs_bio.c b/bsd/vfs/vfs_bio.c index cdccdc828..c1019a327 100644 --- a/bsd/vfs/vfs_bio.c +++ b/bsd/vfs/vfs_bio.c @@ -132,6 +132,8 @@ static buf_t buf_create_shadow_internal(buf_t bp, boolean_t force_copy, int bdwrite_internal(buf_t, int); +extern void disk_conditioner_delay(buf_t, int, int, uint64_t); + /* zone allocated buffer headers */ static void bufzoneinit(void); static void bcleanbuf_thread_init(void); @@ -491,10 +493,16 @@ bufattr_markmeta(bufattr_t bap) { } int +#if !CONFIG_EMBEDDED bufattr_delayidlesleep(bufattr_t bap) +#else /* !CONFIG_EMBEDDED */ +bufattr_delayidlesleep(__unused bufattr_t bap) +#endif /* !CONFIG_EMBEDDED */ { +#if !CONFIG_EMBEDDED if ( (bap->ba_flags & BA_DELAYIDLESLEEP) ) return 1; +#endif /* !CONFIG_EMBEDDED */ return 0; } @@ -2629,12 +2637,13 @@ buf_brelse(buf_t bp) if (upl == NULL) { if ( !ISSET(bp->b_flags, B_INVAL)) { - kret = ubc_create_upl(bp->b_vp, + kret = ubc_create_upl_kernel(bp->b_vp, ubc_blktooff(bp->b_vp, bp->b_lblkno), bp->b_bufsize, &upl, NULL, - UPL_PRECIOUS); + UPL_PRECIOUS, + VM_KERN_MEMORY_FILE); if (kret != KERN_SUCCESS) panic("brelse: Failed to create UPL"); @@ -3034,12 +3043,13 @@ start: case BLK_READ: upl_flags |= UPL_PRECIOUS; if (UBCINFOEXISTS(bp->b_vp) && bp->b_bufsize) { - kret = ubc_create_upl(vp, + kret = ubc_create_upl_kernel(vp, ubc_blktooff(vp, bp->b_lblkno), bp->b_bufsize, &upl, &pl, - upl_flags); + upl_flags, + VM_KERN_MEMORY_FILE); if (kret != KERN_SUCCESS) panic("Failed to create UPL"); @@ -3183,12 +3193,13 @@ start: f_offset = ubc_blktooff(vp, blkno); upl_flags |= UPL_PRECIOUS; - kret = ubc_create_upl(vp, + kret = ubc_create_upl_kernel(vp, f_offset, bp->b_bufsize, &upl, &pl, - upl_flags); + upl_flags, + VM_KERN_MEMORY_FILE); if (kret != KERN_SUCCESS) panic("Failed to create UPL"); @@ -3968,6 +3979,8 @@ buf_biodone(buf_t bp) { mount_t mp; struct bufattr *bap; + struct timeval real_elapsed; + uint64_t real_elapsed_usec = 0; KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 387)) | DBG_FUNC_START, bp, bp->b_datap, bp->b_flags, 0, 0); @@ -4035,6 +4048,11 @@ buf_biodone(buf_t bp) buf_kernel_addrperm_addr(bp), (uintptr_t)VM_KERNEL_ADDRPERM(bp->b_vp), bp->b_resid, bp->b_error, 0); } + microuptime(&real_elapsed); + timevalsub(&real_elapsed, &bp->b_timestamp_tv); + real_elapsed_usec = real_elapsed.tv_sec * USEC_PER_SEC + real_elapsed.tv_usec; + disk_conditioner_delay(bp, 1, bp->b_bcount, real_elapsed_usec); + /* * I/O was done, so don't believe * the DIRTY state from VM anymore... @@ -4484,12 +4502,13 @@ brecover_data(buf_t bp) upl_flags |= UPL_WILL_MODIFY; } - kret = ubc_create_upl(vp, + kret = ubc_create_upl_kernel(vp, ubc_blktooff(vp, bp->b_lblkno), bp->b_bufsize, &upl, &pl, - upl_flags); + upl_flags, + VM_KERN_MEMORY_FILE); if (kret != KERN_SUCCESS) panic("Failed to create UPL"); diff --git a/bsd/vfs/vfs_cache.c b/bsd/vfs/vfs_cache.c index 47cf243ba..b24dbc590 100644 --- a/bsd/vfs/vfs_cache.c +++ b/bsd/vfs/vfs_cache.c @@ -388,12 +388,12 @@ vnode_issubdir(vnode_t vp, vnode_t dvp, int *is_subdir, vfs_context_t ctx) } /* - * This function builds the path to a filename in "buff". The - * length of the buffer *INCLUDING* the trailing zero byte is - * returned in outlen. NOTE: the length includes the trailing - * zero byte and thus the length is one greater than what strlen - * would return. This is important and lots of code elsewhere - * in the kernel assumes this behavior. + * This function builds the path in "buff" from the supplied vnode. + * The length of the buffer *INCLUDING* the trailing zero byte is + * returned in outlen. NOTE: the length includes the trailing zero + * byte and thus the length is one greater than what strlen would + * return. This is important and lots of code elsewhere in the kernel + * assumes this behavior. * * This function can call vnop in file system if the parent vnode * does not exist or when called for hardlinks via volfs path. @@ -410,9 +410,19 @@ vnode_issubdir(vnode_t vp, vnode_t dvp, int *is_subdir, vfs_context_t ctx) * cross over mount points during building the path. * * passed in vp must have a valid io_count reference + * + * If parent vnode is non-NULL it also must have an io count. This + * allows build_path_with_parent to be safely called for operations + * unlink, rmdir and rename that already have io counts on the target + * and the directory. In this way build_path_with_parent does not have + * to try and obtain an additional io count on the parent. Taking an + * io count ont the parent can lead to dead lock if a forced unmount + * occures at the right moment. For a fuller explaination on how this + * can occur see the comment for vn_getpath_with_parent. + * */ int -build_path(vnode_t first_vp, char *buff, int buflen, int *outlen, int flags, vfs_context_t ctx) +build_path_with_parent(vnode_t first_vp, vnode_t parent_vp, char *buff, int buflen, int *outlen, int flags, vfs_context_t ctx) { vnode_t vp, tvp; vnode_t vp_with_iocount; @@ -587,7 +597,7 @@ again: NAME_CACHE_UNLOCK(); - if (vp != first_vp && vp != vp_with_iocount) { + if (vp != first_vp && vp != parent_vp && vp != vp_with_iocount) { if (vp_with_iocount) { vnode_put(vp_with_iocount); vp_with_iocount = NULLVP; @@ -678,7 +688,7 @@ bad_news: NAME_CACHE_UNLOCK(); - if (vp != first_vp && vp != vp_with_iocount) { + if (vp != first_vp && vp != parent_vp && vp != vp_with_iocount) { if (vp_with_iocount) { vnode_put(vp_with_iocount); vp_with_iocount = NULLVP; @@ -745,6 +755,11 @@ out: return (ret); } +int +build_path(vnode_t first_vp, char *buff, int buflen, int *outlen, int flags, vfs_context_t ctx) +{ + return (build_path_with_parent(first_vp, NULL, buff, buflen, outlen, flags, ctx)); +} /* * return NULLVP if vp's parent doesn't @@ -1362,7 +1377,7 @@ skiprsrcfork: #if CONFIG_MACF - /* + /* * Name cache provides authorization caching (see below) * that will short circuit MAC checks in lookup(). * We must perform MAC check here. On denial @@ -1685,7 +1700,7 @@ cache_lookup_locked(vnode_t dvp, struct componentname *cnp) ncpp = NCHHASH(dvp, cnp->cn_hash); LIST_FOREACH(ncp, ncpp, nc_hash) { if ((ncp->nc_dvp == dvp) && (ncp->nc_hashval == hashval)) { - if (memcmp(ncp->nc_name, cnp->cn_nameptr, namelen) == 0 && ncp->nc_name[namelen] == 0) + if (strncmp(ncp->nc_name, cnp->cn_nameptr, namelen) == 0 && ncp->nc_name[namelen] == 0) break; } } @@ -1772,7 +1787,7 @@ relook: ncpp = NCHHASH(dvp, cnp->cn_hash); LIST_FOREACH(ncp, ncpp, nc_hash) { if ((ncp->nc_dvp == dvp) && (ncp->nc_hashval == hashval)) { - if (memcmp(ncp->nc_name, cnp->cn_nameptr, namelen) == 0 && ncp->nc_name[namelen] == 0) + if (strncmp(ncp->nc_name, cnp->cn_nameptr, namelen) == 0 && ncp->nc_name[namelen] == 0) break; } } @@ -2211,7 +2226,7 @@ resize_namecache(u_int newsize) } static void -cache_delete(struct namecache *ncp, int age_entry) +cache_delete(struct namecache *ncp, int free_entry) { NCHSTAT(ncs_deletes); @@ -2232,16 +2247,13 @@ cache_delete(struct namecache *ncp, int age_entry) */ ncp->nc_hash.le_prev = NULL; - if (age_entry) { - /* - * make it the next one available - * for cache_enter's use - */ - TAILQ_REMOVE(&nchead, ncp, nc_entry); - TAILQ_INSERT_HEAD(&nchead, ncp, nc_entry); - } vfs_removename(ncp->nc_name); ncp->nc_name = NULL; + if (free_entry) { + TAILQ_REMOVE(&nchead, ncp, nc_entry); + FREE_ZONE(ncp, sizeof(*ncp), M_CACHE); + numcache--; + } } @@ -2475,7 +2487,7 @@ add_name_internal(const char *name, uint32_t len, u_int hashval, boolean_t need_ lck_mtx_lock_spin(&strcache_mtx_locks[lock_index]); for (entry = head->lh_first; entry != NULL; chain_len++, entry = entry->hash_chain.le_next) { - if (memcmp(entry->str, name, len) == 0 && entry->str[len] == 0) { + if (strncmp(entry->str, name, len) == 0 && entry->str[len] == 0) { entry->refcount++; break; } diff --git a/bsd/vfs/vfs_cluster.c b/bsd/vfs/vfs_cluster.c index 70eecc5ff..df1e9569f 100644 --- a/bsd/vfs/vfs_cluster.c +++ b/bsd/vfs/vfs_cluster.c @@ -98,6 +98,8 @@ #include <stdbool.h> +#include <vfs/vfs_disk_conditioner.h> + #if 0 #undef KERNEL_DEBUG #define KERNEL_DEBUG KERNEL_DEBUG_CONSTANT @@ -266,17 +268,23 @@ int (*bootcache_contains_block)(dev_t device, u_int64_t blkno) = NULL; #define WRITE_BEHIND 1 #define WRITE_BEHIND_SSD 1 +#if CONFIG_EMBEDDED +#define PREFETCH 1 +#define PREFETCH_SSD 1 +uint32_t speculative_prefetch_max = (2048 * 1024); /* maximum bytes in a specluative read-ahead */ +uint32_t speculative_prefetch_max_iosize = (512 * 1024); /* maximum I/O size to use in a specluative read-ahead */ +#else #define PREFETCH 3 #define PREFETCH_SSD 2 uint32_t speculative_prefetch_max = (MAX_UPL_SIZE_BYTES * 3); /* maximum bytes in a specluative read-ahead */ uint32_t speculative_prefetch_max_iosize = (512 * 1024); /* maximum I/O size to use in a specluative read-ahead on SSDs*/ +#endif #define IO_SCALE(vp, base) (vp->v_mount->mnt_ioscale * (base)) #define MAX_CLUSTER_SIZE(vp) (cluster_max_io_size(vp->v_mount, CL_WRITE)) -#define MAX_PREFETCH(vp, size, is_ssd) (size * IO_SCALE(vp, ((is_ssd && !ignore_is_ssd) ? PREFETCH_SSD : PREFETCH))) +#define MAX_PREFETCH(vp, size, is_ssd) (size * IO_SCALE(vp, ((is_ssd) ? PREFETCH_SSD : PREFETCH))) -int ignore_is_ssd = 0; int speculative_reads_disabled = 0; /* @@ -494,8 +502,8 @@ cluster_io_present_in_BC(vnode_t vp, off_t f_offset) size_t io_size; int (*bootcache_check_fn)(dev_t device, u_int64_t blkno) = bootcache_contains_block; - if (bootcache_check_fn) { - if (VNOP_BLOCKMAP(vp, f_offset, PAGE_SIZE, &blkno, &io_size, NULL, VNODE_READ, NULL)) + if (bootcache_check_fn && vp->v_mount && vp->v_mount->mnt_devvp) { + if (VNOP_BLOCKMAP(vp, f_offset, PAGE_SIZE, &blkno, &io_size, NULL, VNODE_READ | VNODE_BLOCKMAP_NO_TRACK, NULL)) return(0); if (io_size == 0) @@ -1189,7 +1197,7 @@ cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int no } else { max_cluster_size = MAX_CLUSTER_SIZE(vp); - if ((vp->v_mount->mnt_kern_flag & MNTK_SSD) && !ignore_is_ssd) + if (disk_conditioner_mount_is_ssd(vp->v_mount)) scale = WRITE_THROTTLE_SSD; else scale = WRITE_THROTTLE; @@ -1249,8 +1257,8 @@ cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int no * Create a UPL to lock the pages in the cache whilst the * write is in progress. */ - ubc_create_upl(vp, f_offset, non_rounded_size, &cached_upl, - NULL, UPL_SET_LITE); + ubc_create_upl_kernel(vp, f_offset, non_rounded_size, &cached_upl, + NULL, UPL_SET_LITE, VM_KERN_MEMORY_FILE); /* * Attach this UPL to the other UPL so that we can find it @@ -1971,7 +1979,7 @@ cluster_read_ahead(vnode_t vp, struct cl_extent *extent, off_t filesize, struct return; } - max_prefetch = MAX_PREFETCH(vp, cluster_max_io_size(vp->v_mount, CL_READ), (vp->v_mount->mnt_kern_flag & MNTK_SSD)); + max_prefetch = MAX_PREFETCH(vp, cluster_max_io_size(vp->v_mount, CL_READ), disk_conditioner_mount_is_ssd(vp->v_mount)); if (max_prefetch > speculative_prefetch_max) max_prefetch = speculative_prefetch_max; @@ -2516,8 +2524,7 @@ next_dwrite: pages_in_pl = 0; upl_size = upl_needed_size; upl_flags = UPL_FILE_IO | UPL_COPYOUT_FROM | UPL_NO_SYNC | - UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE - | UPL_MEMORY_TAG_MAKE(VM_KERN_MEMORY_FILE); + UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE; kret = vm_map_get_upl(map, (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)), @@ -2526,6 +2533,7 @@ next_dwrite: NULL, &pages_in_pl, &upl_flags, + VM_KERN_MEMORY_FILE, force_data_sync); if (kret != KERN_SUCCESS) { @@ -2789,13 +2797,12 @@ next_cwrite: pages_in_pl = 0; upl_size = upl_needed_size; upl_flags = UPL_FILE_IO | UPL_COPYOUT_FROM | UPL_NO_SYNC | - UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE - | UPL_MEMORY_TAG_MAKE(VM_KERN_MEMORY_FILE); + UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE; vm_map_t map = UIO_SEG_IS_USER_SPACE(uio->uio_segflg) ? current_map() : kernel_map; kret = vm_map_get_upl(map, (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)), - &upl_size, &upl[cur_upl], NULL, &pages_in_pl, &upl_flags, 0); + &upl_size, &upl[cur_upl], NULL, &pages_in_pl, &upl_flags, VM_KERN_MEMORY_FILE, 0); if (kret != KERN_SUCCESS) { /* @@ -3178,12 +3185,13 @@ cluster_write_copy(vnode_t vp, struct uio *uio, u_int32_t io_req_size, off_t old * The UPL_WILL_MODIFY flag lets the UPL subsystem know * that we intend to modify these pages. */ - kret = ubc_create_upl(vp, + kret = ubc_create_upl_kernel(vp, upl_f_offset, upl_size, &upl, &pl, - UPL_SET_LITE | (( uio!=NULL && (uio->uio_flags & UIO_FLAGS_IS_COMPRESSED_FILE)) ? 0 : UPL_WILL_MODIFY)); + UPL_SET_LITE | (( uio!=NULL && (uio->uio_flags & UIO_FLAGS_IS_COMPRESSED_FILE)) ? 0 : UPL_WILL_MODIFY), + VM_KERN_MEMORY_FILE); if (kret != KERN_SUCCESS) panic("cluster_write_copy: failed to get pagelist"); @@ -3535,7 +3543,7 @@ check_cluster: n = 0; if (n == 0) { - if (vp->v_mount->mnt_kern_flag & MNTK_SSD) + if (disk_conditioner_mount_is_ssd(vp->v_mount)) n = WRITE_BEHIND_SSD; else n = WRITE_BEHIND; @@ -3777,7 +3785,7 @@ cluster_read_copy(vnode_t vp, struct uio *uio, u_int32_t io_req_size, off_t file bflag |= CL_ENCRYPTED; max_io_size = cluster_max_io_size(vp->v_mount, CL_READ); - max_prefetch = MAX_PREFETCH(vp, max_io_size, (vp->v_mount->mnt_kern_flag & MNTK_SSD)); + max_prefetch = MAX_PREFETCH(vp, max_io_size, disk_conditioner_mount_is_ssd(vp->v_mount)); max_rd_size = max_prefetch; last_request_offset = uio->uio_offset + io_req_size; @@ -3974,12 +3982,13 @@ cluster_read_copy(vnode_t vp, struct uio *uio, u_int32_t io_req_size, off_t file KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 33)) | DBG_FUNC_START, upl, (int)upl_f_offset, upl_size, start_offset, 0); - kret = ubc_create_upl(vp, + kret = ubc_create_upl_kernel(vp, upl_f_offset, upl_size, &upl, &pl, - UPL_FILE_IO | UPL_SET_LITE); + UPL_FILE_IO | UPL_SET_LITE, + VM_KERN_MEMORY_FILE); if (kret != KERN_SUCCESS) panic("cluster_read_copy: failed to get pagelist"); @@ -4651,8 +4660,7 @@ next_dread: for (force_data_sync = 0; force_data_sync < 3; force_data_sync++) { pages_in_pl = 0; upl_size = upl_needed_size; - upl_flags = UPL_FILE_IO | UPL_NO_SYNC | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE - | UPL_MEMORY_TAG_MAKE(VM_KERN_MEMORY_FILE); + upl_flags = UPL_FILE_IO | UPL_NO_SYNC | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE; if (no_zero_fill) upl_flags |= UPL_NOZEROFILL; if (force_data_sync) @@ -4660,7 +4668,7 @@ next_dread: kret = vm_map_create_upl(map, (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)), - &upl_size, &upl, NULL, &pages_in_pl, &upl_flags); + &upl_size, &upl, NULL, &pages_in_pl, &upl_flags, VM_KERN_MEMORY_FILE); if (kret != KERN_SUCCESS) { KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 72)) | DBG_FUNC_END, @@ -4925,8 +4933,7 @@ next_cread: pages_in_pl = 0; upl_size = upl_needed_size; - upl_flags = UPL_FILE_IO | UPL_NO_SYNC | UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE - | UPL_MEMORY_TAG_MAKE(VM_KERN_MEMORY_FILE); + upl_flags = UPL_FILE_IO | UPL_NO_SYNC | UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE; KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 92)) | DBG_FUNC_START, @@ -4935,7 +4942,7 @@ next_cread: vm_map_t map = UIO_SEG_IS_USER_SPACE(uio->uio_segflg) ? current_map() : kernel_map; kret = vm_map_get_upl(map, (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)), - &upl_size, &upl[cur_upl], NULL, &pages_in_pl, &upl_flags, 0); + &upl_size, &upl[cur_upl], NULL, &pages_in_pl, &upl_flags, VM_KERN_MEMORY_FILE, 0); KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 92)) | DBG_FUNC_END, (int)upl_offset, upl_size, io_size, kret, 0); @@ -5103,12 +5110,12 @@ cluster_io_type(struct uio *uio, int *io_type, u_int32_t *io_length, u_int32_t m else upl_size = (u_int32_t)iov_len; - upl_flags = UPL_QUERY_OBJECT_TYPE | UPL_MEMORY_TAG_MAKE(VM_KERN_MEMORY_FILE); + upl_flags = UPL_QUERY_OBJECT_TYPE; vm_map_t map = UIO_SEG_IS_USER_SPACE(uio->uio_segflg) ? current_map() : kernel_map; if ((vm_map_get_upl(map, (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)), - &upl_size, &upl, NULL, NULL, &upl_flags, 0)) != KERN_SUCCESS) { + &upl_size, &upl, NULL, NULL, &upl_flags, VM_KERN_MEMORY_FILE, 0)) != KERN_SUCCESS) { /* * the user app must have passed in an invalid address */ @@ -5177,10 +5184,15 @@ advisory_read_ext(vnode_t vp, off_t filesize, off_t f_offset, int resid, int (*c max_io_size = cluster_max_io_size(vp->v_mount, CL_READ); - if ((vp->v_mount->mnt_kern_flag & MNTK_SSD) && !ignore_is_ssd) { +#if CONFIG_EMBEDDED + if (max_io_size > speculative_prefetch_max_iosize) + max_io_size = speculative_prefetch_max_iosize; +#else + if (disk_conditioner_mount_is_ssd(vp->v_mount)) { if (max_io_size > speculative_prefetch_max_iosize) max_io_size = speculative_prefetch_max_iosize; } +#endif KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 60)) | DBG_FUNC_START, (int)f_offset, resid, (int)filesize, 0, 0); @@ -5239,12 +5251,13 @@ advisory_read_ext(vnode_t vp, off_t filesize, off_t f_offset, int resid, int (*c KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 61)) | DBG_FUNC_START, upl, (int)upl_f_offset, upl_size, start_offset, 0); - kret = ubc_create_upl(vp, + kret = ubc_create_upl_kernel(vp, upl_f_offset, upl_size, &upl, &pl, - UPL_RET_ONLY_ABSENT | UPL_SET_LITE); + UPL_RET_ONLY_ABSENT | UPL_SET_LITE, + VM_KERN_MEMORY_FILE); if (kret != KERN_SUCCESS) return(retval); issued_io = 0; @@ -5754,12 +5767,13 @@ cluster_push_now(vnode_t vp, struct cl_extent *cl, off_t EOF, int flags, int (*c else upl_flags = UPL_COPYOUT_FROM | UPL_RET_ONLY_DIRTY | UPL_SET_LITE; - kret = ubc_create_upl(vp, + kret = ubc_create_upl_kernel(vp, upl_f_offset, upl_size, &upl, &pl, - upl_flags); + upl_flags, + VM_KERN_MEMORY_FILE); if (kret != KERN_SUCCESS) panic("cluster_push: failed to get pagelist"); @@ -5988,12 +6002,13 @@ cluster_align_phys_io(vnode_t vp, struct uio *uio, addr64_t usr_paddr, u_int32_t */ upl_flags |= UPL_FILE_IO; } - kret = ubc_create_upl(vp, + kret = ubc_create_upl_kernel(vp, uio->uio_offset & ~PAGE_MASK_64, PAGE_SIZE, &upl, &pl, - upl_flags); + upl_flags, + VM_KERN_MEMORY_FILE); if (kret != KERN_SUCCESS) return(EINVAL); diff --git a/bsd/vfs/vfs_disk_conditioner.c b/bsd/vfs/vfs_disk_conditioner.c new file mode 100644 index 000000000..8cc7237c6 --- /dev/null +++ b/bsd/vfs/vfs_disk_conditioner.c @@ -0,0 +1,235 @@ +/* + * Copyright (c) 2016 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include <sys/fsctl.h> +#include <stdbool.h> +#include <sys/time.h> +#include <sys/buf.h> +#include <sys/mount_internal.h> +#include <sys/vnode_internal.h> +#include <sys/buf_internal.h> + +#include <kern/kalloc.h> + +#include <sys/kauth.h> +#include <IOKit/IOBSD.h> + +#include <vfs/vfs_disk_conditioner.h> + +#define DISK_CONDITIONER_SET_ENTITLEMENT "com.apple.private.dmc.set" + +// number of total blocks for a mount +#define BLK_MAX(mp) ((mp->mnt_vfsstat.f_blocks * mp->mnt_vfsstat.f_bsize) / (mp->mnt_devblocksize)) + +// approx. time to spin up an idle HDD +#define DISK_SPINUP_SEC (8) + +// idle period until assumed disk spin down +#define DISK_IDLE_SEC (10 * 60) + +struct _disk_conditioner_info_t { + boolean_t enabled; // if other fields have any effect + uint64_t access_time_usec; // maximum latency before an I/O transfer begins + uint64_t read_throughput_mbps; // throughput of an I/O read + uint64_t write_throughput_mbps; // throughput of an I/O write + boolean_t is_ssd; // behave like an SSD (for both conditioning and affecting behavior in other parts of VFS) + daddr64_t last_blkno; // approx. last transfered block for simulating seek times + struct timeval last_io_timestamp; // the last time an I/O completed +}; + +void disk_conditioner_delay(buf_t, int, int, uint64_t); +void disk_conditioner_unmount(mount_t mp); + +extern void throttle_info_mount_reset_period(mount_t, int isssd); + +static double +weighted_scale_factor(double scale) +{ + // 0 to 1 increasing quickly from 0. This weights smaller blkdiffs higher to add a type of minimum latency + // I would like to use log(10) / 2.0 + 1, but using different approximation due to no math library + // y = (x-1)^3 + 1 + double x_m1 = scale - 1; + return x_m1 * x_m1 * x_m1 + 1; +} + +void +disk_conditioner_delay(buf_t bp, int extents, int total_size, uint64_t already_elapsed_usec) +{ + mount_t mp; + uint64_t delay_usec; + daddr64_t blkdiff; + daddr64_t last_blkno; + double access_time_scale; + struct _disk_conditioner_info_t *info = NULL; + struct timeval elapsed; + struct timeval start; + + mp = buf_vnode(bp)->v_mount; + if (!mp) { + return; + } + + info = mp->mnt_disk_conditioner_info; + if (!info || !info->enabled) { + return; + } + + if (!info->is_ssd) { + // calculate approximate seek time based on difference in block number + last_blkno = info->last_blkno; + blkdiff = bp->b_blkno > last_blkno ? bp->b_blkno - last_blkno : last_blkno - bp->b_blkno; + info->last_blkno = bp->b_blkno + bp->b_bcount; + } else { + blkdiff = BLK_MAX(mp); + } + + // scale access time by (distance in blocks from previous I/O / maximum blocks) + access_time_scale = weighted_scale_factor((double)blkdiff / BLK_MAX(mp)); + // most cases should pass in extents==1 for optimal delay calculation, otherwise just multiply delay by extents + delay_usec = (uint64_t)(((uint64_t)extents * info->access_time_usec) * access_time_scale); + + if (info->read_throughput_mbps && (bp->b_flags & B_READ)) { + delay_usec += (uint64_t)(total_size / ((double)(info->read_throughput_mbps * 1024 * 1024 / 8) / USEC_PER_SEC)); + } else if (info->write_throughput_mbps && !(bp->b_flags & B_READ)) { + delay_usec += (uint64_t)(total_size / ((double)(info->write_throughput_mbps * 1024 * 1024 / 8) / USEC_PER_SEC)); + } + + // try simulating disk spinup based on time since last I/O + if (!info->is_ssd) { + microuptime(&elapsed); + timevalsub(&elapsed, &info->last_io_timestamp); + // avoid this delay right after boot (assuming last_io_timestamp is 0 and disk is already spinning) + if (elapsed.tv_sec > DISK_IDLE_SEC && info->last_io_timestamp.tv_sec != 0) { + delay_usec += DISK_SPINUP_SEC * USEC_PER_SEC; + } + } + + if (delay_usec <= already_elapsed_usec) { + microuptime(&info->last_io_timestamp); + return; + } + + delay_usec -= already_elapsed_usec; + + while (delay_usec) { + microuptime(&start); + delay(delay_usec); + microuptime(&elapsed); + timevalsub(&elapsed, &start); + if (elapsed.tv_sec * USEC_PER_SEC < delay_usec) { + delay_usec -= elapsed.tv_sec * USEC_PER_SEC; + } else { + break; + } + if ((uint64_t)elapsed.tv_usec < delay_usec) { + delay_usec -= elapsed.tv_usec; + } else { + break; + } + } + + microuptime(&info->last_io_timestamp); +} + +int +disk_conditioner_get_info(mount_t mp, disk_conditioner_info *uinfo) +{ + struct _disk_conditioner_info_t *info; + + if (!mp) { + return EINVAL; + } + + info = mp->mnt_disk_conditioner_info; + + if (!info) { + return 0; + } + + uinfo->enabled = info->enabled; + uinfo->access_time_usec = info->access_time_usec; + uinfo->read_throughput_mbps = info->read_throughput_mbps; + uinfo->write_throughput_mbps = info->write_throughput_mbps; + uinfo->is_ssd = info->is_ssd; + + return 0; +} + +int +disk_conditioner_set_info(mount_t mp, disk_conditioner_info *uinfo) +{ + struct _disk_conditioner_info_t *info; + + if (!kauth_cred_issuser(kauth_cred_get()) || !IOTaskHasEntitlement(current_task(), DISK_CONDITIONER_SET_ENTITLEMENT)) { + return EPERM; + } + + if (!mp) { + return EINVAL; + } + + info = mp->mnt_disk_conditioner_info; + if (!info) { + info = mp->mnt_disk_conditioner_info = kalloc(sizeof(struct _disk_conditioner_info_t)); + bzero(info, sizeof(struct _disk_conditioner_info_t)); + } + + info->enabled = uinfo->enabled; + info->access_time_usec = uinfo->access_time_usec; + info->read_throughput_mbps = uinfo->read_throughput_mbps; + info->write_throughput_mbps = uinfo->write_throughput_mbps; + info->is_ssd = uinfo->is_ssd; + microuptime(&info->last_io_timestamp); + + // make sure throttling picks up the new periods + throttle_info_mount_reset_period(mp, info->is_ssd); + + return 0; +} + +void +disk_conditioner_unmount(mount_t mp) +{ + if (!mp->mnt_disk_conditioner_info) { + return; + } + kfree(mp->mnt_disk_conditioner_info, sizeof(struct _disk_conditioner_info_t)); + mp->mnt_disk_conditioner_info = NULL; +} + +boolean_t +disk_conditioner_mount_is_ssd(mount_t mp) +{ + struct _disk_conditioner_info_t *info = mp->mnt_disk_conditioner_info; + + if (!info || !info->enabled) { + return (mp->mnt_kern_flag & MNTK_SSD); + } + + return info->is_ssd; +} diff --git a/bsd/vfs/vfs_disk_conditioner.h b/bsd/vfs/vfs_disk_conditioner.h new file mode 100644 index 000000000..85feb15b5 --- /dev/null +++ b/bsd/vfs/vfs_disk_conditioner.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2016 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef _VFS_DISK_CONDITIONER_H_ +#define _VFS_DISK_CONDITIONER_H_ + +#ifdef KERNEL_PRIVATE + +#include <sys/fsctl.h> +int disk_conditioner_get_info(mount_t, disk_conditioner_info *); +int disk_conditioner_set_info(mount_t, disk_conditioner_info *); + +boolean_t disk_conditioner_mount_is_ssd(mount_t); + +#endif /* KERNEL_PRIVATE */ + +#endif /* !_VFS_DISK_CONDITIONER_H_ */ diff --git a/bsd/vfs/vfs_fsevents.c b/bsd/vfs/vfs_fsevents.c index 143f3cc11..f2e6b0bc3 100644 --- a/bsd/vfs/vfs_fsevents.c +++ b/bsd/vfs/vfs_fsevents.c @@ -62,6 +62,7 @@ #include <bsm/audit_kevents.h> #include <pexpert/pexpert.h> +#include <libkern/section_keywords.h> typedef struct kfs_event { LIST_ENTRY(kfs_event) kevent_list; @@ -397,7 +398,7 @@ add_fsevent(int type, vfs_context_t ctx, ...) // (as long as it's not an event type that can never be the // same as a previous event) // - if (type != FSE_CREATE_FILE && type != FSE_DELETE && type != FSE_RENAME && type != FSE_EXCHANGE && type != FSE_CHOWN && type != FSE_DOCID_CHANGED && type != FSE_DOCID_CREATED) { + if (type != FSE_CREATE_FILE && type != FSE_DELETE && type != FSE_RENAME && type != FSE_EXCHANGE && type != FSE_CHOWN && type != FSE_DOCID_CHANGED && type != FSE_DOCID_CREATED && type != FSE_CLONE) { void *ptr=NULL; int vid=0, was_str=0, nlen=0; @@ -465,7 +466,7 @@ add_fsevent(int type, vfs_context_t ctx, ...) kfse = zalloc_noblock(event_zone); - if (kfse && (type == FSE_RENAME || type == FSE_EXCHANGE)) { + if (kfse && (type == FSE_RENAME || type == FSE_EXCHANGE || type == FSE_CLONE)) { kfse_dest = zalloc_noblock(event_zone); if (kfse_dest == NULL) { did_alloc = 1; @@ -541,7 +542,7 @@ add_fsevent(int type, vfs_context_t ctx, ...) kfse->type = type; kfse->abstime = now; kfse->pid = p->p_pid; - if (type == FSE_RENAME || type == FSE_EXCHANGE) { + if (type == FSE_RENAME || type == FSE_EXCHANGE || type == FSE_CLONE) { memset(kfse_dest, 0, sizeof(kfs_event)); kfse_dest->refcount = 1; OSBitOrAtomic16(KFSE_BEING_CREATED, &kfse_dest->flags); @@ -1625,6 +1626,63 @@ fmod_watch(fs_event_watcher *watcher, struct uio *uio) void fsevent_unmount(__unused struct mount *mp, __unused vfs_context_t ctx) { +#if CONFIG_EMBEDDED + dev_t dev = mp->mnt_vfsstat.f_fsid.val[0]; + int error, waitcount = 0; + struct timespec ts = {1, 0}; + + // wait for any other pending unmounts to complete + lock_watch_table(); + while (fsevent_unmount_dev != 0) { + error = msleep((caddr_t)&fsevent_unmount_dev, &watch_table_lock, PRIBIO, "fsevent_unmount_wait", &ts); + if (error == EWOULDBLOCK) + error = 0; + if (!error && (++waitcount >= 10)) { + error = EWOULDBLOCK; + printf("timeout waiting to signal unmount pending for dev %d (fsevent_unmount_dev %d)\n", dev, fsevent_unmount_dev); + } + if (error) { + // there's a problem, bail out + unlock_watch_table(); + return; + } + } + if (fs_event_type_watchers[FSE_UNMOUNT_PENDING] == 0) { + // nobody watching for unmount pending events + unlock_watch_table(); + return; + } + // this is now the current unmount pending + fsevent_unmount_dev = dev; + fsevent_unmount_ack_count = fs_event_type_watchers[FSE_UNMOUNT_PENDING]; + unlock_watch_table(); + + // send an event to notify the watcher they need to get off the mount + error = add_fsevent(FSE_UNMOUNT_PENDING, ctx, FSE_ARG_DEV, dev, FSE_ARG_DONE); + + // wait for acknowledgment(s) (give up if it takes too long) + lock_watch_table(); + waitcount = 0; + while (fsevent_unmount_dev == dev) { + error = msleep((caddr_t)&fsevent_unmount_dev, &watch_table_lock, PRIBIO, "fsevent_unmount_pending", &ts); + if (error == EWOULDBLOCK) + error = 0; + if (!error && (++waitcount >= 10)) { + error = EWOULDBLOCK; + printf("unmount pending ack timeout for dev %d\n", dev); + } + if (error) { + // there's a problem, bail out + if (fsevent_unmount_dev == dev) { + fsevent_unmount_dev = 0; + fsevent_unmount_ack_count = 0; + } + wakeup((caddr_t)&fsevent_unmount_dev); + break; + } + } + unlock_watch_table(); +#endif } @@ -1982,24 +2040,25 @@ filt_fsevent_process(struct knote *kn, struct filt_process_s *data, struct keven return res; } -struct filterops fsevent_filtops = { - .f_isfd = 1, - .f_attach = NULL, - .f_detach = filt_fsevent_detach, +SECURITY_READ_ONLY_EARLY(struct filterops) fsevent_filtops = { + .f_isfd = 1, + .f_attach = NULL, + .f_detach = filt_fsevent_detach, .f_event = filt_fsevent, .f_touch = filt_fsevent_touch, .f_process = filt_fsevent_process, }; static int -fseventsf_kqfilter(__unused struct fileproc *fp, __unused struct knote *kn, __unused vfs_context_t ctx) +fseventsf_kqfilter(__unused struct fileproc *fp, __unused struct knote *kn, + __unused struct kevent_internal_s *kev, __unused vfs_context_t ctx) { fsevent_handle *fseh = (struct fsevent_handle *)fp->f_fglob->fg_data; int res; kn->kn_hook = (void*)fseh; kn->kn_hookid = 1; - kn->kn_filtid = EVFILTID_FSEVENT; + kn->kn_filtid = EVFILTID_FSEVENT; lock_watch_table(); @@ -2101,7 +2160,7 @@ parse_buffer_and_add_events(const char *buffer, int bufsize, vfs_context_t ctx, path_len = ptr - path; - if (type != FSE_RENAME && type != FSE_EXCHANGE) { + if (type != FSE_RENAME && type != FSE_EXCHANGE && type != FSE_CLONE) { event_start = ptr; // record where the next event starts err = add_fsevent(type, ctx, FSE_ARG_STRING, path_len, path, FSE_ARG_FINFO, finfo, FSE_ARG_DONE); diff --git a/bsd/vfs/vfs_lookup.c b/bsd/vfs/vfs_lookup.c index 128a8ce04..55b86f9e6 100644 --- a/bsd/vfs/vfs_lookup.c +++ b/bsd/vfs/vfs_lookup.c @@ -182,6 +182,7 @@ namei(struct nameidata *ndp) #if CONFIG_VOLFS int volfs_restarts = 0; #endif + size_t bytes_copied = 0; fdp = p->p_fd; @@ -244,10 +245,10 @@ vnode_recycled: retry_copy: if (UIO_SEG_IS_USER_SPACE(ndp->ni_segflg)) { error = copyinstr(ndp->ni_dirp, cnp->cn_pnbuf, - cnp->cn_pnlen, (size_t *)&ndp->ni_pathlen); + cnp->cn_pnlen, &bytes_copied); } else { error = copystr(CAST_DOWN(void *, ndp->ni_dirp), cnp->cn_pnbuf, - cnp->cn_pnlen, (size_t *)&ndp->ni_pathlen); + cnp->cn_pnlen, &bytes_copied); } if (error == ENAMETOOLONG && !(cnp->cn_flags & HASBUF)) { MALLOC_ZONE(cnp->cn_pnbuf, caddr_t, MAXPATHLEN, M_NAMEI, M_WAITOK); @@ -258,11 +259,14 @@ retry_copy: cnp->cn_flags |= HASBUF; cnp->cn_pnlen = MAXPATHLEN; + bytes_copied = 0; goto retry_copy; } if (error) goto error_out; + ndp->ni_pathlen = bytes_copied; + bytes_copied = 0; /* * Since the name cache may contain positive entries of @@ -366,6 +370,21 @@ retry_copy: ndp->ni_vp = NULLVP; for (;;) { +#if CONFIG_MACF + /* + * Give MACF policies a chance to reject the lookup + * before performing any filesystem operations. + * This hook is called before resolving the path and + * again each time a symlink is encountered. + * NB: policies receive path information as supplied + * by the caller and thus cannot be trusted. + */ + error = mac_vnode_check_lookup_preflight(ctx, dp, cnp->cn_nameptr, cnp->cn_namelen); + if (error) { + goto error_out; + } +#endif + ndp->ni_startdir = dp; if ( (error = lookup(ndp)) ) { @@ -458,6 +477,7 @@ namei_compound_available(vnode_t dp, struct nameidata *ndp) return 0; } + static int lookup_authorize_search(vnode_t dp, struct componentname *cnp, int dp_authorized_in_cache, vfs_context_t ctx) { @@ -531,6 +551,7 @@ lookup_handle_rsrc_fork(vnode_t dp, struct nameidata *ndp, struct componentname { vnode_t svp = NULLVP; enum nsoperation nsop; + int nsflags; int error; if (dp->v_type != VREG) { @@ -567,8 +588,13 @@ lookup_handle_rsrc_fork(vnode_t dp, struct nameidata *ndp, struct componentname error = EPERM; goto out; } + + nsflags = 0; + if (cnp->cn_flags & CN_RAW_ENCRYPTED) + nsflags |= NS_GETRAWENCRYPTED; + /* Ask the file system for the resource fork. */ - error = vnode_getnamedstream(dp, &svp, XATTR_RESOURCEFORK_NAME, nsop, 0, ctx); + error = vnode_getnamedstream(dp, &svp, XATTR_RESOURCEFORK_NAME, nsop, nsflags, ctx); /* During a create, it OK for stream vnode to be missing. */ if (error == ENOATTR || error == ENOENT) { diff --git a/bsd/vfs/vfs_subr.c b/bsd/vfs/vfs_subr.c index 6b16ca6cb..5acfa82d4 100644 --- a/bsd/vfs/vfs_subr.c +++ b/bsd/vfs/vfs_subr.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -110,7 +110,6 @@ #include <miscfs/fifofs/fifo.h> #include <string.h> -#include <machine/spl.h> #include <machine/machine_routines.h> #include <kern/assert.h> @@ -127,7 +126,9 @@ #include <kern/kalloc.h> /* kalloc()/kfree() */ #include <kern/clock.h> /* delay_for_interval() */ #include <libkern/OSAtomic.h> /* OSAddAtomic() */ +#if !CONFIG_EMBEDDED #include <console/video_console.h> +#endif #ifdef JOE_DEBUG #include <libkern/OSDebug.h> @@ -139,6 +140,9 @@ #include <security/mac_framework.h> #endif +#include <vfs/vfs_disk_conditioner.h> +#include <libkern/section_keywords.h> + extern lck_grp_t *vnode_lck_grp; extern lck_attr_t *vnode_lck_attr; @@ -173,6 +177,8 @@ extern void memory_object_mark_io_tracking( /* XXX next protptype should be from <nfs/nfs.h> */ extern int nfs_vinvalbuf(vnode_t, int, vfs_context_t, int); +extern int paniclog_append_noflush(const char *format, ...); + /* XXX next prototytype should be from libsa/stdlib.h> but conflicts libkern */ __private_extern__ void qsort( void * array, @@ -180,10 +186,7 @@ __private_extern__ void qsort( size_t member_size, int (*)(const void *, const void *)); -extern kern_return_t adjust_vm_object_cache(vm_size_t oval, vm_size_t nval); __private_extern__ void vntblinit(void); -__private_extern__ kern_return_t reset_vmobjectcache(unsigned int val1, - unsigned int val2); __private_extern__ int unlink1(vfs_context_t, vnode_t, user_addr_t, enum uio_seg, int); @@ -315,25 +318,6 @@ static int nummounts = 0; ragevnodes--; \ } while(0) - -/* - * vnodetarget hasn't been used in a long time, but - * it was exported for some reason... I'm leaving in - * place for now... it should be deprecated out of the - * exports and removed eventually. - */ -u_int32_t vnodetarget; /* target for vnreclaim() */ -#define VNODE_FREE_TARGET 20 /* Default value for vnodetarget */ - -/* - * We need quite a few vnodes on the free list to sustain the - * rapid stat() the compilation process does, and still benefit from the name - * cache. Having too few vnodes on the free list causes serious disk - * thrashing as we cycle through them. - */ -#define VNODE_FREE_MIN CONFIG_VNODE_FREE_MIN /* freelist should have at least this many */ - - static void async_work_continue(void); /* @@ -350,21 +334,12 @@ vntblinit(void) TAILQ_INIT(&vnode_async_work_list); TAILQ_INIT(&mountlist); - if (!vnodetarget) - vnodetarget = VNODE_FREE_TARGET; - microuptime(&rage_tv); rage_limit = desiredvnodes / 100; if (rage_limit < RAGE_LIMIT_MIN) rage_limit = RAGE_LIMIT_MIN; - /* - * Scale the vm_object_cache to accomodate the vnodes - * we want to cache - */ - (void) adjust_vm_object_cache(0, desiredvnodes - VNODE_FREE_MIN); - /* * create worker threads */ @@ -372,26 +347,6 @@ vntblinit(void) thread_deallocate(thread); } -/* Reset the VM Object Cache with the values passed in */ -__private_extern__ kern_return_t -reset_vmobjectcache(unsigned int val1, unsigned int val2) -{ - vm_size_t oval = val1 - VNODE_FREE_MIN; - vm_size_t nval; - - if (val1 == val2) { - return KERN_SUCCESS; - } - - if(val2 < VNODE_FREE_MIN) - nval = 0; - else - nval = val2 - VNODE_FREE_MIN; - - return(adjust_vm_object_cache(oval, nval)); -} - - /* the timeout is in 10 msecs */ int vnode_waitforwrites(vnode_t vp, int output_target, int slpflag, int slptimeout, const char *msg) { @@ -618,6 +573,7 @@ vnode_iterate_clear(mount_t mp) mp->mnt_lflag &= ~MNT_LITER; } +#if !CONFIG_EMBEDDED #include <i386/panic_hooks.h> @@ -629,28 +585,28 @@ struct vnode_iterate_panic_hook { static void vnode_iterate_panic_hook(panic_hook_t *hook_) { - extern int kdb_log(const char *fmt, ...); struct vnode_iterate_panic_hook *hook = (struct vnode_iterate_panic_hook *)hook_; panic_phys_range_t range; uint64_t phys; if (panic_phys_range_before(hook->mp, &phys, &range)) { - kdb_log("mp = %p, phys = %p, prev (%p: %p-%p)\n", + paniclog_append_noflush("mp = %p, phys = %p, prev (%p: %p-%p)\n", hook->mp, phys, range.type, range.phys_start, range.phys_start + range.len); } else { - kdb_log("mp = %p, phys = %p, prev (!)\n", hook->mp, phys); + paniclog_append_noflush("mp = %p, phys = %p, prev (!)\n", hook->mp, phys); } if (panic_phys_range_before(hook->vp, &phys, &range)) { - kdb_log("vp = %p, phys = %p, prev (%p: %p-%p)\n", + paniclog_append_noflush("vp = %p, phys = %p, prev (%p: %p-%p)\n", hook->vp, phys, range.type, range.phys_start, range.phys_start + range.len); } else { - kdb_log("vp = %p, phys = %p, prev (!)\n", hook->vp, phys); + paniclog_append_noflush("vp = %p, phys = %p, prev (!)\n", hook->vp, phys); } panic_dump_mem((void *)(((vm_offset_t)hook->mp -4096) & ~4095), 12288); } +#endif //CONFIG_EMBEDDED int vnode_iterate(mount_t mp, int flags, int (*callout)(struct vnode *, void *), @@ -685,14 +641,18 @@ vnode_iterate(mount_t mp, int flags, int (*callout)(struct vnode *, void *), return(ret); } +#if !CONFIG_EMBEDDED struct vnode_iterate_panic_hook hook; hook.mp = mp; hook.vp = NULL; panic_hook(&hook.hook, vnode_iterate_panic_hook); +#endif /* iterate over all the vnodes */ while (!TAILQ_EMPTY(&mp->mnt_workerqueue)) { vp = TAILQ_FIRST(&mp->mnt_workerqueue); +#if !CONFIG_EMBEDDED hook.vp = vp; +#endif TAILQ_REMOVE(&mp->mnt_workerqueue, vp, v_mntvnodes); TAILQ_INSERT_TAIL(&mp->mnt_vnodelist, vp, v_mntvnodes); vid = vp->v_id; @@ -743,7 +703,9 @@ vnode_iterate(mount_t mp, int flags, int (*callout)(struct vnode *, void *), } out: +#if !CONFIG_EMBEDDED panic_unhook(&hook.hook); +#endif (void)vnode_iterate_reloadq(mp); vnode_iterate_clear(mp); mount_unlock(mp); @@ -1161,12 +1123,14 @@ vfs_mountroot(void) mp->mnt_kern_flag |= MNTK_UNMOUNT_PREFLIGHT; } +#if !CONFIG_EMBEDDED uint32_t speed; - if (MNTK_VIRTUALDEV & mp->mnt_kern_flag) speed = 128; - else if (MNTK_SSD & mp->mnt_kern_flag) speed = 7*256; - else speed = 256; + if (MNTK_VIRTUALDEV & mp->mnt_kern_flag) speed = 128; + else if (disk_conditioner_mount_is_ssd(mp)) speed = 7*256; + else speed = 256; vc_progress_setdiskspeed(speed); +#endif /* * Probe root file system for additional features. */ @@ -1452,7 +1416,6 @@ bdevvp(dev_t dev, vnode_t *vpp) return (0); } - /* * Check to see if the new vnode represents a special device * for which we already have a vnode (either because of @@ -2671,6 +2634,42 @@ vn_getpath_fsenter(struct vnode *vp, char *pathbuf, int *len) return build_path(vp, pathbuf, *len, len, 0, vfs_context_current()); } +/* + * vn_getpath_fsenter_with_parent will reenter the file system to fine the path of the + * vnode. It requires that there are IO counts on both the vnode and the directory vnode. + * + * vn_getpath_fsenter is called by MAC hooks to authorize operations for every thing, but + * unlink, rmdir and rename. For these operation the MAC hook calls vn_getpath. This presents + * problems where if the path can not be found from the name cache, those operations can + * erroneously fail with EPERM even though the call should succeed. When removing or moving + * file system objects with operations such as unlink or rename, those operations need to + * take IO counts on the target and containing directory. Calling vn_getpath_fsenter from a + * MAC hook from these operations during forced unmount operations can lead to dead + * lock. This happens when the operation starts, IO counts are taken on the containing + * directories and targets. Before the MAC hook is called a forced unmount from another + * thread takes place and blocks on the on going operation's directory vnode in vdrain. + * After which, the MAC hook gets called and calls vn_getpath_fsenter. vn_getpath_fsenter + * is called with the understanding that there is an IO count on the target. If in + * build_path the directory vnode is no longer in the cache, then the parent object id via + * vnode_getattr from the target is obtain and used to call VFS_VGET to get the parent + * vnode. The file system's VFS_VGET then looks up by inode in its hash and tries to get + * an IO count. But VFS_VGET "sees" the directory vnode is in vdrain and can block + * depending on which version and how it calls the vnode_get family of interfaces. + * + * N.B. A reasonable interface to use is vnode_getwithvid. This interface was modified to + * call vnode_getiocount with VNODE_DRAINO, so it will happily get an IO count and not + * cause issues, but there is no guarantee that all or any file systems are doing that. + * + * vn_getpath_fsenter_with_parent can enter the file system safely since there is a known + * IO count on the directory vnode by calling build_path_with_parent. + */ + +int +vn_getpath_fsenter_with_parent(struct vnode *dvp, struct vnode *vp, char *pathbuf, int *len) +{ + return build_path_with_parent(vp, dvp, pathbuf, *len, len, 0, vfs_context_current()); +} + int vn_getcdhash(struct vnode *vp, off_t offset, unsigned char *cdhash) { @@ -3347,6 +3346,13 @@ vfs_init_io_attributes(vnode_t devvp, mount_t mp) */ if ((cs_info.flags & DK_CORESTORAGE_PIN_YOUR_METADATA)) mp->mnt_ioflags |= MNT_IOFLAGS_FUSION_DRIVE; + } else { + /* Check for APFS Fusion */ + dk_apfs_flavour_t flavour; + if ((VNOP_IOCTL(devvp, DKIOCGETAPFSFLAVOUR, (caddr_t)&flavour, 0, ctx) == 0) && + (flavour == DK_APFS_FUSION)) { + mp->mnt_ioflags |= MNT_IOFLAGS_FUSION_DRIVE; + } } #if CONFIG_IOSCHED @@ -3625,7 +3631,7 @@ sysctl_vfs_ctlbyfsid(__unused struct sysctl_oid *oidp, void *arg1, int arg2, sfs.f_owner = sp->f_owner; #ifdef NFSCLIENT if (mp->mnt_kern_flag & MNTK_TYPENAME_OVERRIDE) { - strlcpy(&sfs.f_fstypename[0], &mp->fstypename_override[0], MFSTYPENAMELEN); + strlcpy(&sfs.f_fstypename[0], &mp->fstypename_override[0], MFSNAMELEN); } else #endif { @@ -3684,9 +3690,9 @@ sysctl_vfs_ctlbyfsid(__unused struct sysctl_oid *oidp, void *arg1, int arg2, sfs.f_fsid = sp->f_fsid; sfs.f_owner = sp->f_owner; -#ifdef NFS_CLIENT +#ifdef NFSCLIENT if (mp->mnt_kern_flag & MNTK_TYPENAME_OVERRIDE) { - strlcpy(&sfs.f_fstypename[0], &mp->fstypename_override[0], MFSTYPENAMELEN); + strlcpy(&sfs.f_fstypename[0], &mp->fstypename_override[0], MFSNAMELEN); } else #endif { @@ -3708,21 +3714,21 @@ out: return (error); } -static int filt_fsattach(struct knote *kn); +static int filt_fsattach(struct knote *kn, struct kevent_internal_s *kev); static void filt_fsdetach(struct knote *kn); static int filt_fsevent(struct knote *kn, long hint); static int filt_fstouch(struct knote *kn, struct kevent_internal_s *kev); static int filt_fsprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); -struct filterops fs_filtops = { - .f_attach = filt_fsattach, - .f_detach = filt_fsdetach, - .f_event = filt_fsevent, +SECURITY_READ_ONLY_EARLY(struct filterops) fs_filtops = { + .f_attach = filt_fsattach, + .f_detach = filt_fsdetach, + .f_event = filt_fsevent, .f_touch = filt_fstouch, .f_process = filt_fsprocess, }; static int -filt_fsattach(struct knote *kn) +filt_fsattach(struct knote *kn, __unused struct kevent_internal_s *kev) { lck_mtx_lock(fs_klist_lock); KNOTE_ATTACH(&fs_klist, kn); @@ -3908,6 +3914,16 @@ SYSCTL_NODE(_vfs_generic, VFS_CONF, conf, CTLFLAG_RD | CTLFLAG_LOCKED, sysctl_vfs_generic_conf, ""); +/* Indicate that the root file system unmounted cleanly */ +static int vfs_root_unmounted_cleanly = 0; +SYSCTL_INT(_vfs_generic, OID_AUTO, root_unmounted_cleanly, CTLFLAG_RD, &vfs_root_unmounted_cleanly, 0, "Root filesystem was unmounted cleanly"); + +void +vfs_set_root_unmounted_cleanly(void) +{ + vfs_root_unmounted_cleanly = 1; +} + /* * Print vnode state. */ @@ -6371,7 +6387,7 @@ vn_authorize_rmdir(vnode_t dvp, vnode_t vp, struct componentname *cnp, vfs_conte int vnode_attr_authorize_dir_clone(struct vnode_attr *vap, kauth_action_t action, struct vnode_attr *dvap, __unused vnode_t sdvp, mount_t mp, - dir_clone_authorizer_op_t vattr_op, vfs_context_t ctx, + dir_clone_authorizer_op_t vattr_op, uint32_t flags, vfs_context_t ctx, __unused void *reserved) { int error; @@ -6403,8 +6419,9 @@ vnode_attr_authorize_dir_clone(struct vnode_attr *vap, kauth_action_t action, VATTR_WANTED(vap, va_acl); if (dvap) VATTR_WANTED(dvap, va_gid); + } else if (dvap && (flags & VNODE_CLONEFILE_NOOWNERCOPY)) { + VATTR_WANTED(dvap, va_gid); } - return (0); } else if (vattr_op == OP_VATTR_CLEANUP) { return (0); /* Nothing to do for now */ @@ -6420,7 +6437,7 @@ vnode_attr_authorize_dir_clone(struct vnode_attr *vap, kauth_action_t action, * vn_attribute_prepare should be able to accept attributes as well as * vnodes but for now we do this inline. */ - if (!is_suser) { + if (!is_suser || (flags & VNODE_CLONEFILE_NOOWNERCOPY)) { /* * If the filesystem is mounted IGNORE_OWNERSHIP and an explicit * owner is set, that owner takes ownership of all new files. @@ -6454,12 +6471,12 @@ vnode_attr_authorize_dir_clone(struct vnode_attr *vap, kauth_action_t action, /* Inherit SF_RESTRICTED bit from destination directory only */ if (VATTR_IS_ACTIVE(vap, va_flags)) { VATTR_SET(vap, va_flags, - ((vap->va_flags & ~SF_RESTRICTED))); /* Turn off from source */ + ((vap->va_flags & ~(UF_DATAVAULT | SF_RESTRICTED)))); /* Turn off from source */ if (VATTR_IS_ACTIVE(dvap, va_flags)) VATTR_SET(vap, va_flags, - vap->va_flags | (dvap->va_flags & SF_RESTRICTED)); + vap->va_flags | (dvap->va_flags & (UF_DATAVAULT | SF_RESTRICTED))); } else if (VATTR_IS_ACTIVE(dvap, va_flags)) { - VATTR_SET(vap, va_flags, (dvap->va_flags & SF_RESTRICTED)); + VATTR_SET(vap, va_flags, (dvap->va_flags & (UF_DATAVAULT | SF_RESTRICTED))); } return (0); @@ -7959,7 +7976,8 @@ static int vnode_authattr_new_internal(vnode_t dvp, struct vnode_attr *vap, int noauth, uint32_t *defaulted_fieldsp, vfs_context_t ctx) { int error; - int has_priv_suser, ismember, defaulted_owner, defaulted_group, defaulted_mode, inherit_restricted; + int has_priv_suser, ismember, defaulted_owner, defaulted_group, defaulted_mode; + uint32_t inherit_flags; kauth_cred_t cred; guid_t changer; mount_t dmp; @@ -7973,7 +7991,7 @@ vnode_authattr_new_internal(vnode_t dvp, struct vnode_attr *vap, int noauth, uin defaulted_owner = defaulted_group = defaulted_mode = 0; - inherit_restricted = 0; + inherit_flags = 0; /* * Require that the filesystem support extended security to apply any. @@ -8038,9 +8056,8 @@ vnode_authattr_new_internal(vnode_t dvp, struct vnode_attr *vap, int noauth, uin /* Determine if SF_RESTRICTED should be inherited from the parent * directory. */ - if (VATTR_IS_SUPPORTED(&dva, va_flags) && - (dva.va_flags & SF_RESTRICTED)) { - inherit_restricted = 1; + if (VATTR_IS_SUPPORTED(&dva, va_flags)) { + inherit_flags = dva.va_flags & (UF_DATAVAULT | SF_RESTRICTED); } /* default mode is everything, masked with current umask */ @@ -8167,11 +8184,11 @@ vnode_authattr_new_internal(vnode_t dvp, struct vnode_attr *vap, int noauth, uin } } out: - if (inherit_restricted) { + if (inherit_flags) { /* Apply SF_RESTRICTED to the file if its parent directory was * restricted. This is done at the end so that root is not * required if this flag is only set due to inheritance. */ - VATTR_SET(vap, va_flags, (vap->va_flags | SF_RESTRICTED)); + VATTR_SET(vap, va_flags, (vap->va_flags | inherit_flags)); } if (defaulted_fieldsp) { if (defaulted_mode) { @@ -8264,7 +8281,8 @@ vnode_authattr(vnode_t vp, struct vnode_attr *vap, kauth_action_t *actionp, vfs_ VATTR_IS_ACTIVE(vap, va_change_time) || VATTR_IS_ACTIVE(vap, va_modify_time) || VATTR_IS_ACTIVE(vap, va_access_time) || - VATTR_IS_ACTIVE(vap, va_backup_time)) { + VATTR_IS_ACTIVE(vap, va_backup_time) || + VATTR_IS_ACTIVE(vap, va_addedtime)) { VATTR_WANTED(&ova, va_uid); #if 0 /* enable this when we support UUIDs as official owners */ @@ -8333,7 +8351,8 @@ vnode_authattr(vnode_t vp, struct vnode_attr *vap, kauth_action_t *actionp, vfs_ VATTR_IS_ACTIVE(vap, va_change_time) || VATTR_IS_ACTIVE(vap, va_modify_time) || VATTR_IS_ACTIVE(vap, va_access_time) || - VATTR_IS_ACTIVE(vap, va_backup_time)) { + VATTR_IS_ACTIVE(vap, va_backup_time) || + VATTR_IS_ACTIVE(vap, va_addedtime)) { /* * The owner and root may set any timestamps they like, * provided that the file is not immutable. The owner still needs @@ -9130,8 +9149,6 @@ static char *__vpath(vnode_t vp, char *str, int len, int depth) return dst; } -extern int kdb_printf(const char *format, ...) __printflike(1,2); - #define SANE_VNODE_PRINT_LIMIT 5000 void panic_print_vnodes(void) { @@ -9142,7 +9159,7 @@ void panic_print_vnodes(void) char *nm; char vname[257]; - kdb_printf("\n***** VNODES *****\n" + paniclog_append_noflush("\n***** VNODES *****\n" "TYPE UREF ICNT PATH\n"); /* NULL-terminate the path name */ @@ -9154,7 +9171,7 @@ void panic_print_vnodes(void) TAILQ_FOREACH(mnt, &mountlist, mnt_list) { if (!ml_validate_nofault((vm_offset_t)mnt, sizeof(mount_t))) { - kdb_printf("Unable to iterate the mount list %p - encountered an invalid mount pointer %p \n", + paniclog_append_noflush("Unable to iterate the mount list %p - encountered an invalid mount pointer %p \n", &mountlist, mnt); break; } @@ -9162,7 +9179,7 @@ void panic_print_vnodes(void) TAILQ_FOREACH(vp, &mnt->mnt_vnodelist, v_mntvnodes) { if (!ml_validate_nofault((vm_offset_t)vp, sizeof(vnode_t))) { - kdb_printf("Unable to iterate the vnode list %p - encountered an invalid vnode pointer %p \n", + paniclog_append_noflush("Unable to iterate the vnode list %p - encountered an invalid vnode pointer %p \n", &mnt->mnt_vnodelist, vp); break; } @@ -9171,7 +9188,7 @@ void panic_print_vnodes(void) return; type = __vtype(vp->v_type); nm = __vpath(vp, vname, sizeof(vname)-1, 0); - kdb_printf("%s %0d %0d %s\n", + paniclog_append_noflush("%s %0d %0d %s\n", type, vp->v_usecount, vp->v_iocount, nm); } } diff --git a/bsd/vfs/vfs_syscalls.c b/bsd/vfs/vfs_syscalls.c index ee016dac6..788314616 100644 --- a/bsd/vfs/vfs_syscalls.c +++ b/bsd/vfs/vfs_syscalls.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 1995-2016 Apple Inc. All rights reserved. + * Copyright (c) 1995-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -107,6 +107,8 @@ #include <machine/limits.h> #include <miscfs/specfs/specdev.h> +#include <vfs/vfs_disk_conditioner.h> + #include <security/audit/audit.h> #include <bsm/audit_kevents.h> @@ -143,6 +145,8 @@ FREE_ZONE((x), MAXPATHLEN, M_NAMEI); #endif /* CONFIG_FSE */ +extern void disk_conditioner_unmount(mount_t mp); + /* struct for checkdirs iteration */ struct cdirargs { vnode_t olddp; @@ -303,6 +307,75 @@ mount(proc_t p, struct mount_args *uap, __unused int32_t *retval) return (__mac_mount(p, &muap, retval)); } +int +fmount(__unused proc_t p, struct fmount_args *uap, __unused int32_t *retval) +{ + struct componentname cn; + vfs_context_t ctx = vfs_context_current(); + size_t dummy = 0; + int error; + int flags = uap->flags; + char fstypename[MFSNAMELEN]; + char *labelstr = NULL; /* regular mount call always sets it to NULL for __mac_mount() */ + vnode_t pvp; + vnode_t vp; + + AUDIT_ARG(fd, uap->fd); + AUDIT_ARG(fflags, flags); + /* fstypename will get audited by mount_common */ + + /* Sanity check the flags */ + if (flags & (MNT_IMGSRC_BY_INDEX|MNT_ROOTFS)) { + return (ENOTSUP); + } + + if (flags & MNT_UNION) { + return (EPERM); + } + + error = copyinstr(uap->type, fstypename, MFSNAMELEN, &dummy); + if (error) { + return (error); + } + + if ((error = file_vnode(uap->fd, &vp)) != 0) { + return (error); + } + + if ((error = vnode_getwithref(vp)) != 0) { + file_drop(uap->fd); + return (error); + } + + pvp = vnode_getparent(vp); + if (pvp == NULL) { + vnode_put(vp); + file_drop(uap->fd); + return (EINVAL); + } + + memset(&cn, 0, sizeof(struct componentname)); + MALLOC(cn.cn_pnbuf, char *, MAXPATHLEN, M_TEMP, M_WAITOK); + cn.cn_pnlen = MAXPATHLEN; + + if((error = vn_getpath(vp, cn.cn_pnbuf, &cn.cn_pnlen)) != 0) { + FREE(cn.cn_pnbuf, M_TEMP); + vnode_put(pvp); + vnode_put(vp); + file_drop(uap->fd); + return (error); + } + + error = mount_common(fstypename, pvp, vp, &cn, uap->data, flags, 0, labelstr, FALSE, ctx); + + FREE(cn.cn_pnbuf, M_TEMP); + vnode_put(pvp); + vnode_put(vp); + file_drop(uap->fd); + + return (error); +} + void vfs_notify_mount(vnode_t pdvp) { @@ -603,6 +676,7 @@ mount_common(char *fstypename, vnode_t pvp, vnode_t vp, vfsp = mp->mnt_vtable; goto update; } + /* * For non-root users, silently enforce MNT_NOSUID and MNT_NODEV, and * MNT_NOEXEC if mount point is already MNT_NOEXEC. @@ -690,6 +764,7 @@ mount_common(char *fstypename, vnode_t pvp, vnode_t vp, #endif /* NFSCLIENT || DEVFS */ update: + /* * Set the mount level flags. */ @@ -713,7 +788,7 @@ update: #if SECURE_KERNEL #if !CONFIG_MNT_SUID /* - * On release builds of iOS based platforms, always enforce NOSUID and NODEV on + * On release builds of iOS based platforms, always enforce NOSUID on * all mounts. We do this here because we can catch update mounts as well as * non-update mounts in this case. */ @@ -1959,6 +2034,9 @@ dounmount(struct mount *mp, int flags, int withref, vfs_context_t ctx) } } + /* free disk_conditioner_info structure for this mount */ + disk_conditioner_unmount(mp); + IOBSDMountChange(mp, kIOMountChangeUnmount); #if CONFIG_TRIGGERS @@ -2400,7 +2478,9 @@ quotactl(proc_t p, struct quotactl_args *uap, __unused int32_t *retval) /* uap->arg is a pointer to a dqblk structure we need to copy out to */ if (error == 0) { if (proc_is64bit(p)) { - struct user_dqblk my_dqblk64 = {.dqb_bhardlimit = 0}; + struct user_dqblk my_dqblk64; + + memset(&my_dqblk64, 0, sizeof(my_dqblk64)); munge_dqblk(&my_dqblk, &my_dqblk64, TRUE); error = copyout((caddr_t)&my_dqblk64, uap->arg, sizeof (my_dqblk64)); } @@ -3532,7 +3612,13 @@ open1(vfs_context_t ctx, struct nameidata *ndp, int uflags, TRUE); } } else if (secluded_for_filecache == 2) { +#if __arm64__ +#define DYLD_SHARED_CACHE_NAME "dyld_shared_cache_arm64" +#elif __arm__ +#define DYLD_SHARED_CACHE_NAME "dyld_shared_cache_armv7" +#else /* not implemented... */ +#endif if (!strncmp(vp->v_name, DYLD_SHARED_CACHE_NAME, strlen(DYLD_SHARED_CACHE_NAME)) || @@ -4918,10 +5004,10 @@ lseek(proc_t p, struct lseek_args *uap, off_t *retval) case L_SET: break; case SEEK_HOLE: - error = VNOP_IOCTL(vp, FSCTL_FIOSEEKHOLE, (caddr_t)&offset, 0, ctx); + error = VNOP_IOCTL(vp, FSIOC_FIOSEEKHOLE, (caddr_t)&offset, 0, ctx); break; case SEEK_DATA: - error = VNOP_IOCTL(vp, FSCTL_FIOSEEKDATA, (caddr_t)&offset, 0, ctx); + error = VNOP_IOCTL(vp, FSIOC_FIOSEEKDATA, (caddr_t)&offset, 0, ctx); break; default: error = EINVAL; @@ -6864,6 +6950,7 @@ clonefile_internal(vnode_t fvp, boolean_t data_read_authorised, int dst_dirfd, uint32_t defaulted; struct vnode_attr va; struct vnode_attr nva; + uint32_t vnop_flags; v_type = vnode_vtype(fvp); switch (v_type) { @@ -6952,6 +7039,7 @@ clonefile_internal(vnode_t fvp, boolean_t data_read_authorised, int dst_dirfd, attr_cleanup = TRUE; } + vnop_flags = VNODE_CLONEFILE_DEFAULT; /* * We've got initial values for all security parameters, * If we are superuser, then we can change owners to be the @@ -6959,22 +7047,24 @@ clonefile_internal(vnode_t fvp, boolean_t data_read_authorised, int dst_dirfd, * WRITE_SECURITY privileges so all other fields can be taken * from source as well. */ - if (vfs_context_issuser(ctx)) { + if (!(flags & CLONE_NOOWNERCOPY) && vfs_context_issuser(ctx)) { if (VATTR_IS_SUPPORTED(&va, va_uid)) VATTR_SET(&nva, va_uid, va.va_uid); if (VATTR_IS_SUPPORTED(&va, va_gid)) VATTR_SET(&nva, va_gid, va.va_gid); + } else { + vnop_flags |= VNODE_CLONEFILE_NOOWNERCOPY; } + if (VATTR_IS_SUPPORTED(&va, va_mode)) VATTR_SET(&nva, va_mode, va.va_mode); if (VATTR_IS_SUPPORTED(&va, va_flags)) { VATTR_SET(&nva, va_flags, - ((va.va_flags & ~SF_RESTRICTED) | /* Turn off from source */ - (nva.va_flags & SF_RESTRICTED))); + ((va.va_flags & ~(UF_DATAVAULT | SF_RESTRICTED)) | /* Turn off from source */ + (nva.va_flags & (UF_DATAVAULT | SF_RESTRICTED)))); } - error = VNOP_CLONEFILE(fvp, tdvp, &tvp, cnp, &nva, - VNODE_CLONEFILE_DEFAULT, ctx); + error = VNOP_CLONEFILE(fvp, tdvp, &tvp, cnp, &nva, vnop_flags, ctx); if (!error && tvp) { int update_flags = 0; @@ -7019,6 +7109,17 @@ clonefile_internal(vnode_t fvp, boolean_t data_read_authorised, int dst_dirfd, } if (need_fsevent(fsevent, tvp)) { + /* + * The following is a sequence of three explicit events. + * A pair of FSE_CLONE events representing the source and destination + * followed by an FSE_CREATE_[FILE | DIR] for the destination. + * fseventsd may coalesce the destination clone and create events + * into a single event resulting in the following sequence for a client + * FSE_CLONE (src) + * FSE_CLONE | FSE_CREATE (dst) + */ + add_fsevent(FSE_CLONE, ctx, FSE_ARG_VNODE, fvp, FSE_ARG_VNODE, tvp, + FSE_ARG_DONE); add_fsevent(fsevent, ctx, FSE_ARG_VNODE, tvp, FSE_ARG_DONE); } @@ -7052,7 +7153,7 @@ clonefileat(__unused proc_t p, struct clonefileat_args *uap, vfs_context_t ctx = vfs_context_current(); /* Check that the flags are valid. */ - if (uap->flags & ~CLONE_NOFOLLOW) + if (uap->flags & ~(CLONE_NOFOLLOW | CLONE_NOOWNERCOPY)) return (EINVAL); AUDIT_ARG(fd, uap->src_dirfd); @@ -7082,6 +7183,10 @@ fclonefileat(__unused proc_t p, struct fclonefileat_args *uap, int error; vfs_context_t ctx = vfs_context_current(); + /* Check that the flags are valid. */ + if (uap->flags & ~(CLONE_NOFOLLOW | CLONE_NOOWNERCOPY)) + return (EINVAL); + AUDIT_ARG(fd, uap->src_fd); error = fp_getfvp(p, uap->src_fd, &fp, &fvp); if (error) @@ -8156,6 +8261,14 @@ rmdir(__unused proc_t p, struct rmdir_args *uap, __unused int32_t *retval) #define DIRENT64_LEN(namlen) \ ((sizeof(struct direntry) + (namlen) - (MAXPATHLEN-1) + 7) & ~7) +/* Get dirent length padded to 4 byte alignment */ +#define DIRENT_LEN(namelen) \ + ((sizeof(struct dirent) + (namelen + 1) - (__DARWIN_MAXNAMLEN + 1) + 3) & ~3) + +/* Get the end of this dirent */ +#define DIRENT_END(dep) \ + (((char *)(dep)) + (dep)->d_reclen - 1) + errno_t vnode_readdir64(struct vnode *vp, struct uio *uio, int flags, int *eofflag, int *numdirent, vfs_context_t ctxp) @@ -8174,9 +8287,13 @@ vnode_readdir64(struct vnode *vp, struct uio *uio, int flags, int *eofflag, int error; /* - * Our kernel buffer needs to be smaller since re-packing - * will expand each dirent. The worse case (when the name - * length is 3) corresponds to a struct direntry size of 32 + * We're here because the underlying file system does not + * support direnties or we mounted denying support so we must + * fall back to dirents and convert them to direntries. + * + * Our kernel buffer needs to be smaller since re-packing will + * expand each dirent. The worse case (when the name length + * is 3 or less) corresponds to a struct direntry size of 32 * bytes (8-byte aligned) and a struct dirent size of 12 bytes * (4-byte aligned). So having a buffer that is 3/8 the size * will prevent us from reading more than we can pack. @@ -8209,6 +8326,15 @@ vnode_readdir64(struct vnode *vp, struct uio *uio, int flags, int *eofflag, while (error == 0 && (char *)dep < ((char *)bufptr + bytesread)) { size_t enbufsize = DIRENT64_LEN(dep->d_namlen); + if (DIRENT_END(dep) > ((char *)bufptr + bytesread) || + DIRENT_LEN(dep->d_namlen) > dep->d_reclen) { + printf("%s: %s: Bad dirent recived from directory %s\n", __func__, + vp->v_mount->mnt_vfsstat.f_mntonname, + vp->v_name ? vp->v_name : "<unknown>"); + error = EIO; + break; + } + bzero(entry64, enbufsize); /* Convert a dirent to a dirent64. */ entry64->d_ino = dep->d_ino; @@ -9890,6 +10016,52 @@ static int process_namespace_fsctl(nspace_type_t nspace_type, int is64bit, u_int return wait_for_namespace_event(&nhd, nspace_type); } +static unsigned long +fsctl_bogus_command_compat(unsigned long cmd) +{ + + switch (cmd) { + case IOCBASECMD(FSIOC_SYNC_VOLUME): + return (FSIOC_SYNC_VOLUME); + case IOCBASECMD(FSIOC_ROUTEFS_SETROUTEID): + return (FSIOC_ROUTEFS_SETROUTEID); + case IOCBASECMD(FSIOC_SET_PACKAGE_EXTS): + return (FSIOC_SET_PACKAGE_EXTS); + case IOCBASECMD(FSIOC_NAMESPACE_HANDLER_GET): + return (FSIOC_NAMESPACE_HANDLER_GET); + case IOCBASECMD(FSIOC_OLD_SNAPSHOT_HANDLER_GET): + return (FSIOC_OLD_SNAPSHOT_HANDLER_GET); + case IOCBASECMD(FSIOC_SNAPSHOT_HANDLER_GET_EXT): + return (FSIOC_SNAPSHOT_HANDLER_GET_EXT); + case IOCBASECMD(FSIOC_NAMESPACE_HANDLER_UPDATE): + return (FSIOC_NAMESPACE_HANDLER_UPDATE); + case IOCBASECMD(FSIOC_NAMESPACE_HANDLER_UNBLOCK): + return (FSIOC_NAMESPACE_HANDLER_UNBLOCK); + case IOCBASECMD(FSIOC_NAMESPACE_HANDLER_CANCEL): + return (FSIOC_NAMESPACE_HANDLER_CANCEL); + case IOCBASECMD(FSIOC_NAMESPACE_HANDLER_SET_SNAPSHOT_TIME): + return (FSIOC_NAMESPACE_HANDLER_SET_SNAPSHOT_TIME); + case IOCBASECMD(FSIOC_NAMESPACE_ALLOW_DMG_SNAPSHOT_EVENTS): + return (FSIOC_NAMESPACE_ALLOW_DMG_SNAPSHOT_EVENTS); + case IOCBASECMD(FSIOC_SET_FSTYPENAME_OVERRIDE): + return (FSIOC_SET_FSTYPENAME_OVERRIDE); + case IOCBASECMD(DISK_CONDITIONER_IOC_GET): + return (DISK_CONDITIONER_IOC_GET); + case IOCBASECMD(DISK_CONDITIONER_IOC_SET): + return (DISK_CONDITIONER_IOC_SET); + case IOCBASECMD(FSIOC_FIOSEEKHOLE): + return (FSIOC_FIOSEEKHOLE); + case IOCBASECMD(FSIOC_FIOSEEKDATA): + return (FSIOC_FIOSEEKDATA); + case IOCBASECMD(SPOTLIGHT_IOC_GET_MOUNT_TIME): + return (SPOTLIGHT_IOC_GET_MOUNT_TIME); + case IOCBASECMD(SPOTLIGHT_IOC_GET_LAST_MTIME): + return (SPOTLIGHT_IOC_GET_LAST_MTIME); + } + + return (cmd); +} + /* * Make a filesystem-specific control call: */ @@ -9905,6 +10077,8 @@ fsctl_internal(proc_t p, vnode_t *arg_vp, u_long cmd, user_addr_t udata, u_long caddr_t data, memp; vnode_t vp = *arg_vp; + cmd = fsctl_bogus_command_compat(cmd); + size = IOCPARM_LEN(cmd); if (size > IOCPARM_MAX) return (EINVAL); @@ -9912,34 +10086,6 @@ fsctl_internal(proc_t p, vnode_t *arg_vp, u_long cmd, user_addr_t udata, u_long memp = NULL; - - /* - * ensure the buffer is large enough for underlying calls - */ -#ifndef HFSIOC_GETPATH - typedef char pn_t[MAXPATHLEN]; -#define HFSIOC_GETPATH _IOWR('h', 13, pn_t) -#endif - -#ifndef HFS_GETPATH -#define HFS_GETPATH IOCBASECMD(HFSIOC_GETPATH) -#endif - if (IOCBASECMD(cmd) == HFS_GETPATH) { - /* Round up to MAXPATHLEN regardless of user input */ - size = MAXPATHLEN; - } - else if (vp->v_tag == VT_CIFS) { - /* - * XXX Until fsctl's length encoding can be - * XXX fixed properly. - */ - if (IOCBASECMD(cmd) == _IOWR('z', 19, 0) && size < 1432) { - size = 1432; /* sizeof(struct UniqueSMBShareID) */ - } else if (IOCBASECMD(cmd) == _IOWR('z', 28, 0) && size < 308) { - size = 308; /* sizeof(struct smbDebugTestPB) */ - } - } - if (size > sizeof (stkbuf)) { if ((memp = (caddr_t)kalloc(size)) == 0) return ENOMEM; data = memp; @@ -9980,9 +10126,9 @@ fsctl_internal(proc_t p, vnode_t *arg_vp, u_long cmd, user_addr_t udata, u_long } /* Check to see if it's a generic command */ - switch (IOCBASECMD(cmd)) { + switch (cmd) { - case FSCTL_SYNC_VOLUME: { + case FSIOC_SYNC_VOLUME: { mount_t mp = vp->v_mount; int arg = *(uint32_t*)data; @@ -10022,7 +10168,7 @@ fsctl_internal(proc_t p, vnode_t *arg_vp, u_long cmd, user_addr_t udata, u_long } break; - case FSCTL_ROUTEFS_SETROUTEID: { + case FSIOC_ROUTEFS_SETROUTEID: { #if ROUTEFS char routepath[MAXPATHLEN]; size_t len = 0; @@ -10043,7 +10189,7 @@ fsctl_internal(proc_t p, vnode_t *arg_vp, u_long cmd, user_addr_t udata, u_long } break; - case FSCTL_SET_PACKAGE_EXTS: { + case FSIOC_SET_PACKAGE_EXTS: { user_addr_t ext_strings; uint32_t num_entries; uint32_t max_width; @@ -10075,23 +10221,23 @@ fsctl_internal(proc_t p, vnode_t *arg_vp, u_long cmd, user_addr_t udata, u_long break; /* namespace handlers */ - case FSCTL_NAMESPACE_HANDLER_GET: { + case FSIOC_NAMESPACE_HANDLER_GET: { error = process_namespace_fsctl(NSPACE_HANDLER_NSPACE, is64bit, size, data); } break; /* Snapshot handlers */ - case FSCTL_OLD_SNAPSHOT_HANDLER_GET: { + case FSIOC_OLD_SNAPSHOT_HANDLER_GET: { error = process_namespace_fsctl(NSPACE_HANDLER_SNAPSHOT, is64bit, size, data); } break; - case FSCTL_SNAPSHOT_HANDLER_GET_EXT: { + case FSIOC_SNAPSHOT_HANDLER_GET_EXT: { error = process_namespace_fsctl(NSPACE_HANDLER_SNAPSHOT, is64bit, size, data); } break; - case FSCTL_NAMESPACE_HANDLER_UPDATE: { + case FSIOC_NAMESPACE_HANDLER_UPDATE: { uint32_t token, val; int i; @@ -10133,7 +10279,7 @@ fsctl_internal(proc_t p, vnode_t *arg_vp, u_long cmd, user_addr_t udata, u_long } break; - case FSCTL_NAMESPACE_HANDLER_UNBLOCK: { + case FSIOC_NAMESPACE_HANDLER_UNBLOCK: { uint32_t token, val; int i; @@ -10181,7 +10327,7 @@ fsctl_internal(proc_t p, vnode_t *arg_vp, u_long cmd, user_addr_t udata, u_long } break; - case FSCTL_NAMESPACE_HANDLER_CANCEL: { + case FSIOC_NAMESPACE_HANDLER_CANCEL: { uint32_t token, val; int i; @@ -10229,7 +10375,7 @@ fsctl_internal(proc_t p, vnode_t *arg_vp, u_long cmd, user_addr_t udata, u_long } break; - case FSCTL_NAMESPACE_HANDLER_SET_SNAPSHOT_TIME: { + case FSIOC_NAMESPACE_HANDLER_SET_SNAPSHOT_TIME: { if ((error = suser(kauth_cred_get(), &(current_proc()->p_acflag)))) { break; } @@ -10245,7 +10391,7 @@ fsctl_internal(proc_t p, vnode_t *arg_vp, u_long cmd, user_addr_t udata, u_long } break; - case FSCTL_NAMESPACE_ALLOW_DMG_SNAPSHOT_EVENTS: + case FSIOC_NAMESPACE_ALLOW_DMG_SNAPSHOT_EVENTS: { if ((error = suser(kauth_cred_get(), &(current_proc()->p_acflag)))) { break; @@ -10261,7 +10407,7 @@ fsctl_internal(proc_t p, vnode_t *arg_vp, u_long cmd, user_addr_t udata, u_long } break; - case FSCTL_SET_FSTYPENAME_OVERRIDE: + case FSIOC_SET_FSTYPENAME_OVERRIDE: { if ((error = suser(kauth_cred_get(), &(current_proc()->p_acflag)))) { break; @@ -10287,9 +10433,19 @@ fsctl_internal(proc_t p, vnode_t *arg_vp, u_long cmd, user_addr_t udata, u_long } break; + case DISK_CONDITIONER_IOC_GET: { + error = disk_conditioner_get_info(vp->v_mount, (disk_conditioner_info *)data); + } + break; + + case DISK_CONDITIONER_IOC_SET: { + error = disk_conditioner_set_info(vp->v_mount, (disk_conditioner_info *)data); + } + break; + default: { /* Invoke the filesystem-specific code */ - error = VNOP_IOCTL(vp, IOCBASECMD(cmd), data, options, ctx); + error = VNOP_IOCTL(vp, cmd, data, options, ctx); } } /* end switch stmt */ @@ -10922,10 +11078,7 @@ out: /* * Obtain the full pathname of a file system object by id. - * - * This is a private SPI used by the File Manager. */ -__private_extern__ int fsgetpath(__unused proc_t p, struct fsgetpath_args *uap, user_ssize_t *retval) { @@ -11563,11 +11716,7 @@ snapshot_revert(int dirfd, user_addr_t name, __unused uint32_t flags, #define APFSIOC_REVERT_TO_SNAPSHOT _IOW('J', 1, u_int64_t) #endif -#ifndef APFS_REVERT_TO_SNAPSHOT -#define APFS_REVERT_TO_SNAPSHOT IOCBASECMD(APFSIOC_REVERT_TO_SNAPSHOT) -#endif - - error = VNOP_IOCTL(namend.ni_vp, APFS_REVERT_TO_SNAPSHOT, (caddr_t) NULL, + error = VNOP_IOCTL(namend.ni_vp, APFSIOC_REVERT_TO_SNAPSHOT, (caddr_t) NULL, 0, ctx); vnode_put(namend.ni_vp); @@ -11731,7 +11880,7 @@ snapshot_mount(int dirfd, user_addr_t name, user_addr_t directory, smnt_data.sm_mp = mp; smnt_data.sm_cnp = &snapndp->ni_cnd; error = mount_common(mp->mnt_vfsstat.f_fstypename, pvp, vp, - &dirndp->ni_cnd, CAST_USER_ADDR_T(&smnt_data), 0, + &dirndp->ni_cnd, CAST_USER_ADDR_T(&smnt_data), flags & MNT_DONTBROWSE, KERNEL_MOUNT_SNAPSHOT, NULL, FALSE, ctx); } diff --git a/bsd/vfs/vfs_vnops.c b/bsd/vfs/vfs_vnops.c index 7a1a2b216..797573d75 100644 --- a/bsd/vfs/vfs_vnops.c +++ b/bsd/vfs/vfs_vnops.c @@ -112,18 +112,19 @@ int ubc_setcred(struct vnode *, struct proc *); #endif #include <IOKit/IOBSD.h> +#include <libkern/section_keywords.h> static int vn_closefile(struct fileglob *fp, vfs_context_t ctx); static int vn_ioctl(struct fileproc *fp, u_long com, caddr_t data, - vfs_context_t ctx); + vfs_context_t ctx); static int vn_read(struct fileproc *fp, struct uio *uio, int flags, - vfs_context_t ctx); + vfs_context_t ctx); static int vn_write(struct fileproc *fp, struct uio *uio, int flags, - vfs_context_t ctx); + vfs_context_t ctx); static int vn_select( struct fileproc *fp, int which, void * wql, - vfs_context_t ctx); + vfs_context_t ctx); static int vn_kqfilt_add(struct fileproc *fp, struct knote *kn, - vfs_context_t ctx); + struct kevent_internal_s *kev, vfs_context_t ctx); static void filt_vndetach(struct knote *kn); static int filt_vnode(struct knote *kn, long hint); static int filt_vnode_common(struct knote *kn, vnode_t vp, long hint); @@ -147,10 +148,10 @@ const struct fileops vnops = { static int filt_vntouch(struct knote *kn, struct kevent_internal_s *kev); static int filt_vnprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); -struct filterops vnode_filtops = { - .f_isfd = 1, - .f_attach = NULL, - .f_detach = filt_vndetach, +SECURITY_READ_ONLY_EARLY(struct filterops) vnode_filtops = { + .f_isfd = 1, + .f_attach = NULL, + .f_detach = filt_vndetach, .f_event = filt_vnode, .f_touch = filt_vntouch, .f_process = filt_vnprocess, @@ -1342,7 +1343,7 @@ vn_stat_noauth(struct vnode *vp, void *sbptr, kauth_filesec_t *xsec, int isstat6 }; if (isstat64 != 0) { sb64->st_mode = mode; - sb64->st_nlink = VATTR_IS_SUPPORTED(&va, va_nlink) ? (u_int16_t)va.va_nlink : 1; + sb64->st_nlink = VATTR_IS_SUPPORTED(&va, va_nlink) ? va.va_nlink > UINT16_MAX ? UINT16_MAX : (u_int16_t)va.va_nlink : 1; sb64->st_uid = va.va_uid; sb64->st_gid = va.va_gid; sb64->st_rdev = va.va_rdev; @@ -1360,7 +1361,7 @@ vn_stat_noauth(struct vnode *vp, void *sbptr, kauth_filesec_t *xsec, int isstat6 sb64->st_blocks = roundup(va.va_total_alloc, 512) / 512; } else { sb->st_mode = mode; - sb->st_nlink = VATTR_IS_SUPPORTED(&va, va_nlink) ? (u_int16_t)va.va_nlink : 1; + sb->st_nlink = VATTR_IS_SUPPORTED(&va, va_nlink) ? va.va_nlink > UINT16_MAX ? UINT16_MAX : (u_int16_t)va.va_nlink : 1; sb->st_uid = va.va_uid; sb->st_gid = va.va_gid; sb->st_rdev = va.va_rdev; @@ -1686,12 +1687,13 @@ vn_pathconf(vnode_t vp, int name, int32_t *retval, vfs_context_t ctx) } static int -vn_kqfilt_add(struct fileproc *fp, struct knote *kn, vfs_context_t ctx) +vn_kqfilt_add(struct fileproc *fp, struct knote *kn, + struct kevent_internal_s *kev, vfs_context_t ctx) { struct vnode *vp; int error = 0; int result = 0; - + vp = (struct vnode *)fp->f_fglob->fg_data; /* @@ -1709,7 +1711,7 @@ vn_kqfilt_add(struct fileproc *fp, struct knote *kn, vfs_context_t ctx) } else if (!vnode_isreg(vp)) { if (vnode_ischr(vp)) { - result = spec_kqfilter(vp, kn); + result = spec_kqfilter(vp, kn, kev); if ((kn->kn_flags & EV_ERROR) == 0) { /* claimed by a special device */ vnode_put(vp); diff --git a/bsd/vfs/vfs_xattr.c b/bsd/vfs/vfs_xattr.c index bd38c5f51..b47ec5553 100644 --- a/bsd/vfs/vfs_xattr.c +++ b/bsd/vfs/vfs_xattr.c @@ -376,7 +376,7 @@ xattr_validatename(const char *name) if (name == NULL || name[0] == '\0') { return (EINVAL); } - namelen = strnlen(name, XATTR_MAXNAMELEN); + namelen = strlen(name); if (name[namelen] != '\0') return (ENAMETOOLONG); @@ -407,10 +407,14 @@ vnode_getnamedstream(vnode_t vp, vnode_t *svpp, const char *name, enum nsoperati { int error; - if (vp->v_mount->mnt_kern_flag & MNTK_NAMED_STREAMS) + if (vp->v_mount->mnt_kern_flag & MNTK_NAMED_STREAMS) { error = VNOP_GETNAMEDSTREAM(vp, svpp, name, op, flags, context); - else - error = default_getnamedstream(vp, svpp, name, op, context); + } else { + if (flags) + error = ENOTSUP; + else + error = default_getnamedstream(vp, svpp, name, op, context); + } if (error == 0) { uint32_t streamflags = VISNAMEDSTREAM; @@ -1602,7 +1606,7 @@ default_getxattr(vnode_t vp, const char *name, uio_t uio, size_t *size, int error; fileflags = FREAD; - if (bcmp(name, XATTR_RESOURCEFORK_NAME, sizeof(XATTR_RESOURCEFORK_NAME)) == 0) { + if (strcmp(name, XATTR_RESOURCEFORK_NAME) == 0) { isrsrcfork = 1; /* * Open the file locked (shared) since the Carbon @@ -1623,7 +1627,7 @@ default_getxattr(vnode_t vp, const char *name, uio_t uio, size_t *size, } /* Get the Finder Info. */ - if (bcmp(name, XATTR_FINDERINFO_NAME, sizeof(XATTR_FINDERINFO_NAME)) == 0) { + if (strcmp(name, XATTR_FINDERINFO_NAME) == 0) { if (ainfo.finderinfo == NULL || ainfo.emptyfinderinfo) { error = ENOATTR; @@ -2113,7 +2117,7 @@ default_removexattr(vnode_t vp, const char *name, __unused int options, vfs_cont int error; fileflags = FREAD | FWRITE; - if (bcmp(name, XATTR_RESOURCEFORK_NAME, sizeof(XATTR_RESOURCEFORK_NAME)) == 0) { + if (strncmp(name, XATTR_RESOURCEFORK_NAME, sizeof(XATTR_RESOURCEFORK_NAME)) == 0) { isrsrcfork = 1; /* * Open the file locked (exclusive) since the Carbon @@ -2140,7 +2144,7 @@ default_removexattr(vnode_t vp, const char *name, __unused int options, vfs_cont ++attrcount; /* Clear the Finder Info. */ - if (bcmp(name, XATTR_FINDERINFO_NAME, sizeof(XATTR_FINDERINFO_NAME)) == 0) { + if (strncmp(name, XATTR_FINDERINFO_NAME, sizeof(XATTR_FINDERINFO_NAME)) == 0) { if (ainfo.finderinfo == NULL || ainfo.emptyfinderinfo) { error = ENOATTR; goto out; @@ -2380,7 +2384,9 @@ default_listxattr(vnode_t vp, uio_t uio, size_t *size, __unused int options, vfs count = ainfo.attrhdr->num_attrs; for (i = 0, entry = ainfo.attr_entry; i < count && ATTR_VALID(entry, ainfo); i++) { if (xattr_protected((const char *)entry->name) || - xattr_validatename((const char *)entry->name) != 0) { + ((entry->namelen < XATTR_MAXNAMELEN) && + (entry->name[entry->namelen] == '\0') && + (xattr_validatename((const char *)entry->name) != 0))) { entry = ATTR_NEXT(entry); continue; } diff --git a/bsd/vm/vm_compressor_backing_file.c b/bsd/vm/vm_compressor_backing_file.c index 9a83330fb..295d023fa 100644 --- a/bsd/vm/vm_compressor_backing_file.c +++ b/bsd/vm/vm_compressor_backing_file.c @@ -182,8 +182,7 @@ vm_swapfile_io(vnode_t vp, uint64_t offset, uint64_t start, int npages, int flag int upl_control_flags = 0; upl_size_t upl_size = 0; - upl_create_flags = UPL_SET_INTERNAL | UPL_SET_LITE - | UPL_MEMORY_TAG_MAKE(VM_KERN_MEMORY_OSFMK); + upl_create_flags = UPL_SET_INTERNAL | UPL_SET_LITE; #if ENCRYPTED_SWAP upl_control_flags = UPL_IOSYNC | UPL_PAGING_ENCRYPTED; @@ -201,7 +200,8 @@ vm_swapfile_io(vnode_t vp, uint64_t offset, uint64_t start, int npages, int flag &upl, NULL, &count, - &upl_create_flags); + &upl_create_flags, + VM_KERN_MEMORY_OSFMK); if (kr != KERN_SUCCESS || (upl_size != io_size)) { panic("vm_map_create_upl failed with %d\n", kr); @@ -322,17 +322,17 @@ u_int32_t vnode_trim_list (vnode_t vp, struct trim_list *tl, boolean_t route_onl * in each call to ensure that the entire range is covered. */ error = VNOP_BLOCKMAP (vp, current_offset, remaining_length, - &io_blockno, &io_bytecount, NULL, VNODE_READ, NULL); + &io_blockno, &io_bytecount, NULL, VNODE_READ | VNODE_BLOCKMAP_NO_TRACK, NULL); if (error) { goto trim_exit; } + if (io_blockno != -1) { + extents[trim_index].offset = (uint64_t) io_blockno * (u_int64_t) blocksize; + extents[trim_index].length = io_bytecount; - extents[trim_index].offset = (uint64_t) io_blockno * (u_int64_t) blocksize; - extents[trim_index].length = io_bytecount; - - trim_index++; - + trim_index++; + } if (trim_index == MAX_BATCH_TO_TRIM) { if (vp->v_mount->mnt_ioflags & MNT_IOFLAGS_CSUNMAP_SUPPORTED) { diff --git a/bsd/vm/vm_unix.c b/bsd/vm/vm_unix.c index 69aeca4ab..8048a6b80 100644 --- a/bsd/vm/vm_unix.c +++ b/bsd/vm/vm_unix.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2010 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -37,9 +37,6 @@ * is included in support of clause 2.2 (b) of the Apple Public License, * Version 2.0. */ - -#include <meta_features.h> - #include <vm/vm_options.h> #include <kern/task.h> @@ -82,6 +79,9 @@ #include <sys/kas_info.h> #include <sys/socket.h> #include <sys/socketvar.h> +#if NECP +#include <net/necp.h> +#endif /* NECP */ #include <security/audit/audit.h> #include <security/mac.h> @@ -92,8 +92,6 @@ #include <vm/vm_kern.h> #include <vm/vm_pageout.h> -#include <machine/spl.h> - #include <mach/shared_region.h> #include <vm/vm_shared_region.h> @@ -101,6 +99,9 @@ #include <sys/kern_memorystatus.h> +#if CONFIG_MACF +#include <security/mac_framework.h> +#endif int _shared_region_map_and_slide(struct proc*, int, unsigned int, struct shared_file_mapping_np*, uint32_t, user_addr_t, user_addr_t); int shared_region_copyin_mappings(struct proc*, user_addr_t, unsigned int, struct shared_file_mapping_np *); @@ -145,6 +146,22 @@ SYSCTL_INT(_vm, OID_AUTO, region_footprint, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_reg #endif /* DEVELOPMENT || DEBUG */ +#if CONFIG_EMBEDDED + +#if DEVELOPMENT || DEBUG +extern int panic_on_unsigned_execute; +SYSCTL_INT(_vm, OID_AUTO, panic_on_unsigned_execute, CTLFLAG_RW | CTLFLAG_LOCKED, &panic_on_unsigned_execute, 0, ""); +#endif /* DEVELOPMENT || DEBUG */ + +extern int log_executable_mem_entry; +extern int cs_executable_create_upl; +extern int cs_executable_mem_entry; +extern int cs_executable_wire; +SYSCTL_INT(_vm, OID_AUTO, log_executable_mem_entry, CTLFLAG_RD | CTLFLAG_LOCKED, &log_executable_mem_entry, 0, ""); +SYSCTL_INT(_vm, OID_AUTO, cs_executable_create_upl, CTLFLAG_RD | CTLFLAG_LOCKED, &cs_executable_create_upl, 0, ""); +SYSCTL_INT(_vm, OID_AUTO, cs_executable_mem_entry, CTLFLAG_RD | CTLFLAG_LOCKED, &cs_executable_mem_entry, 0, ""); +SYSCTL_INT(_vm, OID_AUTO, cs_executable_wire, CTLFLAG_RD | CTLFLAG_LOCKED, &cs_executable_wire, 0, ""); +#endif /* CONFIG_EMBEDDED */ #if DEVELOPMENT || DEBUG extern int radar_20146450; @@ -156,7 +173,38 @@ SYSCTL_INT(_vm, OID_AUTO, macho_printf, CTLFLAG_RW | CTLFLAG_LOCKED, &macho_prin extern int apple_protect_pager_data_request_debug; SYSCTL_INT(_vm, OID_AUTO, apple_protect_pager_data_request_debug, CTLFLAG_RW | CTLFLAG_LOCKED, &apple_protect_pager_data_request_debug, 0, ""); - +#if __arm__ || __arm64__ +/* These are meant to support the page table accounting unit test. */ +extern unsigned int arm_hardware_page_size; +extern unsigned int arm_pt_desc_size; +extern unsigned int arm_pt_root_size; +extern unsigned int free_page_size_tt_count; +extern unsigned int free_two_page_size_tt_count; +extern unsigned int free_tt_count; +extern unsigned int inuse_user_tteroot_count; +extern unsigned int inuse_kernel_tteroot_count; +extern unsigned int inuse_user_ttepages_count; +extern unsigned int inuse_kernel_ttepages_count; +extern unsigned int inuse_user_ptepages_count; +extern unsigned int inuse_kernel_ptepages_count; +SYSCTL_UINT(_vm, OID_AUTO, native_hw_pagesize, CTLFLAG_RD | CTLFLAG_LOCKED, &arm_hardware_page_size, 0, ""); +SYSCTL_UINT(_vm, OID_AUTO, arm_pt_desc_size, CTLFLAG_RD | CTLFLAG_LOCKED, &arm_pt_desc_size, 0, ""); +SYSCTL_UINT(_vm, OID_AUTO, arm_pt_root_size, CTLFLAG_RD | CTLFLAG_LOCKED, &arm_pt_root_size, 0, ""); +SYSCTL_UINT(_vm, OID_AUTO, free_1page_tte_root, CTLFLAG_RD | CTLFLAG_LOCKED, &free_page_size_tt_count, 0, ""); +SYSCTL_UINT(_vm, OID_AUTO, free_2page_tte_root, CTLFLAG_RD | CTLFLAG_LOCKED, &free_two_page_size_tt_count, 0, ""); +SYSCTL_UINT(_vm, OID_AUTO, free_tte_root, CTLFLAG_RD | CTLFLAG_LOCKED, &free_tt_count, 0, ""); +SYSCTL_UINT(_vm, OID_AUTO, user_tte_root, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_user_tteroot_count, 0, ""); +SYSCTL_UINT(_vm, OID_AUTO, kernel_tte_root, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_kernel_tteroot_count, 0, ""); +SYSCTL_UINT(_vm, OID_AUTO, user_tte_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_user_ttepages_count, 0, ""); +SYSCTL_UINT(_vm, OID_AUTO, kernel_tte_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_kernel_ttepages_count, 0, ""); +SYSCTL_UINT(_vm, OID_AUTO, user_pte_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_user_ptepages_count, 0, ""); +SYSCTL_UINT(_vm, OID_AUTO, kernel_pte_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_kernel_ptepages_count, 0, ""); +#endif /* __arm__ || __arm64__ */ + +#if __arm64__ +extern int fourk_pager_data_request_debug; +SYSCTL_INT(_vm, OID_AUTO, fourk_pager_data_request_debug, CTLFLAG_RW | CTLFLAG_LOCKED, &fourk_pager_data_request_debug, 0, ""); +#endif /* __arm64__ */ #endif /* DEVELOPMENT || DEBUG */ SYSCTL_INT(_vm, OID_AUTO, vm_do_collapse_compressor, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.do_collapse_compressor, 0, ""); @@ -203,6 +251,12 @@ extern int allow_stack_exec, allow_data_exec; SYSCTL_INT(_vm, OID_AUTO, allow_stack_exec, CTLFLAG_RW | CTLFLAG_LOCKED, &allow_stack_exec, 0, ""); SYSCTL_INT(_vm, OID_AUTO, allow_data_exec, CTLFLAG_RW | CTLFLAG_LOCKED, &allow_data_exec, 0, ""); +#if __arm64__ +extern int fourk_binary_compatibility_unsafe; +extern int fourk_binary_compatibility_allow_wx; +SYSCTL_INT(_vm, OID_AUTO, fourk_binary_compatibility_unsafe, CTLFLAG_RW | CTLFLAG_LOCKED, &fourk_binary_compatibility_unsafe, 0, ""); +SYSCTL_INT(_vm, OID_AUTO, fourk_binary_compatibility_allow_wx, CTLFLAG_RW | CTLFLAG_LOCKED, &fourk_binary_compatibility_allow_wx, 0, ""); +#endif /* __arm64__ */ #endif /* DEVELOPMENT || DEBUG */ static const char *prot_values[] = { @@ -242,8 +296,13 @@ int shared_region_unnest_log_count_threshold = 5; * Shared cache path enforcement. */ +#ifndef CONFIG_EMBEDDED static int scdir_enforce = 1; static char scdir_path[] = "/var/db/dyld/"; +#else +static int scdir_enforce = 0; +static char scdir_path[] = "/System/Library/Caches/com.apple.dyld/"; +#endif #ifndef SECURE_KERNEL SYSCTL_INT(_vm, OID_AUTO, enforce_shared_cache_dir, CTLFLAG_RW | CTLFLAG_LOCKED, &scdir_enforce, 0, ""); @@ -325,12 +384,12 @@ vslock( vm_map_t map; map = current_map(); - kret = vm_map_wire(map, + kret = vm_map_wire_kernel(map, vm_map_trunc_page(addr, vm_map_page_mask(map)), vm_map_round_page(addr+len, vm_map_page_mask(map)), - VM_PROT_READ | VM_PROT_WRITE | VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_BSD), + VM_PROT_READ | VM_PROT_WRITE, VM_KERN_MEMORY_BSD, FALSE); switch (kret) { @@ -784,6 +843,14 @@ task_for_pid( extmod_statistics_incr_task_for_pid(p->task); sright = (void *) convert_task_to_port(p->task); + + /* Check if the task has been corpsified */ + if (is_corpsetask(p->task)) { + ipc_port_release_send(sright); + error = KERN_FAILURE; + goto tfpout; + } + tret = ipc_port_copyout_send( sright, get_task_ipcspace(current_task())); @@ -921,6 +988,7 @@ pid_suspend(struct proc *p __unused, struct pid_suspend_args *args, int *ret) } target = targetproc->task; +#ifndef CONFIG_EMBEDDED if (target != TASK_NULL) { mach_port_t tfpport; @@ -947,6 +1015,7 @@ pid_suspend(struct proc *p __unused, struct pid_suspend_args *args, int *ret) } } } +#endif task_reference(target); error = task_pidsuspend(target); @@ -1005,6 +1074,7 @@ pid_resume(struct proc *p __unused, struct pid_resume_args *args, int *ret) } target = targetproc->task; +#ifndef CONFIG_EMBEDDED if (target != TASK_NULL) { mach_port_t tfpport; @@ -1031,7 +1101,13 @@ pid_resume(struct proc *p __unused, struct pid_resume_args *args, int *ret) } } } +#endif +#if CONFIG_EMBEDDED +#if SOCKETS + resume_proc_sockets(targetproc); +#endif /* SOCKETS */ +#endif /* CONFIG_EMBEDDED */ task_reference(target); @@ -1062,6 +1138,149 @@ out: return error; } +#if CONFIG_EMBEDDED +/* + * Freeze the specified process (provided in args->pid), or find and freeze a PID. + * When a process is specified, this call is blocking, otherwise we wake up the + * freezer thread and do not block on a process being frozen. + */ +kern_return_t +pid_hibernate(struct proc *p __unused, struct pid_hibernate_args *args, int *ret) +{ + int error = 0; + proc_t targetproc = PROC_NULL; + int pid = args->pid; + +#ifndef CONFIG_FREEZE + #pragma unused(pid) +#else + +#if CONFIG_MACF + error = mac_proc_check_suspend_resume(p, MAC_PROC_CHECK_HIBERNATE); + if (error) { + error = EPERM; + goto out; + } +#endif + + /* + * If a pid has been provided, we obtain the process handle and call task_for_pid_posix_check(). + */ + + if (pid >= 0) { + targetproc = proc_find(pid); + + if (targetproc == PROC_NULL) { + error = ESRCH; + goto out; + } + + if (!task_for_pid_posix_check(targetproc)) { + error = EPERM; + goto out; + } + } + + if (pid == -2) { + vm_pageout_anonymous_pages(); + } else if (pid == -1) { + memorystatus_on_inactivity(targetproc); + } else { + error = memorystatus_freeze_process_sync(targetproc); + } + +out: + +#endif /* CONFIG_FREEZE */ + + if (targetproc != PROC_NULL) + proc_rele(targetproc); + *ret = error; + return error; +} +#endif /* CONFIG_EMBEDDED */ + +#if SOCKETS +static int +shutdown_sockets_callout(proc_t p, void *arg) +{ + struct pid_shutdown_sockets_args *args = arg; + int pid = args->pid; + int level = args->level; + struct filedesc *fdp; + struct fileproc *fp; + int i; + + proc_fdlock(p); + fdp = p->p_fd; + for (i = 0; i < fdp->fd_nfiles; i++) { + fp = fdp->fd_ofiles[i]; + if (fp == NULL || (fdp->fd_ofileflags[i] & UF_RESERVED) != 0) { + continue; + } + if (FILEGLOB_DTYPE(fp->f_fglob) == DTYPE_SOCKET) { + struct socket *so = (struct socket *)fp->f_fglob->fg_data; + if (p->p_pid == pid || so->last_pid == pid || + ((so->so_flags & SOF_DELEGATED) && so->e_pid == pid)) { + /* Call networking stack with socket and level */ + (void) socket_defunct(p, so, level); + } + } +#if NECP + else if (FILEGLOB_DTYPE(fp->f_fglob) == DTYPE_NETPOLICY && + p->p_pid == pid) { + necp_defunct_client(p, fp); + } +#endif /* NECP */ + } + proc_fdunlock(p); + + return (PROC_RETURNED); +} + +int +pid_shutdown_sockets(struct proc *p __unused, struct pid_shutdown_sockets_args *args, int *ret) +{ + int error = 0; + proc_t targetproc = PROC_NULL; + int pid = args->pid; + int level = args->level; + + if (level != SHUTDOWN_SOCKET_LEVEL_DISCONNECT_SVC && + level != SHUTDOWN_SOCKET_LEVEL_DISCONNECT_ALL) { + error = EINVAL; + goto out; + } + +#if CONFIG_MACF + error = mac_proc_check_suspend_resume(p, MAC_PROC_CHECK_SHUTDOWN_SOCKETS); + if (error) { + error = EPERM; + goto out; + } +#endif + + targetproc = proc_find(pid); + if (targetproc == PROC_NULL) { + error = ESRCH; + goto out; + } + + if (!task_for_pid_posix_check(targetproc)) { + error = EPERM; + goto out; + } + + proc_iterate(PROC_ALLPROCLIST | PROC_NOWAITTRANS, shutdown_sockets_callout, args, NULL, NULL); + +out: + if (targetproc != PROC_NULL) + proc_rele(targetproc); + *ret = error; + return error; +} + +#endif /* SOCKETS */ static int sysctl_settfp_policy(__unused struct sysctl_oid *oidp, void *arg1, @@ -1739,9 +1958,8 @@ extern unsigned int vm_pageout_freed_from_cleaned; SYSCTL_UINT(_vm, OID_AUTO, pageout_freed_from_cleaned, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_freed_from_cleaned, 0, ""); /* counts of pages entering the cleaned queue */ -extern unsigned int vm_pageout_enqueued_cleaned, vm_pageout_enqueued_cleaned_from_inactive_clean, vm_pageout_enqueued_cleaned_from_inactive_dirty; +extern unsigned int vm_pageout_enqueued_cleaned, vm_pageout_enqueued_cleaned_from_inactive_dirty; SYSCTL_UINT(_vm, OID_AUTO, pageout_enqueued_cleaned, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_enqueued_cleaned, 0, ""); /* sum of next two */ -SYSCTL_UINT(_vm, OID_AUTO, pageout_enqueued_cleaned_from_inactive_clean, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_enqueued_cleaned_from_inactive_clean, 0, ""); SYSCTL_UINT(_vm, OID_AUTO, pageout_enqueued_cleaned_from_inactive_dirty, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_enqueued_cleaned_from_inactive_dirty, 0, ""); /* counts of pages leaving the cleaned queue */ @@ -1760,6 +1978,35 @@ extern int64_t vm_prefault_nb_pages, vm_prefault_nb_bailout; SYSCTL_QUAD(_vm, OID_AUTO, prefault_nb_pages, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_prefault_nb_pages, ""); SYSCTL_QUAD(_vm, OID_AUTO, prefault_nb_bailout, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_prefault_nb_bailout, ""); +#if defined (__x86_64__) +extern unsigned int vm_clump_promote_threshold; +SYSCTL_UINT(_vm, OID_AUTO, vm_clump_promote_threshold, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_clump_promote_threshold, 0, "clump size threshold for promotes"); +#if DEVELOPMENT || DEBUG +extern unsigned long vm_clump_stats[]; +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats1, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[1], "free page allocations from clump of 1 page"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats2, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[2], "free page allocations from clump of 2 pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats3, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[3], "free page allocations from clump of 3 pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats4, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[4], "free page allocations from clump of 4 pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats5, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[5], "free page allocations from clump of 5 pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats6, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[6], "free page allocations from clump of 6 pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats7, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[7], "free page allocations from clump of 7 pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats8, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[8], "free page allocations from clump of 8 pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats9, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[9], "free page allocations from clump of 9 pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats10, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[10], "free page allocations from clump of 10 pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats11, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[11], "free page allocations from clump of 11 pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats12, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[12], "free page allocations from clump of 12 pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats13, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[13], "free page allocations from clump of 13 pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats14, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[14], "free page allocations from clump of 14 pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats15, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[15], "free page allocations from clump of 15 pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats16, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[16], "free page allocations from clump of 16 pages"); +extern unsigned long vm_clump_allocs, vm_clump_inserts, vm_clump_inrange, vm_clump_promotes; +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_alloc, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_allocs, "free page allocations"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_inserts, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_inserts, "free page insertions"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_inrange, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_inrange, "free page insertions that are part of vm_pages"); +SYSCTL_LONG(_vm, OID_AUTO, vm_clump_promotes, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_promotes, "pages promoted to head"); +#endif /* if DEVELOPMENT || DEBUG */ +#endif /* #if defined (__x86_64__) */ + #if CONFIG_SECLUDED_MEMORY SYSCTL_UINT(_vm, OID_AUTO, num_tasks_can_use_secluded_mem, CTLFLAG_RD | CTLFLAG_LOCKED, &num_tasks_can_use_secluded_mem, 0, ""); @@ -1782,11 +2029,7 @@ SYSCTL_UINT(_vm, OID_AUTO, page_secluded_grab_failure_dirty, CTLFLAG_RD | CTLFLA SYSCTL_UINT(_vm, OID_AUTO, page_secluded_grab_for_iokit, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded.grab_for_iokit, 0, ""); SYSCTL_UINT(_vm, OID_AUTO, page_secluded_grab_for_iokit_success, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded.grab_for_iokit_success, 0, ""); -extern uint64_t vm_pageout_freed_from_secluded; -extern uint64_t vm_pageout_secluded_reactivated; extern uint64_t vm_pageout_secluded_burst_count; -SYSCTL_QUAD(_vm, OID_AUTO, pageout_freed_from_secluded, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_freed_from_secluded, ""); -SYSCTL_QUAD(_vm, OID_AUTO, pageout_secluded_reactivated, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_secluded_reactivated, "Secluded pages reactivated"); /* sum of all reactivated AND busy and nolock (even though those actually get reDEactivated */ SYSCTL_QUAD(_vm, OID_AUTO, pageout_secluded_burst_count, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_secluded_burst_count, ""); #endif /* CONFIG_SECLUDED_MEMORY */ @@ -1923,3 +2166,44 @@ kas_info(struct proc *p, return 0; #endif /* !SECURE_KERNEL */ } + + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wcast-qual" +#pragma clang diagnostic ignored "-Wunused-function" + +static void asserts() { + static_assert(sizeof(vm_min_kernel_address) == sizeof(unsigned long)); + static_assert(sizeof(vm_max_kernel_address) == sizeof(unsigned long)); +} + +SYSCTL_ULONG(_vm, OID_AUTO, vm_min_kernel_address, CTLFLAG_RD, (unsigned long *) &vm_min_kernel_address, ""); +SYSCTL_ULONG(_vm, OID_AUTO, vm_max_kernel_address, CTLFLAG_RD, (unsigned long *) &vm_max_kernel_address, ""); +#pragma clang diagnostic pop + +extern uint32_t vm_page_pages; +SYSCTL_UINT(_vm, OID_AUTO, pages, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_pages, 0, ""); + +#if (__arm__ || __arm64__) && (DEVELOPMENT || DEBUG) +extern void pmap_footprint_suspend(vm_map_t map, boolean_t suspend); +static int +sysctl_vm_footprint_suspend SYSCTL_HANDLER_ARGS +{ +#pragma unused(oidp, arg1, arg2) + int error = 0; + int new_value; + + if (req->newptr == USER_ADDR_NULL) { + return 0; + } + error = SYSCTL_IN(req, &new_value, sizeof(int)); + if (error) { + return error; + } + pmap_footprint_suspend(current_map(), new_value); + return 0; +} +SYSCTL_PROC(_vm, OID_AUTO, footprint_suspend, + CTLTYPE_INT|CTLFLAG_WR|CTLFLAG_ANYBODY|CTLFLAG_LOCKED|CTLFLAG_MASKED, + 0, 0, &sysctl_vm_footprint_suspend, "I", ""); +#endif /* (__arm__ || __arm64__) && (DEVELOPMENT || DEBUG) */ diff --git a/bsd/vm/vnode_pager.c b/bsd/vm/vnode_pager.c index b70717b92..69dad2981 100644 --- a/bsd/vm/vnode_pager.c +++ b/bsd/vm/vnode_pager.c @@ -73,14 +73,13 @@ #include <kern/assert.h> #include <sys/kdebug.h> -#include <machine/spl.h> - #include <nfs/rpcv2.h> #include <nfs/nfsproto.h> #include <nfs/nfs.h> #include <vm/vm_protos.h> +#include <vfs/vfs_disk_conditioner.h> void vnode_pager_throttle() @@ -93,13 +92,10 @@ vnode_pager_throttle() throttle_lowpri_io(1); } - boolean_t vnode_pager_isSSD(vnode_t vp) { - if (vp->v_mount->mnt_kern_flag & MNTK_SSD) - return (TRUE); - return (FALSE); + return disk_conditioner_mount_is_ssd(vp->v_mount); } #if CONFIG_IOSCHED @@ -251,7 +247,7 @@ u_int32_t vnode_trim ( * in each call to ensure that the entire range is covered. */ error = VNOP_BLOCKMAP (vp, current_offset, remaining_length, - &io_blockno, &io_bytecount, NULL, VNODE_READ, NULL); + &io_blockno, &io_bytecount, NULL, VNODE_READ | VNODE_BLOCKMAP_NO_TRACK, NULL); if (error) { goto trim_exit; @@ -367,7 +363,7 @@ vnode_pageout(struct vnode *vp, else request_flags = UPL_UBC_PAGEOUT | UPL_RET_ONLY_DIRTY; - if (ubc_create_upl(vp, f_offset, size, &upl, &pl, request_flags) != KERN_SUCCESS) { + if (ubc_create_upl_kernel(vp, f_offset, size, &upl, &pl, request_flags, VM_KERN_MEMORY_FILE) != KERN_SUCCESS) { result = PAGER_ERROR; error_ret = EINVAL; goto out; @@ -601,7 +597,7 @@ vnode_pagein( } goto out; } - ubc_create_upl(vp, f_offset, size, &upl, &pl, UPL_UBC_PAGEIN | UPL_RET_ONLY_ABSENT); + ubc_create_upl_kernel(vp, f_offset, size, &upl, &pl, UPL_UBC_PAGEIN | UPL_RET_ONLY_ABSENT, VM_KERN_MEMORY_FILE); if (upl == (upl_t)NULL) { result = PAGER_ABSENT; diff --git a/config/BSDKernel.arm.exports b/config/BSDKernel.arm.exports new file mode 100644 index 000000000..e4e0cf3fd --- /dev/null +++ b/config/BSDKernel.arm.exports @@ -0,0 +1,16 @@ +_file_vnode +_mbuf_data +_mbuf_len +_mbuf_next +_mbuf_nextpkt +_mbuf_pkthdr_header +_mbuf_pkthdr_len +_mbuf_pkthdr_rcvif +_mbuf_pkthdr_setheader +_mbuf_setlen +_mbuf_setnextpkt +_mbuf_type +_proc_ucred +_rootvnode +_suser +_ubc_setcred diff --git a/config/BSDKernel.arm64.exports b/config/BSDKernel.arm64.exports new file mode 100644 index 000000000..e4e0cf3fd --- /dev/null +++ b/config/BSDKernel.arm64.exports @@ -0,0 +1,16 @@ +_file_vnode +_mbuf_data +_mbuf_len +_mbuf_next +_mbuf_nextpkt +_mbuf_pkthdr_header +_mbuf_pkthdr_len +_mbuf_pkthdr_rcvif +_mbuf_pkthdr_setheader +_mbuf_setlen +_mbuf_setnextpkt +_mbuf_type +_proc_ucred +_rootvnode +_suser +_ubc_setcred diff --git a/config/BSDKernel.exports b/config/BSDKernel.exports index f595bb171..83659ed3c 100644 --- a/config/BSDKernel.exports +++ b/config/BSDKernel.exports @@ -574,7 +574,7 @@ _timevalfix _timevalsub _tvtoabstime _ubc_blktooff -_ubc_create_upl +_ubc_create_upl:_ubc_create_upl_external _ubc_getcred _ubc_getsize _ubc_msync @@ -720,6 +720,7 @@ _vnode_isinuse _vnode_islnk _vnode_ismount _vnode_ismountedon +_vnode_isnamedstream _vnode_isnocache _vnode_isnoreadahead _vnode_israge diff --git a/config/IOKit.arm.exports b/config/IOKit.arm.exports new file mode 100644 index 000000000..f4ee08125 --- /dev/null +++ b/config/IOKit.arm.exports @@ -0,0 +1,309 @@ +_IOPanic +_OSSynchronizeIO +_PE_arm_debug_panic_hook +__Z16IODTFindSlotNameP15IORegistryEntrym +__Z16IODTSetResolvingP15IORegistryEntryPFlmPmS1_EPFvS0_PhS4_S4_E +__Z17IODTGetCellCountsP15IORegistryEntryPmS1_ +__Z22IODTResolveAddressCellP15IORegistryEntryPmS1_S1_ +__Z23IODTFindMatchingEntriesP15IORegistryEntrymPKc +__ZN10IOWorkLoop19workLoopWithOptionsEm +__ZN10IOWorkLoop9sleepGateEPvym +__ZN10IOWorkLoop9sleepGateEPvm +__ZN11IOCatalogue11findDriversEP12OSDictionaryPl +__ZN11IOCatalogue11findDriversEP9IOServicePl +__ZN11IODataQueue11withEntriesEmm +__ZN11IODataQueue12withCapacityEm +__ZN11IODataQueue15initWithEntriesEmm +__ZN11IODataQueue16initWithCapacityEm +__ZN11IODataQueue7enqueueEPvm +__ZN11IOMemoryMap10getAddressEv +__ZN11IOMemoryMap18getPhysicalSegmentEmPm +__ZN11IOMemoryMap19setMemoryDescriptorEP18IOMemoryDescriptory +__ZN11IOMemoryMap7getSizeEv +__ZN11IOMemoryMap8redirectEP18IOMemoryDescriptormm +__ZN11IOMemoryMap8redirectEP18IOMemoryDescriptormy +__ZN12IODMACommand11OutputBig32EPS_NS_9Segment64EPvm +__ZN12IODMACommand11OutputBig64EPS_NS_9Segment64EPvm +__ZN12IODMACommand11synchronizeEm +__ZN12IODMACommand12OutputHost32EPS_NS_9Segment64EPvm +__ZN12IODMACommand12OutputHost64EPS_NS_9Segment64EPvm +__ZN12IODMACommand14OutputLittle32EPS_NS_9Segment64EPvm +__ZN12IODMACommand14OutputLittle64EPS_NS_9Segment64EPvm +__ZN12IODMACommand15genIOVMSegmentsEPFbPS_NS_9Segment64EPvmEPyS2_Pm +__ZN12IODMACommand15genIOVMSegmentsEPyPvPm +__ZN12IODMACommand16createCopyBufferE11IODirectiony +__ZN12IODMACommand17withSpecificationEPFbPS_NS_9Segment64EPvmEPKNS_14SegmentOptionsEjP8IOMapperS2_ +__ZN12IODMACommand21initWithSpecificationEPFbPS_NS_9Segment64EPvmEPKNS_14SegmentOptionsEjP8IOMapperS2_ +__ZN12IODMACommand24prepareWithSpecificationEPFbPS_NS_9Segment64EPvmEPKNS_14SegmentOptionsEjP8IOMapperyybb +__ZN12IODMACommand17withSpecificationEPFbPS_NS_9Segment64EPvmEhyNS_14MappingOptionsEymP8IOMapperS2_ +__ZN12IODMACommand21initWithSpecificationEPFbPS_NS_9Segment64EPvmEhyNS_14MappingOptionsEymP8IOMapperS2_ +__ZN12IODMACommand24prepareWithSpecificationEPFbPS_NS_9Segment64EPvmEhyNS_14MappingOptionsEymP8IOMapperyybb +__ZN12IODMACommand8transferEmyPvy +__ZN12IOUserClient12initWithTaskEP4taskPvm +__ZN12IOUserClient12initWithTaskEP4taskPvmP12OSDictionary +__ZN12IOUserClient15mapClientMemoryEmP4taskmj +__ZN12IOUserClient15sendAsyncResultEPjiPPvm +__ZN12IOUserClient17mapClientMemory64EmP4taskmy +__ZN12IOUserClient17sendAsyncResult64EPyiS0_m +__ZN12IOUserClient19clientMemoryForTypeEmPmPP18IOMemoryDescriptor +__ZN12IOUserClient19setAsyncReference64EPyP8ipc_portyy +__ZN12IOUserClient19setAsyncReference64EPyP8ipc_portyyP4task +__ZN12IOUserClient23getExternalTrapForIndexEm +__ZN12IOUserClient24getNotificationSemaphoreEmPP9semaphore +__ZN12IOUserClient24getTargetAndTrapForIndexEPP9IOServicem +__ZN12IOUserClient24registerNotificationPortEP8ipc_portmm +__ZN12IOUserClient24registerNotificationPortEP8ipc_portmy +__ZN12IOUserClient25getExternalMethodForIndexEm +__ZN12IOUserClient26getTargetAndMethodForIndexEPP9IOServicem +__ZN12IOUserClient28sendAsyncResult64WithOptionsEPyiS0_mm +__ZN12IOUserClient30getExternalAsyncMethodForIndexEm +__ZN12IOUserClient31getAsyncTargetAndMethodForIndexEPP9IOServicem +__ZN13IOCommandGate12commandSleepEPvym +__ZN13IOCommandGate12commandSleepEPvm +__ZN13IOCommandPool11commandPoolEP9IOServiceP10IOWorkLoopm +__ZN13IOCommandPool4initEP9IOServiceP10IOWorkLoopm +__ZN13IOEventSource9sleepGateEPvym +__ZN13IOEventSource9sleepGateEPvm +__ZN13_IOServiceJob8startJobEP9IOServiceim +__ZN14IODeviceMemory12withSubRangeEPS_mm +__ZN14IODeviceMemory13arrayFromListEPNS_11InitElementEm +__ZN14IODeviceMemory9withRangeEmm +__ZN14IOMemoryCursor17withSpecificationEPFvNS_15PhysicalSegmentEPvmEmmm +__ZN14IOMemoryCursor19genPhysicalSegmentsEP18IOMemoryDescriptormPvmmPm +__ZN14IOMemoryCursor21initWithSpecificationEPFvNS_15PhysicalSegmentEPvmEmmm +__ZN14IOPMrootDomain17setSleepSupportedEm +__ZN14IOPMrootDomain19sysPowerDownHandlerEPvS0_mP9IOServiceS0_j +__ZN14IOPMrootDomain20claimSystemWakeEventEP9IOServicemPKcP8OSObject +__ZN14IOPMrootDomain24receivePowerNotificationEm +__ZN14IOPMrootDomain27displayWranglerNotificationEPvS0_mP9IOServiceS0_j +__ZN15IODMAController13getControllerEP9IOServicem +__ZN15IODMAController16notifyDMACommandEP16IODMAEventSourceP12IODMACommandimy +__ZN15IODMAController20createControllerNameEm +__ZN15IODMAController21registerDMAControllerEm +__ZN16IODMAEventSource14dmaEventSourceEP8OSObjectP9IOServicePFvS1_PS_P12IODMACommandimyES8_m +__ZN16IODMAEventSource15startDMACommandEP12IODMACommand11IODirectionmm +__ZN16IODMAEventSource16notifyDMACommandEP12IODMACommandimy +__ZN16IODMAEventSource4initEP8OSObjectP9IOServicePFvS1_PS_P12IODMACommandimyES8_m +__ZN16IORangeAllocator10deallocateEmm +__ZN16IORangeAllocator12allocElementEm +__ZN16IORangeAllocator13allocateRangeEmm +__ZN16IORangeAllocator14deallocElementEm +__ZN16IORangeAllocator28setFragmentCapacityIncrementEm +__ZN16IORangeAllocator4initEmmmm +__ZN16IORangeAllocator8allocateEmPmm +__ZN16IORangeAllocator9withRangeEmmmm +__ZN17IOBigMemoryCursor13outputSegmentEN14IOMemoryCursor15PhysicalSegmentEPvm +__ZN17IOBigMemoryCursor17withSpecificationEmmm +__ZN17IOBigMemoryCursor21initWithSpecificationEmmm +__ZN17IOSharedDataQueue11withEntriesEmm +__ZN17IOSharedDataQueue12getQueueSizeEv +__ZN17IOSharedDataQueue12setQueueSizeEm +__ZN17IOSharedDataQueue12withCapacityEm +__ZN17IOSharedDataQueue16initWithCapacityEm +__ZN17IOSharedDataQueue7dequeueEPvPm +__ZN17IOSharedDataQueue7enqueueEPvm +__ZN18IOMemoryDescriptor10setMappingEP4taskjm +__ZN18IOMemoryDescriptor10withRangesEP14IOVirtualRangem11IODirectionP4taskb +__ZN18IOMemoryDescriptor10writeBytesEmPKvm +__ZN18IOMemoryDescriptor11makeMappingEPS_P4taskjmmm +__ZN18IOMemoryDescriptor11withAddressEPvm11IODirection +__ZN18IOMemoryDescriptor11withAddressEjm11IODirectionP4task +__ZN18IOMemoryDescriptor11withOptionsEPvmmP4taskmP8IOMapper +__ZN18IOMemoryDescriptor12setPurgeableEmPm +__ZN18IOMemoryDescriptor12withSubRangeEPS_mm11IODirection +__ZN18IOMemoryDescriptor13getPageCountsEPmS0_ +__ZN18IOMemoryDescriptor14initWithRangesEP14IOVirtualRangem11IODirectionP4taskb +__ZN18IOMemoryDescriptor15initWithAddressEPvm11IODirection +__ZN18IOMemoryDescriptor15initWithAddressEjm11IODirectionP4task +__ZN18IOMemoryDescriptor15initWithOptionsEPvmmP4taskmP8IOMapper +__ZN18IOMemoryDescriptor16getSourceSegmentEmPm +__ZN18IOMemoryDescriptor16performOperationEmmm +__ZN18IOMemoryDescriptor16withAddressRangeEyymP4task +__ZN18IOMemoryDescriptor17getVirtualSegmentEmPm +__ZN18IOMemoryDescriptor17withAddressRangesEP14IOAddressRangemmP4task +__ZN18IOMemoryDescriptor18getPhysicalSegmentEmPm +__ZN18IOMemoryDescriptor18getPhysicalSegmentEmPmm +__ZN18IOMemoryDescriptor18withPhysicalRangesEP15IOPhysicalRangem11IODirectionb +__ZN18IOMemoryDescriptor19createMappingInTaskEP4taskymyy +__ZN18IOMemoryDescriptor19withPhysicalAddressEmm11IODirection +__ZN18IOMemoryDescriptor20getPhysicalSegment64EmPm +__ZN18IOMemoryDescriptor22initWithPhysicalRangesEP15IOPhysicalRangem11IODirectionb +__ZN18IOMemoryDescriptor23initWithPhysicalAddressEmm11IODirection +__ZN18IOMemoryDescriptor3mapEP4taskjmmm +__ZN18IOMemoryDescriptor3mapEm +__ZN18IOMemoryDescriptor5doMapEP7_vm_mapPjmmm +__ZN18IOMemoryDescriptor6setTagEm +__ZN18IOMemoryDescriptor7doUnmapEP7_vm_mapjm +__ZN18IOMemoryDescriptor9readBytesEmPvm +__ZN18IORegistryIterator11iterateOverEP15IORegistryEntryPK15IORegistryPlanem +__ZN18IORegistryIterator11iterateOverEPK15IORegistryPlanem +__ZN18IOTimerEventSource10setTimeoutEjyy +__ZN18IOTimerEventSource10setTimeoutEy +__ZN18IOTimerEventSource10setTimeoutE13mach_timespec +__ZN18IOTimerEventSource10setTimeoutEmm +__ZN18IOTimerEventSource10wakeAtTimeEjyy +__ZN18IOTimerEventSource10wakeAtTimeEy +__ZN18IOTimerEventSource10wakeAtTimeE13mach_timespec +__ZN18IOTimerEventSource10wakeAtTimeEmm +__ZN18IOTimerEventSource12setTimeoutMSEm +__ZN18IOTimerEventSource12setTimeoutUSEm +__ZN18IOTimerEventSource12wakeAtTimeMSEm +__ZN18IOTimerEventSource12wakeAtTimeUSEm +__ZN18IOTimerEventSource15setTimeoutTicksEm +__ZN18IOTimerEventSource15wakeAtTimeTicksEm +__ZN20IOLittleMemoryCursor13outputSegmentEN14IOMemoryCursor15PhysicalSegmentEPvm +__ZN20IOLittleMemoryCursor17withSpecificationEmmm +__ZN20IOLittleMemoryCursor21initWithSpecificationEmmm +__ZN20RootDomainUserClient15setPreventativeEmm +__ZN20RootDomainUserClient26getTargetAndMethodForIndexEPP9IOServicem +__ZN21IOInterruptController10initVectorElP17IOInterruptVector +__ZN21IOInterruptController11causeVectorElP17IOInterruptVector +__ZN21IOInterruptController12enableVectorElP17IOInterruptVector +__ZN21IOInterruptController13getVectorTypeElP17IOInterruptVector +__ZN21IOInterruptController17disableVectorHardElP17IOInterruptVector +__ZN21IOInterruptController17vectorCanBeSharedElP17IOInterruptVector +__ZN21IOInterruptController28timeStampInterruptHandlerEndElP17IOInterruptVector +__ZN21IOInterruptController30timeStampInterruptHandlerStartElP17IOInterruptVector +__ZN21IONaturalMemoryCursor13outputSegmentEN14IOMemoryCursor15PhysicalSegmentEPvm +__ZN21IONaturalMemoryCursor17withSpecificationEmmm +__ZN21IONaturalMemoryCursor21initWithSpecificationEmmm +__ZN21IOSubMemoryDescriptor11makeMappingEP18IOMemoryDescriptorP4taskjmmm +__ZN21IOSubMemoryDescriptor12initSubRangeEP18IOMemoryDescriptormm11IODirection +__ZN21IOSubMemoryDescriptor12setPurgeableEmPm +__ZN21IOSubMemoryDescriptor12withSubRangeEP18IOMemoryDescriptormmm +__ZN21IOSubMemoryDescriptor18getPhysicalSegmentEmPmm +__ZN21IOSubMemoryDescriptor7prepareE11IODirection +__ZN21IOSubMemoryDescriptor8completeE11IODirection +__ZN23IOMultiMemoryDescriptor15withDescriptorsEPP18IOMemoryDescriptorm11IODirectionb +__ZN23IOMultiMemoryDescriptor18getPhysicalSegmentEmPmm +__ZN23IOMultiMemoryDescriptor19initWithDescriptorsEPP18IOMemoryDescriptorm11IODirectionb +__ZN23IOMultiMemoryDescriptor7prepareE11IODirection +__ZN23IOMultiMemoryDescriptor8completeE11IODirection +__ZN24IOBufferMemoryDescriptor11appendBytesEPKvj +__ZN24IOBufferMemoryDescriptor11withOptionsEmjj +__ZN24IOBufferMemoryDescriptor12setDirectionE11IODirection +__ZN24IOBufferMemoryDescriptor12withCapacityEj11IODirectionb +__ZN24IOBufferMemoryDescriptor13initWithBytesEPKvj11IODirectionb +__ZN24IOBufferMemoryDescriptor14getBytesNoCopyEjj +__ZN24IOBufferMemoryDescriptor15initWithOptionsEmjj +__ZN24IOBufferMemoryDescriptor15initWithOptionsEmjjP4task +__ZN24IOBufferMemoryDescriptor17getVirtualSegmentEmPm +__ZN24IOBufferMemoryDescriptor17inTaskWithOptionsEP4taskmjj +__ZN24IOBufferMemoryDescriptor20initWithPhysicalMaskEP4taskmyyy +__ZN24IOBufferMemoryDescriptor22inTaskWithPhysicalMaskEP4taskmyy +__ZN24IOBufferMemoryDescriptor9setLengthEj +__ZN24IOBufferMemoryDescriptor9withBytesEPKvj11IODirectionb +__ZN25IOGeneralMemoryDescriptor11setPositionEm +__ZN25IOGeneralMemoryDescriptor11wireVirtualE11IODirection +__ZN25IOGeneralMemoryDescriptor12setPurgeableEmPm +__ZN25IOGeneralMemoryDescriptor13mapIntoKernelEj +__ZN25IOGeneralMemoryDescriptor14initWithRangesEP14IOVirtualRangem11IODirectionP4taskb +__ZN25IOGeneralMemoryDescriptor15initWithAddressEPvm11IODirection +__ZN25IOGeneralMemoryDescriptor15initWithAddressEjm11IODirectionP4task +__ZN25IOGeneralMemoryDescriptor15initWithOptionsEPvmmP4taskmP8IOMapper +__ZN25IOGeneralMemoryDescriptor15unmapFromKernelEv +__ZN25IOGeneralMemoryDescriptor16getSourceSegmentEmPm +__ZN25IOGeneralMemoryDescriptor17getVirtualSegmentEmPm +__ZN25IOGeneralMemoryDescriptor18getPhysicalSegmentEmPm +__ZN25IOGeneralMemoryDescriptor18getPhysicalSegmentEmPmm +__ZN25IOGeneralMemoryDescriptor20getPhysicalSegment64EmPm +__ZN25IOGeneralMemoryDescriptor22initWithPhysicalRangesEP15IOPhysicalRangem11IODirectionb +__ZN25IOGeneralMemoryDescriptor23initWithPhysicalAddressEmm11IODirection +__ZN25IOGeneralMemoryDescriptor5doMapEP7_vm_mapPjmmm +__ZN25IOGeneralMemoryDescriptor7doUnmapEP7_vm_mapjm +__ZN25IOGeneralMemoryDescriptor7prepareE11IODirection +__ZN25IOGeneralMemoryDescriptor8completeE11IODirection +__ZN29IOInterleavedMemoryDescriptor12withCapacityEm11IODirection +__ZN29IOInterleavedMemoryDescriptor16initWithCapacityEm11IODirection +__ZN29IOInterleavedMemoryDescriptor18getPhysicalSegmentEmPmm +__ZN29IOInterleavedMemoryDescriptor19setMemoryDescriptorEP18IOMemoryDescriptormm +__ZN29IOInterleavedMemoryDescriptor22clearMemoryDescriptorsE11IODirection +__ZN29IOInterleavedMemoryDescriptor7prepareE11IODirection +__ZN29IOInterleavedMemoryDescriptor8completeE11IODirection +__ZN8IOPMprot10gMetaClassE +__ZN8IOPMprot10superClassE +__ZN8IOPMprot9MetaClassC1Ev +__ZN8IOPMprot9MetaClassC2Ev +__ZN8IOPMprot9metaClassE +__ZN8IOPMprotC1EPK11OSMetaClass +__ZN8IOPMprotC1Ev +__ZN8IOPMprotC2EPK11OSMetaClass +__ZN8IOPMprotC2Ev +__ZN8IOPMprotD0Ev +__ZN8IOPMprotD2Ev +__ZN9IOService10adjustBusyEl +__ZN9IOService10handleOpenEPS_mPv +__ZN9IOService10systemWakeEv +__ZN9IOService10youAreRootEv +__ZN9IOService11_adjustBusyEl +__ZN9IOService11handleCloseEPS_m +__ZN9IOService11tellClientsEi +__ZN9IOService12clampPowerOnEm +__ZN9IOService12didTerminateEPS_mPb +__ZN9IOService12requestProbeEm +__ZN9IOService12updateReportEP19IOReportChannelListjPvS2_ +__ZN9IOService12waitForStateEmmP13mach_timespec +__ZN9IOService13getPMworkloopEv +__ZN9IOService13messageClientEmP8OSObjectPvj +__ZN9IOService13newUserClientEP4taskPvmP12OSDictionaryPP12IOUserClient +__ZN9IOService13newUserClientEP4taskPvmPP12IOUserClient +__ZN9IOService13startMatchingEm +__ZN9IOService13waitMatchIdleEm +__ZN9IOService13willTerminateEPS_m +__ZN9IOService14doServiceMatchEm +__ZN9IOService14messageClientsEmPvj +__ZN9IOService14newTemperatureElPS_ +__ZN9IOService14setPowerParentEP17IOPowerConnectionbm +__ZN9IOService15addNotificationEPK8OSSymbolP12OSDictionaryPFbPvS5_PS_ES5_S5_l +__ZN9IOService15configureReportEP19IOReportChannelListjPvS2_ +__ZN9IOService15nextIdleTimeoutEyyj +__ZN9IOService15registerServiceEm +__ZN9IOService15tellChangeDown1Em +__ZN9IOService15tellChangeDown2Em +__ZN9IOService15terminateClientEPS_m +__ZN9IOService15terminatePhase1Em +__ZN9IOService15terminateWorkerEm +__ZN9IOService16ack_timer_tickedEv +__ZN9IOService16command_receivedEPvS0_S0_S0_ +__ZN9IOService16didYouWakeSystemEv +__ZN9IOService16registerInterestEPK8OSSymbolPFiPvS3_mPS_S3_jES3_S3_ +__ZN9IOService16requestTerminateEPS_m +__ZN9IOService16setCPUSnoopDelayEm +__ZN9IOService18doServiceTerminateEm +__ZN9IOService18matchPropertyTableEP12OSDictionaryPl +__ZN9IOService18requireMaxBusStallEm +__ZN9IOService18settleTimerExpiredEv +__ZN9IOService18systemWillShutdownEm +__ZN9IOService19deliverNotificationEPK8OSSymbolmm +__ZN9IOService22PM_Clamp_Timer_ExpiredEv +__ZN9IOService22powerDomainDidChangeToEmP17IOPowerConnection +__ZN9IOService23acknowledgeNotificationEPvm +__ZN9IOService23addMatchingNotificationEPK8OSSymbolP12OSDictionaryPFbPvS5_PS_P10IONotifierES5_S5_l +__ZN9IOService23powerDomainWillChangeToEmP17IOPowerConnection +__ZN9IOService23scheduleTerminatePhase2Em +__ZN9IOService23tellClientsWithResponseEi +__ZN9IOService24PM_idle_timer_expirationEv +__ZN9IOService24mapDeviceMemoryWithIndexEjm +__ZN9IOService26temperatureCriticalForZoneEPS_ +__ZN9IOService27serializedAllowPowerChange2Em +__ZN9IOService28serializedCancelPowerChange2Em +__ZN9IOService4openEPS_mPv +__ZN9IOService5closeEPS_m +__ZN9IOService5probeEPS_Pl +__ZN9IOService6PMfreeEv +__ZN9IOService7messageEmPS_Pv +__ZN9IOService8finalizeEm +__ZN9IOService9terminateEm +__ZNK11IOCatalogue13serializeDataEmP11OSSerialize +__ZNK15IORegistryEntry11getPropertyEPK8OSStringPK15IORegistryPlanem +__ZNK15IORegistryEntry11getPropertyEPK8OSSymbolPK15IORegistryPlanem +__ZNK15IORegistryEntry11getPropertyEPKcPK15IORegistryPlanem +__ZNK15IORegistryEntry12copyPropertyEPK8OSStringPK15IORegistryPlanem +__ZNK15IORegistryEntry12copyPropertyEPK8OSSymbolPK15IORegistryPlanem +__ZNK15IORegistryEntry12copyPropertyEPKcPK15IORegistryPlanem +__ZNK18IOMemoryDescriptor19dmaCommandOperationEmPvj +__ZNK25IOGeneralMemoryDescriptor19dmaCommandOperationEmPvj +__ZNK8IOPMprot12getMetaClassEv +__ZNK8IOPMprot9MetaClass5allocEv +__ZTV8IOPMprot +__ZTVN8IOPMprot9MetaClassE diff --git a/config/IOKit.arm64.exports b/config/IOKit.arm64.exports new file mode 100644 index 000000000..ed271b62c --- /dev/null +++ b/config/IOKit.arm64.exports @@ -0,0 +1,230 @@ +_OSSynchronizeIO +_PE_arm_debug_panic_hook +__Z16IODTFindSlotNameP15IORegistryEntryj +__Z16IODTSetResolvingP15IORegistryEntryPFxjPjS1_EPFvS0_PhS4_S4_E +__Z17IODTGetCellCountsP15IORegistryEntryPjS1_ +__Z22IODTResolveAddressCellP15IORegistryEntryPjPyS2_ +__Z23IODTFindMatchingEntriesP15IORegistryEntryjPKc +__ZN10IOWorkLoop19workLoopWithOptionsEj +__ZN10IOWorkLoop9sleepGateEPvj +__ZN10IOWorkLoop9sleepGateEPvyj +__ZN11IOCatalogue11findDriversEP12OSDictionaryPi +__ZN11IOCatalogue11findDriversEP9IOServicePi +__ZN11IODataQueue11withEntriesEjj +__ZN11IODataQueue12withCapacityEj +__ZN11IODataQueue15initWithEntriesEjj +__ZN11IODataQueue16initWithCapacityEj +__ZN11IODataQueue7enqueueEPvj +__ZN11IOMemoryMap18getPhysicalSegmentEyPyj +__ZN11IOMemoryMap19setMemoryDescriptorEP18IOMemoryDescriptory +__ZN11IOMemoryMap8redirectEP18IOMemoryDescriptorjy +__ZN12IODMACommand11OutputBig32EPS_NS_9Segment64EPvj +__ZN12IODMACommand11OutputBig64EPS_NS_9Segment64EPvj +__ZN12IODMACommand11synchronizeEj +__ZN12IODMACommand12OutputHost32EPS_NS_9Segment64EPvj +__ZN12IODMACommand12OutputHost64EPS_NS_9Segment64EPvj +__ZN12IODMACommand14OutputLittle32EPS_NS_9Segment64EPvj +__ZN12IODMACommand14OutputLittle64EPS_NS_9Segment64EPvj +__ZN12IODMACommand15genIOVMSegmentsEPFbPS_NS_9Segment64EPvjEPyS2_Pj +__ZN12IODMACommand15genIOVMSegmentsEPyPvPj +__ZN12IODMACommand16createCopyBufferEjy +__ZN12IODMACommand17withSpecificationEPFbPS_NS_9Segment64EPvjEPKNS_14SegmentOptionsEjP8IOMapperS2_ +__ZN12IODMACommand21initWithSpecificationEPFbPS_NS_9Segment64EPvjEPKNS_14SegmentOptionsEjP8IOMapperS2_ +__ZN12IODMACommand24prepareWithSpecificationEPFbPS_NS_9Segment64EPvjEPKNS_14SegmentOptionsEjP8IOMapperyybb +__ZN12IODMACommand17withSpecificationEPFbPS_NS_9Segment64EPvjEhyNS_14MappingOptionsEyjP8IOMapperS2_ +__ZN12IODMACommand21initWithSpecificationEPFbPS_NS_9Segment64EPvjEhyNS_14MappingOptionsEyjP8IOMapperS2_ +__ZN12IODMACommand24prepareWithSpecificationEPFbPS_NS_9Segment64EPvjEhyNS_14MappingOptionsEyjP8IOMapperyybb +__ZN12IODMACommand8transferEjyPvy +__ZN12IOUserClient12initWithTaskEP4taskPvj +__ZN12IOUserClient12initWithTaskEP4taskPvjP12OSDictionary +__ZN12IOUserClient15sendAsyncResultEPjiPPvj +__ZN12IOUserClient17mapClientMemory64EjP4taskjy +__ZN12IOUserClient17sendAsyncResult64EPyiS0_j +__ZN12IOUserClient19clientMemoryForTypeEjPjPP18IOMemoryDescriptor +__ZN12IOUserClient19setAsyncReference64EPyP8ipc_portyy +__ZN12IOUserClient19setAsyncReference64EPyP8ipc_portyyP4task +__ZN12IOUserClient23getExternalTrapForIndexEj +__ZN12IOUserClient24getNotificationSemaphoreEjPP9semaphore +__ZN12IOUserClient24getTargetAndTrapForIndexEPP9IOServicej +__ZN12IOUserClient24registerNotificationPortEP8ipc_portjj +__ZN12IOUserClient24registerNotificationPortEP8ipc_portjy +__ZN12IOUserClient25getExternalMethodForIndexEj +__ZN12IOUserClient26getTargetAndMethodForIndexEPP9IOServicej +__ZN12IOUserClient28sendAsyncResult64WithOptionsEPyiS0_jj +__ZN12IOUserClient30getExternalAsyncMethodForIndexEj +__ZN12IOUserClient31getAsyncTargetAndMethodForIndexEPP9IOServicej +__ZN13IOCommandGate12commandSleepEPvj +__ZN13IOCommandGate12commandSleepEPvyj +__ZN13IOCommandPool11commandPoolEP9IOServiceP10IOWorkLoopj +__ZN13IOCommandPool4initEP9IOServiceP10IOWorkLoopj +__ZN13IOEventSource9sleepGateEPvj +__ZN13IOEventSource9sleepGateEPvyj +__ZN13_IOServiceJob8startJobEP9IOServiceij +__ZN14IODeviceMemory12withSubRangeEPS_yy +__ZN14IODeviceMemory13arrayFromListEPNS_11InitElementEj +__ZN14IODeviceMemory9withRangeEyy +__ZN14IOMemoryCursor17withSpecificationEPFvNS_15PhysicalSegmentEPvjEyyy +__ZN14IOMemoryCursor19genPhysicalSegmentsEP18IOMemoryDescriptoryPvjjPy +__ZN14IOMemoryCursor21initWithSpecificationEPFvNS_15PhysicalSegmentEPvjEyyy +__ZN14IOPMrootDomain17setSleepSupportedEj +__ZN14IOPMrootDomain19sysPowerDownHandlerEPvS0_jP9IOServiceS0_m +__ZN14IOPMrootDomain20claimSystemWakeEventEP9IOServicejPKcP8OSObject +__ZN14IOPMrootDomain24receivePowerNotificationEj +__ZN14IOPMrootDomain27displayWranglerNotificationEPvS0_jP9IOServiceS0_m +__ZN15IODMAController13getControllerEP9IOServicej +__ZN15IODMAController16notifyDMACommandEP16IODMAEventSourceP12IODMACommandiyy +__ZN15IODMAController20createControllerNameEj +__ZN15IODMAController21registerDMAControllerEj +__ZN16IODMAEventSource14dmaEventSourceEP8OSObjectP9IOServicePFvS1_PS_P12IODMACommandiyyES8_j +__ZN16IODMAEventSource15startDMACommandEP12IODMACommandjyy +__ZN16IODMAEventSource16notifyDMACommandEP12IODMACommandiyy +__ZN16IODMAEventSource4initEP8OSObjectP9IOServicePFvS1_PS_P12IODMACommandiyyES8_j +__ZN16IORangeAllocator10deallocateEyy +__ZN16IORangeAllocator12allocElementEj +__ZN16IORangeAllocator13allocateRangeEyy +__ZN16IORangeAllocator14deallocElementEj +__ZN16IORangeAllocator28setFragmentCapacityIncrementEj +__ZN16IORangeAllocator4initEyyjj +__ZN16IORangeAllocator8allocateEyPyy +__ZN16IORangeAllocator9withRangeEyyjj +__ZN17IOBigMemoryCursor13outputSegmentEN14IOMemoryCursor15PhysicalSegmentEPvj +__ZN17IOBigMemoryCursor17withSpecificationEyyy +__ZN17IOBigMemoryCursor21initWithSpecificationEyyy +__ZN17IOSharedDataQueue11withEntriesEjj +__ZN17IOSharedDataQueue12getQueueSizeEv +__ZN17IOSharedDataQueue12setQueueSizeEj +__ZN17IOSharedDataQueue12withCapacityEj +__ZN17IOSharedDataQueue16initWithCapacityEj +__ZN17IOSharedDataQueue7dequeueEPvPj +__ZN17IOSharedDataQueue7enqueueEPvj +__ZN18IOMemoryDescriptor10setMappingEP4taskyj +__ZN18IOMemoryDescriptor10writeBytesEyPKvy +__ZN18IOMemoryDescriptor11makeMappingEPS_P4taskyjyy +__ZN18IOMemoryDescriptor11withAddressEPvyj +__ZN18IOMemoryDescriptor11withOptionsEPvjjP4taskjP8IOMapper +__ZN18IOMemoryDescriptor12setPurgeableEjPj +__ZN18IOMemoryDescriptor13getPageCountsEPyS0_ +__ZN18IOMemoryDescriptor15initWithOptionsEPvjjP4taskjP8IOMapper +__ZN18IOMemoryDescriptor16performOperationEjyy +__ZN18IOMemoryDescriptor16withAddressRangeEyyjP4task +__ZN18IOMemoryDescriptor17withAddressRangesEP14IOVirtualRangejjP4task +__ZN18IOMemoryDescriptor19createMappingInTaskEP4taskyjyy +__ZN18IOMemoryDescriptor19withPhysicalAddressEyyj +__ZN18IOMemoryDescriptor3mapEj +__ZN18IOMemoryDescriptor5doMapEP7_vm_mapPyjyy +__ZN18IOMemoryDescriptor6setTagEj +__ZN18IOMemoryDescriptor7doUnmapEP7_vm_mapyy +__ZN18IOMemoryDescriptor9readBytesEyPvy +__ZN18IORegistryIterator11iterateOverEP15IORegistryEntryPK15IORegistryPlanej +__ZN18IORegistryIterator11iterateOverEPK15IORegistryPlanej +__ZN18IOTimerEventSource10setTimeoutEjj +__ZN18IOTimerEventSource10setTimeoutEjyy +__ZN18IOTimerEventSource10setTimeoutEy +__ZN18IOTimerEventSource10wakeAtTimeEjj +__ZN18IOTimerEventSource10wakeAtTimeEjyy +__ZN18IOTimerEventSource10wakeAtTimeEy +__ZN18IOTimerEventSource12setTimeoutMSEj +__ZN18IOTimerEventSource12setTimeoutUSEj +__ZN18IOTimerEventSource12wakeAtTimeMSEj +__ZN18IOTimerEventSource12wakeAtTimeUSEj +__ZN18IOTimerEventSource15setTimeoutTicksEj +__ZN18IOTimerEventSource15wakeAtTimeTicksEj +__ZN20IOLittleMemoryCursor13outputSegmentEN14IOMemoryCursor15PhysicalSegmentEPvj +__ZN20IOLittleMemoryCursor17withSpecificationEyyy +__ZN20IOLittleMemoryCursor21initWithSpecificationEyyy +__ZN20RootDomainUserClient15setPreventativeEjj +__ZN20RootDomainUserClient26getTargetAndMethodForIndexEPP9IOServicej +__ZN21IOInterruptController10initVectorEiP17IOInterruptVector +__ZN21IOInterruptController11causeVectorEiP17IOInterruptVector +__ZN21IOInterruptController12enableVectorEiP17IOInterruptVector +__ZN21IOInterruptController13getVectorTypeEiP17IOInterruptVector +__ZN21IOInterruptController17disableVectorHardEiP17IOInterruptVector +__ZN21IOInterruptController17vectorCanBeSharedEiP17IOInterruptVector +__ZN21IOInterruptController28timeStampInterruptHandlerEndEiP17IOInterruptVector +__ZN21IOInterruptController30timeStampInterruptHandlerStartEiP17IOInterruptVector +__ZN21IONaturalMemoryCursor13outputSegmentEN14IOMemoryCursor15PhysicalSegmentEPvj +__ZN21IONaturalMemoryCursor17withSpecificationEyyy +__ZN21IONaturalMemoryCursor21initWithSpecificationEyyy +__ZN21IOSubMemoryDescriptor11makeMappingEP18IOMemoryDescriptorP4taskyjyy +__ZN21IOSubMemoryDescriptor12initSubRangeEP18IOMemoryDescriptoryyj +__ZN21IOSubMemoryDescriptor12setPurgeableEjPj +__ZN21IOSubMemoryDescriptor12withSubRangeEP18IOMemoryDescriptoryyj +__ZN21IOSubMemoryDescriptor18getPhysicalSegmentEyPyj +__ZN21IOSubMemoryDescriptor7prepareEj +__ZN21IOSubMemoryDescriptor8completeEj +__ZN23IOMultiMemoryDescriptor15withDescriptorsEPP18IOMemoryDescriptorjjb +__ZN23IOMultiMemoryDescriptor19initWithDescriptorsEPP18IOMemoryDescriptorjjb +__ZN23IOMultiMemoryDescriptor7prepareEj +__ZN23IOMultiMemoryDescriptor8completeEj +__ZN24IOBufferMemoryDescriptor11appendBytesEPKvm +__ZN24IOBufferMemoryDescriptor11withOptionsEjmm +__ZN24IOBufferMemoryDescriptor12setDirectionEj +__ZN24IOBufferMemoryDescriptor12withCapacityEmjb +__ZN24IOBufferMemoryDescriptor14getBytesNoCopyEmm +__ZN24IOBufferMemoryDescriptor17inTaskWithOptionsEP4taskjmm +__ZN24IOBufferMemoryDescriptor20initWithPhysicalMaskEP4taskjyyy +__ZN24IOBufferMemoryDescriptor22inTaskWithPhysicalMaskEP4taskjyy +__ZN24IOBufferMemoryDescriptor9setLengthEm +__ZN24IOBufferMemoryDescriptor9withBytesEPKvmjb +__ZN25IOGeneralMemoryDescriptor11wireVirtualEj +__ZN25IOGeneralMemoryDescriptor12setPurgeableEjPj +__ZN25IOGeneralMemoryDescriptor15initWithOptionsEPvjjP4taskjP8IOMapper +__ZN25IOGeneralMemoryDescriptor18getPhysicalSegmentEyPyj +__ZN25IOGeneralMemoryDescriptor5doMapEP7_vm_mapPyjyy +__ZN25IOGeneralMemoryDescriptor7doUnmapEP7_vm_mapyy +__ZN25IOGeneralMemoryDescriptor7prepareEj +__ZN25IOGeneralMemoryDescriptor8completeEj +__ZN29IOInterleavedMemoryDescriptor12withCapacityEyj +__ZN29IOInterleavedMemoryDescriptor16initWithCapacityEyj +__ZN29IOInterleavedMemoryDescriptor19setMemoryDescriptorEP18IOMemoryDescriptoryy +__ZN29IOInterleavedMemoryDescriptor22clearMemoryDescriptorsEj +__ZN29IOInterleavedMemoryDescriptor7prepareEj +__ZN29IOInterleavedMemoryDescriptor8completeEj +__ZN9IOService10adjustBusyEi +__ZN9IOService10handleOpenEPS_jPv +__ZN9IOService11_adjustBusyEi +__ZN9IOService11handleCloseEPS_j +__ZN9IOService12didTerminateEPS_jPb +__ZN9IOService12requestProbeEj +__ZN9IOService12updateReportEP19IOReportChannelListjPvS2_ +__ZN9IOService13messageClientEjP8OSObjectPvm +__ZN9IOService13newUserClientEP4taskPvjP12OSDictionaryPP12IOUserClient +__ZN9IOService13newUserClientEP4taskPvjPP12IOUserClient +__ZN9IOService13startMatchingEj +__ZN9IOService13waitMatchIdleEj +__ZN9IOService13willTerminateEPS_j +__ZN9IOService14doServiceMatchEj +__ZN9IOService14messageClientsEjPvm +__ZN9IOService15addNotificationEPK8OSSymbolP12OSDictionaryPFbPvS5_PS_ES5_S5_i +__ZN9IOService15configureReportEP19IOReportChannelListjPvS2_ +__ZN9IOService15nextIdleTimeoutEyyj +__ZN9IOService15registerServiceEj +__ZN9IOService15terminateClientEPS_j +__ZN9IOService15terminatePhase1Ej +__ZN9IOService15terminateWorkerEj +__ZN9IOService16registerInterestEPK8OSSymbolPFiPvS3_jPS_S3_mES3_S3_ +__ZN9IOService16requestTerminateEPS_j +__ZN9IOService16setCPUSnoopDelayEj +__ZN9IOService18doServiceTerminateEj +__ZN9IOService18matchPropertyTableEP12OSDictionaryPi +__ZN9IOService18requireMaxBusStallEj +__ZN9IOService18systemWillShutdownEj +__ZN9IOService19deliverNotificationEPK8OSSymboljj +__ZN9IOService23acknowledgeNotificationEPvj +__ZN9IOService23addMatchingNotificationEPK8OSSymbolP12OSDictionaryPFbPvS5_PS_P10IONotifierES5_S5_i +__ZN9IOService23scheduleTerminatePhase2Ej +__ZN9IOService24mapDeviceMemoryWithIndexEjj +__ZN9IOService4openEPS_jPv +__ZN9IOService5closeEPS_j +__ZN9IOService5probeEPS_Pi +__ZN9IOService7messageEjPS_Pv +__ZN9IOService8finalizeEj +__ZN9IOService9terminateEj +__ZNK15IORegistryEntry11getPropertyEPK8OSStringPK15IORegistryPlanej +__ZNK15IORegistryEntry11getPropertyEPK8OSSymbolPK15IORegistryPlanej +__ZNK15IORegistryEntry11getPropertyEPKcPK15IORegistryPlanej +__ZNK15IORegistryEntry12copyPropertyEPK8OSStringPK15IORegistryPlanej +__ZNK15IORegistryEntry12copyPropertyEPK8OSSymbolPK15IORegistryPlanej +__ZNK15IORegistryEntry12copyPropertyEPKcPK15IORegistryPlanej +__ZNK18IOMemoryDescriptor19dmaCommandOperationEjPvj +__ZNK25IOGeneralMemoryDescriptor19dmaCommandOperationEjPvj diff --git a/config/IOKit.exports b/config/IOKit.exports index ec0b12790..be6ca8013 100644 --- a/config/IOKit.exports +++ b/config/IOKit.exports @@ -104,7 +104,6 @@ __Z17IODeviceTreeAllocPv __Z17IOServiceOrderingPK15OSMetaClassBaseS1_Pv __Z18IODTCompareNubNamePK15IORegistryEntryP8OSStringPS3_ __Z19printDictionaryKeysP12OSDictionaryPc -__Z20IODTMakeNVDescriptorP15IORegistryEntryP17IONVRAMDescriptor __Z20IODTMatchNubWithKeysP15IORegistryEntryPKc __Z21IODTResolveAddressingP15IORegistryEntryPKcP14IODeviceMemory __Z27IODTInterruptControllerNameP15IORegistryEntry @@ -694,6 +693,7 @@ __ZN18IOMemoryDescriptor18getPhysicalAddressEv __ZN18IOMemoryDescriptor30withPersistentMemoryDescriptorEPS_ __ZN18IOMemoryDescriptor4freeEv __ZN18IOMemoryDescriptor6getTagEv +__ZN18IOMemoryDescriptor8getFlagsEv __ZN18IOMemoryDescriptor8redirectEP4taskb __ZN18IOMemoryDescriptor9MetaClassC1Ev __ZN18IOMemoryDescriptor9MetaClassC2Ev @@ -725,11 +725,14 @@ __ZN18IORegistryIteratorD2Ev __ZN18IOTimerEventSource10gMetaClassE __ZN18IOTimerEventSource10superClassE __ZN18IOTimerEventSource11setWorkLoopEP10IOWorkLoop +__ZN18IOTimerEventSource12checkForWorkEv __ZN18IOTimerEventSource13cancelTimeoutEv __ZN18IOTimerEventSource14setTimeoutFuncEv __ZN18IOTimerEventSource16timerEventSourceEP8OSObjectPFvS1_PS_E +__ZN18IOTimerEventSource16timerEventSourceEjP8OSObjectPFvS1_PS_E __ZN18IOTimerEventSource4freeEv __ZN18IOTimerEventSource4initEP8OSObjectPFvS1_PS_E +__ZN18IOTimerEventSource4initEjP8OSObjectPFvS1_PS_E __ZN18IOTimerEventSource6enableEv __ZN18IOTimerEventSource7disableEv __ZN18IOTimerEventSource7timeoutEPv @@ -810,6 +813,7 @@ __ZN21IOInterruptController16getInterruptTypeEP9IOServiceiPi __ZN21IOInterruptController17registerInterruptEP9IOServiceiPvPFvS2_S2_S2_iES2_ __ZN21IOInterruptController19unregisterInterruptEP9IOServicei __ZN21IOInterruptController26getInterruptHandlerAddressEv +__ZN21IOInterruptController26timeStampSpuriousInterruptEv __ZN21IOInterruptController9MetaClassC1Ev __ZN21IOInterruptController9MetaClassC2Ev __ZN21IOInterruptController9metaClassE @@ -1024,7 +1028,6 @@ __ZN9IOService13addPowerChildEPS_ __ZN9IOService13askChangeDownEm __ZN9IOService13checkResourceEP8OSObject __ZN9IOService13getPowerStateEv -__ZN9IOService13invokeNotiferEP18_IOServiceNotifier __ZN9IOService13matchLocationEPS_ __ZN9IOService13setPowerStateEmPS_ __ZN9IOService14activityTickleEmm @@ -1439,7 +1442,6 @@ _gIODTModelKey _gIODTNWInterruptMappingKey _gIODTNameKey _gIODTPHandleKey -_gIODTPersistKey _gIODTPlane _gIODTRangeKey _gIODTSizeCellKey @@ -1485,7 +1487,7 @@ _gIOServiceKey _gIOServicePlane _gIOTerminatedNotification _gIOUserClientClassKey -_gOFVariables +_gIOWillTerminateNotification _gPlatformInterruptControllerName _registerPrioritySleepWakeInterest _registerSleepWakeInterest diff --git a/config/IOKit.x86_64.exports b/config/IOKit.x86_64.exports index 3aadfffa4..d050685e3 100644 --- a/config/IOKit.x86_64.exports +++ b/config/IOKit.x86_64.exports @@ -257,8 +257,10 @@ __ZN18IOMemoryDescriptor9readBytesEyPvy __ZN18IORegistryIterator11iterateOverEP15IORegistryEntryPK15IORegistryPlanej __ZN18IORegistryIterator11iterateOverEPK15IORegistryPlanej __ZN18IOTimerEventSource10setTimeoutEjj +__ZN18IOTimerEventSource10setTimeoutEjyy __ZN18IOTimerEventSource10setTimeoutEy __ZN18IOTimerEventSource10wakeAtTimeEjj +__ZN18IOTimerEventSource10wakeAtTimeEjyy __ZN18IOTimerEventSource10wakeAtTimeEy __ZN18IOTimerEventSource12setTimeoutMSEj __ZN18IOTimerEventSource12setTimeoutUSEj @@ -266,9 +268,6 @@ __ZN18IOTimerEventSource12wakeAtTimeMSEj __ZN18IOTimerEventSource12wakeAtTimeUSEj __ZN18IOTimerEventSource15setTimeoutTicksEj __ZN18IOTimerEventSource15wakeAtTimeTicksEj -__ZN18IOTimerEventSource28_RESERVEDIOTimerEventSource0Ev -__ZN18IOTimerEventSource28_RESERVEDIOTimerEventSource1Ev -__ZN18IOTimerEventSource28_RESERVEDIOTimerEventSource2Ev __ZN18IOTimerEventSource28_RESERVEDIOTimerEventSource3Ev __ZN18IOTimerEventSource28_RESERVEDIOTimerEventSource4Ev __ZN18IOTimerEventSource28_RESERVEDIOTimerEventSource5Ev @@ -291,6 +290,8 @@ __ZN21IOInterruptController31_RESERVEDIOInterruptController2Ev __ZN21IOInterruptController31_RESERVEDIOInterruptController3Ev __ZN21IOInterruptController31_RESERVEDIOInterruptController4Ev __ZN21IOInterruptController31_RESERVEDIOInterruptController5Ev +__ZN21IOInterruptController28timeStampInterruptHandlerEndEiP17IOInterruptVector +__ZN21IOInterruptController30timeStampInterruptHandlerStartEiP17IOInterruptVector __ZN21IONaturalMemoryCursor13outputSegmentEN14IOMemoryCursor15PhysicalSegmentEPvj __ZN21IONaturalMemoryCursor17withSpecificationEyyy __ZN21IONaturalMemoryCursor21initWithSpecificationEyyy diff --git a/config/Libkern.arm.exports b/config/Libkern.arm.exports new file mode 100644 index 000000000..051d0d07b --- /dev/null +++ b/config/Libkern.arm.exports @@ -0,0 +1,4 @@ +_OSAddAtomic64 +_OSCompareAndSwap64 +__ZN12OSOrderedSet12withCapacityEjPFlPK15OSMetaClassBaseS2_PvES3_ +__ZN12OSOrderedSet16initWithCapacityEjPFlPK15OSMetaClassBaseS2_PvES3_ diff --git a/config/Libkern.arm64.exports b/config/Libkern.arm64.exports new file mode 100644 index 000000000..cc07f5dc2 --- /dev/null +++ b/config/Libkern.arm64.exports @@ -0,0 +1,5 @@ +_OSAddAtomic64 +_OSCompareAndSwap64 +_PAGE_SHIFT_CONST +__ZN12OSOrderedSet12withCapacityEjPFiPK15OSMetaClassBaseS2_PvES3_ +__ZN12OSOrderedSet16initWithCapacityEjPFiPK15OSMetaClassBaseS2_PvES3_ diff --git a/config/Libkern.exports b/config/Libkern.exports index 93dfdd556..6769ed75b 100644 --- a/config/Libkern.exports +++ b/config/Libkern.exports @@ -591,8 +591,17 @@ __Znwm ___bzero ___cxa_pure_virtual ___llvm_profile_runtime +___memcpy_chk +___memmove_chk +___memset_chk ___stack_chk_fail ___stack_chk_guard +___strlcpy_chk +___strlcat_chk +___strncpy_chk +___strncat_chk +___strcpy_chk +___strcat_chk __os_log_default __os_log_internal _adler32 @@ -684,6 +693,7 @@ _memcmp _memcpy _memmove _memset +_memset_s _ml_at_interrupt_context _ml_get_interrupts_enabled _ml_set_interrupts_enabled diff --git a/config/MACFramework.arm.exports b/config/MACFramework.arm.exports new file mode 100644 index 000000000..e69de29bb diff --git a/config/MACFramework.arm64.exports b/config/MACFramework.arm64.exports new file mode 100644 index 000000000..e69de29bb diff --git a/config/MASTER b/config/MASTER index cdacdd750..2fa158923 100644 --- a/config/MASTER +++ b/config/MASTER @@ -72,20 +72,13 @@ options NO_DIRECT_RPC # for untyped mig servers # options LOOP # loopback support # <loop> options VLAN # # <vlan> options BOND # # <bond> +options IF_FAKE # # <if_fake> options AH_ALL_CRYPTO # AH all crypto algs # <ah_all_crypto> options IPCOMP_ZLIB # IP compression using zlib # <ipcomp_zlib> options PF # Packet Filter # <pf> -options PF_ALTQ # PF ALTQ (Alternate Queueing) # <pf_altq> options PF_ECN # PF use ECN marking # <pf_ecn> options PFLOG # PF log interface # <pflog> -options PKTSCHED_CBQ # CBQ packet scheduler # <pktsched_cbq> -options PKTSCHED_HFSC # H-FSC packet scheduler # <pktsched_hfsc> -options PKTSCHED_PRIQ # PRIQ packet scheduler # <pktsched_priq> -options PKTSCHED_FAIRQ # FAIRQ packet scheduler # <pktsched_fairq> options MEASURE_BW # interface bandwidth measurement # <measure_bw> -options CLASSQ_BLUE # BLUE queueing algorithm # <classq_blue> -options CLASSQ_RED # RED queueing algorithm # <classq_red> -options CLASSQ_RIO # RIO queueing algorithm # <classq_rio> options DUMMYNET # dummynet support # <dummynet> options TRAFFIC_MGT # traffic management support # <traffic_mgt> options MULTICAST # Internet Protocol Class-D $ @@ -193,7 +186,6 @@ options CONFIG_KN_HASHSIZE=20 # <bsmall> # # configurable vfs related resources # CONFIG_VNODES - used to pre allocate vnode related resources -# CONFIG_VNODE_FREE_MIN - mininmum number of free vnodes # CONFIG_NC_HASH - name cache hash table allocation # CONFIG_VFS_NAMES - name strings # @@ -208,12 +200,6 @@ options CONFIG_VNODES=263168 # <medium> options CONFIG_VNODES=10240 # <small> options CONFIG_VNODES=750 # <bsmall> -options CONFIG_VNODE_FREE_MIN=500 # <large,xlarge> -options CONFIG_VNODE_FREE_MIN=300 # <medium> -options CONFIG_VNODE_FREE_MIN=200 # <small> -options CONFIG_VNODE_FREE_MIN=100 # <xsmall> -options CONFIG_VNODE_FREE_MIN=75 # <bsmall> - options CONFIG_NC_HASH=5120 # <large,xlarge> options CONFIG_NC_HASH=4096 # <medium> options CONFIG_NC_HASH=2048 # <small,xsmall> @@ -295,8 +281,12 @@ options CONFIG_MFCTBLSIZ=16 # <bsmall> # # configurable kernel message buffer size # -options CONFIG_MSG_BSIZE=4096 # <bsmall,small,xsmall> -options CONFIG_MSG_BSIZE=16384 # <medium,large,xlarge> +options CONFIG_MSG_BSIZE_REL=4096 # <bsmall,small,xsmall> +options CONFIG_MSG_BSIZE_DEV=4096 # <bsmall,small,xsmall> +options CONFIG_MSG_BSIZE_REL=16384 # <medium,large,xlarge> +options CONFIG_MSG_BSIZE_DEV=131072 # <medium,large,xlarge> +options CONFIG_MSG_BSIZE=CONFIG_MSG_BSIZE_REL # <!development,debug> +options CONFIG_MSG_BSIZE=CONFIG_MSG_BSIZE_DEV # <development,debug> # # maximum size of the per-process Mach IPC table @@ -304,14 +294,11 @@ options CONFIG_MSG_BSIZE=16384 # <medium,large,xlarge> options CONFIG_IPC_TABLE_ENTRIES_STEPS=64 # 137898 entries # <bsmall,small,xsmall> options CONFIG_IPC_TABLE_ENTRIES_STEPS=256 # 300714 entries # <medium,large,xlarge> - # # configurable kernel - use these options to strip strings from panic # and printf calls. -# no_panic_str - saves around 50K of kernel footprint. # no_printf_str - saves around 45K of kernel footprint. # -options CONFIG_NO_PANIC_STRINGS # <no_panic_str> options CONFIG_NO_PRINTF_STRINGS # <no_printf_str> options CONFIG_NO_KPRINTF_STRINGS # <no_kprintf_str> @@ -399,7 +386,8 @@ options CONFIG_IO_ACCOUNTING # <config_io_accounting> # For now debug is enabled wherever inheritance is # options IMPORTANCE_INHERITANCE # <importance_inheritance> -options IMPORTANCE_DEBUG # <importance_inheritance> +options IMPORTANCE_TRACE # <importance_trace> +options IMPORTANCE_DEBUG # <importance_debug> options CONFIG_TELEMETRY # <config_telemetry> @@ -415,6 +403,11 @@ options CONFIG_ECC_LOGGING # <config_ecc_logging> # options CONFIG_COREDUMP # <config_coredump> +# +# Vnode guards +# +options CONFIG_VNGUARD # <config_vnguard> + # # Ethernet (ARP) # @@ -465,6 +458,7 @@ pseudo-device systrace 1 init systrace_init # <config_dtrace> pseudo-device fbt 1 init fbt_init # <config_dtrace> pseudo-device profile_prvd 1 init profile_init # <config_dtrace> + # # IOKit configuration options # @@ -525,7 +519,6 @@ options CONFIG_AUDIT # Kernel auditing # <config_audit> # forcibly suspending tasks when the demand exceeds supply. This # option should be on. # -options MACH_RT options TASK_SWAPPER # <task_swapper_disabled> # @@ -607,6 +600,7 @@ options CONFIG_DTRACE # # <config_dtrace> options KPERF # <kperf> options KPC # <kpc> + options PGO # <pgo> # MACH_COUNTERS enables code that handles various counters in the system. @@ -716,12 +710,6 @@ options OS_REASON_DEBUG # <os_reason_debug> # options CONFIG_ATM # <config_atm> -# -# Kernel Voucher Attr Manager for BANK -# -options CONFIG_BANK # <config_bank> - - # Group related tasks together into coalitions options CONFIG_COALITIONS # <config_coalitions> @@ -741,3 +729,9 @@ options VIDEO_CONSOLE # uni-directional output over framebuffer # Syscall options # options CONFIG_REQUIRES_U32_MUNGING # incoming U32 argument structures must be munged to match U64 # <config_requires_u32_munging> + +# +# copyout() instrumentation +# +options COPYOUT_SHIM # Shim for copyout memory analysis via kext #<copyout_shim> + diff --git a/config/MASTER.arm b/config/MASTER.arm new file mode 100644 index 000000000..b534c3569 --- /dev/null +++ b/config/MASTER.arm @@ -0,0 +1,86 @@ +# +# Mach Operating System +# Copyright (c) 1986 Carnegie-Mellon University +# Copyright 2001-2016 Apple Inc. +# +# All rights reserved. The CMU software License Agreement +# specifies the terms and conditions for use and redistribution. +# +###################################################################### +# +# Master Apple configuration file (see the master machine independent +# configuration file for a description of the file format). +# +###################################################################### +# +# Standard Apple OS Configurations: +# -------- ----- -- --------------- +# +# KERNEL_BASE = [ arm xsmall config_embedded ] +# KERNEL_RELEASE = [ KERNEL_BASE ] +# KERNEL_DEV = [ KERNEL_BASE development mach_assert config_xnupost proc_ref_debug os_reason_debug ] +# KERNEL_DEBUG = [ KERNEL_BASE debug mach_assert config_xnupost config_ltable_stats config_ltable_debug config_waitq_stats config_waitq_debug ] +# BSD_BASE = [ mach_bsd config_workqueue psynch config_proc_uuid_policy ] +# BSD_RELEASE = [ BSD_BASE no_printf_str no_kprintf_str secure_kernel ] +# BSD_DEV = [ BSD_BASE config_imageboot config_coredump pgo config_vnguard ] +# BSD_DEBUG = [ BSD_BASE config_imageboot config_coredump pgo config_vnguard ] +# FILESYS_BASE = [ devfs fifo fs_compression config_protect config_fse routefs quota namedstreams ] +# FILESYS_RELEASE= [ FILESYS_BASE ] +# FILESYS_DEV = [ FILESYS_BASE fdesc ] +# FILESYS_DEBUG = [ FILESYS_BASE fdesc ] +# NFS = [ nfsclient nfsserver ] +# SKYWALK_BASE = [ skywalk config_nexus_user_pipe config_nexus_kernel_pipe config_nexus_monitor config_nexus_flowswitch config_nexus_netif ] +# SKYWALK_RELEASE = [ SKYWALK_BASE ] +# SKYWALK_DEV = [ SKYWALK_BASE ] +# SKYWALK_DEBUG = [ SKYWALK_BASE ] +# NETWORKING = [ inet tcpdrop_synfin bpfilter inet6 ipv6send if_bridge traffic_mgt dummynet ah_all_crypto if_fake ] +# VPN = [ ipsec flow_divert necp content_filter ] +# PF = [ pf ] +# MULTIPATH = [ multipath mptcp ] +# IOKIT_BASE = [ iokit iokitcpp no_kextd no_kernel_hid config_sleep ] +# IOKIT_RELEASE = [ IOKIT_BASE ] +# IOKIT_DEV = [ IOKIT_BASE iokitstats iotracking ] +# IOKIT_DEBUG = [ IOKIT_BASE iokitstats iotracking ] +# LIBKERN_BASE = [ libkerncpp config_kec_fips zlib crypto_sha2 ] +# LIBKERN_RELEASE =[ LIBKERN_BASE ] +# LIBKERN_DEV = [ LIBKERN_BASE iotracking ] +# LIBKERN_DEBUG = [ LIBKERN_BASE iotracking ] +# PERF_DBG_BASE = [ mach_kdp config_serial_kdp kperf kpc MONOTONIC_BASE ] +# PERF_DBG_RELEASE=[ PERF_DBG_BASE ist_kdebug ] +# PERF_DBG_DEV = [ PERF_DBG_BASE config_dtrace zleaks kdp_interactive_debugging interrupt_masked_debug ] +# PERF_DBG_DEBUG = [ PERF_DBG_BASE config_dtrace zleaks kdp_interactive_debugging interrupt_masked_debug ] +# MACH_BASE = [ mach slidable vc_progress_white mdebug ipc_debug importance_inheritance config_atm config_coalitions config_library_validation config_iosched config_telemetry config_sysdiagnose ] +# MACH_RELEASE = [ MACH_BASE config_skip_precise_user_kernel_time debugger_for_zone_info ] +# MACH_DEV = [ MACH_BASE task_zone_info config_io_accounting importance_trace ] +# MACH_DEBUG = [ MACH_BASE task_zone_info config_io_accounting importance_trace importance_debug ] +# SCHED_BASE = [ config_sched_traditional config_sched_multiq ] +# SCHED_RELEASE = [ SCHED_BASE ] +# SCHED_DEV = [ SCHED_BASE ] +# SCHED_DEBUG = [ SCHED_BASE config_sched_grrr config_sched_proto ] +# VM_BASE = [ vm_pressure_events jetsam memorystatus config_code_decryption config_cs_validation_bitmap ] +# VM_RELEASE = [ VM_BASE ] +# VM_DEV = [ VM_BASE dynamic_codesigning ] +# VM_DEBUG = [ VM_BASE dynamic_codesigning ] +# SECURITY = [ config_macf ] +# RELEASE = [ KERNEL_RELEASE BSD_RELEASE FILESYS_RELEASE SKYWALK_RELEASE NETWORKING PF MULTIPATH VPN IOKIT_RELEASE LIBKERN_RELEASE PERF_DBG_RELEASE MACH_RELEASE SCHED_RELEASE VM_RELEASE SECURITY ] +# DEVELOPMENT = [ KERNEL_DEV BSD_DEV FILESYS_DEV NFS SKYWALK_DEV NETWORKING PF MULTIPATH VPN IOKIT_DEV LIBKERN_DEV PERF_DBG_DEV MACH_DEV SCHED_DEV VM_DEV SECURITY ] +# DEBUG = [ KERNEL_DEBUG BSD_DEBUG FILESYS_DEBUG SKYWALK_DEBUG NETWORKING PF MULTIPATH VPN IOKIT_DEBUG LIBKERN_DEBUG PERF_DBG_DEBUG MACH_DEBUG SCHED_DEBUG VM_DEBUG SECURITY ] +# +###################################################################### +# +machine "arm" # <arm> + +makeoptions OSFMK_MACHINE = "arm" # <mach> + +options COUNT_SYSCALLS # count bsd system calls # <countcalls> + +options SLIDABLE=1 # Use PIE-assembly in *.s # <slidable> +options TRASH_VFP_ON_SAVE # <debug,trash_vfp> + +options CONFIG_VNODES=1024 # <xsmall> + +options CONFIG_FREEZE_SUSPENDED_MIN=4 # <xsmall> + +options CONFIG_MACH_APPROXIMATE_TIME + +options INTERRUPT_MASKED_DEBUG=1 # # <interrupt_masked_debug> diff --git a/config/MASTER.arm64 b/config/MASTER.arm64 new file mode 100644 index 000000000..ae4eb4903 --- /dev/null +++ b/config/MASTER.arm64 @@ -0,0 +1,92 @@ +# +# Mach Operating System +# Copyright (c) 1986 Carnegie-Mellon University +# Copyright 2001-2016 Apple Inc. +# +# All rights reserved. The CMU software License Agreement +# specifies the terms and conditions for use and redistribution. +# +###################################################################### +# +# Master Apple configuration file (see the master machine independent +# configuration file for a description of the file format). +# +###################################################################### +# +# Standard Apple OS Configurations: +# -------- ----- -- --------------- +# +# KERNEL_BASE = [ arm64 xsmall config_embedded config_requires_u32_munging ] +# KERNEL_RELEASE = [ KERNEL_BASE ] +# KERNEL_DEV = [ KERNEL_BASE development mach_assert config_xnupost proc_ref_debug os_reason_debug pgtrace ] +# KERNEL_DEBUG = [ KERNEL_BASE debug mach_assert config_xnupost config_ltable_stats config_ltable_debug config_waitq_stats config_waitq_debug pgtrace ] +# BSD_BASE = [ mach_bsd config_workqueue psynch config_proc_uuid_policy config_personas ] +# BSD_RELEASE = [ BSD_BASE no_printf_str no_kprintf_str secure_kernel ] +# BSD_DEV = [ BSD_BASE config_imageboot config_coredump pgo config_vnguard ] +# BSD_DEBUG = [ BSD_BASE config_imageboot config_coredump pgo config_vnguard ] +# FILESYS_BASE = [ devfs fifo fs_compression config_protect config_fse routefs quota namedstreams ] +# FILESYS_RELEASE= [ FILESYS_BASE ] +# FILESYS_DEV = [ FILESYS_BASE fdesc ] +# FILESYS_DEBUG = [ FILESYS_BASE fdesc ] +# NFS = [ nfsclient nfsserver ] +# SKYWALK_BASE = [ skywalk config_nexus_user_pipe config_nexus_kernel_pipe config_nexus_monitor config_nexus_flowswitch config_nexus_netif ] +# SKYWALK_RELEASE = [ SKYWALK_BASE ] +# SKYWALK_DEV = [ SKYWALK_BASE ] +# SKYWALK_DEBUG = [ SKYWALK_BASE ] +# NETWORKING = [ inet tcpdrop_synfin bpfilter inet6 ipv6send if_bridge traffic_mgt dummynet ah_all_crypto packet_mangler if_fake ] +# VPN = [ ipsec flow_divert necp content_filter ] +# PF = [ pf ] +# MULTIPATH = [ multipath mptcp ] +# IOKIT_BASE = [ iokit iokitcpp no_kextd no_kernel_hid config_sleep ] +# IOKIT_RELEASE = [ IOKIT_BASE ] +# IOKIT_DEV = [ IOKIT_BASE iokitstats iotracking ] +# IOKIT_DEBUG = [ IOKIT_BASE iokitstats iotracking] +# LIBKERN_BASE = [ libkerncpp config_kec_fips zlib crypto_sha2 ] +# LIBKERN_RELEASE =[ LIBKERN_BASE ] +# LIBKERN_DEV = [ LIBKERN_BASE iotracking ] +# LIBKERN_DEBUG = [ LIBKERN_BASE iotracking ] +# PERF_DBG_BASE = [ mach_kdp config_serial_kdp MONOTONIC_BASE kperf kpc ] +# PERF_DBG_RELEASE=[ PERF_DBG_BASE ist_kdebug ] +# PERF_DBG_DEV = [ PERF_DBG_BASE config_dtrace zleaks kdp_interactive_debugging alternate_debugger interrupt_masked_debug ] +# PERF_DBG_DEBUG = [ PERF_DBG_BASE config_dtrace zleaks kdp_interactive_debugging alternate_debugger interrupt_masked_debug ] +# MACH_BASE = [ mach slidable config_ecc_logging vc_progress_white mdebug ipc_debug importance_inheritance config_atm config_coalitions config_iosched config_library_validation config_sysdiagnose config_telemetry config_mach_bridge_recv_time] +# MACH_RELEASE = [ MACH_BASE config_skip_precise_user_kernel_time debugger_for_zone_info ] +# MACH_DEV = [ MACH_BASE task_zone_info config_io_accounting importance_trace ] +# MACH_DEBUG = [ MACH_BASE task_zone_info config_io_accounting importance_trace importance_debug ] +# SCHED_BASE = [ config_sched_traditional config_sched_multiq config_sched_deferred_ast ] +# SCHED_RELEASE = [ SCHED_BASE ] +# SCHED_DEV = [ SCHED_BASE ] +# SCHED_DEBUG = [ SCHED_BASE config_sched_grrr config_sched_proto ] +# VM_BASE = [ vm_pressure_events jetsam freeze memorystatus config_code_decryption phantom_cache config_secluded_memory config_background_queue config_cs_validation_bitmap] +# VM_RELEASE = [ VM_BASE ] +# VM_DEV = [ VM_BASE dynamic_codesigning ] +# VM_DEBUG = [ VM_BASE dynamic_codesigning ] +# SECURITY = [ config_macf kernel_integrity ] +# RELEASE = [ KERNEL_RELEASE BSD_RELEASE FILESYS_RELEASE SKYWALK_RELEASE NETWORKING PF MULTIPATH VPN IOKIT_RELEASE LIBKERN_RELEASE PERF_DBG_RELEASE MACH_RELEASE SCHED_RELEASE VM_RELEASE SECURITY ] +# DEVELOPMENT = [ KERNEL_DEV BSD_DEV FILESYS_DEV NFS SKYWALK_DEV NETWORKING PF MULTIPATH VPN IOKIT_DEV LIBKERN_DEV PERF_DBG_DEV MACH_DEV SCHED_DEV VM_DEV SECURITY ] +# DEBUG = [ KERNEL_DEBUG BSD_DEBUG FILESYS_DEBUG SKYWALK_DEBUG NETWORKING PF MULTIPATH VPN IOKIT_DEBUG LIBKERN_DEBUG PERF_DBG_DEBUG MACH_DEBUG SCHED_DEBUG VM_DEBUG SECURITY ] +# KASAN = [ DEVELOPMENT ] +# +###################################################################### +# +machine "arm64" # <arm64> + +makeoptions OSFMK_MACHINE = "arm64" # <mach> + +options COUNT_SYSCALLS # count bsd system calls # <countcalls> +options TRASH_VFP_ON_SAVE # <debug,trash_vfp> +options ALTERNATE_DEBUGGER # <alternate_debugger> + +options CONFIG_VNODES=1024 # <xsmall> + +options CONFIG_FREEZE_SUSPENDED_MIN=4 # <xsmall> + +options CONFIG_MACH_APPROXIMATE_TIME + +options CONFIG_KERNEL_INTEGRITY # <kernel_integrity> + +options INTERRUPT_MASKED_DEBUG=1 # # <interrupt_masked_debug> + +options CONFIG_PGTRACE # <pgtrace> +options CONFIG_PGTRACE_NONKEXT # <pgtrace_nonkext> +pseudo-device pgtrace 1 init pgtrace_dev_init # <pgtrace_nonkext> diff --git a/config/MASTER.x86_64 b/config/MASTER.x86_64 index 441be7312..0c8b7a4e5 100644 --- a/config/MASTER.x86_64 +++ b/config/MASTER.x86_64 @@ -1,7 +1,7 @@ # # Mach Operating System # Copyright (c) 1986 Carnegie-Mellon University -# Copyright 2001-2013 Apple Inc. +# Copyright 2001-2016 Apple Inc. # # All rights reserved. The CMU software License Agreement # specifies the terms and conditions for use and redistribution. @@ -20,17 +20,18 @@ # KERNEL_RELEASE = [ KERNEL_BASE ] # KERNEL_DEV = [ KERNEL_BASE development mach_assert config_xnupost proc_ref_debug os_reason_debug ] # KERNEL_DEBUG = [ KERNEL_BASE debug mach_assert config_xnupost config_ltable_stats config_ltable_debug config_waitq_stats config_waitq_debug ] -# BSD = [ mach_bsd sysv_sem sysv_msg sysv_shm config_imageboot config_workqueue psynch config_proc_uuid_policy config_coredump pgo ] +# BSD_BASE = [ mach_bsd sysv_sem sysv_msg sysv_shm config_imageboot config_workqueue psynch config_proc_uuid_policy config_coredump pgo ] +# BSD_RELEASE = [ BSD_BASE ] +# BSD_DEV = [ BSD_BASE config_vnguard ] +# BSD_DEBUG = [ BSD_BASE config_vnguard ] # FILESYS_BASE = [ devfs fdesc config_dev_kmem config_fse quota namedstreams config_protect fifo config_volfs fs_compression config_imgsrc_access config_triggers config_ext_resolver config_searchfs config_appledouble nullfs config_mnt_suid ] # FILESYS_RELEASE= [ FILESYS_BASE ] # FILESYS_DEV = [ FILESYS_BASE ] # FILESYS_DEBUG = [ FILESYS_BASE ] # NFS = [ nfsclient nfsserver ] -# NETWORKING = [ inet inet6 ipv6send tcpdrop_synfin bpfilter dummynet traffic_mgt sendfile ah_all_crypto bond vlan gif stf ifnet_input_chk config_mbuf_jumbo if_bridge ipcomp_zlib MULTIPATH packet_mangler ] +# NETWORKING = [ inet inet6 ipv6send tcpdrop_synfin bpfilter dummynet traffic_mgt sendfile ah_all_crypto bond vlan gif stf ifnet_input_chk config_mbuf_jumbo if_bridge ipcomp_zlib MULTIPATH packet_mangler if_fake ] # VPN = [ ipsec flow_divert necp content_filter ] # PF = [ pf pflog ] -# PKTSCHED = [ pktsched_cbq pktsched_fairq pktsched_hfsc pktsched_priq ] -# CLASSQ = [ classq_blue classq_red classq_rio ] # MULTIPATH = [ multipath mptcp ] # IOKIT_BASE = [ iokit iokitcpp hibernation config_sleep iokitstats hypervisor ] # IOKIT_RELEASE = [ IOKIT_BASE ] @@ -40,20 +41,21 @@ # LIBKERN_RELEASE =[ LIBKERN_BASE ] # LIBKERN_DEV = [ LIBKERN_BASE iotracking ] # LIBKERN_DEBUG = [ LIBKERN_BASE iotracking ] -# PERF_DBG = [ config_dtrace mach_kdp config_serial_kdp kdp_interactive_debugging kperf kpc zleaks config_gzalloc ] -# MACH_BASE = [ mach config_kext_basement mdebug ipc_debug config_mca config_vmx config_mtrr config_lapic config_telemetry importance_inheritance config_atm config_bank config_coalitions hypervisor config_iosched config_sysdiagnose ] +# PERF_DBG = [ config_dtrace mach_kdp config_serial_kdp kdp_interactive_debugging kperf kpc zleaks config_gzalloc MONOTONIC_BASE ] +# MACH_BASE = [ mach config_kext_basement mdebug ipc_debug config_mca config_vmx config_mtrr config_lapic config_telemetry importance_inheritance config_atm config_coalitions hypervisor config_iosched config_sysdiagnose config_mach_bridge_send_time copyout_shim ] # MACH_RELEASE = [ MACH_BASE ] -# MACH_DEV = [ MACH_BASE task_zone_info ] -# MACH_DEBUG = [ MACH_BASE task_zone_info ] +# MACH_DEV = [ MACH_BASE task_zone_info importance_trace ] +# MACH_DEBUG = [ MACH_BASE task_zone_info importance_trace importance_debug ] # SCHED_BASE = [ config_sched_traditional config_sched_multiq config_sched_sfi ] # SCHED_RELEASE = [ SCHED_BASE ] # SCHED_DEV = [ SCHED_BASE ] # SCHED_DEBUG = [ SCHED_BASE config_sched_grrr config_sched_proto ] # VM = [ vm_pressure_events memorystatus dynamic_codesigning config_code_decryption encrypted_swap phantom_cache config_background_queue] # SECURITY = [ config_macf config_audit config_csr ] -# RELEASE = [ KERNEL_RELEASE BSD FILESYS_RELEASE NFS SKYWALK_RELEASE NETWORKING PF VPN IOKIT_RELEASE LIBKERN_RELEASE PERF_DBG MACH_RELEASE SCHED_RELEASE VM SECURITY ] -# DEVELOPMENT = [ KERNEL_DEV BSD FILESYS_DEV NFS SKYWALK_DEV NETWORKING PF VPN IOKIT_DEV LIBKERN_DEV PERF_DBG MACH_DEV SCHED_DEV VM SECURITY ] -# DEBUG = [ KERNEL_DEBUG BSD FILESYS_DEBUG NFS SKYWALK_DEBUG NETWORKING PF VPN IOKIT_DEBUG LIBKERN_DEBUG PERF_DBG MACH_DEBUG SCHED_DEBUG VM SECURITY ] +# RELEASE = [ KERNEL_RELEASE BSD_RELEASE FILESYS_RELEASE NFS SKYWALK_RELEASE NETWORKING PF VPN IOKIT_RELEASE LIBKERN_RELEASE PERF_DBG MACH_RELEASE SCHED_RELEASE VM SECURITY ] +# DEVELOPMENT = [ KERNEL_DEV BSD_DEV FILESYS_DEV NFS SKYWALK_DEV NETWORKING PF VPN IOKIT_DEV LIBKERN_DEV PERF_DBG MACH_DEV SCHED_DEV VM SECURITY ] +# DEBUG = [ KERNEL_DEBUG BSD_DEBUG FILESYS_DEBUG NFS SKYWALK_DEBUG NETWORKING PF VPN IOKIT_DEBUG LIBKERN_DEBUG PERF_DBG MACH_DEBUG SCHED_DEBUG VM SECURITY ] +# KASAN = [ DEVELOPMENT ] # ###################################################################### # diff --git a/config/Mach.arm.exports b/config/Mach.arm.exports new file mode 100644 index 000000000..f5f0e735d --- /dev/null +++ b/config/Mach.arm.exports @@ -0,0 +1,2 @@ +_mach_msg_send_from_kernel +_semaphore_timedwait diff --git a/config/Mach.arm64.exports b/config/Mach.arm64.exports new file mode 100644 index 000000000..cc31a814e --- /dev/null +++ b/config/Mach.arm64.exports @@ -0,0 +1 @@ +_semaphore_timedwait diff --git a/config/Mach.exports b/config/Mach.exports index 09ca16fb4..a33253ff5 100644 --- a/config/Mach.exports +++ b/config/Mach.exports @@ -1,3 +1,4 @@ +_absolutetime_to_continuoustime _absolutetime_to_nanoseconds _assert_wait _assert_wait_deadline @@ -12,6 +13,7 @@ _clock_get_uptime _clock_interval_to_absolutetime_interval _clock_interval_to_deadline _clock_timebase_info +_continuoustime_to_absolutetime _current_task _current_thread _kernel_task @@ -45,6 +47,7 @@ _task_reference _thread_block _thread_block_parameter _thread_call_allocate +_thread_call_allocate_with_options _thread_call_cancel _thread_call_enter _thread_call_enter1 @@ -57,5 +60,7 @@ _thread_reference _thread_terminate _thread_tid _thread_wakeup_prim +_vm_kernel_addrhash:_vm_kernel_addrhash_external +_vm_kernel_addrhide _vm_kernel_addrperm_external _vm_kernel_unslide_or_perm_external diff --git a/config/Makefile b/config/Makefile index d88a78568..36b16f259 100644 --- a/config/Makefile +++ b/config/Makefile @@ -13,6 +13,7 @@ INSTALL_KEXT_DIR = $(DSTROOT)$(INSTALL_EXTENSIONS_DIR) KEXT_PLIST_LIST = \ System.kext/Info.plist \ + System.kext/PlugIns/Kasan.kext/Info.plist \ System.kext/PlugIns/AppleNMI.kext/Info.plist \ System.kext/PlugIns/ApplePlatformFamily.kext/Info.plist \ System.kext/PlugIns/IONVRAMFamily.kext/Info.plist \ @@ -128,12 +129,15 @@ $(DSTROOT)/$(KRESDIR)/$(MD_SUPPORTED_KPI_FILENAME) $(DSTROOT)/$(KRESDIR)/$(MI_SU @echo "$(ColorH)INSTALL$(Color0) $(ColorF)$*$(Color0)" $(_v)$(INSTALL) $(INSTALL_FLAGS) $< $@ +ifneq ($(INSTALL_KASAN_ONLY),1) do_config_install:: $(SYMROOT_INSTALL_KEXT_MACHO_FILES) \ $(SYMROOT_INSTALL_KEXT_PLISTS) \ $(DSTROOT_INSTALL_KEXT_MACHO_FILES) \ $(DSTROOT_INSTALL_KEXT_PLISTS) \ $(DSTROOT)/$(KRESDIR)/$(MD_SUPPORTED_KPI_FILENAME) \ $(DSTROOT)/$(KRESDIR)/$(MI_SUPPORTED_KPI_FILENAME) +endif + $(OBJPATH)/all-kpi.exp: $(EXPORTS_FILES) $(_v)$(SOURCE)/generate_linker_exports.sh $@ $+ diff --git a/config/MasterVersion b/config/MasterVersion index a02b1d1cb..d697dff22 100644 --- a/config/MasterVersion +++ b/config/MasterVersion @@ -1,4 +1,4 @@ -16.7.0 +17.0.0 # The first line of this file contains the master version number for the kernel. # All other instances of the kernel version in xnu are derived from this file. diff --git a/config/Private.arm.exports b/config/Private.arm.exports new file mode 100644 index 000000000..0b393134f --- /dev/null +++ b/config/Private.arm.exports @@ -0,0 +1,21 @@ +__ZN17IONVRAMController* +__ZTV17IONVRAMController +_IOCPURunPlatformActiveActions +_IOCPURunPlatformQuiesceActions +_PE_get_default +_PE_reboot_on_panic +_PE_mark_hwaccess +_ml_arm_sleep +_ml_get_abstime_offset +_ml_get_conttime_offset +_ml_get_wake_timebase +_proc_getcdhash +_cpu_broadcast_xcall +_cpu_xcall +_cpu_number +_enable_kernel_vfp_context +_PE_consistent_debug_register +_ml_static_ptovirt +_ml_static_mfree +_sched_perfcontrol_register_callbacks +_sched_perfcontrol_update_recommended_cores diff --git a/config/Private.arm64.exports b/config/Private.arm64.exports new file mode 100644 index 000000000..ab9007317 --- /dev/null +++ b/config/Private.arm64.exports @@ -0,0 +1,34 @@ +_IOCPURunPlatformActiveActions +_IOCPURunPlatformQuiesceActions +_PE_consistent_debug_register +_PE_get_default +_PE_reboot_on_panic +_PE_mark_hwaccess +__ZN17IONVRAMController* +__ZTV17IONVRAMController +_cpu_broadcast_xcall +_cpu_xcall +_cpu_cluster_id +_cpu_number +_cpu_qos_update_register +_ecc_log_record_event +_ml_arm_sleep +_ml_get_abstime_offset +_ml_get_conttime_offset +_ml_get_wake_timebase +_ml_thread_is64bit +_pe_shmcon_set_child +_proc_getcdhash +_sched_perfcontrol_register_callbacks +_sched_perfcontrol_update_recommended_cores +_sched_perfcontrol_thread_group_recommend +_sched_perfcontrol_update_callback_deadline +_ml_static_ptovirt +_ml_static_mfree +_ex_cb_register +_pgtrace_init +_pgtrace_start +_pgtrace_stop +_pgtrace_active +_pgtrace_add_probe +_pgtrace_clear_probe diff --git a/config/Private.exports b/config/Private.exports index 7d563fe0c..5ce2ff9c6 100644 --- a/config/Private.exports +++ b/config/Private.exports @@ -52,6 +52,8 @@ _cdevsw_setkqueueok _chudxnu_platform_ptr _clalloc _clfree +_cluster_unlock_direct_read +_cluster_lock_direct_read _cons_cinput _convert_port_to_task_suspension_token _convert_task_suspension_token_to_port @@ -67,15 +69,15 @@ _cpx_flush _cpx_free _cpx_has_key _cpx_init -_cpx_is_sep_wrapped_key _cpx_is_composite_key +_cpx_is_sep_wrapped_key _cpx_iv_aes_ctx _cpx_key _cpx_key_len _cpx_max_key_len _cpx_set_aes_iv_key -_cpx_set_is_sep_wrapped_key _cpx_set_is_composite_key +_cpx_set_is_sep_wrapped_key _cpx_set_key_len _cpx_set_use_offset_for_iv _cpx_set_synthetic_offset_for_iv @@ -101,19 +103,23 @@ _csblob_get_addr _csblob_get_base_offset _csblob_get_cdhash _csblob_get_entitlements +_csblob_get_flags +_csblob_get_hashtype _csblob_get_identity _csblob_get_platform_binary -_csblob_get_flags _csblob_get_teamid +_csblob_get_signer_type _csblob_get_size _csfg_get_cdhash _csfg_get_path _csfg_get_platform_binary _csfg_get_prod_signed +_csfg_get_signer_type _csfg_get_teamid _csproc_get_blob _csproc_get_platform_binary _csproc_get_prod_signed +_csproc_get_signer_type _csproc_get_teamid _csvnode_get_blob _csvnode_get_teamid @@ -130,7 +136,9 @@ _gpu_describe _gpu_fceiling_cb_register _gpu_submission_telemetry _hz +_iflt_attach_internal _ifnet_allocate_extended +_ifnet_allocate_internal _ifnet_bandwidths _ifnet_clone_attach _ifnet_clone_detach @@ -176,15 +184,8 @@ _ifnet_set_rcvq_maxlen _ifnet_set_sndq_maxlen _ifnet_start _ifnet_subfamily -_ifnet_transmit_burst_end -_ifnet_transmit_burst_start _ifnet_tx_compl _ifnet_tx_compl_status -_ifnet_set_packetpreamblelen -_ifnet_packetpreamblelen -_ifnet_maxpacketpreamblelen -_ifnet_set_fastlane_capable -_ifnet_get_fastlane_capable _ifnet_get_unsent_bytes _ifnet_get_buffer_status _ifnet_normalise_unsent_data @@ -198,6 +199,8 @@ _io_rate_update_register _ip_gre_output _ip_gre_register_input _ipc_port_release_send +_ipf_addv4_internal +_ipf_addv6_internal _kauth_cred_getgroups _kauth_cred_grnam2guid _kauth_cred_guid2grnam @@ -214,11 +217,18 @@ _kdp_register_link _kdp_set_interface _kdp_unregister_link _kdp_unregister_send_receive +_kern_allocation_get_name +_kern_allocation_name_allocate +_kern_allocation_name_release +_thread_set_allocation_name _kern_asl_msg _kern_asl_msg_va +_kern_coredump_log +_kern_register_coredump_helper _kern_config_is_development _kern_stack_snapshot_with_reason _kernel_debug_string +_kevent_id_internal _kevent_qos_internal _kevent_qos_internal_bind _kevent_qos_internal_unbind @@ -239,13 +249,12 @@ _m_prepend_2 _m_pullup _m_split _m_trailingspace:_mbuf_trailingspace -_mac_proc_set_enforce -_mach_vm_allocate +_mach_vm_allocate:_mach_vm_allocate_external _mach_vm_behavior_set _mach_vm_deallocate -_mach_vm_map +_mach_vm_map:_mach_vm_map_external _mach_vm_protect -_mach_vm_remap +_mach_vm_remap:_mach_vm_remap_external _mbuf_add_drvaux _mbuf_del_drvaux _mbuf_find_drvaux @@ -347,18 +356,21 @@ _sbappendaddr _sbappendrecord _sbflush _sbspace +_sflt_register_internal _soabort _socantrcvmore _socantsendmore +_sock_accept_internal _sock_catchevents _sock_getlistener _sock_gettclassopt +_sock_iskernel _sock_release _sock_retain _sock_settclassopt _sock_setupcall _sock_setupcalls -_sock_iskernel +_sock_socket_internal _sodisconnect _sofree _sofreelastref @@ -378,6 +390,7 @@ _strnstr _sysdiagnose_notify_user _termioschars _thread_call_allocate_with_priority +_thread_call_allocate_with_qos _thread_call_cancel_wait _thread_clear_eager_preempt _thread_dispatchqaddr @@ -432,10 +445,11 @@ _vfs_getattr _vfs_getbyid _vfs_mntlabel _vfs_nativexattrs +_vfs_set_root_unmounted_cleanly _vfs_setcompoundopen _vfs_throttle_mask _vfs_vnodecovered -_vm_fault +_vm_fault:_vm_fault_external _vm_map_copy_copy _vm_map_copy_discard _vm_map_copyin @@ -450,6 +464,7 @@ _vm_map_trunc_page_mask _vm_map_wire_and_extract:_vm_map_wire_and_extract_external _vm_page_wire_count _vn_getpath_fsenter +_vn_getpath_fsenter_with_parent _vn_searchfs_inappropriate_name _vnode_create_empty _vnode_initialize @@ -501,6 +516,7 @@ _throttle_io_will_be_throttled _ubc_is_mapped_writable _ubc_setsize_ex _ubc_upl_range_needed +_upl_get_size _vfs_context_current _vfs_context_issuser _vfs_context_kernel @@ -514,7 +530,6 @@ _vnode_getname_printable _vnode_getfromfd _vnode_isautocandidate _vnode_isfastdevicecandidate -_vnode_isnamedstream _vnode_putname_printable _vnode_setautocandidate _vnode_setdirty @@ -567,6 +582,8 @@ _qf_put _dqfileinit _dqreclaim _zalloc +_zalloc_noblock +_zdestroy _zfree _zinit _zone_change diff --git a/config/Private.x86_64.exports b/config/Private.x86_64.exports index bfe836f99..5341bdbfe 100644 --- a/config/Private.x86_64.exports +++ b/config/Private.x86_64.exports @@ -1,4 +1,6 @@ _IOGetBootKeyStoreData +_IOGetAPFSKeyStoreData +_IOSetAPFSKeyStoreData _SHA256_Final _SHA256_Init _SHA256_Update @@ -52,3 +54,13 @@ _PE_reboot_on_panic _file_vnode _proc_ucred _suser + +#For copyout_shim private KPI +_cos_kernel_unslide +_cos_kernel_reslide +_register_copyout_shim + +#Allow kexts introspect the kernel's layout in memory +_getsegdatafromheader +_getsegbynamefromheader +__mh_execute_header diff --git a/config/System.kext/PlugIns/Kasan.kext/Info.plist b/config/System.kext/PlugIns/Kasan.kext/Info.plist new file mode 100644 index 000000000..69d83ce43 --- /dev/null +++ b/config/System.kext/PlugIns/Kasan.kext/Info.plist @@ -0,0 +1,34 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<plist version="1.0"> +<dict> + <key>CFBundleDevelopmentRegion</key> + <string>English</string> + <key>CFBundleExecutable</key> + <string>Kasan</string> + <key>CFBundleGetInfoString</key> + <string>Kasan Pseudoextension, Apple Computer Inc, ###KERNEL_VERSION_LONG###</string> + <key>CFBundleIdentifier</key> + <string>com.apple.kpi.kasan</string> + <key>CFBundleInfoDictionaryVersion</key> + <string>6.0</string> + <key>CFBundleName</key> + <string>Kasan Pseudoextension</string> + <key>CFBundlePackageType</key> + <string>KEXT</string> + <key>CFBundleShortVersionString</key> + <string>###KERNEL_VERSION_SHORT###</string> + <key>CFBundleSignature</key> + <string>????</string> + <key>CFBundleVersion</key> + <string>###KERNEL_VERSION_LONG###</string> + <key>OSBundleCompatibleVersion</key> + <string>8.0.0b1</string> + <key>OSBundleRequired</key> + <string>Root</string> + <key>OSKernelResource</key> + <true/> + <key>OSBundleAllowUserLoad</key> + <true/> +</dict> +</plist> diff --git a/config/Unsupported.arm.exports b/config/Unsupported.arm.exports new file mode 100644 index 000000000..6f33928e1 --- /dev/null +++ b/config/Unsupported.arm.exports @@ -0,0 +1,24 @@ +__ZN9IODTNVRAM17getOWVariableInfoEmPPK8OSSymbolPmS4_ +__ZN9IODTNVRAM19convertObjectToPropEPhPmPK8OSSymbolP8OSObject +__ZN9IODTNVRAM19convertPropToObjectEPhmS0_mPPK8OSSymbolPP8OSObject +__ZN9IODTNVRAM19searchNVRAMPropertyEP17IONVRAMDescriptorPm +__ZN9IODTNVRAM19unescapeBytesToDataEPKhm +_bsd_set_dependency_capable +_clock_get_system_value +_kdp_register_callout +_kdp_set_ip_and_mac_addresses +_logwakeup +_mach_msg_rpc_from_kernel +_mach_msg_send_from_kernel_with_options:_mach_msg_send_from_kernel_with_options_legacy +_ml_stack_remaining +_serial_getc +_serial_init +_serial_putc +_text_crypter_create_hook_set +_vm_map_copyout +_ml_get_cpu_count +_ml_get_boot_cpu_number +_ml_get_cpu_number +_ml_get_max_cpu_number +_ml_dbgwrap_halt_cpu_with_state +_vm_map:_vm_map_external diff --git a/config/Unsupported.arm64.exports b/config/Unsupported.arm64.exports new file mode 100644 index 000000000..38e1a8c0e --- /dev/null +++ b/config/Unsupported.arm64.exports @@ -0,0 +1,40 @@ +__ZN9IODTNVRAM17getOWVariableInfoEjPPK8OSSymbolPjS4_ +__ZN9IODTNVRAM19convertObjectToPropEPhPjPK8OSSymbolP8OSObject +__ZN9IODTNVRAM19convertPropToObjectEPhjS0_jPPK8OSSymbolPP8OSObject +__ZN9IODTNVRAM19searchNVRAMPropertyEP17IONVRAMDescriptorPj +__ZN9IODTNVRAM19unescapeBytesToDataEPKhj +_bsd_set_dependency_capable +_kdp_register_callout +_kdp_set_ip_and_mac_addresses +_logwakeup +_ml_stack_remaining +_serial_getc +_serial_init +_serial_putc +_text_crypter_create_hook_set +_vm_map_copyout +_kpc_register_pm_handler +_kpc_reserve_pm_counters +_kpc_release_pm_counters +_kpc_pm_acknowledge +_kpc_get_running +_kpc_set_running +_kpc_get_cpu_counters +_kpc_get_shadow_counters +_kpc_get_config +_kpc_set_config +_kpc_get_period +_kpc_set_period +_kpc_get_actionid +_kpc_set_actionid +_ml_cpu_signal +_ml_cpu_signal_deferred +_ml_cpu_signal_retract +_ml_get_cpu_count +_ml_get_boot_cpu_number +_ml_get_cpu_number +_ml_get_max_cpu_number +_ml_lockdown_handler_register +_ml_dbgwrap_halt_cpu_with_state +_vm_map:_vm_map_external + diff --git a/config/Unsupported.exports b/config/Unsupported.exports index 70375325a..9840aa96c 100644 --- a/config/Unsupported.exports +++ b/config/Unsupported.exports @@ -82,6 +82,12 @@ _host_get_exception_ports _host_priv_self _hz _ipc_kernel_map +_iflt_attach_internal +_ifnet_allocate_internal +_ifnet_set_fastlane_capable +_ifnet_get_fastlane_capable +_ipf_addv4_internal +_ipf_addv6_internal _kalloc:_kalloc_external _kauth_cred_issuser _kauth_cred_label_update @@ -151,8 +157,11 @@ _putc _rc4_crypt _rc4_init _securelevel +_sflt_register_internal _sha1_hardware_hook _sleep +_sock_accept_internal +_sock_socket_internal _stack_privilege _task_get_special_port _task_resume @@ -164,9 +173,8 @@ _tsleep _ubc_cs_blob_get _vfs_context_current _vfs_update_vfsstat -_vm_allocate +_vm_allocate:_vm_allocate_external _vm_deallocate -_vm_map _vm_map_deallocate _vm_map_unwire _vm_map_wire:_vm_map_wire_external @@ -174,7 +182,6 @@ _set_vm_privilege _vm_protect _vm_region _vm_region_object_create -_vnode_isnamedstream _vnode_tag _vnop_getnamedstream_desc _vnop_kqfilt_add_desc diff --git a/config/Unsupported.x86_64.exports b/config/Unsupported.x86_64.exports index 0f3ed92d1..57c5bb71e 100644 --- a/config/Unsupported.x86_64.exports +++ b/config/Unsupported.x86_64.exports @@ -18,6 +18,7 @@ _dsmos_page_transform_hook _gPEEFIRuntimeServices _gPEEFISystemTable _hibernate_vm_lock +_hibernate_vm_lock_end _hibernate_vm_unlock _kdp_register_callout _kdp_set_ip_and_mac_addresses @@ -42,4 +43,5 @@ _sock_retain _tmrCvt _tsc_get_info _PE_state +_vm_map diff --git a/iokit/IOKit/IOCPU.h b/iokit/IOKit/IOCPU.h index a9ae2e605..25d2398de 100644 --- a/iokit/IOKit/IOCPU.h +++ b/iokit/IOKit/IOCPU.h @@ -51,10 +51,6 @@ enum { kIOCPUStateCount }; -class IOCPUInterruptController; - -extern IOCPUInterruptController *gIOCPUInterruptController; - class IOCPU : public IOService { OSDeclareAbstractStructors(IOCPU); @@ -76,8 +72,6 @@ protected: virtual void setCPUState(UInt32 cpuState); public: - static void initCPUs(void); - virtual bool start(IOService *provider) APPLE_KEXT_OVERRIDE; virtual OSObject *getProperty(const OSSymbol *aKey) const APPLE_KEXT_OVERRIDE; virtual bool setProperty(const OSSymbol *aKey, OSObject *anObject) APPLE_KEXT_OVERRIDE; @@ -116,6 +110,7 @@ extern "C" kern_return_t IOCPURunPlatformQuiesceActions(void); extern "C" kern_return_t IOCPURunPlatformActiveActions(void); extern "C" kern_return_t IOCPURunPlatformHaltRestartActions(uint32_t message); extern "C" kern_return_t IOCPURunPlatformPanicActions(uint32_t message); +extern "C" kern_return_t IOCPURunPlatformPanicSyncAction(void *addr, size_t len); class IOCPUInterruptController : public IOInterruptController { @@ -126,8 +121,8 @@ private: protected: int numCPUs; - IOCPU **cpus; - + int numSources; + struct ExpansionData { }; ExpansionData *reserved; @@ -152,7 +147,8 @@ public: virtual IOReturn handleInterrupt(void *refCon, IOService *nub, int source) APPLE_KEXT_OVERRIDE; - OSMetaClassDeclareReservedUnused(IOCPUInterruptController, 0); + virtual IOReturn initCPUInterruptController(int sources, int cpus); + OSMetaClassDeclareReservedUnused(IOCPUInterruptController, 1); OSMetaClassDeclareReservedUnused(IOCPUInterruptController, 2); OSMetaClassDeclareReservedUnused(IOCPUInterruptController, 3); diff --git a/iokit/IOKit/IODeviceTreeSupport.h b/iokit/IOKit/IODeviceTreeSupport.h index 531202f41..b10c5553c 100644 --- a/iokit/IOKit/IODeviceTreeSupport.h +++ b/iokit/IOKit/IODeviceTreeSupport.h @@ -71,8 +71,13 @@ enum { OSCollectionIterator * IODTFindMatchingEntries( IORegistryEntry * from, IOOptionBits options, const char * keys ); +#if !defined(__arm64__) typedef SInt32 (*IODTCompareAddressCellFunc) (UInt32 cellCount, UInt32 left[], UInt32 right[]); +#else +typedef SInt64 (*IODTCompareAddressCellFunc) + (UInt32 cellCount, UInt32 left[], UInt32 right[]); +#endif typedef void (*IODTNVLocationFunc) (IORegistryEntry * entry, diff --git a/iokit/IOKit/IOEventSource.h b/iokit/IOKit/IOEventSource.h index 66ee9054a..44502a12e 100644 --- a/iokit/IOKit/IOEventSource.h +++ b/iokit/IOKit/IOEventSource.h @@ -122,6 +122,21 @@ protected: Is this event source enabled to deliver requests to the work-loop. */ bool enabled; +#if XNU_KERNEL_PRIVATE + + enum + { + kPassive = 0x0001, + kActive = 0x0002, + }; + uint8_t eventSourceReserved1[1]; + uint16_t flags; +#if __LP64__ + uint8_t eventSourceReserved2[4]; +#endif /* __LP64__ */ + +#endif /* XNU_KERNEL_PRIVATE */ + /*! @var workLoop What is the work-loop for this event source. */ IOWorkLoop *workLoop; @@ -147,9 +162,7 @@ protected: @abstract Primary initialiser for the IOEventSource class. @param owner Owner of this instance of an event source. Used as the first parameter -of the action callout. Owner will generally be an OSObject it doesn't have to -be as no member functions will be called directly in it. It can just be a -refcon for a client routine. +of the action callout. Owner must be an OSObject. @param action Pointer to C call out function. Action is a pointer to a C function that gets called when this event source has outstanding work. It will usually diff --git a/iokit/IOKit/IOHibernatePrivate.h b/iokit/IOKit/IOHibernatePrivate.h index 5fc9cf13e..cf8aa46df 100644 --- a/iokit/IOKit/IOHibernatePrivate.h +++ b/iokit/IOKit/IOHibernatePrivate.h @@ -35,6 +35,9 @@ extern "C" { #ifdef KERNEL #include <libkern/crypto/aes.h> #include <uuid/uuid.h> +#include <kern/debug.h> + +extern int kdb_printf(const char *format, ...) __printflike(1,2); #endif #ifndef __IOKIT_IOHIBERNATEPRIVATE_H @@ -308,8 +311,7 @@ void IOCloseDebugDataFile(); IOReturn IOHibernateIOKitSleep(void); IOReturn IOHibernateSystemHasSlept(void); IOReturn IOHibernateSystemWake(void); -IOReturn IOHibernateSystemPostWake(void); -void IOHibernateSystemPostWakeTrim(void * p1, void * p2); +IOReturn IOHibernateSystemPostWake(bool now); uint32_t IOHibernateWasScreenLocked(void); void IOHibernateSetScreenLocked(uint32_t lockState); void IOHibernateSetWakeCapabilities(uint32_t capability); @@ -362,6 +364,10 @@ void hibernate_vm_lock(void); void hibernate_vm_unlock(void); +void +hibernate_vm_lock_end(void); +boolean_t +hibernate_vm_locks_are_safe(void); // mark pages not to be saved, based on VM system accounting void @@ -435,11 +441,15 @@ extern uint8_t gIOHibernateRestoreStack[]; extern uint8_t gIOHibernateRestoreStackEnd[]; extern IOHibernateImageHeader * gIOHibernateCurrentHeader; +#define HIBLOGFROMPANIC(fmt, args...) \ + { if (kernel_debugger_entry_count) { kdb_printf(fmt, ## args); } } + #define HIBLOG(fmt, args...) \ - { kprintf(fmt, ## args); printf(fmt, ## args); } + { if (kernel_debugger_entry_count) { kdb_printf(fmt, ## args); } else { kprintf(fmt, ## args); printf(fmt, ## args); } } #define HIBPRINT(fmt, args...) \ - { kprintf(fmt, ## args); } + { if (kernel_debugger_entry_count) { kdb_printf(fmt, ## args); } else { kprintf(fmt, ## args); } } + #endif /* KERNEL */ diff --git a/iokit/IOKit/IOInterruptAccounting.h b/iokit/IOKit/IOInterruptAccounting.h index 7e03f6bd5..d2715d0b0 100644 --- a/iokit/IOKit/IOInterruptAccounting.h +++ b/iokit/IOKit/IOInterruptAccounting.h @@ -29,6 +29,8 @@ #ifndef __IOKIT_IOINTERRUPTACCOUNTING_H #define __IOKIT_IOINTERRUPTACCOUNTING_H +#include <IOKit/IOReportTypes.h> + /* * This header contains definitions that will be needed by userspace clients of the interrupt accounting * mechanisms. diff --git a/iokit/IOKit/IOInterruptController.h b/iokit/IOKit/IOInterruptController.h index d389a79e3..0a634b056 100644 --- a/iokit/IOKit/IOInterruptController.h +++ b/iokit/IOKit/IOInterruptController.h @@ -109,6 +109,16 @@ public: OSMetaClassDeclareReservedUnused(IOInterruptController, 3); OSMetaClassDeclareReservedUnused(IOInterruptController, 4); OSMetaClassDeclareReservedUnused(IOInterruptController, 5); + +public: + // Generic methods (not to be overriden). + + void timeStampSpuriousInterrupt(void); + void timeStampInterruptHandlerStart(IOInterruptVectorNumber vectorNumber, IOInterruptVector *vector); + void timeStampInterruptHandlerEnd(IOInterruptVectorNumber vectorNumber, IOInterruptVector *vector); + +private: + void timeStampInterruptHandlerInternal(bool isStart, IOInterruptVectorNumber vectorNumber, IOInterruptVector *vector); }; diff --git a/iokit/IOKit/IOKernelReportStructs.h b/iokit/IOKit/IOKernelReportStructs.h index 4018f7995..b15a62527 100644 --- a/iokit/IOKit/IOKernelReportStructs.h +++ b/iokit/IOKit/IOKernelReportStructs.h @@ -1,8 +1,8 @@ /* * Copyright (c) 2012-2014 Apple Computer, Inc. All Rights Reserved. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ @@ -40,12 +40,10 @@ extern "C" { #endif -#define kIOReportAPIVersion 28 - // Drivers participating in IOReporting can advertise channels by // publishing properties in the I/O Kit registry. Various helper // mechanisms exist to produce correctly-formatted legends. -// 12836893 tracks declaring channels in user space. +// 12836893 tracks advertising channels in user space. #define kIOReportLegendPublicKey "IOReportLegendPublic" // bool #define kIOReportLegendKey "IOReportLegend" // arr #define kIOReportLegendChannelsKey "IOReportChannels" // arr @@ -60,203 +58,7 @@ extern "C" { #define kIOReportChannelIDIdx 0 // required #define kIOReportChannelTypeIdx 1 // required #define kIOReportChannelNameIdx 2 // optional - -// We are currently (internally) limited to 15 (broad!) categories. - - -/* - Units / Scaling Factors - - 1. Implementation Details - 2. Unit Constants (kIOReportUnit...) for clients - - Please file radars if you need more units (IOReporting | X) -*/ - -// 1. Implementation Details -// We are likely to someday support IOReporting data as stored binary data. -// Don't change existing values lest that data become unreadable. - -typedef uint64_t IOReportUnits; -#define __IOR_MAKEUNIT(quantity, scale) \ - (((IOReportUnits)quantity << 56) | (uint64_t)scale) -#define IOREPORT_GETUNIT_QUANTITY(unit) \ - ((IOReportQuantity)((uint64_t)unit >> 56) & 0xff) -#define IOREPORT_GETUNIT_SCALE(unit) \ - ((IOReportScaleFactor)unit & 0x00ffffffffffffff) - -// 8b quantity + 32b const + 8b * 2^10 + 8b * 2^n + 8b cardinal + 8b unused -typedef uint8_t IOReportQuantity; // SI "quantity" is what's measured -typedef uint64_t IOReportScaleFactor; - -// See <http://en.wikipedia.org/wiki/SI_base_unit> for a list -// of quantities and their symbols. -enum { - // used by state reports, etc - kIOReportQuantityUndefined = 0, - - kIOReportQuantityTime = 1, // Seconds - kIOReportQuantityPower = 2, // Watts - kIOReportQuantityEnergy = 3, // Joules - kIOReportQuantityCurrent = 4, // Amperes - kIOReportQuantityVoltage = 5, // Volts - kIOReportQuantityCapacitance = 6, // Farad - kIOReportQuantityInductance = 7, // Henry - kIOReportQuantityFrequency = 8, // Hertz - kIOReportQuantityData = 9, // bits/bytes (see scale) - kIOReportQuantityTemperature = 10, // Celsius (not Kelvin :) - - kIOReportQuantityEventCount = 100, - kIOReportQuantityPacketCount = 101 -}; - - -/* A number of units end up with both IEC (2^n) and SI (10^n) scale factors. - For example, the "MB" of a 1.44 MB floppy or a 1024MHz clock. We - thus support separate 2^n and 10^n factors. The exponent encoding - scheme is modeled loosely on single-precision IEEE 754. - */ -#define kIOReportScaleConstMask 0x000000007fffffff // constant ("uint31") -#define kIOReportScaleOneOver (1LL << 31) // 1/constant -#define kIOReportExpBase (-127) // support base^(-n) -#define kIOReportExpZeroOffset -(kIOReportExpBase) // max exponent = 128 -#define kIOReportScaleSIShift 32 // * 10^n -#define kIOReportScaleSIMask 0x000000ff00000000 -#define kIOReportScaleIECShift 40 // * 2^n -#define kIOReportScaleIECMask 0x0000ff0000000000 -#define kIOReportCardinalShift 48 // placeholders -#define kIOReportCardinalMask 0x00ff000000000000 - -/* - Scales are described as a factor times unity: - 1ms = kIOReportScaleMilli * s - - A value expressed in a scaled unit can be scaled to unity via - multiplication by the constant: - 100ms * kIOReportScaleMilli [1e-3] = 0.1s. -*/ - -// SI / decimal -#define kIOReportScalePico ((-12LL + kIOReportExpZeroOffset) \ - << kIOReportScaleSIShift) -#define kIOReportScaleNano ((-9LL + kIOReportExpZeroOffset) \ - << kIOReportScaleSIShift) -#define kIOReportScaleMicro ((-6LL + kIOReportExpZeroOffset) \ - << kIOReportScaleSIShift) -#define kIOReportScaleMilli ((-3LL + kIOReportExpZeroOffset) \ - << kIOReportScaleSIShift) -#define kIOReportScaleUnity 0 // 10^0 = 2^0 = 1 -// unity = 0 is a special case for which we give up exp = -127 -#define kIOReportScaleKilo ((3LL + kIOReportExpZeroOffset) \ - << kIOReportScaleSIShift) -#define kIOReportScaleMega ((6LL + kIOReportExpZeroOffset) \ - << kIOReportScaleSIShift) -#define kIOReportScaleGiga ((9LL + kIOReportExpZeroOffset) \ - << kIOReportScaleSIShift) -#define kIOReportScaleTera ((12LL + kIOReportExpZeroOffset) \ - << kIOReportScaleSIShift) - -// IEC / computer / binary -// It's not clear we'll ever use 2^(-n), but 1..2^~120 should suffice. -#define kIOReportScaleBits kIOReportScaleUnity -#define kIOReportScaleBytes ((3LL + kIOReportExpZeroOffset) \ - << kIOReportScaleIECShift) -// (bytes have to be added to the exponents up front, can't just OR in) -#define kIOReportScaleKibi ((10LL + kIOReportExpZeroOffset) \ - << kIOReportScaleIECShift) -#define kIOReportScaleKiBytes ((13LL + kIOReportExpZeroOffset) \ - << kIOReportScaleIECShift) -#define kIOReportScaleMebi ((20LL + kIOReportExpZeroOffset) \ - << kIOReportScaleIECShift) -#define kIOReportScaleMiBytes ((23LL + kIOReportExpZeroOffset) \ - << kIOReportScaleIECShift) -#define kIOReportScaleGibi ((30LL + kIOReportExpZeroOffset) \ - << kIOReportScaleIECShift) -#define kIOReportScaleGiBytes ((33LL + kIOReportExpZeroOffset) \ - << kIOReportScaleIECShift) -#define kIOReportScaleTebi ((40LL + kIOReportExpZeroOffset) \ - << kIOReportScaleIECShift) -#define kIOReportScaleTiBytes ((43LL + kIOReportExpZeroOffset) \ - << kIOReportScaleIECShift) -// can't encode more than 2^125 (keeping bits & bytes inside -126..128) -// Also, IOReportScaleValue() is currently limited internally by uint64_t. - - -// Cardinal values, to be filled in appropriately. -// Add values in increasing order. -#define kIOReportScaleMachHWTicks (1LL << kIOReportCardinalShift) -#define kIOReportScaleHWPageSize (2LL << kIOReportCardinalShift) - -// page scales: 2 pages * 4ikB/page = 8096 bytes -#define kIOReportScale4KiB (4 | kIOReportScaleKiBytes) -#define kIOReportScale8KiB (8 | kIOReportScaleKiBytes) - -// Clock frequencies scales (units add seconds). -// 1 GHz ticks are 1 ns: 1000 ticks * 1e-6 = 1e-3s -// The '1' is a no-op, but allows a custom label. -#define kIOReportScale1GHz (1 | kIOReportScaleNano) -// 24MHz ticks are 1/24 of a microsecond: (1/24 * kIOReportScaleMicro [1e-6])s -// So for example, 240 24Mticks * 1/24 * 1e-6 = .00001s [1e-5]s -#define kIOReportScale24MHz (kIOReportScaleOneOver|24 |kIOReportScaleMicro) - -// --- END: implementation details - -// 2. Units Constants -// --- BEGIN: units constants driver writers might use -#define kIOReportUnitNone __IOR_MAKEUNIT(kIOReportQuantityUndefined, \ - kIOReportScaleUnity) - -#define kIOReportUnit_s __IOR_MAKEUNIT(kIOReportQuantityTime, \ - kIOReportScaleUnity) -#define kIOReportUnit_ms __IOR_MAKEUNIT(kIOReportQuantityTime, \ - kIOReportScaleMilli) -#define kIOReportUnit_us __IOR_MAKEUNIT(kIOReportQuantityTime, \ - kIOReportScaleMicro) -#define kIOReportUnit_ns __IOR_MAKEUNIT(kIOReportQuantityTime, \ - kIOReportScaleNano) - -#define kIOReportUnit_J __IOR_MAKEUNIT(kIOReportQuantityEnergy, \ - kIOReportScaleUnity) -#define kIOReportUnit_mJ __IOR_MAKEUNIT(kIOReportQuantityEnergy, \ - kIOReportScaleMilli) -#define kIOReportUnit_uJ __IOR_MAKEUNIT(kIOReportQuantityEnergy, \ - kIOReportScaleMicro) -#define kIOReportUnit_nJ __IOR_MAKEUNIT(kIOReportQuantityEnergy, \ - kIOReportScaleNano) -#define kIOReportUnit_pJ __IOR_MAKEUNIT(kIOReportQuantityEnergy, \ - kIOReportScalePico) - -#define kIOReportUnitHWTicks __IOR_MAKEUNIT(kIOReportQuantityTime, \ - kIOReportScaleMachHWTicks) -#define kIOReportUnit24MHzTicks __IOR_MAKEUNIT(kIOReportQuantityTime, \ - kIOReportScale24MHz) -#define kIOReportUnit1GHzTicks __IOR_MAKEUNIT(kIOReportQuantityTime, \ - kIOReportScale1GHz) - -#define kIOReportUnitBits __IOR_MAKEUNIT(kIOReportQuantityData, \ - kIOReportScaleBits) -#define kIOReportUnitBytes __IOR_MAKEUNIT(kIOReportQuantityData, \ - kIOReportScaleBytes) -#define kIOReportUnit_KiB __IOR_MAKEUNIT(kIOReportQuantityData, \ - kIOReportScaleKiBytes) -#define kIOReportUnit_MiB __IOR_MAKEUNIT(kIOReportQuantityData, \ - kIOReportScaleMiBytes) -#define kIOReportUnit_GiB __IOR_MAKEUNIT(kIOReportQuantityData, \ - kIOReportScaleGiBytes) -#define kIOReportUnit_TiB __IOR_MAKEUNIT(kIOReportQuantityData, \ - kIOReportScaleTiBytes) - -#define kIOReportUnitEvents __IOR_MAKEUNIT(kIOReportQuantityEventCount, \ - kIOReportScaleUnity) - -#define kIOReportUnitPackets __IOR_MAKEUNIT(kIOReportQuantityPacketCount, \ - kIOReportScaleUnity) - -// Please file radars if you need more units (IOReporting | X) - -// --- END: unit constants driver writers might use - /* Histogram Segment Configuration Currently supports 2 types of scaling to compute bucket upper bounds, linear or exponential. @@ -280,7 +82,7 @@ typedef struct { uint64_t variance; uint64_t reserved; } __attribute((packed)) IONormDistReportValues; - + #ifdef __cplusplus } #endif diff --git a/iokit/IOKit/IOKernelReporters.h b/iokit/IOKit/IOKernelReporters.h index 5257ca081..bbde3d817 100644 --- a/iokit/IOKit/IOKernelReporters.h +++ b/iokit/IOKit/IOKernelReporters.h @@ -147,7 +147,7 @@ protected: */ virtual bool init(IOService *reportingService, IOReportChannelType channelType, - IOReportUnits unit); + IOReportUnit unit); public: @@ -736,7 +736,7 @@ private: static IOReportLegendEntry* legendWith(OSArray *channelIDs, OSArray *channelNames, IOReportChannelType channelType, - IOReportUnits unit); + IOReportUnit unit); // protected instance variables (want to get rid of these) protected: @@ -761,7 +761,7 @@ protected: // private instance variables private: - IOReportUnits _unit; + IOReportUnit _unit; int _enabled; // 'enabled' if _enabled > 0 @@ -805,7 +805,7 @@ public: */ static IOSimpleReporter* with(IOService *reportingService, IOReportCategories categories, - IOReportUnits unit); + IOReportUnit unit); /*! @function IOSimpleReporter::setValue @abstract Thread safely set a channel's value @@ -860,7 +860,7 @@ protected: */ virtual bool initWith(IOService *reportingService, IOReportCategories categories, - IOReportUnits unit); + IOReportUnit unit); private: @@ -902,7 +902,7 @@ public: static IOStateReporter* with(IOService *reportingService, IOReportCategories categories, int nstates, - IOReportUnits unit = kIOReportUnitHWTicks); + IOReportUnit unit = kIOReportUnitHWTicks); /*! @function IOStateReporter::setStateID @abstract Assign a non-default ID to a state @@ -1300,7 +1300,7 @@ protected: */ virtual bool initWith(IOService *reportingService, IOReportCategories categories, - int16_t nstates, IOReportUnits unit); + int16_t nstates, IOReportUnit unit); /*! @function IOStateReporter::handleSwapPrepare @@ -1482,7 +1482,7 @@ FIXME: need more explanation of the config IOReportCategories categories, uint64_t channelID, const char *channelName, - IOReportUnits unit, + IOReportUnit unit, int nSegments, IOHistogramSegmentConfig *config); @@ -1558,7 +1558,7 @@ protected: IOReportCategories categories, uint64_t channelID, const OSSymbol *channelName, - IOReportUnits unit, + IOReportUnit unit, int nSegments, IOHistogramSegmentConfig *config); diff --git a/iokit/IOKit/IOKitDebug.h b/iokit/IOKit/IOKitDebug.h index 87467f3c9..a6c64b8eb 100644 --- a/iokit/IOKit/IOKitDebug.h +++ b/iokit/IOKit/IOKitDebug.h @@ -81,7 +81,7 @@ enum { // debug aids - change behaviour kIONoFreeObjects = 0x00100000ULL, - kIOLogSynchronous = 0x00200000ULL, // IOLog completes synchronously +// kIOLogSynchronous = 0x00200000ULL, // IOLog completes synchronously -- obsolete kIOTracking = 0x00400000ULL, kIOWaitQuietPanics = 0x00800000ULL, kIOWaitQuietBeforeRoot = 0x01000000ULL, @@ -90,6 +90,29 @@ enum { _kIODebugTopFlag = 0x8000000000000000ULL // force enum to be 64 bits }; +enum { + kIOKitDebugUserOptions = 0 + | kIOLogAttach + | kIOLogProbe + | kIOLogStart + | kIOLogRegister + | kIOLogMatch + | kIOLogConfig + | kIOLogYield + | kIOLogPower + | kIOLogMapping + | kIOLogCatalogue + | kIOLogTracePower + | kIOLogDebugPower + | kOSLogRegistryMods + | kIOLogPMRootDomain + | kOSRegistryModsMode + | kIOLogHibernate + | kIOSleepWakeWdogOff + | kIOKextSpinDump + | kIOWaitQuietPanics +}; + enum { kIOTraceInterrupts = 0x00000001ULL, // Trace primary interrupts kIOTraceWorkLoops = 0x00000002ULL, // Trace workloop activity @@ -143,7 +166,9 @@ struct IOKitDiagnosticsParameters size_t size; uint64_t value; uint32_t options; - uint32_t reserved[3]; + uint32_t tag; + uint32_t zsize; + uint32_t reserved[8]; }; typedef struct IOKitDiagnosticsParameters IOKitDiagnosticsParameters; @@ -166,7 +191,7 @@ struct IOTrackingCallSiteInfo #define kIOWireTrackingName "IOWire" #define kIOMapTrackingName "IOMap" -#if KERNEL && IOTRACKING +#if XNU_KERNEL_PRIVATE && IOTRACKING struct IOTrackingQueue; struct IOTrackingCallSite; @@ -214,7 +239,7 @@ IOTrackingQueue * IOTrackingQueueAlloc(const char * name, uintptr_t btEntry, size_t allocSize, size_t minCaptureSize, uint32_t type, uint32_t numSiteQs); void IOTrackingQueueFree(IOTrackingQueue * head); -void IOTrackingAdd(IOTrackingQueue * head, IOTracking * mem, size_t size, bool address); +void IOTrackingAdd(IOTrackingQueue * head, IOTracking * mem, size_t size, bool address, vm_tag_t tag); void IOTrackingRemove(IOTrackingQueue * head, IOTracking * mem, size_t size); void IOTrackingAddUser(IOTrackingQueue * queue, IOTrackingUser * mem, vm_size_t size); void IOTrackingRemoveUser(IOTrackingQueue * head, IOTrackingUser * tracking); @@ -231,7 +256,7 @@ extern IOTrackingQueue * gIOMallocTracking; extern IOTrackingQueue * gIOWireTracking; extern IOTrackingQueue * gIOMapTracking; -#endif /* KERNEL && IOTRACKING */ +#endif /* XNU_KERNEL_PRIVATE && IOTRACKING */ enum { diff --git a/iokit/IOKit/IOKitKeys.h b/iokit/IOKit/IOKitKeys.h index 240ec58e9..44ed11807 100644 --- a/iokit/IOKit/IOKitKeys.h +++ b/iokit/IOKit/IOKitKeys.h @@ -104,6 +104,7 @@ #define kIOMatchedNotification "IOServiceMatched" #define kIOFirstMatchNotification "IOServiceFirstMatch" #define kIOTerminatedNotification "IOServiceTerminate" +#define kIOWillTerminateNotification "IOServiceWillTerminate" // IOService interest notification types #define kIOGeneralInterest "IOGeneralInterest" diff --git a/iokit/IOKit/IOLib.h b/iokit/IOKit/IOLib.h index 4a8ae78d6..ce3689190 100644 --- a/iokit/IOKit/IOLib.h +++ b/iokit/IOKit/IOLib.h @@ -80,7 +80,7 @@ typedef void (*IOThreadFunc)(void *argument); @param size Size of the memory requested. @result Pointer to the allocated memory, or zero on failure. */ -void * IOMalloc(vm_size_t size); +void * IOMalloc(vm_size_t size) __attribute__((alloc_size(1))); /*! @function IOFree @abstract Frees memory allocated with IOMalloc. @@ -99,7 +99,7 @@ void IOFree(void * address, vm_size_t size); @param alignment Byte count of the alignment for the memory. For example, pass 256 to get memory allocated at an address with bit 0-7 zero. @result Pointer to the allocated memory, or zero on failure. */ -void * IOMallocAligned(vm_size_t size, vm_offset_t alignment); +void * IOMallocAligned(vm_size_t size, vm_offset_t alignment) __attribute__((alloc_size(1))); /*! @function IOFreeAligned @abstract Frees memory allocated with IOMallocAligned. @@ -118,7 +118,7 @@ void IOFreeAligned(void * address, vm_size_t size); @result Virtual address of the allocated memory, or zero on failure. */ void * IOMallocContiguous(vm_size_t size, vm_size_t alignment, - IOPhysicalAddress * physicalAddress) __attribute__((deprecated)); + IOPhysicalAddress * physicalAddress) __attribute__((deprecated)) __attribute__((alloc_size(1))); /*! @function IOFreeContiguous @abstract Deprecated - use IOBufferMemoryDescriptor. Frees memory allocated with IOMallocContiguous. @@ -136,7 +136,7 @@ void IOFreeContiguous(void * address, vm_size_t size) __attribute__((deprecate @param alignment Byte count of the alignment for the memory. For example, pass 256 to get memory allocated at an address with bits 0-7 zero. @result Pointer to the allocated memory, or zero on failure. */ -void * IOMallocPageable(vm_size_t size, vm_size_t alignment); +void * IOMallocPageable(vm_size_t size, vm_size_t alignment) __attribute__((alloc_size(1))); /*! @function IOFreePageable @abstract Frees memory allocated with IOMallocPageable. @@ -309,7 +309,8 @@ __attribute__((format(printf, 1, 2))); @param format A printf() style format string (see printf(3) documentation). @param ap stdarg(3) style variable arguments. */ -void IOLogv(const char *format, va_list ap); +void IOLogv(const char *format, va_list ap) +__attribute__((format(printf, 1, 0))); #ifndef _FN_KPRINTF #define _FN_KPRINTF diff --git a/iokit/IOKit/IOMapper.h b/iokit/IOKit/IOMapper.h index f63f5463a..4fe1ccf43 100644 --- a/iokit/IOKit/IOMapper.h +++ b/iokit/IOKit/IOMapper.h @@ -59,6 +59,7 @@ class IOMapper : public IOService // Give the platform expert access to setMapperRequired(); friend class IOPlatformExpert; friend class IOMemoryDescriptor; + friend class IOGeneralMemoryDescriptor; private: enum SystemMapperState { @@ -69,7 +70,8 @@ private: }; protected: #ifdef XNU_KERNEL_PRIVATE - uint64_t __reservedA[7]; + uint64_t __reservedA[6]; + kern_allocation_name_t fAllocName; uint32_t __reservedB; uint32_t fPageSize; #else diff --git a/iokit/IOKit/IOMemoryDescriptor.h b/iokit/IOKit/IOMemoryDescriptor.h index c284aaa12..0572ef51d 100644 --- a/iokit/IOKit/IOMemoryDescriptor.h +++ b/iokit/IOKit/IOMemoryDescriptor.h @@ -73,18 +73,6 @@ enum IODirection kIODirectionCompleteWithDataValid = 0x00000080, }; - -#if XNU_KERNEL_PRIVATE -enum -{ - // prepare/complete() notify DMA command active - kIODirectionDMACommand = 0x00000100, - kIODirectionDMACommandMask = 0x0001FE00, - kIODirectionDMACommandShift = 9, -}; -#endif - - #ifdef __LP64__ typedef IOOptionBits IODirection; #endif /* __LP64__ */ @@ -124,6 +112,7 @@ enum { #ifdef XNU_KERNEL_PRIVATE kIOMemoryMapCopyOnWrite = 0x00020000, #endif + kIOMemoryRemote = 0x00040000, kIOMemoryThreadSafe = 0x00100000, // Shared with Buffer MD kIOMemoryClearEncrypt = 0x00200000, // Shared with Buffer MD kIOMemoryUseReserve = 0x00800000, // Shared with Buffer MD @@ -217,6 +206,7 @@ enum { kIOMDSetDMAInactive = kIOMDDMAActive, kIOMDAddDMAMapSpec = 0x04000000, kIOMDDMAMap = 0x05000000, + kIOMDDMAUnmap = 0x06000000, kIOMDDMACommandOperationMask = 0xFF000000, }; struct IOMDDMACharacteristics { @@ -286,17 +276,22 @@ protected: #ifdef XNU_KERNEL_PRIVATE public: struct IOMemoryReference * _memRef; + vm_tag_t _kernelTag; + vm_tag_t _userTag; + int16_t _dmaReferences; + uint16_t _internalFlags; + kern_allocation_name_t _mapName; protected: -#else - void * __iomd_reserved5; -#endif +#else /* XNU_KERNEL_PRIVATE */ + void * __iomd_reserved5; + uint16_t __iomd_reserved1[4]; + uintptr_t __iomd_reserved2; +#endif /* XNU_KERNEL_PRIVATE */ -#ifdef __LP64__ - uint64_t __iomd_reserved1; - uint64_t __iomd_reserved2; - uint64_t __iomd_reserved3; - uint64_t __iomd_reserved4; -#else /* !__LP64__ */ + uintptr_t __iomd_reserved3; + uintptr_t __iomd_reserved4; + +#ifndef __LP64__ IODirection _direction; /* use _flags instead */ #endif /* !__LP64__ */ IOByteCount _length; /* length of all ranges */ @@ -399,6 +394,16 @@ typedef IOOptionBits DMACommandOps; uint64_t length, uint64_t * mapAddress, uint64_t * mapLength); + IOReturn dmaUnmap( + IOMapper * mapper, + IODMACommand * command, + uint64_t offset, + uint64_t mapAddress, + uint64_t mapLength); + void dmaMapRecord( + IOMapper * mapper, + IODMACommand * command, + uint64_t mapLength); void setVMTags(vm_tag_t kernelTag, vm_tag_t userTag); vm_tag_t getVMTag(vm_map_t map); @@ -617,6 +622,13 @@ public: virtual IOOptionBits getTag( void ); +/*! @function getFlags + @abstract Accessor to the retrieve the options the memory descriptor was created with. + @discussion Accessor to the retrieve the options the memory descriptor was created with, and flags with its state. These bits are defined by the kIOMemory* enum. + @result The flags bitfield. */ + + uint64_t getFlags(void); + /*! @function readBytes @abstract Copy data from the memory descriptor's buffer to the specified buffer. @discussion This method copies data from the memory descriptor's memory at the given offset, to the caller's buffer. The memory descriptor MUST have the kIODirectionOut direcction bit set and be prepared. kIODirectionOut means that this memory descriptor will be output to an external device, so readBytes is used to get memory into a local buffer for a PIO transfer to the device. diff --git a/iokit/IOKit/IOPlatformExpert.h b/iokit/IOKit/IOPlatformExpert.h index 52e1c366b..936809c7c 100644 --- a/iokit/IOKit/IOPlatformExpert.h +++ b/iokit/IOKit/IOPlatformExpert.h @@ -58,13 +58,16 @@ enum { kPEUPSDelayHaltCPU, kPEPanicRestartCPU, kPEPanicSync, - kPEPagingOff + kPEPagingOff, + kPEPanicBegin, + kPEPanicEnd }; extern int (*PE_halt_restart)(unsigned int type); extern int PEHaltRestart(unsigned int type); // Save the Panic Info. Returns the number of bytes saved. extern UInt32 PESavePanicInfo(UInt8 *buffer, UInt32 length); +extern void PESavePanicInfoAction(void *buffer, size_t length); extern long PEGetGMTTimeOfDay( void ); extern void PESetGMTTimeOfDay( long secs ); diff --git a/iokit/IOKit/IOPolledInterface.h b/iokit/IOKit/IOPolledInterface.h index 3b3a663ec..cd1ba5986 100644 --- a/iokit/IOKit/IOPolledInterface.h +++ b/iokit/IOKit/IOPolledInterface.h @@ -54,7 +54,8 @@ enum enum { kIOPolledWrite = 1, - kIOPolledRead = 2 + kIOPolledRead = 2, + kIOPolledFlush = 3 }; typedef void (*IOPolledCompletionAction)( void * target, @@ -203,6 +204,8 @@ extern __C IOReturn IOPolledFileRead(IOPolledFileIOVars * vars, uint8_t * bytes, IOByteCount size, IOPolledFileCryptVars * cryptvars); +extern __C IOReturn IOPolledFileFlush(IOPolledFileIOVars * vars); + extern __C IOReturn IOPolledFilePollersOpen(IOPolledFileIOVars * vars, uint32_t state, bool abortable); extern __C IOReturn IOPolledFilePollersClose(IOPolledFileIOVars * vars, uint32_t state); diff --git a/iokit/IOKit/IOReportTypes.h b/iokit/IOKit/IOReportTypes.h index 8cd6e7328..3d65c3480 100644 --- a/iokit/IOKit/IOReportTypes.h +++ b/iokit/IOKit/IOReportTypes.h @@ -1,8 +1,8 @@ /* * Copyright (c) 2012-2014 Apple Computer, Inc. All Rights Reserved. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ @@ -60,7 +60,7 @@ extern "C" { needed. Groups and subgroups are a more extensible mechanism for aggregating channels produced by different drivers. */ -typedef uint16_t IOReportCategories; +typedef uint16_t IOReportCategories; #define kIOReportCategoryPower (1 << 1) // and energy #define kIOReportCategoryTraffic (1 << 2) // I/O at any level #define kIOReportCategoryPerformance (1 << 3) // e.g. cycles/byte @@ -69,9 +69,8 @@ typedef uint16_t IOReportCategories; #define kIOReportCategoryField (1 << 8) // consider logging // future categories TBD -#define kIOReportCategoryInterrupt (1 << 14) // TBR: 15850269 #define kIOReportCategoryDebug (1 << 15) -#define kIOReportInvalidCategory UINT16_MAX +#define kIOReportInvalidCategory UINT16_MAX // IOReportChannelType.report_format @@ -201,6 +200,210 @@ typedef struct { IOReportElementValues values; } __attribute((packed)) IOReportElement; + + +/* + IOReporting unit type and constants +*/ + +// 1. Mechanism + +// Assume encoded units could be stored in binary format: don't +// change existing values. + +typedef uint64_t IOReportUnit; +typedef uint64_t IOReportUnits; // deprecated typo, please switch +#define __IOR_MAKEUNIT(quantity, scale) \ + (((IOReportUnit)quantity << 56) | (uint64_t)scale) +#define IOREPORT_GETUNIT_QUANTITY(unit) \ + ((IOReportQuantity)((uint64_t)unit >> 56) & 0xff) +#define IOREPORT_GETUNIT_SCALE(unit) \ + ((IOReportScaleFactor)unit & 0x00ffffffffffffff) + +// 8b quantity ID | 32b const val + 8b*2^10 + 8b*2^n | 8b cardinal | 8b unused +typedef uint8_t IOReportQuantity; // SI "quantity" is what's measured +typedef uint64_t IOReportScaleFactor; + +// See <http://en.wikipedia.org/wiki/SI_base_unit> for a list +// of quantities and their symbols. +enum { + // used by state reports, etc + kIOReportQuantityUndefined = 0, + + kIOReportQuantityTime = 1, // Seconds + kIOReportQuantityPower = 2, // Watts + kIOReportQuantityEnergy = 3, // Joules + kIOReportQuantityCurrent = 4, // Amperes + kIOReportQuantityVoltage = 5, // Volts + kIOReportQuantityCapacitance = 6, // Farad + kIOReportQuantityInductance = 7, // Henry + kIOReportQuantityFrequency = 8, // Hertz + kIOReportQuantityData = 9, // bits/bytes (see scale) + kIOReportQuantityTemperature = 10, // Celsius (not Kelvin :) + + kIOReportQuantityEventCount = 100, + kIOReportQuantityPacketCount = 101, + kIOReportQuantityCPUInstrs = 102 +}; + + +/* A number of units end up with both IEC (2^n) and SI (10^n) scale factors. + For example, the "MB" of a 1.44 MB floppy or a 1024MHz clock. We + thus support separate 2^n and 10^n factors. The exponent encoding + scheme is modeled loosely on single-precision IEEE 754. + */ +#define kIOReportScaleConstMask 0x000000007fffffff // constant ("uint31") +#define kIOReportScaleOneOver (1LL << 31) // 1/constant +#define kIOReportExpBase (-127) // support base^(-n) +#define kIOReportExpZeroOffset -(kIOReportExpBase) // max exponent = 128 +#define kIOReportScaleSIShift 32 // * 10^n +#define kIOReportScaleSIMask 0x000000ff00000000 +#define kIOReportScaleIECShift 40 // * 2^n +#define kIOReportScaleIECMask 0x0000ff0000000000 +#define kIOReportCardinalShift 48 // placeholders +#define kIOReportCardinalMask 0x00ff000000000000 + + +/* + Scales are described as a factor times unity: + 1ms = kIOReportScaleMilli * s + + A value expressed in a scaled unit can be scaled to unity via + multiplication by the constant: + 100ms * kIOReportScaleMilli [1e-3] = 0.1s. +*/ + +// SI / decimal +#define kIOReportScalePico ((-12LL + kIOReportExpZeroOffset) \ + << kIOReportScaleSIShift) +#define kIOReportScaleNano ((-9LL + kIOReportExpZeroOffset) \ + << kIOReportScaleSIShift) +#define kIOReportScaleMicro ((-6LL + kIOReportExpZeroOffset) \ + << kIOReportScaleSIShift) +#define kIOReportScaleMilli ((-3LL + kIOReportExpZeroOffset) \ + << kIOReportScaleSIShift) +#define kIOReportScaleUnity 0 // 10^0 = 2^0 = 1 +// unity = 0 is a special case for which we give up exp = -127 +#define kIOReportScaleKilo ((3LL + kIOReportExpZeroOffset) \ + << kIOReportScaleSIShift) +#define kIOReportScaleMega ((6LL + kIOReportExpZeroOffset) \ + << kIOReportScaleSIShift) +#define kIOReportScaleGiga ((9LL + kIOReportExpZeroOffset) \ + << kIOReportScaleSIShift) +#define kIOReportScaleTera ((12LL + kIOReportExpZeroOffset) \ + << kIOReportScaleSIShift) + +// IEC / computer / binary +// It's not clear we'll ever use 2^(-n), but 1..2^~120 should suffice. +#define kIOReportScaleBits kIOReportScaleUnity +#define kIOReportScaleBytes ((3LL + kIOReportExpZeroOffset) \ + << kIOReportScaleIECShift) +// (bytes have to be added to the exponents up front, can't just OR in) +#define kIOReportScaleKibi ((10LL + kIOReportExpZeroOffset) \ + << kIOReportScaleIECShift) +#define kIOReportScaleKiBytes ((13LL + kIOReportExpZeroOffset) \ + << kIOReportScaleIECShift) +#define kIOReportScaleMebi ((20LL + kIOReportExpZeroOffset) \ + << kIOReportScaleIECShift) +#define kIOReportScaleMiBytes ((23LL + kIOReportExpZeroOffset) \ + << kIOReportScaleIECShift) +#define kIOReportScaleGibi ((30LL + kIOReportExpZeroOffset) \ + << kIOReportScaleIECShift) +#define kIOReportScaleGiBytes ((33LL + kIOReportExpZeroOffset) \ + << kIOReportScaleIECShift) +#define kIOReportScaleTebi ((40LL + kIOReportExpZeroOffset) \ + << kIOReportScaleIECShift) +#define kIOReportScaleTiBytes ((43LL + kIOReportExpZeroOffset) \ + << kIOReportScaleIECShift) +// can't encode more than 2^125 (keeping bits & bytes inside -126..128) +// Also, IOReportScaleValue() is currently limited internally by uint64_t. + + +// Cardinal values, to be filled in appropriately. +// Add values in increasing order. +#define kIOReportScaleMachHWTicks (1LL << kIOReportCardinalShift) +#define kIOReportScaleHWPageSize (2LL << kIOReportCardinalShift) + +// page scales: 2 pages * 4ikB/page = 8096 bytes +#define kIOReportScale4KiB (4 | kIOReportScaleKiBytes) +#define kIOReportScale8KiB (8 | kIOReportScaleKiBytes) +#define kIOReportScale16KiB (16 | kIOReportScaleKiBytes) + +// Clock frequency scales (units add seconds). +// 1 GHz ticks are 1 ns: 1000 ticks * 1e-6 = 1e-3s +// This '1' is a no-op for scaling, but allows a custom label. +#define kIOReportScale1GHz (1 | kIOReportScaleNano) +// 24MHz ticks are 1/24 of a microsecond: (1/24 * kIOReportScaleMicro [1e-6])s +// So for example, 240 24Mticks * 1/24 * 1e-6 = .00001s [1e-5]s +#define kIOReportScale24MHz (kIOReportScaleOneOver|24 |kIOReportScaleMicro) + +// --- END: units mechanism + + +// 2. Unit constants +#define kIOReportUnitNone __IOR_MAKEUNIT(kIOReportQuantityUndefined, \ + kIOReportScaleUnity) + +#define kIOReportUnit_s __IOR_MAKEUNIT(kIOReportQuantityTime, \ + kIOReportScaleUnity) +#define kIOReportUnit_ms __IOR_MAKEUNIT(kIOReportQuantityTime, \ + kIOReportScaleMilli) +#define kIOReportUnit_us __IOR_MAKEUNIT(kIOReportQuantityTime, \ + kIOReportScaleMicro) +#define kIOReportUnit_ns __IOR_MAKEUNIT(kIOReportQuantityTime, \ + kIOReportScaleNano) + +#define kIOReportUnit_J __IOR_MAKEUNIT(kIOReportQuantityEnergy, \ + kIOReportScaleUnity) +#define kIOReportUnit_mJ __IOR_MAKEUNIT(kIOReportQuantityEnergy, \ + kIOReportScaleMilli) +#define kIOReportUnit_uJ __IOR_MAKEUNIT(kIOReportQuantityEnergy, \ + kIOReportScaleMicro) +#define kIOReportUnit_nJ __IOR_MAKEUNIT(kIOReportQuantityEnergy, \ + kIOReportScaleNano) +#define kIOReportUnit_pJ __IOR_MAKEUNIT(kIOReportQuantityEnergy, \ + kIOReportScalePico) + +#define kIOReportUnitHWTicks __IOR_MAKEUNIT(kIOReportQuantityTime, \ + kIOReportScaleMachHWTicks) +#define kIOReportUnit24MHzTicks __IOR_MAKEUNIT(kIOReportQuantityTime, \ + kIOReportScale24MHz) +#define kIOReportUnit1GHzTicks __IOR_MAKEUNIT(kIOReportQuantityTime, \ + kIOReportScale1GHz) + +#define kIOReportUnitBits __IOR_MAKEUNIT(kIOReportQuantityData, \ + kIOReportScaleBits) +#define kIOReportUnitBytes __IOR_MAKEUNIT(kIOReportQuantityData, \ + kIOReportScaleBytes) +#define kIOReportUnit_KiB __IOR_MAKEUNIT(kIOReportQuantityData, \ + kIOReportScaleKiBytes) +#define kIOReportUnit_MiB __IOR_MAKEUNIT(kIOReportQuantityData, \ + kIOReportScaleMiBytes) +#define kIOReportUnit_GiB __IOR_MAKEUNIT(kIOReportQuantityData, \ + kIOReportScaleGiBytes) +#define kIOReportUnit_TiB __IOR_MAKEUNIT(kIOReportQuantityData, \ + kIOReportScaleTiBytes) + +#define kIOReportUnitEvents __IOR_MAKEUNIT(kIOReportQuantityEventCount, \ + kIOReportScaleUnity) + +#define kIOReportUnitPackets __IOR_MAKEUNIT(kIOReportQuantityPacketCount, \ + kIOReportScaleUnity) + +#define kIOReportUnitInstrs __IOR_MAKEUNIT(kIOReportQuantityCPUInstrs, \ + kIOReportScaleUnity) +#define kIOReportUnit_KI __IOR_MAKEUNIT(kIOReportQuantityCPUInstrs, \ + kIOReportScaleKilo) +#define kIOReportUnit_MI __IOR_MAKEUNIT(kIOReportQuantityCPUInstrs, \ + kIOReportScaleMega) +#define kIOReportUnit_GI __IOR_MAKEUNIT(kIOReportQuantityCPUInstrs, \ + kIOReportScaleGiga) + +// Please file bugs (xnu | IOReporting) for additional units. + +// --- END: unit constants + + #ifdef __cplusplus } #endif diff --git a/iokit/IOKit/IOReturn.h b/iokit/IOKit/IOReturn.h index 464b84a08..048020784 100644 --- a/iokit/IOKit/IOReturn.h +++ b/iokit/IOKit/IOReturn.h @@ -52,14 +52,14 @@ typedef kern_return_t IOReturn; #define sub_iokit_firewire err_sub(2) #define sub_iokit_block_storage err_sub(4) #define sub_iokit_graphics err_sub(5) -#define sub_iokit_networking err_sub(6) +#define sub_iokit_networking err_sub(6) #define sub_iokit_bluetooth err_sub(8) #define sub_iokit_pmu err_sub(9) #define sub_iokit_acpi err_sub(10) #define sub_iokit_smbus err_sub(11) #define sub_iokit_ahci err_sub(12) #define sub_iokit_powermanagement err_sub(13) -#define sub_iokit_hidsystem err_sub(14) +#define sub_iokit_hidsystem err_sub(14) #define sub_iokit_scsi err_sub(16) #define sub_iokit_usbaudio err_sub(17) //#define sub_iokit_pccard err_sub(21) @@ -67,7 +67,12 @@ typedef kern_return_t IOReturn; #define sub_iokit_nvme err_sub(28) #endif #define sub_iokit_thunderbolt err_sub(29) -#define sub_iokit_platform err_sub(0x2A) +#define sub_iokit_graphics_acceleration err_sub(30) +#define sub_iokit_keystore err_sub(31) +#ifdef PRIVATE +#define sub_iokit_smc err_sub(32) +#endif +#define sub_iokit_platform err_sub(0x2A) #define sub_iokit_audio_video err_sub(0x45) #define sub_iokit_baseband err_sub(0x80) #define sub_iokit_HDA err_sub(254) diff --git a/iokit/IOKit/IOService.h b/iokit/IOKit/IOService.h index 35e7ec20e..b45162f87 100644 --- a/iokit/IOKit/IOService.h +++ b/iokit/IOKit/IOService.h @@ -131,6 +131,7 @@ extern const OSSymbol * gIOFirstPublishNotification; extern const OSSymbol * gIOMatchedNotification; extern const OSSymbol * gIOFirstMatchNotification; extern const OSSymbol * gIOTerminatedNotification; +extern const OSSymbol * gIOWillTerminateNotification; extern const OSSymbol * gIOGeneralInterest; extern const OSSymbol * gIOBusyInterest; @@ -639,6 +640,11 @@ public: virtual void unlockForArbitration( void ); +#ifdef XNU_KERNEL_PRIVATE + static uint32_t isLockedForArbitration(IOService * service); +#endif /* XNU_KERNEL_PRIVATE */ + + /*! @function terminateClient @abstract Passes a termination up the stack. @discussion When an IOService object is made inactive the default behavior is to also make any of its clients that have it as their only provider inactive, in this way recursing the termination up the driver stack. This method allows a terminated IOService object to override this behavior. Note the client may also override this behavior by overriding its @link terminate terminate@/link method. @@ -727,7 +733,8 @@ public: <br> <code>gIOFirstPublishNotification</code> Delivered when an IOService object is registered, but only once per IOService instance. Some IOService objects may be reregistered when their state is changed. <br> <code>gIOMatchedNotification</code> Delivered when an IOService object has been matched with all client drivers, and they have been probed and started. <br> <code>gIOFirstMatchNotification</code> Delivered when an IOService object has been matched with all client drivers, but only once per IOService instance. Some IOService objects may be reregistered when their state is changed. -<br> <code>gIOTerminatedNotification</code> Delivered after an IOService object has been terminated, during its finalize stage. +<br> <code>gIOWillTerminateNotification</code> Delivered after an IOService object has been terminated, during its finalize stage. Delivered after any matching on the service has finished. +<br> <code>gIOTerminatedNotification</code> Delivered immediately when an IOService object has been terminated, making it inactive. @param matching A matching dictionary to restrict notifications to only matching IOService objects. The dictionary will be released when the notification is removed, consuming the passed-in reference. @param handler A C function callback to deliver notifications. @param target An instance reference for the callback's use. @@ -750,7 +757,8 @@ public: <br> <code>gIOFirstPublishNotification</code> Delivered when an IOService object is registered, but only once per IOService instance. Some IOService objects may be reregistered when their state is changed. <br> <code>gIOMatchedNotification</code> Delivered when an IOService object has been matched with all client drivers, and they have been probed and started. <br> <code>gIOFirstMatchNotification</code> Delivered when an IOService object has been matched with all client drivers, but only once per IOService instance. Some IOService objects may be reregistered when their state is changed. -<br> <code>gIOTerminatedNotification</code> Delivered after an IOService object has been terminated, during its finalize stage. +<br> <code>gIOWillTerminateNotification</code> Delivered after an IOService object has been terminated, during its finalize stage. Delivered after any matching on the service has finished. +<br> <code>gIOTerminatedNotification</code> Delivered immediately when an IOService object has been terminated, making it inactive. @param matching A matching dictionary to restrict notifications to only matching IOService objects. The dictionary is retained while the notification is installed. (Differs from addNotification). @param handler A C function callback to deliver notifications. @param target An instance reference for the callback's use. @@ -1371,7 +1379,11 @@ private: void deliverNotification( const OSSymbol * type, IOOptionBits orNewState, IOOptionBits andNewState ); - bool invokeNotifer( class _IOServiceNotifier * notify ); + OSArray * copyNotifiers(const OSSymbol * type, + IOOptionBits orNewState, IOOptionBits andNewState); + + bool invokeNotifiers(OSArray ** willSend); + bool invokeNotifier( class _IOServiceNotifier * notify ); APPLE_KEXT_COMPATIBILITY_VIRTUAL void unregisterAllInterest( void ); @@ -1828,7 +1840,7 @@ public: void reset_watchdog_timer( void ); void start_watchdog_timer ( void ); bool stop_watchdog_timer ( void ); - IOReturn registerInterestForNotifer( IONotifier *notify, const OSSymbol * typeOfInterest, + IOReturn registerInterestForNotifier( IONotifier *notify, const OSSymbol * typeOfInterest, IOServiceInterestHandler handler, void * target, void * ref ); static IOWorkLoop * getIOPMWorkloop( void ); diff --git a/iokit/IOKit/IOTimeStamp.h b/iokit/IOKit/IOTimeStamp.h index 1c7cf7e57..955505f73 100644 --- a/iokit/IOKit/IOTimeStamp.h +++ b/iokit/IOKit/IOTimeStamp.h @@ -136,6 +136,7 @@ IOTimeStamp(uintptr_t csc, /* DBG_IOKIT/DBG_IOINTC codes */ #define IOINTC_HANDLER 1 /* 0x05000004 */ +#define IOINTC_SPURIOUS 2 /* 0x05000008 */ /* DBG_IOKIT/DBG_IOWORKLOOP codes */ #define IOWL_CLIENT 1 /* 0x05010004 */ diff --git a/iokit/IOKit/IOTimerEventSource.h b/iokit/IOKit/IOTimerEventSource.h index 538c4991b..49b5f257b 100644 --- a/iokit/IOKit/IOTimerEventSource.h +++ b/iokit/IOKit/IOTimerEventSource.h @@ -47,6 +47,56 @@ __END_DECLS #include <IOKit/IOEventSource.h> #include <IOKit/IOTypes.h> +/*! + @enum IOTimerEventSource constructor options + @abstract Constants defining behavior of the IOTimerEventSource. + @constant kIOTimerEventSourceOptionsPriorityHigh Importance above everything but realtime. + Thread calls allocated with this priority execute at extremely high priority, + above everything but realtime threads. They are generally executed in serial. + Though they may execute concurrently under some circumstances, no fan-out is implied. + These work items should do very small amounts of work or risk disrupting system + responsiveness. + @constant kIOTimerEventSourceOptionsPriorityKernelHigh Importance higher than most kernel + threads. + @constant kIOTimerEventSourceOptionsPriorityKernel Importance similar to that of normal kernel + threads. + @constant kIOTimerEventSourceOptionsPriorityUser Importance similar to that of normal user threads. + @constant kIOTimerEventSourceOptionsPriorityLow Very low importance. + @constant kIOTimerEventSourceOptionsPriorityWorkLoop Run the callout on the thread of the IOWorkLoop + the event source has been added to. + @constant kIOTimerEventSourceOptionsAllowReenter Allow the callout to be rescheduled and potentially + re-entered, if the IOWorkLoop lock has been released (eg. with commandSleep) during its invocation. + @constant kIOTimerEventSourceOptionsDefault Recommended default options. + */ +enum +{ + kIOTimerEventSourceOptionsPriorityMask = 0x000000ff, + kIOTimerEventSourceOptionsPriorityHigh = 0x00000000, + kIOTimerEventSourceOptionsPriorityKernelHigh = 0x00000001, + kIOTimerEventSourceOptionsPriorityKernel = 0x00000002, + kIOTimerEventSourceOptionsPriorityUser = 0x00000003, + kIOTimerEventSourceOptionsPriorityLow = 0x00000004, + kIOTimerEventSourceOptionsPriorityWorkLoop = 0x000000ff, + + kIOTimerEventSourceOptionsAllowReenter = 0x00000100, + + kIOTimerEventSourceOptionsDefault = kIOTimerEventSourceOptionsPriorityKernelHigh +}; + +#define IOTIMEREVENTSOURCEOPTIONS_DEFINED 1 + +/*! + @enum IOTimerEventSource setTimeout/wakeAtTime options + @abstract Constants defining behavior of a scheduled call from IOTimerEventSource. + @constant kIOTimeOptionsWithLeeway Use the leeway parameter to the call. + @constant kIOTimeOptionsContinuous Use mach_continuous_time() to generate the callback. +*/ +enum +{ + kIOTimeOptionsWithLeeway = 0x00000020, + kIOTimeOptionsContinuous = 0x00000100, +}; + /*! @class IOTimerEventSource : public IOEventSource @abstract Time based event source mechanism. @@ -56,6 +106,7 @@ __END_DECLS <br><br> Remember the system doesn't guarantee the accuracy of the callout. It is possible that a higher priority thread is running which will delay the execution of the action routine. In fact the thread will be made runable at the exact requested time, within the accuracy of the CPU's decrementer based interrupt, but the scheduler will then control execution. */ + class IOTimerEventSource : public IOEventSource { OSDeclareDefaultStructors(IOTimerEventSource) @@ -73,6 +124,7 @@ protected: struct ExpansionData { SInt32 calloutGeneration; + SInt32 calloutGenerationSignaled; IOWorkLoop * workLoop; }; @@ -105,11 +157,17 @@ public: @param sender The object that timed out. */ typedef void (*Action)(OSObject *owner, IOTimerEventSource *sender); + static IOTimerEventSource * + timerEventSource(OSObject *owner, Action action = 0); + /*! @function timerEventSource @abstract Allocates and returns an initialized timer instance. + @param options Mask of kIOTimerEventSourceOptions* options. + @param owner The object that that will be passed to the Action callback. + @param action 'C' Function pointer for the callout routine of this event source. */ static IOTimerEventSource * - timerEventSource(OSObject *owner, Action action = 0); + timerEventSource(uint32_t options, OSObject *owner, Action action = 0); /*! @function init @abstract Initializes the timer with an owner, and a handler to call when the timeout expires. @@ -126,6 +184,11 @@ public: @discussion When disable returns the action will not be called until the next time enable(qv) is called. */ virtual void disable() APPLE_KEXT_OVERRIDE; +/*! @function checkForWork + @abstract Pure Virtual member function used by IOWorkLoop for issuing a client calls. + @discussion This function called when the work-loop is ready to check for any work to do and then to call out the owner/action. + @result Return true if this function needs to be called again before all its outstanding events have been processed. */ + virtual bool checkForWork() APPLE_KEXT_OVERRIDE; /*! @function setTimeoutTicks @abstract Setup a callback at after the delay in scheduler ticks. See wakeAtTime(AbsoluteTime). @@ -197,7 +260,7 @@ public: /*! @function wakeAtTime @abstract Setup a callback at this absolute time. - @discussion Starts the timer, which will expire at abstime. After it expires, the timer will call the 'action' registered in the init() function. This timer is not periodic, a further call is needed to reset and restart the timer after it expires. + @discussion Starts the timer, which will expire at abstime. After it expires, the timer will call the 'action' registered in the init() function. This timer is not periodic, a further call is needed to reset and restart the timer after it expires. @param abstime Absolute Time when to wake up, counted in 'decrementer' units and starts at zero when system boots. @result kIOReturnSuccess if everything is fine, kIOReturnNoResources if action hasn't been declared by init or IOEventSource::setAction (qqv). */ virtual IOReturn wakeAtTime(AbsoluteTime abstime); @@ -207,13 +270,36 @@ public: @discussion Clear down any oustanding calls. By the time this function completes it is guaranteed that the action will not be called again. */ virtual void cancelTimeout(); +/*! @function init + @abstract Initializes the timer with an owner, and a handler to call when the timeout expires. + */ + virtual bool init(uint32_t options, OSObject *inOwner, Action inAction); + +/*! @function setTimeout + @abstract Setup a callback at after the delay in decrementer ticks. See wakeAtTime(AbsoluteTime). + @param options see kIOTimeOptionsWithLeeway and kIOTimeOptionsContinuous + @param interval Delay from now to wake up in decrementer ticks. + @param leeway Allowable leeway to wake time, if the kIOTimeOptionsWithLeeway option is set + @result kIOReturnSuccess if everything is fine, kIOReturnNoResources if action hasn't been declared. */ + virtual IOReturn setTimeout(uint32_t options, AbsoluteTime interval, AbsoluteTime leeway); + +/*! @function wakeAtTime + @abstract Setup a callback at this absolute time. + @discussion Starts the timer, which will expire at abstime. After it expires, the timer will call the 'action' registered in the init() function. This timer is not periodic, a further call is needed to reset and restart the timer after it expires. + @param options see kIOTimeOptionsWithLeeway and kIOTimeOptionsContinuous + @param abstime Absolute Time when to wake up, counted in 'decrementer' units and starts at zero when system boots. + @param leeway Allowable leeway to wake time, if the kIOTimeOptionsWithLeeway option is set + @result kIOReturnSuccess if everything is fine, kIOReturnNoResources if action hasn't been declared by init or IOEventSource::setAction (qqv). */ + virtual IOReturn wakeAtTime(uint32_t options, AbsoluteTime abstime, AbsoluteTime leeway); + private: - static void timeoutAndRelease(void *self, void *wl); + static void timeoutAndRelease(void *self, void *c); + static void timeoutSignaled(void *self, void *c); private: - OSMetaClassDeclareReservedUnused(IOTimerEventSource, 0); - OSMetaClassDeclareReservedUnused(IOTimerEventSource, 1); - OSMetaClassDeclareReservedUnused(IOTimerEventSource, 2); + OSMetaClassDeclareReservedUsed(IOTimerEventSource, 0); + OSMetaClassDeclareReservedUsed(IOTimerEventSource, 1); + OSMetaClassDeclareReservedUsed(IOTimerEventSource, 2); OSMetaClassDeclareReservedUnused(IOTimerEventSource, 3); OSMetaClassDeclareReservedUnused(IOTimerEventSource, 4); OSMetaClassDeclareReservedUnused(IOTimerEventSource, 5); diff --git a/iokit/IOKit/IOTypes.h b/iokit/IOKit/IOTypes.h index 6d21a1294..62b5a6b08 100644 --- a/iokit/IOKit/IOTypes.h +++ b/iokit/IOKit/IOTypes.h @@ -192,7 +192,8 @@ enum { kIOWriteThruCache = 2, kIOCopybackCache = 3, kIOWriteCombineCache = 4, - kIOCopybackInnerCache = 5 + kIOCopybackInnerCache = 5, + kIOPostedWrite = 6 }; // IOMemory mapping options @@ -207,6 +208,7 @@ enum { kIOMapCopybackCache = kIOCopybackCache << kIOMapCacheShift, kIOMapWriteCombineCache = kIOWriteCombineCache << kIOMapCacheShift, kIOMapCopybackInnerCache = kIOCopybackInnerCache << kIOMapCacheShift, + kIOMapPostedWrite = kIOPostedWrite << kIOMapCacheShift, kIOMapUserOptionsMask = 0x00000fff, diff --git a/iokit/IOKit/Makefile b/iokit/IOKit/Makefile index 008d33e7e..327365f21 100644 --- a/iokit/IOKit/Makefile +++ b/iokit/IOKit/Makefile @@ -41,19 +41,20 @@ NOT_KF_MI_HEADERS = $(NOT_EXPORT_HEADERS) \ IOCommandQueue.h IOLocksPrivate.h \ IOSyncer.h AppleKeyStoreInterface.h \ IOStatistics.h IOStatisticsPrivate.h \ - IOKernelReporters.h + IOKernelReporters.h \ + IOInterruptAccounting.h # These should be additionally installed in IOKit.framework's public Headers -INSTALL_MI_LIST = IOBSD.h IOKitKeys.h IOKitServer.h IOReturn.h\ - IOSharedLock.h IOTypes.h OSMessageNotification.h\ - IODataQueueShared.h IOMessage.h IOInterruptAccounting.h\ +INSTALL_MI_LIST = IOBSD.h IOKitKeys.h IOKitServer.h IOReturn.h \ + IOSharedLock.h IOTypes.h OSMessageNotification.h \ + IODataQueueShared.h IOMessage.h # These should be additionally installed in IOKit.framework's PrivateHeaders -INSTALL_MI_LCL_LIST = IOKitKeysPrivate.h IOHibernatePrivate.h \ - IOLocksPrivate.h IOStatistics.h \ - AppleKeyStoreInterface.h \ - IOReportTypes.h IOKernelReportStructs.h \ - IOReportMacros.h +INSTALL_MI_LCL_LIST = IOKitKeysPrivate.h IOHibernatePrivate.h \ + IOLocksPrivate.h IOStatistics.h \ + AppleKeyStoreInterface.h \ + IOReportTypes.h IOKernelReportStructs.h \ + IOReportMacros.h IOInterruptAccounting.h INSTALL_MI_DIR = . diff --git a/iokit/IOKit/pwr_mgt/IOPM.h b/iokit/IOKit/pwr_mgt/IOPM.h index 03daffd06..f9ca6ed8a 100644 --- a/iokit/IOKit/pwr_mgt/IOPM.h +++ b/iokit/IOKit/pwr_mgt/IOPM.h @@ -614,6 +614,26 @@ enum { #define kIOPMPSAdapterDetailsPMUConfigurationKey "PMUConfiguration" #define kIOPMPSAdapterDetailsVoltage "AdapterVoltage" +// values for kIOPSPowerAdapterFamilyKey +enum { + kIOPSFamilyCodeDisconnected = 0, + kIOPSFamilyCodeUnsupported = kIOReturnUnsupported, + kIOPSFamilyCodeFirewire = iokit_family_err(sub_iokit_firewire, 0), + kIOPSFamilyCodeUSBHost = iokit_family_err(sub_iokit_usb, 0), + kIOPSFamilyCodeUSBHostSuspended = iokit_family_err(sub_iokit_usb, 1), + kIOPSFamilyCodeUSBDevice = iokit_family_err(sub_iokit_usb, 2), + kIOPSFamilyCodeUSBAdapter = iokit_family_err(sub_iokit_usb, 3), + kIOPSFamilyCodeUSBChargingPortDedicated = iokit_family_err(sub_iokit_usb, 4), + kIOPSFamilyCodeUSBChargingPortDownstream = iokit_family_err(sub_iokit_usb, 5), + kIOPSFamilyCodeUSBChargingPort = iokit_family_err(sub_iokit_usb, 6), + kIOPSFamilyCodeUSBUnknown = iokit_family_err(sub_iokit_usb, 7), + kIOPSFamilyCodeAC = iokit_family_err(sub_iokit_pmu, 0), + kIOPSFamilyCodeExternal = iokit_family_err(sub_iokit_pmu, 1), + kIOPSFamilyCodeExternal2 = iokit_family_err(sub_iokit_pmu, 2), + kIOPSFamilyCodeExternal3 = iokit_family_err(sub_iokit_pmu, 3), + kIOPSFamilyCodeExternal4 = iokit_family_err(sub_iokit_pmu, 4), +}; + // Battery's time remaining estimate is invalid this long (seconds) after a wake #define kIOPMPSInvalidWakeSecondsKey "BatteryInvalidWakeSeconds" diff --git a/iokit/IOKit/pwr_mgt/IOPMPrivate.h b/iokit/IOKit/pwr_mgt/IOPMPrivate.h index 4e498388c..b3f7b3397 100644 --- a/iokit/IOKit/pwr_mgt/IOPMPrivate.h +++ b/iokit/IOKit/pwr_mgt/IOPMPrivate.h @@ -103,6 +103,9 @@ enum { #define kIOPMMessageSystemSleepPreventers \ iokit_family_msg(sub_iokit_powermanagement, 0x430) +#define kIOPMMessageLaunchBootSpinDump \ + iokit_family_msg(sub_iokit_powermanagement, 0x440) + /* @enum SystemSleepReasons * @abstract The potential causes for system sleep as logged in the system event record. */ @@ -623,6 +626,13 @@ enum { */ #define kIOPMAutoPowerOffTimerKey "AutoPowerOff Timer" +/* kIOPMDeepSleepTimerKey + * Key refers to a CFNumberRef that indicates the time in seconds until the + * expiration of the Standby delay period. This value should be used + * to program a wake alarm before system sleep. + */ +#define kIOPMDeepSleepTimerKey "Standby Timer" + /* kIOPMUserWakeAlarmScheduledKey * Key refers to a boolean value that indicates if an user alarm was scheduled * or pending. @@ -706,6 +716,7 @@ struct IOPMSystemSleepPolicyVariables uint32_t hibernateMode; // current hibernate mode uint32_t standbyDelay; // standby delay in seconds + uint32_t standbyTimer; // standby timer in seconds uint32_t poweroffDelay; // auto-poweroff delay in seconds uint32_t scheduledAlarms; // bitmask of scheduled alarm types uint32_t poweroffTimer; // auto-poweroff timer in seconds @@ -788,7 +799,18 @@ enum { kIOPMWakeEventUserPME = 0x00000400, kIOPMWakeEventSleepTimer = 0x00000800, kIOPMWakeEventBatteryLow = 0x00001000, - kIOPMWakeEventDarkPME = 0x00002000 + kIOPMWakeEventDarkPME = 0x00002000, + kIOPMWakeEventWifi = 0x00004000, + kIOPMWakeEventRTCSystem = 0x00008000, // Maintenance RTC wake + kIOPMWakeEventUSBCPlugin = 0x00010000, // USB-C Plugin + kIOPMWakeEventHID = 0x00020000, + kIOPMWakeEventBluetooth = 0x00040000, + kIOPMWakeEventDFR = 0x00080000, + kIOPMWakeEventSD = 0x00100000, // SD card + kIOPMWakeEventLANWake = 0x00200000, // Wake on Lan + kIOPMWakeEventLANPlugin = 0x00400000, // Ethernet media sense + kIOPMWakeEventThunderbolt = 0x00800000, + kIOPMWakeEventRTCUser = 0x01000000, // User requested RTC wake }; /*! diff --git a/iokit/IOKit/pwr_mgt/IOPMlog.h b/iokit/IOKit/pwr_mgt/IOPMlog.h index 351ddad2d..9e93f2379 100644 --- a/iokit/IOKit/pwr_mgt/IOPMlog.h +++ b/iokit/IOKit/pwr_mgt/IOPMlog.h @@ -87,5 +87,7 @@ enum PMLogEnum { kPMLogDrvResponseDelay, // 59 0x050700ec kPMLogPCIDevChangeStart, // 60 0x050700f0 kPMLogPCIDevChangeDone, // 61 0x050700f4 + kPMLogSleepWakeMessage, // 62 0x050700f8 + kPMLogDrvPSChangeDelay, // 63 0x050700fc kIOPMlogLastEvent }; diff --git a/iokit/IOKit/pwr_mgt/RootDomain.h b/iokit/IOKit/pwr_mgt/RootDomain.h index c417c9a34..1e447d199 100644 --- a/iokit/IOKit/pwr_mgt/RootDomain.h +++ b/iokit/IOKit/pwr_mgt/RootDomain.h @@ -44,9 +44,7 @@ class IOPMPowerStateQueue; class RootDomainUserClient; class PMAssertionsTracker; -#define OBFUSCATE(x) \ - (((((uintptr_t)(x)) >= VM_MIN_KERNEL_AND_KEXT_ADDRESS) && (((uintptr_t)(x)) < VM_MAX_KERNEL_ADDRESS)) ? \ - ((void *)(VM_KERNEL_ADDRPERM(x))) : (void *)(x)) +#define OBFUSCATE(x) (void *)VM_KERNEL_UNSLIDE_OR_PERM(x) #endif @@ -490,7 +488,7 @@ public: IOReturn setMaintenanceWakeCalendar( const IOPMCalendarStruct * calendar ); - IOReturn getSystemSleepType( uint32_t * sleepType ); + IOReturn getSystemSleepType(uint32_t * sleepType, uint32_t * standbyTimer); // Handle callbacks from IOService::systemWillShutdown() void acknowledgeSystemWillShutdown( IOService * from ); @@ -511,8 +509,12 @@ public: void kdebugTrace(uint32_t event, uint64_t regId, uintptr_t param1, uintptr_t param2, uintptr_t param3 = 0); - void tracePoint( uint8_t point ); - void traceDetail(uint32_t msgType, uint32_t msgIndex, uintptr_t handler); + void tracePoint(uint8_t point); + void traceDetail(uint32_t msgType, uint32_t msgIndex, uint32_t delay); + void traceDetail(OSObject *notifier); + void traceAckDelay(OSObject *notifier, uint32_t response, uint32_t delay_ms); + + void startSpinDump(uint32_t spindumpKind); bool systemMessageFilter( void * object, void * arg1, void * arg2, void * arg3 ); diff --git a/iokit/Kernel/IOBufferMemoryDescriptor.cpp b/iokit/Kernel/IOBufferMemoryDescriptor.cpp index 5641d3b9e..b3ff13dda 100644 --- a/iokit/Kernel/IOBufferMemoryDescriptor.cpp +++ b/iokit/Kernel/IOBufferMemoryDescriptor.cpp @@ -66,7 +66,8 @@ enum { kInternalFlagPhysical = 0x00000001, kInternalFlagPageSized = 0x00000002, - kInternalFlagPageAllocated = 0x00000004 + kInternalFlagPageAllocated = 0x00000004, + kInternalFlagInit = 0x00000008 }; /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ @@ -289,6 +290,11 @@ bool IOBufferMemoryDescriptor::initWithPhysicalMask( inTask, iomdOptions, /* System mapper */ 0)) return false; + _internalFlags |= kInternalFlagInit; +#if IOTRACKING + if (!(options & kIOMemoryPageable)) trackingAccumSize(capacity); +#endif /* IOTRACKING */ + // give any system mapper the allocation params if (kIOReturnSuccess != dmaCommandOperation(kIOMDAddDMAMapSpec, &mapSpec, sizeof(mapSpec))) @@ -488,19 +494,26 @@ void IOBufferMemoryDescriptor::free() map->release(); } + if ((options & kIOMemoryPageable) + || (kInternalFlagPageSized & internalFlags)) size = round_page(size); + +#if IOTRACKING + if (!(options & kIOMemoryPageable) + && buffer + && (kInternalFlagInit & _internalFlags)) trackingAccumSize(-size); +#endif /* IOTRACKING */ + /* super::free may unwire - deallocate buffer afterwards */ super::free(); if (options & kIOMemoryPageable) { #if IOALLOCDEBUG - OSAddAtomicLong(-(round_page(size)), &debug_iomallocpageable_size); + OSAddAtomicLong(-size, &debug_iomallocpageable_size); #endif } else if (buffer) { - if (kInternalFlagPageSized & internalFlags) size = round_page(size); - if (kInternalFlagPhysical & internalFlags) { IOKernelFreePhysical((mach_vm_address_t) buffer, size); diff --git a/iokit/Kernel/IOCPU.cpp b/iokit/Kernel/IOCPU.cpp index fac8021d5..afbad6666 100644 --- a/iokit/Kernel/IOCPU.cpp +++ b/iokit/Kernel/IOCPU.cpp @@ -30,10 +30,9 @@ extern "C" { #include <machine/machine_routines.h> #include <pexpert/pexpert.h> #include <kern/cpu_number.h> +extern void kperf_kernel_configure(char *); } -#include <machine/machine_routines.h> - #include <IOKit/IOLib.h> #include <IOKit/IOPlatformExpert.h> #include <IOKit/pwr_mgt/RootDomain.h> @@ -61,13 +60,17 @@ struct iocpu_platform_action_entry const char * name; void * refcon0; void * refcon1; + boolean_t callout_in_progress; struct iocpu_platform_action_entry * alloc_list; }; typedef struct iocpu_platform_action_entry iocpu_platform_action_entry_t; /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#define kBootCPUNumber 0 +static IOLock *gIOCPUsLock; +static OSArray *gIOCPUs; +static const OSSymbol *gIOCPUStateKey; +static OSString *gIOCPUStateNames[kIOCPUStateCount]; enum { @@ -116,7 +119,7 @@ iocpu_remove_platform_action(iocpu_platform_action_entry_t * entry) static kern_return_t iocpu_run_platform_actions(queue_head_t * queue, uint32_t first_priority, uint32_t last_priority, - void * param1, void * param2, void * param3) + void * param1, void * param2, void * param3, boolean_t allow_nested_callouts) { kern_return_t ret = KERN_SUCCESS; kern_return_t result = KERN_SUCCESS; @@ -128,7 +131,16 @@ iocpu_run_platform_actions(queue_head_t * queue, uint32_t first_priority, uint32 if ((pri >= first_priority) && (pri <= last_priority)) { //kprintf("[%p]", next->action); - ret = (*next->action)(next->refcon0, next->refcon1, pri, param1, param2, param3, next->name); + if (!allow_nested_callouts && !next->callout_in_progress) + { + next->callout_in_progress = TRUE; + ret = (*next->action)(next->refcon0, next->refcon1, pri, param1, param2, param3, next->name); + next->callout_in_progress = FALSE; + } + else if (allow_nested_callouts) + { + ret = (*next->action)(next->refcon0, next->refcon1, pri, param1, param2, param3, next->name); + } } if (KERN_SUCCESS == result) result = ret; @@ -142,14 +154,14 @@ extern "C" kern_return_t IOCPURunPlatformQuiesceActions(void) { return (iocpu_run_platform_actions(&gActionQueues[kQueueQuiesce], 0, 0U-1, - NULL, NULL, NULL)); + NULL, NULL, NULL, TRUE)); } extern "C" kern_return_t IOCPURunPlatformActiveActions(void) { return (iocpu_run_platform_actions(&gActionQueues[kQueueActive], 0, 0U-1, - NULL, NULL, NULL)); + NULL, NULL, NULL, TRUE)); } extern "C" kern_return_t @@ -157,15 +169,27 @@ IOCPURunPlatformHaltRestartActions(uint32_t message) { if (!gActionQueues[kQueueHaltRestart].next) return (kIOReturnNotReady); return (iocpu_run_platform_actions(&gActionQueues[kQueueHaltRestart], 0, 0U-1, - (void *)(uintptr_t) message, NULL, NULL)); + (void *)(uintptr_t) message, NULL, NULL, TRUE)); } extern "C" kern_return_t IOCPURunPlatformPanicActions(uint32_t message) { + // Don't allow nested calls of panic actions + if (!gActionQueues[kQueuePanic].next) return (kIOReturnNotReady); + return (iocpu_run_platform_actions(&gActionQueues[kQueuePanic], 0, 0U-1, + (void *)(uintptr_t) message, NULL, NULL, FALSE)); +} + + +extern "C" kern_return_t +IOCPURunPlatformPanicSyncAction(void *addr, size_t len) +{ + // Don't allow nested calls of panic actions if (!gActionQueues[kQueuePanic].next) return (kIOReturnNotReady); return (iocpu_run_platform_actions(&gActionQueues[kQueuePanic], 0, 0U-1, - (void *)(uintptr_t) message, NULL, NULL)); + (void *)(uintptr_t)(kPEPanicSync), addr, (void *)(uintptr_t)len, FALSE)); + } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ @@ -232,6 +256,7 @@ IOInstallServicePlatformAction(IOService * service, uint32_t qidx) entry->priority = priority; entry->refcon0 = service; entry->refcon1 = (void *) key; + entry->callout_in_progress = FALSE; iocpu_add_platform_action(queue, entry); } @@ -241,11 +266,25 @@ IOInstallServicePlatformAction(IOService * service, uint32_t qidx) void IOCPUInitialize(void) { + gIOCPUsLock = IOLockAlloc(); + gIOCPUs = OSArray::withCapacity(1); + for (uint32_t qidx = kQueueSleep; qidx < kQueueCount; qidx++) { queue_init(&gActionQueues[qidx]); } + gIOCPUStateKey = OSSymbol::withCStringNoCopy("IOCPUState"); + + gIOCPUStateNames[kIOCPUStateUnregistered] = + OSString::withCStringNoCopy("Unregistered"); + gIOCPUStateNames[kIOCPUStateUninitalized] = + OSString::withCStringNoCopy("Uninitalized"); + gIOCPUStateNames[kIOCPUStateStopped] = + OSString::withCStringNoCopy("Stopped"); + gIOCPUStateNames[kIOCPUStateRunning] = + OSString::withCStringNoCopy("Running"); + gIOPlatformSleepActionKey = gActionSymbols[kQueueSleep] = OSSymbol::withCStringNoCopy(kIOPlatformSleepActionKey); gIOPlatformWakeActionKey = gActionSymbols[kQueueWake] @@ -263,9 +302,13 @@ IOCPUInitialize(void) IOReturn IOInstallServicePlatformActions(IOService * service) { + IOLockLock(gIOCPUsLock); + IOInstallServicePlatformAction(service, kQueueHaltRestart); IOInstallServicePlatformAction(service, kQueuePanic); + IOLockUnlock(gIOCPUsLock); + return (kIOReturnSuccess); } @@ -275,6 +318,8 @@ IORemoveServicePlatformActions(IOService * service) iocpu_platform_action_entry_t * entry; iocpu_platform_action_entry_t * next; + IOLockLock(gIOCPUsLock); + for (uint32_t qidx = kQueueSleep; qidx < kQueueCount; qidx++) { next = (typeof(entry)) queue_first(&gActionQueues[qidx]); @@ -290,6 +335,8 @@ IORemoveServicePlatformActions(IOService * service) } } + IOLockUnlock(gIOCPUsLock); + return (kIOReturnSuccess); } @@ -340,16 +387,49 @@ void PE_cpu_machine_init(cpu_id_t target, boolean_t bootb) { IOCPU *targetCPU = OSDynamicCast(IOCPU, (OSObject *)target); - if (targetCPU) targetCPU->initCPU(bootb); + if (targetCPU) { + targetCPU->initCPU(bootb); +#if defined(__arm__) || defined(__arm64__) + if (!bootb && (targetCPU->getCPUNumber() == (UInt32)master_cpu)) ml_set_is_quiescing(false); +#endif /* defined(__arm__) || defined(__arm64__) */ + } } void PE_cpu_machine_quiesce(cpu_id_t target) { IOCPU *targetCPU = OSDynamicCast(IOCPU, (OSObject *)target); + if (targetCPU) { +#if defined(__arm__) || defined(__arm64__) + if (targetCPU->getCPUNumber() == (UInt32)master_cpu) ml_set_is_quiescing(true); +#endif /* defined(__arm__) || defined(__arm64__) */ + targetCPU->quiesceCPU(); + } +} + +#if defined(__arm__) || defined(__arm64__) +static perfmon_interrupt_handler_func pmi_handler = 0; - if (targetCPU) targetCPU->quiesceCPU(); +kern_return_t PE_cpu_perfmon_interrupt_install_handler(perfmon_interrupt_handler_func handler) +{ + pmi_handler = handler; + + return KERN_SUCCESS; } +void PE_cpu_perfmon_interrupt_enable(cpu_id_t target, boolean_t enable) +{ + IOCPU *targetCPU = OSDynamicCast(IOCPU, (OSObject *)target); + + if (targetCPU) { + if (enable) { + targetCPU->getProvider()->registerInterrupt(1, targetCPU, (IOInterruptAction)pmi_handler, 0); + targetCPU->getProvider()->enableInterrupt(1); + } else { + targetCPU->getProvider()->disableInterrupt(1); + } + } +} +#endif /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ @@ -367,10 +447,6 @@ OSMetaClassDefineReservedUnused(IOCPU, 7); /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -static OSArray *gIOCPUs; -static const OSSymbol *gIOCPUStateKey; -static OSString *gIOCPUStateNames[kIOCPUStateCount]; - void IOCPUSleepKernel(void) { long cnt, numCPUs; @@ -415,7 +491,7 @@ void IOCPUSleepKernel(void) } iocpu_run_platform_actions(&gActionQueues[kQueueSleep], 0, 0U-1, - NULL, NULL, NULL); + NULL, NULL, NULL, TRUE); rootDomain->tracePoint( kIOPMTracePointSleepCPUs ); @@ -429,7 +505,7 @@ void IOCPUSleepKernel(void) // We make certain that the bootCPU is the last to sleep // We'll skip it for now, and halt it after finishing the // non-boot CPU's. - if (target->getCPUNumber() == kBootCPUNumber) + if (target->getCPUNumber() == (UInt32)master_cpu) { bootCPU = target; } else if (target->getCPUState() == kIOCPUStateRunning) @@ -439,7 +515,7 @@ void IOCPUSleepKernel(void) } assert(bootCPU != NULL); - assert(cpu_number() == 0); + assert(cpu_number() == master_cpu); console_suspend(); @@ -453,7 +529,7 @@ void IOCPUSleepKernel(void) console_resume(); iocpu_run_platform_actions(&gActionQueues[kQueueWake], 0, 0U-1, - NULL, NULL, NULL); + NULL, NULL, NULL, TRUE); iocpu_platform_action_entry_t * entry; for (uint32_t qidx = kQueueSleep; qidx <= kQueueActive; qidx++) @@ -472,47 +548,31 @@ void IOCPUSleepKernel(void) for (cnt = 0; cnt < numCPUs; cnt++) { target = OSDynamicCast(IOCPU, gIOCPUs->getObject(cnt)); - + // Skip the already-woken boot CPU. - if ((target->getCPUNumber() != kBootCPUNumber) - && (target->getCPUState() == kIOCPUStateStopped)) - { - processor_start(target->getMachProcessor()); + if (target->getCPUNumber() != (UInt32)master_cpu) { + if (target->getCPUState() == kIOCPUStateRunning) + panic("Spurious wakeup of cpu %u", (unsigned int)(target->getCPUNumber())); + + if (target->getCPUState() == kIOCPUStateStopped) + processor_start(target->getMachProcessor()); } } } -void IOCPU::initCPUs(void) -{ - if (gIOCPUs == 0) { - gIOCPUs = OSArray::withCapacity(1); - - gIOCPUStateKey = OSSymbol::withCStringNoCopy("IOCPUState"); - - gIOCPUStateNames[kIOCPUStateUnregistered] = - OSString::withCStringNoCopy("Unregistered"); - gIOCPUStateNames[kIOCPUStateUninitalized] = - OSString::withCStringNoCopy("Uninitalized"); - gIOCPUStateNames[kIOCPUStateStopped] = - OSString::withCStringNoCopy("Stopped"); - gIOCPUStateNames[kIOCPUStateRunning] = - OSString::withCStringNoCopy("Running"); - } -} - bool IOCPU::start(IOService *provider) { OSData *busFrequency, *cpuFrequency, *timebaseFrequency; if (!super::start(provider)) return false; - initCPUs(); - _cpuGroup = gIOCPUs; cpuNub = provider; + IOLockLock(gIOCPUsLock); gIOCPUs->setObject(this); - + IOLockUnlock(gIOCPUsLock); + // Correct the bus, cpu and timebase frequencies in the device tree. if (gPEClockFrequencyInfo.bus_frequency_hz < 0x100000000ULL) { busFrequency = OSData::withBytesNoCopy((void *)&gPEClockFrequencyInfo.bus_clock_rate_hz, 4); @@ -658,7 +718,6 @@ processor_t IOCPU::getMachProcessor(void) OSDefineMetaClassAndStructors(IOCPUInterruptController, IOInterruptController); -OSMetaClassDefineReservedUnused(IOCPUInterruptController, 0); OSMetaClassDefineReservedUnused(IOCPUInterruptController, 1); OSMetaClassDefineReservedUnused(IOCPUInterruptController, 2); OSMetaClassDefineReservedUnused(IOCPUInterruptController, 3); @@ -669,28 +728,29 @@ OSMetaClassDefineReservedUnused(IOCPUInterruptController, 5); /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - IOReturn IOCPUInterruptController::initCPUInterruptController(int sources) +{ + return initCPUInterruptController(sources, sources); +} + +IOReturn IOCPUInterruptController::initCPUInterruptController(int sources, int cpus) { int cnt; if (!super::init()) return kIOReturnInvalid; - - numCPUs = sources; - - cpus = (IOCPU **)IOMalloc(numCPUs * sizeof(IOCPU *)); - if (cpus == 0) return kIOReturnNoMemory; - bzero(cpus, numCPUs * sizeof(IOCPU *)); - - vectors = (IOInterruptVector *)IOMalloc(numCPUs * sizeof(IOInterruptVector)); + + numSources = sources; + numCPUs = cpus; + + vectors = (IOInterruptVector *)IOMalloc(numSources * sizeof(IOInterruptVector)); if (vectors == 0) return kIOReturnNoMemory; - bzero(vectors, numCPUs * sizeof(IOInterruptVector)); - - // Allocate locks for the - for (cnt = 0; cnt < numCPUs; cnt++) { + bzero(vectors, numSources * sizeof(IOInterruptVector)); + + // Allocate a lock for each vector + for (cnt = 0; cnt < numSources; cnt++) { vectors[cnt].interruptLock = IOLockAlloc(); if (vectors[cnt].interruptLock == NULL) { - for (cnt = 0; cnt < numCPUs; cnt++) { + for (cnt = 0; cnt < numSources; cnt++) { if (vectors[cnt].interruptLock != NULL) IOLockFree(vectors[cnt].interruptLock); } @@ -698,7 +758,20 @@ IOReturn IOCPUInterruptController::initCPUInterruptController(int sources) } } - ml_init_max_cpus(numCPUs); + ml_init_max_cpus(numSources); + +#if KPERF + /* + * kperf allocates based on the number of CPUs and requires them to all be + * accounted for. + */ + boolean_t found_kperf = FALSE; + char kperf_config_str[64]; + found_kperf = PE_parse_boot_arg_str("kperf", kperf_config_str, sizeof(kperf_config_str)); + if (found_kperf && kperf_config_str[0] != '\0') { + kperf_kernel_configure(kperf_config_str); + } +#endif return kIOReturnSuccess; } @@ -724,8 +797,8 @@ void IOCPUInterruptController::setCPUInterruptProperties(IOService *service) return; // Create the interrupt specifer array. - specifier = OSArray::withCapacity(numCPUs); - for (cnt = 0; cnt < numCPUs; cnt++) { + specifier = OSArray::withCapacity(numSources); + for (cnt = 0; cnt < numSources; cnt++) { tmpLong = cnt; tmpData = OSData::withBytes(&tmpLong, sizeof(tmpLong)); specifier->setObject(tmpData); @@ -733,8 +806,8 @@ void IOCPUInterruptController::setCPUInterruptProperties(IOService *service) }; // Create the interrupt controller array. - controller = OSArray::withCapacity(numCPUs); - for (cnt = 0; cnt < numCPUs; cnt++) { + controller = OSArray::withCapacity(numSources); + for (cnt = 0; cnt < numSources; cnt++) { controller->setObject(gPlatformInterruptControllerName); } @@ -750,15 +823,18 @@ void IOCPUInterruptController::enableCPUInterrupt(IOCPU *cpu) IOInterruptHandler handler = OSMemberFunctionCast( IOInterruptHandler, this, &IOCPUInterruptController::handleInterrupt); + assert(numCPUs > 0); + ml_install_interrupt_handler(cpu, cpu->getCPUNumber(), this, handler, 0); - // Ensure that the increment is seen by all processors - OSIncrementAtomic(&enabledCPUs); + IOTakeLock(vectors[0].interruptLock); + ++enabledCPUs; if (enabledCPUs == numCPUs) { - IOService::cpusRunning(); - thread_wakeup(this); - } + IOService::cpusRunning(); + thread_wakeup(this); + } + IOUnlock(vectors[0].interruptLock); } IOReturn IOCPUInterruptController::registerInterrupt(IOService *nub, @@ -768,39 +844,42 @@ IOReturn IOCPUInterruptController::registerInterrupt(IOService *nub, void *refCon) { IOInterruptVector *vector; - - if (source >= numCPUs) return kIOReturnNoResources; - + + if (source >= numSources) return kIOReturnNoResources; + vector = &vectors[source]; - + // Get the lock for this vector. IOTakeLock(vector->interruptLock); - + // Make sure the vector is not in use. if (vector->interruptRegistered) { IOUnlock(vector->interruptLock); return kIOReturnNoResources; } - + // Fill in vector with the client's info. vector->handler = handler; vector->nub = nub; vector->source = source; vector->target = target; vector->refCon = refCon; - + // Get the vector ready. It starts hard disabled. vector->interruptDisabledHard = 1; vector->interruptDisabledSoft = 1; vector->interruptRegistered = 1; - + IOUnlock(vector->interruptLock); - + + IOTakeLock(vectors[0].interruptLock); if (enabledCPUs != numCPUs) { assert_wait(this, THREAD_UNINT); + IOUnlock(vectors[0].interruptLock); thread_block(THREAD_CONTINUE_NULL); - } - + } else + IOUnlock(vectors[0].interruptLock); + return kIOReturnSuccess; } diff --git a/iokit/Kernel/IOCommandGate.cpp b/iokit/Kernel/IOCommandGate.cpp index 2308daf1b..6c1f45767 100644 --- a/iokit/Kernel/IOCommandGate.cpp +++ b/iokit/Kernel/IOCommandGate.cpp @@ -207,7 +207,7 @@ IOReturn IOCommandGate::runAction(Action inAction, bool trace = ( gIOKitTrace & kIOTraceCommandGates ) ? true : false; if (trace) IOTimeStampStartConstant(IODBG_CMDQ(IOCMDQ_ACTION), - VM_KERNEL_UNSLIDE(inAction), (uintptr_t) owner); + VM_KERNEL_ADDRHIDE(inAction), VM_KERNEL_ADDRHIDE(owner)); IOStatisticsActionCall(); @@ -218,7 +218,7 @@ IOReturn IOCommandGate::runAction(Action inAction, *sleepersP -= kSleepersActions; if (trace) IOTimeStampEndConstant(IODBG_CMDQ(IOCMDQ_ACTION), - VM_KERNEL_UNSLIDE(inAction), (uintptr_t) owner); + VM_KERNEL_ADDRHIDE(inAction), VM_KERNEL_ADDRHIDE(owner)); if (kSleepersRemoved == ((kSleepersActionsMask|kSleepersRemoved) & *sleepersP)) { @@ -257,7 +257,7 @@ IOReturn IOCommandGate::attemptAction(Action inAction, if (trace) IOTimeStampStartConstant(IODBG_CMDQ(IOCMDQ_ACTION), - VM_KERNEL_UNSLIDE(inAction), (uintptr_t) owner); + VM_KERNEL_ADDRHIDE(inAction), VM_KERNEL_ADDRHIDE(owner)); IOStatisticsActionCall(); @@ -265,7 +265,7 @@ IOReturn IOCommandGate::attemptAction(Action inAction, if (trace) IOTimeStampEndConstant(IODBG_CMDQ(IOCMDQ_ACTION), - VM_KERNEL_UNSLIDE(inAction), (uintptr_t) owner); + VM_KERNEL_ADDRHIDE(inAction), VM_KERNEL_ADDRHIDE(owner)); } wl->openGate(); diff --git a/iokit/Kernel/IOCommandQueue.cpp b/iokit/Kernel/IOCommandQueue.cpp index 3a184bf94..1b4f3ce23 100644 --- a/iokit/Kernel/IOCommandQueue.cpp +++ b/iokit/Kernel/IOCommandQueue.cpp @@ -166,14 +166,14 @@ bool IOCommandQueue::checkForWork() if (trace) IOTimeStampStartConstant(IODBG_CMDQ(IOCMDQ_ACTION), - (uintptr_t) action, (uintptr_t) owner); + VM_KERNEL_ADDRHIDE(action), VM_KERNEL_ADDRHIDE(owner)); IOStatisticsActionCall(); (*(IOCommandQueueAction) action)(owner, field0, field1, field2, field3); if (trace) IOTimeStampEndConstant(IODBG_CMDQ(IOCMDQ_ACTION), - (uintptr_t) action, (uintptr_t) owner); + VM_KERNEL_ADDRHIDE(action), VM_KERNEL_ADDRHIDE(owner)); return (consumerIndex != producerIndex); } diff --git a/iokit/Kernel/IODMACommand.cpp b/iokit/Kernel/IODMACommand.cpp index 936ae0879..5feadeb14 100644 --- a/iokit/Kernel/IODMACommand.cpp +++ b/iokit/Kernel/IODMACommand.cpp @@ -379,15 +379,7 @@ IODMACommand::setMemoryDescriptor(const IOMemoryDescriptor *mem, bool autoPrepar fInternalState->fNewMD = true; mem->retain(); fMemory = mem; - if (fMapper) - { -#if IOTRACKING - fInternalState->fTag = IOMemoryTag(kernel_map); - __IODEQUALIFY(IOMemoryDescriptor *, mem)->prepare((IODirection) - (kIODirectionDMACommand | (fInternalState->fTag << kIODirectionDMACommandShift))); - IOTrackingAdd(gIOWireTracking, &fInternalState->fWireTracking, fMemory->getLength(), false); -#endif /* IOTRACKING */ - } + if (!fMapper) mem->dmaCommandOperation(kIOMDSetDMAActive, this, 0); if (autoPrepare) { err = prepare(); if (err) { @@ -407,14 +399,7 @@ IODMACommand::clearMemoryDescriptor(bool autoComplete) if (fMemory) { while (fActive) complete(); - if (fMapper) - { -#if IOTRACKING - __IODEQUALIFY(IOMemoryDescriptor *, fMemory)->complete((IODirection) - (kIODirectionDMACommand | (fInternalState->fTag << kIODirectionDMACommandShift))); - IOTrackingRemove(gIOWireTracking, &fInternalState->fWireTracking, fMemory->getLength()); -#endif /* IOTRACKING */ - } + if (!fMapper) fMemory->dmaCommandOperation(kIOMDSetDMAInactive, this, 0); fMemory->release(); fMemory = 0; } @@ -455,7 +440,7 @@ IODMACommand::segmentOp( IODMACommandInternal * state = target->reserved; - if (target->fNumAddressBits && (target->fNumAddressBits < 64) && (state->fLocalMapperAlloc || !target->fMapper)) + if (target->fNumAddressBits && (target->fNumAddressBits < 64) && (state->fLocalMapperAllocValid || !target->fMapper)) maxPhys = (1ULL << target->fNumAddressBits); else maxPhys = 0; @@ -464,7 +449,6 @@ IODMACommand::segmentOp( address = segment.fIOVMAddr; length = segment.fLength; - assert(address); assert(length); if (!state->fMisaligned) @@ -610,7 +594,8 @@ IODMACommand::walkAll(UInt8 op) op &= ~kWalkPreflight; - state->fDoubleBuffer = (state->fMisaligned || (kWalkDoubleBuffer & op)); + state->fDoubleBuffer = (state->fMisaligned || state->fForceDoubleBuffer); + state->fForceDoubleBuffer = false; if (state->fDoubleBuffer) state->fCopyPageCount = atop_64(round_page(state->fPreparedLength)); @@ -835,6 +820,7 @@ IODMACommand::prepare(UInt64 offset, UInt64 length, bool flushCache, bool synchr state->fNextRemapPage = NULL; state->fCopyMD = 0; state->fLocalMapperAlloc = 0; + state->fLocalMapperAllocValid = false; state->fLocalMapperAllocLength = 0; state->fLocalMapper = (fMapper && (fMapper != IOMapper::gSystem)); @@ -880,8 +866,7 @@ IODMACommand::prepare(UInt64 offset, UInt64 length, bool flushCache, bool synchr mapArgs.fMapSpec.numAddressBits = fNumAddressBits ? fNumAddressBits : 64; mapArgs.fLength = state->fPreparedLength; const IOMemoryDescriptor * md = state->fCopyMD; - if (md) { mapArgs.fOffset = 0; } - else + if (md) { mapArgs.fOffset = 0; } else { md = fMemory; mapArgs.fOffset = state->fPreparedOffset; @@ -892,6 +877,7 @@ IODMACommand::prepare(UInt64 offset, UInt64 length, bool flushCache, bool synchr if (kIOReturnSuccess == ret) { state->fLocalMapperAlloc = mapArgs.fAlloc; + state->fLocalMapperAllocValid = true; state->fLocalMapperAllocLength = mapArgs.fAllocLength; state->fMapContig = mapArgs.fMapContig; } @@ -907,17 +893,21 @@ IODMACommand::complete(bool invalidateCache, bool synchronize) { IODMACommandInternal * state = fInternalState; IOReturn ret = kIOReturnSuccess; + IOMemoryDescriptor * copyMD; if (fActive < 1) return kIOReturnNotReady; if (!--fActive) { + copyMD = state->fCopyMD; + if (copyMD) copyMD->retain(); + if (IS_NONCOHERENT(fMappingOptions) && invalidateCache) { - if (state->fCopyMD) + if (copyMD) { - state->fCopyMD->performOperation(kIOMemoryIncoherentIOFlush, 0, state->fPreparedLength); + copyMD->performOperation(kIOMemoryIncoherentIOFlush, 0, state->fPreparedLength); } else { @@ -933,17 +923,30 @@ IODMACommand::complete(bool invalidateCache, bool synchronize) op |= kWalkSyncIn; ret = walkAll(op); } - if (state->fLocalMapperAlloc) + + if (state->fLocalMapperAllocValid) { - if (state->fLocalMapperAllocLength) + IOMDDMAMapArgs mapArgs; + bzero(&mapArgs, sizeof(mapArgs)); + mapArgs.fMapper = fMapper; + mapArgs.fCommand = this; + mapArgs.fAlloc = state->fLocalMapperAlloc; + mapArgs.fAllocLength = state->fLocalMapperAllocLength; + const IOMemoryDescriptor * md = copyMD; + if (md) { mapArgs.fOffset = 0; } + else { - fMapper->iovmUnmapMemory(getIOMemoryDescriptor(), this, - state->fLocalMapperAlloc, state->fLocalMapperAllocLength); + md = fMemory; + mapArgs.fOffset = state->fPreparedOffset; } + + ret = md->dmaCommandOperation(kIOMDDMAUnmap, &mapArgs, sizeof(mapArgs)); + state->fLocalMapperAlloc = 0; + state->fLocalMapperAllocValid = false; state->fLocalMapperAllocLength = 0; } - + if (copyMD) copyMD->release(); state->fPrepared = false; } @@ -981,14 +984,13 @@ IODMACommand::synchronize(IOOptionBits options) op = 0; if (kForceDoubleBuffer & options) { - if (state->fDoubleBuffer) - return kIOReturnSuccess; - if (state->fCursor) - state->fCursor = false; - else - ret = walkAll(kWalkComplete); + if (state->fDoubleBuffer) return kIOReturnSuccess; + ret = complete(false /* invalidateCache */, true /* synchronize */); + state->fCursor = false; + state->fForceDoubleBuffer = true; + ret = prepare(state->fPreparedOffset, state->fPreparedLength, false /* flushCache */, true /* synchronize */); - op |= kWalkPrepare | kWalkPreflight | kWalkDoubleBuffer; + return (ret); } else if (state->fCursor) return kIOReturnSuccess; @@ -1132,17 +1134,18 @@ IODMACommand::genIOVMSegments(uint32_t op, return kIOReturnOverrun; if ((offset == internalState->fPreparedOffset) || (offset != state->fOffset) || internalState->fNewMD) { - state->fOffset = 0; - state->fIOVMAddr = 0; - internalState->fNextRemapPage = NULL; - internalState->fNewMD = false; - state->fMapped = (0 != fMapper); - mdOp = kIOMDFirstSegment; + state->fOffset = 0; + internalState->fIOVMAddrValid = state->fIOVMAddr = 0; + internalState->fNextRemapPage = NULL; + internalState->fNewMD = false; + state->fMapped = (0 != fMapper); + mdOp = kIOMDFirstSegment; }; UInt32 segIndex = 0; UInt32 numSegments = *numSegmentsP; Segment64 curSeg = { 0, 0 }; + bool curSegValid = false; addr64_t maxPhys; if (fNumAddressBits && (fNumAddressBits < 64)) @@ -1151,17 +1154,17 @@ IODMACommand::genIOVMSegments(uint32_t op, maxPhys = 0; maxPhys--; - while (state->fIOVMAddr || (state->fOffset < memLength)) + while (internalState->fIOVMAddrValid || (state->fOffset < memLength)) { // state = next seg - if (!state->fIOVMAddr) { + if (!internalState->fIOVMAddrValid) { IOReturn rtn; state->fOffset = offset; state->fLength = memLength - offset; - if (internalState->fMapContig && internalState->fLocalMapperAlloc) + if (internalState->fMapContig && internalState->fLocalMapperAllocValid) { state->fIOVMAddr = internalState->fLocalMapperAlloc + offset - internalState->fPreparedOffset; rtn = kIOReturnSuccess; @@ -1193,39 +1196,40 @@ IODMACommand::genIOVMSegments(uint32_t op, if (rtn == kIOReturnSuccess) { - assert(state->fIOVMAddr); + internalState->fIOVMAddrValid = true; assert(state->fLength); - if ((curSeg.fIOVMAddr + curSeg.fLength) == state->fIOVMAddr) { + if (curSegValid && ((curSeg.fIOVMAddr + curSeg.fLength) == state->fIOVMAddr)) { UInt64 length = state->fLength; offset += length; curSeg.fLength += length; - state->fIOVMAddr = 0; + internalState->fIOVMAddrValid = state->fIOVMAddr = 0; } } else if (rtn == kIOReturnOverrun) - state->fIOVMAddr = state->fLength = 0; // At end + internalState->fIOVMAddrValid = state->fIOVMAddr = state->fLength = 0; // At end else return rtn; } // seg = state, offset = end of seg - if (!curSeg.fIOVMAddr) + if (!curSegValid) { - UInt64 length = state->fLength; - offset += length; - curSeg.fIOVMAddr = state->fIOVMAddr; - curSeg.fLength = length; - state->fIOVMAddr = 0; + UInt64 length = state->fLength; + offset += length; + curSeg.fIOVMAddr = state->fIOVMAddr; + curSeg.fLength = length; + curSegValid = true; + internalState->fIOVMAddrValid = state->fIOVMAddr = 0; } - if (!state->fIOVMAddr) + if (!internalState->fIOVMAddrValid) { // maxPhys if ((kWalkClient & op) && (curSeg.fIOVMAddr + curSeg.fLength - 1) > maxPhys) { if (internalState->fCursor) { - curSeg.fIOVMAddr = 0; + curSegValid = curSeg.fIOVMAddr = 0; ret = kIOReturnMessageTooLarge; break; } @@ -1237,6 +1241,7 @@ IODMACommand::genIOVMSegments(uint32_t op, DEBG("trunc %qx, %qx-> %qx\n", curSeg.fIOVMAddr, curSeg.fLength, newLength); remain = curSeg.fLength - newLength; state->fIOVMAddr = newLength + curSeg.fIOVMAddr; + internalState->fIOVMAddrValid = true; curSeg.fLength = newLength; state->fLength = remain; offset -= remain; @@ -1264,6 +1269,7 @@ IODMACommand::genIOVMSegments(uint32_t op, curSeg.fIOVMAddr = ptoa_64(vm_page_get_phys_page(remap)) + (addr & PAGE_MASK); + curSegValid = true; internalState->fNextRemapPage = vm_page_get_next(remap); newLength = PAGE_SIZE - (addr & PAGE_MASK); @@ -1271,6 +1277,7 @@ IODMACommand::genIOVMSegments(uint32_t op, { remain = curSeg.fLength - newLength; state->fIOVMAddr = addr + newLength; + internalState->fIOVMAddrValid = true; curSeg.fLength = newLength; state->fLength = remain; offset -= remain; @@ -1288,6 +1295,7 @@ IODMACommand::genIOVMSegments(uint32_t op, leftover += curSeg.fLength - fMaxSegmentSize; curSeg.fLength = fMaxSegmentSize; state->fIOVMAddr = curSeg.fLength + curSeg.fIOVMAddr; + internalState->fIOVMAddrValid = true; } // alignment current length @@ -1298,6 +1306,7 @@ IODMACommand::genIOVMSegments(uint32_t op, leftover += reduce; curSeg.fLength -= reduce; state->fIOVMAddr = curSeg.fLength + curSeg.fIOVMAddr; + internalState->fIOVMAddrValid = true; } // alignment next address @@ -1308,6 +1317,7 @@ IODMACommand::genIOVMSegments(uint32_t op, leftover += reduce; curSeg.fLength -= reduce; state->fIOVMAddr = curSeg.fLength + curSeg.fIOVMAddr; + internalState->fIOVMAddrValid = true; } if (leftover) @@ -1336,7 +1346,7 @@ IODMACommand::genIOVMSegments(uint32_t op, if (misaligned) { if (misaligned) DEBG("cursor misaligned %qx:%qx\n", curSeg.fIOVMAddr, curSeg.fLength); - curSeg.fIOVMAddr = 0; + curSegValid = curSeg.fIOVMAddr = 0; ret = kIOReturnNotAligned; break; } @@ -1346,23 +1356,23 @@ IODMACommand::genIOVMSegments(uint32_t op, { curSeg.fLength -= (offset - memLength); offset = memLength; - state->fIOVMAddr = state->fLength = 0; // At end + internalState->fIOVMAddrValid = state->fIOVMAddr = state->fLength = 0; // At end break; } } - if (state->fIOVMAddr) { + if (internalState->fIOVMAddrValid) { if ((segIndex + 1 == numSegments)) break; ret = (*outSegFunc)(reference, this, curSeg, segmentsP, segIndex++); - curSeg.fIOVMAddr = 0; + curSegValid = curSeg.fIOVMAddr = 0; if (kIOReturnSuccess != ret) break; } } - if (curSeg.fIOVMAddr) { + if (curSegValid) { ret = (*outSegFunc)(reference, this, curSeg, segmentsP, segIndex++); } @@ -1385,7 +1395,7 @@ IODMACommand::clientOutputSegment( if (target->fNumAddressBits && (target->fNumAddressBits < 64) && ((segment.fIOVMAddr + segment.fLength - 1) >> target->fNumAddressBits) - && (target->reserved->fLocalMapperAlloc || !target->fMapper)) + && (target->reserved->fLocalMapperAllocValid || !target->fMapper)) { DEBG("kIOReturnMessageTooLarge(fNumAddressBits) %qx, %qx\n", segment.fIOVMAddr, segment.fLength); ret = kIOReturnMessageTooLarge; diff --git a/iokit/Kernel/IODeviceTreeSupport.cpp b/iokit/Kernel/IODeviceTreeSupport.cpp index e115584ce..1a3b426d8 100644 --- a/iokit/Kernel/IODeviceTreeSupport.cpp +++ b/iokit/Kernel/IODeviceTreeSupport.cpp @@ -37,7 +37,11 @@ #include <pexpert/device_tree.h> +#if __arm64__ +typedef UInt64 dtptr_t; +#else typedef UInt32 dtptr_t; +#endif #include <machine/machine_routines.h> @@ -55,8 +59,9 @@ int IODTGetDefault(const char *key, void *infoAddr, unsigned int infoSize ); const IORegistryPlane * gIODTPlane; -static OSArray * gIODTPHandles; -static OSArray * gIODTPHandleMap; +static OSArray * gIODTPHandles; +static OSArray * gIODTPHandleMap; +static OSData * gIODTResolvers; const OSSymbol * gIODTNameKey; const OSSymbol * gIODTUnitKey; @@ -80,6 +85,8 @@ const OSSymbol * gIODTNWInterruptMappingKey; OSDictionary * gIODTSharedInterrupts; +static IOLock * gIODTResolversLock; + static IORegistryEntry * MakeReferenceTable( DTEntry dtEntry, bool copy ); static void AddPHandle( IORegistryEntry * regEntry ); static void FreePhysicalMemory( vm_offset_t * range ); @@ -91,7 +98,7 @@ IODeviceTreeAlloc( void * dtTop ) IORegistryEntry * parent; IORegistryEntry * child; IORegistryIterator * regIter; - DTEntryIterator iter; + OpaqueDTEntryIterator iter; DTEntry dtChild; DTEntry mapEntry; OSArray * stack; @@ -135,6 +142,9 @@ IODeviceTreeAlloc( void * dtTop ) gIODTPHandles = OSArray::withCapacity( 1 ); gIODTPHandleMap = OSArray::withCapacity( 1 ); + gIODTResolvers = OSData::withCapacity(16); + + gIODTResolversLock = IOLockAlloc(); gIODTInterruptCellKey = OSSymbol::withCStringNoCopy("#interrupt-cells"); @@ -142,7 +152,7 @@ IODeviceTreeAlloc( void * dtTop ) assert( gIODTDefaultInterruptController && gIODTNWInterruptMappingKey && gIODTAAPLInterruptsKey && gIODTPHandleKey && gIODTInterruptParentKey - && gIODTPHandles && gIODTPHandleMap + && gIODTPHandles && gIODTPHandleMap && gIODTResolvers && gIODTResolversLock && gIODTInterruptCellKey ); @@ -154,21 +164,21 @@ IODeviceTreeAlloc( void * dtTop ) parent = MakeReferenceTable( (DTEntry)dtTop, freeDT ); stack = OSArray::withObjects( (const OSObject **) &parent, 1, 10 ); - DTCreateEntryIterator( (DTEntry)dtTop, &iter ); + DTInitEntryIterator( (DTEntry)dtTop, &iter ); do { parent = (IORegistryEntry *)stack->getObject( stack->getCount() - 1); //parent->release(); stack->removeObject( stack->getCount() - 1); - while( kSuccess == DTIterateEntries( iter, &dtChild) ) { + while( kSuccess == DTIterateEntries( &iter, &dtChild) ) { child = MakeReferenceTable( dtChild, freeDT ); child->attachToParent( parent, gIODTPlane); AddPHandle( child ); - if( kSuccess == DTEnterEntry( iter, dtChild)) { + if( kSuccess == DTEnterEntry( &iter, dtChild)) { stack->setObject( parent); parent = child; } @@ -177,10 +187,10 @@ IODeviceTreeAlloc( void * dtTop ) } } while( stack->getCount() - && (kSuccess == DTExitEntry( iter, &dtChild))); + && (kSuccess == DTExitEntry( &iter, &dtChild))); stack->release(); - DTDisposeEntryIterator( iter); + assert(kSuccess != DTExitEntry(&iter, &dtChild)); // parent is now root of the created tree @@ -338,7 +348,7 @@ MakeReferenceTable( DTEntry dtEntry, bool copy ) const OSSymbol *nameKey; OSData *data; const OSSymbol *sym; - DTPropertyIterator dtIter; + OpaqueDTPropertyIterator dtIter; void *prop; unsigned int propSize; char *name; @@ -354,12 +364,12 @@ MakeReferenceTable( DTEntry dtEntry, bool copy ) } if( regEntry && - (kSuccess == DTCreatePropertyIterator( dtEntry, &dtIter))) { + (kSuccess == DTInitPropertyIterator( dtEntry, &dtIter))) { kernelOnly = (kSuccess == DTGetProperty(dtEntry, "kernel-only", &prop, &propSize)); propTable = regEntry->getPropertyTable(); - while( kSuccess == DTIterateProperties( dtIter, &name)) { + while( kSuccess == DTIterateProperties( &dtIter, &name)) { if( kSuccess != DTGetProperty( dtEntry, name, &prop, &propSize )) continue; @@ -404,7 +414,6 @@ MakeReferenceTable( DTEntry dtEntry, bool copy ) regEntry->setLocation( location ); } } - DTDisposePropertyIterator( dtIter); } return( regEntry); @@ -926,29 +935,57 @@ OSCollectionIterator * IODTFindMatchingEntries( IORegistryEntry * from, struct IODTPersistent { IODTCompareAddressCellFunc compareFunc; - IODTNVLocationFunc locationFunc; }; void IODTSetResolving( IORegistryEntry * regEntry, IODTCompareAddressCellFunc compareFunc, - IODTNVLocationFunc locationFunc ) + IODTNVLocationFunc locationFunc __unused ) { - IODTPersistent persist; - OSData *prop; - - persist.compareFunc = compareFunc; - persist.locationFunc = locationFunc; - prop = OSData::withBytes( &persist, sizeof(persist)); - if( !prop) - return; - - prop->setSerializable(false); - regEntry->setProperty( gIODTPersistKey, prop); - prop->release(); + IODTPersistent persist; + IODTPersistent * entry; + OSNumber * num; + unsigned int index, count; + + IOLockLock(gIODTResolversLock); + + count = (gIODTResolvers->getLength() / sizeof(IODTPersistent)); + entry = (typeof(entry)) gIODTResolvers->getBytesNoCopy(); + for (index = 0; index < count; index++) + { + if (compareFunc == entry->compareFunc) break; + entry++; + } + if (index == count) + { + persist.compareFunc = compareFunc; + if (!gIODTResolvers->appendBytes(&persist, sizeof(IODTPersistent))) panic("IODTSetResolving"); + } + + IOLockUnlock(gIODTResolversLock); + + num = OSNumber::withNumber(index, 32); + regEntry->setProperty(gIODTPersistKey, num); + OSSafeReleaseNULL(num); + return; } -#if defined(__arm__) || defined(__i386__) || defined(__x86_64__) +#if defined(__arm64__) +static SInt64 DefaultCompare( UInt32 cellCount, UInt32 left[], UInt32 right[] ) +{ + SInt64 diff = 0; + + if (cellCount == 2) { + diff = IOPhysical32(left[1], left[0]) - IOPhysical32(right[1], right[0]); + } else if (cellCount == 1) { + diff = ( left[0] - right[0] ); + } else { + panic("DefaultCompare only knows how to handle 1 or 2 cells."); + } + + return diff; +} +#elif defined(__arm__) || defined(__i386__) || defined(__x86_64__) static SInt32 DefaultCompare( UInt32 cellCount, UInt32 left[], UInt32 right[] ) { cellCount--; @@ -965,11 +1002,19 @@ static void AddLengthToCells( UInt32 numCells, UInt32 *cells, UInt64 offset) cells[0] += (UInt32)offset; } else { +#if defined(__arm64__) || defined(__arm__) + UInt64 sum = cells[numCells - 2] + offset; + cells[numCells - 2] = (UInt32)sum; + if (sum > UINT32_MAX) { + cells[numCells - 1] += (UInt32)(sum >> 32); + } +#else UInt64 sum = cells[numCells - 1] + offset; cells[numCells - 1] = (UInt32)sum; if (sum > UINT32_MAX) { cells[numCells - 2] += (UInt32)(sum >> 32); } +#endif } } @@ -978,7 +1023,11 @@ static IOPhysicalAddress CellsValue( UInt32 numCells, UInt32 *cells) if (numCells == 1) { return IOPhysical32( 0, cells[0] ); } else { +#if defined(__arm64__) || defined(arm) + return IOPhysical32( cells[numCells - 1], cells[numCells - 2] ); +#else return IOPhysical32( cells[numCells - 2], cells[numCells - 1] ); +#endif } } @@ -998,12 +1047,15 @@ void IODTGetCellCounts( IORegistryEntry * regEntry, // Range[]: child-addr our-addr child-len // #cells: child ours child -bool IODTResolveAddressCell( IORegistryEntry * regEntry, +bool IODTResolveAddressCell( IORegistryEntry * startEntry, UInt32 cellsIn[], IOPhysicalAddress * phys, IOPhysicalLength * lenOut ) { - IORegistryEntry *parent; - OSData *prop; + IORegistryEntry * parent; + IORegistryEntry * regEntry; + OSData * prop; + OSNumber * num; + unsigned int index, count; // cells in addresses at regEntry UInt32 sizeCells, addressCells; // cells in addresses below regEntry @@ -1023,6 +1075,7 @@ bool IODTResolveAddressCell( IORegistryEntry * regEntry, IODTPersistent *persist; IODTCompareAddressCellFunc compare; + regEntry = startEntry; IODTGetCellCounts( regEntry, &childSizeCells, &childAddressCells ); childCells = childAddressCells + childSizeCells; @@ -1039,10 +1092,11 @@ bool IODTResolveAddressCell( IORegistryEntry * regEntry, /* end of the road */ *phys = CellsValue( childAddressCells, cell ); *phys += offset; + if (regEntry != startEntry) regEntry->release(); break; } - parent = regEntry->getParentEntry( gIODTPlane ); + parent = regEntry->copyParentEntry( gIODTPlane ); IODTGetCellCounts( parent, &sizeCells, &addressCells ); if( (propLen = prop->getLength())) { @@ -1051,13 +1105,25 @@ bool IODTResolveAddressCell( IORegistryEntry * regEntry, range = startRange; endRanges = range + (propLen / sizeof(UInt32)); - prop = (OSData *) regEntry->getProperty( gIODTPersistKey ); - if( prop) { - persist = (IODTPersistent *) prop->getBytesNoCopy(); - compare = persist->compareFunc; - } else if (addressCells == childAddressCells) { + compare = NULL; + num = OSDynamicCast(OSNumber, regEntry->getProperty(gIODTPersistKey)); + if (num) + { + IOLockLock(gIODTResolversLock); + index = num->unsigned32BitValue(); + count = gIODTResolvers->getLength() / sizeof(IODTPersistent); + if (index < count) + { + persist = ((IODTPersistent *) gIODTResolvers->getBytesNoCopy()) + index; + compare = persist->compareFunc; + } + IOLockUnlock(gIODTResolversLock); + } + + if (!compare && (addressCells == childAddressCells)) { compare = DefaultCompare; - } else { + } + if (!compare) { panic("There is no mixed comparison function yet..."); } @@ -1125,6 +1191,7 @@ bool IODTResolveAddressCell( IORegistryEntry * regEntry, } /* else zero length range => pass thru to parent */ + if (regEntry != startEntry) regEntry->release(); regEntry = parent; childSizeCells = sizeCells; childAddressCells = addressCells; @@ -1150,107 +1217,45 @@ OSArray * IODTResolveAddressing( IORegistryEntry * regEntry, OSArray *array; IODeviceMemory *range; - parentEntry = regEntry->getParentEntry( gIODTPlane ); - addressProperty = (OSData *) regEntry->getProperty( addressPropertyName ); - if( (0 == addressProperty) || (0 == parentEntry)) - return( 0); - - IODTGetCellCounts( parentEntry, &sizeCells, &addressCells ); - if( 0 == sizeCells) - return( 0); - - cells = sizeCells + addressCells; - reg = (UInt32 *) addressProperty->getBytesNoCopy(); - num = addressProperty->getLength() / (4 * cells); - - array = OSArray::withCapacity( 1 ); - if( 0 == array) - return( 0); - - for( i = 0; i < num; i++) { - if( IODTResolveAddressCell( parentEntry, reg, &phys, &len )) { - range = 0; - if( parent) - range = IODeviceMemory::withSubRange( parent, - phys - parent->getPhysicalSegment(0, 0, kIOMemoryMapperNone), len ); - if( 0 == range) - range = IODeviceMemory::withRange( phys, len ); - if( range) - array->setObject( range ); + array = 0; + do + { + parentEntry = regEntry->copyParentEntry( gIODTPlane ); + addressProperty = (OSData *) regEntry->getProperty( addressPropertyName ); + if( (0 == addressProperty) || (0 == parentEntry)) break; + + IODTGetCellCounts( parentEntry, &sizeCells, &addressCells ); + if( 0 == sizeCells) break; + + cells = sizeCells + addressCells; + reg = (UInt32 *) addressProperty->getBytesNoCopy(); + num = addressProperty->getLength() / (4 * cells); + + array = OSArray::withCapacity( 1 ); + if( 0 == array) break; + + for( i = 0; i < num; i++) { + if( IODTResolveAddressCell( parentEntry, reg, &phys, &len )) { + range = 0; + if( parent) + range = IODeviceMemory::withSubRange( parent, + phys - parent->getPhysicalSegment(0, 0, kIOMemoryMapperNone), len ); + if( 0 == range) + range = IODeviceMemory::withRange( phys, len ); + if( range) + array->setObject( range ); + } + reg += cells; } - reg += cells; - } - - regEntry->setProperty( gIODeviceMemoryKey, array); - array->release(); /* ??? */ - - return( array); -} - -static void IODTGetNVLocation( - IORegistryEntry * parent, - IORegistryEntry * regEntry, - UInt8 * busNum, UInt8 * deviceNum, UInt8 * functionNum ) -{ - OSData *prop; - IODTPersistent *persist; - UInt32 *cell; - - prop = (OSData *) parent->getProperty( gIODTPersistKey ); - if( prop) { - persist = (IODTPersistent *) prop->getBytesNoCopy(); - (*persist->locationFunc)( regEntry, busNum, deviceNum, functionNum ); - } else { - prop = (OSData *) regEntry->getProperty( "reg" ); - *functionNum = 0; - if( prop) { - cell = (UInt32 *) prop->getBytesNoCopy(); - *busNum = 3; - *deviceNum = 0x1f & (cell[ 0 ] >> 24); - } else { - *busNum = 0; - *deviceNum = 0; - } + regEntry->setProperty( gIODeviceMemoryKey, array); + array->release(); /* ??? */ } - return; -} + while (false); -/* - * Try to make the same messed up descriptor as Mac OS - */ - -IOReturn IODTMakeNVDescriptor( IORegistryEntry * regEntry, - IONVRAMDescriptor * hdr ) -{ - IORegistryEntry *parent; - UInt32 level; - UInt32 bridgeDevices; - UInt8 busNum; - UInt8 deviceNum; - UInt8 functionNum; - - hdr->format = 1; - hdr->marker = 0; - - for(level = 0, bridgeDevices = 0; - (parent = regEntry->getParentEntry( gIODTPlane )) && (level < 7); level++ ) { - - IODTGetNVLocation( parent, regEntry, - &busNum, &deviceNum, &functionNum ); - if( level) - bridgeDevices |= ((deviceNum & 0x1f) << ((level - 1) * 5)); - else { - hdr->busNum = busNum; - hdr->deviceNum = deviceNum; - hdr->functionNum = functionNum; - } - regEntry = parent; - } - hdr->bridgeCount = level - 2; - hdr->bridgeDevices = bridgeDevices; + OSSafeReleaseNULL(parentEntry); - return( kIOReturnSuccess ); + return (array); } OSData * IODTFindSlotName( IORegistryEntry * regEntry, UInt32 deviceNumber ) @@ -1266,40 +1271,43 @@ OSData * IODTFindSlotName( IORegistryEntry * regEntry, UInt32 deviceNumber ) UInt32 mask; data = (OSData *) regEntry->getProperty("AAPL,slot-name"); - if( data) - return( data); - parent = regEntry->getParentEntry( gIODTPlane ); - if( !parent) - return( 0 ); - data = OSDynamicCast( OSData, parent->getProperty("slot-names")); - if( !data) - return( 0 ); - if( data->getLength() <= 4) - return( 0 ); - - bits = (UInt32 *) data->getBytesNoCopy(); - mask = *bits; - if( (0 == (mask & (1 << deviceNumber)))) - return( 0 ); - - names = (char *)(bits + 1); - lastName = names + (data->getLength() - 4); - - for( i = 0; (i <= deviceNumber) && (names < lastName); i++ ) { - - if( mask & (1 << i)) { - nlen = 1 + strnlen(names, lastName - names); - if( i == deviceNumber) { - data = OSData::withBytesNoCopy(names, nlen); - if( data) { - regEntry->setProperty("AAPL,slot-name", data); - ret = data; - data->release(); - } - } else - names += nlen; + if (data) return (data); + + do + { + parent = regEntry->copyParentEntry( gIODTPlane ); + if (!parent) break; + + data = OSDynamicCast( OSData, parent->getProperty("slot-names")); + if (!data) break; + if (data->getLength() <= 4) break; + + bits = (UInt32 *) data->getBytesNoCopy(); + mask = *bits; + if ((0 == (mask & (1 << deviceNumber)))) break; + + names = (char *)(bits + 1); + lastName = names + (data->getLength() - 4); + + for( i = 0; (i <= deviceNumber) && (names < lastName); i++ ) { + + if( mask & (1 << i)) { + nlen = 1 + strnlen(names, lastName - names); + if( i == deviceNumber) { + data = OSData::withBytesNoCopy(names, nlen); + if( data) { + regEntry->setProperty("AAPL,slot-name", data); + ret = data; + data->release(); + } + } else + names += nlen; + } } } + while (false); + + OSSafeReleaseNULL(parent); return( ret ); } diff --git a/iokit/Kernel/IOFilterInterruptEventSource.cpp b/iokit/Kernel/IOFilterInterruptEventSource.cpp index 83b6ed6e0..c6f79e91d 100644 --- a/iokit/Kernel/IOFilterInterruptEventSource.cpp +++ b/iokit/Kernel/IOFilterInterruptEventSource.cpp @@ -131,12 +131,12 @@ void IOFilterInterruptEventSource::signalInterrupt() producerCount++; if (trace) - IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), (uintptr_t) this, (uintptr_t) owner); + IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner)); signalWorkAvailable(); if (trace) - IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), (uintptr_t) this, (uintptr_t) owner); + IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner)); } @@ -160,7 +160,7 @@ void IOFilterInterruptEventSource::normalInterruptOccurred if (trace) IOTimeStampStartConstant(IODBG_INTES(IOINTES_FILTER), - VM_KERNEL_UNSLIDE(filterAction), (uintptr_t) owner, (uintptr_t) this, (uintptr_t) workLoop); + VM_KERNEL_UNSLIDE(filterAction), VM_KERNEL_ADDRHIDE(owner), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop)); if (IOInterruptEventSource::reserved->statistics) { if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelTimeIndex)) { @@ -184,7 +184,8 @@ void IOFilterInterruptEventSource::normalInterruptOccurred if (trace) IOTimeStampEndConstant(IODBG_INTES(IOINTES_FILTER), - VM_KERNEL_UNSLIDE(filterAction), (uintptr_t) owner, (uintptr_t) this, (uintptr_t) workLoop); + VM_KERNEL_ADDRHIDE(filterAction), VM_KERNEL_ADDRHIDE(owner), + VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop)); if (filterRes) signalInterrupt(); @@ -200,7 +201,7 @@ void IOFilterInterruptEventSource::disableInterruptOccurred if (trace) IOTimeStampStartConstant(IODBG_INTES(IOINTES_FILTER), - VM_KERNEL_UNSLIDE(filterAction), (uintptr_t) owner, (uintptr_t) this, (uintptr_t) workLoop); + VM_KERNEL_UNSLIDE(filterAction), VM_KERNEL_ADDRHIDE(owner), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop)); if (IOInterruptEventSource::reserved->statistics) { if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelTimeIndex)) { @@ -224,7 +225,7 @@ void IOFilterInterruptEventSource::disableInterruptOccurred if (trace) IOTimeStampEndConstant(IODBG_INTES(IOINTES_FILTER), - VM_KERNEL_UNSLIDE(filterAction), (uintptr_t) owner, (uintptr_t) this, (uintptr_t) workLoop); + VM_KERNEL_UNSLIDE(filterAction), VM_KERNEL_ADDRHIDE(owner), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop)); if (filterRes) { prov->disableInterrupt(source); /* disable the interrupt */ diff --git a/iokit/Kernel/IOHibernateIO.cpp b/iokit/Kernel/IOHibernateIO.cpp index 953c6d445..66b566ca9 100644 --- a/iokit/Kernel/IOHibernateIO.cpp +++ b/iokit/Kernel/IOHibernateIO.cpp @@ -174,6 +174,7 @@ to restrict I/O ops. #include <machine/pal_hibernate.h> #include <i386/tsc.h> #include <i386/cpuid.h> +#include <san/kasan.h> extern "C" addr64_t kvtophys(vm_offset_t va); extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va); @@ -213,7 +214,7 @@ static IOLock * gDebugImageLock; #endif /* defined(__i386__) || defined(__x86_64__) */ static IOLock * gFSLock; -static uint32_t gFSState; + uint32_t gFSState; static thread_call_t gIOHibernateTrimCalloutEntry; static IOPolledFileIOVars gFileVars; static IOHibernateVars gIOHibernateVars; @@ -235,6 +236,7 @@ enum static IOReturn IOHibernateDone(IOHibernateVars * vars); static IOReturn IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature); static void IOSetBootImageNVRAM(OSData * data); +static void IOHibernateSystemPostWakeTrim(void * p1, void * p2); /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ @@ -635,20 +637,37 @@ IOHibernateSystemSleep(void) rtcVars.signature[3] = 'L'; rtcVars.revision = 1; bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey)); - if (gIOHibernateBootSignature[0]) + + if (gIOChosenEntry + && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateBootSignatureKey))) + && (sizeof(rtcVars.booterSignature) <= data->getLength())) + { + bcopy(data->getBytesNoCopy(), &rtcVars.booterSignature[0], sizeof(rtcVars.booterSignature)); + } + else if (gIOHibernateBootSignature[0]) { char c; uint8_t value = 0; - for (uint32_t i = 0; - (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1)); - i++) + uint32_t in, out, digits; + for (in = out = digits = 0; + (c = gIOHibernateBootSignature[in]) && (in < sizeof(gIOHibernateBootSignature)); + in++) { - if (c >= 'a') c -= 'a' - 10; - else if (c >= 'A') c -= 'A' - 10; - else if (c >= '0') c -= '0'; - else continue; + if ((c >= 'a') && (c <= 'f')) c -= 'a' - 10; + else if ((c >= 'A') && (c <= 'F')) c -= 'A' - 10; + else if ((c >= '0') && (c <= '9')) c -= '0'; + else + { + if (c == '=') out = digits = value = 0; + continue; + } value = (value << 4) | c; - if (i & 1) rtcVars.booterSignature[i >> 1] = value; + if (digits & 1) + { + rtcVars.booterSignature[out++] = value; + if (out >= sizeof(rtcVars.booterSignature)) break; + } + digits++; } } data = OSData::withBytes(&rtcVars, sizeof(rtcVars)); @@ -1272,7 +1291,7 @@ IOHibernateDone(IOHibernateVars * vars) return (kIOReturnSuccess); } -void +static void IOHibernateSystemPostWakeTrim(void * p1, void * p2) { // invalidate & close the image file @@ -1293,7 +1312,7 @@ IOHibernateSystemPostWakeTrim(void * p1, void * p2) } IOReturn -IOHibernateSystemPostWake(void) +IOHibernateSystemPostWake(bool now) { gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature; IOLockLock(gFSLock); @@ -1301,11 +1320,14 @@ IOHibernateSystemPostWake(void) else if (kFSOpened != gFSState) gFSState = kFSIdle; else { - AbsoluteTime deadline; - gFSState = kFSTrimDelay; - clock_interval_to_deadline(TRIM_DELAY, kMillisecondScale, &deadline ); - thread_call_enter1_delayed(gIOHibernateTrimCalloutEntry, NULL, deadline); + if (now) IOHibernateSystemPostWakeTrim(NULL, NULL); + else + { + AbsoluteTime deadline; + clock_interval_to_deadline(TRIM_DELAY, kMillisecondScale, &deadline ); + thread_call_enter1_delayed(gIOHibernateTrimCalloutEntry, NULL, deadline); + } } IOLockUnlock(gFSLock); @@ -1478,7 +1500,7 @@ hibernate_write_image(void) if (kIOHibernateModeSleep & gIOHibernateMode) kdebug_enable = save_kdebug_enable; - KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START, 0, 0, 0, 0, 0); + KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START); IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate); restore1Sum = sum1 = sum2 = 0; @@ -1749,6 +1771,16 @@ hibernate_write_image(void) pageCount -= atop_32(segLen); } +#if KASAN + vm_size_t shadow_pages_free = atop_64(shadow_ptop) - atop_64(shadow_pnext); + + /* no need to save unused shadow pages */ + hibernate_set_page_state(vars->page_list, vars->page_list_wired, + atop_64(shadow_pnext), + shadow_pages_free, + kIOHibernatePageStateFree); +#endif + src = (uint8_t *) vars->srcBuffer->getBytesNoCopy(); compressed = src + page_size; scratch = compressed + page_size; @@ -2040,8 +2072,8 @@ hibernate_write_image(void) // should we come back via regular wake, set the state in memory. gIOHibernateState = kIOHibernateStateInactive; - KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END, - wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted, 0, 0); + KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END, wiredPagesEncrypted, + wiredPagesClear, dirtyPagesEncrypted); if (kIOReturnSuccess == err) { @@ -2123,6 +2155,11 @@ hibernate_machine_init(void) gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize; gIOHibernateStats->image1Pages = pagesDone; + /* HIBERNATE_stats */ + KDBG(IOKDBG_CODE(DBG_HIBERNATE, 14), gIOHibernateStats->smcStart, + gIOHibernateStats->booterStart, gIOHibernateStats->booterDuration, + gIOHibernateStats->trampolineDuration); + HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n", gIOHibernateStats->booterStart, gIOHibernateStats->smcStart, @@ -2403,7 +2440,7 @@ hibernate_machine_init(void) nsec / 1000000ULL, nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0); - KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_NONE, pagesRead, pagesDone, 0, 0, 0); + KDBG(IOKDBG_CODE(DBG_HIBERNATE, 2), pagesRead, pagesDone); } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ diff --git a/iokit/Kernel/IOHibernateRestoreKernel.c b/iokit/Kernel/IOHibernateRestoreKernel.c index 017d4d4f8..fc5a1b7f2 100644 --- a/iokit/Kernel/IOHibernateRestoreKernel.c +++ b/iokit/Kernel/IOHibernateRestoreKernel.c @@ -73,17 +73,7 @@ extern void acpi_wake_prot_entry(void); /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #if defined(__i386__) || defined(__x86_64__) - -#define rdtsc(lo,hi) \ - __asm__ volatile("lfence; rdtsc; lfence" : "=a" (lo), "=d" (hi)) - -static inline uint64_t rdtsc64(void) -{ - uint64_t lo, hi; - rdtsc(lo, hi); - return ((hi) << 32) | (lo); -} - +#include <i386/proc_reg.h> #else static inline uint64_t rdtsc64(void) @@ -409,7 +399,7 @@ store_one_page(uint32_t procFlags, uint32_t * src, uint32_t compressedSize, s = *src; d = (uint32_t *)(uintptr_t)dst; - if (!s) bzero((void *) dst, PAGE_SIZE); + if (!s) __nosan_bzero((void *) dst, PAGE_SIZE); else for (i = 0; i < (PAGE_SIZE / sizeof(int32_t)); i++) *d++ = s; } } @@ -465,7 +455,7 @@ hibernate_kernel_entrypoint(uint32_t p1, debug_code(kIOHibernateRestoreCodeImageStart, headerPhys); - memcpy(gIOHibernateCurrentHeader, + __nosan_memcpy(gIOHibernateCurrentHeader, (void *) pal_hib_map(IMAGE_AREA, headerPhys), sizeof(IOHibernateImageHeader)); @@ -638,7 +628,7 @@ hibernate_kernel_entrypoint(uint32_t p1, // alloc new buffer page bufferPage = hibernate_page_list_grab(map, &nextFree); dst = (uint32_t *)pal_hib_map(DEST_COPY_AREA, ptoa_64(bufferPage)); - memcpy(dst, src, compressedSize); + __nosan_memcpy(dst, src, compressedSize); } if (copyPageIndex > ((PAGE_SIZE >> 2) - 3)) { diff --git a/iokit/Kernel/IOHistogramReporter.cpp b/iokit/Kernel/IOHistogramReporter.cpp index 929a830f2..4c288ccdb 100644 --- a/iokit/Kernel/IOHistogramReporter.cpp +++ b/iokit/Kernel/IOHistogramReporter.cpp @@ -43,7 +43,7 @@ IOHistogramReporter::with(IOService *reportingService, IOReportCategories categories, uint64_t channelID, const char *channelName, - IOReportUnits unit, + IOReportUnit unit, int nSegments, IOHistogramSegmentConfig *config) { @@ -74,7 +74,7 @@ IOHistogramReporter::initWith(IOService *reportingService, IOReportCategories categories, uint64_t channelID, const OSSymbol *channelName, - IOReportUnits unit, + IOReportUnit unit, int nSegments, IOHistogramSegmentConfig *config) { @@ -272,34 +272,35 @@ IOHistogramReporter::free(void) IOReportLegendEntry* IOHistogramReporter::handleCreateLegend(void) { - OSData *tmpConfigData; - OSDictionary *tmpDict; - IOReportLegendEntry *legendEntry = NULL; + IOReportLegendEntry *rval = NULL, *legendEntry = NULL; + OSData *tmpConfigData = NULL; + OSDictionary *tmpDict; // no refcount legendEntry = super::handleCreateLegend(); + if (!legendEntry) goto finish; - if (legendEntry) { - - PREFL_MEMOP_PANIC(_segmentCount, IOHistogramSegmentConfig); - tmpConfigData = OSData::withBytes(_histogramSegmentsConfig, - (unsigned)_segmentCount * - (unsigned)sizeof(IOHistogramSegmentConfig)); - if (!tmpConfigData) { - legendEntry->release(); - goto finish; - } - - tmpDict = OSDynamicCast(OSDictionary, legendEntry->getObject(kIOReportLegendInfoKey)); - if (!tmpDict) { - legendEntry->release(); - goto finish; - } - - tmpDict->setObject(kIOReportLegendConfigKey, tmpConfigData); - } - + PREFL_MEMOP_PANIC(_segmentCount, IOHistogramSegmentConfig); + tmpConfigData = OSData::withBytes(_histogramSegmentsConfig, + (unsigned)_segmentCount * + sizeof(IOHistogramSegmentConfig)); + if (!tmpConfigData) goto finish; + + tmpDict = OSDynamicCast(OSDictionary, + legendEntry->getObject(kIOReportLegendInfoKey)); + if (!tmpDict) goto finish; + + tmpDict->setObject(kIOReportLegendConfigKey, tmpConfigData); + + // success + rval = legendEntry; + finish: - return legendEntry; + if (tmpConfigData) tmpConfigData->release(); + if (!rval && legendEntry) { + legendEntry->release(); + } + + return rval; } IOReturn diff --git a/iokit/Kernel/IOInterruptAccounting.cpp b/iokit/Kernel/IOInterruptAccounting.cpp index 8130cce23..bfaf153b2 100644 --- a/iokit/Kernel/IOInterruptAccounting.cpp +++ b/iokit/Kernel/IOInterruptAccounting.cpp @@ -30,10 +30,12 @@ #include <IOKit/IOKernelReporters.h> uint32_t gInterruptAccountingStatisticBitmask = +#if !defined(__arm__) /* Disable timestamps for older ARM platforms; they are expensive. */ IA_GET_ENABLE_BIT(kInterruptAccountingFirstLevelTimeIndex) | IA_GET_ENABLE_BIT(kInterruptAccountingSecondLevelCPUTimeIndex) | IA_GET_ENABLE_BIT(kInterruptAccountingSecondLevelSystemTimeIndex) | +#endif IA_GET_ENABLE_BIT(kInterruptAccountingFirstLevelCountIndex) | IA_GET_ENABLE_BIT(kInterruptAccountingSecondLevelCountIndex); diff --git a/iokit/Kernel/IOInterruptController.cpp b/iokit/Kernel/IOInterruptController.cpp index fb0aa23b0..81f07dcab 100644 --- a/iokit/Kernel/IOInterruptController.cpp +++ b/iokit/Kernel/IOInterruptController.cpp @@ -397,6 +397,54 @@ void IOInterruptController::causeVector(IOInterruptVectorNumber /*vectorNumber*/ { } +void IOInterruptController::timeStampSpuriousInterrupt(void) +{ + uint64_t providerID = 0; + IOService * provider = getProvider(); + + if (provider) { + providerID = provider->getRegistryEntryID(); + } + + IOTimeStampConstant(IODBG_INTC(IOINTC_SPURIOUS), providerID); +} + +void IOInterruptController::timeStampInterruptHandlerInternal(bool isStart, IOInterruptVectorNumber vectorNumber, IOInterruptVector *vector) +{ + uint64_t providerID = 0; + vm_offset_t unslidHandler = 0; + vm_offset_t unslidTarget = 0; + + IOService * provider = getProvider(); + + if (provider) { + providerID = provider->getRegistryEntryID(); + } + + if (vector) { + unslidHandler = VM_KERNEL_UNSLIDE((vm_offset_t)vector->handler); + unslidTarget = VM_KERNEL_UNSLIDE_OR_PERM((vm_offset_t)vector->target); + } + + + if (isStart) { + IOTimeStampStartConstant(IODBG_INTC(IOINTC_HANDLER), (uintptr_t)vectorNumber, (uintptr_t)unslidHandler, + (uintptr_t)unslidTarget, (uintptr_t)providerID); + } else { + IOTimeStampEndConstant(IODBG_INTC(IOINTC_HANDLER), (uintptr_t)vectorNumber, (uintptr_t)unslidHandler, + (uintptr_t)unslidTarget, (uintptr_t)providerID); + } +} + +void IOInterruptController::timeStampInterruptHandlerStart(IOInterruptVectorNumber vectorNumber, IOInterruptVector *vector) +{ + timeStampInterruptHandlerInternal(true, vectorNumber, vector); +} + +void IOInterruptController::timeStampInterruptHandlerEnd(IOInterruptVectorNumber vectorNumber, IOInterruptVector *vector) +{ + timeStampInterruptHandlerInternal(false, vectorNumber, vector); +} /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ @@ -691,26 +739,23 @@ IOReturn IOSharedInterruptController::handleInterrupt(void * /*refCon*/, OSMemoryBarrier(); #endif - if (!vector->interruptDisabledSoft) { - - // Call the handler if it exists. - if (vector->interruptRegistered) { - - bool trace = (gIOKitTrace & kIOTraceInterrupts) ? true : false; - - if (trace) - IOTimeStampStartConstant(IODBG_INTC(IOINTC_HANDLER), - (uintptr_t) vectorNumber, (uintptr_t) vector->handler, (uintptr_t)vector->target); - - // Call handler. - vector->handler(vector->target, vector->refCon, vector->nub, vector->source); - - if (trace) - IOTimeStampEndConstant(IODBG_INTC(IOINTC_HANDLER), - (uintptr_t) vectorNumber, (uintptr_t) vector->handler, (uintptr_t)vector->target); - - } - } + if (!vector->interruptDisabledSoft) { + + // Call the handler if it exists. + if (vector->interruptRegistered) { + + bool trace = (gIOKitTrace & kIOTraceInterrupts) ? true : false; + + if (trace) + timeStampInterruptHandlerStart(vectorNumber, vector); + + // Call handler. + vector->handler(vector->target, vector->refCon, vector->nub, vector->source); + + if (trace) + timeStampInterruptHandlerEnd(vectorNumber, vector); + } + } vector->interruptActive = 0; } diff --git a/iokit/Kernel/IOInterruptEventSource.cpp b/iokit/Kernel/IOInterruptEventSource.cpp index 1636405f6..a410de27e 100644 --- a/iokit/Kernel/IOInterruptEventSource.cpp +++ b/iokit/Kernel/IOInterruptEventSource.cpp @@ -308,7 +308,8 @@ bool IOInterruptEventSource::checkForWork() { if (trace) IOTimeStampStartConstant(IODBG_INTES(IOINTES_ACTION), - VM_KERNEL_UNSLIDE(intAction), (uintptr_t) owner, (uintptr_t) this, (uintptr_t) workLoop); + VM_KERNEL_ADDRHIDE(intAction), VM_KERNEL_ADDRHIDE(owner), + VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop)); if (reserved->statistics) { if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) { @@ -341,7 +342,8 @@ bool IOInterruptEventSource::checkForWork() if (trace) IOTimeStampEndConstant(IODBG_INTES(IOINTES_ACTION), - VM_KERNEL_UNSLIDE(intAction), (uintptr_t) owner, (uintptr_t) this, (uintptr_t) workLoop); + VM_KERNEL_ADDRHIDE(intAction), VM_KERNEL_ADDRHIDE(owner), + VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop)); consumerCount = cacheProdCount; if (autoDisable && !explicitDisable) @@ -352,7 +354,8 @@ bool IOInterruptEventSource::checkForWork() { if (trace) IOTimeStampStartConstant(IODBG_INTES(IOINTES_ACTION), - VM_KERNEL_UNSLIDE(intAction), (uintptr_t) owner, (uintptr_t) this, (uintptr_t) workLoop); + VM_KERNEL_ADDRHIDE(intAction), VM_KERNEL_ADDRHIDE(owner), + VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop)); if (reserved->statistics) { if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) { @@ -385,7 +388,8 @@ bool IOInterruptEventSource::checkForWork() if (trace) IOTimeStampEndConstant(IODBG_INTES(IOINTES_ACTION), - VM_KERNEL_UNSLIDE(intAction), (uintptr_t) owner, (uintptr_t) this, (uintptr_t) workLoop); + VM_KERNEL_ADDRHIDE(intAction), VM_KERNEL_ADDRHIDE(owner), + VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop)); consumerCount = cacheProdCount; if (autoDisable && !explicitDisable) @@ -404,7 +408,7 @@ void IOInterruptEventSource::normalInterruptOccurred producerCount++; if (trace) - IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), (uintptr_t) this, (uintptr_t) owner); + IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner)); if (reserved->statistics) { if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelCountIndex)) { @@ -415,7 +419,7 @@ void IOInterruptEventSource::normalInterruptOccurred signalWorkAvailable(); if (trace) - IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), (uintptr_t) this, (uintptr_t) owner); + IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner)); } void IOInterruptEventSource::disableInterruptOccurred @@ -429,7 +433,7 @@ void IOInterruptEventSource::disableInterruptOccurred producerCount++; if (trace) - IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), (uintptr_t) this, (uintptr_t) owner); + IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner)); if (reserved->statistics) { if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelCountIndex)) { @@ -440,7 +444,7 @@ void IOInterruptEventSource::disableInterruptOccurred signalWorkAvailable(); if (trace) - IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), (uintptr_t) this, (uintptr_t) owner); + IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner)); } void IOInterruptEventSource::interruptOccurred diff --git a/iokit/Kernel/IOKitDebug.cpp b/iokit/Kernel/IOKitDebug.cpp index 22b315da4..0cf42b685 100644 --- a/iokit/Kernel/IOKitDebug.cpp +++ b/iokit/Kernel/IOKitDebug.cpp @@ -62,9 +62,23 @@ SInt64 gIOKitTrace = 0; #define IODEBUG_CTLFLAGS CTLFLAG_RD #endif -SYSCTL_QUAD(_debug, OID_AUTO, iokit, IODEBUG_CTLFLAGS | CTLFLAG_LOCKED, &gIOKitDebug, "boot_arg io"); SYSCTL_QUAD(_debug, OID_AUTO, iotrace, CTLFLAG_RW | CTLFLAG_LOCKED, &gIOKitTrace, "trace io"); +static int +sysctl_debug_iokit +(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req) +{ + SInt64 newValue; + int changed, error = sysctl_io_number(req, gIOKitDebug, sizeof(gIOKitDebug), &newValue, &changed); + if (changed) { + gIOKitDebug = ((gIOKitDebug & ~kIOKitDebugUserOptions) | (newValue & kIOKitDebugUserOptions)); + } + return (error); +} + +SYSCTL_PROC(_debug, OID_AUTO, iokit, + CTLTYPE_QUAD | IODEBUG_CTLFLAGS | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED, + &gIOKitDebug, 0, sysctl_debug_iokit, "Q", "boot_arg io"); int debug_malloc_size; int debug_iomalloc_size; @@ -245,6 +259,7 @@ struct IOTrackingCallSite IOTrackingQueue * queue; uint32_t crc; + vm_tag_t tag; uint32_t count; size_t size[2]; uintptr_t bt[kIOTrackingCallSiteBTs]; @@ -259,6 +274,7 @@ struct IOTrackingLeaksRef uint32_t zoneSize; uint32_t count; uint32_t found; + uint32_t foundzlen; size_t bytes; }; @@ -500,7 +516,7 @@ IOTrackingRemoveUser(IOTrackingQueue * queue, IOTrackingUser * mem) uint64_t gIOTrackingAddTime; void -IOTrackingAdd(IOTrackingQueue * queue, IOTracking * mem, size_t size, bool address) +IOTrackingAdd(IOTrackingQueue * queue, IOTracking * mem, size_t size, bool address, vm_tag_t tag) { IOTrackingCallSite * site; uint32_t crc, num; @@ -522,6 +538,7 @@ IOTrackingAdd(IOTrackingQueue * queue, IOTracking * mem, size_t size, bool addre que = &queue->sites[crc % queue->numSiteQs]; queue_iterate(que, site, IOTrackingCallSite *, link) { + if (tag != site->tag) continue; if (crc == site->crc) break; } @@ -534,6 +551,7 @@ IOTrackingAdd(IOTrackingQueue * queue, IOTracking * mem, size_t size, bool addre site->queue = queue; site->crc = crc; site->count = 0; + site->tag = tag; memset(&site->size[0], 0, sizeof(site->size)); bcopy(&bt[1], &site->bt[0], num * sizeof(site->bt[0])); assert(num <= kIOTrackingCallSiteBTs); @@ -587,6 +605,7 @@ IOTrackingRemove(IOTrackingQueue * queue, IOTracking * mem, size_t size) queue->siteCount--; kfree(mem->site, sizeof(IOTrackingCallSite)); } + mem->site = NULL; } IOTRecursiveLockUnlock(&queue->lock); } @@ -608,7 +627,7 @@ IOTrackingAlloc(IOTrackingQueue * queue, uintptr_t address, size_t size) tracking->address = address; tracking->size = size; - IOTrackingAdd(queue, &tracking->tracking, size, true); + IOTrackingAdd(queue, &tracking->tracking, size, true, VM_KERN_MEMORY_NONE); } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ @@ -790,14 +809,14 @@ IOTrackingLeakScan(void * refcon) uint64_t vaddr, vincr; ppnum_t ppn; uintptr_t ptr, addr, vphysaddr, inst; - size_t size; + size_t size, origsize; uint32_t baseIdx, lim, ptrIdx, count; boolean_t is; AbsoluteTime deadline; - instances = ref->instances; - count = ref->count; - size = ref->zoneSize; + instances = ref->instances; + count = ref->count; + size = origsize = ref->zoneSize; for (deadline = 0, vaddr = VM_MIN_KERNEL_AND_KEXT_ADDRESS; ; @@ -835,13 +854,14 @@ IOTrackingLeakScan(void * refcon) } else if (kInstanceFlagAddress & inst) { - addr = ~((IOTrackingAddress *)instance)->address; - size = ((IOTrackingAddress *)instance)->size; + addr = ~((IOTrackingAddress *)instance)->address; + origsize = size = ((IOTrackingAddress *)instance)->size; + if (!size) size = 1; } else { - addr = (uintptr_t) (instance + 1); - size = instance->site->queue->allocSize; + addr = (uintptr_t) (instance + 1); + origsize = size = instance->site->queue->allocSize; } if ((ptr >= addr) && (ptr < (addr + size)) @@ -853,6 +873,7 @@ IOTrackingLeakScan(void * refcon) inst |= kInstanceFlagReferenced; instances[baseIdx + (lim >> 1)] = inst; ref->found++; + if (!origsize) ref->foundzlen++; } break; } @@ -948,7 +969,7 @@ IOTrackingLeaks(OSData * data) { ref.bytes = 0; IOTrackingLeakScan(&ref); - IOLog("leaks(%d) scanned %ld MB, instance count %d, found %d\n", idx, ref.bytes / 1024 / 1024, count, ref.found); + IOLog("leaks(%d) scanned %ld MB, instance count %d, found %d (zlen %d)\n", idx, ref.bytes / 1024 / 1024, count, ref.found, ref.foundzlen); if (count <= ref.found) break; } @@ -1022,8 +1043,9 @@ SkipName(uint32_t options, const char * name, size_t namesLen, const char * name /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -kern_return_t +static kern_return_t IOTrackingDebug(uint32_t selector, uint32_t options, uint64_t value, + uint32_t intag, uint32_t inzsize, const char * names, size_t namesLen, size_t size, OSObject ** result) { @@ -1119,6 +1141,7 @@ IOTrackingDebug(uint32_t selector, uint32_t options, uint64_t value, break; } + case kIOTrackingGetTracking: { if (kIOTrackingQueueTypeMap & queue->type) break; @@ -1135,11 +1158,51 @@ IOTrackingDebug(uint32_t selector, uint32_t options, uint64_t value, assert(idx < num); idx++; - if (size && ((site->size[0] + site->size[1]) < size)) continue; + size_t tsize[2]; + uint32_t count = site->count; + tsize[0] = site->size[0]; + tsize[1] = site->size[1]; + + if (intag || inzsize) + { + uintptr_t addr; + vm_size_t size, zoneSize; + vm_tag_t tag; + + if (kIOTrackingQueueTypeAlloc & queue->type) + { + addresses = false; + count = 0; + tsize[0] = tsize[1] = 0; + queue_iterate(&site->instances, instance, IOTracking *, link) + { + if (instance == site->addresses) addresses = true; + + if (addresses) addr = ~((IOTrackingAddress *)instance)->address; + else addr = (uintptr_t) (instance + 1); + + kr = vm_kern_allocation_info(addr, &size, &tag, &zoneSize); + if (KERN_SUCCESS != kr) continue; + + if ((VM_KERN_MEMORY_NONE != intag) && (intag != tag)) continue; + if (inzsize && (inzsize != zoneSize)) continue; + + count++; + tsize[0] += size; + } + } + else + { + if (!intag || inzsize || (intag != site->tag)) continue; + } + } + + if (!count) continue; + if (size && ((tsize[0] + tsize[1]) < size)) continue; - siteInfo.count = site->count; - siteInfo.size[0] = site->size[0]; - siteInfo.size[1] = site->size[1]; + siteInfo.count = count; + siteInfo.size[0] = tsize[0]; + siteInfo.size[1] = tsize[1]; CopyOutKernelBacktrace(site, &siteInfo); data->appendBytes(&siteInfo, sizeof(siteInfo)); @@ -1320,7 +1383,7 @@ IOReturn IOKitDiagnosticsClient::externalMethod(uint32_t selector, IOExternalMet namesLen = args->structureInputSize - sizeof(IOKitDiagnosticsParameters); if (namesLen) names = (typeof(names))(params + 1); - ret = IOTrackingDebug(selector, params->options, params->value, names, namesLen, params->size, &result); + ret = IOTrackingDebug(selector, params->options, params->value, params->tag, params->zsize, names, namesLen, params->size, &result); if ((kIOReturnSuccess == ret) && args->structureVariableOutputData) *args->structureVariableOutputData = result; else if (result) result->release(); diff --git a/iokit/Kernel/IOKitKernelInternal.h b/iokit/Kernel/IOKitKernelInternal.h index 630b39fb5..85507aa65 100644 --- a/iokit/Kernel/IOKitKernelInternal.h +++ b/iokit/Kernel/IOKitKernelInternal.h @@ -141,11 +141,9 @@ struct IODMACommandInternal UInt8 fDoubleBuffer; UInt8 fNewMD; UInt8 fLocalMapper; - - vm_tag_t fTag; -#if IOTRACKING - IOTracking fWireTracking; -#endif /* IOTRACKING */ + UInt8 fLocalMapperAllocValid; + UInt8 fIOVMAddrValid; + UInt8 fForceDoubleBuffer; vm_page_t fCopyPageAlloc; vm_page_t fCopyNext; @@ -217,6 +215,8 @@ extern "C" struct timeval gIOLastWakeTime; extern clock_sec_t gIOConsoleLockTime; +extern bool gCPUsRunning; + extern OSSet * gIORemoveOnReadProperties; extern "C" void IOKitInitializeTime( void ); @@ -230,6 +230,7 @@ extern "C" OSString * IOCopyLogNameForPID(int pid); extern const OSSymbol * gIOCreateEFIDevicePathSymbol; extern "C" void IOSetKeyStoreData(IOMemoryDescriptor * data); +extern "C" void IOSetAPFSKeyStoreData(IOMemoryDescriptor* data); #endif extern const OSSymbol * gAKSGetKey; diff --git a/iokit/Kernel/IOLib.cpp b/iokit/Kernel/IOLib.cpp index 73a0c67a8..c4a63b9dd 100644 --- a/iokit/Kernel/IOLib.cpp +++ b/iokit/Kernel/IOLib.cpp @@ -60,6 +60,7 @@ #include <IOKit/IOStatisticsPrivate.h> #include <os/log_private.h> #include <sys/msgbuf.h> +#include <console/serial_protos.h> #if IOKITSTATS @@ -191,7 +192,9 @@ void IOLibInit(void) &gIOKitPageableSpace.maps[0].address, kIOPageableMapSize, TRUE, - VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_IOKIT), + VM_FLAGS_ANYWHERE, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_IOKIT, &gIOKitPageableSpace.maps[0].map); if (ret != KERN_SUCCESS) panic("failed to allocate iokit pageable map\n"); @@ -281,7 +284,7 @@ void * IOMalloc(vm_size_t size) bzero(&hdr->tracking, sizeof(hdr->tracking)); hdr->tracking.address = ~(((uintptr_t) address) + sizeofIOLibMallocHeader); hdr->tracking.size = size; - IOTrackingAdd(gIOMallocTracking, &hdr->tracking.tracking, size, true); + IOTrackingAdd(gIOMallocTracking, &hdr->tracking.tracking, size, true, VM_KERN_MEMORY_NONE); } #endif address = (typeof(address)) (((uintptr_t) address) + sizeofIOLibMallocHeader); @@ -420,7 +423,7 @@ void * IOMallocAligned(vm_size_t size, vm_size_t alignment) bzero(&hdr->tracking, sizeof(hdr->tracking)); hdr->tracking.address = ~address; hdr->tracking.size = size; - IOTrackingAdd(gIOMallocTracking, &hdr->tracking.tracking, size, true); + IOTrackingAdd(gIOMallocTracking, &hdr->tracking.tracking, size, true, VM_KERN_MEMORY_NONE); } #endif } else @@ -525,6 +528,9 @@ IOKernelFreePhysical(mach_vm_address_t address, mach_vm_size_t size) #endif } +#if __arm__ || __arm64__ +extern unsigned long gPhysBase, gPhysSize; +#endif mach_vm_address_t IOKernelAllocateWithPhysicalRestrict(mach_vm_size_t size, mach_vm_address_t maxPhys, @@ -560,6 +566,13 @@ IOKernelAllocateWithPhysicalRestrict(mach_vm_size_t size, mach_vm_address_t maxP if (!contiguous) { +#if __arm__ || __arm64__ + if (maxPhys >= (mach_vm_address_t)(gPhysBase + gPhysSize)) + { + maxPhys = 0; + } + else +#endif if (maxPhys <= 0xFFFFFFFF) { maxPhys = 0; @@ -613,7 +626,7 @@ IOKernelAllocateWithPhysicalRestrict(mach_vm_size_t size, mach_vm_address_t maxP bzero(&hdr->tracking, sizeof(hdr->tracking)); hdr->tracking.address = ~address; hdr->tracking.size = size; - IOTrackingAdd(gIOMallocTracking, &hdr->tracking.tracking, size, true); + IOTrackingAdd(gIOMallocTracking, &hdr->tracking.tracking, size, true, VM_KERN_MEMORY_NONE); } #endif } else @@ -778,7 +791,9 @@ kern_return_t IOIteratePageableMaps(vm_size_t size, &min, segSize, TRUE, - VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_IOKIT), + VM_FLAGS_ANYWHERE, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_IOKIT, &map); if( KERN_SUCCESS != kr) { lck_mtx_unlock( gIOKitPageableSpace.lock ); @@ -1132,12 +1147,7 @@ void IOPause(unsigned nanoseconds) /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -static void _iolog_consputc(int ch, void *arg __unused) -{ - cons_putc_locked(ch); -} - -static void _IOLogv(const char *format, va_list ap, void *caller); +static void _IOLogv(const char *format, va_list ap, void *caller) __printflike(1,0); __attribute__((noinline,not_tail_called)) void IOLog(const char *format, ...) @@ -1160,16 +1170,18 @@ void IOLogv(const char *format, va_list ap) void _IOLogv(const char *format, va_list ap, void *caller) { va_list ap2; - - /* Ideally not called at interrupt context or with interrupts disabled. Needs further validate */ - /* assert(TRUE == ml_get_interrupts_enabled()); */ + struct console_printbuf_state info_data; + console_printbuf_state_init(&info_data, TRUE, TRUE); va_copy(ap2, ap); os_log_with_args(OS_LOG_DEFAULT, OS_LOG_TYPE_DEFAULT, format, ap, caller); - __doprnt(format, ap2, _iolog_consputc, NULL, 16, TRUE); + __doprnt(format, ap2, console_printbuf_putc, &info_data, 16, TRUE); + console_printbuf_clear(&info_data); va_end(ap2); + + assertf(ml_get_interrupts_enabled() || ml_is_quiescing() || debug_mode_active() || !gCPUsRunning, "IOLog called with interrupts disabled"); } #if !__LP64__ diff --git a/iokit/Kernel/IOMapper.cpp b/iokit/Kernel/IOMapper.cpp index 89a00c921..7f944e831 100644 --- a/iokit/Kernel/IOMapper.cpp +++ b/iokit/Kernel/IOMapper.cpp @@ -30,6 +30,7 @@ #include <IOKit/IODMACommand.h> #include <libkern/c++/OSData.h> #include <libkern/OSDebug.h> +#include <mach_debug/zone_info.h> #include "IOKitKernelInternal.h" __BEGIN_DECLS @@ -142,20 +143,16 @@ IOMapper * IOMapper::copyMapperForDeviceWithIndex(IOService * device, unsigned i OSDictionary * matching; obj = device->copyProperty("iommu-parent"); - if (!obj) - return (NULL); + if (!obj) return (NULL); - if ((mapper = OSDynamicCast(IOMapper, obj))) - return (mapper); + if ((mapper = OSDynamicCast(IOMapper, obj))) goto found; if ((data = OSDynamicCast(OSData, obj))) { - if (index >= data->getLength() / sizeof(UInt32)) - goto done; + if (index >= data->getLength() / sizeof(UInt32)) goto done; data = OSData::withBytesNoCopy((UInt32 *)data->getBytesNoCopy() + index, sizeof(UInt32)); - if (!data) - goto done; + if (!data) goto done; matching = IOService::propertyMatching(gIOMapperIDKey, data); data->release(); @@ -166,12 +163,31 @@ IOMapper * IOMapper::copyMapperForDeviceWithIndex(IOService * device, unsigned i if (matching) { mapper = OSDynamicCast(IOMapper, IOService::waitForMatchingService(matching)); - matching->release(); + matching->release(); } done: - if (obj) - obj->release(); + if (obj) obj->release(); +found: + if (mapper) + { + if (!mapper->fAllocName) + { + char name[MACH_ZONE_NAME_MAX_LEN]; + char kmodname[KMOD_MAX_NAME]; + vm_tag_t tag; + uint32_t kmodid; + + tag = IOMemoryTag(kernel_map); + if (!(kmodid = vm_tag_get_kext(tag, &kmodname[0], KMOD_MAX_NAME))) + { + snprintf(kmodname, sizeof(kmodname), "%d", tag); + } + snprintf(name, sizeof(name), "%s.DMA.%s", kmodname, device->getName()); + mapper->fAllocName = kern_allocation_name_allocate(name, 16); + } + } + return (mapper); } diff --git a/iokit/Kernel/IOMemoryDescriptor.cpp b/iokit/Kernel/IOMemoryDescriptor.cpp index fa735f3e4..3b59323b8 100644 --- a/iokit/Kernel/IOMemoryDescriptor.cpp +++ b/iokit/Kernel/IOMemoryDescriptor.cpp @@ -104,8 +104,6 @@ static IORecursiveLock * gIOMemoryLock; #define DEBG(fmt, args...) {} #endif -#define IOMD_DEBUG_DMAACTIVE 1 - /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ // Some data structures and accessor macros used by the initWithOptions @@ -142,15 +140,14 @@ struct ioGMDData uint64_t fPreparationID; #if IOTRACKING IOTracking fWireTracking; - struct vm_tag_set fWireTags; - struct vm_tag_set_entry fWireTagsEntries[kMaxWireTags]; #endif /* IOTRACKING */ unsigned int fPageCnt; uint8_t fDMAMapNumAddressBits; - vm_tag_t fAllocTag; unsigned char fDiscontig:1; unsigned char fCompletionError:1; - unsigned char _resv:6; + unsigned char fMappedBaseValid:1; + unsigned char _resv:3; + unsigned char fDMAAccess:2; /* variable length arrays */ upl_page_info_t fPageList[1] @@ -170,6 +167,8 @@ struct ioGMDData #define computeDataSize(p, u) \ (offsetof(ioGMDData, fPageList) + p * sizeof(upl_page_info_t) + u * sizeof(ioPLBlock)) +enum { kIOMemoryHostOrRemote = kIOMemoryHostOnly | kIOMemoryRemote }; + /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #define next_page(a) ( trunc_page(a) + PAGE_SIZE ) @@ -274,6 +273,14 @@ purgeableControlBits(IOOptionBits newState, vm_purgable_t * control, int * state err = kIOReturnBadArgument; break; } + + if (*control == VM_PURGABLE_SET_STATE) { + // let VM know this call is from the kernel and is allowed to alter + // the volatility of the memory entry even if it was created with + // MAP_MEM_PURGABLE_KERNEL_ONLY + *control = VM_PURGABLE_SET_STATE_FROM_KERNEL; + } + return (err); } @@ -328,6 +335,10 @@ vmProtForCacheMode(IOOptionBits cacheMode) SET_MAP_MEM(MAP_MEM_INNERWBACK, prot); break; + case kIOPostedWrite: + SET_MAP_MEM(MAP_MEM_POSTED, prot); + break; + case kIODefaultCache: default: SET_MAP_MEM(MAP_MEM_NOOP, prot); @@ -363,6 +374,10 @@ pagerFlagsForCacheMode(IOOptionBits cacheMode) pagerFlags = DEVICE_PAGER_COHERENT; break; + case kIOPostedWrite: + pagerFlags = DEVICE_PAGER_CACHE_INHIB | DEVICE_PAGER_COHERENT | DEVICE_PAGER_GUARDED | DEVICE_PAGER_EARLY_ACK; + break; + case kIODefaultCache: default: pagerFlags = -1U; @@ -517,7 +532,9 @@ IOGeneralMemoryDescriptor::memoryReferenceCreate( pagerFlags = IODefaultCacheBits(nextAddr); if (DEVICE_PAGER_CACHE_INHIB & pagerFlags) { - if (DEVICE_PAGER_GUARDED & pagerFlags) + if (DEVICE_PAGER_EARLY_ACK & pagerFlags) + mode = kIOPostedWrite; + else if (DEVICE_PAGER_GUARDED & pagerFlags) mode = kIOInhibitCache; else mode = kIOWriteCombineCache; @@ -554,7 +571,7 @@ IOGeneralMemoryDescriptor::memoryReferenceCreate( { // IOBufferMemoryDescriptor alloc - set flags for entry + object create prot |= MAP_MEM_NAMED_CREATE; - if (kIOMemoryBufferPurgeable & _flags) prot |= MAP_MEM_PURGABLE; + if (kIOMemoryBufferPurgeable & _flags) prot |= (MAP_MEM_PURGABLE | MAP_MEM_PURGABLE_KERNEL_ONLY); if (kIOMemoryUseReserve & _flags) prot |= MAP_MEM_GRAB_SECLUDED; prot |= VM_PROT_WRITE; @@ -703,8 +720,9 @@ IOMemoryDescriptorMapAlloc(vm_map_t map, void * _ref) (vm_map_offset_t) 0, (((ref->options & kIOMapAnywhere) ? VM_FLAGS_ANYWHERE - : VM_FLAGS_FIXED) - | VM_MAKE_TAG(ref->tag)), + : VM_FLAGS_FIXED)), + VM_MAP_KERNEL_FLAGS_NONE, + ref->tag, IPC_PORT_NULL, (memory_object_offset_t) 0, false, /* copy */ @@ -846,6 +864,15 @@ IOGeneralMemoryDescriptor::memoryReferenceMap( } } + /* + * If the memory is associated with a device pager but doesn't have a UPL, + * it will be immediately faulted in through the pager via populateDevicePager(). + * kIOMapPrefault is redundant in that case, so don't try to use it for UPL + * operations. + */ + if ((reserved != NULL) && (reserved->dp.devicePager) && (_memoryEntries == NULL) && (_wireCount != 0)) + options &= ~kIOMapPrefault; + /* * Prefaulting is only possible if we wired the memory earlier. Check the * memory type, and the underlying data. @@ -856,11 +883,9 @@ IOGeneralMemoryDescriptor::memoryReferenceMap( * The memory must have been wired by calling ::prepare(), otherwise * we don't have the UPL. Without UPLs, pages cannot be pre-faulted */ - assert(map != kernel_map); assert(_wireCount != 0); assert(_memoryEntries != NULL); - if ((map == kernel_map) || - (_wireCount == 0) || + if ((_wireCount == 0) || (_memoryEntries == NULL)) { return kIOReturnBadArgument; @@ -930,17 +955,23 @@ IOGeneralMemoryDescriptor::memoryReferenceMap( chunk = entry->size - entryOffset; if (chunk) { + vm_map_kernel_flags_t vmk_flags; + + vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; + vmk_flags.vmkf_iokit_acct = TRUE; /* iokit accounting */ + if (chunk > remain) chunk = remain; if (options & kIOMapPrefault) { UInt nb_pages = round_page(chunk) / PAGE_SIZE; + err = vm_map_enter_mem_object_prefault(map, &mapAddr, chunk, 0 /* mask */, - (VM_FLAGS_FIXED - | VM_FLAGS_OVERWRITE - | VM_MAKE_TAG(tag) - | VM_FLAGS_IOKIT_ACCT), /* iokit accounting */ + (VM_FLAGS_FIXED + | VM_FLAGS_OVERWRITE), + vmk_flags, + tag, entry->entry, entryOffset, prot, // cur @@ -958,9 +989,9 @@ IOGeneralMemoryDescriptor::memoryReferenceMap( &mapAddr, chunk, 0 /* mask */, (VM_FLAGS_FIXED - | VM_FLAGS_OVERWRITE - | VM_MAKE_TAG(tag) - | VM_FLAGS_IOKIT_ACCT), /* iokit accounting */ + | VM_FLAGS_OVERWRITE), + vmk_flags, + tag, entry->entry, entryOffset, false, // copy @@ -1042,7 +1073,7 @@ IOGeneralMemoryDescriptor::memoryReferenceSetPurgeable( err = purgeableControlBits(newState, &control, &state); if (KERN_SUCCESS != err) break; - err = mach_memory_entry_purgable_control(entries->entry, control, &state); + err = memory_entry_purgeable_control_internal(entries->entry, control, &state); if (KERN_SUCCESS != err) break; err = purgeableStateBits(&state); if (KERN_SUCCESS != err) break; @@ -1469,7 +1500,7 @@ IOGeneralMemoryDescriptor::initWithOptions(void * buffers, } // Grab the appropriate mapper - if (kIOMemoryHostOnly & options) options |= kIOMemoryMapperNone; + if (kIOMemoryHostOrRemote & options) options |= kIOMemoryMapperNone; if (kIOMemoryMapperNone & options) mapper = 0; // No Mapper else if (mapper == kIOMapperSystem) { @@ -1486,6 +1517,7 @@ IOGeneralMemoryDescriptor::initWithOptions(void * buffers, _direction = (IODirection) (_flags & kIOMemoryDirectionMask); #endif /* !__LP64__ */ + _dmaReferences = 0; __iomd_reservedA = 0; __iomd_reservedB = 0; _highestPage = 0; @@ -1509,7 +1541,20 @@ IOGeneralMemoryDescriptor::initWithOptions(void * buffers, if (!initMemoryEntries(dataSize, mapper)) return (false); dataP = getDataP(_memoryEntries); dataP->fPageCnt = 0; - + switch (kIOMemoryDirectionMask & options) + { + case kIODirectionOut: + dataP->fDMAAccess = kIODMAMapReadAccess; + break; + case kIODirectionIn: + dataP->fDMAAccess = kIODMAMapWriteAccess; + break; + case kIODirectionNone: + case kIODirectionOutIn: + default: + panic("bad dir for upl 0x%x\n", (int) options); + break; + } // _wireCount++; // UPLs start out life wired _length = count; @@ -1568,7 +1613,9 @@ IOGeneralMemoryDescriptor::initWithOptions(void * buffers, case kIOMemoryTypeVirtual64: case kIOMemoryTypePhysical64: if (count == 1 +#ifndef __arm__ && (((IOAddressRange *) buffers)->address + ((IOAddressRange *) buffers)->length) <= 0x100000000ULL +#endif ) { if (kIOMemoryTypeVirtual64 == type) type = kIOMemoryTypeVirtual; @@ -1604,6 +1651,7 @@ IOGeneralMemoryDescriptor::initWithOptions(void * buffers, break; } } + _rangesCount = count; // Find starting address within the vector of ranges Ranges vec = _ranges; @@ -1631,10 +1679,11 @@ IOGeneralMemoryDescriptor::initWithOptions(void * buffers, _length = totalLength; _pages = pages; - _rangesCount = count; // Auto-prepare memory at creation time. // Implied completion when descriptor is free-ed + + if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type)) _wireCount++; // Physical MDs are, by definition, wired else { /* kIOMemoryTypeVirtual | kIOMemoryTypeVirtual64 | kIOMemoryTypeUIO */ @@ -1648,6 +1697,12 @@ IOGeneralMemoryDescriptor::initWithOptions(void * buffers, dataP = getDataP(_memoryEntries); dataP->fPageCnt = _pages; + if (((_task != kernel_task) || (kIOMemoryBufferPageable & _flags)) + && (VM_KERN_MEMORY_NONE == _kernelTag)) + { + _kernelTag = IOMemoryTag(kernel_map); + } + if ( (kIOMemoryPersistent & _flags) && !_memRef) { IOReturn @@ -1682,10 +1737,10 @@ void IOGeneralMemoryDescriptor::free() if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type)) { ioGMDData * dataP; - if (_memoryEntries && (dataP = getDataP(_memoryEntries)) && dataP->fMappedBase) + if (_memoryEntries && (dataP = getDataP(_memoryEntries)) && dataP->fMappedBaseValid) { - dataP->fMapper->iovmUnmapMemory(this, NULL, dataP->fMappedBase, dataP->fMappedLength); - dataP->fMappedBase = 0; + dmaUnmap(dataP->fMapper, NULL, 0, dataP->fMappedBase, dataP->fMappedLength); + dataP->fMappedBaseValid = dataP->fMappedBase = 0; } } else @@ -1774,6 +1829,11 @@ IOOptionBits IOMemoryDescriptor::getTag( void ) return( _tag); } +uint64_t IOMemoryDescriptor::getFlags(void) +{ + return (_flags); +} + #ifndef __LP64__ #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wdeprecated-declarations" @@ -1810,6 +1870,9 @@ IOByteCount IOMemoryDescriptor::readBytes return 0; } + assert (!(kIOMemoryRemote & _flags)); + if (kIOMemoryRemote & _flags) return (0); + if (kIOMemoryThreadSafe & _flags) LOCK; @@ -1861,6 +1924,9 @@ IOByteCount IOMemoryDescriptor::writeBytes return 0; } + assert (!(kIOMemoryRemote & _flags)); + if (kIOMemoryRemote & _flags) return (0); + if (kIOMemoryThreadSafe & _flags) LOCK; @@ -1961,22 +2027,21 @@ uint64_t IOMemoryDescriptor::getPreparationID( void ) void IOMemoryDescriptor::setVMTags(vm_tag_t kernelTag, vm_tag_t userTag) { - if (!getKernelReserved()) return; - reserved->kernelTag = kernelTag; - reserved->userTag = userTag; + _kernelTag = kernelTag; + _userTag = userTag; } vm_tag_t IOMemoryDescriptor::getVMTag(vm_map_t map) { - if (!reserved - || (VM_KERN_MEMORY_NONE == reserved->kernelTag) - || (VM_KERN_MEMORY_NONE == reserved->userTag)) + if (vm_kernel_map_is_kernel(map)) { - return (IOMemoryTag(map)); + if (VM_KERN_MEMORY_NONE != _kernelTag) return (_kernelTag); } - - if (vm_kernel_map_is_kernel(map)) return (reserved->kernelTag); - return (reserved->userTag); + else + { + if (VM_KERN_MEMORY_NONE != _userTag) return (_userTag); + } + return (IOMemoryTag(map)); } IOReturn IOGeneralMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void *vData, UInt dataSize) const @@ -2015,27 +2080,38 @@ IOReturn IOGeneralMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void * && ((dataP->fMappedBase + _length) > (1ULL << dataP->fDMAMapNumAddressBits)); remap |= (dataP->fDMAMapAlignment > page_size); - if (remap || !dataP->fMappedBase) + if (remap || !dataP->fMappedBaseValid) { -// if (dataP->fMappedBase) OSReportWithBacktrace("kIOMDDMAMap whole %d remap %d params %d\n", whole, remap, params); +// if (dataP->fMappedBaseValid) OSReportWithBacktrace("kIOMDDMAMap whole %d remap %d params %d\n", whole, remap, params); err = md->dmaMap(data->fMapper, data->fCommand, &data->fMapSpec, data->fOffset, data->fLength, &data->fAlloc, &data->fAllocLength); - if (keepMap && (kIOReturnSuccess == err) && !dataP->fMappedBase) + if (keepMap && (kIOReturnSuccess == err) && !dataP->fMappedBaseValid) { - dataP->fMappedBase = data->fAlloc; - dataP->fMappedLength = data->fAllocLength; - data->fAllocLength = 0; // IOMD owns the alloc now + dataP->fMappedBase = data->fAlloc; + dataP->fMappedBaseValid = true; + dataP->fMappedLength = data->fAllocLength; + data->fAllocLength = 0; // IOMD owns the alloc now } } else { data->fAlloc = dataP->fMappedBase; data->fAllocLength = 0; // give out IOMD map + md->dmaMapRecord(data->fMapper, data->fCommand, dataP->fMappedLength); } data->fMapContig = !dataP->fDiscontig; } - return (err); } + if (kIOMDDMAUnmap == op) + { + if (dataSize < sizeof(IOMDDMAMapArgs)) + return kIOReturnUnderrun; + IOMDDMAMapArgs * data = (IOMDDMAMapArgs *) vData; + + err = md->dmaUnmap(data->fMapper, data->fCommand, data->fOffset, data->fAlloc, data->fAllocLength); + + return kIOReturnSuccess; + } if (kIOMDAddDMAMapSpec == op) { @@ -2083,9 +2159,24 @@ IOReturn IOGeneralMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void * } } - return kIOReturnSuccess; + return kIOReturnSuccess; + } - } else if (kIOMDWalkSegments != op) + else if (kIOMDDMAActive == op) + { + if (params) + { + int16_t prior; + prior = OSAddAtomic16(1, &md->_dmaReferences); + if (!prior) md->_mapName = NULL; + } + else + { + if (md->_dmaReferences) OSAddAtomic16(-1, &md->_dmaReferences); + else panic("_dmaReferences underflow"); + } + } + else if (kIOMDWalkSegments != op) return kIOReturnBadArgument; // Get the next segment @@ -2104,10 +2195,12 @@ IOReturn IOGeneralMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void * UInt offset = isP->fIO.fOffset; bool mapped = isP->fIO.fMapped; + if (mapped && (kIOMemoryRemote & _flags)) return (kIOReturnNotAttached); + if (IOMapper::gSystem && mapped && (!(kIOMemoryHostOnly & _flags)) - && (!_memoryEntries || !getDataP(_memoryEntries)->fMappedBase)) -// && (_memoryEntries && !getDataP(_memoryEntries)->fMappedBase)) + && (!_memoryEntries || !getDataP(_memoryEntries)->fMappedBaseValid)) +// && (_memoryEntries && !getDataP(_memoryEntries)->fMappedBaseValid)) { if (!_memoryEntries && !md->initMemoryEntries(computeDataSize(0, 0), kIOMapperWaitSystem)) return (kIOReturnNoMemory); @@ -2121,6 +2214,7 @@ IOReturn IOGeneralMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void * mapSpec.alignment = dataP->fDMAMapAlignment; err = md->dmaMap(dataP->fMapper, NULL, &mapSpec, 0, _length, &dataP->fMappedBase, &dataP->fMappedLength); if (kIOReturnSuccess != err) return (err); + dataP->fMappedBaseValid = true; } } @@ -2157,7 +2251,7 @@ IOReturn IOGeneralMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void * address = physP[ind - 1].address + len - length; if (true && mapped && _memoryEntries - && (dataP = getDataP(_memoryEntries)) && dataP->fMappedBase) + && (dataP = getDataP(_memoryEntries)) && dataP->fMappedBaseValid) { address = dataP->fMappedBase + offset; } @@ -2194,7 +2288,7 @@ IOReturn IOGeneralMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void * address = physP[ind - 1].address + len - length; if (true && mapped && _memoryEntries - && (dataP = getDataP(_memoryEntries)) && dataP->fMappedBase) + && (dataP = getDataP(_memoryEntries)) && dataP->fMappedBaseValid) { address = dataP->fMappedBase + offset; } @@ -2245,7 +2339,7 @@ IOReturn IOGeneralMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void * // If a mapped address is requested and this is a pre-mapped IOPL // then just need to compute an offset relative to the mapped base. - if (mapped && dataP->fMappedBase) { + if (mapped && dataP->fMappedBaseValid) { offset += (ioplInfo.fPageOffset & PAGE_MASK); address = trunc_page_64(dataP->fMappedBase) + ptoa_64(ioplInfo.fMappedPage) + offset; continue; // Done leave do/while(false) now @@ -2368,7 +2462,7 @@ IOGeneralMemoryDescriptor::getPhysicalSegment(IOByteCount offset, IOByteCount *l state->fOffset = offset; state->fLength = _length - offset; - state->fMapped = (0 == (options & kIOMemoryMapperNone)) && !(_flags & kIOMemoryHostOnly); + state->fMapped = (0 == (options & kIOMemoryMapperNone)) && !(_flags & kIOMemoryHostOrRemote); ret = dmaCommandOperation(kIOMDFirstSegment, _state, sizeof(_state)); @@ -2573,8 +2667,19 @@ IOMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void *vData, UInt data data->fMapContig = true; err = md->dmaMap(data->fMapper, data->fCommand, &data->fMapSpec, data->fOffset, data->fLength, &data->fAlloc, &data->fAllocLength); + return (err); } + else if (kIOMDDMAUnmap == op) + { + if (dataSize < sizeof(IOMDDMAMapArgs)) + return kIOReturnUnderrun; + IOMDDMAMapArgs * data = (IOMDDMAMapArgs *) vData; + + err = md->dmaUnmap(data->fMapper, data->fCommand, data->fOffset, data->fAlloc, data->fAllocLength); + + return (kIOReturnSuccess); + } else return kIOReturnBadArgument; return kIOReturnSuccess; @@ -2589,6 +2694,9 @@ IOGeneralMemoryDescriptor::setPurgeable( IOOptionBits newState, vm_purgable_t control; int state; + assert (!(kIOMemoryRemote & _flags)); + if (kIOMemoryRemote & _flags) return (kIOReturnNotAttached); + if (_memRef) { err = super::setPurgeable(newState, oldState); @@ -2612,7 +2720,14 @@ IOGeneralMemoryDescriptor::setPurgeable( IOOptionBits newState, break; } else + { curMap = get_task_map(_task); + if (NULL == curMap) + { + err = KERN_INVALID_ARGUMENT; + break; + } + } // can only do one range Ranges vec = _ranges; @@ -2624,7 +2739,7 @@ IOGeneralMemoryDescriptor::setPurgeable( IOOptionBits newState, err = purgeableControlBits(newState, &control, &state); if (kIOReturnSuccess != err) break; - err = mach_vm_purgable_control(curMap, addr, control, &state); + err = vm_map_purgable_control(curMap, addr, control, &state); if (oldState) { if (kIOReturnSuccess == err) @@ -2659,6 +2774,9 @@ IOReturn IOMemoryDescriptor::getPageCounts( IOByteCount * residentPageCount, { IOReturn err = kIOReturnNotReady; + assert (!(kIOMemoryRemote & _flags)); + if (kIOMemoryRemote & _flags) return (kIOReturnNotAttached); + if (kIOMemoryThreadSafe & _flags) LOCK; if (_memRef) err = IOGeneralMemoryDescriptor::memoryReferenceGetPageCounts(_memRef, residentPageCount, dirtyPageCount); else @@ -2680,8 +2798,13 @@ IOReturn IOMemoryDescriptor::getPageCounts( IOByteCount * residentPageCount, } +#if defined(__arm__) || defined(__arm64__) +extern "C" void dcache_incoherent_io_flush64(addr64_t pa, unsigned int count, unsigned int remaining, unsigned int *res); +extern "C" void dcache_incoherent_io_store64(addr64_t pa, unsigned int count, unsigned int remaining, unsigned int *res); +#else /* defined(__arm__) || defined(__arm64__) */ extern "C" void dcache_incoherent_io_flush64(addr64_t pa, unsigned int count); extern "C" void dcache_incoherent_io_store64(addr64_t pa, unsigned int count); +#endif /* defined(__arm__) || defined(__arm64__) */ static void SetEncryptOp(addr64_t pa, unsigned int count) { @@ -2713,15 +2836,41 @@ IOReturn IOMemoryDescriptor::performOperation( IOOptionBits options, IOByteCount remaining; unsigned int res; void (*func)(addr64_t pa, unsigned int count) = 0; +#if defined(__arm__) || defined(__arm64__) + void (*func_ext)(addr64_t pa, unsigned int count, unsigned int remaining, unsigned int *result) = 0; +#endif + + assert (!(kIOMemoryRemote & _flags)); + if (kIOMemoryRemote & _flags) return (kIOReturnNotAttached); switch (options) { case kIOMemoryIncoherentIOFlush: +#if defined(__arm__) || defined(__arm64__) + func_ext = &dcache_incoherent_io_flush64; +#if __ARM_COHERENT_IO__ + func_ext(0, 0, 0, &res); + return kIOReturnSuccess; +#else /* __ARM_COHERENT_IO__ */ + break; +#endif /* __ARM_COHERENT_IO__ */ +#else /* defined(__arm__) || defined(__arm64__) */ func = &dcache_incoherent_io_flush64; break; +#endif /* defined(__arm__) || defined(__arm64__) */ case kIOMemoryIncoherentIOStore: +#if defined(__arm__) || defined(__arm64__) + func_ext = &dcache_incoherent_io_store64; +#if __ARM_COHERENT_IO__ + func_ext(0, 0, 0, &res); + return kIOReturnSuccess; +#else /* __ARM_COHERENT_IO__ */ + break; +#endif /* __ARM_COHERENT_IO__ */ +#else /* defined(__arm__) || defined(__arm64__) */ func = &dcache_incoherent_io_store64; break; +#endif /* defined(__arm__) || defined(__arm64__) */ case kIOMemorySetEncrypted: func = &SetEncryptOp; @@ -2731,8 +2880,13 @@ IOReturn IOMemoryDescriptor::performOperation( IOOptionBits options, break; } +#if defined(__arm__) || defined(__arm64__) + if ((func == 0) && (func_ext == 0)) + return (kIOReturnUnsupported); +#else /* defined(__arm__) || defined(__arm64__) */ if (!func) return (kIOReturnUnsupported); +#endif /* defined(__arm__) || defined(__arm64__) */ if (kIOMemoryThreadSafe & _flags) LOCK; @@ -2753,7 +2907,19 @@ IOReturn IOMemoryDescriptor::performOperation( IOOptionBits options, if (dstLen > remaining) dstLen = remaining; +#if defined(__arm__) || defined(__arm64__) + if (func) + (*func)(dstAddr64, dstLen); + if (func_ext) { + (*func_ext)(dstAddr64, dstLen, remaining, &res); + if (res != 0x0UL) { + remaining = 0; + break; + } + } +#else /* defined(__arm__) || defined(__arm64__) */ (*func)(dstAddr64, dstLen); +#endif /* defined(__arm__) || defined(__arm64__) */ offset += dstLen; remaining -= dstLen; @@ -2774,6 +2940,18 @@ IOReturn IOMemoryDescriptor::performOperation( IOOptionBits options, #define io_kernel_static_start vm_kernel_stext #define io_kernel_static_end vm_kernel_etext +#elif defined(__arm__) || defined(__arm64__) + +extern vm_offset_t static_memory_end; + +#if defined(__arm64__) +#define io_kernel_static_start vm_kext_base +#else /* defined(__arm64__) */ +#define io_kernel_static_start vm_kernel_stext +#endif /* defined(__arm64__) */ + +#define io_kernel_static_end static_memory_end + #else #error io_kernel_static_end is undefined for this architecture #endif @@ -2818,43 +2996,6 @@ io_get_kernel_static_upl( return ((page >= pageCount) ? kIOReturnSuccess : kIOReturnVMError); } -/* - * - */ -#if IOTRACKING -static void -IOMemoryDescriptorUpdateWireOwner(ioGMDData * dataP, OSData * memoryEntries, vm_tag_t tag) -{ - ioPLBlock *ioplList; - UInt ind, count; - vm_tag_t prior; - - count = getNumIOPL(memoryEntries, dataP); - if (!count) return; - ioplList = getIOPLList(dataP); - - if (VM_KERN_MEMORY_NONE == tag) tag = dataP->fAllocTag; - assert(VM_KERN_MEMORY_NONE != tag); - - for (ind = 0; ind < count; ind++) - { - if (!ioplList[ind].fIOPL) continue; - prior = iopl_set_tag(ioplList[ind].fIOPL, tag); - if (VM_KERN_MEMORY_NONE == dataP->fAllocTag) dataP->fAllocTag = prior; -#if 0 - if (tag != prior) - { - char name[2][48]; - vm_tag_get_kext(prior, &name[0][0], sizeof(name[0])); - vm_tag_get_kext(tag, &name[1][0], sizeof(name[1])); - IOLog("switched %48s to %48s\n", name[0], name[1]); - } -#endif - } -} -#endif /* IOTRACKING */ - - IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection) { IOOptionBits type = _flags & kIOMemoryTypeMask; @@ -2862,28 +3003,33 @@ IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection) ioGMDData *dataP; upl_page_info_array_t pageInfo; ppnum_t mapBase; + vm_tag_t tag = VM_KERN_MEMORY_NONE; assert(kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type); if ((kIODirectionOutIn & forDirection) == kIODirectionNone) forDirection = (IODirection) (forDirection | getDirection()); + dataP = getDataP(_memoryEntries); upl_control_flags_t uplFlags; // This Mem Desc's default flags for upl creation switch (kIODirectionOutIn & forDirection) { - case kIODirectionOut: - // Pages do not need to be marked as dirty on commit - uplFlags = UPL_COPYOUT_FROM; - break; + case kIODirectionOut: + // Pages do not need to be marked as dirty on commit + uplFlags = UPL_COPYOUT_FROM; + dataP->fDMAAccess = kIODMAMapReadAccess; + break; - case kIODirectionIn: - default: - uplFlags = 0; // i.e. ~UPL_COPYOUT_FROM - break; - } - dataP = getDataP(_memoryEntries); + case kIODirectionIn: + dataP->fDMAAccess = kIODMAMapWriteAccess; + uplFlags = 0; // i.e. ~UPL_COPYOUT_FROM + break; - if (kIODirectionDMACommand & forDirection) assert(_wireCount); + default: + dataP->fDMAAccess = kIODMAMapReadAccess | kIODMAMapWriteAccess; + uplFlags = 0; // i.e. ~UPL_COPYOUT_FROM + break; + } if (_wireCount) { @@ -2896,11 +3042,13 @@ IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection) else { IOMapper *mapper; + mapper = dataP->fMapper; - dataP->fMappedBase = 0; + dataP->fMappedBaseValid = dataP->fMappedBase = 0; uplFlags |= UPL_SET_IO_WIRE | UPL_SET_LITE; - uplFlags |= UPL_MEMORY_TAG_MAKE(getVMTag(kernel_map)); + tag = _kernelTag; + if (VM_KERN_MEMORY_NONE == tag) tag = IOMemoryTag(kernel_map); if (kIODirectionPrepareToPhys32 & forDirection) { @@ -3004,7 +3152,8 @@ IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection) &iopl.fIOPL, baseInfo, &numPageInfo, - &ioplFlags); + &ioplFlags, + tag); } else { assert(theMap); @@ -3014,7 +3163,8 @@ IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection) &iopl.fIOPL, baseInfo, &numPageInfo, - &ioplFlags); + &ioplFlags, + tag); } if (error != KERN_SUCCESS) goto abortExit; @@ -3071,23 +3221,13 @@ IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection) } #if IOTRACKING - if (kIOReturnSuccess == error) + if (!(_flags & kIOMemoryAutoPrepare) && (kIOReturnSuccess == error)) { - vm_tag_t tag; - - dataP = getDataP(_memoryEntries); - if (forDirection & kIODirectionDMACommand) tag = (forDirection & kIODirectionDMACommandMask) >> kIODirectionDMACommandShift; - else tag = IOMemoryTag(kernel_map); - - if (!_wireCount) vm_tag_set_init(&dataP->fWireTags, kMaxWireTags); - vm_tag_set_enter(&dataP->fWireTags, kMaxWireTags, tag); - - IOMemoryDescriptorUpdateWireOwner(dataP, _memoryEntries, tag); - if (!_wireCount) - { - //if (!(_flags & kIOMemoryAutoPrepare)) - IOTrackingAdd(gIOWireTracking, &dataP->fWireTracking, ptoa(_pages), false); - } + dataP = getDataP(_memoryEntries); + if (!dataP->fWireTracking.link.next) + { + IOTrackingAdd(gIOWireTracking, &dataP->fWireTracking, ptoa(_pages), false, tag); + } } #endif /* IOTRACKING */ @@ -3145,6 +3285,7 @@ bool IOGeneralMemoryDescriptor::initMemoryEntries(size_t size, IOMapper * mapper dataP->fPreparationID = kIOPreparationIDUnprepared; dataP->fDiscontig = false; dataP->fCompletionError = false; + dataP->fMappedBaseValid = false; return (true); } @@ -3158,16 +3299,83 @@ IOReturn IOMemoryDescriptor::dmaMap( uint64_t * mapAddress, uint64_t * mapLength) { - IOReturn ret; + IOReturn err; uint32_t mapOptions; mapOptions = 0; mapOptions |= kIODMAMapReadAccess; if (!(kIOMemoryPreparedReadOnly & _flags)) mapOptions |= kIODMAMapWriteAccess; - ret = mapper->iovmMapMemory(this, offset, length, mapOptions, + err = mapper->iovmMapMemory(this, offset, length, mapOptions, mapSpec, command, NULL, mapAddress, mapLength); + if (kIOReturnSuccess == err) dmaMapRecord(mapper, command, *mapLength); + + return (err); +} + +void IOMemoryDescriptor::dmaMapRecord( + IOMapper * mapper, + IODMACommand * command, + uint64_t mapLength) +{ + kern_allocation_name_t alloc; + int16_t prior; + + if ((alloc = mapper->fAllocName) /* && mapper != IOMapper::gSystem */) + { + kern_allocation_update_size(mapper->fAllocName, mapLength); + } + + if (!command) return; + prior = OSAddAtomic16(1, &_dmaReferences); + if (!prior) + { + if (alloc && (VM_KERN_MEMORY_NONE != _kernelTag)) + { + _mapName = alloc; + mapLength = _length; + kern_allocation_update_subtotal(alloc, _kernelTag, mapLength); + } + else _mapName = NULL; + } +} + +IOReturn IOMemoryDescriptor::dmaUnmap( + IOMapper * mapper, + IODMACommand * command, + uint64_t offset, + uint64_t mapAddress, + uint64_t mapLength) +{ + IOReturn ret; + kern_allocation_name_t alloc; + kern_allocation_name_t mapName; + int16_t prior; + + mapName = 0; + prior = 0; + if (command) + { + mapName = _mapName; + if (_dmaReferences) prior = OSAddAtomic16(-1, &_dmaReferences); + else panic("_dmaReferences underflow"); + } + + if (!mapLength) return (kIOReturnSuccess); + + ret = mapper->iovmUnmapMemory(this, command, mapAddress, mapLength); + + if ((alloc = mapper->fAllocName)) + { + kern_allocation_update_size(alloc, -mapLength); + if ((1 == prior) && mapName && (VM_KERN_MEMORY_NONE != _kernelTag)) + { + mapLength = _length; + kern_allocation_update_subtotal(mapName, _kernelTag, -mapLength); + } + } + return (ret); } @@ -3186,6 +3394,7 @@ IOReturn IOGeneralMemoryDescriptor::dmaMap( *mapAddress = 0; if (kIOMemoryHostOnly & _flags) return (kIOReturnSuccess); + if (kIOMemoryRemote & _flags) return (kIOReturnNotAttached); if ((type == kIOMemoryTypePhysical) || (type == kIOMemoryTypePhysical64) || offset || (length != _length)) @@ -3217,8 +3426,8 @@ IOReturn IOGeneralMemoryDescriptor::dmaMap( mapOptions |= kIODMAMapPageListFullyOccupied; } - mapOptions |= kIODMAMapReadAccess; - if (!(kIOMemoryPreparedReadOnly & _flags)) mapOptions |= kIODMAMapWriteAccess; + assert(dataP->fDMAAccess); + mapOptions |= dataP->fDMAAccess; // Check for direct device non-paged memory if (ioplList->fFlags & kIOPLOnDevice) mapOptions |= kIODMAMapPhysicallyContiguous; @@ -3231,6 +3440,8 @@ IOReturn IOGeneralMemoryDescriptor::dmaMap( }; err = mapper->iovmMapMemory(this, offset, length, mapOptions, &mapSpec, command, &dmaPageList, mapAddress, mapLength); + + if (kIOReturnSuccess == err) dmaMapRecord(mapper, command, *mapLength); } return (err); @@ -3254,20 +3465,17 @@ IOReturn IOGeneralMemoryDescriptor::prepare(IODirection forDirection) if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type)) return kIOReturnSuccess; + assert (!(kIOMemoryRemote & _flags)); + if (kIOMemoryRemote & _flags) return (kIOReturnNotAttached); + if (_prepareLock) IOLockLock(_prepareLock); - if (kIODirectionDMACommand & forDirection) - { -#if IOMD_DEBUG_DMAACTIVE - OSIncrementAtomic(&__iomd_reservedA); -#endif /* IOMD_DEBUG_DMAACTIVE */ - } if (kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type) { error = wireVirtual(forDirection); } - if ((kIOReturnSuccess == error) && !(kIODirectionDMACommand & forDirection)) + if (kIOReturnSuccess == error) { if (1 == ++_wireCount) { @@ -3300,6 +3508,9 @@ IOReturn IOGeneralMemoryDescriptor::complete(IODirection forDirection) if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type)) return kIOReturnSuccess; + assert (!(kIOMemoryRemote & _flags)); + if (kIOMemoryRemote & _flags) return (kIOReturnNotAttached); + if (_prepareLock) IOLockLock(_prepareLock); do { @@ -3308,26 +3519,6 @@ IOReturn IOGeneralMemoryDescriptor::complete(IODirection forDirection) dataP = getDataP(_memoryEntries); if (!dataP) break; -#if IOMD_DEBUG_DMAACTIVE - if (kIODirectionDMACommand & forDirection) - { - if (__iomd_reservedA) OSDecrementAtomic(&__iomd_reservedA); - else panic("kIOMDSetDMAInactive"); - } -#endif /* IOMD_DEBUG_DMAACTIVE */ -#if IOTRACKING - if (kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type) - { - vm_tag_t tag; - - if (forDirection & kIODirectionDMACommand) tag = (forDirection & kIODirectionDMACommandMask) >> kIODirectionDMACommandShift; - else tag = IOMemoryTag(kernel_map); - vm_tag_set_remove(&dataP->fWireTags, kMaxWireTags, tag, &tag); - IOMemoryDescriptorUpdateWireOwner(dataP, _memoryEntries, tag); - } - if (kIODirectionDMACommand & forDirection) break; -#endif /* IOTRACKING */ - if (kIODirectionCompleteWithError & forDirection) dataP->fCompletionError = true; if ((kIOMemoryClearEncrypt & _flags) && (1 == _wireCount)) @@ -3346,30 +3537,28 @@ IOReturn IOGeneralMemoryDescriptor::complete(IODirection forDirection) // kIODirectionCompleteWithDataValid & forDirection if (kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type) { + vm_tag_t tag; + tag = getVMTag(kernel_map); for (ind = 0; ind < count; ind++) { - if (ioplList[ind].fIOPL) iopl_valid_data(ioplList[ind].fIOPL); + if (ioplList[ind].fIOPL) iopl_valid_data(ioplList[ind].fIOPL, tag); } } } else { -#if IOMD_DEBUG_DMAACTIVE - if (__iomd_reservedA) panic("complete() while dma active"); -#endif /* IOMD_DEBUG_DMAACTIVE */ + if (_dmaReferences) panic("complete() while dma active"); - if (dataP->fMappedBase) { - dataP->fMapper->iovmUnmapMemory(this, NULL, dataP->fMappedBase, dataP->fMappedLength); - dataP->fMappedBase = 0; + if (dataP->fMappedBaseValid) { + dmaUnmap(dataP->fMapper, NULL, 0, dataP->fMappedBase, dataP->fMappedLength); + dataP->fMappedBaseValid = dataP->fMappedBase = 0; } - // Only complete iopls that we created which are for TypeVirtual - if (kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type) { #if IOTRACKING - //if (!(_flags & kIOMemoryAutoPrepare)) - { - IOTrackingRemove(gIOWireTracking, &dataP->fWireTracking, ptoa(_pages)); - } + if (dataP->fWireTracking.link.next) IOTrackingRemove(gIOWireTracking, &dataP->fWireTracking, ptoa(_pages)); #endif /* IOTRACKING */ + // Only complete iopls that we created which are for TypeVirtual + if (kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type) + { for (ind = 0; ind < count; ind++) if (ioplList[ind].fIOPL) { if (dataP->fCompletionError) @@ -3385,7 +3574,7 @@ IOReturn IOGeneralMemoryDescriptor::complete(IODirection forDirection) (void) _memoryEntries->initWithBytes(dataP, computeDataSize(0, 0)); // == setLength() dataP->fPreparationID = kIOPreparationIDUnprepared; - dataP->fAllocTag = VM_KERN_MEMORY_NONE; + _flags &= ~kIOMemoryPreparedReadOnly; } } } @@ -3422,6 +3611,9 @@ IOReturn IOGeneralMemoryDescriptor::doMap( if ((offset >= _length) || ((offset + length) > _length)) return( kIOReturnBadArgument ); + assert (!(kIOMemoryRemote & _flags)); + if (kIOMemoryRemote & _flags) return (0); + if (vec.v) getAddrLenForInd(range0Addr, range0Len, type, vec, 0); @@ -3479,12 +3671,11 @@ IOReturn IOGeneralMemoryDescriptor::doMap( size = round_page(mapping->fLength); flags = UPL_COPYOUT_FROM | UPL_SET_INTERNAL - | UPL_SET_LITE | UPL_SET_IO_WIRE | UPL_BLOCK_ACCESS - | UPL_MEMORY_TAG_MAKE(getVMTag(kernel_map)); + | UPL_SET_LITE | UPL_SET_IO_WIRE | UPL_BLOCK_ACCESS; if (KERN_SUCCESS != memory_object_iopl_request(_memRef->entries[0].entry, 0, &size, &redirUPL2, NULL, NULL, - &flags)) + &flags, getVMTag(kernel_map))) redirUPL2 = NULL; for (lock_count = 0; @@ -3747,11 +3938,14 @@ IOReturn IOMemoryDescriptor::populateDevicePager( // faulting in later can't take place from interrupt level. if ((addressMap == kernel_map) && !(kIOMemoryRedirected & _flags)) { - vm_fault(addressMap, - (vm_map_offset_t)trunc_page_64(address), - VM_PROT_READ|VM_PROT_WRITE, - FALSE, THREAD_UNINT, NULL, - (vm_map_offset_t)0); + err = vm_fault(addressMap, + (vm_map_offset_t)trunc_page_64(address), + options & kIOMapReadOnly ? VM_PROT_READ : VM_PROT_READ|VM_PROT_WRITE, + FALSE, VM_KERN_MEMORY_NONE, + THREAD_UNINT, NULL, + (vm_map_offset_t)0); + + if (KERN_SUCCESS != err) break; } sourceOffset += segLen - pageOffset; @@ -4103,8 +4297,7 @@ IOReturn IOMemoryMap::wireRange( prot = (kIODirectionOutIn & options); if (prot) { - prot |= VM_PROT_MEMORY_TAG_MAKE(fMemory->getVMTag(kernel_map)); - kr = vm_map_wire(fAddressMap, start, end, prot, FALSE); + kr = vm_map_wire_kernel(fAddressMap, start, end, prot, fMemory->getVMTag(kernel_map), FALSE); } else { @@ -4266,11 +4459,10 @@ IOReturn IOMemoryMap::redirect(IOMemoryDescriptor * newBackingMemory, { upl_size_t size = round_page(fLength); upl_control_flags_t flags = UPL_COPYOUT_FROM | UPL_SET_INTERNAL - | UPL_SET_LITE | UPL_SET_IO_WIRE | UPL_BLOCK_ACCESS - | UPL_MEMORY_TAG_MAKE(fMemory->getVMTag(kernel_map)); + | UPL_SET_LITE | UPL_SET_IO_WIRE | UPL_BLOCK_ACCESS; if (KERN_SUCCESS != memory_object_iopl_request(fMemory->_memRef->entries[0].entry, 0, &size, &fRedirUPL, NULL, NULL, - &flags)) + &flags, fMemory->getVMTag(kernel_map))) fRedirUPL = 0; if (physMem) diff --git a/iokit/Kernel/IONVRAM.cpp b/iokit/Kernel/IONVRAM.cpp index 6a819b459..bf7a07032 100644 --- a/iokit/Kernel/IONVRAM.cpp +++ b/iokit/Kernel/IONVRAM.cpp @@ -139,9 +139,11 @@ void IODTNVRAM::initNVRAMImage(void) // Look through the partitions to find the OF, MacOS partitions. while (currentOffset < kIODTNVRAMImageSize) { currentLength = ((UInt16 *)(_nvramImage + currentOffset))[1] * 16; - + + if (currentLength < 16) break; partitionOffset = currentOffset + 16; partitionLength = currentLength - 16; + if ((partitionOffset + partitionLength) > kIODTNVRAMImageSize) break; if (strncmp((const char *)_nvramImage + currentOffset + 4, kIODTNVRAMOFPartitionName, 12) == 0) { @@ -582,7 +584,7 @@ IOReturn IODTNVRAM::readNVRAMPartition(const OSSymbol *partitionID, IOByteCount length) { OSNumber *partitionOffsetNumber, *partitionLengthNumber; - UInt32 partitionOffset, partitionLength; + UInt32 partitionOffset, partitionLength, end; partitionOffsetNumber = (OSNumber *)_nvramPartitionOffsets->getObject(partitionID); @@ -595,8 +597,8 @@ IOReturn IODTNVRAM::readNVRAMPartition(const OSSymbol *partitionID, partitionOffset = partitionOffsetNumber->unsigned32BitValue(); partitionLength = partitionLengthNumber->unsigned32BitValue(); - if ((buffer == 0) || (length == 0) || - (offset + length > partitionLength)) + if (os_add_overflow(offset, length, &end)) return kIOReturnBadArgument; + if ((buffer == 0) || (length == 0) || (end > partitionLength)) return kIOReturnBadArgument; bcopy(_nvramImage + partitionOffset + offset, buffer, length); @@ -609,7 +611,7 @@ IOReturn IODTNVRAM::writeNVRAMPartition(const OSSymbol *partitionID, IOByteCount length) { OSNumber *partitionOffsetNumber, *partitionLengthNumber; - UInt32 partitionOffset, partitionLength; + UInt32 partitionOffset, partitionLength, end; partitionOffsetNumber = (OSNumber *)_nvramPartitionOffsets->getObject(partitionID); @@ -622,8 +624,8 @@ IOReturn IODTNVRAM::writeNVRAMPartition(const OSSymbol *partitionID, partitionOffset = partitionOffsetNumber->unsigned32BitValue(); partitionLength = partitionLengthNumber->unsigned32BitValue(); - if ((buffer == 0) || (length == 0) || - (offset + length > partitionLength)) + if (os_add_overflow(offset, length, &end)) return kIOReturnBadArgument; + if ((buffer == 0) || (length == 0) || (end > partitionLength)) return kIOReturnBadArgument; bcopy(buffer, _nvramImage + partitionOffset + offset, length); @@ -736,7 +738,6 @@ IOReturn IODTNVRAM::initOFVariables(void) } } - // Create the 'aapl,panic-info' property if needed. if (_piImage != 0) { propDataLength = *(UInt32 *)_piImage; if ((propDataLength != 0) && (propDataLength <= (_piPartitionSize - 4))) { @@ -824,6 +825,7 @@ enum { kOWVariableOffsetString = 17 }; +static const OFVariable gOFVariables[] = { {"little-endian?", kOFVariableTypeBoolean, kOFVariablePermUserRead, 0}, {"real-mode?", kOFVariableTypeBoolean, kOFVariablePermUserRead, 1}, @@ -872,12 +874,22 @@ OFVariable gOFVariables[] = { {"security-password", kOFVariableTypeData, kOFVariablePermRootOnly, -1}, {"boot-image", kOFVariableTypeData, kOFVariablePermUserWrite, -1}, {"com.apple.System.fp-state", kOFVariableTypeData, kOFVariablePermKernelOnly, -1}, +#if CONFIG_EMBEDDED + {"backlight-level", kOFVariableTypeData, kOFVariablePermUserWrite, -1}, + {"com.apple.System.sep.art", kOFVariableTypeData, kOFVariablePermKernelOnly, -1}, + {"com.apple.System.boot-nonce", kOFVariableTypeString, kOFVariablePermKernelOnly, -1}, + {"darkboot", kOFVariableTypeBoolean, kOFVariablePermUserWrite, -1}, + {"acc-mb-ld-lifetime", kOFVariableTypeNumber, kOFVariablePermKernelOnly, -1}, + {"acc-cm-override-charger-count", kOFVariableTypeNumber, kOFVariablePermKernelOnly, -1}, + {"acc-cm-override-count", kOFVariableTypeNumber, kOFVariablePermKernelOnly, -1}, + {"enter-tdm-mode", kOFVariableTypeBoolean, kOFVariablePermUserWrite, -1}, +#endif {0, kOFVariableTypeData, kOFVariablePermUserRead, -1} }; UInt32 IODTNVRAM::getOFVariableType(const OSSymbol *propSymbol) const { - OFVariable *ofVar; + const OFVariable *ofVar; ofVar = gOFVariables; while (1) { @@ -891,7 +903,7 @@ UInt32 IODTNVRAM::getOFVariableType(const OSSymbol *propSymbol) const UInt32 IODTNVRAM::getOFVariablePerm(const OSSymbol *propSymbol) const { - OFVariable *ofVar; + const OFVariable *ofVar; ofVar = gOFVariables; while (1) { @@ -906,7 +918,7 @@ UInt32 IODTNVRAM::getOFVariablePerm(const OSSymbol *propSymbol) const bool IODTNVRAM::getOWVariableInfo(UInt32 variableNumber, const OSSymbol **propSymbol, UInt32 *propType, UInt32 *propOffset) { - OFVariable *ofVar; + const OFVariable *ofVar; ofVar = gOFVariables; while (1) { @@ -999,7 +1011,7 @@ bool IODTNVRAM::convertObjectToProp(UInt8 *buffer, UInt32 *length, const OSSymbol *propSymbol, OSObject *propObject) { const UInt8 *propName; - UInt32 propNameLength, propDataLength; + UInt32 propNameLength, propDataLength, remaining; UInt32 propType, tmpValue; OSBoolean *tmpBoolean = 0; OSNumber *tmpNumber = 0; @@ -1043,29 +1055,30 @@ bool IODTNVRAM::convertObjectToProp(UInt8 *buffer, UInt32 *length, // Copy the property name equal sign. buffer += snprintf((char *)buffer, *length, "%s=", propName); - + remaining = *length - propNameLength - 1; + switch (propType) { case kOFVariableTypeBoolean : if (tmpBoolean->getValue()) { - strlcpy((char *)buffer, "true", *length - propNameLength); + strlcpy((char *)buffer, "true", remaining); } else { - strlcpy((char *)buffer, "false", *length - propNameLength); + strlcpy((char *)buffer, "false", remaining); } break; case kOFVariableTypeNumber : tmpValue = tmpNumber->unsigned32BitValue(); if (tmpValue == 0xFFFFFFFF) { - strlcpy((char *)buffer, "-1", *length - propNameLength); + strlcpy((char *)buffer, "-1", remaining); } else if (tmpValue < 1000) { - snprintf((char *)buffer, *length - propNameLength, "%d", (uint32_t)tmpValue); + snprintf((char *)buffer, remaining, "%d", (uint32_t)tmpValue); } else { - snprintf((char *)buffer, *length - propNameLength, "0x%x", (uint32_t)tmpValue); + snprintf((char *)buffer, remaining, "0x%x", (uint32_t)tmpValue); } break; case kOFVariableTypeString : - strlcpy((char *)buffer, tmpString->getCStringNoCopy(), *length - propNameLength); + strlcpy((char *)buffer, tmpString->getCStringNoCopy(), remaining); break; case kOFVariableTypeData : diff --git a/iokit/Kernel/IOPMrootDomain.cpp b/iokit/Kernel/IOPMrootDomain.cpp index 103a073c3..a28aa990f 100644 --- a/iokit/Kernel/IOPMrootDomain.cpp +++ b/iokit/Kernel/IOPMrootDomain.cpp @@ -57,6 +57,8 @@ #include <sys/vnode.h> #include <sys/vnode_internal.h> #include <sys/fcntl.h> +#include <os/log.h> +#include <pexpert/protos.h> #include <sys/time.h> #include "IOServicePrivate.h" // _IOServiceInterestNotifier @@ -83,14 +85,23 @@ __END_DECLS #define LOG(x...) \ do { kprintf(LOG_PREFIX x); } while (false) +#if DEVELOPMENT #define DLOG(x...) do { \ if (kIOLogPMRootDomain & gIOKitDebug) \ kprintf(LOG_PREFIX x); \ + else \ + os_log(OS_LOG_DEFAULT, LOG_PREFIX x); \ } while (false) +#else +#define DLOG(x...) do { \ + if (kIOLogPMRootDomain & gIOKitDebug) \ + kprintf(LOG_PREFIX x); \ +} while (false) +#endif #define DMSG(x...) do { \ if (kIOLogPMRootDomain & gIOKitDebug) { \ - kprintf(LOG_PREFIX x); IOLog(x); \ + kprintf(LOG_PREFIX x); \ } \ } while (false) @@ -185,7 +196,7 @@ static const OSSymbol *sleepMessagePEFunction = NULL; #define kIOPMSystemCapabilitiesKey "System Capabilities" #define kIORequestWranglerIdleKey "IORequestIdle" -#define kDefaultWranglerIdlePeriod 25 // in milliseconds +#define kDefaultWranglerIdlePeriod 1000 // in milliseconds #define kIOSleepWakeDebugKey "Persistent-memory-note" #define kIOEFIBootRomFailureKey "wake-failure" @@ -307,6 +318,14 @@ static UInt32 gWillShutdown = 0; static UInt32 gPagingOff = 0; static UInt32 gSleepWakeUUIDIsSet = false; static uint32_t gAggressivesState = 0; +static uint32_t gHaltTimeMaxLog; +static uint32_t gHaltTimeMaxPanic; +IOLock * gHaltLogLock; +static char * gHaltLog; +enum { kHaltLogSize = 2048 }; +static size_t gHaltLogPos; +static uint64_t gHaltStartTime; + uuid_string_t bootsessionuuid_string; @@ -349,10 +368,10 @@ enum { kInformableCount = 2 }; -const OSSymbol *gIOPMStatsApplicationResponseTimedOut; -const OSSymbol *gIOPMStatsApplicationResponseCancel; -const OSSymbol *gIOPMStatsApplicationResponseSlow; -const OSSymbol *gIOPMStatsApplicationResponsePrompt; +const OSSymbol *gIOPMStatsResponseTimedOut; +const OSSymbol *gIOPMStatsResponseCancel; +const OSSymbol *gIOPMStatsResponseSlow; +const OSSymbol *gIOPMStatsResponsePrompt; const OSSymbol *gIOPMStatsDriverPSChangeSlow; #define kBadPMFeatureID 0 @@ -560,53 +579,115 @@ static void IOPMRootDomainWillShutdown(void) } } -extern "C" +extern "C" IONotifier * registerSleepWakeInterest(IOServiceInterestHandler handler, void * self, void * ref) { - IONotifier * registerSleepWakeInterest(IOServiceInterestHandler handler, void * self, void * ref) - { - return gRootDomain->registerInterest( gIOGeneralInterest, handler, self, ref ); - } + return gRootDomain->registerInterest( gIOGeneralInterest, handler, self, ref ); +} - IONotifier * registerPrioritySleepWakeInterest(IOServiceInterestHandler handler, void * self, void * ref) - { - return gRootDomain->registerInterest( gIOPriorityPowerStateInterest, handler, self, ref ); - } +extern "C" IONotifier * registerPrioritySleepWakeInterest(IOServiceInterestHandler handler, void * self, void * ref) +{ + return gRootDomain->registerInterest( gIOPriorityPowerStateInterest, handler, self, ref ); +} - IOReturn acknowledgeSleepWakeNotification(void * PMrefcon) - { - return gRootDomain->allowPowerChange ( (unsigned long)PMrefcon ); - } +extern "C" IOReturn acknowledgeSleepWakeNotification(void * PMrefcon) +{ + return gRootDomain->allowPowerChange ( (unsigned long)PMrefcon ); +} - IOReturn vetoSleepWakeNotification(void * PMrefcon) - { - return gRootDomain->cancelPowerChange ( (unsigned long)PMrefcon ); - } +extern "C" IOReturn vetoSleepWakeNotification(void * PMrefcon) +{ + return gRootDomain->cancelPowerChange ( (unsigned long)PMrefcon ); +} - IOReturn rootDomainRestart ( void ) - { - return gRootDomain->restartSystem(); +extern "C" IOReturn rootDomainRestart ( void ) +{ + return gRootDomain->restartSystem(); +} + +extern "C" IOReturn rootDomainShutdown ( void ) +{ + return gRootDomain->shutdownSystem(); +} + +static void halt_log_putc(char c) +{ + if (gHaltLogPos >= (kHaltLogSize - 1)) return; + gHaltLog[gHaltLogPos++] = c; +} + +extern "C" void +_doprnt_log(const char *fmt, + va_list *argp, + void (*putc)(char), + int radix); + +static int +halt_log(const char *fmt, ...) +{ + va_list listp; + + va_start(listp, fmt); + _doprnt_log(fmt, &listp, &halt_log_putc, 16); + va_end(listp); + + return (0); +} + +extern "C" void +halt_log_enter(const char * what, const void * pc, uint64_t time) +{ + uint64_t nano, millis; + + if (!gHaltLog) return; + absolutetime_to_nanoseconds(time, &nano); + millis = nano / 1000000ULL; + if (millis < 100) return; + + IOLockLock(gHaltLogLock); + if (pc) { + halt_log("%s: %qd ms @ 0x%lx, ", what, millis, VM_KERNEL_UNSLIDE(pc)); + OSKext::printKextsInBacktrace((vm_offset_t *) &pc, 1, &halt_log, + OSKext::kPrintKextsLock | OSKext::kPrintKextsUnslide | OSKext::kPrintKextsTerse); + } else { + halt_log("%s: %qd ms\n", what, millis, VM_KERNEL_UNSLIDE(pc)); } + IOLockUnlock(gHaltLogLock); +} - IOReturn rootDomainShutdown ( void ) +extern uint32_t gFSState; + +extern "C" void IOSystemShutdownNotification(void) +{ + uint64_t startTime; + + IOLockLock(gHaltLogLock); + if (!gHaltLog) { - return gRootDomain->shutdownSystem(); + gHaltLog = IONew(char, kHaltLogSize); + gHaltStartTime = mach_absolute_time(); + if (gHaltLog) halt_log_putc('\n'); } + IOLockUnlock(gHaltLogLock); - void IOSystemShutdownNotification(void) - { - IOPMRootDomainWillShutdown(); + startTime = mach_absolute_time(); + IOPMRootDomainWillShutdown(); + halt_log_enter("IOPMRootDomainWillShutdown", 0, mach_absolute_time() - startTime); #if HIBERNATION - IOHibernateSystemPostWake(); + startTime = mach_absolute_time(); + IOHibernateSystemPostWake(true); + halt_log_enter("IOHibernateSystemPostWake", 0, mach_absolute_time() - startTime); +#endif + if (OSCompareAndSwap(0, 1, &gPagingOff)) + { +#if !CONFIG_EMBEDDED + gRootDomain->handlePlatformHaltRestart(kPEPagingOff); #endif - if (OSCompareAndSwap(0, 1, &gPagingOff)) - { - gRootDomain->handlePlatformHaltRestart(kPEPagingOff); - } } - - int sync_internal(void); } + +extern "C" int sync_internal(void); + /* A device is always in the highest power state which satisfies its driver, its policy-maker, and any power children it has, but within the constraint @@ -715,7 +796,7 @@ void IOPMrootDomain::swdDebugSetup( ) swd_DebugImageSetup = TRUE; if (CAP_GAIN(kIOPMSystemCapabilityGraphics) || (CAP_LOSS(kIOPMSystemCapabilityGraphics))) { - IOHibernateSystemPostWakeTrim((void*)1, NULL); + IOHibernateSystemPostWake(true); } IOOpenDebugDataFile(kSleepWakeStackBinFilename, SWD_BUF_SIZE); } @@ -769,7 +850,7 @@ static void disk_sync_callout( thread_call_param_t p0, thread_call_param_t p1 ) else { swdDebugTeardownCallout(p0, NULL); - IOHibernateSystemPostWake(); + IOHibernateSystemPostWake(false); if (gRootDomain) gRootDomain->sleepWakeDebugSaveSpinDumpFile(); @@ -781,16 +862,18 @@ static void disk_sync_callout( thread_call_param_t p0, thread_call_param_t p1 ) } //****************************************************************************** -static UInt32 computeDeltaTimeMS( const AbsoluteTime * startTime ) +static UInt32 computeDeltaTimeMS( const AbsoluteTime * startTime, AbsoluteTime * elapsedTime ) { AbsoluteTime endTime; UInt64 nano = 0; clock_get_uptime(&endTime); - if (CMP_ABSOLUTETIME(&endTime, startTime) > 0) + if (CMP_ABSOLUTETIME(&endTime, startTime) <= 0) *elapsedTime = 0; + else { SUB_ABSOLUTETIME(&endTime, startTime); absolutetime_to_nanoseconds(endTime, &nano); + *elapsedTime = endTime; } return (UInt32)(nano / 1000000ULL); @@ -807,12 +890,12 @@ sysctl_sleepwaketime SYSCTL_HANDLER_ARGS if (p == kernproc) { return sysctl_io_opaque(req, swt, sizeof(*swt), NULL); } else if(proc_is64bit(p)) { - struct user64_timeval t; + struct user64_timeval t = {}; t.tv_sec = swt->tv_sec; t.tv_usec = swt->tv_usec; return sysctl_io_opaque(req, &t, sizeof(t), NULL); } else { - struct user32_timeval t; + struct user32_timeval t = {}; t.tv_sec = swt->tv_sec; t.tv_usec = swt->tv_usec; return sysctl_io_opaque(req, &t, sizeof(t), NULL); @@ -850,7 +933,9 @@ static SYSCTL_PROC(_kern, OID_AUTO, willshutdown, 0, 0, sysctl_willshutdown, "I", ""); extern struct sysctl_oid sysctl__kern_iokittest; +extern struct sysctl_oid sysctl__debug_iokit; +#if !CONFIG_EMBEDDED static int sysctl_progressmeterenable @@ -888,6 +973,7 @@ static SYSCTL_PROC(_kern, OID_AUTO, progressmeter, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED, 0, 0, sysctl_progressmeter, "I", ""); +#endif /* !CONFIG_EMBEDDED */ @@ -1002,10 +1088,10 @@ bool IOPMrootDomain::start( IOService * nub ) gIOPMUserTriggeredFullWakeKey = OSSymbol::withCStringNoCopy(kIOPMUserTriggeredFullWakeKey); gIOPMUserIsActiveKey = OSSymbol::withCStringNoCopy(kIOPMUserIsActiveKey); - gIOPMStatsApplicationResponseTimedOut = OSSymbol::withCString(kIOPMStatsResponseTimedOut); - gIOPMStatsApplicationResponseCancel = OSSymbol::withCString(kIOPMStatsResponseCancel); - gIOPMStatsApplicationResponseSlow = OSSymbol::withCString(kIOPMStatsResponseSlow); - gIOPMStatsApplicationResponsePrompt = OSSymbol::withCString(kIOPMStatsResponsePrompt); + gIOPMStatsResponseTimedOut = OSSymbol::withCString(kIOPMStatsResponseTimedOut); + gIOPMStatsResponseCancel = OSSymbol::withCString(kIOPMStatsResponseCancel); + gIOPMStatsResponseSlow = OSSymbol::withCString(kIOPMStatsResponseSlow); + gIOPMStatsResponsePrompt = OSSymbol::withCString(kIOPMStatsResponsePrompt); gIOPMStatsDriverPSChangeSlow = OSSymbol::withCString(kIOPMStatsDriverPSChangeSlow); sleepSupportedPEFunction = OSSymbol::withCString("IOPMSetSleepSupported"); @@ -1034,6 +1120,8 @@ bool IOPMrootDomain::start( IOService * nub ) PE_parse_boot_argn("darkwake", &gDarkWakeFlags, sizeof(gDarkWakeFlags)); PE_parse_boot_argn("noidle", &gNoIdleFlag, sizeof(gNoIdleFlag)); + PE_parse_boot_argn("haltmspanic", &gHaltTimeMaxPanic, sizeof(gHaltTimeMaxPanic)); + PE_parse_boot_argn("haltmslog", &gHaltTimeMaxLog, sizeof(gHaltTimeMaxLog)); queue_init(&aggressivesQueue); aggressivesThreadCall = thread_call_allocate(handleAggressivesFunction, this); @@ -1043,6 +1131,7 @@ bool IOPMrootDomain::start( IOService * nub ) featuresDictLock = IOLockAlloc(); settingsCtrlLock = IOLockAlloc(); wakeEventLock = IOLockAlloc(); + gHaltLogLock = IOLockAlloc(); setPMRootDomain(this); extraSleepTimer = thread_call_allocate( @@ -1228,11 +1317,14 @@ bool IOPMrootDomain::start( IOService * nub ) sysctl_register_oid(&sysctl__kern_waketime); sysctl_register_oid(&sysctl__kern_willshutdown); sysctl_register_oid(&sysctl__kern_iokittest); + sysctl_register_oid(&sysctl__debug_iokit); sysctl_register_oid(&sysctl__hw_targettype); +#if !CONFIG_EMBEDDED sysctl_register_oid(&sysctl__kern_progressmeterenable); sysctl_register_oid(&sysctl__kern_progressmeter); sysctl_register_oid(&sysctl__kern_wakereason); +#endif /* !CONFIG_EMBEDDED */ sysctl_register_oid(&sysctl__kern_consoleoptions); sysctl_register_oid(&sysctl__kern_progressoptions); @@ -1386,6 +1478,7 @@ IOReturn IOPMrootDomain::setProperties( OSObject * props_obj ) setProperty(key, b); } else if (key->isEqualTo(kIOPMDeepSleepDelayKey) || + key->isEqualTo(kIOPMDeepSleepTimerKey) || key->isEqualTo(kIOPMAutoPowerOffDelayKey) || key->isEqualTo(kIOPMAutoPowerOffTimerKey)) { @@ -2191,6 +2284,9 @@ IOReturn IOPMrootDomain::privateSleepSystem( uint32_t sleepReason ) void IOPMrootDomain::powerChangeDone( unsigned long previousPowerState ) { +#if !__i386__ && !__x86_64__ + uint64_t timeSinceReset = 0; +#endif uint64_t now; ASSERT_GATED(); DLOG("PowerChangeDone: %u->%u\n", @@ -2273,7 +2369,7 @@ void IOPMrootDomain::powerChangeDone( unsigned long previousPowerState ) clock_sec_t wakeSecs; clock_usec_t wakeMicrosecs; - clock_initialize_calendar(); + clock_wakeup_calendar(); clock_get_calendar_microtime(&wakeSecs, &wakeMicrosecs); gIOLastWakeTime.tv_sec = wakeSecs; @@ -2438,7 +2534,9 @@ void IOPMrootDomain::powerChangeDone( unsigned long previousPowerState ) } } #else /* !__i386__ && !__x86_64__ */ - kdebugTrace(kPMLogSystemWake, 0, ml_get_wake_timebase() >> 32, ml_get_wake_timebase()); + timeSinceReset = ml_get_time_since_reset(); + + kdebugTrace(kPMLogSystemWake, 0, timeSinceReset >> 32, timeSinceReset); // stay awake for at least 30 seconds wranglerTickled = true; fullWakeReason = kFullWakeReasonLocalUser; @@ -2547,16 +2645,23 @@ bool IOPMrootDomain::updatePreventIdleSleepList( #if defined(__i386__) || defined(__x86_64__) if (addNotRemove && (service == wrangler) && !checkSystemCanSustainFullWake()) { + DLOG("Cannot cancel idle sleep\n"); return false; // do not idle-cancel } #endif - MSG("prevent idle sleep list: %s%c (%u)\n", - service->getName(), - (addNotRemove) ? '+' : '-', newCount); return true; } +//****************************************************************************** +// startSpinDump +//****************************************************************************** + +void IOPMrootDomain::startSpinDump(uint32_t spindumpKind) +{ + messageClients(kIOPMMessageLaunchBootSpinDump, (void *)(uintptr_t)spindumpKind); +} + //****************************************************************************** // preventSystemSleepListUpdate // @@ -2681,21 +2786,23 @@ bool IOPMrootDomain::tellChangeDown( unsigned long stateNum ) tracePoint( kIOPMTracePointSleepPriorityClients ); } - if ((SLEEP_STATE == stateNum) && !ignoreTellChangeDown) - { + if (!ignoreTellChangeDown) { userActivityAtSleep = userActivityCount; - hibernateAborted = false; DLOG("tellChangeDown::userActivityAtSleep %d\n", userActivityAtSleep); - // Direct callout into OSKext so it can disable kext unloads - // during sleep/wake to prevent deadlocks. - OSKextSystemSleepOrWake( kIOMessageSystemWillSleep ); + if (SLEEP_STATE == stateNum) { + hibernateAborted = false; - IOService::updateConsoleUsers(NULL, kIOMessageSystemWillSleep); + // Direct callout into OSKext so it can disable kext unloads + // during sleep/wake to prevent deadlocks. + OSKextSystemSleepOrWake( kIOMessageSystemWillSleep ); - // Two change downs are sent by IOServicePM. Ignore the 2nd. - // But tellClientsWithResponse() must be called for both. - ignoreTellChangeDown = true; + IOService::updateConsoleUsers(NULL, kIOMessageSystemWillSleep); + + // Two change downs are sent by IOServicePM. Ignore the 2nd. + // But tellClientsWithResponse() must be called for both. + ignoreTellChangeDown = true; + } } return super::tellClientsWithResponse( kIOMessageSystemWillSleep ); @@ -3224,7 +3331,9 @@ void IOPMrootDomain::willNotifyPowerChildren( IOPMPowerStateIndex newPowerState tasks_system_suspend(tasksSuspended); clock_interval_to_deadline(10, kSecondScale, &deadline); +#if !CONFIG_EMBEDDED vm_pageout_wait(AbsoluteTime_to_scalar(&deadline)); +#endif /* !CONFIG_EMBEDDED */ } #if HIBERNATION @@ -4012,6 +4121,7 @@ bool IOPMrootDomain::evaluateSystemSleepPolicy( uint32_t standbyDelay = 0; uint32_t powerOffDelay = 0; uint32_t powerOffTimer = 0; + uint32_t standbyTimer = 0; uint32_t mismatch; bool standbyEnabled; bool powerOffEnabled; @@ -4031,9 +4141,11 @@ bool IOPMrootDomain::evaluateSystemSleepPolicy( && (getProperty(kIOPMAutoPowerOffEnabledKey) == kOSBooleanTrue)); if (!getSleepOption(kIOPMAutoPowerOffTimerKey, &powerOffTimer)) powerOffTimer = powerOffDelay; + if (!getSleepOption(kIOPMDeepSleepTimerKey, &standbyTimer)) + standbyTimer = standbyDelay; - DLOG("phase %d, standby %d delay %u, poweroff %d delay %u timer %u, hibernate 0x%x\n", - sleepPhase, standbyEnabled, standbyDelay, + DLOG("phase %d, standby %d delay %u timer %u, poweroff %d delay %u timer %u, hibernate 0x%x\n", + sleepPhase, standbyEnabled, standbyDelay, standbyTimer, powerOffEnabled, powerOffDelay, powerOffTimer, *hibMode); // pmset level overrides @@ -4069,7 +4181,7 @@ bool IOPMrootDomain::evaluateSystemSleepPolicy( currentFactors |= kIOPMSleepFactorACPower; if (lowBatteryCondition) currentFactors |= kIOPMSleepFactorBatteryLow; - if (!standbyDelay) + if (!standbyDelay || !standbyTimer) currentFactors |= kIOPMSleepFactorStandbyNoDelay; if (standbyNixed || !standbyEnabled) currentFactors |= kIOPMSleepFactorStandbyDisabled; @@ -4134,6 +4246,7 @@ bool IOPMrootDomain::evaluateSystemSleepPolicy( gSleepPolicyVars->sleepReason = lastSleepReason; gSleepPolicyVars->sleepPhase = sleepPhase; gSleepPolicyVars->standbyDelay = standbyDelay; + gSleepPolicyVars->standbyTimer = standbyTimer; gSleepPolicyVars->poweroffDelay = powerOffDelay; gSleepPolicyVars->scheduledAlarms = _scheduledAlarms | _userScheduledAlarm; gSleepPolicyVars->poweroffTimer = powerOffTimer; @@ -4332,7 +4445,7 @@ void IOPMrootDomain::evaluateSystemSleepPolicyFinal( void ) if (evaluateSystemSleepPolicy(¶ms, kIOPMSleepPhase2, &hibernateMode)) { if ((kIOPMSleepTypeStandby == params.sleepType) - && gIOHibernateStandbyDisabled + && gIOHibernateStandbyDisabled && gSleepPolicyVars && (!(kIOPMSleepFactorStandbyForced & gSleepPolicyVars->sleepFactors))) { standbyNixed = true; @@ -4446,11 +4559,11 @@ bool IOPMrootDomain::getSleepOption( const char * key, uint32_t * option ) if (optionsProp) optionsProp->release(); - return true; + return ok; } #endif /* HIBERNATION */ -IOReturn IOPMrootDomain::getSystemSleepType( uint32_t * sleepType ) +IOReturn IOPMrootDomain::getSystemSleepType( uint32_t * sleepType, uint32_t * standbyTimer ) { #if HIBERNATION IOPMSystemSleepParameters params; @@ -4463,7 +4576,7 @@ IOReturn IOPMrootDomain::getSystemSleepType( uint32_t * sleepType ) OSMemberFunctionCast(IOWorkLoop::Action, this, &IOPMrootDomain::getSystemSleepType), (OSObject *) this, - (void *) sleepType); + (void *) sleepType, (void *) standbyTimer); return ret; } @@ -4474,6 +4587,11 @@ IOReturn IOPMrootDomain::getSystemSleepType( uint32_t * sleepType ) if (ok) { *sleepType = params.sleepType; + if (!getSleepOption(kIOPMDeepSleepTimerKey, standbyTimer) && + !getSleepOption(kIOPMDeepSleepDelayKey, standbyTimer)) { + DLOG("Standby delay is not set\n"); + *standbyTimer = 0; + } return kIOReturnSuccess; } #endif @@ -4503,7 +4621,7 @@ platformHaltRestartApplier( OSObject * object, void * context ) { IOPowerStateChangeNotification notify; HaltRestartApplierContext * ctx; - AbsoluteTime startTime; + AbsoluteTime startTime, elapsedTime; uint32_t deltaTime; ctx = (HaltRestartApplierContext *) context; @@ -4516,7 +4634,7 @@ platformHaltRestartApplier( OSObject * object, void * context ) clock_get_uptime(&startTime); ctx->RootDomain->messageClient( ctx->MessageType, object, (void *)¬ify ); - deltaTime = computeDeltaTimeMS(&startTime); + deltaTime = computeDeltaTimeMS(&startTime, &elapsedTime); if ((deltaTime > kPMHaltTimeoutMS) || (gIOKitDebug & kIOLogPMRootDomain)) @@ -4531,6 +4649,7 @@ platformHaltRestartApplier( OSObject * object, void * context ) { LOG("%s handler %p took %u ms\n", ctx->LogString, OBFUSCATE(notifier->handler), deltaTime); + halt_log_enter(ctx->LogString, (const void *) notifier->handler, elapsedTime); } } @@ -4548,7 +4667,7 @@ static void quiescePowerTreeCallback( void * target, void * param ) void IOPMrootDomain::handlePlatformHaltRestart( UInt32 pe_type ) { HaltRestartApplierContext ctx; - AbsoluteTime startTime; + AbsoluteTime startTime, elapsedTime; uint32_t deltaTime; memset(&ctx, 0, sizeof(ctx)); @@ -4624,13 +4743,28 @@ void IOPMrootDomain::handlePlatformHaltRestart( UInt32 pe_type ) } } IOLockUnlock(gPMHaltLock); - - deltaTime = computeDeltaTimeMS(&quiesceTime); + deltaTime = computeDeltaTimeMS(&quiesceTime, &elapsedTime); DLOG("PM quiesce took %u ms\n", deltaTime); + halt_log_enter("Quiesce", NULL, elapsedTime); } - deltaTime = computeDeltaTimeMS(&startTime); + deltaTime = computeDeltaTimeMS(&startTime, &elapsedTime); LOG("%s all drivers took %u ms\n", ctx.LogString, deltaTime); + + halt_log_enter(ctx.LogString, NULL, elapsedTime); + if (gHaltLog) gHaltLog[gHaltLogPos] = 0; + + deltaTime = computeDeltaTimeMS(&gHaltStartTime, &elapsedTime); + LOG("%s total %u ms\n", ctx.LogString, deltaTime); + + if (gHaltLog && gHaltTimeMaxLog && (deltaTime >= gHaltTimeMaxLog)) + { + printf("%s total %d ms:%s\n", ctx.LogString, deltaTime, gHaltLog); + } + if (gHaltLog && gHaltTimeMaxPanic && (deltaTime >= gHaltTimeMaxPanic)) + { + panic("%s total %d ms:%s\n", ctx.LogString, deltaTime, gHaltLog); + } } //****************************************************************************** @@ -5132,10 +5266,20 @@ void IOPMrootDomain::handleOurPowerChangeDone( if (!CAP_CURRENT(kIOPMSystemCapabilityGraphics) && CAP_CURRENT(kIOPMSystemCapabilityCPU)) { +#if !CONFIG_EMBEDDED pmPowerStateQueue->submitPowerEvent( kPowerEventPolicyStimulus, (void *) kStimulusDarkWakeReentry, _systemStateGeneration ); +#else + // On embedded, there are no factors that can prolong a + // "darkWake" when a power down is vetoed. We need to + // promote to "fullWake" at least once so that factors + // that prevent idle sleep can assert themselves if required + pmPowerStateQueue->submitPowerEvent( + kPowerEventPolicyStimulus, + (void *) kStimulusDarkWakeActivityTickle); +#endif } // Revert device desire to max. @@ -5604,6 +5748,9 @@ protected: uint32_t ackTimeoutCnt; uint32_t msgType; // Message pending ack + uint64_t uuid0; + uint64_t uuid1; + const OSSymbol *identifier; }; OSDefineMetaClassAndStructors(IOPMServiceInterestNotifier, _IOServiceInterestNotifier) @@ -5633,7 +5780,7 @@ IONotifier * IOPMrootDomain::registerInterest( if (!notifier) return NULL; if (notifier->init()) { - rc = super::registerInterestForNotifer(notifier, typeOfInterest, handler, target, ref); + rc = super::registerInterestForNotifier(notifier, typeOfInterest, handler, target, ref); } if (rc != kIOReturnSuccess) { notifier->release(); @@ -5661,6 +5808,28 @@ IONotifier * IOPMrootDomain::registerInterest( } } + OSData *data = NULL; + uint8_t *uuid = NULL; + OSKext *kext = OSKext::lookupKextWithAddress((vm_address_t)handler); + if (kext) { + data = kext->copyUUID(); + } + if (data && (data->getLength() == sizeof(uuid_t))) { + uuid = (uint8_t *)(data->getBytesNoCopy()); + + notifier->uuid0 = ((uint64_t)(uuid[0]) << 56) | ((uint64_t)(uuid[1]) << 48) | ((uint64_t)(uuid[2]) << 40)| + ((uint64_t)(uuid[3]) << 32) | ((uint64_t)(uuid[4]) << 24) | ((uint64_t)(uuid[5]) << 16) | + ((uint64_t)(uuid[6]) << 8) | (uuid[7]); + notifier->uuid1 = ((uint64_t)(uuid[8]) << 56) | ((uint64_t)(uuid[9]) << 48) | ((uint64_t)(uuid[10]) << 40)| + ((uint64_t)(uuid[11]) << 32) | ((uint64_t)(uuid[12]) << 24) | ((uint64_t)(uuid[13]) << 16) | + ((uint64_t)(uuid[14]) << 8) | (uuid[15]); + + notifier->identifier = kext->getIdentifier(); + + } + if (kext) kext->release(); + if (data) data->release(); + return notifier; } @@ -5737,9 +5906,6 @@ bool IOPMrootDomain::systemMessageFilter( // app has not replied yet, wait for it *((OSObject **) arg3) = kOSBooleanFalse; - if (notifier) { - notifier->msgType = context->messageType; - } } allow = true; @@ -5757,9 +5923,6 @@ bool IOPMrootDomain::systemMessageFilter( if (object == (OSObject *) systemCapabilityNotifier) { allow = true; - if (notifier) { - notifier->msgType = context->messageType; - } break; } @@ -5775,9 +5938,6 @@ bool IOPMrootDomain::systemMessageFilter( if ((object == (OSObject *) systemCapabilityNotifier) && CAP_HIGHEST(kIOPMSystemCapabilityGraphics) && (fullToDarkReason == kIOPMSleepReasonIdle)) { - if (notifier) { - notifier->msgType = context->messageType; - } allow = true; } break; @@ -5805,8 +5965,6 @@ bool IOPMrootDomain::systemMessageFilter( else *((OSObject **) arg3) = kOSBooleanTrue; } - - notifier->msgType = context->messageType; } } else if ((context->notifyType == kNotifyPriority) && @@ -5828,6 +5986,9 @@ bool IOPMrootDomain::systemMessageFilter( _joinedCapabilityClients = 0; } } + if (notifier) { + notifier->msgType = context->messageType; + } return allow; } @@ -6329,20 +6490,22 @@ void IOPMrootDomain::handleDisplayPowerOn( ) void IOPMrootDomain::dispatchPowerEvent( uint32_t event, void * arg0, uint64_t arg1 ) { - DLOG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); ASSERT_GATED(); switch (event) { case kPowerEventFeatureChanged: + DMSG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); messageClients(kIOPMMessageFeatureChange, this); break; case kPowerEventReceivedPowerNotification: + DMSG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); handlePowerNotification( (UInt32)(uintptr_t) arg0 ); break; case kPowerEventSystemBootCompleted: + DLOG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); if (systemBooting) { systemBooting = false; @@ -6382,6 +6545,7 @@ void IOPMrootDomain::dispatchPowerEvent( break; case kPowerEventSystemShutdown: + DLOG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); if (kOSBooleanTrue == (OSBoolean *) arg0) { /* We set systemShutdown = true during shutdown @@ -6396,18 +6560,20 @@ void IOPMrootDomain::dispatchPowerEvent( systemShutdown = true; } else { /* - A shutdown was initiated, but then the shutdown - was cancelled, clearing systemShutdown to false here. - */ + A shutdown was initiated, but then the shutdown + was cancelled, clearing systemShutdown to false here. + */ systemShutdown = false; } break; case kPowerEventUserDisabledSleep: + DLOG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); userDisabledAllSleep = (kOSBooleanTrue == (OSBoolean *) arg0); break; case kPowerEventRegisterSystemCapabilityClient: + DLOG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); if (systemCapabilityNotifier) { systemCapabilityNotifier->release(); @@ -6422,6 +6588,7 @@ void IOPMrootDomain::dispatchPowerEvent( [[clang::fallthrough]]; case kPowerEventRegisterKernelCapabilityClient: + DLOG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); if (!_joinedCapabilityClients) _joinedCapabilityClients = OSSet::withCapacity(8); if (arg0) @@ -6437,6 +6604,7 @@ void IOPMrootDomain::dispatchPowerEvent( break; case kPowerEventPolicyStimulus: + DMSG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); if (arg0) { int stimulus = (uintptr_t) arg0; @@ -6445,6 +6613,7 @@ void IOPMrootDomain::dispatchPowerEvent( break; case kPowerEventAssertionCreate: + DMSG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); if (pmAssertions) { pmAssertions->handleCreateAssertion((OSData *)arg0); } @@ -6452,25 +6621,30 @@ void IOPMrootDomain::dispatchPowerEvent( case kPowerEventAssertionRelease: + DMSG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); if (pmAssertions) { pmAssertions->handleReleaseAssertion(arg1); } break; case kPowerEventAssertionSetLevel: + DMSG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); if (pmAssertions) { pmAssertions->handleSetAssertionLevel(arg1, (IOPMDriverAssertionLevel)(uintptr_t)arg0); } break; case kPowerEventQueueSleepWakeUUID: + DLOG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); handleQueueSleepWakeUUID((OSObject *)arg0); break; case kPowerEventPublishSleepWakeUUID: + DLOG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); handlePublishSleepWakeUUID((bool)arg0); break; case kPowerEventSetDisplayPowerOn: + DLOG("power event %u args %p 0x%llx\n", event, OBFUSCATE(arg0), arg1); if (!wrangler) break; if (arg1 != 0) { @@ -6655,6 +6829,7 @@ void IOPMrootDomain::handlePowerNotification( UInt32 msg ) */ if (msg & kIOPMClamshellOpened) { + DLOG("Clamshell opened\n"); // Received clamshel open message from clamshell controlling driver // Update our internal state and tell general interest clients clamshellClosed = false; @@ -6686,6 +6861,7 @@ void IOPMrootDomain::handlePowerNotification( UInt32 msg ) */ if (msg & kIOPMClamshellClosed) { + DLOG("Clamshell closed\n"); // Received clamshel open message from clamshell controlling driver // Update our internal state and tell general interest clients clamshellClosed = true; @@ -6708,6 +6884,7 @@ void IOPMrootDomain::handlePowerNotification( UInt32 msg ) */ if (msg & kIOPMSetDesktopMode) { + DLOG("Desktop mode\n"); desktopMode = (0 != (msg & kIOPMSetValue)); msg &= ~(kIOPMSetDesktopMode | kIOPMSetValue); @@ -6761,6 +6938,7 @@ void IOPMrootDomain::handlePowerNotification( UInt32 msg ) */ if (msg & kIOPMEnableClamshell) { + DLOG("Clamshell enabled\n"); // Re-evaluate the lid state // System should sleep on external display disappearance // in lid closed operation. @@ -6780,6 +6958,7 @@ void IOPMrootDomain::handlePowerNotification( UInt32 msg ) */ if (msg & kIOPMDisableClamshell) { + DLOG("Clamshell disabled\n"); clamshellDisabled = true; sendClientClamshellNotification(); } @@ -6800,6 +6979,7 @@ void IOPMrootDomain::handlePowerNotification( UInt32 msg ) */ if (msg & kIOPMPowerButton) { + DLOG("Powerbutton press\n"); if (!wranglerAsleep) { OSString *pbs = OSString::withCString("DisablePowerButtonSleep"); @@ -6835,7 +7015,6 @@ void IOPMrootDomain::evaluatePolicy( int stimulus, uint32_t arg ) uint32_t u32; } flags; - DLOG("evaluatePolicy( %d, 0x%x )\n", stimulus, arg); ASSERT_GATED(); flags.u32 = 0; @@ -6843,6 +7022,7 @@ void IOPMrootDomain::evaluatePolicy( int stimulus, uint32_t arg ) switch (stimulus) { case kStimulusDisplayWranglerSleep: + DLOG("evaluatePolicy( %d, 0x%x )\n", stimulus, arg); if (!wranglerAsleep) { // first transition to wrangler sleep or lower @@ -6851,11 +7031,13 @@ void IOPMrootDomain::evaluatePolicy( int stimulus, uint32_t arg ) break; case kStimulusDisplayWranglerWake: + DLOG("evaluatePolicy( %d, 0x%x )\n", stimulus, arg); displayIdleForDemandSleep = false; wranglerAsleep = false; break; case kStimulusEnterUserActiveState: + DLOG("evaluatePolicy( %d, 0x%x )\n", stimulus, arg); if (_preventUserActive) { DLOG("user active dropped\n"); @@ -6881,6 +7063,7 @@ void IOPMrootDomain::evaluatePolicy( int stimulus, uint32_t arg ) break; case kStimulusLeaveUserActiveState: + DLOG("evaluatePolicy( %d, 0x%x )\n", stimulus, arg); if (userIsActive) { userIsActive = false; @@ -6895,6 +7078,7 @@ void IOPMrootDomain::evaluatePolicy( int stimulus, uint32_t arg ) case kStimulusAggressivenessChanged: { + DMSG("evaluatePolicy( %d, 0x%x )\n", stimulus, arg); unsigned long minutesToIdleSleep = 0; unsigned long minutesToDisplayDim = 0; unsigned long minutesDelta = 0; @@ -6947,6 +7131,7 @@ void IOPMrootDomain::evaluatePolicy( int stimulus, uint32_t arg ) } break; case kStimulusDemandSystemSleep: + DLOG("evaluatePolicy( %d, 0x%x )\n", stimulus, arg); displayIdleForDemandSleep = true; if (wrangler && wranglerIdleSettings) { @@ -6963,10 +7148,12 @@ void IOPMrootDomain::evaluatePolicy( int stimulus, uint32_t arg ) break; case kStimulusAllowSystemSleepChanged: + DLOG("evaluatePolicy( %d, 0x%x )\n", stimulus, arg); flags.bit.adjustPowerState = true; break; case kStimulusDarkWakeActivityTickle: + DLOG("evaluatePolicy( %d, 0x%x )\n", stimulus, arg); // arg == true implies real and not self generated wrangler tickle. // Update wake type on PM work loop instead of the tickle thread to // eliminate the possibility of an early tickle clobbering the wake @@ -6990,6 +7177,7 @@ void IOPMrootDomain::evaluatePolicy( int stimulus, uint32_t arg ) case kStimulusDarkWakeEntry: case kStimulusDarkWakeReentry: + DLOG("evaluatePolicy( %d, 0x%x )\n", stimulus, arg); // Any system transitions since the last dark wake transition // will invalid the stimulus. @@ -7006,6 +7194,10 @@ void IOPMrootDomain::evaluatePolicy( int stimulus, uint32_t arg ) { clock_get_uptime(&userBecameInactiveTime); flags.bit.evaluateDarkWake = true; + if (activitySinceSleep()) { + DLOG("User activity recorded while going to darkwake\n"); + reportUserInput(); + } } // Always accelerate disk spindown while in dark wake, @@ -7017,6 +7209,7 @@ void IOPMrootDomain::evaluatePolicy( int stimulus, uint32_t arg ) break; case kStimulusDarkWakeEvaluate: + DMSG("evaluatePolicy( %d, 0x%x )\n", stimulus, arg); if (systemDarkWake) { flags.bit.evaluateDarkWake = true; @@ -7024,6 +7217,7 @@ void IOPMrootDomain::evaluatePolicy( int stimulus, uint32_t arg ) break; case kStimulusNoIdleSleepPreventers: + DMSG("evaluatePolicy( %d, 0x%x )\n", stimulus, arg); flags.bit.adjustPowerState = true; break; @@ -7438,23 +7632,29 @@ void IOPMrootDomain::pmStatsRecordApplicationResponse( if (object && (notify = OSDynamicCast(IOPMServiceInterestNotifier, object))) { - if (response->isEqualTo(gIOPMStatsApplicationResponseTimedOut)) + if (response->isEqualTo(gIOPMStatsResponseTimedOut)) notify->ackTimeoutCnt++; else notify->ackTimeoutCnt = 0; } - if (response->isEqualTo(gIOPMStatsApplicationResponsePrompt) || + if (response->isEqualTo(gIOPMStatsResponsePrompt) || (_systemTransitionType == kSystemTransitionNone) || (_systemTransitionType == kSystemTransitionNewCapClient)) return; if (response->isEqualTo(gIOPMStatsDriverPSChangeSlow)) { - kdebugTrace(kPMLogDrvResponseDelay, id, messageType, delay_ms); + kdebugTrace(kPMLogDrvPSChangeDelay, id, messageType, delay_ms); } else if (notify) { - kdebugTrace(kPMLogAppResponseDelay, id, notify->msgType, delay_ms); + // User space app or kernel capability client + if (id) { + kdebugTrace(kPMLogAppResponseDelay, id, notify->msgType, delay_ms); + } + else { + kdebugTrace(kPMLogDrvResponseDelay, notify->uuid0, messageType, delay_ms); + } notify->msgType = 0; } @@ -7471,6 +7671,10 @@ void IOPMrootDomain::pmStatsRecordApplicationResponse( msgNum->release(); } + if (!name && notify && notify->identifier) { + name = notify->identifier->getCStringNoCopy(); + } + if (name && (strlen(name) > 0)) { appname = OSSymbol::withCString(name); @@ -7480,8 +7684,11 @@ void IOPMrootDomain::pmStatsRecordApplicationResponse( } } + if (!id && notify) { + id = notify->uuid0; + } if (id != 0) { - pidNum = OSNumber::withNumber(id, 32); + pidNum = OSNumber::withNumber(id, 64); if (pidNum) { responseDescription->setObject(_statsPIDKey, pidNum); pidNum->release(); @@ -7632,14 +7839,57 @@ void IOPMrootDomain::tracePoint( uint8_t point ) pmTracer->tracePoint(point); } -void IOPMrootDomain::traceDetail(uint32_t msgType, uint32_t msgIndex, uintptr_t handler) +void IOPMrootDomain::traceDetail(OSObject *object) { + IOPMServiceInterestNotifier *notifier = OSDynamicCast(IOPMServiceInterestNotifier, object); + if (!notifier) { + DLOG("Unknown notifier\n"); + return; + } + if (!systemBooting) { - uint32_t detail = ((msgIndex & 0xff) << 24) | - ((msgType & 0xfff) << 12) | - (handler & 0xfff); + pmTracer->traceDetail( notifier->uuid0 >> 32 ); + kdebugTrace(kPMLogSleepWakeMessage, pmTracer->getTracePhase(), notifier->msgType, notifier->uuid0, notifier->uuid1); + if (notifier->identifier) { + DLOG("trace point 0x%02x msg 0x%x to %s\n", pmTracer->getTracePhase(), notifier->msgType, + notifier->identifier->getCStringNoCopy()); + } + else { + DLOG("trace point 0x%02x msg 0x%x\n", pmTracer->getTracePhase(), notifier->msgType); + } + } + +} + + +void IOPMrootDomain::traceAckDelay(OSObject *object, uint32_t response, uint32_t delay_ms) +{ + IOPMServiceInterestNotifier *notifier = OSDynamicCast(IOPMServiceInterestNotifier, object); + if (!notifier) { + DLOG("Unknown notifier\n"); + return; + } + + if (!systemBooting) { + kdebugTrace(kPMLogDrvResponseDelay, notifier->uuid0, notifier->uuid1, response, delay_ms); + if (notifier->identifier) { + DLOG("Response from %s took %d ms(response:%d)\n", + notifier->identifier->getCStringNoCopy(), delay_ms, response); + } + else { + DLOG("Response from kext UUID %llx-%llx took %d ms(response:%d)\n", + notifier->uuid0, notifier->uuid1, delay_ms, response); + } + } +} + +void IOPMrootDomain::traceDetail(uint32_t msgType, uint32_t msgIndex, uint32_t delay) +{ + if (!systemBooting) { + uint32_t detail = ((msgType & 0xffff) << 16) | (delay & 0xffff); pmTracer->traceDetail( detail ); - kdebugTrace(kPMLogSleepWakeTracePoint, 0, pmTracer->getTracePhase(), msgType, handler & 0xfff); + kdebugTrace(kPMLogSleepWakeTracePoint, pmTracer->getTracePhase(), msgType, delay); + DLOG("trace point 0x%02x msgType 0x%x detail 0x%08x\n", pmTracer->getTracePhase(), msgType, delay); } } @@ -7940,10 +8190,10 @@ void PMTraceWorker::tracePoint(uint8_t phase) void PMTraceWorker::traceDetail(uint32_t detail) { - + if (detail == traceData32) { + return; + } traceData32 = detail; - DLOG("trace point 0x%02x detail 0x%08x\n", tracePhase, traceData32); - RTC_TRACE(); } @@ -8121,7 +8371,7 @@ void PMHaltWorker::work( PMHaltWorker * me ) { IOService * service; OSSet * inner; - AbsoluteTime startTime; + AbsoluteTime startTime, elapsedTime; UInt32 deltaTime; bool timeout; @@ -8170,7 +8420,7 @@ void PMHaltWorker::work( PMHaltWorker * me ) IOLockUnlock(me->lock); } - deltaTime = computeDeltaTimeMS(&startTime); + deltaTime = computeDeltaTimeMS(&startTime, &elapsedTime); if ((deltaTime > kPMHaltTimeoutMS) || timeout || (gIOKitDebug & kIOLogPMRootDomain)) { @@ -8179,6 +8429,10 @@ void PMHaltWorker::work( PMHaltWorker * me ) "PowerOff" : "Restart", service->getName(), service->getRegistryEntryID(), (uint32_t) deltaTime ); + halt_log_enter( + (gPMHaltMessageType == kIOMessageSystemWillPowerOff) ? "PowerOff" : "Restart", + OSMemberFunctionCast(const void *, service, &IOService::systemWillShutdown), + elapsedTime); } service->release(); @@ -8651,6 +8905,29 @@ void IOPMrootDomain::acceptSystemWakeEvents( bool accept ) else { _acceptSystemWakeEvents = false; +#if CONFIG_EMBEDDED + logWakeReason = gWakeReasonSysctlRegistered; +#if DEVELOPMENT + static int panic_allowed = -1; + + if ((panic_allowed == -1) && + (PE_parse_boot_argn("swd_wakereason_panic", &panic_allowed, sizeof(panic_allowed)) == false)) { + panic_allowed = 1; + } + + if (panic_allowed) { + size_t i = 0; + // Panic if wake reason is null or empty + for (i = 0; (i < strlen(gWakeReasonString)); i++) { + if ((gWakeReasonString[i] != ' ') && (gWakeReasonString[i] != '\t')) + break; + } + if (i >= strlen(gWakeReasonString)) { + panic("Wake reason is empty\n"); + } + } +#endif +#endif } WAKEEVENT_UNLOCK(); @@ -8706,6 +8983,9 @@ void IOPMrootDomain::claimSystemWakeEvent( // Lazy registration until the platform driver stops registering // the same name. gWakeReasonSysctlRegistered = true; +#if CONFIG_EMBEDDED + sysctl_register_oid(&sysctl__kern_wakereason); +#endif } if (_acceptSystemWakeEvents) { @@ -9355,6 +9635,7 @@ void IOPMrootDomain::takeStackshot(bool wdogTrigger, bool isOSXWatchdog, bool is swd_hdr * hdr = NULL; addr64_t data[3]; int wdog_panic = -1; + int stress_rack = -1; int cnt = 0; pid_t pid = 0; kern_return_t kr = KERN_SUCCESS; @@ -9385,9 +9666,10 @@ void IOPMrootDomain::takeStackshot(bool wdogTrigger, bool isOSXWatchdog, bool is if (wdogTrigger) { PE_parse_boot_argn("swd_panic", &wdog_panic, sizeof(wdog_panic)); - if (wdog_panic == 1) { + PE_parse_boot_argn("stress-rack", &stress_rack, sizeof(stress_rack)); + if ((wdog_panic == 1) || (stress_rack == 1)) { // If boot-arg specifies to panic then panic. - panic("Sleep/Wake hang detected\n"); + panic("Sleep/Wake hang detected"); return; } else if (swd_flags & SWD_BOOT_BY_SW_WDOG) { @@ -9395,6 +9677,10 @@ void IOPMrootDomain::takeStackshot(bool wdogTrigger, bool isOSXWatchdog, bool is // then don't trigger again until at least 1 successful sleep & wake. if (!(sleepCnt && (displayWakeCnt || darkWakeCnt))) { IOLog("Shutting down due to repeated Sleep/Wake failures\n"); + if (!tasksSuspended) { + tasksSuspended = TRUE; + tasks_system_suspend(true); + } PEHaltRestart(kPEHaltCPU); return; } @@ -9537,8 +9823,13 @@ exit: gRootDomain->swd_lock = 0; if (wdogTrigger) { - IOLog("Restarting to collect Sleep wake debug logs\n"); - PEHaltRestart(kPERestartCPU); + IOLog("Restarting to collect Sleep wake debug logs\n"); + if (!tasksSuspended) { + tasksSuspended = TRUE; + tasks_system_suspend(true); + } + + PEHaltRestart(kPERestartCPU); } else { logBufMap = sleepWakeDebugRetrieve(); @@ -9846,8 +10137,8 @@ uint32_t IOPMrootDomain::checkForValidDebugData(const char *fname, vfs_context_t vfs_context_ucred(*ctx), (int *) 0, vfs_context_proc(*ctx)); if (rc != 0) { - IOLog("sleepWakeDebugDumpFromFile: Failed to read header size %lu(rc=%d) from %s\n", - round_page(sizeof(IOHibernateImageHeader)), rc, fname); + IOLog("sleepWakeDebugDumpFromFile: Failed to read header size %llu(rc=%d) from %s\n", + mach_vm_round_page(sizeof(IOHibernateImageHeader)), rc, fname); error = SWD_FILEOP_ERROR; goto err; } @@ -9951,15 +10242,15 @@ void IOPMrootDomain::sleepWakeDebugDumpFromFile( ) hdrOffset = ((IOHibernateImageHeader *)tmpBuf)->deviceBlockSize; - DLOG("Reading swd_hdr len 0x%lx offset 0x%lx\n", round_page(sizeof(swd_hdr)), trunc_page(hdrOffset)); + DLOG("Reading swd_hdr len 0x%llx offset 0x%lx\n", mach_vm_round_page(sizeof(swd_hdr)), trunc_page(hdrOffset)); /* Read the sleep/wake debug header(swd_hdr) */ rc = vn_rdwr(UIO_READ, vp, (char *)tmpBuf, round_page(sizeof(swd_hdr)), trunc_page(hdrOffset), UIO_SYSSPACE, IO_SKIP_ENCRYPTION|IO_SYNC|IO_NODELOCKED|IO_UNIT|IO_NOCACHE, vfs_context_ucred(ctx), (int *) 0, vfs_context_proc(ctx)); if (rc != 0) { - DMSG("sleepWakeDebugDumpFromFile: Failed to debug read header size %lu. rc=%d\n", - round_page(sizeof(swd_hdr)), rc); + DMSG("sleepWakeDebugDumpFromFile: Failed to debug read header size %llu. rc=%d\n", + mach_vm_round_page(sizeof(swd_hdr)), rc); swd_flags |= SWD_FILEOP_ERROR; goto exit; } @@ -10236,7 +10527,7 @@ void IOPMrootDomain::sleepWakeDebugTrig(bool restart) (wdog_panic == 0)) { return; } - panic("Sleep/Wake hang detected\n"); + panic("Sleep/Wake hang detected"); return; } } diff --git a/iokit/Kernel/IOPlatformExpert.cpp b/iokit/Kernel/IOPlatformExpert.cpp index c83a71fd5..008afdb78 100644 --- a/iokit/Kernel/IOPlatformExpert.cpp +++ b/iokit/Kernel/IOPlatformExpert.cpp @@ -42,6 +42,7 @@ #include <IOKit/IOKitDiagnosticsUserClient.h> #include <IOKit/system.h> +#include <sys/csr.h> #include <libkern/c++/OSContainers.h> #include <libkern/crypto/sha1.h> @@ -53,6 +54,16 @@ extern "C" { #include <uuid/uuid.h> } +#if defined(__x86_64__) +/* + * This will eventually be properly exported in + * <rdar://problem/31181482> ER: Expose coprocessor version (T208/T290) in a kernel/kext header + * although we'll always need to hardcode this here since we won't be able to include whatever + * header this ends up in. + */ +#define kCoprocessorMinVersion 0x00020000 +#endif + void printDictionaryKeys (OSDictionary * inDictionary, char * inMsg); static void getCStringForObject(OSObject *inObj, char *outStr, size_t outStrLen); @@ -99,15 +110,28 @@ bool IOPlatformExpert::start( IOService * provider ) IORangeAllocator * physicalRanges; OSData * busFrequency; uint32_t debugFlags; + +#if defined(__x86_64__) + IORegistryEntry *platform_entry = NULL; + OSData *coprocessor_version_obj = NULL; + uint64_t coprocessor_version = 0; +#endif if (!super::start(provider)) return false; - // Override the mapper present flag is requested by boot arguments. - if (PE_parse_boot_argn("dart", &debugFlags, sizeof (debugFlags)) && (debugFlags == 0)) - removeProperty(kIOPlatformMapperPresentKey); - if (PE_parse_boot_argn("-x", &debugFlags, sizeof (debugFlags))) - removeProperty(kIOPlatformMapperPresentKey); + // Override the mapper present flag is requested by boot arguments, if SIP disabled. +#if CONFIG_CSR + if (csr_check(CSR_ALLOW_UNRESTRICTED_FS) == 0) +#endif /* CONFIG_CSR */ + { + if (PE_parse_boot_argn("dart", &debugFlags, sizeof (debugFlags)) && (debugFlags == 0)) + removeProperty(kIOPlatformMapperPresentKey); +#if DEBUG || DEVELOPMENT + if (PE_parse_boot_argn("-x", &debugFlags, sizeof (debugFlags))) + removeProperty(kIOPlatformMapperPresentKey); +#endif /* DEBUG || DEVELOPMENT */ + } // Register the presence or lack thereof a system // PCI address mapper with the IOMapper class @@ -142,7 +166,21 @@ bool IOPlatformExpert::start( IOService * provider ) serNoString->release(); } } - + +#if defined(__x86_64__) + platform_entry = IORegistryEntry::fromPath(kIODeviceTreePlane ":/efi/platform"); + if (platform_entry != NULL) { + coprocessor_version_obj = OSDynamicCast(OSData, platform_entry->getProperty("apple-coprocessor-version")); + if ((coprocessor_version_obj != NULL) && (coprocessor_version_obj->getLength() <= sizeof(coprocessor_version))) { + memcpy(&coprocessor_version, coprocessor_version_obj->getBytesNoCopy(), coprocessor_version_obj->getLength()); + if (coprocessor_version >= kCoprocessorMinVersion) { + coprocessor_paniclog_flush = TRUE; + } + } + platform_entry->release(); + } +#endif /* defined(__x86_64__) */ + return( configure(provider) ); } @@ -260,9 +298,11 @@ int IOPlatformExpert::haltRestart(unsigned int type) type = kPEHaltCPU; } +#if !CONFIG_EMBEDDED // On ARM kPEPanicRestartCPU is supported in the drivers if (type == kPEPanicRestartCPU) type = kPERestartCPU; +#endif if (PE_halt_restart) return (*PE_halt_restart)(type); else return -1; @@ -738,10 +778,16 @@ static void IOShutdownNotificationsTimedOut( thread_call_param_t p0, thread_call_param_t p1) { +#ifdef CONFIG_EMBEDDED + /* 30 seconds has elapsed - panic */ + panic("Halt/Restart Timed Out"); + +#else /* ! CONFIG_EMBEDDED */ int type = (int)(long)p0; /* 30 seconds has elapsed - resume shutdown */ if(gIOPlatform) gIOPlatform->haltRestart(type); +#endif /* CONFIG_EMBEDDED */ } @@ -783,6 +829,7 @@ int PEHaltRestart(unsigned int type) IORegistryEntry *node; OSData *data; uint32_t timeout = 30; + static boolean_t panic_begin_called = FALSE; if(type == kPEHaltCPU || type == kPERestartCPU || type == kPEUPSDelayHaltCPU) { @@ -797,7 +844,11 @@ int PEHaltRestart(unsigned int type) the timer expires. If the device wants a different timeout, use that value instead of 30 seconds. */ +#if CONFIG_EMBEDDED +#define RESTART_NODE_PATH "/defaults" +#else #define RESTART_NODE_PATH "/chosen" +#endif node = IORegistryEntry::fromPath( RESTART_NODE_PATH, gIODTPlane ); if ( node ) { data = OSDynamicCast( OSData, node->getProperty( "halt-restart-timeout" ) ); @@ -822,6 +873,12 @@ int PEHaltRestart(unsigned int type) } else if(type == kPEPanicRestartCPU || type == kPEPanicSync) { + if (type == kPEPanicRestartCPU) { + // Notify any listeners that we're done collecting + // panic data before we call through to do the restart + IOCPURunPlatformPanicActions(kPEPanicEnd); + } + // Do an initial sync to flush as much panic data as possible, // in case we have a problem in one of the platorm panic handlers. // After running the platform handlers, do a final sync w/ @@ -830,6 +887,15 @@ int PEHaltRestart(unsigned int type) IOCPURunPlatformPanicActions(type); PE_sync_panic_buffers(); } + else if (type == kPEPanicEnd) { + IOCPURunPlatformPanicActions(type); + } else if (type == kPEPanicBegin) { + // Only call the kPEPanicBegin callout once + if (!panic_begin_called) { + panic_begin_called = TRUE; + IOCPURunPlatformPanicActions(type); + } + } if (gIOPlatform) return gIOPlatform->haltRestart(type); else return -1; @@ -841,6 +907,11 @@ UInt32 PESavePanicInfo(UInt8 *buffer, UInt32 length) else return 0; } +void PESavePanicInfoAction(void *buffer, size_t length) +{ + IOCPURunPlatformPanicSyncAction(buffer, length); + return; +} inline static int init_gIOOptionsEntry(void) @@ -1050,6 +1121,41 @@ void IOPlatformExpert::registerNVRAMController(IONVRAMController * caller) OSString * string = 0; uuid_string_t uuid; +#if CONFIG_EMBEDDED + entry = IORegistryEntry::fromPath( "/chosen", gIODTPlane ); + if ( entry ) + { + OSData * data1; + + data1 = OSDynamicCast( OSData, entry->getProperty( "unique-chip-id" ) ); + if ( data1 && data1->getLength( ) == 8 ) + { + OSData * data2; + + data2 = OSDynamicCast( OSData, entry->getProperty( "chip-id" ) ); + if ( data2 && data2->getLength( ) == 4 ) + { + SHA1_CTX context; + uint8_t digest[ SHA_DIGEST_LENGTH ]; + const uuid_t space = { 0xA6, 0xDD, 0x4C, 0xCB, 0xB5, 0xE8, 0x4A, 0xF5, 0xAC, 0xDD, 0xB6, 0xDC, 0x6A, 0x05, 0x42, 0xB8 }; + + SHA1Init( &context ); + SHA1Update( &context, space, sizeof( space ) ); + SHA1Update( &context, data1->getBytesNoCopy( ), data1->getLength( ) ); + SHA1Update( &context, data2->getBytesNoCopy( ), data2->getLength( ) ); + SHA1Final( digest, &context ); + + digest[ 6 ] = ( digest[ 6 ] & 0x0F ) | 0x50; + digest[ 8 ] = ( digest[ 8 ] & 0x3F ) | 0x80; + + uuid_unparse( digest, uuid ); + string = OSString::withCString( uuid ); + } + } + + entry->release( ); + } +#else /* !CONFIG_EMBEDDED */ entry = IORegistryEntry::fromPath( "/efi/platform", gIODTPlane ); if ( entry ) { @@ -1074,6 +1180,7 @@ void IOPlatformExpert::registerNVRAMController(IONVRAMController * caller) entry->release( ); } +#endif /* !CONFIG_EMBEDDED */ if ( string == 0 ) { diff --git a/iokit/Kernel/IOPolledInterface.cpp b/iokit/Kernel/IOPolledInterface.cpp index f0bb31618..2c9dfba68 100644 --- a/iokit/Kernel/IOPolledInterface.cpp +++ b/iokit/Kernel/IOPolledInterface.cpp @@ -336,9 +336,14 @@ IOStartPolledIO(IOPolledFilePollers * vars, poller = (IOPolledInterface *) vars->pollers->getObject(0); err = poller->startIO(operation, bufferOffset, deviceOffset, length, completion); - if (err) - HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err); - + if (err) { + if (kernel_debugger_entry_count) { + HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err); + } else { + HIBLOGFROMPANIC("IOPolledInterface::IOStartPolledIO(0x%p, %d, 0x%x, 0x%llx, %llu) : poller->startIO(%d, 0x%x, 0x%llx, %llu, completion) returned 0x%x", + vars, operation, bufferOffset, deviceOffset, length, operation, bufferOffset, deviceOffset, length, err); + } + } return (err); } @@ -462,6 +467,8 @@ IOCopyMediaForDev(dev_t device) return (result); } +#define APFSMEDIA_GETHIBERKEY "getHiberKey" + static IOReturn IOGetVolumeCryptKey(dev_t block_dev, OSString ** pKeyUUID, uint8_t * volumeCryptKey, size_t keySize) @@ -478,30 +485,49 @@ IOGetVolumeCryptKey(dev_t block_dev, OSString ** pKeyUUID, part = IOCopyMediaForDev(block_dev); if (!part) return (kIOReturnNotFound); - err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false, - (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL); - if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID) + // Try APFS first { -// IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy()); + uuid_t volUuid = {0}; + size_t sizeOut = 0; + err = part->callPlatformFunction(APFSMEDIA_GETHIBERKEY, false, &volUuid, volumeCryptKey, &keySize, &sizeOut); + if (err == kIOReturnSuccess) { + // No need to create uuid string if it's not requested + if (pKeyUUID) { + uuid_string_t volUuidStr; + uuid_unparse(volUuid, volUuidStr); + *pKeyUUID = OSString::withCString(volUuidStr); + } - if (!sKeyStore) - sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane); - if (sKeyStore) - err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID); - else - err = kIOReturnNoResources; - if (kIOReturnSuccess == err) - err = sKeyStore->callPlatformFunction(gAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL); - if (kIOReturnSuccess != err) - IOLog("volume key err 0x%x\n", err); - else - { - if (vek.key.keybytecount < keySize) keySize = vek.key.keybytecount; - bcopy(&vek.key.keybytes[0], volumeCryptKey, keySize); - } - bzero(&vek, sizeof(vek)); + part->release(); + return kIOReturnSuccess; + } + } + // Then old CS path + err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false, + (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL); + if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID) + { +// IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy()); + + if (!sKeyStore) + sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane); + if (sKeyStore) + err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID); + else + err = kIOReturnNoResources; + if (kIOReturnSuccess == err) + err = sKeyStore->callPlatformFunction(gAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL); + if (kIOReturnSuccess != err) + IOLog("volume key err 0x%x\n", err); + else + { + if (vek.key.keybytecount < keySize) keySize = vek.key.keybytecount; + bcopy(&vek.key.keybytes[0], volumeCryptKey, keySize); + } + bzero(&vek, sizeof(vek)); } + part->release(); if (pKeyUUID) *pKeyUUID = keyUUID; @@ -521,7 +547,7 @@ IOPolledFileOpen(const char * filename, IOReturn err = kIOReturnSuccess; IOPolledFileIOVars * vars; _OpenFileContext ctx; - OSData * extentsData; + OSData * extentsData = NULL; OSNumber * num; IOService * part = 0; dev_t block_dev; @@ -642,6 +668,7 @@ IOPolledFileOpen(const char * filename, { HIBLOG("error 0x%x opening polled file\n", err); IOPolledFileClose(&vars, 0, 0, 0, 0, 0); + if (extentsData) extentsData->release(); } if (part) part->release(); @@ -734,6 +761,11 @@ IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position) vars->position = position; + if (position > vars->fileSize) { + HIBLOG("IOPolledFileSeek: called to seek to 0x%llx greater than file size of 0x%llx\n", vars->position, vars->fileSize); + return kIOReturnNoSpace; + } + while (position >= extentMap->length) { position -= extentMap->length; @@ -760,7 +792,7 @@ IOPolledFileWrite(IOPolledFileIOVars * vars, IOPolledFileCryptVars * cryptvars) { IOReturn err = kIOReturnSuccess; - IOByteCount copy; + IOByteCount copy, original_size = size; bool flush = false; do @@ -844,8 +876,11 @@ if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", //if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length); err = IOStartPolledIO(vars->pollers, kIOPolledWrite, vars->bufferHalf, offset, length); - if (kIOReturnSuccess != err) + if (kIOReturnSuccess != err) { + HIBLOGFROMPANIC("IOPolledFileWrite(0x%p, 0x%p, %llu, 0x%p) : IOStartPolledIO(0x%p, kIOPolledWrite, %llu, 0x%llx, %d) returned 0x%x\n", + vars, bytes, (uint64_t) original_size, cryptvars, vars->pollers, (uint64_t) vars->bufferHalf, offset, length, err); break; + } vars->pollers->io = true; vars->extentRemaining -= vars->bufferOffset; @@ -879,6 +914,29 @@ if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +IOReturn +IOPolledFileFlush(IOPolledFileIOVars * vars) +{ + // Only supported by the underlying polled mode driver on embedded currently (expect kIOReturnUnsupported on other platforms) + IOReturn err = kIOReturnSuccess; + + err = IOPolledFilePollersIODone(vars->pollers, true); + if (kIOReturnSuccess != err) + return err; + + err = IOStartPolledIO(vars->pollers, kIOPolledFlush, 0, 0, 0); + if (kIOReturnSuccess != err) { + HIBLOGFROMPANIC("IOPolledFileFlush(0x%p) : IOStartPolledIO(0x%p, kIOPolledFlush, 0, 0, 0) returned 0x%x\n", + vars, vars->pollers, err); + return err; + } + vars->pollers->io = true; + + return err; +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + IOReturn IOPolledFileRead(IOPolledFileIOVars * vars, uint8_t * bytes, IOByteCount size, diff --git a/iokit/Kernel/IOReporter.cpp b/iokit/Kernel/IOReporter.cpp index 81b6bfb94..dd6ccf764 100644 --- a/iokit/Kernel/IOReporter.cpp +++ b/iokit/Kernel/IOReporter.cpp @@ -161,7 +161,7 @@ finish: bool IOReporter::init(IOService *reportingService, IOReportChannelType channelType, - IOReportUnits unit) + IOReportUnit unit) { bool success = false; @@ -189,7 +189,9 @@ IOReporter::init(IOService *reportingService, _channelType = channelType; // FIXME: need to look up dynamically if (unit == kIOReportUnitHWTicks) { -#if defined(__i386__) || defined(__x86_64__) +#if defined(__arm__) || defined(__arm64__) + unit = kIOReportUnit24MHzTicks; +#elif defined(__i386__) || defined(__x86_64__) // Most, but not all Macs use 1GHz unit = kIOReportUnit1GHzTicks; #else @@ -988,7 +990,7 @@ finish: IOReporter::legendWith(OSArray *channelIDs, OSArray *channelNames, IOReportChannelType channelType, - IOReportUnits unit) + IOReportUnit unit) { unsigned int cnt, chCnt; uint64_t type64; diff --git a/iokit/Kernel/IOService.cpp b/iokit/Kernel/IOService.cpp index 57323e108..40055c5c9 100644 --- a/iokit/Kernel/IOService.cpp +++ b/iokit/Kernel/IOService.cpp @@ -51,6 +51,7 @@ #include <IOKit/IOInterruptAccountingPrivate.h> #include <IOKit/IOKernelReporters.h> #include <IOKit/AppleKeyStoreInterface.h> +#include <IOKit/pwr_mgt/RootDomain.h> #include <IOKit/IOCPU.h> #include <mach/sync_policy.h> #include <IOKit/assert.h> @@ -63,7 +64,10 @@ #define LOG kprintf //#define LOG IOLog #define MATCH_DEBUG 0 -#define OBFUSCATE(x) ((void *)(VM_KERNEL_ADDRPERM(x))) +#define IOSERVICE_OBFUSCATE(x) ((void *)(VM_KERNEL_ADDRPERM(x))) + +// disabled since lockForArbitration() can be held externally +#define DEBUG_NOTIFIER_LOCKED 0 #include "IOServicePrivate.h" #include "IOKitKernelInternal.h" @@ -115,6 +119,9 @@ const OSSymbol * gIOPathMatchKey; const OSSymbol * gIOMatchCategoryKey; const OSSymbol * gIODefaultMatchCategoryKey; const OSSymbol * gIOMatchedServiceCountKey; +#if !CONFIG_EMBEDDED +const OSSymbol * gIOServiceLegacyMatchingRegistryIDKey; +#endif const OSSymbol * gIOMapperIDKey; const OSSymbol * gIOUserClientClassKey; @@ -148,6 +155,7 @@ const OSSymbol * gIOFirstPublishNotification; const OSSymbol * gIOMatchedNotification; const OSSymbol * gIOFirstMatchNotification; const OSSymbol * gIOTerminatedNotification; +const OSSymbol * gIOWillTerminateNotification; const OSSymbol * gIOGeneralInterest; const OSSymbol * gIOBusyInterest; @@ -179,7 +187,7 @@ static int gOutstandingJobs; static int gNumConfigThreads; static int gNumWaitingThreads; static IOLock * gIOServiceBusyLock; -static bool gCPUsRunning; +bool gCPUsRunning; static thread_t gIOTerminateThread; static UInt32 gIOTerminateWork; @@ -314,6 +322,10 @@ void IOService::initialize( void ) kIODefaultMatchCategoryKey ); gIOMatchedServiceCountKey = OSSymbol::withCStringNoCopy( kIOMatchedServiceCountKey ); +#if !CONFIG_EMBEDDED + gIOServiceLegacyMatchingRegistryIDKey = OSSymbol::withCStringNoCopy( + kIOServiceLegacyMatchingRegistryIDKey ); +#endif gIOUserClientClassKey = OSSymbol::withCStringNoCopy( kIOUserClientClassKey ); @@ -356,6 +368,8 @@ void IOService::initialize( void ) kIOFirstMatchNotification ); gIOTerminatedNotification = OSSymbol::withCStringNoCopy( kIOTerminatedNotification ); + gIOWillTerminateNotification = OSSymbol::withCStringNoCopy( + kIOWillTerminateNotification ); gIOServiceKey = OSSymbol::withCStringNoCopy( kIOServiceClass); gIOConsoleLockedKey = OSSymbol::withCStringNoCopy( kIOConsoleLockedKey); @@ -464,6 +478,24 @@ static UInt64 getDebugFlags( OSDictionary * props ) return( debugFlags ); } + +static UInt64 getDebugFlags( IOService * inst ) +{ + OSObject * prop; + OSNumber * debugProp; + UInt64 debugFlags; + + prop = inst->copyProperty(gIOKitDebugKey); + debugProp = OSDynamicCast(OSNumber, prop); + if( debugProp) + debugFlags = debugProp->unsigned64BitValue(); + else + debugFlags = gIOKitDebug; + + OSSafeReleaseNULL(prop); + + return( debugFlags ); +} #endif /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ @@ -790,7 +822,7 @@ void IOService::startMatching( IOOptionBits options ) if ( options & kIOServiceAsynchronous ) sync = false; - needConfig = (0 == (__state[1] & (kIOServiceNeedConfigState | kIOServiceConfigState))) + needConfig = (0 == (__state[1] & (kIOServiceNeedConfigState | kIOServiceConfigRunning))) && (0 == (__state[0] & kIOServiceInactiveState)); __state[1] |= kIOServiceNeedConfigState; @@ -1549,6 +1581,41 @@ void IOService::unlockForArbitration( void ) IOUnlock( gArbitrationLockQueueLock ); } +uint32_t IOService::isLockedForArbitration(IOService * service) +{ +#if DEBUG_NOTIFIER_LOCKED + uint32_t count; + ArbitrationLockQueueElement * active; + + // lock global access + IOLockLock(gArbitrationLockQueueLock); + + // determine whether this object is already locked (ie. on active queue) + count = 0; + queue_iterate(&gArbitrationLockQueueActive, + active, + ArbitrationLockQueueElement *, + link) + { + if ((active->thread == IOThreadSelf()) + && (!service || (active->service == service))) + { + count += 0x10000; + count += active->count; + } + } + + IOLockUnlock(gArbitrationLockQueueLock); + + return (count); + +#else /* DEBUG_NOTIFIER_LOCKED */ + + return (0); + +#endif /* DEBUG_NOTIFIER_LOCKED */ +} + void IOService::applyToProviders( IOServiceApplierFunction applier, void * context ) { @@ -1625,12 +1692,13 @@ applyToInterestNotifiers(const IORegistryEntry *target, OSObjectApplierFunction applier, void * context ) { - OSArray * copyArray = 0; + OSArray * copyArray = 0; + OSObject * prop; LOCKREADNOTIFY(); - IOCommand *notifyList = - OSDynamicCast( IOCommand, target->getProperty( typeOfInterest )); + prop = target->copyProperty(typeOfInterest); + IOCommand *notifyList = OSDynamicCast(IOCommand, prop); if( notifyList) { copyArray = OSArray::withCapacity(1); @@ -1653,6 +1721,8 @@ applyToInterestNotifiers(const IORegistryEntry *target, (*applier)(next, context); copyArray->release(); } + + OSSafeReleaseNULL(prop); } void IOService::applyToInterested( const OSSymbol * typeOfInterest, @@ -1717,7 +1787,7 @@ IONotifier * IOService::registerInterest( const OSSymbol * typeOfInterest, if (!notify) return NULL; if(notify->init()) { - rc = registerInterestForNotifer(notify, typeOfInterest, + rc = registerInterestForNotifier(notify, typeOfInterest, handler, target, ref); } @@ -1729,7 +1799,7 @@ IONotifier * IOService::registerInterest( const OSSymbol * typeOfInterest, return( notify ); } -IOReturn IOService::registerInterestForNotifer( IONotifier *svcNotify, const OSSymbol * typeOfInterest, +IOReturn IOService::registerInterestForNotifier( IONotifier *svcNotify, const OSSymbol * typeOfInterest, IOServiceInterestHandler handler, void * target, void * ref ) { IOReturn rc = kIOReturnSuccess; @@ -1806,11 +1876,27 @@ static void cleanInterestList( OSObject * head ) void IOService::unregisterAllInterest( void ) { - cleanInterestList( getProperty( gIOGeneralInterest )); - cleanInterestList( getProperty( gIOBusyInterest )); - cleanInterestList( getProperty( gIOAppPowerStateInterest )); - cleanInterestList( getProperty( gIOPriorityPowerStateInterest )); - cleanInterestList( getProperty( gIOConsoleSecurityInterest )); + OSObject * prop; + + prop = copyProperty(gIOGeneralInterest); + cleanInterestList(prop); + OSSafeReleaseNULL(prop); + + prop = copyProperty(gIOBusyInterest); + cleanInterestList(prop); + OSSafeReleaseNULL(prop); + + prop = copyProperty(gIOAppPowerStateInterest); + cleanInterestList(prop); + OSSafeReleaseNULL(prop); + + prop = copyProperty(gIOPriorityPowerStateInterest); + cleanInterestList(prop); + OSSafeReleaseNULL(prop); + + prop = copyProperty(gIOConsoleSecurityInterest); + cleanInterestList(prop); + OSSafeReleaseNULL(prop); } /* @@ -2046,7 +2132,10 @@ bool IOService::terminatePhase1( IOOptionBits options ) if( victim == this) startPhase2 = didInactive; if (didInactive) { - victim->deliverNotification( gIOTerminatedNotification, 0, 0xffffffff ); + OSArray * notifiers; + notifiers = victim->copyNotifiers(gIOTerminatedNotification, 0, 0xffffffff); + victim->invokeNotifiers(¬ifiers); + IOUserClient::destroyUserReferences( victim ); iter = victim->getClientIterator(); @@ -2533,8 +2622,10 @@ void IOService::terminateWorker( IOOptionBits options ) (uintptr_t) victim->__state[1], (uintptr_t) 0); - doPhase2 = (0 == (victim->__state[1] & kIOServiceTermPhase2State)) - && (0 == (victim->__state[1] & kIOServiceConfigState)); + doPhase2 = (0 == (victim->__state[1] & + (kIOServiceTermPhase1State + | kIOServiceTermPhase2State + | kIOServiceConfigState))); if (doPhase2 && (iter = victim->getClientIterator())) { while (doPhase2 && (client = (IOService *) iter->getNextObject())) { @@ -2567,6 +2658,10 @@ void IOService::terminateWorker( IOOptionBits options ) victim, (void *)(uintptr_t) options, NULL ); } + OSArray * notifiers; + notifiers = victim->copyNotifiers(gIOWillTerminateNotification, 0, 0xffffffff); + victim->invokeNotifiers(¬ifiers); + if( 0 == victim->getClient()) { // no clients - will go to finalize @@ -2932,23 +3027,30 @@ static SInt32 IOServiceObjectOrder( const OSObject * entry, void * ref) _IOServiceNotifier * notify; OSSymbol * key = (OSSymbol *) ref; OSNumber * offset; + OSObject * prop; + SInt32 result; + prop = 0; + result = kIODefaultProbeScore; if( (dict = OSDynamicCast( OSDictionary, entry))) offset = OSDynamicCast(OSNumber, dict->getObject( key )); else if( (notify = OSDynamicCast( _IOServiceNotifier, entry))) return( notify->priority ); - else if( (service = OSDynamicCast( IOService, entry))) - offset = OSDynamicCast(OSNumber, service->getProperty( key )); + { + prop = service->copyProperty(key); + offset = OSDynamicCast(OSNumber, prop); + } else { assert( false ); offset = 0; } - if( offset) - return( (SInt32) offset->unsigned32BitValue()); - else - return( kIODefaultProbeScore ); + if (offset) result = offset->unsigned32BitValue(); + + OSSafeReleaseNULL(prop); + + return (result); } SInt32 IOServiceOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref ) @@ -3003,14 +3105,22 @@ IOService * IOService::getClientWithCategory( const OSSymbol * category ) return (service); } -bool IOService::invokeNotifer( _IOServiceNotifier * notify ) +bool IOService::invokeNotifier( _IOServiceNotifier * notify ) { _IOServiceNotifierInvocation invocation; bool willNotify; bool ret = true; - invocation.thread = current_thread(); +#if DEBUG_NOTIFIER_LOCKED + uint32_t count; + if ((count = isLockedForArbitration(0))) + { + IOLog("[%s, 0x%x]\n", notify->type->getCStringNoCopy(), count); + panic("[%s, 0x%x]\n", notify->type->getCStringNoCopy(), count); + } +#endif /* DEBUG_NOTIFIER_LOCKED */ + LOCKWRITENOTIFY(); willNotify = (0 != (kIOServiceNotifyEnable & notify->state)); @@ -3037,6 +3147,27 @@ bool IOService::invokeNotifer( _IOServiceNotifier * notify ) return( ret ); } +bool IOService::invokeNotifiers(OSArray ** willSend) +{ + OSArray * array; + _IOServiceNotifier * notify; + bool ret = true; + + array = *willSend; + if (!array) return (true); + *willSend = 0; + + for( unsigned int idx = 0; + (notify = (_IOServiceNotifier *) array->getObject(idx)); + idx++) { + ret &= invokeNotifier(notify); + } + array->release(); + + return (ret); +} + + /* * Alloc and probe matching classes, * called on the provider instance @@ -3074,10 +3205,7 @@ void IOService::probeCandidates( OSOrderedSet * matches ) if( (notify = OSDynamicCast( _IOServiceNotifier, nextMatch ))) { - lockForArbitration(); - if( 0 == (__state[0] & kIOServiceInactiveState)) - invokeNotifer( notify ); - unlockForArbitration(); + if (0 == (__state[0] & kIOServiceInactiveState)) invokeNotifier( notify ); nextMatch->release(); nextMatch = 0; continue; @@ -3174,7 +3302,7 @@ void IOService::probeCandidates( OSOrderedSet * matches ) if( !symbol) continue; - //IOLog("%s alloc (symbol %p props %p)\n", symbol->getCStringNoCopy(), OBFUSCATE(symbol), OBFUSCATE(props)); + //IOLog("%s alloc (symbol %p props %p)\n", symbol->getCStringNoCopy(), IOSERVICE_OBFUSCATE(symbol), IOSERVICE_OBFUSCATE(props)); // alloc the driver instance inst = (IOService *) OSMetaClass::allocClassWithName( symbol); @@ -3280,7 +3408,7 @@ void IOService::probeCandidates( OSOrderedSet * matches ) startList->removeObject(inst); #if IOMATCHDEBUG - debugFlags = getDebugFlags( inst->getPropertyTable() ); + debugFlags = getDebugFlags( inst ); if( debugFlags & kIOLogStart) { if( started) @@ -3422,11 +3550,15 @@ bool IOService::addNeededResource( const char * key ) OSString * newKey; bool ret; - resourcesProp = getProperty( gIOResourceMatchKey ); + resourcesProp = copyProperty( gIOResourceMatchKey ); + if (!resourcesProp) return(false); newKey = OSString::withCString( key ); - if( (0 == resourcesProp) || (0 == newKey)) + if (!newKey) + { + resourcesProp->release(); return( false); + } set = OSDynamicCast( OSSet, resourcesProp ); if( !set) { @@ -3441,6 +3573,7 @@ bool IOService::addNeededResource( const char * key ) newKey->release(); ret = setProperty( gIOResourceMatchKey, set ); set->release(); + resourcesProp->release(); return( ret ); } @@ -3467,12 +3600,12 @@ bool IOService::checkResource( OSObject * matching ) } if( gIOKitDebug & kIOLogConfig) - LOG("config(%p): stalling %s\n", OBFUSCATE(IOThreadSelf()), getName()); + LOG("config(%p): stalling %s\n", IOSERVICE_OBFUSCATE(IOThreadSelf()), getName()); waitForService( table ); if( gIOKitDebug & kIOLogConfig) - LOG("config(%p): waking\n", OBFUSCATE(IOThreadSelf()) ); + LOG("config(%p): waking\n", IOSERVICE_OBFUSCATE(IOThreadSelf()) ); return( true ); } @@ -3484,7 +3617,7 @@ bool IOService::checkResources( void ) OSIterator * iter; bool ok; - resourcesProp = getProperty( gIOResourceMatchKey ); + resourcesProp = copyProperty( gIOResourceMatchKey ); if( 0 == resourcesProp) return( true ); @@ -3500,6 +3633,8 @@ bool IOService::checkResources( void ) } else ok = checkResource( resourcesProp ); + OSSafeReleaseNULL(resourcesProp); + return( ok ); } @@ -3543,6 +3678,7 @@ void IOService::doServiceMatch( IOOptionBits options ) bool keepGuessing = true; bool reRegistered = true; bool didRegister; + OSArray * notifiers[2] = {0}; // job->nub->deliverNotification( gIOPublishNotification, // kIOServiceRegisteredState, 0xffffffff ); @@ -3556,12 +3692,12 @@ void IOService::doServiceMatch( IOOptionBits options ) lockForArbitration(); if( 0 == (__state[0] & kIOServiceFirstPublishState)) { getMetaClass()->addInstance(this); - deliverNotification( gIOFirstPublishNotification, + notifiers[0] = copyNotifiers(gIOFirstPublishNotification, kIOServiceFirstPublishState, 0xffffffff ); } LOCKREADNOTIFY(); __state[1] &= ~kIOServiceNeedConfigState; - __state[1] |= kIOServiceConfigState; + __state[1] |= kIOServiceConfigState | kIOServiceConfigRunning; didRegister = (0 == (kIOServiceRegisteredState & __state[0])); __state[0] |= kIOServiceRegisteredState; @@ -3583,6 +3719,7 @@ void IOService::doServiceMatch( IOOptionBits options ) UNLOCKNOTIFY(); unlockForArbitration(); + invokeNotifiers(¬ifiers[0]); if (keepGuessing && matches->getCount() && (kIOReturnSuccess == getResources())) { @@ -3613,15 +3750,22 @@ void IOService::doServiceMatch( IOOptionBits options ) if (resourceKeys) setProperty(gIOResourceMatchedKey, resourceKeys); - deliverNotification( gIOMatchedNotification, - kIOServiceMatchedState, 0xffffffff ); + notifiers[0] = copyNotifiers(gIOMatchedNotification, + kIOServiceMatchedState, 0xffffffff); if( 0 == (__state[0] & kIOServiceFirstMatchState)) - deliverNotification( gIOFirstMatchNotification, - kIOServiceFirstMatchState, 0xffffffff ); + notifiers[1] = copyNotifiers(gIOFirstMatchNotification, + kIOServiceFirstMatchState, 0xffffffff); } + __state[1] &= ~kIOServiceConfigRunning; + unlockForArbitration(); + if (resourceKeys) resourceKeys->release(); + invokeNotifiers(¬ifiers[0]); + invokeNotifiers(¬ifiers[1]); + + lockForArbitration(); __state[1] &= ~kIOServiceConfigState; scheduleTerminatePhase2(); @@ -3785,24 +3929,24 @@ IOReturn IOService::waitForState( UInt32 mask, UInt32 value, return( kIOReturnSuccess ); } -#if NO_KEXTD -#define WAITING_KEXTD false -#else -extern bool gIOKextdClearedBusy; -#define WAITING_KEXTD (false == gIOKextdClearedBusy) -#endif - IOReturn IOService::waitQuiet( uint64_t timeout ) { IOReturn ret; uint32_t loops; - char * string = NULL; - size_t len; + char * string = NULL; + char * panicString = NULL; + size_t len; + size_t panicStringLen; uint64_t time; uint64_t nano; + bool kextdWait; + bool dopanic; + + enum { kTimeoutExtensions = 4 }; time = mach_absolute_time(); - for (loops = 0; loops < 2; loops++) + kextdWait = false; + for (loops = 0; loops < kTimeoutExtensions; loops++) { ret = waitForState( kIOServiceBusyStateMask, 0, timeout ); @@ -3810,15 +3954,13 @@ IOReturn IOService::waitQuiet( uint64_t timeout ) { time = mach_absolute_time() - time; absolutetime_to_nanoseconds(*(AbsoluteTime *)&time, &nano); - IOLog("busy extended ok[%d], (%llds, %llds), kextd wait(%d): %s\n", - loops, timeout / 1000000000ULL, nano / 1000000000ULL, WAITING_KEXTD, - string ? string : ""); + IOLog("busy extended ok[%d], (%llds, %llds)\n", + loops, timeout / 1000000000ULL, nano / 1000000000ULL); break; } else if (kIOReturnTimeout != ret) break; else if (timeout < 41000000000) break; - if (!loops) { IORegistryIterator * iter; OSOrderedSet * set; @@ -3829,17 +3971,23 @@ IOReturn IOService::waitQuiet( uint64_t timeout ) size_t l; len = 256; - string = IONew(char, len); + panicStringLen = 256; + if (!string) string = IONew(char, len); + if (!panicString) panicString = IONew(char, panicStringLen); set = NULL; + kextdWait = OSKext::isWaitingKextd(); iter = IORegistryIterator::iterateOver(this, gIOServicePlane, kIORegistryIterateRecursively); leaves = OSOrderedSet::withCapacity(4); if (iter) set = iter->iterateAll(); - if (string && leaves && set) + if (string && panicString && leaves && set) { + string[0] = panicString[0] = 0; + set->setObject(this); while ((next = (IOService *) set->getLastObject())) { if (next->getBusyState()) { + if (kIOServiceModuleStallState & next->__state[1]) kextdWait = true; leaves->setObject(next); nextParent = next; while ((nextParent = nextParent->getProvider())) @@ -3864,21 +4012,20 @@ IOReturn IOService::waitQuiet( uint64_t timeout ) OSSafeReleaseNULL(set); OSSafeReleaseNULL(iter); } - if (loops && (kIOWaitQuietPanics & gIOKitDebug)) - { - panic("busy timeout[%d], (%llds), kextd wait(%d): %s", - loops, timeout / 1000000000ULL, WAITING_KEXTD, - string ? string : ""); - } - else - { - IOLog("busy timeout[%d], (%llds), kextd wait(%d): %s\n", - loops, timeout / 1000000000ULL, WAITING_KEXTD, - string ? string : ""); - } + + dopanic = ((loops >= (kTimeoutExtensions - 1)) && (kIOWaitQuietPanics & gIOKitDebug)); + snprintf(panicString, panicStringLen, + "%s[%d], (%llds): %s", + kextdWait ? "kextd stall" : "busy timeout", + loops, timeout / 1000000000ULL, + string ? string : ""); + IOLog("%s\n", panicString); + if (dopanic) panic("%s", panicString); + else if (!loops) getPMRootDomain()->startSpinDump(1); } - if (string) IODelete(string, char, 256); + if (string) IODelete(string, char, 256); + if (panicString) IODelete(panicString, char, panicStringLen); return (ret); } @@ -3948,7 +4095,7 @@ void _IOConfigThread::main(void * arg, wait_result_t result) if( gIOKitDebug & kIOLogConfig) LOG("config(%p): starting on %s, %d\n", - OBFUSCATE(IOThreadSelf()), job->nub->getName(), job->type); + IOSERVICE_OBFUSCATE(IOThreadSelf()), job->nub->getName(), job->type); switch( job->type) { @@ -3958,7 +4105,7 @@ void _IOConfigThread::main(void * arg, wait_result_t result) default: LOG("config(%p): strange type (%d)\n", - OBFUSCATE(IOThreadSelf()), job->type ); + IOSERVICE_OBFUSCATE(IOThreadSelf()), job->type ); break; } @@ -3982,7 +4129,7 @@ void _IOConfigThread::main(void * arg, wait_result_t result) } while( alive ); if( gIOKitDebug & kIOLogConfig) - LOG("config(%p): terminating\n", OBFUSCATE(IOThreadSelf()) ); + LOG("config(%p): terminating\n", IOSERVICE_OBFUSCATE(IOThreadSelf()) ); self->release(); } @@ -4243,8 +4390,8 @@ OSObject * IOService::copyExistingServices( OSDictionary * matching, OSSerialize * s2 = OSSerialize::withCapacity(128); current->serialize(s1); _current->serialize(s2); - kprintf("**mismatch** %p %p\n%s\n%s\n%s\n", OBFUSCATE(current), - OBFUSCATE(_current), s->text(), s1->text(), s2->text()); + kprintf("**mismatch** %p %p\n%s\n%s\n%s\n", IOSERVICE_OBFUSCATE(current), + IOSERVICE_OBFUSCATE(_current), s->text(), s1->text(), s2->text()); s1->release(); s2->release(); } @@ -4329,6 +4476,7 @@ IONotifier * IOService::setNotification( if( notify) { notify->handler = handler; notify->target = target; + notify->type = type; notify->matching = matching; matching->retain(); if (handler == &_IOServiceMatchingNotificationHandler) @@ -4386,7 +4534,7 @@ IONotifier * IOService::doInstallNotification( else if (type == gIOFirstMatchNotification) inState = kIOServiceFirstMatchState; - else if( type == gIOTerminatedNotification) + else if ((type == gIOTerminatedNotification) || (type == gIOWillTerminateNotification)) inState = 0; else return( 0 ); @@ -4484,14 +4632,14 @@ IONotifier * IOService::addMatchingNotification( if (!ret) return (0); // send notifications for existing set - if (existing) { - - while( (next = (IOService *) existing->getNextObject())) { - - next->lockForArbitration(); + if (existing) + { + while( (next = (IOService *) existing->getNextObject())) + { if( 0 == (next->__state[0] & kIOServiceInactiveState)) - next->invokeNotifer( notify ); - next->unlockForArbitration(); + { + next->invokeNotifier( notify ); + } } existing->release(); } @@ -4596,6 +4744,12 @@ IOService * IOService::waitForService( OSDictionary * matching, void IOService::deliverNotification( const OSSymbol * type, IOOptionBits orNewState, IOOptionBits andNewState ) +{ + panic("deliverNotification"); +} + +OSArray * IOService::copyNotifiers(const OSSymbol * type, + IOOptionBits orNewState, IOOptionBits andNewState ) { _IOServiceNotifier * notify; OSIterator * iter; @@ -4604,7 +4758,8 @@ void IOService::deliverNotification( const OSSymbol * type, lockForArbitration(); if( (0 == (__state[0] & kIOServiceInactiveState)) - || (type == gIOTerminatedNotification)) { + || (type == gIOTerminatedNotification) + || (type == gIOWillTerminateNotification)) { LOCKREADNOTIFY(); @@ -4624,21 +4779,14 @@ void IOService::deliverNotification( const OSSymbol * type, } iter->release(); } - __state[0] = (__state[0] | orNewState) & andNewState; - UNLOCKNOTIFY(); } - if( willSend) { - for( unsigned int idx = 0; - (notify = (_IOServiceNotifier *) willSend->getObject(idx)); - idx++) { - invokeNotifer( notify ); - } - willSend->release(); - } unlockForArbitration(); + + return (willSend); + } IOOptionBits IOService::getState( void ) const @@ -5461,8 +5609,10 @@ bool IOService::matchPassive(OSDictionary * table, uint32_t options) assert( table ); +#if !CONFIG_EMBEDDED OSArray* aliasServiceRegIds = NULL; IOService* foundAlternateService = NULL; +#endif #if MATCH_DEBUG OSDictionary * root = table; @@ -5534,8 +5684,10 @@ bool IOService::matchPassive(OSDictionary * table, uint32_t options) } if(matchParent == true) { +#if !CONFIG_EMBEDDED // check if service has an alias to search its other "parents" if a parent match isn't found - OSNumber* alternateRegistryID = OSDynamicCast(OSNumber, where->getProperty(kIOServiceLegacyMatchingRegistryIDKey)); + OSObject * prop = where->copyProperty(gIOServiceLegacyMatchingRegistryIDKey); + OSNumber * alternateRegistryID = OSDynamicCast(OSNumber, prop); if(alternateRegistryID != NULL) { if(aliasServiceRegIds == NULL) { @@ -5543,12 +5695,15 @@ bool IOService::matchPassive(OSDictionary * table, uint32_t options) } aliasServiceRegIds->setObject(alternateRegistryID); } + OSSafeReleaseNULL(prop); +#endif } else { break; } where = where->getProvider(); +#if !CONFIG_EMBEDDED if(where == NULL) { // there were no matching parent services, check to see if there are aliased services that have a matching parent if(aliasServiceRegIds != NULL) { @@ -5570,11 +5725,14 @@ bool IOService::matchPassive(OSDictionary * table, uint32_t options) } } } +#endif } while( where != NULL ); +#if !CONFIG_EMBEDDED OSSafeReleaseNULL(foundAlternateService); OSSafeReleaseNULL(aliasServiceRegIds); +#endif #if MATCH_DEBUG if (where != this) @@ -5596,30 +5754,35 @@ IOReturn IOService::newUserClient( task_t owningTask, void * securityID, { const OSSymbol *userClientClass = 0; IOUserClient *client; + OSObject *prop; OSObject *temp; if (kIOReturnSuccess == newUserClient( owningTask, securityID, type, handler )) return kIOReturnSuccess; // First try my own properties for a user client class name - temp = getProperty(gIOUserClientClassKey); - if (temp) { - if (OSDynamicCast(OSSymbol, temp)) - userClientClass = (const OSSymbol *) temp; - else if (OSDynamicCast(OSString, temp)) { - userClientClass = OSSymbol::withString((OSString *) temp); + prop = copyProperty(gIOUserClientClassKey); + if (prop) { + if (OSDynamicCast(OSSymbol, prop)) + userClientClass = (const OSSymbol *) prop; + else if (OSDynamicCast(OSString, prop)) { + userClientClass = OSSymbol::withString((OSString *) prop); if (userClientClass) - setProperty(kIOUserClientClassKey, + setProperty(gIOUserClientClassKey, (OSObject *) userClientClass); } } // Didn't find one so lets just bomb out now without further ado. if (!userClientClass) + { + OSSafeReleaseNULL(prop); return kIOReturnUnsupported; + } // This reference is consumed by the IOServiceOpen call temp = OSMetaClass::allocClassWithName(userClientClass); + OSSafeReleaseNULL(prop); if (!temp) return kIOReturnNoMemory; @@ -6303,7 +6466,9 @@ IOReturn IOService::addInterruptStatistics(IOInterruptAccountingData * statistic /* * We now need to add the legend for this reporter to the registry. */ - legend = IOReportLegend::with(OSDynamicCast(OSArray, getProperty(kIOReportLegendKey))); + OSObject * prop = copyProperty(kIOReportLegendKey); + legend = IOReportLegend::with(OSDynamicCast(OSArray, prop)); + OSSafeReleaseNULL(prop); /* * Note that while we compose the subgroup name, we do not need to diff --git a/iokit/Kernel/IOServicePM.cpp b/iokit/Kernel/IOServicePM.cpp index e7fbb0802..6fc90a603 100644 --- a/iokit/Kernel/IOServicePM.cpp +++ b/iokit/Kernel/IOServicePM.cpp @@ -181,7 +181,11 @@ do { \ #define ns_per_us 1000 #define k30Seconds (30*us_per_s) #define k5Seconds ( 5*us_per_s) +#if CONFIG_EMBEDDED +#define kCanSleepMaxTimeReq k5Seconds +#else #define kCanSleepMaxTimeReq k30Seconds +#endif #define kMaxTimeRequested k30Seconds #define kMinAckTimeoutTicks (10*1000000) #define kIOPMTardyAckSPSKey "IOPMTardyAckSetPowerState" @@ -241,6 +245,9 @@ do { \ // use message tracer to log messages longer than (ns): #define LOG_APP_RESPONSE_MSG_TRACER (3 * 1000ULL * 1000ULL * 1000ULL) +// log kext responses longer than (ns): +#define LOG_KEXT_RESPONSE_TIMES (100ULL * 1000ULL * 1000ULL) + enum { kReserveDomainPower = 1 }; @@ -5289,6 +5296,10 @@ bool IOService::ackTimerTick( void ) PM_ERROR("%s::setPowerState(%p, %lu -> %lu) timed out after %d ms\n", fName, OBFUSCATE(this), fCurrentPowerState, fHeadNotePowerState, NS_TO_MS(nsec)); +#if DEBUG && CONFIG_EMBEDDED + panic("%s::setPowerState(%p, %lu -> %lu) timed out after %d ms", + fName, this, fCurrentPowerState, fHeadNotePowerState, NS_TO_MS(nsec)); +#else if (gIOKitDebug & kIOLogDebugPower) { panic("%s::setPowerState(%p, %lu -> %lu) timed out after %d ms", @@ -5299,6 +5310,7 @@ bool IOService::ackTimerTick( void ) // Unblock state machine and pretend driver has acked. done = true; } +#endif } else { // still waiting, set timer again start_ack_timer(); @@ -5772,7 +5784,7 @@ static void logAppTimeouts( OSObject * object, void * arg ) // TODO: record message type if possible IOService::getPMRootDomain()->pmStatsRecordApplicationResponse( - gIOPMStatsApplicationResponseTimedOut, + gIOPMStatsResponseTimedOut, name, 0, (30*1000), pid, object); } @@ -5915,7 +5927,6 @@ bool IOService::tellClientsWithResponse( int messageType ) applyToInterested( gIOGeneralInterest, pmTellClientWithResponse, (void *) &context ); - fNotifyClientArray = context.notifyClients; break; case kNotifyPriority: @@ -5936,7 +5947,6 @@ bool IOService::tellClientsWithResponse( int messageType ) case kNotifyCapabilityChangeApps: applyToInterested( gIOAppPowerStateInterest, pmTellCapabilityAppWithResponse, (void *) &context ); - fNotifyClientArray = context.notifyClients; if(context.messageType == kIOMessageCanSystemSleep) { maxTimeOut = kCanSleepMaxTimeReq; @@ -5954,6 +5964,7 @@ bool IOService::tellClientsWithResponse( int messageType ) pmTellCapabilityClientWithResponse, (void *) &context ); break; } + fNotifyClientArray = context.notifyClients; // do we have to wait for somebody? if ( !checkForDone() ) @@ -6097,6 +6108,8 @@ void IOService::pmTellClientWithResponse( OSObject * object, void * arg ) _IOServiceInterestNotifier * notifier; uint32_t msgIndex, msgRef, msgType; IOReturn retCode; + AbsoluteTime start, end; + uint64_t nsec; if (context->messageFilter && !context->messageFilter(context->us, object, context, 0, 0)) @@ -6137,6 +6150,9 @@ void IOService::pmTellClientWithResponse( OSObject * object, void * arg ) OBFUSCATE(object), OBFUSCATE(notifier->handler)); } + if (0 == context->notifyClients) + context->notifyClients = OSArray::withCapacity( 32 ); + notify.powerRef = (void *)(uintptr_t) msgRef; notify.returnValue = 0; notify.stateNumber = context->stateNumber; @@ -6144,15 +6160,18 @@ void IOService::pmTellClientWithResponse( OSObject * object, void * arg ) if (context->enableTracing && (notifier != 0)) { - getPMRootDomain()->traceDetail(msgType, msgIndex, (uintptr_t) notifier->handler); + getPMRootDomain()->traceDetail(notifier); } + clock_get_uptime(&start); retCode = context->us->messageClient(msgType, object, (void *) ¬ify, sizeof(notify)); + clock_get_uptime(&end); if (kIOReturnSuccess == retCode) { if (0 == notify.returnValue) { OUR_PMLog(kPMLogClientAcknowledge, msgRef, (uintptr_t) object); + context->responseArray->setObject(msgIndex, replied); } else { replied = kOSBooleanFalse; if ( notify.returnValue > context->maxTimeRequested ) @@ -6169,14 +6188,39 @@ void IOService::pmTellClientWithResponse( OSObject * object, void * arg ) else context->maxTimeRequested = notify.returnValue; } + // + // Track time taken to ack, by storing the timestamp of + // callback completion + OSNumber * num; + num = OSNumber::withNumber(AbsoluteTime_to_scalar(&end), sizeof(uint64_t) * 8); + if (num) { + context->responseArray->setObject(msgIndex, num); + num->release(); + } + else { + context->responseArray->setObject(msgIndex, replied); + } } - } else { + + if (context->enableTracing) { + SUB_ABSOLUTETIME(&end, &start); + absolutetime_to_nanoseconds(end, &nsec); + + if ((nsec > LOG_KEXT_RESPONSE_TIMES) || (notify.returnValue != 0)) { + getPMRootDomain()->traceAckDelay(notifier, notify.returnValue/1000, NS_TO_MS(nsec)); + } + } + } + else { // not a client of ours // so we won't be waiting for response OUR_PMLog(kPMLogClientAcknowledge, msgRef, 0); + context->responseArray->setObject(msgIndex, replied); + } + if (context->notifyClients) { + context->notifyClients->setObject(msgIndex, object); } - context->responseArray->setObject(msgIndex, replied); } //********************************************************************************* @@ -6278,6 +6322,8 @@ void IOService::pmTellCapabilityClientWithResponse( _IOServiceInterestNotifier * notifier; uint32_t msgIndex, msgRef, msgType; IOReturn retCode; + AbsoluteTime start, end; + uint64_t nsec; memset(&msgArg, 0, sizeof(msgArg)); if (context->messageFilter && @@ -6295,6 +6341,9 @@ void IOService::pmTellCapabilityClientWithResponse( return; } + if (0 == context->notifyClients) { + context->notifyClients = OSArray::withCapacity( 32 ); + } notifier = OSDynamicCast(_IOServiceInterestNotifier, object); msgType = context->messageType; msgIndex = context->responseArray->getCount(); @@ -6324,11 +6373,13 @@ void IOService::pmTellCapabilityClientWithResponse( if (context->enableTracing && (notifier != 0)) { - getPMRootDomain()->traceDetail(msgType, msgIndex, (uintptr_t) notifier->handler); + getPMRootDomain()->traceDetail(notifier); } + clock_get_uptime(&start); retCode = context->us->messageClient( msgType, object, (void *) &msgArg, sizeof(msgArg)); + clock_get_uptime(&end); if ( kIOReturnSuccess == retCode ) { @@ -6336,6 +6387,7 @@ void IOService::pmTellCapabilityClientWithResponse( { // client doesn't want time to respond OUR_PMLog(kPMLogClientAcknowledge, msgRef, (uintptr_t) object); + context->responseArray->setObject(msgIndex, replied); } else { @@ -6346,14 +6398,35 @@ void IOService::pmTellCapabilityClientWithResponse( { context->maxTimeRequested = kCapabilityClientMaxWait; PM_ERROR("%s: client %p returned %u for %s\n", - context->us->getName(), - notifier ? (void *) OBFUSCATE(notifier->handler) : OBFUSCATE(object), - msgArg.maxWaitForReply, - getIOMessageString(msgType)); + context->us->getName(), + notifier ? (void *) OBFUSCATE(notifier->handler) : OBFUSCATE(object), + msgArg.maxWaitForReply, + getIOMessageString(msgType)); } else context->maxTimeRequested = msgArg.maxWaitForReply; } + + // Track time taken to ack, by storing the timestamp of + // callback completion + OSNumber * num; + num = OSNumber::withNumber(AbsoluteTime_to_scalar(&end), sizeof(uint64_t) * 8); + if (num) { + context->responseArray->setObject(msgIndex, num); + num->release(); + } + else { + context->responseArray->setObject(msgIndex, replied); + } + } + + if (context->enableTracing) { + SUB_ABSOLUTETIME(&end, &start); + absolutetime_to_nanoseconds(end, &nsec); + + if ((nsec > LOG_KEXT_RESPONSE_TIMES) || (msgArg.maxWaitForReply != 0)) { + getPMRootDomain()->traceAckDelay(notifier, msgArg.maxWaitForReply/1000, NS_TO_MS(nsec)); + } } } else @@ -6361,9 +6434,12 @@ void IOService::pmTellCapabilityClientWithResponse( // not a client of ours // so we won't be waiting for response OUR_PMLog(kPMLogClientAcknowledge, msgRef, 0); + context->responseArray->setObject(msgIndex, replied); + } + if (context->notifyClients) { + context->notifyClients->setObject(msgIndex, object); } - context->responseArray->setObject(msgIndex, replied); } //********************************************************************************* @@ -6630,47 +6706,51 @@ bool IOService::responseValid( uint32_t refcon, int pid ) uint64_t nsec; char name[128]; - name[0] = '\0'; - proc_name(pid, name, sizeof(name)); clock_get_uptime(&now); AbsoluteTime_to_scalar(&start) = num->unsigned64BitValue(); SUB_ABSOLUTETIME(&now, &start); absolutetime_to_nanoseconds(now, &nsec); + if (pid != 0) { + name[0] = '\0'; + proc_name(pid, name, sizeof(name)); + + if (nsec > LOG_APP_RESPONSE_TIMES) + { + IOLog("PM response took %d ms (%d, %s)\n", NS_TO_MS(nsec), + pid, name); + } + + + if (nsec > LOG_APP_RESPONSE_MSG_TRACER) + { + // TODO: populate the messageType argument + getPMRootDomain()->pmStatsRecordApplicationResponse( + gIOPMStatsResponseSlow, + name, 0, NS_TO_MS(nsec), pid, object); + } + else + { + getPMRootDomain()->pmStatsRecordApplicationResponse( + gIOPMStatsResponsePrompt, + name, 0, NS_TO_MS(nsec), pid, object); + } + } + else { + getPMRootDomain()->traceAckDelay(object, 0, NS_TO_MS(nsec)); + } + if (kIOLogDebugPower & gIOKitDebug) { PM_LOG("Ack(%u) %u ms\n", (uint32_t) ordinalComponent, NS_TO_MS(nsec)); } - - // > 100 ms - if (nsec > LOG_APP_RESPONSE_TIMES) - { - IOLog("PM response took %d ms (%d, %s)\n", NS_TO_MS(nsec), - pid, name); - } - - if (nsec > LOG_APP_RESPONSE_MSG_TRACER) - { - // TODO: populate the messageType argument - getPMRootDomain()->pmStatsRecordApplicationResponse( - gIOPMStatsApplicationResponseSlow, - name, 0, NS_TO_MS(nsec), pid, object); - } - else - { - getPMRootDomain()->pmStatsRecordApplicationResponse( - gIOPMStatsApplicationResponsePrompt, - name, 0, NS_TO_MS(nsec), pid, object); - } - - theFlag = kOSBooleanFalse; } else if (object) { getPMRootDomain()->pmStatsRecordApplicationResponse( - gIOPMStatsApplicationResponsePrompt, + gIOPMStatsResponsePrompt, 0, 0, 0, pid, object); } @@ -7855,7 +7935,7 @@ bool IOService::actionPMReplyQueue( IOPMRequest * request, IOPMRequestQueue * qu OSString * name = (OSString *) request->fArg2; getPMRootDomain()->pmStatsRecordApplicationResponse( - gIOPMStatsApplicationResponseCancel, + gIOPMStatsResponseCancel, name ? name->getCStringNoCopy() : "", 0, 0, (int)(uintptr_t) request->fArg1, 0); } @@ -8101,7 +8181,9 @@ const char * IOService::getIOMessageString( uint32_t msg ) MSG_ENTRY( kIOMessageSystemWillRestart ), MSG_ENTRY( kIOMessageSystemWillPowerOn ), MSG_ENTRY( kIOMessageSystemCapabilityChange ), - MSG_ENTRY( kIOPMMessageLastCallBeforeSleep ) + MSG_ENTRY( kIOPMMessageLastCallBeforeSleep ), + MSG_ENTRY( kIOMessageSystemPagingOff ), + { 0, NULL } }; return IOFindNameForValue(msg, msgNames); diff --git a/iokit/Kernel/IOServicePMPrivate.h b/iokit/Kernel/IOServicePMPrivate.h index f332c23ee..0dbc58aca 100644 --- a/iokit/Kernel/IOServicePMPrivate.h +++ b/iokit/Kernel/IOServicePMPrivate.h @@ -549,10 +549,10 @@ enum { // PM Statistics & Diagnostics //****************************************************************************** -extern const OSSymbol *gIOPMStatsApplicationResponseTimedOut; -extern const OSSymbol *gIOPMStatsApplicationResponseCancel; -extern const OSSymbol *gIOPMStatsApplicationResponseSlow; -extern const OSSymbol *gIOPMStatsApplicationResponsePrompt; +extern const OSSymbol *gIOPMStatsResponseTimedOut; +extern const OSSymbol *gIOPMStatsResponseCancel; +extern const OSSymbol *gIOPMStatsResponseSlow; +extern const OSSymbol *gIOPMStatsResponsePrompt; extern const OSSymbol *gIOPMStatsDriverPSChangeSlow; //****************************************************************************** diff --git a/iokit/Kernel/IOServicePrivate.h b/iokit/Kernel/IOServicePrivate.h index 4ad23fa1a..2041d0806 100644 --- a/iokit/Kernel/IOServicePrivate.h +++ b/iokit/Kernel/IOServicePrivate.h @@ -64,6 +64,7 @@ enum { kIOServiceRecursing = 0x00100000, kIOServiceNeedWillTerminate = 0x00080000, kIOServiceWaitDetachState = 0x00040000, + kIOServiceConfigRunning = 0x00020000, }; // notify state @@ -88,6 +89,7 @@ public: OSOrderedSet * whence; OSDictionary * matching; + const OSSymbol * type; IOServiceMatchingNotificationHandler handler; IOServiceNotificationHandler compatHandler; void * target; diff --git a/iokit/Kernel/IOSimpleReporter.cpp b/iokit/Kernel/IOSimpleReporter.cpp index 6707bda89..de430f566 100644 --- a/iokit/Kernel/IOSimpleReporter.cpp +++ b/iokit/Kernel/IOSimpleReporter.cpp @@ -37,7 +37,7 @@ OSDefineMetaClassAndStructors(IOSimpleReporter, IOReporter); IOSimpleReporter* IOSimpleReporter::with(IOService *reportingService, IOReportCategories categories, - IOReportUnits unit) + IOReportUnit unit) { IOSimpleReporter *reporter, *rval = NULL; @@ -65,7 +65,7 @@ finish: bool IOSimpleReporter::initWith(IOService *reportingService, IOReportCategories categories, - IOReportUnits unit) + IOReportUnit unit) { // fully specify the channel type for the superclass IOReportChannelType channelType = { diff --git a/iokit/Kernel/IOStartIOKit.cpp b/iokit/Kernel/IOStartIOKit.cpp index f9d9fc018..6eb48b713 100644 --- a/iokit/Kernel/IOStartIOKit.cpp +++ b/iokit/Kernel/IOStartIOKit.cpp @@ -77,7 +77,7 @@ void IOKitInitializeTime( void ) IOService::resourceMatching("IONVRAM"), &t ); #endif - clock_initialize_calendar(); + clock_initialize_calendar(); } void iokit_post_constructor_init(void) @@ -112,9 +112,6 @@ void iokit_post_constructor_init(void) } } -// From <osfmk/kern/debug.c> -extern int debug_mode; - /***** * Pointer into bootstrap KLD segment for functions never used past startup. */ @@ -137,10 +134,6 @@ void StartIOKit( void * p1, void * p2, void * p3, void * p4 ) // Compat for boot-args gIOKitTrace |= (gIOKitDebug & kIOTraceCompatBootArgs); - // Check for the log synchronous bit set in io - if (gIOKitDebug & kIOLogSynchronous) - debug_mode = true; - if( PE_parse_boot_argn( "pmtimeout", &debugFlags, sizeof (debugFlags) )) gCanSleepTimeout = debugFlags; // diff --git a/iokit/Kernel/IOStateReporter.cpp b/iokit/Kernel/IOStateReporter.cpp index f6e0b7340..e1214dc55 100644 --- a/iokit/Kernel/IOStateReporter.cpp +++ b/iokit/Kernel/IOStateReporter.cpp @@ -40,7 +40,7 @@ IOStateReporter* IOStateReporter::with(IOService *reportingService, IOReportCategories categories, int nstates, - IOReportUnits unit/* = kIOReportUnitHWTicks*/) + IOReportUnit unit/* = kIOReportUnitHWTicks*/) { IOStateReporter *reporter, *rval = NULL; @@ -68,7 +68,7 @@ bool IOStateReporter::initWith(IOService *reportingService, IOReportCategories categories, int16_t nstates, - IOReportUnits unit) + IOReportUnit unit) { bool success = false; diff --git a/iokit/Kernel/IOStatistics.cpp b/iokit/Kernel/IOStatistics.cpp index b67de2511..f3771602a 100644 --- a/iokit/Kernel/IOStatistics.cpp +++ b/iokit/Kernel/IOStatistics.cpp @@ -815,8 +815,11 @@ int IOStatistics::getUserClientStatistics(sysctl_req *req) goto exit; } - SYSCTL_IN(req, &requestedLoadTag, sizeof(requestedLoadTag)); - + error = SYSCTL_IN(req, &requestedLoadTag, sizeof(requestedLoadTag)); + if (error) { + goto exit; + } + LOG(2, "IOStatistics::getUserClientStatistics - requesting kext w/load tag: %d\n", requestedLoadTag); buffer = (char*)kalloc(calculatedSize); diff --git a/iokit/Kernel/IOStringFuncs.c b/iokit/Kernel/IOStringFuncs.c index d536b7243..c4f9458fe 100644 --- a/iokit/Kernel/IOStringFuncs.c +++ b/iokit/Kernel/IOStringFuncs.c @@ -86,7 +86,6 @@ long strtol(const char *nptr, char **endptr, int base); unsigned long strtoul(const char *nptr, char **endptr, int base); quad_t strtoq(const char *nptr, char **endptr, int base); u_quad_t strtouq(const char *nptr, char **endptr, int base); -char *strchr(const char *str, int ch); char *strncat(char *s1, const char *s2, unsigned long n); @@ -469,19 +468,6 @@ strtouq(const char *nptr, } -/* - * - */ - -char *strchr(const char *str, int ch) -{ - do { - if (*str == ch) - return(__CAST_AWAY_QUALIFIER(str, const, char *)); - } while (*str++); - return ((char *) 0); -} - /* * */ diff --git a/iokit/Kernel/IOTimerEventSource.cpp b/iokit/Kernel/IOTimerEventSource.cpp index df939da91..bb7acea43 100644 --- a/iokit/Kernel/IOTimerEventSource.cpp +++ b/iokit/Kernel/IOTimerEventSource.cpp @@ -47,9 +47,9 @@ __END_DECLS #define super IOEventSource OSDefineMetaClassAndStructors(IOTimerEventSource, IOEventSource) -OSMetaClassDefineReservedUnused(IOTimerEventSource, 0); -OSMetaClassDefineReservedUnused(IOTimerEventSource, 1); -OSMetaClassDefineReservedUnused(IOTimerEventSource, 2); +OSMetaClassDefineReservedUsed(IOTimerEventSource, 0); +OSMetaClassDefineReservedUsed(IOTimerEventSource, 1); +OSMetaClassDefineReservedUsed(IOTimerEventSource, 2); OSMetaClassDefineReservedUnused(IOTimerEventSource, 3); OSMetaClassDefineReservedUnused(IOTimerEventSource, 4); OSMetaClassDefineReservedUnused(IOTimerEventSource, 5); @@ -95,6 +95,28 @@ do { \ // Timeout handler function. This function is called by the kernel when // the timeout interval expires. // + +static __inline__ void +InvokeAction(IOTimerEventSource::Action action, IOTimerEventSource * ts, + OSObject * owner, IOWorkLoop * workLoop) +{ + bool trace = (gIOKitTrace & kIOTraceTimers) ? true : false; + + if (trace) + IOTimeStampStartConstant(IODBG_TIMES(IOTIMES_ACTION), + VM_KERNEL_ADDRHIDE(action), VM_KERNEL_ADDRHIDE(owner)); + + (*action)(owner, ts); + +#if CONFIG_DTRACE + DTRACE_TMR3(iotescallout__expire, Action, action, OSObject, owner, void, workLoop); +#endif + + if (trace) + IOTimeStampEndConstant(IODBG_TIMES(IOTIMES_ACTION), + VM_KERNEL_UNSLIDE(action), VM_KERNEL_ADDRHIDE(owner)); +} + void IOTimerEventSource::timeout(void *self) { IOTimerEventSource *me = (IOTimerEventSource *) self; @@ -113,20 +135,7 @@ void IOTimerEventSource::timeout(void *self) doit = (Action) me->action; if (doit && me->enabled && AbsoluteTime_to_scalar(&me->abstime)) { - bool trace = (gIOKitTrace & kIOTraceTimers) ? true : false; - - if (trace) - IOTimeStampStartConstant(IODBG_TIMES(IOTIMES_ACTION), - VM_KERNEL_UNSLIDE(doit), (uintptr_t) me->owner); - - (*doit)(me->owner, me); -#if CONFIG_DTRACE - DTRACE_TMR3(iotescallout__expire, Action, doit, OSObject, me->owner, void, me->workLoop); -#endif - - if (trace) - IOTimeStampEndConstant(IODBG_TIMES(IOTIMES_ACTION), - VM_KERNEL_UNSLIDE(doit), (uintptr_t) me->owner); + InvokeAction(doit, me, me->owner, me->workLoop); } IOStatisticsOpenGate(); wl->openGate(); @@ -155,20 +164,7 @@ void IOTimerEventSource::timeoutAndRelease(void * self, void * c) doit = (Action) me->action; if (doit && (me->reserved->calloutGeneration == count)) { - bool trace = (gIOKitTrace & kIOTraceTimers) ? true : false; - - if (trace) - IOTimeStampStartConstant(IODBG_TIMES(IOTIMES_ACTION), - VM_KERNEL_UNSLIDE(doit), (uintptr_t) me->owner); - - (*doit)(me->owner, me); -#if CONFIG_DTRACE - DTRACE_TMR3(iotescallout__expire, Action, doit, OSObject, me->owner, void, me->workLoop); -#endif - - if (trace) - IOTimeStampEndConstant(IODBG_TIMES(IOTIMES_ACTION), - VM_KERNEL_UNSLIDE(doit), (uintptr_t) me->owner); + InvokeAction(doit, me, me->owner, me->workLoop); } IOStatisticsOpenGate(); wl->openGate(); @@ -179,13 +175,99 @@ void IOTimerEventSource::timeoutAndRelease(void * self, void * c) me->release(); } +// -- work loop delivery + +bool IOTimerEventSource::checkForWork() +{ + Action doit; + + if (reserved + && (reserved->calloutGenerationSignaled == reserved->calloutGeneration) + && enabled && (doit = (Action) action)) + { + reserved->calloutGenerationSignaled = ~reserved->calloutGeneration; + InvokeAction(doit, this, owner, workLoop); + } + + return false; +} + +void IOTimerEventSource::timeoutSignaled(void * self, void * c) +{ + IOTimerEventSource *me = (IOTimerEventSource *) self; + + me->reserved->calloutGenerationSignaled = (SInt32)(long) c; + if (me->enabled) me->signalWorkAvailable(); +} + +// -- + void IOTimerEventSource::setTimeoutFunc() { + thread_call_priority_t pri; + uint32_t options; + + if (reserved) panic("setTimeoutFunc already %p, %p", this, reserved); + // reserved != 0 means IOTimerEventSource::timeoutAndRelease is being used, // not a subclassed implementation reserved = IONew(ExpansionData, 1); - calloutEntry = (void *) thread_call_allocate((thread_call_func_t) &IOTimerEventSource::timeoutAndRelease, - (thread_call_param_t) this); + reserved->calloutGenerationSignaled = ~reserved->calloutGeneration; + options = abstime; + abstime = 0; + + thread_call_options_t tcoptions = 0; + thread_call_func_t func = NULL; + + switch (kIOTimerEventSourceOptionsPriorityMask & options) + { + case kIOTimerEventSourceOptionsPriorityHigh: + pri = THREAD_CALL_PRIORITY_HIGH; + func = &IOTimerEventSource::timeoutAndRelease; + break; + + case kIOTimerEventSourceOptionsPriorityKernel: + pri = THREAD_CALL_PRIORITY_KERNEL; + func = &IOTimerEventSource::timeoutAndRelease; + break; + + case kIOTimerEventSourceOptionsPriorityKernelHigh: + pri = THREAD_CALL_PRIORITY_KERNEL_HIGH; + func = &IOTimerEventSource::timeoutAndRelease; + break; + + case kIOTimerEventSourceOptionsPriorityUser: + pri = THREAD_CALL_PRIORITY_USER; + func = &IOTimerEventSource::timeoutAndRelease; + break; + + case kIOTimerEventSourceOptionsPriorityLow: + pri = THREAD_CALL_PRIORITY_LOW; + func = &IOTimerEventSource::timeoutAndRelease; + break; + + case kIOTimerEventSourceOptionsPriorityWorkLoop: + pri = THREAD_CALL_PRIORITY_KERNEL; + tcoptions |= THREAD_CALL_OPTIONS_SIGNAL; + if (kIOTimerEventSourceOptionsAllowReenter & options) break; + func = &IOTimerEventSource::timeoutSignaled; + break; + + default: + break; + } + + assertf(func, "IOTimerEventSource options 0x%x", options); + if (!func) return; // init will fail + + if (THREAD_CALL_OPTIONS_SIGNAL & tcoptions) flags |= kActive; + else flags |= kPassive; + + if (!(kIOTimerEventSourceOptionsAllowReenter & options)) tcoptions |= THREAD_CALL_OPTIONS_ONCE; + + calloutEntry = (void *) thread_call_allocate_with_options(func, + (thread_call_param_t) this, pri, tcoptions); + assert(calloutEntry); } bool IOTimerEventSource::init(OSObject *inOwner, Action inAction) @@ -202,12 +284,18 @@ bool IOTimerEventSource::init(OSObject *inOwner, Action inAction) return true; } +bool IOTimerEventSource::init(uint32_t options, OSObject *inOwner, Action inAction) +{ + abstime = options; + return (init(inOwner, inAction)); +} + IOTimerEventSource * -IOTimerEventSource::timerEventSource(OSObject *inOwner, Action inAction) +IOTimerEventSource::timerEventSource(uint32_t inOptions, OSObject *inOwner, Action inAction) { IOTimerEventSource *me = new IOTimerEventSource; - if (me && !me->init(inOwner, inAction)) { + if (me && !me->init(inOptions, inOwner, inAction)) { me->release(); return 0; } @@ -215,11 +303,25 @@ IOTimerEventSource::timerEventSource(OSObject *inOwner, Action inAction) return me; } +#define _thread_call_cancel(tc) ((kActive & flags) ? thread_call_cancel_wait((tc)) : thread_call_cancel((tc))) + +IOTimerEventSource * +IOTimerEventSource::timerEventSource(OSObject *inOwner, Action inAction) +{ + return (IOTimerEventSource::timerEventSource( + kIOTimerEventSourceOptionsPriorityKernelHigh, + inOwner, inAction)); +} + void IOTimerEventSource::free() { if (calloutEntry) { + __assert_only bool freed; + cancelTimeout(); - thread_call_free((thread_call_t) calloutEntry); + + freed = thread_call_free((thread_call_t) calloutEntry); + assert(freed); } if (reserved) @@ -232,9 +334,9 @@ void IOTimerEventSource::cancelTimeout() { if (reserved) reserved->calloutGeneration++; - bool active = thread_call_cancel((thread_call_t) calloutEntry); + bool active = _thread_call_cancel((thread_call_t) calloutEntry); AbsoluteTime_to_scalar(&abstime) = 0; - if (active && reserved) + if (active && reserved && (kPassive & flags)) { release(); workLoop->release(); @@ -252,9 +354,9 @@ void IOTimerEventSource::disable() { if (reserved) reserved->calloutGeneration++; - bool active = thread_call_cancel((thread_call_t) calloutEntry); + bool active = _thread_call_cancel((thread_call_t) calloutEntry); super::disable(); - if (active && reserved) + if (active && reserved && (kPassive & flags)) { release(); workLoop->release(); @@ -302,11 +404,17 @@ IOReturn IOTimerEventSource::setTimeout(mach_timespec_t interval) IOReturn IOTimerEventSource::setTimeout(AbsoluteTime interval) { AbsoluteTime end; + clock_absolutetime_interval_to_deadline(interval, &end); + return wakeAtTime(end); +} - clock_get_uptime(&end); - ADD_ABSOLUTETIME(&end, &interval); +IOReturn IOTimerEventSource::setTimeout(uint32_t options, + AbsoluteTime abstime, AbsoluteTime leeway) +{ + AbsoluteTime end; + clock_continuoustime_interval_to_deadline(abstime, &end); + return wakeAtTime(options, end, leeway); - return wakeAtTime(end); } IOReturn IOTimerEventSource::wakeAtTimeTicks(UInt32 ticks) @@ -355,6 +463,11 @@ void IOTimerEventSource::setWorkLoop(IOWorkLoop *inWorkLoop) } IOReturn IOTimerEventSource::wakeAtTime(AbsoluteTime inAbstime) +{ + return wakeAtTime(0, inAbstime, 0); +} + +IOReturn IOTimerEventSource::wakeAtTime(uint32_t options, AbsoluteTime inAbstime, AbsoluteTime leeway) { if (!action) return kIOReturnNoResources; @@ -362,21 +475,33 @@ IOReturn IOTimerEventSource::wakeAtTime(AbsoluteTime inAbstime) abstime = inAbstime; if ( enabled && AbsoluteTime_to_scalar(&inAbstime) && AbsoluteTime_to_scalar(&abstime) && workLoop ) { + uint32_t tcoptions = 0; + + if (kIOTimeOptionsWithLeeway & options) tcoptions |= THREAD_CALL_DELAY_LEEWAY; + if (kIOTimeOptionsContinuous & options) tcoptions |= THREAD_CALL_CONTINUOUS; + if (reserved) { - retain(); - workLoop->retain(); + if (kPassive & flags) + { + retain(); + workLoop->retain(); + } reserved->workLoop = workLoop; reserved->calloutGeneration++; - if (thread_call_enter1_delayed((thread_call_t) calloutEntry, - (void *)(uintptr_t) reserved->calloutGeneration, inAbstime)) + if (thread_call_enter_delayed_with_leeway((thread_call_t) calloutEntry, + (void *)(uintptr_t) reserved->calloutGeneration, inAbstime, leeway, tcoptions) + && (kPassive & flags)) { release(); workLoop->release(); } } else - thread_call_enter_delayed((thread_call_t) calloutEntry, inAbstime); + { + thread_call_enter_delayed_with_leeway((thread_call_t) calloutEntry, + NULL, inAbstime, leeway, tcoptions); + } } return kIOReturnSuccess; diff --git a/iokit/Kernel/IOUserClient.cpp b/iokit/Kernel/IOUserClient.cpp index 985e3b201..b16a516bb 100644 --- a/iokit/Kernel/IOUserClient.cpp +++ b/iokit/Kernel/IOUserClient.cpp @@ -647,6 +647,7 @@ public: void * reference, vm_size_t referenceSize, bool clientIs64 ); virtual void free() APPLE_KEXT_OVERRIDE; + void invalidatePort(void); static bool _handler( void * target, void * ref, IOService * newService, IONotifier * notifier ); @@ -680,6 +681,7 @@ public: bool clientIs64 ); virtual void free() APPLE_KEXT_OVERRIDE; + void invalidatePort(void); static IOReturn _handler( void * target, void * ref, UInt32 messageType, IOService * provider, @@ -785,6 +787,11 @@ bool IOServiceUserNotification::init( mach_port_t port, natural_t type, return( true ); } +void IOServiceUserNotification::invalidatePort(void) +{ + if (pingMsg) pingMsg->msgHdr.msgh_remote_port = MACH_PORT_NULL; +} + void IOServiceUserNotification::free( void ) { PingMsg * _pingMsg; @@ -941,6 +948,11 @@ bool IOServiceMessageUserNotification::init( mach_port_t port, natural_t type, return( true ); } +void IOServiceMessageUserNotification::invalidatePort(void) +{ + if (pingMsg) pingMsg->msgHdr.msgh_remote_port = MACH_PORT_NULL; +} + void IOServiceMessageUserNotification::free( void ) { PingMsg * _pingMsg; @@ -1922,38 +1934,36 @@ kern_return_t is_io_object_get_superclass( io_name_t obj_name, io_name_t class_name) { - const OSMetaClass* my_obj = NULL; - const OSMetaClass* superclass = NULL; - const OSSymbol *my_name = NULL; - const char *my_cstr = NULL; + IOReturn ret; + const OSMetaClass * meta; + const OSMetaClass * super; + const OSSymbol * name; + const char * cstr; - if (!obj_name || !class_name) - return (kIOReturnBadArgument); + if (!obj_name || !class_name) return (kIOReturnBadArgument); + if (master_port != master_device_port) return( kIOReturnNotPrivileged); - if( master_port != master_device_port) - return( kIOReturnNotPrivileged); + ret = kIOReturnNotFound; + meta = 0; + do + { + name = OSSymbol::withCString(obj_name); + if (!name) break; + meta = OSMetaClass::copyMetaClassWithName(name); + if (!meta) break; + super = meta->getSuperClass(); + if (!super) break; + cstr = super->getClassName(); + if (!cstr) break; + strlcpy(class_name, cstr, sizeof(io_name_t)); + ret = kIOReturnSuccess; + } + while (false); - my_name = OSSymbol::withCString(obj_name); - - if (my_name) { - my_obj = OSMetaClass::getMetaClassWithName(my_name); - my_name->release(); - } - if (my_obj) { - superclass = my_obj->getSuperClass(); - } - - if (!superclass) { - return( kIOReturnNotFound ); - } + OSSafeReleaseNULL(name); + if (meta) meta->releaseMetaClass(); - my_cstr = superclass->getClassName(); - - if (my_cstr) { - strlcpy(class_name, my_cstr, sizeof(io_name_t)); - return( kIOReturnSuccess ); - } - return (kIOReturnNotFound); + return (ret); } /* Routine io_object_get_bundle_identifier */ @@ -1962,38 +1972,36 @@ kern_return_t is_io_object_get_bundle_identifier( io_name_t obj_name, io_name_t bundle_name) { - const OSMetaClass* my_obj = NULL; - const OSSymbol *my_name = NULL; - const OSSymbol *identifier = NULL; - const char *my_cstr = NULL; + IOReturn ret; + const OSMetaClass * meta; + const OSSymbol * name; + const OSSymbol * identifier; + const char * cstr; - if (!obj_name || !bundle_name) - return (kIOReturnBadArgument); + if (!obj_name || !bundle_name) return (kIOReturnBadArgument); + if (master_port != master_device_port) return( kIOReturnNotPrivileged); - if( master_port != master_device_port) - return( kIOReturnNotPrivileged); - - my_name = OSSymbol::withCString(obj_name); - - if (my_name) { - my_obj = OSMetaClass::getMetaClassWithName(my_name); - my_name->release(); - } + ret = kIOReturnNotFound; + meta = 0; + do + { + name = OSSymbol::withCString(obj_name); + if (!name) break; + meta = OSMetaClass::copyMetaClassWithName(name); + if (!meta) break; + identifier = meta->getKmodName(); + if (!identifier) break; + cstr = identifier->getCStringNoCopy(); + if (!cstr) break; + strlcpy(bundle_name, identifier->getCStringNoCopy(), sizeof(io_name_t)); + ret = kIOReturnSuccess; + } + while (false); - if (my_obj) { - identifier = my_obj->getKmodName(); - } - if (!identifier) { - return( kIOReturnNotFound ); - } - - my_cstr = identifier->getCStringNoCopy(); - if (my_cstr) { - strlcpy(bundle_name, identifier->getCStringNoCopy(), sizeof(io_name_t)); - return( kIOReturnSuccess ); - } + OSSafeReleaseNULL(name); + if (meta) meta->releaseMetaClass(); - return (kIOReturnBadArgument); + return (ret); } /* Routine io_object_conforms_to */ @@ -2328,7 +2336,8 @@ static kern_return_t internal_io_service_add_notification( else if( (sym == gIOMatchedNotification) || (sym == gIOFirstMatchNotification)) userMsgType = kIOServiceMatchedNotificationType; - else if( sym == gIOTerminatedNotification) + else if ((sym == gIOTerminatedNotification) + || (sym == gIOWillTerminateNotification)) userMsgType = kIOServiceTerminatedNotificationType; else userMsgType = kLastIOKitNotificationType; @@ -2337,7 +2346,6 @@ static kern_return_t internal_io_service_add_notification( if( userNotify && !userNotify->init( port, userMsgType, reference, referenceSize, client64)) { - iokit_release_port_send(port); userNotify->release(); userNotify = 0; } @@ -2355,6 +2363,13 @@ static kern_return_t internal_io_service_add_notification( } while( false ); + if ((kIOReturnSuccess != err) && userNotify) + { + userNotify->invalidatePort(); + userNotify->release(); + userNotify = 0; + } + if( sym) sym->release(); if( dict) @@ -2530,7 +2545,6 @@ static kern_return_t internal_io_service_add_interest_notification( reference, referenceSize, kIOUserNotifyMaxMessageSize, client64 )) { - iokit_release_port_send(port); userNotify->release(); userNotify = 0; } @@ -2550,6 +2564,13 @@ static kern_return_t internal_io_service_add_interest_notification( } while( false ); + if ((kIOReturnSuccess != err) && userNotify) + { + userNotify->invalidatePort(); + userNotify->release(); + userNotify = 0; + } + return( err ); } @@ -3974,6 +3995,8 @@ kern_return_t is_io_connect_async_method args.asyncReference = reference; args.asyncReferenceCount = referenceCnt; + args.structureVariableOutputData = 0; + args.scalarInput = scalar_input; args.scalarInputCount = scalar_inputCnt; args.structureInput = inband_input; @@ -5120,7 +5143,7 @@ kern_return_t is_io_catalog_get_data( vm_size_t size; size = s->getLength(); - kr = vm_allocate(kernel_map, &data, size, VM_FLAGS_ANYWHERE); + kr = vm_allocate_kernel(kernel_map, &data, size, VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_IOKIT); if ( kr == kIOReturnSuccess ) { bcopy(s->text(), (void *)data, size); kr = vm_map_copyin(kernel_map, (vm_map_address_t)data, diff --git a/iokit/Kernel/IOWorkLoop.cpp b/iokit/Kernel/IOWorkLoop.cpp index e3896d38b..a5eb85181 100644 --- a/iokit/Kernel/IOWorkLoop.cpp +++ b/iokit/Kernel/IOWorkLoop.cpp @@ -224,7 +224,7 @@ void IOWorkLoop::free() is = IOSimpleLockLockDisableInterrupt(workToDoLock); SETP(&fFlags, kLoopTerminate); - thread_wakeup_one((void *) &workToDo); + thread_wakeup_thread((void *) &workToDo, workThread); IOSimpleLockUnlockEnableInterrupt(workToDoLock, is); openGate(); @@ -350,7 +350,7 @@ void IOWorkLoop::disableAllInterrupts() const goto abort; if (traceWL) - IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this); + IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_WORK), VM_KERNEL_ADDRHIDE(this)); bool more; do { @@ -363,12 +363,12 @@ void IOWorkLoop::disableAllInterrupts() const for (IOEventSource *evnt = eventChain; evnt; evnt = evnt->getNext()) { if (traceES) - IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt); + IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_CLIENT), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(evnt)); more |= evnt->checkForWork(); if (traceES) - IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt); + IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_CLIENT), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(evnt)); if (ISSETP(&fFlags, kLoopTerminate)) goto abort; @@ -382,7 +382,7 @@ void IOWorkLoop::disableAllInterrupts() const res = true; if (traceWL) - IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this); + IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_WORK), VM_KERNEL_ADDRHIDE(this)); abort: openGate(); @@ -445,7 +445,7 @@ void IOWorkLoop::signalWorkAvailable() if (workToDoLock) { IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock); workToDo = true; - thread_wakeup_one((void *) &workToDo); + thread_wakeup_thread((void *) &workToDo, workThread); IOSimpleLockUnlockEnableInterrupt(workToDoLock, is); } } @@ -633,7 +633,10 @@ IOWorkLoop::eventSourcePerformsWork(IOEventSource *inEventSource) * checkForWork() here. We're just testing to see if it's the same or not. * */ - if (controlG) { + + if (IOEventSource::kPassive & inEventSource->flags) result = false; + else if (IOEventSource::kActive & inEventSource->flags) result = true; + else if (controlG) { void * ptr1; void * ptr2; diff --git a/iokit/Kernel/RootDomainUserClient.cpp b/iokit/Kernel/RootDomainUserClient.cpp index ecd0cc1f3..7a909d998 100644 --- a/iokit/Kernel/RootDomainUserClient.cpp +++ b/iokit/Kernel/RootDomainUserClient.cpp @@ -198,7 +198,7 @@ IOReturn RootDomainUserClient::secureSetUserAssertionLevels( } IOReturn RootDomainUserClient::secureGetSystemSleepType( - uint32_t *outSleepType) + uint32_t *outSleepType, uint32_t *sleepTimer) { int admin_priv = 0; IOReturn ret; @@ -207,7 +207,7 @@ IOReturn RootDomainUserClient::secureGetSystemSleepType( admin_priv = (kIOReturnSuccess == ret); if (admin_priv && fOwner) { - ret = fOwner->getSystemSleepType(outSleepType); + ret = fOwner->getSystemSleepType(outSleepType, sleepTimer); } else { ret = kIOReturnNotPrivileged; } @@ -333,10 +333,11 @@ IOReturn RootDomainUserClient::externalMethod( break; case kPMGetSystemSleepType: - if (1 == arguments->scalarOutputCount) + if (2 == arguments->scalarOutputCount) { ret = this->secureGetSystemSleepType( - (uint32_t *) &arguments->scalarOutput[0]); + (uint32_t *) &arguments->scalarOutput[0], + (uint32_t *) &arguments->scalarOutput[1]); } break; diff --git a/iokit/Kernel/RootDomainUserClient.h b/iokit/Kernel/RootDomainUserClient.h index b37f71029..f4b815bce 100644 --- a/iokit/Kernel/RootDomainUserClient.h +++ b/iokit/Kernel/RootDomainUserClient.h @@ -67,7 +67,7 @@ private: IOReturn secureSetUserAssertionLevels(uint32_t assertionBitfield); - IOReturn secureGetSystemSleepType( uint32_t *sleepType ); + IOReturn secureGetSystemSleepType( uint32_t *sleepType, uint32_t *sleepTimer); public: diff --git a/iokit/Kernel/i386/IOKeyStoreHelper.cpp b/iokit/Kernel/i386/IOKeyStoreHelper.cpp index a1d41b8d0..17ebea802 100644 --- a/iokit/Kernel/i386/IOKeyStoreHelper.cpp +++ b/iokit/Kernel/i386/IOKeyStoreHelper.cpp @@ -47,6 +47,13 @@ IOGetBootKeyStoreData(void); void IOSetKeyStoreData(IOMemoryDescriptor * data); +// APFS +static volatile UInt32 apfsKeyFetched = 0; +static IOMemoryDescriptor* apfsKeyData = NULL; + +IOMemoryDescriptor* IOGetAPFSKeyStoreData(); +void IOSetAPFSKeyStoreData(IOMemoryDescriptor* data); + __END_DECLS #if 1 @@ -102,3 +109,52 @@ IOGetBootKeyStoreData(void) return memoryDescriptor; } + +// APFS volume key fetcher + +// Store in-memory key (could be used by IOHibernateDone) +void +IOSetAPFSKeyStoreData(IOMemoryDescriptor* data) +{ + // Do not allow re-fetching of the boot_args key by passing NULL here. + if (data != NULL) + { + apfsKeyData = data; + apfsKeyFetched = 0; + } +} + +// Retrieve any key we may have (stored in boot_args or by Hibernate) +IOMemoryDescriptor* +IOGetAPFSKeyStoreData() +{ + // Check if someone got the key before us + if (!OSCompareAndSwap(0, 1, &apfsKeyFetched)) + return NULL; + + // Do we have in-memory key? + if (apfsKeyData) + { + IOMemoryDescriptor* data = apfsKeyData; + apfsKeyData = NULL; + return data; + } + + // Looks like there was no in-memory key and it's the first call - try boot_args + boot_args* args = (boot_args*)PE_state.bootArgs; + + DEBG("%s: data at address %u size %u\n", __func__, args->apfsDataStart, args->apfsDataSize); + if (args->apfsDataStart == 0) + return NULL; + + // We have the key in the boot_args, create IOMemoryDescriptor for the blob + IOAddressRange ranges; + ranges.address = args->apfsDataStart; + ranges.length = args->apfsDataSize; + + const IOOptionBits options = kIODirectionInOut | kIOMemoryTypePhysical64 | kIOMemoryMapperNone; + + IOMemoryDescriptor* memoryDescriptor = IOMemoryDescriptor::withOptions(&ranges, 1, 0, NULL, options); + DEBG("%s: memory descriptor %p\n", __func__, memoryDescriptor); + return memoryDescriptor; +} diff --git a/iokit/Tests/TestIOMemoryDescriptor.cpp b/iokit/Tests/TestIOMemoryDescriptor.cpp index 59ce35546..11780e5f2 100644 --- a/iokit/Tests/TestIOMemoryDescriptor.cpp +++ b/iokit/Tests/TestIOMemoryDescriptor.cpp @@ -110,6 +110,204 @@ static int IOMultMemoryDescriptorTest(int newValue) return (0); } + + +// <rdar://problem/30102458> +static int +IODMACommandForceDoubleBufferTest(int newValue) +{ + IOReturn ret; + IOBufferMemoryDescriptor * bmd; + IODMACommand * dma; + uint32_t dir, data; + IODMACommand::SegmentOptions segOptions = + { + .fStructSize = sizeof(segOptions), + .fNumAddressBits = 64, + .fMaxSegmentSize = 0x2000, + .fMaxTransferSize = 128*1024, + .fAlignment = 1, + .fAlignmentLength = 1, + .fAlignmentInternalSegments = 1 + }; + IODMACommand::Segment64 segments[1]; + UInt32 numSegments; + UInt64 dmaOffset; + + + for (dir = kIODirectionIn; ; dir++) + { + bmd = IOBufferMemoryDescriptor::inTaskWithOptions(kernel_task, + dir | kIOMemoryPageable, ptoa(8)); + assert(bmd); + + ((uint32_t*) bmd->getBytesNoCopy())[0] = 0x53535300 | dir; + + ret = bmd->prepare((IODirection) dir); + assert(kIOReturnSuccess == ret); + + dma = IODMACommand::withSpecification(kIODMACommandOutputHost64, &segOptions, + kIODMAMapOptionMapped, + NULL, NULL); + assert(dma); + ret = dma->setMemoryDescriptor(bmd, true); + assert(kIOReturnSuccess == ret); + + ret = dma->synchronize(IODMACommand::kForceDoubleBuffer | kIODirectionOut); + assert(kIOReturnSuccess == ret); + + dmaOffset = 0; + numSegments = 1; + ret = dma->gen64IOVMSegments(&dmaOffset, &segments[0], &numSegments); + assert(kIOReturnSuccess == ret); + assert(1 == numSegments); + + if (kIODirectionOut & dir) + { + data = ((uint32_t*) bmd->getBytesNoCopy())[0]; + assertf((0x53535300 | dir) == data, "mismatch 0x%x", data); + } + if (kIODirectionIn & dir) + { + IOMappedWrite32(segments[0].fIOVMAddr, 0x11223300 | dir); + } + + ret = dma->clearMemoryDescriptor(true); + assert(kIOReturnSuccess == ret); + dma->release(); + + bmd->complete((IODirection) dir); + + if (kIODirectionIn & dir) + { + data = ((uint32_t*) bmd->getBytesNoCopy())[0]; + assertf((0x11223300 | dir) == data, "mismatch 0x%x", data); + } + + bmd->release(); + + if (dir == kIODirectionInOut) break; + } + + return (0); +} + +// <rdar://problem/30102458> +static int +IOMemoryRemoteTest(int newValue) +{ + IOReturn ret; + IOMemoryDescriptor * md; + IOByteCount offset, length; + addr64_t addr; + uint32_t idx; + + IODMACommand * dma; + IODMACommand::SegmentOptions segOptions = + { + .fStructSize = sizeof(segOptions), + .fNumAddressBits = 64, + .fMaxSegmentSize = 0x2000, + .fMaxTransferSize = 128*1024, + .fAlignment = 1, + .fAlignmentLength = 1, + .fAlignmentInternalSegments = 1 + }; + IODMACommand::Segment64 segments[1]; + UInt32 numSegments; + UInt64 dmaOffset; + + IOAddressRange ranges[2] = { + { 0x1234567890123456ULL, 0x1000 }, { 0x5432109876543210, 0x2000 }, + }; + + md = IOMemoryDescriptor::withAddressRanges(&ranges[0], 2, kIODirectionOutIn|kIOMemoryRemote, TASK_NULL); + assert(md); + +// md->map(); +// md->readBytes(0, &idx, sizeof(idx)); + + ret = md->prepare(kIODirectionOutIn); + assert(kIOReturnSuccess == ret); + + printf("remote md flags 0x%qx, r %d\n", + md->getFlags(), (0 != (kIOMemoryRemote & md->getFlags()))); + + for (offset = 0, idx = 0; true; offset += length, idx++) + { + addr = md->getPhysicalSegment(offset, &length, 0); + if (!length) break; + assert(idx < 2); + assert(addr == ranges[idx].address); + assert(length == ranges[idx].length); + } + assert(offset == md->getLength()); + + dma = IODMACommand::withSpecification(kIODMACommandOutputHost64, &segOptions, + kIODMAMapOptionUnmapped | kIODMAMapOptionIterateOnly, + NULL, NULL); + assert(dma); + ret = dma->setMemoryDescriptor(md, true); + assert(kIOReturnSuccess == ret); + + for (dmaOffset = 0, idx = 0; dmaOffset < md->getLength(); idx++) + { + numSegments = 1; + ret = dma->gen64IOVMSegments(&dmaOffset, &segments[0], &numSegments); + assert(kIOReturnSuccess == ret); + assert(1 == numSegments); + assert(idx < 2); + assert(segments[0].fIOVMAddr == ranges[idx].address); + assert(segments[0].fLength == ranges[idx].length); + } + assert(dmaOffset == md->getLength()); + + ret = dma->clearMemoryDescriptor(true); + assert(kIOReturnSuccess == ret); + dma->release(); + md->complete(kIODirectionOutIn); + md->release(); + + return (0); +} + +static IOReturn +IOMemoryPrefaultTest(uint32_t options) +{ + IOBufferMemoryDescriptor * bmd; + IOMemoryMap * map; + IOReturn kr; + uint32_t data; + uint32_t * p; + IOSimpleLock * lock; + + lock = IOSimpleLockAlloc(); + assert(lock); + + bmd = IOBufferMemoryDescriptor::inTaskWithOptions(current_task(), + kIODirectionOutIn | kIOMemoryPageable, ptoa(8)); + assert(bmd); + kr = bmd->prepare(); + assert(KERN_SUCCESS == kr); + + map = bmd->map(kIOMapPrefault); + assert(map); + + p = (typeof(p)) map->getVirtualAddress(); + IOSimpleLockLock(lock); + data = p[0]; + IOSimpleLockUnlock(lock); + + IOLog("IOMemoryPrefaultTest %d\n", data); + + map->release(); + bmd->release(); + IOSimpleLockFree(lock); + + return (kIOReturnSuccess); +} + + // <rdar://problem/26375234> static IOReturn ZeroLengthTest(int newValue) @@ -159,7 +357,6 @@ IODirectionPrepareNoZeroFillTest(int newValue) return (0); } - // <rdar://problem/28190483> static IOReturn IOMemoryMapTest(uint32_t options) @@ -236,6 +433,31 @@ IOMemoryMapCopyOnWriteTest(int newValue) return (0); } +static int +AllocationNameTest(int newValue) +{ + IOMemoryDescriptor * bmd; + kern_allocation_name_t name, prior; + + name = kern_allocation_name_allocate("com.apple.iokit.test", 0); + assert(name); + + prior = thread_set_allocation_name(name); + + bmd = IOBufferMemoryDescriptor::inTaskWithOptions(TASK_NULL, + kIODirectionOutIn | kIOMemoryPageable | kIOMemoryKernelUserShared, + ptoa(13)); + assert(bmd); + bmd->prepare(); + + thread_set_allocation_name(prior); + kern_allocation_name_release(name); + + if (newValue != 7) bmd->release(); + + return (0); +} + int IOMemoryDescriptorTest(int newValue) { int result; @@ -466,6 +688,12 @@ int IOMemoryDescriptorTest(int newValue) } #endif + result = IODMACommandForceDoubleBufferTest(newValue); + if (result) return (result); + + result = AllocationNameTest(newValue); + if (result) return (result); + result = IOMemoryMapCopyOnWriteTest(newValue); if (result) return (result); @@ -481,6 +709,12 @@ int IOMemoryDescriptorTest(int newValue) result = BadFixedAllocTest(newValue); if (result) return (result); + result = IOMemoryRemoteTest(newValue); + if (result) return (result); + + result = IOMemoryPrefaultTest(newValue); + if (result) return (result); + IOGeneralMemoryDescriptor * md; vm_offset_t data[2]; vm_size_t bsize = 16*1024*1024; @@ -488,7 +722,7 @@ int IOMemoryDescriptorTest(int newValue) kern_return_t kr; data[0] = data[1] = 0; - kr = vm_allocate(kernel_map, &data[0], bsize, VM_FLAGS_ANYWHERE); + kr = vm_allocate_kernel(kernel_map, &data[0], bsize, VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_IOKIT); assert(KERN_SUCCESS == kr); vm_inherit(kernel_map, data[0] + ptoa(1), ptoa(1), VM_INHERIT_NONE); diff --git a/iokit/Tests/Tests.cpp b/iokit/Tests/Tests.cpp index bc2d05b69..6336b67ac 100644 --- a/iokit/Tests/Tests.cpp +++ b/iokit/Tests/Tests.cpp @@ -177,6 +177,48 @@ #include <libkern/c++/OSData.h> #include "Tests.h" +#include <IOKit/IOTimerEventSource.h> +#include <IOKit/IOWorkLoop.h> + +#if DEVELOPMENT || DEBUG + +static uint64_t gIOWorkLoopTestDeadline; + +static void +TESAction(OSObject * owner, IOTimerEventSource * tes) +{ + if (mach_absolute_time() < gIOWorkLoopTestDeadline) tes->setTimeout(1, kMicrosecondScale); +} + +static int +IOWorkLoopTest(int newValue) +{ + IOReturn err; + uint32_t idx; + IOWorkLoop * wl; + IOTimerEventSource * tes; + + wl = IOWorkLoop::workLoop(); + assert(wl); + tes = IOTimerEventSource::timerEventSource(kIOTimerEventSourceOptionsPriorityWorkLoop, wl, &TESAction); + assert(tes); + err = wl->addEventSource(tes); + assert(kIOReturnSuccess == err); + clock_interval_to_deadline(2000, kMillisecondScale, &gIOWorkLoopTestDeadline); + for (idx = 0; mach_absolute_time() < gIOWorkLoopTestDeadline; idx++) + { + tes->setTimeout(idx & 1023, kNanosecondScale); + } + tes->cancelTimeout(); + wl->removeEventSource(tes); + tes->release(); + wl->release(); + + return (0); +} + +#endif /* DEVELOPMENT || DEBUG */ + static int sysctl_iokittest(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req) { @@ -194,7 +236,13 @@ sysctl_iokittest(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused data->release(); } - if (changed && newValue) error = IOMemoryDescriptorTest(newValue); + if (changed && newValue) + { + error = IOWorkLoopTest(newValue); + assert(KERN_SUCCESS == error); + error = IOMemoryDescriptorTest(newValue); + assert(KERN_SUCCESS == error); + } #endif /* DEVELOPMENT || DEBUG */ return (error); diff --git a/iokit/bsddev/DINetBootHook.cpp b/iokit/bsddev/DINetBootHook.cpp index 62865b0a1..6ca295a81 100644 --- a/iokit/bsddev/DINetBootHook.cpp +++ b/iokit/bsddev/DINetBootHook.cpp @@ -140,7 +140,7 @@ extern "C" { dev_p <- device number generated from major/minor numbers Comments: */ -int di_root_image(const char *path, char devname[], dev_t *dev_p) +int di_root_image(const char *path, char *devname, size_t devsz, dev_t *dev_p) { IOReturn res = 0; IOService * controller = 0; @@ -196,8 +196,7 @@ int di_root_image(const char *path, char devname[], dev_t *dev_p) myDevName = OSDynamicCast(OSString, controller->getProperty(kDIRootImageDevNameKey)); if (myDevName) { - /* rootdevice is 16 chars in bsd_init.c */ - strlcpy(devname, myDevName->getCStringNoCopy(), 16); + strlcpy(devname, myDevName->getCStringNoCopy(), devsz); } else { IOLog("could not get %s\n", kDIRootImageDevNameKey); res = kIOReturnError; @@ -216,6 +215,66 @@ NoIOHDIXController: return res; } +int +di_root_ramfile_buf(void *buf, size_t bufsz, char *devname, size_t devsz, dev_t *dev_p) +{ + IOReturn res = 0; + IOService *controller = 0; + OSNumber *myResult = 0; + OSString *myDevName = 0; + OSNumber *myDevT = 0; + IOMemoryDescriptor *mem = 0; + + mem = IOMemoryDescriptor::withAddress(buf, bufsz, kIODirectionInOut); + assert(mem); + + controller = di_load_controller(); + if (controller) { + /* attach the image */ + controller->setProperty(kDIRootRamFileKey, mem); + controller->release(); + } else { + res = kIOReturnNotFound; + goto out; + } + + myResult = OSDynamicCast(OSNumber, controller->getProperty(kDIRootImageResultKey)); + res = kIOReturnError; + if (myResult) { + res = myResult->unsigned32BitValue(); + } + + if (res) { + IOLog("%s is 0x%08X/%d\n", kDIRootImageResultKey, res, res); + goto out; + } + + myDevT = OSDynamicCast(OSNumber, controller->getProperty(kDIRootImageDevTKey)); + if (myDevT) + *dev_p = myDevT->unsigned32BitValue(); + else { + IOLog("could not get %s\n", kDIRootImageDevTKey); + res = kIOReturnError; + goto out; + } + + myDevName = OSDynamicCast(OSString, controller->getProperty(kDIRootImageDevNameKey)); + if (myDevName) { + strlcpy(devname, myDevName->getCStringNoCopy(), devsz); + } else { + IOLog("could not get %s\n", kDIRootImageDevNameKey); + res = kIOReturnError; + goto out; + } + +out: + if (res) { + OSSafeReleaseNULL(mem); + } + + return res; +} + void di_root_ramfile( IORegistryEntry * entry ) { OSData * data; diff --git a/iokit/bsddev/DINetBootHook.h b/iokit/bsddev/DINetBootHook.h index 2f44361b0..172679524 100644 --- a/iokit/bsddev/DINetBootHook.h +++ b/iokit/bsddev/DINetBootHook.h @@ -94,8 +94,9 @@ extern "C" { dev_p <- combination of major/minor node Comments: */ -int di_root_image(const char *path, char devname[], dev_t *dev_p); +int di_root_image(const char *path, char *devname, size_t devsz, dev_t *dev_p); void di_root_ramfile( IORegistryEntry * entry ); +int di_root_ramfile_buf(void *buf, size_t bufsz, char *devname, size_t devsz, dev_t *dev_p); #ifdef __cplusplus }; diff --git a/iokit/bsddev/IOKitBSDInit.cpp b/iokit/bsddev/IOKitBSDInit.cpp index a35aa5495..f68eebd1e 100644 --- a/iokit/bsddev/IOKitBSDInit.cpp +++ b/iokit/bsddev/IOKitBSDInit.cpp @@ -58,7 +58,13 @@ extern int mdevgetrange(int devid, uint64_t *base, uint64_t *size); extern void di_root_ramfile(IORegistryEntry * entry); -#if DEVELOPMENT +#if CONFIG_EMBEDDED +#define IOPOLLED_COREFILE (CONFIG_KDP_INTERACTIVE_DEBUGGING) +#define kIOCoreDumpPath "/private/var/vm/kernelcore" +#define kIOCoreDumpSize 350ULL*1024ULL*1024ULL +// leave free space on volume: +#define kIOCoreDumpFreeSize 350ULL*1024ULL*1024ULL +#elif DEVELOPMENT #define IOPOLLED_COREFILE 1 // no sizing #define kIOCoreDumpSize 0ULL @@ -657,6 +663,37 @@ bool IORamDiskBSDRoot(void) void IOSecureBSDRoot(const char * rootName) { +#if CONFIG_EMBEDDED + int tmpInt; + IOReturn result; + IOPlatformExpert *pe; + OSDictionary *matching; + const OSSymbol *functionName = OSSymbol::withCStringNoCopy("SecureRootName"); + + matching = IOService::serviceMatching("IOPlatformExpert"); + assert(matching); + pe = (IOPlatformExpert *) IOService::waitForMatchingService(matching, 30ULL * kSecondScale); + matching->release(); + assert(pe); + // Returns kIOReturnNotPrivileged is the root device is not secure. + // Returns kIOReturnUnsupported if "SecureRootName" is not implemented. + result = pe->callPlatformFunction(functionName, false, (void *)rootName, (void *)0, (void *)0, (void *)0); + functionName->release(); + OSSafeReleaseNULL(pe); + + if (result == kIOReturnNotPrivileged) { + mdevremoveall(); + } else if (result == kIOReturnSuccess) { + // If we are booting with a secure root, and we have the right + // boot-arg, we will want to panic on exception triage. This + // behavior is intended as a debug aid (we can look at why an + // exception occured in the kernel debugger). + if (PE_parse_boot_argn("-panic_on_exception_triage", &tmpInt, sizeof(tmpInt))) { + panic_on_exception_triage = 1; + } + } + +#endif // CONFIG_EMBEDDED } void * @@ -848,6 +885,43 @@ IOBSDMountChange(struct mount * mp, uint32_t op) } while (false); +#if CONFIG_EMBEDDED + uint64_t flags; + char path[128]; + int pathLen; + vnode_t vn; + int result; + + switch (op) + { + case kIOMountChangeMount: + case kIOMountChangeDidResize: + + if (gIOPolledCoreFileVars) break; + flags = vfs_flags(mp); + if (MNT_RDONLY & flags) break; + if (!(MNT_LOCAL & flags)) break; + + vn = vfs_vnodecovered(mp); + if (!vn) break; + pathLen = sizeof(path); + result = vn_getpath(vn, &path[0], &pathLen); + vnode_put(vn); + if (0 != result) break; + if (!pathLen) break; + if (0 != bcmp(path, kIOCoreDumpPath, pathLen - 1)) break; + IOOpenPolledCoreFile(kIOCoreDumpPath); + break; + + case kIOMountChangeUnmount: + case kIOMountChangeWillResize: + if (gIOPolledCoreFileVars && (mp == kern_file_mount(gIOPolledCoreFileVars->fileRef))) + { + IOClosePolledCoreFile(); + } + break; + } +#endif /* CONFIG_EMBEDDED */ #endif /* IOPOLLED_COREFILE */ } diff --git a/iokit/conf/Makefile.arm b/iokit/conf/Makefile.arm new file mode 100644 index 000000000..184148ea4 --- /dev/null +++ b/iokit/conf/Makefile.arm @@ -0,0 +1,18 @@ +###################################################################### +#BEGIN Machine dependent Makefile fragment for arm +###################################################################### + +IODMACommand.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +IODataQueue.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +IOMemoryDescriptor.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +IONVRAM.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +IOPMrootDomain.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +IOSharedDataQueue.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +IOUserClient.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align + +# Files that must go in the __HIB segment: +HIB_FILES= + +###################################################################### +#END Machine dependent Makefile fragment for arm +###################################################################### diff --git a/iokit/conf/Makefile.arm64 b/iokit/conf/Makefile.arm64 new file mode 100644 index 000000000..184148ea4 --- /dev/null +++ b/iokit/conf/Makefile.arm64 @@ -0,0 +1,18 @@ +###################################################################### +#BEGIN Machine dependent Makefile fragment for arm +###################################################################### + +IODMACommand.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +IODataQueue.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +IOMemoryDescriptor.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +IONVRAM.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +IOPMrootDomain.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +IOSharedDataQueue.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +IOUserClient.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align + +# Files that must go in the __HIB segment: +HIB_FILES= + +###################################################################### +#END Machine dependent Makefile fragment for arm +###################################################################### diff --git a/iokit/conf/Makefile.template b/iokit/conf/Makefile.template index 4777a86b0..c29d59de6 100644 --- a/iokit/conf/Makefile.template +++ b/iokit/conf/Makefile.template @@ -18,6 +18,7 @@ include $(MakeInc_def) # CFLAGS+= -include meta_features.h -DDRIVER_PRIVATE \ -DIOKIT_KERNEL_PRIVATE -DIOMATCHDEBUG=1 -DIOALLOCDEBUG=1 +SFLAGS+= -include meta_features.h #-DIOKITDEBUG=-1 CWARNFLAGS = $(CWARNFLAGS_STD) -Wno-unused-parameter diff --git a/iokit/conf/files.arm b/iokit/conf/files.arm new file mode 100644 index 000000000..7269e46f8 --- /dev/null +++ b/iokit/conf/files.arm @@ -0,0 +1,4 @@ +iokit/Families/IONVRAM/IONVRAMController.cpp optional iokitcpp + +# Power Domains +iokit/Kernel/IOPMrootDomain.cpp optional iokitcpp diff --git a/iokit/conf/files.arm64 b/iokit/conf/files.arm64 new file mode 100644 index 000000000..7269e46f8 --- /dev/null +++ b/iokit/conf/files.arm64 @@ -0,0 +1,4 @@ +iokit/Families/IONVRAM/IONVRAMController.cpp optional iokitcpp + +# Power Domains +iokit/Kernel/IOPMrootDomain.cpp optional iokitcpp diff --git a/libkdd/kcdata.h b/libkdd/kcdata.h deleted file mode 120000 index f5573542b..000000000 --- a/libkdd/kcdata.h +++ /dev/null @@ -1 +0,0 @@ -./osfmk/kern/kcdata.h \ No newline at end of file diff --git a/libkdd/kcdata.h b/libkdd/kcdata.h new file mode 100644 index 000000000..741c7f864 --- /dev/null +++ b/libkdd/kcdata.h @@ -0,0 +1,1139 @@ +/* + * Copyright (c) 2015 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + + +/* + * + * THE KCDATA MANIFESTO + * + * Kcdata is a self-describing data serialization format. It is meant to get + * nested data structures out of xnu with minimum fuss, but also for that data + * to be easy to parse. It is also meant to allow us to add new fields and + * evolve the data format without breaking old parsers. + * + * Kcdata is a permanent data format suitable for long-term storage including + * in files. It is very important that we continue to be able to parse old + * versions of kcdata-based formats. To this end, there are several + * invariants you MUST MAINTAIN if you alter this file. + * + * * None of the magic numbers should ever be a byteswap of themselves or + * of any of the other magic numbers. + * + * * Never remove any type. + * + * * All kcdata structs must be packed, and must exclusively use fixed-size + * types. + * + * * Never change the definition of any type, except to add new fields to + * the end. + * + * * If you do add new fields to the end of a type, do not actually change + * the definition of the old structure. Instead, define a new structure + * with the new fields. See thread_snapshot_v3 as an example. This + * provides source compatibility for old readers, and also documents where + * the potential size cutoffs are. + * + * * If you change libkdd, or kcdata.py run the unit tests under libkdd. + * + * * If you add a type or extend an existing one, add a sample test to + * libkdd/tests so future changes to libkdd will always parse your struct + * correctly. + * + * For example to add a field to this: + * + * struct foobar { + * uint32_t baz; + * uint32_t quux; + * } __attribute__ ((packed)); + * + * Make it look like this: + * + * struct foobar { + * uint32_t baz; + * uint32_t quux; + * ///////// end version 1 of foobar. sizeof(struct foobar) was 8 //////// + * uint32_t frozzle; + * } __attribute__ ((packed)); + * + * If you are parsing kcdata formats, you MUST + * + * * Check the length field of each struct, including array elements. If the + * struct is longer than you expect, you must ignore the extra data. + * + * * Ignore any data types you do not understand. + * + * Additionally, we want to be as forward compatible as we can. Meaning old + * tools should still be able to use new data whenever possible. To this end, + * you should: + * + * * Try not to add new versions of types that supplant old ones. Instead + * extend the length of existing types or add supplemental types. + * + * * Try not to remove information from existing kcdata formats, unless + * removal was explicitly asked for. For example it is fine to add a + * stackshot flag to remove unwanted information, but you should not + * remove it from the default stackshot if the new flag is absent. + * + * * (TBD) If you do break old readers by removing information or + * supplanting old structs, then increase the major version number. + * + * + * + * The following is a description of the kcdata format. + * + * + * The format for data is setup in a generic format as follows + * + * Layout of data structure: + * + * | 8 - bytes | + * | type = MAGIC | LENGTH | + * | 0 | + * | type | size | + * | flags | + * | data | + * |___________data____________| + * | type | size | + * | flags | + * |___________data____________| + * | type = END | size=0 | + * | 0 | + * + * + * The type field describes what kind of data is passed. For example type = TASK_CRASHINFO_UUID means the following data is a uuid. + * These types need to be defined in task_corpses.h for easy consumption by userspace inspection tools. + * + * Some range of types is reserved for special types like ints, longs etc. A cool new functionality made possible with this + * extensible data format is that kernel can decide to put more information as required without requiring user space tools to + * re-compile to be compatible. The case of rusage struct versions could be introduced without breaking existing tools. + * + * Feature description: Generic data with description + * ------------------- + * Further more generic data with description is very much possible now. For example + * + * - kcdata_add_uint64_with_description(cdatainfo, 0x700, "NUM MACH PORTS"); + * - and more functions that allow adding description. + * The userspace tools can then look at the description and print the data even if they are not compiled with knowledge of the field apriori. + * + * Example data: + * 0000 57 f1 ad de 00 00 00 00 00 00 00 00 00 00 00 00 W............... + * 0010 01 00 00 00 00 00 00 00 30 00 00 00 00 00 00 00 ........0....... + * 0020 50 49 44 00 00 00 00 00 00 00 00 00 00 00 00 00 PID............. + * 0030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + * 0040 9c 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + * 0050 01 00 00 00 00 00 00 00 30 00 00 00 00 00 00 00 ........0....... + * 0060 50 41 52 45 4e 54 20 50 49 44 00 00 00 00 00 00 PARENT PID...... + * 0070 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + * 0080 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + * 0090 ed 58 91 f1 + * + * Feature description: Container markers for compound data + * ------------------ + * If a given kernel data type is complex and requires adding multiple optional fields inside a container + * object for a consumer to understand arbitrary data, we package it using container markers. + * + * For example, the stackshot code gathers information and describes the state of a given task with respect + * to many subsystems. It includes data such as io stats, vm counters, process names/flags and syscall counts. + * + * kcdata_add_container_marker(kcdata_p, KCDATA_TYPE_CONTAINER_BEGIN, STACKSHOT_KCCONTAINER_TASK, task_uniqueid); + * // add multiple data, or add_<type>_with_description()s here + * + * kcdata_add_container_marker(kcdata_p, KCDATA_TYPE_CONTAINER_END, STACKSHOT_KCCONTAINER_TASK, task_uniqueid); + * + * Feature description: Custom Data formats on demand + * -------------------- + * With the self describing nature of format, the kernel provider can describe a data type (uniquely identified by a number) and use + * it in the buffer for sending data. The consumer can parse the type information and have knowledge of describing incoming data. + * Following is an example of how we can describe a kernel specific struct sample_disk_io_stats in buffer. + * + * struct sample_disk_io_stats { + * uint64_t disk_reads_count; + * uint64_t disk_reads_size; + * uint64_t io_priority_count[4]; + * uint64_t io_priority_size; + * } __attribute__ ((packed)); + * + * + * struct kcdata_subtype_descriptor disk_io_stats_def[] = { + * {KCS_SUBTYPE_FLAGS_NONE, KC_ST_UINT64, 0 * sizeof(uint64_t), sizeof(uint64_t), "disk_reads_count"}, + * {KCS_SUBTYPE_FLAGS_NONE, KC_ST_UINT64, 1 * sizeof(uint64_t), sizeof(uint64_t), "disk_reads_size"}, + * {KCS_SUBTYPE_FLAGS_ARRAY, KC_ST_UINT64, 2 * sizeof(uint64_t), KCS_SUBTYPE_PACK_SIZE(4, sizeof(uint64_t)), "io_priority_count"}, + * {KCS_SUBTYPE_FLAGS_ARRAY, KC_ST_UINT64, (2 + 4) * sizeof(uint64_t), sizeof(uint64_t), "io_priority_size"}, + * }; + * + * Now you can add this custom type definition into the buffer as + * kcdata_add_type_definition(kcdata_p, KCTYPE_SAMPLE_DISK_IO_STATS, "sample_disk_io_stats", + * &disk_io_stats_def[0], sizeof(disk_io_stats_def)/sizeof(struct kcdata_subtype_descriptor)); + * + */ + + +#ifndef _KCDATA_H_ +#define _KCDATA_H_ + +#include <stdint.h> +#include <string.h> +#include <uuid/uuid.h> + +#define KCDATA_DESC_MAXLEN 32 /* including NULL byte at end */ + +#define KCDATA_FLAGS_STRUCT_PADDING_MASK 0xf +#define KCDATA_FLAGS_STRUCT_HAS_PADDING 0x80 + +/* + * kcdata aligns elements to 16 byte boundaries. + */ +#define KCDATA_ALIGNMENT_SIZE 0x10 + +struct kcdata_item { + uint32_t type; + uint32_t size; /* len(data) */ + /* flags. + * + * For structures: + * padding = flags & 0xf + * has_padding = (flags & 0x80) >> 7 + * + * has_padding is needed to disambiguate cases such as + * thread_snapshot_v2 and thread_snapshot_v3. Their + * respective sizes are 0x68 and 0x70, and thread_snapshot_v2 + * was emmitted by old kernels *before* we started recording + * padding. Since legacy thread_snapsht_v2 and modern + * thread_snapshot_v3 will both record 0 for the padding + * flags, we need some other bit which will be nonzero in the + * flags to disambiguate. + * + * This is why we hardcode a special case for + * STACKSHOT_KCTYPE_THREAD_SNAPSHOT into the iterator + * functions below. There is only a finite number of such + * hardcodings which will ever be needed. They can occur + * when: + * + * * We have a legacy structure that predates padding flags + * + * * which we want to extend without changing the kcdata type + * + * * by only so many bytes as would fit in the space that + * was previously unused padding. + * + * For containers: + * container_id = flags + * + * For arrays: + * element_count = flags & UINT32_MAX + * element_type = (flags >> 32) & UINT32_MAX + */ + uint64_t flags; + char data[]; /* must be at the end */ +}; + +typedef struct kcdata_item * kcdata_item_t; + +enum KCDATA_SUBTYPE_TYPES { KC_ST_CHAR = 1, KC_ST_INT8, KC_ST_UINT8, KC_ST_INT16, KC_ST_UINT16, KC_ST_INT32, KC_ST_UINT32, KC_ST_INT64, KC_ST_UINT64 }; +typedef enum KCDATA_SUBTYPE_TYPES kctype_subtype_t; + +/* + * A subtype description structure that defines + * how a compound data is laid out in memory. This + * provides on the fly definition of types and consumption + * by the parser. + */ +struct kcdata_subtype_descriptor { + uint8_t kcs_flags; +#define KCS_SUBTYPE_FLAGS_NONE 0x0 +#define KCS_SUBTYPE_FLAGS_ARRAY 0x1 +/* Force struct type even if only one element. + * + * Normally a kcdata_type_definition is treated as a structure if it has + * more than one subtype descriptor. Otherwise it is treated as a simple + * type. For example libkdd will represent a simple integer 42 as simply + * 42, but it will represent a structure containing an integer 42 as + * {"field_name": 42}.. + * + * If a kcdata_type_definition has only single subtype, then it will be + * treated as a structure iff KCS_SUBTYPE_FLAGS_STRUCT is set. If it has + * multiple subtypes, it will always be treated as a structure. + * + * KCS_SUBTYPE_FLAGS_MERGE has the opposite effect. If this flag is used then + * even if there are multiple elements, they will all be treated as individual + * properties of the parent dictionary. + */ +#define KCS_SUBTYPE_FLAGS_STRUCT 0x2 /* force struct type even if only one element */ +#define KCS_SUBTYPE_FLAGS_MERGE 0x4 /* treat as multiple elements of parents instead of struct */ + uint8_t kcs_elem_type; /* restricted to kctype_subtype_t */ + uint16_t kcs_elem_offset; /* offset in struct where data is found */ + uint32_t kcs_elem_size; /* size of element (or) packed state for array type */ + char kcs_name[KCDATA_DESC_MAXLEN]; /* max 31 bytes for name of field */ +}; + +typedef struct kcdata_subtype_descriptor * kcdata_subtype_descriptor_t; + +/* + * In case of array of basic c types in kctype_subtype_t, + * size is packed in lower 16 bits and + * count is packed in upper 16 bits of kcs_elem_size field. + */ +#define KCS_SUBTYPE_PACK_SIZE(e_count, e_size) (((e_count)&0xffffu) << 16 | ((e_size)&0xffffu)) + +static inline uint32_t +kcs_get_elem_size(kcdata_subtype_descriptor_t d) +{ + if (d->kcs_flags & KCS_SUBTYPE_FLAGS_ARRAY) { + /* size is composed as ((count &0xffff)<<16 | (elem_size & 0xffff)) */ + return (uint32_t)((d->kcs_elem_size & 0xffff) * ((d->kcs_elem_size & 0xffff0000)>>16)); + } + return d->kcs_elem_size; +} + +static inline uint32_t +kcs_get_elem_count(kcdata_subtype_descriptor_t d) +{ + if (d->kcs_flags & KCS_SUBTYPE_FLAGS_ARRAY) + return (d->kcs_elem_size >> 16) & 0xffff; + return 1; +} + +static inline int +kcs_set_elem_size(kcdata_subtype_descriptor_t d, uint32_t size, uint32_t count) +{ + if (count > 1) { + /* means we are setting up an array */ + if (size > 0xffff || count > 0xffff) + return -1; //invalid argument + d->kcs_elem_size = ((count & 0xffff) << 16 | (size & 0xffff)); + } + else + { + d->kcs_elem_size = size; + } + return 0; +} + +struct kcdata_type_definition { + uint32_t kct_type_identifier; + uint32_t kct_num_elements; + char kct_name[KCDATA_DESC_MAXLEN]; + struct kcdata_subtype_descriptor kct_elements[]; +}; + + +/* chunk type definitions. 0 - 0x7ff are reserved and defined here + * NOTE: Please update kcdata/libkdd/kcdtypes.c if you make any changes + * in STACKSHOT_KCTYPE_* types. + */ + +/* + * Types with description value. + * these will have KCDATA_DESC_MAXLEN-1 length string description + * and rest of kcdata_iter_size() - KCDATA_DESC_MAXLEN bytes as data + */ +#define KCDATA_TYPE_INVALID 0x0u +#define KCDATA_TYPE_STRING_DESC 0x1u +#define KCDATA_TYPE_UINT32_DESC 0x2u +#define KCDATA_TYPE_UINT64_DESC 0x3u +#define KCDATA_TYPE_INT32_DESC 0x4u +#define KCDATA_TYPE_INT64_DESC 0x5u +#define KCDATA_TYPE_BINDATA_DESC 0x6u + +/* + * Compound type definitions + */ +#define KCDATA_TYPE_ARRAY 0x11u /* Array of data OBSOLETE DONT USE THIS*/ +#define KCDATA_TYPE_TYPEDEFINTION 0x12u /* Meta type that describes a type on the fly. */ +#define KCDATA_TYPE_CONTAINER_BEGIN \ + 0x13u /* Container type which has corresponding CONTAINER_END header. \ + * KCDATA_TYPE_CONTAINER_BEGIN has type in the data segment. \ + * Both headers have (uint64_t) ID for matching up nested data. \ + */ +#define KCDATA_TYPE_CONTAINER_END 0x14u + +#define KCDATA_TYPE_ARRAY_PAD0 0x20u /* Array of data with 0 byte of padding*/ +#define KCDATA_TYPE_ARRAY_PAD1 0x21u /* Array of data with 1 byte of padding*/ +#define KCDATA_TYPE_ARRAY_PAD2 0x22u /* Array of data with 2 byte of padding*/ +#define KCDATA_TYPE_ARRAY_PAD3 0x23u /* Array of data with 3 byte of padding*/ +#define KCDATA_TYPE_ARRAY_PAD4 0x24u /* Array of data with 4 byte of padding*/ +#define KCDATA_TYPE_ARRAY_PAD5 0x25u /* Array of data with 5 byte of padding*/ +#define KCDATA_TYPE_ARRAY_PAD6 0x26u /* Array of data with 6 byte of padding*/ +#define KCDATA_TYPE_ARRAY_PAD7 0x27u /* Array of data with 7 byte of padding*/ +#define KCDATA_TYPE_ARRAY_PAD8 0x28u /* Array of data with 8 byte of padding*/ +#define KCDATA_TYPE_ARRAY_PAD9 0x29u /* Array of data with 9 byte of padding*/ +#define KCDATA_TYPE_ARRAY_PADa 0x2au /* Array of data with a byte of padding*/ +#define KCDATA_TYPE_ARRAY_PADb 0x2bu /* Array of data with b byte of padding*/ +#define KCDATA_TYPE_ARRAY_PADc 0x2cu /* Array of data with c byte of padding*/ +#define KCDATA_TYPE_ARRAY_PADd 0x2du /* Array of data with d byte of padding*/ +#define KCDATA_TYPE_ARRAY_PADe 0x2eu /* Array of data with e byte of padding*/ +#define KCDATA_TYPE_ARRAY_PADf 0x2fu /* Array of data with f byte of padding*/ + +/* + * Generic data types that are most commonly used + */ +#define KCDATA_TYPE_LIBRARY_LOADINFO 0x30u /* struct dyld_uuid_info_32 */ +#define KCDATA_TYPE_LIBRARY_LOADINFO64 0x31u /* struct dyld_uuid_info_64 */ +#define KCDATA_TYPE_TIMEBASE 0x32u /* struct mach_timebase_info */ +#define KCDATA_TYPE_MACH_ABSOLUTE_TIME 0x33u /* uint64_t */ +#define KCDATA_TYPE_TIMEVAL 0x34u /* struct timeval64 */ +#define KCDATA_TYPE_USECS_SINCE_EPOCH 0x35u /* time in usecs uint64_t */ +#define KCDATA_TYPE_PID 0x36u /* int32_t */ +#define KCDATA_TYPE_PROCNAME 0x37u /* char * */ +#define KCDATA_TYPE_NESTED_KCDATA 0x38u /* nested kcdata buffer */ + +#define KCDATA_TYPE_BUFFER_END 0xF19158EDu + +/* MAGIC numbers defined for each class of chunked data + * + * To future-proof against big-endian arches, make sure none of these magic + * numbers are byteswaps of each other + */ + +#define KCDATA_BUFFER_BEGIN_CRASHINFO 0xDEADF157u /* owner: corpses/task_corpse.h */ + /* type-range: 0x800 - 0x8ff */ +#define KCDATA_BUFFER_BEGIN_STACKSHOT 0x59a25807u /* owner: sys/stackshot.h */ + /* type-range: 0x900 - 0x93f */ +#define KCDATA_BUFFER_BEGIN_DELTA_STACKSHOT 0xDE17A59Au /* owner: sys/stackshot.h */ + /* type-range: 0x940 - 0x9ff */ +#define KCDATA_BUFFER_BEGIN_OS_REASON 0x53A20900u /* owner: sys/reason.h */ + /* type-range: 0x1000-0x103f */ +#define KCDATA_BUFFER_BEGIN_XNUPOST_CONFIG 0x1e21c09fu /* owner: osfmk/tests/kernel_tests.c */ + /* type-range: 0x1040-0x105f */ + +/* next type range number available 0x1060 */ +/**************** definitions for XNUPOST *********************/ +#define XNUPOST_KCTYPE_TESTCONFIG 0x1040 + +/**************** definitions for stackshot *********************/ + +/* This value must always match IO_NUM_PRIORITIES defined in thread_info.h */ +#define STACKSHOT_IO_NUM_PRIORITIES 4 +/* This value must always match MAXTHREADNAMESIZE used in bsd */ +#define STACKSHOT_MAX_THREAD_NAME_SIZE 64 + +/* + * NOTE: Please update kcdata/libkdd/kcdtypes.c if you make any changes + * in STACKSHOT_KCTYPE_* types. + */ +#define STACKSHOT_KCTYPE_IOSTATS 0x901u /* io_stats_snapshot */ +#define STACKSHOT_KCTYPE_GLOBAL_MEM_STATS 0x902u /* struct mem_and_io_snapshot */ +#define STACKSHOT_KCCONTAINER_TASK 0x903u +#define STACKSHOT_KCCONTAINER_THREAD 0x904u +#define STACKSHOT_KCTYPE_TASK_SNAPSHOT 0x905u /* task_snapshot_v2 */ +#define STACKSHOT_KCTYPE_THREAD_SNAPSHOT 0x906u /* thread_snapshot_v2, thread_snapshot_v3 */ +#define STACKSHOT_KCTYPE_DONATING_PIDS 0x907u /* int[] */ +#define STACKSHOT_KCTYPE_SHAREDCACHE_LOADINFO 0x908u /* same as KCDATA_TYPE_LIBRARY_LOADINFO64 */ +#define STACKSHOT_KCTYPE_THREAD_NAME 0x909u /* char[] */ +#define STACKSHOT_KCTYPE_KERN_STACKFRAME 0x90Au /* struct stack_snapshot_frame32 */ +#define STACKSHOT_KCTYPE_KERN_STACKFRAME64 0x90Bu /* struct stack_snapshot_frame64 */ +#define STACKSHOT_KCTYPE_USER_STACKFRAME 0x90Cu /* struct stack_snapshot_frame32 */ +#define STACKSHOT_KCTYPE_USER_STACKFRAME64 0x90Du /* struct stack_snapshot_frame64 */ +#define STACKSHOT_KCTYPE_BOOTARGS 0x90Eu /* boot args string */ +#define STACKSHOT_KCTYPE_OSVERSION 0x90Fu /* os version string */ +#define STACKSHOT_KCTYPE_KERN_PAGE_SIZE 0x910u /* kernel page size in uint32_t */ +#define STACKSHOT_KCTYPE_JETSAM_LEVEL 0x911u /* jetsam level in uint32_t */ +#define STACKSHOT_KCTYPE_DELTA_SINCE_TIMESTAMP 0x912u /* timestamp used for the delta stackshot */ + +#define STACKSHOT_KCTYPE_TASK_DELTA_SNAPSHOT 0x940u /* task_delta_snapshot_v2 */ +#define STACKSHOT_KCTYPE_THREAD_DELTA_SNAPSHOT 0x941u /* thread_delta_snapshot_v2 */ + +#define STACKSHOT_KCTYPE_KERN_STACKLR 0x913u /* uint32_t */ +#define STACKSHOT_KCTYPE_KERN_STACKLR64 0x914u /* uint64_t */ +#define STACKSHOT_KCTYPE_USER_STACKLR 0x915u /* uint32_t */ +#define STACKSHOT_KCTYPE_USER_STACKLR64 0x916u /* uint64_t */ +#define STACKSHOT_KCTYPE_NONRUNNABLE_TIDS 0x917u /* uint64_t */ +#define STACKSHOT_KCTYPE_NONRUNNABLE_TASKS 0x918u /* uint64_t */ +#define STACKSHOT_KCTYPE_CPU_TIMES 0x919u /* struct stackshot_cpu_times */ +#define STACKSHOT_KCTYPE_STACKSHOT_DURATION 0x91au /* struct stackshot_duration */ +#define STACKSHOT_KCTYPE_STACKSHOT_FAULT_STATS 0x91bu /* struct stackshot_fault_stats */ +#define STACKSHOT_KCTYPE_KERNELCACHE_LOADINFO 0x91cu /* kernelcache UUID -- same as KCDATA_TYPE_LIBRARY_LOADINFO64 */ +#define STACKSHOT_KCTYPE_THREAD_WAITINFO 0x91du /* struct stackshot_thread_waitinfo */ +#define STACKSHOT_KCTYPE_THREAD_GROUP_SNAPSHOT 0x91eu /* struct thread_group_snapshot */ +#define STACKSHOT_KCTYPE_THREAD_GROUP 0x91fu /* uint64_t */ +#define STACKSHOT_KCTYPE_JETSAM_COALITION_SNAPSHOT 0x920u /* struct jetsam_coalition_snapshot */ +#define STACKSHOT_KCTYPE_JETSAM_COALITION 0x921u /* uint64_t */ +#define STACKSHOT_KCTYPE_INSTRS_CYCLES 0x923u /* struct instrs_cycles_snapshot */ + +#define STACKSHOT_KCTYPE_THREAD_POLICY_VERSION 0x922u /* THREAD_POLICY_INTERNAL_STRUCT_VERSION in uint32 */ + +struct stack_snapshot_frame32 { + uint32_t lr; + uint32_t sp; +}; + +struct stack_snapshot_frame64 { + uint64_t lr; + uint64_t sp; +}; + +struct dyld_uuid_info_32 { + uint32_t imageLoadAddress; /* base address image is mapped at */ + uuid_t imageUUID; +}; + +struct dyld_uuid_info_64 { + uint64_t imageLoadAddress; /* XXX image slide */ + uuid_t imageUUID; +}; + +struct dyld_uuid_info_64_v2 { + uint64_t imageLoadAddress; /* XXX image slide */ + uuid_t imageUUID; + /* end of version 1 of dyld_uuid_info_64. sizeof v1 was 24 */ + uint64_t imageSlidBaseAddress; /* slid base address of image */ +}; + +struct user32_dyld_uuid_info { + uint32_t imageLoadAddress; /* base address image is mapped into */ + uuid_t imageUUID; /* UUID of image */ +}; + +struct user64_dyld_uuid_info { + uint64_t imageLoadAddress; /* base address image is mapped into */ + uuid_t imageUUID; /* UUID of image */ +}; + +enum task_snapshot_flags { + kTaskRsrcFlagged = 0x4, // In the EXC_RESOURCE danger zone? + kTerminatedSnapshot = 0x8, + kPidSuspended = 0x10, // true for suspended task + kFrozen = 0x20, // true for hibernated task (along with pidsuspended) + kTaskDarwinBG = 0x40, + kTaskExtDarwinBG = 0x80, + kTaskVisVisible = 0x100, + kTaskVisNonvisible = 0x200, + kTaskIsForeground = 0x400, + kTaskIsBoosted = 0x800, + kTaskIsSuppressed = 0x1000, + kTaskIsTimerThrottled = 0x2000, /* deprecated */ + kTaskIsImpDonor = 0x4000, + kTaskIsLiveImpDonor = 0x8000, + kTaskIsDirty = 0x10000, + kTaskWqExceededConstrainedThreadLimit = 0x20000, + kTaskWqExceededTotalThreadLimit = 0x40000, + kTaskWqFlagsAvailable = 0x80000, + kTaskUUIDInfoFaultedIn = 0x100000, /* successfully faulted in some UUID info */ + kTaskUUIDInfoMissing = 0x200000, /* some UUID info was paged out */ + kTaskUUIDInfoTriedFault = 0x400000, /* tried to fault in UUID info */ + kTaskSharedRegionInfoUnavailable = 0x800000, /* shared region info unavailable */ +}; + +enum thread_snapshot_flags { + kHasDispatchSerial = 0x4, + kStacksPCOnly = 0x8, /* Stack traces have no frame pointers. */ + kThreadDarwinBG = 0x10, /* Thread is darwinbg */ + kThreadIOPassive = 0x20, /* Thread uses passive IO */ + kThreadSuspended = 0x40, /* Thread is suspended */ + kThreadTruncatedBT = 0x80, /* Unmapped pages caused truncated backtrace */ + kGlobalForcedIdle = 0x100, /* Thread performs global forced idle */ + kThreadFaultedBT = 0x200, /* Some thread stack pages were faulted in as part of BT */ + kThreadTriedFaultBT = 0x400, /* We tried to fault in thread stack pages as part of BT */ + kThreadOnCore = 0x800, /* Thread was on-core when we entered debugger context */ + kThreadIdleWorker = 0x1000, /* Thread is an idle libpthread worker thread */ + kThreadMain = 0x2000, /* Thread is the main thread */ +}; + +struct mem_and_io_snapshot { + uint32_t snapshot_magic; + uint32_t free_pages; + uint32_t active_pages; + uint32_t inactive_pages; + uint32_t purgeable_pages; + uint32_t wired_pages; + uint32_t speculative_pages; + uint32_t throttled_pages; + uint32_t filebacked_pages; + uint32_t compressions; + uint32_t decompressions; + uint32_t compressor_size; + int32_t busy_buffer_count; + uint32_t pages_wanted; + uint32_t pages_reclaimed; + uint8_t pages_wanted_reclaimed_valid; // did mach_vm_pressure_monitor succeed? +} __attribute__((packed)); + +/* SS_TH_* macros are for ths_state */ +#define SS_TH_WAIT 0x01 /* queued for waiting */ +#define SS_TH_SUSP 0x02 /* stopped or requested to stop */ +#define SS_TH_RUN 0x04 /* running or on runq */ +#define SS_TH_UNINT 0x08 /* waiting uninteruptibly */ +#define SS_TH_TERMINATE 0x10 /* halted at termination */ +#define SS_TH_TERMINATE2 0x20 /* added to termination queue */ +#define SS_TH_IDLE 0x80 /* idling processor */ + +struct thread_snapshot_v2 { + uint64_t ths_thread_id; + uint64_t ths_wait_event; + uint64_t ths_continuation; + uint64_t ths_total_syscalls; + uint64_t ths_voucher_identifier; + uint64_t ths_dqserialnum; + uint64_t ths_user_time; + uint64_t ths_sys_time; + uint64_t ths_ss_flags; + uint64_t ths_last_run_time; + uint64_t ths_last_made_runnable_time; + uint32_t ths_state; + uint32_t ths_sched_flags; + int16_t ths_base_priority; + int16_t ths_sched_priority; + uint8_t ths_eqos; + uint8_t ths_rqos; + uint8_t ths_rqos_override; + uint8_t ths_io_tier; +} __attribute__((packed)); + +struct thread_snapshot_v3 { + uint64_t ths_thread_id; + uint64_t ths_wait_event; + uint64_t ths_continuation; + uint64_t ths_total_syscalls; + uint64_t ths_voucher_identifier; + uint64_t ths_dqserialnum; + uint64_t ths_user_time; + uint64_t ths_sys_time; + uint64_t ths_ss_flags; + uint64_t ths_last_run_time; + uint64_t ths_last_made_runnable_time; + uint32_t ths_state; + uint32_t ths_sched_flags; + int16_t ths_base_priority; + int16_t ths_sched_priority; + uint8_t ths_eqos; + uint8_t ths_rqos; + uint8_t ths_rqos_override; + uint8_t ths_io_tier; + uint64_t ths_thread_t; +} __attribute__((packed)); + + +struct thread_snapshot_v4 { + uint64_t ths_thread_id; + uint64_t ths_wait_event; + uint64_t ths_continuation; + uint64_t ths_total_syscalls; + uint64_t ths_voucher_identifier; + uint64_t ths_dqserialnum; + uint64_t ths_user_time; + uint64_t ths_sys_time; + uint64_t ths_ss_flags; + uint64_t ths_last_run_time; + uint64_t ths_last_made_runnable_time; + uint32_t ths_state; + uint32_t ths_sched_flags; + int16_t ths_base_priority; + int16_t ths_sched_priority; + uint8_t ths_eqos; + uint8_t ths_rqos; + uint8_t ths_rqos_override; + uint8_t ths_io_tier; + uint64_t ths_thread_t; + uint64_t ths_requested_policy; + uint64_t ths_effective_policy; +} __attribute__((packed)); + + +struct thread_group_snapshot { + uint64_t tgs_id; + char tgs_name[16]; +} __attribute__((packed)); + +enum coalition_flags { + kCoalitionTermRequested = 0x1, + kCoalitionTerminated = 0x2, + kCoalitionReaped = 0x4, + kCoalitionPrivileged = 0x8, +}; + +struct jetsam_coalition_snapshot { + uint64_t jcs_id; + uint64_t jcs_flags; + uint64_t jcs_thread_group; + uint64_t jcs_leader_task_uniqueid; +} __attribute__((packed)); + +struct instrs_cycles_snapshot { + uint64_t ics_instructions; + uint64_t ics_cycles; +} __attribute__((packed)); + +struct thread_delta_snapshot_v2 { + uint64_t tds_thread_id; + uint64_t tds_voucher_identifier; + uint64_t tds_ss_flags; + uint64_t tds_last_made_runnable_time; + uint32_t tds_state; + uint32_t tds_sched_flags; + int16_t tds_base_priority; + int16_t tds_sched_priority; + uint8_t tds_eqos; + uint8_t tds_rqos; + uint8_t tds_rqos_override; + uint8_t tds_io_tier; +} __attribute__ ((packed)); + +struct io_stats_snapshot +{ + /* + * I/O Statistics + * XXX: These fields must be together. + */ + uint64_t ss_disk_reads_count; + uint64_t ss_disk_reads_size; + uint64_t ss_disk_writes_count; + uint64_t ss_disk_writes_size; + uint64_t ss_io_priority_count[STACKSHOT_IO_NUM_PRIORITIES]; + uint64_t ss_io_priority_size[STACKSHOT_IO_NUM_PRIORITIES]; + uint64_t ss_paging_count; + uint64_t ss_paging_size; + uint64_t ss_non_paging_count; + uint64_t ss_non_paging_size; + uint64_t ss_data_count; + uint64_t ss_data_size; + uint64_t ss_metadata_count; + uint64_t ss_metadata_size; + /* XXX: I/O Statistics end */ + +} __attribute__ ((packed)); + +struct task_snapshot_v2 { + uint64_t ts_unique_pid; + uint64_t ts_ss_flags; + uint64_t ts_user_time_in_terminated_threads; + uint64_t ts_system_time_in_terminated_threads; + uint64_t ts_p_start_sec; + uint64_t ts_task_size; + uint64_t ts_max_resident_size; + uint32_t ts_suspend_count; + uint32_t ts_faults; + uint32_t ts_pageins; + uint32_t ts_cow_faults; + uint32_t ts_was_throttled; + uint32_t ts_did_throttle; + uint32_t ts_latency_qos; + int32_t ts_pid; + char ts_p_comm[32]; +} __attribute__ ((packed)); + +struct task_delta_snapshot_v2 { + uint64_t tds_unique_pid; + uint64_t tds_ss_flags; + uint64_t tds_user_time_in_terminated_threads; + uint64_t tds_system_time_in_terminated_threads; + uint64_t tds_task_size; + uint64_t tds_max_resident_size; + uint32_t tds_suspend_count; + uint32_t tds_faults; + uint32_t tds_pageins; + uint32_t tds_cow_faults; + uint32_t tds_was_throttled; + uint32_t tds_did_throttle; + uint32_t tds_latency_qos; +} __attribute__ ((packed)); + +struct stackshot_cpu_times { + uint64_t user_usec; + uint64_t system_usec; +} __attribute__((packed)); + +struct stackshot_duration { + uint64_t stackshot_duration; + uint64_t stackshot_duration_outer; +} __attribute__((packed)); + +struct stackshot_fault_stats { + uint32_t sfs_pages_faulted_in; /* number of pages faulted in using KDP fault path */ + uint64_t sfs_time_spent_faulting; /* MATUs spent faulting */ + uint64_t sfs_system_max_fault_time; /* MATUs fault time limit per stackshot */ + uint8_t sfs_stopped_faulting; /* we stopped decompressing because we hit the limit */ +} __attribute__((packed)); + +typedef struct stackshot_thread_waitinfo { + uint64_t owner; /* The thread that owns the object */ + uint64_t waiter; /* The thread that's waiting on the object */ + uint64_t context; /* A context uniquely identifying the object */ + uint8_t wait_type; /* The type of object that the thread is waiting on */ +} __attribute__((packed)) thread_waitinfo_t; + +#define STACKSHOT_WAITOWNER_KERNEL (UINT64_MAX - 1) +#define STACKSHOT_WAITOWNER_PORT_LOCKED (UINT64_MAX - 2) +#define STACKSHOT_WAITOWNER_PSET_LOCKED (UINT64_MAX - 3) +#define STACKSHOT_WAITOWNER_INTRANSIT (UINT64_MAX - 4) +#define STACKSHOT_WAITOWNER_MTXSPIN (UINT64_MAX - 5) +#define STACKSHOT_WAITOWNER_THREQUESTED (UINT64_MAX - 6) /* workloop waiting for a new worker thread */ +#define STACKSHOT_WAITOWNER_SUSPENDED (UINT64_MAX - 7) /* workloop is suspended */ + + +/**************** definitions for crashinfo *********************/ + +/* + * NOTE: Please update kcdata/libkdd/kcdtypes.c if you make any changes + * in TASK_CRASHINFO_* types. + */ + +/* FIXME some of these types aren't clean (fixed width, packed, and defined *here*) */ + +#define TASK_CRASHINFO_BEGIN KCDATA_BUFFER_BEGIN_CRASHINFO +#define TASK_CRASHINFO_STRING_DESC KCDATA_TYPE_STRING_DESC +#define TASK_CRASHINFO_UINT32_DESC KCDATA_TYPE_UINT32_DESC +#define TASK_CRASHINFO_UINT64_DESC KCDATA_TYPE_UINT64_DESC + +#define TASK_CRASHINFO_EXTMODINFO 0x801 +#define TASK_CRASHINFO_BSDINFOWITHUNIQID 0x802 /* struct proc_uniqidentifierinfo */ +#define TASK_CRASHINFO_TASKDYLD_INFO 0x803 +#define TASK_CRASHINFO_UUID 0x804 +#define TASK_CRASHINFO_PID 0x805 +#define TASK_CRASHINFO_PPID 0x806 +#define TASK_CRASHINFO_RUSAGE 0x807 /* struct rusage DEPRECATED do not use. + This struct has longs in it */ +#define TASK_CRASHINFO_RUSAGE_INFO 0x808 /* struct rusage_info_v3 from resource.h */ +#define TASK_CRASHINFO_PROC_NAME 0x809 /* char * */ +#define TASK_CRASHINFO_PROC_STARTTIME 0x80B /* struct timeval64 */ +#define TASK_CRASHINFO_USERSTACK 0x80C /* uint64_t */ +#define TASK_CRASHINFO_ARGSLEN 0x80D +#define TASK_CRASHINFO_EXCEPTION_CODES 0x80E /* mach_exception_data_t */ +#define TASK_CRASHINFO_PROC_PATH 0x80F /* string of len MAXPATHLEN */ +#define TASK_CRASHINFO_PROC_CSFLAGS 0x810 /* uint32_t */ +#define TASK_CRASHINFO_PROC_STATUS 0x811 /* char */ +#define TASK_CRASHINFO_UID 0x812 /* uid_t */ +#define TASK_CRASHINFO_GID 0x813 /* gid_t */ +#define TASK_CRASHINFO_PROC_ARGC 0x814 /* int */ +#define TASK_CRASHINFO_PROC_FLAGS 0x815 /* unsigned int */ +#define TASK_CRASHINFO_CPUTYPE 0x816 /* cpu_type_t */ +#define TASK_CRASHINFO_WORKQUEUEINFO 0x817 /* struct proc_workqueueinfo */ +#define TASK_CRASHINFO_RESPONSIBLE_PID 0x818 /* pid_t */ +#define TASK_CRASHINFO_DIRTY_FLAGS 0x819 /* int */ +#define TASK_CRASHINFO_CRASHED_THREADID 0x81A /* uint64_t */ +#define TASK_CRASHINFO_COALITION_ID 0x81B /* uint64_t */ +#define TASK_CRASHINFO_UDATA_PTRS 0x81C /* uint64_t */ +#define TASK_CRASHINFO_MEMORY_LIMIT 0x81D /* uint64_t */ + +#define TASK_CRASHINFO_END KCDATA_TYPE_BUFFER_END + +/**************** definitions for os reasons *********************/ + +#define EXIT_REASON_SNAPSHOT 0x1001 +#define EXIT_REASON_USER_DESC 0x1002 /* string description of reason */ +#define EXIT_REASON_USER_PAYLOAD 0x1003 /* user payload data */ +#define EXIT_REASON_CODESIGNING_INFO 0x1004 +#define EXIT_REASON_WORKLOOP_ID 0x1005 +#define EXIT_REASON_DISPATCH_QUEUE_NO 0x1006 + +struct exit_reason_snapshot { + uint32_t ers_namespace; + uint64_t ers_code; + /* end of version 1 of exit_reason_snapshot. sizeof v1 was 12 */ + uint64_t ers_flags; +} __attribute__((packed)); + +#define EXIT_REASON_CODESIG_PATH_MAX 1024 + +struct codesigning_exit_reason_info { + uint64_t ceri_virt_addr; + uint64_t ceri_file_offset; + char ceri_pathname[EXIT_REASON_CODESIG_PATH_MAX]; + char ceri_filename[EXIT_REASON_CODESIG_PATH_MAX]; + uint64_t ceri_codesig_modtime_secs; + uint64_t ceri_codesig_modtime_nsecs; + uint64_t ceri_page_modtime_secs; + uint64_t ceri_page_modtime_nsecs; + uint8_t ceri_path_truncated; + uint8_t ceri_object_codesigned; + uint8_t ceri_page_codesig_validated; + uint8_t ceri_page_codesig_tainted; + uint8_t ceri_page_codesig_nx; + uint8_t ceri_page_wpmapped; + uint8_t ceri_page_slid; + uint8_t ceri_page_dirty; + uint32_t ceri_page_shadow_depth; +} __attribute__((packed)); + +#define EXIT_REASON_USER_DESC_MAX_LEN 1024 +#define EXIT_REASON_PAYLOAD_MAX_LEN 2048 +/**************** safe iterators *********************/ + +typedef struct kcdata_iter { + kcdata_item_t item; + void *end; +} kcdata_iter_t; + + +static inline +kcdata_iter_t kcdata_iter(void *buffer, unsigned long size) { + kcdata_iter_t iter; + iter.item = (kcdata_item_t) buffer; + iter.end = (void*) (((uintptr_t)buffer) + size); + return iter; +} + +static inline +kcdata_iter_t kcdata_iter_unsafe(void *buffer) __attribute__((deprecated)); + +static inline +kcdata_iter_t kcdata_iter_unsafe(void *buffer) { + kcdata_iter_t iter; + iter.item = (kcdata_item_t) buffer; + iter.end = (void*) (uintptr_t) ~0; + return iter; +} + +static const kcdata_iter_t kcdata_invalid_iter = { .item = 0, .end = 0 }; + +static inline +int kcdata_iter_valid(kcdata_iter_t iter) { + return + ( (uintptr_t)iter.item + sizeof(struct kcdata_item) <= (uintptr_t)iter.end ) && + ( (uintptr_t)iter.item + sizeof(struct kcdata_item) + iter.item->size <= (uintptr_t)iter.end); +} + + +static inline +kcdata_iter_t kcdata_iter_next(kcdata_iter_t iter) { + iter.item = (kcdata_item_t) (((uintptr_t)iter.item) + sizeof(struct kcdata_item) + (iter.item->size)); + return iter; +} + +static inline uint32_t +kcdata_iter_type(kcdata_iter_t iter) +{ + if ((iter.item->type & ~0xfu) == KCDATA_TYPE_ARRAY_PAD0) + return KCDATA_TYPE_ARRAY; + else + return iter.item->type; +} + +static inline uint32_t +kcdata_calc_padding(uint32_t size) +{ + /* calculate number of bits to add to size to get something divisible by 16 */ + return (-size) & 0xf; +} + +static inline uint32_t +kcdata_flags_get_padding(uint64_t flags) +{ + return flags & KCDATA_FLAGS_STRUCT_PADDING_MASK; +} + +/* see comment above about has_padding */ +static inline int +kcdata_iter_is_legacy_item(kcdata_iter_t iter, uint32_t legacy_size) +{ + uint32_t legacy_size_padded = legacy_size + kcdata_calc_padding(legacy_size); + return (iter.item->size == legacy_size_padded && + (iter.item->flags & (KCDATA_FLAGS_STRUCT_PADDING_MASK | KCDATA_FLAGS_STRUCT_HAS_PADDING)) == 0); + +} + +static inline uint32_t +kcdata_iter_size(kcdata_iter_t iter) +{ + uint32_t legacy_size = 0; + + switch (kcdata_iter_type(iter)) { + case KCDATA_TYPE_ARRAY: + case KCDATA_TYPE_CONTAINER_BEGIN: + return iter.item->size; + case STACKSHOT_KCTYPE_THREAD_SNAPSHOT: { + legacy_size = sizeof(struct thread_snapshot_v2); + if (kcdata_iter_is_legacy_item(iter, legacy_size)) { + return legacy_size; + } + + goto not_legacy; + } + case STACKSHOT_KCTYPE_SHAREDCACHE_LOADINFO: { + legacy_size = sizeof(struct dyld_uuid_info_64); + if (kcdata_iter_is_legacy_item(iter, legacy_size)) { + return legacy_size; + } + + goto not_legacy; + } +not_legacy: + default: + if (iter.item->size < kcdata_flags_get_padding(iter.item->flags)) + return 0; + else + return iter.item->size - kcdata_flags_get_padding(iter.item->flags); + } +} + +static inline uint64_t +kcdata_iter_flags(kcdata_iter_t iter) +{ + return iter.item->flags; +} + +static inline +void * kcdata_iter_payload(kcdata_iter_t iter) { + return &iter.item->data; +} + + +static inline +uint32_t kcdata_iter_array_elem_type(kcdata_iter_t iter) { + return (iter.item->flags >> 32) & UINT32_MAX; +} + +static inline +uint32_t kcdata_iter_array_elem_count(kcdata_iter_t iter) { + return (iter.item->flags) & UINT32_MAX; +} + +/* KCDATA_TYPE_ARRAY is ambiguous about the size of the array elements. Size is + * calculated as total_size / elements_count, but total size got padded out to a + * 16 byte alignment. New kernels will generate KCDATA_TYPE_ARRAY_PAD* instead + * to explicitly tell us how much padding was used. Here we have a fixed, never + * to be altered list of the sizes of array elements that were used before I + * discovered this issue. If you find a KCDATA_TYPE_ARRAY that is not one of + * these types, treat it as invalid data. */ + +static inline +uint32_t +kcdata_iter_array_size_switch(kcdata_iter_t iter) { + switch(kcdata_iter_array_elem_type(iter)) { + case KCDATA_TYPE_LIBRARY_LOADINFO: + return sizeof(struct dyld_uuid_info_32); + case KCDATA_TYPE_LIBRARY_LOADINFO64: + return sizeof(struct dyld_uuid_info_64); + case STACKSHOT_KCTYPE_KERN_STACKFRAME: + case STACKSHOT_KCTYPE_USER_STACKFRAME: + return sizeof(struct stack_snapshot_frame32); + case STACKSHOT_KCTYPE_KERN_STACKFRAME64: + case STACKSHOT_KCTYPE_USER_STACKFRAME64: + return sizeof(struct stack_snapshot_frame64); + case STACKSHOT_KCTYPE_DONATING_PIDS: + return sizeof(int32_t); + case STACKSHOT_KCTYPE_THREAD_DELTA_SNAPSHOT: + return sizeof(struct thread_delta_snapshot_v2); + // This one is only here to make some unit tests work. It should be OK to + // remove. + case TASK_CRASHINFO_CRASHED_THREADID: + return sizeof(uint64_t); + default: + return 0; + } +} + +static inline +int kcdata_iter_array_valid(kcdata_iter_t iter) { + if (!kcdata_iter_valid(iter)) + return 0; + if (kcdata_iter_type(iter) != KCDATA_TYPE_ARRAY) + return 0; + if (kcdata_iter_array_elem_count(iter) == 0) + return iter.item->size == 0; + if (iter.item->type == KCDATA_TYPE_ARRAY) { + uint32_t elem_size = kcdata_iter_array_size_switch(iter); + if (elem_size == 0) + return 0; + /* sizes get aligned to the nearest 16. */ + return + kcdata_iter_array_elem_count(iter) <= iter.item->size / elem_size && + iter.item->size % kcdata_iter_array_elem_count(iter) < 16; + } else { + return + (iter.item->type & 0xf) <= iter.item->size && + kcdata_iter_array_elem_count(iter) <= iter.item->size - (iter.item->type & 0xf) && + (iter.item->size - (iter.item->type & 0xf)) % kcdata_iter_array_elem_count(iter) == 0; + } +} + + +static inline +uint32_t kcdata_iter_array_elem_size(kcdata_iter_t iter) { + if (iter.item->type == KCDATA_TYPE_ARRAY) + return kcdata_iter_array_size_switch(iter); + if (kcdata_iter_array_elem_count(iter) == 0) + return 0; + return (iter.item->size - (iter.item->type & 0xf)) / kcdata_iter_array_elem_count(iter); +} + +static inline +int kcdata_iter_container_valid(kcdata_iter_t iter) { + return + kcdata_iter_valid(iter) && + kcdata_iter_type(iter) == KCDATA_TYPE_CONTAINER_BEGIN && + iter.item->size >= sizeof(uint32_t); +} + +static inline +uint32_t kcdata_iter_container_type(kcdata_iter_t iter) { + return * (uint32_t *) kcdata_iter_payload(iter); +} + +static inline +uint64_t kcdata_iter_container_id(kcdata_iter_t iter) { + return iter.item->flags; +} + + +#define KCDATA_ITER_FOREACH(iter) for(; kcdata_iter_valid(iter) && iter.item->type != KCDATA_TYPE_BUFFER_END; iter = kcdata_iter_next(iter)) +#define KCDATA_ITER_FOREACH_FAILED(iter) (!kcdata_iter_valid(iter) || (iter).item->type != KCDATA_TYPE_BUFFER_END) + +static inline +kcdata_iter_t +kcdata_iter_find_type(kcdata_iter_t iter, uint32_t type) +{ + KCDATA_ITER_FOREACH(iter) + { + if (kcdata_iter_type(iter) == type) + return iter; + } + return kcdata_invalid_iter; +} + +static inline +int kcdata_iter_data_with_desc_valid(kcdata_iter_t iter, uint32_t minsize) { + return + kcdata_iter_valid(iter) && + kcdata_iter_size(iter) >= KCDATA_DESC_MAXLEN + minsize && + ((char*)kcdata_iter_payload(iter))[KCDATA_DESC_MAXLEN-1] == 0; +} + +static inline +char *kcdata_iter_string(kcdata_iter_t iter, uint32_t offset) { + if (offset > kcdata_iter_size(iter)) { + return NULL; + } + uint32_t maxlen = kcdata_iter_size(iter) - offset; + char *s = ((char*)kcdata_iter_payload(iter)) + offset; + if (strnlen(s, maxlen) < maxlen) { + return s; + } else { + return NULL; + } +} + +static inline void kcdata_iter_get_data_with_desc(kcdata_iter_t iter, char **desc_ptr, void **data_ptr, uint32_t *size_ptr) { + if (desc_ptr) + *desc_ptr = (char *)kcdata_iter_payload(iter); + if (data_ptr) + *data_ptr = (void *)((uintptr_t)kcdata_iter_payload(iter) + KCDATA_DESC_MAXLEN); + if (size_ptr) + *size_ptr = kcdata_iter_size(iter) - KCDATA_DESC_MAXLEN; +} + +#endif diff --git a/libkdd/kcdtypes.c b/libkdd/kcdtypes.c index 6b1ac415e..8fe7aee23 100644 --- a/libkdd/kcdtypes.c +++ b/libkdd/kcdtypes.c @@ -325,6 +325,8 @@ kcdata_get_typedescription(unsigned type_id, uint8_t * buffer, uint32_t buffer_s _SUBTYPE(KC_ST_UINT8, struct thread_snapshot_v3, ths_rqos_override); _SUBTYPE(KC_ST_UINT8, struct thread_snapshot_v3, ths_io_tier); _SUBTYPE(KC_ST_UINT64, struct thread_snapshot_v3, ths_thread_t); + _SUBTYPE(KC_ST_UINT64, struct thread_snapshot_v4, ths_requested_policy); + _SUBTYPE(KC_ST_UINT64, struct thread_snapshot_v4, ths_effective_policy); setup_type_definition(retval, type_id, i, "thread_snapshot"); break; @@ -442,6 +444,13 @@ kcdata_get_typedescription(unsigned type_id, uint8_t * buffer, uint32_t buffer_s break; } + case STACKSHOT_KCTYPE_THREAD_POLICY_VERSION: { + i = 0; + setup_subtype_description(&subtypes[i++], KC_ST_UINT32, 0, "thread_policy_version"); + setup_type_definition(retval, type_id, i, "thread_policy_version"); + break; + } + case STACKSHOT_KCTYPE_JETSAM_LEVEL: { i = 0; setup_subtype_description(&subtypes[i++], KC_ST_UINT32, 0, "jetsam_level"); @@ -563,6 +572,46 @@ kcdata_get_typedescription(unsigned type_id, uint8_t * buffer, uint32_t buffer_s break; } + case STACKSHOT_KCTYPE_THREAD_GROUP_SNAPSHOT: { + i = 0; + _SUBTYPE(KC_ST_UINT64, struct thread_group_snapshot, tgs_id); + _SUBTYPE_ARRAY(KC_ST_CHAR, struct thread_group_snapshot, tgs_name, 16); + setup_type_definition(retval, type_id, i, "thread_group_snapshot"); + break; + } + + case STACKSHOT_KCTYPE_THREAD_GROUP: { + i = 0; + setup_subtype_description(&subtypes[i++], KC_ST_UINT64, 0, "thread_group"); + setup_type_definition(retval, type_id, i, "thread_group"); + break; + }; + + case STACKSHOT_KCTYPE_JETSAM_COALITION_SNAPSHOT: { + i = 0; + _SUBTYPE(KC_ST_UINT64, struct jetsam_coalition_snapshot, jcs_id); + _SUBTYPE(KC_ST_UINT64, struct jetsam_coalition_snapshot, jcs_flags); + _SUBTYPE(KC_ST_UINT64, struct jetsam_coalition_snapshot, jcs_thread_group); + _SUBTYPE(KC_ST_UINT64, struct jetsam_coalition_snapshot, jcs_leader_task_uniqueid); + setup_type_definition(retval, type_id, i, "jetsam_coalition_snapshot"); + break; + } + + case STACKSHOT_KCTYPE_JETSAM_COALITION: { + i = 0; + setup_subtype_description(&subtypes[i++], KC_ST_UINT64, 0, "jetsam_coalition"); + setup_type_definition(retval, type_id, i, "jetsam_coalition"); + break; + }; + + case STACKSHOT_KCTYPE_INSTRS_CYCLES: { + i = 0; + _SUBTYPE(KC_ST_UINT64, struct instrs_cycles_snapshot, ics_instructions); + _SUBTYPE(KC_ST_UINT64, struct instrs_cycles_snapshot, ics_cycles); + setup_type_definition(retval, type_id, i, "instrs_cycles_snapshot"); + break; + } + case TASK_CRASHINFO_PROC_STARTTIME: { i = 0; _SUBTYPE(KC_ST_INT64, struct timeval64, tv_sec); @@ -736,6 +785,20 @@ kcdata_get_typedescription(unsigned type_id, uint8_t * buffer, uint32_t buffer_s break; + case EXIT_REASON_WORKLOOP_ID: { + i = 0; + setup_subtype_description(&subtypes[i++], KC_ST_UINT64, 0, "exit_reason_workloop_id"); + setup_type_definition(retval, type_id, i, "exit_reason_workloop_id"); + break; + } + + case EXIT_REASON_DISPATCH_QUEUE_NO: { + i = 0; + setup_subtype_description(&subtypes[i++], KC_ST_UINT64, 0, "exit_reason_dispatch_queue_no"); + setup_type_definition(retval, type_id, i, "exit_reason_dispatch_queue_no"); + break; + } + } default: diff --git a/libkdd/kdd.xcodeproj/project.pbxproj b/libkdd/kdd.xcodeproj/project.pbxproj index 7e55392bf..5a2b63714 100644 --- a/libkdd/kdd.xcodeproj/project.pbxproj +++ b/libkdd/kdd.xcodeproj/project.pbxproj @@ -23,6 +23,8 @@ 0860F87B1BFC3857007E1301 /* stackshot-sample-tailspin-2.plist.gz in Resources */ = {isa = PBXBuildFile; fileRef = 0860F8791BFC3845007E1301 /* stackshot-sample-tailspin-2.plist.gz */; }; 086395B51BF5655D005ED913 /* kdd_main.m in Sources */ = {isa = PBXBuildFile; fileRef = 086395B41BF5655D005ED913 /* kdd_main.m */; }; 086395B91BF565A2005ED913 /* libkdd.a in Frameworks */ = {isa = PBXBuildFile; fileRef = C91C93C71ACB58B700119B60 /* libkdd.a */; }; + 088C36E01EF323C300ABB2E0 /* stackshot-sample-thread-policy in Resources */ = {isa = PBXBuildFile; fileRef = 088C36DF1EF323AE00ABB2E0 /* stackshot-sample-thread-policy */; }; + 088C36E11EF323C300ABB2E0 /* stackshot-sample-thread-policy.plist.gz in Resources */ = {isa = PBXBuildFile; fileRef = 088C36DE1EF323AE00ABB2E0 /* stackshot-sample-thread-policy.plist.gz */; }; 08A4C94C1C4701B800D5F010 /* KCDEmbeddedBufferDescription.m in Sources */ = {isa = PBXBuildFile; fileRef = 08A4C94B1C4701B800D5F010 /* KCDEmbeddedBufferDescription.m */; }; 08A4C94F1C470F1C00D5F010 /* nested-sample in Resources */ = {isa = PBXBuildFile; fileRef = 08A4C94D1C470F0900D5F010 /* nested-sample */; }; 08A4C9501C470F1C00D5F010 /* nested-sample.plist in Resources */ = {isa = PBXBuildFile; fileRef = 08A4C94E1C470F0900D5F010 /* nested-sample.plist */; }; @@ -58,6 +60,10 @@ 13DBA26A1CAB1BA000227EB2 /* stackshot-sample-sharedcachev2 in Resources */ = {isa = PBXBuildFile; fileRef = 13DBA2691CAB1B9C00227EB2 /* stackshot-sample-sharedcachev2 */; }; 13F3DA9C1C7C1BEE00ACFFCC /* corpse-twr-sample-v2 in Resources */ = {isa = PBXBuildFile; fileRef = 13F3DA9B1C7C1BE700ACFFCC /* corpse-twr-sample-v2 */; }; 13F3DA9E1C7C1C6600ACFFCC /* corpse-twr-sample-v2.plist.gz in Resources */ = {isa = PBXBuildFile; fileRef = 13F3DA9D1C7C1C6000ACFFCC /* corpse-twr-sample-v2.plist.gz */; }; + 1862B0341E7A083F0005ADF4 /* stackshot-sample-thread-groups in Resources */ = {isa = PBXBuildFile; fileRef = 1862B0321E7A083F0005ADF4 /* stackshot-sample-thread-groups */; }; + 1862B0351E7A083F0005ADF4 /* stackshot-sample-thread-groups.plist.gz in Resources */ = {isa = PBXBuildFile; fileRef = 1862B0331E7A083F0005ADF4 /* stackshot-sample-thread-groups.plist.gz */; }; + 18E592981E9451A20018612A /* stackshot-sample-coalitions in Resources */ = {isa = PBXBuildFile; fileRef = 18E592961E9451A20018612A /* stackshot-sample-coalitions */; }; + 18E592991E9451A20018612A /* stackshot-sample-coalitions.plist.gz in Resources */ = {isa = PBXBuildFile; fileRef = 18E592971E9451A20018612A /* stackshot-sample-coalitions.plist.gz */; }; C91C93CB1ACB58B700119B60 /* kdd.h in Headers */ = {isa = PBXBuildFile; fileRef = C91C93CA1ACB58B700119B60 /* kdd.h */; settings = {ATTRIBUTES = (Private, ); }; }; C91C93CD1ACB58B700119B60 /* kdd.m in Sources */ = {isa = PBXBuildFile; fileRef = C91C93CC1ACB58B700119B60 /* kdd.m */; }; C91C93E41ACB598700119B60 /* KCDBasicTypeDescription.h in Headers */ = {isa = PBXBuildFile; fileRef = C91C93E01ACB598700119B60 /* KCDBasicTypeDescription.h */; }; @@ -67,6 +73,8 @@ C9C5C68C1ACDAFDB00BE0E5E /* kcdtypes.c in Sources */ = {isa = PBXBuildFile; fileRef = C9C5C68B1ACDAFDB00BE0E5E /* kcdtypes.c */; }; C9D7B53F1D1B41D700F1019D /* xnupost_testconfig-sample.plist.gz in Resources */ = {isa = PBXBuildFile; fileRef = C9D7B53D1D1B41D700F1019D /* xnupost_testconfig-sample.plist.gz */; }; C9D7B5401D1B41D700F1019D /* xnupost_testconfig-sample in Resources */ = {isa = PBXBuildFile; fileRef = C9D7B53E1D1B41D700F1019D /* xnupost_testconfig-sample */; }; + C9DCEF011F01C3810000BD02 /* stackshot-sample-instrs-cycles in Resources */ = {isa = PBXBuildFile; fileRef = C9DCEF001F01C3790000BD02 /* stackshot-sample-instrs-cycles */; }; + C9DCEF021F01C3810000BD02 /* stackshot-sample-instrs-cycles.plist.gz in Resources */ = {isa = PBXBuildFile; fileRef = C9DCEEFF1F01C3790000BD02 /* stackshot-sample-instrs-cycles.plist.gz */; }; C9DE39141ACB5A540020F4A3 /* kcdata_core.m in Sources */ = {isa = PBXBuildFile; fileRef = C9DE39131ACB5A540020F4A3 /* kcdata_core.m */; }; /* End PBXBuildFile section */ @@ -117,6 +125,8 @@ 0860F8791BFC3845007E1301 /* stackshot-sample-tailspin-2.plist.gz */ = {isa = PBXFileReference; lastKnownFileType = archive.gzip; name = "stackshot-sample-tailspin-2.plist.gz"; path = "tests/stackshot-sample-tailspin-2.plist.gz"; sourceTree = SOURCE_ROOT; }; 086395B21BF5655D005ED913 /* kdd */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = kdd; sourceTree = BUILT_PRODUCTS_DIR; }; 086395B41BF5655D005ED913 /* kdd_main.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = kdd_main.m; sourceTree = "<group>"; }; + 088C36DE1EF323AE00ABB2E0 /* stackshot-sample-thread-policy.plist.gz */ = {isa = PBXFileReference; lastKnownFileType = archive.gzip; path = "stackshot-sample-thread-policy.plist.gz"; sourceTree = "<group>"; }; + 088C36DF1EF323AE00ABB2E0 /* stackshot-sample-thread-policy */ = {isa = PBXFileReference; lastKnownFileType = file; path = "stackshot-sample-thread-policy"; sourceTree = "<group>"; }; 08A4C94A1C47019E00D5F010 /* KCDEmbeddedBufferDescription.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = KCDEmbeddedBufferDescription.h; sourceTree = "<group>"; }; 08A4C94B1C4701B800D5F010 /* KCDEmbeddedBufferDescription.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = KCDEmbeddedBufferDescription.m; sourceTree = "<group>"; }; 08A4C94D1C470F0900D5F010 /* nested-sample */ = {isa = PBXFileReference; lastKnownFileType = file; name = "nested-sample"; path = "tests/nested-sample"; sourceTree = SOURCE_ROOT; }; @@ -153,6 +163,10 @@ 13EADC171C4DCDA100468D97 /* test-twr-sample.plist.gz */ = {isa = PBXFileReference; lastKnownFileType = archive.gzip; name = "test-twr-sample.plist.gz"; path = "tests/test-twr-sample.plist.gz"; sourceTree = SOURCE_ROOT; }; 13F3DA9B1C7C1BE700ACFFCC /* corpse-twr-sample-v2 */ = {isa = PBXFileReference; lastKnownFileType = file; name = "corpse-twr-sample-v2"; path = "tests/corpse-twr-sample-v2"; sourceTree = SOURCE_ROOT; }; 13F3DA9D1C7C1C6000ACFFCC /* corpse-twr-sample-v2.plist.gz */ = {isa = PBXFileReference; lastKnownFileType = archive.gzip; name = "corpse-twr-sample-v2.plist.gz"; path = "tests/corpse-twr-sample-v2.plist.gz"; sourceTree = SOURCE_ROOT; }; + 1862B0321E7A083F0005ADF4 /* stackshot-sample-thread-groups */ = {isa = PBXFileReference; lastKnownFileType = file; path = "stackshot-sample-thread-groups"; sourceTree = "<group>"; }; + 1862B0331E7A083F0005ADF4 /* stackshot-sample-thread-groups.plist.gz */ = {isa = PBXFileReference; lastKnownFileType = archive.gzip; path = "stackshot-sample-thread-groups.plist.gz"; sourceTree = "<group>"; }; + 18E592961E9451A20018612A /* stackshot-sample-coalitions */ = {isa = PBXFileReference; lastKnownFileType = file; path = "stackshot-sample-coalitions"; sourceTree = "<group>"; }; + 18E592971E9451A20018612A /* stackshot-sample-coalitions.plist.gz */ = {isa = PBXFileReference; lastKnownFileType = archive.gzip; path = "stackshot-sample-coalitions.plist.gz"; sourceTree = "<group>"; }; C91C93C71ACB58B700119B60 /* libkdd.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libkdd.a; sourceTree = BUILT_PRODUCTS_DIR; }; C91C93CA1ACB58B700119B60 /* kdd.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = kdd.h; sourceTree = "<group>"; }; C91C93CC1ACB58B700119B60 /* kdd.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = kdd.m; sourceTree = "<group>"; }; @@ -163,6 +177,8 @@ C9C5C68B1ACDAFDB00BE0E5E /* kcdtypes.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = kcdtypes.c; sourceTree = "<group>"; }; C9D7B53D1D1B41D700F1019D /* xnupost_testconfig-sample.plist.gz */ = {isa = PBXFileReference; lastKnownFileType = archive.gzip; path = "xnupost_testconfig-sample.plist.gz"; sourceTree = "<group>"; }; C9D7B53E1D1B41D700F1019D /* xnupost_testconfig-sample */ = {isa = PBXFileReference; lastKnownFileType = file; path = "xnupost_testconfig-sample"; sourceTree = "<group>"; }; + C9DCEEFF1F01C3790000BD02 /* stackshot-sample-instrs-cycles.plist.gz */ = {isa = PBXFileReference; lastKnownFileType = archive.gzip; path = "stackshot-sample-instrs-cycles.plist.gz"; sourceTree = "<group>"; }; + C9DCEF001F01C3790000BD02 /* stackshot-sample-instrs-cycles */ = {isa = PBXFileReference; lastKnownFileType = file; path = "stackshot-sample-instrs-cycles"; sourceTree = "<group>"; }; C9DE39131ACB5A540020F4A3 /* kcdata_core.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = kcdata_core.m; sourceTree = "<group>"; }; /* End PBXFileReference section */ @@ -198,6 +214,14 @@ 08603F351BF69EDE007D3784 /* tests */ = { isa = PBXGroup; children = ( + C9DCEF001F01C3790000BD02 /* stackshot-sample-instrs-cycles */, + C9DCEEFF1F01C3790000BD02 /* stackshot-sample-instrs-cycles.plist.gz */, + 088C36DF1EF323AE00ABB2E0 /* stackshot-sample-thread-policy */, + 088C36DE1EF323AE00ABB2E0 /* stackshot-sample-thread-policy.plist.gz */, + 18E592961E9451A20018612A /* stackshot-sample-coalitions */, + 18E592971E9451A20018612A /* stackshot-sample-coalitions.plist.gz */, + 1862B0321E7A083F0005ADF4 /* stackshot-sample-thread-groups */, + 1862B0331E7A083F0005ADF4 /* stackshot-sample-thread-groups.plist.gz */, C9D7B53D1D1B41D700F1019D /* xnupost_testconfig-sample.plist.gz */, C9D7B53E1D1B41D700F1019D /* xnupost_testconfig-sample */, 04C64AC91D25C43400C6C781 /* stackshot-with-waitinfo */, @@ -374,12 +398,12 @@ isa = PBXProject; attributes = { LastSwiftUpdateCheck = 0730; - LastUpgradeCheck = 0730; + LastUpgradeCheck = 0830; ORGANIZATIONNAME = "Vishal Patel"; TargetAttributes = { 08603F331BF69EDE007D3784 = { CreatedOnToolsVersion = 7.3; - LastSwiftMigration = 0800; + LastSwiftMigration = 0830; }; 086395B11BF5655D005ED913 = { CreatedOnToolsVersion = 7.3; @@ -413,9 +437,14 @@ isa = PBXResourcesBuildPhase; buildActionMask = 2147483647; files = ( + C9DCEF011F01C3810000BD02 /* stackshot-sample-instrs-cycles in Resources */, + C9DCEF021F01C3810000BD02 /* stackshot-sample-instrs-cycles.plist.gz in Resources */, + 088C36E01EF323C300ABB2E0 /* stackshot-sample-thread-policy in Resources */, + 088C36E11EF323C300ABB2E0 /* stackshot-sample-thread-policy.plist.gz in Resources */, 045F7F131D2ADE8000B4808B /* stackshot-with-waitinfo.plist.gz in Resources */, 045F7F121D2ADE7C00B4808B /* stackshot-with-waitinfo in Resources */, 08A4C94F1C470F1C00D5F010 /* nested-sample in Resources */, + 1862B0341E7A083F0005ADF4 /* stackshot-sample-thread-groups in Resources */, 08A4C9501C470F1C00D5F010 /* nested-sample.plist in Resources */, 13D6C5D21C4DDDBE005E617C /* test-twr-sample in Resources */, 13D6C5D01C4DDDB6005E617C /* corpse-twr-sample in Resources */, @@ -440,9 +469,11 @@ 13CC08441CB97F8D00EA6069 /* stackshot-fault-stats in Resources */, 13F3DA9C1C7C1BEE00ACFFCC /* corpse-twr-sample-v2 in Resources */, 13D6C5D31C4DDE0D005E617C /* test-twr-sample.plist.gz in Resources */, + 1862B0351E7A083F0005ADF4 /* stackshot-sample-thread-groups.plist.gz in Resources */, 1368F0851C87E06A00940FC6 /* exitreason-codesigning.plist.gz in Resources */, 08C9D83D1BFFF8E100DF6C05 /* exitreason-sample in Resources */, 08C9D83E1BFFF8E100DF6C05 /* exitreason-sample.plist.gz in Resources */, + 18E592981E9451A20018612A /* stackshot-sample-coalitions in Resources */, 08B4808B1BF9474A00B4AAE0 /* corpse-sample in Resources */, 13D6C5D11C4DDDB8005E617C /* corpse-twr-sample.plist.gz in Resources */, 08B4808C1BF9474A00B4AAE0 /* corpse-sample.plist.gz in Resources */, @@ -458,6 +489,7 @@ 08B4807B1BF8297500B4AAE0 /* stackshot-sample-old-arrays.plist.gz in Resources */, 0843EE941BF6BAC100CD4150 /* stackshot-sample.plist.gz in Resources */, 0843EE921BF6AFC600CD4150 /* stackshot-sample in Resources */, + 18E592991E9451A20018612A /* stackshot-sample-coalitions.plist.gz in Resources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -518,13 +550,12 @@ ENABLE_TESTABILITY = YES; INFOPLIST_FILE = tests/Info.plist; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks @loader_path/../Frameworks"; - MACOSX_DEPLOYMENT_TARGET = 10.11; PRODUCT_BUNDLE_IDENTIFIER = apple.com.Tests; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = macosx; SWIFT_OBJC_BRIDGING_HEADER = tests/kdd_bridge.h; SWIFT_OPTIMIZATION_LEVEL = "-Onone"; - SWIFT_VERSION = 2.3; + SWIFT_VERSION = 3.0; }; name = Debug; }; @@ -536,12 +567,11 @@ COMBINE_HIDPI_IMAGES = YES; INFOPLIST_FILE = tests/Info.plist; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks @loader_path/../Frameworks"; - MACOSX_DEPLOYMENT_TARGET = 10.11; PRODUCT_BUNDLE_IDENTIFIER = apple.com.Tests; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = macosx; SWIFT_OBJC_BRIDGING_HEADER = tests/kdd_bridge.h; - SWIFT_VERSION = 2.3; + SWIFT_VERSION = 3.0; }; name = Release; }; diff --git a/libkdd/tests/Tests.swift b/libkdd/tests/Tests.swift index e2225f35a..cd7f46ea0 100644 --- a/libkdd/tests/Tests.swift +++ b/libkdd/tests/Tests.swift @@ -20,61 +20,60 @@ import Foundation // Swift's bridging to uuid_t is awkward. -func nsuuid2uuid_t(nsuuid : NSUUID) -> uuid_t { +func nsuuid2uuid_t(_ nsuuid : NSUUID) -> uuid_t { let dat = nsuuid2array(nsuuid) return nsarray2uuid(dat) } -func nsarray2uuid(x : AnyObject) -> uuid_t { - let a = x as! NSArray - return uuid_t(UInt8(a[0] as! Int), - UInt8(a[1] as! Int), - UInt8(a[2] as! Int), - UInt8(a[3] as! Int), - UInt8(a[4] as! Int), - UInt8(a[5] as! Int), - UInt8(a[6] as! Int), - UInt8(a[7] as! Int), - UInt8(a[8] as! Int), - UInt8(a[9] as! Int), - UInt8(a[10] as! Int), - UInt8(a[11] as! Int), - UInt8(a[12] as! Int), - UInt8(a[13] as! Int), - UInt8(a[14] as! Int), - UInt8(a[15] as! Int)) +func nsarray2uuid(_ a : [Int]) -> uuid_t { + return uuid_t(UInt8(a[0]), + UInt8(a[1]), + UInt8(a[2]), + UInt8(a[3]), + UInt8(a[4]), + UInt8(a[5]), + UInt8(a[6]), + UInt8(a[7]), + UInt8(a[8]), + UInt8(a[9]), + UInt8(a[10]), + UInt8(a[11]), + UInt8(a[12]), + UInt8(a[13]), + UInt8(a[14]), + UInt8(a[15])) } -func nsuuid2array(uuid : NSUUID) -> [Int] { +func nsuuid2array(_ uuid: NSUUID) -> [Int] { var ret = [Int]() - let ptr = UnsafeMutablePointer<UInt8>.alloc(16) + let ptr = UnsafeMutablePointer<UInt8>.allocate(capacity: 16) - defer { ptr.dealloc(16) } + defer { ptr.deallocate(capacity:16) } - uuid.getUUIDBytes(ptr) + uuid.getBytes(ptr) for i in 0..<16 { ret.append(Int(ptr[i])) } return ret } -func decompress(data:NSData) throws -> NSData { +func decompress(_ data:NSData) throws -> NSData { var stream = z_stream(next_in: nil, avail_in: 0, total_in: 0, next_out: nil, avail_out: 0, total_out: 0, msg: nil, state: nil, zalloc: nil, zfree: nil, opaque: nil, data_type: 0, adler: 0, reserved: 0) let bufsize : Int = 1000 - let buffer = UnsafeMutablePointer<UInt8>.alloc(bufsize) - defer { buffer.dealloc(bufsize) } + let buffer = UnsafeMutablePointer<UInt8>.allocate(capacity: bufsize) + defer { buffer.deallocate(capacity:bufsize) } let output = NSMutableData() stream.next_out = buffer stream.avail_out = UInt32(bufsize) - stream.next_in = UnsafeMutablePointer(data.bytes) + stream.next_in = UnsafeMutablePointer(mutating:data.bytes.assumingMemoryBound(to:Bytef.self)) stream.avail_in = UInt32(data.length) - inflateInit2_(&stream, 16+MAX_WBITS, ZLIB_VERSION, Int32(sizeof(z_stream))) + inflateInit2_(&stream, 16+MAX_WBITS, ZLIB_VERSION, Int32(MemoryLayout<z_stream>.size)) while (true) { let z = inflate(&stream, Z_NO_FLUSH); if (z == Z_OK || z == Z_STREAM_END) { - output.appendBytes(buffer, length: bufsize - Int(stream.avail_out)) + output.append(buffer, length: bufsize - Int(stream.avail_out)) stream.avail_out = UInt32(bufsize) stream.next_out = buffer if (z == Z_STREAM_END) { @@ -87,6 +86,12 @@ func decompress(data:NSData) throws -> NSData { } +extension Dictionary { + func value(forKeyPath s:String) -> Any? { + return (self as NSDictionary).value(forKeyPath:s) + } +} + class Tests: XCTestCase { @@ -100,9 +105,9 @@ class Tests: XCTestCase { super.tearDown() } - func parseBuffer(buffer:NSData) throws -> NSDictionary { + func parseBuffer(_ buffer:NSData) throws -> [AnyHashable:Any] { var error : NSError? - guard let dict = parseKCDataBuffer(UnsafeMutablePointer(buffer.bytes), UInt32(buffer.length), &error) + guard let dict = parseKCDataBuffer(UnsafeMutablePointer(mutating:buffer.bytes.assumingMemoryBound(to:UInt8.self)), UInt32(buffer.length), &error) else { XCTAssert(error != nil) throw error! @@ -110,7 +115,7 @@ class Tests: XCTestCase { return dict } - func testPaddingFlags(pad : Int) { + func testPaddingFlags(_ pad : Int) { let buffer = NSMutableData(capacity:1000)! var item = kcdata_item() @@ -118,22 +123,22 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) item.type = UInt32(KCDATA_TYPE_LIBRARY_LOADINFO) item.flags = UInt64(pad) - item.size = UInt32(sizeof(dyld_uuid_info_32)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout<dyld_uuid_info_32>.size) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) - let uuid = NSUUID(UUIDString: "de305d54-75b4-431b-adb2-eb6b9e546014")! + let uuid = NSUUID(uuidString: "de305d54-75b4-431b-adb2-eb6b9e546014")! var payload = dyld_uuid_info_32(imageLoadAddress: 42, imageUUID: nsuuid2uuid_t(uuid)) - buffer.appendBytes(&payload, length:sizeof(dyld_uuid_info_32)) + buffer.append(&payload, length:MemoryLayout<dyld_uuid_info_32>.size) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) guard let dict = try? self.parseBuffer(buffer) else { XCTFail(); return; } @@ -143,8 +148,8 @@ class Tests: XCTestCase { uuidarray.removeLast() } - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??["imageLoadAddress"] == 42) - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??["imageUUID"] == uuidarray) + XCTAssert(dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info.imageLoadAddress") as? Int == 42) + XCTAssert(dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info.imageUUID") as! [Int] == uuidarray) } func testPaddingFlags() { @@ -152,7 +157,6 @@ class Tests: XCTestCase { testPaddingFlags(i) } } - func testBootArgs() { let s = "hello, I am some boot args" @@ -163,23 +167,22 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) item.type = UInt32(STACKSHOT_KCTYPE_BOOTARGS) item.flags = 0 item.size = UInt32(s.utf8.count + 1) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) - s.nulTerminatedUTF8.withUnsafeBufferPointer({ - buffer.appendBytes($0.baseAddress, length:s.utf8.count + 1) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) + s.utf8CString.withUnsafeBufferPointer({ + buffer.append($0.baseAddress!, length:s.utf8.count + 1) }) - item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) guard let dict = try? self.parseBuffer(buffer) else { XCTFail(); return; } - XCTAssert(dict["kcdata_crashinfo"]?["boot_args"] == s) + XCTAssert(dict.value(forKeyPath:"kcdata_crashinfo.boot_args") as? String == s) } func testBootArgsMissingNul() { @@ -192,20 +195,20 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) item.type = UInt32(STACKSHOT_KCTYPE_BOOTARGS) item.flags = 0 item.size = UInt32(s.utf8.count) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) - s.nulTerminatedUTF8.withUnsafeBufferPointer({ - buffer.appendBytes($0.baseAddress, length:s.utf8.count) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) + s.utf8CString.withUnsafeBufferPointer({ + buffer.append($0.baseAddress!, length:s.utf8.count) }) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) XCTAssert( (try? self.parseBuffer(buffer)) == nil ) } @@ -218,28 +221,28 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) item.type = UInt32(KCDATA_TYPE_LIBRARY_LOADINFO) item.flags = 0 - item.size = UInt32(sizeof(dyld_uuid_info_32)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout<dyld_uuid_info_32>.size) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) - let uuid = NSUUID(UUIDString: "de305d54-75b4-431b-adb2-eb6b9e546014")! + let uuid = NSUUID(uuidString: "de305d54-75b4-431b-adb2-eb6b9e546014")! var payload = dyld_uuid_info_32(imageLoadAddress: 42, imageUUID: nsuuid2uuid_t(uuid)) - buffer.appendBytes(&payload, length:sizeof(dyld_uuid_info_32)) + buffer.append(&payload, length:MemoryLayout<dyld_uuid_info_32>.size) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) guard let dict = try? self.parseBuffer(buffer) else { XCTFail(); return; } - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??["imageLoadAddress"] == 42) - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??["imageUUID"] == nsuuid2array(uuid)) + XCTAssert(dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info.imageLoadAddress") as? Int == 42) + XCTAssert(dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info.imageUUID") as! [Int] == nsuuid2array(uuid)) } func testLoadInfoWrongSize() { @@ -252,29 +255,29 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) item.type = UInt32(KCDATA_TYPE_LIBRARY_LOADINFO) item.flags = 0 - item.size = UInt32(sizeof(dyld_uuid_info_32)) - 1 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout<dyld_uuid_info_32>.size) - 1 + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) - let uuid = NSUUID(UUIDString: "de305d54-75b4-431b-adb2-eb6b9e546014")! + let uuid = NSUUID(uuidString: "de305d54-75b4-431b-adb2-eb6b9e546014")! var payload = dyld_uuid_info_32(imageLoadAddress: 42, imageUUID: nsuuid2uuid_t(uuid)) - buffer.appendBytes(&payload, length:sizeof(dyld_uuid_info_32) - 1) + buffer.append(&payload, length:MemoryLayout<dyld_uuid_info_32>.size - 1) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) guard let dict = try? self.parseBuffer(buffer) else { XCTFail(); return; } - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??["imageLoadAddress"] == 42) + XCTAssert(dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info.imageLoadAddress") as? Int == 42) var uuidarray = nsuuid2array(uuid) uuidarray.removeLast() - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??["imageUUID"] == uuidarray) + XCTAssert(dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info.imageUUID") as! [Int] == uuidarray) } func testLoadInfoWayWrongSize() { @@ -287,27 +290,26 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) item.type = UInt32(KCDATA_TYPE_LIBRARY_LOADINFO) item.flags = 0 - item.size = UInt32(sizeof(dyld_uuid_info_32)) - 16 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout<dyld_uuid_info_32>.size) - 16 + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) - let uuid = NSUUID(UUIDString: "de305d54-75b4-431b-adb2-eb6b9e546014")! + let uuid = NSUUID(uuidString: "de305d54-75b4-431b-adb2-eb6b9e546014")! var payload = dyld_uuid_info_32(imageLoadAddress: 42, imageUUID: nsuuid2uuid_t(uuid)) - buffer.appendBytes(&payload, length:sizeof(dyld_uuid_info_32) - 16) + buffer.append(&payload, length:MemoryLayout<dyld_uuid_info_32>.size - 16) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) - + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) guard let dict = try? self.parseBuffer(buffer) else { XCTFail(); return; } - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??["imageLoadAddress"] == 42) - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??["imageUUID"] == nil) + XCTAssert(dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info.imageLoadAddress") as? Int == 42) + XCTAssert(dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info.imageUUID") == nil) } func testLoadInfoPreposterousWrongSize() { @@ -320,25 +322,25 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) item.type = UInt32(KCDATA_TYPE_LIBRARY_LOADINFO) item.flags = 0 item.size = UInt32(1) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) var payload = UInt8(42) - buffer.appendBytes(&payload, length:1) + buffer.append(&payload, length:1) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) guard let dict = try? self.parseBuffer(buffer) else { XCTFail(); return; } - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??["imageLoadAddress"] == nil) - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??["imageUUID"] == nil) + XCTAssert(dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info.imageLoadAddress") == nil) + XCTAssert(dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info.imageUUID") == nil) } @@ -349,43 +351,43 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) item.type = UInt32(KCDATA_TYPE_ARRAY_PAD0) + UInt32(pad) item.flags = UInt64(STACKSHOT_KCTYPE_DONATING_PIDS) << 32 | UInt64(n) - item.size = UInt32(n * sizeof(UInt32) + pad) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(n * MemoryLayout<UInt32>.size + pad) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) for i in 0..<n { var payload = UInt32(42 * i) - buffer.appendBytes(&payload, length:sizeof(UInt32)) + buffer.append(&payload, length:MemoryLayout<UInt32>.size) } for i in 0..<pad { var payload = UInt8(42-i) - buffer.appendBytes(&payload, length:sizeof(UInt8)) + buffer.append(&payload, length:MemoryLayout<UInt8>.size) } item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) guard let dict = try? self.parseBuffer(buffer) else { XCTFail(); return; } - XCTAssert(dict["kcdata_crashinfo"]?["donating_pids"]??.count == n) + XCTAssert((dict.value(forKeyPath:"kcdata_crashinfo.donating_pids") as! [Any]).count == n) for i in 0..<n { let x = dict["kcdata_crashinfo"] as? NSDictionary let y = x?["donating_pids"] as? NSArray - XCTAssert((y?[i]) as? NSObject == 42 * i) + XCTAssert((y?[i]) as? Int == 42 * i) } } func testNewArrays() { - self.testNewArray(0,pad:0) + self.testNewArray(n:0,pad:0) for i in 1..<20 { for pad in 0..<16 { - self.testNewArray(i, pad:pad) + self.testNewArray(n:i, pad:pad) } } } @@ -398,39 +400,43 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) item.type = UInt32(KCDATA_TYPE_ARRAY_PAD0) item.flags = UInt64(KCDATA_TYPE_LIBRARY_LOADINFO) << 32 | UInt64(n) - item.size = UInt32(n * sizeof(dyld_uuid_info_32)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(n * MemoryLayout<dyld_uuid_info_32>.size) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) - let uuid = NSUUID(UUIDString: "de305d54-75b4-431b-adb2-eb6b9e546014")! + let uuid = NSUUID(uuidString: "de305d54-75b4-431b-adb2-eb6b9e546014")! for i in 0..<n { var payload = dyld_uuid_info_32(imageLoadAddress:UInt32(i+42), imageUUID: nsuuid2uuid_t(uuid)) - buffer.appendBytes(&payload, length:sizeof(dyld_uuid_info_32)) + buffer.append(&payload, length:MemoryLayout<dyld_uuid_info_32>.size) } item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) guard let dict = try? self.parseBuffer(buffer) else { XCTFail(); return; } - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??.count == n) + XCTAssert((dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info") as! [Any]).count == n) for i in 0..<n { - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??[i]?["imageLoadAddress"] == 42+i) - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??[i]?["imageUUID"] == nsuuid2array(uuid)) + guard let loadinfo = dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info") as? [Any] + else { XCTFail(); return; } + guard let loadinfo_i = loadinfo[i] as? [AnyHashable:Any] + else { XCTFail(); return; } + XCTAssert(loadinfo_i["imageLoadAddress"] as? Int == 42 + i) + XCTAssert(loadinfo_i["imageUUID"] as! [Int] == nsuuid2array(uuid)) } } func testArrayLoadInfo() { for n in 0..<20 { - testArrayLoadInfo(n) + testArrayLoadInfo(n: n) } } @@ -445,35 +451,39 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) item.type = UInt32(KCDATA_TYPE_ARRAY_PAD0) item.flags = UInt64(KCDATA_TYPE_LIBRARY_LOADINFO) << 32 | UInt64(n) - item.size = UInt32(n * (sizeof(dyld_uuid_info_32) - wrong)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(n * (MemoryLayout<dyld_uuid_info_32>.size - wrong)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) - let uuid = NSUUID(UUIDString: "de305d54-75b4-431b-adb2-eb6b9e546014")! + let uuid = NSUUID(uuidString: "de305d54-75b4-431b-adb2-eb6b9e546014")! for i in 0..<n { var payload = dyld_uuid_info_32(imageLoadAddress:UInt32(i+42), imageUUID: nsuuid2uuid_t(uuid)) - buffer.appendBytes(&payload, length:sizeof(dyld_uuid_info_32)-wrong) + buffer.append(&payload, length:MemoryLayout<dyld_uuid_info_32>.size-wrong) } item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) - + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) var uuidarray = nsuuid2array(uuid) uuidarray.removeLast() guard let dict = try? self.parseBuffer(buffer) - else { XCTFail(); return; } - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??.count == n) + else { XCTFail(); return; } + XCTAssert((dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info") as! [Any]).count == n) for i in 0..<n { - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??[i]?["imageLoadAddress"] == 42+i) - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??[i]?["imageUUID"] == uuidarray) + guard let loadinfo = dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info") as? [Any] + else { XCTFail(); return; } + guard let loadinfo_i = loadinfo[i] as? [AnyHashable:Any] + else { XCTFail(); return; } + XCTAssert(loadinfo_i["imageLoadAddress"] as? Int == 42 + i) + XCTAssert(loadinfo_i["imageUUID"] as! [Int] == uuidarray) } + } @@ -488,31 +498,36 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) item.type = UInt32(KCDATA_TYPE_ARRAY_PAD0) item.flags = UInt64(KCDATA_TYPE_LIBRARY_LOADINFO) << 32 | UInt64(n) - item.size = UInt32(n * (sizeof(dyld_uuid_info_32) - wrong)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(n * (MemoryLayout<dyld_uuid_info_32>.size - wrong)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) - let uuid = NSUUID(UUIDString: "de305d54-75b4-431b-adb2-eb6b9e546014")! + let uuid = NSUUID(uuidString: "de305d54-75b4-431b-adb2-eb6b9e546014")! for i in 0..<n { var payload = dyld_uuid_info_32(imageLoadAddress:UInt32(i+42), imageUUID: nsuuid2uuid_t(uuid)) - buffer.appendBytes(&payload, length:sizeof(dyld_uuid_info_32)-wrong) + buffer.append(&payload, length:MemoryLayout<dyld_uuid_info_32>.size-wrong) } item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) + guard let dict = try? self.parseBuffer(buffer) - else { XCTFail(); return; } - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??.count == n) + else { XCTFail(); return; } + XCTAssert((dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info") as! [Any]).count == n) for i in 0..<n { - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??[i]?["imageLoadAddress"] == 42+i) - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??[i]?["imageUUID"] == nil) + guard let loadinfo = dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info") as? [Any] + else { XCTFail(); return; } + guard let loadinfo_i = loadinfo[i] as? [AnyHashable:Any] + else { XCTFail(); return; } + XCTAssert(loadinfo_i["imageLoadAddress"] as? Int == 42 + i) + XCTAssert(loadinfo_i["imageUUID"] == nil) } } @@ -527,29 +542,34 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) item.type = UInt32(KCDATA_TYPE_ARRAY_PAD0) item.flags = UInt64(KCDATA_TYPE_LIBRARY_LOADINFO) << 32 | UInt64(n) - item.size = UInt32(n * (sizeof(dyld_uuid_info_32) - wrong)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(n * (MemoryLayout<dyld_uuid_info_32>.size - wrong)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) for i in 0..<n { var payload = UInt8(42*i) - buffer.appendBytes(&payload, length:1) + buffer.append(&payload, length:1) } item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) + guard let dict = try? self.parseBuffer(buffer) - else { XCTFail(); return; } - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??.count == n) + else { XCTFail(); return; } + XCTAssert((dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info") as! [Any]).count == n) for i in 0..<n { - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??[i]?["imageLoadAddress"] == nil) - XCTAssert(dict["kcdata_crashinfo"]?["dyld_load_info"]??[i]?["imageUUID"] == nil) + guard let loadinfo = dict.value(forKeyPath:"kcdata_crashinfo.dyld_load_info") as? [Any] + else { XCTFail(); return; } + guard let loadinfo_i = loadinfo[i] as? [AnyHashable:Any] + else { XCTFail(); return; } + XCTAssert(loadinfo_i["imageLoadAddress"] == nil) + XCTAssert(loadinfo_i["imageUUID"] == nil) } } @@ -562,43 +582,43 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) item.type = UInt32(TASK_CRASHINFO_CRASHED_THREADID) item.flags = 0 - item.size = UInt32(sizeof(UInt64)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout<UInt64>.size) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) var payload : UInt64 = 42 - buffer.appendBytes(&payload, length:sizeof(UInt64)) + buffer.append(&payload, length:MemoryLayout<UInt64>.size) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) let buffer2 = NSMutableData(capacity:1000)! item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer2.appendBytes(&item, length: sizeof(kcdata_item)) + buffer2.append(&item, length: MemoryLayout<kcdata_item>.size) item.type = UInt32(KCDATA_TYPE_NESTED_KCDATA) item.flags = 0 item.size = UInt32(buffer.length) - buffer2.appendBytes(&item, length: sizeof(kcdata_item)) - buffer2.appendData(buffer) + buffer2.append(&item, length: MemoryLayout<kcdata_item>.size) + buffer2.append(buffer as Data) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer2.appendBytes(&item, length: sizeof(kcdata_item)) + buffer2.append(&item, length: MemoryLayout<kcdata_item>.size) guard let dict2 = try? self.parseBuffer(buffer2) else { XCTFail(); return; } - XCTAssert(dict2["kcdata_crashinfo"]?["kcdata_crashinfo"]??["crashed_threadid"] == 42) + XCTAssert(dict2.value(forKeyPath:"kcdata_crashinfo.kcdata_crashinfo.crashed_threadid") as? Int == 42) } @@ -610,27 +630,27 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) item.type = UInt32(TASK_CRASHINFO_CRASHED_THREADID) item.flags = 0 - item.size = UInt32(sizeof(UInt64)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout<UInt64>.size) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) var payload : UInt64 = 42 - buffer.appendBytes(&payload, length:sizeof(UInt64)) + buffer.append(&payload, length:MemoryLayout<UInt64>.size) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) guard let dict = try? self.parseBuffer(buffer) else { XCTFail(); return; } - XCTAssert(dict["kcdata_crashinfo"]!["crashed_threadid"] == 42) + XCTAssert(dict.value(forKeyPath:"kcdata_crashinfo.crashed_threadid") as? Int == 42) } - + func testRepeatedKey() { // test a repeated item of the same key causes error @@ -642,28 +662,28 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) item.type = UInt32(TASK_CRASHINFO_CRASHED_THREADID) item.flags = 0 - item.size = UInt32(sizeof(UInt64)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout<UInt64>.size) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) var payload : UInt64 = 42 - buffer.appendBytes(&payload, length:sizeof(UInt64)) + buffer.append(&payload, length:MemoryLayout<UInt64>.size) item.type = UInt32(TASK_CRASHINFO_CRASHED_THREADID) item.flags = 0 - item.size = UInt32(sizeof(UInt64)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout<UInt64>.size) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) payload = 42 - buffer.appendBytes(&payload, length:sizeof(UInt64)) + buffer.append(&payload, length:MemoryLayout<UInt64>.size) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) XCTAssert( (try? self.parseBuffer(buffer)) == nil ) } @@ -679,40 +699,39 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) item.type = UInt32(KCDATA_TYPE_CONTAINER_BEGIN) item.flags = 0 - item.size = UInt32(sizeof(UInt32)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout<UInt32>.size) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) payload32 = UInt32(STACKSHOT_KCCONTAINER_TASK) - buffer.appendBytes(&payload32, length:sizeof(UInt32)) + buffer.append(&payload32, length:MemoryLayout<UInt32>.size) item.type = UInt32(TASK_CRASHINFO_CRASHED_THREADID) item.flags = 0 - item.size = UInt32(sizeof(UInt64)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout<UInt64>.size) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) payload64 = 42 - buffer.appendBytes(&payload64, length:sizeof(UInt64)) + buffer.append(&payload64, length:MemoryLayout<UInt64>.size) item.type = UInt32(KCDATA_TYPE_CONTAINER_END) item.flags = 0 - item.size = UInt32(sizeof(UInt32)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout<UInt32>.size) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) payload32 = UInt32(STACKSHOT_KCCONTAINER_TASK) - buffer.appendBytes(&payload32, length:sizeof(UInt32)) + buffer.append(&payload32, length:MemoryLayout<UInt32>.size) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) guard let dict = try? self.parseBuffer(buffer) - else { XCTFail(); return; } - - XCTAssert(dict["kcdata_crashinfo"]?["task_snapshots"]??["0"]??["crashed_threadid"] == 42) + else { XCTFail(); return; } + XCTAssert(dict.value(forKeyPath: "kcdata_crashinfo.task_snapshots.0.crashed_threadid") as? Int == 42) } func testRepeatedContainer() { @@ -727,55 +746,55 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) item.type = UInt32(KCDATA_TYPE_CONTAINER_BEGIN) item.flags = 0 - item.size = UInt32(sizeof(UInt32)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout<UInt32>.size) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) payload32 = UInt32(STACKSHOT_KCCONTAINER_TASK) - buffer.appendBytes(&payload32, length:sizeof(UInt32)) + buffer.append(&payload32, length:MemoryLayout<UInt32>.size) item.type = UInt32(TASK_CRASHINFO_CRASHED_THREADID) item.flags = 0 - item.size = UInt32(sizeof(UInt64)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout<UInt64>.size) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) payload64 = 42 - buffer.appendBytes(&payload64, length:sizeof(UInt64)) + buffer.append(&payload64, length:MemoryLayout<UInt64>.size) item.type = UInt32(KCDATA_TYPE_CONTAINER_END) item.flags = 0 - item.size = UInt32(sizeof(UInt32)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout<UInt32>.size) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) payload32 = UInt32(STACKSHOT_KCCONTAINER_TASK) - buffer.appendBytes(&payload32, length:sizeof(UInt32)) + buffer.append(&payload32, length:MemoryLayout<UInt32>.size) item.type = UInt32(KCDATA_TYPE_CONTAINER_BEGIN) item.flags = 0 - item.size = UInt32(sizeof(UInt32)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout<UInt32>.size) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) payload32 = UInt32(STACKSHOT_KCCONTAINER_TASK) - buffer.appendBytes(&payload32, length:sizeof(UInt32)) + buffer.append(&payload32, length:MemoryLayout<UInt32>.size) item.type = UInt32(TASK_CRASHINFO_CRASHED_THREADID) item.flags = 0 - item.size = UInt32(sizeof(UInt64)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout<UInt64>.size) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) payload64 = 42 - buffer.appendBytes(&payload64, length:sizeof(UInt64)) + buffer.append(&payload64, length:MemoryLayout<UInt64>.size) item.type = UInt32(KCDATA_TYPE_CONTAINER_END) item.flags = 0 - item.size = UInt32(sizeof(UInt32)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout<UInt32>.size) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) payload32 = UInt32(STACKSHOT_KCCONTAINER_TASK) - buffer.appendBytes(&payload32, length:sizeof(UInt32)) + buffer.append(&payload32, length:MemoryLayout<UInt32>.size) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) XCTAssert( (try? self.parseBuffer(buffer)) == nil ) } @@ -791,26 +810,26 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) item.type = UInt32(KCDATA_TYPE_CONTAINER_BEGIN) item.flags = 0 - item.size = UInt32(sizeof(UInt32)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout<UInt32>.size) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) payload32 = UInt32(STACKSHOT_KCCONTAINER_TASK) - buffer.appendBytes(&payload32, length:sizeof(UInt32)) + buffer.append(&payload32, length:MemoryLayout<UInt32>.size) item.type = UInt32(TASK_CRASHINFO_CRASHED_THREADID) item.flags = 0 - item.size = UInt32(sizeof(UInt64)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout<UInt64>.size) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) payload64 = 42 - buffer.appendBytes(&payload64, length:sizeof(UInt64)) + buffer.append(&payload64, length:MemoryLayout<UInt64>.size) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) XCTAssert( (try? self.parseBuffer(buffer)) == nil ) } @@ -825,21 +844,21 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) item.type = UInt32(KCDATA_TYPE_CONTAINER_BEGIN) item.flags = 0 - item.size = UInt32(sizeof(UInt32)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout<UInt32>.size) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) payload32 = UInt32(STACKSHOT_KCCONTAINER_TASK) - buffer.appendBytes(&payload32, length:sizeof(UInt32)) + buffer.append(&payload32, length:MemoryLayout<UInt32>.size) item.type = UInt32(TASK_CRASHINFO_CRASHED_THREADID) item.flags = 0 - item.size = UInt32(sizeof(UInt64)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout<UInt64>.size) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) payload64 = 42 - buffer.appendBytes(&payload64, length:sizeof(UInt64)) + buffer.append(&payload64, length:MemoryLayout<UInt64>.size) XCTAssert( (try? self.parseBuffer(buffer)) == nil ) } @@ -854,15 +873,15 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) item.type = UInt32(TASK_CRASHINFO_CRASHED_THREADID) item.flags = 0 - item.size = UInt32(sizeof(UInt64)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(MemoryLayout<UInt64>.size) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) var payload : UInt64 = 42 - buffer.appendBytes(&payload, length:sizeof(UInt64)) + buffer.append(&payload, length:MemoryLayout<UInt64>.size) XCTAssert( (try? self.parseBuffer(buffer)) == nil ) } @@ -876,20 +895,20 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) item.type = UInt32(TASK_CRASHINFO_CRASHED_THREADID) item.flags = 0 item.size = 99999 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) var payload : UInt64 = 42 - buffer.appendBytes(&payload, length:sizeof(UInt64)) + buffer.append(&payload, length:MemoryLayout<UInt64>.size) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) XCTAssert( (try? self.parseBuffer(buffer)) == nil ) } @@ -905,41 +924,41 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) item.type = UInt32(KCDATA_TYPE_ARRAY) item.flags = UInt64(TASK_CRASHINFO_CRASHED_THREADID) << 32 | UInt64(n) - item.size = UInt32(n * sizeof(UInt64)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(n * MemoryLayout<UInt64>.size) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) for i in 0..<n { var payload : UInt64 = UInt64(i) - buffer.appendBytes(&payload, length:sizeof(UInt64)) + buffer.append(&payload, length:MemoryLayout<UInt64>.size) } item.type = UInt32(KCDATA_TYPE_ARRAY) item.flags = UInt64(TASK_CRASHINFO_CRASHED_THREADID) << 32 | UInt64(n) - item.size = UInt32(n * sizeof(UInt64)) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(n * MemoryLayout<UInt64>.size) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) for i in 0..<n { var payload : UInt64 = UInt64(i) - buffer.appendBytes(&payload, length:sizeof(UInt64)) + buffer.append(&payload, length:MemoryLayout<UInt64>.size) } item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) guard let dict = try? self.parseBuffer(buffer) else { XCTFail(); return } - XCTAssert( 2*n == dict["kcdata_crashinfo"]!["crashed_threadid"]!!.count) + XCTAssert( 2*n == (dict.value(forKeyPath:"kcdata_crashinfo.crashed_threadid") as! [Any]).count) for i in 0..<2*n { let x = dict["kcdata_crashinfo"] as? NSDictionary let y = x?["crashed_threadid"] as? NSArray - XCTAssert((y?[i]) as? NSObject == i % n) + XCTAssert((y?[i]) as? Int == i % n) } } @@ -951,46 +970,47 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) - + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) + item.type = UInt32(KCDATA_TYPE_ARRAY) item.flags = UInt64(TASK_CRASHINFO_CRASHED_THREADID) << 32 | UInt64(n) - item.size = UInt32(n * sizeof(UInt64) + pad) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(n * MemoryLayout<UInt64>.size + pad) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) for i in 0..<n { var payload : UInt64 = UInt64(i) - buffer.appendBytes(&payload, length:sizeof(UInt64)) + buffer.append(&payload, length:MemoryLayout<UInt64>.size) } for _ in 0..<pad { var payload : UInt8 = 0 - buffer.appendBytes(&payload, length:1) + buffer.append(&payload, length:1) } item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) guard let dict = try? self.parseBuffer(buffer) else { XCTFail(); return; } - - XCTAssert( n == dict["kcdata_crashinfo"]?["crashed_threadid"]??.count) + + XCTAssert( n == (dict.value(forKeyPath:"kcdata_crashinfo.crashed_threadid") as! [Any]).count) + for i in 0..<n { let x = dict["kcdata_crashinfo"] as? NSDictionary let y = x?["crashed_threadid"] as? NSArray - XCTAssert((y?[i]) as? NSObject == i) + XCTAssert((y?[i]) as? Int == i) } } - + func testReadThreadidArray() { // test that we can correctly read old arrays with a variety of sizes and paddings - self.testReadThreadidArray(0, pad:0) + self.testReadThreadidArray(n: 0, pad:0) for n in 1..<100 { for pad in 0..<16 { - self.testReadThreadidArray(n, pad:pad) + self.testReadThreadidArray(n: n, pad:pad) } } } @@ -999,63 +1019,63 @@ class Tests: XCTestCase { /// for old style arrays, if the element size is determined by the type. If the array of that size element at the given count doesn't fit, then parsing should fail let n = 1 - + let buffer = NSMutableData(capacity:1000)! - + var item = kcdata_item() - + item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) - + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) + item.type = UInt32(KCDATA_TYPE_ARRAY) item.flags = UInt64(TASK_CRASHINFO_CRASHED_THREADID) << 32 | UInt64(n) item.size = UInt32(4) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) - + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) + var payload : UInt32 = UInt32(42) - buffer.appendBytes(&payload, length:sizeof(UInt32)) - + buffer.append(&payload, length:MemoryLayout<UInt32>.size) + item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) XCTAssert( (try? self.parseBuffer(buffer)) == nil ) } - + func testReadThreadidArrayWrongSize5() { /// if the count is bigger than the buffer, parsing will just fail - + let n = 5 - + let buffer = NSMutableData(capacity:1000)! - + var item = kcdata_item() - + item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) - + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) + item.type = UInt32(KCDATA_TYPE_ARRAY) item.flags = UInt64(TASK_CRASHINFO_CRASHED_THREADID) << 32 | UInt64(n) item.size = UInt32(4) - buffer.appendBytes(&item, length: sizeof(kcdata_item)) - + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) + var payload : UInt32 = UInt32(42) - buffer.appendBytes(&payload, length:sizeof(UInt32)) - + buffer.append(&payload, length:MemoryLayout<UInt32>.size) + item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) - + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) + XCTAssert( (try? self.parseBuffer(buffer)) == nil ) } - + func testReadThreadidArrayPaddedSize() { // test that we can tolerate a little padding at the end of an array let n = 5 @@ -1067,33 +1087,33 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) item.type = UInt32(KCDATA_TYPE_ARRAY) item.flags = UInt64(TASK_CRASHINFO_CRASHED_THREADID) << 32 | UInt64(n) - item.size = UInt32(n * sizeof(UInt64)) + 1 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(n * MemoryLayout<UInt64>.size) + 1 + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) for i in 0..<n { var payload : UInt64 = UInt64(i) - buffer.appendBytes(&payload, length:sizeof(UInt64)) + buffer.append(&payload, length:MemoryLayout<UInt64>.size) } var payload : UInt8 = 0 - buffer.appendBytes(&payload, length:1) + buffer.append(&payload, length:1) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) guard let dict = try? self.parseBuffer(buffer) else { XCTFail(); return; } - XCTAssert( n == dict["kcdata_crashinfo"]?["crashed_threadid"]??.count) + XCTAssert( n == (dict.value(forKeyPath:"kcdata_crashinfo.crashed_threadid") as! [Any]).count) for i in 0..<n { let x = dict["kcdata_crashinfo"] as? NSDictionary let y = x?["crashed_threadid"] as? NSArray - XCTAssert((y?[i]) as? NSObject == i) + XCTAssert((y?[i]) as? Int == i) } } @@ -1108,36 +1128,35 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) item.type = UInt32(KCDATA_TYPE_ARRAY) item.flags = UInt64(TASK_CRASHINFO_CRASHED_THREADID) << 32 | UInt64(n) - item.size = UInt32(n * sizeof(UInt64)) + 15 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + item.size = UInt32(n * MemoryLayout<UInt64>.size) + 15 + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) for i in 0..<n { var payload : UInt64 = UInt64(i) - buffer.appendBytes(&payload, length:sizeof(UInt64)) + buffer.append(&payload, length:MemoryLayout<UInt64>.size) } - for i in 0..<15 { - i; + for _ in 0..<15 { var payload : UInt8 = 0 - buffer.appendBytes(&payload, length:1) + buffer.append(&payload, length:1) } item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) guard let dict = try? self.parseBuffer(buffer) else { XCTFail(); return; } - XCTAssert( n == dict["kcdata_crashinfo"]?["crashed_threadid"]??.count) + XCTAssert( n == (dict.value(forKeyPath:"kcdata_crashinfo.crashed_threadid") as! [Any]).count) for i in 0..<n { let x = dict["kcdata_crashinfo"] as? NSDictionary let y = x?["crashed_threadid"] as? NSArray - XCTAssert((y?[i]) as? NSObject == i) + XCTAssert((y?[i]) as? Int == i) } } @@ -1150,59 +1169,59 @@ class Tests: XCTestCase { item.type = KCDATA_BUFFER_BEGIN_CRASHINFO item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) item.type = UInt32(TASK_CRASHINFO_CRASHED_THREADID) item.flags = 0 item.size = size - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) var payload : UInt64 = 42 - buffer.appendBytes(&payload, length:Int(size)) + buffer.append(&payload, length:Int(size)) item.type = KCDATA_TYPE_BUFFER_END item.flags = 0 item.size = 0 - buffer.appendBytes(&item, length: sizeof(kcdata_item)) + buffer.append(&item, length: MemoryLayout<kcdata_item>.size) guard let dict = try? self.parseBuffer(buffer) else { XCTFail(); return; } - XCTAssert(dict["kcdata_crashinfo"]?["crashed_threadid"] == nil) + XCTAssert(dict.value(forKeyPath:"kcdata_crashinfo.crashed_threadid") == nil) } func testReadThreadidWrongSize0() { - self.testReadThreadidWrongSize(0) + self.testReadThreadidWrongSize(size: 0) } func testReadThreadidWrongSize7() { - self.testReadThreadidWrongSize(7) + self.testReadThreadidWrongSize(size: 7) } - func dataWithResource(name:String) -> NSData? { - guard let filename = NSBundle(forClass: self.classForCoder).pathForResource(name, ofType: nil) + func dataWithResource(_ name:String) -> NSData? { + guard let filename = Bundle(for: self.classForCoder).path(forResource: name, ofType: nil) else { return nil } return NSData(contentsOfFile:filename)! } - - func testSampleStackshot(name : String) { + + func testSampleStackshot(_ name : String) { // check that we agree with sample file guard let sampledata = self.dataWithResource(name) else { XCTFail(); return } var dict : NSDictionary? - dict = try? self.parseBuffer(sampledata) + dict = try? self.parseBuffer(sampledata) as NSDictionary if (dict == nil) { - if let decoded = NSData(base64EncodedData: sampledata, options:.IgnoreUnknownCharacters) { - dict = try? self.parseBuffer(decoded) + if let decoded = NSData(base64Encoded: sampledata as Data, options:.ignoreUnknownCharacters) { + dict = try? self.parseBuffer(decoded) as NSDictionary } } if (dict == nil) { if let decompressed = try? decompress(sampledata) { - dict = try? self.parseBuffer(decompressed) + dict = try? self.parseBuffer(decompressed) as NSDictionary } } @@ -1214,9 +1233,9 @@ class Tests: XCTestCase { self.dataWithResource(name + ".plist") else {XCTFail(); return} - var dict2 = try? NSPropertyListSerialization.propertyListWithData(plistdata, options: NSPropertyListReadOptions.Immutable, format: nil) + var dict2 = try? PropertyListSerialization.propertyList(from: plistdata as Data, options: [], format: nil) if dict2 == nil { - dict2 = try? NSPropertyListSerialization.propertyListWithData(decompress(plistdata), options: .Immutable, format: nil) + dict2 = try? PropertyListSerialization.propertyList(from:decompress(plistdata) as Data, options:[], format: nil) } XCTAssert(dict2 != nil) @@ -1227,19 +1246,19 @@ class Tests: XCTestCase { #if os(OSX) - let kcdatapy = NSBundle(forClass: self.classForCoder).pathForResource("kcdata.py", ofType: nil) + let kcdatapy = Bundle(for: self.classForCoder).path(forResource: "kcdata.py", ofType: nil) - let task = NSTask() + let task = Process() task.launchPath = kcdatapy task.arguments = ["-p", - NSBundle(forClass:self.classForCoder).pathForResource(name, ofType: nil)!] - let pipe = NSPipe() + Bundle(for:self.classForCoder).path(forResource: name, ofType: nil)!] + let pipe = Pipe() task.standardOutput = pipe task.launch() let data = pipe.fileHandleForReading.readDataToEndOfFile() - guard let dict3 = try? NSPropertyListSerialization.propertyListWithData(data, options: .Immutable, format: nil) as? NSDictionary + guard let dict3 = try? PropertyListSerialization.propertyList(from:data, options:[], format: nil) as? NSDictionary else { XCTFail(); return } XCTAssert(dict == dict3) @@ -1282,7 +1301,7 @@ class Tests: XCTestCase { func testSampleExitReason() { self.testSampleStackshot("exitreason-sample") } - + func testSampleThreadT() { self.testSampleStackshot("stackshot-sample-ths-thread-t") } @@ -1315,6 +1334,14 @@ class Tests: XCTestCase { self.testSampleStackshot("exitreason-codesigning") } + func testSampleThreadGroups() { + self.testSampleStackshot("stackshot-sample-thread-groups") + } + + func testSampleCoalitions() { + self.testSampleStackshot("stackshot-sample-coalitions") + } + func testStackshotSharedcacheV2() { self.testSampleStackshot("stackshot-sample-sharedcachev2") } @@ -1335,6 +1362,14 @@ class Tests: XCTestCase { self.testSampleStackshot("stackshot-with-waitinfo") } + func testStackshotWithThreadPolicy() { + self.testSampleStackshot("stackshot-sample-thread-policy") + } + + func testStackshotWithInstrsCycles() { + self.testSampleStackshot("stackshot-sample-instrs-cycles") + } + func testTrivial() { } } diff --git a/libkdd/tests/stackshot-sample-coalitions b/libkdd/tests/stackshot-sample-coalitions new file mode 100644 index 0000000000000000000000000000000000000000..3f837018bd1341c006d37507a784ce7e75c1b8fb GIT binary patch literal 14448 zcmb_j50n(enJ@k=e*|0+0SW4eh|0mT3kxj&beCO_l?7H<{uy=YndzBrccy#n?%8G4 z#EAkT2Lx6`jEE}|K%OFjL^Nl3+@0RL^F@=3#)qDd#~C$wzDV%iJzg&HF85XSS3NVo zg+MM>@AXvwe!s7)s;j>Gs=9l(@5()^1f`t(q12O(aym9iK*{rsSgY6K_|2BxoJbi- zFW~iWuSk!QKT+yhKdwvWfA;!}71!74{AxOp*Ztdxt)(*WMOfCqfepM4&o|S})C}J| zU;VN^nSNEjWUIXXivB{B64vdPn&f{y<W5z7DeBdM{>3k@Qu^hjXXu)Z@<K{=hTCr0 zkp-q}o2kfh)AcOJj#Nx5D=#ab5oyYrk(!JfiBv`^DrZcooH1#7q_M6kGO4^`+RTWT zGBc)c`7_qoS+mTE-Z!ys?((?}iyIftUDVWEvt;4a%I2o3@`@?c!awv|tLh(2^*2(z z>s9SS#LZ|nIlFvKMP+3q<~5sk!f|6}+O+-VxaAqqlsUU1G9hbwM#5}vRjq9H%{6|+ zGhNF_&7K@-kNdMHR7Avpt+L2Tbd=Je`}bRPF8jQH>AOmw4Ej5deYl4xG4R88<Yi(C z9fjAwy7_SG-iAJ+^yzI%??>&AkWG$`OGy!FG)#!Yltt)T*%q1l1zES>-rbmDqNEs# zK7g@Yv>yQWeE|;yjO7gmoo=ZpsZJ_ju)ZM$nmoM>@KC_m9_q@XG^hv(F#<5QcO>YS z10Dq!@8??3uK*kYjQ2Yl^eX|60gU}I7WAtC-wYW0YaHmq0gnfa{aFq=_NSbGh3OlG zfX*ucV}DNt{TjekfN?x#fPNj|S%4#eZv}ik;A+4)o^wFI0dO5)9PfFc-w1d<VDyJO zK)(ra17P%*MWEjTxDhb=(-P4C25=K#^ta`pPXN3Ua2epc08a#bH(>PVdqJ-Nyc#h2 zdlYoJ-V~;IA)s@d4++radQ_NN3IUzte7O(wX@G6O(*b7y<9u>Kp9$CpjPtDx^x1&d z0LJ<F0O+>?eh6?i;BN!20h|Mj^Z60bYXPqXjPrdx=yL&Y0F3cqBj~pS-UJxq#TL-( z0e1pk0C+oKj3+xlUkG?7V2n541AQ^zU4Stj?FRi$z+HebUhM&WDd4?;F`n%MeHq{b zfHB@31bqeImjGitJOsLW_$lBH7o5vejF(41|0duc0>*gyD(LqBeho0j+aH784EPPe zs{tPeYydt17~}OnfF1+<Cg3>We*$a*{%61mz;6Ri0{$uB7QpWUwgCSeFs>KB0KFCP z`+!q`KLDHtd=jt?_#?m$;Ew_0dUXo)`vIQ@>;nD-um|{4z&_y50cQdK25=kTF95d# z{w?4&fX@K#0DKnk1AzYn@PmN=6YxWT&jFUt--YS-g@De#4fqd$9|rt2;2hxp0sI}n zqDL|DNHG*WL4OqV-k`4qy#(}ip!WlPJ?H~Ke+=}2pl<+uFzAniUJCj~(1(Ejx1bLN zeG}-zK;I1d2++5HJ`(irf<6lLt)O2E`Zmxbpm%~k8uTYX9|QV!(8q%QB<MGTz611e zpg#rrc+hu(UIzNppqGRGJ<unC{tW1qpzi{GD(KIGUIqGY&}V@D9O$z^?*jc+(4Pmr z8uUG&&jI}f(Ca|o3;I0JzYqF+(D#9U2k84jZvg!O=!-yq5%fmT4}#tV`b(fM2mNKx zSAu>B^t(WR1@yZ?KMeZ4p#K1L6Z9jXCqX|7dK&asL3cnu2KxP=zXtlvgnwQtgsk`4 z{&_L@4v@#Xm?(XoNV-ne=8a}gHZkHx#<$wS?Pah0rA24Gi%=f);Zv$N<s`Xq;!2ob zM(@|>w$W?&nk4-Q(U<vlD`CyE+;qF)nhlN-zQ<0Ram%1<+Dum!rf)6bnX(j5*D!#~ zF+AU;_xo--<2q3@9=-=QJkO$!A-2yK3}pL+MIRenr_G876%1lN=D4Pj^;>lLgPHf* zEk72rLR_wS(@&?vUJKs!ONqMJw`^(%rz<#5Q#(x~9r8moFYnhi(~cipbEW33@KzNw zlzG{2ZDz`eS$>Bs>?-D4Oe2+Vz4}z3l$c{Dtb%bF&h{CnoqISSMlf&q#ZM|DnV0p3 zjVJCK%%{nv{AN4hF&@QvtJx7X3hKRv`64GF?po#(M%qes*p5&2#0+0t$9!EjooO^| zOSdY*yl18gn<K8*yp!;yPh~T4`VtlwqnWpC@`hCYnj4rmVlmV699MNFabuWIWQ&Ry zH|lcY<b~At(nYP9DaPt@sMc1hm7%zaPkbjEYw=8<+M^gbZ)KtpY>hkZcD&58ebW`U z1hhz+zF)Z49>;u5CX+Jj>uShF<xCR#eJJ_4nV%IxjA#4ySaJI&FmJ?tmqLMM^VpOz zzck|#H^(u8X(uLXd&~8+wi!BOxh~&w%@p+<&80Q6rV8e5({Fd&R&wg3?RdWWjwU4; zf1<K<Zq{VBH@w!2>8353u8xdNmCXAVMFKvT*%UtaS`69HP@2ko)bLvPRAti`P(K@f zP_$yEGjFAhARsZG!8ykPC(T#Q40+o(Xkxo*!=@|6EX`AUQdYjHvzeFU;uchVEAzgU z3Hwq~w=o|}IoWu|u(<n0HS=jFN)fA_re#aqNeWTJe8y*oVmycQ1Qn4<86EMm`IE}x zrl@7!NG4q~siuTt>X={LR8#9x-_<%9i`-DmWj>nKBNY(yn9sOzgC;5?A#Ue0+h_~N zL{jsaPy1=pP0Em)Q6Zw9`DiL@`i>KZEd74a@=f}X?Z(T38y5|1U+bjPwKVl)8)aG; zWI{y@J$Z>+r1?bg1X#>`ERk`|M1BYwnV0i1>d*qk=}yk+<FQY5gaf>U`Bc6L#Vlps zXfN)!Cg!F8>lh%ZWz5@#O)F25Y6(Kna^^FwMcpY@FfaWxPF<(fmAV|W#Z1vmVklN= zzM%Y+kqzI|ku#^fjHR~&;+t%rN$8RT;%??$KjxYlhg>?CeD^T#nX#<PYmubxW#0C( zZp!l=mnMrWv6=aF-J+#SOfN%|T{U*K=9QnTXr>xqFkdU@#R9V<)<Uzd(G^kVJ;RPi zgV3WGy*{Y@L&}P}M$n)7Zj9n-C-l-CxS0{7J&>I7BEjXW`KT`@j;5YUeyrH9bkJF- z+J8GaFRjD#O<}Qp&<_+SIR@RwJk7f%ihb0ZGMxKDzaO(yw3QtfxLk_uX`PZQZRmAr z=4q9ql{4uYbcLjBZ4dtAFrS}COKA4ZSAWAHrMM9mtt{_n`xq^nyw)iL?O^$G)SU~x z{;%M5VTlp9oP~zaGiqt={38F%a{2WO8=Pb^>;};m@^y{!k4PvO*fFz4v@`E@q%*#g z4ja3Md5X%Z4&RDtK}RTRf3U>!SUwPzW5?oV!pNqAK7Ek+BLAdue2Dou6t9{bM-PYi z7V}BdDe#i~dnjH9x$83GVYY9v;;u=PSZ+UQ=@&WXjd*-fuyod|Aom^Sqn494X^Trs zwwm*gFrTq9W~yML`KT_(O9d+w57b(1PgAR~q3f7Wq!Vh1q!uY=z2+%2q&woK+~i2* zG3L`Wkuq5?+!1eJ-jMr~m_fnbPz62Cd^B9q0%D^shc@kn3$T7a6f<1D)4*^O+tUIf zx9;IBY-V1@LURoTK8}c6m~YE(wH5PS=7V{<Gzd%!9kG@93_VEXOBZ4r^Rm*IL%n0! zDn4{FPy21XR4e8QZJ#a<pxc?xk|(Gr!?E&7ZSSN~PJ4c3*ulKKuU6Y>r(0G5^eN_- z%B>`y%WNl~*QQLv&Sn}ZNYn1OQ9R8&jY~e55U1bMw42eW<Fux!yOm!&!}fH&+&wT7 zVi%tU{Z=3BqG^BrEc4-xo#I<_#ufTKOq;`w!*<Vc?iKCNJ<-K{oOY9i1MocaLA`U- z>RD^V#2)5d@=P65CG`UH9*s-k^V(kK<+@3+BkH8&<Ao)@ukAZL-m3;H!oJWxU8a1x z?EWlGKCz$q)@Z(OBz1s!YFmq0r*49_>b2s<?tFby))ohu598PbS-lW1F;CBH^p}MB zIR7&9!Mc!Nl@2i<ca&esJ*j-W7O!ZYR-(XVgC+hj^GTYbvJRL0PCfwsfO#*QOwuLc z%^lG^JV+d6UOqrOUN-8*T#H^Hh*z~eg@y_A@JG#=(C!-I81wm^MnJsAe2sekvS^b_ z6Jwd)9SU(J>7{gB&33)|^9g5t2<PZr#~*Fiqq?86_~nm()RRqBmKpx3+A5vxv8IP2 za=-4S^`o@OKi5}?<4lxrF3ZIC5&9a;uc!9N_UiTw1cM%=cM+FkKC4@oOR9E^(|Bv( zna}lR!wY4lgZ`|=Jbibi6V=(O;I65(Q>w%xS}%WJoqeh6IbEgBbpKazExn=tO9;z2 zl;i8P9Ye9V^QyjQ>F-H5->8`&*Ui{s{%et-`6bgoS0P0HEvihKzmDDr^rp6a)%o}D z&8dC+!K=Byb6m>S*Mo7_g+}P4nhwbhBik{z4m&8z+BGZ4>q<HwlX(8=Z7SdU_@gQx z_104L=_Y=zkFqBdm9FFZ^!OInCIpV}uXMw6qt(dj;T4gfyw9{A!0x|Y!FAn*fEg5C z|2uVdQX6(nR%hY&mBWD^_V(FHfzI}N-1K;t&^(aoV&kpts(N;+asKJLkt*lQgF*^^ zcYzQ#|7@>e(kvM5{CLk0!4r8$ub#h@<K3V7znbiFbjUQ%N;bKMO%D>TU+dVmNjrGQ zUbb{jS(-N66b(JorZqbmSa<(L|G5)PB802y(EhWU{paJIJ(XXK*c0TZou&=)Ubefp ztNirAo0NX~#UMYhJ;*QSKSQYunRLJE@#uSr<Du<>Qbaia^t!Hjov*40JyKD6JjS2A z;~Gg^XgoeV)8j8354i=-KbPwIF~{+U4KKF1nc!+VE~Dcwna_=fHmG@Bwe8ZziHcY0 zQpZd8yB@C_E^)lHovu&E%MqG@e8m;TgdVSt9;!I+{E_ierjFV_y??%V`C4&t;;-%B z5&kp%WSdU9L4KimpsT&q{?+}Y`*#q<0hulqpR`?Iou3akX#(<btBMKTzhf@@T5T<G zK1}2{q+fNOJ@0&oIePyncb;k&QKldKE@ejGd^o1pYi{OWJ6_c%w{^;Uyja}Qy>hYf z!u2nVE;$)NyQfg^E(Od5uG8k>^%t<an6Bfk_7m-2B?E#QF6Lj_PPa$T-?w0#wEnZ| z;vJ#(uVG`(zu(dRb<68*ADnOf%llU?`PV}FZsOQwLDtklUz*~&;tS;HUq>G)HjEH# zq(gSc|0Ujj{G9p!vmWwz=<zu{m>NXsV&kLjdK@^T;@=PdZbymSk4^|K@Ams&tBa`~ zax+iKn|S{5xsBi4cl_f==Z&ueXs=qB<a~|LA>Bjv=WpkZQ2T^YJA?f}7w;2DR_S8% zP5ZCD&+h+Ev|F{{RR5OAziXS`@WPH8H%@HZM34Ww*Gk`$?U8@OBiGXkS>uUwYTx_$ z>-$fBG<?>3$D5Dd_T;hM_lR7uAAWMr*DsFkKYix<=2ud8uk5{3RF77FqIC86J*U2` zuY6<2!<&B7xM<rSsbt05+ZHD$ch30Z%b)CTt$OS84dTQ^#sAB(p;LeM?yU9u<{emJ z9s2z_aiU!DZ#QiIxcckzJx?9q@!t=8y8DdCO;vpEokw3E_;kg-bwi(ew)J@1JdvBG z_zfE~Z<<w8j=bDt?f=Q{yXc`QSa;?g8#1oToO$~B$qRb+e)AN)4jnr{-tQNeFL-Zj z&CK_HG@#c1WbtQTit5i4zvaw5UsSy?v-XKkK3MnfZ_N3L><XE5|LW(;k-Y!UDV4rI rj1NwB@P{cGZ_4S|#5%nP=Klwz55nj9hdlZJfNZL=FIR5+su2G#{>w$z literal 0 HcmV?d00001 diff --git a/libkdd/tests/stackshot-sample-coalitions.plist.gz b/libkdd/tests/stackshot-sample-coalitions.plist.gz new file mode 100644 index 0000000000000000000000000000000000000000..110c456fde759a0f13ce0286c31c0819ff5d5c07 GIT binary patch literal 6794 zcmZXYWmHsew8jaMQc4<00U2QEmXuDVyBUxeMjE6$hVDi>1?diH2?>!JLO?(m2I(5& zj(^u(>#lXboOeC%@7>RS&OYbMIrLB*oMp;i256XW?v~b$Zto!OKsQV9dnXXk8e-{W z?`{uqcH?^QWbfvVhA=huf{7#t9A1+xca7}mGW(`XxExG)eq&x0rM0^_NFh+`=Nst~ zR9-s1y}@EOS3-M)RXO2Wem-~jwv80m@2cvk@M!1H`l*wrESwbxZLK?a0sj$8?vhn9 zbb_dc>^^wY_S_&ld%Lge8rL<Z9{!9+8m5)1ze1iM?#dI&8};iF65dp?7mtkC?b<bR z7f1$36x55wCrI7iI9%*kfvXD=5+D#r0?xw^O_QD5$jRf3j+%F?YyMVg-L_cq$wEFu z*-LY4a0284a~#Yo5V<!g*zM~@`?7=uTE=qh;ilUHiNEe`h`f|qzTGe4gke=?Ob%=n ztO<Z8ojzntLw}rnI$yo$GF;(&=VCa|J83_|mE6QgLFMl_WmXpxVKO@;_z6E7Xmrz^ zC)lK+d@-N5%8Gz3TpUau`^OQ2?{iw7ex-0xZfF9nnhRi6c9`DoJ*{+XxQADE5{4U^ zbMepa@%Sc^15IadOs{sXHl_li1vKwgXm5NJlwZT;RkPtkRZ0&AnjTp5Z?O=%lT`+s za93Th=F#MFahk#fK}+3X?IlH{SG;Xqg4Lg&M}eNcyKEQBp}m2gp_tD@X$3-MpNAeZ zaP;$15;hJl38W2qSAbw_<gdIxvtW$skZ*0IBQ_0Uu}zq4H9)*ft{c-pY~uF!I2q6= z6Oq_)c{#4<W7|wj6a>4j$BC*X7}|!hktPHLyC+A9s?1XVSC#mplt_XLWmLldzm7=D zsYs!Ek8W@BB#P3xMr%x<dNXD~2c&nrM_D?+s9ID*%>q$Wg!i>*s0bqZL8zTv4rx?` zZ)86iGbm#-$E?JEouEimq>0rPh#h*II6kmp!ie%iFR$cOkmF*ueD<FK&VL4B{~5gg z&w!Dg&qYK(EYd`hV7CD%{+q;lJ0&YN(o~WFj*>?tm)NZQRK(ryy))t}K3^m;&qq-^ zx@g<*r4aA(B~MUm+5w#(soU~x9uqOMhg39?3-_YV6ZY&V={(YO1Ug@#7?U5PCb3*| zbK6(Zw69UFa8VNE>p5d6O(&`mj=9nSzNQP%+4hYsADti8U*!gL{`tR3b%h~o)&^zG z)=935ubmZJ(Mgb4fm3v3q~oB5svyJttImvc%Ks|A@V{;Sfb6cQ5WM<0BxEwF=1}&M zuWAZm+{_o@Y<=Wu`hQLO2$B{liXIv3<owH5&n-vXQt>%w!bbcmCJ}|680(N}{B@N7 zn<xs%4tR+gL(gACYOIr>0pcqnfA!kRJj%*RllN8D*QlfL!5*UR#bgz4z8vC|7DIz; z)?+laWQ8}j{8pPn_<&vJB39$sP?ThiVBF4Y;^BP~873N~;tuP#v?M4ZT_qn`v=Oql zq9h=6<4Fm40lM>i`#s>rG0(){`n#-NV%{elm%_FukeY+_%)Q?FlR#@$G{rHrLyI!h zf_`Yhimo_@fl{ai^N?X|wzSwGWhpkwQmMU<R?f1+T@r%I#rp(6-xiulWFnR-k~Nu0 z88J9Obbd9fJW=VSK>7=&lr>E&C*R|K>&}<M%ljlclF{L$l)v?#57i8<oNkZ%H0u6% z>-uFkEu%X8_T10^%S=KjtFxo3I9A?3rI`(R0aElLYQCNoz4_`iJjlaL7g6yoBN<!G zmY*qt5v1%zbbhTh9QM^oVUVXpZI7Vn^}51X+989=8Mwp=B^At7u=&y^qnP|@0M;F$ z6|)(xA5pcau^scM<w<I-xnpaWC!~t54$bYoDAn-d^|(=L|8Dvhow!r{MG}7z)R#?r zDYo*vq6*a?tNpVi*Wu1gG5?a1W3Dm{dVpgW^e>|M51sr&ga44*KScYlYUIDFU;mKr zKSckp3Wc&gN}Tq&%4}%?c^%M7Lkd9x)UZSgl0<O-?nR@H8!C=5{fXhNR0b<-GF$oC zYULz}ME{1V+4F0~xaj}7PFap^l7!}ZY2{!q><W*R)PC*gD-&B5Vzx8wC|t7E#gs6h zXe;xOX_S{wcb_SFD0F4?J;X!Q;qNf_7$ri>CiH-&P4=~H`;Te|asySpz20p5)Z@=i z%Nv=bzu7M6VSC63BM04|ndM7HA8B0?#^`D|4nc-qIOGYB1{w<qH#yHXc<;-8pSQ2~ z^HSJ{HD%tp-DTT8EZ;iVVk^!kou3}~$=kiumB{hohas0P13ZhCG8#{_te3t7PG!gH zHL1f2dMP+Q=$PFiuMP%!CH-y?<GX~t`P>=-58{{PyM;rIlq;<l(*-**a{`8g2edQt zt{E)Q8k-k#m+7;){{3$b^j|7#eJ|7OdNdR2!+9a7nsRx-fAIDU)|z0Kw=Na0u-Yai zF6})nA9Mc`VnkvttPko`(i61zZT#tSzgWwlMa$N~M)iCy25&`rQIm*W2_L<>AjU}V z=i~JES8ut@JS;A8L;Oqu^Gl{i-Ht}t+y;VAt{=y|a;-ndm?16MR2P;#&Gfo*5yQn_ zMk6`fHEo?^6LQCn9r2s?8RRiO8;jWOTy($0QVJfuPiCDhThorVICDsE*}8cnk6pda zF)6!eVAN}7ykM;yvFOjf<8(+tmgU&{rh`DWUtojQaa;wP&{rZI>WTKNoDQ*UH`wk> z+MZPMX3c&uuIuR@n%N(?iefAh<RPe<CGde;>r~B^*q5+*SZT&Qbfe6lU0MGF8Qd(H zS@O`|9-)+4?VtXI9Lw_OxU!2^sPL+Bs+G@UOwHv4gap}}fb;yA!ts0WS%UY7krZz9 z0|~N5jI#ojV|b6mGI1#fd)~rcYZX#FzT(uc1)YM=q@GmhFvK~z&%rmenIkr}xlPKc zsXCvu4?*^WYw#wfS=R8I=mJMI3!#?ALO@0{8C3Z?ys+j@l3nXuT$bx-DH%_8W3X%6 zKc(sHb@J?)St9{GErKHXRMM-!fhsp(CKaLsmrT<@UO@L1I=qw8d7hs;Hebjq8{^)U zr^JawxaPYHc#ja58xxjTS9W``&=no%*U%<t<%gJQBE~B39^aIji8)UPR-Ln*sW!a8 zTS0=(No7eU&tN}%^-sR<Qn?Y+02*=wTElf6KNlB#w0fL&uic$eO;6LoEwcIhU3qjS zzT9We%j2qZ+q3b*QG<mKXLvRNwT!}qud#AgL8E~A-9v4}leq+%y-|Y)j^8~W({Fzh z%fA_Jw$a!?Lb}EGv+})L3wcbM<7)%jGhUrPmyM4TU2rSK7xJWWUB($!Dp7(j!natL zFDvNdIta$IT9A_t9P<)$j}@E5Eq0YFFRyf+*BhU-wEq#AFvp^DawXZwS2ohuMWUba z8h{t&dF@+ioo@7oswcyFYvIY{J!hKEpYUtWFKKx@3y$`FX-JHF<A+!r-n>mR_n)5N zne9eM&nx0LCHW$Pz9-=G?gf2mOXN1(`%zOKu^d${8B}3c665i~d+<EpDMF=vEm}24 zD9%?U;gH7^0uzLWHQ5knR2L!QtIO*dkQ3d>62Wp)(g~9^<^-7H=^8n3ED9fi?D^f3 zL7i0kgHZa#7U2x^F8vX3UW?8Gu6g`q1v}+II5@ng^k{;P-O{|e>U;7J*Ye-;kN_`Q zv?0&Afd`KH5mP!`S&bRfueKhLT^WH(rf_Xh^#r%&<Rr`5KJUu2pV*$5J+ho$J3S#q zoNbwG!1Qc_HrFO2%>eIMpB@T@6Wcji+G0fHNen|OZ9b^h4(aOGjIBw_sArPQiBM+m z@MHrW!rldv{w_22Du|N#Sud|o@D9v~wXX?y6+;P2wnrQ-OYj)#{JO-AkE9257mYwl z7T0{JksU^4wHnywkp%#z6eFgOaa$f`*<~Nu=PpKT-LX7`syyLh{%M-st<+Grk1xGv z;#Sk^lv<&x4P!><iP~~ebSkX&{bZ>im@Ovs^sTa~Z_kT5QNnG$*;%F@<6F%td_qsb zm+MRHYbCfqqo-L%OL$d_RQ?sER@@v@zVd&zLq*lKguLK1MlVg(E$et*s0upf4P!1+ zFV*g{3kW3$eqPEWcNnL2L^Jeh&u4GZR_G@GOcKYrj!;va${r6c9Vx0VE0QlV#C$9@ z+uZ?G)o!n<DfP%FKY*TU+Uk9`y(RRw9kH&_tQWm~`{4!4+D7vy>RJUok6yjYFkpEN z$EHklIUYp$RAgW-FG((ngw%=oTf>cd@Z;FmM%{YM^!&AYj+j`x4am-XEeK1jCvamq z)udNWU39nwGyHkk?sNFvGmpqfztqDF{ln0`+tkDCP4B`#4vy1T?5i@xu~hL$j@6zk zMq};N!Y;61`XR?sEuOmhYh!@(@~|WESM@(EfLvR|PuQL1I_=`O@EP|!|K}rl)gf=6 zT?Q@|?g7QCg2pnnzY8vS$X*9jXt0Qll);EWi_R<Udmb$A+jD_=4lR2Td~0==E;no& zhOT0yEIm(i&Gx<lJ!0P&JRAO4-|avSZ)74Y_Nn+&<$WNF<zm=mU(XbxIuC?I<_i;M z-lb3J6^%yE)qjelPLWzvNlLCU*p?-8P|*&S^{kiGj-A{SLeRPIr9Gz}>atPDPCuRw z<J#BNOoXb5bn+i@wUJ7m`1{sAS-WJnJKwKz-*J<ABC9$!u=1&54bgmL1x>DB&(=Jk zHPaX{Ih5PURVclr%46^E+?F<?S6D1vF)c<KiTVE7g%lKc<}{~bXcS%%h16%+8Me?* z?tWXbdQVQkblcI;z(OU=%4#Tdc3;%)&HKJCUU5y_CV@tzwshr_Z^dSrNxjTl?s+=1 z5A870ZWjB_kHYRmE3F~h&4Tb0F6el+l8K>#qwo99N`_}QYtf#Iu?tt8A5He-k)z$J zL#_d3{v7$p(@|@4`mPapjK7eYZn4!=jO=>Li&DD4f*6T~b52-3hiQ<6l-bX4vyX2v z4)yDkIWC4fZ3A5qZy$?S9^rejk-<h|3{9K$p&>X#=S26{iXGy0$}>8k)%<jM8EXx< z9)JJ9K~BffWT7UdMZ)mu>G2X575{Iw0$i?sJ5#@<y>f^5R2<*s)-D!g4!56<f{02* z)Edd4G#nTGhPj^a?=3jF68Rc9Vjy2OU&C-Y_x7s-K(NmurdY+KTw43h<)si)n+8md zGaX*c=L(mcC4E`G4%c`0WK+5rMgGzHzu&vGYZZ0c4YiEVGnKNsPU#c#3gmOV;YHIq z2zHzH7_uaaBWwj$n4WD`=FtPbp?yi}oQzmEPneS#+#x8bH7*^|rOo%tvun-AtVHrE z<q2imzTV%76+2MhFR2#$pc^|YmY|MqiWS+m!pMcKcYHdy?LAaT;ru1qld;LE`BB0+ zEe^tsXgRU`yZ1(zo{AD%&Na!>E)+&3`FUz>7x(!><BM&>S^5J1`!t~lS<u)=)5VuL z5lNofUK|&qY%;J2zZ16(?0uh)`Plma-7d*~I*}^vAqFyJj5Y)Iemlpxy9shzdrB#* z6O1u)E*;pKXo3u|03_f^rLJqdmnd_gOEZT2NB|DbQzspyTREyf?PUK%22q5q9TEn6 z%*&dCHrF7f4fB4jV1apLb#-gCWh<9caTOg9EP1DP8NA6S#iW}yQ`}awcQd%4slK$1 z`{_aMmCP9DV{G%=(yPsUxT0Bw?W$A{cP(wMR62JLqd!`Yh;=J@@wP8=)5z|_1cg^- z9q-_Kf^TgrDWUxL3ogUE!G9DqDFLV0bQi%t*&|$r{qqI(!e4FG!T}_Ph;Q%SP!->q zxXybuT~A$qxE3W^#w-jM_53}NMg=E~y->7j>fu|><t_HeYNeY5f{a{VD9`}3-#Dw< z%Z@jvwvy9|AcDW&Z;O7S@2GwDc+Am;V9rf=e=lNwT#C!yIA*RU5aVewAICwhz)UvG z1k2@Pbf3V?WwYxq`V3YF#a$frGtTWhCGJdsbC7dj+3!ai;&`vOKRs7a)AWg6Xpx&a zehrG*vG=jk&jTIzrd|)0+s2<x<EM`FUYffM9QkaTIRW(Z2bNp`f6Sn3?ZDe(ZWnMW zMaMT)ia&+m4B3Se)o3*O04}*eIx8ufkZJs*cU<oD&iZnR^`6{|N?V>3U+=*jjaLrK zthklSXAy_%4N>#fz(4CnqwV><^2D<(l$wU^kFIL8Vs?`vdU<h>eM!eQ?GYpXFbWs9 znmgl&y$2=uZTg%z;n_Vc(TrUUX6TZgmxHN^+K5rF)BLy==5&+0{mAdwod>W|p1y~V z4C4oj7`qaCtrkCWpxjnjkY|={s$sxDCe~m^#@UKB!j18XVY5CFI0wVTEvJQ7X?$2( z5hjjpAXjoVXE5NjWMjrY$|F3~mV1|x{Jdx{Z~9Z2&r*niGakeTFW5<oIvz}(Mw6Y! zdrHN)EIicAp~9CK4J<{_V5e>Jh5sbhItr-SuIj@2q6sLH=!k0Km98#(-ycaC);w&` zAD$20Y(RepBUM|JG9gO*0*ZOTvtrNz|B%X*C_Z@@lZ;Jgh0zyZy5(vuA!5A!gF>qW zP1a-?Ole-lAC=*Z$E@zN<O;FSWqD-tBl-2AwS*GBf(k#;miiuXwDqz8=xM#v*M+;- zr<Y`M)n!pF(Fe_5E7+>mQCA^=T4JHKTEM#fQm4JPg`<ymii?%dB6|Pxva4arSr4r> z@~_0)dYET@r0eNT+m?6TFu|dgXSFLg-k$xp+;mrHzk>l2yz3buS9h~_g~f9$5$}Fb zE+x0H9t$F$6?tgCLmC}FbaCT8l25ZyE^aCJ=wxRbT@b=*g*BB?8EaQ{+!D<60Q$+A zx`A-Vod_;y1NzUmv#ZuHxk>J;3~U~Au@O<p$vTdJEY0sFEOLux9as?eaAh|f5|-5x zNtLZyrKkK$rT8nuQr>P@(=~wp`XBC}f`fRLGUbk8_>7yET4L?MflLsAZ6~1r{T;3{ zzR<YRmo<z&x)?zJpi_+<U{E0CFlaqOVNMzn;2aMm8_Z_i6INLF(yL95`BX&|ms6x_ zW>!Mg%_1IT#)U3s5yP*Gzrm^Gx^--!WTa2kt{NZU5>FCkZ>v_EKAz2G8G@0lVp(&% z{_Ty1W6S5<bX~j+)Ypw^T&)8K4#u-gD`VZ4%#&z-O?{-xc&?b1p8*xyU=29SI&}?N zNX;y$ptwNPc&3F@riBng``reB$H?fVB26MDYR+T1={^A}mPN#~s~?MB#4CPad=1eO zYMV*tX3Rb5%M>%!6YLH%7+}d;7OF3Yk_6SO=ugKs=a#4&_c=wWEPRPGdU{eDXRY2! zq#Btj=|(d`!~7YrswmduPC5ZJY#@9}qD-i+S<``fgrq%f9O<8bU>1R`(Tep2su|wC z5{T#Vkn~j0Lu2C7=CnSvo7Vw^5t3&W8G7K(Q~~0DfK9XmDlyffUh>6r&3Nk$RJ>x+ zHsHwC%@YJ0rmzfBlVpV$2VB4x*6p=<3+SzSl)WyTchTxDxfv#e<=eFnZy!?DhV+?% zJ<?rh+BJ?!NhU#Z-%F8C6xJmc+(3Ks>eA#w#vQoeRFxn(Kc{ihokMKXl`>)RX`C4> zYcLtTH^XpFktvO=H+D;^mwV0X6g=ZzDs`N3*_=TqXV#wr77PqumaGSAHz5(Xzj6iq z)=88YA;!aB?#_1@&0VDBa%Ye0XGH1izT@|+`DqVs?NOb&8$Pp@43k@>JYuNm0&~U% z#)RIQr6ahBOnHVn!-D)=+$_bV9NUO7zqt32UgWRcQKK<-Sm60S`1U$Jyi%S$;qR-b zgc<NWoS#uJBzzOnn+UwUtu*06=Up%mc{n~@qA)t$0yIVK;mT2{CDpZ$SGP>a>;%<^ z<&!dZZrNAJ7zxWuJw~TtVOD|<_b+rZY5Zm}1o1fL*QfXFZUi53S?k^*o#dy+Nic3p zPFJnL-1fL9%j(2`%&OdbFVIG1z}7wwy4m6-@;gOpQCyUb-OSe>r%aOlP6La>t`E?r z2>_D6M?)t5F*W`}byE@5v^}r&q?CO_Tr$??Doyo02S#YeaL2>=t?Q?c?3NnV*%)ge zBj#(<*Vl8{dqFM+2|DV{4$>MRYSo8@EKci-8de5G&gL3cRQiZe`u(mwCUb6MT=N5O zW$@9Y!Ik+B7ODFuvn|IL@R*jfS8kjZYQjUUG>L9)s4(hm2)OOSPky=ES%b@|MA{wF z&-E7t3C(Rbefi|lUyQdf(ez2>ZXh&{4@(V?XKMis$18I1_o5SASzylFUe3mvF07@8 z);Ph3=Qg#_lXT1HqpW!-@-Rl|AtT^~_r9EpWvJ>#>wC=H8NM7v>^ZA07WpS3y@&N# zD72yL+tNPXYn!+!^v@JipavrKoT3%mwCgs}UOX3zB_Y&*C2u3KZG9S<HlATc5@!_# z0*D`JVW20Kw4?(~fxOZ?1<Q1wtwx-jnahArH)1Skiu2&DXSjl=>mOV2;u2T{Vy*jZ ze(*}B_LZ7Xng(dv!Yi?7!fCZKSLsheaSW*G-*|T<Y$WJXo6EX?A-$>PmhR~WtkhR} z%1_OIzb~oMK)qKnzIj}J;^s18z_=-Z_+fh2PBBpt_N!lO_EcMs8EJ<{9t%;U^F*sy z;7KdDdm`{E-o+fxgog<shPxl32X2Mpt6og`J(`<91S_-I`S}$Z0y7SPK~3PAXw0wd z7BmGGKlhMhwjMKs9!9~Oe#6=KW$h|vje(#S59i>g_4*LbBkUwP*f7RJ$~hf)%)oqk z%^xW;9j?)_?h!67CE5F~HEJ(J59D`l;3m9m(U}i#5VbJW3&gxE+$?|4GhF^Gbx;3; Mcbojo4;}4)0Q}5Q+yDRo literal 0 HcmV?d00001 diff --git a/libkdd/tests/stackshot-sample-instrs-cycles b/libkdd/tests/stackshot-sample-instrs-cycles new file mode 100644 index 0000000000000000000000000000000000000000..bca394042264d2b630c6fafe2e5b134f9e051bd7 GIT binary patch literal 625 zcmV-%0*?J3iwFpvhEZ7n19Nm?V{3D0Z*(nlVQp}1Wi4rLb98cZEn|6OY-Mu*V~<!A z$p8cm3=qHs#0Egz1LYT&BqnDUXXKZ}XXeGH<s_yTGoS!w1x5~JK3@F=nJElt;3XQL z8LGbnroXbdBsDi4XiP~eHv5^-^b2w_FbE*oj}8RT<sDGv`8gqB(Ff(bBo>ut<|%lm z7UiYpD1@aJ6=&w>DHxjT8R!{UDTJq{D0r3TDHs_7g{+KCtxPNw0$f5Aj0_CTtrc<; zOG>ON@=A40Oe{@wQ!<N6D)pU0J$+pC^OLglU0lOlef$G_UHwAh9fN$$OyWZ<3=B=t z+{?!aabi1^uTY#?l$n@gYpjr%o0e&-Yp9T19G{w(mS2>dnwy$e5}%S;oS2l8YHO&F zlA2VSZfj6sl5Ak205i{s0pfpH_|`(h90rWh<O8nC{6lt+DVn_FPJs=LL9B3j4xoEM zmNnoAe{>)WH4l~!m^o49S)uU(vk#_@(E+=8wu0W^^n8N>nk)|p!sTFQ$e`%~nOU5Z znp!}FdzGQ-rvZmN%s<h55Izf<f7pQh0wnz~br%Zi*+2aM|KGz`5=`GiGxvcs&t^s- zeRmNfia4VJg!Vg}^&Le2hXR-?c_3tAVPN=c^;-Yl!b$CtXzni-L@|y*g8`ENKrt!? zRJ-CDXA{V*4Sl~GL3E6rKNE<yk^A0=>JkV*51%=UAfbKpiyW91LGw38dU2QsHZ~yq z?8z;2%*>i?&9hvM{ThMdVEbVDVd+2$&AzqD5PH5IgvMPS<(HOVDNxYMv$qiwKO*x1 L<`az&Fa-brYP1-x literal 0 HcmV?d00001 diff --git a/libkdd/tests/stackshot-sample-instrs-cycles.plist.gz b/libkdd/tests/stackshot-sample-instrs-cycles.plist.gz new file mode 100644 index 0000000000000000000000000000000000000000..b35b02a3166bf9995fd9d4b084ad2fb45f27886d GIT binary patch literal 1630 zcmV-k2BG;MiwFpP5=&VC19Nm?V{3D0Z*(nlVQp}1Wi4rLb98cZEn|6OY-MvUaBOLF zbO4=Id2AI$7~h#a3W754QIVst>wyOfeWkCo6e|T=6)2^a7I?H}-R@5Nw%xb;x-(l^ z6rrNlTM+?~OHiW5s2EL*ffy5G)EEt~96{w!6HN^Mfl-4;JZAPiS|El$+`s1g%{Slo z`_64mJ2s)QSO!2Hp-$5>sG$>Tn4Ki)(UWMDD9K9Ru;G`C7<uVsqfjI|dhEFIs&?hr ztF~_2zGJ73qNJBloZ@dN*4rGToiH$Em`R=5DcouhtlMsz$2HHIWx$~35NHLyL2b{~ z(FBq-d~E3^vx0T2*RgcRGpx`~a|foxNa+sl#tx%=p3$^z>1gbKnDhdQeI1P+x?)I{ z!wPDUPMx?$n&XU8Bp}!%I<Z|7>p1P1$waHiUKl<nj5@LJGS6wF9W$L(IN_0Q?30jq zHg2Ppcn)(ICIG>K{L9B&(P&}UOEtPd3j0VztE0n!CPuD)?cjuhiIY?{VW*hA#>RP7 z%tLstVFAyvEFTl*G(UtEJGM2K<ueDrfslH^hXnLPWZKTm)z>VZUR-==^0n7pf5Ys# z>n@&A9gVSFLVco}y{3c7U@jOHn)ESK8_Z72dC59B<;I()R^0MG$xkbc6-_UW$4h3E zmdz}mbu+3%OVMZO6ZAIPhGw93r~;Lu4pfbbQ7!6=f*qB@Xh9G_A3F?G&QaCnwmHa| zIc7o_M_}@jlgzDE4aGA`OAb}vcKaRk7J4*crh~8~OncM?fn6jAydC`PNa$_8!JZGy zUr?haLYTKoQ*HwIIT9f!QGV^Bx;yI^H#9C;x-7B0skvo^ewWc|TDVQM)b^y^(do3O zShd^)O_JQ;a7IM8x|mVha9EqLW(sdvy(~TL+n#UJ-WCq!sPk5jaM>{k)&0N?V`~l{ z^^#D?DTFh4)<n)z)^0ce%eaiYJeHOTgZ5H}g}G7JXmzrU9QqD}&YDJCHa(ZxZV(Px z^Eu>bmIpb<gCBYbgQ6Za9L~}-9EapshW8#DCEvDi6Gv>1x#D9xhD+UEz&QIXZp?hP zv5OTPUTN4=XS=|q2!!qN*2oE8BQ<haQ}j&72enAb7joY}xi2b!-N(;GPv?zWc*h{n z|MadCg{R8@if90xIA3i|`oKWC%O@9ayr?kgmoK71!R~E)f}XqY?aOqptatp7+kL~2 zyqxJ>byn>A^NGEgdsd$n;i6lbGi%lk-kzCsbLKvki6;@d|A7blAA0zaM<09qi6@_0 z|MUj+8Fiz!X*27hXEjyb^4xN^x%f(AL|Q1X%Sye?2pr1R8?PwEuGy=z3@5lAyH2IJ zTHxAU0nV<?d?u%i9-U2RnBs7`H8U_f@=V}0=Lrw1>{05l4hkU>kTiDLcJTuC4yiP3 zK|T{<tFXn)wr|RI84cDN$tF6~8cR4I?tT<IShh82I%{UA-!3%bV;;&-jA=edK4Xnd zRM1EqjMGse^1cuih6zU_l>j{yO)Q91j27hjzP;$(IXWuSvcpzUtQn~`yJBjQ#wv-~ zM)Xab;u)}P!rQ;1NV9Nj&|VSiiJP&w7O50;)Kz8pD{WU>5dIFdCD~tqR#ZAIHZ4}J zEyb2LKXA3_MGPu0E-8<fX?0Z%+Voga=`1Z}P+H#O22<nlnenNX?bF`ExsCH`stUc< zj>4+yCDk>H>K0blHt2Ke7tV<54P~*SlIY(BhwE50BZNu9Y@tE0g?oh!!kfZ-LPq!z zKo9|wK_Qq6YJdSukOVHEpd0jn)nF~?2kXI$U?2D#8~_90FgOa1f$yLQ3t%CvfK{*- z8n6dG3Ll40!u4<i+z2<rEpRK`4tK&`@CCRBz64)^ufuoXe)tjm5*~-&!c*{9F;CRQ zxHwb1Ra_+2i>t+caf|r6xL-UV9v6R-L}{cnMJkgPNXsNFxzajmv-E<rU;0$~OgbVR zmA;a`mcEsKmVT3m%j4yVvMR^q61h@dEH9H=WK(XFZP}Ltxkv7k@0a`Kr{oRtCV8j4 zTizpoEFYA=kx$EiD8m$0xk;I=)GJh3tMn^dlzqyF$|2>5a!mO~`Cj=!Ii;Lde$R{N cmFB&gon{DfWBt5v-j}!cUjbccR1gUO0GKu%nE(I) literal 0 HcmV?d00001 diff --git a/libkdd/tests/stackshot-sample-thread-groups b/libkdd/tests/stackshot-sample-thread-groups new file mode 100644 index 0000000000000000000000000000000000000000..91eb05c6ae9f403e2e6b25a7544aba24b6e20e7f GIT binary patch literal 6784 zcmbW5dypJO8NmC<lQ)4t01*}uAHXel$t8hcB$vB{oaB<@?!rTmP47(aZnHDflbN2| zTR=HP<tY%vNECTQP~IYnqG%Dy7g{O7f3%8{(vs4$u&iLgA67{jzwZ9N+1WE7Mc2)B z_piIZ{^skizwX%u7w@`+L7FlDARPctA3V313ZztXubDQKuE?lT3wS+soc_-#^_U@f zddKmx^z)8b=$QrPK;6uL1M;+;Y7tgI4K4OJf7j$NX3>F+9RWG%k7Dxdh}rFH@+Mn2 z9NJ?j-Gq3UCoLJ3&KF4}g3^T|Nu`QP{VU6T<-S#=i-cbq<w>c3IsC0!-oI*~uQWD1 zSz6lHzw(??8i-hEsaH)$nJBC36=x2wyKvpc&10kMHceIr$46HTR3^{r>t9|_;V_tg zE7WgA=A{)8)l}k%P(-@o%am6GvAVy+!<t;(+h6L<qLkM}rD-*+XfdrzzNlvP)qT_b z0|O;5h`rTIOQ}dC4^}TLp<K$ALc!RJ?EPsdn>C=c3<|~ezh3?8mBlqPx)<ATd_(_v z&nM>5aJCTMrv$Z`oYZaUiKAb6xY9Ul$@SB}^OuaLI~l9b7`Y!Yt&13Y8nP2%EG6d$ zRAY>EsDgXe6O~2iS>zAg=dqSkf<_eIiI~af7Au7^y#dS{sCvh3=VPBOV9<3_uq1<Z z_{Z1+<Ski~>tzzQcp^3`&K4pMg)cdbnG%T=Mz2;&g^`ld3<qI<p6UeFItk-MRfW$E z&Uq?fGejB(9D=+i!PycuC4D!+p~yXz2%hN%$v+Hv+LGGKDPxBd7di}ctGTr`O<-az zBTxhrERjw2y%G^B?b?nc4*Clo=KYT%F7sElWHEAMw<!@QPwKWAbTsmY;6X=w#*RVm zsi-Ck>#_uStXgPc2aZL~wTC5_9gj6LZ>}V;_aP6#rF12#QMfw+c~i8jyfE+kk#{)> zuoIEjcqoH*q%_Rqam`LbKAeT|7>^`rRYIPMpr{Tz8F{Fxu)<n~_;w$l{;H;puVt|h z5z=R;AeRw%Nzjq?LFC-?M4GC^8X4HB$ZJ{GwCqD9#|Q6(H8HL$J;6Rqa$wdb%!*aF zZ=s@9<~35GK_A=nQS1OK+w@f{^2;*PB4MW?uM4f)ST@<|SRRbyK#U9zg6o<c#?C;_ zQxINop<d)IuUr2z@`*T+QGKoAZrialNsdf(7KvO^AIXu42*5(Hm!^%xe&mtREtNE3 z6W1e^YG&f5NhbbOjdYye#7mQS+KfdK%G8_}&^F*OGb>oslwyC{;6~BBX$AJL^0a|P zo2|qO*pq7~+LE)8%aFT(g!NUJV;oQ+jyfmjk>;?mlaNO+l6{Og=o83}s;iM3`zM8o zKaO0>IJc!i=OXt4mH9E3XnwW^x%1~1Y|DnP;7Axm9&2<Qtgpqq1|s5sxBc>lrDb2R zA>_PXPek3iz9qxRH%|@@C1ATD6-#h8whnnUqm^=CJ@Pp5Ic!v{F?Jr7B0iO`i9s8X zhZ=Tb-GtoOhKLd5)gTjEsXSzzk6da27n#H_yBB90kq@ab9D*%obTnzyWz0ql+N8iX zA@^!=B5EB|Z$=Im0}W&H1sg*ibh=t{0rIHUy<x_Y8-J#~YtRI8<3B#U8X02}c_bPl z08hs%+kypfsvwOe9n0)O<R%VO6&#qDF2X$G5zJ7Bw{W59V&uHly<V3P?{+u#N#t>j zG<4un<VE=b&)8)oCk>qE;~RVmc`Cdt!L!Jq%aOa~;~v~C%uZqz<QHMw80WrJqnvH+ z?r)-&Rg5DawekEsa0)ozHV9Tl4pAIVjC#VgvKrC(5Dsse7#C>_!NeMy-ZDzFBuF7( zVJx7dd&um&^F`Zh!0sMPSdH{C_mDtV6YlH`ThAt+cRLCNosNHwgI8~068k&-CqPSx z8l#F`fgJYTB*Z=lWhQKE)<nJ*_Qj-90ah6mwAby2A$EXs%ACNtQW0_w4mCWBY@0$Z zZ8T3oAGZbQVCv{^HD=qezq6ZD-`J8IIBiFsFKRps+}w2jkBe9(+J+P>fjoF(XhWC( zXDEMUbfc=*b2G3h<h)5M=Rhm(KRjkWE+9xmo)~1)$l-n6k&VMBLZ$66)+)^Hb0zlA zZyMkal{jHnksRX>g{iMbUKgrB88i2Pld5^dcJ<G=jCnk1$~+1%b`5eoA8~17HiMk| z{w8;Hj@Te~E%K^Vp@1u{htfWaydnKWK(H~_Shy;%>qw5d4|p7cF1PxA4tdwUfPM6N z>L2l-t))i>`U3TbOLg&pydHTitl1+KX13%8;t;~ZwlB=($F$stJeIKt3KwGfUX0;@ zbP`<0o2mb-b-x99$|Jw(t{Rqn5qZetG|tlehJGu_NtC(j=TqE<+{9)v4S^Wf=62+8 zf2O0?lI_SdSScIDFvi`19PE(GHt0*pO<XEE;+@1{-e#m)pi<pv^=0H>ht6RHYzOkG z&L@E-UqNnO%2Qx{8QHjc7xEPJFMg-&MDF6xh`Z&&z51)jhXTQ)EFMcl47cJj_BG_@ zzP%Pcz^Y(x*v0H_<h&a87EiFRBljUPJ5}5fb`Ns+?touJI&tY<<aKaO)0U_pFjw^Z zkatc%2ku8+gMO;jIcawxpD<TvESu~bSmtru32wk1K;9D7s#47mwx#vE2a!)HbXcr^ z6Z5p|dm&}tLhi$@x42*rA@@{RhPeY#-j=~;We+0<`<mMr));#POKx4(+FQgB_t>Mz z;X4}d&hCVI47s^iKrE}Oz<eG__BirpwIkc0Cy;~8h8VWWj!b3|`!@1C9`>4P7<=-6 z`@<*25c>}D1a>r?nFc+DJk9ELSo(bO?;>~Si@nX-w(n8@SsxxxBR3!7D$S~?mq_>_ zg8cybp#A=m@G%D)Z_5b#A<2QQ-U}a>5Sw~i;GdL^g2EB-^dOjXAho!B9@J-`Nb!pL z_24fLOHa7t$l{36U;N5mtzNhmN9=wkZ!Ux5?E(RRL9zgIBNOgS5$r>_dYiBG*!&ok zk1G~>K%e^n%piPaE*AQpvZ~#50M_qw<NrYIh$*|7`jy-`;(7k08&CHL#*Vb||2-dj zytLTX|7HIJd+AYIQu9cHI~mEoKg0Rv&a&@(sGI#bgTtub-rMgPI{C3zD~CKJ7QDW1 z-@c_J7W0EK=P!3RkUCf0`TT$>!%&-up9H@U%mteqzV$3C|2UZ6tp5y(G4*09L1hg* z#*BwT+mC<x{RfP!mwMj{O@90CUAF!wtZS^ZvTKJfUFjs8kNl~t5h{xjkR8Z=%NDtY z|7Snyw|C&yb;f>g?zn5dvELuf3+yK~FFme{_84P8y6v|Mm+#Mhr@}89^I(nlZuz~H zKihuo!^qN3P6@I;c;>)k?6>BZ&VFN$yk_m!c;e^Qey?Nwtw&rPy4&<_?M3fR_Il=B z+Kc+_J*L}U71|%92<E51K)cofvRC`L{rel)YxA2IoM+Z!w!L~Lx+f?>b-(ty;*D;5 zVg0?@i{6{;_58cE7xmlM+ikD6F&B#u>V7}6*C{`FXM0s1+P}T%dxWIEt-W5G8yQc` z{R`{w)n4@8X1=ZPnC;XL4`Y8Tn|U6;;qM2%eDAr>?!2;f;^WOTYHasgOYBZKX8qXq z%a*NwaL?rNd+u)Rfsw}={_2Vi*Ihnu<&U0z_UgSarrV!`RX@(~N&Wn)-^6F%w6kZ& n8SVSGy-4xE%7XUeF@DTI*`%~&Gxnd;T>g)XxBt_X-2eXq=#6?S literal 0 HcmV?d00001 diff --git a/libkdd/tests/stackshot-sample-thread-groups.plist.gz b/libkdd/tests/stackshot-sample-thread-groups.plist.gz new file mode 100644 index 0000000000000000000000000000000000000000..d4ebd8f86aaf272067461bebb6cff575a97159bd GIT binary patch literal 4312 zcmV;}5GU^+iwFo3;>lP519Nm?V{3D0Z*(nlVQp}1Wi51Qa%Ev;EoX9Xb#QYoaBOLF zbO7yL-E!MFvc8^u3Z2~B=0B;NBwLQ1J>!~W#+5j`b*@wp2}xL!1P1`EXrF!o%CbcP z1O>ZONv1M2RaQg+-2l43ZZx{VSAYLfm6JQnIHC3H7yhaDVuI_O7NlOje)0L{A1B$1 zzrXq0tN*$D<KpK3uis6Yl5jD({`~g+)y3q+$=TU?)0Ft^?DFPva{d15)6GQoad!6Z z_ZO2FD<PVfXJ_m6`V^^Oo#wPsKjdfEj5e5w&3oC|iEMCMh~kALv+cddj{LSDxp?!p zUtZnf&70f2KmyIUK>00SQSs_beJdZ{|7iQGIi+HT*plxa|MH3pCdc$9=QCU{D9dq$ zYcVSbNAnWD_9v)Xkk=>v<fN@RTHx6&W;HHn0)G*c0?*s!YwwF6MU%X2^4GzHV@6Q< zI-I;ZyYG2N``=h_RLx3!hs#Iuq!xII*&FZG+3w@6!9Ar0E%A(#|6sk5pM+VYH>^;; znxQ$TWh?MZkjl_B4nl95MNtq8v{%jMh~pWN)AHDUH=WyG7PzL>!^6kk!(OM`-_~t~ znc=s75cgR)>n$1O_sgy=*(G9YQcr&CrfKre?XM<&D#gZoIr$eZCLa)+__6%|W$eF< zyvg<D%_Q*r^nWM3#0?fiy!=wP_>|87eRlcopYPuPasA=l?>Dpaj~|j~c9VI2tSuv7 zA%=@tF6Ri(icML}N{R~WERjks;d}Y3^P*swbHgMgY3v1FW|+3F@u!j$Z{=cGI*6jF z9GV{oLw$Vye090k5HW_fANQ6b?Mu_%qre}0GW^I-rbCaW@n9owkcl`MWFj2wE*z#5 zc*DaTWWx7`TMcrN1%uuB!~DgEdUW=%Y6n{KfP1r|Wk~eY<$;ILOLb+}Th6G24Q4<a zIr7BT0jXz;677TQ10IA)kV&)}Y2A>*K-SP`!v(Gs*h_pHAZfrgf%xr=*QinS_>i{x zdrI=~<R0$K>fYC;{Z^v@(35U2^QS==XNRIUti^|<d%AomjD4**RHeQgAIfB30c=hE z%+@qM1cV-jSvk&9Ysc2M!J+!~HPFUXXdCUwBzrcNBYm0oo9Z0zY5y<#0PqZvi;`&? zd1;jCi}nPOv&w~S1yWnf%(?)PO;V>eqVb{GI+`Otu@Mb4`_&tb?O|Y5N^8HqW*htB zWF>2O`jvz&%Yq~T!M7l=DOYS8Z)!z@&?XmCTfa7`h^>g4Sr;#}aWsO9hRNQ?c^mQt zR`}BWmflmVUiwpgL|;STQa_EOFoF>{tLFN?bp@w3GZEV?X5d*n_N@qKUBA$7sI7+0 z+R_G{cB7~5BzjL1>r>yxut}7M1+MpGD=D;dHVGFb-xs!h$=rITeAdMwa@y8nU##}L zJZ%bkEqgot-WW13O!Y0xV{0P4TZU1=Ez~6*Xo_j*z`{fnM^jzudVQ=J5!l8>qRWV> zgF!CRBnk~4?5+>FS`8yqcCs-r(NpjJi7}wWuxE43qeOQx4usCYur_EjziJ`menAI^ zXYwqkHVs03eT{rqSfCMc_d^})wdR8hoq35j4K$$Ibx_OAYJ--fUfMWFbqi_eV2}rC zek0%4(SO&+I0J`q9j)y!?Q|L^S)#=v&JH%Fy`8NMpM+A&^=yIKQgE||2bxc?Ygv$D zCRU6JQ4Z+kP)i%VEY-J+yLP4A;Me(P_9rzrHp7T)SfMWx$~Y-hFn4%*1Gz;@Q!|jr zUK%-BRn=>H9}O%~Tj#6dv{2Dp+NhZEex$J1hqN3_GJ$h54^cKv{4Dkcr(FerNo}5h zXn5}SgDE|NLk*-n_G2mgt>PdGvOyMk%Ntx579|*5sry)t3OB0|kie`WwOn>3YHmGk zm0%D`rcp9vJ!RvzCV#frebQQvnVETJ<|CAja}tM%*@Vg-*N8u8nVA_a{;0vhN||v( zsLOx|QRIa^Dw?yMH*Gh<12233cfGJ&dm!po!?SJK&7!4UG7TcrmO<n7e&kC=+s3>p zFxWoBl#LBltoW=u9ogCK0meg<3%=_hr?nt;+Yi?s$XymiNh;UKi+#^a!z?hyG<H1} ze{Rp8>b7!R0o58G!0f*ITbfDlMUefYoudhshzkkT1)j0CuGN8C|FmjgF48ddrg1Dc z1U9z1&Sk&0E@RoJ>@7)6``}$i>}k`Yn$74PW=vY4Crl2{R}x1wM0RTZ9O*QfgMG{6 zO|7j}Y<_4VZS6MPwqUj)->YyR!Fzj8&$LH8Xf6gJNXeX(x<LNt%Z;~H_yiTGRD;P; zSq@F1+;iBb@WHeUHBY49U>(vOZKW8nS?30k#nY5@jfj{@H>W;gX8bhtv&0ZpGC*}V zyzcmCwqQszF9%VW9t3YYC=W)nAH9vUNNpSp<GMFh%~0D!?pVAe(4B5=6z)t-!z`XA zPi89gjL|#*raV({Ym6zA(Dawt64L8jk67*DzjN0kkXm|_JpXh(e!3oRu7{k}>C_k+ z{^@%B#axfrc;4>rdPHg`6ehOeALV)^#<QJcJ&)Lva%&U7Z{m2!fBfi<M-oaS|GhaL zlBwuxc|+gI@ldBA-?`%<_ZUCjj-PJF*Ks>SD}4WSJN_zehcOM~Np6Q1O^q2hPHu-y zlgFHnATj3cd{dVr$qYG*Ka$HKp(P6qhnwHj<>;6Sjj19(tjm$AL*Vb+;|N0i?DvOo zIc)Ii=yLdxpQdqS%8feS<uG_|NBJBkiT|tk97#4cW-1=xbJ(WkNANkU*5B@PSet%- z4o9k|x;(|-@Y77?4H+}Eo%{`BF5<BcM;uSR?<vS3H8Ts7&`_K4Ej*4ae5%J0_+Ao? z8dP_tBwQt#3AvAD%9uhA!IYkMay(Op(v0UZ^#op?;r;A=-YZ$_7q9L66S{M-C##3` zSYpZhSTvj>v-^3CdiuW0=h`||y%er}&X1<6l|qitol6B(<F1S6)7DB1M`)u0HG<sX z=Tp|o9!DrYpQ4r#`PC_ERhr`xRnjHEvT8N2BZNOo<bS6O%K7pzEAdJ651qC#`fF(l z|3O%_Mog7xJfDF>032hW!Ud7L1}2?Cfa%vNm`A8DcxTNy0BNnza#W+$UyfMSFgnK& z9Jt~c;+#mYqZSavT>))DN-RfyCmk|~mDegt#^~o%+ObxwAZWV+9A6WWgHUz`IxMOR zz=&wI3}b|)*_1nb5HWYfxDH|%uBZSZ?~HKrFGdvvX;*lw2n4d}7cU%PR8*#YF1Ohb z^j&dng;5Ej*jgR*IH=SM0^vvKms5;TZs-~WN42E2Q6nO90IIHVqC^a0w9Z(fY9?m& zf&<WY1rqti9DxXzD~`bU#@gi!qb>rfNV%!0ir3@_z&qB0dt)42wpDYDY62kQTAALs zsV*TV-4#eGI+vcAinKw%N$Bhv(S=Z(k+vxih`rWt6dK`x)Y2y|LFV2S)li;e&MAYi zn%d#{sJ68bR9!)B8;=zmwFM+OM0mp$+>n&FTdWSpo*n0K1vsIC($XcD!VW;)72KqQ zc3aSOU7V_B41fbyh+ATz4q~27*A?Ts>kh9j&*k<{#a#e6aQ2$GTy_ZOT+105^_+4D zs;<cPsZoF4QUqaCv_hQG*(OZ1H3pe)SAbg+hD(X+4WnNm=)1yOjm4U>Te$^Z)|3M{ zQQ9%xF=6w$T6uusYp0Ckigw$mLSL@#n}lG%72}jA1wq#p<J8rc@-GBwS8$snz5*cY zifq!Jb%`SoLv}@&?sg)G0KRs=(2;RWT7iw-6=77;4kESGGE~c1060W(MVMU5dPyME zOuM$|7(>p0W?s8yU>sW-2%Dy*MUKvQN!y~4RuzJ{E4pkqsB5Gm=yVCdfGf^40>WKS zTwSPzobRSYo8t7>;Ix2HGiNtyxny_=vn@|taqf?s^9v>o%LQ!+#AZdm?Rd<pwGa-B zc30)7w6Ow_5@45<x<Xrx?jQ#23Tl<8Fk7m#!3H48s@?fIy3A#Z1*IV82dQ)HNnkFw z;>GC}<N!EuMVSj)RTmg09+0{MOc&~f2!5r`UeuytcmZKdwcAO?NEd{Q8s=<LOY$3a znIT53-FP~>%nOJZgnmo=5&KY(NL%)lT!TWw3IuIecd5oJTs9EuoVlWz+!d6(ArP*y zcIWOf!Ob}Z*pO*?@Z)B)mdm}AS`8A%*HTJHy|guipQ2qKH%b;{D>8L<j}{PA-Cd){ zH7-#Lz=3l<C@;6$Z2-mAZnYRcn#@}UliZ=*(S6+Z59NZ~sfY&v2d=R8ultDZM^q5{ z0U&2`>I!c6bD+);AeO)t;XZWa{|~EkQt>N*mpXODICT+iMV1T+3L*c_c~zIcfBJ}d zL+b)U{|qmw8N(mmvXWz{ptkS10^o&;0R4tH`3jNx8e|Tf_lYX4uOxGZK<uoqUCuTx zgfMx-t_b+JMf)iCslP(7;0l3T1o$MiPg<Z403hsZH|HM_54u78R(=FZo!4IgGseQ( zvc)%)mJqaE5m4k1dnjMK$avK6$h&KUK_uTfWxlH(+$M^^FyIP%q&r6DLha`u=(}Rx zqw04W2*H9Y>TONWR`WsTz&S%~Q|ATVUQ!OifGhZY#0_Pl3r0hbcTT0a_!VeF?>nc` zUwwE_mrFp*b<U<ow;*fo3VT-{F0a))E4jMj9-l+h%RB-6w)ue&6X2Xv&o@;gXaz&u z74{xdfYl2R8wkmF&adx|JV6HUig!y)LFl@|+#KY1*LP0s&k?&NFePcup}TFa)V+%V zkP+%TC;6kI_<hU870e-w@0{&FC$z%yV-B<o&UybUQZSrTrYZwZ2|#+L{+hV)Xl7I5 zTs~t6WD1-&xBC$Wh!BhKTnK<_RBqHOv_K}nIm|~mP)3#Eox&ru7uBN6C8IEfsm_TW zq%NvWffWXUFyIP<4QX%*RIKS-5YQkP568I<paJnLoofI%s*5>DYOwEI0Z^faH!X*$ zfp&$shEY`m4no^Gbk|~mzsOA#h=Fs?{$F8`Jb3-RonwU6MMK@u3C!V*?_38Up%sL# zE2OCd1GwFja}j`&x7(VorMSaHO`OXBB&@=ezdNV@-+`3AJ171tY{{uK;p!b<5a)!x zbLu~rY7TVGgDb#o1N2whQbvf<S>L$|;G)E+Zkua{8^rK61PiWkr^<KVB97<M#$GUh zlUn`7-J@RgygGqO!10~S5(>=4R{lYb37=4$Pzz~qKtR{IE@3IZRtzv&mN0kBI2R^> z<UZ-|QXa)VIbym7$#wIcixNJmdR7R+&KuUI0Xvy-u1Z+r`JB?*3b9)dX?LzlxT8SZ zMd!kVe#t&S&bIGdnNZ*o!?-ujbqP7GPP=H)>7OmhZy{K41v+VARb2=GY3J&Mt)KC> z%dEXX0M}&r&SeUG!|#h*A;M(N<q9gdmv5>$ExQ8>2o_v{?{*Hd_RbC-%2)WZ)4Z#j z79vdRTrcr(Ds`en0D=SOdWpwLIB%K@2o_wykKsE4l?disIKkWHQaYI+v|Vv;n~ec> zGUr@9!9ng)@SW=?)SZcxw{xB|(ttpYb49~>7fT@${E66nzJg%ES@-#>IFa8jaecBL z+}!P5cHDKS-l)K5oYXm<;fCgquPxsX@2Ho&r_vKrFH-OEwd?rtZ&MO3Kl~TUeKbyl Gl>h)`PGWcf literal 0 HcmV?d00001 diff --git a/libkdd/tests/stackshot-sample-thread-policy b/libkdd/tests/stackshot-sample-thread-policy new file mode 100644 index 0000000000000000000000000000000000000000..e74ba80691a8f8918ef67d30dc357a1950eef681 GIT binary patch literal 1274 zcmV<W1O@vaiwFQ;??PDs1C>{OOdDkwe+w-XVY5JSgJf_nSp+lS3KTX7&MY7xxRDL8 z#>jE&9Z)FO+7CW-$vS0|WXn>cQ!~HVhS?U2l3=1{TSkl{(||golEptnjoGqf3+jTH zP4T_=<I=u6SbZ+nU!V8+xaWO-&wGVr$BCx^ln}s@0B;`Nhl%bFh|XSrw<jRD6rs~C zcKJb!q;du@VYGrv1~V9exN5xDB>_eZ@V=911tT#28pK2rKSJY&`~kU7fN27<-t!X( z*2DOE7{4(E087;Kjqpx`>-o^WlwbuG#;z*o;JO*wPeZ&P+QV@95HZ(?z5$oQHOf9k zc5`jA&+qamoUNjWFXAh?mhK?;co6D#&Q@Ak%2(R$TysqeSHjyWsyMee;FA@nEJ&iF zh?Rp%u&}IbYk6TsVOMFVBuczku|+DBT)x0i@%Gk+rkY~Uo}S{G+P2!Joy||w?r0Gl zTgrv<GGMTRZeRxMA+Alq{h@sI2VB*5&gbz2+_K+a&2y5xC)icZ58BFPu2=F3ePY1b zEhu82TrDd+D8XCS`fDJ!8D9Co3Q8IK72)J$c2vLZjJ@FE=HE=aOdxwGO;tHFtdEPT z56mYOtFU-(`p&h<((jI0r*5A<_>KVA-(B7ZKqkp&Anu7N2)OA%oX?bkb4AyS(TwoL zkB&T~;)k=_o>%eK&FyHOLFiBH$luax|05Rw9~`i~Fi8&715r8MVo-5*OGT30FS|Y7 zK3PFlu8Gu9PtXAC8I9WCRTKGQX+Wd+G#yCGBM4@QJrl%=llpbk;cvVl)*+1J%QfAB z9Ma5GFGBhpwNKnedT%n(y@#~ts{MV2q&JfOUhl{INh1BF`=NfX|8g{R@Ve<Ar|`x8 zeap+s;?*ReCucAJJ*Hxt^y!j{1`A&AoO4X5bF)l@?rsyjOt2@(!_T7pAVeCm>o|gJ zaz1H2KUO`A`xV)SFdyLgQr{B~K;P$HTKkncM-4|FM92!uR|Yk3xS}U%VGH@#pQ!gS zdD^Lg><=Z4mX3nE9}dN(#o>K4YL;~LnXB3HDYFul@<91(kL6DjmOtvZZtnrq--%ft z2*Lipz|@sW@Uz{x|C37_5Ylqa=Y2c99^B1e9=we3rCJvu{LwdX#(o4We{T?skhx^} z%Q%4fvtL-8)#H!$qn+tTB+h<Nzf(WQ&a-S}&Kr@zfkt#NY@2`0L;WCpAAe=Tnbqcb z-3L8!&GXczM1&U$%)0&fzw;cVhA?jMn^f;ei?uVN`Gu499Y{PY7VW&?HyTF`^LXD2 znj@ZX%3p-xZ#+)^sNWpJU-Jt5Isa`%NB6Z8t*A80-wz9yb>~@oAC1MwpYR^GXV%5X zAJ5ds^7lS9gz<@VQGE*MKSSe~JU#S_mz-n^<M53c=!umkYR1guQUlrdq81;^-#Eix zn54fKf7I{P+1}x3{-*kL%L4;DM$s?kC+^xcDpLMN|Ju0f`SWBQ9*&2<kG(kZyo}G` zz4#09-<#Bk&yA=P7M8zsTDV1j9@BMKqhy7+l_c)x5j*9N^1uN+lWLc2>Ox1>8r5_6 zmoq0@c9t9+D<4^SwQAl0n!iT--fAkjk@Beh=CPT_-paZ#-b(Y)r{^&Jjl7L5m#$!3 k`Tqe=Frf1XeJcK4w)~WX`MKeE`S)o1e`Xy@XRr(a02fZ3KL7v# literal 0 HcmV?d00001 diff --git a/libkdd/tests/stackshot-sample-thread-policy.plist.gz b/libkdd/tests/stackshot-sample-thread-policy.plist.gz new file mode 100644 index 0000000000000000000000000000000000000000..bcd465fa9216e9bcb2039b116efdd9678d6f31ce GIT binary patch literal 2631 zcmV-N3b^$jiwFQ_??PDs1MOUGQ`|Tb{+#>@FW+JGE|~+iHVapkJ8qY%g{`|!E@K<U zhsU1y1;YLMmFErHdKt|OflF#qNrlAjR;$(BdY+bMPJaG6$-Kv;tkNPsJ%C5xz)SL| zh|~Q3^x)?D?}y^x=d+I|Kb`;n>-B%GF1%@$R<(C^^Xc;QuinAoX!Of;nkA#r`SrPX zb@};^YftJJjV^vW@D9dxJv|<co}QkL!uGABsF<`bR->!3m?mZYd?}S3N`a%ejt}H8 zbL~BI<p0KLRG)qPaPp8mpFKozScgGXhtWedF6xs}`@izx;-&dpw?$D0VR>IIAAdNh z>Qc_>?6FQysaMM9SyEM}z>AaH=Kd6Xg(UGF;%P7m>u4P0;Uqata&R(Q$SpNLBxRmt z!8E*2f-3!A@-jf0*U5cSo)O@)lhN{DOSwrXwT8D<ku`M^)agVc2_e8j04Nps1`Vy? zZCE8in%@;K8kUovzZoZaF<CwA=<g|=zneFcq+I*%$*AYvqFOAG;@<PHd`ffgpEECc zf6m|XAeT!Fj=k$~<Nc$N|E3<q#~2(_>Rp{*dk8@OH!lmHN*Q}e5Qljl9)Hc7LqdFZ z$Pe%FT^z<B<RU(dQyDj-PdA@0&qu}We@EvRe_mYve)X@5->!py3l=b<tXVY<%OnmW z=|HLcIg5j=2xF@Y(}`T*OZnC>aa_tk(Rhdp;d7rur19!Z{EsY+KgqQ+tAGS>Oa+4k zQXeyuqMMt~=N;*=EW`PaFPab;NOuMi!uZHHK0*{3A7SGo$bHiz3XS@PF6`)|#3%*K zvof1*wgNJ<o3POeNR3iH9jxbQwQz%US{1V(ZpyIk<<)8+Kz#tzs-VgSg{&~8MuYV8 zAQT1~(mc4!!fw%A#X9C#Q<8e3(=^r$Y7>%%ZaUy|RVS0S`pY6}5Jjq39aaxPm50-I z*vgdiu#;n}CyR=zFB<)1>`JXIBi%?L6cWOrQUNSQU^sL~o4|^eZq^KQqyKam==}__ zX%;q7r7ZL+!!VQkCDJAenQc_1c=VH6^>(+O+Kn1!eQ#Ks#R%DqQc88NY<%cej$Psn z`$+x~Kr`Cr@kY7L7u&8o!Zx)w58Fe+njF}+tm<C$!8vufHJYe35{ElhPt><Q<<^8% z*Tx)cy(jY^DQ0C@;`KD@##uKg=QX<uW{^E9n!N5tRA-8cDLu&^&h83gj5ySU@k%<C zb-zr)xU!56DO8-OQD8PgV1k&shCLlmWm+dTD;QALN~geR1p$m>Wb}kwt!bGSQpvN% zh+bxQtDG9sozOEsQ=|Q5o*HNOqfTS7GHboTSV(9JH2MFMhg@jtuZ!IT0x%?YM%yPz z9j=LaD160%inwvLeTW6*#*oV;3X}|0av)pQ0jA7Xqrhl`Bo9b%br)UgSZjb<PY0sL zp%J12B4rcq6)Xk!*=khXrsh*{7dBa48M0jOth&w5IH<>EQP-KlQ&8_DDQg*#`$)lR zOR-Gi-#mH_z7%>{qpg$htE^{L8Yg*eSy~1F2HZIORC;SN>4`2VcNR8zG>(tr<T1&L z=_F~_{)LYBbja<altv_x-Ui}xN(HxAAyG=2>m4A3S**iTdIOpSWI*<*MlXmFK|-I& zt!PtClRUO0fpMi4RHtqDwCqY!x=EVLWp886l+Rq&H(!MWOO^9<FfKavsQ~6sZSwR> zlWgw3rb<fvG)lc8jA8kk69oV+H9m#aD&h3TAf+3kl!kG;t!qf3raGWmN1-!}g1e(> zGn?W7aChJ95!hxB^}X6A!GsRCwQtXSX%Y4%tXC0Xk2KiPo8D*>g0pU&b`0kzY5{OP zpzEZ<VgvJ;($yL&F=4n3reZ*GZW^o@PT^;E?^p-i{$U>uxq&&0<%U&xW8=+e&M^dy zc+NclZHUg%3Ja!lx5L2(O9ZbUUW35Z;x(w<>&jO>u7cT_$ua~DV{1DOO9fGp*J<AL zyR!r53qlwtm?9`JLr4&%+uYR{f04yJ4YRzNZ0N~WI=Q<`qB?z)vnsMQGR+AB%F(dx zJJ2)$kzH_O5tAuhr`@9?XVbH=swEJOlb~$!JiN^Y=V4l}wkIVV_>@AXb_<5erCzH_ z2oQvlEK=nVp{uG)B;Ki7sA6|9!$R9FT<O`)!L3+OJSJr+(fSr{uA;FtT|urJN$-*| zI@sKiCBU~zw=I5?$8B=Ym2K(g=!$8ZuuBz-^3un-T^;HHlSbTFqT40~!i*x{Cq9D^ zb6XVDMIC1CtfDZ>Ouj@y)0*fgSwV0cZ1j<BJsyijMnoCRm}7dkUFJW9X&p#rXx_iN z&f&y|2y5C*>W(m*;Ot;2xGO^ihY$Ak-WD-ekPW8O<)O0jFh)UJhM+EPM*S^kEA5c^ zm=RrI@1nhJ^a~`TOpA7UUbLJtw7PM74D&_|h(L}s-7!ZBggD~@>w4SYo&*^)*?|Q? z2nke9!n+C_xsYkEcjiKJqg5Npn{y#dOf1%&4Iw5Zc=;Bb$i@xe_Ot?P54B#G6)6{t zG|#u60pbu+<?@GR7EbgC3Df@M$d4c+Z6Vv4Az@}h56%<`amtwH>~v3(_9ST__I;D2 z_IOpoFr}H=RkbHcKORY105GCOyRIw=YcK9>PLkyI&l$wlD&a(u2t~wd(>j*4K$cW{ znd#klk`NdYBB&GM-SZ@^_TA}G3;W)P9zkDw!^wpkK@F~ZL+D^j;f<gJ_8CMB66=<| z9y+L3Q8vd862Azph06}m!B$_tXXpTAiW0#I)!q%<6gpt2xr*Qh9c&K%9-xDD6SqSL ztLV4E4Q*vY)}&es8npCzJJ7%((`<?xf(D3f2CYL3vfffFh<XRKfB*;J|EjABluw~3 z=4_{3?hish?V0Giq6N~n*WL&%P>I8ZDJ9qTQUODn()$*zRO@Ywb-f}Xh<cByBk$Nr z-Thh#@hMwti9;_H8Y`K5FI8p&uyw7rZl!*!`z88Gzj80tut2Y2ztQ14HByaT6xT*- zd-83I1II?Ht}txaNR^u_Gatq78>#z7>JPz&X17fU2T0qu*pv<J8>!o{q4#N|&cPQm z;x89UM2u&e_-F=jS5I8h=df7vFqwv9Rim011s%KP%7wu`Du_I(%i4SCVqT{58r z5$z|XU3d}XSbFk(c+n0Q?X7#qCb#V^N9c}MOep8tOG%sZBCtCzqH-~9UfG%#G0i!V z;|&x$F9N#K`MSJ_x5P-Z@wneYu}(oQKh=Kzem82g>m3uwWO=lNf-^O8yQgBMMi?Tn zp4{xI(ViN8$JA(kP{a_m_Wk$N=m(@mGyDboU9OnS@ZxJxBfz!arLieBvIgMV^P(B9 z6vm6Q?@5g`U;eYFMy}em-xuV_jXFPfzBjD@4PuK}pO?o#`EtIM89ghO3l_ETxc%O} pWLiXHRciK0d<GzGuRr;)7Y9o_UjCkah2z=BzW})J&e>T~006D&3b6nH literal 0 HcmV?d00001 diff --git a/libkern/OSKextLib.cpp b/libkern/OSKextLib.cpp index 00264ecf3..5b9ee7b4a 100644 --- a/libkern/OSKextLib.cpp +++ b/libkern/OSKextLib.cpp @@ -439,10 +439,10 @@ void kext_dump_panic_lists(int (*printf_func)(const char * fmt, ...)) void kmod_panic_dump(vm_offset_t * addr, unsigned int cnt) { - extern int kdb_printf(const char *format, ...) __printflike(1,2); + extern int paniclog_append_noflush(const char *format, ...) __printflike(1,2); + + OSKext::printKextsInBacktrace(addr, cnt, &paniclog_append_noflush, 0); - OSKext::printKextsInBacktrace(addr, cnt, &kdb_printf, - /* takeLock? */ false, false); return; } @@ -455,7 +455,9 @@ kmod_dump_log( unsigned int cnt, boolean_t doUnslide) { - OSKext::printKextsInBacktrace(addr, cnt, &printf, /* lock? */ true, doUnslide); + uint32_t flags = OSKext::kPrintKextsLock; + if (doUnslide) flags |= OSKext::kPrintKextsUnslide; + OSKext::printKextsInBacktrace(addr, cnt, &printf, flags); } void * diff --git a/libkern/c++/OSData.cpp b/libkern/c++/OSData.cpp index a542ee603..fda3dd7c9 100644 --- a/libkern/c++/OSData.cpp +++ b/libkern/c++/OSData.cpp @@ -76,8 +76,8 @@ bool OSData::initWithCapacity(unsigned int inCapacity) if (inCapacity < page_size) data = (void *) kalloc_container(inCapacity); else { kern_return_t kr; - inCapacity = round_page_32(inCapacity); - kr = kmem_alloc(kernel_map, (vm_offset_t *)&data, inCapacity, IOMemoryTag(kernel_map)); + if (round_page_overflow(inCapacity, &inCapacity)) kr = KERN_RESOURCE_SHORTAGE; + else kr = kmem_alloc(kernel_map, (vm_offset_t *)&data, inCapacity, IOMemoryTag(kernel_map)); if (KERN_SUCCESS != kr) data = NULL; } if (!data) diff --git a/libkern/c++/OSKext.cpp b/libkern/c++/OSKext.cpp index d30dbe5b4..797fd38a2 100644 --- a/libkern/c++/OSKext.cpp +++ b/libkern/c++/OSKext.cpp @@ -71,6 +71,8 @@ extern "C" { #include <IOKit/IOStatisticsPrivate.h> #include <IOKit/IOBSD.h> +#include <san/kasan.h> + #if PRAGMA_MARK #pragma mark External & Internal Function Protos #endif @@ -121,6 +123,9 @@ static void * GetAppleTEXTHashForKext(OSKext * theKext, OSDictionary *theInfoDic #define VM_MAPPED_KEXTS 1 #define KASLR_KEXT_DEBUG 0 #define KASLR_IOREG_DEBUG 0 +#elif __arm__ || __arm64__ +#define VM_MAPPED_KEXTS 0 +#define KASLR_KEXT_DEBUG 0 #else #error Unsupported architecture #endif @@ -314,7 +319,7 @@ kmod_info_t g_kernel_kmod_info = { /* version */ "0", // filled in in OSKext::initialize() /* reference_count */ -1, // never adjusted; kernel never unloads /* reference_list */ NULL, - /* address */ NULL, + /* address */ 0, /* size */ 0, // filled in in OSKext::initialize() /* hdr_size */ 0, /* start */ 0, @@ -681,6 +686,10 @@ OSKext::initialize(void) } PE_parse_boot_argn("keepsyms", &sKeepSymbols, sizeof(sKeepSymbols)); +#if KASAN_DYNAMIC_BLACKLIST + /* needed for function lookup */ + sKeepSymbols = true; +#endif /* Set up an OSKext instance to represent the kernel itself. */ @@ -812,6 +821,11 @@ OSKext::removeKextBootstrap(void) kernel_segment_command_t * seg_to_remove = NULL; +#if __arm__ || __arm64__ + const char * dt_segment_name = NULL; + void * segment_paddress = NULL; + int segment_size = 0; +#endif /* This must be the very first thing done by this function. */ @@ -854,7 +868,21 @@ OSKext::removeKextBootstrap(void) OSRuntimeUnloadCPPForSegment(seg_to_remove); } -#if __i386__ || __x86_64__ +#if __arm__ || __arm64__ +#if !(defined(KERNEL_INTEGRITY_KTRR)) + /* Free the memory that was set up by bootx. + */ + dt_segment_name = "Kernel-__KLD"; + if (0 == IODTGetLoaderInfo(dt_segment_name, &segment_paddress, &segment_size)) { + /* We cannot free this with KTRR enabled, as we cannot + * update the permissions on the KLD range this late + * in the boot process. + */ + IODTFreeLoaderInfo(dt_segment_name, (void *)segment_paddress, + (int)segment_size); + } +#endif /* !(defined(KERNEL_INTEGRITY_KTRR)) */ +#elif __i386__ || __x86_64__ /* On x86, use the mapping data from the segment load command to * unload KLD directly. * This may invalidate any assumptions about "avail_start" @@ -890,6 +918,9 @@ OSKext::removeKextBootstrap(void) * managed memory, then copy the segment back in. */ #if CONFIG_KXLD +#if (__arm__ || __arm64__) +#error CONFIG_KXLD not expected for this arch +#endif if (!sKeepSymbols) { kern_return_t mem_result; void *seg_copy = NULL; @@ -930,6 +961,8 @@ OSKext::removeKextBootstrap(void) &seg_offset, seg_length, /* mask */ 0, VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_NONE, (ipc_port_t)NULL, (vm_object_offset_t) 0, /* copy */ FALSE, @@ -956,7 +989,9 @@ OSKext::removeKextBootstrap(void) kmem_free(kernel_map, seg_copy_offset, seg_length); } #else /* we are not CONFIG_KXLD */ +#if !(__arm__ || __arm64__) #error CONFIG_KXLD is expected for this arch +#endif /***** * Dump the LINKEDIT segment, unless keepsyms is set. @@ -3427,8 +3462,20 @@ bool OSKext::isKextWithIdentifierLoaded(const char * kextIdentifier) OSReturn OSKext::removeKext( OSKext * aKext, +#if CONFIG_EMBEDDED + __unused +#endif bool terminateServicesAndRemovePersonalitiesFlag) { +#if CONFIG_EMBEDDED + OSKextLog(aKext, + kOSKextLogErrorLevel | + kOSKextLogKextBookkeepingFlag, + "removeKext() called for %s, not supported on embedded", + aKext->getIdentifier() ? aKext->getIdentifierCString() : "unknown kext"); + + return kOSReturnSuccess; +#else /* CONFIG_EMBEDDED */ OSReturn result = kOSKextReturnInUse; OSKext * checkKext = NULL; // do not release @@ -3524,6 +3571,7 @@ OSKext::removeKext( finish: IORecursiveLockUnlock(sKextLock); return result; +#endif /* CONFIG_EMBEDDED */ } /********************************************************************* @@ -4112,9 +4160,30 @@ finish: /********************************************************************* *********************************************************************/ +#if defined (__arm__) +#include <arm/arch.h> +#endif #if defined (__x86_64__) #define ARCHNAME "x86_64" +#elif defined (__arm64__) +#define ARCHNAME "arm64" +#elif defined (__arm__) + +#if defined (__ARM_ARCH_7S__) +#define ARCHNAME "armv7s" +#elif defined (__ARM_ARCH_7F__) +#define ARCHNAME "armv7f" +#elif defined (__ARM_ARCH_7K__) +#define ARCHNAME "armv7k" +#elif defined (_ARM_ARCH_7) /* umbrella for all remaining */ +#define ARCHNAME "armv7" +#elif defined (_ARM_ARCH_6) /* umbrella for all armv6 */ +#define ARCHNAME "armv6" +#endif + +#elif defined (__arm64__) +#define ARCHNAME "arm64" #else #error architecture not supported #endif @@ -4700,6 +4769,7 @@ OSKext::load( } bzero(account, sizeof(*account)); account->loadTag = kmod_info->id; + account->site.refcount = 0; account->site.flags = VM_TAG_KMOD; account->kext = this; @@ -5404,6 +5474,21 @@ register_kmod: goto finish; } +#if KASAN + kasan_load_kext((vm_offset_t)linkedExecutable->getBytesNoCopy(), + linkedExecutable->getLength(), getIdentifierCString()); +#else + if (lookupSection(KASAN_GLOBAL_SEGNAME, KASAN_GLOBAL_SECTNAME)) { + OSKextLog(this, + kOSKextLogErrorLevel | kOSKextLogLoadFlag, + "KASAN: cannot load KASAN-ified kext %s on a non-KASAN kernel\n", + getIdentifierCString() + ); + result = KERN_FAILURE; + goto finish; + } +#endif + result = kOSReturnSuccess; finish: @@ -5679,7 +5764,40 @@ OSKext::unregisterWithDTrace(void) * called only by loadExecutable() *********************************************************************/ #if !VM_MAPPED_KEXTS +#if defined(__arm__) || defined(__arm64__) +static inline kern_return_t +OSKext_protect( + vm_map_t map, + vm_map_offset_t start, + vm_map_offset_t end, + vm_prot_t new_prot, + boolean_t set_max) +{ +#pragma unused(map) + assert(map == kernel_map); // we can handle KEXTs arising from the PRELINK segment and no others + assert(start <= end); + if (start >= end) + return KERN_SUCCESS; // Punt segments of length zero (e.g., headers) or less (i.e., blunders) + else if (set_max) + return KERN_SUCCESS; // Punt set_max, as there's no mechanism to record that state + else + return ml_static_protect(start, end - start, new_prot); +} + +static inline kern_return_t +OSKext_wire( + vm_map_t map, + vm_map_offset_t start, + vm_map_offset_t end, + vm_prot_t access_type, + boolean_t user_wire) +{ +#pragma unused(map,start,end,access_type,user_wire) + return KERN_SUCCESS; // No-op as PRELINK kexts are cemented into physical memory at boot +} +#else #error Unrecognized architecture +#endif #else static inline kern_return_t OSKext_protect( @@ -5703,7 +5821,7 @@ OSKext_wire( vm_prot_t access_type, boolean_t user_wire) { - return vm_map_wire(map, start, end, access_type | VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_KEXT), user_wire); + return vm_map_wire_kernel(map, start, end, access_type, VM_KERN_MEMORY_KEXT, user_wire); } #endif @@ -5754,6 +5872,11 @@ OSKext::setVMAttributes(bool protect, bool wire) seg = firstsegfromheader((kernel_mach_header_t *)kmod_info->address); while (seg) { +#if __arm__ + /* We build all ARM kexts, so we can ensure they are aligned */ + assert((seg->vmaddr & PAGE_MASK) == 0); + assert((seg->vmsize & PAGE_MASK) == 0); +#endif start = round_page(seg->vmaddr); end = trunc_page(seg->vmaddr + seg->vmsize); @@ -6332,7 +6455,17 @@ OSKext::unload(void) goto finish; } - if (hasOSMetaClassInstances()) { + if (!isLoaded()) { + result = kOSReturnSuccess; + goto finish; + } + + if (isKernelComponent()) { + result = kOSKextReturnInvalidArgument; + goto finish; + } + + if (metaClasses && !OSMetaClass::removeClasses(metaClasses)) { OSKextLog(this, kOSKextLogErrorLevel | kOSKextLogLoadFlag | kOSKextLogKextBookkeepingFlag, @@ -6343,16 +6476,6 @@ OSKext::unload(void) result = kOSKextReturnInUse; goto finish; } - - if (!isLoaded()) { - result = kOSReturnSuccess; - goto finish; - } - - if (isKernelComponent()) { - result = kOSKextReturnInvalidArgument; - goto finish; - } /* Note that the kext is unloading before running any code that * might be in the kext (request callbacks, module stop function). @@ -6474,6 +6597,10 @@ OSKext::unload(void) /* Unwire and free the linked executable. */ if (linkedExecutable) { +#if KASAN + kasan_unload_kext((vm_offset_t)linkedExecutable->getBytesNoCopy(), linkedExecutable->getLength()); +#endif + #if VM_MAPPED_KEXTS if (!isInterface()) { kernel_segment_command_t *seg = NULL; @@ -8234,7 +8361,6 @@ size_t OSKextPgoMetadataSize(OSKext *kext) return position; } - int OSKextGrabPgoDataLocked(OSKext *kext, bool metadata, uuid_t instance_uuid, @@ -8242,7 +8368,6 @@ int OSKextGrabPgoDataLocked(OSKext *kext, char *pBuffer, uint64_t bufferSize) { - int err = 0; kernel_section_t *sect_prf_data = NULL; @@ -8767,6 +8892,7 @@ OSKext::copyInfo(OSArray * infoKeys) segp->filesize = 0; } } + #if 0 OSKextLog(/* kext */ NULL, kOSKextLogErrorLevel | @@ -8788,7 +8914,7 @@ OSKext::copyInfo(OSArray * infoKeys) } #endif segp->vmaddr = VM_KERNEL_UNSLIDE(segp->vmaddr); - + for (secp = firstsect(segp); secp != NULL; secp = nextsect(segp, secp)) { secp->addr = VM_KERNEL_UNSLIDE(secp->addr); } @@ -9472,6 +9598,17 @@ finish: return result; } + +/********************************************************************* +* Busy timeout triage +*********************************************************************/ +/* static */ +bool +OSKext::isWaitingKextd(void) +{ + return sRequestCallbackRecords && sRequestCallbackRecords->getCount(); +} + /********************************************************************* * Assumes sKextLock is held. *********************************************************************/ @@ -10741,15 +10878,14 @@ OSKext::printKextsInBacktrace( vm_offset_t * addr, unsigned int cnt, int (* printf_func)(const char *fmt, ...), - bool lockFlag, - bool doUnslide) + uint32_t flags) { addr64_t summary_page = 0; addr64_t last_summary_page = 0; bool found_kmod = false; u_int i = 0; - if (lockFlag) { + if (kPrintKextsLock & flags) { if (!sKextSummariesLock) return; IOLockLock(sKextSummariesLock); } @@ -10782,15 +10918,17 @@ OSKext::printKextsInBacktrace( } if (!found_kmod) { - (*printf_func)(" Kernel Extensions in backtrace:\n"); + if (!(kPrintKextsTerse & flags)) { + (*printf_func)(" Kernel Extensions in backtrace:\n"); + } found_kmod = true; } - printSummary(summary, printf_func, doUnslide); + printSummary(summary, printf_func, flags); } finish: - if (lockFlag) { + if (kPrintKextsLock & flags) { IOLockUnlock(sKextSummariesLock); } @@ -10919,7 +11057,7 @@ static void findSummaryUUID( void OSKext::printSummary( OSKextLoadedKextSummary * summary, int (* printf_func)(const char *fmt, ...), - bool doUnslide) + uint32_t flags) { kmod_reference_t * kmod_ref = NULL; uuid_string_t uuid; @@ -10931,15 +11069,18 @@ void OSKext::printSummary( } (void) uuid_unparse(summary->uuid, uuid); - if (doUnslide) { + if (kPrintKextsUnslide & flags) { tmpAddr = VM_KERNEL_UNSLIDE(summary->address); } else { tmpAddr = summary->address; } - (*printf_func)(" %s(%s)[%s]@0x%llx->0x%llx\n", + (*printf_func)("%s%s(%s)[%s]@0x%llx->0x%llx\n", + (kPrintKextsTerse & flags) ? "" : " ", summary->name, version, uuid, tmpAddr, tmpAddr + summary->size - 1); + + if (kPrintKextsTerse & flags) return; /* print dependency info */ for (kmod_ref = (kmod_reference_t *) summary->reference_list; @@ -10950,7 +11091,7 @@ void OSKext::printSummary( if (pmap_find_phys(kernel_pmap, (addr64_t)((uintptr_t)kmod_ref)) == 0) { (*printf_func)(" kmod dependency scan stopped " "due to missing dependency page: %p\n", - doUnslide ? (void *)VM_KERNEL_UNSLIDE(kmod_ref) : kmod_ref); + (kPrintKextsUnslide & flags) ? (void *)VM_KERNEL_UNSLIDE(kmod_ref) : kmod_ref); break; } rinfo = kmod_ref->info; @@ -10958,7 +11099,7 @@ void OSKext::printSummary( if (pmap_find_phys(kernel_pmap, (addr64_t)((uintptr_t)rinfo)) == 0) { (*printf_func)(" kmod dependency scan stopped " "due to missing kmod page: %p\n", - doUnslide ? (void *)VM_KERNEL_UNSLIDE(rinfo) : rinfo); + (kPrintKextsUnslide & flags) ? (void *)VM_KERNEL_UNSLIDE(rinfo) : rinfo); break; } @@ -10969,7 +11110,7 @@ void OSKext::printSummary( /* locate UUID in gLoadedKextSummaries */ findSummaryUUID(rinfo->id, uuid); - if (doUnslide) { + if (kPrintKextsUnslide & flags) { tmpAddr = VM_KERNEL_UNSLIDE(rinfo->address); } else { @@ -11597,11 +11738,14 @@ OSKextGetAllocationSiteForCaller(uintptr_t address) { OSKextActiveAccount * active; vm_allocation_site_t * site; + vm_allocation_site_t * releasesite; + uint32_t baseIdx; uint32_t lim; IOSimpleLockLock(sKextAccountsLock); - site = NULL; + site = releasesite = NULL; + // bsearch sKextAccounts list for (baseIdx = 0, lim = sKextAccountsCount; lim; lim >>= 1) { @@ -11609,7 +11753,7 @@ OSKextGetAllocationSiteForCaller(uintptr_t address) if ((address >= active->address) && (address < active->address_end)) { site = &active->account->site; - if (!site->tag) vm_tag_alloc_locked(site); + if (!site->tag) vm_tag_alloc_locked(site, &releasesite); break; } else if (address > active->address) @@ -11621,12 +11765,13 @@ OSKextGetAllocationSiteForCaller(uintptr_t address) // else move left } IOSimpleLockUnlock(sKextAccountsLock); + if (releasesite) kern_allocation_name_release(releasesite); return (site); } extern "C" uint32_t -OSKextGetKmodIDForSite(vm_allocation_site_t * site, char * name, vm_size_t namelen) +OSKextGetKmodIDForSite(const vm_allocation_site_t * site, char * name, vm_size_t namelen) { OSKextAccount * account = (typeof(account)) site; const char * kname; diff --git a/libkern/c++/OSMetaClass.cpp b/libkern/c++/OSMetaClass.cpp index 5e25aa5fd..3d7c2f6e4 100644 --- a/libkern/c++/OSMetaClass.cpp +++ b/libkern/c++/OSMetaClass.cpp @@ -110,6 +110,7 @@ IOLock * sStalledClassesLock = NULL; struct ExpansionData { OSOrderedSet * instances; OSKext * kext; + uint32_t retain; #if IOTRACKING IOTrackingQueue * tracking; #endif @@ -656,7 +657,11 @@ OSMetaClass::postModLoad(void * loadHandle) /* Log this error here so we can include the class name. * xxx - we should look up the other kext that defines the class */ +#if CONFIG_EMBEDDED + panic( +#else OSKextLog(myKext, kOSMetaClassLogSpec, +#endif /* CONFIG_EMBEDDED */ "OSMetaClass: Kext %s class %s is a duplicate;" "kext %s already has a class by that name.", sStalled->kextIdentifier, (const char *)me->className, @@ -944,6 +949,43 @@ OSMetaClass::considerUnloads() OSKext::considerUnloads(); } +/********************************************************************* +*********************************************************************/ +bool +OSMetaClass::removeClasses(OSCollection * metaClasses) +{ + OSCollectionIterator * classIterator; + OSMetaClass * checkClass; + bool result; + + classIterator = OSCollectionIterator::withCollection(metaClasses); + if (!classIterator) return (false); + + IOLockLock(sAllClassesLock); + + result = false; + do + { + while ((checkClass = (OSMetaClass *)classIterator->getNextObject()) + && !checkClass->getInstanceCount() + && !checkClass->reserved->retain) {} + if (checkClass) break; + classIterator->reset(); + while ((checkClass = (OSMetaClass *)classIterator->getNextObject())) + { + sAllClassesDict->removeObject(checkClass->className); + } + result = true; + } + while (false); + + IOLockUnlock(sAllClassesLock); + OSSafeReleaseNULL(classIterator); + + return (result); +} + + /********************************************************************* *********************************************************************/ const OSMetaClass * @@ -964,17 +1006,48 @@ OSMetaClass::getMetaClassWithName(const OSSymbol * name) return retMeta; } +/********************************************************************* +*********************************************************************/ +const OSMetaClass * +OSMetaClass::copyMetaClassWithName(const OSSymbol * name) +{ + const OSMetaClass * meta; + + if (!name) return (0); + + meta = 0; + IOLockLock(sAllClassesLock); + if (sAllClassesDict) { + meta = (OSMetaClass *) sAllClassesDict->getObject(name); + if (meta) OSIncrementAtomic(&meta->reserved->retain); + } + IOLockUnlock(sAllClassesLock); + + return (meta); +} + +/********************************************************************* +*********************************************************************/ +void +OSMetaClass::releaseMetaClass() const +{ + OSDecrementAtomic(&reserved->retain); +} + /********************************************************************* *********************************************************************/ OSObject * OSMetaClass::allocClassWithName(const OSSymbol * name) { - OSObject * result = 0; - - const OSMetaClass * const meta = getMetaClassWithName(name); + const OSMetaClass * meta; + OSObject * result; - if (meta) { + result = 0; + meta = copyMetaClassWithName(name); + if (meta) + { result = meta->alloc(); + meta->releaseMetaClass(); } return result; @@ -1246,7 +1319,7 @@ void OSMetaClass::trackedInstance(OSObject * instance) const { IOTracking * mem = (typeof(mem)) instance; mem--; - return (IOTrackingAdd(reserved->tracking, mem, classSize, false)); + return (IOTrackingAdd(reserved->tracking, mem, classSize, false, VM_KERN_MEMORY_NONE)); } void OSMetaClass::trackedFree(OSObject * instance) const diff --git a/libkern/c++/OSUnserializeXML.cpp b/libkern/c++/OSUnserializeXML.cpp index 7df203dc7..e44bee927 100644 --- a/libkern/c++/OSUnserializeXML.cpp +++ b/libkern/c++/OSUnserializeXML.cpp @@ -159,7 +159,8 @@ #include <libkern/c++/OSContainers.h> #include <libkern/c++/OSLib.h> -#define MAX_OBJECTS 65535 +#define MAX_OBJECTS 131071 +#define MAX_REFED_OBJECTS 65535 #define YYSTYPE object_t * #define YYPARSE_PARAM state @@ -192,6 +193,7 @@ typedef struct parser_state { OSString **errorString; // parse error with line OSObject *parsedObject; // resultant object of parsed text int parsedObjectCount; + int retrievedObjectCount; } parser_state_t; #define STATE ((parser_state_t *)state) @@ -1632,6 +1634,11 @@ yyreduce: #line 246 "OSUnserializeXML.y" { (yyval) = retrieveObject(STATE, (yyvsp[(1) - (1)])->idref); if ((yyval)) { + STATE->retrievedObjectCount++; + if (STATE->retrievedObjectCount > MAX_REFED_OBJECTS) { + yyerror("maximum object reference count"); + YYERROR; + } (yyval)->object->retain(); } else { yyerror("forward reference detected"); @@ -2835,6 +2842,7 @@ OSUnserializeXML(const char *buffer, OSString **errorString) state->errorString = errorString; state->parsedObject = 0; state->parsedObjectCount = 0; + state->retrievedObjectCount = 0; (void)yyparse((void *)state); diff --git a/libkern/c++/Tests/TestSerialization/test1/test1_main.cpp b/libkern/c++/Tests/TestSerialization/test1/test1_main.cpp old mode 100755 new mode 100644 diff --git a/libkern/conf/Makefile.arm b/libkern/conf/Makefile.arm new file mode 100644 index 000000000..04938ae54 --- /dev/null +++ b/libkern/conf/Makefile.arm @@ -0,0 +1,20 @@ +###################################################################### +#BEGIN Machine dependent Makefile fragment for arm +###################################################################### + +# The following files cast opaque pointers to more specific +# structures +OBJS_NO_CAST_ALIGN = kxld_kext.o kxld_reloc.o kxld_sect.o kxld_seg.o \ + kxld_state.o kxld_sym.o kxld_symtab.o kxld_util.o \ + kxld_srcversion.o kxld_splitinfolc.o kxld_uuid.o kxld_vtable.o uuid.o + +$(foreach file,$(OBJS_NO_CAST_ALIGN),$(eval $(call add_perfile_cflags,$(file),-Wno-cast-align))) + +OSKext.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align -Wno-error=shadow +OSMetaClass.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +OSKextLib.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +OSUnserialize.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align + +###################################################################### +#END Machine dependent Makefile fragment for arm +###################################################################### diff --git a/libkern/conf/Makefile.arm64 b/libkern/conf/Makefile.arm64 new file mode 100644 index 000000000..04938ae54 --- /dev/null +++ b/libkern/conf/Makefile.arm64 @@ -0,0 +1,20 @@ +###################################################################### +#BEGIN Machine dependent Makefile fragment for arm +###################################################################### + +# The following files cast opaque pointers to more specific +# structures +OBJS_NO_CAST_ALIGN = kxld_kext.o kxld_reloc.o kxld_sect.o kxld_seg.o \ + kxld_state.o kxld_sym.o kxld_symtab.o kxld_util.o \ + kxld_srcversion.o kxld_splitinfolc.o kxld_uuid.o kxld_vtable.o uuid.o + +$(foreach file,$(OBJS_NO_CAST_ALIGN),$(eval $(call add_perfile_cflags,$(file),-Wno-cast-align))) + +OSKext.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align -Wno-error=shadow +OSMetaClass.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +OSKextLib.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align +OSUnserialize.cpo_CXXWARNFLAGS_ADD += -Wno-cast-align + +###################################################################### +#END Machine dependent Makefile fragment for arm +###################################################################### diff --git a/libkern/conf/Makefile.template b/libkern/conf/Makefile.template index f434fc023..5291268bc 100644 --- a/libkern/conf/Makefile.template +++ b/libkern/conf/Makefile.template @@ -17,6 +17,7 @@ include $(MakeInc_def) # CFLAGS # CFLAGS+= -include meta_features.h -DLIBKERN_KERNEL_PRIVATE -DOSALLOCDEBUG=1 +SFLAGS+= -include meta_features.h # Objects that don't want -Wcast-align warning (8474835) OSKextLib.cpo_CXXWARNFLAGS_ADD = -Wno-cast-align diff --git a/libkern/conf/files b/libkern/conf/files index 6f4f78550..5867b9373 100644 --- a/libkern/conf/files +++ b/libkern/conf/files @@ -61,7 +61,6 @@ libkern/zlib/adler32.c optional zlib libkern/zlib/compress.c optional zlib libkern/zlib/crc32.c optional zlib libkern/zlib/deflate.c optional zlib -#libkern/zlib/gzio.c not needed for kernel optional zlib libkern/zlib/infback.c optional zlib libkern/zlib/inffast.c optional zlib libkern/zlib/inflate.c optional zlib @@ -79,6 +78,7 @@ libkern/crypto/corecrypto_aes.c optional crypto libkern/crypto/corecrypto_aesxts.c optional crypto libkern/crypto/corecrypto_rand.c optional crypto libkern/crypto/corecrypto_rsa.c optional crypto +libkern/crypto/corecrypto_chacha20poly1305.c optional crypto libkern/stack_protector.c standard diff --git a/libkern/conf/files.arm b/libkern/conf/files.arm new file mode 100644 index 000000000..28a08b01f --- /dev/null +++ b/libkern/conf/files.arm @@ -0,0 +1,3 @@ +libkern/zlib/arm/inffastS.s optional zlib +libkern/zlib/arm/adler32vec.s optional zlib + diff --git a/libkern/crypto/corecrypto_chacha20poly1305.c b/libkern/crypto/corecrypto_chacha20poly1305.c new file mode 100644 index 000000000..8957b0708 --- /dev/null +++ b/libkern/crypto/corecrypto_chacha20poly1305.c @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2017 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include <corecrypto/ccchacha20poly1305.h> +#include <libkern/crypto/crypto_internal.h> +#include <libkern/crypto/chacha20poly1305.h> + +static ccchacha20poly1305_fns_t fns(void) +{ + return g_crypto_funcs->ccchacha20poly1305_fns; +} + +static const struct ccchacha20poly1305_info *info(void) +{ + return fns()->info(); +} + +int chacha20poly1305_init(chacha20poly1305_ctx *ctx, const uint8_t *key) +{ + return fns()->init(info(), ctx, key); +} + +int chacha20poly1305_reset(chacha20poly1305_ctx *ctx) +{ + return fns()->reset(info(), ctx); +} + +int chacha20poly1305_setnonce(chacha20poly1305_ctx *ctx, const uint8_t *nonce) +{ + return fns()->setnonce(info(), ctx, nonce); +} + +int chacha20poly1305_incnonce(chacha20poly1305_ctx *ctx, uint8_t *nonce) +{ + return fns()->incnonce(info(), ctx, nonce); +} + +int chacha20poly1305_aad(chacha20poly1305_ctx *ctx, size_t nbytes, const void *aad) +{ + return fns()->aad(info(), ctx, nbytes, aad); +} + +int chacha20poly1305_encrypt(chacha20poly1305_ctx *ctx, size_t nbytes, const void *ptext, void *ctext) +{ + return fns()->encrypt(info(), ctx, nbytes, ptext, ctext); +} + +int chacha20poly1305_finalize(chacha20poly1305_ctx *ctx, uint8_t *tag) +{ + return fns()->finalize(info(), ctx, tag); +} + +int chacha20poly1305_decrypt(chacha20poly1305_ctx *ctx, size_t nbytes, const void *ctext, void *ptext) +{ + return fns()->decrypt(info(), ctx, nbytes, ctext, ptext); +} + +int chacha20poly1305_verify(chacha20poly1305_ctx *ctx, const uint8_t *tag) +{ + return fns()->verify(info(), ctx, tag); +} diff --git a/libkern/firehose/chunk_private.h b/libkern/firehose/chunk_private.h index ca5fe069a..b4fbcd74a 100644 --- a/libkern/firehose/chunk_private.h +++ b/libkern/firehose/chunk_private.h @@ -52,7 +52,8 @@ typedef union { uint8_t fcp_stream; uint8_t fcp_flag_full : 1; uint8_t fcp_flag_io : 1; - uint8_t _fcp_flag_unused : 6; + uint8_t fcp_quarantined : 1; + uint8_t _fcp_flag_unused : 5; }; } firehose_chunk_pos_u; diff --git a/libkern/firehose/firehose_types_private.h b/libkern/firehose/firehose_types_private.h index 785ac177d..ea1f91279 100644 --- a/libkern/firehose/firehose_types_private.h +++ b/libkern/firehose/firehose_types_private.h @@ -91,6 +91,7 @@ OS_ENUM(firehose_tracepoint_namespace, uint8_t, firehose_tracepoint_namespace_trace = 0x03, firehose_tracepoint_namespace_log = 0x04, firehose_tracepoint_namespace_metadata = 0x05, + firehose_tracepoint_namespace_signpost = 0x06, ); /*! @@ -130,13 +131,6 @@ OS_ENUM(firehose_tracepoint_flags, uint16_t, _firehose_tracepoint_flags_pc_style__unused6 = 0x0006 << 1, _firehose_tracepoint_flags_pc_style__unused7 = 0x0007 << 1, _firehose_tracepoint_flags_base_has_unique_pid = 0x0010, - - _firehose_tracepoint_flags_base_main_executable __deprecated = - _firehose_tracepoint_flags_pc_style_main_exe, - _firehose_tracepoint_flags_base_shared_cache __deprecated = - _firehose_tracepoint_flags_pc_style_shared_cache, - _firehose_tracepoint_flags_base_caller_pc __deprecated = - _firehose_tracepoint_flags_pc_style_absolute, ); /*! @@ -207,8 +201,8 @@ OS_ENUM(_firehose_tracepoint_type_log, firehose_tracepoint_type_t, OS_ENUM(_firehose_tracepoint_flags_log, uint16_t, _firehose_tracepoint_flags_log_has_private_data = 0x0100, _firehose_tracepoint_flags_log_has_subsystem = 0x0200, - _firehose_tracepoint_flags_log_has_rules = 0x0400, - _firehose_tracepoint_flags_log_has_oversize = 0x0800, + _firehose_tracepoint_flags_log_has_rules = 0x0400, + _firehose_tracepoint_flags_log_has_oversize = 0x0800, ); /*! @@ -223,6 +217,36 @@ OS_ENUM(_firehose_tracepoint_type_metadata, firehose_tracepoint_type_t, _firehose_tracepoint_type_metadata_kext = 0x03, ); +/*! + * @enum firehose_tracepoint_type_signpost_t + * + * @abstract + * Types of Log tracepoints (namespace signpost). + */ +OS_ENUM(_firehose_tracepoint_type_signpost, firehose_tracepoint_type_t, + _firehose_tracepoint_type_signpost_event = 0x00, + _firehose_tracepoint_type_signpost_interval_begin = 0x01, + _firehose_tracepoint_type_signpost_interval_end = 0x02, + + _firehose_tracepoint_type_signpost_scope_mask = 0xc0, + _firehose_tracepoint_type_signpost_scope_thread = 0x40, + _firehose_tracepoint_type_signpost_scope_process = 0x80, + _firehose_tracepoint_type_signpost_scope_system = 0xc0, +); + +/*! + * @enum firehose_tracepoint_flags_signpost_t + * + * @abstract + * Flags for Log tracepoints (namespace signpost). + */ +OS_ENUM(_firehose_tracepoint_flags_signpost, uint16_t, + _firehose_tracepoint_flags_signpost_has_private_data = 0x0100, + _firehose_tracepoint_flags_signpost_has_subsystem = 0x0200, + _firehose_tracepoint_flags_signpost_has_rules = 0x0400, + _firehose_tracepoint_flags_signpost_has_oversize = 0x0800, +); + /* MIG firehose push reply structure */ typedef struct firehose_push_reply_s { uint64_t fpr_mem_flushed_pos; diff --git a/libkern/gen/OSDebug.cpp b/libkern/gen/OSDebug.cpp index 305cfc3cb..f11263631 100644 --- a/libkern/gen/OSDebug.cpp +++ b/libkern/gen/OSDebug.cpp @@ -56,6 +56,11 @@ extern boolean_t doprnt_hide_pointers; extern void kmod_dump_log(vm_offset_t *addr, unsigned int cnt, boolean_t doUnslide); extern addr64_t kvtophys(vm_offset_t va); +#if __arm__ +extern int copyinframe(vm_address_t fp, char *frame); +#elif defined(__arm64__) +extern int copyinframe(vm_address_t fp, char *frame, boolean_t is64bit); +#endif __END_DECLS @@ -96,7 +101,7 @@ void OSReportWithBacktrace(const char *str, ...) { char buf[128]; - void *bt[9]; + void *bt[9] = {}; const unsigned cnt = sizeof(bt) / sizeof(bt[0]); va_list listp; @@ -217,6 +222,42 @@ pad: for ( ; frame_index < maxAddrs; frame_index++) bt[frame_index] = (void *) 0; +#elif __arm__ || __arm64__ + uint32_t i = 0; + uintptr_t frameb[2]; + uintptr_t fp = 0; + + // get the current frame pointer for this thread +#if defined(__arm__) +#define OSBacktraceFrameAlignOK(x) (((x) & 0x3) == 0) + __asm__ volatile("mov %0,r7" : "=r" (fp)); +#elif defined(__arm64__) +#define OSBacktraceFrameAlignOK(x) (((x) & 0xf) == 0) + __asm__ volatile("mov %0, fp" : "=r" (fp)); +#else +#error Unknown architecture. +#endif + + // now crawl up the stack recording the link value of each frame + do { + // check bounds + if ((fp == 0) || (!OSBacktraceFrameAlignOK(fp)) || (fp > VM_MAX_KERNEL_ADDRESS) || (fp < VM_MIN_KERNEL_AND_KEXT_ADDRESS)) { + break; + } + // safely read frame +#ifdef __arm64__ + if (copyinframe(fp, (char*)frameb, TRUE) != 0) { +#else + if (copyinframe(fp, (char*)frameb) != 0) { +#endif + break; + } + + // No need to use copyin as this is always a kernel address, see check above + bt[i] = (void*)frameb[1]; // link register + fp = frameb[0]; + } while (++i < maxAddrs); + frame= i; #else #error arch #endif diff --git a/libkern/kmod/libkmodtest/libkmodtest.h b/libkern/kmod/libkmodtest/libkmodtest.h index cd0eb4401..3ce25ed12 100644 --- a/libkern/kmod/libkmodtest/libkmodtest.h +++ b/libkern/kmod/libkmodtest/libkmodtest.h @@ -36,4 +36,4 @@ class testlibkmod : public IOService { IOService *provider, SInt32 *score ); -}; \ No newline at end of file +}; diff --git a/libkern/kxld/kxld_object.c b/libkern/kxld/kxld_object.c index f8fc526bc..3413a2ce7 100644 --- a/libkern/kxld/kxld_object.c +++ b/libkern/kxld/kxld_object.c @@ -48,6 +48,7 @@ #include <mach-o/loader.h> #include <mach-o/nlist.h> #include <mach-o/reloc.h> +#include <os/overflow.h> #define DEBUG_ASSERT_COMPONENT_NAME_STRING "kxld" #include <AssertMacros.h> @@ -384,6 +385,14 @@ get_target_machine_info(KXLDObject *object, cpu_type_t cputype __unused, object->cpusubtype = CPU_SUBTYPE_X86_64_ALL; #endif return KERN_SUCCESS; +#elif defined(__arm__) + object->cputype = CPU_TYPE_ARM; + object->cpusubtype = CPU_SUBTYPE_ARM_ALL; + return KERN_SUCCESS; +#elif defined(__arm64__) + object->cputype = CPU_TYPE_ARM64; + object->cpusubtype = CPU_SUBTYPE_ARM64_ALL; + return KERN_SUCCESS; #else kxld_log(kKxldLogLinking, kKxldLogErr, kKxldLogArchNotSupported, _mh_execute_header->cputype); @@ -495,8 +504,10 @@ get_macho_slice_for_arch(KXLDObject *object, u_char *file, u_long size) if (fat->magic == FAT_MAGIC) { struct fat_arch *arch = NULL; + u_long arch_size; + boolean_t ovr = os_mul_and_add_overflow(fat->nfat_arch, sizeof(*archs), sizeof(*fat), &arch_size); - require_action(size >= (sizeof(*fat) + (fat->nfat_arch * sizeof(*archs))), + require_action(!ovr && size >= arch_size, finish, rval=KERN_FAILURE; kxld_log(kKxldLogLinking, kKxldLogErr, kKxldLogTruncatedMachO)); @@ -569,6 +580,7 @@ init_from_final_linked_image(KXLDObject *object, u_int *filetype_out, struct symtab_command *symtab_hdr = NULL; struct uuid_command *uuid_hdr = NULL; struct version_min_command *versionmin_hdr = NULL; + struct build_version_command *build_version_hdr = NULL; struct source_version_command *source_version_hdr = NULL; u_long base_offset = 0; u_long offset = 0; @@ -702,6 +714,10 @@ init_from_final_linked_image(KXLDObject *object, u_int *filetype_out, versionmin_hdr = (struct version_min_command *) cmd_hdr; kxld_versionmin_init_from_macho(&object->versionmin, versionmin_hdr); break; + case LC_BUILD_VERSION: + build_version_hdr = (struct build_version_command *)cmd_hdr; + kxld_versionmin_init_from_build_cmd(&object->versionmin, build_version_hdr); + break; case LC_SOURCE_VERSION: source_version_hdr = (struct source_version_command *) (void *) cmd_hdr; kxld_srcversion_init_from_macho(&object->srcversion, source_version_hdr); @@ -734,6 +750,9 @@ init_from_final_linked_image(KXLDObject *object, u_int *filetype_out, kxld_splitinfolc_init_from_macho(&object->splitinfolc, split_info_hdr); } break; + case LC_NOTE: + /* binary blob of data */ + break; case LC_CODE_SIGNATURE: case LC_DYLD_INFO: case LC_DYLD_INFO_ONLY: @@ -1052,6 +1071,15 @@ init_from_object(KXLDObject *object) case LC_DYLIB_CODE_SIGN_DRS: /* Various metadata that might be stored in the linkedit segment */ break; + case LC_NOTE: + /* bag-of-bits carried with the binary: ignore */ + break; + case LC_BUILD_VERSION: + /* should be able to ignore build version commands */ + kxld_log(kKxldLogLinking, kKxldLogWarn, + "Ignoring LC_BUILD_VERSION (%u) in MH_OBJECT kext: (platform:%d)", + cmd_hdr->cmd, ((struct build_version_command *)cmd_hdr)->platform); + break; case LC_VERSION_MIN_MACOSX: case LC_VERSION_MIN_IPHONEOS: case LC_VERSION_MIN_TVOS: @@ -1180,7 +1208,7 @@ get_macho_header_size(const KXLDObject *object) } if (object->versionmin.has_versionmin) { - header_size += kxld_versionmin_get_macho_header_size(); + header_size += kxld_versionmin_get_macho_header_size(&object->versionmin); } if (object->srcversion.has_srcversion) { diff --git a/libkern/kxld/kxld_util.c b/libkern/kxld/kxld_util.c index eee5dee72..480a454c4 100644 --- a/libkern/kxld/kxld_util.c +++ b/libkern/kxld/kxld_util.c @@ -784,7 +784,8 @@ kxld_strstr(const char *s, const char *find) #if KERNEL char c, sc; size_t len; - + if (!s || !find) + return s; if ((c = *find++) != 0) { len = strlen(find); do { diff --git a/libkern/kxld/kxld_util.h b/libkern/kxld/kxld_util.h index 5b55b4d8b..d8be6faef 100644 --- a/libkern/kxld/kxld_util.h +++ b/libkern/kxld/kxld_util.h @@ -196,7 +196,7 @@ boolean_t kxld_is_32_bit(cpu_type_t) __attribute__((const, visibility("hidden"))); const char * kxld_strstr(const char *s, const char *find) - __attribute__((pure, nonnull, visibility("hidden"))); + __attribute__((pure, visibility("hidden"))); /******************************************************************************* * Debugging diff --git a/libkern/kxld/kxld_versionmin.c b/libkern/kxld/kxld_versionmin.c index abbfaed6a..36c22203f 100644 --- a/libkern/kxld/kxld_versionmin.c +++ b/libkern/kxld/kxld_versionmin.c @@ -63,6 +63,31 @@ kxld_versionmin_init_from_macho(KXLDversionmin *versionmin, struct version_min_c versionmin->has_versionmin = TRUE; } +void +kxld_versionmin_init_from_build_cmd(KXLDversionmin *versionmin, struct build_version_command *src) +{ + check(versionmin); + check(src); + switch (src->platform) { + case PLATFORM_MACOS: + versionmin->platform = kKxldVersionMinMacOSX; + break; + case PLATFORM_IOS: + versionmin->platform = kKxldVersionMiniPhoneOS; + break; + case PLATFORM_TVOS: + versionmin->platform = kKxldVersionMinAppleTVOS; + break; + case PLATFORM_WATCHOS: + versionmin->platform = kKxldVersionMinWatchOS; + break; + default: + return; + } + versionmin->version = src->minos; + versionmin->has_versionmin = TRUE; +} + /******************************************************************************* *******************************************************************************/ void @@ -74,8 +99,9 @@ kxld_versionmin_clear(KXLDversionmin *versionmin) /******************************************************************************* *******************************************************************************/ u_long -kxld_versionmin_get_macho_header_size(void) +kxld_versionmin_get_macho_header_size(__unused const KXLDversionmin *versionmin) { + /* TODO: eventually we can just use struct build_version_command */ return sizeof(struct version_min_command); } @@ -92,6 +118,7 @@ kxld_versionmin_export_macho(const KXLDversionmin *versionmin, u_char *buf, check(buf); check(header_offset); + require_action(sizeof(*versionminhdr) <= header_size - *header_offset, finish, rval=KERN_FAILURE); versionminhdr = (struct version_min_command *) ((void *) (buf + *header_offset)); @@ -111,6 +138,8 @@ kxld_versionmin_export_macho(const KXLDversionmin *versionmin, u_char *buf, case kKxldVersionMinWatchOS: versionminhdr->cmd = LC_VERSION_MIN_WATCHOS; break; + default: + goto finish; } versionminhdr->cmdsize = (uint32_t) sizeof(*versionminhdr); versionminhdr->version = versionmin->version; diff --git a/libkern/kxld/kxld_versionmin.h b/libkern/kxld/kxld_versionmin.h index ff9c02124..8b3df067e 100644 --- a/libkern/kxld/kxld_versionmin.h +++ b/libkern/kxld/kxld_versionmin.h @@ -58,6 +58,9 @@ struct kxld_versionmin { void kxld_versionmin_init_from_macho(KXLDversionmin *versionmin, struct version_min_command *src) __attribute__((nonnull, visibility("hidden"))); +void kxld_versionmin_init_from_build_cmd(KXLDversionmin *versionmin, struct build_version_command *src) + __attribute__((nonnull, visibility("hidden"))); + void kxld_versionmin_clear(KXLDversionmin *versionmin) __attribute__((nonnull, visibility("hidden"))); @@ -65,7 +68,7 @@ void kxld_versionmin_clear(KXLDversionmin *versionmin) * Accessors *******************************************************************************/ -u_long kxld_versionmin_get_macho_header_size(void) +u_long kxld_versionmin_get_macho_header_size(const KXLDversionmin *versionmin) __attribute__((pure, visibility("hidden"))); kern_return_t diff --git a/libkern/kxld/tests/loadtest.py b/libkern/kxld/tests/loadtest.py old mode 100644 new mode 100755 diff --git a/libkern/libkern/OSAtomic.h b/libkern/libkern/OSAtomic.h index f4a2a6736..76b945443 100644 --- a/libkern/libkern/OSAtomic.h +++ b/libkern/libkern/OSAtomic.h @@ -676,13 +676,21 @@ extern void OSSpinLockUnlock(volatile OSSpinLock * lock); * @discussion * The OSSynchronizeIO routine ensures orderly load and store operations to noncached memory mapped I/O devices. It executes the eieio instruction on PowerPC processors. */ +#if defined(__arm__) || defined(__arm64__) +extern void OSSynchronizeIO(void); +#else static __inline__ void OSSynchronizeIO(void) { } +#endif #if defined(KERNEL_PRIVATE) -#if defined(__i386__) || defined(__x86_64__) +#if defined(__arm__) || defined(__arm64__) +static inline void OSMemoryBarrier(void) { + __asm__ volatile("dmb ish" ::: "memory"); +} +#elif defined(__i386__) || defined(__x86_64__) #if defined(XNU_KERNEL_PRIVATE) static inline void OSMemoryBarrier(void) { __asm__ volatile("mfence" ::: "memory"); diff --git a/libkern/libkern/OSByteOrder.h b/libkern/libkern/OSByteOrder.h index 8ae2c33b8..2a1d1da5d 100644 --- a/libkern/libkern/OSByteOrder.h +++ b/libkern/libkern/OSByteOrder.h @@ -41,6 +41,8 @@ #if (defined(__i386__) || defined(__x86_64__)) #include <libkern/i386/OSByteOrder.h> +#elif defined (__arm__) || defined(__arm64__) +#include <libkern/arm/OSByteOrder.h> #else #include <libkern/machine/OSByteOrder.h> #endif diff --git a/libkern/libkern/OSKextLib.h b/libkern/libkern/OSKextLib.h index 4c863af70..a08218c2e 100644 --- a/libkern/libkern/OSKextLib.h +++ b/libkern/libkern/OSKextLib.h @@ -238,6 +238,12 @@ __BEGIN_DECLS */ #define kOSKextReturnStopping libkern_kext_err(0x1a) +/*! + * @define kOSKextReturnSystemPolicy + * @abstract The kext was prevented from loading due to system policy. + */ +#define kOSKextReturnSystemPolicy libkern_kext_err(0x1b) + #if PRAGMA_MARK #pragma mark - /********************************************************************/ @@ -914,7 +920,7 @@ OSKextGrabPgoData(uuid_t uuid, * Call this function before trapping into the debugger to call OSKextResetPgoCounters. */ void -OSKextResetPgoCountersLock(); +OSKextResetPgoCountersLock(void); /*! * @function OSKextResetPgoCountersUnlock @@ -923,7 +929,7 @@ OSKextResetPgoCountersLock(); * Call this function after trapping into the debugger to call OSKextResetPgoCounters. */ void -OSKextResetPgoCountersUnlock(); +OSKextResetPgoCountersUnlock(void); /*! * @function OSKextResetPgoCounters @@ -932,7 +938,7 @@ OSKextResetPgoCountersUnlock(); * context, while holding OSKextResetPgoCountersLock(). */ void -OSKextResetPgoCounters(); +OSKextResetPgoCounters(void); #if PRAGMA_MARK diff --git a/libkern/libkern/OSKextLibPrivate.h b/libkern/libkern/OSKextLibPrivate.h index 2557007ea..fd08744ed 100644 --- a/libkern/libkern/OSKextLibPrivate.h +++ b/libkern/libkern/OSKextLibPrivate.h @@ -926,7 +926,7 @@ void OSKextLoadedKextSummariesUpdated(void); #ifdef XNU_KERNEL_PRIVATE extern const vm_allocation_site_t * OSKextGetAllocationSiteForCaller(uintptr_t address); -extern uint32_t OSKextGetKmodIDForSite(vm_allocation_site_t * site, +extern uint32_t OSKextGetKmodIDForSite(const vm_allocation_site_t * site, char * name, vm_size_t namelen); extern void OSKextFreeSite(vm_allocation_site_t * site); diff --git a/libkern/libkern/OSMalloc.h b/libkern/libkern/OSMalloc.h index 04a5ba6f2..2d14ce94d 100644 --- a/libkern/libkern/OSMalloc.h +++ b/libkern/libkern/OSMalloc.h @@ -200,7 +200,7 @@ extern void OSMalloc_Tagfree(OSMallocTag tag); */ extern void * OSMalloc( uint32_t size, - OSMallocTag tag); + OSMallocTag tag) __attribute__((alloc_size(1))); /*! @@ -211,7 +211,7 @@ extern void * OSMalloc( */ extern void * OSMalloc_nowait( uint32_t size, - OSMallocTag tag); + OSMallocTag tag) __attribute__((alloc_size(1))); /*! @@ -241,7 +241,7 @@ extern void * OSMalloc_nowait( */ extern void * OSMalloc_noblock( uint32_t size, - OSMallocTag tag); + OSMallocTag tag) __attribute__((alloc_size(1))); /*! diff --git a/libkern/libkern/_OSByteOrder.h b/libkern/libkern/_OSByteOrder.h index 3ceec32eb..5ffcba282 100644 --- a/libkern/libkern/_OSByteOrder.h +++ b/libkern/libkern/_OSByteOrder.h @@ -66,6 +66,9 @@ #include <libkern/i386/_OSByteOrder.h> #endif +#if defined (__arm__) || defined(__arm64__) +#include <libkern/arm/OSByteOrder.h> +#endif #define __DARWIN_OSSwapInt16(x) \ diff --git a/libkern/libkern/arm/Makefile b/libkern/libkern/arm/Makefile new file mode 100644 index 000000000..acfa028c8 --- /dev/null +++ b/libkern/libkern/arm/Makefile @@ -0,0 +1,21 @@ +export MakeInc_cmd=${SRCROOT}/makedefs/MakeInc.cmd +export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def +export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule +export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir + +include $(MakeInc_cmd) +include $(MakeInc_def) + +DATAFILES = \ + OSByteOrder.h + +INSTALL_MD_LIST = ${DATAFILES} + +INSTALL_MD_DIR = libkern/arm + +EXPORT_MD_LIST = ${DATAFILES} + +EXPORT_MD_DIR = libkern/arm + +include $(MakeInc_rule) +include $(MakeInc_dir) diff --git a/libkern/libkern/arm/OSByteOrder.h b/libkern/libkern/arm/OSByteOrder.h new file mode 100644 index 000000000..81279a1f8 --- /dev/null +++ b/libkern/libkern/arm/OSByteOrder.h @@ -0,0 +1,147 @@ +/* + * Copyright (c) 1999-2007 Apple Inc. All rights reserved. + */ + +#ifndef _OS_OSBYTEORDERARM_H +#define _OS_OSBYTEORDERARM_H + +#include <stdint.h> +#include <arm/arch.h> /* for _ARM_ARCH_6 */ +#include <sys/_types/_os_inline.h> + +/* Generic byte swapping functions. */ + +OS_INLINE +uint16_t +_OSSwapInt16( + uint16_t data +) +{ + /* Reduces to 'rev16' with clang */ + return (uint16_t)(data << 8 | data >> 8); +} + +OS_INLINE +uint32_t +_OSSwapInt32( + uint32_t data +) +{ +#if defined(__llvm__) + data = __builtin_bswap32(data); +#else + /* This actually generates the best code */ + data = (((data ^ (data >> 16 | (data << 16))) & 0xFF00FFFF) >> 8) ^ (data >> 8 | data << 24); +#endif + + return data; +} + +OS_INLINE +uint64_t +_OSSwapInt64( + uint64_t data +) +{ +#if defined(__llvm__) + return __builtin_bswap64(data); +#else + union { + uint64_t ull; + uint32_t ul[2]; + } u; + + /* This actually generates the best code */ + u.ul[0] = (uint32_t)(data >> 32); + u.ul[1] = (uint32_t)(data & 0xffffffff); + u.ul[0] = _OSSwapInt32(u.ul[0]); + u.ul[1] = _OSSwapInt32(u.ul[1]); + return u.ull; +#endif +} + +/* Functions for byte reversed loads. */ + +OS_INLINE +uint16_t +OSReadSwapInt16( + const volatile void * base, + uintptr_t offset +) +{ + uint16_t result; + + result = *(volatile uint16_t *)((volatile uintptr_t)base + offset); + return _OSSwapInt16(result); +} + +OS_INLINE +uint32_t +OSReadSwapInt32( + const volatile void * base, + uintptr_t offset +) +{ + uint32_t result; + + result = *(volatile uint32_t *)((volatile uintptr_t)base + offset); + return _OSSwapInt32(result); +} + +OS_INLINE +uint64_t +OSReadSwapInt64( + const volatile void * base, + uintptr_t offset +) +{ + volatile uint32_t * inp; + union ullc { + uint64_t ull; + uint32_t ul[2]; + } outv; + + inp = (volatile uint32_t *)((volatile uintptr_t)base + offset); + outv.ul[0] = inp[1]; + outv.ul[1] = inp[0]; + outv.ul[0] = _OSSwapInt32(outv.ul[0]); + outv.ul[1] = _OSSwapInt32(outv.ul[1]); + return outv.ull; +} + +/* Functions for byte reversed stores. */ + +OS_INLINE +void +OSWriteSwapInt16( + volatile void * base, + uintptr_t offset, + uint16_t data +) +{ + *(volatile uint16_t *)((volatile uintptr_t)base + offset) = _OSSwapInt16(data); +} + +OS_INLINE +void +OSWriteSwapInt32( + volatile void * base, + uintptr_t offset, + uint32_t data +) +{ + *(volatile uint32_t *)((volatile uintptr_t)base + offset) = _OSSwapInt32(data); +} + +OS_INLINE +void +OSWriteSwapInt64( + volatile void * base, + uintptr_t offset, + uint64_t data +) +{ + *(volatile uint64_t *)((volatile uintptr_t)base + offset) = _OSSwapInt64(data); +} + +#endif /* ! _OS_OSBYTEORDERARM_H */ diff --git a/libkern/libkern/c++/OSData.h b/libkern/libkern/c++/OSData.h index e25079f80..031a056a3 100644 --- a/libkern/libkern/c++/OSData.h +++ b/libkern/libkern/c++/OSData.h @@ -391,7 +391,7 @@ public: * * @abstract * Deallocates or releases any resources - * used by the OSDictionary instance. + * used by the OSData instance. * * @discussion * This function should not be called directly; diff --git a/libkern/libkern/c++/OSKext.h b/libkern/libkern/c++/OSKext.h index 4d4cea4ec..2fc70eb1c 100644 --- a/libkern/libkern/c++/OSKext.h +++ b/libkern/libkern/c++/OSKext.h @@ -528,12 +528,18 @@ private: /* panic() support. */ +public: + enum { + kPrintKextsLock = 0x01, + kPrintKextsUnslide = 0x02, + kPrintKextsTerse = 0x04 + }; static void printKextsInBacktrace( vm_offset_t * addr, unsigned int cnt, int (* printf_func)(const char *fmt, ...), - bool lockFlag, - bool doUnslide); + uint32_t flags); +private: static OSKextLoadedKextSummary *summaryForAddress(const uintptr_t addr); static void *kextForAddress(const void *addr); static boolean_t summaryIsInBacktrace( @@ -543,7 +549,7 @@ private: static void printSummary( OSKextLoadedKextSummary * summary, int (* printf_func)(const char *fmt, ...), - bool doUnslide); + uint32_t flags); static int saveLoadedKextPanicListTyped( const char * prefix, @@ -625,6 +631,8 @@ public: OSCollectionIterator * theIterator); static void createExcludeListFromPrelinkInfo(OSArray * theInfoArray); + static bool isWaitingKextd(void); + virtual bool setAutounloadEnabled(bool flag); virtual const OSSymbol * getIdentifier(void); diff --git a/libkern/libkern/c++/OSMetaClass.h b/libkern/libkern/c++/OSMetaClass.h index 4660b240e..322e5238a 100644 --- a/libkern/libkern/c++/OSMetaClass.h +++ b/libkern/libkern/c++/OSMetaClass.h @@ -41,7 +41,8 @@ class OSDictionary; class OSSerialize; #ifdef XNU_KERNEL_PRIVATE class OSOrderedSet; -#endif +class OSCollection; +#endif /* XNU_KERNEL_PRIVATE */ /*! @@ -59,14 +60,22 @@ class OSOrderedSet; #ifdef XNU_KERNEL_PRIVATE +#ifdef CONFIG_EMBEDDED +#define APPLE_KEXT_VTABLE_PADDING 0 +#else /* CONFIG_EMBEDDED */ /*! @parseOnly */ #define APPLE_KEXT_VTABLE_PADDING 1 +#endif /* CONFIG_EMBEDDED */ #else /* XNU_KERNEL_PRIVATE */ #include <TargetConditionals.h> +#if TARGET_OS_EMBEDDED +#define APPLE_KEXT_VTABLE_PADDING 0 +#else /* TARGET_OS_EMBEDDED */ /*! @parseOnly */ #define APPLE_KEXT_VTABLE_PADDING 1 +#endif /* TARGET_OS_EMBEDDED */ #endif /* XNU_KERNEL_PRIVATE */ @@ -75,6 +84,8 @@ class OSOrderedSet; #if defined(__LP64__) /*! @parseOnly */ #define APPLE_KEXT_LEGACY_ABI 0 +#elif defined(__arm__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 2)) +#define APPLE_KEXT_LEGACY_ABI 0 #else #define APPLE_KEXT_LEGACY_ABI 1 #endif @@ -346,7 +357,43 @@ _ptmf2ptf(const OSMetaClassBase *self, void (OSMetaClassBase::*func)(void)) } #else /* !APPLE_KEXT_LEGACY_ABI */ -#if defined(__i386__) || defined(__x86_64__) +#if defined(__arm__) || defined(__arm64__) +typedef long int ptrdiff_t; +/* + * Ugly reverse engineered ABI. Where does it come from? Nobody knows. + * <rdar://problem/5641129> gcc 4.2-built ARM kernel panics with multiple inheritance (no, really) + */ +static inline _ptf_t +_ptmf2ptf(const OSMetaClassBase *self, void (OSMetaClassBase::*func)(void)) +{ + struct ptmf_t { + _ptf_t fPFN; + ptrdiff_t delta; + }; + union { + void (OSMetaClassBase::*fIn)(void); + struct ptmf_t pTMF; + } map; + + + map.fIn = func; + + if (map.pTMF.delta & 1) { + // virtual + union { + const OSMetaClassBase *fObj; + _ptf_t **vtablep; + } u; + u.fObj = self; + + // Virtual member function so dereference table + return *(_ptf_t *)(((uintptr_t)*u.vtablep) + (uintptr_t)map.pTMF.fPFN); + } else { + // Not virtual, i.e. plain member func + return map.pTMF.fPFN; + } +} +#elif defined(__i386__) || defined(__x86_64__) // Slightly less arcane and slightly less evil code to do // the same for kexts compiled with the standard Itanium C++ @@ -926,13 +973,43 @@ public: * @abstract * Look up a metaclass in the run-time type information system. * - * @param name The name of the desired class's metaclass. + * @param name The name of the desired class's metaclass. * * @result * A pointer to the metaclass object if found, <code>NULL</code> otherwise. */ static const OSMetaClass * getMetaClassWithName(const OSSymbol * name); +#if XNU_KERNEL_PRIVATE + + /*! + * @function copyMetaClassWithName + * + * @abstract + * Look up a metaclass in the run-time type information system. + * + * @param name The name of the desired class's metaclass. + * + * @result + * A pointer to the metaclass object if found, <code>NULL</code> otherwise. + * The metaclass will be protected from unloading until releaseMetaClass() + * is called. + */ + static const OSMetaClass * copyMetaClassWithName(const OSSymbol * name); + /*! + * @function releaseMetaClass + * + * @abstract + * Releases reference obtained from copyMetaClassWithName(). + * + * @discussion + * The metaclass will be protected from unloading until releaseMetaClass() + * is called. + */ + void releaseMetaClass() const; + +#endif /* XNU_KERNEL_PRIVATE */ + protected: /*! * @function retain @@ -1263,6 +1340,9 @@ public: */ static void considerUnloads(); +#if XNU_KERNEL_PRIVATE + static bool removeClasses(OSCollection * metaClasses); +#endif /* XNU_KERNEL_PRIVATE */ /*! * @function allocClassWithName @@ -1569,7 +1649,7 @@ private: OSMetaClassInstanceApplierFunction applier, void * context); public: -#endif +#endif /* XNU_KERNEL_PRIVATE */ /* Not to be included in headerdoc. * @@ -2086,8 +2166,8 @@ public: void trackedFree(OSObject * instance) const; void trackedAccumSize(OSObject * instance, size_t size) const; struct IOTrackingQueue * getTracking() const; -#endif -#endif +#endif /* IOTRACKING */ +#endif /* XNU_KERNEL_PRIVATE */ private: // Obsolete APIs diff --git a/libkern/libkern/crypto/Makefile b/libkern/libkern/crypto/Makefile index e701850eb..0fdcb6258 100644 --- a/libkern/libkern/crypto/Makefile +++ b/libkern/libkern/crypto/Makefile @@ -8,7 +8,7 @@ include $(MakeInc_def) DATAFILES = md5.h sha1.h -PRIVATE_DATAFILES = register_crypto.h sha2.h des.h aes.h aesxts.h rand.h rsa.h +PRIVATE_DATAFILES = register_crypto.h sha2.h des.h aes.h aesxts.h rand.h rsa.h chacha20poly1305.h INSTALL_KF_MI_LIST = ${DATAFILES} diff --git a/libkern/libkern/crypto/chacha20poly1305.h b/libkern/libkern/crypto/chacha20poly1305.h new file mode 100644 index 000000000..598f59746 --- /dev/null +++ b/libkern/libkern/crypto/chacha20poly1305.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2017 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef _CHACHA20POLY1305_H +#define _CHACHA20POLY1305_H + +#if defined(__cplusplus) +extern "C" +{ +#endif + +#include <corecrypto/ccchacha20poly1305.h> + +typedef ccchacha20poly1305_ctx chacha20poly1305_ctx; + +int chacha20poly1305_init(chacha20poly1305_ctx *ctx, const uint8_t *key); +int chacha20poly1305_reset(chacha20poly1305_ctx *ctx); +int chacha20poly1305_setnonce(chacha20poly1305_ctx *ctx, const uint8_t *nonce); +int chacha20poly1305_incnonce(chacha20poly1305_ctx *ctx, uint8_t *nonce); +int chacha20poly1305_aad(chacha20poly1305_ctx *ctx, size_t nbytes, const void *aad); +int chacha20poly1305_encrypt(chacha20poly1305_ctx *ctx, size_t nbytes, const void *ptext, void *ctext); +int chacha20poly1305_finalize(chacha20poly1305_ctx *ctx, uint8_t *tag); +int chacha20poly1305_decrypt(chacha20poly1305_ctx *ctx, size_t nbytes, const void *ctext, void *ptext); +int chacha20poly1305_verify(chacha20poly1305_ctx *ctx, const uint8_t *tag); + +#if defined(__cplusplus) +} +#endif + +#endif diff --git a/libkern/libkern/crypto/register_crypto.h b/libkern/libkern/crypto/register_crypto.h index 6041ebb31..a29592700 100644 --- a/libkern/libkern/crypto/register_crypto.h +++ b/libkern/libkern/crypto/register_crypto.h @@ -39,6 +39,7 @@ extern "C" { #include <corecrypto/ccrc4.h> #include <corecrypto/ccrng.h> #include <corecrypto/ccrsa.h> +#include <corecrypto/ccchacha20poly1305.h> /* Function types */ @@ -68,7 +69,19 @@ typedef int (*ccgcm_init_with_iv_fn_t)(const struct ccmode_gcm *mode, ccgcm_ctx size_t key_nbytes, const void *key, const void *iv); typedef int (*ccgcm_inc_iv_fn_t)(const struct ccmode_gcm *mode, ccgcm_ctx *ctx, void *iv); - + +typedef const struct ccchacha20poly1305_fns { + const struct ccchacha20poly1305_info *(*info)(void); + int (*init)(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, const uint8_t *key); + int (*reset)(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx); + int (*setnonce)(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, const uint8_t *nonce); + int (*incnonce)(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, uint8_t *nonce); + int (*aad)(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, size_t nbytes, const void *aad); + int (*encrypt)(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, size_t nbytes, const void *ptext, void *ctext); + int (*finalize)(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, uint8_t *tag); + int (*decrypt)(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, size_t nbytes, const void *ctext, void *ptext); + int (*verify)(const struct ccchacha20poly1305_info *info, ccchacha20poly1305_ctx *ctx, const uint8_t *tag); +} *ccchacha20poly1305_fns_t; /* pbkdf2 */ typedef void (*ccpbkdf2_hmac_fn_t)(const struct ccdigest_info *di, @@ -130,6 +143,7 @@ typedef struct crypto_functions { const struct ccmode_ecb *ccaes_ecb_decrypt; const struct ccmode_cbc *ccaes_cbc_encrypt; const struct ccmode_cbc *ccaes_cbc_decrypt; + const struct ccmode_ctr *ccaes_ctr_crypt; const struct ccmode_xts *ccaes_xts_encrypt; const struct ccmode_xts *ccaes_xts_decrypt; const struct ccmode_gcm *ccaes_gcm_encrypt; @@ -137,6 +151,8 @@ typedef struct crypto_functions { ccgcm_init_with_iv_fn_t ccgcm_init_with_iv_fn; ccgcm_inc_iv_fn_t ccgcm_inc_iv_fn; + + ccchacha20poly1305_fns_t ccchacha20poly1305_fns; /* DES, ecb and cbc */ const struct ccmode_ecb *ccdes_ecb_encrypt; diff --git a/libkern/net/inet_aton.c b/libkern/net/inet_aton.c index 9c0d94ae3..ff518373e 100644 --- a/libkern/net/inet_aton.c +++ b/libkern/net/inet_aton.c @@ -41,7 +41,7 @@ int inet_aton(const char *cp, struct in_addr *addr) { u_long parts[4]; - in_addr_t val; + in_addr_t val = 0; const char *c; char *endptr; int gotend, n; diff --git a/libkern/os/Makefile b/libkern/os/Makefile index 80a933fb9..390b9b861 100644 --- a/libkern/os/Makefile +++ b/libkern/os/Makefile @@ -16,9 +16,11 @@ KERNELFILES = \ overflow.h PRIVATE_KERNELFILES = \ - object_private.h + object_private.h \ + reason_private.h -PRIVATE_DATAFILES = +PRIVATE_DATAFILES = \ + reason_private.h INSTALL_MI_LIST = \ overflow.h diff --git a/libkern/os/log.c b/libkern/os/log.c index a2638fee5..a019b7bd9 100644 --- a/libkern/os/log.c +++ b/libkern/os/log.c @@ -72,7 +72,12 @@ uint32_t oslog_s_error_count = 0; uint32_t oslog_s_metadata_msgcount = 0; static bool oslog_boot_done = false; -extern boolean_t oslog_early_boot_complete; +extern boolean_t early_boot_complete; + +#ifdef XNU_KERNEL_PRIVATE +bool startup_serial_logging_active = true; +uint64_t startup_serial_num_procs = 300; +#endif /* XNU_KERNEL_PRIVATE */ // XXX firehose_tracepoint_id_t @@ -166,9 +171,15 @@ _os_log_with_args_internal(os_log_t oslog, os_log_type_t type, if (format[0] == '\0') { return; } - /* cf. r24974766 & r25201228*/ - safe = (!oslog_early_boot_complete || oslog_is_safe()); - logging = (!(logging_config & ATM_TRACE_DISABLE) || !(logging_config & ATM_TRACE_OFF)); + + /* early boot can log to dmesg for later replay (27307943) */ + safe = (!early_boot_complete || oslog_is_safe()); + + if (logging_config & ATM_TRACE_DISABLE || logging_config & ATM_TRACE_OFF) { + logging = false; + } else { + logging = true; + } if (oslog != &_os_log_replay) { _os_log_to_msgbuf_internal(format, args, safe, logging); diff --git a/libkern/os/log.h b/libkern/os/log.h index a26a129a1..1da91a8f3 100644 --- a/libkern/os/log.h +++ b/libkern/os/log.h @@ -48,7 +48,12 @@ __BEGIN_DECLS extern void *__dso_handle; -OS_ALWAYS_INLINE static inline void _os_log_verify_format_str(__unused const char *msg, ...) __attribute__((format(os_trace, 1, 2))); +#ifdef XNU_KERNEL_PRIVATE +extern bool startup_serial_logging_active; +extern uint64_t startup_serial_num_procs; +#endif /* XNU_KERNEL_PRIVATE */ + +OS_ALWAYS_INLINE static inline void _os_log_verify_format_str(__unused const char *msg, ...) __attribute__((format(os_log, 1, 2))); OS_ALWAYS_INLINE static inline void _os_log_verify_format_str(__unused const char *msg, ...) { /* placeholder */ } #if OS_OBJECT_USE_OBJC @@ -485,6 +490,13 @@ os_log_debug_enabled(os_log_t log); __asm__(""); /* avoid tailcall */ \ }) +#ifdef XNU_KERNEL_PRIVATE +#define os_log_with_startup_serial(log, format, ...) __extension__({ \ + if (startup_serial_logging_active) { printf(format, ##__VA_ARGS__); } \ + else { os_log(log, format, ##__VA_ARGS__); } \ +}) +#endif /* XNU_KERNEL_PRIVATE */ + /*! * @function _os_log_internal * diff --git a/libkern/os/overflow.h b/libkern/os/overflow.h index 8d0fd9949..05ddbef39 100644 --- a/libkern/os/overflow.h +++ b/libkern/os/overflow.h @@ -131,6 +131,15 @@ __os_warn_unused(__const bool x) _s | _t; \ })) +/* os_mul3_overflow(a, b, c) -> (a * b * c) */ +#define os_mul3_overflow(a, b, c, res) __os_warn_unused(__extension__({ \ + __typeof(*(res)) _tmp; \ + bool _s, _t; \ + _s = os_mul_overflow((a), (b), &_tmp); \ + _t = os_mul_overflow((c), _tmp, (res)); \ + _s | _t; \ +})) + /* os_add_and_mul_overflow(a, b, x) -> (a + b)*x */ #define os_add_and_mul_overflow(a, b, x, res) __os_warn_unused(__extension__({ \ __typeof(*(res)) _tmp; \ @@ -149,4 +158,6 @@ __os_warn_unused(__const bool x) _s | _t; \ })) +#define os_convert_overflow(a, res) os_add_overflow((a), 0, (res)) + #endif /* _OS_OVERFLOW_H */ diff --git a/libkern/os/reason_private.h b/libkern/os/reason_private.h new file mode 100644 index 000000000..477bceeed --- /dev/null +++ b/libkern/os/reason_private.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef OS_REASON_PRIVATE_H +#define OS_REASON_PRIVATE_H + +#include <sys/reason.h> +#include <os/base.h> + +/* Codes in the OS_REASON_LIBSYSTEM namespace */ + +OS_ENUM(os_reason_libsystem_code, uint64_t, + OS_REASON_LIBSYSTEM_CODE_WORKLOOP_OWNERSHIP_LEAK = 1, + OS_REASON_LIBSYSTEM_CODE_FAULT = 2, /* generated by os_log_fault */ +); + +#ifndef KERNEL + +/* + * similar to abort_with_payload, but for faults. + * + * [EBUSY] too many corpses are being generated at the moment + * [EQFULL] the process used all its user fault quota + * [ENOTSUP] generating simulated abort with reason is disabled + * [EPERM] generating simulated abort with reason for this namespace is not turned on + */ +int +os_fault_with_payload(uint32_t reason_namespace, uint64_t reason_code, + void *payload, uint32_t payload_size, const char *reason_string, + uint64_t reason_flags); + +#endif // !KERNEL + +#endif // OS_REASON_PRIVATE_H diff --git a/libkern/zlib/gzio.c b/libkern/zlib/gzio.c deleted file mode 100644 index 9c87cdd0f..000000000 --- a/libkern/zlib/gzio.c +++ /dev/null @@ -1,1031 +0,0 @@ -/* - * Copyright (c) 2008-2016 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ -/* gzio.c -- IO on .gz files - * Copyright (C) 1995-2005 Jean-loup Gailly. - * For conditions of distribution and use, see copyright notice in zlib.h - * - * Compile this file with -DNO_GZCOMPRESS to avoid the compression code. - */ - -/* @(#) $Id$ */ - -#include <stdio.h> - -#include "zutil.h" - -#ifdef NO_DEFLATE /* for compatibility with old definition */ -# define NO_GZCOMPRESS -#endif - -#ifndef NO_DUMMY_DECL -struct internal_state {int dummy;}; /* for buggy compilers */ -#endif - -#ifndef Z_BUFSIZE -# ifdef MAXSEG_64K -# define Z_BUFSIZE 4096 /* minimize memory usage for 16-bit DOS */ -# else -# define Z_BUFSIZE 16384 -# endif -#endif -#ifndef Z_PRINTF_BUFSIZE -# define Z_PRINTF_BUFSIZE 4096 -#endif - -#ifdef __MVS__ -# pragma map (fdopen , "\174\174FDOPEN") - FILE *fdopen(int, const char *); -#endif - -#ifndef STDC -extern voidp malloc OF((uInt size)); -extern void free OF((voidpf ptr)); -#endif - -#define ALLOC(size) malloc(size) -#define TRYFREE(p) {if (p) free(p);} - -static int const gz_magic[2] = {0x1f, 0x8b}; /* gzip magic header */ - -/* gzip flag byte */ -#define ASCII_FLAG 0x01 /* bit 0 set: file probably ascii text */ -#define HEAD_CRC 0x02 /* bit 1 set: header CRC present */ -#define EXTRA_FIELD 0x04 /* bit 2 set: extra field present */ -#define ORIG_NAME 0x08 /* bit 3 set: original file name present */ -#define COMMENT 0x10 /* bit 4 set: file comment present */ -#define RESERVED 0xE0 /* bits 5..7: reserved */ - -typedef struct gz_stream { - z_stream stream; - int z_err; /* error code for last stream operation */ - int z_eof; /* set if end of input file */ - FILE *file; /* .gz file */ - Byte *inbuf; /* input buffer */ - Byte *outbuf; /* output buffer */ - uLong crc; /* crc32 of uncompressed data */ - char *msg; /* error message */ - char *path; /* path name for debugging only */ - int transparent; /* 1 if input file is not a .gz file */ - char mode; /* 'w' or 'r' */ - z_off_t start; /* start of compressed data in file (header skipped) */ - z_off_t in; /* bytes into deflate or inflate */ - z_off_t out; /* bytes out of deflate or inflate */ - int back; /* one character push-back */ - int last; /* true if push-back is last character */ -} gz_stream; - - -local gzFile gz_open OF((const char *path, const char *mode, int fd)); -local int do_flush OF((gzFile file, int flush)); -local int get_byte OF((gz_stream *s)); -local void check_header OF((gz_stream *s)); -local int destroy OF((gz_stream *s)); -local void putLong OF((FILE *file, uLong x)); -local uLong getLong OF((gz_stream *s)); - -/* =========================================================================== - Opens a gzip (.gz) file for reading or writing. The mode parameter - is as in fopen ("rb" or "wb"). The file is given either by file descriptor - or path name (if fd == -1). - gz_open returns NULL if the file could not be opened or if there was - insufficient memory to allocate the (de)compression state; errno - can be checked to distinguish the two cases (if errno is zero, the - zlib error is Z_MEM_ERROR). -*/ -local gzFile -gz_open(const char *path, const char *mode, int fd) -{ - int err; - int level = Z_DEFAULT_COMPRESSION; /* compression level */ - int strategy = Z_DEFAULT_STRATEGY; /* compression strategy */ - char *p = (char*)mode; - gz_stream *s; - char fmode[80]; /* copy of mode, without the compression level */ - char *m = fmode; - - if (!path || !mode) return Z_NULL; - - s = (gz_stream *)ALLOC(sizeof(gz_stream)); - if (!s) return Z_NULL; - - s->stream.zalloc = (alloc_func)0; - s->stream.zfree = (free_func)0; - s->stream.opaque = (voidpf)0; - s->stream.next_in = s->inbuf = Z_NULL; - s->stream.next_out = s->outbuf = Z_NULL; - s->stream.avail_in = s->stream.avail_out = 0; - s->file = NULL; - s->z_err = Z_OK; - s->z_eof = 0; - s->in = 0; - s->out = 0; - s->back = EOF; - s->crc = z_crc32(0L, Z_NULL, 0); - s->msg = NULL; - s->transparent = 0; - - s->path = (char*)ALLOC(strlen(path)+1); - if (s->path == NULL) { - return destroy(s), (gzFile)Z_NULL; - } - strcpy(s->path, path); /* do this early for debugging */ - - s->mode = '\0'; - do { - if (*p == 'r') s->mode = 'r'; - if (*p == 'w' || *p == 'a') s->mode = 'w'; - if (*p >= '0' && *p <= '9') { - level = *p - '0'; - } else if (*p == 'f') { - strategy = Z_FILTERED; - } else if (*p == 'h') { - strategy = Z_HUFFMAN_ONLY; - } else if (*p == 'R') { - strategy = Z_RLE; - } else { - *m++ = *p; /* copy the mode */ - } - } while (*p++ && m != fmode + sizeof(fmode)); - if (s->mode == '\0') return destroy(s), (gzFile)Z_NULL; - - if (s->mode == 'w') { -#ifdef NO_GZCOMPRESS - err = Z_STREAM_ERROR; -#else - err = deflateInit2(&(s->stream), level, - Z_DEFLATED, -MAX_WBITS, DEF_MEM_LEVEL, strategy); - /* windowBits is passed < 0 to suppress zlib header */ - - s->stream.next_out = s->outbuf = (Byte*)ALLOC(Z_BUFSIZE); -#endif - if (err != Z_OK || s->outbuf == Z_NULL) { - return destroy(s), (gzFile)Z_NULL; - } - } else { - s->stream.next_in = s->inbuf = (Byte*)ALLOC(Z_BUFSIZE); - - err = inflateInit2(&(s->stream), -MAX_WBITS); - /* windowBits is passed < 0 to tell that there is no zlib header. - * Note that in this case inflate *requires* an extra "dummy" byte - * after the compressed stream in order to complete decompression and - * return Z_STREAM_END. Here the gzip CRC32 ensures that 4 bytes are - * present after the compressed stream. - */ - if (err != Z_OK || s->inbuf == Z_NULL) { - return destroy(s), (gzFile)Z_NULL; - } - } - s->stream.avail_out = Z_BUFSIZE; - - errno = 0; - s->file = fd < 0 ? F_OPEN(path, fmode) : (FILE*)fdopen(fd, fmode); - - if (s->file == NULL) { - return destroy(s), (gzFile)Z_NULL; - } - if (s->mode == 'w') { - /* Write a very simple .gz header: - */ - fprintf(s->file, "%c%c%c%c%c%c%c%c%c%c", gz_magic[0], gz_magic[1], - Z_DEFLATED, 0 /*flags*/, 0,0,0,0 /*time*/, 0 /*xflags*/, OS_CODE); - s->start = 10L; - /* We use 10L instead of ftell(s->file) to because ftell causes an - * fflush on some systems. This version of the library doesn't use - * start anyway in write mode, so this initialization is not - * necessary. - */ - } else { - check_header(s); /* skip the .gz header */ - s->start = ftell(s->file) - s->stream.avail_in; - } - - return (gzFile)s; -} - -/* =========================================================================== - Opens a gzip (.gz) file for reading or writing. -*/ -gzFile ZEXPORT -gzopen(const char *path, const char *mode) -{ - return gz_open (path, mode, -1); -} - -/* =========================================================================== - Associate a gzFile with the file descriptor fd. fd is not dup'ed here - to mimic the behavio(u)r of fdopen. -*/ -gzFile ZEXPORT -gzdopen(int fd, const char *mode) -{ - char name[46]; /* allow for up to 128-bit integers */ - - if (fd < 0) return (gzFile)Z_NULL; - sprintf(name, "<fd:%d>", fd); /* for debugging */ - - return gz_open (name, mode, fd); -} - -/* =========================================================================== - * Update the compression level and strategy - */ -int ZEXPORT -gzsetparams(gzFile file, int level, int strategy) -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL || s->mode != 'w') return Z_STREAM_ERROR; - - /* Make room to allow flushing */ - if (s->stream.avail_out == 0) { - - s->stream.next_out = s->outbuf; - if (fwrite(s->outbuf, 1, Z_BUFSIZE, s->file) != Z_BUFSIZE) { - s->z_err = Z_ERRNO; - } - s->stream.avail_out = Z_BUFSIZE; - } - - return deflateParams (&(s->stream), level, strategy); -} - -/* =========================================================================== - Read a byte from a gz_stream; update next_in and avail_in. Return EOF - for end of file. - IN assertion: the stream s has been sucessfully opened for reading. -*/ -local int -get_byte(gz_stream *s) -{ - if (s->z_eof) return EOF; - if (s->stream.avail_in == 0) { - errno = 0; - s->stream.avail_in = (uInt)fread(s->inbuf, 1, Z_BUFSIZE, s->file); - if (s->stream.avail_in == 0) { - s->z_eof = 1; - if (ferror(s->file)) s->z_err = Z_ERRNO; - return EOF; - } - s->stream.next_in = s->inbuf; - } - s->stream.avail_in--; - return *(s->stream.next_in)++; -} - -/* =========================================================================== - Check the gzip header of a gz_stream opened for reading. Set the stream - mode to transparent if the gzip magic header is not present; set s->err - to Z_DATA_ERROR if the magic header is present but the rest of the header - is incorrect. - IN assertion: the stream s has already been created sucessfully; - s->stream.avail_in is zero for the first time, but may be non-zero - for concatenated .gz files. -*/ -local void -check_header(gz_stream *s) -{ - int method; /* method byte */ - int flags; /* flags byte */ - uInt len; - int c; - - /* Assure two bytes in the buffer so we can peek ahead -- handle case - where first byte of header is at the end of the buffer after the last - gzip segment */ - len = s->stream.avail_in; - if (len < 2) { - if (len) s->inbuf[0] = s->stream.next_in[0]; - errno = 0; - len = (uInt)fread(s->inbuf + len, 1, Z_BUFSIZE >> len, s->file); - if (len == 0 && ferror(s->file)) s->z_err = Z_ERRNO; - s->stream.avail_in += len; - s->stream.next_in = s->inbuf; - if (s->stream.avail_in < 2) { - s->transparent = s->stream.avail_in; - return; - } - } - - /* Peek ahead to check the gzip magic header */ - if (s->stream.next_in[0] != gz_magic[0] || - s->stream.next_in[1] != gz_magic[1]) { - s->transparent = 1; - return; - } - s->stream.avail_in -= 2; - s->stream.next_in += 2; - - /* Check the rest of the gzip header */ - method = get_byte(s); - flags = get_byte(s); - if (method != Z_DEFLATED || (flags & RESERVED) != 0) { - s->z_err = Z_DATA_ERROR; - return; - } - - /* Discard time, xflags and OS code: */ - for (len = 0; len < 6; len++) (void)get_byte(s); - - if ((flags & EXTRA_FIELD) != 0) { /* skip the extra field */ - len = (uInt)get_byte(s); - len += ((uInt)get_byte(s))<<8; - /* len is garbage if EOF but the loop below will quit anyway */ - while (len-- != 0 && get_byte(s) != EOF) ; - } - if ((flags & ORIG_NAME) != 0) { /* skip the original file name */ - while ((c = get_byte(s)) != 0 && c != EOF) ; - } - if ((flags & COMMENT) != 0) { /* skip the .gz file comment */ - while ((c = get_byte(s)) != 0 && c != EOF) ; - } - if ((flags & HEAD_CRC) != 0) { /* skip the header crc */ - for (len = 0; len < 2; len++) (void)get_byte(s); - } - s->z_err = s->z_eof ? Z_DATA_ERROR : Z_OK; -} - - /* =========================================================================== - * Cleanup then free the given gz_stream. Return a zlib error code. - Try freeing in the reverse order of allocations. - */ -local int -destroy(gz_stream *s) -{ - int err = Z_OK; - - if (!s) return Z_STREAM_ERROR; - - TRYFREE(s->msg); - - if (s->stream.state != NULL) { - if (s->mode == 'w') { -#ifdef NO_GZCOMPRESS - err = Z_STREAM_ERROR; -#else - err = deflateEnd(&(s->stream)); -#endif - } else if (s->mode == 'r') { - err = inflateEnd(&(s->stream)); - } - } - if (s->file != NULL && fclose(s->file)) { -#ifdef ESPIPE - if (errno != ESPIPE) /* fclose is broken for pipes in HP/UX */ -#endif - err = Z_ERRNO; - } - if (s->z_err < 0) err = s->z_err; - - TRYFREE(s->inbuf); - TRYFREE(s->outbuf); - TRYFREE(s->path); - TRYFREE(s); - return err; -} - -/* =========================================================================== - Reads the given number of uncompressed bytes from the compressed file. - gzread returns the number of bytes actually read (0 for end of file). -*/ -int ZEXPORT -gzread(gzFile file, voidp buf, unsigned len) -{ - gz_stream *s = (gz_stream*)file; - Bytef *start = (Bytef*)buf; /* starting point for crc computation */ - Byte *next_out; /* == stream.next_out but not forced far (for MSDOS) */ - - if (s == NULL || s->mode != 'r') return Z_STREAM_ERROR; - - if (s->z_err == Z_DATA_ERROR || s->z_err == Z_ERRNO) return -1; - if (s->z_err == Z_STREAM_END) return 0; /* EOF */ - - next_out = (Byte*)buf; - s->stream.next_out = (Bytef*)buf; - s->stream.avail_out = len; - - if (s->stream.avail_out && s->back != EOF) { - *next_out++ = s->back; - s->stream.next_out++; - s->stream.avail_out--; - s->back = EOF; - s->out++; - start++; - if (s->last) { - s->z_err = Z_STREAM_END; - return 1; - } - } - - while (s->stream.avail_out != 0) { - - if (s->transparent) { - /* Copy first the lookahead bytes: */ - uInt n = s->stream.avail_in; - if (n > s->stream.avail_out) n = s->stream.avail_out; - if (n > 0) { - zmemcpy(s->stream.next_out, s->stream.next_in, n); - next_out += n; - s->stream.next_out = next_out; - s->stream.next_in += n; - s->stream.avail_out -= n; - s->stream.avail_in -= n; - } - if (s->stream.avail_out > 0) { - s->stream.avail_out -= - (uInt)fread(next_out, 1, s->stream.avail_out, s->file); - } - len -= s->stream.avail_out; - s->in += len; - s->out += len; - if (len == 0) s->z_eof = 1; - return (int)len; - } - if (s->stream.avail_in == 0 && !s->z_eof) { - - errno = 0; - s->stream.avail_in = (uInt)fread(s->inbuf, 1, Z_BUFSIZE, s->file); - if (s->stream.avail_in == 0) { - s->z_eof = 1; - if (ferror(s->file)) { - s->z_err = Z_ERRNO; - break; - } - } - s->stream.next_in = s->inbuf; - } - s->in += s->stream.avail_in; - s->out += s->stream.avail_out; - s->z_err = inflate(&(s->stream), Z_NO_FLUSH); - s->in -= s->stream.avail_in; - s->out -= s->stream.avail_out; - - if (s->z_err == Z_STREAM_END) { - /* Check CRC and original size */ - s->crc = z_crc32(s->crc, start, (uInt)(s->stream.next_out - start)); - start = s->stream.next_out; - - if (getLong(s) != s->crc) { - s->z_err = Z_DATA_ERROR; - } else { - (void)getLong(s); - /* The uncompressed length returned by above getlong() may be - * different from s->out in case of concatenated .gz files. - * Check for such files: - */ - check_header(s); - if (s->z_err == Z_OK) { - inflateReset(&(s->stream)); - s->crc = z_crc32(0L, Z_NULL, 0); - } - } - } - if (s->z_err != Z_OK || s->z_eof) break; - } - s->crc = z_crc32(s->crc, start, (uInt)(s->stream.next_out - start)); - - if (len == s->stream.avail_out && - (s->z_err == Z_DATA_ERROR || s->z_err == Z_ERRNO)) - return -1; - return (int)(len - s->stream.avail_out); -} - - -/* =========================================================================== - Reads one byte from the compressed file. gzgetc returns this byte - or -1 in case of end of file or error. -*/ -int ZEXPORT -gzgetc(gzFile file) -{ - unsigned char c; - - return gzread(file, &c, 1) == 1 ? c : -1; -} - - -/* =========================================================================== - Push one byte back onto the stream. -*/ -int ZEXPORT -gzungetc(int c, gzFile file) -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL || s->mode != 'r' || c == EOF || s->back != EOF) return EOF; - s->back = c; - s->out--; - s->last = (s->z_err == Z_STREAM_END); - if (s->last) s->z_err = Z_OK; - s->z_eof = 0; - return c; -} - - -/* =========================================================================== - Reads bytes from the compressed file until len-1 characters are - read, or a newline character is read and transferred to buf, or an - end-of-file condition is encountered. The string is then terminated - with a null character. - gzgets returns buf, or Z_NULL in case of error. - - The current implementation is not optimized at all. -*/ -char * ZEXPORT -gzgets(gzFile file, char *buf, int len) -{ - char *b = buf; - if (buf == Z_NULL || len <= 0) return Z_NULL; - - while (--len > 0 && gzread(file, buf, 1) == 1 && *buf++ != '\n') ; - *buf = '\0'; - return b == buf && len > 0 ? Z_NULL : b; -} - - -#ifndef NO_GZCOMPRESS -/* =========================================================================== - Writes the given number of uncompressed bytes into the compressed file. - gzwrite returns the number of bytes actually written (0 in case of error). -*/ -int ZEXPORT -gzwrite(gzFile file, voidpc buf, unsigned len) -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL || s->mode != 'w') return Z_STREAM_ERROR; - - s->stream.next_in = (Bytef*)buf; - s->stream.avail_in = len; - - while (s->stream.avail_in != 0) { - - if (s->stream.avail_out == 0) { - - s->stream.next_out = s->outbuf; - if (fwrite(s->outbuf, 1, Z_BUFSIZE, s->file) != Z_BUFSIZE) { - s->z_err = Z_ERRNO; - break; - } - s->stream.avail_out = Z_BUFSIZE; - } - s->in += s->stream.avail_in; - s->out += s->stream.avail_out; - s->z_err = deflate(&(s->stream), Z_NO_FLUSH); - s->in -= s->stream.avail_in; - s->out -= s->stream.avail_out; - if (s->z_err != Z_OK) break; - } - s->crc = z_crc32(s->crc, (const Bytef *)buf, len); - - return (int)(len - s->stream.avail_in); -} - - -/* =========================================================================== - Converts, formats, and writes the args to the compressed file under - control of the format string, as in fprintf. gzprintf returns the number of - uncompressed bytes actually written (0 in case of error). -*/ -#ifdef STDC -#include <stdarg.h> - -int ZEXPORTVA -gzprintf(gzFile file, const char *format, /* args */ ...) -{ - char buf[Z_PRINTF_BUFSIZE]; - va_list va; - int len; - - buf[sizeof(buf) - 1] = 0; - va_start(va, format); -#ifdef NO_vsnprintf -# ifdef HAS_vsprintf_void - (void)vsprintf(buf, format, va); - va_end(va); - for (len = 0; len < sizeof(buf); len++) - if (buf[len] == 0) break; -# else - len = vsprintf(buf, format, va); - va_end(va); -# endif -#else -# ifdef HAS_vsnprintf_void - (void)vsnprintf(buf, sizeof(buf), format, va); - va_end(va); - len = strlen(buf); -# else - len = vsnprintf(buf, sizeof(buf), format, va); - va_end(va); -# endif -#endif - if (len <= 0 || len >= (int)sizeof(buf) || buf[sizeof(buf) - 1] != 0) - return 0; - return gzwrite(file, buf, (unsigned)len); -} -#else /* not ANSI C */ - -int ZEXPORTVA -gzprintf(gzFile file, const char *format, int a1, int a2, int a3, int a4, - int a5, int a6, int a7, int a8, int a9, int a10, int a11, int a12, - int a13, int a14, int a15, int a16, int a17, int a18, int a19, int a20) -{ - char buf[Z_PRINTF_BUFSIZE]; - int len; - - buf[sizeof(buf) - 1] = 0; -#ifdef NO_snprintf -# ifdef HAS_sprintf_void - sprintf(buf, format, a1, a2, a3, a4, a5, a6, a7, a8, - a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); - for (len = 0; len < sizeof(buf); len++) - if (buf[len] == 0) break; -# else - len = sprintf(buf, format, a1, a2, a3, a4, a5, a6, a7, a8, - a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); -# endif -#else -# ifdef HAS_snprintf_void - snprintf(buf, sizeof(buf), format, a1, a2, a3, a4, a5, a6, a7, a8, - a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); - len = strlen(buf); -# else - len = snprintf(buf, sizeof(buf), format, a1, a2, a3, a4, a5, a6, a7, a8, - a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); -# endif -#endif - if (len <= 0 || len >= sizeof(buf) || buf[sizeof(buf) - 1] != 0) - return 0; - return gzwrite(file, buf, len); -} -#endif - -/* =========================================================================== - Writes c, converted to an unsigned char, into the compressed file. - gzputc returns the value that was written, or -1 in case of error. -*/ -int ZEXPORT -gzputc(gzFile file, int c) -{ - unsigned char cc = (unsigned char) c; /* required for big endian systems */ - - return gzwrite(file, &cc, 1) == 1 ? (int)cc : -1; -} - - -/* =========================================================================== - Writes the given null-terminated string to the compressed file, excluding - the terminating null character. - gzputs returns the number of characters written, or -1 in case of error. -*/ -int ZEXPORT -gzputs(gzFile file, const char *s) -{ - return gzwrite(file, (char*)s, (unsigned)strlen(s)); -} - - -/* =========================================================================== - Flushes all pending output into the compressed file. The parameter - flush is as in the deflate() function. -*/ -local int -do_flush(gzFile file, int flush) -{ - uInt len; - int done = 0; - gz_stream *s = (gz_stream*)file; - - if (s == NULL || s->mode != 'w') return Z_STREAM_ERROR; - - s->stream.avail_in = 0; /* should be zero already anyway */ - - for (;;) { - len = Z_BUFSIZE - s->stream.avail_out; - - if (len != 0) { - if ((uInt)fwrite(s->outbuf, 1, len, s->file) != len) { - s->z_err = Z_ERRNO; - return Z_ERRNO; - } - s->stream.next_out = s->outbuf; - s->stream.avail_out = Z_BUFSIZE; - } - if (done) break; - s->out += s->stream.avail_out; - s->z_err = deflate(&(s->stream), flush); - s->out -= s->stream.avail_out; - - /* Ignore the second of two consecutive flushes: */ - if (len == 0 && s->z_err == Z_BUF_ERROR) s->z_err = Z_OK; - - /* deflate has finished flushing only when it hasn't used up - * all the available space in the output buffer: - */ - done = (s->stream.avail_out != 0 || s->z_err == Z_STREAM_END); - - if (s->z_err != Z_OK && s->z_err != Z_STREAM_END) break; - } - return s->z_err == Z_STREAM_END ? Z_OK : s->z_err; -} - -int ZEXPORT -gzflush(gzFile file, int flush) -{ - gz_stream *s = (gz_stream*)file; - int err = do_flush (file, flush); - - if (err) return err; - fflush(s->file); - return s->z_err == Z_STREAM_END ? Z_OK : s->z_err; -} -#endif /* NO_GZCOMPRESS */ - -/* =========================================================================== - Sets the starting position for the next gzread or gzwrite on the given - compressed file. The offset represents a number of bytes in the - gzseek returns the resulting offset location as measured in bytes from - the beginning of the uncompressed stream, or -1 in case of error. - SEEK_END is not implemented, returns error. - In this version of the library, gzseek can be extremely slow. -*/ -z_off_t ZEXPORT -gzseek(gzFile file, z_off_t offset, int whence) -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL || whence == SEEK_END || - s->z_err == Z_ERRNO || s->z_err == Z_DATA_ERROR) { - return -1L; - } - - if (s->mode == 'w') { -#ifdef NO_GZCOMPRESS - return -1L; -#else - if (whence == SEEK_SET) { - offset -= s->in; - } - if (offset < 0) return -1L; - - /* At this point, offset is the number of zero bytes to write. */ - if (s->inbuf == Z_NULL) { - s->inbuf = (Byte*)ALLOC(Z_BUFSIZE); /* for seeking */ - if (s->inbuf == Z_NULL) return -1L; - zmemzero(s->inbuf, Z_BUFSIZE); - } - while (offset > 0) { - uInt size = Z_BUFSIZE; - if (offset < Z_BUFSIZE) size = (uInt)offset; - - size = gzwrite(file, s->inbuf, size); - if (size == 0) return -1L; - - offset -= size; - } - return s->in; -#endif - } - /* Rest of function is for reading only */ - - /* compute absolute position */ - if (whence == SEEK_CUR) { - offset += s->out; - } - if (offset < 0) return -1L; - - if (s->transparent) { - /* map to fseek */ - s->back = EOF; - s->stream.avail_in = 0; - s->stream.next_in = s->inbuf; - if (fseek(s->file, offset, SEEK_SET) < 0) return -1L; - - s->in = s->out = offset; - return offset; - } - - /* For a negative seek, rewind and use positive seek */ - if (offset >= s->out) { - offset -= s->out; - } else if (gzrewind(file) < 0) { - return -1L; - } - /* offset is now the number of bytes to skip. */ - - if (offset != 0 && s->outbuf == Z_NULL) { - s->outbuf = (Byte*)ALLOC(Z_BUFSIZE); - if (s->outbuf == Z_NULL) return -1L; - } - if (offset && s->back != EOF) { - s->back = EOF; - s->out++; - offset--; - if (s->last) s->z_err = Z_STREAM_END; - } - while (offset > 0) { - int size = Z_BUFSIZE; - if (offset < Z_BUFSIZE) size = (int)offset; - - size = gzread(file, s->outbuf, (uInt)size); - if (size <= 0) return -1L; - offset -= size; - } - return s->out; -} - -/* =========================================================================== - Rewinds input file. -*/ -int ZEXPORT -gzrewind(gzFile file) -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL || s->mode != 'r') return -1; - - s->z_err = Z_OK; - s->z_eof = 0; - s->back = EOF; - s->stream.avail_in = 0; - s->stream.next_in = s->inbuf; - s->crc = z_crc32(0L, Z_NULL, 0); - if (!s->transparent) (void)inflateReset(&s->stream); - s->in = 0; - s->out = 0; - return fseek(s->file, s->start, SEEK_SET); -} - -/* =========================================================================== - Returns the starting position for the next gzread or gzwrite on the - given compressed file. This position represents a number of bytes in the - uncompressed data stream. -*/ -z_off_t ZEXPORT -gztell(gzFile file) -{ - return gzseek(file, 0L, SEEK_CUR); -} - -/* =========================================================================== - Returns 1 when EOF has previously been detected reading the given - input stream, otherwise zero. -*/ -int ZEXPORT -gzeof(gzFile file) -{ - gz_stream *s = (gz_stream*)file; - - /* With concatenated compressed files that can have embedded - * crc trailers, z_eof is no longer the only/best indicator of EOF - * on a gz_stream. Handle end-of-stream error explicitly here. - */ - if (s == NULL || s->mode != 'r') return 0; - if (s->z_eof) return 1; - return s->z_err == Z_STREAM_END; -} - -/* =========================================================================== - Returns 1 if reading and doing so transparently, otherwise zero. -*/ -int ZEXPORT -gzdirect(gzFile file) -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL || s->mode != 'r') return 0; - return s->transparent; -} - -/* =========================================================================== - Outputs a long in LSB order to the given file -*/ -local void -putLong(FILE *file, uLong x) -{ - int n; - for (n = 0; n < 4; n++) { - fputc((int)(x & 0xff), file); - x >>= 8; - } -} - -/* =========================================================================== - Reads a long in LSB order from the given gz_stream. Sets z_err in case - of error. -*/ -local uLong -getLong(gz_stream *s) -{ - uLong x = (uLong)get_byte(s); - int c; - - x += ((uLong)get_byte(s))<<8; - x += ((uLong)get_byte(s))<<16; - c = get_byte(s); - if (c == EOF) s->z_err = Z_DATA_ERROR; - x += ((uLong)c)<<24; - return x; -} - -/* =========================================================================== - Flushes all pending output if necessary, closes the compressed file - and deallocates all the (de)compression state. -*/ -int ZEXPORT -gzclose(gzFile file) -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL) return Z_STREAM_ERROR; - - if (s->mode == 'w') { -#ifdef NO_GZCOMPRESS - return Z_STREAM_ERROR; -#else - if (do_flush (file, Z_FINISH) != Z_OK) - return destroy((gz_stream*)file); - - putLong (s->file, s->crc); - putLong (s->file, (uLong)(s->in & 0xffffffff)); -#endif - } - return destroy((gz_stream*)file); -} - -#ifdef STDC -# define zstrerror(errnum) strerror(errnum) -#else -# define zstrerror(errnum) "" -#endif - -/* =========================================================================== - Returns the error message for the last error which occurred on the - given compressed file. errnum is set to zlib error number. If an - error occurred in the file system and not in the compression library, - errnum is set to Z_ERRNO and the application may consult errno - to get the exact error code. -*/ -const char * ZEXPORT -gzerror(gzFile file, int *errnum) -{ - char *m; - gz_stream *s = (gz_stream*)file; - - if (s == NULL) { - *errnum = Z_STREAM_ERROR; - return (const char*)ERR_MSG(Z_STREAM_ERROR); - } - *errnum = s->z_err; - if (*errnum == Z_OK) return (const char*)""; - - m = (char*)(*errnum == Z_ERRNO ? zstrerror(errno) : s->stream.msg); - - if (m == NULL || *m == '\0') m = (char*)ERR_MSG(s->z_err); - - TRYFREE(s->msg); - s->msg = (char*)ALLOC(strlen(s->path) + strlen(m) + 3); - if (s->msg == Z_NULL) return (const char*)ERR_MSG(Z_MEM_ERROR); - strcpy(s->msg, s->path); - strcat(s->msg, ": "); - strcat(s->msg, m); - return (const char*)s->msg; -} - -/* =========================================================================== - Clear the error and end-of-file flags, and do the same for the real file. -*/ -void ZEXPORT -gzclearerr(gzFile file) -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL) return; - if (s->z_err != Z_STREAM_END) s->z_err = Z_OK; - s->z_eof = 0; - clearerr(s->file); -} diff --git a/libsa/bootstrap.cpp b/libsa/bootstrap.cpp index 1e60c8ccf..80a7508f3 100644 --- a/libsa/bootstrap.cpp +++ b/libsa/bootstrap.cpp @@ -30,6 +30,9 @@ extern "C" { #include <libkern/kernel_mach_header.h> #include <libkern/prelink.h> +#if CONFIG_EMBEDDED +extern uuid_t kernelcache_uuid; +#endif } #include <libkern/version.h> @@ -243,6 +246,9 @@ KLDBootstrap::readPrelinkedExtensions( OSDictionary * prelinkInfoDict = NULL; // do not release OSString * errorString = NULL; // must release OSKext * theKernel = NULL; // must release +#if CONFIG_EMBEDDED + OSData * kernelcacheUUID = NULL; // do not release +#endif kernel_segment_command_t * prelinkTextSegment = NULL; // see code kernel_segment_command_t * prelinkInfoSegment = NULL; // see code @@ -368,6 +374,19 @@ KLDBootstrap::readPrelinkedExtensions( ramDiskBoot = IORamDiskBSDRoot(); #endif /* NO_KEXTD */ +#if CONFIG_EMBEDDED + /* Copy in the kernelcache UUID */ + kernelcacheUUID = OSDynamicCast(OSData, + prelinkInfoDict->getObject(kPrelinkInfoKCIDKey)); + if (!kernelcacheUUID) { + bzero(&kernelcache_uuid, sizeof(kernelcache_uuid)); + } else if (kernelcacheUUID->getLength() != sizeof(kernelcache_uuid)) { + panic("kernelcacheUUID length is %d, expected %lu", kernelcacheUUID->getLength(), + sizeof(kernelcache_uuid)); + } else { + memcpy((void *)&kernelcache_uuid, (void *)kernelcacheUUID->getBytesNoCopy(), kernelcacheUUID->getLength()); + } +#endif /* CONFIG_EMBEDDED */ infoDictArray = OSDynamicCast(OSArray, prelinkInfoDict->getObject(kPrelinkInfoDictionaryKey)); @@ -383,7 +402,9 @@ KLDBootstrap::readPrelinkedExtensions( /* Create dictionary of excluded kexts */ +#ifndef CONFIG_EMBEDDED OSKext::createExcludeListFromPrelinkInfo(infoDictArray); +#endif /* Create OSKext objects for each info dictionary. */ for (i = 0; i < infoDictArray->getCount(); ++i) { @@ -432,7 +453,13 @@ KLDBootstrap::readPrelinkedExtensions( OSNumber *lengthNum = OSDynamicCast(OSNumber, infoDict->getObject(kPrelinkExecutableSizeKey)); if (addressNum && lengthNum) { +#if __arm__ || __arm64__ + vm_offset_t data = (vm_offset_t) ((addressNum->unsigned64BitValue()) + vm_kernel_slide); + vm_size_t length = (vm_size_t) (lengthNum->unsigned32BitValue()); + ml_static_mfree(data, length); +#else #error Pick the right way to free prelinked data on this arch +#endif } infoDictArray->removeObject(i--); @@ -605,7 +632,9 @@ KLDBootstrap::readBooterExtensions(void) /* Create dictionary of excluded kexts */ +#ifndef CONFIG_EMBEDDED OSKext::createExcludeListFromBooterData(propertyDict, keyIterator); +#endif keyIterator->reset(); while ( ( deviceTreeName = diff --git a/libsa/conf/Makefile.arm b/libsa/conf/Makefile.arm new file mode 100644 index 000000000..1c1cef911 --- /dev/null +++ b/libsa/conf/Makefile.arm @@ -0,0 +1,10 @@ +###################################################################### +#BEGIN Machine dependent Makefile fragment for arm +###################################################################### + +# Bootstrap __KLD files must be Mach-O for "setsegname" +$(foreach file,$(OBJS),$(eval $(file)_CFLAGS_ADD += $(CFLAGS_NOLTO_FLAG))) + +###################################################################### +#END Machine dependent Makefile fragment for arm +###################################################################### diff --git a/libsa/conf/Makefile.arm64 b/libsa/conf/Makefile.arm64 new file mode 100644 index 000000000..1c1cef911 --- /dev/null +++ b/libsa/conf/Makefile.arm64 @@ -0,0 +1,10 @@ +###################################################################### +#BEGIN Machine dependent Makefile fragment for arm +###################################################################### + +# Bootstrap __KLD files must be Mach-O for "setsegname" +$(foreach file,$(OBJS),$(eval $(file)_CFLAGS_ADD += $(CFLAGS_NOLTO_FLAG))) + +###################################################################### +#END Machine dependent Makefile fragment for arm +###################################################################### diff --git a/libsa/conf/Makefile.template b/libsa/conf/Makefile.template index ebfd0d2a8..bc570dde5 100644 --- a/libsa/conf/Makefile.template +++ b/libsa/conf/Makefile.template @@ -17,6 +17,7 @@ include $(MakeInc_def) # CFLAGS # CFLAGS+= -include meta_features.h -DLIBSA_KERNEL_PRIVATE +SFLAGS+= -include meta_features.h # # Directories for mig generated files diff --git a/libsa/conf/files.arm b/libsa/conf/files.arm new file mode 100644 index 000000000..e69de29bb diff --git a/libsa/conf/files.arm64 b/libsa/conf/files.arm64 new file mode 100644 index 000000000..e69de29bb diff --git a/libsa/lastkerneldataconst.c b/libsa/lastkerneldataconst.c index 9b8db0b51..580756f0f 100644 --- a/libsa/lastkerneldataconst.c +++ b/libsa/lastkerneldataconst.c @@ -38,7 +38,12 @@ * alignment and no straight forward way to specify section ordering. */ +#if defined(__arm64__) +/* PAGE_SIZE on ARM64 is an expression derived from a non-const global variable */ +#define PAD_SIZE PAGE_MAX_SIZE +#else #define PAD_SIZE PAGE_SIZE +#endif static const uint8_t __attribute__((section("__DATA,__const"))) data_const_padding[PAD_SIZE] = {[0 ... PAD_SIZE-1] = 0xFF}; const vm_offset_t __attribute__((section("__DATA,__data"))) _lastkerneldataconst = (vm_offset_t)&data_const_padding[0]; diff --git a/libsyscall/Libsyscall.xcconfig b/libsyscall/Libsyscall.xcconfig index 181fe1f4e..105571937 100644 --- a/libsyscall/Libsyscall.xcconfig +++ b/libsyscall/Libsyscall.xcconfig @@ -1,7 +1,7 @@ #include "<DEVELOPER_DIR>/Makefiles/CoreOS/Xcode/BSD.xcconfig" BUILD_VARIANTS = normal -SUPPORTED_PLATFORMS = macosx iphoneos iphoneosnano tvos appletvos watchos +SUPPORTED_PLATFORMS = macosx iphoneos iphoneosnano tvos appletvos watchos bridgeos ONLY_ACTIVE_ARCH = NO DEAD_CODE_STRIPPING = YES DEBUG_INFORMATION_FORMAT = dwarf-with-dsym @@ -20,6 +20,7 @@ OTHER_CFLAGS[sdk=iphoneos*] = $(inherited) -DNO_SYSCALL_LEGACY OTHER_CFLAGS[sdk=watchos*] = $(inherited) -DNO_SYSCALL_LEGACY OTHER_CFLAGS[sdk=tvos*] = $(inherited) -DNO_SYSCALL_LEGACY OTHER_CFLAGS[sdk=appletvos*] = $(inherited) -DNO_SYSCALL_LEGACY +OTHER_CFLAGS[sdk=bridgeos*] = $(inherited) -DNO_SYSCALL_LEGACY GCC_PREPROCESSOR_DEFINITIONS = CF_OPEN_SOURCE CF_EXCLUDE_CSTD_HEADERS DEBUG _FORTIFY_SOURCE=0 HEADER_SEARCH_PATHS = $(PROJECT_DIR)/mach $(PROJECT_DIR)/os $(PROJECT_DIR)/wrappers $(PROJECT_DIR)/wrappers/string $(PROJECT_DIR)/wrappers/libproc $(PROJECT_DIR)/wrappers/libproc/spawn $(BUILT_PRODUCTS_DIR)/internal_hdr/include $(BUILT_PRODUCTS_DIR)/mig_hdr/local/include $(BUILT_PRODUCTS_DIR)/mig_hdr/include $(SDKROOT)/System/Library/Frameworks/System.framework/PrivateHeaders WARNING_CFLAGS = -Wmost @@ -27,11 +28,12 @@ GCC_TREAT_WARNINGS_AS_ERRORS = YES GCC_WARN_ABOUT_MISSING_NEWLINE = YES CODE_SIGN_IDENTITY = - DYLIB_CURRENT_VERSION = $(RC_ProjectSourceVersion) -DYLIB_LDFLAGS = -umbrella System -all_load +DYLIB_LDFLAGS = -umbrella System -all_load -lCrashReporterClient DYLIB_LDFLAGS[sdk=iphoneos*] = $(inherited) -Wl,-sectalign,__DATA,__data,1000 DYLIB_LDFLAGS[sdk=watchos*] = $(inherited) -Wl,-sectalign,__DATA,__data,1000 DYLIB_LDFLAGS[sdk=tvos*] = $(inherited) -Wl,-sectalign,__DATA,__data,1000 DYLIB_LDFLAGS[sdk=appletvos*] = $(inherited) -Wl,-sectalign,__DATA,__data,1000 +DYLIB_LDFLAGS[sdk=bridgeos*] = $(inherited) -Wl,-sectalign,__DATA,__data,1000 OTHER_LDFLAGS = INSTALLHDRS_SCRIPT_PHASE = YES INSTALLHDRS_COPY_PHASE = YES diff --git a/libsyscall/Libsyscall.xcodeproj/project.pbxproj b/libsyscall/Libsyscall.xcodeproj/project.pbxproj index 1f54c0158..8175500a0 100644 --- a/libsyscall/Libsyscall.xcodeproj/project.pbxproj +++ b/libsyscall/Libsyscall.xcodeproj/project.pbxproj @@ -111,6 +111,11 @@ 3F538F891A659C5600B37EFD /* persona.c in Sources */ = {isa = PBXBuildFile; fileRef = 3F538F881A659C5600B37EFD /* persona.c */; }; 401BB71A1BCAE57B005080D3 /* os_channel.c in Sources */ = {isa = PBXBuildFile; fileRef = 401BB7161BCAE539005080D3 /* os_channel.c */; settings = {COMPILER_FLAGS = "-fno-builtin"; }; }; 401BB71C1BCAE57B005080D3 /* os_nexus.c in Sources */ = {isa = PBXBuildFile; fileRef = 401BB7181BCAE539005080D3 /* os_nexus.c */; settings = {COMPILER_FLAGS = "-fno-builtin"; }; }; + 402AF43F1E5CD88600F1A4B9 /* cpu_in_cksum_gen.c in Sources */ = {isa = PBXBuildFile; fileRef = 402AF43E1E5CD88100F1A4B9 /* cpu_in_cksum_gen.c */; settings = {COMPILER_FLAGS = "-fno-builtin"; }; }; + 403C7CEE1E1F4E4400D6FEEF /* os_packet.c in Sources */ = {isa = PBXBuildFile; fileRef = 405FA3381E0C669D007D66EA /* os_packet.c */; settings = {COMPILER_FLAGS = "-fno-builtin"; }; }; + 406E0B721E4ACD2000295EA3 /* cpu_copy_in_cksum.s in Sources */ = {isa = PBXBuildFile; fileRef = 40DD162F1E4ACCAA003297CC /* cpu_copy_in_cksum.s */; }; + 409A78321E4EB3E300E0699B /* cpu_in_cksum.s in Sources */ = {isa = PBXBuildFile; fileRef = 409A78301E4EB3D900E0699B /* cpu_in_cksum.s */; }; + 40DF0F741E5CD7BB0035A864 /* cpu_copy_in_cksum_gen.c in Sources */ = {isa = PBXBuildFile; fileRef = 40DF0F731E5CD7B30035A864 /* cpu_copy_in_cksum_gen.c */; settings = {COMPILER_FLAGS = "-fno-builtin"; }; }; 435F3CAA1B06B7BA005ED9EF /* work_interval.c in Sources */ = {isa = PBXBuildFile; fileRef = 435F3CA91B06B7BA005ED9EF /* work_interval.c */; }; 467DAFD4157E8AF200CE68F0 /* guarded_open_np.c in Sources */ = {isa = PBXBuildFile; fileRef = 467DAFD3157E8AF200CE68F0 /* guarded_open_np.c */; }; 4BDD5F1D1891AB2F004BF300 /* mach_approximate_time.c in Sources */ = {isa = PBXBuildFile; fileRef = 4BDD5F1B1891AB2F004BF300 /* mach_approximate_time.c */; }; @@ -123,6 +128,7 @@ 74F3290B18EB269400B2B70E /* vm_page_size.h in CopyFiles */ = {isa = PBXBuildFile; fileRef = 7466C923170CB99B004557CC /* vm_page_size.h */; }; 7AE28FDF18AC41B1006A5626 /* csr.c in Sources */ = {isa = PBXBuildFile; fileRef = 7AE28FDE18AC41B1006A5626 /* csr.c */; }; 9002401118FC9A7F00D73BFA /* renamex.c in Sources */ = {isa = PBXBuildFile; fileRef = 906AA2D018F74CD1001C681A /* renamex.c */; }; + 92197BAF1EAD8F2C003994B9 /* utimensat.c in Sources */ = {isa = PBXBuildFile; fileRef = 92197BAE1EAD8DF2003994B9 /* utimensat.c */; }; 925559921CBC23C300E527CE /* mach_boottime.c in Sources */ = {isa = PBXBuildFile; fileRef = 925559911CBBBBB300E527CE /* mach_boottime.c */; }; 928336A11B83ED9100873B90 /* thread_register_state.c in Sources */ = {isa = PBXBuildFile; fileRef = 928336A01B83ED7800873B90 /* thread_register_state.c */; }; 9299E14A1B841E74005B7350 /* thread_state.h in Headers */ = {isa = PBXBuildFile; fileRef = 928336A21B8412C100873B90 /* thread_state.h */; }; @@ -130,6 +136,11 @@ 929FD46F1C5711DB0087B9C8 /* mach_timebase_info.c in Sources */ = {isa = PBXBuildFile; fileRef = 929FD46E1C5711CF0087B9C8 /* mach_timebase_info.c */; }; 978228281B8678DC008385AC /* pselect-darwinext.c in Sources */ = {isa = PBXBuildFile; fileRef = 978228271B8678CB008385AC /* pselect-darwinext.c */; }; 978228291B8678DF008385AC /* pselect-darwinext-cancel.c in Sources */ = {isa = PBXBuildFile; fileRef = 978228261B8678C2008385AC /* pselect-darwinext-cancel.c */; }; + 9CCF28271E68E993002EE6CD /* pid_shutdown_networking.c in Sources */ = {isa = PBXBuildFile; fileRef = 9CCF28261E68E993002EE6CD /* pid_shutdown_networking.c */; }; + A50845861DDA69AC0041C0E0 /* thread_self_restrict.h in CopyFiles */ = {isa = PBXBuildFile; fileRef = A50BD52E1DDA548F006622C8 /* thread_self_restrict.h */; }; + A50845871DDA69C90041C0E0 /* thread_self_restrict.h in CopyFiles */ = {isa = PBXBuildFile; fileRef = A50BD52E1DDA548F006622C8 /* thread_self_restrict.h */; }; + A50BD52F1DDA548F006622C8 /* thread_self_restrict.h in Headers */ = {isa = PBXBuildFile; fileRef = A50BD52E1DDA548F006622C8 /* thread_self_restrict.h */; }; + A50BD5301DDA5500006622C8 /* thread_self_restrict.h in Headers */ = {isa = PBXBuildFile; fileRef = A50BD52E1DDA548F006622C8 /* thread_self_restrict.h */; }; A59CB95616669EFB00B064B3 /* stack_logging_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = A59CB95516669DB700B064B3 /* stack_logging_internal.h */; }; A59CB9581666A1A200B064B3 /* munmap.c in Sources */ = {isa = PBXBuildFile; fileRef = A59CB9571666A1A200B064B3 /* munmap.c */; }; BA0D9FB1199031AD007E8A73 /* kdebug_trace.c in Sources */ = {isa = PBXBuildFile; fileRef = BA0D9FB0199031AD007E8A73 /* kdebug_trace.c */; }; @@ -239,7 +250,6 @@ E453AF3917013F1B00F2C94C /* spawn_private.h in Headers */ = {isa = PBXBuildFile; fileRef = E4D45C3E16FB20970002AF25 /* spawn_private.h */; settings = {ATTRIBUTES = (Private, ); }; }; E453AF3A17013F4C00F2C94C /* stack_logging_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = A59CB95516669DB700B064B3 /* stack_logging_internal.h */; }; E4D45C2416F856900002AF25 /* __commpage_gettimeofday.c in Sources */ = {isa = PBXBuildFile; fileRef = E4D45C2116F856900002AF25 /* __commpage_gettimeofday.c */; }; - E4D45C2516F856900002AF25 /* __commpage_gettimeofday.s in Sources */ = {isa = PBXBuildFile; fileRef = E4D45C2216F856900002AF25 /* __commpage_gettimeofday.s */; }; E4D45C2616F856900002AF25 /* mach_absolute_time.s in Sources */ = {isa = PBXBuildFile; fileRef = E4D45C2316F856900002AF25 /* mach_absolute_time.s */; }; E4D45C2E16F868ED0002AF25 /* libproc.c in Sources */ = {isa = PBXBuildFile; fileRef = E4D45C2A16F868ED0002AF25 /* libproc.c */; }; E4D45C2F16F868ED0002AF25 /* libproc.h in Headers */ = {isa = PBXBuildFile; fileRef = E4D45C2B16F868ED0002AF25 /* libproc.h */; settings = {ATTRIBUTES = (Public, ); }; }; @@ -377,6 +387,7 @@ dstPath = "$(OS_PRIVATE_HEADERS_FOLDER_PATH)"; dstSubfolderSpec = 0; files = ( + A50845871DDA69C90041C0E0 /* thread_self_restrict.h in CopyFiles */, C9FD8508166D6BD400963B73 /* tsd.h in CopyFiles */, ); runOnlyForDeploymentPostprocessing = 1; @@ -387,6 +398,7 @@ dstPath = "$(OS_PRIVATE_HEADERS_FOLDER_PATH)"; dstSubfolderSpec = 0; files = ( + A50845861DDA69AC0041C0E0 /* thread_self_restrict.h in CopyFiles */, C9A3D6EB1672AD1000A5CAA3 /* tsd.h in CopyFiles */, ); runOnlyForDeploymentPostprocessing = 1; @@ -484,6 +496,11 @@ 3F538F881A659C5600B37EFD /* persona.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = persona.c; sourceTree = "<group>"; }; 401BB7161BCAE539005080D3 /* os_channel.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = os_channel.c; path = skywalk/os_channel.c; sourceTree = "<group>"; }; 401BB7181BCAE539005080D3 /* os_nexus.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = os_nexus.c; path = skywalk/os_nexus.c; sourceTree = "<group>"; }; + 402AF43E1E5CD88100F1A4B9 /* cpu_in_cksum_gen.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = cpu_in_cksum_gen.c; path = skywalk/cpu_in_cksum_gen.c; sourceTree = "<group>"; }; + 405FA3381E0C669D007D66EA /* os_packet.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = os_packet.c; path = skywalk/os_packet.c; sourceTree = "<group>"; }; + 409A78301E4EB3D900E0699B /* cpu_in_cksum.s */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.asm; name = cpu_in_cksum.s; path = skywalk/cpu_in_cksum.s; sourceTree = "<group>"; }; + 40DD162F1E4ACCAA003297CC /* cpu_copy_in_cksum.s */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.asm; name = cpu_copy_in_cksum.s; path = skywalk/cpu_copy_in_cksum.s; sourceTree = "<group>"; }; + 40DF0F731E5CD7B30035A864 /* cpu_copy_in_cksum_gen.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = cpu_copy_in_cksum_gen.c; path = skywalk/cpu_copy_in_cksum_gen.c; sourceTree = "<group>"; }; 435F3CA91B06B7BA005ED9EF /* work_interval.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = work_interval.c; sourceTree = "<group>"; }; 467DAFD3157E8AF200CE68F0 /* guarded_open_np.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = guarded_open_np.c; sourceTree = "<group>"; }; 4BDD5F1B1891AB2F004BF300 /* mach_approximate_time.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = mach_approximate_time.c; sourceTree = "<group>"; }; @@ -493,12 +510,15 @@ 7466C923170CB99B004557CC /* vm_page_size.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = vm_page_size.h; sourceTree = "<group>"; }; 7AE28FDE18AC41B1006A5626 /* csr.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = csr.c; sourceTree = "<group>"; }; 906AA2D018F74CD1001C681A /* renamex.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = renamex.c; sourceTree = "<group>"; }; + 92197BAE1EAD8DF2003994B9 /* utimensat.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = utimensat.c; sourceTree = "<group>"; }; 925559911CBBBBB300E527CE /* mach_boottime.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = mach_boottime.c; sourceTree = "<group>"; }; 928336A01B83ED7800873B90 /* thread_register_state.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = thread_register_state.c; sourceTree = "<group>"; }; 928336A21B8412C100873B90 /* thread_state.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = thread_state.h; sourceTree = "<group>"; }; 929FD46E1C5711CF0087B9C8 /* mach_timebase_info.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = mach_timebase_info.c; sourceTree = "<group>"; }; 978228261B8678C2008385AC /* pselect-darwinext-cancel.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = "pselect-darwinext-cancel.c"; sourceTree = "<group>"; }; 978228271B8678CB008385AC /* pselect-darwinext.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = "pselect-darwinext.c"; sourceTree = "<group>"; }; + 9CCF28261E68E993002EE6CD /* pid_shutdown_networking.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = pid_shutdown_networking.c; sourceTree = "<group>"; }; + A50BD52E1DDA548F006622C8 /* thread_self_restrict.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = thread_self_restrict.h; sourceTree = "<group>"; }; A59CB95516669DB700B064B3 /* stack_logging_internal.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = stack_logging_internal.h; sourceTree = "<group>"; }; A59CB9571666A1A200B064B3 /* munmap.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = munmap.c; sourceTree = "<group>"; }; BA0D9FB0199031AD007E8A73 /* kdebug_trace.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = kdebug_trace.c; sourceTree = "<group>"; }; @@ -521,7 +541,6 @@ C99A4F4E1305B1B70054B7B7 /* __get_cpu_capabilities.s */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.asm; path = __get_cpu_capabilities.s; sourceTree = "<group>"; }; C99A4F511305B43F0054B7B7 /* init_cpu_capabilities.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = init_cpu_capabilities.c; sourceTree = "<group>"; }; C9C1824F15338C0B00933F23 /* alloc_once.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = alloc_once.c; sourceTree = "<group>"; }; - C9D9BCBF114B00600000D8B9 /* .open_source_exclude */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = .open_source_exclude; sourceTree = "<group>"; }; C9D9BCC5114B00600000D8B9 /* clock_priv.defs */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.mig; path = clock_priv.defs; sourceTree = "<group>"; }; C9D9BCC6114B00600000D8B9 /* clock_reply.defs */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.mig; path = clock_reply.defs; sourceTree = "<group>"; }; C9D9BCC7114B00600000D8B9 /* clock_sleep.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = clock_sleep.c; sourceTree = "<group>"; }; @@ -588,7 +607,6 @@ E4216C301822D404006F2632 /* mach_voucher.defs */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.mig; path = mach_voucher.defs; sourceTree = "<group>"; }; E453AF341700FD3C00F2C94C /* getiopolicy_np.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = getiopolicy_np.c; sourceTree = "<group>"; }; E4D45C2116F856900002AF25 /* __commpage_gettimeofday.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = __commpage_gettimeofday.c; sourceTree = "<group>"; }; - E4D45C2216F856900002AF25 /* __commpage_gettimeofday.s */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.asm; path = __commpage_gettimeofday.s; sourceTree = "<group>"; }; E4D45C2316F856900002AF25 /* mach_absolute_time.s */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.asm; path = mach_absolute_time.s; sourceTree = "<group>"; }; E4D45C2A16F868ED0002AF25 /* libproc.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = libproc.c; sourceTree = "<group>"; }; E4D45C2B16F868ED0002AF25 /* libproc.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = libproc.h; sourceTree = "<group>"; }; @@ -649,7 +667,6 @@ 08FB7795FE84155DC02AAC07 /* mach */ = { isa = PBXGroup; children = ( - C9D9BCBE114B00600000D8B9 /* arm */, 247A08FF11F8E18000E4693F /* abort.h */, C9D9BCC5114B00600000D8B9 /* clock_priv.defs */, C9D9BCC6114B00600000D8B9 /* clock_reply.defs */, @@ -751,14 +768,20 @@ isa = PBXGroup; children = ( E4D45C2116F856900002AF25 /* __commpage_gettimeofday.c */, + C99A4F4E1305B1B70054B7B7 /* __get_cpu_capabilities.s */, + 24A7C5CB11FF973C007669EB /* _errno.h */, 24E47824120881DF009A384D /* _libc_funcptr.c */, 247A08B311F8B05900E4693F /* _libkernel_init.c */, + 247A08B211F8B05900E4693F /* _libkernel_init.h */, + 248BA04A121C8EE4008C073F /* cancelable */, FB50F1B315AB7DE700F814BA /* carbon_delete.c */, E214BDC71C2E34E200CEE8A3 /* clonefile.c */, 2BA88DCB1810A3CE00EB63F6 /* coalition.c */, 7AE28FDE18AC41B1006A5626 /* csr.c */, E2A0F3331C3B17D100A11F8A /* fs_snapshot.c */, + C6C40121174154D9000AE69F /* gethostuuid_private.h */, C6C4012017415384000AE69F /* gethostuuid.c */, + C639F0E41741C09A00A39F47 /* gethostuuid.h */, E453AF341700FD3C00F2C94C /* getiopolicy_np.c */, 72B1E6EC190723DB00FB3FA2 /* guarded_open_dprotected_np.c */, 467DAFD3157E8AF200CE68F0 /* guarded_open_np.c */, @@ -766,13 +789,18 @@ 248BA07F121DA36B008C073F /* ioctl.c */, BA0D9FB0199031AD007E8A73 /* kdebug_trace.c */, 248BA081121DA4F3008C073F /* kill.c */, + 24A7C6951200AF8A007669EB /* legacy */, + E4D45C2916F868ED0002AF25 /* libproc */, + E4D45C2316F856900002AF25 /* mach_absolute_time.s */, 4BDD5F1B1891AB2F004BF300 /* mach_approximate_time.c */, + 4BDD5F1C1891AB2F004BF300 /* mach_approximate_time.s */, 925559911CBBBBB300E527CE /* mach_boottime.c */, 72FB18801B437F7A00181A5B /* mach_continuous_time.c */, 14FE60EB1B7D3BED00ACB44C /* mach_get_times.c */, 929FD46E1C5711CF0087B9C8 /* mach_timebase_info.c */, 030B179A135377B400DAD1F0 /* open_dprotected_np.c */, 3F538F881A659C5600B37EFD /* persona.c */, + 9CCF28261E68E993002EE6CD /* pid_shutdown_networking.c */, C6BEE9171806840200D25AAB /* posix_sem_obsolete.c */, BA9973461C3B4C8A00B14D8C /* quota_obsolete.c */, 24B8C2611237F53900D36CC3 /* remove-counter.c */, @@ -784,28 +812,18 @@ C962B16B18DBA2C80031244A /* setpriority.c */, C6460B7B182025DF00F73CCA /* sfi.c */, 24B223B3121DFF12007DAEDE /* sigsuspend-base.c */, + 401BB7141BCAE523005080D3 /* skywalk */, + E4D45C3B16FB20970002AF25 /* spawn */, 13B598931A142F5900DB2D5A /* stackshot.c */, + E4D7E55216F8776300F92D8D /* string */, 13D932CB1C7B9DE600158FA1 /* terminate_with_reason.c */, 928336A01B83ED7800873B90 /* thread_register_state.c */, + 2419382912135FE1003CDE41 /* unix03 */, 248AA962122C7B2A0085F5B1 /* unlink.c */, 29A59AE5183B110C00E8B896 /* unlinkat.c */, - 435F3CA91B06B7BA005ED9EF /* work_interval.c */, - 24A7C5CB11FF973C007669EB /* _errno.h */, - 247A08B211F8B05900E4693F /* _libkernel_init.h */, - C6C40121174154D9000AE69F /* gethostuuid_private.h */, - C639F0E41741C09A00A39F47 /* gethostuuid.h */, - E4D45C2216F856900002AF25 /* __commpage_gettimeofday.s */, - C99A4F4E1305B1B70054B7B7 /* __get_cpu_capabilities.s */, - E4D45C2316F856900002AF25 /* mach_absolute_time.s */, - 4BDD5F1C1891AB2F004BF300 /* mach_approximate_time.s */, + 92197BAE1EAD8DF2003994B9 /* utimensat.c */, 374A36E214748EE400AAF39D /* varargs_wrappers.s */, - 248BA04A121C8EE4008C073F /* cancelable */, - 24A7C6951200AF8A007669EB /* legacy */, - E4D45C2916F868ED0002AF25 /* libproc */, - 401BB7141BCAE523005080D3 /* skywalk */, - E4D45C3B16FB20970002AF25 /* spawn */, - E4D7E55216F8776300F92D8D /* string */, - 2419382912135FE1003CDE41 /* unix03 */, + 435F3CA91B06B7BA005ED9EF /* work_interval.c */, ); path = wrappers; sourceTree = "<group>"; @@ -943,6 +961,11 @@ 401BB7141BCAE523005080D3 /* skywalk */ = { isa = PBXGroup; children = ( + 405FA3381E0C669D007D66EA /* os_packet.c */, + 40DD162F1E4ACCAA003297CC /* cpu_copy_in_cksum.s */, + 409A78301E4EB3D900E0699B /* cpu_in_cksum.s */, + 40DF0F731E5CD7B30035A864 /* cpu_copy_in_cksum_gen.c */, + 402AF43E1E5CD88100F1A4B9 /* cpu_in_cksum_gen.c */, 401BB7161BCAE539005080D3 /* os_channel.c */, 401BB7181BCAE539005080D3 /* os_nexus.c */, ); @@ -964,18 +987,11 @@ children = ( C9C1824F15338C0B00933F23 /* alloc_once.c */, C9EE57F51669673D00337E4B /* tsd.h */, + A50BD52E1DDA548F006622C8 /* thread_self_restrict.h */, ); path = os; sourceTree = "<group>"; }; - C9D9BCBE114B00600000D8B9 /* arm */ = { - isa = PBXGroup; - children = ( - C9D9BCBF114B00600000D8B9 /* .open_source_exclude */, - ); - path = arm; - sourceTree = "<group>"; - }; C9D9BCD8114B00600000D8B9 /* mach */ = { isa = PBXGroup; children = ( @@ -1061,6 +1077,7 @@ C6D3EFBC16542C510052CF30 /* mach_interface.h in Headers */, C6D3EFBD16542C510052CF30 /* port_obj.h in Headers */, C6D3EFBE16542C510052CF30 /* sync.h in Headers */, + A50BD52F1DDA548F006622C8 /* thread_self_restrict.h in Headers */, C6D3EFC116542C510052CF30 /* vm_task.h in Headers */, C6D3EFC216542C510052CF30 /* key_defs.h in Headers */, C6D3EFC316542C510052CF30 /* ls_defs.h in Headers */, @@ -1095,6 +1112,7 @@ C9D9BD29114B00600000D8B9 /* mach_interface.h in Headers */, C9D9BD2B114B00600000D8B9 /* port_obj.h in Headers */, C9D9BD2C114B00600000D8B9 /* sync.h in Headers */, + A50BD5301DDA5500006622C8 /* thread_self_restrict.h in Headers */, C9D9BD2F114B00600000D8B9 /* vm_task.h in Headers */, C9D9BD50114B00600000D8B9 /* key_defs.h in Headers */, C9D9BD51114B00600000D8B9 /* ls_defs.h in Headers */, @@ -1283,12 +1301,14 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + 403C7CEE1E1F4E4400D6FEEF /* os_packet.c in Sources */, E214BDC81C2E358300CEE8A3 /* clonefile.c in Sources */, C9D9BD19114B00600000D8B9 /* clock_priv.defs in Sources */, C9D9BD1A114B00600000D8B9 /* clock_reply.defs in Sources */, C9D9BD1C114B00600000D8B9 /* clock.defs in Sources */, C9D9BD22114B00600000D8B9 /* exc.defs in Sources */, C9D9BD30114B00600000D8B9 /* host_priv.defs in Sources */, + 409A78321E4EB3E300E0699B /* cpu_in_cksum.s in Sources */, C9D9BD31114B00600000D8B9 /* host_security.defs in Sources */, C9D9BD35114B00600000D8B9 /* lock_set.defs in Sources */, C9D9BD38114B00600000D8B9 /* mach_host.defs in Sources */, @@ -1319,7 +1339,6 @@ C9D9BD3C114B00600000D8B9 /* mach_msg.c in Sources */, C9D9BD3E114B00600000D8B9 /* mach_traps.s in Sources */, C9D9BD41114B00600000D8B9 /* mig_allocate.c in Sources */, - E4D45C2516F856900002AF25 /* __commpage_gettimeofday.s in Sources */, C9D9BD42114B00600000D8B9 /* mig_deallocate.c in Sources */, BA9973471C3B4C9A00B14D8C /* quota_obsolete.c in Sources */, E4D45C2416F856900002AF25 /* __commpage_gettimeofday.c in Sources */, @@ -1348,6 +1367,7 @@ 24A7C5C211FF8DA6007669EB /* lchown.c in Sources */, 24A7C5C311FF8DA6007669EB /* listen.c in Sources */, 24A7C5C411FF8DA6007669EB /* recvfrom.c in Sources */, + 92197BAF1EAD8F2C003994B9 /* utimensat.c in Sources */, C962B16E18DBB43F0031244A /* thread_act.c in Sources */, 24A7C5C511FF8DA6007669EB /* recvmsg.c in Sources */, 24A7C5C611FF8DA6007669EB /* sendmsg.c in Sources */, @@ -1373,6 +1393,7 @@ C6BEE9181806840200D25AAB /* posix_sem_obsolete.c in Sources */, 248BA082121DA4F3008C073F /* kill.c in Sources */, 248BA085121DA5E4008C073F /* kill.c in Sources */, + 9CCF28271E68E993002EE6CD /* pid_shutdown_networking.c in Sources */, 2BA88DCC1810A3CE00EB63F6 /* coalition.c in Sources */, 248BA087121DA72D008C073F /* mmap.c in Sources */, 7AE28FDF18AC41B1006A5626 /* csr.c in Sources */, @@ -1398,10 +1419,13 @@ 248AA965122C7C330085F5B1 /* rmdir.c in Sources */, 435F3CAA1B06B7BA005ED9EF /* work_interval.c in Sources */, 248AA967122C7CDA0085F5B1 /* rename.c in Sources */, + 406E0B721E4ACD2000295EA3 /* cpu_copy_in_cksum.s in Sources */, 24B8C2621237F53900D36CC3 /* remove-counter.c in Sources */, C99A4F501305B2BD0054B7B7 /* __get_cpu_capabilities.s in Sources */, 978228291B8678DF008385AC /* pselect-darwinext-cancel.c in Sources */, + 40DF0F741E5CD7BB0035A864 /* cpu_copy_in_cksum_gen.c in Sources */, C99A4F531305B43F0054B7B7 /* init_cpu_capabilities.c in Sources */, + 402AF43F1E5CD88600F1A4B9 /* cpu_in_cksum_gen.c in Sources */, 030B179B135377B400DAD1F0 /* open_dprotected_np.c in Sources */, E4D45C3116F868ED0002AF25 /* proc_listpidspath.c in Sources */, 374A36E314748F1300AAF39D /* varargs_wrappers.s in Sources */, @@ -1480,6 +1504,7 @@ DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; MAP_PLATFORM = "$(MAP_PLATFORM_$(PLATFORM_NAME))"; MAP_PLATFORM_appletvos = iPhoneOS; + MAP_PLATFORM_bridgeos = iPhoneOS; MAP_PLATFORM_iphoneos = iPhoneOS; MAP_PLATFORM_iphoneosnano = iPhoneOS; MAP_PLATFORM_macosx = MacOSX; diff --git a/libsyscall/Platforms/iPhoneOS/arm/syscall.map b/libsyscall/Platforms/iPhoneOS/arm/syscall.map new file mode 100644 index 000000000..2466d41b2 --- /dev/null +++ b/libsyscall/Platforms/iPhoneOS/arm/syscall.map @@ -0,0 +1,77 @@ +_accept$NOCANCEL ___accept_nocancel +_aio_suspend$NOCANCEL ___aio_suspend_nocancel +_close$NOCANCEL ___close_nocancel +_connect$NOCANCEL ___connect_nocancel +_fstat ___fstat64 +_fstat64 +_fstatat ___fstatat64 +_fstatat64 +_fstatfs ___fstatfs64 +_fstatfs64 +_fstatx_np ___fstatx64_np +_fstatx64_np +_fsync$NOCANCEL ___fsync_nocancel +_getfsstat ___getfsstat64 +_getfsstat64 +_getmntinfo ___getmntinfo64 +_getmntinfo64 +_lstat ___lstat64 +_lstat64 +_lstatx_np ___lstatx64_np +_lstatx64_np +_msgrcv$NOCANCEL ___msgrcv_nocancel +_msgsnd$NOCANCEL ___msgsnd_nocancel +_msync$NOCANCEL ___msync_nocancel +_msgsys ___msgsys +_open$NOCANCEL ___open_nocancel +_openat$NOCANCEL ___openat_nocancel +_poll$NOCANCEL ___poll_nocancel +_pread$NOCANCEL ___pread_nocancel +_pwrite$NOCANCEL ___pwrite_nocancel +_read$NOCANCEL ___read_nocancel +_readv$NOCANCEL ___readv_nocancel +_recvfrom$NOCANCEL ___recvfrom_nocancel +_recvmsg$NOCANCEL ___recvmsg_nocancel +_select$DARWIN_EXTSN ___select +_select$DARWIN_EXTSN$NOCANCEL ___select_nocancel +_sem_wait$NOCANCEL ___sem_wait_nocancel +_semsys ___semsys +_sendmsg$NOCANCEL ___sendmsg_nocancel +_sendto$NOCANCEL ___sendto_nocancel +_stat ___stat64 +_stat64 +_statfs ___statfs64 +_statfs64 +_statx_np ___statx64_np +_statx64_np +_waitid$NOCANCEL ___waitid_nocancel +_write$NOCANCEL ___write_nocancel +_writev$NOCANCEL ___writev_nocancel + +_accept ___accept +_bind ___bind +_getattrlist ___getattrlist +_getpeername ___getpeername +_getsockname ___getsockname +_lchown ___lchown +_listen ___listen +_recvfrom ___recvfrom +_recvmsg ___recvmsg +_sendmsg ___sendmsg +_sendto ___sendto +_setattrlist ___setattrlist +_socketpair ___socketpair + +_mprotect ___mprotect +_setregid ___setregid +_setreuid ___setreuid +_open ___open +_openat ___openat +_connect ___connect +_msync ___msync +_sem_open ___sem_open +_semctl ___semctl +_msgctl ___msgctl +_shmctl ___shmctl +_shmsys ___shmsys +_shm_open ___shm_open diff --git a/libsyscall/Platforms/iPhoneOS/arm64/syscall.map b/libsyscall/Platforms/iPhoneOS/arm64/syscall.map new file mode 100644 index 000000000..20eb08fae --- /dev/null +++ b/libsyscall/Platforms/iPhoneOS/arm64/syscall.map @@ -0,0 +1,67 @@ +_accept$NOCANCEL ___accept_nocancel +_aio_suspend$NOCANCEL ___aio_suspend_nocancel +_close$NOCANCEL ___close_nocancel +_connect$NOCANCEL ___connect_nocancel +_fstat ___fstat64 +_fstat64 +_fstatat ___fstatat64 +_fstatat64 +_fstatfs ___fstatfs64 +_fstatfs64 +_fstatx_np ___fstatx64_np +_fstatx64_np +_fsync$NOCANCEL ___fsync_nocancel +_getfsstat ___getfsstat64 +_getfsstat64 +_getmntinfo ___getmntinfo64 +_getmntinfo64 +_lstat ___lstat64 +_lstat64 +_lstatx_np ___lstatx64_np +_lstatx64_np +_msgrcv$NOCANCEL ___msgrcv_nocancel +_msgsnd$NOCANCEL ___msgsnd_nocancel +_msync$NOCANCEL ___msync_nocancel +_poll$NOCANCEL ___poll_nocancel +_pread$NOCANCEL ___pread_nocancel +_pwrite$NOCANCEL ___pwrite_nocancel +_read$NOCANCEL ___read_nocancel +_readv$NOCANCEL ___readv_nocancel +_recvfrom$NOCANCEL ___recvfrom_nocancel +_recvmsg$NOCANCEL ___recvmsg_nocancel +_select$DARWIN_EXTSN ___select +_select$DARWIN_EXTSN$NOCANCEL ___select_nocancel +_sem_wait$NOCANCEL ___sem_wait_nocancel +_sendmsg$NOCANCEL ___sendmsg_nocancel +_sendto$NOCANCEL ___sendto_nocancel +_stat ___stat64 +_stat64 +_statfs ___statfs64 +_statfs64 +_statx_np ___statx64_np +_statx64_np +_waitid$NOCANCEL ___waitid_nocancel +_write$NOCANCEL ___write_nocancel +_writev$NOCANCEL ___writev_nocancel + +_accept ___accept +_bind ___bind +_getattrlist ___getattrlist +_getpeername ___getpeername +_getsockname ___getsockname +_lchown ___lchown +_listen ___listen +_recvfrom ___recvfrom +_recvmsg ___recvmsg +_sendmsg ___sendmsg +_sendto ___sendto +_setattrlist ___setattrlist +_socketpair ___socketpair + +_mprotect ___mprotect +_setregid ___setregid +_setreuid ___setreuid +_connect ___connect +_msync ___msync +_msgctl ___msgctl +_shmctl ___shmctl diff --git a/libsyscall/custom/SYS.h b/libsyscall/custom/SYS.h index 2c66bbf4d..ff93f852a 100644 --- a/libsyscall/custom/SYS.h +++ b/libsyscall/custom/SYS.h @@ -162,6 +162,317 @@ LEAF(pseudo, 0) ;\ PSEUDO(pseudo, name, nargs, cerror) ;\ ret +#elif defined(__arm__) + +#include <architecture/arm/asm_help.h> +#include <mach/arm/syscall_sw.h> + +/* + * ARM system call interface: + * + * swi 0x80 + * args: r0-r6 + * return code: r0 + * on error, carry bit is set in the psr, otherwise carry bit is cleared. + */ + +/* + * Macros. + */ + +/* + * until we update the architecture project, these live here + */ + +#if defined(__DYNAMIC__) +#define MI_GET_ADDRESS(reg,var) \ + ldr reg, 4f ;\ +3: ldr reg, [pc, reg] ;\ + b 5f ;\ +4: .long 6f - (3b + 8) ;\ +5: ;\ + .non_lazy_symbol_pointer ;\ +6: ;\ + .indirect_symbol var ;\ + .long 0 ;\ + .text ;\ + .align 2 +#else +#define MI_GET_ADDRESS(reg,var) \ + ldr reg, 3f ;\ + b 4f ;\ +3: .long var ;\ +4: +#endif + +#if defined(__DYNAMIC__) +#define MI_BRANCH_EXTERNAL(var) \ + .globl var ;\ + MI_GET_ADDRESS(ip, var) ;\ + bx ip +#else +#define MI_BRANCH_EXTERNAL(var) ;\ + .globl var ;\ + b var +#endif + +#if defined(__DYNAMIC__) +#define MI_CALL_EXTERNAL(var) \ + .globl var ;\ + MI_GET_ADDRESS(ip,var) ;\ + blx ip +#else +#define MI_CALL_EXTERNAL(var) \ + .globl var ;\ + bl var +#endif + +#define MI_ENTRY_POINT(name) \ + .align 2 ;\ + .globl name ;\ + .text ;\ +name: + +/* load the syscall number into r12 and trap */ +#define DO_SYSCALL(num) \ + .if (((num) & 0xff) == (num)) ;\ + mov r12, #(num) ;\ + .elseif (((num) & 0x3fc) == (num)) ;\ + mov r12, #(num) ;\ + .else ;\ + mov r12, #((num) & 0xffffff00) /* top half of the syscall number */ ;\ + orr r12, r12, #((num) & 0xff) /* bottom half */ ;\ + .endif ;\ + swi #SWI_SYSCALL + +/* simple syscalls (0 to 4 args) */ +#define SYSCALL_0to4(name, cerror) \ + MI_ENTRY_POINT(_##name) ;\ + DO_SYSCALL(SYS_##name) ;\ + bxcc lr /* return if carry is clear (no error) */ ; \ +1: MI_BRANCH_EXTERNAL(_##cerror) + +/* syscalls with 5 args is different, because of the single arg register load */ +#define SYSCALL_5(name, cerror) \ + MI_ENTRY_POINT(_##name) ;\ + mov ip, sp /* save a pointer to the args */ ; \ + stmfd sp!, { r4-r5 } /* save r4-r5 */ ;\ + ldr r4, [ip] /* load 5th arg */ ; \ + DO_SYSCALL(SYS_##name) ;\ + ldmfd sp!, { r4-r5 } /* restore r4-r5 */ ; \ + bxcc lr /* return if carry is clear (no error) */ ; \ +1: MI_BRANCH_EXTERNAL(_##cerror) + +/* syscalls with 6 to 12 args. kernel may have to read from stack */ +#define SYSCALL_6to12(name, save_regs, arg_regs, cerror) \ + MI_ENTRY_POINT(_##name) ;\ + mov ip, sp /* save a pointer to the args */ ; \ + stmfd sp!, { save_regs } /* callee saved regs */ ;\ + ldmia ip, { arg_regs } /* load arg regs */ ; \ + DO_SYSCALL(SYS_##name) ;\ + ldmfd sp!, { save_regs } /* restore callee saved regs */ ; \ + bxcc lr /* return if carry is clear (no error) */ ; \ +1: MI_BRANCH_EXTERNAL(_##cerror) + +#define COMMA , + +#if __BIGGEST_ALIGNMENT__ > 4 + +/* For the armv7k ABI, the alignment requirements may add padding. So we + * let the kernel figure it out and push extra on the stack to avoid un-needed + * copy-ins */ + + /* We'll also use r8 for moving arguments */ + +#define SYSCALL_0(name) SYSCALL_0to4(name) +#define SYSCALL_1(name) SYSCALL_0to4(name) +#define SYSCALL_2(name) SYSCALL_0to4(name) +#define SYSCALL_3(name) SYSCALL_0to4(name) +#define SYSCALL_4(name) SYSCALL_6to12(name, r4-r5, r4-r5) +#undef SYSCALL_5 +#define SYSCALL_5(name) SYSCALL_6to12(name, r4-r5, r4-r5) +#define SYSCALL_6(name) SYSCALL_6to12(name, r4-r6 COMMA r8, r4-r6 COMMA r8) +#define SYSCALL_7(name) SYSCALL_6to12(name, r4-r6 COMMA r8, r4-r6 COMMA r8) +#define SYSCALL_8(name) SYSCALL_6to12(name, r4-r6 COMMA r8, r4-r6 COMMA r8) +#define SYSCALL_12(name) SYSCALL_6to12(name, r4-r6 COMMA r8, r4-r6 COMMA r8) + +#else // !(__BIGGEST_ALIGNMENT__ > 4) (the normal arm32 ABI case) + +#define SYSCALL_0(name) SYSCALL_0to4(name) +#define SYSCALL_1(name) SYSCALL_0to4(name) +#define SYSCALL_2(name) SYSCALL_0to4(name) +#define SYSCALL_3(name) SYSCALL_0to4(name) +#define SYSCALL_4(name) SYSCALL_0to4(name) +/* SYSCALL_5 declared above */ +#define SYSCALL_6(name) SYSCALL_6to12(name, r4-r5, r4-r5) +#define SYSCALL_7(name) SYSCALL_6to12(name, r4-r6 COMMA r8, r4-r6) +#define SYSCALL_8(name) SYSCALL_6to12(name, r4-r6 COMMA r8, r4-r6) /* 8th on stack */ +#define SYSCALL_12(name) SYSCALL_6to12(name, r4-r6 COMMA r8, r4-r6) /* 8th-12th on stack */ + +#endif // __BIGGEST_ALIGNMENT__ > 4 + +/* select the appropriate syscall code, based on the number of arguments */ +#ifndef __SYSCALL_32BIT_ARG_BYTES +#define SYSCALL(name, nargs, cerror) SYSCALL_##nargs(name, cerror) +#define SYSCALL_NONAME(name, nargs, cerror) SYSCALL_NONAME_##nargs(name, cerror) +#else +#if __SYSCALL_32BIT_ARG_BYTES < 20 +#define SYSCALL(name, nargs, cerror) SYSCALL_0to4(name, cerror) +#define SYSCALL_NONAME(name, nargs, cerror) SYSCALL_NONAME_0to4(name, cerror) +#elif __SYSCALL_32BIT_ARG_BYTES == 20 +#define SYSCALL(name, nargs, cerror) SYSCALL_5(name, cerror) +#define SYSCALL_NONAME(name, nargs, cerror) SYSCALL_NONAME_5(name, cerror) +#elif __SYSCALL_32BIT_ARG_BYTES == 24 +#define SYSCALL(name, nargs, cerror) SYSCALL_6(name, cerror) +#define SYSCALL_NONAME(name, nargs, cerror) SYSCALL_NONAME_6(name, cerror) +#elif __SYSCALL_32BIT_ARG_BYTES == 28 +#define SYSCALL(name, nargs, cerror) SYSCALL_7(name, cerror) +#define SYSCALL_NONAME(name, nargs, cerror) SYSCALL_NONAME_7(name, cerror) +#elif __SYSCALL_32BIT_ARG_BYTES == 32 +#define SYSCALL(name, nargs, cerror) SYSCALL_8(name, cerror) +#define SYSCALL_NONAME(name, nargs, cerror) SYSCALL_NONAME_8(name, cerror) +#elif __SYSCALL_32BIT_ARG_BYTES == 36 +#define SYSCALL(name, nargs, cerror) SYSCALL_8(name, cerror) +#define SYSCALL_NONAME(name, nargs, cerror) SYSCALL_NONAME_8(name, cerror) +#elif __SYSCALL_32BIT_ARG_BYTES == 44 +#define SYSCALL(name, nargs, cerror) SYSCALL_8(name, cerror) +#define SYSCALL_NONAME(name, nargs, cerror) SYSCALL_NONAME_8(name, cerror) +#elif __SYSCALL_32BIT_ARG_BYTES == 48 +#define SYSCALL(name, nargs, cerror) SYSCALL_12(name, cerror) +#define SYSCALL_NONAME(name, nargs, cerror) SYSCALL_NONAME_12(name, cerror) +#endif +#endif + +#define SYSCALL_NONAME_0to4(name, cerror) \ + DO_SYSCALL(SYS_##name) ;\ + bcc 1f /* branch if carry bit is clear (no error) */ ; \ + MI_BRANCH_EXTERNAL(_##cerror) /* call cerror */ ; \ +1: + +#define SYSCALL_NONAME_5(name, cerror) \ + mov ip, sp /* save a pointer to the args */ ; \ + stmfd sp!, { r4-r5 } /* save r4-r5 */ ;\ + ldr r4, [ip] /* load 5th arg */ ; \ + DO_SYSCALL(SYS_##name) ;\ + ldmfd sp!, { r4-r5 } /* restore r4-r7 */ ; \ + bcc 1f /* branch if carry bit is clear (no error) */ ; \ + MI_BRANCH_EXTERNAL(_##cerror) /* call cerror */ ; \ +1: + +#define SYSCALL_NONAME_6to12(name, save_regs, arg_regs, cerror) \ + mov ip, sp /* save a pointer to the args */ ; \ + stmfd sp!, { save_regs } /* callee save regs */ ;\ + ldmia ip, { arg_regs } /* load arguments */ ; \ + DO_SYSCALL(SYS_##name) ;\ + ldmfd sp!, { save_regs } /* restore callee saved regs */ ; \ + bcc 1f /* branch if carry bit is clear (no error) */ ; \ + MI_BRANCH_EXTERNAL(_##cerror) /* call cerror */ ; \ +1: + + +#if __BIGGEST_ALIGNMENT__ > 4 + +/* For the armv7k ABI, the alignment requirements may add padding. So we + * let the kernel figure it out and push extra on the stack to avoid un-needed + * copy-ins. We are relying on arguments that aren't in registers starting + * 32 bytes from sp. We also use r8 like in the mach case. */ + +#define SYSCALL_NONAME_0(name, cerror) SYSCALL_NONAME_0to4(name, cerror) +#define SYSCALL_NONAME_1(name, cerror) SYSCALL_NONAME_0to4(name, cerror) +#define SYSCALL_NONAME_2(name, cerror) SYSCALL_NONAME_0to4(name, cerror) +#define SYSCALL_NONAME_3(name, cerror) SYSCALL_NONAME_0to4(name, cerror) +#define SYSCALL_NONAME_4(name, cerror) SYSCALL_NONAME_6to12(name, r4-r5, r4-r5, cerror) +#undef SYSCALL_NONAME_5 +#define SYSCALL_NONAME_5(name, cerror) SYSCALL_NONAME_6to12(name, r4-r5, r4-r5, cerror) +#define SYSCALL_NONAME_6(name, cerror) SYSCALL_NONAME_6to12(name, r4-r6 COMMA r8, r4-r6 COMMA r8, cerror) +#define SYSCALL_NONAME_7(name, cerror) SYSCALL_NONAME_6to12(name, r4-r6 COMMA r8, r4-r6 COMMA r8, cerror) +#define SYSCALL_NONAME_8(name, cerror) SYSCALL_NONAME_6to12(name, r4-r6 COMMA r8, r4-r6 COMMA r8, cerror) +#define SYSCALL_NONAME_12(name, cerror) SYSCALL_NONAME_6to12(name, r4-r6 COMMA r8, r4-r6 COMMA r8, cerror) + +#else // !(__BIGGEST_ALIGNMENT__ > 4) (the normal arm32 ABI case) + +#define SYSCALL_NONAME_0(name, cerror) SYSCALL_NONAME_0to4(name, cerror) +#define SYSCALL_NONAME_1(name, cerror) SYSCALL_NONAME_0to4(name, cerror) +#define SYSCALL_NONAME_2(name, cerror) SYSCALL_NONAME_0to4(name, cerror) +#define SYSCALL_NONAME_3(name, cerror) SYSCALL_NONAME_0to4(name, cerror) +#define SYSCALL_NONAME_4(name, cerror) SYSCALL_NONAME_0to4(name, cerror) +/* SYSCALL_NONAME_5 declared above */ +#define SYSCALL_NONAME_6(name, cerror) SYSCALL_NONAME_6to12(name, r4-r5, r4-r5, cerror) +#define SYSCALL_NONAME_7(name, cerror) SYSCALL_NONAME_6to12(name, r4-r6 COMMA r8, r4-r6, cerror) +#define SYSCALL_NONAME_8(name, cerror) SYSCALL_NONAME_6to12(name, r4-r6 COMMA r8, r4-r6, cerror) +#define SYSCALL_NONAME_12(name, cerror) SYSCALL_NONAME_6to12(name, r4-r6 COMMA r8, r4-r6, cerror) + +#endif // __BIGGEST_ALIGNMENT__ > 4 + +#define PSEUDO(pseudo, name, nargs, cerror) \ + .globl pseudo ;\ + .text ;\ + .align 2 ;\ +pseudo: ;\ + SYSCALL_NONAME(name, nargs, cerror) + +#define __SYSCALL2(pseudo, name, nargs, cerror) \ + PSEUDO(pseudo, name, nargs, cerror) ;\ + bx lr + +#define __SYSCALL(pseudo, name, nargs) \ + PSEUDO(pseudo, name, nargs, cerror) ;\ + bx lr + +#elif defined(__arm64__) + +#include <mach/arm/syscall_sw.h> +#include <mach/arm/vm_param.h> +#include <mach/arm64/asm.h> + +/* + * ARM64 system call interface: + * + * TBD + */ + +#define DO_SYSCALL(num, cerror) \ + mov x16, #(num) %%\ + svc #SWI_SYSCALL %%\ + b.cc 2f %%\ + PUSH_FRAME %%\ + bl _##cerror %%\ + POP_FRAME %%\ +2: + +#define MI_GET_ADDRESS(reg,var) \ + adrp reg, var@page %%\ + add reg, reg, var@pageoff %% + +#define MI_CALL_EXTERNAL(sym) \ + .globl sym %% \ + bl sym + +#define SYSCALL_NONAME(name, nargs, cerror) \ + DO_SYSCALL(SYS_##name, cerror) %% \ +1: + +#define MI_ENTRY_POINT(name) \ + .text %% \ + .align 2 %% \ + .globl name %% \ +name: + +#define PSEUDO(pseudo, name, nargs, cerror) \ + .text %% \ + .align 2 %% \ + .globl pseudo %% \ + pseudo: %% \ + SYSCALL_NONAME(name, nargs, cerror) + +#define __SYSCALL(pseudo, name, nargs) \ + PSEUDO(pseudo, name, nargs, cerror) %% \ + ret + +#define __SYSCALL2(pseudo, name, nargs, cerror) \ + PSEUDO(pseudo, name, nargs, cerror) %% \ + ret + #else #error Unsupported architecture #endif diff --git a/libsyscall/custom/__fork.s b/libsyscall/custom/__fork.s index 5857ab2ad..dc517a1a2 100644 --- a/libsyscall/custom/__fork.s +++ b/libsyscall/custom/__fork.s @@ -100,6 +100,59 @@ L2: addq $24, %rsp // restore the stack ret +#elif defined(__arm__) + +MI_ENTRY_POINT(___fork) + stmfd sp!, {r4, r7, lr} + add r7, sp, #4 + + mov r1, #1 // prime results + mov r12, #SYS_fork + swi #SWI_SYSCALL // make the syscall + bcs Lbotch // error? + + cmp r1, #0 // parent (r1=0) or child(r1=1) + beq Lparent + + //child here... + MI_GET_ADDRESS(r3, __current_pid) + mov r0, #0 + str r0, [r3] // clear cached pid in child + ldmfd sp!, {r4, r7, pc} + +Lbotch: + MI_CALL_EXTERNAL(_cerror) // jump here on error + mov r0,#-1 // set the error + // fall thru +Lparent: + ldmfd sp!, {r4, r7, pc} // pop and return + +#elif defined(__arm64__) + +#include <mach/arm64/asm.h> + +MI_ENTRY_POINT(___fork) + PUSH_FRAME + // ARM moves a 1 in to r1 here, but I can't see why. + mov x16, #SYS_fork // Syscall code + svc #SWI_SYSCALL // Trap to kernel + b.cs Lbotch // Carry bit indicates failure + cbz x1, Lparent // x1 == 0 indicates that we are the parent + + // Child + MI_GET_ADDRESS(x9, __current_pid) // Get address of cached "current pid" + mov w0, #0 + str w0, [x9] // Clear cached current pid + POP_FRAME // And done + ret + +Lbotch: + MI_CALL_EXTERNAL(_cerror) // Handle error + mov w0, #-1 // Return value is -1 +Lparent: + POP_FRAME // Return + ret + #else #error Unsupported architecture #endif diff --git a/libsyscall/custom/__getpid.s b/libsyscall/custom/__getpid.s index 2768d9b82..a048f48aa 100644 --- a/libsyscall/custom/__getpid.s +++ b/libsyscall/custom/__getpid.s @@ -97,6 +97,77 @@ LEAF(___getpid, 0) movl %edx, %eax ret +#elif defined(__arm__) + +#include <arm/arch.h> + + .data + .globl __current_pid + .align 2 +__current_pid: + /* Cached pid. Possible values: + * 0: no value cached + * > 0: cached PID of current process + * < 0: negative number of vforks in progress + * INT_MIN: for pre-ARMv6, "looking" value (0x80000000) + */ + .long 0 + +MI_ENTRY_POINT(___getpid) + ldr r3, L__current_pid +L1: add r3, pc, r3 // r3 = &__current_pid + ldr r0, [r3] // get the cached pid + cmp r0, #0 + bxgt lr // if positive, return it + + SYSCALL_NONAME(getpid, 0, cerror_nocancel) + +#ifdef _ARM_ARCH_6 + ldrex r2, [r3] // see if we can cache it + cmp r2, #0 // we can't if there are any... + bxlt lr // ...vforks in progress + strex r2, r0, [r3] // ignore conflicts +#else + mov r1, #0x80000000 // load "looking" value + swp r2, r1, [r3] // look at the value, lock others out + cmp r2, r1 // anyone else trying to look? + bxeq lr // yes, so return immediately/ + cmp r2, #0 // see if we can cache it + streq r0, [r3] // if zero, we can + strne r2, [r3] // otherwise restore previous value +#endif + + bx lr + +L__current_pid: + .long __current_pid - (L1+8) + +#elif defined(__arm64__) + .data + .globl __current_pid + .align 2 +__current_pid: + /* cached pid. possible values: + * 0: no value cached + * > 0: cached pid of current process + * < 0: negative number of vforks in progress + * int_min: for pre-armv6, "looking" value (0x80000000) + */ + .long 0 + +MI_ENTRY_POINT(___getpid) + MI_GET_ADDRESS(x9, __current_pid) // Get address of cached value + ldr w0, [x9] // Load it + cmp w0, #0 // See if there's a cached value + b.ls L_notcached // If not, make syscall + ret // Else, we're done +L_notcached: + SYSCALL_NONAME(getpid, 0, cerror_nocancel) + ldxr w10, [x9] // Exclusive load + cbnz w10, L_done // Unless unset, don't even try + stxr wzr, w0, [x9] // Try to store, but don't care if we fail (someone will win, or not) +L_done: + ret // Done #else #error Unsupported architecture #endif diff --git a/libsyscall/custom/__gettimeofday.s b/libsyscall/custom/__gettimeofday.s index 8712a2094..47aada592 100644 --- a/libsyscall/custom/__gettimeofday.s +++ b/libsyscall/custom/__gettimeofday.s @@ -91,6 +91,30 @@ LABEL(___gettimeofday) 2: ret +#elif defined(__arm__) + +__SYSCALL2(___gettimeofday_with_mach, gettimeofday, 3, cerror_nocancel) + +.text +.align 2 +.globl ___gettimeofday +___gettimeofday: + mov r2, #0x0 + SYSCALL_NONAME(gettimeofday, 3, cerror_nocancel) + bx lr + +#elif defined(__arm64__) + +__SYSCALL2(___gettimeofday_with_mach, gettimeofday, 3, cerror_nocancel) + +.text +.align 2 +.globl ___gettimeofday +___gettimeofday: + movz x2, #0x0 + SYSCALL_NONAME(gettimeofday, 3, cerror_nocancel) + ret + #else #error Unsupported architecture #endif diff --git a/libsyscall/custom/__kdebug_trace_string.s b/libsyscall/custom/__kdebug_trace_string.s index e3543e6bf..81cf375b5 100644 --- a/libsyscall/custom/__kdebug_trace_string.s +++ b/libsyscall/custom/__kdebug_trace_string.s @@ -36,6 +36,14 @@ __SYSCALL(___kdebug_trace_string, kdebug_trace_string, 3) __SYSCALL_INT(___kdebug_trace_string, kdebug_trace_string, 3) +#elif defined(__arm__) + +__SYSCALL(___kdebug_trace_string, kdebug_trace_string, 4) + +#elif defined(__arm64__) + +__SYSCALL(___kdebug_trace_string, kdebug_trace_string, 3) + #else #error Unsupported architecture #endif diff --git a/libsyscall/custom/__lseek.s b/libsyscall/custom/__lseek.s index b051cc5a4..7c7f41eb2 100644 --- a/libsyscall/custom/__lseek.s +++ b/libsyscall/custom/__lseek.s @@ -36,6 +36,14 @@ __SYSCALL(___lseek, lseek, 3) __SYSCALL_INT(___lseek, lseek, 3) +#elif defined(__arm__) + +__SYSCALL(___lseek, lseek, 4) + +#elif defined(__arm64__) + +__SYSCALL(___lseek, lseek, 3) + #else #error Unsupported architecture #endif diff --git a/libsyscall/custom/__pipe.s b/libsyscall/custom/__pipe.s index d375dddbd..0c527d5ea 100644 --- a/libsyscall/custom/__pipe.s +++ b/libsyscall/custom/__pipe.s @@ -46,6 +46,25 @@ PSEUDO(___pipe, pipe, 0, cerror_nocancel) xorl %eax, %eax ret +#elif defined(__arm__) + +MI_ENTRY_POINT(___pipe) + mov r3,r0 // save fildes across syscall + SYSCALL_NONAME(pipe, 0, cerror_nocancel) + str r0, [r3, #0] + str r1, [r3, #4] + mov r0,#0 + bx lr + +#elif defined(__arm64__) + +MI_ENTRY_POINT(___pipe) + mov x9, x0 // Stash FD array + SYSCALL_NONAME(pipe, 0, cerror_nocancel) + stp w0, w1, [x9] // Save results + mov x0, #0 // Success + ret // Done + #else #error Unsupported architecture #endif diff --git a/libsyscall/custom/__ptrace.s b/libsyscall/custom/__ptrace.s index bdcbec9fb..607a26e87 100644 --- a/libsyscall/custom/__ptrace.s +++ b/libsyscall/custom/__ptrace.s @@ -50,6 +50,25 @@ LEAF(___ptrace, 0) UNIX_SYSCALL_NONAME(ptrace, 4, cerror) ret +#elif defined(__arm__) + +MI_ENTRY_POINT(___ptrace) + MI_GET_ADDRESS(ip,_errno) + str r8, [sp, #-4]! + mov r8, #0 + str r8, [ip] + ldr r8, [sp], #4 + SYSCALL_NONAME(ptrace, 4, cerror) + bx lr + +#elif defined(__arm64__) + +MI_ENTRY_POINT(___ptrace) + MI_GET_ADDRESS(x9,_errno) + str wzr, [x9] + SYSCALL_NONAME(ptrace, 4, cerror) + ret + #else #error Unsupported architecture #endif diff --git a/libsyscall/custom/__sigaltstack.s b/libsyscall/custom/__sigaltstack.s index d5f1803ff..7c4fceaf5 100644 --- a/libsyscall/custom/__sigaltstack.s +++ b/libsyscall/custom/__sigaltstack.s @@ -36,6 +36,14 @@ __SYSCALL(___sigaltstack, sigaltstack, 3) __SYSCALL_INT(___sigaltstack, sigaltstack, 3) +#elif defined(__arm__) + +__SYSCALL(___sigaltstack, sigaltstack, 3) + +#elif defined(__arm64__) + +__SYSCALL(___sigaltstack, sigaltstack, 3) + #else #error Unsupported architecture #endif diff --git a/libsyscall/custom/__sigreturn.s b/libsyscall/custom/__sigreturn.s index 16d5be4fc..a6a24404e 100644 --- a/libsyscall/custom/__sigreturn.s +++ b/libsyscall/custom/__sigreturn.s @@ -36,6 +36,14 @@ __SYSCALL(___sigreturn, sigreturn, 2) __SYSCALL_INT(___sigreturn, sigreturn, 2) +#elif defined(__arm__) + +__SYSCALL(___sigreturn, sigreturn, 2) + +#elif defined(__arm64__) + +__SYSCALL(___sigreturn, sigreturn, 2) + #else #error Unsupported architecture #endif diff --git a/libsyscall/custom/__syscall.s b/libsyscall/custom/__syscall.s index f00894425..81e49f11a 100644 --- a/libsyscall/custom/__syscall.s +++ b/libsyscall/custom/__syscall.s @@ -50,6 +50,24 @@ END(___syscall) // that value anyway. __SYSCALL(___syscall, syscall, 0); +#elif defined(__arm__) + +__SYSCALL(___syscall, syscall, 7) + +#elif defined(__arm64__) + +/* + * Ignore nominal number of arguments: just pop from stack and let the kernel + * interpret. + */ +#include <mach/arm64/asm.h> +MI_ENTRY_POINT(___syscall) + ldp x1, x2, [sp] + ldp x3, x4, [sp, #16] + ldp x5, x6, [sp, #32] + ldr x7, [sp, #48] + DO_SYSCALL(SYS_syscall, cerror) + ret #else #error Unsupported architecture #endif diff --git a/libsyscall/custom/__thread_selfid.s b/libsyscall/custom/__thread_selfid.s index d84a5305f..86c17638a 100644 --- a/libsyscall/custom/__thread_selfid.s +++ b/libsyscall/custom/__thread_selfid.s @@ -36,4 +36,12 @@ __SYSCALL(___thread_selfid, thread_selfid, 0) __SYSCALL_INT(___thread_selfid, thread_selfid, 0) +#elif defined(__arm__) + +__SYSCALL(___thread_selfid, thread_selfid, 0) + +#elif defined(__arm64__) + +__SYSCALL(___thread_selfid, thread_selfid, 0) + #endif diff --git a/libsyscall/custom/__thread_selfusage.s b/libsyscall/custom/__thread_selfusage.s index 064c5bad9..ec83854e4 100644 --- a/libsyscall/custom/__thread_selfusage.s +++ b/libsyscall/custom/__thread_selfusage.s @@ -36,4 +36,12 @@ __SYSCALL(___thread_selfusage, thread_selfusage, 0) __SYSCALL_INT(___thread_selfusage, thread_selfusage, 0) +#elif defined(__arm__) + +__SYSCALL(___thread_selfusage, thread_selfusage, 0) + +#elif defined(__arm64__) + +__SYSCALL(___thread_selfusage, thread_selfusage, 0) + #endif diff --git a/libsyscall/custom/__vfork.s b/libsyscall/custom/__vfork.s index 8449d25e4..65a781efd 100644 --- a/libsyscall/custom/__vfork.s +++ b/libsyscall/custom/__vfork.s @@ -128,6 +128,107 @@ L2: addl $1, (%rdx) jmp *%rdi +#elif defined(__arm__) + +#include <arm/arch.h> + + .globl cerror + MI_ENTRY_POINT(___vfork) + + MI_GET_ADDRESS(r3, __current_pid) // get address of __current_pid +#ifdef _ARM_ARCH_6 +L0: + ldrex r1, [r3] + subs r1, r1, #1 // if __current_pid <= 0, decrement it + movpl r1, #-1 // otherwise put -1 in there + strex r2, r1, [r3] + cmp r2, #0 + bne L0 +#else + mov r2, #0x80000000 // load "looking" value +L0: + swp r1, r2, [r3] // look at the value, lock others out + cmp r1, r2 // anyone else trying to look? + beq L0 // yes, so wait our turn + subs r1, r1, #1 // if __current_pid <= 0, decrement it + movpl r1, #-1 // otherwise put -1 in there + str r1, [r3] +#endif + + mov r1, #1 // prime results + mov r12, #SYS_vfork + swi #SWI_SYSCALL // make the syscall + bcs Lbotch // error? + cmp r1, #0 // parent (r1=0) or child(r1=1) + beq Lparent + + //child here... + mov r0, #0 + bx lr // return + +Lbotch: + MI_CALL_EXTERNAL(_cerror) // jump here on error + mov r0,#-1 // set the error + // reload values clobbered by cerror (so we can treat them as live in Lparent) + MI_GET_ADDRESS(r3, __current_pid) // get address of __current_pid +#ifndef _ARM_ARCH_6 + mov r2, #0x80000000 // load "looking" value +#endif + // fall thru + +Lparent: +#ifdef _ARM_ARCH_6 + ldrex r1, [r3] + add r1, r1, #1 // we're back, decrement vfork count + strex r2, r1, [r3] + cmp r2, #0 + bne Lparent +#else + swp r1, r2, [r3] // look at the value, lock others out + cmp r1, r2 // anyone else trying to look? + beq Lparent // yes, so wait our turn + add r1, r1, #1 // we're back, decrement vfork count + str r1, [r3] +#endif + + bx lr // return + +#elif defined(__arm64__) + + MI_ENTRY_POINT(___vfork) + + MI_GET_ADDRESS(x9, __current_pid) +Ltry_set_vfork: + ldxr w10, [x9] // Get old current pid value (exclusive) + mov w11, #-1 // Will be -1 if current value is positive + subs w10, w10, #1 // Subtract one + csel w12, w11, w10, pl // If >= 0, set to -1, else set to (current - 1) + stxr w13, w12, [x9] // Attempt exclusive store to current pid + cbnz w13, Ltry_set_vfork // If store failed, retry + + // ARM sets r1 to 1 here. I don't see why. + mov w16, #SYS_vfork // Set syscall code + svc #SWI_SYSCALL + b.cs Lbotch + cbz w1, Lparent + + // Child + mov w0, #0 + ret + + // Error case +Lbotch: + bl _cerror // Update errno + mov w0, #-1 // Set return value + MI_GET_ADDRESS(x9, __current_pid) // Reload current pid address + // Fall through +Lparent: + ldxr w10, [x9] // Exclusive load current pid value + add w10, w10, #1 // Increment (i.e. decrement vfork count) + stxr w11, w10, [x9] // Attempt exclusive store of updated vfork count + cbnz w11, Lparent // If exclusive store failed, retry + ret // Done, return + #else #error Unsupported architecture #endif diff --git a/libsyscall/custom/custom.s b/libsyscall/custom/custom.s index 56a95cf5b..de2e0c6bd 100644 --- a/libsyscall/custom/custom.s +++ b/libsyscall/custom/custom.s @@ -122,6 +122,26 @@ __thread_set_tsd_base: MACHDEP_SYSCALL_TRAP ret +#elif defined(__arm__) + + .align 2 + .globl __thread_set_tsd_base +__thread_set_tsd_base: + mov r3, #2 + mov r12, #0x80000000 + swi #SWI_SYSCALL + bx lr + +#elif defined(__arm64__) + + .align 2 + .globl __thread_set_tsd_base +__thread_set_tsd_base: + mov x3, #2 + mov x16, #0x80000000 + svc #SWI_SYSCALL + ret + #else #error unknown architecture #endif diff --git a/libsyscall/mach/err_libkern.sub b/libsyscall/mach/err_libkern.sub index f419d04fa..f865b9d7c 100644 --- a/libsyscall/mach/err_libkern.sub +++ b/libsyscall/mach/err_libkern.sub @@ -89,6 +89,7 @@ static const char * const err_codes_libkern_kext[] = { "(libkern/kext) kext is in use or retained (cannot unload)", /* 0x18 */ "(libkern/kext) kext request timed out", /* 0x19 */ "(libkern/kext) kext is stopping and cannot issue requests", /* 0x1a */ + "(libkern/kext) system policy prevents loading", /* 0x1b */ }; /* libkern is err_system(0x37) */ diff --git a/libsyscall/mach/host.c b/libsyscall/mach/host.c index a781a09ad..651d148d8 100644 --- a/libsyscall/mach/host.c +++ b/libsyscall/mach/host.c @@ -46,16 +46,33 @@ kern_return_t host_get_multiuser_config_flags(host_t host __unused, uint32_t *multiuser_flags) { +#if TARGET_OS_EMBEDDED + volatile uint32_t *multiuser_flag_address = (volatile uint32_t *)(uintptr_t)(_COMM_PAGE_MULTIUSER_CONFIG); + *multiuser_flags = *multiuser_flag_address; + return KERN_SUCCESS; +#else (void)multiuser_flags; return KERN_NOT_SUPPORTED; +#endif } kern_return_t host_check_multiuser_mode(host_t host __unused, uint32_t *multiuser_mode) { +#if TARGET_OS_EMBEDDED + uint32_t multiuser_flags; + kern_return_t kr; + + kr = host_get_multiuser_config_flags(host, &multiuser_flags); + if (kr != KERN_SUCCESS) + return kr; + *multiuser_mode = (multiuser_flags & kIsMultiUserDevice) == kIsMultiUserDevice; + return KERN_SUCCESS; +#else (void)multiuser_mode; return KERN_NOT_SUPPORTED; +#endif } extern kern_return_t diff --git a/libsyscall/mach/mach_init.c b/libsyscall/mach/mach_init.c index 90a42ceb5..338f7c95b 100644 --- a/libsyscall/mach/mach_init.c +++ b/libsyscall/mach/mach_init.c @@ -115,6 +115,12 @@ _mach_fork_child(void) return 0; } +#if defined(__arm__) || defined(__arm64__) +#if !defined(_COMM_PAGE_USER_PAGE_SHIFT_64) && defined(_COMM_PAGE_UNUSED0) +#define _COMM_PAGE_USER_PAGE_SHIFT_32 (_COMM_PAGE_UNUSED0) +#define _COMM_PAGE_USER_PAGE_SHIFT_64 (_COMM_PAGE_UNUSED0+1) +#endif +#endif void mach_init_doit(void) @@ -136,7 +142,13 @@ mach_init_doit(void) } if (vm_page_shift == 0) { +#if defined(__arm64__) + vm_page_shift = *(uint8_t*) _COMM_PAGE_USER_PAGE_SHIFT_64; +#elif defined(__arm__) + vm_page_shift = *(uint8_t*) _COMM_PAGE_USER_PAGE_SHIFT_32; +#else vm_page_shift = vm_kernel_page_shift; +#endif vm_page_size = 1 << vm_page_shift; vm_page_mask = vm_page_size - 1; } diff --git a/libsyscall/mach/string.h b/libsyscall/mach/string.h index b3c00458e..7d668126a 100644 --- a/libsyscall/mach/string.h +++ b/libsyscall/mach/string.h @@ -39,8 +39,8 @@ // of Libc's string.h (which no one should be using bar MIG) in order // to override their use of memcpy. -int _mach_snprintf(char *buffer, int length, const char *fmt, ...); -int _mach_vsnprintf(char *buffer, int length, const char *fmt, va_list ap); +int _mach_snprintf(char *buffer, int length, const char *fmt, ...) __printflike(3, 4); +int _mach_vsnprintf(char *buffer, int length, const char *fmt, va_list ap) __printflike(3, 0); // Actually in memcpy.c but MIG likes to include string.h diff --git a/libsyscall/os/tsd.h b/libsyscall/os/tsd.h index 0e064b954..d49087f14 100644 --- a/libsyscall/os/tsd.h +++ b/libsyscall/os/tsd.h @@ -38,15 +38,35 @@ #define __TSD_THREAD_SELF 0 #define __TSD_ERRNO 1 #define __TSD_MIG_REPLY 2 +#define __TSD_MACH_THREAD_SELF 3 +#define __TSD_THREAD_QOS_CLASS 4 +#define __TSD_RETURN_TO_KERNEL 5 +/* slot 6 is reserved for Windows/WINE compatibility reasons */ #define __TSD_SEMAPHORE_CACHE 9 +#ifdef __arm__ +#include <arm/arch.h> +#endif __attribute__((always_inline)) static __inline__ unsigned int _os_cpu_number(void) { - /* Not yet implemented */ - return 0; +#if defined(__arm__) && defined(_ARM_ARCH_6) + uintptr_t p; + __asm__("mrc p15, 0, %[p], c13, c0, 3" : [p] "=&r" (p)); + return (unsigned int)(p & 0x3ul); +#elif defined(__arm64__) + uint64_t p; + __asm__("mrs %[p], TPIDRRO_EL0" : [p] "=&r" (p)); + return (unsigned int)p & 0x7; +#elif defined(__x86_64__) || defined(__i386__) + struct { uintptr_t p1, p2; } p; + __asm__("sidt %[p]" : [p] "=&m" (p)); + return (unsigned int)(p.p1 & 0xfff); +#else +#error _os_cpu_number not implemented on this architecture +#endif } #if defined(__i386__) || defined(__x86_64__) @@ -84,6 +104,28 @@ _os_tsd_set_direct(unsigned long slot, void *val) } #endif +#elif defined(__arm__) || defined(__arm64__) + +__attribute__((always_inline, pure)) +static __inline__ void** +_os_tsd_get_base(void) +{ +#if defined(__arm__) && defined(_ARM_ARCH_6) + uintptr_t tsd; + __asm__("mrc p15, 0, %0, c13, c0, 3" : "=r" (tsd)); + tsd &= ~0x3ul; /* lower 2-bits contain CPU number */ +#elif defined(__arm__) && defined(_ARM_ARCH_5) + register uintptr_t tsd asm ("r9"); +#elif defined(__arm64__) + uint64_t tsd; + __asm__("mrs %0, TPIDRRO_EL0" : "=r" (tsd)); + tsd &= ~0x7ull; +#endif + + return (void**)(uintptr_t)tsd; +} +#define _os_tsd_get_base() _os_tsd_get_base() + #else #error _os_tsd_get_base not implemented on this architecture #endif diff --git a/libsyscall/wrappers/__commpage_gettimeofday.c b/libsyscall/wrappers/__commpage_gettimeofday.c index 4967a2f8d..0ebfc5318 100644 --- a/libsyscall/wrappers/__commpage_gettimeofday.c +++ b/libsyscall/wrappers/__commpage_gettimeofday.c @@ -24,56 +24,93 @@ #include <sys/time.h> #include <mach/mach_time.h> #include <machine/cpu_capabilities.h> +#include <os/overflow.h> +#include <kern/arithmetic_128.h> int __commpage_gettimeofday(struct timeval *); __attribute__((visibility("hidden"))) int __commpage_gettimeofday_internal(struct timeval *tp, uint64_t *tbr_out); -#if defined(__x86_64__) || defined(__i386__) +int +__commpage_gettimeofday(struct timeval *tp) +{ + return __commpage_gettimeofday_internal(tp, NULL); +} -// XXX: must be kept in sync with __commpage_gettimeofday.s int __commpage_gettimeofday_internal(struct timeval *tp, uint64_t *tbr_out) { - volatile uint32_t *gtod_generation_p = _COMM_PAGE_GTOD_GENERATION; - volatile uint64_t *gtod_sec_base_p = _COMM_PAGE_GTOD_SEC_BASE; - volatile uint64_t *gtod_ns_base_p = _COMM_PAGE_GTOD_NS_BASE; - - uint64_t tbr, gen, tod_secs, tod_nsecs, elapsed; - while(1) { - gen = *gtod_generation_p; - tbr = mach_absolute_time(); - tod_secs = *gtod_sec_base_p; - tod_nsecs = *gtod_ns_base_p; - uint64_t gen2 = *gtod_generation_p; - if(__builtin_expect(gen, gen2) == gen2) - break; + uint64_t now, over; + uint64_t delta,frac; + uint64_t TimeStamp_tick; + uint64_t TimeStamp_sec; + uint64_t TimeStamp_frac; + uint64_t Tick_scale; + uint64_t Ticks_per_sec; + + volatile uint64_t *gtod_TimeStamp_tick_p; + volatile uint64_t *gtod_TimeStamp_sec_p; + volatile uint64_t *gtod_TimeStamp_frac_p; + volatile uint64_t *gtod_Ticks_scale_p; + volatile uint64_t *gtod_Ticks_per_sec_p; + + new_commpage_timeofday_data_t *commpage_timeofday_datap; + + commpage_timeofday_datap = (new_commpage_timeofday_data_t *)_COMM_PAGE_NEWTIMEOFDAY_DATA; + + gtod_TimeStamp_tick_p = &commpage_timeofday_datap->TimeStamp_tick; + gtod_TimeStamp_sec_p = &commpage_timeofday_datap->TimeStamp_sec; + gtod_TimeStamp_frac_p = &commpage_timeofday_datap->TimeStamp_frac; + gtod_Ticks_scale_p = &commpage_timeofday_datap->Ticks_scale; + gtod_Ticks_per_sec_p = &commpage_timeofday_datap->Ticks_per_sec; + + do { + TimeStamp_tick = *gtod_TimeStamp_tick_p; + TimeStamp_sec = *gtod_TimeStamp_sec_p; + TimeStamp_frac = *gtod_TimeStamp_frac_p; + Tick_scale = *gtod_Ticks_scale_p; + Ticks_per_sec = *gtod_Ticks_per_sec_p; + + /* + * This call contains an instruction barrier which will ensure that the + * second read of the abs time isn't speculated above the reads of the + * other values above + */ + now = mach_absolute_time(); + } while (TimeStamp_tick != *gtod_TimeStamp_tick_p); + + if (TimeStamp_tick == 0) + return(1); + + delta = now - TimeStamp_tick; + + /* If more than one second force a syscall */ + if (delta >= Ticks_per_sec) + return(1); + + tp->tv_sec = TimeStamp_sec; + + over = multi_overflow(Tick_scale, delta); + if(over){ + tp->tv_sec += over; } - if (gen == 0) return KERN_FAILURE; - elapsed = tbr - tod_nsecs; - - unsigned long secs; - uint32_t nsec; -#if defined(__x86_64__) - secs = elapsed/NSEC_PER_SEC; - nsec = elapsed % NSEC_PER_SEC; -#elif defined(__i386__) - uint32_t secs1, secs2; - secs1 = elapsed >> 32; - secs2 = elapsed; - __asm__ ( - "divl %4" - : "=a" (secs), "=d" (nsec) - : "0" (secs2), "1" (secs1), "rm" (NSEC_PER_SEC) - ); -#endif /* __i386 or __x86_64__ */ - tp->tv_sec = tod_secs + secs; - tp->tv_usec = nsec / NSEC_PER_USEC; - - if (tbr_out) *tbr_out = tbr; - - return KERN_SUCCESS; -} -#endif + /* Sum scale*delta to TimeStamp_frac, if it overflows increment sec */ + frac = TimeStamp_frac; + frac += Tick_scale * delta; + if( TimeStamp_frac > frac ) + tp->tv_sec++; + + /* + * Convert frac (64 bit frac of a sec) to usec + * usec = frac * USEC_PER_SEC / 2^64 + */ + tp->tv_usec = ((uint64_t)1000000 * (uint32_t)(frac >> 32)) >> 32; + + if (tbr_out) { + *tbr_out = now; + } + + return(0); +} diff --git a/libsyscall/wrappers/__commpage_gettimeofday.s b/libsyscall/wrappers/__commpage_gettimeofday.s deleted file mode 100644 index da920f28b..000000000 --- a/libsyscall/wrappers/__commpage_gettimeofday.s +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright (c) 2003-2007 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -#include <sys/appleapiopts.h> -#include <machine/cpu_capabilities.h> - -#define NSEC_PER_SEC 1000*1000*1000 -#define NSEC_PER_USEC 1000 - -#if defined(__i386__) - - .align 4 - .globl ___commpage_gettimeofday -___commpage_gettimeofday: - push %ebp - mov %esp,%ebp - push %esi - push %ebx -0: - movl _COMM_PAGE_GTOD_GENERATION,%esi /* get generation (0 if disabled) */ - testl %esi,%esi /* disabled? */ - jz 4f - - call _mach_absolute_time /* get nanotime in %edx:%eax */ - - sub _COMM_PAGE_GTOD_NS_BASE,%eax - sbb _COMM_PAGE_GTOD_NS_BASE+4,%edx - mov _COMM_PAGE_GTOD_SEC_BASE,%ebx /* load all the data before checking generation */ - mov $ NSEC_PER_SEC,%ecx - - cmpl _COMM_PAGE_GTOD_GENERATION,%esi /* has time data changed out from under us? */ - jne 0b - - div %ecx - add %eax,%ebx - - mov $ NSEC_PER_USEC,%ecx - mov %edx,%eax - xor %edx,%edx - div %ecx - - mov 8(%ebp),%ecx - mov %ebx,(%ecx) - mov %eax,4(%ecx) - xor %eax,%eax -3: - pop %ebx - pop %esi - pop %ebp - ret -4: /* fail */ - movl $1,%eax - jmp 3b - -#elif defined(__x86_64__) - - .align 4, 0x90 - .globl ___commpage_gettimeofday -___commpage_gettimeofday: -// %rdi = ptr to timeval - pushq %rbp // set up a frame for backtraces - pushq %r12 // push callee-saved registers we want to use - pushq %r13 - pushq %r14 - subq $8, %rsp - movq %rsp,%rbp - movq %rdi,%r12 // save ptr to timeval - movq $(_COMM_PAGE_TIME_DATA_START),%r13 -0: - movl _GTOD_GENERATION(%r13),%r14d // get generation (0 if disabled) - testl %r14d,%r14d // disabled? - jz 4f - - call _mach_absolute_time // get %rax <- nanotime() - - movl _GTOD_SEC_BASE(%r13),%r8d // get _COMM_PAGE_TIMESTAMP - subq _GTOD_NS_BASE(%r13),%rax // generate nanoseconds since timestamp - cmpl _GTOD_GENERATION(%r13),%r14d // has data changed out from under us? - jne 0b - - movl $ NSEC_PER_SEC,%ecx - movq %rax,%rdx - shrq $32,%rdx // get high half of delta in %edx - divl %ecx // %eax <- seconds since timestamp, %edx <- nanoseconds - addl %eax,%r8d // add seconds elapsed to timestamp seconds - - movl $ NSEC_PER_USEC,%ecx - movl %edx,%eax - xorl %edx,%edx - divl %ecx // divide residual ns by 1000 to get residual us in %eax - - movq %r8,(%r12) // store 64-bit seconds into timeval - movl %eax,8(%r12) // store 32-bit useconds into timeval - xorl %eax,%eax // return 0 for success -3: - addq $8, %rsp - popq %r14 - popq %r13 - popq %r12 - popq %rbp - ret -4: // fail - movl $1,%eax - jmp 3b - -#endif diff --git a/libsyscall/wrappers/__get_cpu_capabilities.s b/libsyscall/wrappers/__get_cpu_capabilities.s index de177986a..86b0ee2a7 100644 --- a/libsyscall/wrappers/__get_cpu_capabilities.s +++ b/libsyscall/wrappers/__get_cpu_capabilities.s @@ -47,6 +47,31 @@ __get_cpu_capabilities: movl _COMM_PAGE_CPU_CAPABILITIES64+4, %edx ret +#elif defined(__arm__) + + .text + .align 2 + .globl __get_cpu_capabilities +__get_cpu_capabilities: + mov r0, #(_COMM_PAGE_CPU_CAPABILITIES & 0x000000ff) + orr r0, r0, #(_COMM_PAGE_CPU_CAPABILITIES & 0x0000ff00) + orr r0, r0, #(_COMM_PAGE_CPU_CAPABILITIES & 0x00ff0000) + orr r0, r0, #(_COMM_PAGE_CPU_CAPABILITIES & 0xff000000) + ldr r0, [r0] + bx lr + +#elif defined(__arm64__) + + .text + .align 2 + .globl __get_cpu_capabilities +__get_cpu_capabilities: + ldr x0, Lcommpage_cc_addr + ldr w0, [x0] + ret +Lcommpage_cc_addr: +.quad _COMM_PAGE_CPU_CAPABILITIES + #else #error Unsupported architecture #endif diff --git a/libsyscall/wrappers/_libkernel_init.c b/libsyscall/wrappers/_libkernel_init.c index 16d7e1917..3eb67853d 100644 --- a/libsyscall/wrappers/_libkernel_init.c +++ b/libsyscall/wrappers/_libkernel_init.c @@ -26,10 +26,20 @@ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ +#include <TargetConditionals.h> +#include <stdbool.h> +#include <strings.h> +#include <unistd.h> #include "_libkernel_init.h" extern int mach_init(void); +#if TARGET_OS_OSX +__attribute__((visibility("default"))) +extern bool _os_xbs_chrooted; +bool _os_xbs_chrooted; +#endif + /* dlsym() funcptr is for legacy support in exc_catcher */ void* (*_dlsym)(void*, const char*) __attribute__((visibility("hidden"))); diff --git a/libsyscall/wrappers/coalition.c b/libsyscall/wrappers/coalition.c index 92a0bda27..627da2261 100644 --- a/libsyscall/wrappers/coalition.c +++ b/libsyscall/wrappers/coalition.c @@ -50,3 +50,14 @@ int coalition_info_resource_usage(uint64_t cid, struct coalition_resource_usage { return __coalition_info(COALITION_INFO_RESOURCE_USAGE, &cid, cru, &sz); } + +int coalition_info_set_name(uint64_t cid, const char *name, size_t size) +{ + return __coalition_info(COALITION_INFO_SET_NAME, &cid, (void *)name, &size); +} + +int coalition_info_set_efficiency(uint64_t cid, uint64_t flags) +{ + size_t size = sizeof(flags); + return __coalition_info(COALITION_INFO_SET_EFFICIENCY, &cid, (void *)&flags, &size); +} diff --git a/libsyscall/wrappers/init_cpu_capabilities.c b/libsyscall/wrappers/init_cpu_capabilities.c index 7eecac6bf..8414c1dfa 100644 --- a/libsyscall/wrappers/init_cpu_capabilities.c +++ b/libsyscall/wrappers/init_cpu_capabilities.c @@ -38,4 +38,16 @@ _init_cpu_capabilities( void ) _cpu_capabilities = _get_cpu_capabilities(); } +#elif defined(__arm__) || defined(__arm64__) + +extern int _get_cpu_capabilities(void); + +int _cpu_capabilities = 0; +int _cpu_has_altivec = 0; // DEPRECATED: use _cpu_capabilities instead + +void +_init_cpu_capabilities( void ) +{ +} + #endif diff --git a/libsyscall/wrappers/libproc/libproc.c b/libsyscall/wrappers/libproc/libproc.c index cc0321e2d..255664d9c 100644 --- a/libsyscall/wrappers/libproc/libproc.c +++ b/libsyscall/wrappers/libproc/libproc.c @@ -170,6 +170,17 @@ proc_pidfileportinfo(int pid, uint32_t fileport, int flavor, void *buffer, int b return (retval); } +int +proc_piddynkqueueinfo(int pid, int flavor, kqueue_id_t kq_id, void *buffer, int buffersize) +{ + int ret; + + if ((ret = __proc_info(PROC_INFO_CALL_PIDDYNKQUEUEINFO, pid, flavor, (uint64_t)kq_id, buffer, buffersize)) == -1) { + return 0; + } + + return ret; +} int proc_name(int pid, void * buffer, uint32_t buffersize) @@ -199,7 +210,7 @@ proc_name(int pid, void * buffer, uint32_t buffersize) int proc_regionfilename(int pid, uint64_t address, void * buffer, uint32_t buffersize) { - int retval = 0, len; + int retval; struct proc_regionwithpathinfo reginfo; if (buffersize < MAXPATHLEN) { @@ -209,17 +220,9 @@ proc_regionfilename(int pid, uint64_t address, void * buffer, uint32_t buffersiz retval = proc_pidinfo(pid, PROC_PIDREGIONPATHINFO, (uint64_t)address, ®info, sizeof(struct proc_regionwithpathinfo)); if (retval != -1) { - len = (int)strlen(®info.prp_vip.vip_path[0]); - if (len != 0) { - if (len > MAXPATHLEN) - len = MAXPATHLEN; - bcopy(®info.prp_vip.vip_path[0], buffer, len); - return(len); - } - return(0); + return ((int)(strlcpy(buffer, reginfo.prp_vip.vip_path, MAXPATHLEN))); } - return(0); - + return(0); } int @@ -557,90 +560,67 @@ proc_disable_wakemon(pid_t pid) int proc_list_uptrs(int pid, uint64_t *buf, uint32_t bufsz) { - int i, j; - int nfds, nkns; - int count = 0; - int knote_max = 4096; /* arbitrary starting point */ - - /* if buffer is empty, this call simply counts the knotes */ - if (bufsz > 0 && buf == NULL) { - errno = EFAULT; - return -1; - } + return __proc_info(PROC_INFO_CALL_PIDINFO, pid, PROC_PIDLISTUPTRS, 0, + buf, bufsz); +} - /* get the list of FDs for this process */ - struct proc_fdinfo fdlist[OPEN_MAX+1]; - nfds = proc_pidinfo(pid, PROC_PIDLISTFDS, 0, &fdlist[1], OPEN_MAX*sizeof(struct proc_fdinfo)); - if (nfds < 0 || nfds > OPEN_MAX) { - return -1; - } +int +proc_list_dynkqueueids(int pid, kqueue_id_t *buf, uint32_t bufsz) +{ + return __proc_info(PROC_INFO_CALL_PIDINFO, pid, PROC_PIDLISTDYNKQUEUES, 0, + buf, bufsz); +} - /* Add FD -1, the implicit workq kqueue */ - fdlist[0].proc_fd = -1; - fdlist[0].proc_fdtype = PROX_FDTYPE_KQUEUE; - nfds++; - struct kevent_extinfo *kqext = malloc(knote_max * sizeof(struct kevent_extinfo)); - if (!kqext) { - errno = ENOMEM; - return -1; - } +int +proc_setcpu_percentage(pid_t pid, int action, int percentage) +{ + proc_policy_cpuusage_attr_t attr; - for (i = 0; i < nfds; i++) { - if (fdlist[i].proc_fdtype != PROX_FDTYPE_KQUEUE) { - continue; - } + bzero(&attr, sizeof(proc_policy_cpuusage_attr_t)); + attr.ppattr_cpu_attr = action; + attr.ppattr_cpu_percentage = percentage; + if (__process_policy(PROC_POLICY_SCOPE_PROCESS, PROC_POLICY_ACTION_APPLY, PROC_POLICY_RESOURCE_USAGE, PROC_POLICY_RUSAGE_CPU, (proc_policy_attribute_t*)&attr, pid, (uint64_t)0) != -1) + return(0); + else + return(errno); +} - again: - nkns = __proc_info(PROC_INFO_CALL_PIDFDINFO, pid, PROC_PIDFDKQUEUE_EXTINFO, - (uint64_t)fdlist[i].proc_fd, kqext, knote_max * sizeof(struct kevent_extinfo)); - if (nkns < 0) { - if (errno == EBADF) { - /* the FD table can change after enumerating the FDs */ - errno = EAGAIN; - } - free(kqext); - return -1; - } +int +proc_clear_cpulimits(pid_t pid) +{ + if (__process_policy(PROC_POLICY_SCOPE_PROCESS, PROC_POLICY_ACTION_RESTORE, PROC_POLICY_RESOURCE_USAGE, PROC_POLICY_RUSAGE_CPU, NULL, pid, (uint64_t)0) != -1) + return(0); + else + return(errno); +} - if (nkns > knote_max) { - /* there are more knotes than we requested - try again with a - * larger buffer */ - free(kqext); - knote_max = nkns + 32; /* small margin in case of extra knotes */ - kqext = malloc(knote_max * sizeof(struct kevent_extinfo)); - if (!kqext) { - errno = ENOMEM; - return -1; - } - goto again; - } +#if TARGET_OS_EMBEDDED - for (j = 0; j < nkns; j++) { - if (kqext[j].kqext_kev.udata == 0) { - continue; - } +int +proc_setcpu_deadline(pid_t pid, int action, uint64_t deadline) +{ + proc_policy_cpuusage_attr_t attr; - if (bufsz >= sizeof(uint64_t)) { - *buf++ = kqext[j].kqext_kev.udata; - bufsz -= sizeof(uint64_t); - } - count++; - } - } + bzero(&attr, sizeof(proc_policy_cpuusage_attr_t)); + attr.ppattr_cpu_attr = action; + attr.ppattr_cpu_attr_deadline = deadline; + if (__process_policy(PROC_POLICY_SCOPE_PROCESS, PROC_POLICY_ACTION_APPLY, PROC_POLICY_RESOURCE_USAGE, PROC_POLICY_RUSAGE_CPU, (proc_policy_attribute_t*)&attr, pid, (uint64_t)0) != -1) + return(0); + else + return(errno); - free(kqext); - return count; } -int -proc_setcpu_percentage(pid_t pid, int action, int percentage) +int +proc_setcpu_percentage_withdeadline(pid_t pid, int action, int percentage, uint64_t deadline) { proc_policy_cpuusage_attr_t attr; bzero(&attr, sizeof(proc_policy_cpuusage_attr_t)); attr.ppattr_cpu_attr = action; attr.ppattr_cpu_percentage = percentage; + attr.ppattr_cpu_attr_deadline = deadline; if (__process_policy(PROC_POLICY_SCOPE_PROCESS, PROC_POLICY_ACTION_APPLY, PROC_POLICY_RESOURCE_USAGE, PROC_POLICY_RUSAGE_CPU, (proc_policy_attribute_t*)&attr, pid, (uint64_t)0) != -1) return(0); else @@ -648,14 +628,87 @@ proc_setcpu_percentage(pid_t pid, int action, int percentage) } int -proc_clear_cpulimits(pid_t pid) +proc_appstate(int pid, int * appstatep) { - if (__process_policy(PROC_POLICY_SCOPE_PROCESS, PROC_POLICY_ACTION_RESTORE, PROC_POLICY_RESOURCE_USAGE, PROC_POLICY_RUSAGE_CPU, NULL, pid, (uint64_t)0) != -1) + int state; + + if (__process_policy(PROC_POLICY_SCOPE_PROCESS, PROC_POLICY_ACTION_GET, PROC_POLICY_APP_LIFECYCLE, PROC_POLICY_APPLIFE_STATE, (proc_policy_attribute_t*)&state, pid, (uint64_t)0) != -1) { + if (appstatep != NULL) + *appstatep = state; + return(0); + } else + return(errno); + +} + +int +proc_setappstate(int pid, int appstate) +{ + int state = appstate; + + switch (state) { + case PROC_APPSTATE_NONE: + case PROC_APPSTATE_ACTIVE: + case PROC_APPSTATE_INACTIVE: + case PROC_APPSTATE_BACKGROUND: + case PROC_APPSTATE_NONUI: + break; + default: + return(EINVAL); + } + if (__process_policy(PROC_POLICY_SCOPE_PROCESS, PROC_POLICY_ACTION_APPLY, PROC_POLICY_APP_LIFECYCLE, PROC_POLICY_APPLIFE_STATE, (proc_policy_attribute_t*)&state, pid, (uint64_t)0) != -1) return(0); else return(errno); } +int +proc_devstatusnotify(int devicestatus) +{ + int state = devicestatus; + + switch (devicestatus) { + case PROC_DEVSTATUS_SHORTTERM: + case PROC_DEVSTATUS_LONGTERM: + break; + default: + return(EINVAL); + } + + if (__process_policy(PROC_POLICY_SCOPE_PROCESS, PROC_POLICY_ACTION_APPLY, PROC_POLICY_APP_LIFECYCLE, PROC_POLICY_APPLIFE_DEVSTATUS, (proc_policy_attribute_t*)&state, getpid(), (uint64_t)0) != -1) { + return(0); + } else + return(errno); + +} + +int +proc_pidbind(int pid, uint64_t threadid, int bind) +{ + int state = bind; + pid_t passpid = pid; + + switch (bind) { + case PROC_PIDBIND_CLEAR: + passpid = getpid(); /* ignore pid on clear */ + break; + case PROC_PIDBIND_SET: + break; + default: + return(EINVAL); + } + if (__process_policy(PROC_POLICY_SCOPE_PROCESS, PROC_POLICY_ACTION_APPLY, PROC_POLICY_APP_LIFECYCLE, PROC_POLICY_APPLIFE_PIDBIND, (proc_policy_attribute_t*)&state, passpid, threadid) != -1) + return(0); + else + return(errno); +} + +int +proc_can_use_foreground_hw(int pid, uint32_t *reason) +{ + return __proc_info(PROC_INFO_CALL_CANUSEFGHW, pid, 0, NULL, reason, sizeof(*reason)); +} +#endif /* TARGET_OS_EMBEDDED */ /* Donate importance to adaptive processes from this process */ @@ -664,11 +717,19 @@ proc_donate_importance_boost() { int rval; +#if TARGET_OS_EMBEDDED + rval = __process_policy(PROC_POLICY_SCOPE_PROCESS, + PROC_POLICY_ACTION_ENABLE, + PROC_POLICY_APPTYPE, + PROC_POLICY_IOS_DONATEIMP, + NULL, getpid(), (uint64_t)0); +#else /* TARGET_OS_EMBEDDED */ rval = __process_policy(PROC_POLICY_SCOPE_PROCESS, PROC_POLICY_ACTION_SET, PROC_POLICY_BOOST, PROC_POLICY_IMP_DONATION, NULL, getpid(), 0); +#endif /* TARGET_OS_EMBEDDED */ if (rval == 0) return (0); @@ -808,6 +869,7 @@ proc_denap_assertion_complete(uint64_t assertion_token) return proc_importance_assertion_complete(assertion_token); } +#if !TARGET_OS_EMBEDDED int proc_clear_vmpressure(pid_t pid) @@ -903,6 +965,7 @@ proc_suppress(__unused pid_t pid, __unused uint64_t *generation) #endif /* !TARGET_IPHONE_SIMULATOR */ +#endif /* !TARGET_OS_EMBEDDED */ diff --git a/libsyscall/wrappers/libproc/libproc.h b/libsyscall/wrappers/libproc/libproc.h index 27633ffa4..3cfcfdc70 100644 --- a/libsyscall/wrappers/libproc/libproc.h +++ b/libsyscall/wrappers/libproc/libproc.h @@ -138,6 +138,10 @@ int proc_terminate(pid_t pid, int *sig); * failure and errno set appropriately. */ int proc_list_uptrs(pid_t pid, uint64_t *buffer, uint32_t buffersize); + +int proc_list_dynkqueueids(int pid, kqueue_id_t *buf, uint32_t bufsz); +int proc_piddynkqueueinfo(int pid, int flavor, kqueue_id_t kq_id, void *buffer, + int buffersize); #endif /* PRIVATE */ __END_DECLS diff --git a/libsyscall/wrappers/libproc/libproc_internal.h b/libsyscall/wrappers/libproc/libproc_internal.h index 7169b7eb4..f18366427 100644 --- a/libsyscall/wrappers/libproc/libproc_internal.h +++ b/libsyscall/wrappers/libproc/libproc_internal.h @@ -41,6 +41,52 @@ int proc_clear_cpulimits(pid_t pid) __OSX_AVAILABLE_STARTING(__MAC_10_12_2, __IP /* CPU limits, applies to current thread only. 0% unsets limit */ int proc_setthread_cpupercent(uint8_t percentage, uint32_t ms_refill) __OSX_AVAILABLE_STARTING(__MAC_10_10, __IPHONE_5_0); +#if TARGET_OS_EMBEDDED + +/* CPU monitor action, continued */ +#define PROC_SETCPU_ACTION_SUSPEND 2 +#define PROC_SETCPU_ACTION_TERMINATE 3 +#define PROC_SETCPU_ACTION_NOTIFY 4 + +int proc_setcpu_deadline(pid_t pid, int action, uint64_t deadline) __OSX_AVAILABLE_STARTING(__MAC_NA, __IPHONE_5_0); +int proc_setcpu_percentage_withdeadline(pid_t pid, int action, int percentage, uint64_t deadline) __OSX_AVAILABLE_STARTING(__MAC_NA, __IPHONE_5_0); + +#define PROC_APPSTATE_NONE 0 +#define PROC_APPSTATE_ACTIVE 1 +#define PROC_APPSTATE_BACKGROUND 2 +#define PROC_APPSTATE_NONUI 3 +#define PROC_APPSTATE_INACTIVE 4 + +int proc_setappstate(int pid, int appstate); +int proc_appstate(int pid, int * appstatep); + +#define PROC_DEVSTATUS_SHORTTERM 1 +#define PROC_DEVSTATUS_LONGTERM 2 + +int proc_devstatusnotify(int devicestatus); + +#define PROC_PIDBIND_CLEAR 0 +#define PROC_PIDBIND_SET 1 +int proc_pidbind(int pid, uint64_t threadid, int bind); + +/* + * High level check to see if a process is allowed to use HW + * resources reserved for foreground applications. + * Returns: + * 1 if the PID is allowed + * 0 if the PID is NOT allowed + * <0 on error + * + * When 0 is returned, 'reason' is set to indicate why + * the pid is not allowed to use foreground-only hardware. + * Reasons returned by the kernel are found in <sys/proc_info.h> + * + * When <0 is returned, errno indicates the reason + * for the failure. + */ +int proc_can_use_foreground_hw(int pid, uint32_t *reason); + +#else /* TARGET_OS_EMBEDDED */ /* resume the process suspend due to low VM resource */ int proc_clear_vmpressure(pid_t pid); @@ -67,6 +113,7 @@ int proc_clear_delayidlesleep(void); int proc_disable_apptype(pid_t pid, int apptype); int proc_enable_apptype(pid_t pid, int apptype); +#endif /* TARGET_OS_EMBEDDED */ /* mark process as importance donating */ int proc_donate_importance_boost(void); diff --git a/libsyscall/wrappers/mach_absolute_time.s b/libsyscall/wrappers/mach_absolute_time.s index 603504a9a..cf93161b9 100644 --- a/libsyscall/wrappers/mach_absolute_time.s +++ b/libsyscall/wrappers/mach_absolute_time.s @@ -150,6 +150,130 @@ _mach_absolute_time: popq %rbp ret +#elif defined(__arm__) + +#include <mach/arm/syscall_sw.h> + +/* + * If userspace access to the timebase is supported (indicated through the commpage), + * directly reads the timebase and uses it and the current timebase offset (also in + * the commpage, and updated whenever the system wakes from sleep) to construct the + * current time value; otherwise, traps to the kernel to handle this. + * + * If we do this in user mode, there are two cases where we may need to redrive the + * read. We do 3 reads (high-low-high) to the timebase, because we only have a + * 32-bit interface to it (despite the use of mrrc). If the high bits change, we + * need to reread the register (as our returned value could otherwise be off by + * 2^32 mach absolute time units). + * + * We do two reads of the offset, before and after the register reads. If the offset + * changes, we have gone to sleep in the midst of doing a read. This case should be + * exceedingly rare, but could result in a terribly inaccurate result, so we need + * to get a fresh timebase value. + */ + .text + .align 2 + .globl _mach_absolute_time +_mach_absolute_time: + movw ip, #((_COMM_PAGE_TIMEBASE_OFFSET) & 0x0000FFFF) + movt ip, #(((_COMM_PAGE_TIMEBASE_OFFSET) >> 16) & 0x0000FFFF) + ldrb r0, [ip, #((_COMM_PAGE_USER_TIMEBASE) - (_COMM_PAGE_TIMEBASE_OFFSET))] + cmp r0, #0 // Are userspace reads supported? + beq _mach_absolute_time_kernel // If not, go to the kernel + isb // Prevent speculation on CNTPCT across calls + // (see ARMV7C.b section B8.1.2, ARMv8 section D6.1.2) + push {r4, r5, r7, lr} // Push a frame + add r7, sp, #8 +L_mach_absolute_time_user: + ldr r4, [ip] // Load offset low bits + ldr r5, [ip, #4] // Load offset high bits + mrrc p15, 0, r3, r1, c14 // Read timebase high to r1 + mrrc p15, 0, r0, r3, c14 // Read timebase low to r0 + mrrc p15, 0, r3, r2, c14 // Read timebase high to r2 + cmp r1, r2 // Did the high bits change? + bne L_mach_absolute_time_user // Loop if timebase high changed + ldr r2, [ip] // Load offset low bits + ldr r3, [ip, #4] // Load offset high bits + eor r4, r2 // Compare our offset values... + eor r5, r3 + orrs r5, r4 + bne L_mach_absolute_time_user // If they changed, try again + adds r0, r0, r2 // Construct mach_absolute_time + adcs r1, r1, r3 + pop {r4, r5, r7, pc} // Pop the frame + + .text + .align 2 + .globl _mach_absolute_time_kernel +_mach_absolute_time_kernel: + mov r12, #-3 // Load the magic MAT number + swi #SWI_SYSCALL + bx lr + + .text + .align 2 + .globl _mach_continuous_time_kernel +_mach_continuous_time_kernel: + mov r12, #-4 // Load the magic MCT number + swi #SWI_SYSCALL + bx lr + +#elif defined(__arm64__) + +#include <mach/arm/syscall_sw.h> + +/* + * If userspace access to the timebase is supported (indicated through the commpage), + * directly reads the timebase and uses it and the current timebase offset (also in + * the commpage, and updated whenever the system wakes from sleep) to construct the + * current time value; otherwise, traps to the kernel to handle this. + * + * If we do this in user mode, we do two reads of the offset, before and after we + * read the register. If the offset changes, we have gone to sleep in the midst of + * doing a read. This case should be exceedingly rare, but could result in a terribly + * inaccurate result, so we need to get a fresh timebase value. + * + * Note that the commpage address construction expects our top 2 bytes to be 0xFFFF. + * If this changes (i.e, we significantly relocate the commpage), this logic will need + * to change as well (use 4 movk instructions rather than cheating with the movn). + */ + .text + .align 2 + .globl _mach_absolute_time +_mach_absolute_time: + movn x3, #(~((_COMM_PAGE_TIMEBASE_OFFSET) >> 32) & 0x000000000000FFFF), lsl #32 + movk x3, #(((_COMM_PAGE_TIMEBASE_OFFSET) >> 16) & 0x000000000000FFFF), lsl #16 + movk x3, #((_COMM_PAGE_TIMEBASE_OFFSET) & 0x000000000000FFFF) + ldrb w2, [x3, #((_COMM_PAGE_USER_TIMEBASE) - (_COMM_PAGE_TIMEBASE_OFFSET))] + cmp x2, #0 // Are userspace reads supported? + b.eq _mach_absolute_time_kernel // If not, go to the kernel + isb // Prevent speculation on CNTPCT across calls + // (see ARMV7C.b section B8.1.2, ARMv8 section D6.1.2) +L_mach_absolute_time_user: + ldr x1, [x3] // Load the offset + mrs x0, CNTPCT_EL0 // Read the timebase + ldr x2, [x3] // Load the offset + cmp x1, x2 // Compare our offset values... + b.ne L_mach_absolute_time_user // If they changed, try again + add x0, x0, x1 // Construct mach_absolute_time + ret + + .text + .align 2 + .globl _mach_absolute_time_kernel +_mach_absolute_time_kernel: + mov w16, #-3 // Load the magic MAT number + svc #SWI_SYSCALL + ret + + .text + .align 2 + .globl _mach_continuous_time_kernel +_mach_continuous_time_kernel: + mov w16, #-4 // Load the magic MCT number + svc #SWI_SYSCALL + ret + #else #error Unsupported architecture #endif diff --git a/libsyscall/wrappers/mach_approximate_time.s b/libsyscall/wrappers/mach_approximate_time.s index 7ef3336f7..f1e6ed871 100644 --- a/libsyscall/wrappers/mach_approximate_time.s +++ b/libsyscall/wrappers/mach_approximate_time.s @@ -29,7 +29,45 @@ #include <sys/appleapiopts.h> #include <machine/cpu_capabilities.h> -#if defined(__i386__) +#if defined(__arm__) + + .text + .align 2 + .globl _mach_approximate_time +_mach_approximate_time: + + movw r0, #((_COMM_PAGE_APPROX_TIME_SUPPORTED>>0)&0x0FFFF) + movt r0, #((_COMM_PAGE_APPROX_TIME_SUPPORTED>>16)&0x0FFFF) + ldrb r0, [r0] // load COMM_PAGE_APPROX_TIME_SUPPORTED + cmp r0, #1 // check if approx time is supported + + bne _mach_absolute_time // if not supported, fall through to + // absolute_time + + movw r2, #((_COMM_PAGE_APPROX_TIME>>0)&0x0FFFF) + movt r2, #((_COMM_PAGE_APPROX_TIME>>16)&0x0FFFF) + + // at this point, r2->COMM_PAGE_APPROX_TIME, which is a 64-bit value. + // Since this is a 32-bit architecture, and the commpage is readonly, + // there is no "guaranteed" atomic way to read all 64-bits with + // hardware protection. Even the ldrd instruction is not guaranteed to + // complete atomically. The solution is to use a 32-bit high/low/high + // read with a consistency check on the high bits. To further + // complicate things, reading the same location in memory back to back + // could trigger a predictive read, which would defeat the purpose of + // doing the consistency check so we insert a data memory barrier to + // prevent this. +_consistency_check: + ldr r1, [r2,#4] // load high + ldr r0, [r2] // load low + dsb // avoid predictive reads that could + // be invalid if interrupted + ldr r3, [r2,#4] // load high again + cmp r1, r3 // check that high1 == high2 + bne _consistency_check // try again if not + bx lr + +#elif defined(__i386__) .text .align 4, 0x90 diff --git a/libsyscall/wrappers/mach_continuous_time.c b/libsyscall/wrappers/mach_continuous_time.c index 4f20664f8..61b996de7 100644 --- a/libsyscall/wrappers/mach_continuous_time.c +++ b/libsyscall/wrappers/mach_continuous_time.c @@ -36,7 +36,9 @@ _mach_continuous_time_base(void) volatile uint64_t *base_ptr = (volatile uint64_t*)_COMM_PAGE_CONT_TIMEBASE; uint64_t read1, read2; read1 = *base_ptr; -#if defined(__i386__) +#if defined(__arm__) + __asm__ volatile("dsb sy" ::: "memory"); +#elif defined(__i386__) __asm__ volatile("lfence" ::: "memory"); #else #error "unsupported arch" @@ -51,6 +53,21 @@ _mach_continuous_time_base(void) #endif // 64-bit } +__attribute__((visibility("hidden"))) +kern_return_t +_mach_continuous_hwclock(uint64_t *cont_time __unused) +{ +#if defined(__arm64__) + uint8_t cont_hwclock = *((uint8_t*)_COMM_PAGE_CONT_HWCLOCK); + uint64_t timebase; + if (cont_hwclock) { + __asm__ volatile("isb\n" "mrs %0, CNTPCT_EL0" : "=r"(timebase)); + *cont_time = timebase; + return KERN_SUCCESS; + } +#endif + return KERN_NOT_SUPPORTED; +} __attribute__((visibility("hidden"))) kern_return_t @@ -63,6 +80,13 @@ _mach_continuous_time(uint64_t* absolute_time, uint64_t* cont_time) do { read1 = *base_ptr; absolute = mach_absolute_time(); +#if defined(__arm__) || defined(__arm64__) + /* + * mach_absolute_time() contains an instruction barrier which will + * prevent the speculation of read2 above this point, so we don't + * need another barrier here. + */ +#endif read2 = *base_ptr; } while (__builtin_expect((read1 != read2), 0)); @@ -76,17 +100,18 @@ uint64_t mach_continuous_time(void) { uint64_t cont_time; - _mach_continuous_time(NULL, &cont_time); + if (_mach_continuous_hwclock(&cont_time) != KERN_SUCCESS) + _mach_continuous_time(NULL, &cont_time); return cont_time; } uint64_t mach_continuous_approximate_time(void) { - /* - * No retry loop here because if we use a slightly too old timebase that's - * okay, we are approximate time anyway. - */ - volatile register uint64_t time_base = _mach_continuous_time_base(); - return time_base + mach_approximate_time(); + /* + * No retry loop here because if we use a slightly too old timebase that's + * okay, we are approximate time anyway. + */ + volatile register uint64_t time_base = _mach_continuous_time_base(); + return time_base + mach_approximate_time(); } diff --git a/libsyscall/wrappers/mach_get_times.c b/libsyscall/wrappers/mach_get_times.c index 37ddfa9fd..b078c8eb8 100644 --- a/libsyscall/wrappers/mach_get_times.c +++ b/libsyscall/wrappers/mach_get_times.c @@ -63,8 +63,12 @@ mach_get_times(uint64_t* absolute_time, uint64_t* cont_time, struct timespec *tp if (__gettimeofday_with_mach(&tv, NULL, &tbr) < 0) { return KERN_FAILURE; } else if (tbr == 0) { +#if !TARGET_OS_EMBEDDED // On an old kernel, likely chroot'ed. (remove next year) tbr = mach_absolute_time(); +#else + __builtin_trap(); +#endif } } diff --git a/libsyscall/wrappers/pid_shutdown_networking.c b/libsyscall/wrappers/pid_shutdown_networking.c new file mode 100644 index 000000000..7d96044be --- /dev/null +++ b/libsyscall/wrappers/pid_shutdown_networking.c @@ -0,0 +1,33 @@ +/* + * Copyright (c) 20017 Apple Computer, Inc. All rights reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ + +#include <sys/proc.h> + +/* + * Wrapper for pid_shutdown_sockets. + */ +int +pid_shutdown_networking(int pid, int level) +{ + return pid_shutdown_sockets(pid, level); +} diff --git a/libsyscall/wrappers/quota_obsolete.c b/libsyscall/wrappers/quota_obsolete.c index 435339b2c..aa566c0c6 100644 --- a/libsyscall/wrappers/quota_obsolete.c +++ b/libsyscall/wrappers/quota_obsolete.c @@ -25,6 +25,7 @@ #include <unistd.h> #include <TargetConditionals.h> +#if !TARGET_OS_EMBEDDED /* * system call stubs are no longer generated for these from * syscalls.master. Instead, provide simple stubs here. @@ -42,3 +43,4 @@ int setquota(void) { return kill(getpid(), SIGSYS); } +#endif /* !TARGET_OS_EMBEDDED */ diff --git a/libsyscall/wrappers/remove-counter.c b/libsyscall/wrappers/remove-counter.c index fe41f2757..f1757a654 100644 --- a/libsyscall/wrappers/remove-counter.c +++ b/libsyscall/wrappers/remove-counter.c @@ -22,6 +22,9 @@ */ #include <sys/types.h> +#if defined(__arm__) +#include <arm/arch.h> +#endif #if defined(__ppc64__) || defined(__i386__) || defined(__x86_64__) static int64_t __remove_counter = 0; @@ -31,11 +34,19 @@ static int32_t __remove_counter = 0; __uint64_t __get_remove_counter(void) { +#if defined(__arm__) && !defined(_ARM_ARCH_6) + return __remove_counter; +#else return __sync_add_and_fetch(&__remove_counter, 0); +#endif } void __inc_remove_counter(void) { +#if defined(__arm__) && !defined(_ARM_ARCH_6) + __remove_counter++; +#else __sync_add_and_fetch(&__remove_counter, 1); +#endif } diff --git a/libsyscall/wrappers/spawn/posix_spawn.c b/libsyscall/wrappers/spawn/posix_spawn.c index be3e94cea..69002dc0b 100644 --- a/libsyscall/wrappers/spawn/posix_spawn.c +++ b/libsyscall/wrappers/spawn/posix_spawn.c @@ -1331,6 +1331,41 @@ posix_spawnattr_getcpumonitor(posix_spawnattr_t * __restrict attr, return (0); } +#if TARGET_OS_EMBEDDED +/* + * posix_spawnattr_setjetsam + * + * Description: Set jetsam attributes for the spawn attribute object + * referred to by 'attr'. + * + * Parameters: flags The flags value to set + * priority Relative jetsam priority + * memlimit Value in megabytes; a memory footprint + * above this level may result in termination. + * Implies both active and inactive limits. + * + * Returns: 0 Success + * + * Note: to be deprecated (not available on desktop) + * + */ +int +posix_spawnattr_setjetsam(posix_spawnattr_t * __restrict attr, + short flags, int priority, int memlimit) +{ + short flags_ext = flags; + + if (flags & POSIX_SPAWN_JETSAM_MEMLIMIT_FATAL) { + flags_ext |= POSIX_SPAWN_JETSAM_MEMLIMIT_ACTIVE_FATAL; + flags_ext |= POSIX_SPAWN_JETSAM_MEMLIMIT_INACTIVE_FATAL; + } else { + flags_ext &= ~POSIX_SPAWN_JETSAM_MEMLIMIT_ACTIVE_FATAL; + flags_ext &= ~POSIX_SPAWN_JETSAM_MEMLIMIT_INACTIVE_FATAL; + } + + return (posix_spawnattr_setjetsam_ext(attr, flags_ext, priority, memlimit, memlimit)); +} +#endif /* TARGET_OS_EMBEDDED */ /* * posix_spawnattr_setjetsam_ext diff --git a/libsyscall/wrappers/spawn/spawn_private.h b/libsyscall/wrappers/spawn/spawn_private.h index ec7f50fb6..bebd58e60 100644 --- a/libsyscall/wrappers/spawn/spawn_private.h +++ b/libsyscall/wrappers/spawn/spawn_private.h @@ -40,6 +40,10 @@ int posix_spawnattr_setcpumonitor(posix_spawnattr_t * __restrict, uint64_t, uint int posix_spawnattr_getcpumonitor(posix_spawnattr_t * __restrict, uint64_t *, uint64_t *) __OSX_AVAILABLE_STARTING(__MAC_10_8, __IPHONE_6_0); int posix_spawnattr_setcpumonitor_default(posix_spawnattr_t * __restrict) __OSX_AVAILABLE_STARTING(__MAC_10_9, __IPHONE_6_0); +#if TARGET_OS_EMBEDDED +int posix_spawnattr_setjetsam(posix_spawnattr_t * __restrict attr, + short flags, int priority, int memlimit) __OSX_AVAILABLE_STARTING(__MAC_NA, __IPHONE_5_0); +#endif /* TARGET_OS_EMBEDDED */ int posix_spawnattr_setjetsam_ext(posix_spawnattr_t * __restrict attr, short flags, int priority, int memlimit_active, int memlimit_inactive) __OSX_AVAILABLE_STARTING(__MAC_10_11, __IPHONE_9_0); diff --git a/libsyscall/wrappers/thread_register_state.c b/libsyscall/wrappers/thread_register_state.c index 2fa478328..e1181d251 100644 --- a/libsyscall/wrappers/thread_register_state.c +++ b/libsyscall/wrappers/thread_register_state.c @@ -44,6 +44,14 @@ thread_get_register_pointer_values(thread_t thread, uintptr_t *sp, size_t *lengt x86_thread_state64_t state = {}; thread_state_flavor_t flavor = x86_THREAD_STATE64; mach_msg_type_number_t count = x86_THREAD_STATE64_COUNT; +#elif defined(__arm__) + arm_thread_state_t state = {}; + thread_state_flavor_t flavor = ARM_THREAD_STATE; + mach_msg_type_number_t count = ARM_THREAD_STATE_COUNT; +#elif defined(__arm64__) + arm_thread_state64_t state = {}; + thread_state_flavor_t flavor = ARM_THREAD_STATE64; + mach_msg_type_number_t count = ARM_THREAD_STATE64_COUNT; #else #error thread_get_register_pointer_values not defined for this architecture #endif @@ -76,7 +84,12 @@ thread_get_register_pointer_values(thread_t thread, uintptr_t *sp, size_t *lengt push_register_value(state.__esi); push_register_value(state.__ebp); #elif defined(__x86_64__) - if (sp) *sp = state.__rsp - 128 /* redzone */; + if (sp) { + if (state.__rsp > 128) + *sp = state.__rsp - 128 /* redzone */; + else + *sp = 0; + } push_register_value(state.__rip); @@ -94,6 +107,29 @@ thread_get_register_pointer_values(thread_t thread, uintptr_t *sp, size_t *lengt push_register_value(state.__r13); push_register_value(state.__r14); push_register_value(state.__r15); +#elif defined(__arm__) + if (sp) *sp = state.__sp; + + push_register_value(state.__pc); + push_register_value(state.__lr); + + for (int i = 0; i < 13; i++){ + push_register_value(state.__r[i]); + } +#elif defined(__arm64__) + if (sp) { + if (state.__sp > 128) + *sp = state.__sp - 128 /* redzone */; + else + *sp = 0; + } + + push_register_value(state.__pc); + push_register_value(state.__lr); + + for (int i = 0; i < 29; i++){ + push_register_value(state.__x[i]); + } #else #error thread_get_register_pointer_values not defined for this architecture #endif diff --git a/libsyscall/wrappers/utimensat.c b/libsyscall/wrappers/utimensat.c new file mode 100644 index 000000000..6deaf45a3 --- /dev/null +++ b/libsyscall/wrappers/utimensat.c @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2006, 2017 Apple Computer, Inc. All rights reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ + +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/attr.h> +#include <sys/time.h> +#include <sys/fcntl.h> +#include <unistd.h> +#include <strings.h> + +extern int __gettimeofday(struct timeval *, struct timezone *); +extern int __commpage_gettimeofday(struct timeval *); + +static struct timespec times_now[2] = { + { .tv_nsec = UTIME_NOW }, + { .tv_nsec = UTIME_NOW } +}; + +/* + * Resolve any UTIME_NOW or UTIME_OMIT and return the attributes buffer and + * attributes to pass. Assumes times_in is writable. + */ +static int +prepare_times_array_and_attrs(struct timespec times_in[2], + struct timespec times_out[2], size_t *times_out_size) +{ + if (times_in[0].tv_nsec == UTIME_OMIT && + times_in[1].tv_nsec == UTIME_OMIT) { + return 0; + } + + if (times_in[0].tv_nsec == UTIME_NOW || + times_in[1].tv_nsec == UTIME_NOW) { + struct timespec now = {}; + { + /* + * TODO: Replace with nanosecond time when available + */ + struct timeval tv; + if (__commpage_gettimeofday(&tv) != 0) { + __gettimeofday(&tv, NULL); + } + TIMEVAL_TO_TIMESPEC(&tv, &now); + } + + if (times_in[0].tv_nsec == UTIME_NOW) { + times_in[0] = now; + } + if (times_in[1].tv_nsec == UTIME_NOW) { + times_in[1] = now; + } + } + + int attrs = 0; + *times_out_size = 0; + struct timespec *times_cursor = times_out; + if (times_in[1].tv_nsec != UTIME_OMIT) { + attrs |= ATTR_CMN_MODTIME; + *times_cursor++ = times_in[1]; + *times_out_size += sizeof(struct timespec); + } + if (times_in[0].tv_nsec != UTIME_OMIT) { + attrs |= ATTR_CMN_ACCTIME; + *times_cursor = times_in[0]; + *times_out_size += sizeof(struct timespec); + } + return attrs; +} + +int +futimens(int fd, const struct timespec _times_in[2]) +{ + struct timespec times_in[2]; + if (_times_in != NULL) { + memcpy(×_in, _times_in, sizeof(times_in)); + } else { + memcpy(×_in, times_now, sizeof(times_in)); + } + + size_t attrbuf_size = 0; + struct timespec times_out[2] = {}; + struct attrlist a = { + .bitmapcount = ATTR_BIT_MAP_COUNT + }; + a.commonattr = prepare_times_array_and_attrs(times_in, times_out, &attrbuf_size); + + return fsetattrlist(fd, &a, ×_out, attrbuf_size, 0); +} + +int +utimensat(int fd, const char *path, const struct timespec _times_in[2], int flags) +{ + struct timespec times_in[2]; + if (_times_in != NULL) { + memcpy(×_in, _times_in, sizeof(times_in)); + } else { + memcpy(×_in, times_now, sizeof(times_in)); + } + + size_t attrbuf_size = 0; + struct timespec times_out[2] = {}; + struct attrlist a = { + .bitmapcount = ATTR_BIT_MAP_COUNT + }; + a.commonattr = prepare_times_array_and_attrs(times_in, times_out, &attrbuf_size); + + int flags_out = 0; + if (flags & AT_SYMLINK_NOFOLLOW) { + flags_out |= FSOPT_NOFOLLOW; + } + + return setattrlistat(fd, path, &a, ×_out, attrbuf_size, flags_out); +} diff --git a/libsyscall/wrappers/varargs_wrappers.s b/libsyscall/wrappers/varargs_wrappers.s index bc6d6c3a4..6a22a5395 100644 --- a/libsyscall/wrappers/varargs_wrappers.s +++ b/libsyscall/wrappers/varargs_wrappers.s @@ -21,3 +21,127 @@ * @APPLE_LICENSE_HEADER_END@ */ +#ifdef __arm64__ + +#include "../custom/SYS.h" +#include <mach/arm64/asm.h> + +/* + * Stubs are to handle the ARM64 ABI for variadic functions' + * not matching the ABI used by the system call handler. + */ + +/* + * sem_t* sem_open(const char *name, int oflag, ...); + * sem_t* __sem_open(const char *name, int oflag, int mode, int value); + */ +MI_ENTRY_POINT(_sem_open) + PUSH_FRAME + ldp x2, x3, [fp, #16] + MI_CALL_EXTERNAL(___sem_open) + POP_FRAME + ret + +/* + * int open(const char *name, int oflag, ...); + * int __open(const char *name, int oflag, int mode, int value); + */ +MI_ENTRY_POINT(_open) + PUSH_FRAME + ldr x2, [fp, #16] + MI_CALL_EXTERNAL(___open) + POP_FRAME + ret + +/* + * int open_nocancel(const char *name, int oflag, ...); + * int __open_nocancel(const char *name, int oflag, int mode); + */ +MI_ENTRY_POINT(_open$NOCANCEL) + PUSH_FRAME + ldr x2, [fp, #16] + MI_CALL_EXTERNAL(___open_nocancel) + POP_FRAME + ret + +/* + * int openat(int fd,const char *name, int oflag, ...); + * int __openat(int fd, const char *name, int oflag, int mode, int value); + */ +MI_ENTRY_POINT(_openat) + PUSH_FRAME + ldr x3, [fp, #16] + MI_CALL_EXTERNAL(___openat) + POP_FRAME + ret + +/* + * int openat_nocancel(int fd, const char *name, int oflag, ...); + * int __openat_nocancel(int fd, const char *name, int oflag, int mode); + */ +MI_ENTRY_POINT(_openat$NOCANCEL) + PUSH_FRAME + ldr x3, [fp, #16] + MI_CALL_EXTERNAL(___openat_nocancel) + POP_FRAME + ret + +/* + * int shm_open(const char *, int, ...); + * int __shm_open(const char*, int oflag, int mode); + */ +MI_ENTRY_POINT(_shm_open) + PUSH_FRAME + ldr x2, [fp, #16] + MI_CALL_EXTERNAL(___shm_open) + POP_FRAME + ret + +/* + * int msgsys(int, ...); + * int __msgsys(int which, int a2, int a3, int a4, int a5); + */ +MI_ENTRY_POINT(_msgsys) + PUSH_FRAME + ldp x1, x2, [fp, #16] + ldp x3, x4, [fp, #32] + MI_CALL_EXTERNAL(___msgsys) + POP_FRAME + ret + +/* + * int semsys(int, ...); + * int __semsys(int which, int a2, int a3, int a4, int a5); + */ +MI_ENTRY_POINT(_semsys) + PUSH_FRAME + ldp x1, x2, [fp, #16] + ldp x3, x4, [fp, #32] + MI_CALL_EXTERNAL(___semsys) + POP_FRAME + ret + +/* + * int semctl(int, int, int, ...); + * int __semctl(int semid, int semnum, int cmd, semun_t arg); + */ + MI_ENTRY_POINT(_semctl) + PUSH_FRAME + ldr x3, [fp, #16] + MI_CALL_EXTERNAL(___semctl) + POP_FRAME + ret + +/* + * int shmsys(int, ...); + * int __shmsys(int which, int a2, int a3, int a4); + */ + MI_ENTRY_POINT(_shmsys) + PUSH_FRAME + ldp x1, x2, [fp, #16] + ldr x3, [fp, #32] + MI_CALL_EXTERNAL(___shmsys) + POP_FRAME + ret + +#endif /* defined(__arm64__) */ diff --git a/libsyscall/wrappers/work_interval.c b/libsyscall/wrappers/work_interval.c index 29dd2ad61..96ca9ef83 100644 --- a/libsyscall/wrappers/work_interval.c +++ b/libsyscall/wrappers/work_interval.c @@ -23,26 +23,41 @@ #include <sys/cdefs.h> #include <sys/types.h> #include <sys/work_interval.h> + +#include <mach/mach.h> #include <mach/mach_time.h> +#include <mach/port.h> + #include <sys/errno.h> #include <stdlib.h> struct work_interval { uint64_t thread_id; uint64_t work_interval_id; + uint32_t create_flags; + mach_port_t wi_port; }; extern uint64_t __thread_selfid(void); -/* Create a new work interval handle (currently for the current thread only). Flags is unused */ +/* Create a new work interval handle (currently for the current thread only). */ int -work_interval_create(work_interval_t *interval_handle, uint32_t flags __unused) +work_interval_create(work_interval_t *interval_handle, uint32_t create_flags) { int ret; - uint64_t work_interval_id; work_interval_t handle; - ret = __work_interval_ctl(WORK_INTERVAL_OPERATION_CREATE, 0, &work_interval_id, sizeof(work_interval_id)); + if (interval_handle == NULL) { + errno = EINVAL; + return -1; + } + + struct work_interval_create_params create_params = { + .wicp_create_flags = create_flags, + }; + + ret = __work_interval_ctl(WORK_INTERVAL_OPERATION_CREATE2, 0, + &create_params, sizeof(create_params)); if (ret == -1) { return ret; } @@ -54,14 +69,18 @@ work_interval_create(work_interval_t *interval_handle, uint32_t flags __unused) } handle->thread_id = __thread_selfid(); - handle->work_interval_id = work_interval_id; + handle->work_interval_id = create_params.wicp_id; + handle->create_flags = create_params.wicp_create_flags; + handle->wi_port = create_params.wicp_port; *interval_handle = handle; return 0; } int -work_interval_notify(work_interval_t interval_handle, uint64_t start, uint64_t finish, uint64_t deadline, uint64_t next_start, uint32_t flags) +work_interval_notify(work_interval_t interval_handle, uint64_t start, + uint64_t finish, uint64_t deadline, uint64_t next_start, + uint32_t notify_flags) { int ret; uint64_t work_interval_id; @@ -70,8 +89,7 @@ work_interval_notify(work_interval_t interval_handle, uint64_t start, uint64_t f .finish = finish, .deadline = deadline, .next_start = next_start, - .flags = flags, - .unused1 = 0 + .notify_flags = notify_flags }; if (interval_handle == NULL) { @@ -79,35 +97,156 @@ work_interval_notify(work_interval_t interval_handle, uint64_t start, uint64_t f return -1; } + notification.create_flags = interval_handle->create_flags; work_interval_id = interval_handle->work_interval_id; - ret = __work_interval_ctl(WORK_INTERVAL_OPERATION_NOTIFY, work_interval_id, ¬ification, sizeof(notification)); + ret = __work_interval_ctl(WORK_INTERVAL_OPERATION_NOTIFY, work_interval_id, + ¬ification, sizeof(notification)); return ret; } int -work_interval_notify_simple(work_interval_t interval_handle, uint64_t start, uint64_t deadline, uint64_t next_start) +work_interval_notify_simple(work_interval_t interval_handle, uint64_t start, + uint64_t deadline, uint64_t next_start) { - return work_interval_notify(interval_handle, start, mach_absolute_time(), deadline, next_start, 0); + return work_interval_notify(interval_handle, start, mach_absolute_time(), + deadline, next_start, 0); } int work_interval_destroy(work_interval_t interval_handle) { - int ret, saved_errno; - uint64_t work_interval_id; + if (interval_handle == NULL) { + errno = EINVAL; + return -1; + } + + if (interval_handle->create_flags & WORK_INTERVAL_FLAG_JOINABLE) { + mach_port_t wi_port = interval_handle->wi_port; + + /* + * A joinable work interval's lifetime is tied to the port lifetime. + * When the last port reference is destroyed, the work interval + * is destroyed via no-senders notification. + * + * Note however that after destroy it can no longer be notified + * because the userspace token is gone. + * + * Additionally, this function does not cause the thread to un-join + * the interval. + */ + kern_return_t kr = mach_port_deallocate(mach_task_self(), wi_port); + + if (kr != KERN_SUCCESS) { + /* + * If the deallocate fails, then someone got their port + * lifecycle wrong and over-released a port right. + * + * Return an error so the client can assert on this, + * and still find the port name in the interval handle. + */ + errno = EINVAL; + return -1; + } + + interval_handle->wi_port = MACH_PORT_NULL; + interval_handle->work_interval_id = 0; + + free(interval_handle); + return 0; + } else { + uint64_t work_interval_id = interval_handle->work_interval_id; + + int ret = __work_interval_ctl(WORK_INTERVAL_OPERATION_DESTROY, + work_interval_id, NULL, 0); + + interval_handle->work_interval_id = 0; + + int saved_errno = errno; + free(interval_handle); + errno = saved_errno; + return ret; + } +} +int +work_interval_join(work_interval_t interval_handle) +{ if (interval_handle == NULL) { errno = EINVAL; return -1; } - work_interval_id = interval_handle->work_interval_id; + if ((interval_handle->create_flags & WORK_INTERVAL_FLAG_JOINABLE) == 0) { + errno = EINVAL; + return -1; + } - ret = __work_interval_ctl(WORK_INTERVAL_OPERATION_DESTROY, work_interval_id, NULL, 0); - saved_errno = errno; - free(interval_handle); - errno = saved_errno; + mach_port_t wi_port = interval_handle->wi_port; - return ret; + if (!MACH_PORT_VALID(wi_port)) { + errno = EINVAL; + return -1; + } + + return work_interval_join_port(wi_port); } + +int +work_interval_join_port(mach_port_t port) +{ + if (port == MACH_PORT_NULL) { + errno = EINVAL; + return -1; + } + + return __work_interval_ctl(WORK_INTERVAL_OPERATION_JOIN, + (uint64_t)port, NULL, 0); +} + +int +work_interval_leave(void) +{ + return __work_interval_ctl(WORK_INTERVAL_OPERATION_JOIN, + (uint64_t)MACH_PORT_NULL, NULL, 0); +} + +int +work_interval_copy_port(work_interval_t interval_handle, mach_port_t *port) +{ + if (port == NULL) { + errno = EINVAL; + return -1; + } + + if (interval_handle == NULL) { + *port = MACH_PORT_NULL; + errno = EINVAL; + return -1; + } + + if ((interval_handle->create_flags & WORK_INTERVAL_FLAG_JOINABLE) == 0) { + *port = MACH_PORT_NULL; + errno = EINVAL; + return -1; + } + + mach_port_t wi_port = interval_handle->wi_port; + + kern_return_t kr = mach_port_mod_refs(mach_task_self(), wi_port, + MACH_PORT_RIGHT_SEND, 1); + + if (kr != KERN_SUCCESS) { + *port = MACH_PORT_NULL; + errno = EINVAL; + return -1; + } + + *port = wi_port; + + return 0; +} + + + + diff --git a/libsyscall/xcodescripts/create-syscalls.pl b/libsyscall/xcodescripts/create-syscalls.pl index 54514f454..81dfc8a8a 100755 --- a/libsyscall/xcodescripts/create-syscalls.pl +++ b/libsyscall/xcodescripts/create-syscalls.pl @@ -222,6 +222,7 @@ sub checkForCustomStubs { foreach my $subarch (@Architectures) { (my $arch = $subarch) =~ s/arm(v.*)/arm/; $arch =~ s/x86_64(.*)/x86_64/; + $arch =~ s/arm64(.*)/arm64/; $$sym{aliases}{$arch} = [] unless $$sym{aliases}{$arch}; push(@{$$sym{aliases}{$arch}}, $$sym{asm_sym}); } @@ -244,6 +245,7 @@ sub readAliases { for my $arch (@Architectures) { (my $new_arch = $arch) =~ s/arm(v.*)/arm/g; $new_arch =~ s/x86_64(.*)/x86_64/g; + $new_arch =~ s/arm64(.*)/arm64/g; push(@a, $new_arch) unless grep { $_ eq $new_arch } @a; } @@ -302,6 +304,7 @@ sub writeStubForSymbol { for my $subarch (@Architectures) { (my $arch = $subarch) =~ s/arm(v.*)/arm/; $arch =~ s/x86_64(.*)/x86_64/; + $arch =~ s/arm64(.*)/arm64/; push(@conditions, "defined(__${arch}__)") unless grep { $_ eq $arch } @{$$symbol{except}}; } @@ -334,6 +337,7 @@ sub writeAliasesForSymbol { foreach my $subarch (@Architectures) { (my $arch = $subarch) =~ s/arm(v.*)/arm/; $arch =~ s/x86_64(.*)/x86_64/; + $arch =~ s/arm64(.*)/arm64/; next unless scalar($$symbol{aliases}{$arch}); diff --git a/libsyscall/xcodescripts/mach_install_mig.sh b/libsyscall/xcodescripts/mach_install_mig.sh index 7457fa195..94cc8dbb7 100755 --- a/libsyscall/xcodescripts/mach_install_mig.sh +++ b/libsyscall/xcodescripts/mach_install_mig.sh @@ -42,17 +42,17 @@ MACH_HEADER_DST="$BUILT_PRODUCTS_DIR/mig_hdr/include/mach" # from old Libsystem makefiles MACHINE_ARCH=`echo $ARCHS | cut -d' ' -f 1` -if [[ ( "$MACHINE_ARCH" = "arm64" || "$MACHINE_ARCH" = "x86_64" || "$MACHINE_ARCH" = "x86_64h" ) && `echo $ARCHS | wc -w` -gt 1 ]] +if [[ ( "$MACHINE_ARCH" =~ ^"arm64" || "$MACHINE_ARCH" =~ ^"x86_64" ) && `echo $ARCHS | wc -w` -gt 1 ]] then # MACHINE_ARCH needs to be a 32-bit arch to generate vm_map_internal.h correctly. MACHINE_ARCH=`echo $ARCHS | cut -d' ' -f 2` - if [[ ( "$MACHINE_ARCH" = "arm64" || "$MACHINE_ARCH" = "x86_64" || "$MACHINE_ARCH" = "x86_64h" ) && `echo $ARCHS | wc -w` -gt 1 ]] + if [[ ( "$MACHINE_ARCH" =~ ^"arm64" || "$MACHINE_ARCH" =~ ^"x86_64" ) && `echo $ARCHS | wc -w` -gt 1 ]] then # MACHINE_ARCH needs to be a 32-bit arch to generate vm_map_internal.h correctly. MACHINE_ARCH=`echo $ARCHS | cut -d' ' -f 3` fi fi -if [[ ( "$MACHINE_ARCH" = "arm64" ) ]] +if [[ ( "$MACHINE_ARCH" =~ ^"arm64" ) ]] then # MACHINE_ARCH *really* needs to be a 32-bit arch to generate vm_map_internal.h correctly, even if there are no 32-bit targets. MACHINE_ARCH="armv7" @@ -88,7 +88,7 @@ MIGS_PRIVATE="" MIGS_DUAL_PUBLIC_PRIVATE="" -if ( echo {iphone,tv,appletv,watch}{os,simulator} iphone{osnano,nanosimulator} | grep -wFq "$PLATFORM_NAME" ) +if ( echo {iphone,tv,appletv,watch,bridge}{os,simulator} iphone{osnano,nanosimulator} | grep -wFq "$PLATFORM_NAME" ) then MIGS_PRIVATE="mach_vm.defs" else diff --git a/makedefs/MakeInc.cmd b/makedefs/MakeInc.cmd index 0619a3324..a70a2d815 100644 --- a/makedefs/MakeInc.cmd +++ b/makedefs/MakeInc.cmd @@ -118,7 +118,7 @@ endif # # Platform options # -SUPPORTED_EMBEDDED_PLATFORMS := iPhoneOS iPhoneOSNano tvOS AppleTVOS WatchOS +SUPPORTED_EMBEDDED_PLATFORMS := iPhoneOS iPhoneOSNano tvOS AppleTVOS WatchOS BridgeOS SUPPORTED_SIMULATOR_PLATFORMS := iPhoneSimulator iPhoneNanoSimulator tvSimulator AppleTVSimulator WatchSimulator SUPPORTED_PLATFORMS := MacOSX $(SUPPORTED_SIMULATOR_PLATFORMS) $(SUPPORTED_EMBEDDED_PLATFORMS) @@ -127,7 +127,7 @@ ifneq ($(filter $(SUPPORTED_EMBEDDED_PLATFORMS),$(PLATFORM)),) ifeq ($(EMBEDDED_DEVICE_MAP),) export EMBEDDED_DEVICE_MAP := $(shell $(XCRUN) -sdk $(SDKROOT) -find embedded_device_map) endif -EDM_DBPATH = $(PLATFORMPATH)/usr/local/standalone/firmware/device_map.db +EDM_DBPATH ?= $(PLATFORMPATH)/usr/local/standalone/firmware/device_map.db endif # Scripts or tools we build ourselves diff --git a/makedefs/MakeInc.def b/makedefs/MakeInc.def index 9edf2627a..4eca357c9 100644 --- a/makedefs/MakeInc.def +++ b/makedefs/MakeInc.def @@ -1,6 +1,6 @@ # -*- mode: makefile;-*- # -# Copyright (C) 1999-2016 Apple Inc. All rights reserved. +# Copyright (C) 1999-2017 Apple Inc. All rights reserved. # # MakeInc.def contains global definitions for building, # linking, and installing files. @@ -14,7 +14,7 @@ SUPPORTED_ARCH_CONFIGS := X86_64 X86_64H # # Kernel Configuration options # -SUPPORTED_KERNEL_CONFIGS = RELEASE DEVELOPMENT DEBUG PROFILE +SUPPORTED_KERNEL_CONFIGS = RELEASE DEVELOPMENT DEBUG PROFILE KASAN # # Machine Configuration options @@ -43,7 +43,7 @@ endif # # Component List # -COMPONENT_LIST = osfmk bsd libkern iokit pexpert libsa security +COMPONENT_LIST = osfmk bsd libkern iokit pexpert libsa security san COMPONENT = $(if $(word 2,$(subst /, ,$(RELATIVE_SOURCE_PATH))),$(word 2,$(subst /, ,$(RELATIVE_SOURCE_PATH))),$(firstword $(subst /, ,$(RELATIVE_SOURCE_PATH)))) COMPONENT_IMPORT_LIST = $(filter-out $(COMPONENT),$(COMPONENT_LIST)) @@ -59,6 +59,8 @@ else ifeq ($(PLATFORM),tvOS) DEPLOYMENT_TARGET_FLAGS = -mtvos-version-min=$(SDKVERSION) else ifeq ($(PLATFORM),AppleTVOS) DEPLOYMENT_TARGET_FLAGS = -mtvos-version-min=$(SDKVERSION) +else ifeq ($(PLATFORM),BridgeOS) + DEPLOYMENT_TARGET_FLAGS = -mbridgeos-version-min=$(SDKVERSION) else ifneq ($(filter $(SUPPORTED_EMBEDDED_PLATFORMS),$(PLATFORM)),) DEPLOYMENT_TARGET_FLAGS = -miphoneos-version-min=$(SDKVERSION) else ifneq ($(filter $(SUPPORTED_SIMULATOR_PLATFORMS),$(PLATFORM)),) @@ -89,11 +91,29 @@ GENASSYM_KCC = $(CC) # Compiler warning flags # -CWARNFLAGS_STD = \ - -Weverything -Werror -Wextra -Wstrict-prototypes \ - -Wmissing-prototypes -Wpointer-arith -Wreturn-type -Wcast-qual \ - -Wwrite-strings -Wswitch -Wshadow -Wcast-align -Wchar-subscripts \ - -Winline -Wnested-externs -Wredundant-decls -Wextra-tokens \ +USE_WERROR := 1 +ifneq ($(BUILD_WERROR),) +USE_WERROR := $(BUILD_WERROR) +endif + +ifeq ($(USE_WERROR),1) +WERROR := -Werror +endif + +# Shared C/C++ warning flags +WARNFLAGS_STD := \ + -Weverything \ + -Wextra \ + $(WERROR) \ + -Wpointer-arith \ + -Wreturn-type \ + -Wcast-qual \ + -Wwrite-strings \ + -Wswitch \ + -Wcast-align \ + -Wchar-subscripts \ + -Wredundant-decls \ + -Wextra-tokens \ -Wunreachable-code \ -Wno-assign-enum \ -Wno-bad-function-cast \ @@ -101,6 +121,15 @@ CWARNFLAGS_STD = \ -Wno-c++-compat \ -Wno-conditional-uninitialized \ -Wno-conversion \ + -Wnull-conversion \ + -Wstring-conversion \ + -Wliteral-conversion \ + -Wnon-literal-null-conversion \ + -Wint-conversion \ + -Wenum-conversion \ + -Wfloat-conversion \ + -Wconstant-conversion \ + -Wpointer-bool-conversion \ -Wno-covered-switch-default \ -Wno-disabled-macro-expansion \ -Wno-documentation-unknown-command \ @@ -122,6 +151,14 @@ CWARNFLAGS_STD = \ -Wno-vla \ -Wno-zero-length-array +CWARNFLAGS_STD = \ + $(WARNFLAGS_STD) \ + -Wstrict-prototypes \ + -Wmissing-prototypes \ + -Wshadow \ + -Winline \ + -Wnested-externs + # Can be overridden in Makefile.template or Makefile.$arch export CWARNFLAGS ?= $(CWARNFLAGS_STD) @@ -130,40 +167,11 @@ $(1)_CWARNFLAGS_ADD += $2 endef CXXWARNFLAGS_STD = \ - -Weverything -Werror -Wextra -Wpointer-arith -Wreturn-type \ - -Wcast-qual -Wwrite-strings -Wswitch -Wcast-align -Wchar-subscripts \ - -Wredundant-decls -Wextra-tokens \ - -Wunreachable-code \ - -Wno-assign-enum \ - -Wno-bad-function-cast \ - -Wno-c++98-compat \ + $(WARNFLAGS_STD) \ -Wno-c++98-compat-pedantic \ - -Wno-c++-compat \ - -Wno-conditional-uninitialized \ - -Wno-conversion \ - -Wno-covered-switch-default \ - -Wno-disabled-macro-expansion \ - -Wno-documentation-unknown-command \ -Wno-exit-time-destructors \ - -Wno-format-non-iso \ - -Wno-format-nonliteral \ -Wno-global-constructors \ - -Wno-reserved-id-macro \ - -Wno-language-extension-token \ - -Wno-missing-variable-declarations \ - -Wno-old-style-cast \ - -Wno-packed \ - -Wno-padded \ - -Wno-partial-availability \ - -Wno-pedantic \ - -Wno-shift-sign-overflow \ - -Wno-switch-enum \ - -Wno-undef \ - -Wno-unused-macros \ - -Wno-used-but-marked-unused \ - -Wno-variadic-macros \ - -Wno-vla \ - -Wno-zero-length-array + -Wno-old-style-cast # overloaded-virtual warnings are non-fatal (9000888) CXXWARNFLAGS_STD += -Wno-error=overloaded-virtual @@ -186,8 +194,8 @@ ARCH_FLAGS_X86_64H = -arch x86_64h # # Default CFLAGS # -ifdef RC_CFLAGS -OTHER_CFLAGS = $(subst $(addprefix -arch ,$(RC_ARCHS)),,$(RC_CFLAGS)) +ifdef RC_NONARCH_CFLAGS +OTHER_CFLAGS = $(RC_NONARCH_CLFAGS) endif # @@ -198,7 +206,7 @@ DSYMKGMACROSDIR = Contents/Resources DSYMLLDBMACROSDIR = Contents/Resources/Python DSYMDWARFDIR = Contents/Resources/DWARF -DEBUG_CFLAGS := -gdwarf-2 +DEBUG_CFLAGS := -g BUILD_DSYM := 1 # @@ -212,6 +220,7 @@ CFLAGS_GEN = $(DEBUG_CFLAGS) -nostdinc \ CFLAGS_RELEASE = CFLAGS_DEVELOPMENT = CFLAGS_DEBUG = +CFLAGS_KASAN = $(CFLAGS_DEVELOPMENT) CFLAGS_PROFILE = -pg CFLAGS_X86_64 = -Dx86_64 -DX86_64 -D__X86_64__ -DLP64 \ @@ -222,12 +231,14 @@ CFLAGS_X86_64H = $(CFLAGS_X86_64) CFLAGS_RELEASEX86_64 = -O2 CFLAGS_DEVELOPMENTX86_64 = -O2 +CFLAGS_KASANX86_64 = $(CFLAGS_DEVELOPMENTX86_64) # No space optimization for the DEBUG kernel for the benefit of gdb: CFLAGS_DEBUGX86_64 = -O0 CFLAGS_PROFILEX86_64 = -O2 CFLAGS_RELEASEX86_64H = -O2 CFLAGS_DEVELOPMENTX86_64H = -O2 +CFLAGS_KASANX86_64H = $(CFLAGS_DEVELOPMENTX86_64H) # No space optimization for the DEBUG kernel for the benefit of gdb: CFLAGS_DEBUGX86_64H = -O0 CFLAGS_PROFILEX86_64H = -O2 @@ -238,6 +249,30 @@ CFLAGS_DEBUGARM = -O0 CFLAGS_PROFILEARM = -O2 + +# +# KASAN support +# + + +ifeq ($(CURRENT_KERNEL_CONFIG),KASAN) +KASAN = 1 +endif + +ifeq ($(KASAN),1) + +BUILD_LTO = 0 +KASAN_SHIFT_X86_64=0xdffffe1000000000 +KASAN_SHIFT_X86_64H=$(KASAN_SHIFT_X86_64) +KASAN_SHIFT=$($(addsuffix $(CURRENT_ARCH_CONFIG),KASAN_SHIFT_)) +KASAN_BLACKLIST=$(OBJROOT)/san/kasan-blacklist-$(CURRENT_ARCH_CONFIG_LC) +CFLAGS_GEN += -DKASAN=1 -DKASAN_SHIFT=$(KASAN_SHIFT) -fsanitize=address \ + -mllvm -asan-globals-live-support \ + -mllvm -asan-mapping-offset=$(KASAN_SHIFT) \ + -fsanitize-blacklist=$(KASAN_BLACKLIST) + +endif + CFLAGS = $(CFLAGS_GEN) \ $($(addsuffix $(CURRENT_MACHINE_CONFIG),MACHINE_FLAGS_$(CURRENT_ARCH_CONFIG)_)) \ $($(addsuffix $(CURRENT_ARCH_CONFIG),ARCH_FLAGS_)) \ @@ -273,6 +308,7 @@ SFLAGS_GEN = -D__ASSEMBLER__ -DASSEMBLER $(OTHER_CFLAGS) SFLAGS_RELEASE = SFLAGS_DEVELOPMENT = +SFLAGS_KASAN = $(SFLAGS_DEVELOPMENT) -DKASAN=1 SFLAGS_DEBUG = SFLAGS_PROFILE = @@ -323,7 +359,8 @@ LDFLAGS_KERNEL_GEN = \ LDFLAGS_KERNEL_SDK = -L$(SDKROOT)/usr/local/lib/kernel -lfirehose_kernel LDFLAGS_KERNEL_RELEASE = -LDFLAGS_KERNEL_DEVELOPMENT = +LDFLAGS_KERNEL_DEVELOPMENT = +LDFLAGS_KERNEL_KASAN = $(LDFLAGS_KERNEL_DEVELOPMENT) LDFLAGS_KERNEL_DEBUG = LDFLAGS_KERNEL_PROFILE = @@ -362,17 +399,29 @@ LDFLAGS_KERNEL_RELEASEX86_64 = \ -Wl,-no_zero_fill_sections \ $(LDFLAGS_NOSTRIP_FLAG) +ifeq ($(KASAN),1) +LDFLAGS_KERNEL_RELEASEX86_64 += \ + -Wl,-sectalign,__HIB,__cstring,0x1000 \ + -Wl,-sectalign,__HIB,__asan_globals,0x1000 \ + -Wl,-sectalign,__HIB,__asan_liveness,0x1000 \ + -Wl,-sectalign,__HIB,__mod_term_func,0x1000 \ + -Wl,-rename_section,__HIB,__mod_init_func,__NULL,__mod_init_func \ + -Wl,-rename_section,__HIB,__eh_frame,__NULL,__eh_frame +endif + # Define KERNEL_BASE_OFFSET so known at compile time: CFLAGS_X86_64 += -DKERNEL_BASE_OFFSET=$(KERNEL_BASE_OFFSET) CFLAGS_X86_64H += -DKERNEL_BASE_OFFSET=$(KERNEL_BASE_OFFSET) LDFLAGS_KERNEL_DEBUGX86_64 = $(LDFLAGS_KERNEL_RELEASEX86_64) LDFLAGS_KERNEL_DEVELOPMENTX86_64 = $(LDFLAGS_KERNEL_RELEASEX86_64) +LDFLAGS_KERNEL_KASANX86_64 = $(LDFLAGS_KERNEL_RELEASEX86_64) LDFLAGS_KERNEL_PROFILEX86_64 = $(LDFLAGS_KERNEL_RELEASEX86_64) LDFLAGS_KERNEL_RELEASEX86_64H = $(LDFLAGS_KERNEL_RELEASEX86_64) LDFLAGS_KERNEL_DEBUGX86_64H = $(LDFLAGS_KERNEL_RELEASEX86_64H) LDFLAGS_KERNEL_DEVELOPMENTX86_64H = $(LDFLAGS_KERNEL_RELEASEX86_64H) +LDFLAGS_KERNEL_KASANX86_64H = $(LDFLAGS_KERNEL_RELEASEX86_64H) LDFLAGS_KERNEL_PROFILEX86_64H = $(LDFLAGS_KERNEL_RELEASEX86_64H) @@ -458,6 +507,7 @@ endif LTO_ENABLED_RELEASE = 1 LTO_ENABLED_DEVELOPMENT = 1 LTO_ENABLED_DEBUG = 0 +LTO_ENABLED_KASAN = 0 ifneq ($(BUILD_LTO),) USE_LTO = $(BUILD_LTO) @@ -553,6 +603,7 @@ EXPDIR = EXPORT_HDRS/$(COMPONENT) # STRIP_FLAGS_RELEASE = -S -x STRIP_FLAGS_DEVELOPMENT = -S +STRIP_FLAGS_KASAN = $(STRIP_FLAGS_DEVELOPMENT) STRIP_FLAGS_DEBUG = -S STRIP_FLAGS_PROFILE = -S -x @@ -577,7 +628,7 @@ MANDIR = /usr/share/man # # DEBUG alias location # -DEVELOPER_EXTRAS_DIR = /AppleInternal/CoreOS/xnu_debug +DEVELOPER_EXTRAS_DIR = /AppleInternal/CoreOS/xnu_$(CURRENT_KERNEL_CONFIG_LC) # # mach_kernel install location @@ -643,6 +694,14 @@ INSTALL_KERNEL_DIR := $(DEVELOPER_EXTRAS_DIR) INSTALL_KERNEL_SYM_DIR := $(DEVELOPER_EXTRAS_DIR) INSTALL_KERNEL_SYM_TO_KDK = 1 INSTALL_XNU_DEBUG_FILES = 1 +else ifeq ($(RC_ProjectName),xnu_kasan) +ifeq ($(filter $(SUPPORTED_EMBEDDED_PLATFORMS),$(PLATFORM)),) +# MacOS +INSTALL_KERNEL_DIR := $(DEVELOPER_EXTRAS_DIR) +INSTALL_KERNEL_SYM_DIR := $(DEVELOPER_EXTRAS_DIR) +endif +INSTALL_KERNEL_SYM_TO_KDK = 1 +INSTALL_KASAN_ONLY = 1 else ifneq ($(filter $(SUPPORTED_EMBEDDED_PLATFORMS),$(PLATFORM)),) INSTALL_KERNEL_SYM_TO_KDK = 1 USE_BINARY_PLIST = 1 diff --git a/makedefs/MakeInc.top b/makedefs/MakeInc.top index c19e8e59f..c552c108a 100644 --- a/makedefs/MakeInc.top +++ b/makedefs/MakeInc.top @@ -56,8 +56,12 @@ endif # Kernel Configuration options # +DEFAULT_PRODUCT_CONFIGS := + ifeq ($(RC_ProjectName),xnu_debug) override DEFAULT_KERNEL_CONFIG := DEBUG +else ifeq ($(RC_ProjectName),xnu_kasan) +override KERNEL_CONFIGS := KASAN else ifneq ($(filter $(SUPPORTED_EMBEDDED_PLATFORMS),$(PLATFORM)),) override DEFAULT_KERNEL_CONFIG := DEVELOPMENT else ifeq ($(PLATFORM),MacOSX) @@ -71,6 +75,11 @@ ifndef KERNEL_CONFIGS KERNEL_CONFIGS := DEFAULT endif +# If PRODUCT_CONFIGS is specified it should override default +ifndef PRODUCT_CONFIGS +PRODUCT_CONFIGS := $(DEFAULT_PRODUCT_CONFIGS) +endif + # # Machine Configuration options # @@ -158,7 +167,7 @@ TARGET_CONFIGS_ALIASES_UC := $(strip $(shell printf "%s" "$(TARGET_CONFIGS_ALIAS # TARGET_CONFIGS is unwieldy for use in Makefiles. Convert them to # "build configurations" which are tuples joined by "^". For # example, "RELEASE I386 DEFAULT DEVELOPMENT ARM DEFAULT" becomes -# "RELEASE^I386^NONE DEVELOPMENT^ARM^S5L8920X", which can be looped +# "RELEASE^I386^NONE DEVELOPMENT^ARM^T8002", which can be looped # over trivially. PRIMARY_BUILD_CONFIGS is the first config # for each architecture, used primarily for machine-dependent recursion. @@ -340,6 +349,9 @@ exporthdrs_md: build_exporthdrs_md_bootstrap .PHONY: installhdrs installhdrs_mi installhdrs_md ifeq ($(RC_ProjectName),xnu_debug) +installhdrs: + @: +else ifeq ($(RC_ProjectName),xnu_kasan) installhdrs: @: else @@ -469,6 +481,8 @@ final_touch_config_timestamps: config_install_bootstrap ifeq ($(RC_ProjectName),xnu_debug) install: install_kernels +else ifeq ($(RC_ProjectName),xnu_kasan) +install: install_config install_kernels else ifeq ($(RC_ProjectName),xnu_headers_Sim) install: installhdrs else diff --git a/osfmk/arm/Makefile b/osfmk/arm/Makefile new file mode 100644 index 000000000..8fd552a10 --- /dev/null +++ b/osfmk/arm/Makefile @@ -0,0 +1,49 @@ +export MakeInc_cmd=${SRCROOT}/makedefs/MakeInc.cmd +export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def +export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule +export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir + +include $(MakeInc_cmd) +include $(MakeInc_def) + +ARM_HEADER_FILES = \ + arch.h \ + atomic.h \ + cpu_number.h \ + cpu_capabilities.h \ + cpuid.h \ + cpuid_internal.h \ + dbgwrap.h \ + io_map_entries.h \ + lock.h \ + locks.h \ + machine_cpu.h \ + machine_cpuid.h \ + machine_routines.h \ + pal_routines.h \ + proc_reg.h \ + smp.h \ + thread.h \ + simple_lock.h + + +INSTALL_MD_DIR = arm + +INSTALL_MD_LCL_LIST = arch.h cpu_capabilities.h + +INSTALL_MD_LIST = arch.h + +INSTALL_KF_MD_LIST = $(ARM_HEADER_FILES) + +INSTALL_KF_MD_LCL_LIST = machine_kpc.h monotonic.h $(ARM_HEADER_FILES) + +EXPORT_MD_LIST = \ + caches_internal.h \ + machine_kpc.h \ + monotonic.h \ + ${ARM_HEADER_FILES} + +EXPORT_MD_DIR = arm + +include $(MakeInc_rule) +include $(MakeInc_dir) diff --git a/osfmk/arm/WKdmCompress_new.s b/osfmk/arm/WKdmCompress_new.s new file mode 100644 index 000000000..7b5f5a195 --- /dev/null +++ b/osfmk/arm/WKdmCompress_new.s @@ -0,0 +1,710 @@ +/* + * Copyright (c) 2000-2013 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +/* + This file contains armv7 hand optimized implementation of WKdm memory page compressor. + + int WKdm_compress (WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, unsigned int bytes_budget); + + input : + src_buf : address of input page (length = 1024 words) + dest_buf : address of output buffer (may not be 16-byte aligned) + scratch : a 16-byte aligned 4k bytes scratch memory provided by the caller, + bytes_budget : a given byte target in compression + + output : + + if the input buffer can be compressed within the given byte budget, the dest_buf is written with compressed data and the function returns with number of bytes for the compressed data + o.w., the function returns -1 to signal that the input data can not be compressed with the given byte budget. + During the scan and tag process, each word that can not be compressed will be written to dest_buf, followed by a 12-bytes header + 256-bytes tag area. + When the functions returns -1, dest_buf is filled with all those words that can not be compressed and should be considered undefined. + The worst-case scenario is that all words can not be compressed. Hence, the minimum size requirement for dest_buf should be 12+256+4096 = 4364 bytes to prevent from memory fault. + + The 4th argument bytes_budget is the target compress budget in bytes. + Should the input page can be compressed within the budget, the compressed data is written to *dest_buf, and the function returns the number of compressed bytes. + Otherwise, the function returns -1 (to signal to the caller that the page can not be compressed). + + WKdm Compression algorithm is briefly stated as follows: + + There is a dynamically updated dictionary consisting of 16 words. Each dictionary word is initialized to 1 at the point of entry to the function. + For a nonzero input word x, its 8-bits (10-bits scaled up) is used to determine a corresponding word from the dictionary, represented by dict_index (4-bits) and dict_word (32-bits). + a. k = (x>>10)&255; // 8-bit hash table index + b. dict_index = hashTable[k]; // 4-bit dictionary index, hashTable[] is fixed + c. dict_word = dictionary[dict_index]; // 32-bit dictionary word, dictionary[] is dynamically updated + + Each input word x is classified/tagged into 4 classes : + 0 : x = 0 + 1 : (x>>10) == (dict_word>>10), bits 10:31 of the input word match a dictionary word + 2 : (x>>10) != (dict_word>>10), the above condition (22 higher bits matched) is not met, meaning a dictionary miss + 3 : (x == dict_word), the exact input word is in the dictionary + + For each class, different numbers of bits are needed for the decompressor to reproduce the original input word. + 0 : 2-bits tag (32->2 compression) + 1 : 2-bits tag + 4-bits dict_index + 10-bits lower bits (32->16 compression) + 2 : 2-bits tag + 32-bits new word (32->34 expansion) + 3 : 2-bits tag + 4-bits dict_index (32->6 compression) + + It is obvious now that WKdm compress algorithm works well for pages where there are lots of zero words (32->2) and/or there are freqeunt repeats of some word patterns (32->6). + + the output bit stream (*dest_buf) consists of + a. 12 bytes header + b. 256 bytes for 1024 packed tags + c. (varying number of) words for new words not matched to dictionary word. + d. (varying number of) 32-bit words for packed 4-bit dict_indices (for class 1 and 3) + e. (varying number of) 32-bit words for packed 10-bit low bits (for class 1) + + the header is actually of 3 words that specify the ending offset (in 32-bit words) from the start of the bit stream of c,d,e, respectively. + Note that there might be padding bits in d (if the number of dict_indices does not divide by 8), and there are 2/12/22 padding bits for packing 3/2/1 low 10-bits in a 32-bit word. + + + The WKdm compress algorithm 1st runs a scan and classification pass, tagging and write unpacked data into temporary buffers. It follows by packing those data into the output buffer. + + The temp buffers are + + uint8_t tempTagsArray[1024]; // temporary saving for tags before final packing + uint8_t tempQPosArray[1024]; // temporary saving for dict_indices before final packing + uint16_t tempLowBitsArray[1024]; // temporary saving for partially matched lower 10 bits before final packing + + Since the new words (that can not matched fully or partially to the dictionary) are stored right after the header and the tags section and need no packing, we directly write them to + the destination buffer. + + uint32_t *new_word = dest_buf+3+64; // 3 words for header, 64 words for tags, new words come right after the tags. + + Now since we are given a byte budget for this compressor, we can monitor the byte usage on the fly in the scanning and tagging pass. + + byte_count = bytes_budget - 12 - 256; // header + tags + + whenever an input word is classified as class + + 2 : byte_count -= 4; + + in 4-bit/10-bit packing, we can also return -1 when byte_budget <=0; + + Note : since there might be extra padding bits for class 1 and 3, it is complicated to track this padding bits on the fly. To compromise, we change class 1 to + + without showing the bit budget management, the pseudo code is given as follows: + + uint8_t *tags=tempTagsArray; + uint8_t *dict=tempQPosArray; + uint8_t *partial=tempLowBitsArray; + + for (i=0;i<1024;i++) { + x = *src_buf++; + if (x == 0) { // zero, 2-bits tag + *tags++ = 0; + } else { + + // find dict_index and dict_word from x + k = (x>>10)&255; + dict_index = hashTable[k]; + dict_word = dictionary[dict_index]; + + if (dict_word == x) { // exactly match + // 2-bits tag + 4-bits table index + *tags++ = 3; + *dict++ = dict_index; + } else if (((x^dict_word)>>10)==0) { // 22 higher bits matched + // 2-bits tag + 4-bits table index + 10-bits lower partial + *tags++ = 1; + *dict++ = dict_index; + *partial++ = x &0x3ff; + dictionary[dict_index] = x; + } else { // not matched + // 2-bits tag + 32-bits new word + *tags++ = 2; + *new_word++ = x; + dictionary[dict_index] = x; + } + } + } + + after this classification/tagging pass is completed, the 3 temp buffers are packed into the output *dest_buf: + + 1. 1024 tags are packed into 256 bytes right after the 12-bytes header + 2. dictionary indices (4-bits each) are packed into are right after the new words section + 3. 3 low 10-bits are packed into a 32-bit word, this is after the dictionary indices section. + + cclee, 11/9/12 + + Added zero page, single value page, sparse page, early abort optimizations + rsrini, 09/14/14 +*/ + .text + .align 4 + + // int WKdm_compress (WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, unsigned int bytes_budget); + +.globl _WKdm_compress_new +_WKdm_compress_new: + +/* + ------------------------- symbolizing register use ----------------------------------- +*/ + + #define src_buf r0 + #define next_input_word r0 + #define dest_buf r1 + #define scratch r2 + #define dictionary sp + #define byte_count r3 + + #define next_tag r12 + + #define remaining r4 + #define next_full_patt r5 + #define dict_location r6 + #define next_qp r8 + #define hashTable r9 + #define next_low_bits r10 + #define eax r11 + #define ecx r12 + #define edx lr + #define rdi r6 + + #define tempTagsArray scratch + #define R11 r0 // only safe to use between phase-1 and phase-2 + #define R13 r4 // only safe to use between phase-1 and phase-2 +/* + ------------------------- allocate scratch memory for local use -------------------------------------- + + need 256*4 (tempTagsArray) + 256*4 (tempQPosArray) + 1024*2 (tempLowBitsArray) + total 4096 bytes + [scratch,#0] : tempTagsArray + [scratch,#1024] : tempQPosArray + [scratch,#2048] : tempLowBitsArray + + [sp,#0] : dictionary + +*/ + + #define TagsArray_offset 0 + #define QPosArray_offset 1024 + #define LowBitsArray_offset 2048 + + #define SV_RETURN 0 // return value when SV, ZV page is found + #define MZV_MAGIC 17185 // magic value used to identify MZV page encoding + #define CHKPT_BYTES 416 // for early aborts: checkpoint after processing this many bytes. Must be in range [4..4096] + #define CHKPT_WORDS (CHKPT_BYTES/4) // checkpoint bytes in words + #define CHKPT_TAG_BYTES (CHKPT_BYTES/16) // size of the tags for CHKPT_BYTES of data + #define CHKPT_SHRUNK_BYTES 426 // for early aborts: max size of compressed stream to allow further processing .. + // .. to disable early aborts, set CHKPT_SHRUNK_BYTES to 4096 +#if CHKPT_BYTES > 4096 + #error CHKPT_BYTES must be <= 4096 +#endif +#if CHKPT_BYTES < 4 + #error CHKPT_BYTES must be >= 4 +#endif + + push {r7,lr} + mov r7, sp + push {r4-r6,r8-r11} + +#if KERNEL + sub sp, sp, #32 + vst1.64 {q0,q1}, [sp] +#endif + + sub sp, sp, #(64+24) // reserve stack space for temps + dictionary + +/* + ----- set up registers and initialize WKdm dictionary ---------- +*/ + // NOTE: ALL THE DICTIONARY VALUES MUST BE INITIALIZED TO ZERO + // THIS IS NEEDED TO EFFICIENTLY DETECT SINGLE VALUE PAGES + mov eax, #0 + + mov next_tag, scratch // &tempTagsArray[0] + vdup.32 q0, eax + + add next_qp, scratch, #QPosArray_offset // next_qp + mov lr, sp + mov remaining, #(CHKPT_WORDS) // remaining input words .. initially set to checkpoint + vst1.64 {q0}, [lr]! + add next_full_patt, dest_buf, #268 // dest_buf + [TAGS_AREA_OFFSET + (4096 / 16)]*4 + vst1.64 {q0}, [lr]! + vst1.64 {q0}, [lr]! + add next_low_bits, scratch, #LowBitsArray_offset // &tempLowBitsArray[0] + vst1.64 {q0}, [lr]! + +#if defined(KERNEL) && !SLIDABLE + adr hashTable, L_table + ldr hashTable, [hashTable] +#else + ldr hashTable, L_table +L_table0: + ldr hashTable, [pc, hashTable] +#endif + +#define EARLYCHECK 0 +#define NORMAL 1 + +#define mode [sp, #64] +#define start_next_full_patt [sp, #68] +#define start_next_input_word [sp, #72] +#define start_next_low_bits [sp, #76] +#define byte_budget [sp, #80] + + mov edx, #EARLYCHECK + str edx, mode // indicate we are yet to evaluate the early aborts + str next_full_patt, start_next_full_patt // remember the start of next_full_patt + str next_input_word, start_next_input_word // remember the start of next_input_word + str next_low_bits, start_next_low_bits // remember the start of next_low_bits + str byte_count, byte_budget // remember the byte budget + + sub byte_count, byte_count, #(12+256) // byte_count - header bytes - tags bytes + b L_scan_loop + + .align 4, 0x90 +L_RECORD_ZERO: + /* we've just detected a zero input word in edx */ + strb edx, [next_tag], #1 // *next_tag++ = ZERO; + subs remaining, remaining, #1 // remaining input words + ble CHECKPOINT // if remaining = 0, break + + /* WKdm compress scan/tag loop */ +L_scan_loop: + ldr edx, [next_input_word], #4 + cmp edx, #0 + beq L_RECORD_ZERO // if (input_word==0) RECORD_ZERO + + /* + now the input word edx is nonzero, we next find the corresponding dictionary word (eax) and dict_location + */ + and eax, edx, #(0xff<<10) // part of input_word for hash table index + lsr eax, eax, #10 // 8-bit index to the Hash Table + ldrb eax, [hashTable, eax] // HASH_TO_DICT_BYTE_OFFSET(input_word) + add dict_location, dictionary, eax // ((char*) dictionary) + HASH_TO_DICT_BYTE_OFFSET(input_word)); + ldr eax, [dictionary, eax] // dict_word = *dict_location; + cmp eax, edx // dict_word vs input_word + beq L_RECORD_EXACT // if identical, RECORD_EXACT + + eor eax, eax, edx + lsrs eax, eax, #10 // HIGH_BITS(dict_word) + beq L_RECORD_PARTIAL // if identical, RECORD_PARTIAL + +L_RECORD_MISS: +/* + if we are here, the input word can not be derived from the dictionary, + we write the input word as a new word, + and update the dictionary with this new word +*/ + + subs byte_count, byte_count, #4 + ble L_budgetExhausted // o.w., return -1 to signal this page is not compressable + str edx, [next_full_patt], #4 // *next_full_patt++ = input_word; + mov eax, #2 + str edx, [dict_location] // *dict_location = input_word + strb eax, [next_tag], #1 // *next_tag++ = 2 for miss + subs remaining, remaining, #1 // remaining input words + bgt L_scan_loop // if bit_count>0, go on the scan/tag pass, + b CHECKPOINT + +L_done_search: + + // SET_QPOS_AREA_START(dest_buf,next_full_patt); + sub eax, next_full_patt, dest_buf // next_full_patt - dest_buf + lsr eax, eax, #2 // offset in 4-bytes + str eax, [dest_buf] // dest_buf[0] = next_full_patt - dest_buf + + + /* -------------------------- packing 1024 tags into 256 bytes ----------------------------------------*/ + // boundary_tmp = WK_pack_2bits(tempTagsArray, (WK_word *) next_tag, dest_buf + HEADER_SIZE_IN_WORDS); + + add rdi, dest_buf, #12 // dest_buf + mov eax, scratch // &tempTagsArray[0] + sub edx, next_tag, scratch // this should be 1024 + + vld1.64 {q0,q1}, [eax,:128]! + subs edx, edx, #32 // pre-decrement by 32 +L_pack_2bits: + subs edx, edx, #32 + vshl.i64 d1, d1, #4 + vshl.i64 d3, d3, #4 + vorr d0, d0, d1 + vorr d2, d2, d3 + vshr.u64 d1, d0, #30 + vshr.u64 d3, d2, #30 + vorr d0, d0, d1 + vorr d2, d2, d3 + vzip.32 d0, d2 + vst1.64 {d0}, [rdi]! + vld1.64 {q0,q1}, [eax,:128]! + bgt L_pack_2bits + vshl.i64 d1, d1, #4 + vshl.i64 d3, d3, #4 + vorr d0, d0, d1 + vorr d2, d2, d3 + vshr.u64 d1, d0, #30 + vshr.u64 d3, d2, #30 + vorr d0, d0, d1 + vorr d2, d2, d3 + vzip.32 d0, d2 + vst1.64 {d0}, [rdi] + + + /* --------------------------------- packing 4-bits dict indices into dest_buf ---------------------------------- */ + + /* 1st, round up number of 4-bits dict_indices to a multiple of 8 and fill in 0 if needed */ + add ecx, scratch, #QPosArray_offset // tempQPosArray + sub eax, next_qp, ecx // eax = num_bytes_to_pack = next_qp - (char *) tempQPosArray; + add eax, eax, #7 // num_bytes_to_pack+7 + lsr eax, eax, #3 // num_packed_words = (num_bytes_to_pack + 7) >> 3 + subs byte_count, byte_count, eax, lsl #2 // byte_count -= 4 * packed_words + blt L_budgetExhausted // o.w., return -1 to signal this page is not compressable + add ecx, ecx, eax, lsl #3 // endQPosArray = tempQPosArray + 2*num_source_words + cmp ecx, next_qp // endQPosArray vs next_qp + bls L16 // if (next_qp >= endQPosArray) skip the following zero paddings + sub eax, ecx, next_qp + mov edx, #0 + tst eax, #4 + beq 1f + str edx, [next_qp], #4 +1: tst eax, #2 + beq 1f + strh edx, [next_qp], #2 +1: tst eax, #1 + beq 1f + strb edx, [next_qp], #1 +1: +L16: + add edx, scratch, #QPosArray_offset // tempQPosArray + mov rdi, next_full_patt // next_full_patt + cmp ecx, edx // endQPosArray vs tempQPosArray + ldr eax, [dest_buf] + bls L20 // if (endQPosArray <= tempQPosArray) skip the following + + /* packing 4-bits dict indices into dest_buf */ +L_pack_4bits: + vld1.64 {d0}, [edx,:64]! // src_next[1]:src_next[0] + vshr.u64 d1, d0, #28 // (src_next[1] << 4) + vorr d0, d0, d1 // src_next[0] | (src_next[1] << 4) + cmp ecx, edx // source_end vs src_next + vstr s0, [rdi] + add rdi, rdi, #4 + bhi L_pack_4bits // while (src_next < source_end) repeat the loop + + /* --------------------------- packing 3 10-bits low bits into a 32-bit word in dest_buf[] ----------------------------------------- */ + // SET_LOW_BITS_AREA_START(dest_buf,boundary_tmp); + sub eax, rdi, dest_buf // boundary_tmp - dest_buf + lsr eax, eax, #2 // boundary_tmp - dest_buf in words +L20: + str eax, [dest_buf,#4] // dest_buf[1] = boundary_tmp - dest_buf + + add ecx, scratch, #LowBitsArray_offset // tempLowBitsArray + sub edx, next_low_bits, ecx // next_low_bits - tempLowBitsArray (in bytes) + lsr edx, edx, #1 // num_tenbits_to_pack (in half-words) + subs edx, edx, #3 // pre-decrement num_tenbits_to_pack by 3 + blt 1f // if num_tenbits_to_pack < 3, skip the following loop +0: + subs byte_count, byte_count, #4 // byte_count -= 4 + ble L_budgetExhausted // o.w., return -1 to signal this page is not compressable + ldr r4,[ecx, #2] // w2:6bits:w1 + ldrh r0,[ecx], #6 // w0 + uxth r5, r4, ror #16 // w2 + uxth r4, r4 // w1 + orr r0, r0, r4, lsl #10 // w1:w0 + subs edx, edx, #3 // num_tenbits_to_pack-=3 + orr r0, r0, r5, lsl #20 // w2:w1:w0 + str r0, [rdi], #4 // pack w0,w1,w2 into 1 dest_buf word + bge 0b // if no less than 3 elements, back to loop head + +1: adds edx, edx, #3 // post-increment num_tenbits_to_pack by 3 + beq 3f // if num_tenbits_to_pack is a multiple of 3, skip the following + subs byte_count, byte_count, #4 // byte_count -= 4 + ble L_budgetExhausted // o.w., return -1 to signal this page is not compressable + ldrh eax,[ecx] // w0 + subs edx, edx, #1 // num_tenbits_to_pack-- + beq 2f // + ldrh edx, [ecx, #2] // w1 + orr eax, eax, edx, lsl #10 // w0 | (w1<<10) + +2: str eax, [rdi], #4 // write the final dest_buf word + +3: sub eax, rdi, dest_buf // boundary_tmp - dest_buf + lsr eax, eax, #2 // boundary_tmp - dest_buf in terms of words + str eax, [dest_buf, #8] // SET_LOW_BITS_AREA_END(dest_buf,boundary_tmp) + lsl r0, eax, #2 // boundary_tmp - dest_buf in terms of bytes + +L_done: + // restore registers and return + + add sp, sp, #(64+24) // skip memory for temps + dictionary +#if KERNEL + vld1.64 {q0,q1}, [sp]! +#endif + pop {r4-r6,r8-r11} + pop {r7,pc} + + .align 4 +L_budgetExhausted: + mov r0, #-1 + b L_done + + + .align 4,0x90 +L_RECORD_EXACT: +/* + we have an exact match of the input word to its corresponding dictionary word + write tag/dict_index to the temorary buffers +*/ + sub edx, dict_location, dictionary // dict_location - dictionary + mov eax, #3 + lsr edx, edx, #2 // divide by 4 for word offset + strb eax, [next_tag], #1 // *next_tag++ = 3 for exact + strb edx, [next_qp], #1 // *next_qp = word offset (4-bit) + subs remaining, remaining, #1 // remaining input words + bgt L_scan_loop // if remaining>0, go on the scan/tag pass, + b CHECKPOINT // if remaining = 0, break + + .align 4,0x90 +L_RECORD_PARTIAL: +/* + we have a partial (high 22-bits) match of the input word to its corresponding dictionary word + write tag/dict_index/low 10 bits to the temorary buffers +*/ + sub eax, dict_location, dictionary // dict_location - dictionary + str edx, [dict_location] // *dict_location = input_word; + lsr eax, eax, #2 // offset in 32-bit word + lsl edx, edx, #22 + strb eax, [next_qp], #1 // update *next_qp++ + mov eax, #1 + lsr edx, edx, #22 // lower 10 bits + strb eax, [next_tag], #1 // *next_tag++ = 1 for partial matched + strh edx, [next_low_bits], #2 // save next_low_bits++ + subs remaining, remaining, #1 // remaining input words + bgt L_scan_loop // if remaining>0, go on the scan/tag pass, + +CHECKPOINT: + ldr eax, mode // load the mode + cmp eax, #EARLYCHECK + beq L_check_compression_ratio // early abort check + +L_check_zero_page: + + ldr eax, start_next_full_patt // check if any dictionary misses in page + cmp eax, next_full_patt + bne L_check_single_value_page + + add eax, scratch, #QPosArray_offset // get start_next_qp + cmp eax, next_qp // check if any partial or exact dictionary matches + + moveq r0, #SV_RETURN // Magic return value + beq L_done + +L_check_single_value_page: + + ldr eax, start_next_full_patt // get # dictionary misses + sub eax, next_full_patt, eax + lsr eax, eax, #2 + + add R11, scratch, #QPosArray_offset // get start_next_qp + sub R11, next_qp, R11 // get # dictionary hits (exact + partial) + + ldr R13, start_next_low_bits + sub R13, next_low_bits, R13 // get # dictionary partial hits + lsrs R13, R13, #1 + + // Single value page if one of the follwoing is true: + // partial == 0 AND hits == 1023 AND miss == 1 AND tag[0] == 2 (i.e. miss) + // partial == 1 AND hits == 1024 AND tag[0] == 1 (i.e. partial) + // + bne 1f // were there 0 partial hits? + + mov edx, #1023 + cmp R11, edx // were there 1023 dictionary hits + bne 1f + + cmp eax, #1 // was there exacly 1 dictionary miss? + bne 1f + + ldrb edx, [tempTagsArray] // read the very 1st tag + cmp edx, #2 // was the very 1st tag a miss? + beq L_is_single_value_page + +1: + cmp R13, #1 // was there 1 partial hit? + bne L_check_mostly_zero + + mov edx, #1024 + cmp R11, edx // were there 1024 dictionary hits + bne L_check_mostly_zero + + ldrb edx, [tempTagsArray] // read the very 1st tag + cmp edx, #1 // was the very 1st tag a partial? + bne L_is_single_value_page + +L_is_single_value_page: + + moveq r0, #SV_RETURN // Magic return value + beq L_done + +L_check_mostly_zero: + // how much space will the sparse packer take? + add eax, eax, R11 // eax += (next_qp - start_next_qp) + mov edx, #6 + mov R11, #4 + mla R11, eax, edx, R11 // R11 = eax * 6 (i.e. 4 byte word + 2 byte offset) + 4 byte for header + + ldr eax, start_next_low_bits + sub eax, next_low_bits, eax // get bytes consumed by lower-10 bits + mov edx, #1365 + mul eax, eax, edx + + ldr edx, start_next_full_patt + sub edx, next_full_patt, edx // get bytes consumed by dictionary misses + add eax, edx, eax, lsr #11 // eax = 2/3*(next_low_bits - start_next_low_bits) + (next_full_patt - start_next_full_patt) + + add edx, scratch, #QPosArray_offset // get start_next_qp + sub edx, next_qp, edx + add eax, eax, edx, lsr #1 // eax += (next_qp - start_next_qp)/2 + mov edx, #(12+256) + add eax, eax, edx // rax += bytes taken by the header + tags + + cmp eax, R11 // is the default packer the better option? + blt L_done_search + + ldr edx, byte_budget + cmp R11, edx // can the sparse packer fit into the given budget? + bgt L_budgetExhausted + +L_sparse_packer: + + mov edx, #MZV_MAGIC + str edx, [dest_buf], #4 // header to indicate a sparse packer + + ldr R13, start_next_input_word // get the starting address of src + mov edx, #0 + mov ecx, #4096 + +1: + ldm R13!, {r2, r3, r5, r6, r7, r8, r9, r10} + + teq r2, #0 + teqeq r3, #0 + teqeq r5, #0 + teqeq r6, #0 + teqeq r7, #0 + teqeq r8, #0 + teqeq r9, #0 + teqeq r10, #0 + + bne 2f + subs ecx, ecx, #32 + add edx, edx, #32 // 16 more bytes have been processed + bne 1b + mov r0, R11 // store the size of the compressed stream + b L_done + +2: + teq r2, #0 + strne r2, [dest_buf], #4 // store the non-0 word in the dest buffer + strhne edx, [dest_buf], #2 // store the byte index + add edx, edx, 4 + + teq r3, #0 + strne r3, [dest_buf], #4 // store the non-0 word in the dest buffer + strhne edx, [dest_buf], #2 // store the byte index + add edx, edx, 4 + + teq r5, #0 + strne r5, [dest_buf], #4 // store the non-0 word in the dest buffer + strhne edx, [dest_buf], #2 // store the byte index + add edx, edx, 4 + + teq r6, #0 + strne r6, [dest_buf], #4 // store the non-0 word in the dest buffer + strhne edx, [dest_buf], #2 // store the byte index + add edx, edx, 4 + + teq r7, #0 + strne r7, [dest_buf], #4 // store the non-0 word in the dest buffer + strhne edx, [dest_buf], #2 // store the byte index + add edx, edx, 4 + + teq r8, #0 + strne r8, [dest_buf], #4 // store the non-0 word in the dest buffer + strhne edx, [dest_buf], #2 // store the byte index + add edx, edx, 4 + + teq r9, #0 + strne r9, [dest_buf], #4 // store the non-0 word in the dest buffer + strhne edx, [dest_buf], #2 // store the byte index + add edx, edx, 4 + + teq r10, #0 + strne r10, [dest_buf], #4 // store the non-0 word in the dest buffer + strhne edx, [dest_buf], #2 // store the byte index + add edx, edx, 4 + + subs ecx, ecx, #32 + bne 1b + mov r0, R11 // store the size of the compressed stream + b L_done + +L_check_compression_ratio: + + mov eax, #NORMAL + str eax, mode + mov remaining, #(1024 - CHKPT_WORDS) // remaining input words to process + cmp remaining, #0 + beq CHECKPOINT // if there are no remaining words to process + + ldr eax, start_next_low_bits + sub eax, next_low_bits, eax // get bytes consumed by lower-10 bits + mov edx, #1365 + mul eax, eax, edx + + ldr edx, start_next_full_patt + sub edx, next_full_patt, edx // get bytes consumed by dictionary misses + add eax, edx, eax, lsr #11 // eax = 2/3*(next_low_bits - start_next_low_bits) + (next_full_patt - start_next_full_patt) + + add edx, scratch, #QPosArray_offset // get start_next_qp + sub edx, next_qp, edx + add eax, eax, edx, lsr #1 // eax += (next_qp - start_next_qp)/2 + mov edx, #(CHKPT_SHRUNK_BYTES - CHKPT_TAG_BYTES) + subs eax, eax, edx // eax += CHKPT_TAG_BYTES; eax -= CHKPT_SHRUNK_BYTES + bgt L_budgetExhausted // if eax is > 0, we need to early abort + b L_scan_loop // we are done + + +#if defined(KERNEL) && !SLIDABLE + .align 2 +L_table: + .long _hashLookupTable_new +#else + .align 2 +L_table: + .long L_Tab$non_lazy_ptr-(L_table0+8) + + .section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers + .align 2 +L_Tab$non_lazy_ptr: + .indirect_symbol _hashLookupTable_new + .long 0 +#endif + diff --git a/osfmk/arm/WKdmData_new.s b/osfmk/arm/WKdmData_new.s new file mode 100644 index 000000000..081059ab2 --- /dev/null +++ b/osfmk/arm/WKdmData_new.s @@ -0,0 +1,289 @@ +/* + * Copyright (c) 2000-2013 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + + .const + .align 4 +.globl _hashLookupTable_new +_hashLookupTable_new: + .byte 0 + .byte 52 + .byte 8 + .byte 56 + .byte 16 + .byte 12 + .byte 28 + .byte 20 + .byte 4 + .byte 36 + .byte 48 + .byte 24 + .byte 44 + .byte 40 + .byte 32 + .byte 60 + .byte 8 + .byte 12 + .byte 28 + .byte 20 + .byte 4 + .byte 60 + .byte 16 + .byte 36 + .byte 24 + .byte 48 + .byte 44 + .byte 32 + .byte 52 + .byte 56 + .byte 40 + .byte 12 + .byte 8 + .byte 48 + .byte 16 + .byte 52 + .byte 60 + .byte 28 + .byte 56 + .byte 32 + .byte 20 + .byte 24 + .byte 36 + .byte 40 + .byte 44 + .byte 4 + .byte 8 + .byte 40 + .byte 60 + .byte 32 + .byte 20 + .byte 44 + .byte 4 + .byte 36 + .byte 52 + .byte 24 + .byte 16 + .byte 56 + .byte 48 + .byte 12 + .byte 28 + .byte 16 + .byte 8 + .byte 40 + .byte 36 + .byte 28 + .byte 32 + .byte 12 + .byte 4 + .byte 44 + .byte 52 + .byte 20 + .byte 24 + .byte 48 + .byte 60 + .byte 56 + .byte 40 + .byte 48 + .byte 8 + .byte 32 + .byte 28 + .byte 36 + .byte 4 + .byte 44 + .byte 20 + .byte 56 + .byte 60 + .byte 24 + .byte 52 + .byte 16 + .byte 12 + .byte 12 + .byte 4 + .byte 48 + .byte 20 + .byte 8 + .byte 52 + .byte 16 + .byte 60 + .byte 24 + .byte 36 + .byte 44 + .byte 28 + .byte 56 + .byte 40 + .byte 32 + .byte 36 + .byte 20 + .byte 24 + .byte 60 + .byte 40 + .byte 44 + .byte 52 + .byte 16 + .byte 32 + .byte 4 + .byte 48 + .byte 8 + .byte 28 + .byte 56 + .byte 12 + .byte 28 + .byte 32 + .byte 40 + .byte 52 + .byte 36 + .byte 16 + .byte 20 + .byte 48 + .byte 8 + .byte 4 + .byte 60 + .byte 24 + .byte 56 + .byte 44 + .byte 12 + .byte 8 + .byte 36 + .byte 24 + .byte 28 + .byte 16 + .byte 60 + .byte 20 + .byte 56 + .byte 32 + .byte 40 + .byte 48 + .byte 12 + .byte 4 + .byte 44 + .byte 52 + .byte 44 + .byte 40 + .byte 12 + .byte 56 + .byte 8 + .byte 36 + .byte 24 + .byte 60 + .byte 28 + .byte 48 + .byte 4 + .byte 32 + .byte 20 + .byte 16 + .byte 52 + .byte 60 + .byte 12 + .byte 24 + .byte 36 + .byte 8 + .byte 4 + .byte 16 + .byte 56 + .byte 48 + .byte 44 + .byte 40 + .byte 52 + .byte 32 + .byte 20 + .byte 28 + .byte 32 + .byte 12 + .byte 36 + .byte 28 + .byte 24 + .byte 56 + .byte 40 + .byte 16 + .byte 52 + .byte 44 + .byte 4 + .byte 20 + .byte 60 + .byte 8 + .byte 48 + .byte 48 + .byte 52 + .byte 12 + .byte 20 + .byte 32 + .byte 44 + .byte 36 + .byte 28 + .byte 4 + .byte 40 + .byte 24 + .byte 8 + .byte 56 + .byte 60 + .byte 16 + .byte 36 + .byte 32 + .byte 8 + .byte 40 + .byte 4 + .byte 52 + .byte 24 + .byte 44 + .byte 20 + .byte 12 + .byte 28 + .byte 48 + .byte 56 + .byte 16 + .byte 60 + .byte 4 + .byte 52 + .byte 60 + .byte 48 + .byte 20 + .byte 16 + .byte 56 + .byte 44 + .byte 24 + .byte 8 + .byte 40 + .byte 12 + .byte 32 + .byte 28 + .byte 36 + .byte 24 + .byte 32 + .byte 12 + .byte 4 + .byte 20 + .byte 16 + .byte 60 + .byte 36 + .byte 28 + .byte 8 + .byte 52 + .byte 40 + .byte 48 + .byte 44 + .byte 56 + diff --git a/osfmk/arm/WKdmDecompress_new.s b/osfmk/arm/WKdmDecompress_new.s new file mode 100644 index 000000000..2dbbd4502 --- /dev/null +++ b/osfmk/arm/WKdmDecompress_new.s @@ -0,0 +1,427 @@ +/* + * Copyright (c) 2000-2013 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +/* + This file contains armv7 hand optimized implementation of WKdm memory page decompressor. + + void WKdm_decompress (WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, __unused__ unsigned int words); + + input : + src_buf : address of input compressed data buffer + dest_buf : address of output decompressed buffer + scratch : a 16-byte aligned 4k bytes scratch memory provided by the caller + words : this argument is not used in the implementation + + output : + + the input buffer is decompressed and the dest_buf is written with decompressed data. + + Am algorithm description of the WKdm compress and bit stream format can be found in the WKdm Compress armv7 assembly code WKdmCompress.s + + The bit stream (*src_buf) consists of + a. 12 bytes header + b. 256 bytes for 1024 packed tags + c. (varying number of) words for new words not matched to dictionary word. + d. (varying number of) 32-bit words for packed 4-bit dict_indices (for class 1 and 3) + e. (varying number of) 32-bit words for packed 10-bit low bits (for class 1) + + where the header (of 3 words) specifies the ending boundaries (in 32-bit words) from the start of the bit stream of c,d,e, respectively. + + The decompressor 1st unpacking the bit stream component b/d/e into temorary buffers. Then it sequentially decodes the decompressed word as follows + + for (i=0;i<1024;i++) { + tag = *next_tag++ + switch (tag) { + case 0 : *dest_buf++ = 0; break; + case 1 : dict_word = dictionary[*dict_index]; dictionary[*dict_index++] = *dest_buf++ = dict_word&0xfffffc00 | *LowBits++; break; + case 2 : x = *new_word++; k = (x>>10)&255; k = hashTable[k]; dictionary[k] = *dest_buf++ = x; break; + case 3 : *dest_buf++ = dictionary[*dict_index++]; break; + } + + cclee, 11/9/12 + + Added zero page, single value page, sparse page, early abort optimizations + rsrini, 09/14/14 + +*/ + + #define MZV_MAGIC 17185 // magic value used to identify MZV page encoding + + #define ZERO 0 + #define PARTIAL_MATCH 1 + #define MISS_TAG 2 + #define MATCH 3 + + .text + .syntax unified + .align 4 + + // void WKdm_decompress (WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, unsigned int bytes); + + .globl _WKdm_decompress_new +_WKdm_decompress_new: + + /* + -------- symbolizing registers -------- + the armv7 code was ported from x86_64 so we name some registers that are used as temp variables with x86_64 register names. + */ + + #define src_buf r0 + #define dest_buf r1 + #define scratch r2 + #define eax r3 + #define ebx r4 + #define hashTable r4 + #define ecx r5 + #define edx r6 + #define n_bytes r8 + #define next_tag r12 + #define tags_counter lr + #define dictionary sp + #define v0 q0 + #define v1 q1 + #define v2 q2 + #define v3 q3 + #define v4 q4 + #define v5 q5 + + // and scratch memory for local variables + + // [sp,#0] : dictionary + // [scratch,#0] : tempTagsArray was 64 + // [scratch,#1024] : tempQPosArray was 1088 + // [scratch,#2048] : tempLowBitsArray was 2112 + + push {r7, lr} + mov r7, sp + push {r4-r6,r8-r11} +#if KERNEL + sub ecx, sp, #96 + sub sp, sp, #96 + vst1.64 {q0,q1},[ecx]! + vst1.64 {q2,q3},[ecx]! + vst1.64 {q4,q5},[ecx]! +#endif + sub sp, sp, #64 // allocate for dictionary + + mov n_bytes, r3 // save the n_bytes passed as function args + ldr eax, [src_buf] // read the 1st word from the header + mov ecx, #MZV_MAGIC + cmp eax, ecx // is the alternate packer used (i.e. is MZV page)? + bne L_default_decompressor // default decompressor was used + + // Mostly Zero Page Handling... + // { + add src_buf, src_buf, 4 // skip the header + mov eax, dest_buf + mov ecx, #4096 // number of bytes to zero out + mov r9, #0 + mov r10, #0 + mov r11, #0 + mov r12, #0 +1: + subs ecx, ecx, #64 + stmia eax!, {r9-r12} + stmia eax!, {r9-r12} + stmia eax!, {r9-r12} + stmia eax!, {r9-r12} + bne 1b + + mov r12, #4 // current byte position in src to read from +2: + ldr eax, [src_buf], #4 // get the word + ldrh edx, [src_buf], #2 // get the index + str eax, [dest_buf, edx] // store non-0 word in the destination buffer + add r12, r12, #6 // 6 more bytes processed + cmp r12, n_bytes // finished processing all the bytes? + bne 2b + b L_done + // } + +L_default_decompressor: + + /* + ---------------------- set up registers and PRELOAD_DICTIONARY --------------------------------- + */ + // NOTE: ALL THE DICTIONARY VALUES MUST BE INITIALIZED TO ZERO TO MIRROR THE COMPRESSOR + vmov.i32 q0, #0 + mov r8, sp + adr ebx, _table_2bits + vst1.64 {q0}, [r8]! + add r10, src_buf, #268 // TAGS_AREA_END + vst1.64 {q0}, [r8]! + add eax, src_buf, #12 // TAGS_AREA_START + vst1.64 {q0}, [r8]! + mov ecx, scratch // tempTagsArray + vst1.64 {q0}, [r8]! + vld1.64 {q0,q1},[ebx,:128] + + + // WK_unpack_2bits(TAGS_AREA_START(src_buf), TAGS_AREA_END(src_buf), tempTagsArray); +/* + unpacking 16 2-bit tags (from a 32-bit word) into 16 bytes + for arm64, this can be done by + 1. read the input 32-bit word into GPR w + 2. duplicate GPR into 4 elements in a vector register v0 + 3. ushl.4s vd, v0, vshift where vshift = {0, -2, -4, -6} + 4. and.4s vd, vd, vmask where vmask = 0x03030303030303030303030303030303 +*/ + +L_WK_unpack_2bits: + vld1.64 {v5}, [eax]! // read 4 32-bit words for 64 2-bit tags + vdup.32 v2, d10[0] // duplicate to 4 elements + vdup.32 v3, d10[1] // duplicate to 4 elements + vdup.32 v4, d11[0] // duplicate to 4 elements + vdup.32 v5, d11[1] // duplicate to 4 elements + vshl.u32 v2, v2, v0 // v0 = {0, -2, -4, -6} + vshl.u32 v3, v3, v0 // v0 = {0, -2, -4, -6} + vshl.u32 v4, v4, v0 // v0 = {0, -2, -4, -6} + vshl.u32 v5, v5, v0 // v0 = {0, -2, -4, -6} + vand v2, v2, v1 // v1 = {3,3,...,3} + vand v3, v3, v1 // v1 = {3,3,...,3} + vand v4, v4, v1 // v1 = {3,3,...,3} + vand v5, v5, v1 // v1 = {3,3,...,3} + vst1.64 {v2,v3}, [ecx,:128]! // write 64 tags into tempTagsArray + cmp r10, eax // TAGS_AREA_END vs TAGS_AREA_START + vst1.64 {v4,v5}, [ecx,:128]! // write 64 tags into tempTagsArray + bhi L_WK_unpack_2bits // if not reach TAGS_AREA_END, repeat L_WK_unpack_2bits + + + // WK_unpack_4bits(QPOS_AREA_START(src_buf), QPOS_AREA_END(src_buf), tempQPosArray); + + ldm src_buf, {r8,r9} // WKdm header qpos start and end + adr ebx, _table_4bits + subs r12, r9, r8 // r12 = (QPOS_AREA_END - QPOS_AREA_START)/4 + add r8, src_buf, r8, lsl #2 // QPOS_AREA_START + add r9, src_buf, r9, lsl #2 // QPOS_AREA_END + bls 1f // if QPOS_AREA_END <= QPOS_AREA_START, skip L_WK_unpack_4bits + add ecx, scratch, #1024 // tempQPosArray + vld1.64 {v0,v1},[ebx,:128] + + subs r12, r12, #1 + bls 2f // do loop of 2 only if w14 >= 5 +L_WK_unpack_4bits: + vld1.64 {d4}, [r8]! // read a 32-bit word for 8 4-bit positions + subs r12, r12, #2 + vmov d5, d4 + vzip.32 d4, d5 + vshl.u32 v2, v2, v0 // v0 = {0, -4, 0, -4} + vand v2, v2, v1 // v1 = {15,15,...,15} + vst1.64 {q2}, [ecx,:128]! + bhi L_WK_unpack_4bits +2: + adds r12, r12, #1 + ble 1f + + ldr r12, [r8], #4 // read a 32-bit word for 8 4-bit positions + vdup.32 d4, r12 // duplicate to 2 elements + vshl.u32 v2, v2, v0 // v0 = {0, -4} + vand v2, v2, v1 // v1 = {15,15,...,15} + vst1.64 {d4}, [ecx,:64]! // write 16 tags into tempTagsArray + +1: + + // WK_unpack_3_tenbits(LOW_BITS_AREA_START(src_buf), LOW_BITS_AREA_END(src_buf), tempLowBitsArray); + + ldr eax, [src_buf,#8] // LOW_BITS_AREA_END offset + add r8, src_buf, eax, lsl #2 // LOW_BITS_AREA_END + cmp r8, r9 // LOW_BITS_AREA_START vs LOW_BITS_AREA_END + add ecx, scratch, #2048 // tempLowBitsArray + add edx, scratch, #4096 // last tenbits + bls 1f // if START>=END, skip L_WK_unpack_3_tenbits + + adr ebx, _table_10bits + vld1.64 {v0,v1},[ebx,:128] + + mov r11, #0x03ff +L_WK_unpack_3_tenbits: + ldr r12, [r9], #4 // read a 32-bit word for 3 low 10-bits + and lr, r11, r12 + strh lr, [ecx], #2 + cmp ecx, edx + and lr, r11, r12, lsr #10 + beq 1f + strh lr, [ecx], #2 + and lr, r11, r12, lsr #20 + strh lr, [ecx], #2 + + cmp r8, r9 // LOW_BITS_AREA_START vs LOW_BITS_AREA_END + bhi L_WK_unpack_3_tenbits // repeat loop if LOW_BITS_AREA_END > next_word + +1: + /* + set up before going to the main decompress loop + */ + + mov next_tag, scratch // tempTagsArray + add r8, scratch, #1024 // next_qpos + add r11, scratch, #2048 // tempLowBitsArray +#if defined(KERNEL) && !SLIDABLE + adr hashTable, L_table + ldr hashTable, [hashTable] +#else + ldr hashTable, L_table +L_table0: + ldr hashTable, [pc, hashTable] +#endif + mov tags_counter, #1024 // tags_counter + + b L_next + + .align 4,0x90 +L_ZERO_TAG: + /* + we can only get here if w9 = 0, meaning this is a zero tag + *dest_buf++ = 0; + */ + subs tags_counter,tags_counter,#1 // tags_counter-- + str r9, [dest_buf], #4 // *dest_buf++ = 0 + ble L_done // if next_tag >= tag_area_end, we're done + + /* WKdm decompress main loop */ +L_next: + ldrb r9, [next_tag], #1 // new tag + cmp r9, #0 + beq L_ZERO_TAG + cmp r9, #2 // partial match tag ? + beq L_MISS_TAG + bgt L_EXACT_TAG + +L_PARTIAL_TAG: + /* + this is a partial match: + dict_word = dictionary[*dict_index]; + dictionary[*dict_index++] = *dest_buf++ = dict_word&0xfffffc00 | *LowBits++; + */ + ldrb edx, [r8], #1 // qpos = *next_qpos++ + ldrh ecx, [r11], #2 // lower 10-bits from *next_low_bits++ + ldr eax, [dictionary, edx, lsl #2] // read dictionary word + subs tags_counter,tags_counter,#1 // tags_counter-- + lsr eax, eax, #10 // clear lower 10 bits + orr eax, ecx, eax, lsl #10 // pad the lower 10-bits from *next_low_bits + str eax, [dictionary,edx,lsl #2] // *dict_location = newly formed word + str eax, [dest_buf], #4 // *dest_buf++ = newly formed word + bgt L_next // repeat loop until next_tag==tag_area_end + +L_done: + + add sp, sp, #64 // deallocate for dictionary + + // release stack memory, restore registers, and return +#if KERNEL + vld1.64 {q0,q1},[sp]! + vld1.64 {q2,q3},[sp]! + vld1.64 {q4,q5},[sp]! +#endif + pop {r4-r6,r8-r11} + pop {r7,pc} + + .align 4,0x90 +L_MISS_TAG: + /* + this is a dictionary miss. + x = *new_word++; + k = (x>>10)&255; + k = hashTable[k]; + dictionary[k] = *dest_buf++ = x; + */ + subs tags_counter,tags_counter,#1 // tags_counter-- + ldr eax, [r10], #4 // w = *next_full_patt++ + lsr edx, eax, #10 // w>>10 + str eax, [dest_buf], #4 // *dest_buf++ = word + and edx, edx, #0x0ff // 8-bit hash table index + ldrb edx, [ebx, edx] // qpos + str eax, [dictionary,edx] // dictionary[qpos] = word + bgt L_next // repeat the loop + b L_done // if next_tag >= tag_area_end, we're done + + .align 4,0x90 + +L_EXACT_TAG: + /* + this is an exact match; + *dest_buf++ = dictionary[*dict_index++]; + */ + + ldrb eax, [r8], #1 // qpos = *next_qpos++ + subs tags_counter,tags_counter,#1 // tags_counter-- + ldr eax, [dictionary,eax,lsl #2] // w = dictionary[qpos] + str eax, [dest_buf], #4 // *dest_buf++ = w + bgt L_next // repeat the loop + b L_done // if next_tag >= tag_area_end, we're done + + + .align 4 + +_table_2bits: + .word 0 + .word -2 + .word -4 + .word -6 + .word 0x03030303 + .word 0x03030303 + .word 0x03030303 + .word 0x03030303 + +_table_4bits: + .word 0 + .word -4 + .word 0 + .word -4 + .word 0x0f0f0f0f + .word 0x0f0f0f0f + .word 0x0f0f0f0f + .word 0x0f0f0f0f + +_table_10bits: + .word 0 + .word -10 + .word -20 + .word 0 + .word 1023 + .word 1023 + .word 1023 + .word 0 + + +#if defined(KERNEL) && !SLIDABLE + .align 2 +L_table: + .long _hashLookupTable_new +#else + .align 2 +L_table: + .long L_Tab$non_lazy_ptr-(L_table0+8) + + .section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers + .align 2 +L_Tab$non_lazy_ptr: + .indirect_symbol _hashLookupTable_new + .long 0 +#endif + diff --git a/osfmk/arm/arch.h b/osfmk/arm/arch.h new file mode 100644 index 000000000..8c38de577 --- /dev/null +++ b/osfmk/arm/arch.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _ARM_ARCH_H +#define _ARM_ARCH_H + +/* Collect the __ARM_ARCH_*__ compiler flags into something easier to use. */ +#if defined (__ARM_ARCH_7A__) || defined (__ARM_ARCH_7S__) || defined (__ARM_ARCH_7F__) || defined (__ARM_ARCH_7K__) +#define _ARM_ARCH_7 +#endif + +#if defined (_ARM_ARCH_7) || defined (__ARM_ARCH_6K__) || defined (__ARM_ARCH_6ZK__) +#define _ARM_ARCH_6K +#endif + +#if defined (_ARM_ARCH_7) || defined (__ARM_ARCH_6Z__) || defined (__ARM_ARCH_6ZK__) +#define _ARM_ARCH_6Z +#endif + +#if defined (__ARM_ARCH_6__) || defined (__ARM_ARCH_6J__) || \ + defined (_ARM_ARCH_6Z) || defined (_ARM_ARCH_6K) +#define _ARM_ARCH_6 +#endif + +#if defined (_ARM_ARCH_6) || defined (__ARM_ARCH_5E__) || \ + defined (__ARM_ARCH_5TE__) || defined (__ARM_ARCH_5TEJ__) +#define _ARM_ARCH_5E +#endif + +#if defined (_ARM_ARCH_5E) || defined (__ARM_ARCH_5__) || \ + defined (__ARM_ARCH_5T__) +#define _ARM_ARCH_5 +#endif + +#if defined (_ARM_ARCH_5) || defined (__ARM_ARCH_4T__) +#define _ARM_ARCH_4T +#endif + +#if defined (_ARM_ARCH_4T) || defined (__ARM_ARCH_4__) +#define _ARM_ARCH_4 +#endif + +#endif diff --git a/osfmk/arm/arm_init.c b/osfmk/arm/arm_init.c new file mode 100644 index 000000000..a9bb6d407 --- /dev/null +++ b/osfmk/arm/arm_init.c @@ -0,0 +1,531 @@ +/* + * Copyright (c) 2007-2009 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ + +#include <debug.h> +#include <mach_ldebug.h> +#include <mach_kdp.h> + +#include <kern/misc_protos.h> +#include <kern/thread.h> +#include <kern/timer_queue.h> +#include <kern/processor.h> +#include <kern/startup.h> +#include <kern/debug.h> +#include <prng/random.h> +#include <machine/machine_routines.h> +#include <machine/commpage.h> +/* ARM64_TODO unify boot.h */ +#if __arm64__ +#include <pexpert/arm64/boot.h> +#elif __arm__ +#include <pexpert/arm/boot.h> +#else +#error Unsupported arch +#endif +#include <pexpert/arm/consistent_debug.h> +#include <pexpert/device_tree.h> +#include <arm/proc_reg.h> +#include <arm/pmap.h> +#include <arm/caches_internal.h> +#include <arm/cpu_internal.h> +#include <arm/cpu_data_internal.h> +#include <arm/misc_protos.h> +#include <arm/machine_cpu.h> +#include <arm/rtclock.h> +#include <vm/vm_map.h> + +#include <libkern/kernel_mach_header.h> +#include <libkern/stack_protector.h> +#include <libkern/section_keywords.h> +#include <san/kasan.h> + +#include <pexpert/pexpert.h> + +#include <console/serial_protos.h> + +#if CONFIG_TELEMETRY +#include <kern/telemetry.h> +#endif +#if MONOTONIC +#include <kern/monotonic.h> +#endif /* MONOTONIC */ + +extern void patch_low_glo(void); +extern int serial_init(void); +extern void sleep_token_buffer_init(void); + +extern vm_offset_t intstack_top; +extern vm_offset_t fiqstack_top; +#if __arm64__ +extern vm_offset_t excepstack_top; +#endif + +extern const char version[]; +extern const char version_variant[]; +extern int disableConsoleOutput; + +#if __ARM_PAN_AVAILABLE__ +SECURITY_READ_ONLY_LATE(boolean_t) arm_pan_enabled = FALSE; /* PAN support on Hurricane and newer HW */ +#endif + +int pc_trace_buf[PC_TRACE_BUF_SIZE] = {0}; +int pc_trace_cnt = PC_TRACE_BUF_SIZE; +int debug_task; + +boolean_t up_style_idle_exit = 0; + + + +#if INTERRUPT_MASKED_DEBUG +boolean_t interrupt_masked_debug = 1; +uint64_t interrupt_masked_timeout = 0xd0000; +#endif + +boot_args const_boot_args __attribute__((section("__DATA, __const"))); +boot_args *BootArgs __attribute__((section("__DATA, __const"))); + +unsigned int arm_diag; +#ifdef APPLETYPHOON +static unsigned cpus_defeatures = 0x0; +extern void cpu_defeatures_set(unsigned int); +#endif + +#if __arm64__ && __ARM_GLOBAL_SLEEP_BIT__ +extern volatile boolean_t arm64_stall_sleep; +#endif + +extern boolean_t force_immediate_debug_halt; + +#define MIN_LOW_GLO_MASK (0x144) + +/* + * Forward definition + */ +void arm_init(boot_args * args); + +#if __arm64__ +unsigned int page_shift_user32; /* for page_size as seen by a 32-bit task */ +#endif /* __arm64__ */ + + +/* + * Routine: arm_init + * Function: + */ +void +arm_init( + boot_args *args) +{ + unsigned int maxmem; + uint32_t memsize; + uint64_t xmaxmem; + thread_t thread; + processor_t my_master_proc; + + /* If kernel integrity is supported, use a constant copy of the boot args. */ + const_boot_args = *args; + BootArgs = &const_boot_args; + + cpu_data_init(&BootCpuData); + + PE_init_platform(FALSE, args); /* Get platform expert set up */ + +#if __arm64__ + { + unsigned int tmp_16k = 0; + +#ifdef XXXX + /* + * Select the advertised kernel page size; without the boot-arg + * we default to the hardware page size for the current platform. + */ + if (PE_parse_boot_argn("-vm16k", &tmp_16k, sizeof(tmp_16k))) + PAGE_SHIFT_CONST = PAGE_MAX_SHIFT; + else + PAGE_SHIFT_CONST = ARM_PGSHIFT; +#else + /* + * Select the advertised kernel page size; with the boot-arg + * use to the hardware page size for the current platform. + */ + int radar_20804515 = 1; /* default: new mode */ + PE_parse_boot_argn("radar_20804515", &radar_20804515, sizeof(radar_20804515)); + if (radar_20804515) { + if (args->memSize > 1ULL*1024*1024*1024) { + /* + * arm64 device with > 1GB of RAM: + * kernel uses 16KB pages. + */ + PAGE_SHIFT_CONST = PAGE_MAX_SHIFT; + } else { + /* + * arm64 device with <= 1GB of RAM: + * kernel uses hardware page size + * (4KB for H6/H7, 16KB for H8+). + */ + PAGE_SHIFT_CONST = ARM_PGSHIFT; + } + /* 32-bit apps always see 16KB page size */ + page_shift_user32 = PAGE_MAX_SHIFT; + } else { + /* kernel page size: */ + if (PE_parse_boot_argn("-use_hwpagesize", &tmp_16k, sizeof(tmp_16k))) + PAGE_SHIFT_CONST = ARM_PGSHIFT; + else + PAGE_SHIFT_CONST = PAGE_MAX_SHIFT; + /* old mode: 32-bit apps see same page size as kernel */ + page_shift_user32 = PAGE_SHIFT_CONST; + } +#endif +#ifdef APPLETYPHOON + if (PE_parse_boot_argn("cpus_defeatures", &cpus_defeatures, sizeof(cpus_defeatures))) { + if ((cpus_defeatures & 0xF) != 0) + cpu_defeatures_set(cpus_defeatures & 0xF); + } +#endif + } +#endif + + ml_parse_cpu_topology(); + + master_cpu = ml_get_boot_cpu_number(); + assert(master_cpu >= 0 && master_cpu <= ml_get_max_cpu_number()); + + BootCpuData.cpu_number = (unsigned short)master_cpu; +#if __arm__ + BootCpuData.cpu_exc_vectors = (vm_offset_t)&ExceptionVectorsTable; +#endif + BootCpuData.intstack_top = (vm_offset_t) & intstack_top; + BootCpuData.istackptr = BootCpuData.intstack_top; + BootCpuData.fiqstack_top = (vm_offset_t) & fiqstack_top; + BootCpuData.fiqstackptr = BootCpuData.fiqstack_top; +#if __arm64__ + BootCpuData.excepstack_top = (vm_offset_t) & excepstack_top; + BootCpuData.excepstackptr = BootCpuData.excepstack_top; +#endif + BootCpuData.cpu_processor = cpu_processor_alloc(TRUE); + BootCpuData.cpu_console_buf = (void *)NULL; + CpuDataEntries[master_cpu].cpu_data_vaddr = &BootCpuData; + CpuDataEntries[master_cpu].cpu_data_paddr = (void *)((uintptr_t)(args->physBase) + + ((uintptr_t)&BootCpuData + - (uintptr_t)(args->virtBase))); + + thread_bootstrap(); + thread = current_thread(); + /* + * Preemption is enabled for this thread so that it can lock mutexes without + * tripping the preemption check. In reality scheduling is not enabled until + * this thread completes, and there are no other threads to switch to, so + * preemption level is not really meaningful for the bootstrap thread. + */ + thread->machine.preemption_count = 0; + thread->machine.CpuDatap = &BootCpuData; +#if __arm__ && __ARM_USER_PROTECT__ + { + unsigned int ttbr0_val, ttbr1_val, ttbcr_val; + __asm__ volatile("mrc p15,0,%0,c2,c0,0\n" : "=r"(ttbr0_val)); + __asm__ volatile("mrc p15,0,%0,c2,c0,1\n" : "=r"(ttbr1_val)); + __asm__ volatile("mrc p15,0,%0,c2,c0,2\n" : "=r"(ttbcr_val)); + thread->machine.uptw_ttb = ttbr0_val; + thread->machine.kptw_ttb = ttbr1_val; + thread->machine.uptw_ttc = ttbcr_val; + } +#endif + BootCpuData.cpu_processor->processor_data.kernel_timer = &thread->system_timer; + BootCpuData.cpu_processor->processor_data.thread_timer = &thread->system_timer; + + cpu_bootstrap(); + + rtclock_early_init(); + + kernel_early_bootstrap(); + + cpu_init(); + + EntropyData.index_ptr = EntropyData.buffer; + + processor_bootstrap(); + my_master_proc = master_processor; + + (void)PE_parse_boot_argn("diag", &arm_diag, sizeof (arm_diag)); + + if (PE_parse_boot_argn("maxmem", &maxmem, sizeof (maxmem))) + xmaxmem = (uint64_t) maxmem *(1024 * 1024); + else if (PE_get_default("hw.memsize", &memsize, sizeof (memsize))) + xmaxmem = (uint64_t) memsize; + else + xmaxmem = 0; + + if (PE_parse_boot_argn("up_style_idle_exit", &up_style_idle_exit, sizeof(up_style_idle_exit))) { + up_style_idle_exit = 1; + } +#if INTERRUPT_MASKED_DEBUG + int wdt_boot_arg = 0; + /* Disable if WDT is disabled or no_interrupt_mask_debug in boot-args */ + if (PE_parse_boot_argn("no_interrupt_masked_debug", &interrupt_masked_debug, + sizeof(interrupt_masked_debug)) || (PE_parse_boot_argn("wdt", &wdt_boot_arg, + sizeof(wdt_boot_arg)) && (wdt_boot_arg == -1))) { + interrupt_masked_debug = 0; + } + + PE_parse_boot_argn("interrupt_masked_debug_timeout", &interrupt_masked_timeout, sizeof(interrupt_masked_timeout)); +#endif + + + + PE_parse_boot_argn("immediate_NMI", &force_immediate_debug_halt, sizeof(force_immediate_debug_halt)); + +#if __ARM_PAN_AVAILABLE__ +#if (DEVELOPMENT || DEBUG) + boolean_t pan; + if (!PE_parse_boot_argn("-pmap_smap_disable", &pan, sizeof(pan))) { + arm_pan_enabled = TRUE; + __builtin_arm_wsr("pan", 1); + set_mmu_control((get_mmu_control()) & ~SCTLR_PAN_UNCHANGED); + } +#else + arm_pan_enabled = TRUE; + __builtin_arm_wsr("pan", 1); + /* SCTLR_EL1.SPAN is clear on RELEASE */ +#endif +#endif /* __ARM_PAN_AVAILABLE__ */ + + arm_vm_init(xmaxmem, args); + + uint32_t debugmode; + if (PE_parse_boot_argn("debug", &debugmode, sizeof(debugmode)) && + ((debugmode & MIN_LOW_GLO_MASK) == MIN_LOW_GLO_MASK)) + patch_low_glo(); + + printf_init(); + panic_init(); +#if __arm64__ && WITH_CLASSIC_S2R + sleep_token_buffer_init(); +#endif + + PE_consistent_debug_inherit(); + + /* setup debugging output if one has been chosen */ + PE_init_kprintf(FALSE); + + kprintf("kprintf initialized\n"); + + serialmode = 0; /* Assume normal keyboard and console */ + if (PE_parse_boot_argn("serial", &serialmode, sizeof(serialmode))) { /* Do we want a serial + * keyboard and/or + * console? */ + kprintf("Serial mode specified: %08X\n", serialmode); + int force_sync = serialmode & SERIALMODE_SYNCDRAIN; + if (force_sync || PE_parse_boot_argn("drain_uart_sync", &force_sync, sizeof(force_sync))) { + if (force_sync) { + serialmode |= SERIALMODE_SYNCDRAIN; + kprintf( + "WARNING: Forcing uart driver to output synchronously." + "printf()s/IOLogs will impact kernel performance.\n" + "You are advised to avoid using 'drain_uart_sync' boot-arg.\n"); + } + } + } + if (kern_feature_override(KF_SERIAL_OVRD)) { + serialmode = 0; + } + + if (serialmode & SERIALMODE_OUTPUT) { /* Start serial if requested */ + (void)switch_to_serial_console(); /* Switch into serial mode */ + disableConsoleOutput = FALSE; /* Allow printfs to happen */ + } + PE_create_console(); + + /* setup console output */ + PE_init_printf(FALSE); + +#if __arm64__ +#if DEBUG + dump_kva_space(); +#endif +#endif + + cpu_machine_idle_init(TRUE); + +#if (__ARM_ARCH__ == 7) + if (arm_diag & 0x8000) + set_mmu_control((get_mmu_control()) ^ SCTLR_PREDIC); +#endif + + PE_init_platform(TRUE, &BootCpuData); + cpu_timebase_init(TRUE); + fiq_context_init(TRUE); + + + /* + * Initialize the stack protector for all future calls + * to C code. Since kernel_bootstrap() eventually + * switches stack context without returning through this + * function, we do not risk failing the check even though + * we mutate the guard word during execution. + */ + __stack_chk_guard = (unsigned long)early_random(); + /* Zero a byte of the protector to guard + * against string vulnerabilities + */ + __stack_chk_guard &= ~(0xFFULL << 8); + machine_startup(args); +} + +/* + * Routine: arm_init_cpu + * Function: + * Re-initialize CPU when coming out of reset + */ + +void +arm_init_cpu( + cpu_data_t *cpu_data_ptr) +{ +#if __ARM_PAN_AVAILABLE__ +#if (DEVELOPMENT || DEBUG) + if (arm_pan_enabled) { + __builtin_arm_wsr("pan", 1); + set_mmu_control((get_mmu_control()) & ~SCTLR_PAN_UNCHANGED); + } +#else + __builtin_arm_wsr("pan", 1); + /* SCTLR_EL1.SPAN is clear on RELEASE */ +#endif +#endif + + cpu_data_ptr->cpu_flags &= ~SleepState; +#if __ARM_SMP__ && defined(ARMA7) + cpu_data_ptr->cpu_CLW_active = 1; +#endif + + machine_set_current_thread(cpu_data_ptr->cpu_active_thread); + +#if __arm64__ + /* Enable asynchronous exceptions */ + __builtin_arm_wsr("DAIFClr", DAIFSC_ASYNCF); +#endif + + cpu_machine_idle_init(FALSE); + + cpu_init(); + +#if (__ARM_ARCH__ == 7) + if (arm_diag & 0x8000) + set_mmu_control((get_mmu_control()) ^ SCTLR_PREDIC); +#endif +#ifdef APPLETYPHOON + if ((cpus_defeatures & (0xF << 4*cpu_data_ptr->cpu_number)) != 0) + cpu_defeatures_set((cpus_defeatures >> 4*cpu_data_ptr->cpu_number) & 0xF); +#endif + /* Initialize the timebase before serial_init, as some serial + * drivers use mach_absolute_time() to implement rate control + */ + cpu_timebase_init(FALSE); + + if (cpu_data_ptr == &BootCpuData) { +#if __arm64__ && __ARM_GLOBAL_SLEEP_BIT__ + /* + * Prevent CPUs from going into deep sleep until all + * CPUs are ready to do so. + */ + arm64_stall_sleep = TRUE; +#endif + serial_init(); + PE_init_platform(TRUE, NULL); + commpage_update_timebase(); + } + + fiq_context_init(TRUE); + cpu_data_ptr->rtcPop = EndOfAllTime; + timer_resync_deadlines(); + +#if DEVELOPMENT || DEBUG + PE_arm_debug_enable_trace(); +#endif + + kprintf("arm_cpu_init(): cpu %d online\n", cpu_data_ptr->cpu_processor->cpu_id); + + if (cpu_data_ptr == &BootCpuData) { +#if CONFIG_TELEMETRY + bootprofile_wake_from_sleep(); +#endif /* CONFIG_TELEMETRY */ +#if MONOTONIC && defined(__arm64__) + mt_wake(); +#endif /* MONOTONIC && defined(__arm64__) */ + } + + slave_main(NULL); +} + +/* + * Routine: arm_init_idle_cpu + * Function: + */ +void __attribute__((noreturn)) +arm_init_idle_cpu( + cpu_data_t *cpu_data_ptr) +{ +#if __ARM_PAN_AVAILABLE__ +#if (DEVELOPMENT || DEBUG) + if (arm_pan_enabled) { + __builtin_arm_wsr("pan", 1); + set_mmu_control((get_mmu_control()) & ~SCTLR_PAN_UNCHANGED); + } +#else + __builtin_arm_wsr("pan", 1); + /* SCTLR_EL1.SPAN is clear on RELEASE */ +#endif +#endif +#if __ARM_SMP__ && defined(ARMA7) + cpu_data_ptr->cpu_CLW_active = 1; +#endif + + machine_set_current_thread(cpu_data_ptr->cpu_active_thread); + +#if __arm64__ + /* Enable asynchronous exceptions */ + __builtin_arm_wsr("DAIFClr", DAIFSC_ASYNCF); +#endif + +#if (__ARM_ARCH__ == 7) + if (arm_diag & 0x8000) + set_mmu_control((get_mmu_control()) ^ SCTLR_PREDIC); +#endif +#ifdef APPLETYPHOON + if ((cpus_defeatures & (0xF << 4*cpu_data_ptr->cpu_number)) != 0) + cpu_defeatures_set((cpus_defeatures >> 4*cpu_data_ptr->cpu_number) & 0xF); +#endif + + fiq_context_init(FALSE); + + cpu_idle_exit(); +} diff --git a/osfmk/arm/arm_timer.c b/osfmk/arm/arm_timer.c new file mode 100644 index 000000000..a38ff056d --- /dev/null +++ b/osfmk/arm/arm_timer.c @@ -0,0 +1,279 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * @APPLE_FREE_COPYRIGHT@ + */ +/* + * File: etimer.c + * Purpose: Routines for handling the machine independent + * event timer. + */ + +#include <mach/mach_types.h> + +#include <kern/clock.h> +#include <kern/thread.h> +#include <kern/processor.h> +#include <kern/macro_help.h> +#include <kern/spl.h> +#include <kern/timer_queue.h> +#include <kern/timer_call.h> + +#include <machine/commpage.h> +#include <machine/machine_routines.h> + +#include <sys/kdebug.h> +#include <arm/cpu_data.h> +#include <arm/cpu_data_internal.h> +#include <arm/cpu_internal.h> + +/* + * Event timer interrupt. + * + * XXX a drawback of this implementation is that events serviced earlier must not set deadlines + * that occur before the entire chain completes. + * + * XXX a better implementation would use a set of generic callouts and iterate over them + */ +void +timer_intr(__unused int inuser, __unused uint64_t iaddr) +{ + uint64_t abstime, new_idle_timeout_ticks; + rtclock_timer_t *mytimer; + cpu_data_t *cpu_data_ptr; + + cpu_data_ptr = getCpuDatap(); + mytimer = &cpu_data_ptr->rtclock_timer; /* Point to the event timer */ + abstime = mach_absolute_time(); /* Get the time now */ + + /* is it time for an idle timer event? */ + if ((cpu_data_ptr->idle_timer_deadline > 0) && (cpu_data_ptr->idle_timer_deadline <= abstime)) { + cpu_data_ptr->idle_timer_deadline = 0x0ULL; + new_idle_timeout_ticks = 0x0ULL; + + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON, MACHDBG_CODE(DBG_MACH_EXCP_DECI, 3) | DBG_FUNC_START, 0, 0, 0, 0, 0); + ((idle_timer_t)cpu_data_ptr->idle_timer_notify)(cpu_data_ptr->idle_timer_refcon, &new_idle_timeout_ticks); + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON, MACHDBG_CODE(DBG_MACH_EXCP_DECI, 3) | DBG_FUNC_END, 0, 0, 0, 0, 0); + + /* if a new idle timeout was requested set the new idle timer deadline */ + if (new_idle_timeout_ticks != 0x0ULL) { + clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline); + } + + abstime = mach_absolute_time(); /* Get the time again since we ran a bit */ + } + + /* has a pending clock timer expired? */ + if (mytimer->deadline <= abstime) { /* Have we expired the + * deadline? */ + mytimer->has_expired = TRUE; /* Remember that we popped */ + mytimer->deadline = EndOfAllTime; /* Set timer request to + * the end of all time + * in case we have no + * more events */ + mytimer->deadline = timer_queue_expire(&mytimer->queue, abstime); + mytimer->has_expired = FALSE; + abstime = mach_absolute_time(); /* Get the time again since we ran a bit */ + } + + uint64_t quantum_deadline = cpu_data_ptr->quantum_timer_deadline; + /* is it the quantum timer expiration? */ + if ((quantum_deadline <= abstime) && (quantum_deadline > 0)) { + cpu_data_ptr->quantum_timer_deadline = 0; + quantum_timer_expire(abstime); + } + + /* Force reload our next deadline */ + cpu_data_ptr->rtcPop = EndOfAllTime; + /* schedule our next deadline */ + timer_resync_deadlines(); +} + +/* + * Set the clock deadline + */ +void +timer_set_deadline(uint64_t deadline) +{ + rtclock_timer_t *mytimer; + spl_t s; + cpu_data_t *cpu_data_ptr; + + s = splclock(); /* no interruptions */ + cpu_data_ptr = getCpuDatap(); + + mytimer = &cpu_data_ptr->rtclock_timer; /* Point to the timer itself */ + mytimer->deadline = deadline; /* Set the new expiration time */ + + timer_resync_deadlines(); + + splx(s); +} + +void +quantum_timer_set_deadline(uint64_t deadline) +{ + cpu_data_t *cpu_data_ptr; + + /* We should've only come into this path with interrupts disabled */ + assert(ml_get_interrupts_enabled() == FALSE); + + cpu_data_ptr = getCpuDatap(); + cpu_data_ptr->quantum_timer_deadline = deadline; + timer_resync_deadlines(); +} + +/* + * Re-evaluate the outstanding deadlines and select the most proximate. + * + * Should be called at splclock. + */ +void +timer_resync_deadlines(void) +{ + uint64_t deadline; + rtclock_timer_t *mytimer; + spl_t s = splclock(); /* No interruptions please */ + cpu_data_t *cpu_data_ptr; + + cpu_data_ptr = getCpuDatap(); + + deadline = 0; + + /* if we have a clock timer set sooner, pop on that */ + mytimer = &cpu_data_ptr->rtclock_timer; /* Point to the timer itself */ + if ((!mytimer->has_expired) && (mytimer->deadline > 0)) + deadline = mytimer->deadline; + + /* if we have a idle timer event coming up, how about that? */ + if ((cpu_data_ptr->idle_timer_deadline > 0) + && (cpu_data_ptr->idle_timer_deadline < deadline)) + deadline = cpu_data_ptr->idle_timer_deadline; + + /* If we have the quantum timer setup, check that */ + if ((cpu_data_ptr->quantum_timer_deadline > 0) + && (cpu_data_ptr->quantum_timer_deadline < deadline)) + deadline = cpu_data_ptr->quantum_timer_deadline; + + if ((deadline == EndOfAllTime) + || ((deadline > 0) && (cpu_data_ptr->rtcPop != deadline))) { + int decr; + + decr = setPop(deadline); + + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + MACHDBG_CODE(DBG_MACH_EXCP_DECI, 1) | DBG_FUNC_NONE, + decr, 2, 0, 0, 0); + } + splx(s); +} + + +boolean_t +timer_resort_threshold(__unused uint64_t skew) { + return FALSE; +} + +mpqueue_head_t * +timer_queue_assign( + uint64_t deadline) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + mpqueue_head_t *queue; + + if (cpu_data_ptr->cpu_running) { + queue = &cpu_data_ptr->rtclock_timer.queue; + + if (deadline < cpu_data_ptr->rtclock_timer.deadline) + timer_set_deadline(deadline); + } + else + queue = &cpu_datap(master_cpu)->rtclock_timer.queue; + + return (queue); +} + +void +timer_queue_cancel( + mpqueue_head_t *queue, + uint64_t deadline, + uint64_t new_deadline) +{ + if (queue == &getCpuDatap()->rtclock_timer.queue) { + if (deadline < new_deadline) + timer_set_deadline(new_deadline); + } +} + +mpqueue_head_t * +timer_queue_cpu(int cpu) +{ + return &cpu_datap(cpu)->rtclock_timer.queue; +} + +void +timer_call_cpu(int cpu, void (*fn)(void *), void *arg) +{ + cpu_signal(cpu_datap(cpu), SIGPxcall, (void *) fn, arg); +} + +void +timer_call_nosync_cpu(int cpu, void (*fn)(void *), void *arg) +{ + /* XXX Needs error checking and retry */ + cpu_signal(cpu_datap(cpu), SIGPxcall, (void *) fn, arg); +} + + +static timer_coalescing_priority_params_ns_t tcoal_prio_params_init = +{ + .idle_entry_timer_processing_hdeadline_threshold_ns = 5000ULL * NSEC_PER_USEC, + .interrupt_timer_coalescing_ilat_threshold_ns = 30ULL * NSEC_PER_USEC, + .timer_resort_threshold_ns = 50 * NSEC_PER_MSEC, + .timer_coalesce_rt_shift = 0, + .timer_coalesce_bg_shift = -5, + .timer_coalesce_kt_shift = 3, + .timer_coalesce_fp_shift = 3, + .timer_coalesce_ts_shift = 3, + .timer_coalesce_rt_ns_max = 0ULL, + .timer_coalesce_bg_ns_max = 100 * NSEC_PER_MSEC, + .timer_coalesce_kt_ns_max = 1 * NSEC_PER_MSEC, + .timer_coalesce_fp_ns_max = 1 * NSEC_PER_MSEC, + .timer_coalesce_ts_ns_max = 1 * NSEC_PER_MSEC, + .latency_qos_scale = {3, 2, 1, -2, -15, -15}, + .latency_qos_ns_max ={1 * NSEC_PER_MSEC, 5 * NSEC_PER_MSEC, 20 * NSEC_PER_MSEC, + 75 * NSEC_PER_MSEC, 10000 * NSEC_PER_MSEC, 10000 * NSEC_PER_MSEC}, + .latency_tier_rate_limited = {FALSE, FALSE, FALSE, FALSE, TRUE, TRUE}, +}; +timer_coalescing_priority_params_ns_t * timer_call_get_priority_params(void) +{ + return &tcoal_prio_params_init; +} diff --git a/osfmk/arm/arm_vm_init.c b/osfmk/arm/arm_vm_init.c new file mode 100644 index 000000000..07bcfb9b2 --- /dev/null +++ b/osfmk/arm/arm_vm_init.c @@ -0,0 +1,537 @@ +/* + * Copyright (c) 2007-2008 Apple Inc. All rights reserved. + * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include <mach_debug.h> +#include <mach_kdp.h> +#include <debug.h> + +#include <mach/vm_types.h> +#include <mach/vm_param.h> +#include <mach/thread_status.h> +#include <kern/misc_protos.h> +#include <kern/assert.h> +#include <kern/cpu_number.h> +#include <kern/thread.h> +#include <vm/vm_map.h> +#include <vm/vm_page.h> +#include <vm/pmap.h> + +#include <arm/proc_reg.h> +#include <arm/caches_internal.h> +#include <arm/pmap.h> +#include <arm/misc_protos.h> +#include <arm/lowglobals.h> + +#include <pexpert/arm/boot.h> + +#include <libkern/kernel_mach_header.h> + +/* + * Denotes the end of xnu. + */ +extern void *last_kernel_symbol; + +/* + * KASLR parameters + */ +vm_offset_t vm_kernel_base; +vm_offset_t vm_kernel_top; +vm_offset_t vm_kernel_stext; +vm_offset_t vm_kernel_etext; +vm_offset_t vm_kernel_slide; +vm_offset_t vm_kernel_slid_base; +vm_offset_t vm_kernel_slid_top; +vm_offset_t vm_kext_base; +vm_offset_t vm_kext_top; +vm_offset_t vm_prelink_stext; +vm_offset_t vm_prelink_etext; +vm_offset_t vm_prelink_sinfo; +vm_offset_t vm_prelink_einfo; +vm_offset_t vm_slinkedit; +vm_offset_t vm_elinkedit; +vm_offset_t vm_prelink_sdata; +vm_offset_t vm_prelink_edata; + +unsigned long gVirtBase, gPhysBase, gPhysSize; /* Used by <mach/arm/vm_param.h> */ + +vm_offset_t mem_size; /* Size of actual physical memory present + * minus any performance buffer and possibly + * limited by mem_limit in bytes */ +uint64_t mem_actual; /* The "One True" physical memory size + * actually, it's the highest physical + * address + 1 */ +uint64_t max_mem; /* Size of physical memory (bytes), adjusted + * by maxmem */ +uint64_t sane_size; /* Memory size to use for defaults + * calculations */ +addr64_t vm_last_addr = VM_MAX_KERNEL_ADDRESS; /* Highest kernel + * virtual address known + * to the VM system */ + +static vm_offset_t segTEXTB; +static unsigned long segSizeTEXT; +static vm_offset_t segDATAB; +static unsigned long segSizeDATA; +static vm_offset_t segLINKB; +static unsigned long segSizeLINK; +static vm_offset_t segKLDB; +static unsigned long segSizeKLD; +static vm_offset_t segLASTB; +static unsigned long segSizeLAST; +static vm_offset_t sectCONSTB; +static unsigned long sectSizeCONST; + +vm_offset_t segPRELINKTEXTB; +unsigned long segSizePRELINKTEXT; +vm_offset_t segPRELINKINFOB; +unsigned long segSizePRELINKINFO; + +static kernel_segment_command_t *segDATA; +static boolean_t doconstro = TRUE; + +vm_offset_t end_kern, etext, sdata, edata; + +/* + * Bootstrap the system enough to run with virtual memory. + * Map the kernel's code and data, and allocate the system page table. + * Page_size must already be set. + * + * Parameters: + * first_avail: first available physical page - + * after kernel page tables + * avail_start: PA of first physical page + * avail_end : PA of last physical page + */ +vm_offset_t first_avail; +vm_offset_t static_memory_end; +pmap_paddr_t avail_start, avail_end; + +#define MEM_SIZE_MAX 0x40000000 + +extern vm_offset_t ExceptionVectorsBase; /* the code we want to load there */ + +/* The translation tables have to be 16KB aligned */ +#define round_x_table(x) \ + (((pmap_paddr_t)(x) + (ARM_PGBYTES<<2) - 1) & ~((ARM_PGBYTES<<2) - 1)) + + +static void +arm_vm_page_granular_helper(vm_offset_t start, vm_offset_t _end, vm_offset_t va, + int pte_prot_APX, int pte_prot_XN) +{ + if (va & ARM_TT_L1_PT_OFFMASK) { /* ragged edge hanging over a ARM_TT_L1_PT_SIZE boundary */ + va &= (~ARM_TT_L1_PT_OFFMASK); + tt_entry_t *tte = &cpu_tte[ttenum(va)]; + tt_entry_t tmplate = *tte; + pmap_paddr_t pa; + pt_entry_t *ppte, ptmp; + unsigned int i; + + pa = va - gVirtBase + gPhysBase; + + if (ARM_TTE_TYPE_TABLE == (tmplate & ARM_TTE_TYPE_MASK)) { + /* pick up the existing page table. */ + ppte = (pt_entry_t *)phystokv((tmplate & ARM_TTE_TABLE_MASK)); + } else { + /* TTE must be reincarnated COARSE. */ + ppte = (pt_entry_t *)phystokv(avail_start); + avail_start += ARM_PGBYTES; + + pmap_init_pte_static_page(kernel_pmap, ppte, pa); + + for (i = 0; i < 4; ++i) + tte[i] = pa_to_tte(kvtophys((vm_offset_t)ppte) + (i * 0x400)) | ARM_TTE_TYPE_TABLE; + } + + /* Apply the desired protections to the specified page range */ + for (i = 0; i < (ARM_PGBYTES / sizeof(*ppte)); i++) { + if (start <= va && va < _end) { + + ptmp = pa | ARM_PTE_AF | ARM_PTE_SH | ARM_PTE_TYPE; + ptmp = ptmp | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT); + ptmp = ptmp | ARM_PTE_AP(pte_prot_APX); + if (pte_prot_XN) + ptmp = ptmp | ARM_PTE_NX; + + ppte[i] = ptmp; + } + + va += ARM_PGBYTES; + pa += ARM_PGBYTES; + } + } +} + +static void +arm_vm_page_granular_prot(vm_offset_t start, unsigned long size, + int tte_prot_XN, int pte_prot_APX, int pte_prot_XN, int forceCoarse) +{ + vm_offset_t _end = start + size; + vm_offset_t align_start = (start + ARM_TT_L1_PT_OFFMASK) & ~ARM_TT_L1_PT_OFFMASK; + vm_offset_t align_end = _end & ~ARM_TT_L1_PT_OFFMASK; + + arm_vm_page_granular_helper(start, _end, start, pte_prot_APX, pte_prot_XN); + + while (align_start < align_end) { + if (forceCoarse) { + arm_vm_page_granular_helper(align_start, align_end, align_start + 1, + pte_prot_APX, pte_prot_XN); + } else { + tt_entry_t *tte = &cpu_tte[ttenum(align_start)]; + for (int i = 0; i < 4; ++i) { + tt_entry_t tmplate = tte[i]; + + tmplate = (tmplate & ~ARM_TTE_BLOCK_APMASK) | ARM_TTE_BLOCK_AP(pte_prot_APX); + tmplate = (tmplate & ~ARM_TTE_BLOCK_NX_MASK); + if (tte_prot_XN) + tmplate = tmplate | ARM_TTE_BLOCK_NX; + + tte[i] = tmplate; + } + } + align_start += ARM_TT_L1_PT_SIZE; + } + + arm_vm_page_granular_helper(start, _end, _end, pte_prot_APX, pte_prot_XN); +} + +static inline void +arm_vm_page_granular_RNX(vm_offset_t start, unsigned long size, int forceCoarse) +{ + arm_vm_page_granular_prot(start, size, 1, AP_RONA, 1, forceCoarse); +} + +static inline void +arm_vm_page_granular_ROX(vm_offset_t start, unsigned long size, int forceCoarse) +{ + arm_vm_page_granular_prot(start, size, 0, AP_RONA, 0, forceCoarse); +} + +static inline void +arm_vm_page_granular_RWNX(vm_offset_t start, unsigned long size, int forceCoarse) +{ + arm_vm_page_granular_prot(start, size, 1, AP_RWNA, 1, forceCoarse); +} + +static inline void +arm_vm_page_granular_RWX(vm_offset_t start, unsigned long size, int forceCoarse) +{ + arm_vm_page_granular_prot(start, size, 0, AP_RWNA, 0, forceCoarse); +} + +void +arm_vm_prot_init(boot_args * args) +{ +#if __ARM_PTE_PHYSMAP__ + boolean_t force_coarse_physmap = TRUE; +#else + boolean_t force_coarse_physmap = FALSE; +#endif + /* + * Enforce W^X protections on segments that have been identified so far. This will be + * further refined for each KEXT's TEXT and DATA segments in readPrelinkedExtensions() + */ + + /* + * Protection on kernel text is loose here to allow shenanigans early on (e.g. copying exception vectors) + * and storing an address into "error_buffer" (see arm_init.c) !?! + * These protections are tightened in arm_vm_prot_finalize() + */ + arm_vm_page_granular_RWX(gVirtBase, segSizeTEXT + (segTEXTB - gVirtBase), FALSE); + + if (doconstro) { + /* + * We map __DATA with 3 calls, so that the __const section can have its + * protections changed independently of the rest of the __DATA segment. + */ + arm_vm_page_granular_RWNX(segDATAB, sectCONSTB - segDATAB, FALSE); + arm_vm_page_granular_RNX(sectCONSTB, sectSizeCONST, FALSE); + arm_vm_page_granular_RWNX(sectCONSTB + sectSizeCONST, (segDATAB + segSizeDATA) - (sectCONSTB + sectSizeCONST), FALSE); + } else { + /* If we aren't protecting const, just map DATA as a single blob. */ + arm_vm_page_granular_RWNX(segDATAB, segSizeDATA, FALSE); + } + + arm_vm_page_granular_ROX(segKLDB, segSizeKLD, force_coarse_physmap); + arm_vm_page_granular_RWNX(segLINKB, segSizeLINK, force_coarse_physmap); + arm_vm_page_granular_RWNX(segLASTB, segSizeLAST, FALSE); // __LAST may be empty, but we cannot assume this + arm_vm_page_granular_RWNX(segPRELINKTEXTB, segSizePRELINKTEXT, TRUE); // Refined in OSKext::readPrelinkedExtensions + arm_vm_page_granular_RWNX(segPRELINKTEXTB + segSizePRELINKTEXT, + end_kern - (segPRELINKTEXTB + segSizePRELINKTEXT), force_coarse_physmap); // PreLinkInfoDictionary + arm_vm_page_granular_RWNX(end_kern, phystokv(args->topOfKernelData) - end_kern, force_coarse_physmap); // Device Tree, RAM Disk (if present), bootArgs + arm_vm_page_granular_RWNX(phystokv(args->topOfKernelData), ARM_PGBYTES * 8, FALSE); // boot_tte, cpu_tte + + /* + * FIXME: Any page table pages that arm_vm_page_granular_* created with ROX entries in the range + * phystokv(args->topOfKernelData) to phystokv(prot_avail_start) should themselves be + * write protected in the static mapping of that range. + * [Page table pages whose page table entries grant execute (X) privileges should themselves be + * marked read-only. This aims to thwart attacks that replace the X entries with vectors to evil code + * (relying on some thread of execution to eventually arrive at what previously was a trusted routine).] + */ + arm_vm_page_granular_RWNX(phystokv(args->topOfKernelData) + ARM_PGBYTES * 8, ARM_PGBYTES, FALSE); /* Excess physMem over 1MB */ + arm_vm_page_granular_RWX(phystokv(args->topOfKernelData) + ARM_PGBYTES * 9, ARM_PGBYTES, FALSE); /* refined in finalize */ + + /* Map the remainder of xnu owned memory. */ + arm_vm_page_granular_RWNX(phystokv(args->topOfKernelData) + ARM_PGBYTES * 10, + static_memory_end - (phystokv(args->topOfKernelData) + ARM_PGBYTES * 10), force_coarse_physmap); /* rest of physmem */ + + /* + * Special case write protection for the mapping of ExceptionVectorsBase (EVB) at 0xFFFF0000. + * Recall that start.s handcrafted a page table page for EVB mapping + */ + pmap_paddr_t p = (pmap_paddr_t)(args->topOfKernelData) + (ARM_PGBYTES * 9); + pt_entry_t *ppte = (pt_entry_t *)phystokv(p); + + int idx = (HIGH_EXC_VECTORS & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT; + pt_entry_t ptmp = ppte[idx]; + + ptmp = (ptmp & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RONA); + + ppte[idx] = ptmp; +} + +void +arm_vm_prot_finalize(boot_args * args) +{ + /* + * Naively we could have: + * arm_vm_page_granular_ROX(segTEXTB, segSizeTEXT, FALSE); + * but, at present, that would miss a 1Mb boundary at the beginning of the segment and + * so would force a (wasteful) coarse page (e.g. when gVirtBase is 0x80000000, segTEXTB is 0x80001000). + */ + arm_vm_page_granular_ROX(gVirtBase, segSizeTEXT + (segTEXTB - gVirtBase), FALSE); + + arm_vm_page_granular_RWNX(phystokv(args->topOfKernelData) + ARM_PGBYTES * 9, ARM_PGBYTES, FALSE); /* commpage, EVB */ + +#ifndef __ARM_L1_PTW__ + FlushPoC_Dcache(); +#endif + flush_mmu_tlb(); +} + +void +arm_vm_init(uint64_t memory_size, boot_args * args) +{ + vm_map_address_t va, off, off_end; + tt_entry_t *tte, *tte_limit; + pmap_paddr_t boot_ttep; + tt_entry_t *boot_tte; + uint32_t mem_segments; + kernel_section_t *sectDCONST; + + /* + * Get the virtual and physical memory base from boot_args. + */ + gVirtBase = args->virtBase; + gPhysBase = args->physBase; + gPhysSize = args->memSize; + mem_size = args->memSize; + if ((memory_size != 0) && (mem_size > memory_size)) + mem_size = memory_size; + if (mem_size > MEM_SIZE_MAX ) + mem_size = MEM_SIZE_MAX; + static_memory_end = gVirtBase + mem_size; + + /* Calculate the nubmer of ~256MB segments of memory */ + mem_segments = (mem_size + 0x0FFFFFFF) >> 28; + + /* + * Copy the boot mmu tt to create system mmu tt. + * System mmu tt start after the boot mmu tt. + * Determine translation table base virtual address: - aligned at end + * of executable. + */ + boot_ttep = args->topOfKernelData; + boot_tte = (tt_entry_t *) phystokv(boot_ttep); + + cpu_ttep = boot_ttep + ARM_PGBYTES * 4; + cpu_tte = (tt_entry_t *) phystokv(cpu_ttep); + + bcopy(boot_tte, cpu_tte, ARM_PGBYTES * 4); + + /* + * Clear out any V==P mappings that may have been established in e.g. start.s + */ + tte = &cpu_tte[ttenum(gPhysBase)]; + tte_limit = &cpu_tte[ttenum(gPhysBase + gPhysSize)]; + + /* Hands off [gVirtBase, gVirtBase + gPhysSize) please. */ + if (gPhysBase < gVirtBase) { + if (gPhysBase + gPhysSize > gVirtBase) + tte_limit = &cpu_tte[ttenum(gVirtBase)]; + } else { + if (gPhysBase < gVirtBase + gPhysSize) + tte = &cpu_tte[ttenum(gVirtBase + gPhysSize)]; + } + + while (tte < tte_limit) { + *tte = ARM_TTE_TYPE_FAULT; + tte++; + } + + /* Skip 6 pages (four L1 + two L2 entries) */ + avail_start = cpu_ttep + ARM_PGBYTES * 6; + avail_end = gPhysBase + mem_size; + + /* + * Now retrieve addresses for end, edata, and etext + * from MACH-O headers for the currently running 32 bit kernel. + */ + segTEXTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__TEXT", &segSizeTEXT); + segDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__DATA", &segSizeDATA); + segLINKB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__LINKEDIT", &segSizeLINK); + segKLDB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__KLD", &segSizeKLD); + segLASTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__LAST", &segSizeLAST); + segPRELINKTEXTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PRELINK_TEXT", &segSizePRELINKTEXT); + segPRELINKINFOB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PRELINK_INFO", &segSizePRELINKINFO); + + etext = (vm_offset_t) segTEXTB + segSizeTEXT; + sdata = (vm_offset_t) segDATAB; + edata = (vm_offset_t) segDATAB + segSizeDATA; + end_kern = round_page(getlastaddr()); /* Force end to next page */ + + /* + * Special handling for the __DATA,__const *section*. + * A page of padding named lastkerneldataconst is at the end of the __DATA,__const + * so we can safely truncate the size. __DATA,__const is also aligned, but + * just in case we will round that to a page, too. + */ + segDATA = getsegbynamefromheader(&_mh_execute_header, "__DATA"); + sectDCONST = getsectbynamefromheader(&_mh_execute_header, "__DATA", "__const"); + sectCONSTB = sectDCONST->addr; + sectSizeCONST = sectDCONST->size; + +#if !SECURE_KERNEL + /* doconstro is true by default, but we allow a boot-arg to disable it */ + (void) PE_parse_boot_argn("dataconstro", &doconstro, sizeof(doconstro)); +#endif + + if (doconstro) { + extern vm_offset_t _lastkerneldataconst; + extern vm_size_t _lastkerneldataconst_padsize; + vm_offset_t sdataconst = sectCONSTB; + + /* this should already be aligned, but so that we can protect we round */ + sectCONSTB = round_page(sectCONSTB); + + /* make sure lastkerneldataconst is really last and the right size */ + if ((_lastkerneldataconst == sdataconst + sectSizeCONST - _lastkerneldataconst_padsize) && + (_lastkerneldataconst_padsize >= PAGE_SIZE)) { + sectSizeCONST = trunc_page(sectSizeCONST); + } else { + /* otherwise see if next section is aligned then protect up to it */ + kernel_section_t *next_sect = nextsect(segDATA, sectDCONST); + + if (next_sect && ((next_sect->addr & PAGE_MASK) == 0)) { + sectSizeCONST = next_sect->addr - sectCONSTB; + } else { + /* lastly just go ahead and truncate so we try to protect something */ + sectSizeCONST = trunc_page(sectSizeCONST); + } + } + + /* sanity check */ + if ((sectSizeCONST == 0) || (sectCONSTB < sdata) || (sectCONSTB + sectSizeCONST) >= edata) { + doconstro = FALSE; + } + } + + vm_set_page_size(); + +#ifndef __ARM_L1_PTW__ + FlushPoC_Dcache(); +#endif + set_mmu_ttb(cpu_ttep); + set_mmu_ttb_alternate(cpu_ttep); + flush_mmu_tlb(); +#if __arm__ && __ARM_USER_PROTECT__ + { + unsigned int ttbr0_val, ttbr1_val, ttbcr_val; + thread_t thread = current_thread(); + + __asm__ volatile("mrc p15,0,%0,c2,c0,0\n" : "=r"(ttbr0_val)); + __asm__ volatile("mrc p15,0,%0,c2,c0,1\n" : "=r"(ttbr1_val)); + __asm__ volatile("mrc p15,0,%0,c2,c0,2\n" : "=r"(ttbcr_val)); + thread->machine.uptw_ttb = ttbr0_val; + thread->machine.kptw_ttb = ttbr1_val; + thread->machine.uptw_ttc = ttbcr_val; + } +#endif + vm_prelink_stext = segPRELINKTEXTB; + vm_prelink_etext = segPRELINKTEXTB + segSizePRELINKTEXT; + vm_prelink_sinfo = segPRELINKINFOB; + vm_prelink_einfo = segPRELINKINFOB + segSizePRELINKINFO; + vm_slinkedit = segLINKB; + vm_elinkedit = segLINKB + segSizeLINK; + + sane_size = mem_size - (avail_start - gPhysBase); + max_mem = mem_size; + vm_kernel_slide = gVirtBase-0x80000000; + vm_kernel_stext = segTEXTB; + vm_kernel_etext = segTEXTB + segSizeTEXT; + vm_kernel_base = gVirtBase; + vm_kernel_top = (vm_offset_t) &last_kernel_symbol; + vm_kext_base = segPRELINKTEXTB; + vm_kext_top = vm_kext_base + segSizePRELINKTEXT; + vm_kernel_slid_base = segTEXTB; + vm_kernel_slid_top = vm_kext_top; + + pmap_bootstrap((gVirtBase+MEM_SIZE_MAX+0x3FFFFF) & 0xFFC00000); + + arm_vm_prot_init(args); + + /* + * To avoid recursing while trying to init the vm_page and object * mechanisms, + * pre-initialize kernel pmap page table pages to cover this address range: + * 2MB + FrameBuffer size + 3MB for each 256MB segment + */ + off_end = (2 + (mem_segments * 3)) << 20; + off_end += (unsigned int) round_page(args->Video.v_height * args->Video.v_rowBytes); + + for (off = 0, va = (gVirtBase+MEM_SIZE_MAX+0x3FFFFF) & 0xFFC00000; off < off_end; off += ARM_TT_L1_PT_SIZE) { + pt_entry_t *ptp; + pmap_paddr_t ptp_phys; + + ptp = (pt_entry_t *) phystokv(avail_start); + ptp_phys = (pmap_paddr_t)avail_start; + avail_start += ARM_PGBYTES; + pmap_init_pte_page(kernel_pmap, ptp, va + off, 2, TRUE); + tte = &cpu_tte[ttenum(va + off)]; + *tte = pa_to_tte((ptp_phys )) | ARM_TTE_TYPE_TABLE;; + *(tte+1) = pa_to_tte((ptp_phys + 0x400)) | ARM_TTE_TYPE_TABLE;; + *(tte+2) = pa_to_tte((ptp_phys + 0x800)) | ARM_TTE_TYPE_TABLE;; + *(tte+3) = pa_to_tte((ptp_phys + 0xC00)) | ARM_TTE_TYPE_TABLE;; + } + + avail_start = (avail_start + PAGE_MASK) & ~PAGE_MASK; + + first_avail = avail_start; + patch_low_glo_static_region(args->topOfKernelData, avail_start - args->topOfKernelData); +} + diff --git a/osfmk/arm/asm.h b/osfmk/arm/asm.h new file mode 100644 index 000000000..f27a8763a --- /dev/null +++ b/osfmk/arm/asm.h @@ -0,0 +1,320 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990,1989 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ + +#ifndef _ARM_ASM_H_ +#define _ARM_ASM_H_ + +#include <arm/arch.h> + +#define FRAME pushl %ebp; movl %esp, %ebp +#define EMARF leave + + +/* There is another definition of ALIGN for .c sources */ +#ifdef ASSEMBLER +#define ALIGN 2 +#endif /* ASSEMBLER */ + +#ifndef FALIGN +#define FALIGN ALIGN +#endif + +#define LB(x,n) n +#if __STDC__ +#ifndef __NO_UNDERSCORES__ +#define LCL(x) L ## x +#define EXT(x) _ ## x +#define LEXT(x) _ ## x ## : +#else +#define LCL(x) .L ## x +#define EXT(x) x +#define LEXT(x) x ## : +#endif +#define LBc(x,n) n ## : +#define LBb(x,n) n ## b +#define LBf(x,n) n ## f +#else /* __STDC__ */ +#ifndef __NO_UNDERSCORES__ +#define LCL(x) L/**/x +#define EXT(x) _/**/x +#define LEXT(x) _/**/x/**/: +#else /* __NO_UNDERSCORES__ */ +#define LCL(x) .L/**/x +#define EXT(x) x +#define LEXT(x) x/**/: +#endif /* __NO_UNDERSCORES__ */ +#define LBc(x,n) n/**/: +#define LBb(x,n) n/**/b +#define LBf(x,n) n/**/f +#endif /* __STDC__ */ + +#define String .asciz +#define Value .word +#define Times(a,b) (a*b) +#define Divide(a,b) (a/b) + +#if 0 /* TOTOJK */ +#ifdef __ELF__ +#define ELF_FUNC(x) .type x,@function +#define ELF_DATA(x) .type x,@object +#define ELF_SIZE(x,s) .size x,s +#else +#define ELF_FUNC(x) +#define ELF_DATA(x) +#define ELF_SIZE(x,s) +#endif +#else +#define ELF_FUNC(x) +#define ELF_DATA(x) +#define ELF_SIZE(x,s) +#endif /* TODOJK */ + +#define Entry(x) .globl EXT(x); ELF_FUNC(EXT(x)); .align FALIGN; LEXT(x) +#define ENTRY(x) Entry(x) MCOUNT +#define ENTRY2(x,y) .globl EXT(x); .globl EXT(y); \ + ELF_FUNC(EXT(x)); ELF_FUNC(EXT(y)); \ + .align FALIGN; LEXT(x); LEXT(y) \ + MCOUNT +#if __STDC__ +#define ASENTRY(x) .globl x; .align FALIGN; x ## : ELF_FUNC(x) MCOUNT +#else +#define ASENTRY(x) .globl x; .align FALIGN; x: ELF_FUNC(x) MCOUNT +#endif /* __STDC__ */ + +#define DATA(x) .globl EXT(x); ELF_DATA(EXT(x)); .align ALIGN; LEXT(x) + +#define End(x) ELF_SIZE(x,.-x) +#define END(x) End(EXT(x)) +#define ENDDATA(x) END(x) +#define Enddata(x) End(x) + +#ifdef ASSEMBLER + +#define MCOUNT + +#else /* NOT ASSEMBLER */ + +/* These defines are here for .c files that wish to reference global symbols + * within __asm__ statements. + */ +#ifndef __NO_UNDERSCORES__ +#define CC_SYM_PREFIX "_" +#else +#define CC_SYM_PREFIX "" +#endif /* __NO_UNDERSCORES__ */ +#endif /* ASSEMBLER */ + +#ifdef ASSEMBLER + +#if defined (_ARM_ARCH_4T) +# define RET bx lr +# define RETeq bxeq lr +# define RETne bxne lr +# ifdef __STDC__ +# define RETc(c) bx##c lr +# else +# define RETc(c) bx/**/c lr +# endif +#else +# define RET mov pc, lr +# define RETeq moveq pc, lr +# define RETne movne pc, lr +# ifdef __STDC__ +# define RETc(c) mov##c pc, lr +# else +# define RETc(c) mov/**/c pc, lr +# endif +#endif + +#if defined (__thumb__) +/* Provide a PI mechanism for thumb branching. */ +# define BRANCH_EXTERN(x) ldr pc, [pc, #-4] ; \ + .long EXT(x) +#else +# define BRANCH_EXTERN(x) b EXT(x) +#endif + +/* + * arg0: Register for thread pointer + */ +.macro READ_THREAD + mrc p15, 0, $0, c13, c0, 4 /* Read TPIDRPRW */ +.endmacro + + +/* Macros for loading up addresses that are external to the .s file. + * LOAD_ADDR: loads the address for (label) into (reg). Not safe for + * loading to the PC. + * LOAD_ADDR_PC: Variant for loading to the PC; load the address of (label) + * into the pc. + * LOAD_ADDR_GEN_DEF: The general definition needed to support loading + * a label address. + * + * Usage: For any label accessed, we require one (and only one) instance + * of LOAD_ADDR_GEN_DEF(label). + * + * Example: + * LOAD_ADDR(r0, arm_init) + * LOAD_ADDR(lr, arm_init_cpu) + * LOAD_ADDR_PC(arm_init) + * ... + * + * LOAD_ADDR_GEN_DEF(arm_init) + * LOAD_ADDR_GEN_DEF(arm_init_cpu) + */ + +#if SLIDABLE +/* Definitions for a position dependent kernel using non-lazy pointers. + */ + +/* TODO: Make this work with thumb .s files. */ +#define PC_INC 0x8 + +/* We need wrapper macros in order to ensure that __LINE__ is expanded. + * + * There is some small potential for duplicate labels here, but because + * we do not export the generated labels, it should not be an issue. + */ + +#define GLUE_LABEL_GUTS(label, tag) L_##label##_##tag##_glue +#define GLUE_LABEL(label, tag) GLUE_LABEL_GUTS(label, tag) + +#define LOAD_ADDR(reg, label) \ + movw reg, :lower16:(label##$non_lazy_ptr - (GLUE_LABEL(label, __LINE__) + PC_INC)) ; \ + movt reg, :upper16:(label##$non_lazy_ptr - (GLUE_LABEL(label, __LINE__) + PC_INC)) ; \ +GLUE_LABEL(label, __LINE__): ; \ + ldr reg, [pc, reg] + +/* Designed with the understanding that directly branching to thumb code + * is unreliable; this should allow for dealing with __thumb__ in + * assembly; the non-thumb variant still needs to provide the glue label + * to avoid failing to build on undefined symbols. + * + * TODO: Make this actually use a scratch register; this macro is convenient + * for translating (ldr pc, [?]) to a slidable format without the risk of + * clobbering registers, but it is also wasteful. + */ +#if defined(__thumb__) +#define LOAD_ADDR_PC(label) \ + stmfd sp!, { r0 } ; \ + stmfd sp!, { r0 } ; \ + LOAD_ADDR(r0, label) ; \ + str r0, [sp, #4] ; \ + ldmfd sp!, { r0 } ; \ + ldmfd sp!, { pc } +#else +#define LOAD_ADDR_PC(label) \ + b EXT(label) +#endif + +#define LOAD_ADDR_GEN_DEF(label) \ + .section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers ; \ + .align 2 ; \ +label##$non_lazy_ptr: ; \ + .indirect_symbol EXT(label) ; \ + .long 0 + +#else /* !SLIDABLE */ + +/* Definitions for a position dependent kernel */ +#define LOAD_ADDR(reg, label) \ + ldr reg, L_##label + +#if defined(__thumb__) +#define LOAD_ADDR_PC(label) \ + ldr pc, L_##label +#else +#define LOAD_ADDR_PC(label) \ + b EXT(label) +#endif + +#define LOAD_ADDR_GEN_DEF(label) \ + .text ; \ + .align 2 ; \ +L_##label: ; \ + .long EXT(label) + +#endif /* SLIDABLE */ + +/* The linker can deal with branching from ARM to thumb in unconditional + * branches, but not in conditional branches. To support this in our + * assembly (which allows us to build xnu without -mno-thumb), use the + * following macros for branching conditionally to external symbols. + * These macros are used just like the corresponding conditional branch + * instructions. + */ + +#define SHIM_LABEL_GUTS(line_num) L_cond_extern_##line_num##_shim +#define SHIM_LABEL(line_num) SHIM_LABEL_GUTS(line_num) + +#define COND_EXTERN_BEQ(label) \ + bne SHIM_LABEL(__LINE__) ; \ + b EXT(label) ; \ +SHIM_LABEL(__LINE__): + +#define COND_EXTERN_BLNE(label) \ + beq SHIM_LABEL(__LINE__) ; \ + bl EXT(label) ; \ +SHIM_LABEL(__LINE__): + +#define COND_EXTERN_BLGT(label) \ + ble SHIM_LABEL(__LINE__) ; \ + bl EXT(label) ; \ +SHIM_LABEL(__LINE__): + +#endif /* ASSEMBLER */ + +#endif /* _ARM_ASM_H_ */ diff --git a/osfmk/arm/atomic.h b/osfmk/arm/atomic.h new file mode 100644 index 000000000..3da426b3d --- /dev/null +++ b/osfmk/arm/atomic.h @@ -0,0 +1,261 @@ +/* + * Copyright (c) 2015 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef _ARM_ATOMIC_H_ +#define _ARM_ATOMIC_H_ + +#include <arm/smp.h> + +// Parameter for __builtin_arm_dmb +#define DMB_NSH 0x7 +#define DMB_ISHLD 0x9 +#define DMB_ISHST 0xa +#define DMB_ISH 0xb +#define DMB_SY 0xf + +// Parameter for __builtin_arm_dsb +#define DSB_NSH 0x7 +#define DSB_ISHLD 0x9 +#define DSB_ISHST 0xa +#define DSB_ISH 0xb +#define DSB_SY 0xf + +// Parameter for __builtin_arm_isb +#define ISB_SY 0xf + +#if __SMP__ + +#define memory_order_consume_smp memory_order_consume +#define memory_order_acquire_smp memory_order_acquire +#define memory_order_release_smp memory_order_release +#define memory_order_acq_rel_smp memory_order_acq_rel +#define memory_order_seq_cst_smp memory_order_seq_cst + +#else + +#define memory_order_consume_smp memory_order_relaxed +#define memory_order_acquire_smp memory_order_relaxed +#define memory_order_release_smp memory_order_relaxed +#define memory_order_acq_rel_smp memory_order_relaxed +#define memory_order_seq_cst_smp memory_order_relaxed + +#endif + +/* + * Atomic operations functions + * + * These static functions are designed for inlining + * It is expected that the memory_order arguments are + * known at compile time. This collapses these + * functions into a simple atomic operation + */ + +static inline boolean_t +memory_order_has_acquire(enum memory_order ord) +{ + switch (ord) { + case memory_order_consume: + case memory_order_acquire: + case memory_order_acq_rel: + case memory_order_seq_cst: + return TRUE; + default: + return FALSE; + } +} + +static inline boolean_t +memory_order_has_release(enum memory_order ord) +{ + switch (ord) { + case memory_order_release: + case memory_order_acq_rel: + case memory_order_seq_cst: + return TRUE; + default: + return FALSE; + } +} + +#ifdef ATOMIC_PRIVATE + +#define clear_exclusive() __builtin_arm_clrex() + +__unused static uint32_t +load_exclusive32(uint32_t *target, enum memory_order ord) +{ + uint32_t value; + +#if __arm__ + if (memory_order_has_release(ord)) { + // Pre-load release barrier + atomic_thread_fence(memory_order_release); + } + value = __builtin_arm_ldrex(target); +#else + if (memory_order_has_acquire(ord)) + value = __builtin_arm_ldaex(target); // ldaxr + else + value = __builtin_arm_ldrex(target); // ldxr +#endif // __arm__ + return value; +} + +__unused static boolean_t +store_exclusive32(uint32_t *target, uint32_t value, enum memory_order ord) +{ + boolean_t err; + +#if __arm__ + err = __builtin_arm_strex(value, target); + if (memory_order_has_acquire(ord)) { + // Post-store acquire barrier + atomic_thread_fence(memory_order_acquire); + } +#else + if (memory_order_has_release(ord)) + err = __builtin_arm_stlex(value, target); // stlxr + else + err = __builtin_arm_strex(value, target); // stxr +#endif // __arm__ + return !err; +} + +__unused static uintptr_t +load_exclusive(uintptr_t *target, enum memory_order ord) +{ +#if !__LP64__ + return load_exclusive32((uint32_t *)target, ord); +#else + uintptr_t value; + + if (memory_order_has_acquire(ord)) + value = __builtin_arm_ldaex(target); // ldaxr + else + value = __builtin_arm_ldrex(target); // ldxr + return value; +#endif // __arm__ +} + +__unused static boolean_t +store_exclusive(uintptr_t *target, uintptr_t value, enum memory_order ord) +{ +#if !__LP64__ + return store_exclusive32((uint32_t *)target, value, ord); +#else + boolean_t err; + + if (memory_order_has_release(ord)) + err = __builtin_arm_stlex(value, target); // stlxr + else + err = __builtin_arm_strex(value, target); // stxr + return !err; +#endif +} + +__unused static boolean_t +atomic_compare_exchange(uintptr_t *target, uintptr_t oldval, uintptr_t newval, + enum memory_order orig_ord, boolean_t wait) +{ + enum memory_order ord = orig_ord; + uintptr_t value; + + +#if __arm__ + ord = memory_order_relaxed; + if (memory_order_has_release(orig_ord)) { + atomic_thread_fence(memory_order_release); + } +#endif + do { + value = load_exclusive(target, ord); + if (value != oldval) { + if (wait) + wait_for_event(); // Wait with monitor held + else + clear_exclusive(); // Clear exclusive monitor + return FALSE; + } + } while (!store_exclusive(target, newval, ord)); +#if __arm__ + if (memory_order_has_acquire(orig_ord)) { + atomic_thread_fence(memory_order_acquire); + } +#endif + return TRUE; +} + +#endif // ATOMIC_PRIVATE + +#if __arm__ +#define os_atomic_rmw_loop(p, ov, nv, m, ...) ({ \ + boolean_t _result = FALSE; uint32_t _err = 0; \ + typeof(atomic_load(p)) *_p = (typeof(atomic_load(p)) *)(p); \ + for (;;) { \ + ov = __builtin_arm_ldrex(_p); \ + __VA_ARGS__; \ + if (!_err && memory_order_has_release(memory_order_##m)) { \ + /* only done for the first loop iteration */ \ + atomic_thread_fence(memory_order_release); \ + } \ + _err = __builtin_arm_strex(nv, _p); \ + if (__builtin_expect(!_err, 1)) { \ + if (memory_order_has_acquire(memory_order_##m)) { \ + atomic_thread_fence(memory_order_acquire); \ + } \ + _result = TRUE; \ + break; \ + } \ + } \ + _result; \ + }) +#else +#define os_atomic_rmw_loop(p, ov, nv, m, ...) ({ \ + boolean_t _result = FALSE; \ + typeof(atomic_load(p)) *_p = (typeof(atomic_load(p)) *)(p); \ + do { \ + if (memory_order_has_acquire(memory_order_##m)) { \ + ov = __builtin_arm_ldaex(_p); \ + } else { \ + ov = __builtin_arm_ldrex(_p); \ + } \ + __VA_ARGS__; \ + if (memory_order_has_release(memory_order_##m)) { \ + _result = !__builtin_arm_stlex(nv, _p); \ + } else { \ + _result = !__builtin_arm_strex(nv, _p); \ + } \ + } while (__builtin_expect(!_result, 0)); \ + _result; \ + }) +#endif + +#define os_atomic_rmw_loop_give_up(expr) \ + ({ __builtin_arm_clrex(); expr; __builtin_trap(); }) + +#endif // _ARM_ATOMIC_H_ diff --git a/osfmk/arm/bcopy.s b/osfmk/arm/bcopy.s new file mode 100644 index 000000000..013232499 --- /dev/null +++ b/osfmk/arm/bcopy.s @@ -0,0 +1,402 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include <arm/proc_reg.h> + +.syntax unified +.text +.align 2 + + .globl _ovbcopy + .globl _memcpy + .globl _bcopy + .globl _memmove + +_bcopy: /* void bcopy(const void *src, void *dest, size_t len); */ +_ovbcopy: + mov r3, r0 + mov r0, r1 + mov r1, r3 + +_memcpy: /* void *memcpy(void *dest, const void *src, size_t len); */ +_memmove: /* void *memmove(void *dest, const void *src, size_t len); */ + /* check for zero len or if the pointers are the same */ + cmp r2, #0 + cmpne r0, r1 + bxeq lr + + /* save r0 (return value), r4 (scratch), and r5 (scratch) */ + stmfd sp!, { r0, r4, r5, r7, lr } + add r7, sp, #12 + + /* check for overlap. r3 <- distance between src & dest */ + subhs r3, r0, r1 + sublo r3, r1, r0 + cmp r3, r2 /* if distance(src, dest) < len, we have overlap */ + blo Loverlap + +Lnormalforwardcopy: + /* are src and dest dissimilarly word aligned? */ + mov r12, r0, lsl #30 + cmp r12, r1, lsl #30 + bne Lnonwordaligned_forward + + /* if len < 64, do a quick forward copy */ + cmp r2, #64 + blt Lsmallforwardcopy + + /* check for 16 byte src/dest unalignment */ + tst r0, #0xf + bne Lsimilarlyunaligned + + /* check for 32 byte dest unalignment */ + tst r0, #(1<<4) + bne Lunaligned_32 + +Lmorethan64_aligned: + /* save some more registers to use in the copy */ + stmfd sp!, { r6, r8, r10, r11 } + + /* pre-subtract 64 from the len counter to avoid an extra compare in the loop */ + sub r2, r2, #64 + +L64loop: + /* copy 64 bytes at a time */ + ldmia r1!, { r3, r4, r5, r6, r8, r10, r11, r12 } + pld [r1, #32] + stmia r0!, { r3, r4, r5, r6, r8, r10, r11, r12 } + ldmia r1!, { r3, r4, r5, r6, r8, r10, r11, r12 } + subs r2, r2, #64 + pld [r1, #32] + stmia r0!, { r3, r4, r5, r6, r8, r10, r11, r12 } + bge L64loop + + /* restore the scratch registers we just saved */ + ldmfd sp!, { r6, r8, r10, r11 } + + /* fix up the len counter (previously subtracted an extra 64 from it) and test for completion */ + adds r2, r2, #64 + beq Lexit + +Llessthan64_aligned: + /* copy 16 bytes at a time until we have < 16 bytes */ + cmp r2, #16 + ldmiage r1!, { r3, r4, r5, r12 } + stmiage r0!, { r3, r4, r5, r12 } + subsge r2, r2, #16 + bgt Llessthan64_aligned + beq Lexit + +Llessthan16_aligned: + mov r2, r2, lsl #28 + msr cpsr_f, r2 + + ldmiami r1!, { r2, r3 } + ldreq r4, [r1], #4 + ldrhcs r5, [r1], #2 + ldrbvs r12, [r1], #1 + + stmiami r0!, { r2, r3 } + streq r4, [r0], #4 + strhcs r5, [r0], #2 + strbvs r12, [r0], #1 + b Lexit + +Lsimilarlyunaligned: + /* both src and dest are unaligned in similar ways, align to dest on 32 byte boundary */ + mov r12, r0, lsl #28 + rsb r12, r12, #0 + msr cpsr_f, r12 + + ldrbvs r3, [r1], #1 + ldrhcs r4, [r1], #2 + ldreq r5, [r1], #4 + + strbvs r3, [r0], #1 + strhcs r4, [r0], #2 + streq r5, [r0], #4 + + ldmiami r1!, { r3, r4 } + stmiami r0!, { r3, r4 } + + subs r2, r2, r12, lsr #28 + beq Lexit + +Lunaligned_32: + /* bring up to dest 32 byte alignment */ + tst r0, #(1 << 4) + ldmiane r1!, { r3, r4, r5, r12 } + stmiane r0!, { r3, r4, r5, r12 } + subne r2, r2, #16 + + /* we should now be aligned, see what copy method we should use */ + cmp r2, #64 + bge Lmorethan64_aligned + b Llessthan64_aligned + +Lbytewise2: + /* copy 2 bytes at a time */ + subs r2, r2, #2 + + ldrb r3, [r1], #1 + ldrbpl r4, [r1], #1 + + strb r3, [r0], #1 + strbpl r4, [r0], #1 + + bhi Lbytewise2 + b Lexit + +Lbytewise: + /* simple bytewise forward copy */ + ldrb r3, [r1], #1 + subs r2, r2, #1 + strb r3, [r0], #1 + bne Lbytewise + b Lexit + +Lsmallforwardcopy: + /* src and dest are word aligned similarly, less than 64 bytes to copy */ + cmp r2, #4 + blt Lbytewise2 + + /* bytewise copy until word aligned */ + tst r1, #3 +Lwordalignloop: + ldrbne r3, [r1], #1 + strbne r3, [r0], #1 + subne r2, r2, #1 + tstne r1, #3 + bne Lwordalignloop + + cmp r2, #16 + bge Llessthan64_aligned + blt Llessthan16_aligned + +Loverlap: + /* src and dest overlap in some way, len > 0 */ + cmp r0, r1 /* if dest > src */ + bhi Loverlap_srclower + +Loverlap_destlower: + /* dest < src, see if we can still do a fast forward copy or fallback to slow forward copy */ + cmp r3, #64 + bge Lnormalforwardcopy /* overlap is greater than one stride of the copy, use normal copy */ + + cmp r3, #2 + bge Lbytewise2 + b Lbytewise + + /* the following routines deal with having to copy in the reverse direction */ +Loverlap_srclower: + /* src < dest, with overlap */ + + /* src += len; dest += len; */ + add r0, r0, r2 + add r1, r1, r2 + + /* we have to copy in reverse no matter what, test if we can we use a large block reverse copy */ + cmp r2, #64 /* less than 64 bytes to copy? */ + cmpgt r3, #64 /* less than 64 bytes of nonoverlap? */ + blt Lbytewise_reverse + + /* test of src and dest are nonword aligned differently */ + mov r3, r0, lsl #30 + cmp r3, r1, lsl #30 + bne Lbytewise_reverse + + /* test if src and dest are non word aligned or dest is non 16 byte aligned */ + tst r0, #0xf + bne Lunaligned_reverse_similarly + + /* test for dest 32 byte alignment */ + tst r0, #(1<<4) + bne Lunaligned_32_reverse_similarly + + /* 64 byte reverse block copy, src and dest aligned */ +Lmorethan64_aligned_reverse: + /* save some more registers to use in the copy */ + stmfd sp!, { r6, r8, r10, r11 } + + /* pre-subtract 64 from the len counter to avoid an extra compare in the loop */ + sub r2, r2, #64 + +L64loop_reverse: + /* copy 64 bytes at a time */ + ldmdb r1!, { r3, r4, r5, r6, r8, r10, r11, r12 } +#if ARCH_ARMv5 || ARCH_ARMv5e || ARCH_ARMv6 + pld [r1, #-32] +#endif + stmdb r0!, { r3, r4, r5, r6, r8, r10, r11, r12 } + ldmdb r1!, { r3, r4, r5, r6, r8, r10, r11, r12 } + subs r2, r2, #64 + pld [r1, #-32] + stmdb r0!, { r3, r4, r5, r6, r8, r10, r11, r12 } + bge L64loop_reverse + + /* restore the scratch registers we just saved */ + ldmfd sp!, { r6, r8, r10, r11 } + + /* fix up the len counter (previously subtracted an extra 64 from it) and test for completion */ + adds r2, r2, #64 + beq Lexit + +Lbytewise_reverse: + ldrb r3, [r1, #-1]! + strb r3, [r0, #-1]! + subs r2, r2, #1 + bne Lbytewise_reverse + b Lexit + +Lunaligned_reverse_similarly: + /* both src and dest are unaligned in similar ways, align to dest on 32 byte boundary */ + mov r12, r0, lsl #28 + msr cpsr_f, r12 + + ldrbvs r3, [r1, #-1]! + ldrhcs r4, [r1, #-2]! + ldreq r5, [r1, #-4]! + + strbvs r3, [r0, #-1]! + strhcs r4, [r0, #-2]! + streq r5, [r0, #-4]! + + ldmdbmi r1!, { r3, r4 } + stmdbmi r0!, { r3, r4 } + + subs r2, r2, r12, lsr #28 + beq Lexit + +Lunaligned_32_reverse_similarly: + /* bring up to dest 32 byte alignment */ + tst r0, #(1 << 4) + ldmdbne r1!, { r3, r4, r5, r12 } + stmdbne r0!, { r3, r4, r5, r12 } + subne r2, r2, #16 + + /* we should now be aligned, see what copy method we should use */ + cmp r2, #64 + bge Lmorethan64_aligned_reverse + b Lbytewise_reverse + + /* the following routines deal with non word aligned copies */ +Lnonwordaligned_forward: + cmp r2, #8 + blt Lbytewise2 /* not worth the effort with less than 24 bytes total */ + + /* bytewise copy until src word aligned */ + tst r1, #3 +Lwordalignloop2: + ldrbne r3, [r1], #1 + strbne r3, [r0], #1 + subne r2, r2, #1 + tstne r1, #3 + bne Lwordalignloop2 + + /* figure out how the src and dest are unaligned */ + and r3, r0, #3 + cmp r3, #2 + blt Lalign1_forward + beq Lalign2_forward + bgt Lalign3_forward + +Lalign1_forward: + /* the dest pointer is 1 byte off from src */ + mov r12, r2, lsr #2 /* number of words we should copy */ + sub r0, r0, #1 + + /* prime the copy */ + ldrb r4, [r0] /* load D[7:0] */ + +Lalign1_forward_loop: + ldr r3, [r1], #4 /* load S */ + orr r4, r4, r3, lsl #8 /* D[31:8] = S[24:0] */ + str r4, [r0], #4 /* save D */ + mov r4, r3, lsr #24 /* D[7:0] = S[31:25] */ + subs r12, r12, #1 + bne Lalign1_forward_loop + + /* finish the copy off */ + strb r4, [r0], #1 /* save D[7:0] */ + + ands r2, r2, #3 + beq Lexit + b Lbytewise2 + +Lalign2_forward: + /* the dest pointer is 2 bytes off from src */ + mov r12, r2, lsr #2 /* number of words we should copy */ + sub r0, r0, #2 + + /* prime the copy */ + ldrh r4, [r0] /* load D[15:0] */ + +Lalign2_forward_loop: + ldr r3, [r1], #4 /* load S */ + orr r4, r4, r3, lsl #16 /* D[31:16] = S[15:0] */ + str r4, [r0], #4 /* save D */ + mov r4, r3, lsr #16 /* D[15:0] = S[31:15] */ + subs r12, r12, #1 + bne Lalign2_forward_loop + + /* finish the copy off */ + strh r4, [r0], #2 /* save D[15:0] */ + + ands r2, r2, #3 + beq Lexit + b Lbytewise2 + +Lalign3_forward: + /* the dest pointer is 3 bytes off from src */ + mov r12, r2, lsr #2 /* number of words we should copy */ + sub r0, r0, #3 + + /* prime the copy */ + ldr r4, [r0] + and r4, r4, #0x00ffffff /* load D[24:0] */ + +Lalign3_forward_loop: + ldr r3, [r1], #4 /* load S */ + orr r4, r4, r3, lsl #24 /* D[31:25] = S[7:0] */ + str r4, [r0], #4 /* save D */ + mov r4, r3, lsr #8 /* D[24:0] = S[31:8] */ + subs r12, r12, #1 + bne Lalign3_forward_loop + + /* finish the copy off */ + strh r4, [r0], #2 /* save D[15:0] */ + mov r4, r4, lsr #16 + strb r4, [r0], #1 /* save D[23:16] */ + + ands r2, r2, #3 + beq Lexit + b Lbytewise2 + +Lexit: + ldmfd sp!, { r0, r4, r5, r7, pc } + diff --git a/osfmk/arm/bsd_arm.c b/osfmk/arm/bsd_arm.c new file mode 100644 index 000000000..5845d01de --- /dev/null +++ b/osfmk/arm/bsd_arm.c @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifdef MACH_BSD +#include <mach_debug.h> +#include <mach_ldebug.h> + +#include <mach/kern_return.h> +#include <mach/mach_traps.h> +#include <mach/thread_status.h> +#include <mach/vm_param.h> + +#include <kern/counters.h> +#include <kern/cpu_data.h> +#include <arm/cpu_data_internal.h> +#include <kern/mach_param.h> +#include <kern/task.h> +#include <kern/thread.h> +#include <kern/sched_prim.h> +#include <kern/misc_protos.h> +#include <kern/assert.h> +#include <kern/spl.h> +#include <kern/syscall_sw.h> +#include <ipc/ipc_port.h> +#include <vm/vm_kern.h> +#include <vm/pmap.h> + +#include <sys/syscall.h> + +kern_return_t +thread_setsinglestep(__unused thread_t thread, __unused int on) +{ + return (KERN_FAILURE); /* XXX TODO */ +} + +#if CONFIG_DTRACE + +vm_offset_t dtrace_get_cpu_int_stack_top(void); + +vm_offset_t +dtrace_get_cpu_int_stack_top(void) +{ + return getCpuDatap()->intstack_top; +} +#endif /* CONFIG_DTRACE */ + +#endif /* MACH_BSD */ diff --git a/osfmk/arm/bzero.s b/osfmk/arm/bzero.s new file mode 100644 index 000000000..3ea2e2288 --- /dev/null +++ b/osfmk/arm/bzero.s @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include <arm/proc_reg.h> + +#include <arm/asm.h> + +/* + * A reasonably well-optimized bzero/memset. Should work equally well on arm11 and arm9 based + * cores. + * + * The algorithm is to align the destination pointer on a 32 byte boundary and then + * blast data 64 bytes at a time, in two stores of 32 bytes per loop. + */ + .syntax unified + .text + .align 2 + +/* + * void *secure_memset(void * addr, int pattern, size_t length) + * + * It is important that this function remains defined in assembly to avoid + * compiler optimizations. + */ +ENTRY(secure_memset) +/* void *memset(void *ptr, int c, size_t len); */ +ENTRY(memset) + /* move len into r1, unpack c into r2 */ + mov r3, r2 + and r1, r1, #0xff + orr r1, r1, r1, lsl #8 + orr r2, r1, r1, lsl #16 + mov r1, r3 + b Lbzeroengine + +/* void bzero(void *ptr, size_t len); */ +ENTRY2(bzero,__bzero) + /* zero out r2 so we can be just like memset(0) */ + mov r2, #0 + +Lbzeroengine: + /* move the base pointer into r12 and leave r0 alone so that we return the original pointer */ + mov r12, r0 + + /* copy r2 into r3 for 64-bit stores */ + mov r3, r2 + + /* check for zero len */ + cmp r1, #0 + bxeq lr + + /* fall back to a bytewise store for less than 32 bytes */ + cmp r1, #32 + blt L_bytewise + + /* check for 32 byte unaligned ptr */ + tst r12, #0x1f + bne L_unaligned + + /* make sure we have more than 64 bytes to zero */ + cmp r1, #64 + blt L_lessthan64aligned + + /* >= 64 bytes of len, 32 byte aligned */ +L_64ormorealigned: + + /* we need some registers, avoid r7 (frame pointer) and r9 (thread register) */ + stmfd sp!, { r4-r6, r8, r10-r11 } + mov r4, r2 + mov r5, r2 + mov r6, r2 + mov r8, r2 + mov r10, r2 + mov r11, r2 + + /* pre-subtract 64 from the len to avoid an extra compare in the loop */ + sub r1, r1, #64 + +L_64loop: + stmia r12!, { r2-r6, r8, r10-r11 } + subs r1, r1, #64 + stmia r12!, { r2-r6, r8, r10-r11 } + bge L_64loop + + /* restore the saved regs */ + ldmfd sp!, { r4-r6, r8, r10-r11 } + + /* check for completion (had previously subtracted an extra 64 from len) */ + adds r1, r1, #64 + bxeq lr + +L_lessthan64aligned: + /* do we have 16 or more bytes left */ + cmp r1, #16 + stmiage r12!, { r2-r3 } + stmiage r12!, { r2-r3 } + subsge r1, r1, #16 + bgt L_lessthan64aligned + bxeq lr + +L_lessthan16aligned: + /* store 0 to 15 bytes */ + mov r1, r1, lsl #28 /* move the remaining len bits [3:0] to the flags area of cpsr */ + msr cpsr_f, r1 + + stmiami r12!, { r2-r3 } /* n is set, store 8 bytes */ + streq r2, [r12], #4 /* z is set, store 4 bytes */ + strhcs r2, [r12], #2 /* c is set, store 2 bytes */ + strbvs r2, [r12], #1 /* v is set, store 1 byte */ + bx lr + +L_bytewise: + /* bytewise copy, 2 bytes at a time, alignment not guaranteed */ + subs r1, r1, #2 + strb r2, [r12], #1 + strbpl r2, [r12], #1 + bhi L_bytewise + bx lr + +L_unaligned: + /* unaligned on 32 byte boundary, store 1-15 bytes until we're 16 byte aligned */ + mov r3, r12, lsl #28 + rsb r3, r3, #0x00000000 + msr cpsr_f, r3 + + strbvs r2, [r12], #1 /* v is set, unaligned in the 1s column */ + strhcs r2, [r12], #2 /* c is set, unaligned in the 2s column */ + streq r2, [r12], #4 /* z is set, unaligned in the 4s column */ + strmi r2, [r12], #4 /* n is set, unaligned in the 8s column */ + strmi r2, [r12], #4 + + subs r1, r1, r3, lsr #28 + bxeq lr + + /* we had previously trashed r3, restore it */ + mov r3, r2 + + /* now make sure we're 32 byte aligned */ + tst r12, #(1 << 4) + stmiane r12!, { r2-r3 } + stmiane r12!, { r2-r3 } + subsne r1, r1, #16 + + /* we're now aligned, check for >= 64 bytes left */ + cmp r1, #64 + bge L_64ormorealigned + b L_lessthan64aligned + diff --git a/osfmk/arm/caches.c b/osfmk/arm/caches.c new file mode 100644 index 000000000..5f37e202d --- /dev/null +++ b/osfmk/arm/caches.c @@ -0,0 +1,753 @@ +/* + * Copyright (c) 2010 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include <mach_assert.h> +#include <mach/vm_types.h> +#include <mach/mach_time.h> +#include <kern/timer.h> +#include <kern/clock.h> +#include <kern/machine.h> +#include <mach/machine.h> +#include <mach/machine/vm_param.h> +#include <mach_kdp.h> +#include <kdp/kdp_udp.h> +#include <arm/caches_internal.h> +#include <arm/cpuid.h> +#include <arm/cpu_data.h> +#include <arm/cpu_data_internal.h> +#include <arm/cpu_internal.h> + +#include <vm/vm_kern.h> +#include <vm/vm_map.h> +#include <vm/pmap.h> + +#include <arm/misc_protos.h> + +/* + * dcache_incoherent_io_flush64() dcache_incoherent_io_store64() result info + */ +#define LWOpDone 1 +#define BWOpDone 3 + +#ifndef __ARM_COHERENT_IO__ + +extern boolean_t up_style_idle_exit; + +void +flush_dcache( + vm_offset_t addr, + unsigned length, + boolean_t phys) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + if (phys) { + unsigned int paddr; + unsigned int vaddr; + + paddr = CAST_DOWN(unsigned int, addr); + if (!isphysmem(paddr)) + return; + vaddr = (unsigned int)phystokv(paddr); + FlushPoC_DcacheRegion( (vm_offset_t) vaddr, length); + + if (cpu_data_ptr->cpu_cache_dispatch != (cache_dispatch_t) NULL) + ((cache_dispatch_t) cpu_data_ptr->cpu_cache_dispatch) ( + cpu_data_ptr->cpu_id, CacheCleanFlushRegion, (unsigned int) paddr, length); + return; + } + if (cpu_data_ptr->cpu_cache_dispatch == (cache_dispatch_t) NULL) { + FlushPoC_DcacheRegion( (vm_offset_t) addr, length); + } else { + addr64_t paddr; + uint32_t count; + + while (length > 0) { + count = PAGE_SIZE - (addr & PAGE_MASK); + if (count > length) + count = length; + FlushPoC_DcacheRegion( (vm_offset_t) addr, count); + paddr = kvtophys(addr); + if (paddr) + ((cache_dispatch_t) cpu_data_ptr->cpu_cache_dispatch) ( + cpu_data_ptr->cpu_id, CacheCleanFlushRegion, (unsigned int) paddr, count); + addr += count; + length -= count; + } + } + return; +} + +void +clean_dcache( + vm_offset_t addr, + unsigned length, + boolean_t phys) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + if (phys) { + unsigned int paddr; + unsigned int vaddr; + + paddr = CAST_DOWN(unsigned int, addr); + if (!isphysmem(paddr)) + return; + + vaddr = (unsigned int)phystokv(paddr); + CleanPoC_DcacheRegion( (vm_offset_t) vaddr, length); + + if (cpu_data_ptr->cpu_cache_dispatch != (cache_dispatch_t) NULL) + ((cache_dispatch_t) cpu_data_ptr->cpu_cache_dispatch) ( + cpu_data_ptr->cpu_id, CacheCleanRegion, paddr, length); + return; + } + + if (cpu_data_ptr->cpu_cache_dispatch == (cache_dispatch_t) NULL) { + CleanPoC_DcacheRegion( (vm_offset_t) addr, length); + } else { + addr64_t paddr; + uint32_t count; + + while (length > 0) { + count = PAGE_SIZE - (addr & PAGE_MASK); + if (count > length) + count = length; + CleanPoC_DcacheRegion( (vm_offset_t) addr, count); + paddr = kvtophys(addr); + if (paddr) + ((cache_dispatch_t) cpu_data_ptr->cpu_cache_dispatch) ( + cpu_data_ptr->cpu_id, CacheCleanRegion, (unsigned int) paddr, count); + addr += count; + length -= count; + } + } + return; +} + +void +flush_dcache_syscall( + vm_offset_t va, + unsigned length) +{ + if ((cache_info()->c_bulksize_op !=0) && (length >= (cache_info()->c_bulksize_op))) { +#if __ARM_SMP__ && defined(ARMA7) + cache_xcall(LWFlush); +#else + FlushPoC_Dcache(); + if (getCpuDatap()->cpu_cache_dispatch != (cache_dispatch_t) NULL) + ((cache_dispatch_t) getCpuDatap()->cpu_cache_dispatch) ( getCpuDatap()->cpu_id, CacheCleanFlush, 0x0UL , 0x0UL); +#endif + } else { + FlushPoC_DcacheRegion( (vm_offset_t) va, length); + } + return; +} + +void +dcache_incoherent_io_flush64( + addr64_t pa, + unsigned int size, + unsigned int remaining, + unsigned int *res) +{ + unsigned int vaddr; + unsigned int paddr = CAST_DOWN(unsigned int, pa); + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + if ((cache_info()->c_bulksize_op !=0) && (remaining >= (cache_info()->c_bulksize_op))) { +#if __ARM_SMP__ && defined (ARMA7) + cache_xcall(LWFlush); +#else + FlushPoC_Dcache(); + if (cpu_data_ptr->cpu_cache_dispatch != (cache_dispatch_t) NULL) + ((cache_dispatch_t) cpu_data_ptr->cpu_cache_dispatch) ( cpu_data_ptr->cpu_id, CacheCleanFlush, 0x0UL , 0x0UL); +#endif + *res = BWOpDone; + } else { + if (isphysmem(paddr)) { + vaddr = (unsigned int)phystokv(pa); + { + FlushPoC_DcacheRegion( (vm_offset_t) vaddr, size); + + if (cpu_data_ptr->cpu_cache_dispatch != (cache_dispatch_t) NULL) + ((cache_dispatch_t) cpu_data_ptr->cpu_cache_dispatch) (cpu_data_ptr->cpu_id, CacheCleanFlushRegion, (unsigned int) pa, size); + } + } else { + /* slow path - pa isn't in the vtop region. Flush one page at a time via cpu_copywindows */ + unsigned int wimg_bits, index; + uint32_t count; + + mp_disable_preemption(); + + while (size > 0) { + count = PAGE_SIZE - (paddr & PAGE_MASK); + if (count > size) + count = size; + + wimg_bits = pmap_cache_attributes((paddr >> PAGE_SHIFT)); + index = pmap_map_cpu_windows_copy((paddr >> PAGE_SHIFT), VM_PROT_READ|VM_PROT_WRITE, wimg_bits); + vaddr = pmap_cpu_windows_copy_addr(cpu_number(), index) | (paddr & PAGE_MASK); + + CleanPoC_DcacheRegion( (vm_offset_t) vaddr, count); + + pmap_unmap_cpu_windows_copy(index); + + paddr += count; + size -= count; + } + + mp_enable_preemption(); + } + } + + return; +} + +void +dcache_incoherent_io_store64( + addr64_t pa, + unsigned int size, + unsigned int remaining, + unsigned int *res) +{ + unsigned int vaddr; + unsigned int paddr = CAST_DOWN(unsigned int, pa); + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + if (isphysmem(paddr)) { + unsigned int wimg_bits = pmap_cache_attributes(paddr >> PAGE_SHIFT); + if ((wimg_bits == VM_WIMG_IO) || (wimg_bits == VM_WIMG_WCOMB)) { + return; + } + } + + if ((cache_info()->c_bulksize_op !=0) && (remaining >= (cache_info()->c_bulksize_op))) { +#if __ARM_SMP__ && defined (ARMA7) + cache_xcall(LWClean); + if (cpu_data_ptr->cpu_cache_dispatch != (cache_dispatch_t) NULL) + ((cache_dispatch_t) cpu_data_ptr->cpu_cache_dispatch) ( cpu_data_ptr->cpu_id, CacheClean, 0x0UL , 0x0UL); +#else + CleanPoC_Dcache(); + if (cpu_data_ptr->cpu_cache_dispatch != (cache_dispatch_t) NULL) + ((cache_dispatch_t) cpu_data_ptr->cpu_cache_dispatch) ( cpu_data_ptr->cpu_id, CacheClean, 0x0UL , 0x0UL); +#endif + *res = BWOpDone; + } else { + if (isphysmem(paddr)) { + vaddr = (unsigned int)phystokv(pa); + { + CleanPoC_DcacheRegion( (vm_offset_t) vaddr, size); + + if (cpu_data_ptr->cpu_cache_dispatch != (cache_dispatch_t) NULL) + ((cache_dispatch_t) cpu_data_ptr->cpu_cache_dispatch) (cpu_data_ptr->cpu_id, CacheCleanRegion, (unsigned int) pa, size); + } + } else { + /* slow path - pa isn't in the vtop region. Flush one page at a time via cpu_copywindows */ + unsigned int wimg_bits, index; + uint32_t count; + + mp_disable_preemption(); + + while (size > 0) { + count = PAGE_SIZE - (paddr & PAGE_MASK); + if (count > size) + count = size; + + wimg_bits = pmap_cache_attributes((paddr >> PAGE_SHIFT)); + index = pmap_map_cpu_windows_copy((paddr >> PAGE_SHIFT), VM_PROT_READ|VM_PROT_WRITE, wimg_bits); + vaddr = pmap_cpu_windows_copy_addr(cpu_number(), index) | (paddr & PAGE_MASK); + + CleanPoC_DcacheRegion( (vm_offset_t) vaddr, count); + + pmap_unmap_cpu_windows_copy(index); + + paddr += count; + size -= count; + } + + mp_enable_preemption(); + } + } + + return; +} + +void +cache_sync_page( + ppnum_t pp +) +{ + pmap_paddr_t paddr = ptoa(pp); + + if (isphysmem(paddr)) { + vm_offset_t vaddr = phystokv(paddr); + + CleanPoU_DcacheRegion(vaddr, PAGE_SIZE); +#ifdef __ARM_IC_NOALIAS_ICACHE__ + InvalidatePoU_IcacheRegion(vaddr, PAGE_SIZE); +#else + InvalidatePoU_Icache(); +#endif + } else { + FlushPoC_Dcache(); + InvalidatePoU_Icache(); + }; +} + +void +platform_cache_init( + void) +{ + cache_info_t *cpuid_cache_info; + unsigned int cache_size = 0x0UL; + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + cpuid_cache_info = cache_info(); + + if (cpu_data_ptr->cpu_cache_dispatch != (cache_dispatch_t) NULL) { + ((cache_dispatch_t) cpu_data_ptr->cpu_cache_dispatch) ( + cpu_data_ptr->cpu_id, CacheControl, CacheControlEnable, 0x0UL); + + if ( cpuid_cache_info->c_l2size == 0x0 ) { + ((cache_dispatch_t) cpu_data_ptr->cpu_cache_dispatch) ( + cpu_data_ptr->cpu_id, CacheConfig, CacheConfigSize , (unsigned int)&cache_size); + cpuid_cache_info->c_l2size = cache_size; + } + } + +} + +void +platform_cache_flush( + void) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + FlushPoC_Dcache(); + + if (cpu_data_ptr->cpu_cache_dispatch != (cache_dispatch_t) NULL) + ((cache_dispatch_t) cpu_data_ptr->cpu_cache_dispatch) ( + cpu_data_ptr->cpu_id, CacheCleanFlush, 0x0UL , 0x0UL); +} + +void +platform_cache_clean( + void) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + CleanPoC_Dcache(); + + if (cpu_data_ptr->cpu_cache_dispatch != (cache_dispatch_t) NULL) + ((cache_dispatch_t) cpu_data_ptr->cpu_cache_dispatch) ( + cpu_data_ptr->cpu_id, CacheClean, 0x0UL , 0x0UL); +} + +void +platform_cache_shutdown( + void) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + CleanPoC_Dcache(); + + if (cpu_data_ptr->cpu_cache_dispatch != (cache_dispatch_t) NULL) + ((cache_dispatch_t) cpu_data_ptr->cpu_cache_dispatch) ( + cpu_data_ptr->cpu_id, CacheShutdown, 0x0UL , 0x0UL); +} + +void +platform_cache_disable(void) +{ + uint32_t sctlr_value = 0; + + /* Disable dcache allocation. */ + __asm__ volatile("mrc p15, 0, %0, c1, c0, 0" + : "=r"(sctlr_value)); + + sctlr_value &= ~SCTLR_DCACHE; + + __asm__ volatile("mcr p15, 0, %0, c1, c0, 0\n" + "isb" + :: "r"(sctlr_value)); + +} + +void +platform_cache_idle_enter( + void) +{ +#if __ARM_SMP__ + platform_cache_disable(); + + /* + * If we're only using a single CPU, just write back any + * dirty cachelines. We can avoid doing housekeeping + * on CPU data that would normally be modified by other + * CPUs. + */ + if (up_style_idle_exit && (real_ncpus == 1)) + CleanPoU_Dcache(); + else { + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + FlushPoU_Dcache(); + + cpu_data_ptr->cpu_CLW_active = 0; + __asm__ volatile("dmb ish"); + cpu_data_ptr->cpu_CLWFlush_req = 0; + cpu_data_ptr->cpu_CLWClean_req = 0; + CleanPoC_DcacheRegion((vm_offset_t) cpu_data_ptr, sizeof(cpu_data_t)); + } +#else + CleanPoU_Dcache(); +#endif + +#if defined (__ARM_SMP__) && defined (ARMA7) + uint32_t actlr_value = 0; + + /* Leave the coherency domain */ + __asm__ volatile("clrex\n" + "mrc p15, 0, %0, c1, c0, 1\n" + : "=r"(actlr_value)); + + actlr_value &= ~0x40; + + __asm__ volatile("mcr p15, 0, %0, c1, c0, 1\n" + /* Ensures any pending fwd request gets serviced and ends up */ + "dsb\n" + /* Forces the processor to re-fetch, so any pending fwd request gets into the core */ + "isb\n" + /* Ensures the second possible pending fwd request ends up. */ + "dsb\n" + :: "r"(actlr_value)); +#endif +} + +void +platform_cache_idle_exit( + void) +{ +#if defined (ARMA7) + uint32_t actlr_value = 0; + + /* Flush L1 caches and TLB before rejoining the coherency domain */ + FlushPoU_Dcache(); + /* + * If we're only using a single CPU, we can avoid flushing the + * I-cache or the TLB, as neither program text nor pagetables + * should have been changed during the idle period. We still + * want to flush the D-cache to PoU (above), as memory contents + * may have been changed by DMA. + */ + if (!up_style_idle_exit || (real_ncpus > 1)) { + InvalidatePoU_Icache(); + flush_core_tlb(); + } + + /* Rejoin the coherency domain */ + __asm__ volatile("mrc p15, 0, %0, c1, c0, 1\n" + : "=r"(actlr_value)); + + actlr_value |= 0x40; + + __asm__ volatile("mcr p15, 0, %0, c1, c0, 1\n" + "isb\n" + :: "r"(actlr_value)); + +#if __ARM_SMP__ + uint32_t sctlr_value = 0; + + /* Enable dcache allocation. */ + __asm__ volatile("mrc p15, 0, %0, c1, c0, 0\n" + : "=r"(sctlr_value)); + + sctlr_value |= SCTLR_DCACHE; + + __asm__ volatile("mcr p15, 0, %0, c1, c0, 0\n" + "isb" + :: "r"(sctlr_value)); + getCpuDatap()->cpu_CLW_active = 1; +#endif +#endif +} + +boolean_t +platform_cache_batch_wimg( + __unused unsigned int new_wimg, + __unused unsigned int size + ) +{ + boolean_t do_cache_op = FALSE; + + if ((cache_info()->c_bulksize_op != 0) && (size >= (cache_info()->c_bulksize_op))) do_cache_op = TRUE; + + return do_cache_op; +} + +void +platform_cache_flush_wimg( + __unused unsigned int new_wimg +) +{ +#if __ARM_SMP__ && defined (ARMA7) + cache_xcall(LWFlush); +#else + FlushPoC_Dcache(); + if (getCpuDatap()->cpu_cache_dispatch != (cache_dispatch_t) NULL) + ((cache_dispatch_t) getCpuDatap()->cpu_cache_dispatch) ( getCpuDatap()->cpu_id, CacheCleanFlush, 0x0UL , 0x0UL); +#endif +} + +#if __ARM_SMP__ && defined(ARMA7) +void +cache_xcall_handler(unsigned int op) +{ + cpu_data_t *cdp; + uint64_t abstime; + + cdp = getCpuDatap(); + + if ((op == LWFlush) && (cdp->cpu_CLWFlush_req > cdp->cpu_CLWFlush_last)) { + FlushPoU_Dcache(); + abstime = ml_get_timebase(); + cdp->cpu_CLWFlush_last = abstime; + cdp->cpu_CLWClean_last = abstime; + } else if ((op == LWClean) && (cdp->cpu_CLWClean_req > cdp->cpu_CLWClean_last)) { + CleanPoU_Dcache(); + abstime = ml_get_timebase(); + cdp->cpu_CLWClean_last = abstime; + } +} + + +void +cache_xcall(unsigned int op) +{ + boolean_t intr; + cpu_data_t *cdp; + cpu_data_t *target_cdp; + unsigned int cpu; + unsigned int signal; + uint64_t abstime; + + intr = ml_set_interrupts_enabled(FALSE); + cdp = getCpuDatap(); + abstime = ml_get_timebase(); + if (op == LWClean) + signal = SIGPLWClean; + else + signal = SIGPLWFlush; + + for (cpu=0; cpu < MAX_CPUS; cpu++) { + + target_cdp = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr; + if(target_cdp == (cpu_data_t *)NULL) + break; + + if (target_cdp->cpu_CLW_active == 0) + continue; + + if (op == LWFlush) + target_cdp->cpu_CLWFlush_req = abstime; + else if (op == LWClean) + target_cdp->cpu_CLWClean_req = abstime; + __asm__ volatile("dmb ish"); + if (target_cdp->cpu_CLW_active == 0) { + if (op == LWFlush) + target_cdp->cpu_CLWFlush_req = 0x0ULL; + else if (op == LWClean) + target_cdp->cpu_CLWClean_req = 0x0ULL; + continue; + } + + if (target_cdp == cdp) + continue; + + if(KERN_SUCCESS != cpu_signal(target_cdp, signal, (void *)NULL, NULL)) { + if (op == LWFlush) + target_cdp->cpu_CLWFlush_req = 0x0ULL; + else if (op == LWClean) + target_cdp->cpu_CLWClean_req = 0x0ULL; + } + if (cpu == real_ncpus) + break; + } + + cache_xcall_handler (op); + + (void) ml_set_interrupts_enabled(intr); + + for (cpu=0; cpu < MAX_CPUS; cpu++) { + + target_cdp = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr; + if(target_cdp == (cpu_data_t *)NULL) + break; + + if (target_cdp == cdp) + continue; + + if (op == LWFlush) + while ((target_cdp->cpu_CLWFlush_req != 0x0ULL) && (target_cdp->cpu_CLWFlush_last < abstime)); + else if (op == LWClean) + while ((target_cdp->cpu_CLWClean_req != 0x0ULL ) && (target_cdp->cpu_CLWClean_last < abstime)); + + if (cpu == real_ncpus) + break; + } + + if (op == LWFlush) + FlushPoC_Dcache(); + else if (op == LWClean) + CleanPoC_Dcache(); +} +#endif + + +#else /* __ARM_COHERENT_IO__ */ + +void +flush_dcache( + __unused vm_offset_t addr, + __unused unsigned length, + __unused boolean_t phys) +{ + __asm__ volatile ("dsb sy"); +} + +void +clean_dcache( + __unused vm_offset_t addr, + __unused unsigned length, + __unused boolean_t phys) +{ + __asm__ volatile ("dsb sy"); +} + +void +flush_dcache_syscall( + __unused vm_offset_t va, + __unused unsigned length) +{ + __asm__ volatile ("dsb sy"); +} + +void +dcache_incoherent_io_flush64( + __unused addr64_t pa, + __unused unsigned int size, + __unused unsigned int remaining, + __unused unsigned int *res) +{ + __asm__ volatile ("dsb sy"); + *res = LWOpDone; + return; +} + +void +dcache_incoherent_io_store64( + __unused addr64_t pa, + __unused unsigned int size, + __unused unsigned int remaining, + __unused unsigned int *res) +{ + __asm__ volatile ("dsb sy"); + *res = LWOpDone; + return; +} + +void +cache_sync_page( + ppnum_t pp +) +{ + pmap_paddr_t paddr = ptoa(pp); + + if (isphysmem(paddr)) { + vm_offset_t vaddr = phystokv(paddr); + +#ifdef __ARM_IC_NOALIAS_ICACHE__ + InvalidatePoU_IcacheRegion(vaddr, PAGE_SIZE); +#else + InvalidatePoU_Icache(); +#endif + } +} + +void +platform_cache_init( + void) +{ +} + +void +platform_cache_flush( + void) +{ +} + +void +platform_cache_clean( + void) +{ +} + +void +platform_cache_shutdown( + void) +{ +} + +void +platform_cache_idle_enter( + void) +{ +} + +void +platform_cache_idle_exit( + void) +{ +} + +boolean_t +platform_cache_batch_wimg( + __unused unsigned int new_wimg, + __unused unsigned int size + ) +{ + return TRUE; +} + +void +platform_cache_flush_wimg( + __unused unsigned int new_wimg) +{ +} + +#endif /* __ARM_COHERENT_IO__ */ diff --git a/osfmk/arm/caches_asm.s b/osfmk/arm/caches_asm.s new file mode 100644 index 000000000..b4e6a94c8 --- /dev/null +++ b/osfmk/arm/caches_asm.s @@ -0,0 +1,362 @@ +/* + * Copyright (c) 2010 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include <machine/asm.h> +#include <arm/proc_reg.h> +#include <arm/pmap.h> +#include <sys/errno.h> +#include "assym.s" + + +/* + * void invalidate_mmu_cache(void) + * + * Invalidate d-cache and i-cache + */ + .text + .align 2 + .globl EXT(invalidate_mmu_cache) +LEXT(invalidate_mmu_cache) + mov r0, #0 + mcr p15, 0, r0, c7, c7, 0 // Invalidate caches + bx lr + +/* + * void invalidate_mmu_dcache(void) + * + * Invalidate d-cache + */ + .text + .align 2 + .globl EXT(invalidate_mmu_dcache) +LEXT(invalidate_mmu_dcache) + mov r0, #0 + mcr p15, 0, r0, c7, c6, 0 // Invalidate dcache + bx lr + +/* + * void invalidate_mmu_dcache_region(vm_offset_t va, unsigned length) + * + * Invalidate d-cache region + */ + .text + .align 2 + .globl EXT(invalidate_mmu_dcache_region) +LEXT(invalidate_mmu_dcache_region) + and r2, r0, #((1<<MMU_CLINE)-1) + bic r0, r0, #((1<<MMU_CLINE)-1) // Cached aligned + add r1, r1, r2 + sub r1, r1, #1 + mov r1, r1, LSR #MMU_CLINE // Set cache line counter +fmdr_loop: + mcr p15, 0, r0, c7, c14, 1 // Invalidate dcache line + add r0, r0, #1<<MMU_CLINE // Get next cache aligned addr + subs r1, r1, #1 // Decrementer cache line counter + bpl fmdr_loop // Loop in counter not null + isb + bx lr + +/* + * void InvalidatePoU_Icache(void) + * + * Invalidate i-cache + */ + .text + .align 2 + .globl EXT(InvalidatePoU_Icache) + .globl EXT(invalidate_mmu_icache) +LEXT(InvalidatePoU_Icache) +LEXT(invalidate_mmu_icache) + mov r0, #0 + mcr p15, 0, r0, c7, c5, 0 // Invalidate icache + bx lr + +/* + * void InvalidatePoU_IcacheRegion(vm_offset_t va, unsigned length) + * + * Invalidate icache region + */ + .text + .align 2 + .globl EXT(InvalidatePoU_IcacheRegion) +LEXT(InvalidatePoU_IcacheRegion) + and r2, r0, #((1<<MMU_I_CLINE)-1) + bic r0, r0, #((1<<MMU_I_CLINE)-1) // Cached aligned + add r1, r1, r2 + sub r1, r1, #1 + mov r1, r1, LSR #MMU_I_CLINE // Set cache line counter +fmir_loop: + mcr p15, 0, r0, c7, c5, 1 // Invalidate icache line + add r0, r0, #1<<MMU_I_CLINE // Get next cache aligned addr + subs r1, r1, #1 // Decrementer cache line counter + bpl fmir_loop // Loop in counter not null + bx lr + +/* + * void CleanPoC_Dcache(void) + * + * Clean all d-caches + */ + .text + .align 2 + .globl EXT(CleanPoC_Dcache) + .globl EXT(clean_mmu_dcache) +LEXT(CleanPoC_Dcache) +LEXT(clean_mmu_dcache) +#if !defined(__ARM_L1_WT_CACHE__) + mov r0, #0 +clean_dcacheway: +clean_dcacheline: + mcr p15, 0, r0, c7, c10, 2 // clean dcache line by way/set + add r0, r0, #1 << MMU_I7SET // increment set index + tst r0, #1 << (MMU_NSET + MMU_I7SET) // look for overflow + beq clean_dcacheline + bic r0, r0, #1 << (MMU_NSET + MMU_I7SET) // clear set overflow + adds r0, r0, #1 << MMU_I7WAY // increment way + bcc clean_dcacheway // loop +#endif +#if __ARM_L2CACHE__ + dsb + mov r0, #2 +clean_l2dcacheway: +clean_l2dcacheline: + mcr p15, 0, r0, c7, c10, 2 // clean dcache line by way/set + add r0, r0, #1 << L2_I7SET // increment set index + tst r0, #1 << (L2_NSET + L2_I7SET) // look for overflow + beq clean_l2dcacheline + bic r0, r0, #1 << (L2_NSET + L2_I7SET) // clear set overflow + adds r0, r0, #1 << L2_I7WAY // increment way + bcc clean_l2dcacheway // loop +#endif + dsb + bx lr + +/* + * void CleanPoU_Dcache(void) + * + * Clean D-cache to Point of Unification + */ + .text + .align 2 + .globl EXT(CleanPoU_Dcache) +LEXT(CleanPoU_Dcache) +#if !defined(__ARM_PoU_WT_CACHE__) + mov r0, #0 +clean_dcacheway_idle: +clean_dcacheline_idle: + mcr p15, 0, r0, c7, c10, 2 // clean dcache line by way/set + add r0, r0, #1 << MMU_I7SET // increment set index + tst r0, #1 << (MMU_NSET + MMU_I7SET) // look for overflow + beq clean_dcacheline_idle + bic r0, r0, #1 << (MMU_NSET + MMU_I7SET) // clear set overflow + adds r0, r0, #1 << MMU_I7WAY // increment way + bcc clean_dcacheway_idle // loop +#endif + dsb + bx lr + +/* + * void CleanPoU_DcacheRegion(vm_offset_t va, unsigned length) + * + * Clean d-cache region to Point of Unification + */ + .text + .align 2 + .globl EXT(CleanPoU_DcacheRegion) +LEXT(CleanPoU_DcacheRegion) +#if !defined(__ARM_PoU_WT_CACHE__) + + and r2, r0, #((1<<MMU_CLINE)-1) + bic r0, r0, #((1<<MMU_CLINE)-1) // Cached aligned + add r1, r1, r2 + sub r1, r1, #1 + mov r1, r1, LSR #MMU_CLINE // Set cache line counter +cudr_loop: + mcr p15, 0, r0, c7, c11, 1 // Clean dcache line to PoU + add r0, r0, #1<<MMU_CLINE // Get next cache aligned addr + subs r1, r1, #1 // Decrementer cache line counter + bpl cudr_loop // Loop in counter not null + +#endif + dsb + bx lr + +/* + * void CleanPoC_DcacheRegion(vm_offset_t va, unsigned length) + * + * Clean d-cache region to Point of Coherency + */ + .text + .align 2 + .globl EXT(CleanPoC_DcacheRegion) + .globl EXT(CleanPoC_DcacheRegion_Force) +LEXT(CleanPoC_DcacheRegion) +LEXT(CleanPoC_DcacheRegion_Force) + and r2, r0, #((1<<MMU_CLINE)-1) + bic r0, r0, #((1<<MMU_CLINE)-1) // Cached aligned + add r1, r1, r2 + sub r1, r1, #1 + mov r1, r1, LSR #MMU_CLINE // Set cache line counter +ccdr_loop: + mcr p15, 0, r0, c7, c10, 1 // Clean dcache line to PoC + add r0, r0, #1<<MMU_CLINE // Get next cache aligned addr + subs r1, r1, #1 // Decrementer cache line counter + bpl ccdr_loop // Loop in counter not null + dsb + bx lr + +/* + * void FlushPoC_Dcache(void) + * + * Clean and Invalidate dcaches to Point of Coherency + */ + .text + .align 2 + .globl EXT(FlushPoC_Dcache) +LEXT(FlushPoC_Dcache) + mov r0, #0 +cleanflush_dcacheway: +cleanflush_dcacheline: + mcr p15, 0, r0, c7, c14, 2 // cleanflush dcache line by way/set + add r0, r0, #1 << MMU_I7SET // increment set index + tst r0, #1 << (MMU_NSET + MMU_I7SET) // look for overflow + beq cleanflush_dcacheline + bic r0, r0, #1 << (MMU_NSET + MMU_I7SET) // clear set overflow + adds r0, r0, #1 << MMU_I7WAY // increment way + bcc cleanflush_dcacheway // loop +#if __ARM_L2CACHE__ + dsb + mov r0, #2 +cleanflush_l2dcacheway: +cleanflush_l2dcacheline: + mcr p15, 0, r0, c7, c14, 2 // cleanflush dcache line by way/set + add r0, r0, #1 << L2_I7SET // increment set index + tst r0, #1 << (L2_NSET + L2_I7SET) // look for overflow + beq cleanflush_l2dcacheline + bic r0, r0, #1 << (L2_NSET + L2_I7SET) // clear set overflow + adds r0, r0, #1 << L2_I7WAY // increment way + bcc cleanflush_l2dcacheway // loop +#endif + dsb + bx lr + +/* + * void FlushPoU_Dcache(void) + * + * Flush D-cache to Point of Unification + */ + .text + .align 2 + .globl EXT(FlushPoU_Dcache) +LEXT(FlushPoU_Dcache) + mov r0, #0 +fpud_way: +fpud_line: + mcr p15, 0, r0, c7, c14, 2 // cleanflush dcache line by way/set + add r0, r0, #1 << MMU_I7SET // increment set index + tst r0, #1 << (MMU_NSET + MMU_I7SET) // look for overflow + beq fpud_line + bic r0, r0, #1 << (MMU_NSET + MMU_I7SET) // clear set overflow + adds r0, r0, #1 << MMU_I7WAY // increment way + bcc fpud_way // loop + dsb + bx lr + +/* + * void FlushPoC_DcacheRegion(vm_offset_t va, unsigned length) + * + * Clean and Invalidate d-cache region to Point of Coherency + */ + .text + .align 2 + .globl EXT(FlushPoC_DcacheRegion) +LEXT(FlushPoC_DcacheRegion) + and r2, r0, #((1<<MMU_CLINE)-1) + bic r0, r0, #((1<<MMU_CLINE)-1) // Cached aligned + add r1, r1, r2 + sub r1, r1, #1 + mov r1, r1, LSR #MMU_CLINE // Set cache line counter +cfmdr_loop: + mcr p15, 0, r0, c7, c14, 1 // Clean & invalidate dcache line + add r0, r0, #1<<MMU_CLINE // Get next cache aligned addr + subs r1, r1, #1 // Decrementer cache line counter + bpl cfmdr_loop // Loop in counter not null + dsb + bx lr + +/* + * void flush_dcache64(addr64_t addr, unsigned length, boolean_t phys) + */ + .text + .align 2 + .globl EXT(flush_dcache64) +LEXT(flush_dcache64) + mov r1, r2 + mov r2, r3 + LOAD_ADDR_PC(flush_dcache) + +/* + * void clean_dcache64(addr64_t addr, unsigned length, boolean_t phys) + */ + .text + .align 2 + .globl EXT(clean_dcache64) +LEXT(clean_dcache64) + mov r1, r2 + mov r2, r3 + LOAD_ADDR_PC(clean_dcache) + +/* + * void invalidate_icache(vm_offset_t va, unsigned length, boolean_t phys) + * void invalidate_icache64(addr64_t va, unsigned length, boolean_t phys) + */ + .text + .align 2 + .globl EXT(invalidate_icache64) + .globl EXT(invalidate_icache) +LEXT(invalidate_icache64) + mov r1, r2 + mov r2, r3 +LEXT(invalidate_icache) + cmp r2, #0 // Is it physical? + COND_EXTERN_BEQ(InvalidatePoU_IcacheRegion) + LOAD_ADDR(r2, gPhysBase) + ldr r2, [r2] + sub r0, r0, r2 + LOAD_ADDR(r2, gVirtBase) + ldr r2, [r2] + add r0, r0, r2 + b EXT(InvalidatePoU_IcacheRegion) + + +#include "globals_asm.h" + +LOAD_ADDR_GEN_DEF(flush_dcache) +LOAD_ADDR_GEN_DEF(clean_dcache) + +/* vim: set ts=4: */ diff --git a/osfmk/arm/caches_internal.h b/osfmk/arm/caches_internal.h new file mode 100644 index 000000000..e8058a858 --- /dev/null +++ b/osfmk/arm/caches_internal.h @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _ARM_CACHES_INTERNAL +#define _ARM_CACHES_INTERNAL 1 + +#include <arm/proc_reg.h> + +#include <kern/kern_types.h> + +extern void flush_dcache_syscall( vm_offset_t addr, unsigned length); + +#ifdef MACH_KERNEL_PRIVATE +extern void flush_dcache(vm_offset_t addr, unsigned count, int phys); +extern void flush_dcache64(addr64_t addr, unsigned count, int phys); +extern void invalidate_icache(vm_offset_t addr, unsigned cnt, int phys); +extern void invalidate_icache64(addr64_t addr, unsigned cnt, int phys); + +#if __ARM_SMP__ && defined(ARMA7) +#define LWFlush 1 +#define LWClean 2 +extern void cache_xcall(unsigned int op); +extern void cache_xcall_handler(unsigned int op); +#endif +#endif +extern void clean_dcache(vm_offset_t addr, unsigned count, int phys); +extern void clean_dcache64(addr64_t addr, unsigned count, int phys); + +extern void CleanPoC_Dcache(void); +extern void CleanPoU_Dcache(void); + +/* + * May not actually perform a flush on platforms + * where AP caches are snooped by all agents on SoC. + * + * This is the one you need unless you really know what + * you're doing. + */ +extern void CleanPoC_DcacheRegion(vm_offset_t va, unsigned length); + +/* + * Always actually flushes the cache, even on platforms + * where AP caches are snooped by all agents. You + * probably don't need to use this. Intended for use in + * panic save routine (where caches will be yanked by reset + * and coherency doesn't help). + */ +extern void CleanPoC_DcacheRegion_Force(vm_offset_t va, unsigned length); + +extern void CleanPoU_DcacheRegion(vm_offset_t va, unsigned length); + +extern void FlushPoC_Dcache(void); +extern void FlushPoU_Dcache(void); +extern void FlushPoC_DcacheRegion(vm_offset_t va, unsigned length); + +#ifdef __arm__ +extern void invalidate_mmu_cache(void); +extern void invalidate_mmu_dcache(void); +extern void invalidate_mmu_dcache_region(vm_offset_t va, unsigned length); +#endif + +extern void InvalidatePoU_Icache(void); +extern void InvalidatePoU_IcacheRegion(vm_offset_t va, unsigned length); + +extern void cache_sync_page(ppnum_t pp); + +extern void platform_cache_init(void); +extern void platform_cache_idle_enter(void); +extern void platform_cache_idle_exit(void); +extern void platform_cache_flush(void); +extern boolean_t platform_cache_batch_wimg(unsigned int new_wimg, unsigned int size); +extern void platform_cache_flush_wimg(unsigned int new_wimg); +extern void platform_cache_clean(void); +extern void platform_cache_shutdown(void); +extern void platform_cache_disable(void); + +#endif /* #ifndef _ARM_CACHES_INTERNAL */ diff --git a/osfmk/arm/commpage/commpage.c b/osfmk/arm/commpage/commpage.c new file mode 100644 index 000000000..520a140d0 --- /dev/null +++ b/osfmk/arm/commpage/commpage.c @@ -0,0 +1,432 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * @APPLE_FREE_COPYRIGHT@ + */ +/* + * File: arm/commpage/commpage.c + * Purpose: Set up and export a RO/RW page + */ +#include <mach/mach_types.h> +#include <mach/machine.h> +#include <mach/vm_map.h> +#include <machine/cpu_capabilities.h> +#include <machine/commpage.h> +#include <machine/pmap.h> +#include <vm/vm_kern.h> +#include <vm/vm_map.h> +#include <vm/vm_protos.h> +#include <ipc/ipc_port.h> +#include <arm/cpuid.h> /* for cpuid_info() & cache_info() */ +#include <arm/rtclock.h> +#include <libkern/OSAtomic.h> +#include <stdatomic.h> + +#include <sys/kdebug.h> + +#if CONFIG_ATM +#include <atm/atm_internal.h> +#endif + +static void commpage_init_cpu_capabilities( void ); +static int commpage_cpus( void ); + +vm_address_t commPagePtr=0; +vm_address_t sharedpage_rw_addr = 0; +uint32_t _cpu_capabilities = 0; + +extern int gARMv81Atomics; /* For sysctl access from BSD side */ + +void +commpage_populate( + void) +{ + uint16_t c2; + int cpufamily; + + sharedpage_rw_addr = pmap_create_sharedpage(); + commPagePtr = (vm_address_t)_COMM_PAGE_BASE_ADDRESS; + + *((uint16_t*)(_COMM_PAGE_VERSION+_COMM_PAGE_RW_OFFSET)) = (uint16_t) _COMM_PAGE_THIS_VERSION; + + commpage_init_cpu_capabilities(); + commpage_set_timestamp(0, 0, 0, 0, 0); + + if (_cpu_capabilities & kCache32) + c2 = 32; + else if (_cpu_capabilities & kCache64) + c2 = 64; + else if (_cpu_capabilities & kCache128) + c2 = 128; + else + c2 = 0; + + *((uint16_t*)(_COMM_PAGE_CACHE_LINESIZE+_COMM_PAGE_RW_OFFSET)) = c2; + *((uint32_t*)(_COMM_PAGE_SPIN_COUNT+_COMM_PAGE_RW_OFFSET)) = 1; + + commpage_update_active_cpus(); + cpufamily = cpuid_get_cpufamily(); + + /* machine_info valid after ml_get_max_cpus() */ + *((uint8_t*)(_COMM_PAGE_PHYSICAL_CPUS+_COMM_PAGE_RW_OFFSET)) = (uint8_t) machine_info.physical_cpu_max; + *((uint8_t*)(_COMM_PAGE_LOGICAL_CPUS+_COMM_PAGE_RW_OFFSET))= (uint8_t) machine_info.logical_cpu_max; + *((uint64_t*)(_COMM_PAGE_MEMORY_SIZE+_COMM_PAGE_RW_OFFSET)) = machine_info.max_mem; + *((uint32_t*)(_COMM_PAGE_CPUFAMILY+_COMM_PAGE_RW_OFFSET)) = (uint32_t)cpufamily; + *((uint32_t*)(_COMM_PAGE_DEV_FIRM+_COMM_PAGE_RW_OFFSET)) = (uint32_t)PE_i_can_has_debugger(NULL); + *((uint8_t*)(_COMM_PAGE_USER_TIMEBASE+_COMM_PAGE_RW_OFFSET)) = user_timebase_allowed(); + *((uint8_t*)(_COMM_PAGE_CONT_HWCLOCK+_COMM_PAGE_RW_OFFSET)) = user_cont_hwclock_allowed(); + *((uint8_t*)(_COMM_PAGE_KERNEL_PAGE_SHIFT+_COMM_PAGE_RW_OFFSET)) = (uint8_t) page_shift; + +#if __arm64__ + *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_32+_COMM_PAGE_RW_OFFSET)) = (uint8_t) page_shift_user32; + *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_64+_COMM_PAGE_RW_OFFSET)) = (uint8_t) SIXTEENK_PAGE_SHIFT; +#elif (__ARM_ARCH_7K__ >= 2) && defined(PLATFORM_WatchOS) + /* enforce 16KB alignment for watch targets with new ABI */ + *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_32+_COMM_PAGE_RW_OFFSET)) = (uint8_t) SIXTEENK_PAGE_SHIFT; + *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_64+_COMM_PAGE_RW_OFFSET)) = (uint8_t) SIXTEENK_PAGE_SHIFT; +#else /* __arm64__ */ + *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_32+_COMM_PAGE_RW_OFFSET)) = (uint8_t) PAGE_SHIFT; + *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_64+_COMM_PAGE_RW_OFFSET)) = (uint8_t) PAGE_SHIFT; +#endif /* __arm64__ */ + + commpage_update_timebase(); + commpage_update_mach_continuous_time(0); + + clock_sec_t secs; + clock_usec_t microsecs; + clock_get_boottime_microtime(&secs, µsecs); + commpage_update_boottime(secs * USEC_PER_SEC + microsecs); + + /* + * set commpage approximate time to zero for initialization. + * scheduler shall populate correct value before running user thread + */ + *((uint64_t *)(_COMM_PAGE_APPROX_TIME+ _COMM_PAGE_RW_OFFSET)) = 0; +#ifdef CONFIG_MACH_APPROXIMATE_TIME + *((uint8_t *)(_COMM_PAGE_APPROX_TIME_SUPPORTED+_COMM_PAGE_RW_OFFSET)) = 1; +#else + *((uint8_t *)(_COMM_PAGE_APPROX_TIME_SUPPORTED+_COMM_PAGE_RW_OFFSET)) = 0; +#endif + + commpage_update_kdebug_state(); + +#if CONFIG_ATM + commpage_update_atm_diagnostic_config(atm_get_diagnostic_config()); +#endif + +} + +struct mu { + uint64_t m; // magic number + int32_t a; // add indicator + int32_t s; // shift amount +}; + +void +commpage_set_timestamp( + uint64_t tbr, + uint64_t secs, + uint64_t frac, + uint64_t scale, + uint64_t tick_per_sec) +{ + new_commpage_timeofday_data_t *commpage_timeofday_datap; + + if (commPagePtr == 0) + return; + + commpage_timeofday_datap = (new_commpage_timeofday_data_t *)(_COMM_PAGE_NEWTIMEOFDAY_DATA+_COMM_PAGE_RW_OFFSET); + + commpage_timeofday_datap->TimeStamp_tick = 0x0ULL; + +#if (__ARM_ARCH__ >= 7) + __asm__ volatile("dmb ish"); +#endif + commpage_timeofday_datap->TimeStamp_sec = secs; + commpage_timeofday_datap->TimeStamp_frac = frac; + commpage_timeofday_datap->Ticks_scale = scale; + commpage_timeofday_datap->Ticks_per_sec = tick_per_sec; + +#if (__ARM_ARCH__ >= 7) + __asm__ volatile("dmb ish"); +#endif + commpage_timeofday_datap->TimeStamp_tick = tbr; +} + +/* + * Update _COMM_PAGE_MEMORY_PRESSURE. Called periodically from vm's compute_memory_pressure() + */ + +void +commpage_set_memory_pressure( + unsigned int pressure ) +{ + if (commPagePtr == 0) + return; + *((uint32_t *)(_COMM_PAGE_MEMORY_PRESSURE+_COMM_PAGE_RW_OFFSET)) = pressure; +} + +/* + * Update _COMM_PAGE_SPIN_COUNT. We might want to reduce when running on a battery, etc. + */ + +void +commpage_set_spin_count( + unsigned int count ) +{ + if (count == 0) /* we test for 0 after decrement, not before */ + count = 1; + + if (commPagePtr == 0) + return; + *((uint32_t *)(_COMM_PAGE_SPIN_COUNT+_COMM_PAGE_RW_OFFSET)) = count; +} + +/* + * Determine number of CPUs on this system. + */ +static int +commpage_cpus( void ) +{ + int cpus; + + cpus = ml_get_max_cpus(); // NB: this call can block + + if (cpus == 0) + panic("commpage cpus==0"); + if (cpus > 0xFF) + cpus = 0xFF; + + return cpus; +} + +/* + * Initialize _cpu_capabilities vector + */ +static void +commpage_init_cpu_capabilities( void ) +{ + uint32_t bits; + int cpus; + ml_cpu_info_t cpu_info; + + bits = 0; + ml_cpu_get_info(&cpu_info); + + switch (cpu_info.cache_line_size) { + case 128: + bits |= kCache128; + break; + case 64: + bits |= kCache64; + break; + case 32: + bits |= kCache32; + break; + default: + break; + } + cpus = commpage_cpus(); + + if (cpus == 1) + bits |= kUP; + + bits |= (cpus << kNumCPUsShift); + + bits |= kFastThreadLocalStorage; // TPIDRURO for TLS +#if __ARM_VFP__ + bits |= kHasVfp; +#if (__ARM_VFP__ >= 3) + bits |= kHasNeon; + +#if defined(__arm64__) + bits |= kHasNeonHPFP; +#else + boolean_t intr = ml_set_interrupts_enabled(FALSE); + unsigned int mvfr1 = get_mvfr1(); + + if (mvfr1 & MVFR_ASIMD_HPFP) + bits |= kHasNeonHPFP; + (void) ml_set_interrupts_enabled(intr); +#endif +#endif +#endif +#if defined(__arm64__) + bits |= kHasFMA; +#endif +#if __ARM_ENABLE_WFE_ +#ifdef __arm64__ + if (arm64_wfe_allowed()) { + bits |= kHasEvent; + } +#else + bits |= kHasEvent; +#endif +#endif +#if __ARM_V8_CRYPTO_EXTENSIONS__ + bits |= kHasARMv8Crypto; +#endif +#ifdef __arm64__ + if ((__builtin_arm_rsr64("ID_AA64ISAR0_EL1") & ID_AA64ISAR0_EL1_ATOMIC_MASK) == ID_AA64ISAR0_EL1_ATOMIC_8_1) { + bits |= kHasARMv81Atomics; + gARMv81Atomics = 1; + } +#endif + _cpu_capabilities = bits; + + *((uint32_t *)(_COMM_PAGE_CPU_CAPABILITIES+_COMM_PAGE_RW_OFFSET)) = _cpu_capabilities; +} + +/* + * Updated every time a logical CPU goes offline/online + */ +void +commpage_update_active_cpus(void) +{ + if (!commPagePtr) + return; + *((uint8_t *)(_COMM_PAGE_ACTIVE_CPUS+_COMM_PAGE_RW_OFFSET)) = processor_avail_count; +} + +/* + * Update the commpage bits for mach_absolute_time and mach_continuous_time (for userspace) + */ +void +commpage_update_timebase(void) +{ + if (commPagePtr) { + *((uint64_t*)(_COMM_PAGE_TIMEBASE_OFFSET+_COMM_PAGE_RW_OFFSET)) = rtclock_base_abstime; + } +} + +/* + * Update the commpage with current kdebug state. This currently has bits for + * global trace state, and typefilter enablement. It is likely additional state + * will be tracked in the future. + * + * INVARIANT: This value will always be 0 if global tracing is disabled. This + * allows simple guard tests of "if (*_COMM_PAGE_KDEBUG_ENABLE) { ... }" + */ +void +commpage_update_kdebug_state(void) +{ + if (commPagePtr) + *((volatile uint32_t*)(_COMM_PAGE_KDEBUG_ENABLE+_COMM_PAGE_RW_OFFSET)) = kdebug_commpage_state(); +} + +/* Ditto for atm_diagnostic_config */ +void +commpage_update_atm_diagnostic_config(uint32_t diagnostic_config) +{ + if (commPagePtr) + *((volatile uint32_t*)(_COMM_PAGE_ATM_DIAGNOSTIC_CONFIG+_COMM_PAGE_RW_OFFSET)) = diagnostic_config; +} + +/* + * Update the commpage data with the state of multiuser mode for + * this device. Allowing various services in userspace to avoid + * IPC in the (more common) non-multiuser environment. + */ +void +commpage_update_multiuser_config(uint32_t multiuser_config) +{ + if (commPagePtr) + *((volatile uint32_t *)(_COMM_PAGE_MULTIUSER_CONFIG+_COMM_PAGE_RW_OFFSET)) = multiuser_config; +} + +/* + * update the commpage data for + * last known value of mach_absolute_time() + */ + +void +commpage_update_mach_approximate_time(uint64_t abstime) +{ +#ifdef CONFIG_MACH_APPROXIMATE_TIME + uintptr_t approx_time_base = (uintptr_t)(_COMM_PAGE_APPROX_TIME + _COMM_PAGE_RW_OFFSET); + uint64_t saved_data; + + if (commPagePtr) { + saved_data = atomic_load_explicit((_Atomic uint64_t *)approx_time_base, + memory_order_relaxed); + if (saved_data < abstime) { + /* ignoring the success/fail return value assuming that + * if the value has been updated since we last read it, + * "someone" has a newer timestamp than us and ours is + * now invalid. */ + atomic_compare_exchange_strong_explicit((_Atomic uint64_t *)approx_time_base, + &saved_data, abstime, memory_order_relaxed, memory_order_relaxed); + } + } +#else +#pragma unused (abstime) +#endif +} + +/* + * update the commpage data's total system sleep time for + * userspace call to mach_continuous_time() + */ +void +commpage_update_mach_continuous_time(uint64_t sleeptime) +{ + if (commPagePtr) { +#ifdef __arm64__ + *((uint64_t *)(_COMM_PAGE_CONT_TIMEBASE + _COMM_PAGE_RW_OFFSET)) = sleeptime; +#else + uint64_t *c_time_base = (uint64_t *)(_COMM_PAGE_CONT_TIMEBASE + _COMM_PAGE_RW_OFFSET); + uint64_t old; + do { + old = *c_time_base; + } while(!OSCompareAndSwap64(old, sleeptime, c_time_base)); +#endif /* __arm64__ */ + } +} + +/* + * update the commpage's value for the boot time + */ +void +commpage_update_boottime(uint64_t value) +{ + if (commPagePtr) { +#ifdef __arm64__ + *((uint64_t *)(_COMM_PAGE_BOOTTIME_USEC + _COMM_PAGE_RW_OFFSET)) = value; +#else + uint64_t *cp = (uint64_t *)(_COMM_PAGE_BOOTTIME_USEC + _COMM_PAGE_RW_OFFSET); + uint64_t old_value; + do { + old_value = *cp; + } while (!OSCompareAndSwap64(old_value, value, cp)); +#endif /* __arm64__ */ + } +} diff --git a/osfmk/arm/commpage/commpage.h b/osfmk/arm/commpage/commpage.h new file mode 100644 index 000000000..69ac59cf7 --- /dev/null +++ b/osfmk/arm/commpage/commpage.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2003-2008 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef _ARM_COMMPAGE_H +#define _ARM_COMMPAGE_H + +#ifndef __ASSEMBLER__ +#include <stdint.h> +#endif /* __ASSEMBLER__ */ + +extern void commpage_set_timestamp(uint64_t tbr, uint64_t secs, uint64_t frac, uint64_t scale, uint64_t tick_per_sec); +#define commpage_disable_timestamp() commpage_set_timestamp( 0, 0, 0, 0, 0 ); +extern void commpage_set_memory_pressure( unsigned int pressure ); +extern void commpage_update_active_cpus(void); +extern void commpage_set_spin_count(unsigned int count); +extern void commpage_update_timebase(void); +extern void commpage_update_mach_approximate_time(uint64_t); +extern void commpage_update_kdebug_state(void); +extern void commpage_update_atm_diagnostic_config(uint32_t); +extern void commpage_update_mach_continuous_time(uint64_t sleeptime); +extern void commpage_update_multiuser_config(uint32_t); +extern void commpage_update_boottime(uint64_t boottime_usec); + +#endif /* _ARM_COMMPAGE_H */ diff --git a/osfmk/arm/commpage/commpage_sigs.h b/osfmk/arm/commpage/commpage_sigs.h new file mode 100644 index 000000000..d392097a1 --- /dev/null +++ b/osfmk/arm/commpage/commpage_sigs.h @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#define BSWAP_32(x) \ + (((x) << 24) & 0xff000000) | \ + (((x) << 8) & 0x00ff0000) | \ + (((x) >> 8) & 0x0000ff00) | \ + (((x) >> 24) & 0x000000ff) + +#define BSWAP_32_OFFSET(x) \ + BSWAP_32(x + _COMM_PAGE_SIGS_OFFSET) + +#define COMMPAGE_SIGS_BEGIN \ +.const_data ; \ +.align 2 ; \ +.private_extern _commpage_sigs_begin ; \ +_commpage_sigs_begin: + +#define COMMPAGE_SIGS_DONE \ +.private_extern _commpage_sigs_end ; \ +_commpage_sigs_end: ; \ + +#define COMMPAGE_SIG_START(x) \ +.private_extern _commpage_sig ## x ; \ +_commpage_sig ## x ## : ; \ + .long BSWAP_32(0x14400000) ; \ + .long BSWAP_32(0x00000001) ; \ + .asciz # x ; \ + .align 2 ; \ + .long BSWAP_32(0x14400000) + +#define COMMPAGE_SIG_END(x) \ + .long BSWAP_32(0x4e800020) ; \ + .long BSWAP_32(0x14400000) ; \ + .long BSWAP_32(0x00000000) ; \ + .asciz # x ; \ + .align 2 ; \ + .long BSWAP_32(0x14400000) + +#define OBJCRTP_SIG_START(x) COMMPAGE_SIG_START(x) + +#define OBJCRTP_SIG_END(x) \ + .long BSWAP_32(0x14400000) ; \ + .long BSWAP_32(0x00000000) ; \ + .asciz # x ; \ + .align 2 ; \ + .long BSWAP_32(0x14400000) + +#define OBJCRTP_SIG_CALL_SUBJECT(x) \ + .long BSWAP_32(0x14400002) ; \ + .long BSWAP_32(0x00000000) ; \ + .long BSWAP_32(0x00040000) ; \ + .long BSWAP_32(0x00000000) ; \ + .asciz # x ; \ + .align 2 ; \ + .long BSWAP_32(0x14400002) + +#define ARG(n) \ + ((((n * 2) + 6) << 20) + 4) + +#define COMMPAGE_SIG_ARG(n) \ + .long BSWAP_32(0x14400001) ; \ + .long BSWAP_32(ARG(n)) ; \ + .long BSWAP_32(0x14400001) + +#define COMMPAGE_SIG_CALL(x, n) \ + .long BSWAP_32(0x14400002) ; \ + .long BSWAP_32(n) ; \ + .long BSWAP_32(0x00000000) ; \ + .long BSWAP_32(0x00000000) ; \ + .asciz # x ; \ + .align 2 ; \ + .long BSWAP_32(0x14400002) + +#define COMMPAGE_SIG_CALL_VOID(x) \ + COMMPAGE_SIG_CALL(x, 0) + +#define COMMPAGE_SIG_CALL_RET0(x) \ + COMMPAGE_SIG_CALL(x, ARG(0)) + +#define COMMPAGE_SIG_CALL_RET1(x) \ + COMMPAGE_SIG_CALL(x, ARG(1)) + +#define COMMPAGE_FAST_TRAP(x) \ + .long BSWAP_32(0x14400005) ; \ + .long BSWAP_32_OFFSET(x) ; \ + .long BSWAP_32(0x14400005) diff --git a/osfmk/arm/conf.c b/osfmk/arm/conf.c new file mode 100644 index 000000000..a9da87124 --- /dev/null +++ b/osfmk/arm/conf.c @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * @APPLE_FREE_COPYRIGHT@ + */ +/* + * Mach Operating System Copyright (c) 1991,1990,1989 Carnegie Mellon + * University All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright notice + * and this permission notice appear in all copies of the software, + * derivative works or modified versions, and any portions thereof, and that + * both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" CONDITION. + * CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR ANY DAMAGES + * WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science Carnegie Mellon University Pittsburgh PA + * 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon the + * rights to redistribute these changes. + */ +/* + * */ + +#include <types.h> +#include <kern/clock.h> +#include <libkern/section_keywords.h> + +/* + * Clock device subsystem configuration. The clock_list[] + * table contains the clock structures for all clocks in + * the system. + */ + +extern const struct clock_ops sysclk_ops, calend_ops; + +/* + * List of clock devices. + */ +SECURITY_READ_ONLY_LATE(struct clock) clock_list[] = { + + /* SYSTEM_CLOCK */ + {&sysclk_ops, 0, 0}, + + /* CALENDAR_CLOCK */ + {&calend_ops, 0, 0}, +}; +int clock_count = sizeof(clock_list) / sizeof(clock_list[0]); diff --git a/osfmk/arm/cpu.c b/osfmk/arm/cpu.c new file mode 100644 index 000000000..46cfcddb7 --- /dev/null +++ b/osfmk/arm/cpu.c @@ -0,0 +1,604 @@ +/* + * Copyright (c) 2007-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * File: arm/cpu.c + * + * cpu specific routines + */ + +#include <kern/kalloc.h> +#include <kern/machine.h> +#include <kern/cpu_number.h> +#include <kern/thread.h> +#include <kern/timer_queue.h> +#include <arm/cpu_data.h> +#include <arm/cpuid.h> +#include <arm/caches_internal.h> +#include <arm/cpu_data_internal.h> +#include <arm/cpu_internal.h> +#include <arm/misc_protos.h> +#include <arm/machine_cpu.h> +#include <arm/rtclock.h> +#include <arm/proc_reg.h> +#include <mach/processor_info.h> +#include <vm/pmap.h> +#include <vm/vm_kern.h> +#include <vm/vm_map.h> +#include <pexpert/arm/board_config.h> +#include <pexpert/arm/protos.h> +#include <sys/kdebug.h> + +#include <machine/atomic.h> + +#if KPC +#include <kern/kpc.h> +#endif + +extern unsigned int resume_idle_cpu; +extern unsigned int start_cpu; + +unsigned int start_cpu_paddr; + +extern boolean_t idle_enable; +extern unsigned int real_ncpus; +extern uint64_t wake_abstime; + +extern void* wfi_inst; +unsigned wfi_fast = 1; +unsigned patch_to_nop = 0xe1a00000; + +void *LowExceptionVectorsAddr; +#define IOS_STATE (((vm_offset_t)LowExceptionVectorsAddr + 0x80)) +#define IOS_STATE_SIZE (0x08UL) +static const uint8_t suspend_signature[] = {'X', 'S', 'O', 'M', 'P', 'S', 'U', 'S'}; +static const uint8_t running_signature[] = {'X', 'S', 'O', 'M', 'N', 'N', 'U', 'R'}; + +/* + * Routine: cpu_bootstrap + * Function: + */ +void +cpu_bootstrap(void) +{ +} + + +/* + * Routine: cpu_sleep + * Function: + */ +void +cpu_sleep(void) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + pmap_switch_user_ttb(kernel_pmap); + cpu_data_ptr->cpu_active_thread = current_thread(); + cpu_data_ptr->cpu_reset_handler = (vm_offset_t) start_cpu_paddr; + cpu_data_ptr->cpu_flags |= SleepState; + cpu_data_ptr->cpu_user_debug = NULL; + + CleanPoC_Dcache(); + + PE_cpu_machine_quiesce(cpu_data_ptr->cpu_id); + +} + +_Atomic uint32_t cpu_idle_count = 0; + +/* + * Routine: cpu_idle + * Function: + */ +void __attribute__((noreturn)) +cpu_idle(void) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + uint64_t new_idle_timeout_ticks = 0x0ULL, lastPop; + + if ((!idle_enable) || (cpu_data_ptr->cpu_signal & SIGPdisabled)) + Idle_load_context(); + if (!SetIdlePop()) + Idle_load_context(); + lastPop = cpu_data_ptr->rtcPop; + + pmap_switch_user_ttb(kernel_pmap); + cpu_data_ptr->cpu_active_thread = current_thread(); + if (cpu_data_ptr->cpu_user_debug) + arm_debug_set(NULL); + cpu_data_ptr->cpu_user_debug = NULL; + + if (cpu_data_ptr->cpu_idle_notify) + ((processor_idle_t) cpu_data_ptr->cpu_idle_notify) (cpu_data_ptr->cpu_id, TRUE, &new_idle_timeout_ticks); + + if (cpu_data_ptr->idle_timer_notify != 0) { + if (new_idle_timeout_ticks == 0x0ULL) { + /* turn off the idle timer */ + cpu_data_ptr->idle_timer_deadline = 0x0ULL; + } else { + /* set the new idle timeout */ + clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline); + } + timer_resync_deadlines(); + if (cpu_data_ptr->rtcPop != lastPop) + SetIdlePop(); + } + +#if KPC + kpc_idle(); +#endif + + platform_cache_idle_enter(); + cpu_idle_wfi((boolean_t) wfi_fast); + platform_cache_idle_exit(); + + ClearIdlePop(TRUE); + cpu_idle_exit(); +} + +/* + * Routine: cpu_idle_exit + * Function: + */ +void +cpu_idle_exit(void) +{ + uint64_t new_idle_timeout_ticks = 0x0ULL; + cpu_data_t *cpu_data_ptr = getCpuDatap(); + +#if KPC + kpc_idle_exit(); +#endif + + + pmap_set_pmap(cpu_data_ptr->cpu_active_thread->map->pmap, current_thread()); + + if (cpu_data_ptr->cpu_idle_notify) + ((processor_idle_t) cpu_data_ptr->cpu_idle_notify) (cpu_data_ptr->cpu_id, FALSE, &new_idle_timeout_ticks); + + if (cpu_data_ptr->idle_timer_notify != 0) { + if (new_idle_timeout_ticks == 0x0ULL) { + /* turn off the idle timer */ + cpu_data_ptr->idle_timer_deadline = 0x0ULL; + } else { + /* set the new idle timeout */ + clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline); + } + timer_resync_deadlines(); + } + + Idle_load_context(); +} + +void +cpu_init(void) +{ + cpu_data_t *cdp = getCpuDatap(); + arm_cpu_info_t *cpu_info_p; + + if (cdp->cpu_type != CPU_TYPE_ARM) { + + cdp->cpu_type = CPU_TYPE_ARM; + + timer_call_queue_init(&cdp->rtclock_timer.queue); + cdp->rtclock_timer.deadline = EndOfAllTime; + + if (cdp == &BootCpuData) { + do_cpuid(); + do_cacheid(); + do_mvfpid(); + } else { + /* + * We initialize non-boot CPUs here; the boot CPU is + * dealt with as part of pmap_bootstrap. + */ + pmap_cpu_data_init(); + } + /* ARM_SMP: Assuming identical cpu */ + do_debugid(); + + cpu_info_p = cpuid_info(); + + /* switch based on CPU's reported architecture */ + switch (cpu_info_p->arm_info.arm_arch) { + case CPU_ARCH_ARMv4T: + case CPU_ARCH_ARMv5T: + cdp->cpu_subtype = CPU_SUBTYPE_ARM_V4T; + break; + case CPU_ARCH_ARMv5TE: + case CPU_ARCH_ARMv5TEJ: + if (cpu_info_p->arm_info.arm_implementor == CPU_VID_INTEL) + cdp->cpu_subtype = CPU_SUBTYPE_ARM_XSCALE; + else + cdp->cpu_subtype = CPU_SUBTYPE_ARM_V5TEJ; + break; + case CPU_ARCH_ARMv6: + cdp->cpu_subtype = CPU_SUBTYPE_ARM_V6; + break; + case CPU_ARCH_ARMv7: + cdp->cpu_subtype = CPU_SUBTYPE_ARM_V7; + break; + case CPU_ARCH_ARMv7f: + cdp->cpu_subtype = CPU_SUBTYPE_ARM_V7F; + break; + case CPU_ARCH_ARMv7s: + cdp->cpu_subtype = CPU_SUBTYPE_ARM_V7S; + break; + case CPU_ARCH_ARMv7k: + cdp->cpu_subtype = CPU_SUBTYPE_ARM_V7K; + break; + default: + cdp->cpu_subtype = CPU_SUBTYPE_ARM_ALL; + break; + } + + cdp->cpu_threadtype = CPU_THREADTYPE_NONE; + } + cdp->cpu_stat.irq_ex_cnt_wake = 0; + cdp->cpu_stat.ipi_cnt_wake = 0; + cdp->cpu_stat.timer_cnt_wake = 0; + cdp->cpu_running = TRUE; + cdp->cpu_sleep_token_last = cdp->cpu_sleep_token; + cdp->cpu_sleep_token = 0x0UL; + +} + +cpu_data_t * +cpu_data_alloc(boolean_t is_boot_cpu) +{ + cpu_data_t *cpu_data_ptr = NULL; + + if (is_boot_cpu) + cpu_data_ptr = &BootCpuData; + else { + void *irq_stack = NULL; + void *fiq_stack = NULL; + + if ((kmem_alloc(kernel_map, (vm_offset_t *)&cpu_data_ptr, sizeof(cpu_data_t), VM_KERN_MEMORY_CPU)) != KERN_SUCCESS) + goto cpu_data_alloc_error; + + bzero((void *)cpu_data_ptr, sizeof(cpu_data_t)); + + if ((irq_stack = kalloc(INTSTACK_SIZE)) == 0) + goto cpu_data_alloc_error; +#if __BIGGEST_ALIGNMENT__ + /* force 16-byte alignment */ + if ((uint32_t)irq_stack & 0x0F) + irq_stack = (void *)((uint32_t)irq_stack + (0x10 - ((uint32_t)irq_stack & 0x0F))); +#endif + cpu_data_ptr->intstack_top = (vm_offset_t)irq_stack + INTSTACK_SIZE ; + cpu_data_ptr->istackptr = cpu_data_ptr->intstack_top; + + if ((fiq_stack = kalloc(PAGE_SIZE)) == 0) + goto cpu_data_alloc_error; +#if __BIGGEST_ALIGNMENT__ + /* force 16-byte alignment */ + if ((uint32_t)fiq_stack & 0x0F) + fiq_stack = (void *)((uint32_t)fiq_stack + (0x10 - ((uint32_t)fiq_stack & 0x0F))); +#endif + cpu_data_ptr->fiqstack_top = (vm_offset_t)fiq_stack + PAGE_SIZE ; + cpu_data_ptr->fiqstackptr = cpu_data_ptr->fiqstack_top; + } + + cpu_data_ptr->cpu_processor = cpu_processor_alloc(is_boot_cpu); + if (cpu_data_ptr->cpu_processor == (struct processor *)NULL) + goto cpu_data_alloc_error; + + return cpu_data_ptr; + +cpu_data_alloc_error: + panic("cpu_data_alloc() failed\n"); + return (cpu_data_t *)NULL; +} + + +void +cpu_data_free(cpu_data_t *cpu_data_ptr) +{ + if (cpu_data_ptr == &BootCpuData) + return; + + cpu_processor_free( cpu_data_ptr->cpu_processor); + kfree( (void *)(cpu_data_ptr->intstack_top - INTSTACK_SIZE), INTSTACK_SIZE); + kfree( (void *)(cpu_data_ptr->fiqstack_top - PAGE_SIZE), PAGE_SIZE); + kmem_free(kernel_map, (vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t)); +} + +void +cpu_data_init(cpu_data_t *cpu_data_ptr) +{ + uint32_t i = 0; + + cpu_data_ptr->cpu_flags = 0; +#if __arm__ + cpu_data_ptr->cpu_exc_vectors = (vm_offset_t)&ExceptionVectorsTable; +#endif + cpu_data_ptr->interrupts_enabled = 0; + cpu_data_ptr->cpu_int_state = 0; + cpu_data_ptr->cpu_pending_ast = AST_NONE; + cpu_data_ptr->cpu_cache_dispatch = (void *) 0; + cpu_data_ptr->rtcPop = EndOfAllTime; + cpu_data_ptr->rtclock_datap = &RTClockData; + cpu_data_ptr->cpu_user_debug = NULL; + cpu_data_ptr->cpu_base_timebase_low = 0; + cpu_data_ptr->cpu_base_timebase_high = 0; + cpu_data_ptr->cpu_idle_notify = (void *) 0; + cpu_data_ptr->cpu_idle_latency = 0x0ULL; + cpu_data_ptr->cpu_idle_pop = 0x0ULL; + cpu_data_ptr->cpu_reset_type = 0x0UL; + cpu_data_ptr->cpu_reset_handler = 0x0UL; + cpu_data_ptr->cpu_reset_assist = 0x0UL; + cpu_data_ptr->cpu_regmap_paddr = 0x0ULL; + cpu_data_ptr->cpu_phys_id = 0x0UL; + cpu_data_ptr->cpu_l2_access_penalty = 0; + cpu_data_ptr->cpu_cluster_type = CLUSTER_TYPE_SMP; + cpu_data_ptr->cpu_cluster_id = 0; + cpu_data_ptr->cpu_l2_id = 0; + cpu_data_ptr->cpu_l2_size = 0; + cpu_data_ptr->cpu_l3_id = 0; + cpu_data_ptr->cpu_l3_size = 0; + + cpu_data_ptr->cpu_signal = SIGPdisabled; + +#if DEBUG || DEVELOPMENT + cpu_data_ptr->failed_xcall = NULL; + cpu_data_ptr->failed_signal = 0; + cpu_data_ptr->failed_signal_count = 0; +#endif + + cpu_data_ptr->cpu_get_fiq_handler = NULL; + cpu_data_ptr->cpu_tbd_hardware_addr = NULL; + cpu_data_ptr->cpu_tbd_hardware_val = NULL; + cpu_data_ptr->cpu_get_decrementer_func = NULL; + cpu_data_ptr->cpu_set_decrementer_func = NULL; + cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH; + cpu_data_ptr->cpu_sleep_token_last = 0x00000000UL; + cpu_data_ptr->cpu_xcall_p0 = NULL; + cpu_data_ptr->cpu_xcall_p1 = NULL; + +#if __ARM_SMP__ && defined(ARMA7) + cpu_data_ptr->cpu_CLWFlush_req = 0x0ULL; + cpu_data_ptr->cpu_CLWFlush_last = 0x0ULL; + cpu_data_ptr->cpu_CLWClean_req = 0x0ULL; + cpu_data_ptr->cpu_CLWClean_last = 0x0ULL; + cpu_data_ptr->cpu_CLW_active = 0x1UL; +#endif + + pmap_cpu_data_t * pmap_cpu_data_ptr = &cpu_data_ptr->cpu_pmap_cpu_data; + + pmap_cpu_data_ptr->cpu_user_pmap = (struct pmap *) NULL; + pmap_cpu_data_ptr->cpu_user_pmap_stamp = 0; + pmap_cpu_data_ptr->cpu_number = PMAP_INVALID_CPU_NUM; + + for (i = 0; i < (sizeof(pmap_cpu_data_ptr->cpu_asid_high_bits) / sizeof(*pmap_cpu_data_ptr->cpu_asid_high_bits)); i++) { + pmap_cpu_data_ptr->cpu_asid_high_bits[i] = 0; + } + cpu_data_ptr->halt_status = CPU_NOT_HALTED; +} + +kern_return_t +cpu_data_register(cpu_data_t *cpu_data_ptr) +{ + int cpu; + + cpu = OSIncrementAtomic((SInt32*)&real_ncpus); + if (real_ncpus > MAX_CPUS) { + return KERN_FAILURE; + } + + cpu_data_ptr->cpu_number = cpu; + CpuDataEntries[cpu].cpu_data_vaddr = cpu_data_ptr; + CpuDataEntries[cpu].cpu_data_paddr = (void *)ml_vtophys( (vm_offset_t)cpu_data_ptr); + return KERN_SUCCESS; +} + +kern_return_t +cpu_start(int cpu) +{ + kprintf("cpu_start() cpu: %d\n", cpu); + if (cpu == cpu_number()) { + cpu_machine_init(); + return KERN_SUCCESS; + } else { +#if __ARM_SMP__ + cpu_data_t *cpu_data_ptr; + thread_t first_thread; + + cpu_data_ptr = CpuDataEntries[cpu].cpu_data_vaddr; + cpu_data_ptr->cpu_reset_handler = (vm_offset_t) start_cpu_paddr; + + cpu_data_ptr->cpu_pmap_cpu_data.cpu_user_pmap = NULL; + + if (cpu_data_ptr->cpu_processor->next_thread != THREAD_NULL) + first_thread = cpu_data_ptr->cpu_processor->next_thread; + else + first_thread = cpu_data_ptr->cpu_processor->idle_thread; + cpu_data_ptr->cpu_active_thread = first_thread; + first_thread->machine.CpuDatap = cpu_data_ptr; + + flush_dcache((vm_offset_t)&CpuDataEntries[cpu], sizeof(cpu_data_entry_t), FALSE); + flush_dcache((vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t), FALSE); + (void) PE_cpu_start(cpu_data_ptr->cpu_id, (vm_offset_t)NULL, (vm_offset_t)NULL); + return KERN_SUCCESS; +#else + return KERN_FAILURE; +#endif + } +} + +void +cpu_timebase_init(boolean_t from_boot __unused) +{ + cpu_data_t *cdp = getCpuDatap(); + + if (cdp->cpu_get_fiq_handler == NULL) { + cdp->cpu_get_fiq_handler = rtclock_timebase_func.tbd_fiq_handler; + cdp->cpu_get_decrementer_func = rtclock_timebase_func.tbd_get_decrementer; + cdp->cpu_set_decrementer_func = rtclock_timebase_func.tbd_set_decrementer; + cdp->cpu_tbd_hardware_addr = (void *)rtclock_timebase_addr; + cdp->cpu_tbd_hardware_val = (void *)rtclock_timebase_val; + } + cdp->cpu_decrementer = 0x7FFFFFFFUL; + cdp->cpu_timebase_low = 0x0UL; + cdp->cpu_timebase_high = 0x0UL; + +#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4) + /* For the newer ARMv7k ABI where 64-bit types are 64-bit aligned, but pointers + * are 32-bit. */ + cdp->cpu_base_timebase_low = rtclock_base_abstime_low; + cdp->cpu_base_timebase_high = rtclock_base_abstime_high; +#else + *((uint64_t *) & cdp->cpu_base_timebase_low) = rtclock_base_abstime; +#endif +} + + +__attribute__((noreturn)) +void +ml_arm_sleep(void) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + if (cpu_data_ptr == &BootCpuData) { + cpu_data_t *target_cdp; + unsigned int cpu; + + for (cpu=0; cpu < MAX_CPUS; cpu++) { + target_cdp = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr; + if(target_cdp == (cpu_data_t *)NULL) + break; + + if (target_cdp == cpu_data_ptr) + continue; + + while (target_cdp->cpu_sleep_token != ARM_CPU_ON_SLEEP_PATH); + } + + /* Now that the other cores have entered the sleep path, set + * the abstime fixup we'll use when we resume.*/ + rtclock_base_abstime = ml_get_timebase(); + wake_abstime = rtclock_base_abstime; + + } else { + platform_cache_disable(); + CleanPoU_Dcache(); + } + cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH; +#if __ARM_SMP__ && defined(ARMA7) + cpu_data_ptr->cpu_CLWFlush_req = 0; + cpu_data_ptr->cpu_CLWClean_req = 0; + __builtin_arm_dmb(DMB_ISH); + cpu_data_ptr->cpu_CLW_active = 0; +#endif + if (cpu_data_ptr == &BootCpuData) { + platform_cache_disable(); + platform_cache_shutdown(); + bcopy((const void *)suspend_signature, (void *)(IOS_STATE), IOS_STATE_SIZE); + } else + CleanPoC_DcacheRegion((vm_offset_t) cpu_data_ptr, sizeof(cpu_data_t)); + + __builtin_arm_dsb(DSB_SY); + while (TRUE) { +#if __ARM_ENABLE_WFE_ + __builtin_arm_wfe(); +#endif + } /* Spin */ +} + +void +cpu_machine_idle_init(boolean_t from_boot) +{ + static const unsigned int *BootArgs_paddr = (unsigned int *)NULL; + static const unsigned int *CpuDataEntries_paddr = (unsigned int *)NULL; + static unsigned int resume_idle_cpu_paddr = (unsigned int )NULL; + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + if (from_boot) { + unsigned int jtag = 0; + unsigned int wfi; + + + if (PE_parse_boot_argn("jtag", &jtag, sizeof (jtag))) { + if (jtag != 0) + idle_enable = FALSE; + else + idle_enable = TRUE; + } else + idle_enable = TRUE; + + if (!PE_parse_boot_argn("wfi", &wfi, sizeof (wfi))) + wfi = 1; + + if (wfi == 0) + bcopy_phys((addr64_t)ml_static_vtop((vm_offset_t)&patch_to_nop), + (addr64_t)ml_static_vtop((vm_offset_t)&wfi_inst), sizeof(unsigned)); + if (wfi == 2) + wfi_fast = 0; + + LowExceptionVectorsAddr = (void *)ml_io_map(ml_vtophys((vm_offset_t)gPhysBase), PAGE_SIZE); + + /* Copy Exception Vectors low, but don't touch the sleep token */ + bcopy((void *)&ExceptionLowVectorsBase, (void *)LowExceptionVectorsAddr, 0x90); + bcopy(((void *)(((vm_offset_t)&ExceptionLowVectorsBase) + 0xA0)), ((void *)(((vm_offset_t)LowExceptionVectorsAddr) + 0xA0)), ARM_PGBYTES - 0xA0); + + start_cpu_paddr = ml_static_vtop((vm_offset_t)&start_cpu); + + BootArgs_paddr = (unsigned int *)ml_static_vtop((vm_offset_t)BootArgs); + bcopy_phys((addr64_t)ml_static_vtop((vm_offset_t)&BootArgs_paddr), + (addr64_t)((unsigned int)(gPhysBase) + + ((unsigned int)&(ResetHandlerData.boot_args) - (unsigned int)&ExceptionLowVectorsBase)), + 4); + + CpuDataEntries_paddr = (unsigned int *)ml_static_vtop((vm_offset_t)CpuDataEntries); + bcopy_phys((addr64_t)ml_static_vtop((vm_offset_t)&CpuDataEntries_paddr), + (addr64_t)((unsigned int)(gPhysBase) + + ((unsigned int)&(ResetHandlerData.cpu_data_entries) - (unsigned int)&ExceptionLowVectorsBase)), + 4); + + CleanPoC_DcacheRegion((vm_offset_t) phystokv((char *) (gPhysBase)), PAGE_SIZE); + + resume_idle_cpu_paddr = (unsigned int)ml_static_vtop((vm_offset_t)&resume_idle_cpu); + + } + + if (cpu_data_ptr == &BootCpuData) { + bcopy(((const void *)running_signature), (void *)(IOS_STATE), IOS_STATE_SIZE); + }; + + cpu_data_ptr->cpu_reset_handler = resume_idle_cpu_paddr; + clean_dcache((vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t), FALSE); +} + +void +machine_track_platform_idle(boolean_t entry) +{ + if (entry) + (void)__c11_atomic_fetch_add(&cpu_idle_count, 1, __ATOMIC_RELAXED); + else + (void)__c11_atomic_fetch_sub(&cpu_idle_count, 1, __ATOMIC_RELAXED); +} + diff --git a/osfmk/arm/cpu_affinity.h b/osfmk/arm/cpu_affinity.h new file mode 100644 index 000000000..531428736 --- /dev/null +++ b/osfmk/arm/cpu_affinity.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifdef KERNEL_PRIVATE +#ifndef _ARM_CPU_AFFINITY_H_ +#define _ARM_CPU_AFFINITY_H_ + +static inline int +ml_get_max_affinity_sets(void) +{ + return 0; +} + +static inline processor_set_t +ml_affinity_to_pset(__unused int affinity_num) +{ + return PROCESSOR_SET_NULL; +} + +#endif /* _ARM_CPU_AFFINITY_H_ */ +#endif /* KERNEL_PRIVATE */ diff --git a/osfmk/arm/cpu_capabilities.h b/osfmk/arm/cpu_capabilities.h new file mode 100644 index 000000000..f1d16dde6 --- /dev/null +++ b/osfmk/arm/cpu_capabilities.h @@ -0,0 +1,212 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifdef PRIVATE + +#ifndef _ARM_CPU_CAPABILITIES_H +#define _ARM_CPU_CAPABILITIES_H + +#ifndef __ASSEMBLER__ +#include <stdint.h> +#ifdef KERNEL_PRIVATE +#include <mach/vm_types.h> +#endif +#endif + +/* + * This is the authoritative way to determine from user mode what + * implementation-specific processor features are available. + * This API only supported for Apple internal use. + * + */ + +/* + * Bit definitions for _cpu_capabilities: + */ +#define kCache32 0x00000010 // cache line size is 32 bytes +#define kCache64 0x00000020 // cache line size is 64 bytes +#define kCache128 0x00000040 // cache line size is 128 bytes +#define kFastThreadLocalStorage 0x00000080 // TLS ptr is kept in a user-mode-readable register +#define kHasNeon 0x00000100 // Advanced SIMD is supported +#define kHasNeonHPFP 0x00000200 // Advanced SIMD half-precision +#define kHasVfp 0x00000400 // VFP is supported +#define kHasEvent 0x00001000 // WFE/SVE and period event wakeup +#define kHasFMA 0x00002000 // Fused multiply add is supported +#define kUP 0x00008000 // set if (kNumCPUs == 1) +#define kNumCPUs 0x00FF0000 // number of CPUs (see _NumCPUs() below) +#define kHasARMv8Crypto 0x01000000 // Optional ARMv8 Crypto extensions +#define kHasARMv81Atomics 0x02000000 // ARMv8.1 Atomic instructions supported + +#define kNumCPUsShift 16 // see _NumCPUs() below + + /* + * Bit definitions for multiuser_config: + */ +#define kIsMultiUserDevice 0x80000000 // this device is in multiuser mode +#define kMultiUserCurrentUserMask 0x7fffffff // the current user UID of the multiuser device + +#ifndef __ASSEMBLER__ +#include <sys/commpage.h> + +extern int _get_cpu_capabilities( void ); + +__inline static +int _NumCPUs( void ) +{ + return (_get_cpu_capabilities() & kNumCPUs) >> kNumCPUsShift; +} + +typedef struct { + volatile uint64_t TimeBase; + volatile uint32_t TimeStamp_sec; + volatile uint32_t TimeStamp_usec; + volatile uint32_t TimeBaseTicks_per_sec; + volatile uint32_t TimeBaseTicks_per_usec; + volatile uint64_t TimeBase_magic; + volatile uint32_t TimeBase_add; + volatile uint32_t TimeBase_shift; +} commpage_timeofday_data_t; + +#endif /* __ASSEMBLER__ */ + + +/* + * The shared kernel/user "comm page(s)": + */ + +#if defined(__arm64__) + +#define _COMM_PAGE64_BASE_ADDRESS (0xfffffff0001fc000ULL) /* Just below the kernel, safely in TTBR1 */ +#define _COMM_PRIV_PAGE64_BASE_ADDRESS (_COMM_PAGE64_BASE_ADDRESS - (PAGE_SIZE)) /* Privileged RO in kernel mode */ + +#define _COMM_PAGE64_AREA_LENGTH (_COMM_PAGE32_AREA_LENGTH) +#define _COMM_PAGE64_AREA_USED (-1) + +// macro to change a user comm page address to one that is accessible from privileged mode +// we can no longer access user memory in privileged mode once PAN is enabled +#define _COMM_PAGE_PRIV(_addr_) ((_addr_) - (_COMM_PAGE_START_ADDRESS) + (_COMM_PRIV_PAGE64_BASE_ADDRESS)) + +#ifdef KERNEL_PRIVATE +extern vm_address_t sharedpage_rw_addr; +#define _COMM_PAGE_RW_OFFSET (0) +#define _COMM_PAGE_AREA_LENGTH (PAGE_SIZE) + +#define _COMM_PAGE_BASE_ADDRESS (sharedpage_rw_addr) +#define _COMM_PAGE_START_ADDRESS (sharedpage_rw_addr) +#else +#define _COMM_PAGE_AREA_LENGTH (4096) + +#define _COMM_PAGE_BASE_ADDRESS _COMM_PAGE64_BASE_ADDRESS +#define _COMM_PAGE_START_ADDRESS _COMM_PAGE64_BASE_ADDRESS +#endif + +#elif defined(__arm__) + +#define _COMM_PAGE64_BASE_ADDRESS (-1) +#define _COMM_PAGE64_AREA_LENGTH (-1) +#define _COMM_PAGE64_AREA_USED (-1) + +// macro to change a user comm page address to one that is accessible from privileged mode +// this macro is stubbed as PAN is not available on AARCH32, +// but this may still be required for compatibility +#define _COMM_PAGE_PRIV(_addr_) (_addr_) + +#ifdef KERNEL_PRIVATE +extern vm_address_t sharedpage_rw_addr; +#define _COMM_PAGE_RW_OFFSET (sharedpage_rw_addr-_COMM_PAGE_BASE_ADDRESS) +#define _COMM_PAGE_AREA_LENGTH (PAGE_SIZE) +#else +#define _COMM_PAGE_AREA_LENGTH (4096) +#endif + +#define _COMM_PAGE_BASE_ADDRESS _COMM_PAGE32_BASE_ADDRESS +#define _COMM_PAGE_START_ADDRESS _COMM_PAGE32_BASE_ADDRESS + +#else +#error Unknown architecture. +#endif + +#define _COMM_PAGE32_BASE_ADDRESS (0xFFFF4000) /* Must be outside of normal map bounds */ +#define _COMM_PAGE32_AREA_LENGTH (_COMM_PAGE_AREA_LENGTH) + +#define _COMM_PAGE_TEXT_START (-1) +#define _COMM_PAGE32_TEXT_START (-1) +#define _COMM_PAGE64_TEXT_START (-1) +#define _COMM_PAGE_PFZ_START_OFFSET (-1) +#define _COMM_PAGE_PFZ_END_OFFSET (-1) + +#define _COMM_PAGE32_OBJC_SIZE 0ULL +#define _COMM_PAGE32_OBJC_BASE 0ULL +#define _COMM_PAGE64_OBJC_SIZE 0ULL +#define _COMM_PAGE64_OBJC_BASE 0ULL + +/* + * data in the comm pages + * apply _COMM_PAGE_PRIV macro to use these in privileged mode + */ +#define _COMM_PAGE_SIGNATURE (_COMM_PAGE_START_ADDRESS+0x000) // first few bytes are a signature +#define _COMM_PAGE_VERSION (_COMM_PAGE_START_ADDRESS+0x01E) // 16-bit version# +#define _COMM_PAGE_THIS_VERSION 3 // version of the commarea format + +#define _COMM_PAGE_CPU_CAPABILITIES (_COMM_PAGE_START_ADDRESS+0x020) // uint32_t _cpu_capabilities +#define _COMM_PAGE_NCPUS (_COMM_PAGE_START_ADDRESS+0x022) // uint8_t number of configured CPUs +#define _COMM_PAGE_USER_PAGE_SHIFT_32 (_COMM_PAGE_START_ADDRESS+0x024) // VM page shift for 32-bit processes +#define _COMM_PAGE_USER_PAGE_SHIFT_64 (_COMM_PAGE_START_ADDRESS+0x025) // VM page shift for 64-bit processes +#define _COMM_PAGE_CACHE_LINESIZE (_COMM_PAGE_START_ADDRESS+0x026) // uint16_t cache line size +#define _COMM_PAGE_SCHED_GEN (_COMM_PAGE_START_ADDRESS+0x028) // uint32_t scheduler generation number (count of pre-emptions) +#define _COMM_PAGE_SPIN_COUNT (_COMM_PAGE_START_ADDRESS+0x02C) // uint32_t max spin count for mutex's +#define _COMM_PAGE_MEMORY_PRESSURE (_COMM_PAGE_START_ADDRESS+0x030) // uint32_t copy of vm_memory_pressure +#define _COMM_PAGE_ACTIVE_CPUS (_COMM_PAGE_START_ADDRESS+0x034) // uint8_t number of active CPUs (hw.activecpu) +#define _COMM_PAGE_PHYSICAL_CPUS (_COMM_PAGE_START_ADDRESS+0x035) // uint8_t number of physical CPUs (hw.physicalcpu_max) +#define _COMM_PAGE_LOGICAL_CPUS (_COMM_PAGE_START_ADDRESS+0x036) // uint8_t number of logical CPUs (hw.logicalcpu_max) +#define _COMM_PAGE_KERNEL_PAGE_SHIFT (_COMM_PAGE_START_ADDRESS+0x037) // uint8_t kernel vm page shift */ +#define _COMM_PAGE_MEMORY_SIZE (_COMM_PAGE_START_ADDRESS+0x038) // uint64_t max memory size */ +#define _COMM_PAGE_TIMEOFDAY_DATA (_COMM_PAGE_START_ADDRESS+0x040) // used by gettimeofday(). Currently, sizeof(commpage_timeofday_data_t) = 40. A new struct is used on gettimeofday but space is reserved on the commpage for compatibility +#define _COMM_PAGE_CPUFAMILY (_COMM_PAGE_START_ADDRESS+0x080) // used by memcpy() resolver +#define _COMM_PAGE_DEV_FIRM (_COMM_PAGE_START_ADDRESS+0x084) // uint32_t handle on PE_i_can_has_debugger +#define _COMM_PAGE_TIMEBASE_OFFSET (_COMM_PAGE_START_ADDRESS+0x088) // uint64_t timebase offset for constructing mach_absolute_time() +#define _COMM_PAGE_USER_TIMEBASE (_COMM_PAGE_START_ADDRESS+0x090) // uint8_t is userspace mach_absolute_time supported (can read the timebase) +#define _COMM_PAGE_CONT_HWCLOCK (_COMM_PAGE_START_ADDRESS+0x091) // uint8_t is always-on hardware clock present for mach_continuous_time() +#define _COMM_PAGE_UNUSED0 (_COMM_PAGE_START_ADDRESS+0x092) // 6 unused bytes +#define _COMM_PAGE_CONT_TIMEBASE (_COMM_PAGE_START_ADDRESS+0x098) // uint64_t base for mach_continuous_time() +#define _COMM_PAGE_BOOTTIME_USEC (_COMM_PAGE_START_ADDRESS+0x0A0) // uint64_t boottime in microseconds + +// aligning to 64byte for cacheline size +#define _COMM_PAGE_APPROX_TIME (_COMM_PAGE_START_ADDRESS+0x0C0) // uint64_t last known mach_absolute_time() +#define _COMM_PAGE_APPROX_TIME_SUPPORTED (_COMM_PAGE_START_ADDRESS+0x0C8) // uint8_t is mach_approximate_time supported +#define _COMM_PAGE_UNUSED1 (_COMM_PAGE_START_ADDRESS+0x0C9) // 55 unused bytes, align next mutable value to a separate cache line + +#define _COMM_PAGE_KDEBUG_ENABLE (_COMM_PAGE_START_ADDRESS+0x100) // uint32_t export kdebug status bits to userspace +#define _COMM_PAGE_ATM_DIAGNOSTIC_CONFIG (_COMM_PAGE_START_ADDRESS+0x104) // uint32_t export "atm_diagnostic_config" to userspace +#define _COMM_PAGE_MULTIUSER_CONFIG (_COMM_PAGE_START_ADDRESS+0x108) // uint32_t export "multiuser_config" to userspace + + +#define _COMM_PAGE_NEWTIMEOFDAY_DATA (_COMM_PAGE_START_ADDRESS+0x120) // used by gettimeofday(). Currently, sizeof(new_commpage_timeofday_data_t) = 40. +#define _COMM_PAGE_END (_COMM_PAGE_START_ADDRESS+0x1000) // end of common page + +#endif /* _ARM_CPU_CAPABILITIES_H */ +#endif /* PRIVATE */ diff --git a/osfmk/arm/cpu_common.c b/osfmk/arm/cpu_common.c new file mode 100644 index 000000000..ac41435d4 --- /dev/null +++ b/osfmk/arm/cpu_common.c @@ -0,0 +1,577 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * File: arm/cpu_common.c + * + * cpu routines common to all supported arm variants + */ + +#include <kern/kalloc.h> +#include <kern/machine.h> +#include <kern/cpu_number.h> +#include <kern/thread.h> +#include <kern/timer_queue.h> +#include <arm/cpu_data.h> +#include <arm/cpuid.h> +#include <arm/caches_internal.h> +#include <arm/cpu_data_internal.h> +#include <arm/cpu_internal.h> +#include <arm/misc_protos.h> +#include <arm/machine_cpu.h> +#include <arm/rtclock.h> +#include <mach/processor_info.h> +#include <machine/atomic.h> +#include <machine/config.h> +#include <vm/vm_kern.h> +#include <vm/vm_map.h> +#include <pexpert/arm/protos.h> +#include <pexpert/device_tree.h> +#include <sys/kdebug.h> +#include <arm/machine_routines.h> +#include <libkern/OSAtomic.h> +#include <chud/chud_xnu.h> +#include <chud/chud_xnu_private.h> + +#if KPERF +void kperf_signal_handler(unsigned int cpu_number); +#endif + +struct processor BootProcessor; + +unsigned int real_ncpus = 1; +boolean_t idle_enable = FALSE; +uint64_t wake_abstime=0x0ULL; + + +cpu_data_t * +cpu_datap(int cpu) +{ + assert(cpu < MAX_CPUS); + return (CpuDataEntries[cpu].cpu_data_vaddr); +} + +kern_return_t +cpu_control(int slot_num, + processor_info_t info, + unsigned int count) +{ + printf("cpu_control(%d,%p,%d) not implemented\n", + slot_num, info, count); + return (KERN_FAILURE); +} + +kern_return_t +cpu_info_count(processor_flavor_t flavor, + unsigned int *count) +{ + + switch (flavor) { + case PROCESSOR_CPU_STAT: + *count = PROCESSOR_CPU_STAT_COUNT; + return (KERN_SUCCESS); + + default: + *count = 0; + return (KERN_FAILURE); + } +} + +kern_return_t +cpu_info(processor_flavor_t flavor, + int slot_num, + processor_info_t info, + unsigned int *count) +{ + switch (flavor) { + case PROCESSOR_CPU_STAT: + { + processor_cpu_stat_t cpu_stat; + cpu_data_t *cpu_data_ptr = CpuDataEntries[slot_num].cpu_data_vaddr; + + if (*count < PROCESSOR_CPU_STAT_COUNT) + return (KERN_FAILURE); + + cpu_stat = (processor_cpu_stat_t) info; + cpu_stat->irq_ex_cnt = cpu_data_ptr->cpu_stat.irq_ex_cnt; + cpu_stat->ipi_cnt = cpu_data_ptr->cpu_stat.ipi_cnt; + cpu_stat->timer_cnt = cpu_data_ptr->cpu_stat.timer_cnt; + cpu_stat->undef_ex_cnt = cpu_data_ptr->cpu_stat.undef_ex_cnt; + cpu_stat->unaligned_cnt = cpu_data_ptr->cpu_stat.unaligned_cnt; + cpu_stat->vfp_cnt = cpu_data_ptr->cpu_stat.vfp_cnt; + cpu_stat->vfp_shortv_cnt = 0; + cpu_stat->data_ex_cnt = cpu_data_ptr->cpu_stat.data_ex_cnt; + cpu_stat->instr_ex_cnt = cpu_data_ptr->cpu_stat.instr_ex_cnt; + + *count = PROCESSOR_CPU_STAT_COUNT; + + return (KERN_SUCCESS); + } + + default: + return (KERN_FAILURE); + } +} + +/* + * Routine: cpu_doshutdown + * Function: + */ +void +cpu_doshutdown(void (*doshutdown) (processor_t), + processor_t processor) +{ + doshutdown(processor); +} + +/* + * Routine: cpu_idle_tickle + * + */ +void +cpu_idle_tickle(void) +{ + boolean_t intr; + cpu_data_t *cpu_data_ptr; + uint64_t new_idle_timeout_ticks = 0x0ULL; + + intr = ml_set_interrupts_enabled(FALSE); + cpu_data_ptr = getCpuDatap(); + + if (cpu_data_ptr->idle_timer_notify != (void *)NULL) { + ((idle_timer_t)cpu_data_ptr->idle_timer_notify)(cpu_data_ptr->idle_timer_refcon, &new_idle_timeout_ticks); + if (new_idle_timeout_ticks != 0x0ULL) { + /* if a new idle timeout was requested set the new idle timer deadline */ + clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline); + } else { + /* turn off the idle timer */ + cpu_data_ptr->idle_timer_deadline = 0x0ULL; + } + timer_resync_deadlines(); + } + (void) ml_set_interrupts_enabled(intr); +} + +static void +cpu_handle_xcall(cpu_data_t *cpu_data_ptr) +{ + broadcastFunc xfunc; + void *xparam; + + __c11_atomic_thread_fence(memory_order_acquire_smp); + /* Come back around if cpu_signal_internal is running on another CPU and has just + * added SIGPxcall to the pending mask, but hasn't yet assigned the call params.*/ + if (cpu_data_ptr->cpu_xcall_p0 != NULL && cpu_data_ptr->cpu_xcall_p1 != NULL) { + xfunc = cpu_data_ptr->cpu_xcall_p0; + xparam = cpu_data_ptr->cpu_xcall_p1; + cpu_data_ptr->cpu_xcall_p0 = NULL; + cpu_data_ptr->cpu_xcall_p1 = NULL; + __c11_atomic_thread_fence(memory_order_acq_rel_smp); + hw_atomic_and_noret(&cpu_data_ptr->cpu_signal, ~SIGPxcall); + xfunc(xparam); + } + +} + +unsigned int +cpu_broadcast_xcall(uint32_t *synch, + boolean_t self_xcall, + broadcastFunc func, + void *parm) +{ + boolean_t intr; + cpu_data_t *cpu_data_ptr; + cpu_data_t *target_cpu_datap; + unsigned int failsig; + int cpu; + int max_cpu; + + intr = ml_set_interrupts_enabled(FALSE); + cpu_data_ptr = getCpuDatap(); + + failsig = 0; + + if (synch != NULL) { + *synch = real_ncpus; + assert_wait((event_t)synch, THREAD_UNINT); + } + + max_cpu = ml_get_max_cpu_number(); + for (cpu=0; cpu <= max_cpu; cpu++) { + target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr; + + if ((target_cpu_datap == NULL) || (target_cpu_datap == cpu_data_ptr)) + continue; + + if(KERN_SUCCESS != cpu_signal(target_cpu_datap, SIGPxcall, (void *)func, parm)) { + failsig++; + } + } + + + if (self_xcall) { + func(parm); + } + + (void) ml_set_interrupts_enabled(intr); + + if (synch != NULL) { + if (hw_atomic_sub(synch, (!self_xcall)? failsig+1 : failsig) == 0) + clear_wait(current_thread(), THREAD_AWAKENED); + else + thread_block(THREAD_CONTINUE_NULL); + } + + if (!self_xcall) + return (real_ncpus - failsig - 1); + else + return (real_ncpus - failsig); +} + +kern_return_t +cpu_xcall(int cpu_number, broadcastFunc func, void *param) +{ + cpu_data_t *target_cpu_datap; + + if ((cpu_number < 0) || (cpu_number > ml_get_max_cpu_number())) + return KERN_INVALID_ARGUMENT; + + target_cpu_datap = (cpu_data_t*)CpuDataEntries[cpu_number].cpu_data_vaddr; + if (target_cpu_datap == NULL) + return KERN_INVALID_ARGUMENT; + + return cpu_signal(target_cpu_datap, SIGPxcall, (void*)func, param); +} + +static kern_return_t +cpu_signal_internal(cpu_data_t *target_proc, + unsigned int signal, + void *p0, + void *p1, + boolean_t defer) +{ + unsigned int Check_SIGPdisabled; + int current_signals; + Boolean swap_success; + boolean_t interruptible = ml_set_interrupts_enabled(FALSE); + cpu_data_t *current_proc = getCpuDatap(); + + /* We'll mandate that only IPIs meant to kick a core out of idle may ever be deferred. */ + if (defer) { + assert(signal == SIGPnop); + } + + if (current_proc != target_proc) + Check_SIGPdisabled = SIGPdisabled; + else + Check_SIGPdisabled = 0; + + if (signal == SIGPxcall) { + do { + current_signals = target_proc->cpu_signal; + if ((current_signals & SIGPdisabled) == SIGPdisabled) { +#if DEBUG || DEVELOPMENT + target_proc->failed_signal = SIGPxcall; + target_proc->failed_xcall = p0; + OSIncrementAtomicLong(&target_proc->failed_signal_count); +#endif + ml_set_interrupts_enabled(interruptible); + return KERN_FAILURE; + } + swap_success = OSCompareAndSwap(current_signals & (~SIGPxcall), current_signals | SIGPxcall, + &target_proc->cpu_signal); + + /* Drain pending xcalls on this cpu; the CPU we're trying to xcall may in turn + * be trying to xcall us. Since we have interrupts disabled that can deadlock, + * so break the deadlock by draining pending xcalls. */ + if (!swap_success && (current_proc->cpu_signal & SIGPxcall)) + cpu_handle_xcall(current_proc); + + } while (!swap_success); + + target_proc->cpu_xcall_p0 = p0; + target_proc->cpu_xcall_p1 = p1; + } else { + do { + current_signals = target_proc->cpu_signal; + if ((Check_SIGPdisabled !=0 ) && (current_signals & Check_SIGPdisabled) == SIGPdisabled) { +#if DEBUG || DEVELOPMENT + target_proc->failed_signal = signal; + OSIncrementAtomicLong(&target_proc->failed_signal_count); +#endif + ml_set_interrupts_enabled(interruptible); + return KERN_FAILURE; + } + + swap_success = OSCompareAndSwap(current_signals, current_signals | signal, + &target_proc->cpu_signal); + } while (!swap_success); + } + + /* + * Issue DSB here to guarantee: 1) prior stores to pending signal mask and xcall params + * will be visible to other cores when the IPI is dispatched, and 2) subsequent + * instructions to signal the other cores will not execute until after the barrier. + * DMB would be sufficient to guarantee 1) but not 2). + */ + __builtin_arm_dsb(DSB_ISH); + + if (!(target_proc->cpu_signal & SIGPdisabled)) { + if (defer) { + PE_cpu_signal_deferred(getCpuDatap()->cpu_id, target_proc->cpu_id); + } else { + PE_cpu_signal(getCpuDatap()->cpu_id, target_proc->cpu_id); + } + } + + ml_set_interrupts_enabled(interruptible); + return (KERN_SUCCESS); +} + +kern_return_t +cpu_signal(cpu_data_t *target_proc, + unsigned int signal, + void *p0, + void *p1) +{ + return cpu_signal_internal(target_proc, signal, p0, p1, FALSE); +} + +kern_return_t +cpu_signal_deferred(cpu_data_t *target_proc) +{ + return cpu_signal_internal(target_proc, SIGPnop, NULL, NULL, TRUE); +} + +void +cpu_signal_cancel(cpu_data_t *target_proc) +{ + /* TODO: Should we care about the state of a core as far as squashing deferred IPIs goes? */ + if (!(target_proc->cpu_signal & SIGPdisabled)) { + PE_cpu_signal_cancel(getCpuDatap()->cpu_id, target_proc->cpu_id); + } +} + +void +cpu_signal_handler(void) +{ + cpu_signal_handler_internal(FALSE); +} + +void +cpu_signal_handler_internal(boolean_t disable_signal) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + unsigned int cpu_signal; + + + cpu_data_ptr->cpu_stat.ipi_cnt++; + cpu_data_ptr->cpu_stat.ipi_cnt_wake++; + + SCHED_STATS_IPI(current_processor()); + + cpu_signal = hw_atomic_or(&cpu_data_ptr->cpu_signal, 0); + + if ((!(cpu_signal & SIGPdisabled)) && (disable_signal == TRUE)) + (void)hw_atomic_or(&cpu_data_ptr->cpu_signal, SIGPdisabled); + else if ((cpu_signal & SIGPdisabled) && (disable_signal == FALSE)) + (void)hw_atomic_and(&cpu_data_ptr->cpu_signal, ~SIGPdisabled); + + while (cpu_signal & ~SIGPdisabled) { + if (cpu_signal & SIGPdec) { + (void)hw_atomic_and(&cpu_data_ptr->cpu_signal, ~SIGPdec); + rtclock_intr(FALSE); + } + if (cpu_signal & SIGPchud) { + (void)hw_atomic_and(&cpu_data_ptr->cpu_signal, ~SIGPchud); + chudxnu_cpu_signal_handler(); + } +#if KPERF + if (cpu_signal & SIGPkptimer) { + (void)hw_atomic_and(&cpu_data_ptr->cpu_signal, ~SIGPkptimer); + kperf_signal_handler((unsigned int)cpu_data_ptr->cpu_number); + } +#endif + if (cpu_signal & SIGPxcall) { + cpu_handle_xcall(cpu_data_ptr); + } + if (cpu_signal & SIGPast) { + (void)hw_atomic_and(&cpu_data_ptr->cpu_signal, ~SIGPast); + ast_check(cpu_data_ptr->cpu_processor); + } + if (cpu_signal & SIGPdebug) { + (void)hw_atomic_and(&cpu_data_ptr->cpu_signal, ~SIGPdebug); + DebuggerXCall(cpu_data_ptr->cpu_int_state); + } +#if __ARM_SMP__ && defined(ARMA7) + if (cpu_signal & SIGPLWFlush) { + (void)hw_atomic_and(&cpu_data_ptr->cpu_signal, ~SIGPLWFlush); + cache_xcall_handler(LWFlush); + } + if (cpu_signal & SIGPLWClean) { + (void)hw_atomic_and(&cpu_data_ptr->cpu_signal, ~SIGPLWClean); + cache_xcall_handler(LWClean); + } +#endif + + cpu_signal = hw_atomic_or(&cpu_data_ptr->cpu_signal, 0); + } +} + +void +cpu_exit_wait(int cpu) +{ + if ( cpu != master_cpu) { + cpu_data_t *cpu_data_ptr; + + cpu_data_ptr = CpuDataEntries[cpu].cpu_data_vaddr; + while (!((*(volatile unsigned int*)&cpu_data_ptr->cpu_sleep_token) == ARM_CPU_ON_SLEEP_PATH)) {}; + } +} + +void +cpu_machine_init(void) +{ + static boolean_t started = FALSE; + cpu_data_t *cpu_data_ptr; + + cpu_data_ptr = getCpuDatap(); + started = ((cpu_data_ptr->cpu_flags & StartedState) == StartedState); + if (cpu_data_ptr->cpu_cache_dispatch != (cache_dispatch_t) NULL) + platform_cache_init(); + PE_cpu_machine_init(cpu_data_ptr->cpu_id, !started); + cpu_data_ptr->cpu_flags |= StartedState; + ml_init_interrupt(); +} + +processor_t +cpu_processor_alloc(boolean_t is_boot_cpu) +{ + processor_t proc; + + if (is_boot_cpu) + return &BootProcessor; + + proc = kalloc(sizeof(*proc)); + if (!proc) + return NULL; + + bzero((void *) proc, sizeof(*proc)); + return proc; +} + +void +cpu_processor_free(processor_t proc) +{ + if (proc != NULL && proc != &BootProcessor) + kfree((void *) proc, sizeof(*proc)); +} + +processor_t +current_processor(void) +{ + return getCpuDatap()->cpu_processor; +} + +processor_t +cpu_to_processor(int cpu) +{ + cpu_data_t *cpu_data = cpu_datap(cpu); + if (cpu_data != NULL) + return cpu_data->cpu_processor; + else + return NULL; +} + +cpu_data_t * +processor_to_cpu_datap(processor_t processor) +{ + cpu_data_t *target_cpu_datap; + + assert(processor->cpu_id < MAX_CPUS); + assert(CpuDataEntries[processor->cpu_id].cpu_data_vaddr != NULL); + + target_cpu_datap = (cpu_data_t*)CpuDataEntries[processor->cpu_id].cpu_data_vaddr; + assert(target_cpu_datap->cpu_processor == processor); + + return target_cpu_datap; +} + +ast_t * +ast_pending(void) +{ + return (&getCpuDatap()->cpu_pending_ast); +} + +cpu_type_t +slot_type(int slot_num) +{ + return (cpu_datap(slot_num)->cpu_type); +} + +cpu_subtype_t +slot_subtype(int slot_num) +{ + return (cpu_datap(slot_num)->cpu_subtype); +} + +cpu_threadtype_t +slot_threadtype(int slot_num) +{ + return (cpu_datap(slot_num)->cpu_threadtype); +} + +cpu_type_t +cpu_type(void) +{ + return (getCpuDatap()->cpu_type); +} + +cpu_subtype_t +cpu_subtype(void) +{ + return (getCpuDatap()->cpu_subtype); +} + +cpu_threadtype_t +cpu_threadtype(void) +{ + return (getCpuDatap()->cpu_threadtype); +} + +int +cpu_number(void) +{ + return (getCpuDatap()->cpu_number); +} + +uint64_t +ml_get_wake_timebase(void) +{ + return wake_abstime; +} + diff --git a/osfmk/arm/cpu_data.h b/osfmk/arm/cpu_data.h new file mode 100644 index 000000000..f35121e35 --- /dev/null +++ b/osfmk/arm/cpu_data.h @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + * + */ + +#ifndef ARM_CPU_DATA +#define ARM_CPU_DATA + +#ifdef MACH_KERNEL_PRIVATE + +#include <mach_assert.h> +#include <kern/assert.h> +#include <kern/kern_types.h> +#include <kern/processor.h> +#include <pexpert/pexpert.h> +#include <arm/thread.h> +#include <arm/proc_reg.h> + +#include <mach/mach_types.h> +#include <machine/thread.h> + + +#define current_thread() current_thread_fast() + +static inline thread_t current_thread_fast(void) +{ + thread_t result; +#if defined(__arm64__) + __asm__ volatile("mrs %0, TPIDR_EL1" : "=r" (result)); +#else + result = (thread_t)__builtin_arm_mrc(15, 0, 13, 0, 4); // TPIDRPRW +#endif + return result; +} + +#if defined(__arm64__) + +static inline vm_offset_t exception_stack_pointer(void) +{ + vm_offset_t result = 0; + __asm__ volatile( + "msr SPSel, #1 \n" + "mov %0, sp \n" + "msr SPSel, #0 \n" + : "=r" (result)); + + return result; +} + +#endif /* defined(__arm64__) */ + +#define getCpuDatap() current_thread()->machine.CpuDatap +#define current_cpu_datap() getCpuDatap() + +extern int get_preemption_level(void); +extern void _enable_preemption_no_check(void); + +#define enable_preemption_no_check() _enable_preemption_no_check() +#define mp_disable_preemption() _disable_preemption() +#define mp_enable_preemption() _enable_preemption() +#define mp_enable_preemption_no_check() _enable_preemption_no_check() + +#endif /* MACH_KERNEL_PRIVATE */ + +#endif /* ARM_CPU_DATA */ diff --git a/osfmk/arm/cpu_data_internal.h b/osfmk/arm/cpu_data_internal.h new file mode 100644 index 000000000..264e7ed96 --- /dev/null +++ b/osfmk/arm/cpu_data_internal.h @@ -0,0 +1,319 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + * + */ + +#ifndef ARM_CPU_DATA_INTERNAL +#define ARM_CPU_DATA_INTERNAL + +#include <mach_assert.h> +#include <kern/assert.h> +#include <kern/kern_types.h> +#include <kern/processor.h> +#include <pexpert/pexpert.h> +#include <arm/dbgwrap.h> +#include <arm/proc_reg.h> +#include <arm/thread.h> +#include <arm/pmap.h> + +#if MONOTONIC +#include <machine/monotonic.h> +#endif /* MONOTONIC */ + +#define NSEC_PER_HZ (NSEC_PER_SEC / 100) + +typedef struct reset_handler_data { + vm_offset_t assist_reset_handler; /* Assist handler phys address */ + vm_offset_t cpu_data_entries; /* CpuDataEntries phys address */ +#if !__arm64__ + vm_offset_t boot_args; /* BootArgs phys address */ +#endif +} reset_handler_data_t; + +extern reset_handler_data_t ResetHandlerData; + +#if __ARM_SMP__ +#ifdef CPU_COUNT +#define MAX_CPUS CPU_COUNT +#else +#define MAX_CPUS 2 +#endif +#else +#define MAX_CPUS 1 +#endif + +#define CPUWINDOWS_MAX 4 +#ifdef __arm__ +#define CPUWINDOWS_BASE 0xFFF00000UL +#else +#define CPUWINDOWS_BASE_MASK 0xFFFFFFFFFFF00000UL +#define CPUWINDOWS_BASE (VM_MAX_KERNEL_ADDRESS & CPUWINDOWS_BASE_MASK) +#endif + +typedef struct cpu_data_entry { + void *cpu_data_paddr; /* Cpu data physical address */ + struct cpu_data *cpu_data_vaddr; /* Cpu data virtual address */ +#if __arm__ + uint32_t cpu_data_offset_8; + uint32_t cpu_data_offset_12; +#elif __arm64__ +#else +#error Check cpu_data_entry padding for this architecture +#endif +} cpu_data_entry_t; + + +typedef struct rtclock_timer { + mpqueue_head_t queue; + uint64_t deadline; + uint32_t is_set:1, + has_expired:1, + :0; +} rtclock_timer_t; + +typedef struct { + uint32_t irq_ex_cnt; + uint32_t irq_ex_cnt_wake; + uint32_t ipi_cnt; + uint32_t ipi_cnt_wake; + uint32_t timer_cnt; + uint32_t timer_cnt_wake; + uint32_t undef_ex_cnt; + uint32_t unaligned_cnt; + uint32_t vfp_cnt; + uint32_t data_ex_cnt; + uint32_t instr_ex_cnt; +} cpu_stat_t; + +typedef struct cpu_data +{ + unsigned short cpu_number; + unsigned short cpu_flags; + vm_offset_t istackptr; + vm_offset_t intstack_top; + vm_offset_t fiqstackptr; + vm_offset_t fiqstack_top; +#if __arm64__ + vm_offset_t excepstackptr; + vm_offset_t excepstack_top; + boolean_t cluster_master; +#endif + boolean_t interrupts_enabled; + thread_t cpu_active_thread; + vm_offset_t cpu_active_stack; + unsigned int cpu_ident; + cpu_id_t cpu_id; + unsigned volatile int cpu_signal; +#if DEBUG || DEVELOPMENT + void *failed_xcall; + unsigned int failed_signal; + volatile long failed_signal_count; +#endif + void *cpu_cache_dispatch; + ast_t cpu_pending_ast; + struct processor *cpu_processor; + int cpu_type; + int cpu_subtype; + int cpu_threadtype; + int cpu_running; + +#ifdef __LP64__ + uint64_t cpu_base_timebase; + uint64_t cpu_timebase; +#else + union { + struct { + uint32_t low; + uint32_t high; + } split; + struct { + uint64_t val; + } raw; + } cbtb; +#define cpu_base_timebase_low cbtb.split.low +#define cpu_base_timebase_high cbtb.split.high + + union { + struct { + uint32_t low; + uint32_t high; + } split; + struct { + uint64_t val; + } raw; + } ctb; +#define cpu_timebase_low ctb.split.low +#define cpu_timebase_high ctb.split.high +#endif + + uint32_t cpu_decrementer; + void *cpu_get_decrementer_func; + void *cpu_set_decrementer_func; + void *cpu_get_fiq_handler; + + void *cpu_tbd_hardware_addr; + void *cpu_tbd_hardware_val; + + void *cpu_console_buf; + void *cpu_chud; + + void *cpu_idle_notify; + uint64_t cpu_idle_latency; + uint64_t cpu_idle_pop; + +#if __arm__ + vm_offset_t cpu_exc_vectors; +#endif + vm_offset_t cpu_reset_handler; + uint32_t cpu_reset_type; + uintptr_t cpu_reset_assist; + + void *cpu_int_state; + IOInterruptHandler interrupt_handler; + void *interrupt_nub; + unsigned int interrupt_source; + void *interrupt_target; + void *interrupt_refCon; + + void *idle_timer_notify; + void *idle_timer_refcon; + uint64_t idle_timer_deadline; + + uint64_t quantum_timer_deadline; + uint64_t rtcPop; + rtclock_timer_t rtclock_timer; + struct _rtclock_data_ *rtclock_datap; + + arm_debug_state_t *cpu_user_debug; /* Current debug state */ + vm_offset_t cpu_debug_interface_map; + + volatile int debugger_active; + + void *cpu_xcall_p0; + void *cpu_xcall_p1; + +#if __ARM_SMP__ && defined(ARMA7) + volatile uint32_t cpu_CLW_active; + volatile uint64_t cpu_CLWFlush_req; + volatile uint64_t cpu_CLWFlush_last; + volatile uint64_t cpu_CLWClean_req; + volatile uint64_t cpu_CLWClean_last; +#endif + + +#if __arm64__ + vm_offset_t coresight_base[CORESIGHT_REGIONS]; +#endif + + /* CCC ARMv8 registers */ + uint64_t cpu_regmap_paddr; + + uint32_t cpu_phys_id; + uint32_t cpu_l2_access_penalty; + void *platform_error_handler; + + int cpu_mcount_off; + + #define ARM_CPU_ON_SLEEP_PATH 0x50535553UL + volatile unsigned int cpu_sleep_token; + unsigned int cpu_sleep_token_last; + + cpu_stat_t cpu_stat; + + volatile int PAB_active; /* Tells the console if we are dumping backtraces */ + +#if KPC + /* double-buffered performance counter data */ + uint64_t *cpu_kpc_buf[2]; + /* PMC shadow and reload value buffers */ + uint64_t *cpu_kpc_shadow; + uint64_t *cpu_kpc_reload; +#endif +#if MONOTONIC + struct mt_cpu cpu_monotonic; +#endif /* MONOTONIC */ + struct prngContext *cpu_prng; + cluster_type_t cpu_cluster_type; + uint32_t cpu_cluster_id; + uint32_t cpu_l2_id; + uint32_t cpu_l2_size; + uint32_t cpu_l3_id; + uint32_t cpu_l3_size; + + struct pmap_cpu_data cpu_pmap_cpu_data; + dbgwrap_thread_state_t halt_state; + enum { + CPU_NOT_HALTED = 0, + CPU_HALTED, + CPU_HALTED_WITH_STATE + } halt_status; +} cpu_data_t; + +/* + * cpu_flags + */ +#define SleepState 0x0800 +#define StartedState 0x1000 + +extern cpu_data_entry_t CpuDataEntries[MAX_CPUS]; +extern cpu_data_t BootCpuData; +extern boot_args *BootArgs; + +#if __arm__ +extern unsigned int *ExceptionLowVectorsBase; +extern unsigned int *ExceptionVectorsTable; +#elif __arm64__ +extern unsigned int LowResetVectorBase; +extern unsigned int LowResetVectorEnd; +#if WITH_CLASSIC_S2R +extern uint8_t SleepToken[8]; +#endif +extern unsigned int LowExceptionVectorBase; +#else +#error Unknown arch +#endif + +extern cpu_data_t *cpu_datap(int cpu); +extern cpu_data_t *cpu_data_alloc(boolean_t is_boot); +extern void cpu_data_init(cpu_data_t *cpu_data_ptr); +extern void cpu_data_free(cpu_data_t *cpu_data_ptr); +extern kern_return_t cpu_data_register(cpu_data_t *cpu_data_ptr); +extern cpu_data_t *processor_to_cpu_datap( processor_t processor); + +#if __arm64__ +typedef struct sysreg_restore +{ + uint64_t tcr_el1; +} sysreg_restore_t; + +extern sysreg_restore_t sysreg_restore; +#endif /* __arm64__ */ + +#endif /* ARM_CPU_DATA_INTERNAL */ diff --git a/osfmk/arm/cpu_internal.h b/osfmk/arm/cpu_internal.h new file mode 100644 index 000000000..227863b4c --- /dev/null +++ b/osfmk/arm/cpu_internal.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +#ifndef _ARM_CPU_INTERNAL_H_ +#define _ARM_CPU_INTERNAL_H_ + + +#include <mach/kern_return.h> +#include <arm/cpu_data_internal.h> + +extern void cpu_bootstrap( + void); + +extern void cpu_init( + void); + +extern void cpu_timebase_init(boolean_t from_boot); + +extern kern_return_t cpu_signal( + cpu_data_t *target, + unsigned int signal, + void *p0, + void *p1); + +extern kern_return_t cpu_signal_deferred( + cpu_data_t *target); + +extern void cpu_signal_cancel( + cpu_data_t *target); + +#define SIGPnop 0x00000000U /* Send IPI with no service */ +#define SIGPdec 0x00000001U /* Request decremeter service */ +#define SIGPchud 0x00000002U /* CHUD CPU Signal request types */ +#define SIGPxcall 0x00000004U /* Call a function on a processor */ +#define SIGPast 0x00000008U /* Request AST check */ +#define SIGPdebug 0x00000010U /* Request Debug call */ +#define SIGPLWFlush 0x00000020UL /* Request LWFlush call */ +#define SIGPLWClean 0x00000040UL /* Request LWClean call */ +#define SIGPkptimer 0x00000100U /* Request kperf timer */ + +#define SIGPdisabled 0x80000000U /* Signal disabled */ + +extern void * chudxnu_cpu_alloc( + boolean_t boot_processor); + +extern void chudxnu_cpu_free( + void *per_proc_chud); + +extern unsigned int real_ncpus; + + +#endif /* _ARM_CPU_INTERNAL_H_ */ diff --git a/osfmk/arm/cpu_number.h b/osfmk/arm/cpu_number.h new file mode 100644 index 000000000..d781fa067 --- /dev/null +++ b/osfmk/arm/cpu_number.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ + +/* + */ + +/* + * Machine-dependent definitions for cpu identification. + * + */ +#ifdef KERNEL_PRIVATE + +#ifndef _ARM_CPU_NUMBER_H_ +#define _ARM_CPU_NUMBER_H_ + +#include <sys/cdefs.h> + +__BEGIN_DECLS + +extern int cpu_number(void); +extern int cpu_cluster_id(void); + +__END_DECLS + +#endif /* _ARM_CPU_NUMBER_H_ */ + +#endif /* KERNEL_PRIVATE */ diff --git a/osfmk/arm/cpuid.c b/osfmk/arm/cpuid.c new file mode 100644 index 000000000..2782475e2 --- /dev/null +++ b/osfmk/arm/cpuid.c @@ -0,0 +1,314 @@ +/* + * Copyright (c) 2007-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ + +#include <pexpert/pexpert.h> +#include <arm/cpuid.h> +#include <arm/cpuid_internal.h> +#include <vm/vm_page.h> +#include "proc_reg.h" + +#include <libkern/section_keywords.h> + +/* Temporary types to aid decoding, + * Everything in Little Endian */ + +typedef struct { + uint32_t + + Ctype1:3, /* 2:0 */ + Ctype2:3, /* 5:3 */ + Ctype3:3, /* 8:6 */ + Ctypes:15, /* 6:23 - Don't Care */ + LoC:3, /* 26-24 - Level of Coherency */ + LoU:3, /* 29:27 - Level of Unification */ + RAZ:2; /* 31:30 - Read-As-Zero */ +} arm_cache_clidr_t; + +typedef union { + arm_cache_clidr_t bits; + uint32_t value; +} arm_cache_clidr_info_t; + + +typedef struct { + uint32_t + + LineSize:3, /* 2:0 - Number of words in cache line */ + Assoc:10, /* 12:3 - Associativity of cache */ + NumSets:15, /* 27:13 - Number of sets in cache */ + c_type:4; /* 31:28 - Cache type */ +} arm_cache_ccsidr_t; + + +typedef union { + arm_cache_ccsidr_t bits; + uint32_t value; +} arm_cache_ccsidr_info_t; + +/* Statics */ + +static SECURITY_READ_ONLY_LATE(arm_cpu_info_t) cpuid_cpu_info; +static SECURITY_READ_ONLY_LATE(cache_info_t) cpuid_cache_info; + +/* Code */ + +__private_extern__ +void +do_cpuid(void) +{ + cpuid_cpu_info.value = machine_read_midr(); +#if (__ARM_ARCH__ == 8) + + cpuid_cpu_info.arm_info.arm_arch = CPU_ARCH_ARMv8; + +#elif (__ARM_ARCH__ == 7) + #ifdef __ARM_SUB_ARCH__ + cpuid_cpu_info.arm_info.arm_arch = __ARM_SUB_ARCH__; + #else + cpuid_cpu_info.arm_info.arm_arch = CPU_ARCH_ARMv7; + #endif +#else + /* 1176 architecture lives in the extended feature register */ + if (cpuid_cpu_info.arm_info.arm_arch == CPU_ARCH_EXTENDED) { + arm_isa_feat1_reg isa = machine_read_isa_feat1(); + + /* + * if isa feature register 1 [15:12] == 0x2, this chip + * supports sign extention instructions, which indicate ARMv6 + */ + if (isa.field.sign_zero_ext_support == 0x2) { + cpuid_cpu_info.arm_info.arm_arch = CPU_ARCH_ARMv6; + } + } +#endif +} + +arm_cpu_info_t * +cpuid_info(void) +{ + return &cpuid_cpu_info; +} + +int +cpuid_get_cpufamily(void) +{ + int cpufamily = 0; + + switch (cpuid_info()->arm_info.arm_implementor) { + case CPU_VID_ARM: + switch (cpuid_info()->arm_info.arm_part) { + case CPU_PART_CORTEXA9: + cpufamily = CPUFAMILY_ARM_14; + break; + case CPU_PART_CORTEXA8: + cpufamily = CPUFAMILY_ARM_13; + break; + case CPU_PART_CORTEXA7: + cpufamily = CPUFAMILY_ARM_15; + break; + case CPU_PART_1136JFS: + case CPU_PART_1176JZFS: + cpufamily = CPUFAMILY_ARM_11; + break; + case CPU_PART_926EJS: + case CPU_PART_920T: + cpufamily = CPUFAMILY_ARM_9; + break; + default: + cpufamily = CPUFAMILY_UNKNOWN; + break; + } + break; + + case CPU_VID_INTEL: + cpufamily = CPUFAMILY_ARM_XSCALE; + break; + + case CPU_VID_APPLE: + switch (cpuid_info()->arm_info.arm_part) { + case CPU_PART_SWIFT: + cpufamily = CPUFAMILY_ARM_SWIFT; + break; + case CPU_PART_CYCLONE: + cpufamily = CPUFAMILY_ARM_CYCLONE; + break; + case CPU_PART_TYPHOON: + case CPU_PART_TYPHOON_CAPRI: + cpufamily = CPUFAMILY_ARM_TYPHOON; + break; + case CPU_PART_TWISTER: + case CPU_PART_TWISTER_ELBA_MALTA: + cpufamily = CPUFAMILY_ARM_TWISTER; + break; + case CPU_PART_HURRICANE: + case CPU_PART_HURRICANE_MYST: + cpufamily = CPUFAMILY_ARM_HURRICANE; + break; + default: + cpufamily = CPUFAMILY_UNKNOWN; + break; + } + break; + + default: + cpufamily = CPUFAMILY_UNKNOWN; + break; + } + + return cpufamily; +} + +void +do_debugid(void) +{ + machine_do_debugid(); +} + +arm_debug_info_t * +arm_debug_info(void) +{ + return machine_arm_debug_info(); +} + +void +do_mvfpid(void) +{ + return machine_do_mvfpid(); +} + +arm_mvfp_info_t +*arm_mvfp_info(void) +{ + return machine_arm_mvfp_info(); +} + +void +do_cacheid(void) +{ + arm_cache_clidr_info_t arm_cache_clidr_info; + arm_cache_ccsidr_info_t arm_cache_ccsidr_info; + + arm_cache_clidr_info.value = machine_read_clidr(); + + + /* Select L1 data/unified cache */ + + machine_write_csselr(CSSELR_L1, CSSELR_DATA_UNIFIED); + arm_cache_ccsidr_info.value = machine_read_ccsidr(); + + cpuid_cache_info.c_unified = (arm_cache_clidr_info.bits.Ctype1 == 0x4) ? 1 : 0; + + switch (arm_cache_ccsidr_info.bits.c_type) { + case 0x1: + cpuid_cache_info.c_type = CACHE_WRITE_ALLOCATION; + break; + case 0x2: + cpuid_cache_info.c_type = CACHE_READ_ALLOCATION; + break; + case 0x4: + cpuid_cache_info.c_type = CACHE_WRITE_BACK; + break; + case 0x8: + cpuid_cache_info.c_type = CACHE_WRITE_THROUGH; + break; + default: + cpuid_cache_info.c_type = CACHE_UNKNOWN; + } + + cpuid_cache_info.c_linesz = 4 * (1<<(arm_cache_ccsidr_info.bits.LineSize + 2)); + cpuid_cache_info.c_assoc = (arm_cache_ccsidr_info.bits.Assoc + 1); + + /* I cache size */ + cpuid_cache_info.c_isize = (arm_cache_ccsidr_info.bits.NumSets + 1) * cpuid_cache_info.c_linesz * cpuid_cache_info.c_assoc; + + /* D cache size */ + cpuid_cache_info.c_dsize = (arm_cache_ccsidr_info.bits.NumSets + 1) * cpuid_cache_info.c_linesz * cpuid_cache_info.c_assoc; + + + if ((arm_cache_clidr_info.bits.Ctype3 == 0x4) || + (arm_cache_clidr_info.bits.Ctype2 == 0x4) || (arm_cache_clidr_info.bits.Ctype2 == 0x2)) { + + if (arm_cache_clidr_info.bits.Ctype3 == 0x4) { + /* Select L3 (LLC) if the SoC is new enough to have that. + * This will be the second-level cache for the highest-performing ACC. */ + machine_write_csselr(CSSELR_L3, CSSELR_DATA_UNIFIED); + } else { + /* Select L2 data cache */ + machine_write_csselr(CSSELR_L2, CSSELR_DATA_UNIFIED); + } + arm_cache_ccsidr_info.value = machine_read_ccsidr(); + + cpuid_cache_info.c_linesz = 4 * (1<<(arm_cache_ccsidr_info.bits.LineSize + 2)); + cpuid_cache_info.c_assoc = (arm_cache_ccsidr_info.bits.Assoc + 1); + cpuid_cache_info.c_l2size = (arm_cache_ccsidr_info.bits.NumSets + 1) * cpuid_cache_info.c_linesz * cpuid_cache_info.c_assoc; + cpuid_cache_info.c_inner_cache_size = cpuid_cache_info.c_dsize; + cpuid_cache_info.c_bulksize_op = cpuid_cache_info.c_l2size; + + /* capri has a 2MB L2 cache unlike every other SoC up to this + * point with a 1MB L2 cache, so to get the same performance + * gain from coloring, we have to double the number of colors. + * Note that in general (and in fact as it's implemented in + * i386/cpuid.c), the number of colors is calculated as the + * cache line size * the number of sets divided by the page + * size. Also note that for H8 devices and up, the page size + * will be 16k instead of 4, which will reduce the number of + * colors required. Thus, this is really a temporary solution + * for capri specifically that we may want to generalize later: + * + * TODO: Are there any special considerations for our unusual + * cache geometries (3MB)? + */ + vm_cache_geometry_colors = ((arm_cache_ccsidr_info.bits.NumSets + 1) * cpuid_cache_info.c_linesz) / PAGE_SIZE; + kprintf(" vm_cache_geometry_colors: %d\n", vm_cache_geometry_colors); + } else { + cpuid_cache_info.c_l2size = 0; + + cpuid_cache_info.c_inner_cache_size = cpuid_cache_info.c_dsize; + cpuid_cache_info.c_bulksize_op = cpuid_cache_info.c_dsize; + } + + kprintf("%s() - %u bytes %s cache (I:%u D:%u (%s)), %u-way assoc, %u bytes/line\n", + __FUNCTION__, + cpuid_cache_info.c_dsize + cpuid_cache_info.c_isize, + ((cpuid_cache_info.c_type == CACHE_WRITE_BACK) ? "WB" : + (cpuid_cache_info.c_type == CACHE_WRITE_THROUGH ? "WT" : "Unknown")), + cpuid_cache_info.c_isize, + cpuid_cache_info.c_dsize, + (cpuid_cache_info.c_unified) ? "unified" : "separate", + cpuid_cache_info.c_assoc, + cpuid_cache_info.c_linesz); +} + +cache_info_t * +cache_info(void) +{ + return &cpuid_cache_info; +} diff --git a/osfmk/arm/cpuid.h b/osfmk/arm/cpuid.h new file mode 100644 index 000000000..07778404b --- /dev/null +++ b/osfmk/arm/cpuid.h @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2007-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ + +/* + * ARM CPU identification + */ + +#ifndef _MACHINE_CPUID_H_ +#define _MACHINE_CPUID_H_ + +#include <stdint.h> +#include <mach/boolean.h> +#include <machine/machine_cpuid.h> + +typedef struct { +uint32_t arm_rev : 4, /* 00:03 revision number */ + arm_part : 12, /* 04:15 primary part number */ + arm_arch : 4, /* 16:19 architecture */ + arm_variant : 4, /* 20:23 variant */ + arm_implementor : 8; /* 24:31 implementor (0x41) */ +} arm_cpuid_bits_t; + +typedef union { + arm_cpuid_bits_t arm_info; /* ARM9xx, ARM11xx, and later processors */ + uint32_t value; +} arm_cpu_info_t; + +/* Implementor codes */ +#define CPU_VID_ARM 0x41 // ARM Limited +#define CPU_VID_DEC 0x44 // Digital Equipment Corporation +#define CPU_VID_MOTOROLA 0x4D // Motorola - Freescale Semiconductor Inc. +#define CPU_VID_MARVELL 0x56 // Marvell Semiconductor Inc. +#define CPU_VID_INTEL 0x69 // Intel ARM parts. +#define CPU_VID_APPLE 0x61 // Apple Inc. + + +/* ARM Architecture Codes */ + +#define CPU_ARCH_ARMv4 0x1 /* ARMv4 */ +#define CPU_ARCH_ARMv4T 0x2 /* ARMv4 + Thumb */ +#define CPU_ARCH_ARMv5 0x3 /* ARMv5 */ +#define CPU_ARCH_ARMv5T 0x4 /* ARMv5 + Thumb */ +#define CPU_ARCH_ARMv5TE 0x5 /* ARMv5 + Thumb + Extensions(?) */ +#define CPU_ARCH_ARMv5TEJ 0x6 /* ARMv5 + Thumb + Extensions(?) + //Jazelle(?) XXX */ +#define CPU_ARCH_ARMv6 0x7 /* ARMv6 */ +#define CPU_ARCH_ARMv7 0x8 /* ARMv7 */ +#define CPU_ARCH_ARMv7f 0x9 /* ARMv7 for Cortex A9 */ +#define CPU_ARCH_ARMv7s 0xa /* ARMv7 for Swift */ +#define CPU_ARCH_ARMv7k 0xb /* ARMv7 for Cortex A7 */ + +#define CPU_ARCH_ARMv8 0xc /* Subtype for CPU_TYPE_ARM64 */ + + +/* special code indicating we need to look somewhere else for the architecture version */ +#define CPU_ARCH_EXTENDED 0xF + +/* ARM Part Numbers */ +/* + * XXX: ARM Todo + * Fill out these part numbers more completely + */ + +/* ARM9 (ARMv4T architecture) */ +#define CPU_PART_920T 0x920 +#define CPU_PART_926EJS 0x926 /* ARM926EJ-S */ + +/* ARM11 (ARMv6 architecture) */ +#define CPU_PART_1136JFS 0xB36 /* ARM1136JF-S or ARM1136J-S */ +#define CPU_PART_1176JZFS 0xB76 /* ARM1176JZF-S */ + +/* G1 (ARMv7 architecture) */ +#define CPU_PART_CORTEXA5 0xC05 + +/* M7 (ARMv7 architecture) */ +#define CPU_PART_CORTEXA7 0xC07 + +/* H2 H3 (ARMv7 architecture) */ +#define CPU_PART_CORTEXA8 0xC08 + +/* H4 (ARMv7 architecture) */ +#define CPU_PART_CORTEXA9 0xC09 + +/* H5 (SWIFT architecture) */ +#define CPU_PART_SWIFT 0x0 + +/* H6 (ARMv8 architecture) */ +#define CPU_PART_CYCLONE 0x1 + +/* H7 (ARMv8 architecture) */ +#define CPU_PART_TYPHOON 0x2 + +/* H7G (ARMv8 architecture) */ +#define CPU_PART_TYPHOON_CAPRI 0x3 + +/* H8 (ARMv8 architecture) */ +#define CPU_PART_TWISTER 0x4 + +/* H8G H8M (ARMv8 architecture) */ +#define CPU_PART_TWISTER_ELBA_MALTA 0x5 + +/* H9 (ARMv8 architecture) */ +#define CPU_PART_HURRICANE 0x6 + +/* H9G (ARMv8 architecture) */ +#define CPU_PART_HURRICANE_MYST 0x7 + + +/* Cache type identification */ + +/* Supported Cache Types */ +typedef enum { + CACHE_WRITE_THROUGH, + CACHE_WRITE_BACK, + CACHE_READ_ALLOCATION, + CACHE_WRITE_ALLOCATION, + CACHE_UNKNOWN +} cache_type_t; + +typedef struct { + boolean_t c_unified; /* unified I & D cache? */ + uint32_t c_isize; /* in Bytes (ARM caches can be 0.5 KB) */ + boolean_t c_i_ppage; /* protected page restriction for I cache + * (see B6-11 in ARM DDI 0100I document). */ + uint32_t c_dsize; /* in Bytes (ARM caches can be 0.5 KB) */ + boolean_t c_d_ppage; /* protected page restriction for I cache + * (see B6-11 in ARM DDI 0100I document). */ + cache_type_t c_type; /* WB or WT */ + uint32_t c_linesz; /* number of bytes */ + uint32_t c_assoc; /* n-way associativity */ + uint32_t c_l2size; /* L2 size, if present */ + uint32_t c_bulksize_op; /* bulk operation size limit. 0 if disabled */ + uint32_t c_inner_cache_size; /* inner dache size */ +} cache_info_t; + +typedef struct { + uint32_t + + RB:4, /* 3:0 - 32x64-bit media register bank supported: 0x2 */ + SP:4, /* 7:4 - Single precision supported in VFPv3: 0x2 */ + DP:4, /* 8:11 - Double precision supported in VFPv3: 0x2 */ + TE:4, /* 12-15 - Only untrapped exception handling can be selected: 0x0 */ + D:4, /* 19:16 - VFP hardware divide supported: 0x1 */ + SR:4, /* 23:20 - VFP hardware square root supported: 0x1 */ + SV:4, /* 27:24 - VFP short vector supported: 0x1 */ + RM:4; /* 31:28 - All VFP rounding modes supported: 0x1 */ +} arm_mvfr0_t; + +typedef union { + arm_mvfr0_t bits; + uint32_t value; +} arm_mvfr0_info_t; + +typedef struct { + uint32_t + + FZ:4, /* 3:0 - Full denormal arithmetic supported for VFP: 0x1 */ + DN:4, /* 7:4 - Propagation of NaN values supported for VFP: 0x1 */ + LS:4, /* 11:8 - Load/store instructions supported for NEON: 0x1 */ + I:4, /* 15:12 - Integer instructions supported for NEON: 0x1 */ + SP:4, /* 19:16 - Single precision floating-point instructions supported for NEON: 0x1 */ + HPFP:4, /* 23:20 - Half precision floating-point instructions supported */ + RSVP:8; /* 31:24 - Reserved */ +} arm_mvfr1_t; + +typedef union { + arm_mvfr1_t bits; + uint32_t value; +} arm_mvfr1_info_t; + +typedef struct { + uint32_t neon; + uint32_t neon_hpfp; +} arm_mvfp_info_t; + +#ifdef __cplusplus +extern "C" { +#endif + +extern void do_cpuid(void); +extern arm_cpu_info_t *cpuid_info(void); +extern int cpuid_get_cpufamily(void); + +extern void do_debugid(void); +extern arm_debug_info_t *arm_debug_info(void); + +extern void do_cacheid(void); +extern cache_info_t *cache_info(void); + +extern void do_mvfpid(void); +extern arm_mvfp_info_t *arm_mvfp_info(void); + +#ifdef __cplusplus +} +#endif + +#endif // _MACHINE_CPUID_H_ diff --git a/osfmk/arm/cpuid_internal.h b/osfmk/arm/cpuid_internal.h new file mode 100644 index 000000000..9778d117b --- /dev/null +++ b/osfmk/arm/cpuid_internal.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2011 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef _ARM_CPUID_INTERNAL_H_ +#define _ARM_CPUID_INTERNAL_H_ + +void machine_do_debugid(void); +arm_debug_info_t *machine_arm_debug_info(void); + +void machine_do_mvfpid(void); +arm_mvfp_info_t *machine_arm_mvfp_info(void); + +uint32_t machine_read_midr(void); +uint32_t machine_read_clidr(void); +uint32_t machine_read_ccsidr(void); + +typedef enum { + CSSELR_L1 = 0x0, + CSSELR_L2 = 0x2, + CSSELR_L3 = 0x4 +} csselr_cache_level; + +typedef enum { + CSSELR_DATA_UNIFIED = 0x0, + CSSELR_INSTR = 0x1 +} csselr_cache_type; + +void machine_write_csselr(csselr_cache_level level, csselr_cache_type type); + +#endif /* _ARM_CPUID_INTERNAL_H_ */ diff --git a/osfmk/arm/cswitch.s b/osfmk/arm/cswitch.s new file mode 100644 index 000000000..7c3812dd0 --- /dev/null +++ b/osfmk/arm/cswitch.s @@ -0,0 +1,290 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include <machine/asm.h> +#include <arm/proc_reg.h> +#include "assym.s" + +/* + * save_vfp_registers + * + * Expects a pointer to the VFP save area in r3; saves the callee-saved registers to that save area. + * Clobbers r2 and r3. + */ +.macro save_vfp_registers +#if __ARM_VFP__ + fmrx r2, fpscr // Get the current FPSCR... + str r2, [r3, VSS_FPSCR] // ...and save it to the save area + add r3, r3, #64 // Only s16-s31 are callee-saved +#if (__ARM_VFP__ >= 3) + vstmia.64 r3!, {d8-d11} + vstmia.64 r3!, {d12-d15} +#else + fstmias r3!, {s16-s31} +#endif /* __ARM_VFP__ >= 3 */ +#endif /* __ARM_VFP__ */ +.endmacro + +/* + * load_vfp_registers + * + * Expects a pointer to the VFP save area in r3; loads the callee-saved registers from that save area. + * Clobbers r2 and r3. + */ +.macro load_vfp_registers +#if __ARM_VFP__ + add r2, r3, #64 // Only s16-s31 are callee-saved +#if (__ARM_VFP__ >= 3) + vldmia.64 r2!, {d8-d11} + vldmia.64 r2!, {d12-d15} +#else + fldmias r2!, {s16-s31} +#endif /* __ARM_VFP__ >= 3 */ + ldr r3, [r3, VSS_FPSCR] // Get our saved FPSCR value... + fmxr fpscr, r3 // ...and restore it +#endif /* __ARM_VFP__ */ +.endmacro + +/* + * void machine_load_context(thread_t thread) + * + * Load the context for the first thread to run on a + * cpu, and go. + */ + .syntax unified + .text + .align 2 + .globl EXT(machine_load_context) + +LEXT(machine_load_context) + mcr p15, 0, r0, c13, c0, 4 // Write TPIDRPRW + ldr r1, [r0, TH_CTH_SELF] + mrc p15, 0, r2, c13, c0, 3 // Read TPIDRURO + and r2, r2, #3 // Extract cpu number + orr r1, r1, r2 // + mcr p15, 0, r1, c13, c0, 3 // Write TPIDRURO + ldr r1, [r0, TH_CTH_DATA] + mcr p15, 0, r1, c13, c0, 2 // Write TPIDRURW + mov r7, #0 // Clear frame pointer + ldr r3, [r0, TH_KSTACKPTR] // Get kernel stack top + mov r0, #0 // no param + add r3, r3, SS_R4 + ldmia r3!, {r4-r14} // Load thread status + bx lr // Return + +/* + * void Call_continuation( void (*continuation)(void), + * void *param, + * wait_result_t wresult, + * vm_offset_t stack_ptr) + */ + .text + .align 5 + .globl EXT(Call_continuation) + +LEXT(Call_continuation) + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + ldr sp, [r9, TH_KSTACKPTR] // Set stack pointer + mov r7, #0 // Clear frame pointer + mov r6,r0 // Load continuation + mov r0,r1 // Set first parameter + mov r1,r2 // Set wait result arg + blx r6 // Branch to continuation + mrc p15, 0, r0, c13, c0, 4 // Read TPIDRPRW + LOAD_ADDR_PC(thread_terminate) + b . // Not reach + + +/* + * thread_t Switch_context(thread_t old, + * void (*cont)(void), + * thread_t new) + */ + .text + .align 5 + .globl EXT(Switch_context) + +LEXT(Switch_context) + teq r1, #0 // Test if blocking on continuaton + bne switch_threads // No need to save GPR/NEON state if we are +#if __ARM_VFP__ + mov r1, r2 // r2 will be clobbered by the save, so preserve it + add r3, r0, ACT_KVFP // Get the kernel VFP save area for the old thread... + save_vfp_registers // ...and save our VFP state to it + mov r2, r1 // Restore r2 (the new thread pointer) +#endif /* __ARM_VFP__ */ + ldr r3, [r0, TH_KSTACKPTR] // Get old kernel stack top + add r3, r3, SS_R4 + stmia r3!, {r4-r14} // Save general registers to pcb +switch_threads: + ldr r3, [r2, TH_KSTACKPTR] // get kernel stack top + mcr p15, 0, r2, c13, c0, 4 // Write TPIDRPRW + ldr r6, [r2, TH_CTH_SELF] + mrc p15, 0, r5, c13, c0, 3 // Read TPIDRURO + and r5, r5, #3 // Extract cpu number + orr r6, r6, r5 + mcr p15, 0, r6, c13, c0, 3 // Write TPIDRURO + ldr r6, [r2, TH_CTH_DATA] + mcr p15, 0, r6, c13, c0, 2 // Write TPIDRURW +load_reg: + add r3, r3, SS_R4 + ldmia r3!, {r4-r14} // Restore new thread status +#if __ARM_VFP__ + add r3, r2, ACT_KVFP // Get the kernel VFP save area for the new thread... + load_vfp_registers // ...and load the saved state +#endif /* __ARM_VFP__ */ + bx lr // Return + +/* + * thread_t Shutdown_context(void (*doshutdown)(processor_t), processor_t processor) + * + */ + .text + .align 5 + .globl EXT(Shutdown_context) + +LEXT(Shutdown_context) + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW +#if __ARM_VFP__ + add r3, r9, ACT_KVFP // Get the kernel VFP save area for the current thread... + save_vfp_registers // ...and save our VFP state to it +#endif + ldr r3, [r9, TH_KSTACKPTR] // Get kernel stack top + add r3, r3, SS_R4 + stmia r3!, {r4-r14} // Save general registers to pcb + cpsid if // Disable FIQ IRQ + + ldr r12, [r9, ACT_CPUDATAP] // Get current cpu + ldr sp, [r12, CPU_ISTACKPTR] // Switch to interrupt stack + LOAD_ADDR_PC(cpu_doshutdown) + +/* + * thread_t Idle_context(void) + * + */ + .text + .align 5 + .globl EXT(Idle_context) + +LEXT(Idle_context) + + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW +#if __ARM_VFP__ + add r3, r9, ACT_KVFP // Get the kernel VFP save area for the current thread... + save_vfp_registers // ...and save our VFP state to it +#endif + ldr r3, [r9, TH_KSTACKPTR] // Get kernel stack top + add r3, r3, SS_R4 + stmia r3!, {r4-r14} // Save general registers to pcb + + ldr r12, [r9, ACT_CPUDATAP] // Get current cpu + ldr sp, [r12, CPU_ISTACKPTR] // Switch to interrupt stack + LOAD_ADDR_PC(cpu_idle) + +/* + * thread_t Idle_context(void) + * + */ + .text + .align 5 + .globl EXT(Idle_load_context) + +LEXT(Idle_load_context) + + mrc p15, 0, r12, c13, c0, 4 // Read TPIDRPRW + ldr r3, [r12, TH_KSTACKPTR] // Get kernel stack top + add r3, r3, SS_R4 + ldmia r3!, {r4-r14} // Restore new thread status +#if __ARM_VFP__ + add r3, r9, ACT_KVFP // Get the kernel VFP save area for the current thread... + load_vfp_registers // ...and load the saved state +#endif + bx lr // Return + +/* + * void vfp_save(struct arm_vfpsaved_state *vfp_ss) + */ + .text + .align 2 + .globl EXT(vfp_save) + +LEXT(vfp_save) +#if __ARM_VFP__ + fmrx r1, fpscr // Get the current FPSCR... + str r1, [r0, VSS_FPSCR] // ...and save it to the save area +#if (__ARM_VFP__ >= 3) + vstmia.64 r0!, {d0-d3} // Save vfp registers + vstmia.64 r0!, {d4-d7} + vstmia.64 r0!, {d8-d11} + vstmia.64 r0!, {d12-d15} + vstmia.64 r0!, {d16-d19} + vstmia.64 r0!, {d20-d23} + vstmia.64 r0!, {d24-d27} + vstmia.64 r0!, {d28-d31} +#else + fstmias r0!, {s0-s31} // Save vfp registers +#endif +#endif /* __ARM_VFP__ */ + bx lr // Return + +/* + * void vfp_load(struct arm_vfpsaved_state *vfp_ss) + * + * Loads the state in vfp_ss into the VFP registers. + */ + .text + .align 2 + .globl EXT(vfp_load) +LEXT(vfp_load) +#if __ARM_VFP__ + /* r0: vfp_ss, r1: unused, r2: unused, r3: unused */ + mov r1, r0 +#if (__ARM_VFP__ >= 3) + vldmia.64 r0!, {d0-d3} // Restore vfp registers + vldmia.64 r0!, {d4-d7} + vldmia.64 r0!, {d8-d11} + vldmia.64 r0!, {d12-d15} + vldmia.64 r0!, {d16-d19} + vldmia.64 r0!, {d20-d23} + vldmia.64 r0!, {d24-d27} + vldmia.64 r0!, {d28-d31} +#else + fldmias r0!, {s0-s31} // Restore vfp registers +#endif /* __ARM_VFP__ >= 3 */ + ldr r1, [r1, VSS_FPSCR] // Get fpscr from the save state... + fmxr fpscr, r1 // ...and load it into the register +#endif /* __ARM_VFP__ */ + bx lr // Return + +#include "globals_asm.h" + +LOAD_ADDR_GEN_DEF(thread_terminate) +LOAD_ADDR_GEN_DEF(cpu_doshutdown) +LOAD_ADDR_GEN_DEF(cpu_idle) + +/* vim: set ts=4: */ + diff --git a/osfmk/arm/data.s b/osfmk/arm/data.s new file mode 100644 index 000000000..b82b50339 --- /dev/null +++ b/osfmk/arm/data.s @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2007-2009 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include <arm/asm.h> +#include <arm/proc_reg.h> +#include "assym.s" + +#if defined(__arm__) +#include "globals_asm.h" +#elif defined(__arm64__) +/* We're fine, use adrp, add */ +#else +#error Unknown architecture. +#endif + + + .section __DATA, __data // Aligned data + +#if __arm64__ + /* + * Exception stack; this is above the interrupt stack so we don't squash the interrupt + * stack on an exception. + */ + .global EXT(excepstack) +LEXT(excepstack) + .space (4096) + .globl EXT(excepstack_top) +LEXT(excepstack_top) +#endif + + /* IRQ stack */ + .globl EXT(intstack) // Boot processor IRQ stack +LEXT(intstack) + .space (4*4096) + .globl EXT(intstack_top) +LEXT(intstack_top) + + + .align 12 // Page aligned Section + + .globl EXT(fiqstack) // Boot processor FIQ stack +LEXT(fiqstack) + .space (4096) // One page size + .globl EXT(fiqstack_top) // Boot processor FIQ stack top +LEXT(fiqstack_top) + + .globl EXT(CpuDataEntries) + .align 12 // Page aligned +LEXT(CpuDataEntries) // Cpu Data Entry Array + .space (cdeSize_NUM*MAX_CPUS_NUM),0 // (filled with 0s) + + .globl EXT(BootCpuData) + .align 12 // Page aligned +LEXT(BootCpuData) // Per cpu data area + .space cdSize_NUM,0 // (filled with 0s) + + .align 3 // unsigned long long aligned Section + .globl EXT(RTClockData) +LEXT(RTClockData) // Real Time clock area + .space RTCLOCKDataSize_NUM,0 // (filled with 0s) + +#if TRASH_VFP_ON_SAVE + .align 4 + .globl EXT(vfptrash_data) +LEXT(vfptrash_data) + .fill 64, 4, 0xca55e77e +#endif + +// Must align to 16K here, due to <rdar://problem/33268668> + .global EXT(kd_early_buffer) + .align 14 +LEXT(kd_early_buffer) // space for kdebug's early event buffer + .space 16*1024,0 + +#if __arm64__ + .section __DATA, __const + +#if defined(KERNEL_INTEGRITY_KTRR) +/* reserve space for read only page tables */ + .align 14 +LEXT(ropagetable_begin) + .space 16*16*1024,0 +#else +LEXT(ropagetable_begin) +#endif /* defined(KERNEL_INTEGRITY_KTRR)*/ + +LEXT(ropagetable_end) + + .globl EXT(ropagetable_begin) + .globl EXT(ropagetable_end) +#endif /* __arm64__ */ + +/* vim: set ts=4: */ diff --git a/osfmk/arm/dbgwrap.c b/osfmk/arm/dbgwrap.c new file mode 100644 index 000000000..73aa8b658 --- /dev/null +++ b/osfmk/arm/dbgwrap.c @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include <arm/dbgwrap.h> + +boolean_t +ml_dbgwrap_cpu_is_halted(int cpu_index __unused) +{ + return FALSE; +} + +dbgwrap_status_t +ml_dbgwrap_wait_cpu_halted(int cpu_index __unused, uint64_t timeout_ns __unused) +{ + return DBGWRAP_ERR_UNSUPPORTED; +} + +dbgwrap_status_t +ml_dbgwrap_halt_cpu(int cpu_index __unused, uint64_t timeout_ns __unused) +{ + return DBGWRAP_ERR_UNSUPPORTED; +} + +dbgwrap_status_t +ml_dbgwrap_halt_cpu_with_state(int cpu_index __unused, uint64_t timeout_ns __unused, dbgwrap_thread_state_t *state __unused) +{ + return DBGWRAP_ERR_UNSUPPORTED; +} + diff --git a/osfmk/arm/dbgwrap.h b/osfmk/arm/dbgwrap.h new file mode 100644 index 000000000..940346719 --- /dev/null +++ b/osfmk/arm/dbgwrap.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#pragma once + +#include <mach/thread_status.h> +#include <sys/cdefs.h> + +__BEGIN_DECLS + +#if defined(__arm64__) +typedef arm_thread_state64_t __attribute__((aligned(16))) dbgwrap_thread_state_t; +#else +typedef arm_thread_state32_t dbgwrap_thread_state_t; +#endif + +typedef enum { + DBGWRAP_ERR_SELF_HALT = -6, + DBGWRAP_ERR_UNSUPPORTED = -5, + DBGWRAP_ERR_INPROGRESS = -4, + DBGWRAP_ERR_INSTR_ERROR = -3, + DBGWRAP_ERR_INSTR_TIMEOUT = -2, + DBGWRAP_ERR_HALT_TIMEOUT = -1, + DBGWRAP_SUCCESS = 0, + DBGWRAP_WARN_ALREADY_HALTED, + DBGWRAP_WARN_CPU_OFFLINE +} dbgwrap_status_t; + +boolean_t ml_dbgwrap_cpu_is_halted(int cpu_index); + +dbgwrap_status_t ml_dbgwrap_wait_cpu_halted(int cpu_index, uint64_t timeout_ns); + +dbgwrap_status_t ml_dbgwrap_halt_cpu(int cpu_index, uint64_t timeout_ns); + +dbgwrap_status_t ml_dbgwrap_halt_cpu_with_state(int cpu_index, uint64_t timeout_ns, dbgwrap_thread_state_t *state); + +__END_DECLS + diff --git a/osfmk/arm/exception.h b/osfmk/arm/exception.h new file mode 100644 index 000000000..bafe40b7d --- /dev/null +++ b/osfmk/arm/exception.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* CMU_ENDHIST */ +/* + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ + +/* + */ + +/* + * ARM Exception Info + */ +#ifndef _ARM_EXCEPTION_H_ +#define _ARM_EXCEPTION_H_ + +#define VECT_RESET 0x0 +#define VECT_UNDEF_INST 0x4 +#define VECT_SWI 0x8 +#define VECT_PREFECT_ABT 0xC +#define VECT_DATA_ABT 0x10 +#define VECT_IRQ 0x18 +#define VECT_FIQ 0x1C +/* can put actual code for FIQ here, avoiding extra fetch */ + + +#endif /* _ARM_EXCEPTION_H_ */ diff --git a/osfmk/arm/genassym.c b/osfmk/arm/genassym.c new file mode 100644 index 000000000..4f3c1b8ba --- /dev/null +++ b/osfmk/arm/genassym.c @@ -0,0 +1,368 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ + +#include <stddef.h> + +#include <mach_ldebug.h> + +/* + * Pass field offsets to assembly code. + */ +#include <kern/ast.h> +#include <kern/thread.h> +#include <kern/task.h> +#include <kern/locks.h> +#include <ipc/ipc_space.h> +#include <ipc/ipc_port.h> +#include <ipc/ipc_pset.h> +#include <kern/host.h> +#include <kern/misc_protos.h> +#include <kern/syscall_sw.h> +#include <arm/thread.h> +#include <mach/arm/vm_param.h> +#include <arm/pmap.h> +#include <arm/trap.h> +#include <arm/cpu_data_internal.h> +#include <arm/cpu_capabilities.h> +#include <arm/cpu_internal.h> +#include <arm/rtclock.h> +#include <machine/commpage.h> +#include <vm/vm_map.h> +#include <pexpert/arm/boot.h> +#include <arm/proc_reg.h> +#include <prng/random.h> + +#if CONFIG_DTRACE +#define NEED_DTRACE_DEFS +#include <../bsd/sys/lockstat.h> +#endif /* CONFIG_DTRACE */ + +/* + * genassym.c is used to produce an + * assembly file which, intermingled with unuseful assembly code, + * has all the necessary definitions emitted. This assembly file is + * then postprocessed with sed to extract only these definitions + * and thus the final assyms.s is created. + * + * This convoluted means is necessary since the structure alignment + * and packing may be different between the host machine and the + * target so we are forced into using the cross compiler to generate + * the values, but we cannot run anything on the target machine. + */ + +#define DECLARE(SYM,VAL) \ + __asm("DEFINITION__define__" SYM ":\t .ascii \"%0\"" : : "n" ((u_int)(VAL))) + + +int main( + int argc, + char ** argv); + +int +main( + int argc, + char **argv) +{ + + DECLARE("T_PREFETCH_ABT", T_PREFETCH_ABT); + DECLARE("T_DATA_ABT", T_DATA_ABT); + + DECLARE("AST_URGENT", AST_URGENT); + DECLARE("AST_PREEMPTION", AST_PREEMPTION); + + DECLARE("TH_RECOVER", offsetof(struct thread, recover)); + DECLARE("TH_CONTINUATION", offsetof(struct thread, continuation)); + DECLARE("TH_KERNEL_STACK", offsetof(struct thread, kernel_stack)); + DECLARE("TH_KSTACKPTR", offsetof(struct thread, machine.kstackptr)); + DECLARE("TH_UTHREAD", offsetof(struct thread, uthread)); + + DECLARE("TASK_MACH_EXC_PORT", + offsetof(struct task, exc_actions[EXC_MACH_SYSCALL].port)); + + /* These fields are being added on demand */ + DECLARE("ACT_TASK", offsetof(struct thread, task)); + DECLARE("ACT_PCBDATA", offsetof(struct thread, machine.PcbData)); +#if __ARM_VFP__ + DECLARE("ACT_UVFP", offsetof(struct thread, machine.uVFPdata)); + DECLARE("ACT_KVFP", offsetof(struct thread, machine.kVFPdata)); +#endif + DECLARE("TH_CTH_SELF", offsetof(struct thread, machine.cthread_self)); + DECLARE("TH_CTH_DATA", offsetof(struct thread, machine.cthread_data)); + DECLARE("ACT_PCBDATA_PC", offsetof(struct thread, machine.PcbData.pc)); + DECLARE("ACT_PCBDATA_R0", offsetof(struct thread, machine.PcbData.r[0])); + DECLARE("ACT_PREEMPT_CNT", offsetof(struct thread, machine.preemption_count)); + DECLARE("ACT_CPUDATAP", offsetof(struct thread, machine.CpuDatap)); + DECLARE("ACT_MAP", offsetof(struct thread, map)); +#if __ARM_USER_PROTECT__ + DECLARE("ACT_UPTW_TTC", offsetof(struct thread, machine.uptw_ttc)); + DECLARE("ACT_UPTW_TTB", offsetof(struct thread, machine.uptw_ttb)); + DECLARE("ACT_KPTW_TTB", offsetof(struct thread, machine.kptw_ttb)); + DECLARE("ACT_ASID", offsetof(struct thread, machine.asid)); +#endif + DECLARE("ACT_DEBUGDATA", offsetof(struct thread, machine.DebugData)); + DECLARE("TH_IOTIER_OVERRIDE", offsetof(struct thread, iotier_override)); + DECLARE("TH_RWLOCK_CNT", offsetof(struct thread, rwlock_count)); + DECLARE("TH_SCHED_FLAGS", offsetof(struct thread, sched_flags)); + DECLARE("TH_SFLAG_RW_PROMOTED", TH_SFLAG_RW_PROMOTED); + + DECLARE("TH_MACH_SYSCALLS", offsetof(struct thread, syscalls_mach)); + DECLARE("TH_UNIX_SYSCALLS", offsetof(struct thread, syscalls_unix)); + DECLARE("TASK_BSD_INFO", offsetof(struct task, bsd_info)); + + DECLARE("MACH_TRAP_TABLE_COUNT", MACH_TRAP_TABLE_COUNT); + DECLARE("MACH_TRAP_TABLE_ENTRY_SIZE", sizeof(mach_trap_t)); + + DECLARE("MAP_PMAP", offsetof(struct _vm_map, pmap)); + + DECLARE("SS_SIZE", sizeof(struct arm_saved_state)); + DECLARE("SS_LR", offsetof(struct arm_saved_state, lr)); + DECLARE("SS_CPSR", offsetof(struct arm_saved_state, cpsr)); + DECLARE("SS_PC", offsetof(struct arm_saved_state, pc)); + DECLARE("SS_R0", offsetof(struct arm_saved_state, r[0])); + DECLARE("SS_R4", offsetof(struct arm_saved_state, r[4])); + DECLARE("SS_R9", offsetof(struct arm_saved_state, r[9])); + DECLARE("SS_R12", offsetof(struct arm_saved_state, r[12])); + DECLARE("SS_SP", offsetof(struct arm_saved_state, sp)); + DECLARE("SS_STATUS", offsetof(struct arm_saved_state, fsr)); + DECLARE("SS_VADDR", offsetof(struct arm_saved_state, far)); + DECLARE("SS_EXC", offsetof(struct arm_saved_state, exception)); + +#if __ARM_VFP__ + DECLARE("VSS_SIZE", sizeof(struct arm_vfpsaved_state)); + DECLARE("VSS_FPSCR", offsetof(struct arm_vfpsaved_state, fpscr)); + DECLARE("VSS_FPEXC", offsetof(struct arm_vfpsaved_state, fpexc)); + + DECLARE("EXC_CTX_SIZE", sizeof(struct arm_saved_state) + + sizeof(struct arm_vfpsaved_state) + + VFPSAVE_ALIGN); + DECLARE("VSS_ALIGN", VFPSAVE_ALIGN); +#else + DECLARE("EXC_CTX_SIZE", sizeof(struct arm_saved_state)); +#endif + + + DECLARE("PGBYTES", ARM_PGBYTES); + DECLARE("PGSHIFT", ARM_PGSHIFT); + DECLARE("PGMASK", ARM_PGMASK); + + DECLARE("VM_MIN_ADDRESS", VM_MIN_ADDRESS); + DECLARE("VM_MAX_ADDRESS", VM_MAX_ADDRESS); + DECLARE("KERNELBASE", VM_MIN_KERNEL_ADDRESS); + DECLARE("KERNEL_STACK_SIZE", KERNEL_STACK_SIZE); + + DECLARE("KERN_INVALID_ADDRESS", KERN_INVALID_ADDRESS); + + DECLARE("MAX_CPUS", MAX_CPUS); + + DECLARE("cdeSize", + sizeof(struct cpu_data_entry)); + + DECLARE("cdSize", + sizeof(struct cpu_data)); + + DECLARE("CPU_ACTIVE_THREAD", + offsetof(cpu_data_t, cpu_active_thread)); + DECLARE("CPU_ACTIVE_STACK", + offsetof(cpu_data_t, cpu_active_stack)); + DECLARE("CPU_ISTACKPTR", + offsetof(cpu_data_t, istackptr)); + DECLARE("CPU_INTSTACK_TOP", + offsetof(cpu_data_t, intstack_top)); + DECLARE("CPU_FIQSTACKPTR", + offsetof(cpu_data_t, fiqstackptr)); + DECLARE("CPU_FIQSTACK_TOP", + offsetof(cpu_data_t, fiqstack_top)); + DECLARE("CPU_NUMBER_GS", + offsetof(cpu_data_t,cpu_number)); + DECLARE("CPU_IDENT", + offsetof(cpu_data_t,cpu_ident)); + DECLARE("CPU_RUNNING", + offsetof(cpu_data_t,cpu_running)); + DECLARE("CPU_MCOUNT_OFF", + offsetof(cpu_data_t,cpu_mcount_off)); + DECLARE("CPU_PENDING_AST", + offsetof(cpu_data_t,cpu_pending_ast)); + DECLARE("CPU_PROCESSOR", + offsetof(cpu_data_t,cpu_processor)); + DECLARE("CPU_CACHE_DISPATCH", + offsetof(cpu_data_t,cpu_cache_dispatch)); + DECLARE("CPU_BASE_TIMEBASE_LOW", + offsetof(cpu_data_t,cpu_base_timebase_low)); + DECLARE("CPU_BASE_TIMEBASE_HIGH", + offsetof(cpu_data_t,cpu_base_timebase_high)); + DECLARE("CPU_TIMEBASE_LOW", + offsetof(cpu_data_t,cpu_timebase_low)); + DECLARE("CPU_TIMEBASE_HIGH", + offsetof(cpu_data_t,cpu_timebase_high)); + DECLARE("CPU_DECREMENTER", + offsetof(cpu_data_t,cpu_decrementer)); + DECLARE("CPU_GET_DECREMENTER_FUNC", + offsetof(cpu_data_t,cpu_get_decrementer_func)); + DECLARE("CPU_SET_DECREMENTER_FUNC", + offsetof(cpu_data_t,cpu_set_decrementer_func)); + DECLARE("CPU_GET_FIQ_HANDLER", + offsetof(cpu_data_t,cpu_get_fiq_handler)); + DECLARE("CPU_TBD_HARDWARE_ADDR", + offsetof(cpu_data_t,cpu_tbd_hardware_addr)); + DECLARE("CPU_TBD_HARDWARE_VAL", + offsetof(cpu_data_t,cpu_tbd_hardware_val)); + DECLARE("CPU_INT_STATE", + offsetof(cpu_data_t,cpu_int_state)); + DECLARE("INTERRUPT_HANDLER", + offsetof(cpu_data_t,interrupt_handler)); + DECLARE("INTERRUPT_TARGET", + offsetof(cpu_data_t,interrupt_target)); + DECLARE("INTERRUPT_REFCON", + offsetof(cpu_data_t,interrupt_refCon)); + DECLARE("INTERRUPT_NUB", + offsetof(cpu_data_t,interrupt_nub)); + DECLARE("INTERRUPT_SOURCE", + offsetof(cpu_data_t,interrupt_source)); + DECLARE("CPU_USER_DEBUG", + offsetof(cpu_data_t, cpu_user_debug)); + DECLARE("CPU_STAT_IRQ", + offsetof(cpu_data_t, cpu_stat.irq_ex_cnt)); + DECLARE("CPU_STAT_IRQ_WAKE", + offsetof(cpu_data_t, cpu_stat.irq_ex_cnt_wake)); + DECLARE("CPU_RESET_HANDLER", + offsetof(cpu_data_t, cpu_reset_handler)); + DECLARE("CPU_RESET_ASSIST", + offsetof(cpu_data_t, cpu_reset_assist)); + DECLARE("RTCLOCK_DATAP", + offsetof(cpu_data_t, rtclock_datap)); +#ifdef __arm__ + DECLARE("CPU_EXC_VECTORS", + offsetof(cpu_data_t, cpu_exc_vectors)); +#endif + + DECLARE("RTCLOCKDataSize", + sizeof(rtclock_data_t)); + DECLARE("RTCLOCK_ADJ_ABSTIME_LOW", + offsetof(rtclock_data_t, rtc_adj.abstime_val.low)); + DECLARE("RTCLOCK_ADJ_ABSTIME_HIGH", + offsetof(rtclock_data_t, rtc_adj.abstime_val.high)); + DECLARE("RTCLOCK_BASE_ABSTIME_LOW", + offsetof(rtclock_data_t, rtc_base.abstime_val.low)); + DECLARE("RTCLOCK_BASE_ABSTIME_HIGH", + offsetof(rtclock_data_t, rtc_base.abstime_val.high)); + DECLARE("RTCLOCK_TB_FUNC", + offsetof(rtclock_data_t, rtc_timebase_func)); + DECLARE("RTCLOCK_TB_ADDR", + offsetof(rtclock_data_t, rtc_timebase_addr)); + DECLARE("RTCLOCK_TB_VAL", + offsetof(rtclock_data_t, rtc_timebase_val)); + + DECLARE("SIGPdec", SIGPdec); + + DECLARE("rhdSize", + sizeof(struct reset_handler_data)); + + DECLARE("CPU_DATA_ENTRIES", offsetof(struct reset_handler_data, cpu_data_entries)); + DECLARE("BOOT_ARGS", offsetof(struct reset_handler_data, boot_args)); + DECLARE("ASSIST_RESET_HANDLER", offsetof(struct reset_handler_data, assist_reset_handler)); + + DECLARE("CPU_DATA_PADDR", offsetof(struct cpu_data_entry, cpu_data_paddr)); + + + DECLARE("INTSTACK_SIZE", INTSTACK_SIZE); + + /* values from kern/timer.h */ + DECLARE("TIMER_LOW", + offsetof(struct timer, low_bits)); + DECLARE("TIMER_HIGH", + offsetof(struct timer, high_bits)); + DECLARE("TIMER_HIGHCHK", + offsetof(struct timer, high_bits_check)); + DECLARE("TIMER_TSTAMP", + offsetof(struct timer, tstamp)); + DECLARE("THREAD_TIMER", + offsetof(struct processor, processor_data.thread_timer)); + DECLARE("KERNEL_TIMER", + offsetof(struct processor, processor_data.kernel_timer)); + DECLARE("SYSTEM_STATE", + offsetof(struct processor, processor_data.system_state)); + DECLARE("USER_STATE", + offsetof(struct processor, processor_data.user_state)); + DECLARE("CURRENT_STATE", + offsetof(struct processor, processor_data.current_state)); + + DECLARE("SYSTEM_TIMER", + offsetof(struct thread, system_timer)); + DECLARE("USER_TIMER", + offsetof(struct thread, user_timer)); + +#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME + DECLARE("PRECISE_USER_KERNEL_TIME", + offsetof(struct thread, precise_user_kernel_time)); +#endif + + DECLARE("BA_VIRT_BASE", + offsetof(struct boot_args, virtBase)); + DECLARE("BA_PHYS_BASE", + offsetof(struct boot_args, physBase)); + DECLARE("BA_MEM_SIZE", + offsetof(struct boot_args, memSize)); + DECLARE("BA_TOP_OF_KERNEL_DATA", + offsetof(struct boot_args, topOfKernelData)); + + DECLARE("ENTROPY_INDEX_PTR", + offsetof(entropy_data_t, index_ptr)); + DECLARE("ENTROPY_BUFFER", + offsetof(entropy_data_t, buffer)); + DECLARE("ENTROPY_DATA_SIZE", sizeof(struct entropy_data)); + + return (0); +} diff --git a/osfmk/arm/globals_asm.h b/osfmk/arm/globals_asm.h new file mode 100644 index 000000000..62fe102b9 --- /dev/null +++ b/osfmk/arm/globals_asm.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +LOAD_ADDR_GEN_DEF(ExceptionVectorsBase) +LOAD_ADDR_GEN_DEF(intstack_top) +LOAD_ADDR_GEN_DEF(fiqstack_top) +LOAD_ADDR_GEN_DEF(gVirtBase) +LOAD_ADDR_GEN_DEF(gPhysBase) +LOAD_ADDR_GEN_DEF(gPhysSize) +LOAD_ADDR_GEN_DEF(EntropyData) +LOAD_ADDR_GEN_DEF(kdebug_enable) +#if CONFIG_TELEMETRY +LOAD_ADDR_GEN_DEF(telemetry_needs_record) +#endif + diff --git a/osfmk/arm/hw_lock_types.h b/osfmk/arm/hw_lock_types.h new file mode 100644 index 000000000..342445c33 --- /dev/null +++ b/osfmk/arm/hw_lock_types.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * Copyright (C) 1998 Apple Computer + * All Rights Reserved + */ +/* + * @OSF_COPYRIGHT@ + */ + +/* + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ + +#ifndef _ARM_HW_LOCK_TYPES_H_ +#define _ARM_HW_LOCK_TYPES_H_ + +struct hslock { + uintptr_t lock_data; +}; + +typedef struct hslock hw_lock_data_t, *hw_lock_t; + +#define hw_lock_addr(hwl) (&((hwl).lock_data)) + +#endif /* _ARM_HW_LOCK_TYPES_H_ */ diff --git a/osfmk/arm/io_map.c b/osfmk/arm/io_map.c new file mode 100644 index 000000000..2aa718001 --- /dev/null +++ b/osfmk/arm/io_map.c @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright notice + * and this permission notice appear in all copies of the software, + * derivative works or modified versions, and any portions thereof, and that + * both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" CONDITION. + * CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR ANY DAMAGES + * WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science Carnegie Mellon University Pittsburgh PA + * 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon the + * rights to redistribute these changes. + */ +/* + */ + +#include <mach/vm_param.h> +#include <vm/vm_kern.h> +#include <vm/vm_map.h> +#include <arm/pmap.h> +#include <arm/io_map_entries.h> +#include <san/kasan.h> + +extern vm_offset_t virtual_space_start; /* Next available kernel VA */ + +/* + * Allocate and map memory for devices that may need to be mapped before + * Mach VM is running. + */ +vm_offset_t +io_map(vm_map_offset_t phys_addr, vm_size_t size, unsigned int flags) +{ + vm_offset_t start, start_offset; + + start_offset = phys_addr & PAGE_MASK; + size += start_offset; + phys_addr -= start_offset; + + if (kernel_map == VM_MAP_NULL) { + /* + * VM is not initialized. Grab memory. + */ + start = virtual_space_start; + virtual_space_start += round_page(size); + + assert(flags == VM_WIMG_WCOMB || flags == VM_WIMG_IO); + + if (flags == VM_WIMG_WCOMB) { + (void) pmap_map_bd_with_options(start, phys_addr, phys_addr + round_page(size), + VM_PROT_READ | VM_PROT_WRITE, PMAP_MAP_BD_WCOMB); + } else { + (void) pmap_map_bd(start, phys_addr, phys_addr + round_page(size), + VM_PROT_READ | VM_PROT_WRITE); + } + } else { + (void) kmem_alloc_pageable(kernel_map, &start, round_page(size), VM_KERN_MEMORY_IOKIT); + (void) pmap_map(start, phys_addr, phys_addr + round_page(size), + VM_PROT_READ | VM_PROT_WRITE, flags); + } +#if KASAN + kasan_notify_address(start + start_offset, size); +#endif + return (start + start_offset); +} + +/* just wrap this since io_map handles it */ + +vm_offset_t +io_map_spec(vm_map_offset_t phys_addr, vm_size_t size, unsigned int flags) +{ + return (io_map(phys_addr, size, flags)); +} diff --git a/osfmk/arm/io_map_entries.h b/osfmk/arm/io_map_entries.h new file mode 100644 index 000000000..1a96d9764 --- /dev/null +++ b/osfmk/arm/io_map_entries.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +#ifdef KERNEL_PRIVATE + +#ifndef _ARM_IO_MAP_ENTRIES +#define _ARM_IO_MAP_ENTRIES + +#include <sys/appleapiopts.h> + +#ifdef __APPLE_API_PRIVATE +extern vm_offset_t io_map( + vm_map_offset_t phys_addr, + vm_size_t size, + unsigned int flags); +extern vm_offset_t io_map_spec(vm_map_offset_t phys_addr, vm_size_t size, unsigned int flags); +#endif /* __APPLE_API_PRIVATE */ + +#endif /* _ARM_IO_MAP_ENTRIES */ + +#endif /* KERNEL_PRIVATE */ + diff --git a/osfmk/arm/kpc_arm.c b/osfmk/arm/kpc_arm.c new file mode 100644 index 000000000..11a544584 --- /dev/null +++ b/osfmk/arm/kpc_arm.c @@ -0,0 +1,986 @@ +/* + * Copyright (c) 2012 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include <mach/mach_types.h> +#include <machine/machine_routines.h> +#include <kern/processor.h> +#include <kern/kalloc.h> +#include <kern/thread.h> +#include <sys/errno.h> +#include <arm/cpu_data_internal.h> +#include <arm/cpu_internal.h> +#include <kern/kpc.h> + +#ifdef ARMA7 +/* PMU v2 based implementation for A7 */ +static uint32_t saved_PMXEVTYPER[MAX_CPUS][KPC_ARM_TOTAL_COUNT]; +static uint32_t saved_PMCNTENSET[MAX_CPUS]; +static uint64_t saved_counter[MAX_CPUS][KPC_ARM_TOTAL_COUNT]; +static uint32_t saved_PMOVSR[MAX_CPUS]; + +static uint32_t kpc_configured = 0; +static uint32_t kpc_xcall_sync; +static uint64_t kpc_running_cfg_pmc_mask = 0; +static uint32_t kpc_running_classes = 0; +static uint32_t kpc_reload_sync; +static uint32_t kpc_enabled_counters = 0; + +static int first_time = 1; + +/* Private */ + +static boolean_t +enable_counter(uint32_t counter) +{ + boolean_t enabled; + uint32_t PMCNTENSET; + /* Cycle counter is MSB; configurable counters reside in LSBs */ + uint32_t mask = (counter == 0) ? (1 << 31) : (1 << (counter - 1)); + + /* Enabled? */ + __asm__ volatile("mrc p15, 0, %0, c9, c12, 1;" : "=r" (PMCNTENSET)); + + enabled = (PMCNTENSET & mask); + if (!enabled) { + /* Counter interrupt enable (PMINTENSET) */ + __asm__ volatile("mcr p15, 0, %0, c9, c14, 1;" : : "r" (mask)); + + /* Individual counter enable set (PMCNTENSET) */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 1;" : : "r" (mask)); + + kpc_enabled_counters++; + + /* 1st enabled counter? Set the master enable bit in PMCR */ + if (kpc_enabled_counters == 1) { + uint32_t PMCR = 1; + __asm__ volatile("mcr p15, 0, %0, c9, c12, 0;" : : "r" (PMCR)); + } + } + + return enabled; +} + +static boolean_t +disable_counter(uint32_t counter) +{ + boolean_t enabled; + uint32_t PMCNTENCLR; + /* Cycle counter is MSB; configurable counters reside in LSBs */ + uint32_t mask = (counter == 0) ? (1 << 31) : (1 << (counter - 1)); + + /* Enabled? */ + __asm__ volatile("mrc p15, 0, %0, c9, c12, 2;" : "=r" (PMCNTENCLR)); + + enabled = (PMCNTENCLR & mask); + if (enabled) { + /* Individual counter enable clear (PMCNTENCLR) */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 2;" : : "r" (mask)); + + /* Counter interrupt disable (PMINTENCLR) */ + __asm__ volatile("mcr p15, 0, %0, c9, c14, 2;" : : "r" (mask)); + + kpc_enabled_counters--; + + /* Last enabled counter? Clear the master enable bit in PMCR */ + if (kpc_enabled_counters == 0) { + uint32_t PMCR = 0; + __asm__ volatile("mcr p15, 0, %0, c9, c12, 0;" : : "r" (PMCR)); + } + } + + return enabled; +} + +static uint64_t +read_counter(uint32_t counter) +{ + uint32_t low = 0; + + switch (counter) { + case 0: + /* Fixed counter */ + __asm__ volatile("mrc p15, 0, %0, c9, c13, 0;" : "=r" (low)); + break; + case 1: + case 2: + case 3: + case 4: + /* Configurable. Set PMSELR... */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (counter - 1)); + /* ...then read PMXEVCNTR */ + __asm__ volatile("mrc p15, 0, %0, c9, c13, 2;" : "=r" (low)); + break; + default: + /* ??? */ + break; + } + + return (uint64_t)low; +} + +static void +write_counter(uint32_t counter, uint64_t value) +{ + uint32_t low = value & 0xFFFFFFFF; + + switch (counter) { + case 0: + /* Fixed counter */ + __asm__ volatile("mcr p15, 0, %0, c9, c13, 0;" : : "r" (low)); + break; + case 1: + case 2: + case 3: + case 4: + /* Configurable. Set PMSELR... */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (counter - 1)); + /* ...then write PMXEVCNTR */ + __asm__ volatile("mcr p15, 0, %0, c9, c13, 2;" : : "r" (low)); + break; + default: + /* ??? */ + break; + } +} + +static uint64_t +kpc_reload_counter(int ctr) +{ + uint64_t old = read_counter(ctr); + write_counter(ctr, FIXED_RELOAD(ctr)); + return old; +} + +static void +set_running_fixed(boolean_t on) +{ + int i; + boolean_t enabled; + int n = KPC_ARM_FIXED_COUNT; + + enabled = ml_set_interrupts_enabled(FALSE); + + for( i = 0; i < n; i++ ) { + if (on) { + enable_counter(i); + } else { + disable_counter(i); + } + } + + ml_set_interrupts_enabled(enabled); +} + +static void +set_running_configurable(uint64_t target_mask, uint64_t state_mask) +{ + uint32_t cfg_count = kpc_configurable_count(), offset = kpc_fixed_count(); + boolean_t enabled; + + enabled = ml_set_interrupts_enabled(FALSE); + + for (uint32_t i = 0; i < cfg_count; ++i) { + if (((1ULL << i) & target_mask) == 0) + continue; + assert(kpc_controls_counter(offset + i)); + + if ((1ULL << i) & state_mask) { + enable_counter(offset + i); + } else { + disable_counter(offset + i); + } + } + + ml_set_interrupts_enabled(enabled); +} + +void kpc_pmi_handler(cpu_id_t source); +void +kpc_pmi_handler(cpu_id_t source) +{ + uint64_t extra; + int ctr; + int enabled; + + enabled = ml_set_interrupts_enabled(FALSE); + + /* The pmi must be delivered to the CPU that generated it */ + if (source != getCpuDatap()->interrupt_nub) { + panic("pmi from IOCPU %p delivered to IOCPU %p", source, getCpuDatap()->interrupt_nub); + } + + for (ctr = 0; + ctr < (KPC_ARM_FIXED_COUNT + KPC_ARM_CONFIGURABLE_COUNT); + ctr++) + { + uint32_t PMOVSR; + uint32_t mask; + + /* check the counter for overflow */ + if (ctr == 0) { + mask = 1 << 31; + } else { + mask = 1 << (ctr - 1); + } + + /* read PMOVSR */ + __asm__ volatile("mrc p15, 0, %0, c9, c12, 3;" : "=r" (PMOVSR)); + + if (PMOVSR & mask) { + extra = kpc_reload_counter(ctr); + + FIXED_SHADOW(ctr) + += (kpc_fixed_max() - FIXED_RELOAD(ctr) + 1 /* wrap */) + extra; + + if (FIXED_ACTIONID(ctr)) + kpc_sample_kperf(FIXED_ACTIONID(ctr)); + + /* clear PMOVSR bit */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 3;" : : "r" (mask)); + } + } + + ml_set_interrupts_enabled(enabled); +} + +static void +kpc_set_running_xcall( void *vstate ) +{ + struct kpc_running_remote *mp_config = (struct kpc_running_remote*) vstate; + assert(mp_config); + + if (kpc_controls_fixed_counters()) + set_running_fixed(mp_config->classes & KPC_CLASS_FIXED_MASK); + + set_running_configurable(mp_config->cfg_target_mask, + mp_config->cfg_state_mask); + + if (hw_atomic_sub(&kpc_xcall_sync, 1) == 0) { + thread_wakeup((event_t) &kpc_xcall_sync); + } +} + +static uint64_t +get_counter_config(uint32_t counter) +{ + uint32_t config = 0; + + switch (counter) { + case 0: + /* Fixed counter accessed via top bit... */ + counter = 31; + /* Write PMSELR.SEL */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (counter)); + /* Read PMXEVTYPER */ + __asm__ volatile("mcr p15, 0, %0, c9, c13, 1;" : "=r" (config)); + break; + case 1: + case 2: + case 3: + case 4: + /* Offset */ + counter -= 1; + /* Write PMSELR.SEL to select the configurable counter */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (counter)); + /* Read PMXEVTYPER to get the config */ + __asm__ volatile("mrc p15, 0, %0, c9, c13, 1;" : "=r" (config)); + break; + default: + break; + } + + return config; +} + +static void +set_counter_config(uint32_t counter, uint64_t config) +{ + switch (counter) { + case 0: + /* Write PMSELR.SEL */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (31)); + /* Write PMXEVTYPER */ + __asm__ volatile("mcr p15, 0, %0, c9, c13, 1;" : : "r" (config & 0xFFFFFFFF)); + break; + case 1: + case 2: + case 3: + case 4: + /* Write PMSELR.SEL */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (counter - 1)); + /* Write PMXEVTYPER */ + __asm__ volatile("mcr p15, 0, %0, c9, c13, 1;" : : "r" (config & 0xFFFFFFFF)); + break; + default: + break; + } +} + +/* Common */ + +void +kpc_arch_init(void) +{ + uint32_t PMCR; + uint32_t event_counters; + + /* read PMOVSR and determine the number of event counters */ + __asm__ volatile("mrc p15, 0, %0, c9, c12, 0;" : "=r" (PMCR)); + event_counters = (PMCR >> 11) & 0x1F; + + assert(event_counters >= KPC_ARM_CONFIGURABLE_COUNT); +} + +uint32_t +kpc_get_classes(void) +{ + return KPC_CLASS_FIXED_MASK | KPC_CLASS_CONFIGURABLE_MASK; +} + +uint32_t +kpc_fixed_count(void) +{ + return KPC_ARM_FIXED_COUNT; +} + +uint32_t +kpc_configurable_count(void) +{ + return KPC_ARM_CONFIGURABLE_COUNT; +} + +uint32_t +kpc_fixed_config_count(void) +{ + return KPC_ARM_FIXED_COUNT; +} + +uint32_t +kpc_configurable_config_count(uint64_t pmc_mask) +{ + assert(kpc_popcount(pmc_mask) <= kpc_configurable_count()); + return kpc_popcount(pmc_mask); +} + +int +kpc_get_fixed_config(kpc_config_t *configv) +{ + configv[0] = get_counter_config(0); + return 0; +} + +uint64_t +kpc_fixed_max(void) +{ + return (1ULL << KPC_ARM_COUNTER_WIDTH) - 1; +} + +uint64_t +kpc_configurable_max(void) +{ + return (1ULL << KPC_ARM_COUNTER_WIDTH) - 1; +} + +int +kpc_get_configurable_counters(uint64_t *counterv, uint64_t pmc_mask) +{ + uint32_t cfg_count = kpc_configurable_count(), offset = kpc_fixed_count(); + + assert(counterv); + + for (uint32_t i = 0; i < cfg_count; ++i) { + uint32_t PMOVSR; + uint32_t mask; + uint64_t ctr; + + if (((1ULL << i) & pmc_mask) == 0) + continue; + ctr = read_counter(i + offset); + + /* check the counter for overflow */ + mask = 1 << i; + + /* read PMOVSR */ + __asm__ volatile("mrc p15, 0, %0, c9, c12, 3;" : "=r" (PMOVSR)); + + if (PMOVSR & mask) { + ctr = CONFIGURABLE_SHADOW(i) + + (kpc_configurable_max() - CONFIGURABLE_RELOAD(i) + 1 /* Wrap */) + + ctr; + } else { + ctr = CONFIGURABLE_SHADOW(i) + + (ctr - CONFIGURABLE_RELOAD(i)); + } + + *counterv++ = ctr; + } + + return 0; +} + +int +kpc_get_fixed_counters(uint64_t *counterv) +{ + uint32_t PMOVSR; + uint32_t mask; + uint64_t ctr; + + /* check the counter for overflow */ + mask = 1 << 31; + + /* read PMOVSR */ + __asm__ volatile("mrc p15, 0, %0, c9, c12, 3;" : "=r" (PMOVSR)); + + ctr = read_counter(0); + + if (PMOVSR & mask) { + ctr = FIXED_SHADOW(0) + + (kpc_fixed_max() - FIXED_RELOAD(0) + 1 /* Wrap */) + + (ctr & 0xFFFFFFFF); + } else { + ctr = FIXED_SHADOW(0) + + (ctr - FIXED_RELOAD(0)); + } + + counterv[0] = ctr; + + return 0; +} +boolean_t +kpc_is_running_fixed(void) +{ + return (kpc_running_classes & KPC_CLASS_FIXED_MASK) == KPC_CLASS_FIXED_MASK; +} + +boolean_t +kpc_is_running_configurable(uint64_t pmc_mask) +{ + assert(kpc_popcount(pmc_mask) <= kpc_configurable_count()); + return ((kpc_running_classes & KPC_CLASS_CONFIGURABLE_MASK) == KPC_CLASS_CONFIGURABLE_MASK) && + ((kpc_running_cfg_pmc_mask & pmc_mask) == pmc_mask); +} + +int +kpc_set_running_arch(struct kpc_running_remote *mp_config) +{ + unsigned int cpu; + + assert(mp_config); + + if (first_time) { + kprintf( "kpc: setting PMI handler\n" ); + PE_cpu_perfmon_interrupt_install_handler(kpc_pmi_handler); + for (cpu = 0; cpu < real_ncpus; cpu++) + PE_cpu_perfmon_interrupt_enable(cpu_datap(cpu)->cpu_id, + TRUE); + first_time = 0; + } + + /* dispatch to all CPUs */ + cpu_broadcast_xcall(&kpc_xcall_sync, TRUE, kpc_set_running_xcall, + mp_config); + + kpc_running_cfg_pmc_mask = mp_config->cfg_state_mask; + kpc_running_classes = mp_config->classes; + kpc_configured = 1; + + return 0; +} + +static void +save_regs(void) +{ + int i; + int cpuid = current_processor()->cpu_id; + uint32_t PMCR = 0; + + __asm__ volatile("dmb ish"); + + /* Clear master enable */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 0;" : : "r" (PMCR)); + + /* Save individual enable state */ + __asm__ volatile("mrc p15, 0, %0, c9, c12, 1;" : "=r" (saved_PMCNTENSET[cpuid])); + + /* Save PMOVSR */ + __asm__ volatile("mrc p15, 0, %0, c9, c12, 3;" : "=r" (saved_PMOVSR[cpuid])); + + /* Select fixed counter with PMSELR.SEL */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (31)); + /* Read PMXEVTYPER */ + __asm__ volatile("mrc p15, 0, %0, c9, c13, 1;" : "=r" (saved_PMXEVTYPER[cpuid][0])); + + /* Save configurable event selections */ + for (i = 0; i < 4; i++) { + /* Select counter with PMSELR.SEL */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (i)); + /* Read PMXEVTYPER */ + __asm__ volatile("mrc p15, 0, %0, c9, c13, 1;" : "=r" (saved_PMXEVTYPER[cpuid][i + 1])); + } + + /* Finally, save count for each counter */ + for (i=0; i < 5; i++) { + saved_counter[cpuid][i] = read_counter(i); + } +} + +static void +restore_regs(void) +{ + int i; + int cpuid = current_processor()->cpu_id; + uint64_t extra; + uint32_t PMCR = 1; + + /* Restore counter values */ + for (i = 0; i < 5; i++) { + /* did we overflow? if so handle it now since we won't get a pmi */ + uint32_t mask; + + /* check the counter for overflow */ + if (i == 0) { + mask = 1 << 31; + } else { + mask = 1 << (i - 1); + } + + if (saved_PMOVSR[cpuid] & mask) { + extra = kpc_reload_counter(i); + + /* + * CONFIGURABLE_* directly follows FIXED, so we can simply + * increment the index here. Although it's ugly. + */ + FIXED_SHADOW(i) + += (kpc_fixed_max() - FIXED_RELOAD(i) + 1 /* Wrap */) + extra; + + if (FIXED_ACTIONID(i)) + kpc_sample_kperf(FIXED_ACTIONID(i)); + } else { + write_counter(i, saved_counter[cpuid][i]); + } + } + + /* Restore configuration - first, the fixed... */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (31)); + /* Write PMXEVTYPER */ + __asm__ volatile("mcr p15, 0, %0, c9, c13, 1;" : : "r" (saved_PMXEVTYPER[cpuid][0])); + + /* ...then the configurable */ + for (i = 0; i < 4; i++) { + /* Select counter with PMSELR.SEL */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 5;" : : "r" (i)); + /* Write PMXEVTYPER */ + __asm__ volatile("mcr p15, 0, %0, c9, c13, 1;" : : "r" (saved_PMXEVTYPER[cpuid][i + 1])); + } + + /* Restore enable state */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 1;" : : "r" (saved_PMCNTENSET[cpuid])); + + /* Counter master re-enable */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 0;" : : "r" (PMCR)); +} + +static void +kpc_set_reload_xcall(void *vmp_config) +{ + struct kpc_config_remote *mp_config = vmp_config; + uint32_t classes = 0, count = 0, offset = kpc_fixed_count(); + uint64_t *new_period = NULL, max = kpc_configurable_max(); + boolean_t enabled; + + assert(mp_config); + assert(mp_config->configv); + classes = mp_config->classes; + new_period = mp_config->configv; + + enabled = ml_set_interrupts_enabled(FALSE); + + if ((classes & KPC_CLASS_FIXED_MASK) && kpc_controls_fixed_counters()) { + /* update shadow counters */ + kpc_get_fixed_counters(&FIXED_SHADOW(0)); + + /* set the new period */ + count = kpc_fixed_count(); + for (uint32_t i = 0; i < count; ++i) { + if (*new_period == 0) + *new_period = kpc_fixed_max(); + FIXED_RELOAD(i) = max - *new_period; + /* reload the counter if possible */ + kpc_reload_counter(i); + /* next period value */ + new_period++; + } + } + + if (classes & KPC_CLASS_CONFIGURABLE_MASK) { + /* + * Update _all_ shadow counters, this cannot be done for only + * selected PMCs. Otherwise, we would corrupt the configurable + * shadow buffer since the PMCs are muxed according to the pmc + * mask. + */ + uint64_t all_cfg_mask = (1ULL << kpc_configurable_count()) - 1; + kpc_get_configurable_counters(&CONFIGURABLE_SHADOW(0), all_cfg_mask); + + /* set the new period */ + count = kpc_configurable_count(); + for (uint32_t i = 0; i < count; ++i) { + /* ignore the counter */ + if (((1ULL << i) & mp_config->pmc_mask) == 0) + continue; + if (*new_period == 0) + *new_period = kpc_configurable_max(); + CONFIGURABLE_RELOAD(i) = max - *new_period; + /* reload the counter */ + kpc_reload_counter(offset + i); + /* next period value */ + new_period++; + } + } + + ml_set_interrupts_enabled(enabled); + + if (hw_atomic_sub(&kpc_reload_sync, 1) == 0) + thread_wakeup((event_t) &kpc_reload_sync); +} + + +int +kpc_set_period_arch(struct kpc_config_remote *mp_config) +{ + /* dispatch to all CPUs */ + cpu_broadcast_xcall(&kpc_reload_sync, TRUE, kpc_set_reload_xcall, mp_config); + + kpc_configured = 1; + + return 0; +} + +int +kpc_get_configurable_config(kpc_config_t *configv, uint64_t pmc_mask) +{ + uint32_t cfg_count = kpc_configurable_count(), offset = kpc_fixed_count(); + + assert(configv); + + for (uint32_t i = 0; i < cfg_count; ++i) + if ((1ULL << i) & pmc_mask) + *configv++ = get_counter_config(i + offset); + + return 0; +} + +static int +kpc_set_configurable_config(kpc_config_t *configv, uint64_t pmc_mask) +{ + uint32_t cfg_count = kpc_configurable_count(), offset = kpc_fixed_count(); + boolean_t enabled; + + assert(configv); + + enabled = ml_set_interrupts_enabled(FALSE); + + for (uint32_t i = 0; i < cfg_count; ++i) { + if (((1ULL << i) & pmc_mask) == 0) + continue; + assert(kpc_controls_counter(i + offset)); + + set_counter_config(i + offset, *configv++); + } + + ml_set_interrupts_enabled(enabled); + + return 0; +} + +static uint32_t kpc_config_sync; +static void +kpc_set_config_xcall(void *vmp_config) +{ + struct kpc_config_remote *mp_config = vmp_config; + kpc_config_t *new_config = NULL; + uint32_t classes = 0ULL; + + assert(mp_config); + assert(mp_config->configv); + classes = mp_config->classes; + new_config = mp_config->configv; + + if (classes & KPC_CLASS_CONFIGURABLE_MASK) { + kpc_set_configurable_config(new_config, mp_config->pmc_mask); + new_config += kpc_popcount(mp_config->pmc_mask); + } + + if (hw_atomic_sub(&kpc_config_sync, 1) == 0) + thread_wakeup((event_t) &kpc_config_sync); +} + +int +kpc_set_config_arch(struct kpc_config_remote *mp_config) +{ + /* dispatch to all CPUs */ + cpu_broadcast_xcall(&kpc_config_sync, TRUE, kpc_set_config_xcall, mp_config); + + kpc_configured = 1; + + return 0; +} + +void +kpc_idle(void) +{ + if (kpc_configured) + save_regs(); +} + +void +kpc_idle_exit(void) +{ + if (kpc_configured) + restore_regs(); +} + +static uint32_t kpc_xread_sync; +static void +kpc_get_curcpu_counters_xcall(void *args) +{ + struct kpc_get_counters_remote *handler = args; + int offset=0, r=0; + + assert(handler); + assert(handler->buf); + + offset = cpu_number() * handler->buf_stride; + r = kpc_get_curcpu_counters(handler->classes, NULL, &handler->buf[offset]); + + /* number of counters added by this CPU, needs to be atomic */ + hw_atomic_add(&(handler->nb_counters), r); + + if (hw_atomic_sub(&kpc_xread_sync, 1) == 0) + thread_wakeup((event_t) &kpc_xread_sync); +} + +int +kpc_get_all_cpus_counters(uint32_t classes, int *curcpu, uint64_t *buf) +{ + int enabled = 0; + + struct kpc_get_counters_remote hdl = { + .classes = classes, .nb_counters = 0, + .buf_stride = kpc_get_counter_count(classes), + .buf = buf + }; + + assert(buf); + + enabled = ml_set_interrupts_enabled(FALSE); + + if (curcpu) + *curcpu = current_processor()->cpu_id; + cpu_broadcast_xcall(&kpc_xread_sync, TRUE, kpc_get_curcpu_counters_xcall, &hdl); + + ml_set_interrupts_enabled(enabled); + + return hdl.nb_counters; +} + +int +kpc_get_pmu_version(void) +{ + return KPC_PMU_ARM_V2; +} + +int +kpc_set_sw_inc( uint32_t mask ) +{ + /* Only works with the configurable counters set to count the increment event (0x0) */ + + /* Write to PMSWINC */ + __asm__ volatile("mcr p15, 0, %0, c9, c12, 4;" : : "r" (mask)); + + return 0; +} + +#else /* !ARMA7 */ + +/* no kpc */ + +void +kpc_arch_init(void) +{ + /* No-op */ +} + +uint32_t +kpc_get_classes(void) +{ + return 0; +} + +uint32_t +kpc_fixed_count(void) +{ + return 0; +} + +uint32_t +kpc_configurable_count(void) +{ + return 0; +} + +uint32_t +kpc_fixed_config_count(void) +{ + return 0; +} + +uint32_t +kpc_configurable_config_count(uint64_t pmc_mask __unused) +{ + return 0; +} + +int +kpc_get_fixed_config(kpc_config_t *configv __unused) +{ + return 0; +} + +uint64_t +kpc_fixed_max(void) +{ + return 0; +} + +uint64_t +kpc_configurable_max(void) +{ + return 0; +} + +int +kpc_get_configurable_config(kpc_config_t *configv __unused, uint64_t pmc_mask __unused) +{ + return ENOTSUP; +} + +int +kpc_get_configurable_counters(uint64_t *counterv __unused, uint64_t pmc_mask __unused) +{ + return ENOTSUP; +} + +int +kpc_get_fixed_counters(uint64_t *counterv __unused) +{ + return 0; +} + +boolean_t +kpc_is_running_fixed(void) +{ + return FALSE; +} + +boolean_t +kpc_is_running_configurable(uint64_t pmc_mask __unused) +{ + return FALSE; +} + +int +kpc_set_running_arch(struct kpc_running_remote *mp_config __unused) +{ + return ENOTSUP; +} + +int +kpc_set_period_arch(struct kpc_config_remote *mp_config __unused) +{ + return ENOTSUP; +} + +int +kpc_set_config_arch(struct kpc_config_remote *mp_config __unused) +{ + return ENOTSUP; +} + +void +kpc_idle(void) +{ + // do nothing +} + +void +kpc_idle_exit(void) +{ + // do nothing +} + +int +kpc_get_all_cpus_counters(uint32_t classes, int *curcpu, uint64_t *buf) +{ +#pragma unused(classes) +#pragma unused(curcpu) +#pragma unused(buf) + + return 0; +} + +int +kpc_set_sw_inc( uint32_t mask __unused ) +{ + return ENOTSUP; +} + +int +kpc_get_pmu_version(void) +{ + return KPC_PMU_ERROR; +} + +#endif + +/* + * RAWPMU isn't implemented for any of the 32-bit ARMs. + */ + +uint32_t +kpc_rawpmu_config_count(void) +{ + return 0; +} + +int +kpc_get_rawpmu_config(__unused kpc_config_t *configv) +{ + return 0; +} diff --git a/osfmk/arm/lock.h b/osfmk/arm/lock.h new file mode 100644 index 000000000..943a5f345 --- /dev/null +++ b/osfmk/arm/lock.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * Copyright (C) 1998 Apple Computer + * All Rights Reserved + */ +/* + * @OSF_COPYRIGHT@ + */ + +/* + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ + +#ifdef KERNEL_PRIVATE + +#ifndef _ARM_LOCK_H_ +#define _ARM_LOCK_H_ + +#warning This header is deprecated. Use <kern/locks.h> instead. + +#endif /* _ARM_LOCK_H_ */ + +#endif /* KERNEL_PRIVATE */ diff --git a/osfmk/arm/locks.h b/osfmk/arm/locks.h new file mode 100644 index 000000000..3a58a7fcc --- /dev/null +++ b/osfmk/arm/locks.h @@ -0,0 +1,332 @@ +/* + * Copyright (c) 2007-2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef _ARM_LOCKS_H_ +#define _ARM_LOCKS_H_ + +#include <kern/kern_types.h> +#ifdef MACH_KERNEL_PRIVATE +#include <arm/hw_lock_types.h> +#endif + + +#ifdef MACH_KERNEL_PRIVATE + +extern unsigned int LcksOpts; + +#define enaLkDeb 0x00000001 /* Request debug in default attribute */ +#define enaLkStat 0x00000002 /* Request statistic in default attribute */ +#define disLkRWPrio 0x00000004 /* Disable RW lock priority promotion */ + +#define disLkType 0x80000000 /* Disable type checking */ +#define disLktypeb 0 +#define disLkThread 0x40000000 /* Disable ownership checking */ +#define disLkThreadb 1 +#define enaLkExtStck 0x20000000 /* Enable extended backtrace */ +#define enaLkExtStckb 2 +#define disLkMyLck 0x10000000 /* Disable recursive lock dectection */ +#define disLkMyLckb 3 + +#endif + +#ifdef MACH_KERNEL_PRIVATE +typedef struct { + struct hslock hwlock; + uintptr_t type; +} lck_spin_t; + +#define lck_spin_data hwlock.lock_data + +#define LCK_SPIN_TAG_DESTROYED 0xdead /* lock marked as Destroyed */ + +#define LCK_SPIN_TYPE 0x00000011 + +#else +#ifdef KERNEL_PRIVATE + +typedef struct { + uintptr_t opaque[2]; +} lck_spin_t; + +#else +typedef struct __lck_spin_t__ lck_spin_t; +#endif // KERNEL_PRIVATE +#endif // MACH_KERNEL_PRIVATE + +#ifdef MACH_KERNEL_PRIVATE +typedef struct _lck_mtx_ { + union { + + uintptr_t lck_mtx_data; /* Thread pointer plus lock bits */ + uintptr_t lck_mtx_tag; /* Tag for type */ + }; /* arm: 4 arm64: 8 */ + union { + struct { + uint16_t lck_mtx_waiters;/* Number of waiters */ + uint8_t lck_mtx_pri; /* Priority to inherit */ + uint8_t lck_mtx_type; /* Type */ + }; + struct { + struct _lck_mtx_ext_ *lck_mtx_ptr; /* Indirect pointer */ + }; + }; /* arm: 4 arm64: 8 */ +} lck_mtx_t; /* arm: 8 arm64: 16 */ + +/* Shared between mutex and read-write locks */ +#define LCK_ILOCK_BIT 0 +#define ARM_LCK_WAITERS_BIT 1 +#define LCK_ILOCK (1 << LCK_ILOCK_BIT) +#define ARM_LCK_WAITERS (1 << ARM_LCK_WAITERS_BIT) + +#define LCK_MTX_TYPE 0x22 /* lock type */ + +#define LCK_MTX_TAG_INDIRECT 0x00001007 /* lock marked as Indirect */ +#define LCK_MTX_TAG_DESTROYED 0x00002007 /* lock marked as Destroyed */ + +#define LCK_FRAMES_MAX 8 + +extern uint64_t MutexSpin; + +typedef struct { + unsigned int type; + vm_offset_t stack[LCK_FRAMES_MAX]; + vm_offset_t thread; +} lck_mtx_deb_t; + +#define MUTEX_TAG 0x4d4d + +typedef struct { + unsigned int lck_mtx_stat_data; +} lck_mtx_stat_t; + +typedef struct _lck_mtx_ext_ { + lck_mtx_t lck_mtx; /* arm: 12 arm64: 24 */ + struct _lck_grp_ *lck_mtx_grp; /* arm: 4 arm64: 8 */ + unsigned int lck_mtx_attr; /* arm: 4 arm64: 4 */ + lck_mtx_stat_t lck_mtx_stat; /* arm: 4 arm64: 4 */ + lck_mtx_deb_t lck_mtx_deb; /* arm: 40 arm64: 80 */ +} lck_mtx_ext_t; /* arm: 64 arm64: 120 */ + +#define LCK_MTX_ATTR_DEBUG 0x1 +#define LCK_MTX_ATTR_DEBUGb 31 +#define LCK_MTX_ATTR_STAT 0x2 +#define LCK_MTX_ATTR_STATb 30 + +#define LCK_MTX_EVENT(lck) ((event_t)(((unsigned int*)(lck))+((sizeof(lck_mtx_t)-1)/sizeof(unsigned int)))) +#define LCK_EVENT_TO_MUTEX(event) ((lck_mtx_t *)(uintptr_t)(((unsigned int *)(event)) - ((sizeof(lck_mtx_t)-1)/sizeof(unsigned int)))) + +#else +#ifdef KERNEL_PRIVATE +typedef struct { + uintptr_t opaque[2]; +} lck_mtx_t; + +typedef struct { +#if defined(__arm64__) + unsigned long opaque[16]; +#else /* __arm__ */ + unsigned int opaque[16]; +#endif +} lck_mtx_ext_t; + +#else +typedef struct __lck_mtx_t__ lck_mtx_t; +#endif +#endif + +#ifdef MACH_KERNEL_PRIVATE + +typedef union { + struct { + uint16_t shared_count; /* No. of shared granted request */ + uint16_t interlock: 1, /* Interlock */ + priv_excl: 1, /* priority for Writer */ + want_upgrade: 1, /* Read-to-write upgrade waiting */ + want_excl: 1, /* Writer is waiting, or locked for write */ + r_waiting: 1, /* Someone is sleeping on lock */ + w_waiting: 1, /* Writer is sleeping on lock */ + can_sleep: 1, /* Can attempts to lock go to sleep? */ + _pad2: 8, /* padding */ + tag_valid: 1; /* Field is actually a tag, not a bitfield */ +#if __arm64__ + uint32_t _pad4; +#endif + }; + struct { + uint32_t data; /* Single word version of bitfields and shared count */ +#if __arm64__ + uint32_t lck_rw_pad4; +#endif + }; +} lck_rw_word_t; + +typedef struct { + lck_rw_word_t word; + thread_t lck_rw_owner; +} lck_rw_t; /* arm: 8 arm64: 16 */ + +#define lck_rw_shared_count word.shared_count +#define lck_rw_interlock word.interlock +#define lck_rw_priv_excl word.priv_excl +#define lck_rw_want_upgrade word.want_upgrade +#define lck_rw_want_excl word.want_excl +#define lck_r_waiting word.r_waiting +#define lck_w_waiting word.w_waiting +#define lck_rw_can_sleep word.can_sleep +#define lck_rw_data word.data +// tag and data reference the same memory. When the tag_valid bit is set, +// the data word should be treated as a tag instead of a bitfield. +#define lck_rw_tag_valid word.tag_valid +#define lck_rw_tag word.data + +#define LCK_RW_SHARED_READER_OFFSET 0 +#define LCK_RW_INTERLOCK_BIT 16 +#define LCK_RW_PRIV_EXCL_BIT 17 +#define LCK_RW_WANT_UPGRADE_BIT 18 +#define LCK_RW_WANT_EXCL_BIT 19 +#define LCK_RW_R_WAITING_BIT 20 +#define LCK_RW_W_WAITING_BIT 21 +#define LCK_RW_CAN_SLEEP_BIT 22 +// 23-30 +#define LCK_RW_TAG_VALID_BIT 31 + +#define LCK_RW_INTERLOCK (1 << LCK_RW_INTERLOCK_BIT) +#define LCK_RW_R_WAITING (1 << LCK_RW_R_WAITING_BIT) +#define LCK_RW_W_WAITING (1 << LCK_RW_W_WAITING_BIT) +#define LCK_RW_WANT_UPGRADE (1 << LCK_RW_WANT_UPGRADE_BIT) +#define LCK_RW_WANT_EXCL (1 << LCK_RW_WANT_EXCL_BIT) +#define LCK_RW_TAG_VALID (1 << LCK_RW_TAG_VALID_BIT) +#define LCK_RW_PRIV_EXCL (1 << LCK_RW_PRIV_EXCL_BIT) +#define LCK_RW_SHARED_MASK (0xffff << LCK_RW_SHARED_READER_OFFSET) +#define LCK_RW_SHARED_READER (0x1 << LCK_RW_SHARED_READER_OFFSET) + +#define LCK_RW_TAG_DESTROYED ((LCK_RW_TAG_VALID | 0xdddddeadu)) /* lock marked as Destroyed */ + +#define LCK_RW_WRITER_EVENT(lck) (event_t)((uintptr_t)(lck)+1) +#define LCK_RW_READER_EVENT(lck) (event_t)((uintptr_t)(lck)+2) +#define WRITE_EVENT_TO_RWLOCK(event) ((lck_rw_t *)((uintptr_t)(event)-1)) +#define READ_EVENT_TO_RWLOCK(event) ((lck_rw_t *)((uintptr_t)(event)-2)) + +#if __ARM_ENABLE_WFE_ + +#define wait_for_event() __builtin_arm_wfe() +#if __arm__ +#define set_event() do{__builtin_arm_dsb(DSB_ISHST);__builtin_arm_sev();}while(0) +#define LOCK_SNOOP_SPINS 4 +#else +#define set_event() do{}while(0) // arm64 sev is implicit in stlxr +#define LOCK_SNOOP_SPINS 0x300 +#endif + +#else + +#define wait_for_event() __builtin_arm_clrex() +#define set_event() do{}while(0) +#define LOCK_SNOOP_SPINS 0x300 + +#endif // __ARM_ENABLE_WFE_ + +#if LOCK_PRIVATE + +#define LOCK_PANIC_TIMEOUT 0xc00000 // 12.5 m ticks = 250ms with 24MHz OSC + +#define LOCK_TRY_DISABLE_INT 1 // Disable interrupts for a quick acquire attempt + +#define PLATFORM_LCK_ILOCK LCK_ILOCK + + +/* + * Lock state to thread pointer + * Clear the bottom bits + */ +#define LCK_MTX_STATE_TO_THREAD(s) (thread_t)(s & ~(LCK_ILOCK | ARM_LCK_WAITERS)) +/* + * Thread pointer to lock state + * arm thread pointers are aligned such that the bottom two bits are clear + */ +#define LCK_MTX_THREAD_TO_STATE(t) ((uintptr_t)t) +/* + * Thread pointer mask + */ +#define LCK_MTX_THREAD_MASK (~(uintptr_t)(LCK_ILOCK | ARM_LCK_WAITERS)) + +#define disable_preemption_for_thread(t) ((volatile thread_t)t)->machine.preemption_count++ + + +__unused static void disable_interrupts_noread(void) +{ +#if __arm__ + __asm__ volatile ("cpsid if" ::: "memory"); // Mask IRQ FIQ +#else + __builtin_arm_wsr64("DAIFSet", (DAIFSC_IRQF | DAIFSC_FIQF)); // Mask IRQ FIQ +#endif +} + +__unused static inline long get_interrupts(void) +{ + long state; + +#if __arm__ + __asm__ volatile ("mrs %[state], cpsr" :[state] "=r" (state)); // Read cpsr +#else + state = __builtin_arm_rsr64("DAIF"); // Read interrupt state +#endif + return state; +} + +__unused static inline long disable_interrupts(void) +{ + long state; + + state = get_interrupts(); // Get previous state + disable_interrupts_noread(); // Disable + return state; +} + +__unused static inline void restore_interrupts(long state) +{ +#if __arm__ + __asm__ volatile ("msr cpsr, %[state]" :: [state] "r" (state) : "cc", "memory"); // Restore CPSR +#elif __arm64__ + __builtin_arm_wsr64("DAIF", state); // Restore masks +#endif +} + +#endif // LOCK_PRIVATE + +#else +#ifdef KERNEL_PRIVATE +typedef struct { + uintptr_t opaque[2]; +} lck_rw_t; +#else +typedef struct __lck_rw_t__ lck_rw_t; +#endif +#endif + +#endif /* _ARM_LOCKS_H_ */ diff --git a/osfmk/arm/locks_arm.c b/osfmk/arm/locks_arm.c new file mode 100644 index 000000000..941ec38fc --- /dev/null +++ b/osfmk/arm/locks_arm.c @@ -0,0 +1,2882 @@ +/* + * Copyright (c) 2007-2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System Copyright (c) 1991,1990,1989,1988,1987 Carnegie + * Mellon University All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright notice + * and this permission notice appear in all copies of the software, + * derivative works or modified versions, and any portions thereof, and that + * both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" CONDITION. + * CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR ANY DAMAGES + * WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science Carnegie Mellon University Pittsburgh PA + * 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon the + * rights to redistribute these changes. + */ +/* + * File: kern/lock.c + * Author: Avadis Tevanian, Jr., Michael Wayne Young + * Date: 1985 + * + * Locking primitives implementation + */ + +#define ATOMIC_PRIVATE 1 +#define LOCK_PRIVATE 1 + +#include <mach_ldebug.h> + +#include <kern/kalloc.h> +#include <kern/locks.h> +#include <kern/misc_protos.h> +#include <kern/thread.h> +#include <kern/processor.h> +#include <kern/sched_prim.h> +#include <kern/xpr.h> +#include <kern/debug.h> +#include <kern/kcdata.h> +#include <string.h> + +#include <arm/cpu_data_internal.h> +#include <arm/proc_reg.h> +#include <arm/smp.h> +#include <machine/atomic.h> +#include <machine/machine_cpu.h> + +#include <sys/kdebug.h> + +/* + * We need only enough declarations from the BSD-side to be able to + * test if our probe is active, and to call __dtrace_probe(). Setting + * NEED_DTRACE_DEFS gets a local copy of those definitions pulled in. + */ +#if CONFIG_DTRACE +#define NEED_DTRACE_DEFS +#include <../bsd/sys/lockstat.h> + +#define DTRACE_RW_SHARED 0x0 //reader +#define DTRACE_RW_EXCL 0x1 //writer +#define DTRACE_NO_FLAG 0x0 //not applicable + +#endif /* CONFIG_DTRACE */ + +#define LCK_RW_LCK_EXCLUSIVE_CODE 0x100 +#define LCK_RW_LCK_EXCLUSIVE1_CODE 0x101 +#define LCK_RW_LCK_SHARED_CODE 0x102 +#define LCK_RW_LCK_SH_TO_EX_CODE 0x103 +#define LCK_RW_LCK_SH_TO_EX1_CODE 0x104 +#define LCK_RW_LCK_EX_TO_SH_CODE 0x105 + + +#define ANY_LOCK_DEBUG (USLOCK_DEBUG || LOCK_DEBUG || MUTEX_DEBUG) + +// Panic in tests that check lock usage correctness +// These are undesirable when in a panic or a debugger is runnning. +#define LOCK_CORRECTNESS_PANIC() (kernel_debugger_entry_count == 0) + +unsigned int LcksOpts = 0; + +#if CONFIG_DTRACE && __SMP__ +extern uint64_t dtrace_spin_threshold; +#endif + +/* Forwards */ + + +#if USLOCK_DEBUG +/* + * Perform simple lock checks. + */ +int uslock_check = 1; +int max_lock_loops = 100000000; +decl_simple_lock_data(extern, printf_lock) +decl_simple_lock_data(extern, panic_lock) +#endif /* USLOCK_DEBUG */ + +extern unsigned int not_in_kdp; + +/* + * We often want to know the addresses of the callers + * of the various lock routines. However, this information + * is only used for debugging and statistics. + */ +typedef void *pc_t; +#define INVALID_PC ((void *) VM_MAX_KERNEL_ADDRESS) +#define INVALID_THREAD ((void *) VM_MAX_KERNEL_ADDRESS) + +#ifdef lint +/* + * Eliminate lint complaints about unused local pc variables. + */ +#define OBTAIN_PC(pc,l) ++pc +#else /* lint */ +#define OBTAIN_PC(pc,l) +#endif /* lint */ + + +/* + * Portable lock package implementation of usimple_locks. + */ + +#if USLOCK_DEBUG +#define USLDBG(stmt) stmt + void usld_lock_init(usimple_lock_t, unsigned short); + void usld_lock_pre(usimple_lock_t, pc_t); + void usld_lock_post(usimple_lock_t, pc_t); + void usld_unlock(usimple_lock_t, pc_t); + void usld_lock_try_pre(usimple_lock_t, pc_t); + void usld_lock_try_post(usimple_lock_t, pc_t); + int usld_lock_common_checks(usimple_lock_t, const char *); +#else /* USLOCK_DEBUG */ +#define USLDBG(stmt) +#endif /* USLOCK_DEBUG */ + +/* + * Owner thread pointer when lock held in spin mode + */ +#define LCK_MTX_SPIN_TAG 0xfffffff0 + + +#define interlock_lock(lock) hw_lock_bit ((hw_lock_bit_t*)(&(lock)->lck_mtx_data), LCK_ILOCK_BIT) +#define interlock_try(lock) hw_lock_bit_try((hw_lock_bit_t*)(&(lock)->lck_mtx_data), LCK_ILOCK_BIT) +#define interlock_unlock(lock) hw_unlock_bit ((hw_lock_bit_t*)(&(lock)->lck_mtx_data), LCK_ILOCK_BIT) +#define lck_rw_ilk_lock(lock) hw_lock_bit ((hw_lock_bit_t*)(&(lock)->lck_rw_tag), LCK_RW_INTERLOCK_BIT) +#define lck_rw_ilk_unlock(lock) hw_unlock_bit((hw_lock_bit_t*)(&(lock)->lck_rw_tag), LCK_RW_INTERLOCK_BIT) + +#define memory_barrier() __c11_atomic_thread_fence(memory_order_acq_rel_smp) +#define load_memory_barrier() __c11_atomic_thread_fence(memory_order_acquire_smp) +#define store_memory_barrier() __c11_atomic_thread_fence(memory_order_release_smp) + +// Enforce program order of loads and stores. +#define ordered_load(target, type) \ + __c11_atomic_load((_Atomic type *)(target), memory_order_relaxed) +#define ordered_store(target, type, value) \ + __c11_atomic_store((_Atomic type *)(target), value, memory_order_relaxed) + +#define ordered_load_mtx(lock) ordered_load(&(lock)->lck_mtx_data, uintptr_t) +#define ordered_store_mtx(lock, value) ordered_store(&(lock)->lck_mtx_data, uintptr_t, (value)) +#define ordered_load_rw(lock) ordered_load(&(lock)->lck_rw_data, uint32_t) +#define ordered_store_rw(lock, value) ordered_store(&(lock)->lck_rw_data, uint32_t, (value)) +#define ordered_load_rw_owner(lock) ordered_load(&(lock)->lck_rw_owner, thread_t) +#define ordered_store_rw_owner(lock, value) ordered_store(&(lock)->lck_rw_owner, thread_t, (value)) +#define ordered_load_hw(lock) ordered_load(&(lock)->lock_data, uintptr_t) +#define ordered_store_hw(lock, value) ordered_store(&(lock)->lock_data, uintptr_t, (value)) +#define ordered_load_bit(lock) ordered_load((lock), uint32_t) +#define ordered_store_bit(lock, value) ordered_store((lock), uint32_t, (value)) + + +// Prevent the compiler from reordering memory operations around this +#define compiler_memory_fence() __asm__ volatile ("" ::: "memory") + +#define LOCK_PANIC_TIMEOUT 0xc00000 +#define NOINLINE __attribute__((noinline)) + + +#if __arm__ +#define interrupts_disabled(mask) (mask & PSR_INTMASK) +#else +#define interrupts_disabled(mask) (mask & DAIF_IRQF) +#endif + + +#if __arm__ +#define enable_fiq() __asm__ volatile ("cpsie f" ::: "memory"); +#define enable_interrupts() __asm__ volatile ("cpsie if" ::: "memory"); +#endif + +/* + * Forward declarations + */ + +static void lck_rw_lock_shared_gen(lck_rw_t *lck); +static void lck_rw_lock_exclusive_gen(lck_rw_t *lck); +static boolean_t lck_rw_lock_shared_to_exclusive_success(lck_rw_t *lck); +static boolean_t lck_rw_lock_shared_to_exclusive_failure(lck_rw_t *lck, uint32_t prior_lock_state); +static void lck_rw_lock_exclusive_to_shared_gen(lck_rw_t *lck, uint32_t prior_lock_state); +static lck_rw_type_t lck_rw_done_gen(lck_rw_t *lck, uint32_t prior_lock_state); +void lck_rw_clear_promotions_x86(thread_t thread); +static boolean_t lck_rw_grab(lck_rw_t *lock, int mode, boolean_t wait); + +/* + * atomic exchange API is a low level abstraction of the operations + * to atomically read, modify, and write a pointer. This abstraction works + * for both Intel and ARMv8.1 compare and exchange atomic instructions as + * well as the ARM exclusive instructions. + * + * atomic_exchange_begin() - begin exchange and retrieve current value + * atomic_exchange_complete() - conclude an exchange + * atomic_exchange_abort() - cancel an exchange started with atomic_exchange_begin() + */ +static uint32_t +atomic_exchange_begin32(uint32_t *target, uint32_t *previous, enum memory_order ord) +{ + uint32_t val; + + val = load_exclusive32(target, ord); + *previous = val; + return val; +} + +static boolean_t +atomic_exchange_complete32(uint32_t *target, uint32_t previous, uint32_t newval, enum memory_order ord) +{ + (void)previous; // Previous not needed, monitor is held + return store_exclusive32(target, newval, ord); +} + +static void +atomic_exchange_abort(void) +{ + clear_exclusive(); +} + +static boolean_t +atomic_test_and_set32(uint32_t *target, uint32_t test_mask, uint32_t set_mask, enum memory_order ord, boolean_t wait) +{ + uint32_t value, prev; + + for ( ; ; ) { + value = atomic_exchange_begin32(target, &prev, ord); + if (value & test_mask) { + if (wait) + wait_for_event(); // Wait with monitor held + else + atomic_exchange_abort(); // Clear exclusive monitor + return FALSE; + } + value |= set_mask; + if (atomic_exchange_complete32(target, prev, value, ord)) + return TRUE; + } +} + +void _disable_preemption(void) +{ + thread_t thread = current_thread(); + unsigned int count; + + count = thread->machine.preemption_count + 1; + ordered_store(&thread->machine.preemption_count, unsigned int, count); +} + +void _enable_preemption(void) +{ + thread_t thread = current_thread(); + long state; + unsigned int count; +#if __arm__ +#define INTERRUPT_MASK PSR_IRQF +#else // __arm__ +#define INTERRUPT_MASK DAIF_IRQF +#endif // __arm__ + + count = thread->machine.preemption_count; + if (count == 0) + panic("Preemption count negative"); // Count will go negative when released + count--; + if (count > 0) + goto update_count; // Preemption is still disabled, just update + state = get_interrupts(); // Get interrupt state + if (state & INTERRUPT_MASK) + goto update_count; // Interrupts are already masked, can't take AST here + + disable_interrupts_noread(); // Disable interrupts + ordered_store(&thread->machine.preemption_count, unsigned int, count); + if (thread->machine.CpuDatap->cpu_pending_ast & AST_URGENT) { +#if __arm__ +#if __ARM_USER_PROTECT__ + uintptr_t up = arm_user_protect_begin(thread); +#endif // __ARM_USER_PROTECT__ + enable_fiq(); +#endif // __arm__ + ast_taken_kernel(); // Handle urgent AST +#if __arm__ +#if __ARM_USER_PROTECT__ + arm_user_protect_end(thread, up, TRUE); +#endif // __ARM_USER_PROTECT__ + enable_interrupts(); + return; // Return early on arm only due to FIQ enabling +#endif // __arm__ + } + restore_interrupts(state); // Enable interrupts + return; + +update_count: + ordered_store(&thread->machine.preemption_count, unsigned int, count); + return; +} + +int get_preemption_level(void) +{ + return current_thread()->machine.preemption_count; +} + +/* Forward declarations for unexported functions that are used externally */ +void hw_lock_bit(hw_lock_bit_t *lock, unsigned int bit); +void hw_unlock_bit(hw_lock_bit_t *lock, unsigned int bit); + +#if __SMP__ +static unsigned int +hw_lock_bit_to_contended(hw_lock_bit_t *lock, uint32_t mask, uint32_t timeout); +#endif + +unsigned int +hw_lock_bit_to(hw_lock_bit_t *lock, unsigned int bit, uint32_t timeout) +{ + unsigned int success = 0; + uint32_t mask = (1 << bit); +#if !__SMP__ + uint32_t state; +#endif + + _disable_preemption(); +#if __SMP__ + if (__improbable(!atomic_test_and_set32(lock, mask, mask, memory_order_acquire, FALSE))) + success = hw_lock_bit_to_contended(lock, mask, timeout); + else + success = 1; +#else // __SMP__ + (void)timeout; + state = ordered_load_bit(lock); + if (!(mask & state)) { + ordered_store_bit(lock, state | mask); + success = 1; + } +#endif // __SMP__ + +#if CONFIG_DTRACE + if (success) + LOCKSTAT_RECORD(LS_LCK_SPIN_LOCK_ACQUIRE, lock, bit); +#endif + + return success; +} + +#if __SMP__ +static unsigned int NOINLINE +hw_lock_bit_to_contended(hw_lock_bit_t *lock, uint32_t mask, uint32_t timeout) +{ + uint64_t end = 0; + int i; +#if CONFIG_DTRACE + uint64_t begin; + boolean_t dtrace_enabled = lockstat_probemap[LS_LCK_SPIN_LOCK_SPIN] != 0; + if (__improbable(dtrace_enabled)) + begin = mach_absolute_time(); +#endif + for ( ; ; ) { + for (i = 0; i < LOCK_SNOOP_SPINS; i++) { + // Always load-exclusive before wfe + // This grabs the monitor and wakes up on a release event + if (atomic_test_and_set32(lock, mask, mask, memory_order_acquire, TRUE)) { + goto end; + } + } + if (end == 0) + end = ml_get_timebase() + timeout; + else if (ml_get_timebase() >= end) + break; + } + return 0; +end: +#if CONFIG_DTRACE + if (__improbable(dtrace_enabled)) { + uint64_t spintime = mach_absolute_time() - begin; + if (spintime > dtrace_spin_threshold) + LOCKSTAT_RECORD2(LS_LCK_SPIN_LOCK_SPIN, lock, spintime, mask); + } +#endif + return 1; +} +#endif // __SMP__ + +void +hw_lock_bit(hw_lock_bit_t *lock, unsigned int bit) +{ + if (hw_lock_bit_to(lock, bit, LOCK_PANIC_TIMEOUT)) + return; +#if __SMP__ + panic("hw_lock_bit(): timed out (%p)", lock); +#else + panic("hw_lock_bit(): interlock held (%p)", lock); +#endif +} + +unsigned int +hw_lock_bit_try(hw_lock_bit_t *lock, unsigned int bit) +{ + long intmask; + uint32_t mask = (1 << bit); +#if !__SMP__ + uint32_t state; +#endif + boolean_t success = FALSE; + + intmask = disable_interrupts(); +#if __SMP__ + // TODO: consider weak (non-looping) atomic test-and-set + success = atomic_test_and_set32(lock, mask, mask, memory_order_acquire, FALSE); +#else + state = ordered_load_bit(lock); + if (!(mask & state)) { + ordered_store_bit(lock, state | mask); + success = TRUE; + } +#endif // __SMP__ + if (success) + disable_preemption(); + restore_interrupts(intmask); + +#if CONFIG_DTRACE + if (success) + LOCKSTAT_RECORD(LS_LCK_SPIN_LOCK_ACQUIRE, lock, bit); +#endif + + return success; +} + +/* + * Routine: hw_unlock_bit + * + * Release spin-lock. The second parameter is the bit number to test and set. + * Decrement the preemption level. + */ +void +hw_unlock_bit(hw_lock_bit_t *lock, unsigned int bit) +{ + uint32_t mask = (1 << bit); +#if !__SMP__ + uint32_t state; +#endif + +#if __SMP__ + __c11_atomic_fetch_and((_Atomic uint32_t *)lock, ~mask, memory_order_release); + set_event(); +#else // __SMP__ + state = ordered_load_bit(lock); + ordered_store_bit(lock, state & ~mask); +#endif // __SMP__ +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_SPIN_UNLOCK_RELEASE, lock, bit); +#endif + enable_preemption(); +} + + +/* + * Routine: lck_spin_alloc_init + */ +lck_spin_t * +lck_spin_alloc_init( + lck_grp_t * grp, + lck_attr_t * attr) +{ + lck_spin_t *lck; + + if ((lck = (lck_spin_t *) kalloc(sizeof(lck_spin_t))) != 0) + lck_spin_init(lck, grp, attr); + + return (lck); +} + +/* + * Routine: lck_spin_free + */ +void +lck_spin_free( + lck_spin_t * lck, + lck_grp_t * grp) +{ + lck_spin_destroy(lck, grp); + kfree((void *) lck, sizeof(lck_spin_t)); +} + +/* + * Routine: lck_spin_init + */ +void +lck_spin_init( + lck_spin_t * lck, + lck_grp_t * grp, + __unused lck_attr_t * attr) +{ + hw_lock_init(&lck->hwlock); + lck->type = LCK_SPIN_TYPE; + lck_grp_reference(grp); + lck_grp_lckcnt_incr(grp, LCK_TYPE_SPIN); + store_memory_barrier(); +} + +/* + * arm_usimple_lock is a lck_spin_t without a group or attributes + */ +void inline +arm_usimple_lock_init(simple_lock_t lck, __unused unsigned short initial_value) +{ + lck->type = LCK_SPIN_TYPE; + hw_lock_init(&lck->hwlock); + store_memory_barrier(); +} + + +/* + * Routine: lck_spin_lock + */ +void +lck_spin_lock(lck_spin_t *lock) +{ +#if DEVELOPMENT || DEBUG + if (lock->type != LCK_SPIN_TYPE) + panic("Invalid spinlock %p", lock); +#endif // DEVELOPMENT || DEBUG + hw_lock_lock(&lock->hwlock); +} + +/* + * Routine: lck_spin_try_lock + */ +int +lck_spin_try_lock(lck_spin_t *lock) +{ + return hw_lock_try(&lock->hwlock); +} + +/* + * Routine: lck_spin_unlock + */ +void +lck_spin_unlock(lck_spin_t *lock) +{ +#if DEVELOPMENT || DEBUG + if ((LCK_MTX_STATE_TO_THREAD(lock->lck_spin_data) != current_thread()) && LOCK_CORRECTNESS_PANIC()) + panic("Spinlock not owned by thread %p = %lx", lock, lock->lck_spin_data); + if (lock->type != LCK_SPIN_TYPE) + panic("Invalid spinlock type %p", lock); +#endif // DEVELOPMENT || DEBUG + hw_lock_unlock(&lock->hwlock); +} + +/* + * Routine: lck_spin_destroy + */ +void +lck_spin_destroy( + lck_spin_t * lck, + lck_grp_t * grp) +{ + if (lck->lck_spin_data == LCK_SPIN_TAG_DESTROYED) + return; + lck->lck_spin_data = LCK_SPIN_TAG_DESTROYED; + lck_grp_lckcnt_decr(grp, LCK_TYPE_SPIN); + lck_grp_deallocate(grp); +} + +/* + * Routine: kdp_lck_spin_is_acquired + * NOT SAFE: To be used only by kernel debugger to avoid deadlock. + */ +boolean_t +kdp_lck_spin_is_acquired(lck_spin_t *lck) { + if (not_in_kdp) { + panic("panic: spinlock acquired check done outside of kernel debugger"); + } + return ((lck->lck_spin_data & ~LCK_SPIN_TAG_DESTROYED) != 0) ? TRUE:FALSE; +} + +/* + * Initialize a usimple_lock. + * + * No change in preemption state. + */ +void +usimple_lock_init( + usimple_lock_t l, + unsigned short tag) +{ +#ifndef MACHINE_SIMPLE_LOCK + USLDBG(usld_lock_init(l, tag)); + hw_lock_init(&l->lck_spin_data); +#else + simple_lock_init((simple_lock_t) l, tag); +#endif +} + + +/* + * Acquire a usimple_lock. + * + * Returns with preemption disabled. Note + * that the hw_lock routines are responsible for + * maintaining preemption state. + */ +void +usimple_lock( + usimple_lock_t l) +{ +#ifndef MACHINE_SIMPLE_LOCK + pc_t pc; + + OBTAIN_PC(pc, l); + USLDBG(usld_lock_pre(l, pc)); + + if (!hw_lock_to(&l->lck_spin_data, LockTimeOut)) /* Try to get the lock + * with a timeout */ + panic("simple lock deadlock detection - l=%p, cpu=%d, ret=%p", &l, cpu_number(), pc); + + USLDBG(usld_lock_post(l, pc)); +#else + simple_lock((simple_lock_t) l); +#endif +} + + +extern void sync(void); + +/* + * Release a usimple_lock. + * + * Returns with preemption enabled. Note + * that the hw_lock routines are responsible for + * maintaining preemption state. + */ +void +usimple_unlock( + usimple_lock_t l) +{ +#ifndef MACHINE_SIMPLE_LOCK + pc_t pc; + + OBTAIN_PC(pc, l); + USLDBG(usld_unlock(l, pc)); + sync(); + hw_lock_unlock(&l->lck_spin_data); +#else + simple_unlock((simple_lock_t) l); +#endif +} + + +/* + * Conditionally acquire a usimple_lock. + * + * On success, returns with preemption disabled. + * On failure, returns with preemption in the same state + * as when first invoked. Note that the hw_lock routines + * are responsible for maintaining preemption state. + * + * XXX No stats are gathered on a miss; I preserved this + * behavior from the original assembly-language code, but + * doesn't it make sense to log misses? XXX + */ +unsigned int +usimple_lock_try( + usimple_lock_t l) +{ +#ifndef MACHINE_SIMPLE_LOCK + pc_t pc; + unsigned int success; + + OBTAIN_PC(pc, l); + USLDBG(usld_lock_try_pre(l, pc)); + if ((success = hw_lock_try(&l->lck_spin_data))) { + USLDBG(usld_lock_try_post(l, pc)); + } + return success; +#else + return (simple_lock_try((simple_lock_t) l)); +#endif +} + +#if USLOCK_DEBUG +/* + * States of a usimple_lock. The default when initializing + * a usimple_lock is setting it up for debug checking. + */ +#define USLOCK_CHECKED 0x0001 /* lock is being checked */ +#define USLOCK_TAKEN 0x0002 /* lock has been taken */ +#define USLOCK_INIT 0xBAA0 /* lock has been initialized */ +#define USLOCK_INITIALIZED (USLOCK_INIT|USLOCK_CHECKED) +#define USLOCK_CHECKING(l) (uslock_check && \ + ((l)->debug.state & USLOCK_CHECKED)) + +/* + * Trace activities of a particularly interesting lock. + */ +void usl_trace(usimple_lock_t, int, pc_t, const char *); + + +/* + * Initialize the debugging information contained + * in a usimple_lock. + */ +void +usld_lock_init( + usimple_lock_t l, + __unused unsigned short tag) +{ + if (l == USIMPLE_LOCK_NULL) + panic("lock initialization: null lock pointer"); + l->lock_type = USLOCK_TAG; + l->debug.state = uslock_check ? USLOCK_INITIALIZED : 0; + l->debug.lock_cpu = l->debug.unlock_cpu = 0; + l->debug.lock_pc = l->debug.unlock_pc = INVALID_PC; + l->debug.lock_thread = l->debug.unlock_thread = INVALID_THREAD; + l->debug.duration[0] = l->debug.duration[1] = 0; + l->debug.unlock_cpu = l->debug.unlock_cpu = 0; + l->debug.unlock_pc = l->debug.unlock_pc = INVALID_PC; + l->debug.unlock_thread = l->debug.unlock_thread = INVALID_THREAD; +} + + +/* + * These checks apply to all usimple_locks, not just + * those with USLOCK_CHECKED turned on. + */ +int +usld_lock_common_checks( + usimple_lock_t l, + const char *caller) +{ + if (l == USIMPLE_LOCK_NULL) + panic("%s: null lock pointer", caller); + if (l->lock_type != USLOCK_TAG) + panic("%s: 0x%x is not a usimple lock", caller, (integer_t) l); + if (!(l->debug.state & USLOCK_INIT)) + panic("%s: 0x%x is not an initialized lock", + caller, (integer_t) l); + return USLOCK_CHECKING(l); +} + + +/* + * Debug checks on a usimple_lock just before attempting + * to acquire it. + */ +/* ARGSUSED */ +void +usld_lock_pre( + usimple_lock_t l, + pc_t pc) +{ + const char *caller = "usimple_lock"; + + + if (!usld_lock_common_checks(l, caller)) + return; + + /* + * Note that we have a weird case where we are getting a lock when we are] + * in the process of putting the system to sleep. We are running with no + * current threads, therefore we can't tell if we are trying to retake a lock + * we have or someone on the other processor has it. Therefore we just + * ignore this test if the locking thread is 0. + */ + + if ((l->debug.state & USLOCK_TAKEN) && l->debug.lock_thread && + l->debug.lock_thread == (void *) current_thread()) { + printf("%s: lock 0x%x already locked (at %p) by", + caller, (integer_t) l, l->debug.lock_pc); + printf(" current thread %p (new attempt at pc %p)\n", + l->debug.lock_thread, pc); + panic("%s", caller); + } + mp_disable_preemption(); + usl_trace(l, cpu_number(), pc, caller); + mp_enable_preemption(); +} + + +/* + * Debug checks on a usimple_lock just after acquiring it. + * + * Pre-emption has been disabled at this point, + * so we are safe in using cpu_number. + */ +void +usld_lock_post( + usimple_lock_t l, + pc_t pc) +{ + int mycpu; + const char *caller = "successful usimple_lock"; + + + if (!usld_lock_common_checks(l, caller)) + return; + + if (!((l->debug.state & ~USLOCK_TAKEN) == USLOCK_INITIALIZED)) + panic("%s: lock 0x%x became uninitialized", + caller, (integer_t) l); + if ((l->debug.state & USLOCK_TAKEN)) + panic("%s: lock 0x%x became TAKEN by someone else", + caller, (integer_t) l); + + mycpu = cpu_number(); + l->debug.lock_thread = (void *) current_thread(); + l->debug.state |= USLOCK_TAKEN; + l->debug.lock_pc = pc; + l->debug.lock_cpu = mycpu; + + usl_trace(l, mycpu, pc, caller); +} + + +/* + * Debug checks on a usimple_lock just before + * releasing it. Note that the caller has not + * yet released the hardware lock. + * + * Preemption is still disabled, so there's + * no problem using cpu_number. + */ +void +usld_unlock( + usimple_lock_t l, + pc_t pc) +{ + int mycpu; + const char *caller = "usimple_unlock"; + + + if (!usld_lock_common_checks(l, caller)) + return; + + mycpu = cpu_number(); + + if (!(l->debug.state & USLOCK_TAKEN)) + panic("%s: lock 0x%x hasn't been taken", + caller, (integer_t) l); + if (l->debug.lock_thread != (void *) current_thread()) + panic("%s: unlocking lock 0x%x, owned by thread %p", + caller, (integer_t) l, l->debug.lock_thread); + if (l->debug.lock_cpu != mycpu) { + printf("%s: unlocking lock 0x%x on cpu 0x%x", + caller, (integer_t) l, mycpu); + printf(" (acquired on cpu 0x%x)\n", l->debug.lock_cpu); + panic("%s", caller); + } + usl_trace(l, mycpu, pc, caller); + + l->debug.unlock_thread = l->debug.lock_thread; + l->debug.lock_thread = INVALID_PC; + l->debug.state &= ~USLOCK_TAKEN; + l->debug.unlock_pc = pc; + l->debug.unlock_cpu = mycpu; +} + + +/* + * Debug checks on a usimple_lock just before + * attempting to acquire it. + * + * Preemption isn't guaranteed to be disabled. + */ +void +usld_lock_try_pre( + usimple_lock_t l, + pc_t pc) +{ + const char *caller = "usimple_lock_try"; + + if (!usld_lock_common_checks(l, caller)) + return; + mp_disable_preemption(); + usl_trace(l, cpu_number(), pc, caller); + mp_enable_preemption(); +} + + +/* + * Debug checks on a usimple_lock just after + * successfully attempting to acquire it. + * + * Preemption has been disabled by the + * lock acquisition attempt, so it's safe + * to use cpu_number. + */ +void +usld_lock_try_post( + usimple_lock_t l, + pc_t pc) +{ + int mycpu; + const char *caller = "successful usimple_lock_try"; + + if (!usld_lock_common_checks(l, caller)) + return; + + if (!((l->debug.state & ~USLOCK_TAKEN) == USLOCK_INITIALIZED)) + panic("%s: lock 0x%x became uninitialized", + caller, (integer_t) l); + if ((l->debug.state & USLOCK_TAKEN)) + panic("%s: lock 0x%x became TAKEN by someone else", + caller, (integer_t) l); + + mycpu = cpu_number(); + l->debug.lock_thread = (void *) current_thread(); + l->debug.state |= USLOCK_TAKEN; + l->debug.lock_pc = pc; + l->debug.lock_cpu = mycpu; + + usl_trace(l, mycpu, pc, caller); +} + + +/* + * For very special cases, set traced_lock to point to a + * specific lock of interest. The result is a series of + * XPRs showing lock operations on that lock. The lock_seq + * value is used to show the order of those operations. + */ +usimple_lock_t traced_lock; +unsigned int lock_seq; + +void +usl_trace( + usimple_lock_t l, + int mycpu, + pc_t pc, + const char *op_name) +{ + if (traced_lock == l) { + XPR(XPR_SLOCK, + "seq %d, cpu %d, %s @ %x\n", + (integer_t) lock_seq, (integer_t) mycpu, + (integer_t) op_name, (integer_t) pc, 0); + lock_seq++; + } +} + + +#endif /* USLOCK_DEBUG */ + +/* + * The C portion of the shared/exclusive locks package. + */ + +/* + * compute the deadline to spin against when + * waiting for a change of state on a lck_rw_t + */ +#if __SMP__ +static inline uint64_t +lck_rw_deadline_for_spin(lck_rw_t *lck) +{ + lck_rw_word_t word; + + word.data = ordered_load_rw(lck); + if (word.can_sleep) { + if (word.r_waiting || word.w_waiting || (word.shared_count > machine_info.max_cpus)) { + /* + * there are already threads waiting on this lock... this + * implies that they have spun beyond their deadlines waiting for + * the desired state to show up so we will not bother spinning at this time... + * or + * the current number of threads sharing this lock exceeds our capacity to run them + * concurrently and since all states we're going to spin for require the rw_shared_count + * to be at 0, we'll not bother spinning since the latency for this to happen is + * unpredictable... + */ + return (mach_absolute_time()); + } + return (mach_absolute_time() + MutexSpin); + } else + return (mach_absolute_time() + (100000LL * 1000000000LL)); +} +#endif // __SMP__ + +static boolean_t +lck_rw_drain_status(lck_rw_t *lock, uint32_t status_mask, boolean_t wait __unused) +{ +#if __SMP__ + uint64_t deadline = 0; + uint32_t data; + + if (wait) + deadline = lck_rw_deadline_for_spin(lock); + + for ( ; ; ) { + data = load_exclusive32(&lock->lck_rw_data, memory_order_acquire_smp); + if ((data & status_mask) == 0) + break; + if (wait) + wait_for_event(); + else + clear_exclusive(); + if (!wait || (mach_absolute_time() >= deadline)) + return FALSE; + } + clear_exclusive(); + return TRUE; +#else + uint32_t data; + + data = ordered_load_rw(lock); + if ((data & status_mask) == 0) + return TRUE; + else + return FALSE; +#endif // __SMP__ +} + +/* + * Spin while interlock is held. + */ +static inline void +lck_rw_interlock_spin(lck_rw_t *lock) +{ +#if __SMP__ + uint32_t data; + + for ( ; ; ) { + data = load_exclusive32(&lock->lck_rw_data, memory_order_relaxed); + if (data & LCK_RW_INTERLOCK) + wait_for_event(); + else { + clear_exclusive(); + return; + } + } +#else + panic("lck_rw_interlock_spin(): Interlock locked %p %x", lock, lock->lck_rw_data); +#endif +} + +/* + * We disable interrupts while holding the RW interlock to prevent an + * interrupt from exacerbating hold time. + * Hence, local helper functions lck_interlock_lock()/lck_interlock_unlock(). + */ +static inline boolean_t +lck_interlock_lock(lck_rw_t *lck) +{ + boolean_t istate; + + istate = ml_set_interrupts_enabled(FALSE); + lck_rw_ilk_lock(lck); + return istate; +} + +static inline void +lck_interlock_unlock(lck_rw_t *lck, boolean_t istate) +{ + lck_rw_ilk_unlock(lck); + ml_set_interrupts_enabled(istate); +} + + +#define LCK_RW_GRAB_WANT 0 +#define LCK_RW_GRAB_SHARED 1 + +static boolean_t +lck_rw_grab(lck_rw_t *lock, int mode, boolean_t wait) +{ + uint64_t deadline = 0; + uint32_t data, prev; + boolean_t do_exch; + +#if __SMP__ + if (wait) + deadline = lck_rw_deadline_for_spin(lock); +#else + wait = FALSE; // Don't spin on UP systems +#endif + + for ( ; ; ) { + data = atomic_exchange_begin32(&lock->lck_rw_data, &prev, memory_order_acquire_smp); + if (data & LCK_RW_INTERLOCK) { + atomic_exchange_abort(); + lck_rw_interlock_spin(lock); + continue; + } + do_exch = FALSE; + if (mode == LCK_RW_GRAB_WANT) { + if ((data & LCK_RW_WANT_EXCL) == 0) { + data |= LCK_RW_WANT_EXCL; + do_exch = TRUE; + } + } else { // LCK_RW_GRAB_SHARED + if (((data & (LCK_RW_WANT_EXCL | LCK_RW_WANT_UPGRADE)) == 0) || + (((data & LCK_RW_SHARED_MASK)) && ((data & LCK_RW_PRIV_EXCL) == 0))) { + data += LCK_RW_SHARED_READER; + do_exch = TRUE; + } + } + if (do_exch) { + if (atomic_exchange_complete32(&lock->lck_rw_data, prev, data, memory_order_acquire_smp)) + return TRUE; + } else { + if (wait) // Non-waiting + wait_for_event(); + else + atomic_exchange_abort(); + if (!wait || (mach_absolute_time() >= deadline)) + return FALSE; + } + } +} + + +/* + * Routine: lck_rw_alloc_init + */ +lck_rw_t * +lck_rw_alloc_init( + lck_grp_t *grp, + lck_attr_t *attr) +{ + lck_rw_t *lck; + + if ((lck = (lck_rw_t *)kalloc(sizeof(lck_rw_t))) != 0) + lck_rw_init(lck, grp, attr); + + return lck; +} + +/* + * Routine: lck_rw_free + */ +void +lck_rw_free( + lck_rw_t *lck, + lck_grp_t *grp) +{ + lck_rw_destroy(lck, grp); + kfree(lck, sizeof(lck_rw_t)); +} + +/* + * Routine: lck_rw_init + */ +void +lck_rw_init( + lck_rw_t *lck, + lck_grp_t *grp, + lck_attr_t *attr) +{ + if (attr == LCK_ATTR_NULL) + attr = &LockDefaultLckAttr; + memset(lck, 0, sizeof(lck_rw_t)); + lck->lck_rw_can_sleep = TRUE; + if ((attr->lck_attr_val & LCK_ATTR_RW_SHARED_PRIORITY) == 0) + lck->lck_rw_priv_excl = TRUE; + + lck_grp_reference(grp); + lck_grp_lckcnt_incr(grp, LCK_TYPE_RW); +} + + +/* + * Routine: lck_rw_destroy + */ +void +lck_rw_destroy( + lck_rw_t *lck, + lck_grp_t *grp) +{ + if (lck->lck_rw_tag == LCK_RW_TAG_DESTROYED) + return; +#if MACH_LDEBUG + lck_rw_assert(lck, LCK_RW_ASSERT_NOTHELD); +#endif + lck->lck_rw_tag = LCK_RW_TAG_DESTROYED; + lck_grp_lckcnt_decr(grp, LCK_TYPE_RW); + lck_grp_deallocate(grp); + return; +} + +/* + * Routine: lck_rw_lock + */ +void +lck_rw_lock( + lck_rw_t *lck, + lck_rw_type_t lck_rw_type) +{ + if (lck_rw_type == LCK_RW_TYPE_SHARED) + lck_rw_lock_shared(lck); + else if (lck_rw_type == LCK_RW_TYPE_EXCLUSIVE) + lck_rw_lock_exclusive(lck); + else + panic("lck_rw_lock(): Invalid RW lock type: %x", lck_rw_type); +} + +/* + * Routine: lck_rw_lock_exclusive + */ +void +lck_rw_lock_exclusive(lck_rw_t *lock) +{ + thread_t thread = current_thread(); + + thread->rwlock_count++; + if (atomic_test_and_set32(&lock->lck_rw_data, + (LCK_RW_SHARED_MASK | LCK_RW_WANT_EXCL | LCK_RW_WANT_UPGRADE | LCK_RW_INTERLOCK), + LCK_RW_WANT_EXCL, memory_order_acquire_smp, FALSE)) { +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_RW_LOCK_EXCL_ACQUIRE, lock, DTRACE_RW_EXCL); +#endif /* CONFIG_DTRACE */ + } else + lck_rw_lock_exclusive_gen(lock); +#if MACH_ASSERT + thread_t owner = ordered_load_rw_owner(lock); + assertf(owner == THREAD_NULL, "state=0x%x, owner=%p", ordered_load_rw(lock), owner); +#endif + ordered_store_rw_owner(lock, thread); +} + +/* + * Routine: lck_rw_lock_shared + */ +void +lck_rw_lock_shared(lck_rw_t *lock) +{ + uint32_t data, prev; + + current_thread()->rwlock_count++; + for ( ; ; ) { + data = atomic_exchange_begin32(&lock->lck_rw_data, &prev, memory_order_acquire_smp); + if (data & (LCK_RW_WANT_EXCL | LCK_RW_WANT_UPGRADE | LCK_RW_INTERLOCK)) { + atomic_exchange_abort(); + lck_rw_lock_shared_gen(lock); + break; + } + data += LCK_RW_SHARED_READER; + if (atomic_exchange_complete32(&lock->lck_rw_data, prev, data, memory_order_acquire_smp)) + break; + cpu_pause(); + } +#if MACH_ASSERT + thread_t owner = ordered_load_rw_owner(lock); + assertf(owner == THREAD_NULL, "state=0x%x, owner=%p", ordered_load_rw(lock), owner); +#endif +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_ACQUIRE, lock, DTRACE_RW_SHARED); +#endif /* CONFIG_DTRACE */ + return; +} + +/* + * Routine: lck_rw_lock_shared_to_exclusive + */ +boolean_t +lck_rw_lock_shared_to_exclusive(lck_rw_t *lock) +{ + uint32_t data, prev; + + for ( ; ; ) { + data = atomic_exchange_begin32(&lock->lck_rw_data, &prev, memory_order_acquire_smp); + if (data & LCK_RW_INTERLOCK) { + atomic_exchange_abort(); + lck_rw_interlock_spin(lock); + continue; + } + if (data & LCK_RW_WANT_UPGRADE) { + data -= LCK_RW_SHARED_READER; + if ((data & LCK_RW_SHARED_MASK) == 0) /* we were the last reader */ + data &= ~(LCK_RW_W_WAITING); /* so clear the wait indicator */ + if (atomic_exchange_complete32(&lock->lck_rw_data, prev, data, memory_order_acquire_smp)) + return lck_rw_lock_shared_to_exclusive_failure(lock, prev); + } else { + data |= LCK_RW_WANT_UPGRADE; /* ask for WANT_UPGRADE */ + data -= LCK_RW_SHARED_READER; /* and shed our read count */ + if (atomic_exchange_complete32(&lock->lck_rw_data, prev, data, memory_order_acquire_smp)) + break; + } + cpu_pause(); + } + /* we now own the WANT_UPGRADE */ + if (data & LCK_RW_SHARED_MASK) /* check to see if all of the readers are drained */ + lck_rw_lock_shared_to_exclusive_success(lock); /* if not, we need to go wait */ +#if MACH_ASSERT + thread_t owner = ordered_load_rw_owner(lock); + assertf(owner == THREAD_NULL, "state=0x%x, owner=%p", ordered_load_rw(lock), owner); +#endif + ordered_store_rw_owner(lock, current_thread()); +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE, lock, 0); +#endif /* CONFIG_DTRACE */ + return TRUE; +} + + +/* + * Routine: lck_rw_lock_shared_to_exclusive_failure + * Function: + * Fast path code has already dropped our read + * count and determined that someone else owns 'lck_rw_want_upgrade' + * if 'lck_rw_shared_count' == 0, its also already dropped 'lck_w_waiting' + * all we need to do here is determine if a wakeup is needed + */ +static boolean_t +lck_rw_lock_shared_to_exclusive_failure( + lck_rw_t *lck, + uint32_t prior_lock_state) +{ + thread_t thread = current_thread(); + uint32_t rwlock_count; + + /* Check if dropping the lock means that we need to unpromote */ + rwlock_count = thread->rwlock_count--; +#if MACH_LDEBUG + if (rwlock_count == 0) { + panic("rw lock count underflow for thread %p", thread); + } +#endif + if ((prior_lock_state & LCK_RW_W_WAITING) && + ((prior_lock_state & LCK_RW_SHARED_MASK) == LCK_RW_SHARED_READER)) { + /* + * Someone else has requested upgrade. + * Since we've released the read lock, wake + * him up if he's blocked waiting + */ + thread_wakeup(LCK_RW_WRITER_EVENT(lck)); + } + + if ((rwlock_count == 1 /* field now 0 */) && (thread->sched_flags & TH_SFLAG_RW_PROMOTED)) { + /* sched_flags checked without lock, but will be rechecked while clearing */ + lck_rw_clear_promotion(thread); + } + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SH_TO_EX_CODE) | DBG_FUNC_NONE, + VM_KERNEL_UNSLIDE_OR_PERM(lck), lck->lck_rw_shared_count, lck->lck_rw_want_upgrade, 0, 0); + + return (FALSE); +} + +/* + * Routine: lck_rw_lock_shared_to_exclusive_success + * Function: + * assembly fast path code has already dropped our read + * count and successfully acquired 'lck_rw_want_upgrade' + * we just need to wait for the rest of the readers to drain + * and then we can return as the exclusive holder of this lock + */ +static boolean_t +lck_rw_lock_shared_to_exclusive_success( + lck_rw_t *lock) +{ + __kdebug_only uintptr_t trace_lck = VM_KERNEL_UNSLIDE_OR_PERM(lock); + int slept = 0; + lck_rw_word_t word; + wait_result_t res; + boolean_t istate; + boolean_t not_shared; + +#if CONFIG_DTRACE + uint64_t wait_interval = 0; + int readers_at_sleep = 0; + boolean_t dtrace_ls_initialized = FALSE; + boolean_t dtrace_rwl_shared_to_excl_spin, dtrace_rwl_shared_to_excl_block, dtrace_ls_enabled = FALSE; +#endif + + while (!lck_rw_drain_status(lock, LCK_RW_SHARED_MASK, FALSE)) { + + word.data = ordered_load_rw(lock); +#if CONFIG_DTRACE + if (dtrace_ls_initialized == FALSE) { + dtrace_ls_initialized = TRUE; + dtrace_rwl_shared_to_excl_spin = (lockstat_probemap[LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN] != 0); + dtrace_rwl_shared_to_excl_block = (lockstat_probemap[LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK] != 0); + dtrace_ls_enabled = dtrace_rwl_shared_to_excl_spin || dtrace_rwl_shared_to_excl_block; + if (dtrace_ls_enabled) { + /* + * Either sleeping or spinning is happening, + * start a timing of our delay interval now. + */ + readers_at_sleep = word.shared_count; + wait_interval = mach_absolute_time(); + } + } +#endif + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SH_TO_EX_SPIN_CODE) | DBG_FUNC_START, + trace_lck, word.shared_count, 0, 0, 0); + + not_shared = lck_rw_drain_status(lock, LCK_RW_SHARED_MASK, TRUE); + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SH_TO_EX_SPIN_CODE) | DBG_FUNC_END, + trace_lck, lock->lck_rw_shared_count, 0, 0, 0); + + if (not_shared) + break; + + /* + * if we get here, the spin deadline in lck_rw_wait_on_status() + * has expired w/o the rw_shared_count having drained to 0 + * check to see if we're allowed to do a thread_block + */ + if (word.can_sleep) { + + istate = lck_interlock_lock(lock); + + word.data = ordered_load_rw(lock); + if (word.shared_count != 0) { + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SH_TO_EX_WAIT_CODE) | DBG_FUNC_START, + trace_lck, word.shared_count, 0, 0, 0); + + word.w_waiting = 1; + ordered_store_rw(lock, word.data); + + thread_set_pending_block_hint(current_thread(), kThreadWaitKernelRWLockUpgrade); + res = assert_wait(LCK_RW_WRITER_EVENT(lock), THREAD_UNINT); + lck_interlock_unlock(lock, istate); + + if (res == THREAD_WAITING) { + res = thread_block(THREAD_CONTINUE_NULL); + slept++; + } + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SH_TO_EX_WAIT_CODE) | DBG_FUNC_END, + trace_lck, res, slept, 0, 0); + } else { + lck_interlock_unlock(lock, istate); + break; + } + } + } +#if CONFIG_DTRACE + /* + * We infer whether we took the sleep/spin path above by checking readers_at_sleep. + */ + if (dtrace_ls_enabled == TRUE) { + if (slept == 0) { + LOCKSTAT_RECORD2(LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN, lock, mach_absolute_time() - wait_interval, 0); + } else { + LOCKSTAT_RECORD4(LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK, lock, + mach_absolute_time() - wait_interval, 1, + (readers_at_sleep == 0 ? 1 : 0), readers_at_sleep); + } + } + LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE, lock, 1); +#endif + return (TRUE); +} + + +/* + * Routine: lck_rw_lock_exclusive_to_shared + */ + +void lck_rw_lock_exclusive_to_shared(lck_rw_t *lock) +{ + uint32_t data, prev; + + assertf(lock->lck_rw_owner == current_thread(), "state=0x%x, owner=%p", lock->lck_rw_data, lock->lck_rw_owner); + ordered_store_rw_owner(lock, THREAD_NULL); + for ( ; ; ) { + data = atomic_exchange_begin32(&lock->lck_rw_data, &prev, memory_order_release_smp); + if (data & LCK_RW_INTERLOCK) { +#if __SMP__ + atomic_exchange_abort(); + lck_rw_interlock_spin(lock); /* wait for interlock to clear */ + continue; +#else + panic("lck_rw_lock_exclusive_to_shared(): Interlock locked (%p): %x", lock, data); +#endif // __SMP__ + } + data += LCK_RW_SHARED_READER; + if (data & LCK_RW_WANT_UPGRADE) + data &= ~(LCK_RW_WANT_UPGRADE); + else + data &= ~(LCK_RW_WANT_EXCL); + if (!((prev & LCK_RW_W_WAITING) && (prev & LCK_RW_PRIV_EXCL))) + data &= ~(LCK_RW_W_WAITING); + if (atomic_exchange_complete32(&lock->lck_rw_data, prev, data, memory_order_release_smp)) + break; + cpu_pause(); + } + return lck_rw_lock_exclusive_to_shared_gen(lock, prev); +} + +/* + * Routine: lck_rw_lock_exclusive_to_shared_gen + * Function: + * Fast path has already dropped + * our exclusive state and bumped lck_rw_shared_count + * all we need to do here is determine if anyone + * needs to be awakened. + */ +static void +lck_rw_lock_exclusive_to_shared_gen( + lck_rw_t *lck, + uint32_t prior_lock_state) +{ + __kdebug_only uintptr_t trace_lck = VM_KERNEL_UNSLIDE_OR_PERM(lck); + lck_rw_word_t fake_lck; + + /* + * prior_lock state is a snapshot of the 1st word of the + * lock in question... we'll fake up a pointer to it + * and carefully not access anything beyond whats defined + * in the first word of a lck_rw_t + */ + fake_lck.data = prior_lock_state; + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EX_TO_SH_CODE) | DBG_FUNC_START, + trace_lck, fake_lck->want_excl, fake_lck->want_upgrade, 0, 0); + + /* + * don't wake up anyone waiting to take the lock exclusively + * since we hold a read count... when the read count drops to 0, + * the writers will be woken. + * + * wake up any waiting readers if we don't have any writers waiting, + * or the lock is NOT marked as rw_priv_excl (writers have privilege) + */ + if (!(fake_lck.priv_excl && fake_lck.w_waiting) && fake_lck.r_waiting) + thread_wakeup(LCK_RW_READER_EVENT(lck)); + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EX_TO_SH_CODE) | DBG_FUNC_END, + trace_lck, lck->lck_rw_want_excl, lck->lck_rw_want_upgrade, lck->lck_rw_shared_count, 0); + +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_RW_LOCK_EXCL_TO_SHARED_DOWNGRADE, lck, 0); +#endif +} + + +/* + * Routine: lck_rw_try_lock + */ +boolean_t +lck_rw_try_lock( + lck_rw_t *lck, + lck_rw_type_t lck_rw_type) +{ + if (lck_rw_type == LCK_RW_TYPE_SHARED) + return lck_rw_try_lock_shared(lck); + else if (lck_rw_type == LCK_RW_TYPE_EXCLUSIVE) + return lck_rw_try_lock_exclusive(lck); + else + panic("lck_rw_try_lock(): Invalid rw lock type: %x", lck_rw_type); + return FALSE; +} + +/* + * Routine: lck_rw_try_lock_shared + */ + +boolean_t lck_rw_try_lock_shared(lck_rw_t *lock) +{ + uint32_t data, prev; + + for ( ; ; ) { + data = atomic_exchange_begin32(&lock->lck_rw_data, &prev, memory_order_acquire_smp); + if (data & LCK_RW_INTERLOCK) { +#if __SMP__ + atomic_exchange_abort(); + lck_rw_interlock_spin(lock); + continue; +#else + panic("lck_rw_try_lock_shared(): Interlock locked (%p): %x", lock, data); +#endif + } + if (data & (LCK_RW_WANT_EXCL | LCK_RW_WANT_UPGRADE)) { + atomic_exchange_abort(); + return FALSE; /* lock is busy */ + } + data += LCK_RW_SHARED_READER; /* Increment reader refcount */ + if (atomic_exchange_complete32(&lock->lck_rw_data, prev, data, memory_order_acquire_smp)) + break; + cpu_pause(); + } +#if MACH_ASSERT + thread_t owner = ordered_load_rw_owner(lock); + assertf(owner == THREAD_NULL, "state=0x%x, owner=%p", ordered_load_rw(lock), owner); +#endif + current_thread()->rwlock_count++; +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE, lock, DTRACE_RW_SHARED); +#endif /* CONFIG_DTRACE */ + return TRUE; +} + + +/* + * Routine: lck_rw_try_lock_exclusive + */ + +boolean_t lck_rw_try_lock_exclusive(lck_rw_t *lock) +{ + uint32_t data, prev; + thread_t thread; + + for ( ; ; ) { + data = atomic_exchange_begin32(&lock->lck_rw_data, &prev, memory_order_acquire_smp); + if (data & LCK_RW_INTERLOCK) { +#if __SMP__ + atomic_exchange_abort(); + lck_rw_interlock_spin(lock); + continue; +#else + panic("lck_rw_try_lock_exclusive(): Interlock locked (%p): %x", lock, data); +#endif + } + if (data & (LCK_RW_SHARED_MASK | LCK_RW_WANT_EXCL | LCK_RW_WANT_UPGRADE)) { + atomic_exchange_abort(); + return FALSE; + } + data |= LCK_RW_WANT_EXCL; + if (atomic_exchange_complete32(&lock->lck_rw_data, prev, data, memory_order_acquire_smp)) + break; + cpu_pause(); + } + thread = current_thread(); + thread->rwlock_count++; +#if MACH_ASSERT + thread_t owner = ordered_load_rw_owner(lock); + assertf(owner == THREAD_NULL, "state=0x%x, owner=%p", ordered_load_rw(lock), owner); +#endif + ordered_store_rw_owner(lock, thread); +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE, lock, DTRACE_RW_EXCL); +#endif /* CONFIG_DTRACE */ + return TRUE; +} + + +/* + * Routine: lck_rw_unlock + */ +void +lck_rw_unlock( + lck_rw_t *lck, + lck_rw_type_t lck_rw_type) +{ + if (lck_rw_type == LCK_RW_TYPE_SHARED) + lck_rw_unlock_shared(lck); + else if (lck_rw_type == LCK_RW_TYPE_EXCLUSIVE) + lck_rw_unlock_exclusive(lck); + else + panic("lck_rw_unlock(): Invalid RW lock type: %d", lck_rw_type); +} + + +/* + * Routine: lck_rw_unlock_shared + */ +void +lck_rw_unlock_shared( + lck_rw_t *lck) +{ + lck_rw_type_t ret; + + assertf(lck->lck_rw_owner == THREAD_NULL, "state=0x%x, owner=%p", lck->lck_rw_data, lck->lck_rw_owner); + assertf(lck->lck_rw_shared_count > 0, "shared_count=0x%x", lck->lck_rw_shared_count); + ret = lck_rw_done(lck); + + if (ret != LCK_RW_TYPE_SHARED) + panic("lck_rw_unlock_shared(): lock %p held in mode: %d", lck, ret); +} + + +/* + * Routine: lck_rw_unlock_exclusive + */ +void +lck_rw_unlock_exclusive( + lck_rw_t *lck) +{ + lck_rw_type_t ret; + + assertf(lck->lck_rw_owner == current_thread(), "state=0x%x, owner=%p", lck->lck_rw_data, lck->lck_rw_owner); + ret = lck_rw_done(lck); + + if (ret != LCK_RW_TYPE_EXCLUSIVE) + panic("lck_rw_unlock_exclusive(): lock %p held in mode: %d", lck, ret); +} + + +/* + * Routine: lck_rw_lock_exclusive_gen + */ +static void +lck_rw_lock_exclusive_gen( + lck_rw_t *lock) +{ + __kdebug_only uintptr_t trace_lck = VM_KERNEL_UNSLIDE_OR_PERM(lock); + lck_rw_word_t word; + int slept = 0; + boolean_t gotlock = 0; + boolean_t not_shared_or_upgrade = 0; + wait_result_t res = 0; + boolean_t istate; + +#if CONFIG_DTRACE + boolean_t dtrace_ls_initialized = FALSE; + boolean_t dtrace_rwl_excl_spin, dtrace_rwl_excl_block, dtrace_ls_enabled= FALSE; + uint64_t wait_interval = 0; + int readers_at_sleep = 0; +#endif + + /* + * Try to acquire the lck_rw_want_excl bit. + */ + while (!lck_rw_grab(lock, LCK_RW_GRAB_WANT, FALSE)) { + +#if CONFIG_DTRACE + if (dtrace_ls_initialized == FALSE) { + dtrace_ls_initialized = TRUE; + dtrace_rwl_excl_spin = (lockstat_probemap[LS_LCK_RW_LOCK_EXCL_SPIN] != 0); + dtrace_rwl_excl_block = (lockstat_probemap[LS_LCK_RW_LOCK_EXCL_BLOCK] != 0); + dtrace_ls_enabled = dtrace_rwl_excl_spin || dtrace_rwl_excl_block; + if (dtrace_ls_enabled) { + /* + * Either sleeping or spinning is happening, + * start a timing of our delay interval now. + */ + readers_at_sleep = lock->lck_rw_shared_count; + wait_interval = mach_absolute_time(); + } + } +#endif + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EX_WRITER_SPIN_CODE) | DBG_FUNC_START, trace_lck, 0, 0, 0, 0); + + gotlock = lck_rw_grab(lock, LCK_RW_GRAB_WANT, TRUE); + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EX_WRITER_SPIN_CODE) | DBG_FUNC_END, trace_lck, 0, 0, gotlock, 0); + + if (gotlock) + break; + /* + * if we get here, the deadline has expired w/o us + * being able to grab the lock exclusively + * check to see if we're allowed to do a thread_block + */ + word.data = ordered_load_rw(lock); + if (word.can_sleep) { + + istate = lck_interlock_lock(lock); + word.data = ordered_load_rw(lock); + + if (word.want_excl) { + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EX_WRITER_WAIT_CODE) | DBG_FUNC_START, trace_lck, 0, 0, 0, 0); + + word.w_waiting = 1; + ordered_store_rw(lock, word.data); + + thread_set_pending_block_hint(current_thread(), kThreadWaitKernelRWLockWrite); + res = assert_wait(LCK_RW_WRITER_EVENT(lock), THREAD_UNINT); + lck_interlock_unlock(lock, istate); + + if (res == THREAD_WAITING) { + res = thread_block(THREAD_CONTINUE_NULL); + slept++; + } + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EX_WRITER_WAIT_CODE) | DBG_FUNC_END, trace_lck, res, slept, 0, 0); + } else { + word.want_excl = 1; + ordered_store_rw(lock, word.data); + lck_interlock_unlock(lock, istate); + break; + } + } + } + /* + * Wait for readers (and upgrades) to finish... + */ + while (!lck_rw_drain_status(lock, LCK_RW_SHARED_MASK | LCK_RW_WANT_UPGRADE, FALSE)) { + +#if CONFIG_DTRACE + /* + * Either sleeping or spinning is happening, start + * a timing of our delay interval now. If we set it + * to -1 we don't have accurate data so we cannot later + * decide to record a dtrace spin or sleep event. + */ + if (dtrace_ls_initialized == FALSE) { + dtrace_ls_initialized = TRUE; + dtrace_rwl_excl_spin = (lockstat_probemap[LS_LCK_RW_LOCK_EXCL_SPIN] != 0); + dtrace_rwl_excl_block = (lockstat_probemap[LS_LCK_RW_LOCK_EXCL_BLOCK] != 0); + dtrace_ls_enabled = dtrace_rwl_excl_spin || dtrace_rwl_excl_block; + if (dtrace_ls_enabled) { + /* + * Either sleeping or spinning is happening, + * start a timing of our delay interval now. + */ + readers_at_sleep = lock->lck_rw_shared_count; + wait_interval = mach_absolute_time(); + } + } +#endif + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EX_READER_SPIN_CODE) | DBG_FUNC_START, trace_lck, 0, 0, 0, 0); + + not_shared_or_upgrade = lck_rw_drain_status(lock, LCK_RW_SHARED_MASK | LCK_RW_WANT_UPGRADE, TRUE); + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EX_READER_SPIN_CODE) | DBG_FUNC_END, trace_lck, 0, 0, not_shared_or_upgrade, 0); + + if (not_shared_or_upgrade) + break; + /* + * if we get here, the deadline has expired w/o us + * being able to grab the lock exclusively + * check to see if we're allowed to do a thread_block + */ + word.data = ordered_load_rw(lock); + if (word.can_sleep) { + + istate = lck_interlock_lock(lock); + word.data = ordered_load_rw(lock); + + if (word.shared_count != 0 || word.want_upgrade) { + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EX_READER_WAIT_CODE) | DBG_FUNC_START, trace_lck, 0, 0, 0, 0); + + word.w_waiting = 1; + ordered_store_rw(lock, word.data); + + thread_set_pending_block_hint(current_thread(), kThreadWaitKernelRWLockWrite); + res = assert_wait(LCK_RW_WRITER_EVENT(lock), THREAD_UNINT); + lck_interlock_unlock(lock, istate); + + if (res == THREAD_WAITING) { + res = thread_block(THREAD_CONTINUE_NULL); + slept++; + } + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EX_READER_WAIT_CODE) | DBG_FUNC_END, trace_lck, res, slept, 0, 0); + } else { + lck_interlock_unlock(lock, istate); + /* + * must own the lock now, since we checked for + * readers or upgrade owner behind the interlock + * no need for a call to 'lck_rw_drain_status' + */ + break; + } + } + } + +#if CONFIG_DTRACE + /* + * Decide what latencies we suffered that are Dtrace events. + * If we have set wait_interval, then we either spun or slept. + * At least we get out from under the interlock before we record + * which is the best we can do here to minimize the impact + * of the tracing. + * If we have set wait_interval to -1, then dtrace was not enabled when we + * started sleeping/spinning so we don't record this event. + */ + if (dtrace_ls_enabled == TRUE) { + if (slept == 0) { + LOCKSTAT_RECORD2(LS_LCK_RW_LOCK_EXCL_SPIN, lock, + mach_absolute_time() - wait_interval, 1); + } else { + /* + * For the blocking case, we also record if when we blocked + * it was held for read or write, and how many readers. + * Notice that above we recorded this before we dropped + * the interlock so the count is accurate. + */ + LOCKSTAT_RECORD4(LS_LCK_RW_LOCK_EXCL_BLOCK, lock, + mach_absolute_time() - wait_interval, 1, + (readers_at_sleep == 0 ? 1 : 0), readers_at_sleep); + } + } + LOCKSTAT_RECORD(LS_LCK_RW_LOCK_EXCL_ACQUIRE, lock, 1); +#endif /* CONFIG_DTRACE */ +} + +/* + * Routine: lck_rw_done + */ + +lck_rw_type_t lck_rw_done(lck_rw_t *lock) +{ + uint32_t data, prev; + boolean_t once = FALSE; + + for ( ; ; ) { + data = atomic_exchange_begin32(&lock->lck_rw_data, &prev, memory_order_release_smp); + if (data & LCK_RW_INTERLOCK) { /* wait for interlock to clear */ +#if __SMP__ + atomic_exchange_abort(); + lck_rw_interlock_spin(lock); + continue; +#else + panic("lck_rw_done(): Interlock locked (%p): %x", lock, data); +#endif // __SMP__ + } + if (data & LCK_RW_SHARED_MASK) { /* lock is held shared */ + assertf(lock->lck_rw_owner == THREAD_NULL, "state=0x%x, owner=%p", lock->lck_rw_data, lock->lck_rw_owner); + data -= LCK_RW_SHARED_READER; + if ((data & LCK_RW_SHARED_MASK) == 0) /* if reader count has now gone to 0, check for waiters */ + goto check_waiters; + } else { /* if reader count == 0, must be exclusive lock */ + if (data & LCK_RW_WANT_UPGRADE) { + data &= ~(LCK_RW_WANT_UPGRADE); + } else { + if (data & LCK_RW_WANT_EXCL) + data &= ~(LCK_RW_WANT_EXCL); + else /* lock is not 'owned', panic */ + panic("Releasing non-exclusive RW lock without a reader refcount!"); + } + if (!once) { + // Only check for holder and clear it once + assertf(lock->lck_rw_owner == current_thread(), "state=0x%x, owner=%p", lock->lck_rw_data, lock->lck_rw_owner); + ordered_store_rw_owner(lock, THREAD_NULL); + once = TRUE; + } +check_waiters: + /* + * test the original values to match what + * lck_rw_done_gen is going to do to determine + * which wakeups need to happen... + * + * if !(fake_lck->lck_rw_priv_excl && fake_lck->lck_w_waiting) + */ + if (prev & LCK_RW_W_WAITING) { + data &= ~(LCK_RW_W_WAITING); + if ((prev & LCK_RW_PRIV_EXCL) == 0) + data &= ~(LCK_RW_R_WAITING); + } else + data &= ~(LCK_RW_R_WAITING); + } + if (atomic_exchange_complete32(&lock->lck_rw_data, prev, data, memory_order_release_smp)) + break; + cpu_pause(); + } + return lck_rw_done_gen(lock, prev); +} + +/* + * Routine: lck_rw_done_gen + * + * called from the assembly language wrapper... + * prior_lock_state is the value in the 1st + * word of the lock at the time of a successful + * atomic compare and exchange with the new value... + * it represents the state of the lock before we + * decremented the rw_shared_count or cleared either + * rw_want_upgrade or rw_want_write and + * the lck_x_waiting bits... since the wrapper + * routine has already changed the state atomically, + * we just need to decide if we should + * wake up anyone and what value to return... we do + * this by examining the state of the lock before + * we changed it + */ +static lck_rw_type_t +lck_rw_done_gen( + lck_rw_t *lck, + uint32_t prior_lock_state) +{ + lck_rw_word_t fake_lck; + lck_rw_type_t lock_type; + thread_t thread; + uint32_t rwlock_count; + + /* + * prior_lock state is a snapshot of the 1st word of the + * lock in question... we'll fake up a pointer to it + * and carefully not access anything beyond whats defined + * in the first word of a lck_rw_t + */ + fake_lck.data = prior_lock_state; + + if (fake_lck.shared_count <= 1) { + if (fake_lck.w_waiting) + thread_wakeup(LCK_RW_WRITER_EVENT(lck)); + + if (!(fake_lck.priv_excl && fake_lck.w_waiting) && fake_lck.r_waiting) + thread_wakeup(LCK_RW_READER_EVENT(lck)); + } + if (fake_lck.shared_count) + lock_type = LCK_RW_TYPE_SHARED; + else + lock_type = LCK_RW_TYPE_EXCLUSIVE; + + /* Check if dropping the lock means that we need to unpromote */ + thread = current_thread(); + rwlock_count = thread->rwlock_count--; +#if MACH_LDEBUG + if (rwlock_count == 0) + panic("rw lock count underflow for thread %p", thread); +#endif + if ((rwlock_count == 1 /* field now 0 */) && (thread->sched_flags & TH_SFLAG_RW_PROMOTED)) { + /* sched_flags checked without lock, but will be rechecked while clearing */ + lck_rw_clear_promotion(thread); + } +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_RW_DONE_RELEASE, lck, lock_type == LCK_RW_TYPE_SHARED ? 0 : 1); +#endif + return lock_type; +} + +/* + * Routine: lck_rw_lock_shared_gen + * Function: + * Fast path code has determined that this lock + * is held exclusively... this is where we spin/block + * until we can acquire the lock in the shared mode + */ +static void +lck_rw_lock_shared_gen( + lck_rw_t *lck) +{ + __kdebug_only uintptr_t trace_lck = VM_KERNEL_UNSLIDE_OR_PERM(lck); + lck_rw_word_t word; + boolean_t gotlock = 0; + int slept = 0; + wait_result_t res = 0; + boolean_t istate; + +#if CONFIG_DTRACE + uint64_t wait_interval = 0; + int readers_at_sleep = 0; + boolean_t dtrace_ls_initialized = FALSE; + boolean_t dtrace_rwl_shared_spin, dtrace_rwl_shared_block, dtrace_ls_enabled = FALSE; +#endif /* CONFIG_DTRACE */ + + while ( !lck_rw_grab(lck, LCK_RW_GRAB_SHARED, FALSE)) { + +#if CONFIG_DTRACE + if (dtrace_ls_initialized == FALSE) { + dtrace_ls_initialized = TRUE; + dtrace_rwl_shared_spin = (lockstat_probemap[LS_LCK_RW_LOCK_SHARED_SPIN] != 0); + dtrace_rwl_shared_block = (lockstat_probemap[LS_LCK_RW_LOCK_SHARED_BLOCK] != 0); + dtrace_ls_enabled = dtrace_rwl_shared_spin || dtrace_rwl_shared_block; + if (dtrace_ls_enabled) { + /* + * Either sleeping or spinning is happening, + * start a timing of our delay interval now. + */ + readers_at_sleep = lck->lck_rw_shared_count; + wait_interval = mach_absolute_time(); + } + } +#endif + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SHARED_SPIN_CODE) | DBG_FUNC_START, + trace_lck, lck->lck_rw_want_excl, lck->lck_rw_want_upgrade, 0, 0); + + gotlock = lck_rw_grab(lck, LCK_RW_GRAB_SHARED, TRUE); + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SHARED_SPIN_CODE) | DBG_FUNC_END, + trace_lck, lck->lck_rw_want_excl, lck->lck_rw_want_upgrade, gotlock, 0); + + if (gotlock) + break; + /* + * if we get here, the deadline has expired w/o us + * being able to grab the lock for read + * check to see if we're allowed to do a thread_block + */ + if (lck->lck_rw_can_sleep) { + + istate = lck_interlock_lock(lck); + + word.data = ordered_load_rw(lck); + if ((word.want_excl || word.want_upgrade) && + ((word.shared_count == 0) || word.priv_excl)) { + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SHARED_WAIT_CODE) | DBG_FUNC_START, + trace_lck, word.want_excl, word.want_upgrade, 0, 0); + + word.r_waiting = 1; + ordered_store_rw(lck, word.data); + + thread_set_pending_block_hint(current_thread(), kThreadWaitKernelRWLockRead); + res = assert_wait(LCK_RW_READER_EVENT(lck), THREAD_UNINT); + lck_interlock_unlock(lck, istate); + + if (res == THREAD_WAITING) { + res = thread_block(THREAD_CONTINUE_NULL); + slept++; + } + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SHARED_WAIT_CODE) | DBG_FUNC_END, + trace_lck, res, slept, 0, 0); + } else { + word.shared_count++; + ordered_store_rw(lck, word.data); + lck_interlock_unlock(lck, istate); + break; + } + } + } + +#if CONFIG_DTRACE + if (dtrace_ls_enabled == TRUE) { + if (slept == 0) { + LOCKSTAT_RECORD2(LS_LCK_RW_LOCK_SHARED_SPIN, lck, mach_absolute_time() - wait_interval, 0); + } else { + LOCKSTAT_RECORD4(LS_LCK_RW_LOCK_SHARED_BLOCK, lck, + mach_absolute_time() - wait_interval, 0, + (readers_at_sleep == 0 ? 1 : 0), readers_at_sleep); + } + } + LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_ACQUIRE, lck, 0); +#endif /* CONFIG_DTRACE */ +} + + +void +lck_rw_assert( + lck_rw_t *lck, + unsigned int type) +{ + switch (type) { + case LCK_RW_ASSERT_SHARED: + if ((lck->lck_rw_shared_count != 0) && + (lck->lck_rw_owner == THREAD_NULL)) { + return; + } + break; + case LCK_RW_ASSERT_EXCLUSIVE: + if ((lck->lck_rw_want_excl || lck->lck_rw_want_upgrade) && + (lck->lck_rw_shared_count == 0) && + (lck->lck_rw_owner == current_thread())) { + return; + } + break; + case LCK_RW_ASSERT_HELD: + if (lck->lck_rw_shared_count != 0) + return; // Held shared + if ((lck->lck_rw_want_excl || lck->lck_rw_want_upgrade) && + (lck->lck_rw_owner == current_thread())) { + return; // Held exclusive + } + break; + case LCK_RW_ASSERT_NOTHELD: + if ((lck->lck_rw_shared_count == 0) && + !(lck->lck_rw_want_excl || lck->lck_rw_want_upgrade) && + (lck->lck_rw_owner == THREAD_NULL)) { + return; + } + break; + default: + break; + } + panic("rw lock (%p)%s held (mode=%u)", lck, (type == LCK_RW_ASSERT_NOTHELD ? "" : " not"), type); +} + + +/* + * Routine: kdp_lck_rw_lock_is_acquired_exclusive + * NOT SAFE: To be used only by kernel debugger to avoid deadlock. + */ +boolean_t +kdp_lck_rw_lock_is_acquired_exclusive(lck_rw_t *lck) { + if (not_in_kdp) { + panic("panic: rw lock exclusive check done outside of kernel debugger"); + } + return ((lck->lck_rw_want_upgrade || lck->lck_rw_want_excl) && (lck->lck_rw_shared_count == 0)) ? TRUE : FALSE; +} + +/* + * The C portion of the mutex package. These routines are only invoked + * if the optimized assembler routines can't do the work. + */ + +/* + * Forward declaration + */ + +void +lck_mtx_ext_init( + lck_mtx_ext_t * lck, + lck_grp_t * grp, + lck_attr_t * attr); + +/* + * Routine: lck_mtx_alloc_init + */ +lck_mtx_t * +lck_mtx_alloc_init( + lck_grp_t * grp, + lck_attr_t * attr) +{ + lck_mtx_t *lck; + + if ((lck = (lck_mtx_t *) kalloc(sizeof(lck_mtx_t))) != 0) + lck_mtx_init(lck, grp, attr); + + return (lck); +} + +/* + * Routine: lck_mtx_free + */ +void +lck_mtx_free( + lck_mtx_t * lck, + lck_grp_t * grp) +{ + lck_mtx_destroy(lck, grp); + kfree((void *) lck, sizeof(lck_mtx_t)); +} + +/* + * Routine: lck_mtx_init + */ +void +lck_mtx_init( + lck_mtx_t * lck, + lck_grp_t * grp, + lck_attr_t * attr) +{ +#ifdef BER_XXX + lck_mtx_ext_t *lck_ext; +#endif + lck_attr_t *lck_attr; + + if (attr != LCK_ATTR_NULL) + lck_attr = attr; + else + lck_attr = &LockDefaultLckAttr; + +#ifdef BER_XXX + if ((lck_attr->lck_attr_val) & LCK_ATTR_DEBUG) { + if ((lck_ext = (lck_mtx_ext_t *) kalloc(sizeof(lck_mtx_ext_t))) != 0) { + lck_mtx_ext_init(lck_ext, grp, lck_attr); + lck->lck_mtx_tag = LCK_MTX_TAG_INDIRECT; + lck->lck_mtx_ptr = lck_ext; + lck->lck_mtx_type = LCK_MTX_TYPE; + } + } else +#endif + { + lck->lck_mtx_ptr = NULL; // Clear any padding in the union fields below + lck->lck_mtx_waiters = 0; + lck->lck_mtx_pri = 0; + lck->lck_mtx_type = LCK_MTX_TYPE; + ordered_store_mtx(lck, 0); + } + lck_grp_reference(grp); + lck_grp_lckcnt_incr(grp, LCK_TYPE_MTX); +} + +/* + * Routine: lck_mtx_init_ext + */ +void +lck_mtx_init_ext( + lck_mtx_t * lck, + lck_mtx_ext_t * lck_ext, + lck_grp_t * grp, + lck_attr_t * attr) +{ + lck_attr_t *lck_attr; + + if (attr != LCK_ATTR_NULL) + lck_attr = attr; + else + lck_attr = &LockDefaultLckAttr; + + if ((lck_attr->lck_attr_val) & LCK_ATTR_DEBUG) { + lck_mtx_ext_init(lck_ext, grp, lck_attr); + lck->lck_mtx_tag = LCK_MTX_TAG_INDIRECT; + lck->lck_mtx_ptr = lck_ext; + lck->lck_mtx_type = LCK_MTX_TYPE; + } else { + lck->lck_mtx_waiters = 0; + lck->lck_mtx_pri = 0; + lck->lck_mtx_type = LCK_MTX_TYPE; + ordered_store_mtx(lck, 0); + } + lck_grp_reference(grp); + lck_grp_lckcnt_incr(grp, LCK_TYPE_MTX); +} + +/* + * Routine: lck_mtx_ext_init + */ +void +lck_mtx_ext_init( + lck_mtx_ext_t * lck, + lck_grp_t * grp, + lck_attr_t * attr) +{ + bzero((void *) lck, sizeof(lck_mtx_ext_t)); + + lck->lck_mtx.lck_mtx_type = LCK_MTX_TYPE; + + if ((attr->lck_attr_val) & LCK_ATTR_DEBUG) { + lck->lck_mtx_deb.type = MUTEX_TAG; + lck->lck_mtx_attr |= LCK_MTX_ATTR_DEBUG; + } + lck->lck_mtx_grp = grp; + + if (grp->lck_grp_attr & LCK_GRP_ATTR_STAT) + lck->lck_mtx_attr |= LCK_MTX_ATTR_STAT; +} + +/* The slow versions */ +static void lck_mtx_lock_contended(lck_mtx_t *lock, thread_t thread, boolean_t interlocked); +static boolean_t lck_mtx_try_lock_contended(lck_mtx_t *lock, thread_t thread); +static void lck_mtx_unlock_contended(lck_mtx_t *lock, thread_t thread, boolean_t interlocked); + +/* + * Routine: lck_mtx_verify + * + * Verify if a mutex is valid + */ +static inline void +lck_mtx_verify(lck_mtx_t *lock) +{ + if (lock->lck_mtx_type != LCK_MTX_TYPE) + panic("Invalid mutex %p", lock); +#if DEVELOPMENT || DEBUG + if (lock->lck_mtx_tag == LCK_MTX_TAG_DESTROYED) + panic("Mutex destroyed %p", lock); +#endif /* DEVELOPMENT || DEBUG */ +} + +/* + * Routine: lck_mtx_check_preemption + * + * Verify preemption is enabled when attempting to acquire a mutex. + */ + +static inline void +lck_mtx_check_preemption(lck_mtx_t *lock) +{ +#if DEVELOPMENT || DEBUG + int pl = get_preemption_level(); + + if (pl != 0) + panic("Attempt to take mutex with preemption disabled. Lock=%p, level=%d", lock, pl); +#else + (void)lock; +#endif +} + +/* + * Routine: lck_mtx_lock + */ +void +lck_mtx_lock(lck_mtx_t *lock) +{ + thread_t thread; + + lck_mtx_verify(lock); + lck_mtx_check_preemption(lock); + thread = current_thread(); + if (atomic_compare_exchange(&lock->lck_mtx_data, 0, LCK_MTX_THREAD_TO_STATE(thread), + memory_order_acquire_smp, FALSE)) { +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_ACQUIRE, lock, 0); +#endif /* CONFIG_DTRACE */ + return; + } + lck_mtx_lock_contended(lock, thread, FALSE); +} + +/* + This is the slow version of mutex locking. + */ +static void NOINLINE +lck_mtx_lock_contended(lck_mtx_t *lock, thread_t thread, boolean_t interlocked) +{ + thread_t holding_thread; + uintptr_t state; + int waiters; + + if (interlocked) + goto interlock_held; + + for ( ; ; ) { + if (atomic_compare_exchange(&lock->lck_mtx_data, 0, LCK_MTX_THREAD_TO_STATE(thread), + memory_order_acquire_smp, FALSE)) + return; + interlock_lock(lock); +interlock_held: + state = ordered_load_mtx(lock); + holding_thread = LCK_MTX_STATE_TO_THREAD(state); + if (holding_thread == NULL) + break; + ordered_store_mtx(lock, (state | LCK_ILOCK | ARM_LCK_WAITERS)); // Set waiters bit and wait + lck_mtx_lock_wait(lock, holding_thread); + } + waiters = lck_mtx_lock_acquire(lock); + state = LCK_MTX_THREAD_TO_STATE(thread); + if (waiters != 0) + state |= ARM_LCK_WAITERS; +#if __SMP__ + state |= LCK_ILOCK; // Preserve interlock + ordered_store_mtx(lock, state); // Set ownership + interlock_unlock(lock); // Release interlock, enable preemption +#else + ordered_store_mtx(lock, state); // Set ownership + enable_preemption(); +#endif + load_memory_barrier(); + +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_ACQUIRE, lock, 0); +#endif /* CONFIG_DTRACE */ +} + +/* + * Common code for mutex locking as spinlock + */ +static inline void +lck_mtx_lock_spin_internal(lck_mtx_t *lock, boolean_t allow_held_as_mutex) +{ + uintptr_t state; + + interlock_lock(lock); + state = ordered_load_mtx(lock); + if (LCK_MTX_STATE_TO_THREAD(state)) { + if (allow_held_as_mutex) + lck_mtx_lock_contended(lock, current_thread(), TRUE); + else + // "Always" variants can never block. If the lock is held and blocking is not allowed + // then someone is mixing always and non-always calls on the same lock, which is + // forbidden. + panic("Attempting to block on a lock taken as spin-always %p", lock); + return; + } + state &= ARM_LCK_WAITERS; // Preserve waiters bit + state |= (LCK_MTX_SPIN_TAG | LCK_ILOCK); // Add spin tag and maintain interlock + ordered_store_mtx(lock, state); + load_memory_barrier(); + +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_SPIN_ACQUIRE, lock, 0); +#endif /* CONFIG_DTRACE */ +} + +/* + * Routine: lck_mtx_lock_spin + */ +void +lck_mtx_lock_spin(lck_mtx_t *lock) +{ + lck_mtx_check_preemption(lock); + lck_mtx_lock_spin_internal(lock, TRUE); +} + +/* + * Routine: lck_mtx_lock_spin_always + */ +void +lck_mtx_lock_spin_always(lck_mtx_t *lock) +{ + lck_mtx_lock_spin_internal(lock, FALSE); +} + +/* + * Routine: lck_mtx_try_lock + */ +boolean_t +lck_mtx_try_lock(lck_mtx_t *lock) +{ + thread_t thread = current_thread(); + + lck_mtx_verify(lock); + if (atomic_compare_exchange(&lock->lck_mtx_data, 0, LCK_MTX_THREAD_TO_STATE(thread), + memory_order_acquire_smp, FALSE)) { +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_MTX_TRY_LOCK_ACQUIRE, lock, 0); +#endif /* CONFIG_DTRACE */ + return TRUE; + } + return lck_mtx_try_lock_contended(lock, thread); +} + +static boolean_t NOINLINE +lck_mtx_try_lock_contended(lck_mtx_t *lock, thread_t thread) +{ + thread_t holding_thread; + uintptr_t state; + int waiters; + +#if __SMP__ + interlock_lock(lock); + state = ordered_load_mtx(lock); + holding_thread = LCK_MTX_STATE_TO_THREAD(state); + if (holding_thread) { + interlock_unlock(lock); + return FALSE; + } +#else + disable_preemption_for_thread(thread); + state = ordered_load_mtx(lock); + if (state & LCK_ILOCK) + panic("Unexpected interlock set (%p)", lock); + holding_thread = LCK_MTX_STATE_TO_THREAD(state); + if (holding_thread) { + enable_preemption(); + return FALSE; + } + state |= LCK_ILOCK; + ordered_store_mtx(lock, state); +#endif // __SMP__ + waiters = lck_mtx_lock_acquire(lock); + state = LCK_MTX_THREAD_TO_STATE(thread); + if (waiters != 0) + state |= ARM_LCK_WAITERS; +#if __SMP__ + state |= LCK_ILOCK; // Preserve interlock + ordered_store_mtx(lock, state); // Set ownership + interlock_unlock(lock); // Release interlock, enable preemption +#else + ordered_store_mtx(lock, state); // Set ownership + enable_preemption(); +#endif + load_memory_barrier(); + return TRUE; +} + +static inline boolean_t +lck_mtx_try_lock_spin_internal(lck_mtx_t *lock, boolean_t allow_held_as_mutex) +{ + uintptr_t state; + + if (!interlock_try(lock)) + return FALSE; + state = ordered_load_mtx(lock); + if(LCK_MTX_STATE_TO_THREAD(state)) { + // Lock is held as mutex + if (allow_held_as_mutex) + interlock_unlock(lock); + else + // "Always" variants can never block. If the lock is held as a normal mutex + // then someone is mixing always and non-always calls on the same lock, which is + // forbidden. + panic("Spin-mutex held as full mutex %p", lock); + return FALSE; + } + state &= ARM_LCK_WAITERS; // Preserve waiters bit + state |= (LCK_MTX_SPIN_TAG | LCK_ILOCK); // Add spin tag and maintain interlock + ordered_store_mtx(lock, state); + load_memory_barrier(); + +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE, lock, 0); +#endif /* CONFIG_DTRACE */ + return TRUE; +} + +/* + * Routine: lck_mtx_try_lock_spin + */ +boolean_t +lck_mtx_try_lock_spin(lck_mtx_t *lock) +{ + return lck_mtx_try_lock_spin_internal(lock, TRUE); +} + +/* + * Routine: lck_mtx_try_lock_spin_always + */ +boolean_t +lck_mtx_try_lock_spin_always(lck_mtx_t *lock) +{ + return lck_mtx_try_lock_spin_internal(lock, FALSE); +} + + + +/* + * Routine: lck_mtx_unlock + */ +void +lck_mtx_unlock(lck_mtx_t *lock) +{ + thread_t thread = current_thread(); + uintptr_t state; + boolean_t ilk_held = FALSE; + + lck_mtx_verify(lock); + + state = ordered_load_mtx(lock); + if (state & LCK_ILOCK) { + if(LCK_MTX_STATE_TO_THREAD(state) == (thread_t)LCK_MTX_SPIN_TAG) + ilk_held = TRUE; // Interlock is held by (presumably) this thread + goto slow_case; + } + // Locked as a mutex + if (atomic_compare_exchange(&lock->lck_mtx_data, LCK_MTX_THREAD_TO_STATE(thread), 0, + memory_order_release_smp, FALSE)) { +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_MTX_UNLOCK_RELEASE, lock, 0); +#endif /* CONFIG_DTRACE */ + return; + } +slow_case: + lck_mtx_unlock_contended(lock, thread, ilk_held); +} + +static void NOINLINE +lck_mtx_unlock_contended(lck_mtx_t *lock, thread_t thread, boolean_t ilk_held) +{ + uintptr_t state; + + if (ilk_held) { + state = ordered_load_mtx(lock); + } else { +#if __SMP__ + interlock_lock(lock); + state = ordered_load_mtx(lock); + if (thread != LCK_MTX_STATE_TO_THREAD(state)) + panic("lck_mtx_unlock(): Attempt to release lock not owned by thread (%p)", lock); +#else + disable_preemption_for_thread(thread); + state = ordered_load_mtx(lock); + if (state & LCK_ILOCK) + panic("lck_mtx_unlock(): Unexpected interlock set (%p)", lock); + if (thread != LCK_MTX_STATE_TO_THREAD(state)) + panic("lck_mtx_unlock(): Attempt to release lock not owned by thread (%p)", lock); + state |= LCK_ILOCK; + ordered_store_mtx(lock, state); +#endif + } + if (state & ARM_LCK_WAITERS) { + lck_mtx_unlock_wakeup(lock, thread); + state = ordered_load_mtx(lock); + } else { + assertf(lock->lck_mtx_pri == 0, "pri=0x%x", lock->lck_mtx_pri); + } + state &= ARM_LCK_WAITERS; // Retain waiters bit +#if __SMP__ + state |= LCK_ILOCK; + ordered_store_mtx(lock, state); + interlock_unlock(lock); +#else + ordered_store_mtx(lock, state); + enable_preemption(); +#endif + +#if CONFIG_DTRACE + LOCKSTAT_RECORD(LS_LCK_MTX_UNLOCK_RELEASE, lock, 0); +#endif /* CONFIG_DTRACE */ +} + +/* + * Routine: lck_mtx_assert + */ +void +lck_mtx_assert(lck_mtx_t *lock, unsigned int type) +{ + thread_t thread, holder; + uintptr_t state; + + state = ordered_load_mtx(lock); + holder = LCK_MTX_STATE_TO_THREAD(state); + if (holder == (thread_t)LCK_MTX_SPIN_TAG) { + // Lock is held in spin mode, owner is unknown. + return; // Punt + } + thread = current_thread(); + if (type == LCK_MTX_ASSERT_OWNED) { + if (thread != holder) + panic("lck_mtx_assert(): mutex (%p) owned", lock); + } else if (type == LCK_MTX_ASSERT_NOTOWNED) { + if (thread == holder) + panic("lck_mtx_assert(): mutex (%p) not owned", lock); + } else + panic("lck_mtx_assert(): invalid arg (%u)", type); +} + +/* + * Routine: lck_mtx_ilk_unlock + */ +boolean_t +lck_mtx_ilk_unlock(lck_mtx_t *lock) +{ + interlock_unlock(lock); + return TRUE; +} + +/* + * Routine: lck_mtx_convert_spin + * + * Convert a mutex held for spin into a held full mutex + */ +void +lck_mtx_convert_spin(lck_mtx_t *lock) +{ + thread_t thread = current_thread(); + uintptr_t state; + int waiters; + + state = ordered_load_mtx(lock); + if (LCK_MTX_STATE_TO_THREAD(state) == thread) + return; // Already owned as mutex, return + if ((state & LCK_ILOCK) == 0 || (LCK_MTX_STATE_TO_THREAD(state) != (thread_t)LCK_MTX_SPIN_TAG)) + panic("lck_mtx_convert_spin: Not held as spinlock (%p)", lock); + state &= ~(LCK_MTX_THREAD_MASK); // Clear the spin tag + ordered_store_mtx(lock, state); + waiters = lck_mtx_lock_acquire(lock); // Acquire to manage priority boosts + state = LCK_MTX_THREAD_TO_STATE(thread); + if (waiters != 0) + state |= ARM_LCK_WAITERS; +#if __SMP__ + state |= LCK_ILOCK; + ordered_store_mtx(lock, state); // Set ownership + interlock_unlock(lock); // Release interlock, enable preemption +#else + ordered_store_mtx(lock, state); // Set ownership + enable_preemption(); +#endif +} + + +/* + * Routine: lck_mtx_destroy + */ +void +lck_mtx_destroy( + lck_mtx_t * lck, + lck_grp_t * grp) +{ + if (lck->lck_mtx_type != LCK_MTX_TYPE) + panic("Destroying invalid mutex %p", lck); + if (lck->lck_mtx_tag == LCK_MTX_TAG_DESTROYED) + panic("Destroying previously destroyed lock %p", lck); + lck_mtx_assert(lck, LCK_MTX_ASSERT_NOTOWNED); + lck->lck_mtx_tag = LCK_MTX_TAG_DESTROYED; + lck_grp_lckcnt_decr(grp, LCK_TYPE_MTX); + lck_grp_deallocate(grp); + return; +} + +/* + * Routine: lck_spin_assert + */ +void +lck_spin_assert(lck_spin_t *lock, unsigned int type) +{ + thread_t thread, holder; + uintptr_t state; + + if (lock->type != LCK_SPIN_TYPE) + panic("Invalid spinlock %p", lock); + + state = lock->lck_spin_data; + holder = (thread_t)(state & ~LCK_ILOCK); + thread = current_thread(); + if (type == LCK_ASSERT_OWNED) { + if (holder == 0) + panic("Lock not owned %p = %lx", lock, state); + if (holder != thread) + panic("Lock not owned by current thread %p = %lx", lock, state); + if ((state & LCK_ILOCK) == 0) + panic("Lock bit not set %p = %lx", lock, state); + } else if (type == LCK_ASSERT_NOTOWNED) { + if (holder != 0) { + if (holder == thread) + panic("Lock owned by current thread %p = %lx", lock, state); + else + panic("Lock %p owned by thread %p", lock, holder); + } + if (state & LCK_ILOCK) + panic("Lock bit set %p = %lx", lock, state); + } else + panic("lck_spin_assert(): invalid arg (%u)", type); +} + +boolean_t +lck_rw_lock_yield_shared(lck_rw_t *lck, boolean_t force_yield) +{ + lck_rw_word_t word; + + lck_rw_assert(lck, LCK_RW_ASSERT_SHARED); + + word.data = ordered_load_rw(lck); + if (word.want_excl || word.want_upgrade || force_yield) { + lck_rw_unlock_shared(lck); + mutex_pause(2); + lck_rw_lock_shared(lck); + return TRUE; + } + + return FALSE; +} + +/* + * Routine: kdp_lck_mtx_lock_spin_is_acquired + * NOT SAFE: To be used only by kernel debugger to avoid deadlock. + */ +boolean_t +kdp_lck_mtx_lock_spin_is_acquired(lck_mtx_t *lck) +{ + uintptr_t state; + + if (not_in_kdp) { + panic("panic: spinlock acquired check done outside of kernel debugger"); + } + state = ordered_load_mtx(lck); + if (state == LCK_MTX_TAG_DESTROYED) + return FALSE; + if (LCK_MTX_STATE_TO_THREAD(state) || (state & LCK_ILOCK)) + return TRUE; + return FALSE; +} + +void +kdp_lck_mtx_find_owner(__unused struct waitq * waitq, event64_t event, thread_waitinfo_t * waitinfo) +{ + lck_mtx_t * mutex = LCK_EVENT_TO_MUTEX(event); + waitinfo->context = VM_KERNEL_UNSLIDE_OR_PERM(mutex); + uintptr_t state = ordered_load_mtx(mutex); + thread_t holder = LCK_MTX_STATE_TO_THREAD(state); + if ((uintptr_t)holder == (uintptr_t)LCK_MTX_SPIN_TAG) { + waitinfo->owner = STACKSHOT_WAITOWNER_MTXSPIN; + } else { + assertf(state != (uintptr_t)LCK_MTX_TAG_DESTROYED, "state=0x%llx", (uint64_t)state); + assertf(state != (uintptr_t)LCK_MTX_TAG_INDIRECT, "state=0x%llx", (uint64_t)state); + waitinfo->owner = thread_tid(holder); + } +} + +void +kdp_rwlck_find_owner(__unused struct waitq * waitq, event64_t event, thread_waitinfo_t * waitinfo) +{ + lck_rw_t *rwlck = NULL; + switch(waitinfo->wait_type) { + case kThreadWaitKernelRWLockRead: + rwlck = READ_EVENT_TO_RWLOCK(event); + break; + case kThreadWaitKernelRWLockWrite: + case kThreadWaitKernelRWLockUpgrade: + rwlck = WRITE_EVENT_TO_RWLOCK(event); + break; + default: + panic("%s was called with an invalid blocking type", __FUNCTION__); + break; + } + waitinfo->context = VM_KERNEL_UNSLIDE_OR_PERM(rwlck); + waitinfo->owner = thread_tid(rwlck->lck_rw_owner); +} diff --git a/osfmk/arm/locore.s b/osfmk/arm/locore.s new file mode 100644 index 000000000..5af8c2a79 --- /dev/null +++ b/osfmk/arm/locore.s @@ -0,0 +1,2041 @@ +/* + * Copyright (c) 2007-2011 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ + +#include <machine/asm.h> +#include <arm/proc_reg.h> +#include <pexpert/arm/board_config.h> +#include <mach/exception_types.h> +#include <mach_kdp.h> +#include <mach_assert.h> +#include <config_dtrace.h> +#include "assym.s" + +#define TRACE_SYSCALL 0 + +/* + * Copied to low physical memory in arm_init, + * so the kernel must be linked virtually at + * 0xc0001000 or higher to leave space for it. + */ + .syntax unified + .text + .align 12 + .globl EXT(ExceptionLowVectorsBase) + +LEXT(ExceptionLowVectorsBase) + adr pc, Lreset_low_vector + b . // Undef + b . // SWI + b . // Prefetch Abort + b . // Data Abort + b . // Address Exception + b . // IRQ + b . // FIQ/DEC +LEXT(ResetPrivateData) + .space (480),0 // (filled with 0s) + // ExceptionLowVectorsBase + 0x200 +Lreset_low_vector: + adr r4, EXT(ResetHandlerData) + ldr r0, [r4, ASSIST_RESET_HANDLER] + movs r0, r0 + blxne r0 + adr r4, EXT(ResetHandlerData) + ldr r1, [r4, CPU_DATA_ENTRIES] + ldr r1, [r1, CPU_DATA_PADDR] + ldr r5, [r1, CPU_RESET_ASSIST] + movs r5, r5 + blxne r5 + adr r4, EXT(ResetHandlerData) + ldr r0, [r4, BOOT_ARGS] + ldr r1, [r4, CPU_DATA_ENTRIES] +#if __ARM_SMP__ +#if defined(ARMA7) + // physical cpu number is stored in MPIDR Affinity level 0 + mrc p15, 0, r6, c0, c0, 5 // Read MPIDR + and r6, r6, #0xFF // Extract Affinity level 0 +#else +#error missing Who Am I implementation +#endif +#else + mov r6, #0 +#endif /* __ARM_SMP__ */ + // physical cpu number matches cpu number +//#if cdeSize != 16 +//#error cpu_data_entry is not 16bytes in size +//#endif + lsl r6, r6, #4 // Get CpuDataEntry offset + add r1, r1, r6 // Get cpu_data_entry pointer + ldr r1, [r1, CPU_DATA_PADDR] + ldr r5, [r1, CPU_RESET_HANDLER] + movs r5, r5 + blxne r5 // Branch to cpu reset handler + b . // Unexpected reset + .globl EXT(ResetHandlerData) +LEXT(ResetHandlerData) + .space (rhdSize_NUM),0 // (filled with 0s) + + + .globl EXT(ExceptionLowVectorsEnd) +LEXT(ExceptionLowVectorsEnd) + + .text + .align 12 + .globl EXT(ExceptionVectorsBase) + +LEXT(ExceptionVectorsBase) + + adr pc, Lexc_reset_vector + adr pc, Lexc_undefined_inst_vector + adr pc, Lexc_swi_vector + adr pc, Lexc_prefetch_abort_vector + adr pc, Lexc_data_abort_vector + adr pc, Lexc_address_exception_vector + adr pc, Lexc_irq_vector +#if __ARM_TIME__ + adr pc, Lexc_decirq_vector +#else /* ! __ARM_TIME__ */ + mov pc, r9 +#endif /* __ARM_TIME__ */ + +Lexc_reset_vector: + b . + .long 0x0 + .long 0x0 + .long 0x0 +Lexc_undefined_inst_vector: + mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW + ldr sp, [sp, ACT_CPUDATAP] // Get current cpu data + ldr sp, [sp, CPU_EXC_VECTORS] // Get exception vector table + ldr pc, [sp, #4] // Branch to exception handler +Lexc_swi_vector: + mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW + ldr sp, [sp, ACT_CPUDATAP] // Get current cpu data + ldr sp, [sp, CPU_EXC_VECTORS] // Get exception vector table + ldr pc, [sp, #8] // Branch to exception handler +Lexc_prefetch_abort_vector: + mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW + ldr sp, [sp, ACT_CPUDATAP] // Get current cpu data + ldr sp, [sp, CPU_EXC_VECTORS] // Get exception vector table + ldr pc, [sp, #0xC] // Branch to exception handler +Lexc_data_abort_vector: + mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW + ldr sp, [sp, ACT_CPUDATAP] // Get current cpu data + ldr sp, [sp, CPU_EXC_VECTORS] // Get exception vector table + ldr pc, [sp, #0x10] // Branch to exception handler +Lexc_address_exception_vector: + mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW + ldr sp, [sp, ACT_CPUDATAP] // Get current cpu data + ldr sp, [sp, CPU_EXC_VECTORS] // Get exception vector table + ldr pc, [sp, #0x14] // Branch to exception handler +Lexc_irq_vector: + mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW + ldr sp, [sp, ACT_CPUDATAP] // Get current cpu data + ldr sp, [sp, CPU_EXC_VECTORS] // Get exception vector table + ldr pc, [sp, #0x18] // Branch to exception handler +#if __ARM_TIME__ +Lexc_decirq_vector: + mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW + ldr sp, [sp, ACT_CPUDATAP] // Get current cpu data + ldr sp, [sp, CPU_EXC_VECTORS] // Get exception vector table + ldr pc, [sp, #0x1C] // Branch to exception handler +#else /* ! __ARM_TIME__ */ + .long 0x0 + .long 0x0 + .long 0x0 + .long 0x0 +#endif /* __ARM_TIME__ */ + + .fill 984, 4, 0 // Push to the 4KB page boundary + + .globl EXT(ExceptionVectorsEnd) +LEXT(ExceptionVectorsEnd) + + +/* + * Targets for the exception vectors; we patch these during boot (to allow + * for position independent code without complicating the vectors; see start.s). + */ + .globl EXT(ExceptionVectorsTable) +LEXT(ExceptionVectorsTable) +Lreset_vector: + .long 0x0 +Lundefined_inst_vector: + .long 0x0 +Lswi_vector: + .long 0x0 +Lprefetch_abort_vector: + .long 0x0 +Ldata_abort_vector: + .long 0x0 +Laddress_exception_vector: + .long 0x0 +Lirq_vector: + .long 0x0 +Ldecirq_vector: + .long 0x0 + + +/* + * First Level Exception Handlers + */ + .text + .align 2 + .globl EXT(fleh_reset) +LEXT(fleh_reset) + b . // Never return + +/* + * First Level Exception Handler for Undefined Instruction. + */ + .text + .align 2 + .globl EXT(fleh_undef) + +LEXT(fleh_undef) + mrs sp, spsr // Check the previous mode + tst sp, #PSR_TF // Is it Thumb? + subeq lr, lr, #4 + subne lr, lr, #2 + tst sp, #0x0f // Is it from user? + bne undef_from_kernel + +undef_from_user: + mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW + add sp, sp, ACT_PCBDATA // Get current thread PCB pointer + + stmia sp, {r0-r12, sp, lr}^ // Save user context on PCB + mov r7, #0 // Zero the frame pointer + nop + + mov r0, sp // Store arm_saved_state pointer + // for argument + + str lr, [sp, SS_PC] // Save user mode pc register + + mrs r4, spsr + str r4, [sp, SS_CPSR] // Save user mode cpsr + + mrs r4, cpsr // Read cpsr + cpsid i, #PSR_SVC_MODE + mrs r3, cpsr // Read cpsr + msr spsr_cxsf, r3 // Set spsr(svc mode cpsr) + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + ldr sp, [r9, TH_KSTACKPTR] // Load kernel stack +#if __ARM_USER_PROTECT__ + ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 + mov r3, #0 // Load kernel asid + mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + and r0, r4, #PSR_MODE_MASK // Extract current mode + cmp r0, #PSR_UND_MODE // Check undef mode + bne EXT(ExceptionVectorPanic) + + mvn r0, #0 + str r0, [r9, TH_IOTIER_OVERRIDE] // Reset IO tier override to -1 before handling abort from userspace + +#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME + bl EXT(timer_state_event_user_to_kernel) + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW +#endif + +#if __ARM_VFP__ + add r0, r9, ACT_UVFP // Get the address of the user VFP save area + bl EXT(vfp_save) // Save the current VFP state to ACT_UVFP + mov r3, #FPSCR_DEFAULT // Load up the default FPSCR value... + fmxr fpscr, r3 // And shove it into FPSCR + add r1, r9, ACT_UVFP // Reload the pointer to the save state + add r0, r9, ACT_PCBDATA // Reload the VFP save state argument +#else + mov r1, #0 // Clear the VFP save state argument + add r0, r9, ACT_PCBDATA // Reload arm_saved_state pointer +#endif + + bl EXT(sleh_undef) // Call second level handler + // sleh will enable interrupt + b load_and_go_user + +undef_from_kernel: + mrs sp, cpsr // Read cpsr + and sp, sp, #PSR_MODE_MASK // Extract current mode + cmp sp, #PSR_UND_MODE // Check undef mode + movne r0, sp + bne EXT(ExceptionVectorPanic) + mrs sp, spsr // Check the previous mode + + /* + * We have a kernel stack already, and I will use it to save contexts + * IRQ is disabled + */ + +#if CONFIG_DTRACE + /* + * See if we came here from IRQ or SVC mode, and go back to that mode + */ + + and sp, sp, #PSR_MODE_MASK + cmp sp, #PSR_IRQ_MODE + bne undef_from_kernel_svc + + cpsid i, #PSR_IRQ_MODE + b handle_undef +#endif + +undef_from_kernel_svc: + cpsid i, #PSR_SVC_MODE + +handle_undef: +#if CONFIG_DTRACE + // We need a frame for backtracing. The LR here is the LR of supervisor mode, not the location where the exception + // took place. We'll store that later after we switch to undef mode and pull out the LR from there. + + // This frame is consumed by fbt_invop. Any changes with the size or location of this frame will probably require + // changes in fbt_invop also. + stmfd sp!, { r7, lr } +#endif + + sub sp, sp, EXC_CTX_SIZE // Reserve for arm_saved_state + + stmia sp, {r0-r12} // Save on supervisor mode stack + str lr, [sp, SS_LR] + +#if CONFIG_DTRACE + add r7, sp, EXC_CTX_SIZE // Save frame pointer +#endif + + mov ip, sp // Stack transfer + + cpsid i, #PSR_UND_MODE + + str lr, [ip, SS_PC] // Save complete + mrs r4, spsr + str r4, [ip, SS_CPSR] + +#if CONFIG_DTRACE + /* + * Go back to previous mode for mode specific regs + */ + and r4, r4, #PSR_MODE_MASK + cmp r4, #PSR_IRQ_MODE + bne handle_undef_from_svc + + cpsid i, #PSR_IRQ_MODE + b handle_undef2 +#endif + +handle_undef_from_svc: + cpsid i, #PSR_SVC_MODE + +handle_undef2: + +/* + sp - stack pointer + ip - stack pointer + r7 - frame pointer state + */ + + +#if CONFIG_DTRACE + ldr r0, [ip, SS_PC] // Get the exception pc to store later +#endif + + add ip, ip, EXC_CTX_SIZE // Send stack pointer to debugger +#if CONFIG_DTRACE + str r0, [ip, #4] + add ip, ip, #8 +#endif + str ip, [sp, SS_SP] // for accessing local variable +#if CONFIG_DTRACE + sub ip, ip, #8 +#endif + sub ip, ip, EXC_CTX_SIZE + +#if __ARM_VFP__ + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + add r0, sp, SS_SIZE // Get vfp state pointer + bic r0, #(VSS_ALIGN_NUM - 1) // Align to arm_vfpsaved_state alignment + add r0, VSS_ALIGN // Get the actual vfp save area + mov r5, r0 // Stash the save area in another register + bl EXT(vfp_save) // Save the current VFP state to the stack + mov r1, r5 // Load the VFP save area argument + mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value... + fmxr fpscr, r4 // And shove it into FPSCR +#else + mov r1, #0 // Clear the facility context argument +#endif +#if __ARM_USER_PROTECT__ + mrc p15, 0, r10, c2, c0, 0 // Get TTBR0 + ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb + cmp r3, r10 + beq 1f + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 +1: + mrc p15, 0, r11, c13, c0, 1 // Save CONTEXTIDR + mov r3, #0 // Load kernel asid + mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + mov r0, sp // Argument + +/* + * For armv7k ABI, the stack needs to be 16-byte aligned + */ +#if __BIGGEST_ALIGNMENT__ > 4 + and r1, sp, #0x0F // sp mod 16-bytes + cmp r1, #4 // need space for the sp on the stack + addlt r1, r1, #0x10 // make room if needed, but keep stack aligned + mov r2, sp // get current sp + sub sp, sp, r1 // align stack + str r2, [sp] // store previous sp on stack +#endif + + bl EXT(sleh_undef) // Call second level handler + +#if __BIGGEST_ALIGNMENT__ > 4 + ldr sp, [sp] // restore stack +#endif + +#if __ARM_USER_PROTECT__ + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + ldr r0, [r9, ACT_KPTW_TTB] // Load kernel ttb + cmp r10, r0 + beq 1f + ldr r10, [r9, ACT_UPTW_TTB] // Load thread ttb + cmp r10, r0 + beq 1f + mcr p15, 0, r10, c2, c0, 0 // Set TTBR0 + ldr r11, [r9, ACT_ASID] // Load thread asid +1: + mcr p15, 0, r11, c13, c0, 1 // set CONTEXTIDR + isb +#endif + b load_and_go_sys + + +/* + * First Level Exception Handler for Software Interrupt + * + * We assert that only user level can use the "SWI" instruction for a system + * call on development kernels, and assume it's true on release. + * + * System call number is stored in r12. + * System call arguments are stored in r0 to r6 and r8 (we skip r7) + * + */ + .text + .align 5 + .globl EXT(fleh_swi) + +LEXT(fleh_swi) + cpsid i, #PSR_ABT_MODE + mov sp, ip // Save ip + cpsid i, #PSR_SVC_MODE + mrs ip, spsr // Check the previous mode + tst ip, #0x0f + cpsid i, #PSR_ABT_MODE + mov ip, sp // Restore ip + cpsid i, #PSR_SVC_MODE + beq swi_from_user + +/* Only user mode can use SWI. Panic if the kernel tries. */ +swi_from_kernel: + sub sp, sp, EXC_CTX_SIZE + stmia sp, {r0-r12} + add r0, sp, EXC_CTX_SIZE + + str r0, [sp, SS_SP] // Save supervisor mode sp + str lr, [sp, SS_LR] // Save supervisor mode lr + + adr r0, L_kernel_swi_panic_str // Load panic messages and panic() + blx EXT(panic) + b . + +swi_from_user: + mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW + add sp, sp, ACT_PCBDATA // Get User PCB + + + /* Check for special mach_absolute_time trap value. + * This is intended to be a super-lightweight call to ml_get_timebase(), which + * is handrolled assembly and does not use the stack, thus not requiring us to setup a kernel stack. */ + cmp r12, #-3 + beq fleh_swi_trap_tb + stmia sp, {r0-r12, sp, lr}^ // Save user context on PCB + mov r7, #0 // Zero the frame pointer + nop + mov r8, sp // Store arm_saved_state pointer + add sp, sp, SS_PC + srsia sp, #PSR_SVC_MODE + mrs r3, cpsr // Read cpsr + msr spsr_cxsf, r3 // Set spsr(svc mode cpsr) + sub r9, sp, ACT_PCBDATA_PC + + ldr sp, [r9, TH_KSTACKPTR] // Load kernel stack + mov r11, r12 // save the syscall vector in a nontrashed register + +#if __ARM_VFP__ + add r0, r9, ACT_UVFP // Get the address of the user VFP save area + bl EXT(vfp_save) // Save the current VFP state to ACT_UVFP + mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value... + fmxr fpscr, r4 // And shove it into FPSCR +#endif +#if __ARM_USER_PROTECT__ + ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 + mov r3, #0 // Load kernel asid + mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + + mvn r0, #0 + str r0, [r9, TH_IOTIER_OVERRIDE] // Reset IO tier override to -1 before handling SWI from userspace + +#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME + bl EXT(timer_state_event_user_to_kernel) + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + add r8, r9, ACT_PCBDATA // Reload arm_saved_state pointer +#endif + ldr r10, [r9, ACT_TASK] // Load the current task + + /* enable interrupts */ + cpsie i // Enable IRQ + + cmp r11, #-4 // Special value for mach_continuous_time + beq fleh_swi_trap_mct + + cmp r11, #0x80000000 + beq fleh_swi_trap +fleh_swi_trap_ret: + +#if TRACE_SYSCALL + /* trace the syscall */ + mov r0, r8 + bl EXT(syscall_trace) +#endif + + bl EXT(mach_kauth_cred_uthread_update) + mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW + /* unix syscall? */ + rsbs r5, r11, #0 // make the syscall positive (if negative) + ble fleh_swi_unix // positive syscalls are unix (note reverse logic here) + +fleh_swi_mach: + /* note that mach_syscall_trace can modify r9, so increment the thread + * syscall count before the call : */ + ldr r2, [r9, TH_MACH_SYSCALLS] + add r2, r2, #1 + str r2, [r9, TH_MACH_SYSCALLS] + + LOAD_ADDR(r1, mach_trap_table) // load mach_trap_table +#if MACH_TRAP_TABLE_ENTRY_SIZE_NUM == 12 + add r11, r5, r5, lsl #1 // syscall * 3 + add r6, r1, r11, lsl #2 // trap_table + syscall * 12 +#elif MACH_TRAP_TABLE_ENTRY_SIZE_NUM == 16 + add r6, r1, r5, lsl #4 // trap_table + syscall * 16 +#elif MACH_TRAP_TABLE_ENTRY_SIZE_NUM == 20 + add r11, r5, r5, lsl #2 // syscall * 5 + add r6, r1, r11, lsl #2 // trap_table + syscall * 20 +#else +#error mach_trap_t size unhandled (see MACH_TRAP_TABLE_ENTRY_SIZE)! +#endif + +#ifndef NO_KDEBUG + LOAD_ADDR(r4, kdebug_enable) + ldr r4, [r4] + movs r4, r4 + movne r0, r8 // ready the reg state pointer as an arg to the call + movne r1, r5 // syscall number as 2nd arg + COND_EXTERN_BLNE(mach_syscall_trace) +#endif + adr lr, fleh_swi_exit // any calls from here on out will return to our exit path + cmp r5, MACH_TRAP_TABLE_COUNT // check syscall number range + bge fleh_swi_mach_error + +/* + * For arm32 ABI where 64-bit types are aligned to even registers and + * 64-bits on stack, we need to unpack registers differently. So + * we use the mungers for marshalling in arguments from user space. + * Currently this is just ARMv7k. + */ +#if __BIGGEST_ALIGNMENT__ > 4 + sub sp, #0x40 // allocate buffer and keep stack 128-bit aligned + // it should be big enough for all syscall arguments + ldr r11, [r6, #8] // get mach_trap_table[call_number].mach_trap_arg_munge32 + teq r11, #0 // check if we have a munger + moveq r0, #0 + movne r0, r8 // ready the reg state pointer as an arg to the call + movne r1, sp // stack will hold arguments buffer + blxne r11 // call munger to get arguments from userspace + adr lr, fleh_swi_exit // any calls from here on out will return to our exit path + teq r0, #0 + bne fleh_swi_mach_error // exit if the munger returned non-zero status +#endif + + ldr r1, [r6, #4] // load the syscall vector + + LOAD_ADDR(r2, kern_invalid) // test to make sure the trap is not kern_invalid + teq r1, r2 + beq fleh_swi_mach_error + +#if __BIGGEST_ALIGNMENT__ > 4 + mov r0, sp // argument buffer on stack + bx r1 // call the syscall handler +#else + mov r0, r8 // ready the reg state pointer as an arg to the call + bx r1 // call the syscall handler +#endif + +fleh_swi_exit64: + str r1, [r8, #4] // top of 64-bit return +fleh_swi_exit: + str r0, [r8] // save the return value +#ifndef NO_KDEBUG + movs r4, r4 + movne r1, r5 + COND_EXTERN_BLNE(mach_syscall_trace_exit) +#endif +#if TRACE_SYSCALL + bl EXT(syscall_trace_exit) +#endif + + mov r0, #1 + bl EXT(throttle_lowpri_io) // throttle_lowpri_io(1); + + bl EXT(thread_exception_return) + b . + +fleh_swi_mach_error: + mov r0, #EXC_SYSCALL + sub r1, sp, #4 + mov r2, #1 + bl EXT(exception_triage) + b . + + .align 5 +fleh_swi_unix: + ldr r1, [r9, TH_UNIX_SYSCALLS] + mov r0, r8 // reg state structure is arg + add r1, r1, #1 + str r1, [r9, TH_UNIX_SYSCALLS] + mov r1, r9 // current thread in arg1 + ldr r2, [r9, TH_UTHREAD] // current uthread in arg2 + ldr r3, [r10, TASK_BSD_INFO] // current proc in arg3 + bl EXT(unix_syscall) + b . + +fleh_swi_trap: + ldmia r8, {r0-r3} + cmp r3, #3 + addls pc, pc, r3, LSL#2 + b fleh_swi_trap_ret + b icache_invalidate_trap + b dcache_flush_trap + b thread_set_cthread_trap + b thread_get_cthread_trap + +icache_invalidate_trap: + add r3, r0, r1 + cmp r3, VM_MAX_ADDRESS + subhi r3, r3, #1<<MMU_CLINE + bhi cache_trap_error + adr r11, cache_trap_jmp + ldr r6, [r9, TH_RECOVER] // Save existing recovery routine + str r11, [r9, TH_RECOVER] +#if __ARM_USER_PROTECT__ + ldr r5, [r9, ACT_UPTW_TTB] // Load thread ttb + mcr p15, 0, r5, c2, c0, 0 // Set TTBR0 + ldr r5, [r9, ACT_ASID] // Load thread asid + mcr p15, 0, r5, c13, c0, 1 // Set CONTEXTIDR + dsb ish + isb +#endif + mov r4, r0 + mov r5, r1 + bl EXT(CleanPoU_DcacheRegion) + mov r0, r4 + mov r1, r5 + bl EXT(InvalidatePoU_IcacheRegion) + mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW +#if __ARM_USER_PROTECT__ + ldr r4, [r9, ACT_KPTW_TTB] // Load kernel ttb + mcr p15, 0, r4, c2, c0, 0 // Set TTBR0 + mov r4, #0 // Load kernel asid + mcr p15, 0, r4, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + str r6, [r9, TH_RECOVER] + bl EXT(thread_exception_return) + b . + +dcache_flush_trap: + add r3, r0, r1 + cmp r3, VM_MAX_ADDRESS + subhi r3, r3, #1<<MMU_CLINE + bhi cache_trap_error + adr r11, cache_trap_jmp + ldr r4, [r9, TH_RECOVER] // Save existing recovery routine + str r11, [r9, TH_RECOVER] +#if __ARM_USER_PROTECT__ + ldr r6, [r9, ACT_UPTW_TTB] // Load thread ttb + mcr p15, 0, r6, c2, c0, 0 // Set TTBR0 + ldr r5, [r9, ACT_ASID] // Load thread asid + mcr p15, 0, r5, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + bl EXT(flush_dcache_syscall) + mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW +#if __ARM_USER_PROTECT__ + ldr r5, [r9, ACT_KPTW_TTB] // Load kernel ttb + mcr p15, 0, r5, c2, c0, 0 // Set TTBR0 + mov r5, #0 // Load kernel asid + mcr p15, 0, r5, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + str r4, [r9, TH_RECOVER] + bl EXT(thread_exception_return) + b . + +thread_set_cthread_trap: + bl EXT(thread_set_cthread_self) + bl EXT(thread_exception_return) + b . + +thread_get_cthread_trap: + bl EXT(thread_get_cthread_self) + mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW + add r1, r9, ACT_PCBDATA // Get User PCB + str r0, [r1, SS_R0] // set return value + bl EXT(thread_exception_return) + b . + +cache_trap_jmp: +#if __ARM_USER_PROTECT__ + mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW + ldr r5, [r9, ACT_KPTW_TTB] // Load kernel ttb + mcr p15, 0, r5, c2, c0, 0 // Set TTBR0 + mov r5, #0 // Load kernel asid + mcr p15, 0, r5, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + mrc p15, 0, r3, c6, c0 // Read Fault Address +cache_trap_error: + mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW + add r0, r9, ACT_PCBDATA // Get User PCB + ldr r1, [r0, SS_PC] // Save user mode pc register as pc + sub r1, r1, #4 // Backtrack current pc + str r1, [r0, SS_PC] // pc at cache assist swi + str r3, [r0, SS_VADDR] // Fault Address + mov r0, #EXC_BAD_ACCESS + mov r2, KERN_INVALID_ADDRESS + sub sp, sp, #8 + mov r1, sp + str r2, [sp] + str r3, [sp, #4] + mov r2, #2 + bl EXT(exception_triage) + b . + +fleh_swi_trap_mct: + bl EXT(mach_continuous_time) + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + add r9, r9, ACT_PCBDATA_R0 // Get User register state + stmia r9, {r0, r1} // set 64-bit return value + bl EXT(thread_exception_return) + b . + +fleh_swi_trap_tb: + str lr, [sp, SS_PC] + bl EXT(ml_get_timebase) // ml_get_timebase() (64-bit return) + ldr lr, [sp, SS_PC] + nop + movs pc, lr // Return to user + + .align 2 +L_kernel_swi_panic_str: + .asciz "fleh_swi: took SWI from kernel mode\n" + .align 2 + +/* + * First Level Exception Handler for Prefetching Abort. + */ + .text + .align 2 + .globl EXT(fleh_prefabt) + +LEXT(fleh_prefabt) + sub lr, lr, #4 + + mrs sp, spsr // For check the previous mode + tst sp, #0x0f // Is it from user? + bne prefabt_from_kernel + +prefabt_from_user: + mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW + add sp, sp, ACT_PCBDATA // Get User PCB + + stmia sp, {r0-r12, sp, lr}^ // Save user context on PCB + mov r7, #0 // Zero the frame pointer + nop + mov r0, sp // Store arm_saved_state pointer + // For argument + str lr, [sp, SS_PC] // Save user mode pc register as pc + mrc p15, 0, r1, c6, c0, 2 // Read IFAR + str r1, [sp, SS_VADDR] // and fault address of pcb + + mrc p15, 0, r5, c5, c0, 1 // Read Fault Status + str r5, [sp, SS_STATUS] // Save fault status register to pcb + + mrs r4, spsr + str r4, [sp, SS_CPSR] // Save user mode cpsr + + mrs r4, cpsr // Read cpsr + cpsid i, #PSR_SVC_MODE + mrs r3, cpsr // Read cpsr + msr spsr_cxsf, r3 // Set spsr(svc mode cpsr) + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + ldr sp, [r9, TH_KSTACKPTR] // Load kernel stack + +#if __ARM_VFP__ + add r0, r9, ACT_UVFP // Get the address of the user VFP save area + bl EXT(vfp_save) // Save the current VFP state to ACT_UVFP + mov r3, #FPSCR_DEFAULT // Load up the default FPSCR value... + fmxr fpscr, r3 // And shove it into FPSCR +#endif +#if __ARM_USER_PROTECT__ + ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 + mov r3, #0 // Load kernel asid + mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + and r0, r4, #PSR_MODE_MASK // Extract current mode + cmp r0, #PSR_ABT_MODE // Check abort mode + bne EXT(ExceptionVectorPanic) + + mvn r0, #0 + str r0, [r9, TH_IOTIER_OVERRIDE] // Reset IO tier override to -1 before handling abort from userspace + +#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME + bl EXT(timer_state_event_user_to_kernel) + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW +#endif + + add r0, r9, ACT_PCBDATA // Reload arm_saved_state pointer + mov r1, T_PREFETCH_ABT // Pass abort type + bl EXT(sleh_abort) // Call second level handler + // Sleh will enable interrupt + b load_and_go_user + +prefabt_from_kernel: + mrs sp, cpsr // Read cpsr + and sp, sp, #PSR_MODE_MASK // Extract current mode + cmp sp, #PSR_ABT_MODE // Check abort mode + movne r0, sp + bne EXT(ExceptionVectorPanic) + mrs sp, spsr // Check the previous mode + + /* + * We have a kernel stack already, and I will use it to save contexts: + * ------------------ + * | VFP saved state | + * |------------------| + * | ARM saved state | + * SP ------------------ + * + * IRQ is disabled + */ + cpsid i, #PSR_SVC_MODE + + sub sp, sp, EXC_CTX_SIZE + stmia sp, {r0-r12} + add r0, sp, EXC_CTX_SIZE + + str r0, [sp, SS_SP] // Save supervisor mode sp + str lr, [sp, SS_LR] // Save supervisor mode lr + + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + +#if __ARM_VFP__ + add r0, sp, SS_SIZE // Get vfp state pointer + bic r0, #(VSS_ALIGN_NUM - 1) // Align to arm_vfpsaved_state alignment + add r0, VSS_ALIGN // Get the actual vfp save area + bl EXT(vfp_save) // Save the current VFP state to the stack + mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value... + fmxr fpscr, r4 // And shove it into FPSCR +#endif +#if __ARM_USER_PROTECT__ + mrc p15, 0, r10, c2, c0, 0 // Get TTBR0 + ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb + cmp r3, r10 + beq 1f + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 +1: + mrc p15, 0, r11, c13, c0, 1 // Save CONTEXTIDR + mov r3, #0 // Load kernel asid + mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + mov ip, sp + + cpsid i, #PSR_ABT_MODE + + str lr, [ip, SS_PC] // Save pc to pc and + + mrc p15, 0, r5, c6, c0, 2 // Read IFAR + str r5, [ip, SS_VADDR] // and fault address of pcb + mrc p15, 0, r5, c5, c0, 1 // Read (instruction) Fault Status + str r5, [ip, SS_STATUS] // Save fault status register to pcb + + mrs r4, spsr + str r4, [ip, SS_CPSR] + + cpsid i, #PSR_SVC_MODE + + mov r0, sp + +/* + * For armv7k ABI, the stack needs to be 16-byte aligned + */ +#if __BIGGEST_ALIGNMENT__ > 4 + and r1, sp, #0x0F // sp mod 16-bytes + cmp r1, #4 // need space for the sp on the stack + addlt r1, r1, #0x10 // make room if needed, but keep stack aligned + mov r2, sp // get current sp + sub sp, sp, r1 // align stack + str r2, [sp] // store previous sp on stack +#endif + + mov r1, T_PREFETCH_ABT // Pass abort type + bl EXT(sleh_abort) // Call second level handler + +#if __BIGGEST_ALIGNMENT__ > 4 + ldr sp, [sp] // restore stack +#endif + + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW +#if __ARM_USER_PROTECT__ + ldr r0, [r9, ACT_KPTW_TTB] // Load kernel ttb + cmp r10, r0 + beq 1f + ldr r10, [r9, ACT_UPTW_TTB] // Load thread ttb + cmp r10, r0 + beq 1f + mcr p15, 0, r10, c2, c0, 0 // Set TTBR0 + ldr r11, [r9, ACT_ASID] // Load thread asid +1: + mcr p15, 0, r11, c13, c0, 1 // set CONTEXTIDR + isb +#endif + + b load_and_go_sys + + +/* + * First Level Exception Handler for Data Abort + */ + .text + .align 2 + .globl EXT(fleh_dataabt) + +LEXT(fleh_dataabt) + sub lr, lr, #8 + + mrs sp, spsr // For check the previous mode + tst sp, #0x0f // Is it from kernel? + bne dataabt_from_kernel + +dataabt_from_user: + mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW + add sp, sp, ACT_PCBDATA // Get User PCB + + stmia sp, {r0-r12, sp, lr}^ // Save user context on PCB + mov r7, #0 // Zero the frame pointer + nop + + mov r0, sp // Store arm_saved_state pointer + // For argument + + str lr, [sp, SS_PC] // Save user mode pc register + + mrs r4, spsr + str r4, [sp, SS_CPSR] // Save user mode cpsr + + mrc p15, 0, r5, c5, c0 // Read Fault Status + mrc p15, 0, r6, c6, c0 // Read Fault Address + str r5, [sp, SS_STATUS] // Save fault status register to pcb + str r6, [sp, SS_VADDR] // Save fault address to pcb + + mrs r4, cpsr // Read cpsr + cpsid i, #PSR_SVC_MODE + mrs r3, cpsr // Read cpsr + msr spsr_cxsf, r3 // Set spsr(svc mode cpsr) + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + ldr sp, [r9, TH_KSTACKPTR] // Load kernel stack + +#if __ARM_VFP__ + add r0, r9, ACT_UVFP // Get the address of the user VFP save area + bl EXT(vfp_save) // Save the current VFP state to ACT_UVFP + mov r3, #FPSCR_DEFAULT // Load up the default FPSCR value... + fmxr fpscr, r3 // And shove it into FPSCR +#endif +#if __ARM_USER_PROTECT__ + ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 + mov r3, #0 // Load kernel asid + mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + and r0, r4, #PSR_MODE_MASK // Extract current mode + cmp r0, #PSR_ABT_MODE // Check abort mode + bne EXT(ExceptionVectorPanic) + + mvn r0, #0 + str r0, [r9, TH_IOTIER_OVERRIDE] // Reset IO tier override to -1 before handling abort from userspace + +#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME + bl EXT(timer_state_event_user_to_kernel) + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW +#endif + + add r0, r9, ACT_PCBDATA // Reload arm_saved_state pointer + mov r1, T_DATA_ABT // Pass abort type + bl EXT(sleh_abort) // Call second level handler + // Sleh will enable irq + b load_and_go_user + +dataabt_from_kernel: + mrs sp, cpsr // Read cpsr + and sp, sp, #PSR_MODE_MASK // Extract current mode + cmp sp, #PSR_ABT_MODE // Check abort mode + movne r0, sp + bne EXT(ExceptionVectorPanic) + mrs sp, spsr // Check the previous mode + + /* + * We have a kernel stack already, and I will use it to save contexts: + * ------------------ + * | VFP saved state | + * |------------------| + * | ARM saved state | + * SP ------------------ + * + * IRQ is disabled + */ + cpsid i, #PSR_SVC_MODE + + sub sp, sp, EXC_CTX_SIZE + stmia sp, {r0-r12} + add r0, sp, EXC_CTX_SIZE + + str r0, [sp, SS_SP] // Save supervisor mode sp + str lr, [sp, SS_LR] // Save supervisor mode lr + + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + +#if __ARM_VFP__ + add r0, sp, SS_SIZE // Get vfp state pointer + bic r0, #(VSS_ALIGN_NUM - 1) // Align to arm_vfpsaved_state alignment + add r0, VSS_ALIGN // Get the actual vfp save area + bl EXT(vfp_save) // Save the current VFP state to the stack + mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value... + fmxr fpscr, r4 // And shove it into FPSCR +#endif + + mov ip, sp + + cpsid i, #PSR_ABT_MODE + + str lr, [ip, SS_PC] + mrs r4, spsr + str r4, [ip, SS_CPSR] + + cpsid i, #PSR_SVC_MODE + +#if __ARM_USER_PROTECT__ + mrc p15, 0, r10, c2, c0, 0 // Get TTBR0 + ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb + cmp r3, r10 + beq 1f + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 +1: + mrc p15, 0, r11, c13, c0, 1 // Save CONTEXTIDR + mov r3, #0 // Load kernel asid + mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + mrc p15, 0, r5, c5, c0 // Read Fault Status + mrc p15, 0, r6, c6, c0 // Read Fault Address + str r5, [sp, SS_STATUS] // Save fault status register to pcb + str r6, [sp, SS_VADDR] // Save fault address to pcb + + mov r0, sp // Argument + +/* + * For armv7k ABI, the stack needs to be 16-byte aligned + */ +#if __BIGGEST_ALIGNMENT__ > 4 + and r1, sp, #0x0F // sp mod 16-bytes + cmp r1, #4 // need space for the sp on the stack + addlt r1, r1, #0x10 // make room if needed, but keep stack aligned + mov r2, sp // get current sp + sub sp, sp, r1 // align stack + str r2, [sp] // store previous sp on stack +#endif + + mov r1, T_DATA_ABT // Pass abort type + bl EXT(sleh_abort) // Call second level handler + +#if __BIGGEST_ALIGNMENT__ > 4 + ldr sp, [sp] // restore stack (removed align padding) +#endif + + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW +#if __ARM_USER_PROTECT__ + ldr r0, [r9, ACT_KPTW_TTB] // Load kernel ttb + cmp r10, r0 + beq 1f + ldr r10, [r9, ACT_UPTW_TTB] // Load thread ttb + cmp r10, r0 + beq 1f + mcr p15, 0, r10, c2, c0, 0 // Set TTBR0 + ldr r11, [r9, ACT_ASID] // Load thread asid +1: + mcr p15, 0, r11, c13, c0, 1 // set CONTEXTIDR + isb +#endif + +load_and_go_sys: + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + + ldr r4, [sp, SS_CPSR] // Load saved cpsr + tst r4, #PSR_IRQF // Test IRQ set + bne lags1 // Branch if IRQ disabled + + cpsid i // Disable IRQ + ldr r2, [r9, ACT_PREEMPT_CNT] // Load preemption count + movs r2, r2 // Test if null + ldr r8, [r9, ACT_CPUDATAP] // Get current cpu + bne lags1 // Branch if count not null + ldr r5, [r8, CPU_PENDING_AST] // Get ASTs + ands r5, r5, AST_URGENT // Get the requests we do honor + beq lags1 // Branch if no ASTs +#if __ARM_USER_PROTECT__ + mrc p15, 0, r10, c2, c0, 0 // Get TTBR0 + ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb + cmp r3, r10 + beq 1f + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 +1: + mrc p15, 0, r11, c13, c0, 1 // Save CONTEXTIDR + mov r3, #0 // Load kernel asid + mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + ldr lr, [sp, SS_LR] // Restore the link register + stmfd sp!, {r7, lr} // Push a fake frame + + /* TODO: Should this be setting r7? I think so. */ + mov r7, sp // Set the frame pointer + +#if __BIGGEST_ALIGNMENT__ > 4 + and r2, sp, #0x0F // sp mod 16-bytes + cmp r2, #4 // need space for the sp on the stack + addlt r2, r2, #0x10 // make room if needed, but keep stack aligned + mov r3, sp // get current sp + sub sp, sp, r2 // align stack + str r3, [sp] // store previous sp on stack +#endif + + bl EXT(ast_taken_kernel) // Handle AST_URGENT + +#if __BIGGEST_ALIGNMENT__ > 4 + ldr sp, [sp] +#endif + + + ldmfd sp!, {r7, lr} // Pop the fake frame + mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW + ldr r8, [r9, ACT_CPUDATAP] // Get current cpu +#if __ARM_USER_PROTECT__ + ldr r0, [r9, ACT_KPTW_TTB] // Load kernel ttb + cmp r10, r0 + beq 1f + ldr r10, [r9, ACT_UPTW_TTB] // Load thread ttb + cmp r10, r0 + beq 1f + mcr p15, 0, r10, c2, c0, 0 // Set TTBR0 + ldr r11, [r9, ACT_ASID] // Load thread asid +1: + mcr p15, 0, r11, c13, c0, 1 // set CONTEXTIDR + isb +#endif +lags1: + ldr lr, [sp, SS_LR] + + mov ip, sp // Save pointer to contexts for abort mode + ldr sp, [ip, SS_SP] // Restore stack pointer + + cpsid if, #PSR_ABT_MODE + + mov sp, ip + + ldr r4, [sp, SS_CPSR] + msr spsr_cxsf, r4 // Restore spsr + + clrex // clear exclusive memory tag +#if __ARM_ENABLE_WFE_ + sev +#endif + +#if __ARM_VFP__ + add r0, sp, SS_SIZE // Get vfp state pointer + bic r0, #(VSS_ALIGN_NUM - 1) // Align to arm_vfpsaved_state alignment + add r0, VSS_ALIGN // Get the actual vfp save area + bl EXT(vfp_load) // Load the desired VFP state from the stack +#endif + + ldr lr, [sp, SS_PC] // Restore lr + + ldmia sp, {r0-r12} // Restore other registers + + movs pc, lr // Return to sys (svc, irq, fiq) + +/* + * First Level Exception Handler for address exception + * Not supported + */ + .text + .align 2 + .globl EXT(fleh_addrexc) + +LEXT(fleh_addrexc) + b . + + +/* + * First Level Exception Handler for IRQ + * Current mode : IRQ + * IRQ and FIQ are always disabled while running in FIQ handler + * We do not permit nested interrupt. + * + * Saving area: from user : PCB. + * from kernel : interrupt stack. + */ + + .text + .align 2 + .globl EXT(fleh_irq) + +LEXT(fleh_irq) + sub lr, lr, #4 + + cpsie a // Re-enable async aborts + + mrs sp, spsr + tst sp, #0x0f // From user? or kernel? + bne fleh_irq_kernel + +fleh_irq_user: + mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW + add sp, sp, ACT_PCBDATA // Get User PCB + stmia sp, {r0-r12, sp, lr}^ + mov r7, #0 // Zero the frame pointer + nop + str lr, [sp, SS_PC] + mrs r4, spsr + str r4, [sp, SS_CPSR] + mov r5, sp // Saved context in r5 + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + ldr r6, [r9, ACT_CPUDATAP] // Get current cpu + ldr sp, [r6, CPU_ISTACKPTR] // Set interrupt stack + cpsid i, #PSR_SVC_MODE + ldr sp, [r9, TH_KSTACKPTR] // Set kernel stack + cpsid i, #PSR_IRQ_MODE + +#if __ARM_VFP__ + add r0, r9, ACT_UVFP // Get the address of the user VFP save area + bl EXT(vfp_save) // Save the current VFP state to ACT_UVFP + mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value... + fmxr fpscr, r4 // And shove it into FPSCR +#endif +#if __ARM_USER_PROTECT__ + ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 + mov r3, #0 // Load kernel asid + mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR + isb +#endif +#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME + bl EXT(timer_state_event_user_to_kernel) + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW +#endif +#if CONFIG_TELEMETRY + LOAD_ADDR(r2, telemetry_needs_record) // Check if a telemetry record was requested... + mov r0, #1 + ldr r2, [r2] + movs r2, r2 + beq 1f + bl EXT(telemetry_mark_curthread) // ...if so, mark the current thread... + mrc p15, 0, r9, c13, c0, 4 // ...and restore the thread pointer from TPIDRPRW +1: +#endif + + b fleh_irq_handler + +fleh_irq_kernel: + cpsid i, #PSR_SVC_MODE + + sub sp, sp, EXC_CTX_SIZE + stmia sp, {r0-r12} + add r0, sp, EXC_CTX_SIZE + + str r0, [sp, SS_SP] // Save supervisor mode sp + str lr, [sp, SS_LR] // Save supervisor mode lr + + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + +#if __ARM_VFP__ + add r0, sp, SS_SIZE // Get vfp state pointer + bic r0, #(VSS_ALIGN_NUM - 1) // Align to arm_vfpsaved_state alignment + add r0, VSS_ALIGN // Get the actual vfp save area + bl EXT(vfp_save) // Save the current VFP state to the stack + mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value... + fmxr fpscr, r4 // And shove it into FPSCR +#endif +#if __ARM_USER_PROTECT__ + mrc p15, 0, r10, c2, c0, 0 // Get TTBR0 + ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 + mrc p15, 0, r11, c13, c0, 1 // Get CONTEXTIDR + mov r3, #0 // Load kernel asid + mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + mov r5, sp // Saved context in r5 + + cpsid i, #PSR_IRQ_MODE + + str lr, [r5, SS_PC] // Save LR as the return PC + mrs r4, spsr + str r4, [r5, SS_CPSR] // Save the cpsr of the interrupted mode + + ldr sp, [r9, ACT_CPUDATAP] // Get current cpu + ldr sp, [sp, CPU_ISTACKPTR] // Set interrupt stack + +#if CONFIG_TELEMETRY + LOAD_ADDR(r2, telemetry_needs_record) // Check if a telemetry record was requested... + mov r0, #0 + ldr r2, [r2] + movs r2, r2 + beq 1f + bl EXT(telemetry_mark_curthread) // ...if so, mark the current thread... + mrc p15, 0, r9, c13, c0, 4 // ...and restore the thread pointer from TPIDRPRW +1: +#endif + +fleh_irq_handler: + ldr r2, [r9, ACT_PREEMPT_CNT] // Load preemption count + add r2, r2, #1 // Increment count + str r2, [r9, ACT_PREEMPT_CNT] // Update preemption count +#ifndef NO_KDEBUG + LOAD_ADDR(r8, kdebug_enable) + ldr r8, [r8] + movs r8, r8 + movne r0, r5 + COND_EXTERN_BLNE(interrupt_trace) +#endif + bl EXT(interrupt_stats) // Record interrupt statistics + mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW + ldr r4, [r9, ACT_CPUDATAP] // Get current cpu + str r5, [r4, CPU_INT_STATE] // Saved context in cpu_int_state + ldr r3, [r4, CPU_STAT_IRQ] // Get IRQ count + add r3, r3, #1 // Increment count + str r3, [r4, CPU_STAT_IRQ] // Update IRQ count + ldr r3, [r4, CPU_STAT_IRQ_WAKE] // Get post-wake IRQ count + add r3, r3, #1 // Increment count + str r3, [r4, CPU_STAT_IRQ_WAKE] // Update post-wake IRQ count + ldr r0, [r4, INTERRUPT_TARGET] + ldr r1, [r4, INTERRUPT_REFCON] + ldr r2, [r4, INTERRUPT_NUB] + ldr r3, [r4, INTERRUPT_SOURCE] + ldr r5, [r4, INTERRUPT_HANDLER] // Call second level exception handler + blx r5 +#ifndef NO_KDEBUG + movs r8, r8 + COND_EXTERN_BLNE(interrupt_trace_exit) +#endif + mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW + bl EXT(ml_get_timebase) // get current timebase + LOAD_ADDR(r3, EntropyData) + ldr r2, [r3, ENTROPY_INDEX_PTR] + add r1, r3, ENTROPY_DATA_SIZE + add r2, r2, #4 + cmp r2, r1 + addge r2, r3, ENTROPY_BUFFER + ldr r4, [r2] + eor r0, r0, r4, ROR #9 + str r0, [r2] // Update gEntropie + str r2, [r3, ENTROPY_INDEX_PTR] + +return_from_irq: + mov r5, #0 + ldr r4, [r9, ACT_CPUDATAP] // Get current cpu + str r5, [r4, CPU_INT_STATE] // Clear cpu_int_state + ldr r2, [r9, ACT_PREEMPT_CNT] // Load preemption count +#if MACH_ASSERT + cmp r2, #0 // verify positive count + bgt 1f + push {r7, lr} + mov r7, sp + adr r0, L_preemption_count_zero_str + blx EXT(panic) + b . +1: +#endif + sub r2, r2, #1 // Decrement count + str r2, [r9, ACT_PREEMPT_CNT] // Update preemption count + + mrs r0, spsr // For check the previous mode + + cpsid i, #PSR_SVC_MODE + + tst r0, #0x0f // Check if the previous is from user + ldreq sp, [r9, TH_KSTACKPTR] // ...If so, reload the kernel stack pointer + beq load_and_go_user // ...and return + +#if __ARM_USER_PROTECT__ + ldr r0, [r9, ACT_KPTW_TTB] // Load kernel ttb + cmp r10, r0 + beq 1f + ldr r10, [r9, ACT_UPTW_TTB] // Load thread ttb + cmp r10, r0 + beq 1f + mcr p15, 0, r10, c2, c0, 0 // Set TTBR0 + ldr r11, [r9, ACT_ASID] // Load thread asid +1: + mcr p15, 0, r11, c13, c0, 1 // set CONTEXTIDR + isb +#endif + b load_and_go_sys + + .align 2 +L_preemption_count_zero_str: + .ascii "locore.s: preemption count is zero \000" + .align 2 +/* + * First Level Exception Handler for DEC + * Current mode : IRQ + * IRQ and FIQ are always disabled while running in FIQ handler + * We do not permit nested interrupt. + * + * Saving area: from user : PCB. + * from kernel : interrupt stack. + */ + + .text + .align 2 + .globl EXT(fleh_decirq) + +LEXT(fleh_decirq) + sub lr, lr, #4 + + cpsie af // Re-enable async aborts/FIQ + + mrs sp, spsr + tst sp, #0x0f // From user? or kernel? + bne fleh_decirq_kernel + +fleh_decirq_user: + mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW + add sp, sp, ACT_PCBDATA // Get User PCB + stmia sp, {r0-r12, sp, lr}^ + mov r7, #0 // Zero the frame pointer + nop + str lr, [sp, SS_PC] + mrs r4, spsr + str r4, [sp, SS_CPSR] + mov r5, sp // Saved context in r5 + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + ldr r6, [r9, ACT_CPUDATAP] // Get current cpu + ldr sp, [r6, CPU_ISTACKPTR] // Set interrupt stack + cpsid i, #PSR_SVC_MODE + ldr sp, [r9, TH_KSTACKPTR] // Set kernel stack + cpsid i, #PSR_IRQ_MODE + +#if __ARM_VFP__ + add r0, r9, ACT_UVFP // Get the address of the user VFP save area + bl EXT(vfp_save) // Save the current VFP state to ACT_UVFP + mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value... + fmxr fpscr, r4 // And shove it into FPSCR +#endif +#if __ARM_USER_PROTECT__ + ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 + mov r3, #0 // Load kernel asid + mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR + isb +#endif +#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME + bl EXT(timer_state_event_user_to_kernel) + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW +#endif +#if CONFIG_TELEMETRY + LOAD_ADDR(r2, telemetry_needs_record) // Check if a telemetry record was requested... + mov r0, #1 + ldr r2, [r2] + movs r2, r2 + beq 1f + bl EXT(telemetry_mark_curthread) // ...if so, mark the current thread... + mrc p15, 0, r9, c13, c0, 4 // ...and restore the thread pointer from TPIDRPRW +1: +#endif + + b fleh_decirq_handler + +fleh_decirq_kernel: + cpsid i, #PSR_SVC_MODE + + sub sp, sp, EXC_CTX_SIZE + stmia sp, {r0-r12} + add r0, sp, EXC_CTX_SIZE + + str r0, [sp, SS_SP] // Save supervisor mode sp + str lr, [sp, SS_LR] // Save supervisor mode lr + + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + +#if __ARM_VFP__ + add r0, sp, SS_SIZE // Get vfp state pointer + bic r0, #(VSS_ALIGN_NUM - 1) // Align to arm_vfpsaved_state alignment + add r0, VSS_ALIGN // Get the actual vfp save area + bl EXT(vfp_save) // Save the current VFP state to the stack + mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value... + fmxr fpscr, r4 // And shove it into FPSCR +#endif +#if __ARM_USER_PROTECT__ + mrc p15, 0, r10, c2, c0, 0 // Get TTBR0 + ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 + mrc p15, 0, r11, c13, c0, 1 // Get CONTEXTIDR + mov r3, #0 // Load kernel asid + mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + mov r5, sp // Saved context in r5 + + cpsid i, #PSR_IRQ_MODE + + str lr, [r5, SS_PC] // Save LR as the return PC + mrs r4, spsr + str r4, [r5, SS_CPSR] // Save the cpsr of the interrupted mode + + ldr sp, [r9, ACT_CPUDATAP] // Get current cpu + ldr sp, [sp, CPU_ISTACKPTR] // Set interrupt stack + +#if CONFIG_TELEMETRY + LOAD_ADDR(r2, telemetry_needs_record) // Check if a telemetry record was requested... + mov r0, #0 + ldr r2, [r2] + movs r2, r2 + beq 1f + bl EXT(telemetry_mark_curthread) // ...if so, mark the current thread... + mrc p15, 0, r9, c13, c0, 4 // ...and restore the thread pointer from TPIDRPRW +1: +#endif + +fleh_decirq_handler: + ldr r2, [r9, ACT_PREEMPT_CNT] // Load preemption count + add r2, r2, #1 // Increment count + str r2, [r9, ACT_PREEMPT_CNT] // Update preemption count + ldr r2, [r9, ACT_CPUDATAP] // Get current cpu + str r5, [r2, CPU_INT_STATE] // Saved context in cpu_int_state + ldr r3, [r2, CPU_STAT_IRQ] // Get IRQ count + add r3, r3, #1 // Increment count + str r3, [r2, CPU_STAT_IRQ] // Update IRQ count + ldr r3, [r2, CPU_STAT_IRQ_WAKE] // Get post-wake IRQ count + add r3, r3, #1 // Increment count + str r3, [r2, CPU_STAT_IRQ_WAKE] // Update post-wake IRQ count +#ifndef NO_KDEBUG + LOAD_ADDR(r4, kdebug_enable) + ldr r4, [r4] + movs r4, r4 + movne r0, r5 // Pass saved context + COND_EXTERN_BLNE(interrupt_trace) +#endif + bl EXT(interrupt_stats) // Record interrupt statistics + mov r0, #0 + bl EXT(rtclock_intr) // Call second level exception handler +#ifndef NO_KDEBUG + movs r4, r4 + COND_EXTERN_BLNE(interrupt_trace_exit) +#endif + + mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW + + b return_from_irq + + +/* + * First Level Exception Handler for FIQ + * Current mode : FIQ + * IRQ and FIQ are always disabled while running in FIQ handler + * We do not permit nested interrupt. + * + * Saving area: from user : PCB. + * from kernel : interrupt stack. + * + * We have 7 added shadow registers in FIQ mode for fast services. + * So only we have to save is just 8 general registers and LR. + * But if the current thread was running on user mode before the FIQ interrupt, + * All user registers be saved for ast handler routine. + */ + .text + .align 2 + .globl EXT(fleh_fiq_generic) + +LEXT(fleh_fiq_generic) + str r11, [r10] // Clear the FIQ source + + ldr r13, [r8, CPU_TIMEBASE_LOW] // Load TBL + adds r13, r13, #1 // Increment TBL + str r13, [r8, CPU_TIMEBASE_LOW] // Store TBL + ldreq r13, [r8, CPU_TIMEBASE_HIGH] // Load TBU + addeq r13, r13, #1 // Increment TBU + streq r13, [r8, CPU_TIMEBASE_HIGH] // Store TBU + subs r12, r12, #1 // Decrement, DEC + str r12, [r8, CPU_DECREMENTER] // Store DEC + subspl pc, lr, #4 // Return unless DEC < 0 + b EXT(fleh_dec) + + .text + .align 2 + .globl EXT(fleh_dec) +LEXT(fleh_dec) + mrs sp, spsr // Get the spsr + sub lr, lr, #4 + tst sp, #0x0f // From user? or kernel? + bne 2f + + /* From user */ + mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW + add sp, sp, ACT_PCBDATA // Get User PCB + + stmia sp, {r0-r12, sp, lr}^ + mov r7, #0 // Zero the frame pointer + nop + str lr, [sp, SS_PC] + + mrs r4, spsr + str r4, [sp, SS_CPSR] + mov r5, sp + sub sp, sp, ACT_PCBDATA // Get User PCB + ldr sp, [sp, ACT_CPUDATAP] // Get current cpu + ldr sp, [sp, CPU_ISTACKPTR] // Set interrupt stack + mov r6, sp + cpsid i, #PSR_SVC_MODE + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + ldr sp, [r9, TH_KSTACKPTR] // Set kernel stack + +#if __ARM_VFP__ + add r0, r9, ACT_UVFP // Get the address of the user VFP save area + bl EXT(vfp_save) // Save the current VFP state to ACT_UVFP + mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value... + fmxr fpscr, r4 // And shove it into FPSCR +#endif +#if __ARM_USER_PROTECT__ + mrc p15, 0, r10, c2, c0, 0 // Get TTBR0 + ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 + mrc p15, 0, r11, c13, c0, 1 // Get CONTEXTIDR + mov r3, #0 // Load kernel asid + mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + mov r0, #1 // Mark this as coming from user context + b 4f + +2: + /* From kernel */ + tst sp, #PSR_IRQF // Test for IRQ masked + bne 3f // We're on the cpu_signal path + + cpsid if, #PSR_SVC_MODE + + sub sp, sp, EXC_CTX_SIZE + stmia sp, {r0-r12} + add r0, sp, EXC_CTX_SIZE + + str r0, [sp, SS_SP] // Save supervisor mode sp + str lr, [sp, SS_LR] // Save supervisor mode lr + + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + +#if __ARM_VFP__ + add r0, sp, SS_SIZE // Get vfp state pointer + bic r0, #(VSS_ALIGN_NUM - 1) // Align to arm_vfpsaved_state alignment + add r0, VSS_ALIGN // Get the actual vfp save area + bl EXT(vfp_save) // Save the current VFP state to the stack + mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value... + fmxr fpscr, r4 // And shove it into FPSCR +#endif +#if __ARM_USER_PROTECT__ + mrc p15, 0, r10, c2, c0, 0 // Get TTBR0 + ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 + mrc p15, 0, r11, c13, c0, 1 // Get CONTEXTIDR + mov r3, #0 // Load kernel asid + mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + mov r5, sp // Saved context in r5 + + cpsid if, #PSR_FIQ_MODE + + mrc p15, 0, r1, c13, c0, 4 // Read TPIDRPRW + + str lr, [r5, SS_PC] // Save LR as the return PC + mrs r4, spsr + str r4, [r5, SS_CPSR] // Save the cpsr of the interrupted mode + + ldr r6, [r1, ACT_CPUDATAP] // Get current cpu + ldr r6, [r6, CPU_ISTACKPTR] // Set interrupt stack + + mov r0, #0 // Mark this as coming from kernel context + b 4f + +3: + /* cpu_signal path */ + mrc p15, 0, sp, c13, c0, 4 // Read TPIDRPRW + ldr sp, [sp, ACT_CPUDATAP] // Get current cpu + ldr sp, [sp, CPU_FIQSTACKPTR] // Set fiq stack + sub sp, sp, EXC_CTX_SIZE + stmia sp, {r0-r12} + str lr, [sp, SS_PC] + mrs r4, spsr + str r4, [sp, SS_CPSR] + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + +#if __ARM_VFP__ + add r0, sp, SS_SIZE // Get vfp state pointer + bic r0, #(VSS_ALIGN_NUM - 1) // Align to arm_vfpsaved_state alignment + add r0, VSS_ALIGN // Get the actual vfp save area + bl EXT(vfp_save) // Save the current VFP state to the stack + mov r4, #FPSCR_DEFAULT // Load up the default FPSCR value... + fmxr fpscr, r4 // And shove it into FPSCR +#endif +#if __ARM_USER_PROTECT__ + mrc p15, 0, r10, c2, c0, 0 // Get TTBR0 + ldr r3, [r9, ACT_KPTW_TTB] // Load kernel ttb + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 + mrc p15, 0, r11, c13, c0, 1 // Get CONTEXTIDR + mov r3, #0 // Load kernel asid + mcr p15, 0, r3, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + mov r0, r8 // Get current cpu in arg 0 + mov r1, SIGPdec // Decrementer signal in arg1 + mov r2, #0 + mov r3, #0 + bl EXT(cpu_signal) // Call cpu_signal + + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + +#if __ARM_VFP__ + add r0, sp, SS_SIZE // Get vfp state pointer + bic r0, #(VSS_ALIGN_NUM - 1) // Align to arm_vfpsaved_state alignment + add r0, VSS_ALIGN // Get the actual vfp save area + bl EXT(vfp_load) // Load the desired VFP state from the stack +#endif + + clrex // clear exclusive memory tag +#if __ARM_ENABLE_WFE_ + sev +#endif +#if __ARM_USER_PROTECT__ + mcr p15, 0, r10, c2, c0, 0 // Set TTBR0 + mcr p15, 0, r11, c13, c0, 1 // Set CONTEXTIDR + isb +#endif + ldr lr, [sp, SS_PC] + ldmia sp, {r0-r12} // Restore saved registers + movs pc, lr // Return from fiq + +4: + cpsid i, #PSR_IRQ_MODE + cpsie f + mov sp, r6 // Restore the stack pointer + msr spsr_cxsf, r4 // Restore the spsr + ldr r2, [r9, ACT_PREEMPT_CNT] // Load preemption count + add r2, r2, #1 // Increment count + str r2, [r9, ACT_PREEMPT_CNT] // Update preemption count + ldr r4, [r9, ACT_CPUDATAP] // Get current cpu + str r5, [r4, CPU_INT_STATE] + ldr r3, [r4, CPU_STAT_IRQ] // Get IRQ count + add r3, r3, #1 // Increment count + str r3, [r4, CPU_STAT_IRQ] // Update IRQ count + ldr r3, [r4, CPU_STAT_IRQ_WAKE] // Get post-wake IRQ count + add r3, r3, #1 // Increment count + str r3, [r4, CPU_STAT_IRQ_WAKE] // Update post-wake IRQ count +#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME + movs r0, r0 + beq 5f + mov r8, r0 // Stash our "from_user" boolean value + bl EXT(timer_state_event_user_to_kernel) + mov r0, r8 // Restore our "from_user" value + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW +5: +#endif +#if CONFIG_TELEMETRY + LOAD_ADDR(r4, telemetry_needs_record) // Check if a telemetry record was requested... + ldr r4, [r4] + movs r4, r4 + beq 6f + bl EXT(telemetry_mark_curthread) // ...if so, mark the current thread... + mrc p15, 0, r9, c13, c0, 4 // ...and restore the thread pointer from TPIDRPRW +6: +#endif + +#ifndef NO_KDEBUG + LOAD_ADDR(r4, kdebug_enable) + ldr r4, [r4] + movs r4, r4 + ldrne r1, [r9, ACT_CPUDATAP] // Get current cpu + ldrne r0, [r1, CPU_INT_STATE] + COND_EXTERN_BLNE(interrupt_trace) +#endif + bl EXT(interrupt_stats) // Record interrupt statistics + mov r0, #0 + bl EXT(rtclock_intr) // Call second level exception handler +#ifndef NO_KDEBUG + movs r4, r4 + COND_EXTERN_BLNE(interrupt_trace_exit) +#endif + + mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW + + b return_from_irq + +/* + * void thread_syscall_return(kern_return_t r0) + * + */ + .text + .align 2 + .globl EXT(thread_syscall_return) + +LEXT(thread_syscall_return) + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + add r1, r9, ACT_PCBDATA // Get User PCB + str r0, [r1, SS_R0] // set return value +#ifndef NO_KDEBUG + LOAD_ADDR(r4, kdebug_enable) + ldr r4, [r4] + movs r4, r4 + beq load_and_go_user + ldr r12, [r1, SS_R12] // Load syscall number + rsbs r1, r12, #0 // make the syscall positive (if negative) + COND_EXTERN_BLGT(mach_syscall_trace_exit) +#endif + b load_and_go_user + +/* + * void thread_exception_return(void) + * void thread_bootstrap_return(void) + * + */ + .text + .globl EXT(thread_exception_return) + .globl EXT(thread_bootstrap_return) + +LEXT(thread_bootstrap_return) +#if CONFIG_DTRACE + bl EXT(dtrace_thread_bootstrap) +#endif + // Fall through + +LEXT(thread_exception_return) + +load_and_go_user: +/* + * Restore user mode states and go back to user mode + */ + cpsid i // Disable irq + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + + mvn r0, #0 + str r0, [r9, TH_IOTIER_OVERRIDE] // Reset IO tier override to -1 before returning to user + + ldr r8, [r9, ACT_CPUDATAP] // Get current cpu + ldr r5, [r8, CPU_PENDING_AST] // Get ASTs + cmp r5, #0 // Test if ASTs pending + beq return_to_user_now // Branch if no ASTs + +#if __BIGGEST_ALIGNMENT__ > 4 + and r2, sp, #0x0F // sp mod 16-bytes + cmp r2, #4 // need space for the sp on the stack + addlt r2, r2, #0x10 // make room if needed, but keep stack aligned + mov r3, sp // get current sp + sub sp, sp, r2 // align stack + str r3, [sp] // store previous sp on stack +#endif + + bl EXT(ast_taken_user) // Handle all ASTs (may continue via thread_exception_return) + +#if __BIGGEST_ALIGNMENT__ > 4 + ldr sp, [sp] // Restore the stack pointer +#endif + + mrc p15, 0, r9, c13, c0, 4 // Reload r9 from TPIDRPRW + b load_and_go_user // Loop back + +return_to_user_now: + +#if MACH_ASSERT +/* + * Assert that the preemption level is zero prior to the return to user space + */ + ldr r1, [r9, ACT_PREEMPT_CNT] // Load preemption count + movs r1, r1 // Test + beq 0f // Continue if zero, or... + adr r0, L_lagu_panic_str // Load the panic string... + blx EXT(panic) // Finally, panic +0: + ldr r2, [r9, TH_RWLOCK_CNT] // Load RW lock count + movs r2, r2 // Test + beq 0f // Continue if zero, or... + adr r0, L_lagu_rwlock_cnt_panic_str // Load the panic string... + mov r1, r9 // Thread argument for panic string + blx EXT(panic) // Finally, panic +#endif + +0: +#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME + bl EXT(timer_state_event_kernel_to_user) + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW + ldr r8, [r9, ACT_CPUDATAP] // Get current cpu data +#endif /* !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME */ +#if __ARM_DEBUG__ >= 6 + ldr r0, [r9, ACT_DEBUGDATA] + ldr r6, [r8, CPU_USER_DEBUG] + cmp r0, r6 // test if debug registers need to be changed + beq 1f + bl EXT(arm_debug_set) // argument is already in r0 + mrc p15, 0, r9, c13, c0, 4 // Read TPIDRPRW +1: +#endif +#if __ARM_VFP__ + add r0, r9, ACT_UVFP // Get the address of the user VFP save area + bl EXT(vfp_load) // Load the desired VFP state from ACT_UVFP +#endif + add r0, r9, ACT_PCBDATA // Get User PCB + ldr r4, [r0, SS_CPSR] // Get saved cpsr + and r3, r4, #PSR_MODE_MASK // Extract current mode + cmp r3, #PSR_USER_MODE // Check user mode + movne r0, r3 + bne EXT(ExceptionVectorPanic) + + msr spsr_cxsf, r4 // Restore spsr(user mode cpsr) + mov sp, r0 // Get User PCB + + clrex // clear exclusive memory tag +#if __ARM_ENABLE_WFE_ + sev +#endif +#if __ARM_USER_PROTECT__ + ldr r3, [r9, ACT_UPTW_TTB] // Load thread ttb + mcr p15, 0, r3, c2, c0, 0 // Set TTBR0 + ldr r2, [r9, ACT_ASID] // Load thread asid + mcr p15, 0, r2, c13, c0, 1 + isb +#endif + ldr lr, [sp, SS_PC] // Restore user mode pc + ldmia sp, {r0-r12, sp, lr}^ // Restore the other user mode registers + nop // Hardware problem + movs pc, lr // Return to user + + .align 2 +L_lagu_panic_str: + .asciz "load_and_go_user: preemption_level %d" + .align 2 + + .align 2 +L_lagu_rwlock_cnt_panic_str: + .asciz "load_and_go_user: RW lock count not 0 on thread %p (%u)" + .align 2 + + .align 2 +L_evimpanic_str: + .ascii "Exception Vector: Illegal Mode: 0x%08X\n\000" + .align 2 + + .text + .align 2 + .globl EXT(ExceptionVectorPanic) + +LEXT(ExceptionVectorPanic) + cpsid i, #PSR_SVC_MODE + mov r1, r0 + adr r0, L_evimpanic_str + blx EXT(panic) + b . + +#include "globals_asm.h" + +LOAD_ADDR_GEN_DEF(mach_trap_table) +LOAD_ADDR_GEN_DEF(kern_invalid) + +/* vim: set ts=4: */ diff --git a/osfmk/arm/loose_ends.c b/osfmk/arm/loose_ends.c new file mode 100644 index 000000000..46aeec6da --- /dev/null +++ b/osfmk/arm/loose_ends.c @@ -0,0 +1,665 @@ +/* + * Copyright (c) 2007-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include <mach_assert.h> +#include <mach/vm_types.h> +#include <mach/mach_time.h> +#include <kern/timer.h> +#include <kern/clock.h> +#include <kern/machine.h> +#include <mach/machine.h> +#include <mach/machine/vm_param.h> +#include <mach_kdp.h> +#include <kdp/kdp_udp.h> +#if !MACH_KDP +#include <kdp/kdp_callout.h> +#endif /* !MACH_KDP */ +#include <arm/cpu_data.h> +#include <arm/cpu_data_internal.h> +#include <arm/caches_internal.h> + +#include <vm/vm_kern.h> +#include <vm/vm_map.h> +#include <vm/pmap.h> + +#include <arm/misc_protos.h> + +#include <sys/errno.h> + +#define INT_SIZE (BYTE_SIZE * sizeof (int)) + + +void +bcopy_phys(addr64_t src, addr64_t dst, vm_size_t bytes) +{ + unsigned int src_index; + unsigned int dst_index; + vm_offset_t src_offset; + vm_offset_t dst_offset; + unsigned int cpu_num; + unsigned int wimg_bits_src, wimg_bits_dst; + ppnum_t pn_src = (src >> PAGE_SHIFT); + ppnum_t pn_dst = (dst >> PAGE_SHIFT); + + wimg_bits_src = pmap_cache_attributes(pn_src); + wimg_bits_dst = pmap_cache_attributes(pn_dst); + + if (mmu_kvtop_wpreflight(phystokv((pmap_paddr_t) dst)) && + ((wimg_bits_src & VM_WIMG_MASK) == VM_WIMG_DEFAULT) && + ((wimg_bits_dst & VM_WIMG_MASK) == VM_WIMG_DEFAULT)) { + /* Fast path - dst is writable and both source and destination have default attributes */ + bcopy((char *)phystokv((pmap_paddr_t) src), (char *)phystokv((pmap_paddr_t) dst), bytes); + return; + } + + src_offset = src & PAGE_MASK; + dst_offset = dst & PAGE_MASK; + + if ((src_offset + bytes) > PAGE_SIZE || (dst_offset + bytes) > PAGE_SIZE) + panic("bcopy extends beyond copy windows"); + + mp_disable_preemption(); + cpu_num = cpu_number(); + src_index = pmap_map_cpu_windows_copy(pn_src, VM_PROT_READ, wimg_bits_src); + dst_index = pmap_map_cpu_windows_copy(pn_dst, VM_PROT_READ|VM_PROT_WRITE, wimg_bits_dst); + + bcopy((char *)(pmap_cpu_windows_copy_addr(cpu_num, src_index)+src_offset), + (char *)(pmap_cpu_windows_copy_addr(cpu_num, dst_index)+dst_offset), + bytes); + + pmap_unmap_cpu_windows_copy(src_index); + pmap_unmap_cpu_windows_copy(dst_index); + mp_enable_preemption(); +} + +void +bzero_phys_nc(addr64_t src64, vm_size_t bytes) +{ + bzero_phys(src64, bytes); +} + +/* Zero bytes starting at a physical address */ +void +bzero_phys(addr64_t src, vm_size_t bytes) +{ + unsigned int wimg_bits; + ppnum_t pn = (src >> PAGE_SHIFT); + + wimg_bits = pmap_cache_attributes(pn); + if ((wimg_bits & VM_WIMG_MASK) == VM_WIMG_DEFAULT) { + /* Fast path - default attributes */ + bzero((char *)phystokv((pmap_paddr_t) src), bytes); + } else { + mp_disable_preemption(); + + unsigned int cpu_num = cpu_number(); + + while (bytes > 0) { + vm_offset_t offset = src & PAGE_MASK; + uint32_t count = PAGE_SIZE - offset; + + if (count > bytes) + count = bytes; + + unsigned int index = pmap_map_cpu_windows_copy(src >> PAGE_SHIFT, VM_PROT_READ | VM_PROT_WRITE, wimg_bits); + + bzero((char *)(pmap_cpu_windows_copy_addr(cpu_num, index) + offset), count); + + pmap_unmap_cpu_windows_copy(index); + + src += count; + bytes -= count; + } + + mp_enable_preemption(); + } +} + +/* + * Read data from a physical address. + */ + + +static unsigned int +ml_phys_read_data(pmap_paddr_t paddr, int size) +{ + unsigned int index; + unsigned int result; + unsigned int wimg_bits; + ppnum_t pn = (paddr >> PAGE_SHIFT); + unsigned char s1; + unsigned short s2; + vm_offset_t copywindow_vaddr = 0; + + mp_disable_preemption(); + wimg_bits = pmap_cache_attributes(pn); + index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ, wimg_bits); + copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_number(), index) | ((uint32_t)paddr & PAGE_MASK);; + + switch (size) { + case 1: + s1 = *(volatile unsigned char *)(copywindow_vaddr); + result = s1; + break; + case 2: + s2 = *(volatile unsigned short *)(copywindow_vaddr); + result = s2; + break; + case 4: + default: + result = *(volatile unsigned int *)(copywindow_vaddr); + break; + } + + pmap_unmap_cpu_windows_copy(index); + mp_enable_preemption(); + + return result; +} + +static unsigned long long +ml_phys_read_long_long(pmap_paddr_t paddr) +{ + unsigned int index; + unsigned int result; + unsigned int wimg_bits; + ppnum_t pn = (paddr >> PAGE_SHIFT); + + mp_disable_preemption(); + wimg_bits = pmap_cache_attributes(pn); + index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ, wimg_bits); + + result = *(volatile unsigned long long *)(pmap_cpu_windows_copy_addr(cpu_number(), index) + | ((uint32_t)paddr & PAGE_MASK)); + + pmap_unmap_cpu_windows_copy(index); + mp_enable_preemption(); + + return result; +} + +unsigned int ml_phys_read( vm_offset_t paddr) +{ + return ml_phys_read_data((pmap_paddr_t)paddr, 4); +} + +unsigned int ml_phys_read_word(vm_offset_t paddr) { + + return ml_phys_read_data((pmap_paddr_t)paddr, 4); +} + +unsigned int ml_phys_read_64(addr64_t paddr64) +{ + return ml_phys_read_data((pmap_paddr_t)paddr64, 4); +} + +unsigned int ml_phys_read_word_64(addr64_t paddr64) +{ + return ml_phys_read_data((pmap_paddr_t)paddr64, 4); +} + +unsigned int ml_phys_read_half(vm_offset_t paddr) +{ + return ml_phys_read_data((pmap_paddr_t)paddr, 2); +} + +unsigned int ml_phys_read_half_64(addr64_t paddr64) +{ + return ml_phys_read_data((pmap_paddr_t)paddr64, 2); +} + +unsigned int ml_phys_read_byte(vm_offset_t paddr) +{ + return ml_phys_read_data((pmap_paddr_t)paddr, 1); +} + +unsigned int ml_phys_read_byte_64(addr64_t paddr64) +{ + return ml_phys_read_data((pmap_paddr_t)paddr64, 1); +} + +unsigned long long ml_phys_read_double(vm_offset_t paddr) +{ + return ml_phys_read_long_long((pmap_paddr_t)paddr); +} + +unsigned long long ml_phys_read_double_64(addr64_t paddr64) +{ + return ml_phys_read_long_long((pmap_paddr_t)paddr64); +} + + + +/* + * Write data to a physical address. + */ + +static void +ml_phys_write_data(pmap_paddr_t paddr, unsigned long data, int size) +{ + unsigned int index; + unsigned int wimg_bits; + ppnum_t pn = (paddr >> PAGE_SHIFT); + vm_offset_t copywindow_vaddr = 0; + + mp_disable_preemption(); + wimg_bits = pmap_cache_attributes(pn); + index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ|VM_PROT_WRITE, wimg_bits); + copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_number(), index) | ((uint32_t) paddr & PAGE_MASK); + + switch (size) { + case 1: + *(volatile unsigned char *)(copywindow_vaddr) = (unsigned char)data; + break; + case 2: + *(volatile unsigned short *)(copywindow_vaddr) = (unsigned short)data; + break; + case 4: + default: + *(volatile unsigned int *)(copywindow_vaddr) = (uint32_t)data; + break; + } + + pmap_unmap_cpu_windows_copy(index); + mp_enable_preemption(); +} + +static void +ml_phys_write_long_long(pmap_paddr_t paddr, unsigned long long data) +{ + unsigned int index; + unsigned int wimg_bits; + ppnum_t pn = (paddr >> PAGE_SHIFT); + + mp_disable_preemption(); + wimg_bits = pmap_cache_attributes(pn); + index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ|VM_PROT_WRITE, wimg_bits); + + *(volatile unsigned long long *)(pmap_cpu_windows_copy_addr(cpu_number(), index) + | ((uint32_t)paddr & PAGE_MASK)) = data; + + pmap_unmap_cpu_windows_copy(index); + mp_enable_preemption(); +} + + + +void ml_phys_write_byte(vm_offset_t paddr, unsigned int data) +{ + ml_phys_write_data((pmap_paddr_t)paddr, data, 1); +} + +void ml_phys_write_byte_64(addr64_t paddr64, unsigned int data) +{ + ml_phys_write_data((pmap_paddr_t)paddr64, data, 1); +} + +void ml_phys_write_half(vm_offset_t paddr, unsigned int data) +{ + ml_phys_write_data((pmap_paddr_t)paddr, data, 2); +} + +void ml_phys_write_half_64(addr64_t paddr64, unsigned int data) +{ + ml_phys_write_data((pmap_paddr_t)paddr64, data, 2); +} + +void ml_phys_write(vm_offset_t paddr, unsigned int data) +{ + ml_phys_write_data((pmap_paddr_t)paddr, data, 4); +} + +void ml_phys_write_64(addr64_t paddr64, unsigned int data) +{ + ml_phys_write_data((pmap_paddr_t)paddr64, data, 4); +} + +void ml_phys_write_word(vm_offset_t paddr, unsigned int data) +{ + ml_phys_write_data((pmap_paddr_t)paddr, data, 4); +} + +void ml_phys_write_word_64(addr64_t paddr64, unsigned int data) +{ + ml_phys_write_data((pmap_paddr_t)paddr64, data, 4); +} + +void ml_phys_write_double(vm_offset_t paddr, unsigned long long data) +{ + ml_phys_write_long_long((pmap_paddr_t)paddr, data); +} + +void ml_phys_write_double_64(addr64_t paddr64, unsigned long long data) +{ + ml_phys_write_long_long((pmap_paddr_t)paddr64, data); +} + + +/* + * Set indicated bit in bit string. + */ +void +setbit(int bitno, int *s) +{ + s[bitno / INT_SIZE] |= 1 << (bitno % INT_SIZE); +} + +/* + * Clear indicated bit in bit string. + */ +void +clrbit(int bitno, int *s) +{ + s[bitno / INT_SIZE] &= ~(1 << (bitno % INT_SIZE)); +} + +/* + * Test if indicated bit is set in bit string. + */ +int +testbit(int bitno, int *s) +{ + return s[bitno / INT_SIZE] & (1 << (bitno % INT_SIZE)); +} + +/* + * Find first bit set in bit string. + */ +int +ffsbit(int *s) +{ + int offset; + + for (offset = 0; !*s; offset += INT_SIZE, ++s); + return offset + __builtin_ctz(*s); +} + +int +ffs(unsigned int mask) +{ + if (mask == 0) + return 0; + + /* + * NOTE: cannot use __builtin_ffs because it generates a call to + * 'ffs' + */ + return 1 + __builtin_ctz(mask); +} + +int +ffsll(unsigned long long mask) +{ + if (mask == 0) + return 0; + + /* + * NOTE: cannot use __builtin_ffsll because it generates a call to + * 'ffsll' + */ + return 1 + __builtin_ctzll(mask); +} + +/* + * Find last bit set in bit string. + */ +int +fls(unsigned int mask) +{ + if (mask == 0) + return 0; + + return (sizeof (mask) << 3) - __builtin_clz(mask); +} + +int +flsll(unsigned long long mask) +{ + if (mask == 0) + return 0; + + return (sizeof (mask) << 3) - __builtin_clzll(mask); +} + +int +bcmp( + const void *pa, + const void *pb, + size_t len) +{ + const char *a = (const char *) pa; + const char *b = (const char *) pb; + + if (len == 0) + return 0; + + do + if (*a++ != *b++) + break; + while (--len); + + return len; +} + +int +memcmp(const void *s1, const void *s2, size_t n) +{ + if (n != 0) { + const unsigned char *p1 = s1, *p2 = s2; + + do { + if (*p1++ != *p2++) + return (*--p1 - *--p2); + } while (--n != 0); + } + return (0); +} + +kern_return_t +copypv(addr64_t source, addr64_t sink, unsigned int size, int which) +{ + kern_return_t retval = KERN_SUCCESS; + void *from, *to; + unsigned int from_wimg_bits, to_wimg_bits; + + from = CAST_DOWN(void *, source); + to = CAST_DOWN(void *, sink); + + if ((which & (cppvPsrc | cppvPsnk)) == 0) /* Make sure that only + * one is virtual */ + panic("copypv: no more than 1 parameter may be virtual\n"); /* Not allowed */ + + if (which & cppvPsrc) + from = (void *)phystokv(from); + if (which & cppvPsnk) + to = (void *)phystokv(to); + + if ((which & (cppvPsrc | cppvKmap)) == 0) /* Source is virtual in + * current map */ + retval = copyin((user_addr_t) from, to, size); + else if ((which & (cppvPsnk | cppvKmap)) == 0) /* Sink is virtual in + * current map */ + retval = copyout(from, (user_addr_t) to, size); + else /* both addresses are physical or kernel map */ + bcopy(from, to, size); + + if (which & cppvFsrc) { + flush_dcache64(source, size, ((which & cppvPsrc) == cppvPsrc)); + } else if (which & cppvPsrc) { + from_wimg_bits = pmap_cache_attributes(source >> PAGE_SHIFT); + if ((from_wimg_bits != VM_WIMG_COPYBACK) && (from_wimg_bits != VM_WIMG_WTHRU)) + flush_dcache64(source, size, TRUE); + } + + if (which & cppvFsnk) { + flush_dcache64(sink, size, ((which & cppvPsnk) == cppvPsnk)); + } else if (which & cppvPsnk) { + to_wimg_bits = pmap_cache_attributes(sink >> PAGE_SHIFT); + if (to_wimg_bits != VM_WIMG_COPYBACK) + flush_dcache64(sink, size, TRUE); + } + return retval; +} + +/* + * Copy sizes bigger than this value will cause a kernel panic. + * + * Yes, this is an arbitrary fixed limit, but it's almost certainly + * a programming error to be copying more than this amount between + * user and wired kernel memory in a single invocation on this + * platform. + */ +const int copysize_limit_panic = (64 * 1024 * 1024); + +/* + * Validate the arguments to copy{in,out} on this platform. + * + * Called when nbytes is "large" e.g. more than a page. Such sizes are + * infrequent, and very large sizes are likely indications of attempts + * to exploit kernel programming errors (bugs). + */ +static int +copy_validate(const user_addr_t user_addr, + uintptr_t kernel_addr, vm_size_t nbytes) +{ + uintptr_t kernel_addr_last = kernel_addr + nbytes; + + if (kernel_addr < VM_MIN_KERNEL_ADDRESS || + kernel_addr > VM_MAX_KERNEL_ADDRESS || + kernel_addr_last < kernel_addr || + kernel_addr_last > VM_MAX_KERNEL_ADDRESS) + panic("%s(%p, %p, %u) - kaddr not in kernel", __func__, + (void *)user_addr, (void *)kernel_addr, nbytes); + + user_addr_t user_addr_last = user_addr + nbytes; + + if (user_addr_last < user_addr || + user_addr_last > VM_MIN_KERNEL_ADDRESS) + return (EFAULT); + + if (__improbable(nbytes > copysize_limit_panic)) + panic("%s(%p, %p, %u) - transfer too large", __func__, + (void *)user_addr, (void *)kernel_addr, nbytes); + + return (0); +} + +int +copyin_validate(const user_addr_t ua, uintptr_t ka, vm_size_t nbytes) +{ + return (copy_validate(ua, ka, nbytes)); +} + +int +copyout_validate(uintptr_t ka, const user_addr_t ua, vm_size_t nbytes) +{ + return (copy_validate(ua, ka, nbytes)); +} + +#if MACH_ASSERT + +extern int copyinframe(vm_address_t fp, char *frame); + +/* + * Machine-dependent routine to fill in an array with up to callstack_max + * levels of return pc information. + */ +void +machine_callstack( + uintptr_t * buf, + vm_size_t callstack_max) +{ + /* Captures the USER call stack */ + uint32_t i=0; + uint32_t frame[2]; + + struct arm_saved_state* state = find_user_regs(current_thread()); + + if (!state) { + while (i<callstack_max) + buf[i++] = 0; + } else { + buf[i++] = (uintptr_t)state->pc; + frame[0] = state->r[7]; + + while (i<callstack_max && frame[0] != 0) { + if (copyinframe(frame[0], (void*) frame)) + break; + buf[i++] = (uintptr_t)frame[1]; + } + + while (i<callstack_max) + buf[i++] = 0; + } +} + +#endif /* MACH_ASSERT */ + +int +clr_be_bit(void) +{ + panic("clr_be_bit"); + return 0; +} + +boolean_t +ml_probe_read( + __unused vm_offset_t paddr, + __unused unsigned int *val) +{ + panic("ml_probe_read() unimplemented"); + return 1; +} + +boolean_t +ml_probe_read_64( + __unused addr64_t paddr, + __unused unsigned int *val) +{ + panic("ml_probe_read_64() unimplemented"); + return 1; +} + + +void +ml_thread_policy( + __unused thread_t thread, + __unused unsigned policy_id, + __unused unsigned policy_info) +{ + // <rdar://problem/7141284>: Reduce print noise + // kprintf("ml_thread_policy() unimplemented\n"); +} + +#if !MACH_KDP +void +kdp_register_callout(kdp_callout_fn_t fn, void *arg) +{ +#pragma unused(fn,arg) +} +#endif diff --git a/osfmk/arm/lowglobals.h b/osfmk/arm/lowglobals.h new file mode 100644 index 000000000..447a28b9a --- /dev/null +++ b/osfmk/arm/lowglobals.h @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * Header files for the Low Memory Globals (lg) + */ +#ifndef _LOW_MEMORY_GLOBALS_H_ +#define _LOW_MEMORY_GLOBALS_H_ + +#ifndef __arm__ +#error Wrong architecture - this file is meant for arm +#endif + +#define LOWGLO_LAYOUT_MAGIC 0xC0DEC0DE + +#pragma pack(4) /* Make sure the structure stays as we defined it */ +typedef struct lowglo { + unsigned char lgVerCode[8]; /* 0xffff1000 System verification code */ + uint32_t lgZero[2]; /* 0xffff1008 Double constant 0 */ + uint32_t lgStext; /* 0xffff1010 Start of kernel text */ + uint32_t lgRsv014[2]; /* 0xffff1014 Reserved */ + uint32_t lgVersion; /* 0xffff101C Pointer to kernel version string */ + uint32_t lgRsv020[216]; /* 0xffff1020 Reserved */ + uint32_t lgKmodptr; /* 0xffff1380 Pointer to kmod, debugging aid */ + uint32_t lgTransOff; /* 0xffff1384 Pointer to kdp_trans_off, debugging aid */ + uint32_t lgRsv388[3]; /* 0xffff1388 Reserved */ + uint32_t lgOSVersion; /* 0xffff1394 Pointer to OS version string */ + uint32_t lgRsv398; /* 0xffff1398 Reserved */ + uint32_t lgRebootFlag; /* 0xffff139C Pointer to debugger reboot trigger */ + uint32_t lgManualPktAddr; /* 0xffff13A0 Pointer to manual packet structure */ + uint32_t lgRsv3A4; /* 0xffff13A4 Reserved */ + uint32_t lgPmapMemQ; /* 0xffff13A8 Pointer to PMAP memory queue */ + uint32_t lgPmapMemPageOffset;/* 0xffff13AC Offset of physical page member in vm_page_with_ppnum_t */ + uint32_t lgPmapMemChainOffset;/*0xffff13B0 Offset of listq in vm_page_t or vm_page_with_ppnum_t */ + uint32_t lgStaticAddr; /* 0xffff13B4 Static allocation address */ + uint32_t lgStaticSize; /* 0xffff13B8 Static allocation size */ + uint32_t lgLayoutMajorVersion; /* 0xffff13BC Lowglo layout major version */ + uint32_t lgLayoutMagic; /* 0xffff13C0 Magic value evaluated to determine if lgLayoutVersion is valid */ + uint32_t lgPmapMemStartAddr; /* 0xffff13C4 Pointer to start of vm_page_t array */ + uint32_t lgPmapMemEndAddr; /* 0xffff13C8 Pointer to end of vm_page_t array */ + uint32_t lgPmapMemPagesize; /* 0xffff13CC size of vm_page_t */ + uint32_t lgPmapMemFirstppnum; /* 0xffff13D0 physical page number of the first vm_page_t in the array */ + uint32_t lgLayoutMinorVersion; /* 0xffff13D4 Lowglo layout minor version */ + uint32_t lgPageShift; /* 0xffff13D8 Number of shifts from page number to size */ +} lowglo; +#pragma pack() + +extern lowglo lowGlo; + +void patch_low_glo(void); +void patch_low_glo_static_region(uint32_t address, uint32_t size); +void patch_low_glo_vm_page_info(void *, void *, uint32_t); + +#endif /* _LOW_MEMORY_GLOBALS_H_ */ diff --git a/osfmk/arm/lowmem_vectors.c b/osfmk/arm/lowmem_vectors.c new file mode 100644 index 000000000..f710eb65f --- /dev/null +++ b/osfmk/arm/lowmem_vectors.c @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2012-2013 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include <mach_kdp.h> +#include <mach/vm_param.h> +#include <arm/lowglobals.h> +#include <vm/vm_object.h> +#include <vm/vm_page.h> + +extern vm_offset_t vm_kernel_stext; +extern void *version; +extern void *kmod; +extern void *kdp_trans_off; +extern void *osversion; +extern void *flag_kdp_trigger_reboot; +extern void *manual_pkt; +extern struct vm_object pmap_object_store; /* store pt pages */ + +lowglo lowGlo __attribute__ ((aligned(PAGE_MAX_SIZE))) = { + .lgVerCode = { 'O','c','t','o','p','u','s',' ' }, + // Increment the major version for changes that break the current Astris + // usage of lowGlo values + // Increment the minor version for changes that provide additonal info/function + // but does not break current usage + .lgLayoutMajorVersion = 3, + .lgLayoutMinorVersion = 0, + .lgLayoutMagic = LOWGLO_LAYOUT_MAGIC, + .lgVersion = (uint32_t)&version, + .lgKmodptr = (uint32_t)&kmod, + .lgPageShift = PAGE_SHIFT, +#if MACH_KDP + .lgTransOff = (uint32_t)&kdp_trans_off, +#endif + .lgOSVersion = (uint32_t)&osversion, +#if MACH_KDP && CONFIG_KDP_INTERACTIVE_DEBUGGING + .lgRebootFlag = (uint32_t)&flag_kdp_trigger_reboot, + .lgManualPktAddr = (uint32_t)&manual_pkt, +#endif + .lgPmapMemQ = (uint32_t)&(pmap_object_store.memq), + .lgPmapMemPageOffset = offsetof(struct vm_page_with_ppnum, phys_page), + .lgPmapMemChainOffset = offsetof(struct vm_page, listq), + .lgPmapMemPagesize = (uint32_t)sizeof(struct vm_page), + + .lgPmapMemStartAddr = -1, + .lgPmapMemEndAddr = -1, + .lgPmapMemFirstppnum = -1 +}; + +void patch_low_glo(void) +{ + lowGlo.lgStext = (uint32_t)vm_kernel_stext; +} + +void patch_low_glo_static_region(uint32_t address, uint32_t size) +{ + lowGlo.lgStaticAddr = address; + lowGlo.lgStaticSize = size; +} + + +void patch_low_glo_vm_page_info(void * start_addr, void * end_addr, uint32_t first_ppnum) +{ + lowGlo.lgPmapMemStartAddr = (uint32_t)start_addr; + lowGlo.lgPmapMemEndAddr = (uint32_t)end_addr; + lowGlo.lgPmapMemFirstppnum = first_ppnum; +} diff --git a/osfmk/arm/lz4_decode_armv7NEON.s b/osfmk/arm/lz4_decode_armv7NEON.s new file mode 100644 index 000000000..f9faa2a55 --- /dev/null +++ b/osfmk/arm/lz4_decode_armv7NEON.s @@ -0,0 +1,348 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include <vm/lz4_assembly_select.h> +#if LZ4_ENABLE_ASSEMBLY_DECODE_ARMV7 + +/* + + int64_t lz4_decode_asm( + uint8_t ** dst_ptr, *dst_ptr points to next output byte to write + uint8_t * dst_begin, points to first valid output byte we can access, dst_begin <= dst + uint8_t * dst_end, "relaxed" end of output buffer (see below) + const uint8_t ** src_ptr, *src_ptr points to next input byte to read + const uint8_t * src_end) "relaxed" end of input buffer (see below) + + We test the position of the pointers only to ensure we don't access past src_end/dst_end + some fixed constant. + We never read before dst_begin. + + Return 0 on success, -1 on failure + On output, (*src_ptr,*dst_ptr) receives the last position in both buffers corresponding to the beginning of a LZ4 instruction. + +*/ + +.globl _lz4_decode_asm + +#define dst r0 // arg0 +#define dst_begin r1 // arg1 +#define dst_end r2 // arg2 +#define src r3 // arg3 + +#define src_end r4 // arg4 + +#define n_matches r5 +#define n_literals r10 +#define copy_src r11 // match/literal copy source +#define copy_dst r8 // match/literal copy destination + +#define aux1 r9 + +#define match_distance r6 + +#define match_permtable r12 +#define match_disttable lr + +#define dst_good [sp, #0] +#define src_good [sp, #4] + +.macro establish_frame + ldr ip, [sp, #0] // read src_end + push {r4-r7, lr} // Save registers + add r7, sp, #12 // Establish stack frame + push {r8-r11} + mov src_end, ip + push {r0, r3} // save dst/src + sub sp, sp, #4+16 // 4 for 16-byte stack alignment, extra 16-bytes for local +.endm + +.macro clear_frame_and_return + add sp, sp, #12+16 // skip r0/r3 + pop {r8-r11} + pop {r4-r7,pc} +.endm + +// copy_1x16 SOURCE_ADDR DESTINATION_ADDR +// Copy 16 bytes, clobber: q0 +.macro copy_1x16 + vld1.8 {q0}, [$0] + vst1.8 {q0}, [$1] +.endm + +// copy_1x16_and_increment SOURCE_ADDR DESTINATION_ADDR +// Copy 16 bytes, and increment both addresses by 16, clobber: q0 +.macro copy_1x16_and_increment + vld1.8 {q0}, [$0]! + vst1.8 {q0}, [$1]! +.endm + +// copy_2x16_and_increment SOURCE_ADDR DESTINATION_ADDR +// Copy 2 times 16 bytes, and increment both addresses by 32, clobber: q0 +.macro copy_2x16_and_increment + vld1.8 {q0}, [$0]! + vst1.8 {q0}, [$1]! + vld1.8 {q0}, [$0]! + vst1.8 {q0}, [$1]! +.endm + +// copy_1x32_and_increment SOURCE_ADDR DESTINATION_ADDR +// Copy 32 bytes, and increment both addresses by 32, clobber: q0,q1 +.macro copy_1x32_and_increment + vld1.8 {q0,q1}, [$0]! + vst1.8 {q0,q1}, [$1]! +.endm + +// If we don't branch, src < src_end after this +.macro check_src_end + cmp src,src_end + bhs L_done // extremely unlikely, DONE when src >= src_end +.endm + +// If we don't branch, dst < dst_end after this +.macro check_dst_end + cmp dst,dst_end + bhs L_done // extremely unlikely, DONE when dst >= dst_end +.endm + +.text +.syntax unified +.thumb +.thumb_func _lz4_decode_asm +.p2align 1 +_lz4_decode_asm: + establish_frame + ldr src,[src] // src = *src_ptr + ldr dst,[dst] // dst = *dst_ptr + + adr match_permtable,L_match_permtable + adr match_disttable,L_match_disttable + +L_decode_command: + // Keep last known good positions in both streams + str dst, dst_good + str src, src_good + + // Check limits + check_src_end + check_dst_end + + // Decode 1-byte command + ldrb aux1,[src],#1 // read command byte LLLLMMMM + lsr n_literals,aux1,#4 // 0000LLLL. n_literals is now 0..15 + and n_matches,aux1,#0xf // 0000MMMM. n_matches is now 0..15 + add n_matches,n_matches,#4 // n_matches is now 4..19 + + // Test number of literals (do not test if n_literals==0, because branch prediction fails on it) + cmp n_literals,#14 + bls L_copy_short_literal // 96% likely: n_literals in 0..14 + // continue to decode_long_literal + + // the number of literals is encoded on more bytes, we need to decode them +L_decode_long_literal: + check_src_end // required here, since we may loop an arbitrarily high number of times + ldrb aux1,[src],#1 + add n_literals,n_literals,aux1 + cmp aux1,#255 + beq L_decode_long_literal // extremely unlikely + // continue to copy_long_literal + + // Copy literals, n_literals >= 15 +L_copy_long_literal: + mov copy_src,src // literal copy origin + mov copy_dst,dst // literal copy destination + add src,src,n_literals + add dst,dst,n_literals + check_src_end // required here, since n_literals can be arbitrarily high + check_dst_end + + // fixed + loop + copy_1x32_and_increment copy_src,copy_dst +L_copy_long_literal_loop: + copy_1x32_and_increment copy_src,copy_dst + cmp dst,copy_dst + bhi L_copy_long_literal_loop // first test occurs after 64 bytes have been copied, and is unlikely to loop back + b L_expand_match + + // Copy literals, n_literals <= 14: copy 16 bytes +L_copy_short_literal: + copy_1x16 src,dst + add src,src,n_literals + add dst,dst,n_literals + // continue to expand match + +L_expand_match: + + // Decode match distance + ldrh match_distance,[src],#2 // 16-bit distance + cbz match_distance,L_fail // distance == 0 is invalid + sub copy_src,dst,match_distance // copy_src is the match copy source + cmp copy_src,dst_begin + blo L_fail // copy_src < dst_begin: FAIL + mov copy_dst,dst // copy_dst is the match copy destination + add dst,dst,n_matches // dst is updated to be the byte after the match; n_matches <= 19 here + + // Do we need to decode a long match? + cmp n_matches,#19 + beq L_decode_long_match // unlikely, n_matches >= 19 encoded on more bytes + cmp n_matches,#16 + bhi L_long_match // unlikely, n_matches == 17 or 18 + // continue to short match (most probable case) + + // Copy match, n_matches <= 16 +L_short_match: + cmp match_distance,#15 + bls L_copy_short_match_small_distance + + // Copy match, n_matches <= 16, match_distance >= 16: copy 16 bytes + copy_1x16 copy_src,copy_dst + b L_decode_command + +L_fail: +mov aux1,#-1 // FAIL +b L_exit + +L_done: +mov aux1,#0 // OK +// continue to L_exit + +L_exit: + +ldr dst,[sp, #20] // get back src_ptr,dst_ptr from stack +ldr src,[sp, #24] // get back src_ptr,dst_ptr from stack +ldr ip, src_good +ldr lr, dst_good +str ip,[src] // *src_ptr = src_good +str lr,[dst] // *dst_ptr = dst_good + +mov r0,aux1 // x0 = return value +clear_frame_and_return + + // Copy match, n_matches <= 16, match_distance < 16: + // load shuffle table, and permute to replicate the pattern on 16 bytes +L_copy_short_match_small_distance: + vld1.8 {q0},[copy_src] + add aux1,match_permtable,match_distance,lsl #5 // index in table + vld1.8 {q1},[aux1] // load only permutation for the low 16 bytes + vtbl.8 d4, {q0}, d2 // low 16 bytes of pattern + vtbl.8 d5, {q0}, d3 // low 16 bytes of pattern + vst1.8 {q2},[copy_dst] + b L_decode_command + + // n_matches == 19: the number of matches in encoded on more bytes, we need to decode them +L_decode_long_match: + check_src_end // required here, since we may loop an arbitrarily high number of times + ldrb aux1,[src],#1 + add dst,dst,aux1 + cmp aux1,#255 + beq L_decode_long_match // very unlikely + check_dst_end // required here, since dst was incremented by a arbitrarily high value + // continue to long_match + + // n_matches > 16 +L_long_match: + cmp match_distance,#31 + bhi L_copy_long_match_32 + cmp match_distance,#15 + bhi L_copy_long_match_16 + + // Copy match, n_matches >= 16, match_distance < 16: + // load shuffle table, and permute to replicate the pattern on 32 bytes +L_copy_long_match_small_distance: + vld1.8 {q1}, [copy_src] // 16 pattern bytes + add aux1,match_permtable,match_distance,lsl #5 // index in table + vld1.8 {q2-q3}, [aux1] // load 32-byte permutation + + vtbl.8 d4, {q1}, d4 // low 16 bytes of pattern + vtbl.8 d5, {q1}, d5 // low 16 bytes of pattern + vtbl.8 d6, {q1}, d6 // low 16 bytes of pattern + vtbl.8 d7, {q1}, d7 // low 16 bytes of pattern + + ldrb aux1,[match_disttable,match_distance] // valid pattern length in aux1 + // fixed + vst1.8 {q2-q3},[copy_dst] + add copy_dst,copy_dst,aux1 +L_copy_long_match_small_distance_loop: + // loop + vst1.8 {q2-q3},[copy_dst] + add copy_dst,copy_dst,aux1 + vst1.8 {q2-q3},[copy_dst] + add copy_dst,copy_dst,aux1 + cmp dst,copy_dst + bhi L_copy_long_match_small_distance_loop + b L_decode_command + + // Copy match, n_matches >= 16, match_distance >= 32 +L_copy_long_match_32: + // fixed + loop + copy_1x16_and_increment copy_src,copy_dst +L_copy_long_match_32_loop: + copy_1x32_and_increment copy_src,copy_dst + cmp dst,copy_dst + bhi L_copy_long_match_32_loop + b L_decode_command + + // Copy match, n_matches >= 16, match_distance >= 16 +L_copy_long_match_16: + // fixed + loop + copy_1x16_and_increment copy_src,copy_dst +L_copy_long_match_16_loop: + copy_2x16_and_increment copy_src,copy_dst + cmp dst,copy_dst + bhi L_copy_long_match_16_loop + b L_decode_command + + +// permutation tables for short distance matches, 32 byte result, for match_distance = 0 to 15 +// value(d)[i] = i%d for i = 0..31 +.p2align 6 +L_match_permtable: +.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 // 0 +.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 // 1 +.byte 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 // 2 +.byte 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1 // 3 +.byte 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 // 4 +.byte 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1 // 5 +.byte 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1 // 6 +.byte 0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3 // 7 +.byte 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7 // 8 +.byte 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4 // 9 +.byte 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1 // 10 +.byte 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 // 11 +.byte 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11, 0, 1, 2, 3, 4, 5, 6, 7 // 12 +.byte 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12, 0, 1, 2, 3, 4, 5 // 13 +.byte 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13, 0, 1, 2, 3 // 14 +.byte 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14, 0, 1 // 15 + +// valid repeating pattern size, for each match_distance = 0 to 15 +// value(d) = 32 - (32%d), is the largest a multiple of d <= 32 +.p2align 6 +L_match_disttable: +.byte 32,32,32,30 // 0 .. 3 +.byte 16,30,30,28 // 4 .. 7 +.byte 16,27,30,22 // 8 .. 11 +.byte 24,26,28,30 // 12 .. 15 + +#endif // LZ4_ENABLE_ASSEMBLY_DECODE_ARMV7 diff --git a/osfmk/arm/lz4_encode_armv7.s b/osfmk/arm/lz4_encode_armv7.s new file mode 100644 index 000000000..370386791 --- /dev/null +++ b/osfmk/arm/lz4_encode_armv7.s @@ -0,0 +1,429 @@ +/* + * Copyright (c) 2016-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include <vm/lz4_assembly_select.h> +#if LZ4_ENABLE_ASSEMBLY_ENCODE_ARMV7 + +/* void lz4_encode_2gb(uint8_t ** dst_ptr, + size_t dst_size, + const uint8_t ** src_ptr, + const uint8_t * src_begin, + size_t src_size, + lz4_hash_entry_t hash_table[LZ4_COMPRESS_HASH_ENTRIES], + int skip_final_literals) */ + +.globl _lz4_encode_2gb +.syntax unified + +#define dst_ptr r0 +#define dst_end r1 +#define src_ptr r2 +#define src_beg r3 +#define src_end r4 +#define table r5 +#define mch_ptr r6 +#define mch_dis r8 +#define mch_len r9 +#define mch_ref r10 + +#define margin 128 + +.macro establish_frame + push {r4-r7, lr} + add r7, sp, #12 + push {r8-r11} + ldrd r4, r5, [sp, #36] + push {r0, r2} + ldr dst_ptr, [r0] + ldr src_ptr, [r2] + subs r1, r1, margin // subtract safety margin from dst_size + bls L_done + add dst_end, dst_ptr, r1 // dst end - margin + sub r4, r4, margin // subtract safety margin from src_size (src_size < margin is detected by check on mch_ptr in match_candidate_loop). + add src_end, src_ptr, r4 // src end - margin. + vmov.i8 q1, #255 // vector of all 1s used to emit +.endm + +.macro clear_frame_and_return + pop {r1, r3} + str dst_ptr, [r1] + str src_ptr, [r3] + pop {r8-r11} + pop {r4-r7, pc} +.endm + +.p2align 4 +_lz4_encode_2gb: + establish_frame +L_next_match: + // Start searching for the next match, starting at the end of the last one. + // [We begin with mch_ptr = src_ptr - 1 because we pre-increment mch_ptr + // within the search loop itself]. Load the hash magic number in lr, and + // zero out mch_len (when we find a match, its length will initially be + // four, but we actually work with the match length minus four at all times). + ldr lr, L_hash + sub mch_ptr, src_ptr, #1 + +L_match_candidate_loop: + // If mch_ptr >= src_end, we are near the end of the source buffer (remember + // that we subtracted margin from src_end, so we are *not* actually past the + // end just yet). + cmp mch_ptr, src_end + bhs L_trailing_literal + + // Load the four-byte word starting at mch_ptr, and get the address of the + // corresponding row of the hash table. + ldr r9, [mch_ptr, #1]! + sub r8, mch_ptr, src_beg + mul r12, r9, lr + mvn r10, #0x7 + and r12, r10, r12, lsr #17 // byte offset of table entry. + + // Load offset and word from hash table row, then update with the new offset + // and word that we just computed. + ldrd r10,r11, [table, r12] + strd r8, r9, [table, r12] + + // At this point, we only know that the hashes of the words match; check to + // see if the words themselves match. If not, move on to the next candidate. + cmp r9, r11 + bne L_match_candidate_loop + + // It's not enough for the words to match; the match distance must also be + // in the representable range (i.e. less than 0x10000). + sub mch_dis, r8, r10 + add mch_ref, src_beg, r10 + cmp mch_dis, #0x10000 + bhs L_match_candidate_loop + + // We have found a match; registers at this point are as follows: + // + // register symbolic name meaning + // r0 dst_ptr pointer into destination buffer where the + // match information will be stored. + // r1 dst_end pointer to the end of the destination buffer, + // less margin. + // r2 src_ptr pointer to the byte after the last match that + // we found, or to the point from which we + // started searching if this is the first match. + // r3 src_beg pointer to the actual start of the buffer. + // r4 src_end pointer to the end of the source buffer, less + // margin. + // r5 table address of hash table. + // r6 mch_ptr pointer to match. + // r8 mch_dis match distance ("D") + // r9 mch_len length of match less four ("M") + // r10 mch_ref pointer to match reference. + // r11 - + // r12 - + // lr - + // + // Attempt to grow the match backwards (typically we only grow backwards by + // a byte or two, if at all, so we use a byte-by-byte scan). + eor mch_len, mch_len +0:cmp mch_ref, src_beg + cmpne mch_ptr, src_ptr + beq 1f + ldrb r11, [mch_ref, #-1] + ldrb r12, [mch_ptr, #-1] + cmp r11, r12 + bne 1f + sub mch_ref, #1 + sub mch_ptr, #1 + add mch_len, #1 + b 0b + + // Now that we have the start of the match, we can compute the literal + // length. Then advance the mch and ref pointers to the end of the match + // and its reference. Because mch_len is the real match length minus four, + // we actually advance to four before the end of the match, but our loop + // to grow the matches uses pre-incremented loads with writeback, so this + // works out correctly. +#define lit_len lr +1:sub lit_len, mch_ptr, src_ptr + add mch_ptr, mch_len + add mch_ref, mch_len + + // Now attempt to grow the match forwards. This is much more common, and + // there is a safety margin at the end of the buffer, so we grow forwards + // in four-byte chunks. +0:ldr r11, [mch_ptr, #4]! + ldr r12, [mch_ref, #4]! + eors r11, r12 + bne 1f + add mch_len, #4 + cmp mch_ptr, src_end + blo 0b + b L_emit_match + // At least one of the bytes in the last comparison did not match. Identify + // which byte had the mismatch and compute the final length (less four). +1:rev r11, r11 + clz r11, r11 + add mch_len, r11, lsr #3 + +L_emit_match: + // Time to emit what we've found! + // + // register symbolic name meaning + // r0 dst_ptr pointer into destination buffer where the + // match information will be stored. + // r1 dst_end pointer to the end of the destination buffer, + // less margin. + // r2 src_ptr pointer to the byte after the last match that + // we found, or to the point from which we + // started searching if this is the first match. + // r3 src_beg pointer to the actual start of the buffer. + // r4 src_end pointer to the end of the source buffer, less + // margin. + // r5 table address of hash table. + // r6 - + // r8 mch_dis match distance ("D") + // r9 mch_len length of match ("M") + // r10 - + // r11 - + // r12 - + // lr lit_len literal length ("L") + // q1 vector of all ones + // + // Synthesize control byte under the assumption that L and M are both less + // than 15, jumping out of the fast path if one of them is not. + cmp lit_len, #15 + orr r10, mch_len, lit_len, lsl #4 + cmplo mch_len, #15 + bhs L_emit_careful + // L and M are both less than 15, which means (a) we use the most compact + // encoding for the match and (b) we do not need to do a bounds check on + // the destination buffer before writing, only before continuing our search. + // Store the command byte. + strb r10, [dst_ptr], #1 + // Copy literal. + vld1.8 q0, [src_ptr] + add src_ptr, lit_len + vst1.8 q0, [dst_ptr] + add dst_ptr, lit_len + // Restore "true" match length before updating src_ptr. + add mch_len, #4 + // Store match distance (D) and update the source pointer. + strh r8, [dst_ptr], #2 + add src_ptr, mch_len + // If we're not into the safety margin of the destination buffer, look for + // another match. + cmp dst_ptr, dst_end + blo L_next_match + // If we *are* into the safety margin of the destination buffer, we're done + // encoding this block; update the source and destination pointers and + // return. +L_done: + clear_frame_and_return + +// Constant island +L_hash: .long 2654435761 +L_magic: .long 0x80808081 + +L_emit_careful: + // Either L or M is >= 15, which means that we don't get to use the compact + // encoding, and that we need to do extra bounds checks while writing. + // + // register symbolic name meaning + // r0 dst_ptr pointer into destination buffer where the + // match information will be stored. + // r1 dst_end pointer to the end of the destination buffer, + // less margin. + // r2 src_ptr pointer to the byte after the last match that + // we found, or to the point from which we + // started searching if this is the first match. + // r3 src_beg pointer to the actual start of the buffer. + // r4 src_end pointer to the end of the source buffer, less + // margin. + // r5 table address of hash table. + // r6 - + // r8 mch_dis match distance ("D") + // r9 mch_len length of match ("M") less four + // r10 - + // r11 - + // r12 - + // lr lit_len literal length ("L") + // q1 vector of all ones + // + // Start by creating the low 4 bits of the control word; M if M < 15, 0xf + // otherwise. We also load 0x80808081, which is the magic number for + // division by 255; this will be required later on. + ldr r12, L_magic + cmp mch_len, #15 + mov r10, mch_len + movhs r10, #0x0f + subs r6, lit_len, #15 + bhs L_large_L + // M is large, but L is < 15. This means we can use the simple approach + // for copying the literal with no bounds checks. + orr r10, lit_len, lsl #4 + strb r10, [dst_ptr], #1 + // Copy literal. + vld1.8 q0, [src_ptr] + add src_ptr, lit_len + vst1.8 q0, [dst_ptr] + add dst_ptr, lit_len + // Store match distance (D). + strh r8, [dst_ptr], #2 + sub r6, mch_len, #15 + b L_large_M + +L_large_L: + // L is large, M may or may not be. We need to encode extra literal length + // bytes and we need to do bounds checks while store both those byte and the + // literal itself. + orr r10, #0xf0 + strb r10, [dst_ptr], #1 + // How many extra literal bytes do we need to store? We need to store + // (L - 15)/255 extra literal bytes of 0xff, plus one more byte that is + // (L - 15)%255. Get these quantities via magic number multiplication: + // (L - 15)*0x80808081 >> (32 + 7) + umull r10, r11, r6, r12 + mov r12, #255 + lsr r10, r11, #7 // (L - 15) / 255 + mls r11, r10, r12, r6 // (L - 15) % 255 + ldr r12, L_magic // may need magic number again for M. + // Compute address dst_ptr will have after storing all 0xff bytes, and + // check that we won't exceed dst_end in doing so. + add r10, dst_ptr, r10 + cmp r10, dst_end + bhs L_done + // There's enough space for all the 0xff bytes, so go ahead and store them. +0:vst1.8 q1, [dst_ptr]! + cmp dst_ptr, r10 + blo 0b + // Store the (L - 15) % 255 byte. + strb r11, [r10], #1 + // Compute the address we'll have reached after storing all literal bytes. + // If that passes dst_end, we're done. + add dst_ptr, r10, lit_len + cmp dst_ptr, dst_end + bhs L_done + // Copy the literal. +0:vld1.8 q0, [src_ptr]! + vst1.8 q0, [r10]! + subs r6, r10, dst_ptr + blo 0b + // Fixup src_ptr, store match distance (D), and check whether or not M is + // bigger than 14. If not, go find the next match. + strh r8, [dst_ptr], #2 + sub src_ptr, r6 + subs r6, mch_len, #15 + bhs L_large_M + // M is small, so we're all done; we just need to update the source pointer + // and we can go look for the next match. + add mch_len, #4 + add src_ptr, mch_len + b L_next_match + +L_large_M: + // Just like with large L, we split (M - 15) into (M - 15) / 255 and + // (M - 15) % 255 via magic number multiply. + umull r10, r11, r6, r12 + mov r12, #255 + lsr r10, r11, #7 // (M - 15) / 255 + mls r11, r10, r12, r6 // (M - 15) % 255 + // Compute address dst_ptr will have after storing all 0xff bytes, and + // check that we won't exceed dst_end in doing so. + add r10, dst_ptr, r10 + cmp r10, dst_end + bhs L_done + // There's enough space for all the 0xff bytes, so go ahead and store them. +0:vst1.8 q1, [dst_ptr]! + cmp dst_ptr, r10 + blo 0b + // Store final M % 255 byte, update dst_ptr and src_ptr, and look for next + // match. + strb r11, [r10] + add mch_len, #4 + add dst_ptr, r10, #1 + add src_ptr, mch_len + b L_next_match + +L_trailing_literal: + // Check if skip_final_literals is set. + ldr r5, [sp, #52] + tst r5, r5 + bne L_done + // Emit a trailing literal that covers the remainder of the source buffer, + // if we can do so without exceeding the bounds of the destination buffer. + add src_end, margin + sub lit_len, src_end, src_ptr + subs r6, lit_len, #15 + bhs L_trailing_literal_long + lsl r10, lit_len, #4 + strb r10, [dst_ptr], #1 + vld1.8 q0, [src_ptr] + mov src_ptr, src_end + vst1.8 q0, [dst_ptr] + add dst_ptr, lit_len + b L_done + +L_trailing_literal_long: + ldr r12, L_magic + mov r10, #0xf0 + add dst_end, margin + strb r10, [dst_ptr], #1 + umull r10, r11, r6, r12 + mov r12, #255 + lsr r10, r11, #7 // (L - 15) / 255 + mls r11, r10, r12, r6 // (L - 15) % 255 + // We want to write out lit_len + (L - 15)/255 + 1 bytes. Check if we have + // space for all of them. + add r10, dst_ptr + add r12, r10, lit_len + cmp r12, dst_end + bhs L_done + // We have enough space, so go ahead and write them all out. Because we + // know that we have enough space, and that the literal is at least 15 bytes, + // we can write the block of 0xffs using vector stores, even without a + // safety margin. +0:vst1.8 q1, [dst_ptr]! + cmp dst_ptr, r10 + blo 0b + // Store the (L - 15) % 255 byte. + strb r11, [r10], #1 + mov dst_ptr, r10 + // Now store the literal itself; here we need to actually be somewhat + // careful to ensure that we don't write past the end of the destination + // buffer or read past the end of the source buffer. + subs lit_len, #16 + blo 1f +0:vld1.8 q0, [src_ptr]! + subs lit_len, #16 + vst1.8 q0, [dst_ptr]! + bhs 0b +1:adds lit_len, #16 + beq L_done +2:ldrb r6, [src_ptr], #1 + subs lit_len, #1 + strb r6, [dst_ptr], #1 + bne 2b + b L_done + +#endif diff --git a/osfmk/arm/machdep_call.c b/osfmk/arm/machdep_call.c new file mode 100644 index 000000000..b77c3c7ce --- /dev/null +++ b/osfmk/arm/machdep_call.c @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * Copyright (c) 1992 NeXT Computer, Inc. + * + * Machine dependent kernel calls. + * + * HISTORY + * + * 17 June 1992 ? at NeXT + * Created. + */ + +#include <kern/thread.h> +#include <mach/mach_types.h> +#include <arm/machdep_call.h> +#if __arm64__ +#include <arm64/machine_machdep.h> +#endif + +extern kern_return_t kern_invalid(void); + +uintptr_t +get_tpidrro(void) +{ + uintptr_t uthread; +#if __arm__ + uthread = __builtin_arm_mrc(15, 0, 13, 0, 3); // TPIDRURO +#else + __asm__ volatile("mrs %0, TPIDRRO_EL0" : "=r" (uthread)); +#endif + return uthread; +} + +void +set_tpidrro(uintptr_t uthread) +{ +#if __arm__ + __builtin_arm_mcr(15, 0, uthread, 13, 0, 3); // TPIDRURO +#else + __asm__ volatile("msr TPIDRRO_EL0, %0" : : "r" (uthread)); +#endif +} + +kern_return_t +thread_set_cthread_self(vm_address_t self) +{ + return machine_thread_set_tsd_base(current_thread(), self); +} + +vm_address_t +thread_get_cthread_self(void) +{ + uintptr_t self; + + self = get_tpidrro(); +#if __arm__ + self &= ~3; + assert( self == current_thread()->machine.cthread_self); + return ((kern_return_t) current_thread()->machine.cthread_self); +#else + self &= MACHDEP_CTHREAD_MASK; + assert( self == current_thread()->machine.cthread_self); + return self; +#endif +} + diff --git a/osfmk/arm/machdep_call.h b/osfmk/arm/machdep_call.h new file mode 100644 index 000000000..1f7f1606f --- /dev/null +++ b/osfmk/arm/machdep_call.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * Copyright (c) 1992 NeXT Computer, Inc. + * + * Machine dependent kernel call table defines. + * + * HISTORY + * + * 17 June 1992 ? at NeXT + * Created. + */ + +typedef union { + kern_return_t (*args_0)(void); + kern_return_t (*args_1)(vm_address_t); + kern_return_t (*args_2)(vm_address_t,vm_address_t); + kern_return_t (*args_3)(vm_address_t,vm_address_t,vm_address_t); + kern_return_t (*args_4)(vm_address_t, vm_address_t,vm_address_t,vm_address_t); + kern_return_t (*args_var)(vm_address_t,...); +} machdep_call_routine_t; + +#define MACHDEP_CALL_ROUTINE(func,args) \ + { { .args_ ## args = func }, args } + +typedef struct { + machdep_call_routine_t routine; + int nargs; +} machdep_call_t; + +extern const machdep_call_t machdep_call_table[]; +extern int machdep_call_count; + +extern vm_address_t thread_get_cthread_self(void); +extern kern_return_t thread_set_cthread_self(vm_address_t); + +// Read and write raw TPIDRURO / TPIDRRO_EL0 +uintptr_t get_tpidrro(void); +void set_tpidrro(uintptr_t); + diff --git a/osfmk/arm/machine_cpu.h b/osfmk/arm/machine_cpu.h new file mode 100644 index 000000000..64d795e70 --- /dev/null +++ b/osfmk/arm/machine_cpu.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _ARM_MACHINE_CPU_H_ +#define _ARM_MACHINE_CPU_H_ + +#include <mach/mach_types.h> +#include <mach/boolean.h> +#include <kern/kern_types.h> +#include <pexpert/pexpert.h> +#include <arm/cpu_data_internal.h> + +extern void cpu_machine_init(void); + +extern kern_return_t cpu_register(int *slot_nump); + +extern void cpu_signal_handler(void); +extern void cpu_signal_handler_internal(boolean_t disable_signal); + +extern void cpu_doshutdown(void (*doshutdown)(processor_t), processor_t processor); + +extern void cpu_idle(void); +extern void cpu_idle_exit(void) __attribute__((noreturn)); +extern void cpu_idle_tickle(void); + +extern void cpu_machine_idle_init(boolean_t from_boot); + +extern void arm_init_cpu(cpu_data_t *args); + +extern void arm_init_idle_cpu(cpu_data_t *args); + +extern void init_cpu_timebase(boolean_t enable_fiq); + +#define cpu_pause() do {} while (0) /* Not for this architecture */ + +#endif /* _ARM_MACHINE_CPU_H_ */ diff --git a/osfmk/arm/machine_cpuid.c b/osfmk/arm/machine_cpuid.c new file mode 100644 index 000000000..232c6e64a --- /dev/null +++ b/osfmk/arm/machine_cpuid.c @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include <arm/cpuid.h> +#include <arm/cpuid_internal.h> +#include <machine/atomic.h> +#include <machine/machine_cpuid.h> +#include <arm/cpu_data_internal.h> + +static arm_mvfp_info_t cpuid_mvfp_info; +static arm_debug_info_t cpuid_debug_info; + +uint32_t +machine_read_midr(void) +{ +#if __arm__ + uint32_t midr = __builtin_arm_mrc(15,0,0,0,0); +#else + uint64_t midr; + __asm__ volatile("mrs %0, MIDR_EL1" : "=r" (midr)); +#endif + return (uint32_t)midr; +} + +uint32_t +machine_read_clidr(void) +{ +#if __arm__ + uint32_t clidr = __builtin_arm_mrc(15,1,0,0,1); +#else + uint64_t clidr; + __asm__ volatile("mrs %0, CLIDR_EL1" : "=r" (clidr)); +#endif + return (uint32_t)clidr; +} + +uint32_t +machine_read_ccsidr(void) +{ +#if __arm__ + uint32_t ccsidr = __builtin_arm_mrc(15,1,0,0,0); +#else + uint64_t ccsidr; + __asm__ volatile("mrs %0, CCSIDR_EL1" : "=r" (ccsidr)); +#endif + return (uint32_t)ccsidr; +} + +#if __arm__ +arm_isa_feat1_reg +machine_read_isa_feat1(void) +{ + arm_isa_feat1_reg isa; + isa.value = __builtin_arm_mrc(15,0,0,2,1); + return isa; +} +#endif // __arm__ + +void +machine_write_csselr(csselr_cache_level level, csselr_cache_type type) +{ +#if __arm__ + uint32_t csselr = (level | type); + __builtin_arm_mcr(15,2,csselr,0,0,0); +#else + uint64_t csselr = (level | type); + __asm__ volatile("msr CSSELR_EL1, %0" : : "r" (csselr)); +#endif + __builtin_arm_isb(ISB_SY); +} + +void +machine_do_debugid(void) +{ +#if __arm__ + arm_cpuid_id_dfr0 id_dfr0; + arm_debug_dbgdidr dbgdidr; + + /* read CPUID ID_DFR0 */ + id_dfr0.value = __builtin_arm_mrc(15,0,0,1,2); + /* read DBGDIDR */ + dbgdidr.value = __builtin_arm_mrc(14,0,0,0,0); + + cpuid_debug_info.coprocessor_core_debug = id_dfr0.debug_feature.coprocessor_core_debug != 0; + cpuid_debug_info.memory_mapped_core_debug = (id_dfr0.debug_feature.memory_mapped_core_debug != 0) + && (getCpuDatap()->cpu_debug_interface_map != 0); + + if (cpuid_debug_info.coprocessor_core_debug || cpuid_debug_info.memory_mapped_core_debug) { + cpuid_debug_info.num_watchpoint_pairs = dbgdidr.debug_id.wrps + 1; + cpuid_debug_info.num_breakpoint_pairs = dbgdidr.debug_id.brps + 1; + } +#else + arm_cpuid_id_aa64dfr0_el1 id_dfr0; + + /* read ID_AA64DFR0_EL1 */ + __asm__ volatile("mrs %0, ID_AA64DFR0_EL1" : "=r"(id_dfr0.value)); + + if (id_dfr0.debug_feature.debug_arch_version) { + cpuid_debug_info.num_watchpoint_pairs = id_dfr0.debug_feature.wrps + 1; + cpuid_debug_info.num_breakpoint_pairs = id_dfr0.debug_feature.brps + 1; + } +#endif +} + +arm_debug_info_t * +machine_arm_debug_info(void) +{ + return &cpuid_debug_info; +} + +void +machine_do_mvfpid() +{ + arm_mvfr0_info_t arm_mvfr0_info; + arm_mvfr1_info_t arm_mvfr1_info; + uint64_t tmp; + +#if __arm__ + (void)tmp; + __asm__ volatile("vmrs %0, mvfr0":"=r"(arm_mvfr0_info.value)); + __asm__ volatile("vmrs %0, mvfr1":"=r"(arm_mvfr1_info.value)); +#else + __asm__ volatile("mrs %0, MVFR0_EL1":"=r"(tmp)); + arm_mvfr0_info.value = (uint32_t)tmp; + + __asm__ volatile("mrs %0, MVFR1_EL1":"=r"(tmp)); + arm_mvfr1_info.value = (uint32_t)tmp; +#endif + + cpuid_mvfp_info.neon = arm_mvfr1_info.bits.SP; + cpuid_mvfp_info.neon_hpfp = arm_mvfr1_info.bits.HPFP; +} + +arm_mvfp_info_t * +machine_arm_mvfp_info(void) +{ + return &cpuid_mvfp_info; +} + diff --git a/osfmk/arm/machine_cpuid.h b/osfmk/arm/machine_cpuid.h new file mode 100644 index 000000000..e50ac9302 --- /dev/null +++ b/osfmk/arm/machine_cpuid.h @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _ARM_MACHINE_CPUID_H_ +#define _ARM_MACHINE_CPUID_H_ + +/* CPU feature identification */ + +typedef struct { +uint32_t arm_32bit_isa : 4, + arm_thumb_ver : 4, + arm_jazelle : 4, + arm_thumb2 : 4, + reserved : 16; +} arm_feature_bits_t; + +typedef union { + arm_feature_bits_t field; + uint32_t value; +} arm_feature0_reg_t; + +// Register 0, subtype 21: Instruction Set Features #1 +typedef struct +{ + uint32_t endianness_support : 4; + uint32_t exception_1_support : 4; + uint32_t exception_2_support : 4; + uint32_t sign_zero_ext_support : 4; + uint32_t if_then_support : 4; + uint32_t immediate_support : 4; + uint32_t interworking_support : 4; + uint32_t jazelle_support : 4; +} +syscp_ID_instructions_feat_1_reg; + +typedef union { + uint32_t value; + syscp_ID_instructions_feat_1_reg field; +} arm_isa_feat1_reg; + +arm_isa_feat1_reg machine_read_isa_feat1(void); + +/* Debug identification */ + +/* ID_DFR0 */ +typedef union { + struct { + uint32_t coprocessor_core_debug : 4, + coprocessor_secure_debug : 4, + memory_mapped_core_debug : 4, + coprocessor_trace_debug : 4, + memory_mapped_trace_debug : 4, + microcontroller_debug : 4; + } debug_feature; + uint32_t value; +} arm_cpuid_id_dfr0; + +/* DBGDIDR */ +typedef union { + struct { + uint32_t revision : 4, + variant : 4, + : 4, + se_imp : 1, + pcsr_imp : 1, + nsuhd_imp : 1, + : 1, + version : 4, + ctx_cmps : 4, + brps : 4, + wrps : 4; + } debug_id; + uint32_t value; +} arm_debug_dbgdidr; + +typedef struct { + boolean_t memory_mapped_core_debug; + boolean_t coprocessor_core_debug; + uint32_t num_watchpoint_pairs; + uint32_t num_breakpoint_pairs; +} arm_debug_info_t; + +#endif /* _ARM_MACHINE_CPUID_H_ */ diff --git a/osfmk/arm/machine_kpc.h b/osfmk/arm/machine_kpc.h new file mode 100644 index 000000000..ec7aed315 --- /dev/null +++ b/osfmk/arm/machine_kpc.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2013 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _MACHINE_ARM_KPC_H +#define _MACHINE_ARM_KPC_H + +#ifdef ARMA7 + +#define KPC_ARM_FIXED_COUNT 1 +#define KPC_ARM_CONFIGURABLE_COUNT 4 + +#define KPC_ARM_TOTAL_COUNT (KPC_ARM_FIXED_COUNT + KPC_ARM_CONFIGURABLE_COUNT) + +#define KPC_ARM_COUNTER_WIDTH 32 + +#else + +#define KPC_ARM_FIXED_COUNT 2 +#define KPC_ARM_CONFIGURABLE_COUNT 6 + +#define KPC_ARM_COUNTER_WIDTH 39 +#define KPC_ARM_COUNTER_MASK ((1ull << KPC_ARM_COUNTER_WIDTH) - 1) +#define KPC_ARM_COUNTER_OVF_BIT (39) +#define KPC_ARM_COUNTER_OVF_MASK (1ull << KPC_ARM_COUNTER_OVF_BIT) + +#endif + +typedef uint64_t kpc_config_t; + +/* Size to the maximum number of counters we could read from every class in one go */ +#define KPC_MAX_COUNTERS (KPC_ARM_FIXED_COUNT + KPC_ARM_CONFIGURABLE_COUNT + 1) + +/* arm32 uses fixed counter shadows */ +#define FIXED_COUNTER_SHADOW (1) + +#endif /* _MACHINE_ARM_KPC_H */ diff --git a/osfmk/arm/machine_routines.c b/osfmk/arm/machine_routines.c new file mode 100644 index 000000000..b86bd643b --- /dev/null +++ b/osfmk/arm/machine_routines.c @@ -0,0 +1,1176 @@ +/* + * Copyright (c) 2007-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include <arm/proc_reg.h> +#include <arm/machine_cpu.h> +#include <arm/cpu_internal.h> +#include <arm/cpuid.h> +#include <arm/io_map_entries.h> +#include <arm/cpu_data.h> +#include <arm/cpu_data_internal.h> +#include <arm/misc_protos.h> +#include <arm/rtclock.h> +#include <arm/caches_internal.h> +#include <console/serial_protos.h> +#include <kern/machine.h> +#include <prng/random.h> +#include <kern/startup.h> +#include <kern/sched.h> +#include <kern/thread.h> +#include <mach/machine.h> +#include <machine/atomic.h> +#include <vm/pmap.h> +#include <vm/vm_page.h> +#include <sys/kdebug.h> +#include <kern/coalition.h> +#include <pexpert/device_tree.h> + +#include <IOKit/IOPlatformExpert.h> +#include <libkern/section_keywords.h> + +#if KPC +#include <kern/kpc.h> +#endif + +static int max_cpus_initialized = 0; +#define MAX_CPUS_SET 0x1 +#define MAX_CPUS_WAIT 0x2 + +static unsigned int avail_cpus = 0; + +uint32_t LockTimeOut; +uint32_t LockTimeOutUsec; +uint64_t MutexSpin; +boolean_t is_clock_configured = FALSE; + +extern int mach_assert; +extern volatile uint32_t debug_enabled; +SECURITY_READ_ONLY_LATE(unsigned int) debug_boot_arg; + +void machine_conf(void); + +void +machine_startup(__unused boot_args * args) +{ + int boot_arg; + +#if MACH_KDP + if (PE_parse_boot_argn("debug", &debug_boot_arg, sizeof (debug_boot_arg)) && + debug_enabled) { +#if DEVELOPMENT || DEBUG + if (debug_boot_arg & DB_HALT) + halt_in_debugger = 1; +#endif + if (debug_boot_arg & DB_NMI) + panicDebugging = TRUE; + } else { + debug_boot_arg = 0; + } +#endif + + PE_parse_boot_argn("assert", &mach_assert, sizeof (mach_assert)); + + if (PE_parse_boot_argn("preempt", &boot_arg, sizeof (boot_arg))) { + default_preemption_rate = boot_arg; + } + if (PE_parse_boot_argn("bg_preempt", &boot_arg, sizeof (boot_arg))) { + default_bg_preemption_rate = boot_arg; + } + + machine_conf(); + + /* + * Kick off the kernel bootstrap. + */ + kernel_bootstrap(); + /* NOTREACHED */ +} + +char * +machine_boot_info( + __unused char *buf, + __unused vm_size_t size) +{ + return (PE_boot_args()); +} + +void +machine_conf(void) +{ + machine_info.memory_size = mem_size; +} + +void +machine_init(void) +{ + debug_log_init(); + clock_config(); + is_clock_configured = TRUE; + if (debug_enabled) + pmap_map_globals(); +} + +void +slave_machine_init(__unused void *param) +{ + cpu_machine_init(); /* Initialize the processor */ + clock_init(); /* Init the clock */ +} + +/* + * Routine: machine_processor_shutdown + * Function: + */ +thread_t +machine_processor_shutdown( + __unused thread_t thread, + void (*doshutdown) (processor_t), + processor_t processor) +{ + return (Shutdown_context(doshutdown, processor)); +} + +/* + * Routine: ml_init_max_cpus + * Function: + */ +void +ml_init_max_cpus(unsigned int max_cpus) +{ + boolean_t current_state; + + current_state = ml_set_interrupts_enabled(FALSE); + if (max_cpus_initialized != MAX_CPUS_SET) { + machine_info.max_cpus = max_cpus; + machine_info.physical_cpu_max = max_cpus; + machine_info.logical_cpu_max = max_cpus; + if (max_cpus_initialized == MAX_CPUS_WAIT) + thread_wakeup((event_t) & max_cpus_initialized); + max_cpus_initialized = MAX_CPUS_SET; + } + (void) ml_set_interrupts_enabled(current_state); +} + +/* + * Routine: ml_get_max_cpus + * Function: + */ +unsigned int +ml_get_max_cpus(void) +{ + boolean_t current_state; + + current_state = ml_set_interrupts_enabled(FALSE); + if (max_cpus_initialized != MAX_CPUS_SET) { + max_cpus_initialized = MAX_CPUS_WAIT; + assert_wait((event_t) & max_cpus_initialized, THREAD_UNINT); + (void) thread_block(THREAD_CONTINUE_NULL); + } + (void) ml_set_interrupts_enabled(current_state); + return (machine_info.max_cpus); +} + +/* + * Routine: ml_init_lock_timeout + * Function: + */ +void +ml_init_lock_timeout(void) +{ + uint64_t abstime; + uint64_t mtxspin; + uint64_t default_timeout_ns = NSEC_PER_SEC>>2; + uint32_t slto; + + if (PE_parse_boot_argn("slto_us", &slto, sizeof (slto))) + default_timeout_ns = slto * NSEC_PER_USEC; + + nanoseconds_to_absolutetime(default_timeout_ns, &abstime); + LockTimeOutUsec = (uint32_t)(abstime / NSEC_PER_USEC); + LockTimeOut = (uint32_t)abstime; + + if (PE_parse_boot_argn("mtxspin", &mtxspin, sizeof (mtxspin))) { + if (mtxspin > USEC_PER_SEC>>4) + mtxspin = USEC_PER_SEC>>4; + nanoseconds_to_absolutetime(mtxspin*NSEC_PER_USEC, &abstime); + } else { + nanoseconds_to_absolutetime(10*NSEC_PER_USEC, &abstime); + } + MutexSpin = abstime; +} + +/* + * This is called from the machine-independent routine cpu_up() + * to perform machine-dependent info updates. + */ +void +ml_cpu_up(void) +{ + hw_atomic_add(&machine_info.physical_cpu, 1); + hw_atomic_add(&machine_info.logical_cpu, 1); +} + +/* + * This is called from the machine-independent routine cpu_down() + * to perform machine-dependent info updates. + */ +void +ml_cpu_down(void) +{ + cpu_data_t *cpu_data_ptr; + + hw_atomic_sub(&machine_info.physical_cpu, 1); + hw_atomic_sub(&machine_info.logical_cpu, 1); + + /* + * If we want to deal with outstanding IPIs, we need to + * do relatively early in the processor_doshutdown path, + * as we pend decrementer interrupts using the IPI + * mechanism if we cannot immediately service them (if + * IRQ is masked). Do so now. + * + * We aren't on the interrupt stack here; would it make + * more sense to disable signaling and then enable + * interrupts? It might be a bit cleaner. + */ + cpu_data_ptr = getCpuDatap(); + cpu_data_ptr->cpu_running = FALSE; + + cpu_signal_handler_internal(TRUE); +} + +/* + * Routine: ml_cpu_get_info + * Function: + */ +void +ml_cpu_get_info(ml_cpu_info_t * ml_cpu_info) +{ + cache_info_t *cpuid_cache_info; + + cpuid_cache_info = cache_info(); + ml_cpu_info->vector_unit = 0; + ml_cpu_info->cache_line_size = cpuid_cache_info->c_linesz; + ml_cpu_info->l1_icache_size = cpuid_cache_info->c_isize; + ml_cpu_info->l1_dcache_size = cpuid_cache_info->c_dsize; + +#if (__ARM_ARCH__ >= 7) + ml_cpu_info->l2_settings = 1; + ml_cpu_info->l2_cache_size = cpuid_cache_info->c_l2size; +#else + ml_cpu_info->l2_settings = 0; + ml_cpu_info->l2_cache_size = 0xFFFFFFFF; +#endif + ml_cpu_info->l3_settings = 0; + ml_cpu_info->l3_cache_size = 0xFFFFFFFF; +} + +unsigned int +ml_get_machine_mem(void) +{ + return (machine_info.memory_size); +} + +/* Return max offset */ +vm_map_offset_t +ml_get_max_offset( + boolean_t is64, + unsigned int option) +{ + unsigned int pmap_max_offset_option = 0; + + switch (option) { + case MACHINE_MAX_OFFSET_DEFAULT: + pmap_max_offset_option = ARM_PMAP_MAX_OFFSET_DEFAULT; + break; + case MACHINE_MAX_OFFSET_MIN: + pmap_max_offset_option = ARM_PMAP_MAX_OFFSET_MIN; + break; + case MACHINE_MAX_OFFSET_MAX: + pmap_max_offset_option = ARM_PMAP_MAX_OFFSET_MAX; + break; + case MACHINE_MAX_OFFSET_DEVICE: + pmap_max_offset_option = ARM_PMAP_MAX_OFFSET_DEVICE; + break; + default: + panic("ml_get_max_offset(): Illegal option 0x%x\n", option); + break; + } + return pmap_max_offset(is64, pmap_max_offset_option); +} + +boolean_t +ml_wants_panic_trap_to_debugger(void) +{ + return FALSE; +} + +void +ml_panic_trap_to_debugger(__unused const char *panic_format_str, + __unused va_list *panic_args, + __unused unsigned int reason, + __unused void *ctx, + __unused uint64_t panic_options_mask, + __unused unsigned long panic_caller) +{ + return; +} + +__attribute__((noreturn)) +void +halt_all_cpus(boolean_t reboot) +{ + if (reboot) { + printf("MACH Reboot\n"); + PEHaltRestart(kPERestartCPU); + } else { + printf("CPU halted\n"); + PEHaltRestart(kPEHaltCPU); + } + while (1); +} + +__attribute__((noreturn)) +void +halt_cpu(void) +{ + halt_all_cpus(FALSE); +} + +/* + * Routine: machine_signal_idle + * Function: + */ +void +machine_signal_idle( + processor_t processor) +{ + cpu_signal(processor_to_cpu_datap(processor), SIGPnop, (void *)NULL, (void *)NULL); + KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_REMOTE_AST), processor->cpu_id, 0 /* nop */, 0, 0, 0); +} + +void +machine_signal_idle_deferred( + processor_t processor) +{ + cpu_signal_deferred(processor_to_cpu_datap(processor)); + KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_REMOTE_DEFERRED_AST), processor->cpu_id, 0 /* nop */, 0, 0, 0); +} + +void +machine_signal_idle_cancel( + processor_t processor) +{ + cpu_signal_cancel(processor_to_cpu_datap(processor)); + KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_REMOTE_CANCEL_AST), processor->cpu_id, 0 /* nop */, 0, 0, 0); +} + +/* + * Routine: ml_install_interrupt_handler + * Function: Initialize Interrupt Handler + */ +void +ml_install_interrupt_handler( + void *nub, + int source, + void *target, + IOInterruptHandler handler, + void *refCon) +{ + cpu_data_t *cpu_data_ptr; + boolean_t current_state; + + current_state = ml_set_interrupts_enabled(FALSE); + cpu_data_ptr = getCpuDatap(); + + cpu_data_ptr->interrupt_nub = nub; + cpu_data_ptr->interrupt_source = source; + cpu_data_ptr->interrupt_target = target; + cpu_data_ptr->interrupt_handler = handler; + cpu_data_ptr->interrupt_refCon = refCon; + + cpu_data_ptr->interrupts_enabled = TRUE; + (void) ml_set_interrupts_enabled(current_state); + + initialize_screen(NULL, kPEAcquireScreen); +} + +/* + * Routine: ml_init_interrupt + * Function: Initialize Interrupts + */ +void +ml_init_interrupt(void) +{ +} + +/* + * Routine: ml_init_timebase + * Function: register and setup Timebase, Decremeter services + */ +void ml_init_timebase( + void *args, + tbd_ops_t tbd_funcs, + vm_offset_t int_address, + vm_offset_t int_value) +{ + cpu_data_t *cpu_data_ptr; + + cpu_data_ptr = (cpu_data_t *)args; + + if ((cpu_data_ptr == &BootCpuData) + && (rtclock_timebase_func.tbd_fiq_handler == (void *)NULL)) { + rtclock_timebase_func = *tbd_funcs; + rtclock_timebase_addr = int_address; + rtclock_timebase_val = int_value; + } +} + +void +ml_parse_cpu_topology(void) +{ + DTEntry entry, child; + OpaqueDTEntryIterator iter; + uint32_t cpu_boot_arg; + int err; + + err = DTLookupEntry(NULL, "/cpus", &entry); + assert(err == kSuccess); + + err = DTInitEntryIterator(entry, &iter); + assert(err == kSuccess); + + while (kSuccess == DTIterateEntries(&iter, &child)) { + +#if MACH_ASSERT + unsigned int propSize; + void *prop = NULL; + if (avail_cpus == 0) { + if (kSuccess != DTGetProperty(child, "state", &prop, &propSize)) + panic("unable to retrieve state for cpu %u", avail_cpus); + + if (strncmp((char*)prop, "running", propSize) != 0) + panic("cpu 0 has not been marked as running!"); + } + assert(kSuccess == DTGetProperty(child, "reg", &prop, &propSize)); + assert(avail_cpus == *((uint32_t*)prop)); +#endif + ++avail_cpus; + } + + cpu_boot_arg = avail_cpus; + if (PE_parse_boot_argn("cpus", &cpu_boot_arg, sizeof(cpu_boot_arg)) && + (avail_cpus > cpu_boot_arg)) + avail_cpus = cpu_boot_arg; + + if (avail_cpus == 0) + panic("No cpus found!"); +} + +unsigned int +ml_get_cpu_count(void) +{ + return avail_cpus; +} + +int +ml_get_boot_cpu_number(void) +{ + return 0; +} + +cluster_type_t +ml_get_boot_cluster(void) +{ + return CLUSTER_TYPE_SMP; +} + +int +ml_get_cpu_number(uint32_t phys_id) +{ + return (int)phys_id; +} + +int +ml_get_max_cpu_number(void) +{ + return avail_cpus - 1; +} + +kern_return_t +ml_processor_register( + ml_processor_info_t * in_processor_info, + processor_t * processor_out, + ipi_handler_t * ipi_handler) +{ + cpu_data_t *this_cpu_datap; + boolean_t is_boot_cpu; + + if (in_processor_info->phys_id >= MAX_CPUS) { + /* + * The physical CPU ID indicates that we have more CPUs than + * this xnu build support. This probably means we have an + * incorrect board configuration. + * + * TODO: Should this just return a failure instead? A panic + * is simply a convenient way to catch bugs in the pexpert + * headers. + */ + panic("phys_id %u is too large for MAX_CPUS (%u)", in_processor_info->phys_id, MAX_CPUS); + } + + /* Fail the registration if the number of CPUs has been limited by boot-arg. */ + if ((in_processor_info->phys_id >= avail_cpus) || + (in_processor_info->log_id > (uint32_t)ml_get_max_cpu_number())) + return KERN_FAILURE; + + if (in_processor_info->log_id != (uint32_t)ml_get_boot_cpu_number()) { + is_boot_cpu = FALSE; + this_cpu_datap = cpu_data_alloc(FALSE); + cpu_data_init(this_cpu_datap); + } else { + this_cpu_datap = &BootCpuData; + is_boot_cpu = TRUE; + } + + this_cpu_datap->cpu_id = in_processor_info->cpu_id; + + this_cpu_datap->cpu_chud = chudxnu_cpu_alloc(is_boot_cpu); + if (this_cpu_datap->cpu_chud == (void *)NULL) + goto processor_register_error; + this_cpu_datap->cpu_console_buf = console_cpu_alloc(is_boot_cpu); + if (this_cpu_datap->cpu_console_buf == (void *)(NULL)) + goto processor_register_error; + + if (!is_boot_cpu) { + if (cpu_data_register(this_cpu_datap) != KERN_SUCCESS) + goto processor_register_error; + } + + this_cpu_datap->cpu_idle_notify = (void *) in_processor_info->processor_idle; + this_cpu_datap->cpu_cache_dispatch = in_processor_info->platform_cache_dispatch; + nanoseconds_to_absolutetime((uint64_t) in_processor_info->powergate_latency, &this_cpu_datap->cpu_idle_latency); + this_cpu_datap->cpu_reset_assist = kvtophys(in_processor_info->powergate_stub_addr); + + this_cpu_datap->idle_timer_notify = (void *) in_processor_info->idle_timer; + this_cpu_datap->idle_timer_refcon = in_processor_info->idle_timer_refcon; + + this_cpu_datap->platform_error_handler = (void *) in_processor_info->platform_error_handler; + this_cpu_datap->cpu_regmap_paddr = in_processor_info->regmap_paddr; + this_cpu_datap->cpu_phys_id = in_processor_info->phys_id; + this_cpu_datap->cpu_l2_access_penalty = in_processor_info->l2_access_penalty; + + if (!is_boot_cpu) { + processor_init((struct processor *)this_cpu_datap->cpu_processor, + this_cpu_datap->cpu_number, processor_pset(master_processor)); + + if (this_cpu_datap->cpu_l2_access_penalty) { + /* + * Cores that have a non-zero L2 access penalty compared + * to the boot processor should be de-prioritized by the + * scheduler, so that threads use the cores with better L2 + * preferentially. + */ + processor_set_primary(this_cpu_datap->cpu_processor, + master_processor); + } + } + + *processor_out = this_cpu_datap->cpu_processor; + *ipi_handler = cpu_signal_handler; + if (in_processor_info->idle_tickle != (idle_tickle_t *) NULL) + *in_processor_info->idle_tickle = (idle_tickle_t) cpu_idle_tickle; + +#if KPC + if (kpc_register_cpu(this_cpu_datap) != TRUE) + goto processor_register_error; +#endif + + if (!is_boot_cpu) + prng_cpu_init(this_cpu_datap->cpu_number); + + return KERN_SUCCESS; + +processor_register_error: +#if KPC + kpc_unregister_cpu(this_cpu_datap); +#endif + if (this_cpu_datap->cpu_chud != (void *)NULL) + chudxnu_cpu_free(this_cpu_datap->cpu_chud); + if (!is_boot_cpu) + cpu_data_free(this_cpu_datap); + return KERN_FAILURE; +} + +void +ml_init_arm_debug_interface( + void * in_cpu_datap, + vm_offset_t virt_address) +{ + ((cpu_data_t *)in_cpu_datap)->cpu_debug_interface_map = virt_address; + do_debugid(); +} + +/* + * Routine: init_ast_check + * Function: + */ +void +init_ast_check( + __unused processor_t processor) +{ +} + +/* + * Routine: cause_ast_check + * Function: + */ +void +cause_ast_check( + processor_t processor) +{ + if (current_processor() != processor) { + cpu_signal(processor_to_cpu_datap(processor), SIGPast, (void *)NULL, (void *)NULL); + KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_REMOTE_AST), processor->cpu_id, 1 /* ast */, 0, 0, 0); + } +} + + +/* + * Routine: ml_at_interrupt_context + * Function: Check if running at interrupt context + */ +boolean_t +ml_at_interrupt_context(void) +{ + vm_offset_t stack_ptr; + vm_offset_t intstack_top_ptr; + + __asm__ volatile("mov %0, sp\n":"=r"(stack_ptr)); + intstack_top_ptr = getCpuDatap()->intstack_top; + return ((stack_ptr < intstack_top_ptr) && (stack_ptr > intstack_top_ptr - INTSTACK_SIZE)); +} + +extern uint32_t cpu_idle_count; + +void ml_get_power_state(boolean_t *icp, boolean_t *pidlep) { + *icp = ml_at_interrupt_context(); + *pidlep = (cpu_idle_count == real_ncpus); +} + +/* + * Routine: ml_cause_interrupt + * Function: Generate a fake interrupt + */ +void +ml_cause_interrupt(void) +{ + return; /* BS_XXX */ +} + +/* Map memory map IO space */ +vm_offset_t +ml_io_map( + vm_offset_t phys_addr, + vm_size_t size) +{ + return (io_map(phys_addr, size, VM_WIMG_IO)); +} + +vm_offset_t +ml_io_map_wcomb( + vm_offset_t phys_addr, + vm_size_t size) +{ + return (io_map(phys_addr, size, VM_WIMG_WCOMB)); +} + +/* boot memory allocation */ +vm_offset_t +ml_static_malloc( + __unused vm_size_t size) +{ + return ((vm_offset_t) NULL); +} + +vm_map_address_t +ml_map_high_window( + vm_offset_t phys_addr, + vm_size_t len) +{ + return pmap_map_high_window_bd(phys_addr, len, VM_PROT_READ | VM_PROT_WRITE); +} + +vm_offset_t +ml_static_ptovirt( + vm_offset_t paddr) +{ + return phystokv(paddr); +} + +vm_offset_t +ml_static_vtop( + vm_offset_t vaddr) +{ + if (((vm_address_t)(vaddr) - gVirtBase) >= gPhysSize) + panic("ml_static_ptovirt(): illegal vaddr: %p\n", (void*)vaddr); + return ((vm_address_t)(vaddr) - gVirtBase + gPhysBase); +} + + +kern_return_t +ml_static_protect( + vm_offset_t vaddr, /* kernel virtual address */ + vm_size_t size, + vm_prot_t new_prot) +{ + pt_entry_t arm_prot = 0; + pt_entry_t arm_block_prot = 0; + vm_offset_t vaddr_cur; + ppnum_t ppn; + kern_return_t result = KERN_SUCCESS; + + if (vaddr < VM_MIN_KERNEL_ADDRESS) + return KERN_FAILURE; + + assert((vaddr & (ARM_PGBYTES - 1)) == 0); /* must be page aligned */ + + if ((new_prot & VM_PROT_WRITE) && (new_prot & VM_PROT_EXECUTE)) { + panic("ml_static_protect(): WX request on %p", (void *) vaddr); + } + + /* Set up the protection bits, and block bits so we can validate block mappings. */ + if (new_prot & VM_PROT_WRITE) { + arm_prot |= ARM_PTE_AP(AP_RWNA); + arm_block_prot |= ARM_TTE_BLOCK_AP(AP_RWNA); + } else { + arm_prot |= ARM_PTE_AP(AP_RONA); + arm_block_prot |= ARM_TTE_BLOCK_AP(AP_RONA); + } + + if (!(new_prot & VM_PROT_EXECUTE)) { + arm_prot |= ARM_PTE_NX; + arm_block_prot |= ARM_TTE_BLOCK_NX; + } + + for (vaddr_cur = vaddr; + vaddr_cur < ((vaddr + size) & ~ARM_PGMASK); + vaddr_cur += ARM_PGBYTES) { + ppn = pmap_find_phys(kernel_pmap, vaddr_cur); + if (ppn != (vm_offset_t) NULL) { + tt_entry_t *ttp = &kernel_pmap->tte[ttenum(vaddr_cur)]; + tt_entry_t tte = *ttp; + + if ((tte & ARM_TTE_TYPE_MASK) != ARM_TTE_TYPE_TABLE) { + if (((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_BLOCK) && + ((tte & (ARM_TTE_BLOCK_APMASK | ARM_TTE_BLOCK_NX_MASK)) == arm_block_prot)) { + /* + * We can support ml_static_protect on a block mapping if the mapping already has + * the desired protections. We still want to run checks on a per-page basis. + */ + continue; + } + + result = KERN_FAILURE; + break; + } + + pt_entry_t *pte_p = (pt_entry_t *) ttetokv(tte) + ptenum(vaddr_cur); + pt_entry_t ptmp = *pte_p; + + ptmp = (ptmp & ~(ARM_PTE_APMASK | ARM_PTE_NX_MASK)) | arm_prot; + *pte_p = ptmp; +#ifndef __ARM_L1_PTW__ + FlushPoC_DcacheRegion((vm_offset_t) pte_p, sizeof(*pte_p)); +#endif + } + } + + if (vaddr_cur > vaddr) + flush_mmu_tlb_region(vaddr, (vm_size_t)(vaddr_cur - vaddr)); + + return result; +} + +/* + * Routine: ml_static_mfree + * Function: + */ +void +ml_static_mfree( + vm_offset_t vaddr, + vm_size_t size) +{ + vm_offset_t vaddr_cur; + ppnum_t ppn; + uint32_t freed_pages = 0; + + /* It is acceptable (if bad) to fail to free. */ + if (vaddr < VM_MIN_KERNEL_ADDRESS) + return; + + assert((vaddr & (PAGE_SIZE - 1)) == 0); /* must be page aligned */ + + for (vaddr_cur = vaddr; + vaddr_cur < trunc_page_32(vaddr + size); + vaddr_cur += PAGE_SIZE) { + ppn = pmap_find_phys(kernel_pmap, vaddr_cur); + if (ppn != (vm_offset_t) NULL) { + /* + * It is not acceptable to fail to update the protections on a page + * we will release to the VM. We need to either panic or continue. + * For now, we'll panic (to help flag if there is memory we can + * reclaim). + */ + if (ml_static_protect(vaddr_cur, PAGE_SIZE, VM_PROT_WRITE | VM_PROT_READ) != KERN_SUCCESS) { + panic("Failed ml_static_mfree on %p", (void *) vaddr_cur); + } +#if 0 + /* + * Must NOT tear down the "V==P" mapping for vaddr_cur as the zone alias scheme + * relies on the persistence of these mappings for all time. + */ + // pmap_remove(kernel_pmap, (addr64_t) vaddr_cur, (addr64_t) (vaddr_cur + PAGE_SIZE)); +#endif + vm_page_create(ppn, (ppn + 1)); + freed_pages++; + } + } + vm_page_lockspin_queues(); + vm_page_wire_count -= freed_pages; + vm_page_wire_count_initial -= freed_pages; + vm_page_unlock_queues(); +#if DEBUG + kprintf("ml_static_mfree: Released 0x%x pages at VA %p, size:0x%llx, last ppn: 0x%x\n", freed_pages, (void *)vaddr, (uint64_t)size, ppn); +#endif +} + + +/* virtual to physical on wired pages */ +vm_offset_t +ml_vtophys(vm_offset_t vaddr) +{ + return kvtophys(vaddr); +} + +/* + * Routine: ml_nofault_copy + * Function: Perform a physical mode copy if the source and destination have + * valid translations in the kernel pmap. If translations are present, they are + * assumed to be wired; e.g., no attempt is made to guarantee that the + * translations obtained remain valid for the duration of the copy process. + */ +vm_size_t +ml_nofault_copy(vm_offset_t virtsrc, vm_offset_t virtdst, vm_size_t size) +{ + addr64_t cur_phys_dst, cur_phys_src; + uint32_t count, nbytes = 0; + + while (size > 0) { + if (!(cur_phys_src = kvtophys(virtsrc))) + break; + if (!(cur_phys_dst = kvtophys(virtdst))) + break; + if (!pmap_valid_address(trunc_page_64(cur_phys_dst)) || + !pmap_valid_address(trunc_page_64(cur_phys_src))) + break; + count = PAGE_SIZE - (cur_phys_src & PAGE_MASK); + if (count > (PAGE_SIZE - (cur_phys_dst & PAGE_MASK))) + count = PAGE_SIZE - (cur_phys_dst & PAGE_MASK); + if (count > size) + count = size; + + bcopy_phys(cur_phys_src, cur_phys_dst, count); + + nbytes += count; + virtsrc += count; + virtdst += count; + size -= count; + } + + return nbytes; +} + +/* + * Routine: ml_validate_nofault + * Function: Validate that ths address range has a valid translations + * in the kernel pmap. If translations are present, they are + * assumed to be wired; i.e. no attempt is made to guarantee + * that the translation persist after the check. + * Returns: TRUE if the range is mapped and will not cause a fault, + * FALSE otherwise. + */ + +boolean_t ml_validate_nofault( + vm_offset_t virtsrc, vm_size_t size) +{ + addr64_t cur_phys_src; + uint32_t count; + + while (size > 0) { + if (!(cur_phys_src = kvtophys(virtsrc))) + return FALSE; + if (!pmap_valid_address(trunc_page_64(cur_phys_src))) + return FALSE; + count = (uint32_t)(PAGE_SIZE - (cur_phys_src & PAGE_MASK)); + if (count > size) + count = (uint32_t)size; + + virtsrc += count; + size -= count; + } + + return TRUE; +} + +void +ml_get_bouncepool_info(vm_offset_t * phys_addr, vm_size_t * size) +{ + *phys_addr = 0; + *size = 0; +} + +/* + * Stubs for CPU Stepper + */ +void +active_rt_threads(__unused boolean_t active) +{ +} + +void +thread_tell_urgency(__unused int urgency, + __unused uint64_t rt_period, + __unused uint64_t rt_deadline, + __unused uint64_t sched_latency, + __unused thread_t nthread) +{ +} + +void +machine_run_count(__unused uint32_t count) +{ +} + +processor_t +machine_choose_processor(__unused processor_set_t pset, processor_t processor) +{ + return (processor); +} + +vm_offset_t +ml_stack_remaining(void) +{ + uintptr_t local = (uintptr_t) &local; + + if (ml_at_interrupt_context()) { + return (local - (getCpuDatap()->intstack_top - INTSTACK_SIZE)); + } else { + return (local - current_thread()->kernel_stack); + } +} + +boolean_t machine_timeout_suspended(void) { + return FALSE; +} + +kern_return_t +ml_interrupt_prewarm(__unused uint64_t deadline) +{ + return KERN_FAILURE; +} + +uint64_t +ml_get_hwclock(void) +{ + uint64_t high_first = 0; + uint64_t high_second = 0; + uint64_t low = 0; + + __builtin_arm_isb(ISB_SY); + + do { + high_first = __builtin_arm_mrrc(15, 0, 14) >> 32; + low = __builtin_arm_mrrc(15, 0, 14) & 0xFFFFFFFFULL; + high_second = __builtin_arm_mrrc(15, 0, 14) >> 32; + } while (high_first != high_second); + + return (high_first << 32) | (low); +} + +boolean_t +ml_delay_should_spin(uint64_t interval) +{ + cpu_data_t *cdp = getCpuDatap(); + + if (cdp->cpu_idle_latency) { + return (interval < cdp->cpu_idle_latency) ? TRUE : FALSE; + } else { + /* + * Early boot, latency is unknown. Err on the side of blocking, + * which should always be safe, even if slow + */ + return FALSE; + } +} + +boolean_t ml_thread_is64bit(thread_t thread) +{ + return (thread_is_64bit(thread)); +} + +void ml_timer_evaluate(void) { +} + +boolean_t +ml_timer_forced_evaluation(void) { + return FALSE; +} + +uint64_t +ml_energy_stat(__unused thread_t t) { + return 0; +} + + +void +ml_gpu_stat_update(__unused uint64_t gpu_ns_delta) { +#if CONFIG_EMBEDDED + /* + * For now: update the resource coalition stats of the + * current thread's coalition + */ + task_coalition_update_gpu_stats(current_task(), gpu_ns_delta); +#endif +} + +uint64_t +ml_gpu_stat(__unused thread_t t) { + return 0; +} + +#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME +static void +timer_state_event(boolean_t switch_to_kernel) +{ + thread_t thread = current_thread(); + if (!thread->precise_user_kernel_time) return; + + processor_data_t *pd = &getCpuDatap()->cpu_processor->processor_data; + uint64_t now = ml_get_timebase(); + + timer_stop(pd->current_state, now); + pd->current_state = (switch_to_kernel) ? &pd->system_state : &pd->user_state; + timer_start(pd->current_state, now); + + timer_stop(pd->thread_timer, now); + pd->thread_timer = (switch_to_kernel) ? &thread->system_timer : &thread->user_timer; + timer_start(pd->thread_timer, now); +} + +void +timer_state_event_user_to_kernel(void) +{ + timer_state_event(TRUE); +} + +void +timer_state_event_kernel_to_user(void) +{ + timer_state_event(FALSE); +} +#endif /* !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME */ + +boolean_t +user_cont_hwclock_allowed(void) +{ + return FALSE; +} + +boolean_t +user_timebase_allowed(void) +{ +#if __ARM_TIME__ + return TRUE; +#else + return FALSE; +#endif +} + +/* + * The following are required for parts of the kernel + * that cannot resolve these functions as inlines: + */ +extern thread_t current_act(void); +thread_t +current_act(void) +{ + return current_thread_fast(); +} + +#undef current_thread +extern thread_t current_thread(void); +thread_t +current_thread(void) +{ + return current_thread_fast(); +} + +#if __ARM_USER_PROTECT__ +uintptr_t +arm_user_protect_begin(thread_t thread) +{ + uintptr_t ttbr0, asid = 0; // kernel asid + + ttbr0 = __builtin_arm_mrc(15,0,2,0,0); // Get TTBR0 + if (ttbr0 != thread->machine.kptw_ttb) { + __builtin_arm_mcr(15,0,thread->machine.kptw_ttb,2,0,0); // Set TTBR0 + __builtin_arm_mcr(15,0,asid,13,0,1); // Set CONTEXTIDR + __builtin_arm_isb(ISB_SY); + } + return ttbr0; +} + +void +arm_user_protect_end(thread_t thread, uintptr_t ttbr0, boolean_t disable_interrupts) +{ + if ((ttbr0 != thread->machine.kptw_ttb) && (thread->machine.uptw_ttb != thread->machine.kptw_ttb)) { + if (disable_interrupts) + __asm__ volatile ("cpsid if" ::: "memory"); // Disable FIQ/IRQ + __builtin_arm_mcr(15,0,thread->machine.uptw_ttb,2,0,0); // Set TTBR0 + __builtin_arm_mcr(15,0,thread->machine.asid,13,0,1); // Set CONTEXTIDR with thread asid + __builtin_arm_dsb(DSB_ISH); + __builtin_arm_isb(ISB_SY); + } +} +#endif // __ARM_USER_PROTECT__ + +void ml_task_set_rop_pid(__unused task_t task, __unused task_t parent_task, __unused boolean_t inherit) +{ + return; +} diff --git a/osfmk/arm/machine_routines.h b/osfmk/arm/machine_routines.h new file mode 100644 index 000000000..3510649b4 --- /dev/null +++ b/osfmk/arm/machine_routines.h @@ -0,0 +1,884 @@ +/* + * Copyright (c) 2007-2015 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ + +#ifndef _ARM_MACHINE_ROUTINES_H_ +#define _ARM_MACHINE_ROUTINES_H_ + +#include <mach/mach_types.h> +#include <mach/boolean.h> +#include <kern/kern_types.h> +#include <pexpert/pexpert.h> + +#include <sys/cdefs.h> +#include <sys/appleapiopts.h> + +#include <stdarg.h> + +__BEGIN_DECLS + +/* Interrupt handling */ + +void ml_cpu_signal(unsigned int cpu_id); +void ml_cpu_signal_deferred_adjust_timer(uint64_t nanosecs); +uint64_t ml_cpu_signal_deferred_get_timer(void); +void ml_cpu_signal_deferred(unsigned int cpu_id); +void ml_cpu_signal_retract(unsigned int cpu_id); + +/* Initialize Interrupts */ +void ml_init_interrupt(void); + +/* Get Interrupts Enabled */ +boolean_t ml_get_interrupts_enabled(void); + +/* Set Interrupts Enabled */ +boolean_t ml_set_interrupts_enabled(boolean_t enable); + +/* Check if running at interrupt context */ +boolean_t ml_at_interrupt_context(void); + +/* Generate a fake interrupt */ +void ml_cause_interrupt(void); + +/* Clear interrupt spin debug state for thread */ +#if INTERRUPT_MASKED_DEBUG +void ml_spin_debug_reset(thread_t thread); +void ml_spin_debug_clear(thread_t thread); +void ml_spin_debug_clear_self(void); +void ml_check_interrupts_disabled_duration(thread_t thread); +#endif + +#ifdef XNU_KERNEL_PRIVATE +extern boolean_t ml_is_quiescing(void); +extern void ml_set_is_quiescing(boolean_t); +extern uint64_t ml_get_booter_memory_size(void); +#endif + +/* Type for the Time Base Enable function */ +typedef void (*time_base_enable_t)(cpu_id_t cpu_id, boolean_t enable); +#if MACH_KERNEL_PRIVATE +/* Type for the Processor Cache Dispatch function */ +typedef void (*cache_dispatch_t)(cpu_id_t cpu_id, unsigned int select, unsigned int param0, unsigned int param1); +#endif + +#define CacheConfig 0x00000000UL +#define CacheControl 0x00000001UL +#define CacheClean 0x00000002UL +#define CacheCleanRegion 0x00000003UL +#define CacheCleanFlush 0x00000004UL +#define CacheCleanFlushRegion 0x00000005UL +#define CacheShutdown 0x00000006UL + +#define CacheControlEnable 0x00000000UL + +#define CacheConfigCCSIDR 0x00000001UL +#define CacheConfigSize 0x00000100UL + +/* Type for the Processor Idle function */ +typedef void (*processor_idle_t)(cpu_id_t cpu_id, boolean_t enter, uint64_t *new_timeout_ticks); + +/* Type for the Idle Tickle function */ +typedef void (*idle_tickle_t)(void); + +/* Type for the Idle Timer function */ +typedef void (*idle_timer_t)(void *refcon, uint64_t *new_timeout_ticks); + +/* Type for the IPI Hander */ +typedef void (*ipi_handler_t)(void); + +/* Type for the Lockdown Hander */ +typedef void (*lockdown_handler_t)(void *); + +/* Type for the Platform specific Error Handler */ +typedef void (*platform_error_handler_t)(void *refcon, vm_offset_t fault_addr); + +/* + * The exception callback (ex_cb) module allows kernel drivers to + * register and receive callbacks for exceptions, and indicate + * actions to be taken by the platform kernel + * Currently this is supported for ARM64 but extending support for ARM32 + * should be straightforward + */ + +/* Supported exception classes for callbacks */ +typedef enum +{ + EXCB_CLASS_ILLEGAL_INSTR_SET, + EXCB_CLASS_MAX // this must be last +} +ex_cb_class_t; + +/* Actions indicated by callbacks to be taken by platform kernel */ +typedef enum +{ + EXCB_ACTION_RERUN, // re-run the faulting instruction + EXCB_ACTION_NONE, // continue normal exception handling +} +ex_cb_action_t; + +/* + * Exception state + * We cannot use a private kernel data structure such as arm_saved_state_t + * The CPSR and ESR are not clobbered when the callback function is invoked so + * those registers can be examined by the callback function; + * the same is done in the platform error handlers + */ +typedef struct +{ + vm_offset_t far; +} +ex_cb_state_t; + +/* callback type definition */ +typedef ex_cb_action_t (*ex_cb_t) ( + ex_cb_class_t cb_class, + void *refcon,// provided at registration + const ex_cb_state_t *state // exception state + ); + +/* + * Callback registration + * Currently we support only one registered callback per class but + * it should be possible to support more callbacks + */ +kern_return_t ex_cb_register( + ex_cb_class_t cb_class, + ex_cb_t cb, + void *refcon ); + +/* + * Called internally by platform kernel to invoke the registered callback for class + */ +ex_cb_action_t ex_cb_invoke( + ex_cb_class_t cb_class, + vm_offset_t far); + + +void ml_parse_cpu_topology(void); + +unsigned int ml_get_cpu_count(void); + +int ml_get_boot_cpu_number(void); + +int ml_get_cpu_number(uint32_t phys_id); + +int ml_get_max_cpu_number(void); + +/* Struct for ml_cpu_get_info */ +struct ml_cpu_info { + unsigned long vector_unit; + unsigned long cache_line_size; + unsigned long l1_icache_size; + unsigned long l1_dcache_size; + unsigned long l2_settings; + unsigned long l2_cache_size; + unsigned long l3_settings; + unsigned long l3_cache_size; +}; +typedef struct ml_cpu_info ml_cpu_info_t; + +typedef enum { + CLUSTER_TYPE_SMP, +} cluster_type_t; + +cluster_type_t ml_get_boot_cluster(void); + +/* Struct for ml_processor_register */ +struct ml_processor_info { + cpu_id_t cpu_id; + vm_offset_t start_paddr; + boolean_t supports_nap; + void *platform_cache_dispatch; + time_base_enable_t time_base_enable; + processor_idle_t processor_idle; + idle_tickle_t *idle_tickle; + idle_timer_t idle_timer; + void *idle_timer_refcon; + vm_offset_t powergate_stub_addr; + uint32_t powergate_stub_length; + uint32_t powergate_latency; + platform_error_handler_t platform_error_handler; + uint64_t regmap_paddr; + uint32_t phys_id; + uint32_t log_id; + uint32_t l2_access_penalty; + uint32_t cluster_id; + cluster_type_t cluster_type; + uint32_t l2_cache_id; + uint32_t l2_cache_size; + uint32_t l3_cache_id; + uint32_t l3_cache_size; +}; +typedef struct ml_processor_info ml_processor_info_t; + +#if defined(PEXPERT_KERNEL_PRIVATE) || defined(MACH_KERNEL_PRIVATE) +/* Struct for ml_init_timebase */ +struct tbd_ops { + void (*tbd_fiq_handler)(void); + uint32_t (*tbd_get_decrementer)(void); + void (*tbd_set_decrementer)(uint32_t dec_value); +}; +typedef struct tbd_ops *tbd_ops_t; +typedef struct tbd_ops tbd_ops_data_t; +#endif + +/* Register a processor */ +kern_return_t ml_processor_register( + ml_processor_info_t *ml_processor_info, + processor_t *processor, + ipi_handler_t *ipi_handler); + +/* Register a lockdown handler */ +kern_return_t ml_lockdown_handler_register(lockdown_handler_t, void *); + +#if XNU_KERNEL_PRIVATE +void ml_lockdown_init(void); + +/* Check if the machine layer wants to intercept a panic call */ +boolean_t ml_wants_panic_trap_to_debugger(void); + +/* Machine layer routine for intercepting panics */ +void ml_panic_trap_to_debugger(const char *panic_format_str, + va_list *panic_args, + unsigned int reason, + void *ctx, + uint64_t panic_options_mask, + unsigned long panic_caller); +#endif /* XNU_KERNEL_PRIVATE */ + +/* Initialize Interrupts */ +void ml_install_interrupt_handler( + void *nub, + int source, + void *target, + IOInterruptHandler handler, + void *refCon); + +vm_offset_t +ml_static_vtop( + vm_offset_t); + +vm_offset_t +ml_static_ptovirt( + vm_offset_t); + +/* Offset required to obtain absolute time value from tick counter */ +uint64_t ml_get_abstime_offset(void); + +/* Offset required to obtain continuous time value from tick counter */ +uint64_t ml_get_conttime_offset(void); + +#ifdef __APPLE_API_UNSTABLE +/* PCI config cycle probing */ +boolean_t ml_probe_read( + vm_offset_t paddr, + unsigned int *val); +boolean_t ml_probe_read_64( + addr64_t paddr, + unsigned int *val); + +/* Read physical address byte */ +unsigned int ml_phys_read_byte( + vm_offset_t paddr); +unsigned int ml_phys_read_byte_64( + addr64_t paddr); + +/* Read physical address half word */ +unsigned int ml_phys_read_half( + vm_offset_t paddr); +unsigned int ml_phys_read_half_64( + addr64_t paddr); + +/* Read physical address word*/ +unsigned int ml_phys_read( + vm_offset_t paddr); +unsigned int ml_phys_read_64( + addr64_t paddr); +unsigned int ml_phys_read_word( + vm_offset_t paddr); +unsigned int ml_phys_read_word_64( + addr64_t paddr); + +unsigned long long ml_io_read(uintptr_t iovaddr, int iovsz); +unsigned int ml_io_read8(uintptr_t iovaddr); +unsigned int ml_io_read16(uintptr_t iovaddr); +unsigned int ml_io_read32(uintptr_t iovaddr); +unsigned long long ml_io_read64(uintptr_t iovaddr); + +/* Read physical address double word */ +unsigned long long ml_phys_read_double( + vm_offset_t paddr); +unsigned long long ml_phys_read_double_64( + addr64_t paddr); + +/* Write physical address byte */ +void ml_phys_write_byte( + vm_offset_t paddr, unsigned int data); +void ml_phys_write_byte_64( + addr64_t paddr, unsigned int data); + +/* Write physical address half word */ +void ml_phys_write_half( + vm_offset_t paddr, unsigned int data); +void ml_phys_write_half_64( + addr64_t paddr, unsigned int data); + +/* Write physical address word */ +void ml_phys_write( + vm_offset_t paddr, unsigned int data); +void ml_phys_write_64( + addr64_t paddr, unsigned int data); +void ml_phys_write_word( + vm_offset_t paddr, unsigned int data); +void ml_phys_write_word_64( + addr64_t paddr, unsigned int data); + +/* Write physical address double word */ +void ml_phys_write_double( + vm_offset_t paddr, unsigned long long data); +void ml_phys_write_double_64( + addr64_t paddr, unsigned long long data); + +void ml_static_mfree( + vm_offset_t, + vm_size_t); + +kern_return_t +ml_static_protect( + vm_offset_t start, + vm_size_t size, + vm_prot_t new_prot); + +/* virtual to physical on wired pages */ +vm_offset_t ml_vtophys( + vm_offset_t vaddr); + +/* Get processor info */ +void ml_cpu_get_info(ml_cpu_info_t *ml_cpu_info); + +#endif /* __APPLE_API_UNSTABLE */ + +#ifdef __APPLE_API_PRIVATE +#ifdef XNU_KERNEL_PRIVATE +vm_size_t ml_nofault_copy( + vm_offset_t virtsrc, + vm_offset_t virtdst, + vm_size_t size); +boolean_t ml_validate_nofault( + vm_offset_t virtsrc, vm_size_t size); +#endif /* XNU_KERNEL_PRIVATE */ +#if defined(PEXPERT_KERNEL_PRIVATE) || defined(MACH_KERNEL_PRIVATE) +/* IO memory map services */ + +/* Map memory map IO space */ +vm_offset_t ml_io_map( + vm_offset_t phys_addr, + vm_size_t size); + +vm_offset_t ml_io_map_wcomb( + vm_offset_t phys_addr, + vm_size_t size); + +void ml_get_bouncepool_info( + vm_offset_t *phys_addr, + vm_size_t *size); + +vm_map_address_t ml_map_high_window( + vm_offset_t phys_addr, + vm_size_t len); + +/* boot memory allocation */ +vm_offset_t ml_static_malloc( + vm_size_t size); + +void ml_init_timebase( + void *args, + tbd_ops_t tbd_funcs, + vm_offset_t int_address, + vm_offset_t int_value); + +uint64_t ml_get_timebase(void); + +void ml_init_lock_timeout(void); + +boolean_t ml_delay_should_spin(uint64_t interval); + +uint32_t ml_get_decrementer(void); + +#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME +void timer_state_event_user_to_kernel(void); +void timer_state_event_kernel_to_user(void); +#endif /* !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME */ + +uint64_t ml_get_hwclock(void); + +#ifdef __arm64__ +boolean_t ml_get_timer_pending(void); +#endif + +void platform_syscall( + struct arm_saved_state *); + +void ml_set_decrementer( + uint32_t dec_value); + +boolean_t is_user_contex( + void); + +void ml_init_arm_debug_interface(void *args, vm_offset_t virt_address); + +/* These calls are only valid if __ARM_USER_PROTECT__ is defined */ +uintptr_t arm_user_protect_begin( + thread_t thread); + +void arm_user_protect_end( + thread_t thread, + uintptr_t up, + boolean_t disable_interrupts); + +#endif /* PEXPERT_KERNEL_PRIVATE || MACH_KERNEL_PRIVATE */ + +/* Zero bytes starting at a physical address */ +void bzero_phys( + addr64_t phys_address, + vm_size_t length); + +void bzero_phys_nc(addr64_t src64, vm_size_t bytes); + +void ml_thread_policy( + thread_t thread, + unsigned policy_id, + unsigned policy_info); + +#define MACHINE_GROUP 0x00000001 +#define MACHINE_NETWORK_GROUP 0x10000000 +#define MACHINE_NETWORK_WORKLOOP 0x00000001 +#define MACHINE_NETWORK_NETISR 0x00000002 + +/* Initialize the maximum number of CPUs */ +void ml_init_max_cpus( + unsigned int max_cpus); + +/* Return the maximum number of CPUs set by ml_init_max_cpus() */ +unsigned int ml_get_max_cpus( + void); + +/* Return the maximum memory size */ +unsigned int ml_get_machine_mem(void); + +#ifdef XNU_KERNEL_PRIVATE +/* Return max offset */ +vm_map_offset_t ml_get_max_offset( + boolean_t is64, + unsigned int option); +#define MACHINE_MAX_OFFSET_DEFAULT 0x01 +#define MACHINE_MAX_OFFSET_MIN 0x02 +#define MACHINE_MAX_OFFSET_MAX 0x04 +#define MACHINE_MAX_OFFSET_DEVICE 0x08 +#endif + +extern void ml_cpu_up(void); +extern void ml_cpu_down(void); +extern void ml_arm_sleep(void); + +extern uint64_t ml_get_wake_timebase(void); +extern uint64_t ml_get_conttime_wake_time(void); + +/* Time since the system was reset (as part of boot/wake) */ +uint64_t ml_get_time_since_reset(void); + +#ifdef XNU_KERNEL_PRIVATE +/* Just a stub on ARM */ +extern kern_return_t ml_interrupt_prewarm(uint64_t deadline); +#define TCOAL_DEBUG(x, a, b, c, d, e) do { } while(0) +#endif /* XNU_KERNEL_PRIVATE */ + +/* Bytes available on current stack */ +vm_offset_t ml_stack_remaining(void); + +#ifdef MACH_KERNEL_PRIVATE +uint32_t get_fpscr(void); +void set_fpscr(uint32_t); + +extern void init_vfp(void); +extern boolean_t get_vfp_enabled(void); +#if (__ARM_VFP__ >= 3) +extern unsigned int get_mvfr0(void); +extern unsigned int get_mvfr1(void); +#endif +extern void arm_debug_set_cp14(arm_debug_state_t *debug_state); +extern void fiq_context_init(boolean_t enable_fiq); + +extern void reenable_async_aborts(void); +extern void cpu_idle_wfi(boolean_t wfi_fast); + +#ifdef MONITOR +#define MONITOR_SET_ENTRY 0x800 /* Set kernel entry point from monitor */ +#define MONITOR_LOCKDOWN 0x801 /* Enforce kernel text/rodata integrity */ +unsigned long monitor_call(uintptr_t callnum, uintptr_t arg1, + uintptr_t arg2, uintptr_t arg3); +#endif /* MONITOR */ + +#if defined(KERNEL_INTEGRITY_KTRR) +void rorgn_stash_range(void); +void rorgn_lockdown(void); +#endif /* defined(KERNEL_INTEGRITY_KTRR)*/ + +#endif /* MACH_KERNEL_PRIVATE */ + +extern uint32_t arm_debug_read_dscr(void); + +extern int set_be_bit(void); +extern int clr_be_bit(void); +extern int be_tracing(void); + +typedef void (*broadcastFunc) (void *); +unsigned int cpu_broadcast_xcall(uint32_t *, boolean_t, broadcastFunc, void *); +kern_return_t cpu_xcall(int, broadcastFunc, void *); + +#ifdef KERNEL_PRIVATE + +/* Interface to be used by the perf. controller to register a callback, in a + * single-threaded fashion. The callback will receive notifications of + * processor performance quality-of-service changes from the scheduler. + */ + +#ifdef __arm64__ +typedef void (*cpu_qos_update_t)(int throughput_qos, uint64_t qos_param1, uint64_t qos_param2); +void cpu_qos_update_register(cpu_qos_update_t); +#endif /* __arm64__ */ + +struct going_on_core { + uint64_t thread_id; + uint16_t qos_class; + uint16_t urgency; /* XCPM compatibility */ + uint32_t is_32_bit : 1; /* uses 32-bit ISA/register state in userspace (which may differ from address space size) */ + uint32_t is_kernel_thread : 1; + uint64_t thread_group_id; + void *thread_group_data; + uint64_t scheduling_latency; /* absolute time between when thread was made runnable and this ctx switch */ + uint64_t start_time; + uint64_t scheduling_latency_at_same_basepri; + uint32_t energy_estimate_nj; /* return: In nanojoules */ + /* smaller of the time between last change to base priority and ctx switch and scheduling_latency */ +}; +typedef struct going_on_core *going_on_core_t; + +struct going_off_core { + uint64_t thread_id; + uint32_t energy_estimate_nj; /* return: In nanojoules */ + uint32_t reserved; + uint64_t end_time; + uint64_t thread_group_id; + void *thread_group_data; +}; +typedef struct going_off_core *going_off_core_t; + +struct thread_group_data { + uint64_t thread_group_id; + void *thread_group_data; + uint32_t thread_group_size; + uint32_t thread_group_flags; +}; +typedef struct thread_group_data *thread_group_data_t; + +struct perfcontrol_max_runnable_latency { + uint64_t max_scheduling_latencies[4 /* THREAD_URGENCY_MAX */]; +}; +typedef struct perfcontrol_max_runnable_latency *perfcontrol_max_runnable_latency_t; + +struct perfcontrol_work_interval { + uint64_t thread_id; + uint16_t qos_class; + uint16_t urgency; + uint32_t flags; // notify + uint64_t work_interval_id; + uint64_t start; + uint64_t finish; + uint64_t deadline; + uint64_t next_start; + uint64_t thread_group_id; + void *thread_group_data; + uint32_t create_flags; +}; +typedef struct perfcontrol_work_interval *perfcontrol_work_interval_t; + + +/* + * Structure to export per-CPU counters as part of the CLPC callout. + * Contains only the fixed CPU counters (instructions and cycles); CLPC + * would call back into XNU to get the configurable counters if needed. + */ +struct perfcontrol_cpu_counters { + uint64_t instructions; + uint64_t cycles; +}; + +/* + * Structure used to pass information about a thread to CLPC + */ +struct perfcontrol_thread_data { + /* + * Energy estimate (return value) + * The field is populated by CLPC and used to update the + * energy estimate of the thread + */ + uint32_t energy_estimate_nj; + /* Perfcontrol class for thread */ + perfcontrol_class_t perfctl_class; + /* Thread ID for the thread */ + uint64_t thread_id; + /* Thread Group ID */ + uint64_t thread_group_id; + /* + * Scheduling latency for threads at the same base priority. + * Calculated by the scheduler and passed into CLPC. The field is + * populated only in the thread_data structure for the thread + * going on-core. + */ + uint64_t scheduling_latency_at_same_basepri; + /* Thread Group data pointer */ + void *thread_group_data; + /* perfctl state pointer */ + void *perfctl_state; +}; + +/* + * All callouts from the scheduler are executed with interrupts + * disabled. Callouts should be implemented in C with minimal + * abstractions, and only use KPI exported by the mach/libkern + * symbolset, restricted to routines like spinlocks and atomic + * operations and scheduler routines as noted below. Spinlocks that + * are used to synchronize data in the perfcontrol_state_t should only + * ever be acquired with interrupts disabled, to avoid deadlocks where + * an quantum expiration timer interrupt attempts to perform a callout + * that attempts to lock a spinlock that is already held. + */ + +/* + * When a processor is switching between two threads (after the + * scheduler has chosen a new thread), the low-level platform layer + * will call this routine, which should perform required timestamps, + * MMIO register reads, or other state switching. No scheduler locks + * are held during this callout. + * + * This function is called with interrupts ENABLED. + */ +typedef void (*sched_perfcontrol_context_switch_t)(perfcontrol_state_t, perfcontrol_state_t); + +/* + * Once the processor has switched to the new thread, the offcore + * callout will indicate the old thread that is no longer being + * run. The thread's scheduler lock is held, so it will not begin + * running on another processor (in the case of preemption where it + * remains runnable) until it completes. If the "thread_terminating" + * boolean is TRUE, this will be the last callout for this thread_id. + */ +typedef void (*sched_perfcontrol_offcore_t)(perfcontrol_state_t, going_off_core_t /* populated by callee */, boolean_t); + +/* + * After the offcore callout and after the old thread can potentially + * start running on another processor, the oncore callout will be + * called with the thread's scheduler lock held. The oncore callout is + * also called any time one of the parameters in the going_on_core_t + * structure changes, like priority/QoS changes, and quantum + * expiration, so the callout must not assume callouts are paired with + * offcore callouts. + */ +typedef void (*sched_perfcontrol_oncore_t)(perfcontrol_state_t, going_on_core_t); + +/* + * Periodically (on hundreds of ms scale), the scheduler will perform + * maintenance and report the maximum latency for runnable (but not currently + * running) threads for each urgency class. + */ +typedef void (*sched_perfcontrol_max_runnable_latency_t)(perfcontrol_max_runnable_latency_t); + +/* + * When the kernel receives information about work intervals from userland, + * it is passed along using this callback. No locks are held, although the state + * object will not go away during the callout. + */ +typedef void (*sched_perfcontrol_work_interval_notify_t)(perfcontrol_state_t, perfcontrol_work_interval_t); + +/* + * These callbacks are used when thread groups are added, removed or properties + * updated. + * No blocking allocations (or anything else blocking) are allowed inside these + * callbacks. No locks allowed in these callbacks as well since the kernel might + * be holding the thread/task locks. + */ +typedef void (*sched_perfcontrol_thread_group_init_t)(thread_group_data_t); +typedef void (*sched_perfcontrol_thread_group_deinit_t)(thread_group_data_t); +typedef void (*sched_perfcontrol_thread_group_flags_update_t)(thread_group_data_t); + +/* + * Sometime after the timeout set by sched_perfcontrol_update_callback_deadline has passed, + * this function will be called, passing the timeout deadline that was previously armed as an argument. + * + * This is called inside context-switch/quantum-interrupt context and must follow the safety rules for that context. + */ +typedef void (*sched_perfcontrol_deadline_passed_t)(uint64_t deadline); + +/* + * Context Switch Callout + * + * Parameters: + * event - The perfcontrol_event for this callout + * cpu_id - The CPU doing the context switch + * timestamp - The timestamp for the context switch + * flags - Flags for other relevant information + * offcore - perfcontrol_data structure for thread going off-core + * oncore - perfcontrol_data structure for thread going on-core + * cpu_counters - perfcontrol_cpu_counters for the CPU doing the switch + */ +typedef void (*sched_perfcontrol_csw_t)( + perfcontrol_event event, uint32_t cpu_id, uint64_t timestamp, uint32_t flags, + struct perfcontrol_thread_data *offcore, struct perfcontrol_thread_data *oncore, + struct perfcontrol_cpu_counters *cpu_counters, __unused void *unused); + + +/* + * Thread State Update Callout + * + * Parameters: + * event - The perfcontrol_event for this callout + * cpu_id - The CPU doing the state update + * timestamp - The timestamp for the state update + * flags - Flags for other relevant information + * thr_data - perfcontrol_data structure for the thread being updated + */ +typedef void (*sched_perfcontrol_state_update_t)( + perfcontrol_event event, uint32_t cpu_id, uint64_t timestamp, uint32_t flags, + struct perfcontrol_thread_data *thr_data, __unused void *unused); + + +/* + * Callers should always use the CURRENT version so that the kernel can detect both older + * and newer structure layouts. New callbacks should always be added at the end of the + * structure, and xnu should expect existing source recompiled against newer headers + * to pass NULL for unimplemented callbacks. Pass NULL as the as the callbacks parameter + * to reset callbacks to their default in-kernel values. + */ + +#define SCHED_PERFCONTROL_CALLBACKS_VERSION_0 (0) /* up-to oncore */ +#define SCHED_PERFCONTROL_CALLBACKS_VERSION_1 (1) /* up-to max_runnable_latency */ +#define SCHED_PERFCONTROL_CALLBACKS_VERSION_2 (2) /* up-to work_interval_notify */ +#define SCHED_PERFCONTROL_CALLBACKS_VERSION_3 (3) /* up-to thread_group_deinit */ +#define SCHED_PERFCONTROL_CALLBACKS_VERSION_4 (4) /* up-to deadline_passed */ +#define SCHED_PERFCONTROL_CALLBACKS_VERSION_5 (5) /* up-to state_update */ +#define SCHED_PERFCONTROL_CALLBACKS_VERSION_6 (6) /* up-to thread_group_flags_update */ +#define SCHED_PERFCONTROL_CALLBACKS_VERSION_CURRENT SCHED_PERFCONTROL_CALLBACKS_VERSION_6 + +struct sched_perfcontrol_callbacks { + unsigned long version; /* Use SCHED_PERFCONTROL_CALLBACKS_VERSION_CURRENT */ + sched_perfcontrol_offcore_t offcore; + sched_perfcontrol_context_switch_t context_switch; + sched_perfcontrol_oncore_t oncore; + sched_perfcontrol_max_runnable_latency_t max_runnable_latency; + sched_perfcontrol_work_interval_notify_t work_interval_notify; + sched_perfcontrol_thread_group_init_t thread_group_init; + sched_perfcontrol_thread_group_deinit_t thread_group_deinit; + sched_perfcontrol_deadline_passed_t deadline_passed; + sched_perfcontrol_csw_t csw; + sched_perfcontrol_state_update_t state_update; + sched_perfcontrol_thread_group_flags_update_t thread_group_flags_update; +}; +typedef struct sched_perfcontrol_callbacks *sched_perfcontrol_callbacks_t; + +extern void sched_perfcontrol_register_callbacks(sched_perfcontrol_callbacks_t callbacks, unsigned long size_of_state); + +/* + * Update the scheduler with the set of cores that should be used to dispatch new threads. + * Non-recommended cores can still be used to field interrupts or run bound threads. + * This should be called with interrupts enabled and no scheduler locks held. + */ +#define ALL_CORES_RECOMMENDED (~(uint32_t)0) + +extern void sched_perfcontrol_update_recommended_cores(uint32_t recommended_cores); +extern void sched_perfcontrol_thread_group_recommend(void *data, cluster_type_t recommendation); + +/* + * Update the deadline after which sched_perfcontrol_deadline_passed will be called. + * Returns TRUE if it successfully canceled a previously set callback, + * and FALSE if it did not (i.e. one wasn't set, or callback already fired / is in flight). + * The callback is automatically canceled when it fires, and does not repeat unless rearmed. + * + * This 'timer' executes as the scheduler switches between threads, on a non-idle core + * + * There can be only one outstanding timer globally. + */ +extern boolean_t sched_perfcontrol_update_callback_deadline(uint64_t deadline); + +typedef enum perfcontrol_callout_type { + PERFCONTROL_CALLOUT_ON_CORE, + PERFCONTROL_CALLOUT_OFF_CORE, + PERFCONTROL_CALLOUT_CONTEXT, + PERFCONTROL_CALLOUT_STATE_UPDATE, + /* Add other callout types here */ + PERFCONTROL_CALLOUT_MAX +} perfcontrol_callout_type_t; + +typedef enum perfcontrol_callout_stat { + PERFCONTROL_STAT_INSTRS, + PERFCONTROL_STAT_CYCLES, + /* Add other stat types here */ + PERFCONTROL_STAT_MAX +} perfcontrol_callout_stat_t; + +uint64_t perfcontrol_callout_stat_avg(perfcontrol_callout_type_t type, + perfcontrol_callout_stat_t stat); + + +#endif /* KERNEL_PRIVATE */ + +boolean_t machine_timeout_suspended(void); +void ml_get_power_state(boolean_t *, boolean_t *); + +boolean_t user_cont_hwclock_allowed(void); +boolean_t user_timebase_allowed(void); +boolean_t ml_thread_is64bit(thread_t thread); +void ml_task_set_rop_pid(task_t task, task_t parent_task, boolean_t inherit); + +#ifdef __arm64__ +void ml_set_align_checking(void); +boolean_t arm64_wfe_allowed(void); +#endif /* __arm64__ */ + +void ml_timer_evaluate(void); +boolean_t ml_timer_forced_evaluation(void); +uint64_t ml_energy_stat(thread_t); +void ml_gpu_stat_update(uint64_t); +uint64_t ml_gpu_stat(thread_t); +#endif /* __APPLE_API_PRIVATE */ + +__END_DECLS + +#endif /* _ARM_MACHINE_ROUTINES_H_ */ diff --git a/osfmk/arm/machine_routines_asm.s b/osfmk/arm/machine_routines_asm.s new file mode 100644 index 000000000..b64d28373 --- /dev/null +++ b/osfmk/arm/machine_routines_asm.s @@ -0,0 +1,1131 @@ +/* + * Copyright (c) 2007-2014 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include <machine/asm.h> +#include <arm/proc_reg.h> +#include <arm/pmap.h> +#include <sys/errno.h> +#include "assym.s" + + .align 2 + .globl EXT(machine_set_current_thread) +LEXT(machine_set_current_thread) + mcr p15, 0, r0, c13, c0, 4 // Write TPIDRPRW + ldr r1, [r0, TH_CTH_SELF] + mrc p15, 0, r2, c13, c0, 3 // Read TPIDRURO + and r2, r2, #3 // Extract cpu number + orr r1, r1, r2 // + mcr p15, 0, r1, c13, c0, 3 // Write TPIDRURO + ldr r1, [r0, TH_CTH_DATA] + mcr p15, 0, r1, c13, c0, 2 // Write TPIDRURW + bx lr + +/* + * void machine_idle(void) + */ + .text + .align 2 + .globl EXT(machine_idle) +LEXT(machine_idle) + cpsid if // Disable FIQ IRQ + mov ip, lr + bl EXT(Idle_context) + mov lr, ip + cpsie if // Enable FIQ IRQ + bx lr + +/* + * void cpu_idle_wfi(boolean_t wfi_fast): + * cpu_idle is the only function that should call this. + */ + .text + .align 2 + .globl EXT(cpu_idle_wfi) +LEXT(cpu_idle_wfi) + mov r1, #32 + mov r2, #1200 + cmp r0, #0 + beq 3f + mov r1, #1 + b 2f + .align 5 +1: + add r0, r0, #1 + mov r1, r2 +2: + +/* + * We export the address of the WFI instruction so that it can be patched; this will be + * ugly from a debugging perspective. + */ + +#if (__ARM_ARCH__ >= 7) + dsb + .globl EXT(wfi_inst) +LEXT(wfi_inst) + wfi +#else + mcr p15, 0, r0, c7, c10, 4 + .globl EXT(wfi_inst) +LEXT(wfi_inst) + mcr p15, 0, r0, c7, c0, 4 +#endif +3: + subs r1, r1, #1 + bne 3b + nop + nop + nop + nop + nop + cmp r0, #0 + beq 1b + bx lr + + .align 2 + .globl EXT(timer_grab) +LEXT(timer_grab) +0: + ldr r2, [r0, TIMER_HIGH] + ldr r3, [r0, TIMER_LOW] +#if __ARM_SMP__ + dmb ish // dmb ish +#endif + ldr r1, [r0, TIMER_HIGHCHK] + cmp r1, r2 + bne 0b + mov r0, r3 + bx lr + + .align 2 + .globl EXT(timer_update) +LEXT(timer_update) + str r1, [r0, TIMER_HIGHCHK] +#if __ARM_SMP__ + dmb ish // dmb ish +#endif + str r2, [r0, TIMER_LOW] +#if __ARM_SMP__ + dmb ish // dmb ish +#endif + str r1, [r0, TIMER_HIGH] + bx lr + + .align 2 + .globl EXT(get_vfp_enabled) +LEXT(get_vfp_enabled) +#if __ARM_VFP__ + fmrx r0, fpexc + and r1, r0, #FPEXC_EN // Extact vfp enable previous state + mov r0, r1, LSR #FPEXC_EN_BIT // Return 1 if enabled, 0 if disabled +#else + mov r0, #0 // return false +#endif + bx lr + +/* This is no longer useful (but is exported, so this may require kext cleanup). */ + .align 2 + .globl EXT(enable_kernel_vfp_context) +LEXT(enable_kernel_vfp_context) + bx lr + +/* uint32_t get_fpscr(void): + * Returns the current state of the FPSCR register. + */ + .align 2 + .globl EXT(get_fpscr) +LEXT(get_fpscr) +#if __ARM_VFP__ + fmrx r0, fpscr +#endif + bx lr + .align 2 + .globl EXT(set_fpscr) +/* void set_fpscr(uint32_t value): + * Set the FPSCR register. + */ +LEXT(set_fpscr) +#if __ARM_VFP__ + fmxr fpscr, r0 +#else + mov r0, #0 +#endif + bx lr + +#if (__ARM_VFP__ >= 3) + .align 2 + .globl EXT(get_mvfr0) +LEXT(get_mvfr0) + vmrs r0, mvfr0 + bx lr + .globl EXT(get_mvfr1) +LEXT(get_mvfr1) + vmrs r0, mvfr1 + bx lr +#endif + +/* + * void OSSynchronizeIO(void) + */ + .text + .align 2 + .globl EXT(OSSynchronizeIO) +LEXT(OSSynchronizeIO) + .align 2 + dsb + bx lr + +/* + * void flush_mmu_tlb(void) + * + * Flush all TLBs + */ + .text + .align 2 + .globl EXT(flush_mmu_tlb) +LEXT(flush_mmu_tlb) + mov r0, #0 +#if __ARM_SMP__ + mcr p15, 0, r0, c8, c3, 0 // Invalidate Inner Shareable entire TLBs +#else + mcr p15, 0, r0, c8, c7, 0 // Invalidate entire TLB +#endif + dsb ish + isb + bx lr + +/* + * void flush_core_tlb(void) + * + * Flush core TLB + */ + .text + .align 2 + .globl EXT(flush_core_tlb) +LEXT(flush_core_tlb) + mov r0, #0 + mcr p15, 0, r0, c8, c7, 0 // Invalidate entire TLB + dsb ish + isb + bx lr + +/* + * void flush_mmu_tlb_entry(uint32_t) + * + * Flush TLB entry + */ + .text + .align 2 + .globl EXT(flush_mmu_tlb_entry) +LEXT(flush_mmu_tlb_entry) +#if __ARM_SMP__ + mcr p15, 0, r0, c8, c3, 1 // Invalidate TLB Inner Shareableentry +#else + mcr p15, 0, r0, c8, c7, 1 // Invalidate TLB entry +#endif + dsb ish + isb + bx lr + +/* + * void flush_mmu_tlb_entries(uint32_t, uint32_t) + * + * Flush TLB entries + */ + .text + .align 2 + .globl EXT(flush_mmu_tlb_entries) +LEXT(flush_mmu_tlb_entries) +1: +#if __ARM_SMP__ + mcr p15, 0, r0, c8, c3, 1 // Invalidate TLB Inner Shareable entry +#else + mcr p15, 0, r0, c8, c7, 1 // Invalidate TLB entry +#endif + add r0, r0, ARM_PGBYTES // Increment to the next page + cmp r0, r1 // Loop if current address < end address + blt 1b + dsb ish // Synchronize + isb + bx lr + + +/* + * void flush_mmu_tlb_mva_entries(uint32_t) + * + * Flush TLB entries for mva + */ + .text + .align 2 + .globl EXT(flush_mmu_tlb_mva_entries) +LEXT(flush_mmu_tlb_mva_entries) +#if __ARM_SMP__ + mcr p15, 0, r0, c8, c3, 3 // Invalidate TLB Inner Shareable entries by mva +#else + mcr p15, 0, r0, c8, c7, 3 // Invalidate TLB Inner Shareable entries by mva +#endif + dsb ish + isb + bx lr + +/* + * void flush_mmu_tlb_asid(uint32_t) + * + * Flush TLB entriesfor requested asid + */ + .text + .align 2 + .globl EXT(flush_mmu_tlb_asid) +LEXT(flush_mmu_tlb_asid) +#if __ARM_SMP__ + mcr p15, 0, r0, c8, c3, 2 // Invalidate TLB Inner Shareable entries by asid +#else + mcr p15, 0, r0, c8, c7, 2 // Invalidate TLB entries by asid +#endif + dsb ish + isb + bx lr + +/* + * void flush_core_tlb_asid(uint32_t) + * + * Flush TLB entries for core for requested asid + */ + .text + .align 2 + .globl EXT(flush_core_tlb_asid) +LEXT(flush_core_tlb_asid) + mcr p15, 0, r0, c8, c7, 2 // Invalidate TLB entries by asid + dsb ish + isb + bx lr + +/* + * Set MMU Translation Table Base + */ + .text + .align 2 + .globl EXT(set_mmu_ttb) +LEXT(set_mmu_ttb) + orr r0, r0, #(TTBR_SETUP & 0xFF) // Setup PTWs memory attribute + orr r0, r0, #(TTBR_SETUP & 0xFF00) // Setup PTWs memory attribute + mcr p15, 0, r0, c2, c0, 0 // write r0 to translation table 0 + dsb ish + isb + bx lr + +/* + * Set MMU Translation Table Base Alternate + */ + .text + .align 2 + .globl EXT(set_mmu_ttb_alternate) +LEXT(set_mmu_ttb_alternate) + orr r0, r0, #(TTBR_SETUP & 0xFF) // Setup PTWs memory attribute + orr r0, r0, #(TTBR_SETUP & 0xFF00) // Setup PTWs memory attribute + mcr p15, 0, r0, c2, c0, 1 // write r0 to translation table 1 + dsb ish + isb + bx lr + +/* + * Set MMU Translation Table Base + */ + .text + .align 2 + .globl EXT(get_mmu_ttb) +LEXT(get_mmu_ttb) + mrc p15, 0, r0, c2, c0, 0 // translation table to r0 + isb + bx lr + +/* + * get MMU control register + */ + .text + .align 2 + .globl EXT(get_aux_control) +LEXT(get_aux_control) + mrc p15, 0, r0, c1, c0, 1 // read aux control into r0 + bx lr // return old bits in r0 + +/* + * set MMU control register + */ + .text + .align 2 + .globl EXT(set_aux_control) +LEXT(set_aux_control) + mcr p15, 0, r0, c1, c0, 1 // write r0 back to aux control + isb + bx lr + + +/* + * get MMU control register + */ + .text + .align 2 + .globl EXT(get_mmu_control) +LEXT(get_mmu_control) + mrc p15, 0, r0, c1, c0, 0 // read mmu control into r0 + bx lr // return old bits in r0 + +/* + * set MMU control register + */ + .text + .align 2 + .globl EXT(set_mmu_control) +LEXT(set_mmu_control) + mcr p15, 0, r0, c1, c0, 0 // write r0 back to mmu control + isb + bx lr + +/* + * MMU kernel virtual to physical address translation + */ + .text + .align 2 + .globl EXT(mmu_kvtop) +LEXT(mmu_kvtop) + mrs r3, cpsr // Read cpsr + cpsid if // Disable FIQ IRQ + mov r1, r0 + mcr p15, 0, r1, c7, c8, 0 // Write V2PCWPR + isb + mrc p15, 0, r0, c7, c4, 0 // Read PAR + ands r2, r0, #0x1 // Test conversion aborted + bne mmu_kvtophys_fail + ands r2, r0, #0x2 // Test super section + mvnne r2, #0xFF000000 + moveq r2, #0x000000FF + orreq r2, r2, #0x00000F00 + bics r0, r0, r2 // Clear lower bits + beq mmu_kvtophys_fail + and r1, r1, r2 + orr r0, r0, r1 + b mmu_kvtophys_ret +mmu_kvtophys_fail: + mov r0, #0 +mmu_kvtophys_ret: + msr cpsr, r3 // Restore cpsr + bx lr + +/* + * MMU user virtual to physical address translation + */ + .text + .align 2 + .globl EXT(mmu_uvtop) +LEXT(mmu_uvtop) + mrs r3, cpsr // Read cpsr + cpsid if // Disable FIQ IRQ + mov r1, r0 + mcr p15, 0, r1, c7, c8, 2 // Write V2PCWUR + isb + mrc p15, 0, r0, c7, c4, 0 // Read PAR + ands r2, r0, #0x1 // Test conversion aborted + bne mmu_uvtophys_fail + ands r2, r0, #0x2 // Test super section + mvnne r2, #0xFF000000 + moveq r2, #0x000000FF + orreq r2, r2, #0x00000F00 + bics r0, r0, r2 // Clear lower bits + beq mmu_uvtophys_fail + and r1, r1, r2 + orr r0, r0, r1 + b mmu_uvtophys_ret +mmu_uvtophys_fail: + mov r0, #0 +mmu_uvtophys_ret: + msr cpsr, r3 // Restore cpsr + bx lr + +/* + * MMU kernel virtual to physical address preflight write access + */ + .text + .align 2 + .globl EXT(mmu_kvtop_wpreflight) +LEXT(mmu_kvtop_wpreflight) + mrs r3, cpsr // Read cpsr + cpsid if // Disable FIQ IRQ + mov r1, r0 + mcr p15, 0, r1, c7, c8, 1 // Write V2PCWPW + isb + mrc p15, 0, r0, c7, c4, 0 // Read PAR + ands r2, r0, #0x1 // Test conversion aborted + bne mmu_kvtophys_wpreflight_fail + ands r2, r0, #0x2 // Test super section + mvnne r2, #0xFF000000 + moveq r2, #0x000000FF + orreq r2, r2, #0x00000F00 + bics r0, r0, r2 // Clear lower bits + beq mmu_kvtophys_wpreflight_fail // Sanity check: successful access must deliver zero low bits + and r1, r1, r2 + orr r0, r0, r1 + b mmu_kvtophys_wpreflight_ret +mmu_kvtophys_wpreflight_fail: + mov r0, #0 +mmu_kvtophys_wpreflight_ret: + msr cpsr, r3 // Restore cpsr + bx lr + +/* + * set context id register + */ +/* + * set context id register + */ + .text + .align 2 + .globl EXT(set_context_id) +LEXT(set_context_id) + mcr p15, 0, r0, c13, c0, 1 + isb + bx lr + +#define COPYIO_HEADER(rUser, kLabel) \ + /* test for zero len */ ;\ + cmp r2, #0 ;\ + moveq r0, #0 ;\ + bxeq lr ;\ + /* test user_addr, user_addr+len to see if it's in kernel space */ ;\ + add r12, rUser, r2 ;\ + cmp r12, KERNELBASE ;\ + bhs kLabel ;\ + cmp r12, rUser ;\ + bcc kLabel + +#define COPYIO_VALIDATE(NAME, SIZE) \ + /* branch around for small sizes */ ;\ + cmp r2, #(SIZE) ;\ + bls L##NAME##_validate_done ;\ + /* call NAME_validate to check the arguments */ ;\ + push {r0, r1, r2, r7, lr} ;\ + add r7, sp, #12 ;\ + blx EXT(NAME##_validate) ;\ + cmp r0, #0 ;\ + addne sp, #12 ;\ + popne {r7, pc} ;\ + pop {r0, r1, r2, r7, lr} ;\ +L##NAME##_validate_done: + +#define COPYIO_SET_RECOVER() \ + /* set recovery address */ ;\ + stmfd sp!, { r4, r5, r6 } ;\ + adr r3, copyio_error ;\ + mrc p15, 0, r12, c13, c0, 4 ;\ + ldr r4, [r12, TH_RECOVER] ;\ + str r3, [r12, TH_RECOVER] + +#if __ARM_USER_PROTECT__ +#define COPYIO_MAP_USER() \ + /* disable interrupts to prevent expansion to 2GB at L1 ;\ + * between loading ttep and storing it in ttbr0.*/ ;\ + mrs r5, cpsr ;\ + cpsid if ;\ + ldr r3, [r12, ACT_UPTW_TTB] ;\ + mcr p15, 0, r3, c2, c0, 0 ;\ + msr cpsr, r5 ;\ + ldr r3, [r12, ACT_ASID] ;\ + mcr p15, 0, r3, c13, c0, 1 ;\ + isb +#else +#define COPYIO_MAP_USER() +#endif + +#define COPYIO_HEADER_KERN() ;\ + /* test for zero len */ ;\ + cmp r2, #0 ;\ + moveq r0, #0 ;\ + bxeq lr + +.macro COPYIO_BODY + /* if len is less than 16 bytes, just do a simple copy */ + cmp r2, #16 + blt L$0_bytewise + /* test for src and dest of the same word alignment */ + orr r3, r0, r1 + tst r3, #3 + bne L$0_bytewise +L$0_wordwise: + sub r2, r2, #16 +L$0_wordwise_loop: + /* 16 bytes at a time */ + ldmia r0!, { r3, r5, r6, r12 } + stmia r1!, { r3, r5, r6, r12 } + subs r2, r2, #16 + bge L$0_wordwise_loop + /* fixup the len and test for completion */ + adds r2, r2, #16 + beq L$0_noerror +L$0_bytewise: + /* copy 2 bytes at a time */ + subs r2, r2, #2 + ldrb r3, [r0], #1 + ldrbpl r12, [r0], #1 + strb r3, [r1], #1 + strbpl r12, [r1], #1 + bhi L$0_bytewise +L$0_noerror: + mov r0, #0 +.endmacro + +#if __ARM_USER_PROTECT__ +#define COPYIO_UNMAP_USER() \ + mrc p15, 0, r12, c13, c0, 4 ;\ + ldr r3, [r12, ACT_KPTW_TTB] ;\ + mcr p15, 0, r3, c2, c0, 0 ;\ + mov r3, #0 ;\ + mcr p15, 0, r3, c13, c0, 1 ;\ + isb +#else +#define COPYIO_UNMAP_USER() \ + mrc p15, 0, r12, c13, c0, 4 +#endif + +#define COPYIO_RESTORE_RECOVER() \ + /* restore the recovery address */ ;\ + str r4, [r12, TH_RECOVER] ;\ + ldmfd sp!, { r4, r5, r6 } + +/* + * int copyinstr( + * const user_addr_t user_addr, + * char *kernel_addr, + * vm_size_t max, + * vm_size_t *actual) + */ + .text + .align 2 + .globl EXT(copyinstr) +LEXT(copyinstr) + stmfd sp!, { r4, r5, r6 } + + mov r6, r3 + add r3, r0, r2 // user_addr + max + cmp r3, KERNELBASE // Check KERNELBASE < user_addr + max + bhs copyinstr_param_error // Drop out if it is + cmp r3, r0 // Check we're copying from user space + bcc copyinstr_param_error // Drop out if we aren't + adr r3, copyinstr_error // Get address for recover + mrc p15, 0, r12, c13, c0, 4 // Read TPIDRPRW + ldr r4, [r12, TH_RECOVER] ;\ + str r3, [r12, TH_RECOVER] + COPYIO_MAP_USER() + mov r12, #0 // Number of bytes copied so far + cmp r2, #0 + beq copyinstr_too_long +copyinstr_loop: + ldrb r3, [r0], #1 // Load a byte from the source (user) + strb r3, [r1], #1 // Store a byte to the destination (kernel) + add r12, r12, #1 + cmp r3, #0 + beq copyinstr_done + cmp r12, r2 // Room to copy more bytes? + bne copyinstr_loop +// +// Ran out of space in the destination buffer, so return ENAMETOOLONG. +// +copyinstr_too_long: + mov r3, #ENAMETOOLONG +copyinstr_done: +// +// When we get here, we have finished copying the string. We came here from +// either the "beq copyinstr_done" above, in which case r4 == 0 (which is also +// the function result for success), or falling through from copyinstr_too_long, +// in which case r4 == ENAMETOOLONG. +// + str r12, [r6] // Save the count for actual + mov r0, r3 // Return error code from r3 +copyinstr_exit: + COPYIO_UNMAP_USER() + str r4, [r12, TH_RECOVER] +copyinstr_exit2: + ldmfd sp!, { r4, r5, r6 } + bx lr + +copyinstr_error: + /* set error, exit routine */ + mov r0, #EFAULT + b copyinstr_exit + +copyinstr_param_error: + /* set error, exit routine */ + mov r0, #EFAULT + b copyinstr_exit2 + +/* + * int copyin(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes) + */ + .text + .align 2 + .globl EXT(copyin) +LEXT(copyin) + COPYIO_HEADER(r0,copyio_kernel) + COPYIO_VALIDATE(copyin,4096) + COPYIO_SET_RECOVER() + COPYIO_MAP_USER() + COPYIO_BODY copyin + COPYIO_UNMAP_USER() + COPYIO_RESTORE_RECOVER() + bx lr + +/* + * int copyout(const char *kernel_addr, user_addr_t user_addr, vm_size_t nbytes) + */ + .text + .align 2 + .globl EXT(copyout) +LEXT(copyout) + COPYIO_HEADER(r1,copyio_kernel) + COPYIO_VALIDATE(copyout,4096) + COPYIO_SET_RECOVER() + COPYIO_MAP_USER() + COPYIO_BODY copyout + COPYIO_UNMAP_USER() + COPYIO_RESTORE_RECOVER() + bx lr + + +/* + * int copyin_word(const user_addr_t user_addr, uint64_t *kernel_addr, vm_size_t nbytes) + */ + .text + .align 2 + .globl EXT(copyin_word) +LEXT(copyin_word) + cmp r2, #4 // Test if size is 4 or 8 + cmpne r2, #8 + bne L_copyin_invalid + sub r3, r2, #1 + tst r0, r3 // Test alignment of user address + bne L_copyin_invalid + + COPYIO_HEADER(r0,L_copyin_word_fault) + COPYIO_SET_RECOVER() + COPYIO_MAP_USER() + + mov r3, #0 // Clear high register + cmp r2, #4 // If size is 4 + ldreq r2, [r0] // Load word from user + ldrdne r2, r3, [r0] // Else Load double word from user + stm r1, {r2, r3} // Store to kernel_addr + mov r0, #0 // Success + + COPYIO_UNMAP_USER() + COPYIO_RESTORE_RECOVER() + bx lr +L_copyin_invalid: + mov r0, #EINVAL + bx lr +L_copyin_word_fault: + mov r0, #EFAULT + bx lr + + +copyio_error: + mov r0, #EFAULT + COPYIO_UNMAP_USER() + str r4, [r12, TH_RECOVER] + ldmfd sp!, { r4, r5, r6 } + bx lr + +/* + * int copyin_kern(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes) + */ + .text + .align 2 + .globl EXT(copyin_kern) +LEXT(copyin_kern) + COPYIO_HEADER_KERN() + b bypass_check + +/* + * int copyout_kern(const char *kernel_addr, user_addr_t user_addr, vm_size_t nbytes) + */ + .text + .align 2 + .globl EXT(copyout_kern) +LEXT(copyout_kern) + COPYIO_HEADER_KERN() + b bypass_check + +copyio_kernel_error: + mov r0, #EFAULT + bx lr + +copyio_kernel: + /* if (current_thread()->map->pmap != kernel_pmap) return EFAULT */ + mrc p15, 0, r12, c13, c0, 4 // Read TPIDRPRW + ldr r3, [r12, ACT_MAP] + ldr r3, [r3, MAP_PMAP] + LOAD_ADDR(ip, kernel_pmap_store) + cmp r3, ip + bne copyio_kernel_error + +bypass_check: + stmfd sp!, { r5, r6 } + COPYIO_BODY copyio_kernel + ldmfd sp!, { r5, r6 } + bx lr + +/* + * int copyinframe(const vm_address_t frame_addr, char *kernel_addr) + * + * Safely copy eight bytes (the fixed top of an ARM frame) from + * either user or kernel memory. + */ + .text + .align 2 + .globl EXT(copyinframe) +LEXT(copyinframe) + COPYIO_SET_RECOVER() + COPYIO_MAP_USER() + ldmia r0, {r2, r3} + stmia r1, {r2, r3} + b Lcopyin_noerror + +/* + * uint32_t arm_debug_read_dscr(void) + */ + .text + .align 2 + .globl EXT(arm_debug_read_dscr) +LEXT(arm_debug_read_dscr) +#if __ARM_DEBUG__ >= 6 + mrc p14, 0, r0, c0, c1 +#else + mov r0, #0 +#endif + bx lr + +/* + * void arm_debug_set_cp14(arm_debug_state_t *debug_state) + * + * Set debug registers to match the current thread state + * (NULL to disable). Assume 6 breakpoints and 2 + * watchpoints, since that has been the case in all cores + * thus far. + */ + .text + .align 2 + .globl EXT(arm_debug_set_cp14) +LEXT(arm_debug_set_cp14) +#if __ARM_DEBUG__ >= 6 + mrc p15, 0, r1, c13, c0, 4 // Read TPIDRPRW + ldr r2, [r1, ACT_CPUDATAP] // Get current cpu + str r0, [r2, CPU_USER_DEBUG] // Set current user debug + + // Lock the debug registers + movw ip, #0xCE55 + movt ip, #0xC5AC + mcr p14, 0, ip, c1, c0, 4 + + // enable monitor mode (needed to set and use debug registers) + mrc p14, 0, ip, c0, c1, 0 + orr ip, ip, #0x8000 // set MDBGen = 1 +#if __ARM_DEBUG__ >= 7 + mcr p14, 0, ip, c0, c2, 2 +#else + mcr p14, 0, ip, c0, c1, 0 +#endif + // first turn off all breakpoints/watchpoints + mov r1, #0 + mcr p14, 0, r1, c0, c0, 5 // BCR0 + mcr p14, 0, r1, c0, c1, 5 // BCR1 + mcr p14, 0, r1, c0, c2, 5 // BCR2 + mcr p14, 0, r1, c0, c3, 5 // BCR3 + mcr p14, 0, r1, c0, c4, 5 // BCR4 + mcr p14, 0, r1, c0, c5, 5 // BCR5 + mcr p14, 0, r1, c0, c0, 7 // WCR0 + mcr p14, 0, r1, c0, c1, 7 // WCR1 + // if (debug_state == NULL) disable monitor mode and return; + cmp r0, #0 + biceq ip, ip, #0x8000 // set MDBGen = 0 +#if __ARM_DEBUG__ >= 7 + mcreq p14, 0, ip, c0, c2, 2 +#else + mcreq p14, 0, ip, c0, c1, 0 +#endif + bxeq lr + ldmia r0!, {r1, r2, r3, ip} + mcr p14, 0, r1, c0, c0, 4 // BVR0 + mcr p14, 0, r2, c0, c1, 4 // BVR1 + mcr p14, 0, r3, c0, c2, 4 // BVR2 + mcr p14, 0, ip, c0, c3, 4 // BVR3 + ldmia r0!, {r1, r2} + mcr p14, 0, r1, c0, c4, 4 // BVR4 + mcr p14, 0, r2, c0, c5, 4 // BVR5 + add r0, r0, #40 // advance to bcr[0] + ldmia r0!, {r1, r2, r3, ip} + mcr p14, 0, r1, c0, c0, 5 // BCR0 + mcr p14, 0, r2, c0, c1, 5 // BCR1 + mcr p14, 0, r3, c0, c2, 5 // BCR2 + mcr p14, 0, ip, c0, c3, 5 // BCR3 + ldmia r0!, {r1, r2} + mcr p14, 0, r1, c0, c4, 5 // BCR4 + mcr p14, 0, r2, c0, c5, 5 // BCR5 + add r0, r0, #40 // advance to wvr[0] + ldmia r0!, {r1, r2} + mcr p14, 0, r1, c0, c0, 6 // WVR0 + mcr p14, 0, r2, c0, c1, 6 // WVR1 + add r0, r0, #56 // advance to wcr[0] + ldmia r0!, {r1, r2} + mcr p14, 0, r1, c0, c0, 7 // WCR0 + mcr p14, 0, r2, c0, c1, 7 // WCR1 + + // Unlock debug registers + mov ip, #0 + mcr p14, 0, ip, c1, c0, 4 +#endif + bx lr + +/* + * void fiq_context_init(boolean_t enable_fiq) + */ + .text + .align 2 + .globl EXT(fiq_context_init) +LEXT(fiq_context_init) + mrs r3, cpsr // Save current CPSR + cmp r0, #0 // Test enable_fiq + bicne r3, r3, #PSR_FIQF // Enable FIQ if not FALSE + mrc p15, 0, r12, c13, c0, 4 // Read TPIDRPRW + ldr r2, [r12, ACT_CPUDATAP] // Get current cpu data + +#if __ARM_TIME__ + /* Despite the fact that we use the physical timebase + * register as the basis for time on our platforms, we + * end up using the virtual timer in order to manage + * deadlines. This is due to the fact that for our + * current platforms, the interrupt generated by the + * physical timer is not hooked up to anything, and is + * therefore dropped on the floor. Therefore, for + * timers to function they MUST be based on the virtual + * timer. + */ + + mov r0, #1 // Enable Timer + mcr p15, 0, r0, c14, c3, 1 // Write to CNTV_CTL + + /* Enable USER access to the physical timebase (PL0PCTEN). + * The rationale for providing access to the physical + * timebase being that the virtual timebase is broken for + * some platforms. Maintaining the offset ourselves isn't + * expensive, so mandate that the userspace implementation + * do timebase_phys+offset rather than trying to propogate + * all of the informaiton about what works up to USER. + */ + mcr p15, 0, r0, c14, c1, 0 // Set CNTKCTL.PL0PCTEN (CNTKCTL[0]) + +#else /* ! __ARM_TIME__ */ + msr cpsr_c, #(PSR_FIQ_MODE|PSR_FIQF|PSR_IRQF) // Change mode to FIQ with FIQ/IRQ disabled + mov r8, r2 // Load the BootCPUData address + ldr r9, [r2, CPU_GET_FIQ_HANDLER] // Load fiq function address + ldr r10, [r2, CPU_TBD_HARDWARE_ADDR] // Load the hardware address + ldr r11, [r2, CPU_TBD_HARDWARE_VAL] // Load the hardware value +#endif /* __ARM_TIME__ */ + + msr cpsr_c, r3 // Restore saved CPSR + bx lr + +/* + * void reenable_async_aborts(void) + */ + .text + .align 2 + .globl EXT(reenable_async_aborts) +LEXT(reenable_async_aborts) + cpsie a // Re-enable async aborts + bx lr + +/* + * uint64_t ml_get_timebase(void) + */ + .text + .align 2 + .globl EXT(ml_get_timebase) +LEXT(ml_get_timebase) + mrc p15, 0, r12, c13, c0, 4 // Read TPIDRPRW + ldr r3, [r12, ACT_CPUDATAP] // Get current cpu data +#if __ARM_TIME__ || __ARM_TIME_TIMEBASE_ONLY__ + isb // Required by ARMV7C.b section B8.1.2, ARMv8 section D6.1.2. +1: + mrrc p15, 0, r3, r1, c14 // Read the Time Base (CNTPCT), high => r1 + mrrc p15, 0, r0, r3, c14 // Read the Time Base (CNTPCT), low => r0 + mrrc p15, 0, r3, r2, c14 // Read the Time Base (CNTPCT), high => r2 + cmp r1, r2 + bne 1b // Loop until both high values are the same + + ldr r3, [r12, ACT_CPUDATAP] // Get current cpu data + ldr r2, [r3, CPU_BASE_TIMEBASE_LOW] // Add in the offset to + adds r0, r0, r2 // convert to + ldr r2, [r3, CPU_BASE_TIMEBASE_HIGH] // mach_absolute_time + adc r1, r1, r2 // +#else /* ! __ARM_TIME__ || __ARM_TIME_TIMEBASE_ONLY__ */ +1: + ldr r2, [r3, CPU_TIMEBASE_HIGH] // Get the saved TBU value + ldr r0, [r3, CPU_TIMEBASE_LOW] // Get the saved TBL value + ldr r1, [r3, CPU_TIMEBASE_HIGH] // Get the saved TBU value + cmp r1, r2 // Make sure TB has not rolled over + bne 1b +#endif /* __ARM_TIME__ */ + bx lr // return + + +/* + * uint32_t ml_get_decrementer(void) + */ + .text + .align 2 + .globl EXT(ml_get_decrementer) +LEXT(ml_get_decrementer) + mrc p15, 0, r12, c13, c0, 4 // Read TPIDRPRW + ldr r3, [r12, ACT_CPUDATAP] // Get current cpu data + ldr r2, [r3, CPU_GET_DECREMENTER_FUNC] // Get get_decrementer_func + cmp r2, #0 + bxne r2 // Call it if there is one +#if __ARM_TIME__ + mrc p15, 0, r0, c14, c3, 0 // Read the Decrementer (CNTV_TVAL) +#else + ldr r0, [r3, CPU_DECREMENTER] // Get the saved dec value +#endif + bx lr // return + + +/* + * void ml_set_decrementer(uint32_t dec_value) + */ + .text + .align 2 + .globl EXT(ml_set_decrementer) +LEXT(ml_set_decrementer) + mrc p15, 0, r12, c13, c0, 4 // Read TPIDRPRW + ldr r3, [r12, ACT_CPUDATAP] // Get current cpu data + ldr r2, [r3, CPU_SET_DECREMENTER_FUNC] // Get set_decrementer_func + cmp r2, #0 + bxne r2 // Call it if there is one +#if __ARM_TIME__ + str r0, [r3, CPU_DECREMENTER] // Save the new dec value + mcr p15, 0, r0, c14, c3, 0 // Write the Decrementer (CNTV_TVAL) +#else + mrs r2, cpsr // Save current CPSR + msr cpsr_c, #(PSR_FIQ_MODE|PSR_FIQF|PSR_IRQF) // Change mode to FIQ with FIQ/IRQ disabled. + mov r12, r0 // Set the DEC value + str r12, [r8, CPU_DECREMENTER] // Store DEC + msr cpsr_c, r2 // Restore saved CPSR +#endif + bx lr + + +/* + * boolean_t ml_get_interrupts_enabled(void) + */ + .text + .align 2 + .globl EXT(ml_get_interrupts_enabled) +LEXT(ml_get_interrupts_enabled) + mrs r2, cpsr + mov r0, #1 + bic r0, r0, r2, lsr #PSR_IRQFb + bx lr + +/* + * Platform Specific Timebase & Decrementer Functions + * + */ + +#if defined(ARM_BOARD_CLASS_S7002) + .text + .align 2 + .globl EXT(fleh_fiq_s7002) +LEXT(fleh_fiq_s7002) + str r11, [r10, #PMGR_INTERVAL_TMR_CTL_OFFSET] // Clear the decrementer interrupt + mvn r13, #0 + str r13, [r8, CPU_DECREMENTER] + b EXT(fleh_dec) + + .text + .align 2 + .globl EXT(s7002_get_decrementer) +LEXT(s7002_get_decrementer) + ldr ip, [r3, CPU_TBD_HARDWARE_ADDR] // Get the hardware address + add ip, ip, #PMGR_INTERVAL_TMR_OFFSET + ldr r0, [ip] // Get the Decrementer + bx lr + + .text + .align 2 + .globl EXT(s7002_set_decrementer) +LEXT(s7002_set_decrementer) + str r0, [r3, CPU_DECREMENTER] // Save the new dec value + ldr ip, [r3, CPU_TBD_HARDWARE_ADDR] // Get the hardware address + str r0, [ip, #PMGR_INTERVAL_TMR_OFFSET] // Store the new Decrementer + bx lr +#endif /* defined(ARM_BOARD_CLASS_S7002) */ + +#if defined(ARM_BOARD_CLASS_T8002) + .text + .align 2 + .globl EXT(fleh_fiq_t8002) +LEXT(fleh_fiq_t8002) + mov r13, #kAICTmrIntStat + str r11, [r10, r13] // Clear the decrementer interrupt + mvn r13, #0 + str r13, [r8, CPU_DECREMENTER] + b EXT(fleh_dec) + + .text + .align 2 + .globl EXT(t8002_get_decrementer) +LEXT(t8002_get_decrementer) + ldr ip, [r3, CPU_TBD_HARDWARE_ADDR] // Get the hardware address + mov r0, #kAICTmrCnt + add ip, ip, r0 + ldr r0, [ip] // Get the Decrementer + bx lr + + .text + .align 2 + .globl EXT(t8002_set_decrementer) +LEXT(t8002_set_decrementer) + str r0, [r3, CPU_DECREMENTER] // Save the new dec value + ldr ip, [r3, CPU_TBD_HARDWARE_ADDR] // Get the hardware address + mov r5, #kAICTmrCnt + str r0, [ip, r5] // Store the new Decrementer + bx lr +#endif /* defined(ARM_BOARD_CLASS_T8002) */ + +LOAD_ADDR_GEN_DEF(kernel_pmap_store) + +#include "globals_asm.h" + +/* vim: set ts=4: */ diff --git a/osfmk/arm/machine_routines_common.c b/osfmk/arm/machine_routines_common.c new file mode 100644 index 000000000..764fb97e0 --- /dev/null +++ b/osfmk/arm/machine_routines_common.c @@ -0,0 +1,614 @@ +/* + * Copyright (c) 2007-2013 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include <arm/machine_cpu.h> +#include <arm/cpu_internal.h> +#include <arm/cpuid.h> +#include <arm/cpu_data.h> +#include <arm/cpu_data_internal.h> +#include <arm/misc_protos.h> +#include <arm/machdep_call.h> +#include <arm/machine_routines.h> +#include <arm/rtclock.h> +#include <kern/machine.h> +#include <kern/thread.h> +#include <kern/thread_group.h> +#include <kern/policy_internal.h> +#include <machine/config.h> + +#if MONOTONIC +#include <kern/monotonic.h> +#include <machine/monotonic.h> +#endif /* MONOTONIC */ + +#include <mach/machine.h> + +#if INTERRUPT_MASKED_DEBUG +extern boolean_t interrupt_masked_debug; +extern uint64_t interrupt_masked_timeout; +#endif + +extern uint64_t mach_absolutetime_asleep; + +static void +sched_perfcontrol_oncore_default(perfcontrol_state_t new_thread_state __unused, going_on_core_t on __unused) +{ +} + +static void +sched_perfcontrol_switch_default(perfcontrol_state_t old_thread_state __unused, perfcontrol_state_t new_thread_state __unused) +{ +} + +static void +sched_perfcontrol_offcore_default(perfcontrol_state_t old_thread_state __unused, going_off_core_t off __unused, boolean_t thread_terminating __unused) +{ +} + +static void +sched_perfcontrol_thread_group_default(thread_group_data_t data __unused) +{ +} + +static void +sched_perfcontrol_max_runnable_latency_default(perfcontrol_max_runnable_latency_t latencies __unused) +{ +} + +static void +sched_perfcontrol_work_interval_notify_default(perfcontrol_state_t thread_state __unused, perfcontrol_work_interval_t work_interval __unused) +{ +} + +static void +sched_perfcontrol_deadline_passed_default(__unused uint64_t deadline) +{ +} + +static void +sched_perfcontrol_csw_default( + __unused perfcontrol_event event, __unused uint32_t cpu_id, __unused uint64_t timestamp, + __unused uint32_t flags, __unused struct perfcontrol_thread_data *offcore, + __unused struct perfcontrol_thread_data *oncore, + __unused struct perfcontrol_cpu_counters *cpu_counters, __unused void *unused) +{ +} + +static void +sched_perfcontrol_state_update_default( + __unused perfcontrol_event event, __unused uint32_t cpu_id, __unused uint64_t timestamp, + __unused uint32_t flags, __unused struct perfcontrol_thread_data *thr_data, + __unused void *unused) +{ +} + +sched_perfcontrol_offcore_t sched_perfcontrol_offcore = sched_perfcontrol_offcore_default; +sched_perfcontrol_context_switch_t sched_perfcontrol_switch = sched_perfcontrol_switch_default; +sched_perfcontrol_oncore_t sched_perfcontrol_oncore = sched_perfcontrol_oncore_default; +sched_perfcontrol_thread_group_init_t sched_perfcontrol_thread_group_init = sched_perfcontrol_thread_group_default; +sched_perfcontrol_thread_group_deinit_t sched_perfcontrol_thread_group_deinit = sched_perfcontrol_thread_group_default; +sched_perfcontrol_thread_group_flags_update_t sched_perfcontrol_thread_group_flags_update = sched_perfcontrol_thread_group_default; +sched_perfcontrol_max_runnable_latency_t sched_perfcontrol_max_runnable_latency = sched_perfcontrol_max_runnable_latency_default; +sched_perfcontrol_work_interval_notify_t sched_perfcontrol_work_interval_notify = sched_perfcontrol_work_interval_notify_default; +sched_perfcontrol_deadline_passed_t sched_perfcontrol_deadline_passed = sched_perfcontrol_deadline_passed_default; +sched_perfcontrol_csw_t sched_perfcontrol_csw = sched_perfcontrol_csw_default; +sched_perfcontrol_state_update_t sched_perfcontrol_state_update = sched_perfcontrol_state_update_default; + +void +sched_perfcontrol_register_callbacks(sched_perfcontrol_callbacks_t callbacks, unsigned long size_of_state) +{ + assert(callbacks == NULL || callbacks->version >= SCHED_PERFCONTROL_CALLBACKS_VERSION_2); + + if (size_of_state > sizeof(struct perfcontrol_state)) { + panic("%s: Invalid required state size %lu", __FUNCTION__, size_of_state); + } + + if (callbacks) { + + + if (callbacks->version >= SCHED_PERFCONTROL_CALLBACKS_VERSION_5) { + if (callbacks->csw != NULL) { + sched_perfcontrol_csw = callbacks->csw; + } else { + sched_perfcontrol_csw = sched_perfcontrol_csw_default; + } + + if (callbacks->state_update != NULL) { + sched_perfcontrol_state_update = callbacks->state_update; + } else { + sched_perfcontrol_state_update = sched_perfcontrol_state_update_default; + } + } + + if (callbacks->version >= SCHED_PERFCONTROL_CALLBACKS_VERSION_4) { + if (callbacks->deadline_passed != NULL) { + sched_perfcontrol_deadline_passed = callbacks->deadline_passed; + } else { + sched_perfcontrol_deadline_passed = sched_perfcontrol_deadline_passed_default; + } + } + + if (callbacks->offcore != NULL) { + sched_perfcontrol_offcore = callbacks->offcore; + } else { + sched_perfcontrol_offcore = sched_perfcontrol_offcore_default; + } + + if (callbacks->context_switch != NULL) { + sched_perfcontrol_switch = callbacks->context_switch; + } else { + sched_perfcontrol_switch = sched_perfcontrol_switch_default; + } + + if (callbacks->oncore != NULL) { + sched_perfcontrol_oncore = callbacks->oncore; + } else { + sched_perfcontrol_oncore = sched_perfcontrol_oncore_default; + } + + if (callbacks->max_runnable_latency != NULL) { + sched_perfcontrol_max_runnable_latency = callbacks->max_runnable_latency; + } else { + sched_perfcontrol_max_runnable_latency = sched_perfcontrol_max_runnable_latency_default; + } + + if (callbacks->work_interval_notify != NULL) { + sched_perfcontrol_work_interval_notify = callbacks->work_interval_notify; + } else { + sched_perfcontrol_work_interval_notify = sched_perfcontrol_work_interval_notify_default; + } + } else { + /* reset to defaults */ + sched_perfcontrol_offcore = sched_perfcontrol_offcore_default; + sched_perfcontrol_switch = sched_perfcontrol_switch_default; + sched_perfcontrol_oncore = sched_perfcontrol_oncore_default; + sched_perfcontrol_thread_group_init = sched_perfcontrol_thread_group_default; + sched_perfcontrol_thread_group_deinit = sched_perfcontrol_thread_group_default; + sched_perfcontrol_thread_group_flags_update = sched_perfcontrol_thread_group_default; + sched_perfcontrol_max_runnable_latency = sched_perfcontrol_max_runnable_latency_default; + sched_perfcontrol_work_interval_notify = sched_perfcontrol_work_interval_notify_default; + sched_perfcontrol_csw = sched_perfcontrol_csw_default; + sched_perfcontrol_state_update = sched_perfcontrol_state_update_default; + } +} + + +static void +machine_switch_populate_perfcontrol_thread_data(struct perfcontrol_thread_data *data, + thread_t thread, + uint64_t same_pri_latency) +{ + bzero(data, sizeof(struct perfcontrol_thread_data)); + data->perfctl_class = thread_get_perfcontrol_class(thread); + data->energy_estimate_nj = 0; + data->thread_id = thread->thread_id; + data->scheduling_latency_at_same_basepri = same_pri_latency; + data->perfctl_state = FIND_PERFCONTROL_STATE(thread); +} + +static void +machine_switch_populate_perfcontrol_cpu_counters(struct perfcontrol_cpu_counters *cpu_counters) +{ +#if MONOTONIC + mt_perfcontrol(&cpu_counters->instructions, &cpu_counters->cycles); +#else /* MONOTONIC */ + cpu_counters->instructions = 0; + cpu_counters->cycles = 0; +#endif /* !MONOTONIC */ +} + +int perfcontrol_callout_stats_enabled = 0; +static _Atomic uint64_t perfcontrol_callout_stats[PERFCONTROL_CALLOUT_MAX][PERFCONTROL_STAT_MAX]; +static _Atomic uint64_t perfcontrol_callout_count[PERFCONTROL_CALLOUT_MAX]; + +#if MONOTONIC +static inline +bool perfcontrol_callout_counters_begin(uint64_t *counters) +{ + if (!perfcontrol_callout_stats_enabled) + return false; + mt_fixed_counts(counters); + return true; +} + +static inline +void perfcontrol_callout_counters_end(uint64_t *start_counters, + perfcontrol_callout_type_t type) +{ + uint64_t end_counters[MT_CORE_NFIXED]; + mt_fixed_counts(end_counters); + atomic_fetch_add_explicit(&perfcontrol_callout_stats[type][PERFCONTROL_STAT_CYCLES], + end_counters[MT_CORE_CYCLES] - start_counters[MT_CORE_CYCLES], memory_order_relaxed); +#ifdef MT_CORE_INSTRS + atomic_fetch_add_explicit(&perfcontrol_callout_stats[type][PERFCONTROL_STAT_INSTRS], + end_counters[MT_CORE_INSTRS] - start_counters[MT_CORE_INSTRS], memory_order_relaxed); +#endif /* defined(MT_CORE_INSTRS) */ + atomic_fetch_add_explicit(&perfcontrol_callout_count[type], 1, memory_order_relaxed); +} +#endif /* MONOTONIC */ + +uint64_t perfcontrol_callout_stat_avg(perfcontrol_callout_type_t type, + perfcontrol_callout_stat_t stat) +{ + if (!perfcontrol_callout_stats_enabled) + return 0; + return (perfcontrol_callout_stats[type][stat] / perfcontrol_callout_count[type]); +} + +void +machine_switch_perfcontrol_context(perfcontrol_event event, + uint64_t timestamp, + uint32_t flags, + uint64_t new_thread_same_pri_latency, + thread_t old, + thread_t new) +{ + if (sched_perfcontrol_switch != sched_perfcontrol_switch_default) { + perfcontrol_state_t old_perfcontrol_state = FIND_PERFCONTROL_STATE(old); + perfcontrol_state_t new_perfcontrol_state = FIND_PERFCONTROL_STATE(new); + sched_perfcontrol_switch(old_perfcontrol_state, new_perfcontrol_state); + } + + if (sched_perfcontrol_csw != sched_perfcontrol_csw_default) { + uint32_t cpu_id = (uint32_t)cpu_number(); + struct perfcontrol_cpu_counters cpu_counters; + struct perfcontrol_thread_data offcore, oncore; + machine_switch_populate_perfcontrol_thread_data(&offcore, old, 0); + machine_switch_populate_perfcontrol_thread_data(&oncore, new, + new_thread_same_pri_latency); + machine_switch_populate_perfcontrol_cpu_counters(&cpu_counters); + +#if MONOTONIC + uint64_t counters[MT_CORE_NFIXED]; + bool ctrs_enabled = perfcontrol_callout_counters_begin(counters); +#endif /* MONOTONIC */ + sched_perfcontrol_csw(event, cpu_id, timestamp, flags, + &offcore, &oncore, &cpu_counters, NULL); +#if MONOTONIC + if (ctrs_enabled) perfcontrol_callout_counters_end(counters, PERFCONTROL_CALLOUT_CONTEXT); +#endif /* MONOTONIC */ + +#if __arm64__ + old->machine.energy_estimate_nj += offcore.energy_estimate_nj; + new->machine.energy_estimate_nj += oncore.energy_estimate_nj; +#endif + } +} + +void +machine_switch_perfcontrol_state_update(perfcontrol_event event, + uint64_t timestamp, + uint32_t flags, + thread_t thread) +{ + if (sched_perfcontrol_state_update == sched_perfcontrol_state_update_default) + return; + uint32_t cpu_id = (uint32_t)cpu_number(); + struct perfcontrol_thread_data data; + machine_switch_populate_perfcontrol_thread_data(&data, thread, 0); + +#if MONOTONIC + uint64_t counters[MT_CORE_NFIXED]; + bool ctrs_enabled = perfcontrol_callout_counters_begin(counters); +#endif /* MONOTONIC */ + sched_perfcontrol_state_update(event, cpu_id, timestamp, flags, + &data, NULL); +#if MONOTONIC + if (ctrs_enabled) perfcontrol_callout_counters_end(counters, PERFCONTROL_CALLOUT_STATE_UPDATE); +#endif /* MONOTONIC */ + +#if __arm64__ + thread->machine.energy_estimate_nj += data.energy_estimate_nj; +#endif +} + +void +machine_thread_going_on_core(thread_t new_thread, + int urgency, + uint64_t sched_latency, + uint64_t same_pri_latency, + uint64_t timestamp) +{ + + if (sched_perfcontrol_oncore == sched_perfcontrol_oncore_default) + return; + struct going_on_core on_core; + perfcontrol_state_t state = FIND_PERFCONTROL_STATE(new_thread); + + on_core.thread_id = new_thread->thread_id; + on_core.energy_estimate_nj = 0; + on_core.qos_class = proc_get_effective_thread_policy(new_thread, TASK_POLICY_QOS); + on_core.urgency = urgency; + on_core.is_32_bit = thread_is_64bit(new_thread) ? FALSE : TRUE; + on_core.is_kernel_thread = new_thread->task == kernel_task; + on_core.scheduling_latency = sched_latency; + on_core.start_time = timestamp; + on_core.scheduling_latency_at_same_basepri = same_pri_latency; + +#if MONOTONIC + uint64_t counters[MT_CORE_NFIXED]; + bool ctrs_enabled = perfcontrol_callout_counters_begin(counters); +#endif /* MONOTONIC */ + sched_perfcontrol_oncore(state, &on_core); +#if MONOTONIC + if (ctrs_enabled) perfcontrol_callout_counters_end(counters, PERFCONTROL_CALLOUT_ON_CORE); +#endif /* MONOTONIC */ + +#if __arm64__ + new_thread->machine.energy_estimate_nj += on_core.energy_estimate_nj; +#endif +} + +void +machine_thread_going_off_core(thread_t old_thread, boolean_t thread_terminating, uint64_t last_dispatch) +{ + if (sched_perfcontrol_offcore == sched_perfcontrol_offcore_default) + return; + struct going_off_core off_core; + perfcontrol_state_t state = FIND_PERFCONTROL_STATE(old_thread); + + off_core.thread_id = old_thread->thread_id; + off_core.energy_estimate_nj = 0; + off_core.end_time = last_dispatch; + +#if MONOTONIC + uint64_t counters[MT_CORE_NFIXED]; + bool ctrs_enabled = perfcontrol_callout_counters_begin(counters); +#endif /* MONOTONIC */ + sched_perfcontrol_offcore(state, &off_core, thread_terminating); +#if MONOTONIC + if (ctrs_enabled) perfcontrol_callout_counters_end(counters, PERFCONTROL_CALLOUT_OFF_CORE); +#endif /* MONOTONIC */ + +#if __arm64__ + old_thread->machine.energy_estimate_nj += off_core.energy_estimate_nj; +#endif +} + + +void +machine_max_runnable_latency(uint64_t bg_max_latency, + uint64_t default_max_latency, + uint64_t realtime_max_latency) +{ + if (sched_perfcontrol_max_runnable_latency == sched_perfcontrol_max_runnable_latency_default) + return; + struct perfcontrol_max_runnable_latency latencies = { + .max_scheduling_latencies = { + [THREAD_URGENCY_NONE] = 0, + [THREAD_URGENCY_BACKGROUND] = bg_max_latency, + [THREAD_URGENCY_NORMAL] = default_max_latency, + [THREAD_URGENCY_REAL_TIME] = realtime_max_latency + } + }; + + sched_perfcontrol_max_runnable_latency(&latencies); +} + +void +machine_work_interval_notify(thread_t thread, + struct kern_work_interval_args* kwi_args) +{ + if (sched_perfcontrol_work_interval_notify == sched_perfcontrol_work_interval_notify_default) + return; + perfcontrol_state_t state = FIND_PERFCONTROL_STATE(thread); + struct perfcontrol_work_interval work_interval = { + .thread_id = thread->thread_id, + .qos_class = proc_get_effective_thread_policy(thread, TASK_POLICY_QOS), + .urgency = kwi_args->urgency, + .flags = kwi_args->notify_flags, + .work_interval_id = kwi_args->work_interval_id, + .start = kwi_args->start, + .finish = kwi_args->finish, + .deadline = kwi_args->deadline, + .next_start = kwi_args->next_start, + .create_flags = kwi_args->create_flags, + }; + sched_perfcontrol_work_interval_notify(state, &work_interval); +} + +void +machine_perfcontrol_deadline_passed(uint64_t deadline) +{ + if (sched_perfcontrol_deadline_passed != sched_perfcontrol_deadline_passed_default) + sched_perfcontrol_deadline_passed(deadline); +} + +#if INTERRUPT_MASKED_DEBUG +/* + * ml_spin_debug_reset() + * Reset the timestamp on a thread that has been unscheduled + * to avoid false alarms. Alarm will go off if interrupts are held + * disabled for too long, starting from now. + */ +void +ml_spin_debug_reset(thread_t thread) +{ + thread->machine.intmask_timestamp = mach_absolute_time(); +} + +/* + * ml_spin_debug_clear() + * Clear the timestamp on a thread that has been unscheduled + * to avoid false alarms + */ +void +ml_spin_debug_clear(thread_t thread) +{ + thread->machine.intmask_timestamp = 0; +} + +/* + * ml_spin_debug_clear_self() + * Clear the timestamp on the current thread to prevent + * false alarms + */ +void +ml_spin_debug_clear_self() +{ + ml_spin_debug_clear(current_thread()); +} + +void +ml_check_interrupts_disabled_duration(thread_t thread) +{ + uint64_t start; + uint64_t now; + + start = thread->machine.intmask_timestamp; + if (start != 0) { + now = mach_absolute_time(); + + if ((now - start) > interrupt_masked_timeout) { + mach_timebase_info_data_t timebase; + clock_timebase_info(&timebase); + +#ifndef KASAN + /* + * Disable the actual panic for KASAN due to the overhead of KASAN itself, leave the rest of the + * mechanism enabled so that KASAN can catch any bugs in the mechanism itself. + */ + panic("Interrupts held disabled for %llu nanoseconds", (((now - start) * timebase.numer)/timebase.denom)); +#endif + } + } + + return; +} +#endif // INTERRUPT_MASKED_DEBUG + + +boolean_t +ml_set_interrupts_enabled(boolean_t enable) +{ + thread_t thread; + uint64_t state; + +#if __arm__ +#define INTERRUPT_MASK PSR_IRQF + state = __builtin_arm_rsr("cpsr"); +#else +#define INTERRUPT_MASK DAIF_IRQF + state = __builtin_arm_rsr("DAIF"); +#endif + if (enable) { +#if INTERRUPT_MASKED_DEBUG + if (interrupt_masked_debug && (state & INTERRUPT_MASK)) { + // Interrupts are currently masked, we will enable them (after finishing this check) + thread = current_thread(); + ml_check_interrupts_disabled_duration(thread); + thread->machine.intmask_timestamp = 0; + } +#endif // INTERRUPT_MASKED_DEBUG + if (get_preemption_level() == 0) { + thread = current_thread(); + while (thread->machine.CpuDatap->cpu_pending_ast & AST_URGENT) { +#if __ARM_USER_PROTECT__ + uintptr_t up = arm_user_protect_begin(thread); +#endif + ast_taken_kernel(); +#if __ARM_USER_PROTECT__ + arm_user_protect_end(thread, up, FALSE); +#endif + } + } +#if __arm__ + __asm__ volatile ("cpsie if" ::: "memory"); // Enable IRQ FIQ +#else + __builtin_arm_wsr("DAIFClr", (DAIFSC_IRQF | DAIFSC_FIQF)); +#endif + } else { +#if __arm__ + __asm__ volatile ("cpsid if" ::: "memory"); // Mask IRQ FIQ +#else + __builtin_arm_wsr("DAIFSet", (DAIFSC_IRQF | DAIFSC_FIQF)); +#endif +#if INTERRUPT_MASKED_DEBUG + if (interrupt_masked_debug && ((state & INTERRUPT_MASK) == 0)) { + // Interrupts were enabled, we just masked them + current_thread()->machine.intmask_timestamp = mach_absolute_time(); + } +#endif + } + return ((state & INTERRUPT_MASK) == 0); +} + +static boolean_t ml_quiescing; + +void ml_set_is_quiescing(boolean_t quiescing) +{ + assert(FALSE == ml_get_interrupts_enabled()); + ml_quiescing = quiescing; +} + +boolean_t ml_is_quiescing(void) +{ + assert(FALSE == ml_get_interrupts_enabled()); + return (ml_quiescing); +} + +uint64_t ml_get_booter_memory_size(void) +{ + enum { kRoundSize = 512*1024*1024ULL }; + uint64_t size; + size = BootArgs->memSizeActual; + if (!size) + { + size = BootArgs->memSize; + size = (size + kRoundSize - 1) & ~(kRoundSize - 1); + size -= BootArgs->memSize; + } + return (size); +} + +uint64_t +ml_get_abstime_offset(void) +{ + return rtclock_base_abstime; +} + +uint64_t +ml_get_conttime_offset(void) +{ + return (rtclock_base_abstime + mach_absolutetime_asleep); +} + +uint64_t +ml_get_time_since_reset(void) +{ + /* The timebase resets across S2R, so just return the raw value. */ + return ml_get_hwclock(); +} + +uint64_t +ml_get_conttime_wake_time(void) +{ + /* The wake time is simply our continuous time offset. */ + return ml_get_conttime_offset(); +} + diff --git a/osfmk/arm/machine_task.c b/osfmk/arm/machine_task.c new file mode 100644 index 000000000..517f4fa1a --- /dev/null +++ b/osfmk/arm/machine_task.c @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ + +#include <kern/task.h> +#include <kern/thread.h> +#include <arm/misc_protos.h> + +extern zone_t ads_zone; + +kern_return_t +machine_task_set_state( + task_t task, + int flavor, + thread_state_t state, + mach_msg_type_number_t state_count) +{ + switch (flavor) { + case ARM_DEBUG_STATE: + { + arm_debug_state_t *tstate = (arm_debug_state_t *) state; + + if (state_count != ARM_DEBUG_STATE_COUNT) { + return KERN_INVALID_ARGUMENT; + } + + if (task->task_debug == NULL) { + task->task_debug = zalloc(ads_zone); + if (task->task_debug == NULL) + return KERN_FAILURE; + } + + copy_debug_state(tstate, (arm_debug_state_t*) task->task_debug, FALSE); + + return KERN_SUCCESS; + } + case THREAD_STATE_NONE: /* Using this flavor to clear task_debug */ + { + if (task->task_debug != NULL) { + zfree(ads_zone, task->task_debug); + task->task_debug = NULL; + + return KERN_SUCCESS; + } + return KERN_FAILURE; + } + default: + { + return KERN_INVALID_ARGUMENT; + } + } + + return KERN_FAILURE; +} + +kern_return_t +machine_task_get_state(task_t task, + int flavor, + thread_state_t state, + mach_msg_type_number_t *state_count) +{ + switch (flavor) { + case ARM_DEBUG_STATE: + { + arm_debug_state_t *tstate = (arm_debug_state_t *) state; + + if (*state_count != ARM_DEBUG_STATE_COUNT) { + return KERN_INVALID_ARGUMENT; + } + + if (task->task_debug == NULL) { + bzero(state, sizeof(*tstate)); + } else { + copy_debug_state((arm_debug_state_t*) task->task_debug, tstate, FALSE); /* FALSE OR TRUE doesn't matter since we are ignoring it for arm */ + } + + return KERN_SUCCESS; + } + default: + { + return KERN_INVALID_ARGUMENT; + } + + } + return KERN_FAILURE; +} + +void +machine_task_terminate(task_t task) +{ + if (task) { + void *task_debug; + + task_debug = task->task_debug; + if (task_debug != NULL) { + task->task_debug = NULL; + zfree(ads_zone, task_debug); + } + } +} + + +kern_return_t +machine_thread_inherit_taskwide( + thread_t thread, + task_t parent_task) +{ + if (parent_task->task_debug) { + int flavor; + mach_msg_type_number_t count; + + flavor = ARM_DEBUG_STATE; + count = ARM_DEBUG_STATE_COUNT; + + return machine_thread_set_state(thread, flavor, parent_task->task_debug, count); + } + + return KERN_SUCCESS; +} + + +void +machine_task_init(__unused task_t new_task, + __unused task_t parent_task, + __unused boolean_t memory_inherit) +{ +} diff --git a/osfmk/arm/machlimits.h b/osfmk/arm/machlimits.h new file mode 100644 index 000000000..0ab749b6b --- /dev/null +++ b/osfmk/arm/machlimits.h @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * HISTORY + * + * Revision 1.1.1.1 1998/09/22 21:05:41 wsanchez + * Import of Mac OS X kernel (~semeria) + * + * Revision 1.1.1.1 1998/03/07 02:26:02 wsanchez + * Import of OSF Mach kernel (~mburg) + * + * Revision 1.1.2.1 1996/12/09 16:55:05 stephen + * nmklinux_1.0b3_shared into pmk1.1 + * New file based on hp_pa + * [1996/12/09 11:09:22 stephen] + * + * $EndLog$ + */ +/* + * Copyright (c) 1988 The Regents of the University of California. + * All rights reserved. + * + * Redistribution and use in source and binary forms are permitted + * provided that the above copyright notice and this paragraph are + * duplicated in all such forms and that any documentation, + * advertising materials, and other materials related to such + * distribution and use acknowledge that the software was developed + * by the University of California, Berkeley. The name of the + * University may not be used to endorse or promote products derived + * from this software without specific prior written permission. + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + * + * @(#)machlimits.h 7.1 (Berkeley) 2/15/89 + */ +#ifndef _MACH_MACHLIMITS_H_ +#define _MACH_MACHLIMITS_H_ + +#define CHAR_BIT 8 /* number of bits in a char */ + +#define SCHAR_MAX 127 /* max value for a signed char */ +#define SCHAR_MIN (-128) /* min value for a signed char */ + +#define UCHAR_MAX 255U /* max value for an unsigned char */ +#define CHAR_MAX 127 /* max value for a char */ +#define CHAR_MIN (-128) /* min value for a char */ + +#define USHRT_MAX 65535U /* max value for an unsigned short */ +#define SHRT_MAX 32767 /* max value for a short */ +#define SHRT_MIN (-32768) /* min value for a short */ + +#define UINT_MAX 0xFFFFFFFFU /* max value for an unsigned int */ +#define INT_MAX 2147483647 /* max value for an int */ +#define INT_MIN (-2147483647-1) /* min value for an int */ + +#ifdef __LP64__ +#define ULONG_MAX 0xffffffffffffffffUL /* max unsigned long */ +#define LONG_MAX 0x7fffffffffffffffL /* max signed long */ +#define LONG_MIN (-0x7fffffffffffffffL-1)/* min signed long */ +#else /* !__LP64__ */ +#define ULONG_MAX 0xffffffffUL /* max value for an unsigned long */ +#define LONG_MAX 2147483647L /* max value for a long */ +#define LONG_MIN (-2147483647L-1) /* min value for a long */ +#endif /* __LP64__ */ + +/* Must be at least two, for internationalization (NLS/KJI) */ +#define MB_LEN_MAX 4 /* multibyte characters */ + +#endif /* _MACH_MACHLIMITS_H_ */ diff --git a/osfmk/arm/machparam.h b/osfmk/arm/machparam.h new file mode 100644 index 000000000..4d5ec30de --- /dev/null +++ b/osfmk/arm/machparam.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ +/* + */ + +/* + * Machine-dependent SPL definitions. + * + * SPLs are true functions on i386, defined elsewhere. + */ + diff --git a/osfmk/arm/misc_protos.h b/osfmk/arm/misc_protos.h new file mode 100644 index 000000000..416bcb2a3 --- /dev/null +++ b/osfmk/arm/misc_protos.h @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ + +#ifndef _ARM_MISC_PROTOS_H_ +#define _ARM_MISC_PROTOS_H_ + +#include <kern/kern_types.h> + +extern processor_t cpu_processor_alloc(boolean_t is_boot_cpu); +extern void cpu_processor_free(processor_t proc); + +extern void machine_startup(__unused boot_args *args) __attribute__((noinline)); +extern void machine_lockdown_preflight(void); +extern void machine_lockdown(void); +extern void arm_vm_init(uint64_t memory_size, boot_args *args); +extern void arm_vm_prot_init(boot_args *args); +extern void arm_vm_prot_finalize(boot_args *args); + + +extern kern_return_t DebuggerXCallEnter(boolean_t); +extern void DebuggerXCallReturn(void); + +#if __arm64__ && DEBUG +extern void dump_kva_space(void); +#endif + +extern void Load_context(thread_t); +extern void Idle_load_context(void) __attribute__((noreturn)); +extern thread_t Switch_context(thread_t, thread_continue_t, thread_t); +extern thread_t Shutdown_context(void (*doshutdown)(processor_t), processor_t processor); +extern void Call_continuation(thread_continue_t, void *, wait_result_t, vm_offset_t); + +extern void DebuggerCall(unsigned int reason, void *ctx); +extern void DebuggerXCall(void *ctx); + +extern int _copyinstr(const user_addr_t user_addr, char *kernel_addr, vm_size_t max, vm_size_t *actual); +extern int copyout_kern(const char *kernel_addr, user_addr_t user_addr, vm_size_t nbytes); +extern int copyin_kern(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes); + +extern void bcopy_phys(addr64_t from, addr64_t to, vm_size_t nbytes); + +extern void dcache_incoherent_io_flush64(addr64_t pa, unsigned int count, unsigned int remaining, unsigned int *res); +extern void dcache_incoherent_io_store64(addr64_t pa, unsigned int count, unsigned int remaining, unsigned int *res); + +#if defined(__arm__) +extern void copy_debug_state(arm_debug_state_t *src, arm_debug_state_t *target, __unused boolean_t all); +#elif defined(__arm64__) +extern void copy_legacy_debug_state(arm_legacy_debug_state_t *src, arm_legacy_debug_state_t *target, __unused boolean_t all); +extern void copy_debug_state32(arm_debug_state32_t *src, arm_debug_state32_t *target, __unused boolean_t all); +extern void copy_debug_state64(arm_debug_state64_t *src, arm_debug_state64_t *target, __unused boolean_t all); + +extern boolean_t debug_legacy_state_is_valid(arm_legacy_debug_state_t *ds); +extern boolean_t debug_state_is_valid32(arm_debug_state32_t *ds); +extern boolean_t debug_state_is_valid64(arm_debug_state64_t *ds); + +extern int copyio_check_user_addr(user_addr_t user_addr, vm_size_t nbytes); +extern int _emulate_swp(user_addr_t addr, uint32_t newval, uint32_t *oldval); +extern int _emulate_swpb(user_addr_t addr, uint8_t newval, uint32_t *oldval); + +/* Top-Byte-Ignore */ +extern boolean_t user_tbi; +#define TBI_MASK 0xff00000000000000 +#define user_tbi_enabled() (user_tbi) +#define tbi_clear(addr) ((addr) & ~(TBI_MASK)) + +#else +#error Unknown architecture. +#endif + +#endif /* _ARM_MISC_PROTOS_H_ */ diff --git a/osfmk/arm/model_dep.c b/osfmk/arm/model_dep.c new file mode 100644 index 000000000..ab930d549 --- /dev/null +++ b/osfmk/arm/model_dep.c @@ -0,0 +1,868 @@ +/* + * Copyright (c) 2007-2013 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include <debug.h> +#include <mach_kdp.h> + +#include <kern/thread.h> +#include <machine/pmap.h> +#include <device/device_types.h> + +#include <mach/vm_param.h> +#include <mach/clock_types.h> +#include <mach/machine.h> +#include <mach/kmod.h> +#include <pexpert/boot.h> +#include <pexpert/pexpert.h> + +#include <kern/misc_protos.h> +#include <kern/startup.h> +#include <kern/clock.h> +#include <kern/debug.h> +#include <kern/processor.h> +#include <kdp/kdp_core.h> +#if ALTERNATE_DEBUGGER +#include <arm64/alternate_debugger.h> +#endif +#include <machine/atomic.h> +#include <machine/trap.h> +#include <kern/spl.h> +#include <pexpert/pexpert.h> +#include <kdp/kdp_callout.h> +#include <kdp/kdp_dyld.h> +#include <kdp/kdp_internal.h> +#include <uuid/uuid.h> +#include <sys/time.h> + +#include <IOKit/IOPlatformExpert.h> + +#include <mach/vm_prot.h> +#include <vm/vm_map.h> +#include <vm/pmap.h> +#include <vm/vm_shared_region.h> +#include <mach/time_value.h> +#include <machine/machparam.h> /* for btop */ + +#include <console/video_console.h> +#include <arm/cpu_data.h> +#include <arm/cpu_data_internal.h> +#include <arm/cpu_internal.h> +#include <arm/misc_protos.h> +#include <libkern/OSKextLibPrivate.h> +#include <vm/vm_kern.h> +#include <kern/kern_cdata.h> + +#if MACH_KDP +void kdp_trap(unsigned int, struct arm_saved_state *); +#endif + +extern kern_return_t do_stackshot(void *); +extern void kdp_snapshot_preflight(int pid, void *tracebuf, + uint32_t tracebuf_size, uint32_t flags, + kcdata_descriptor_t data_p, + boolean_t enable_faulting); +extern int kdp_stack_snapshot_bytes_traced(void); + +/* + * Increment the PANICLOG_VERSION if you change the format of the panic + * log in any way. + */ +#define PANICLOG_VERSION 8 +static struct kcdata_descriptor kc_panic_data; + +extern char firmware_version[]; +extern volatile uint32_t debug_enabled; +extern unsigned int not_in_kdp; + +extern int copyinframe(vm_address_t fp, uint32_t * frame); +extern void kdp_callouts(kdp_event_t event); + +/* #include <sys/proc.h> */ +#define MAXCOMLEN 16 +extern int proc_pid(void *p); +extern void proc_name_kdp(task_t, char *, int); + +extern const char version[]; +extern char osversion[]; +extern uint8_t gPlatformECID[8]; +extern uint32_t gPlatformMemoryID; + +extern uint64_t last_hwaccess_thread; + +/*Choosing the size for gTargetTypeBuffer as 8 and size for gModelTypeBuffer as 32 + since the target name and model name typically doesn't exceed this size */ +extern char gTargetTypeBuffer[8]; +extern char gModelTypeBuffer[32]; + +decl_simple_lock_data(extern,clock_lock) +extern struct timeval gIOLastSleepTime; +extern struct timeval gIOLastWakeTime; +extern boolean_t is_clock_configured; +extern uuid_t kernelcache_uuid; + +/* Definitions for frame pointers */ +#define FP_ALIGNMENT_MASK ((uint32_t)(0x3)) +#define FP_LR_OFFSET ((uint32_t)4) +#define FP_LR_OFFSET64 ((uint32_t)8) +#define FP_MAX_NUM_TO_EVALUATE (50) + +/* Timeout (in nanoseconds) for all processors responding to debug crosscall */ +#define DEBUG_ACK_TIMEOUT ((uint64_t) 10000000) + +/* Forward functions definitions */ +void panic_display_times(void) ; +void panic_print_symbol_name(vm_address_t search); + + +/* Global variables */ +static uint32_t panic_bt_depth; +boolean_t PanicInfoSaved = FALSE; +boolean_t force_immediate_debug_halt = FALSE; +unsigned int debug_ack_timeout_count = 0; +volatile unsigned int debugger_sync = 0; +volatile unsigned int mp_kdp_trap = 0; /* CPUs signalled by the debug CPU will spin on this */ +unsigned int DebugContextCount = 0; + +// Convenient macros to easily validate one or more pointers if +// they have defined types +#define VALIDATE_PTR(ptr) \ + validate_ptr((vm_offset_t)(ptr), sizeof(*(ptr)), #ptr) + +#define VALIDATE_PTR_2(ptr0, ptr1) \ + VALIDATE_PTR(ptr0) && VALIDATE_PTR(ptr1) + +#define VALIDATE_PTR_3(ptr0, ptr1, ptr2) \ + VALIDATE_PTR_2(ptr0, ptr1) && VALIDATE_PTR(ptr2) + +#define VALIDATE_PTR_4(ptr0, ptr1, ptr2, ptr3) \ + VALIDATE_PTR_2(ptr0, ptr1) && VALIDATE_PTR_2(ptr2, ptr3) + +#define GET_MACRO(_1,_2,_3,_4,NAME,...) NAME + +#define VALIDATE_PTR_LIST(...) GET_MACRO(__VA_ARGS__, VALIDATE_PTR_4, VALIDATE_PTR_3, VALIDATE_PTR_2, VALIDATE_PTR)(__VA_ARGS__) + +/* + * Evaluate if a pointer is valid + * Print a message if pointer is invalid + */ +static boolean_t validate_ptr( + vm_offset_t ptr, vm_size_t size, const char * ptr_name) +{ + if (ptr) { + if (ml_validate_nofault(ptr, size)) { + return TRUE; + } else { + paniclog_append_noflush("Invalid %s pointer: %p size: %d\n", + ptr_name, (void *)ptr, (int)size); + return FALSE; + } + } else { + paniclog_append_noflush("NULL %s pointer\n", ptr_name); + return FALSE; + } +} + +/* + * Backtrace a single frame. + */ +static void +print_one_backtrace(pmap_t pmap, vm_offset_t topfp, const char *cur_marker, + boolean_t is_64_bit) +{ + int i = 0; + addr64_t lr; + addr64_t fp; + addr64_t fp_for_ppn; + ppnum_t ppn; + boolean_t dump_kernel_stack; + + fp = topfp; + fp_for_ppn = 0; + ppn = (ppnum_t)NULL; + + if (fp >= VM_MIN_KERNEL_ADDRESS) + dump_kernel_stack = TRUE; + else + dump_kernel_stack = FALSE; + + do { + if ((fp == 0) || ((fp & FP_ALIGNMENT_MASK) != 0)) + break; + if (dump_kernel_stack && ((fp < VM_MIN_KERNEL_ADDRESS) || (fp > VM_MAX_KERNEL_ADDRESS))) + break; + if ((!dump_kernel_stack) && (fp >=VM_MIN_KERNEL_ADDRESS)) + break; + + /* + * Check to see if current address will result in a different + * ppn than previously computed (to avoid recomputation) via + * (addr) ^ fp_for_ppn) >> PAGE_SHIFT) + */ + if ((((fp + FP_LR_OFFSET) ^ fp_for_ppn) >> PAGE_SHIFT) != 0x0U) { + ppn = pmap_find_phys(pmap, fp + FP_LR_OFFSET); + fp_for_ppn = fp + (is_64_bit ? FP_LR_OFFSET64 : FP_LR_OFFSET); + } + if (ppn != (ppnum_t)NULL) { + if (is_64_bit) { + lr = ml_phys_read_double_64(((((vm_offset_t)ppn) << PAGE_SHIFT)) | ((fp + FP_LR_OFFSET64) & PAGE_MASK)); + } else { + lr = ml_phys_read_word(((((vm_offset_t)ppn) << PAGE_SHIFT)) | ((fp + FP_LR_OFFSET) & PAGE_MASK)); + } + } else { + if (is_64_bit) { + paniclog_append_noflush("%s\t Could not read LR from frame at 0x%016llx\n", cur_marker, fp + FP_LR_OFFSET64); + } else { + paniclog_append_noflush("%s\t Could not read LR from frame at 0x%08x\n", cur_marker, (uint32_t)(fp + FP_LR_OFFSET)); + } + break; + } + if (((fp ^ fp_for_ppn) >> PAGE_SHIFT) != 0x0U) { + ppn = pmap_find_phys(pmap, fp); + fp_for_ppn = fp; + } + if (ppn != (ppnum_t)NULL) { + if (is_64_bit) { + fp = ml_phys_read_double_64(((((vm_offset_t)ppn) << PAGE_SHIFT)) | (fp & PAGE_MASK)); + } else { + fp = ml_phys_read_word(((((vm_offset_t)ppn) << PAGE_SHIFT)) | (fp & PAGE_MASK)); + } + } else { + if (is_64_bit) { + paniclog_append_noflush("%s\t Could not read FP from frame at 0x%016llx\n", cur_marker, fp); + } else { + paniclog_append_noflush("%s\t Could not read FP from frame at 0x%08x\n", cur_marker, (uint32_t)fp); + } + break; + } + + if (lr) { + if (is_64_bit) { + paniclog_append_noflush("%s\t lr: 0x%016llx fp: 0x%016llx\n", cur_marker, lr, fp); + } else { + paniclog_append_noflush("%s\t lr: 0x%08x fp: 0x%08x\n", cur_marker, (uint32_t)lr, (uint32_t)fp); + } + } + } while ((++i < FP_MAX_NUM_TO_EVALUATE) && (fp != topfp)); +} + +#define SANE_TASK_LIMIT 256 +#define TOP_RUNNABLE_LIMIT 5 +#define PANICLOG_UUID_BUF_SIZE 256 + +extern void panic_print_vnodes(void); + +static void +do_print_all_backtraces( + const char *message) +{ + int logversion = PANICLOG_VERSION; + thread_t cur_thread = current_thread(); + uintptr_t cur_fp; + task_t task; + int i; + size_t index; + int print_vnodes = 0; + const char *nohilite_thread_marker="\t"; + + /* end_marker_bytes set to 200 for printing END marker + stackshot summary info always */ + int bytes_traced = 0, bytes_remaining = 0, end_marker_bytes = 200; + uint64_t bytes_used = 0ULL; + int err = 0; + char *stackshot_begin_loc = NULL; + +#if defined(__arm__) + __asm__ volatile("mov %0, r7":"=r"(cur_fp)); +#elif defined(__arm64__) + __asm__ volatile("add %0, xzr, fp":"=r"(cur_fp)); +#else +#error Unknown architecture. +#endif + if (panic_bt_depth != 0) + return; + panic_bt_depth++; + + /* Truncate panic string to 1200 bytes -- WDT log can be ~1100 bytes */ + paniclog_append_noflush("Debugger message: %.1200s\n", message); + if (debug_enabled) { + paniclog_append_noflush("Device: %s\n", + ('\0' != gTargetTypeBuffer[0]) ? gTargetTypeBuffer : "Not set yet"); + paniclog_append_noflush("Hardware Model: %s\n", + ('\0' != gModelTypeBuffer[0]) ? gModelTypeBuffer:"Not set yet"); + paniclog_append_noflush("ECID: %02X%02X%02X%02X%02X%02X%02X%02X\n", gPlatformECID[7], + gPlatformECID[6], gPlatformECID[5], gPlatformECID[4], gPlatformECID[3], + gPlatformECID[2], gPlatformECID[1], gPlatformECID[0]); + if (last_hwaccess_thread) { + paniclog_append_noflush("AppleHWAccess Thread: 0x%llx\n", last_hwaccess_thread); + } + } + paniclog_append_noflush("Memory ID: 0x%x\n", gPlatformMemoryID); + paniclog_append_noflush("OS version: %.256s\n", + ('\0' != osversion[0]) ? osversion : "Not set yet"); + paniclog_append_noflush("Kernel version: %.512s\n", version); + paniclog_append_noflush("KernelCache UUID: "); + for (index = 0; index < sizeof(uuid_t); index++) { + paniclog_append_noflush("%02X", kernelcache_uuid[index]); + } + paniclog_append_noflush("\n"); + + paniclog_append_noflush("iBoot version: %.128s\n", firmware_version); + paniclog_append_noflush("secure boot?: %s\n", debug_enabled ? "NO": "YES"); + paniclog_append_noflush("Paniclog version: %d\n", logversion); + + panic_display_kernel_aslr(); + panic_display_times(); + panic_display_zprint(); +#if CONFIG_ZLEAKS + panic_display_ztrace(); +#endif /* CONFIG_ZLEAKS */ +#if CONFIG_ECC_LOGGING + panic_display_ecc_errors(); +#endif /* CONFIG_ECC_LOGGING */ + + // Just print threads with high CPU usage for WDT timeouts + if (strncmp(message, "WDT timeout", 11) == 0) { + thread_t top_runnable[5] = {0}; + thread_t thread; + int total_cpu_usage = 0; + + print_vnodes = 1; + + + for (thread = (thread_t)queue_first(&threads); + VALIDATE_PTR(thread) && !queue_end(&threads, (queue_entry_t)thread); + thread = (thread_t)queue_next(&thread->threads)) { + + total_cpu_usage += thread->cpu_usage; + + // Look for the 5 runnable threads with highest priority + if (thread->state & TH_RUN) { + int k; + thread_t comparison_thread = thread; + + for (k = 0; k < TOP_RUNNABLE_LIMIT; k++) { + if (top_runnable[k] == 0) { + top_runnable[k] = comparison_thread; + break; + } else if (comparison_thread->sched_pri > top_runnable[k]->sched_pri) { + thread_t temp = top_runnable[k]; + top_runnable[k] = comparison_thread; + comparison_thread = temp; + } // if comparison thread has higher priority than previously saved thread + } // loop through highest priority runnable threads + } // Check if thread is runnable + } // Loop through all threads + + // Print the relevant info for each thread identified + paniclog_append_noflush("Total cpu_usage: %d\n", total_cpu_usage); + paniclog_append_noflush("Thread task pri cpu_usage\n"); + + for (i = 0; i < TOP_RUNNABLE_LIMIT; i++) { + + if (top_runnable[i] && VALIDATE_PTR(top_runnable[i]->task) && + validate_ptr((vm_offset_t)top_runnable[i]->task->bsd_info, 1, "bsd_info")) { + + char name[MAXCOMLEN + 1]; + proc_name_kdp(top_runnable[i]->task, name, sizeof(name)); + paniclog_append_noflush("%p %s %d %d\n", + top_runnable[i], name, top_runnable[i]->sched_pri, top_runnable[i]->cpu_usage); + } + } // Loop through highest priority runnable threads + paniclog_append_noflush("\n"); + } // Check if message is "WDT timeout" + + // print current task info + if (VALIDATE_PTR_LIST(cur_thread, cur_thread->task)) { + + task = cur_thread->task; + + if (VALIDATE_PTR_LIST(task->map, task->map->pmap)) { + paniclog_append_noflush("Panicked task %p: %d pages, %d threads: ", + task, task->map->pmap->stats.resident_count, task->thread_count); + } else { + paniclog_append_noflush("Panicked task %p: %d threads: ", + task, task->thread_count); + } + + if (validate_ptr((vm_offset_t)task->bsd_info, 1, "bsd_info")) { + char name[MAXCOMLEN + 1]; + int pid = proc_pid(task->bsd_info); + proc_name_kdp(task, name, sizeof(name)); + paniclog_append_noflush("pid %d: %s", pid, name); + } else { + paniclog_append_noflush("unknown task"); + } + + paniclog_append_noflush("\n"); + } + + if (cur_fp < VM_MAX_KERNEL_ADDRESS) { + paniclog_append_noflush("Panicked thread: %p, backtrace: 0x%llx, tid: %llu\n", + cur_thread, (addr64_t)cur_fp, thread_tid(cur_thread)); +#if __LP64__ + print_one_backtrace(kernel_pmap, cur_fp, nohilite_thread_marker, TRUE); +#else + print_one_backtrace(kernel_pmap, cur_fp, nohilite_thread_marker, FALSE); +#endif + } else { + paniclog_append_noflush("Could not print panicked thread backtrace:" + "frame pointer outside kernel vm.\n"); + } + + paniclog_append_noflush("\n"); + panic_info->eph_panic_log_len = PE_get_offset_into_panic_region(debug_buf_ptr) - panic_info->eph_panic_log_offset; + + if (debug_ack_timeout_count) { + panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_FAILED_DEBUGGERSYNC; + panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr); + paniclog_append_noflush("!! debugger synchronization failed, no stackshot !!\n"); + } else if (stackshot_active()) { + panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_FAILED_NESTED; + panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr); + paniclog_append_noflush("!! panicked during stackshot, skipping panic stackshot !!\n"); + } else { + /* Align the stackshot buffer to an 8-byte address (especially important for armv7k devices) */ + debug_buf_ptr += (8 - ((uintptr_t)debug_buf_ptr % 8)); + stackshot_begin_loc = debug_buf_ptr; + + bytes_remaining = debug_buf_size - (unsigned int)((uintptr_t)stackshot_begin_loc - (uintptr_t)debug_buf_base); + err = kcdata_memory_static_init(&kc_panic_data, (mach_vm_address_t)debug_buf_ptr, + KCDATA_BUFFER_BEGIN_STACKSHOT, bytes_remaining - end_marker_bytes, + KCFLAG_USE_MEMCOPY); + if (err == KERN_SUCCESS) { + kdp_snapshot_preflight(-1, stackshot_begin_loc, bytes_remaining - end_marker_bytes, + (STACKSHOT_GET_GLOBAL_MEM_STATS | STACKSHOT_SAVE_LOADINFO | STACKSHOT_KCDATA_FORMAT | + STACKSHOT_ENABLE_BT_FAULTING | STACKSHOT_ENABLE_UUID_FAULTING | STACKSHOT_FROM_PANIC | + STACKSHOT_NO_IO_STATS | STACKSHOT_THREAD_WAITINFO), &kc_panic_data, 0); + err = do_stackshot(NULL); + bytes_traced = kdp_stack_snapshot_bytes_traced(); + if (bytes_traced > 0 && !err) { + debug_buf_ptr += bytes_traced; + panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_SUCCEEDED; + panic_info->eph_stackshot_offset = PE_get_offset_into_panic_region(stackshot_begin_loc); + panic_info->eph_stackshot_len = bytes_traced; + + panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr); + paniclog_append_noflush("\n** Stackshot Succeeded ** Bytes Traced %d **\n", bytes_traced); + } else { + bytes_used = kcdata_memory_get_used_bytes(&kc_panic_data); + if (bytes_used > 0) { + /* Zero out the stackshot data */ + bzero(stackshot_begin_loc, bytes_used); + panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_FAILED_INCOMPLETE; + + panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr); + paniclog_append_noflush("\n** Stackshot Incomplete ** Bytes Filled %llu **\n", bytes_used); + } else { + bzero(stackshot_begin_loc, bytes_used); + panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_FAILED_ERROR; + + panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr); + paniclog_append_noflush("\n!! Stackshot Failed !! Bytes Traced %d, err %d\n", bytes_traced, err); + } + } + } else { + panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_STACKSHOT_FAILED_ERROR; + panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr); + paniclog_append_noflush("\n!! Stackshot Failed !!\nkcdata_memory_static_init returned %d", err); + } + } + + assert(panic_info->eph_other_log_offset != 0); + + if (print_vnodes != 0) + panic_print_vnodes(); + + panic_bt_depth--; +} + +/* + * Entry to print_all_backtraces is serialized by the debugger lock + */ +static void +print_all_backtraces(const char *message) +{ + unsigned int initial_not_in_kdp = not_in_kdp; + + cpu_data_t * cpu_data_ptr = getCpuDatap(); + + assert(cpu_data_ptr->PAB_active == FALSE); + cpu_data_ptr->PAB_active = TRUE; + + /* + * Because print all backtraces uses the pmap routines, it needs to + * avoid taking pmap locks. Right now, this is conditionalized on + * not_in_kdp. + */ + not_in_kdp = 0; + do_print_all_backtraces(message); + + not_in_kdp = initial_not_in_kdp; + + cpu_data_ptr->PAB_active = FALSE; +} + +void +panic_display_times() +{ + if (kdp_clock_is_locked()) { + paniclog_append_noflush("Warning: clock is locked. Can't get time\n"); + return; + } + + if ((is_clock_configured) && (simple_lock_try(&clock_lock))) { + clock_sec_t secs, boot_secs; + clock_usec_t usecs, boot_usecs; + + simple_unlock(&clock_lock); + + clock_get_calendar_microtime(&secs, &usecs); + clock_get_boottime_microtime(&boot_secs, &boot_usecs); + + paniclog_append_noflush("Epoch Time: sec usec\n"); + paniclog_append_noflush(" Boot : 0x%08x 0x%08x\n", (unsigned int)boot_secs, (unsigned int)boot_usecs); + paniclog_append_noflush(" Sleep : 0x%08x 0x%08x\n", (unsigned int)gIOLastSleepTime.tv_sec, (unsigned int)gIOLastSleepTime.tv_usec); + paniclog_append_noflush(" Wake : 0x%08x 0x%08x\n", (unsigned int)gIOLastWakeTime.tv_sec, (unsigned int)gIOLastWakeTime.tv_usec); + paniclog_append_noflush(" Calendar: 0x%08x 0x%08x\n\n", (unsigned int)secs, (unsigned int)usecs); + } +} + +void panic_print_symbol_name(vm_address_t search) +{ +#pragma unused(search) + // empty stub. Really only used on x86_64. + return; +} + +void +SavePanicInfo( + const char *message, __unused uint64_t panic_options) +{ + + /* This should be initialized by the time we get here */ + assert(panic_info->eph_panic_log_offset != 0); + + if (panic_options & DEBUGGER_OPTION_PANICLOGANDREBOOT) { + panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_BUTTON_RESET_PANIC; + } + + if (panic_options & DEBUGGER_OPTION_COPROC_INITIATED_PANIC) { + panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_COPROC_INITIATED_PANIC; + } + + /* + * On newer targets, panic data is stored directly into the iBoot panic region. + * If we re-enter SavePanicInfo (e.g. on a double panic) on such a target, update the + * panic CRC so that iBoot can hopefully find *something* useful in the panic region. + */ + if (PanicInfoSaved && (debug_buf_base >= (char*)gPanicBase) && (debug_buf_base < (char*)gPanicBase + gPanicSize)) { + unsigned int pi_size = (unsigned int)(debug_buf_ptr - gPanicBase); + PE_save_buffer_to_vram((unsigned char*)gPanicBase, &pi_size); + PE_sync_panic_buffers(); // extra precaution; panic path likely isn't reliable if we're here + } + + if (PanicInfoSaved || (debug_buf_size == 0)) + return; + + PanicInfoSaved = TRUE; + + print_all_backtraces(message); + + assert(panic_info->eph_panic_log_len != 0); + panic_info->eph_other_log_len = PE_get_offset_into_panic_region(debug_buf_ptr) - panic_info->eph_other_log_offset; + + PEHaltRestart(kPEPanicSync); + + /* + * Notifies registered IOPlatformPanicAction callbacks + * (which includes one to disable the memcache) and flushes + * the buffer contents from the cache + */ + paniclog_flush(); +} + +void +paniclog_flush() +{ + unsigned int panicbuf_length = 0; + + panicbuf_length = (unsigned int)(debug_buf_ptr - gPanicBase); + if (!panicbuf_length) + return; + + /* + * Updates the log length of the last part of the panic log. + */ + panic_info->eph_other_log_len = PE_get_offset_into_panic_region(debug_buf_ptr) - panic_info->eph_other_log_offset; + + /* + * Updates the metadata at the beginning of the panic buffer, + * updates the CRC. + */ + PE_save_buffer_to_vram((unsigned char *)gPanicBase, &panicbuf_length); + + /* + * This is currently unused by platform KEXTs on embedded but is + * kept for compatibility with the published IOKit interfaces. + */ + PESavePanicInfo((unsigned char *)gPanicBase, panicbuf_length); + + PE_sync_panic_buffers(); +} + +/* + * @function DebuggerXCallEnter + * + * @abstract IPI other cores so this core can run in a single-threaded context. + * + * @discussion This function should be called with the debugger lock held. It + * signals the other cores to go into a busy loop so this core can run in a + * single-threaded context and inspect kernel memory. + * + * @param proceed_on_sync_failure If true, then go ahead and try to debug even + * if we can't synch with the other cores. This is inherently unsafe and should + * only be used if the kernel is going down in flames anyway. + * + * @result returns KERN_OPERATION_TIMED_OUT if synchronization times out and + * proceed_on_sync_failure is false. + */ +kern_return_t +DebuggerXCallEnter( + boolean_t proceed_on_sync_failure) +{ + uint64_t max_mabs_time, current_mabs_time; + int cpu; + int max_cpu; + cpu_data_t *target_cpu_datap; + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + /* Check for nested debugger entry. */ + cpu_data_ptr->debugger_active++; + if (cpu_data_ptr->debugger_active != 1) + return KERN_SUCCESS; + + /* + * If debugger_sync is not 0, someone responded excessively late to the last + * debug request (we zero the sync variable in the return function). Zero it + * again here. This should prevent us from getting out of sync (heh) and + * timing out on every entry to the debugger if we timeout once. + */ + + debugger_sync = 0; + mp_kdp_trap = 1; + + /* + * We need a barrier here to ensure CPUs see mp_kdp_trap and spin when responding + * to the signal. + */ + __builtin_arm_dmb(DMB_ISH); + + /* + * Try to signal all CPUs (except ourselves, of course). Use debugger_sync to + * synchronize with every CPU that we appeared to signal successfully (cpu_signal + * is not synchronous). + */ + bool cpu_signal_failed = false; + max_cpu = ml_get_max_cpu_number(); + + boolean_t immediate_halt = FALSE; + if (proceed_on_sync_failure && force_immediate_debug_halt) + immediate_halt = TRUE; + + if (!immediate_halt) { + for (cpu=0; cpu <= max_cpu; cpu++) { + target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr; + + if ((target_cpu_datap == NULL) || (target_cpu_datap == cpu_data_ptr)) + continue; + + if(KERN_SUCCESS == cpu_signal(target_cpu_datap, SIGPdebug, (void *)NULL, NULL)) { + (void)hw_atomic_add(&debugger_sync, 1); + } else { + cpu_signal_failed = true; + kprintf("cpu_signal failed in DebuggerXCallEnter\n"); + } + } + + nanoseconds_to_absolutetime(DEBUG_ACK_TIMEOUT, &max_mabs_time); + current_mabs_time = mach_absolute_time(); + max_mabs_time += current_mabs_time; + assert(max_mabs_time > current_mabs_time); + + /* + * Wait for DEBUG_ACK_TIMEOUT ns for a response from everyone we IPI'd. If we + * timeout, that is simply too bad; we don't have a true NMI, and one CPU may be + * uninterruptibly spinning on someone else. The best we can hope for is that + * all other CPUs have either responded or are spinning in a context that is + * debugger safe. + */ + while ((debugger_sync != 0) && (current_mabs_time < max_mabs_time)) + current_mabs_time = mach_absolute_time(); + + } + + if (cpu_signal_failed && !proceed_on_sync_failure) { + DebuggerXCallReturn(); + return KERN_FAILURE; + } else if (immediate_halt || (current_mabs_time >= max_mabs_time)) { + /* + * For the moment, we're aiming for a timeout that the user shouldn't notice, + * but will be sufficient to let the other core respond. + */ + __builtin_arm_dmb(DMB_ISH); + for (cpu=0; cpu <= max_cpu; cpu++) { + target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr; + + if ((target_cpu_datap == NULL) || (target_cpu_datap == cpu_data_ptr)) + continue; + if (!(target_cpu_datap->cpu_signal & SIGPdebug) && !immediate_halt) + continue; + if (proceed_on_sync_failure) { + paniclog_append_noflush("Attempting to forcibly halt cpu %d\n", cpu); + dbgwrap_status_t halt_status = ml_dbgwrap_halt_cpu(cpu, 0); + if (halt_status < 0) + paniclog_append_noflush("Unable to halt cpu %d: %d\n", cpu, halt_status); + else { + if (halt_status > 0) + paniclog_append_noflush("cpu %d halted with warning %d\n", cpu, halt_status); + target_cpu_datap->halt_status = CPU_HALTED; + } + } else + kprintf("Debugger synch pending on cpu %d\n", cpu); + } + if (proceed_on_sync_failure) { + for (cpu = 0; cpu <= max_cpu; cpu++) { + target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr; + + if ((target_cpu_datap == NULL) || (target_cpu_datap == cpu_data_ptr) || + (target_cpu_datap->halt_status == CPU_NOT_HALTED)) + continue; + dbgwrap_status_t halt_status = ml_dbgwrap_halt_cpu_with_state(cpu, + NSEC_PER_SEC, &target_cpu_datap->halt_state); + if ((halt_status < 0) || (halt_status == DBGWRAP_WARN_CPU_OFFLINE)) + paniclog_append_noflush("Unable to obtain state for cpu %d: %d\n", cpu, halt_status); + else + target_cpu_datap->halt_status = CPU_HALTED_WITH_STATE; + } + if (immediate_halt) + paniclog_append_noflush("Immediate halt requested on all cores\n"); + else + paniclog_append_noflush("Debugger synchronization timed out; waited %llu nanoseconds\n", DEBUG_ACK_TIMEOUT); + debug_ack_timeout_count++; + return KERN_SUCCESS; + } else { + DebuggerXCallReturn(); + return KERN_OPERATION_TIMED_OUT; + } + } else { + return KERN_SUCCESS; + } +} + +/* + * @function DebuggerXCallReturn + * + * @abstract Resume normal multicore operation after DebuggerXCallEnter() + * + * @discussion This function should be called with debugger lock held. + */ +void +DebuggerXCallReturn( + void) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + cpu_data_ptr->debugger_active--; + if (cpu_data_ptr->debugger_active != 0) + return; + + mp_kdp_trap = 0; + debugger_sync = 0; + + /* Do we need a barrier here? */ + __builtin_arm_dmb(DMB_ISH); +} + +void +DebuggerXCall( + void *ctx) +{ + boolean_t save_context = FALSE; + vm_offset_t kstackptr = 0; + arm_saved_state_t *regs = (arm_saved_state_t *) ctx; + + if (regs != NULL) { +#if defined(__arm64__) + save_context = PSR64_IS_KERNEL(get_saved_state_cpsr(regs)); +#else + save_context = PSR_IS_KERNEL(regs->cpsr); +#endif + } + + kstackptr = current_thread()->machine.kstackptr; + arm_saved_state_t *state = (arm_saved_state_t *)kstackptr; + + if (save_context) { + /* Save the interrupted context before acknowledging the signal */ + *state = *regs; + } else if (regs) { + /* zero old state so machine_trace_thread knows not to backtrace it */ + set_saved_state_fp(state, 0); + set_saved_state_pc(state, 0); + set_saved_state_lr(state, 0); + set_saved_state_sp(state, 0); + } + + (void)hw_atomic_sub(&debugger_sync, 1); + __builtin_arm_dmb(DMB_ISH); + while (mp_kdp_trap); + + /* Any cleanup for our pushed context should go here */ +} + + +void +DebuggerCall( + unsigned int reason, + void *ctx) +{ +#if !MACH_KDP +#pragma unused(reason,ctx) +#endif /* !MACH_KDP */ + +#if ALTERNATE_DEBUGGER + alternate_debugger_enter(); +#endif + +#if MACH_KDP + kdp_trap(reason, (struct arm_saved_state *)ctx); +#else + /* TODO: decide what to do if no debugger config */ +#endif +} + + diff --git a/osfmk/arm/monotonic.h b/osfmk/arm/monotonic.h new file mode 100644 index 000000000..9b638bfbe --- /dev/null +++ b/osfmk/arm/monotonic.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef ARM_MONOTONIC_H +#define ARM_MONOTONIC_H + +#define MT_NDEVS 0 + +#define MT_CORE_NFIXED 1 +#define MT_CORE_CYCLES 0 +#define MT_CORE_MAXVAL UINT32_MAX + +#endif /* !defined(ARM_MONOTONIC_H) */ diff --git a/osfmk/arm/monotonic_arm.c b/osfmk/arm/monotonic_arm.c new file mode 100644 index 000000000..a5b071631 --- /dev/null +++ b/osfmk/arm/monotonic_arm.c @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include <arm/monotonic.h> +#include <sys/monotonic.h> + +bool mt_core_supported = false; + +void +mt_init(void) +{ +} + +uint64_t +mt_core_snap(__unused unsigned int ctr) +{ + return 0; +} + +struct mt_cpu * +mt_cur_cpu(void) +{ + return &getCpuDatap()->cpu_monotonic; +} + +const struct monotonic_dev monotonic_devs[0]; diff --git a/osfmk/arm/pal_routines.c b/osfmk/arm/pal_routines.c new file mode 100644 index 000000000..535e3cc80 --- /dev/null +++ b/osfmk/arm/pal_routines.c @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2009 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +/* + * file: pal_routines.c + * Platform Abstraction Layer routines for ARM + */ + + +#include <machine/pal_routines.h> +#include <mach/mach_types.h> +#include <pexpert/arm/protos.h> + +/* Serial routines */ +int +pal_serial_init(void) +{ + return serial_init(); +} + +void +pal_serial_putc_nocr(char c) +{ + serial_putc(c); +} + +void +pal_serial_putc(char c) +{ + serial_putc(c); +} + +int +pal_serial_getc(void) +{ + return serial_getc(); +} diff --git a/osfmk/arm/pal_routines.h b/osfmk/arm/pal_routines.h new file mode 100644 index 000000000..f100f6e99 --- /dev/null +++ b/osfmk/arm/pal_routines.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2009 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _ARM_PAL_ROUTINES_H +#define _ARM_PAL_ROUTINES_H + +#include <stdint.h> + +#if defined(__cplusplus) +extern "C" { +#endif + +#ifdef XNU_KERNEL_PRIVATE + +/* No-op */ +#define pal_dbg_set_task_name( x ) do { } while(0) + +#define pal_ast_check(thread) +#define pal_thread_terminate_self(t) + +/* serial / debug output routines */ +extern int pal_serial_init(void); +extern void pal_serial_putc(char a); +extern void pal_serial_putc_nocr(char a); +extern int pal_serial_getc(void); + +#define panic_display_pal_info() do { } while(0) +#define pal_kernel_announce() do { } while(0) + +#endif /* XNU_KERNEL_PRIVATE */ + +/* Allows us to set a property on the IOResources object. Unused on ARM. */ +static inline void +pal_get_resource_property(const char **property_name, + int *property_value) +{ + *property_name = 0; + (void) property_value; +} + +#if defined(__cplusplus) +} +#endif + +#endif /* _ARM_PAL_ROUTINES_H */ diff --git a/osfmk/arm/pcb.c b/osfmk/arm/pcb.c new file mode 100644 index 000000000..2d06b559e --- /dev/null +++ b/osfmk/arm/pcb.c @@ -0,0 +1,406 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include <debug.h> + +#include <types.h> + +#include <mach/mach_types.h> +#include <mach/thread_status.h> +#include <mach/vm_types.h> + +#include <kern/kern_types.h> +#include <kern/task.h> +#include <kern/thread.h> +#include <kern/misc_protos.h> +#include <kern/mach_param.h> +#include <kern/spl.h> +#include <kern/machine.h> +#include <kern/kalloc.h> +#include <kern/kpc.h> + +#include <arm/proc_reg.h> +#include <arm/cpu_data_internal.h> +#include <arm/misc_protos.h> +#include <arm/cpuid.h> + +#include <vm/vm_map.h> +#include <vm/vm_protos.h> + +#include <sys/kdebug.h> + +extern int debug_task; + +zone_t ads_zone; /* zone for debug_state area */ + +/* + * Routine: consider_machine_collect + * + */ +void +consider_machine_collect(void) +{ + pmap_gc(); +} + +/* + * Routine: consider_machine_adjust + * + */ +void +consider_machine_adjust(void) +{ +} + +/* + * Routine: machine_switch_context + * + */ +thread_t +machine_switch_context( + thread_t old, + thread_continue_t continuation, + thread_t new) +{ + thread_t retval; + cpu_data_t *cpu_data_ptr; + +#define machine_switch_context_kprintf(x...) /* kprintf("machine_switch_con + * text: " x) */ + + cpu_data_ptr = getCpuDatap(); + if (old == new) + panic("machine_switch_context"); + + kpc_off_cpu(old); + + pmap_set_pmap(new->map->pmap, new); + + new->machine.CpuDatap = cpu_data_ptr; + + machine_switch_context_kprintf("old= %x contination = %x new = %x\n", old, continuation, new); + retval = Switch_context(old, continuation, new); + assert(retval != NULL); + + return retval; +} + +/* + * Routine: machine_thread_create + * + */ +kern_return_t +machine_thread_create( + thread_t thread, +#if !__ARM_USER_PROTECT__ + __unused +#endif + task_t task) +{ + +#define machine_thread_create_kprintf(x...) /* kprintf("machine_thread_create: " x) */ + + machine_thread_create_kprintf("thread = %x\n", thread); + + if (current_thread() != thread) { + thread->machine.CpuDatap = (cpu_data_t *)0; + } + thread->machine.preemption_count = 0; + thread->machine.cthread_self = 0; + thread->machine.cthread_data = 0; +#if __ARM_USER_PROTECT__ + { + struct pmap *new_pmap = vm_map_pmap(task->map); + + thread->machine.kptw_ttb = ((unsigned int) kernel_pmap->ttep) | TTBR_SETUP; + thread->machine.asid = new_pmap->asid; + if (new_pmap->tte_index_max == NTTES) { + thread->machine.uptw_ttc = 2; + thread->machine.uptw_ttb = ((unsigned int) new_pmap->ttep) | TTBR_SETUP; + } else { + thread->machine.uptw_ttc = 1; + thread->machine.uptw_ttb = ((unsigned int) new_pmap->ttep ) | TTBR_SETUP; + } + } +#endif + machine_thread_state_initialize(thread); + + return (KERN_SUCCESS); +} + +/* + * Routine: machine_thread_destroy + * + */ +void +machine_thread_destroy( + thread_t thread) +{ + + if (thread->machine.DebugData != NULL) { + if (thread->machine.DebugData == getCpuDatap()->cpu_user_debug) + arm_debug_set(NULL); + zfree(ads_zone, thread->machine.DebugData); + } +} + + +/* + * Routine: machine_thread_init + * + */ +void +machine_thread_init(void) +{ + ads_zone = zinit(sizeof(arm_debug_state_t), + THREAD_CHUNK * (sizeof(arm_debug_state_t)), + THREAD_CHUNK * (sizeof(arm_debug_state_t)), + "arm debug state"); +} + + +/* + * Routine: get_useraddr + * + */ +user_addr_t +get_useraddr() +{ + return (current_thread()->machine.PcbData.pc); +} + +/* + * Routine: machine_stack_detach + * + */ +vm_offset_t +machine_stack_detach( + thread_t thread) +{ + vm_offset_t stack; + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_DETACH), + (uintptr_t)thread_tid(thread), thread->priority, thread->sched_pri, 0, 0); + + stack = thread->kernel_stack; + thread->kernel_stack = 0; + thread->machine.kstackptr = 0; + + return (stack); +} + + +/* + * Routine: machine_stack_attach + * + */ +void +machine_stack_attach( + thread_t thread, + vm_offset_t stack) +{ + struct arm_saved_state *savestate; + +#define machine_stack_attach_kprintf(x...) /* kprintf("machine_stack_attach: " x) */ + + KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_ATTACH), + (uintptr_t)thread_tid(thread), thread->priority, thread->sched_pri, 0, 0); + + thread->kernel_stack = stack; + thread->machine.kstackptr = stack + kernel_stack_size - sizeof(struct thread_kernel_state); + thread_initialize_kernel_state(thread); + savestate = (struct arm_saved_state *) thread->machine.kstackptr; + + savestate->lr = (uint32_t) thread_continue; + savestate->sp = thread->machine.kstackptr; + savestate->r[7] = 0x0UL; + savestate->r[9] = (uint32_t) NULL; + savestate->cpsr = PSR_SVC_MODE | PSR_INTMASK; + machine_stack_attach_kprintf("thread = %x pc = %x, sp = %x\n", thread, savestate->lr, savestate->sp); +} + + +/* + * Routine: machine_stack_handoff + * + */ +void +machine_stack_handoff( + thread_t old, + thread_t new) +{ + vm_offset_t stack; + cpu_data_t *cpu_data_ptr; + + kpc_off_cpu(old); + + stack = machine_stack_detach(old); + cpu_data_ptr = getCpuDatap(); + new->kernel_stack = stack; + new->machine.kstackptr = stack + kernel_stack_size - sizeof(struct thread_kernel_state); + if (stack == old->reserved_stack) { + assert(new->reserved_stack); + old->reserved_stack = new->reserved_stack; + new->reserved_stack = stack; + } + + pmap_set_pmap(new->map->pmap, new); + new->machine.CpuDatap = cpu_data_ptr; + machine_set_current_thread(new); + thread_initialize_kernel_state(new); + + return; +} + + +/* + * Routine: call_continuation + * + */ +void +call_continuation( + thread_continue_t continuation, + void *parameter, + wait_result_t wresult) +{ +#define call_continuation_kprintf(x...) /* kprintf("call_continuation_kprintf: + * " x) */ + + call_continuation_kprintf("thread = %x continuation = %x, stack = %x\n", current_thread(), continuation, current_thread()->machine.kstackptr); + Call_continuation(continuation, parameter, wresult, current_thread()->machine.kstackptr); +} + +void arm_debug_set(arm_debug_state_t *debug_state) +{ + /* If this CPU supports the memory-mapped debug interface, use it, otherwise + * attempt the Extended CP14 interface. The two routines need to be kept in sync, + * functionality-wise. + */ + struct cpu_data *cpu_data_ptr; + arm_debug_info_t *debug_info = arm_debug_info(); + boolean_t intr; + + intr = ml_set_interrupts_enabled(FALSE); + cpu_data_ptr = getCpuDatap(); + + // Set current user debug + cpu_data_ptr->cpu_user_debug = debug_state; + + if (debug_info->memory_mapped_core_debug) { + int i; + uintptr_t debug_map = cpu_data_ptr->cpu_debug_interface_map; + + // unlock debug registers + *(volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGLAR) = ARM_DBG_LOCK_ACCESS_KEY; + + // read DBGPRSR to clear the sticky power-down bit (necessary to access debug registers) + *(volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGPRSR); + + // enable monitor mode (needed to set and use debug registers) + *(volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGDSCR) |= ARM_DBGDSCR_MDBGEN; + + // first turn off all breakpoints/watchpoints + for (i = 0; i < 16; i++) { + ((volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGBCR))[i] = 0; + ((volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGWCR))[i] = 0; + } + + // if (debug_state == NULL) disable monitor mode + if (debug_state == NULL) { + *(volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGDSCR) &= ~ARM_DBGDSCR_MDBGEN; + } else { + for (i = 0; i < 16; i++) { + ((volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGBVR))[i] = debug_state->bvr[i]; + ((volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGBCR))[i] = debug_state->bcr[i]; + ((volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGWVR))[i] = debug_state->wvr[i]; + ((volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGWCR))[i] = debug_state->wcr[i]; + } + } + + // lock debug registers + *(volatile uint32_t *)(debug_map + ARM_DEBUG_OFFSET_DBGLAR) = 0; + + } else if (debug_info->coprocessor_core_debug) { + arm_debug_set_cp14(debug_state); + } + + (void) ml_set_interrupts_enabled(intr); + + return; +} + +/* + * Duplicate one arm_debug_state_t to another. "all" parameter + * is ignored in the case of ARM -- Is this the right assumption? + */ +void +copy_debug_state( + arm_debug_state_t *src, + arm_debug_state_t *target, + __unused boolean_t all) +{ + bcopy(src, target, sizeof(arm_debug_state_t)); +} + +kern_return_t +machine_thread_set_tsd_base( + thread_t thread, + mach_vm_offset_t tsd_base) +{ + + if (thread->task == kernel_task) { + return KERN_INVALID_ARGUMENT; + } + + if (tsd_base & 0x3) { + return KERN_INVALID_ARGUMENT; + } + + if (tsd_base > UINT32_MAX) + tsd_base = 0ULL; + + thread->machine.cthread_self = tsd_base; + + /* For current thread, make the TSD base active immediately */ + if (thread == current_thread()) { + + mp_disable_preemption(); + __asm__ volatile( + "mrc p15, 0, r6, c13, c0, 3\n" + "and r6, r6, #3\n" + "orr r6, r6, %0\n" + "mcr p15, 0, r6, c13, c0, 3\n" + : /* output */ + : "r"((uint32_t)tsd_base) /* input */ + : "r6" /* clobbered register */ + ); + mp_enable_preemption(); + + } + + return KERN_SUCCESS; +} diff --git a/osfmk/arm/pmap.c b/osfmk/arm/pmap.c new file mode 100644 index 000000000..327ac9b9f --- /dev/null +++ b/osfmk/arm/pmap.c @@ -0,0 +1,10555 @@ +/* + * Copyright (c) 2011-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include <string.h> +#include <mach_assert.h> +#include <mach_ldebug.h> + +#include <mach/shared_region.h> +#include <mach/vm_param.h> +#include <mach/vm_prot.h> +#include <mach/vm_map.h> +#include <mach/machine/vm_param.h> +#include <mach/machine/vm_types.h> + +#include <mach/boolean.h> +#include <kern/thread.h> +#include <kern/sched.h> +#include <kern/zalloc.h> +#include <kern/kalloc.h> +#include <kern/ledger.h> +#include <kern/misc_protos.h> +#include <kern/spl.h> +#include <kern/xpr.h> + +#include <vm/pmap.h> +#include <vm/vm_map.h> +#include <vm/vm_kern.h> +#include <vm/vm_protos.h> +#include <vm/vm_object.h> +#include <vm/vm_page.h> +#include <vm/vm_pageout.h> +#include <vm/cpm.h> + +#include <libkern/section_keywords.h> + +#include <machine/atomic.h> +#include <machine/thread.h> +#include <machine/lowglobals.h> + +#include <arm/caches_internal.h> +#include <arm/cpu_data.h> +#include <arm/cpu_data_internal.h> +#include <arm/cpu_capabilities.h> +#include <arm/cpu_number.h> +#include <arm/machine_cpu.h> +#include <arm/misc_protos.h> +#include <arm/trap.h> + +#include <libkern/section_keywords.h> + +#if (__ARM_VMSA__ > 7) +#include <arm64/proc_reg.h> +#include <pexpert/arm64/boot.h> +#if CONFIG_PGTRACE +#include <stdint.h> +#include <arm64/pgtrace.h> +#if CONFIG_PGTRACE_NONKEXT +#include <arm64/pgtrace_decoder.h> +#endif // CONFIG_PGTRACE_NONKEXT +#endif +#endif + +#include <pexpert/device_tree.h> + +#include <san/kasan.h> + +#if DEVELOPMENT || DEBUG +#define PMAP_FOOTPRINT_SUSPENDED(pmap) ((pmap)->footprint_suspended) +#else /* DEVELOPMENT || DEBUG */ +#define PMAP_FOOTPRINT_SUSPENDED(pmap) (FALSE) +#endif /* DEVELOPMENT || DEBUG */ + + + +#if DEVELOPMENT || DEBUG +int panic_on_unsigned_execute = 0; +#endif /* DEVELOPMENT || DEBUG */ + + +/* Virtual memory region for early allocation */ +#if (__ARM_VMSA__ == 7) +#define VREGION1_START (VM_HIGH_KERNEL_WINDOW & ~ARM_TT_L1_PT_OFFMASK) +#else +#define VREGION1_HIGH_WINDOW (PE_EARLY_BOOT_VA) +#define VREGION1_START ((VM_MAX_KERNEL_ADDRESS & CPUWINDOWS_BASE_MASK) - VREGION1_HIGH_WINDOW) +#endif +#define VREGION1_SIZE (trunc_page(VM_MAX_KERNEL_ADDRESS - (VREGION1_START))) + +extern unsigned int not_in_kdp; + +extern vm_offset_t first_avail; + +extern pmap_paddr_t avail_start; +extern pmap_paddr_t avail_end; + +extern vm_offset_t virtual_space_start; /* Next available kernel VA */ +extern vm_offset_t virtual_space_end; /* End of kernel address space */ + +extern int hard_maxproc; + +#if (__ARM_VMSA__ > 7) +/* The number of address bits one TTBR can cover. */ +#define PGTABLE_ADDR_BITS (64ULL - T0SZ_BOOT) + +/* + * The bounds on our TTBRs. These are for sanity checking that + * an address is accessible by a TTBR before we attempt to map it. + */ +#define ARM64_TTBR0_MIN_ADDR (0ULL) +#define ARM64_TTBR0_MAX_ADDR (0ULL + (1ULL << PGTABLE_ADDR_BITS) - 1) +#define ARM64_TTBR1_MIN_ADDR (0ULL - (1ULL << PGTABLE_ADDR_BITS)) +#define ARM64_TTBR1_MAX_ADDR (~0ULL) + +/* The level of the root of a page table. */ +const uint64_t arm64_root_pgtable_level = (3 - ((PGTABLE_ADDR_BITS - 1 - ARM_PGSHIFT) / (ARM_PGSHIFT - TTE_SHIFT))); + +/* The number of entries in the root TT of a page table. */ +const uint64_t arm64_root_pgtable_num_ttes = (2 << ((PGTABLE_ADDR_BITS - 1 - ARM_PGSHIFT) % (ARM_PGSHIFT - TTE_SHIFT))); +#else +const uint64_t arm64_root_pgtable_level = 0; +const uint64_t arm64_root_pgtable_num_ttes = 0; +#endif + +struct pmap kernel_pmap_store MARK_AS_PMAP_DATA; +SECURITY_READ_ONLY_LATE(pmap_t) kernel_pmap = &kernel_pmap_store; + +struct vm_object pmap_object_store __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT))); /* store pt pages */ +vm_object_t pmap_object = &pmap_object_store; + +static struct zone *pmap_zone; /* zone of pmap structures */ + +decl_simple_lock_data(, pmaps_lock MARK_AS_PMAP_DATA) +unsigned int pmap_stamp MARK_AS_PMAP_DATA; +queue_head_t map_pmap_list MARK_AS_PMAP_DATA; + +queue_head_t tt_pmap_list MARK_AS_PMAP_DATA; +unsigned int tt_pmap_count MARK_AS_PMAP_DATA; +unsigned int tt_pmap_max MARK_AS_PMAP_DATA; + +decl_simple_lock_data(, pt_pages_lock MARK_AS_PMAP_DATA) +queue_head_t pt_page_list MARK_AS_PMAP_DATA; /* pt page ptd entries list */ + +decl_simple_lock_data(, pmap_pages_lock MARK_AS_PMAP_DATA) + +typedef struct page_free_entry { + struct page_free_entry *next; +} page_free_entry_t; + +#define PAGE_FREE_ENTRY_NULL ((page_free_entry_t *) 0) + +page_free_entry_t *pmap_pages_reclaim_list MARK_AS_PMAP_DATA; /* Reclaimed pt page list */ +unsigned int pmap_pages_request_count MARK_AS_PMAP_DATA; /* Pending requests to reclaim pt page */ +unsigned long long pmap_pages_request_acum MARK_AS_PMAP_DATA; + + +typedef struct tt_free_entry { + struct tt_free_entry *next; +} tt_free_entry_t; + +#define TT_FREE_ENTRY_NULL ((tt_free_entry_t *) 0) + +tt_free_entry_t *free_page_size_tt_list MARK_AS_PMAP_DATA; +unsigned int free_page_size_tt_count MARK_AS_PMAP_DATA; +unsigned int free_page_size_tt_max MARK_AS_PMAP_DATA; +#define FREE_PAGE_SIZE_TT_MAX 4 +tt_free_entry_t *free_two_page_size_tt_list MARK_AS_PMAP_DATA; +unsigned int free_two_page_size_tt_count MARK_AS_PMAP_DATA; +unsigned int free_two_page_size_tt_max MARK_AS_PMAP_DATA; +#define FREE_TWO_PAGE_SIZE_TT_MAX 4 +tt_free_entry_t *free_tt_list MARK_AS_PMAP_DATA; +unsigned int free_tt_count MARK_AS_PMAP_DATA; +unsigned int free_tt_max MARK_AS_PMAP_DATA; + +#define TT_FREE_ENTRY_NULL ((tt_free_entry_t *) 0) + +boolean_t pmap_gc_allowed MARK_AS_PMAP_DATA = TRUE; +boolean_t pmap_gc_forced MARK_AS_PMAP_DATA = FALSE; +boolean_t pmap_gc_allowed_by_time_throttle = TRUE; + +unsigned int inuse_user_ttepages_count MARK_AS_PMAP_DATA = 0; /* non-root, non-leaf user pagetable pages, in units of PAGE_SIZE */ +unsigned int inuse_user_ptepages_count MARK_AS_PMAP_DATA = 0; /* leaf user pagetable pages, in units of PAGE_SIZE */ +unsigned int inuse_user_tteroot_count MARK_AS_PMAP_DATA = 0; /* root user pagetables, in units of PMAP_ROOT_ALLOC_SIZE */ +unsigned int inuse_kernel_ttepages_count MARK_AS_PMAP_DATA = 0; /* non-root, non-leaf kernel pagetable pages, in units of PAGE_SIZE */ +unsigned int inuse_kernel_ptepages_count MARK_AS_PMAP_DATA = 0; /* leaf kernel pagetable pages, in units of PAGE_SIZE */ +unsigned int inuse_kernel_tteroot_count MARK_AS_PMAP_DATA = 0; /* root kernel pagetables, in units of PMAP_ROOT_ALLOC_SIZE */ +unsigned int inuse_pmap_pages_count = 0; /* debugging */ + +SECURITY_READ_ONLY_LATE(tt_entry_t *) invalid_tte = 0; +SECURITY_READ_ONLY_LATE(pmap_paddr_t) invalid_ttep = 0; + +SECURITY_READ_ONLY_LATE(tt_entry_t *) cpu_tte = 0; /* set by arm_vm_init() - keep out of bss */ +SECURITY_READ_ONLY_LATE(pmap_paddr_t) cpu_ttep = 0; /* set by arm_vm_init() - phys tte addr */ + +#if DEVELOPMENT || DEBUG +int nx_enabled = 1; /* enable no-execute protection */ +int allow_data_exec = 0; /* No apps may execute data */ +int allow_stack_exec = 0; /* No apps may execute from the stack */ +#else /* DEVELOPMENT || DEBUG */ +const int nx_enabled = 1; /* enable no-execute protection */ +const int allow_data_exec = 0; /* No apps may execute data */ +const int allow_stack_exec = 0; /* No apps may execute from the stack */ +#endif /* DEVELOPMENT || DEBUG */ + +/* + * pv_entry_t - structure to track the active mappings for a given page + */ +typedef struct pv_entry { + struct pv_entry *pve_next; /* next alias */ + pt_entry_t *pve_ptep; /* page table entry */ +#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4) +/* For the newer ARMv7k ABI where 64-bit types are 64-bit aligned, but pointers + * are 32-bit: + * Since pt_desc is 64-bit aligned and we cast often from pv_entry to + * pt_desc. + */ +} __attribute__ ((aligned(8))) pv_entry_t; +#else +} pv_entry_t; +#endif + +#define PV_ENTRY_NULL ((pv_entry_t *) 0) + +/* + * PMAP LEDGERS: + * We use the least significant bit of the "pve_next" pointer in a "pv_entry" + * as a marker for pages mapped through an "alternate accounting" mapping. + * These macros set, clear and test for this marker and extract the actual + * value of the "pve_next" pointer. + */ +#define PVE_NEXT_ALTACCT ((uintptr_t) 0x1) +#define PVE_NEXT_SET_ALTACCT(pve_next_p) \ + *(pve_next_p) = (struct pv_entry *) (((uintptr_t) *(pve_next_p)) | \ + PVE_NEXT_ALTACCT) +#define PVE_NEXT_CLR_ALTACCT(pve_next_p) \ + *(pve_next_p) = (struct pv_entry *) (((uintptr_t) *(pve_next_p)) & \ + ~PVE_NEXT_ALTACCT) +#define PVE_NEXT_IS_ALTACCT(pve_next) \ + ((((uintptr_t) (pve_next)) & PVE_NEXT_ALTACCT) ? TRUE : FALSE) +#define PVE_NEXT_PTR(pve_next) \ + ((struct pv_entry *)(((uintptr_t) (pve_next)) & \ + ~PVE_NEXT_ALTACCT)) +#if MACH_ASSERT +static void pmap_check_ledgers(pmap_t pmap); +#else +static inline void pmap_check_ledgers(__unused pmap_t pmap) {} +#endif /* MACH_ASSERT */ + +SECURITY_READ_ONLY_LATE(pv_entry_t **) pv_head_table; /* array of pv entry pointers */ + +pv_entry_t *pv_free_list MARK_AS_PMAP_DATA; +pv_entry_t *pv_kern_free_list MARK_AS_PMAP_DATA; +decl_simple_lock_data(,pv_free_list_lock MARK_AS_PMAP_DATA) +decl_simple_lock_data(,pv_kern_free_list_lock MARK_AS_PMAP_DATA) + +decl_simple_lock_data(,phys_backup_lock) + +/* + * pt_desc - structure to keep info on page assigned to page tables + */ +#if (__ARM_VMSA__ == 7) +#define PT_INDEX_MAX 1 +#else +#if (ARM_PGSHIFT == 14) +#define PT_INDEX_MAX 1 +#else +#define PT_INDEX_MAX 4 +#endif +#endif + +#define PT_DESC_REFCOUNT 0x4000U + +typedef struct pt_desc { + queue_chain_t pt_page; + struct { + unsigned short refcnt; + unsigned short wiredcnt; + } pt_cnt[PT_INDEX_MAX]; + struct pmap *pmap; + struct { + vm_offset_t va; + } pt_map[PT_INDEX_MAX]; +} pt_desc_t; + + +#define PTD_ENTRY_NULL ((pt_desc_t *) 0) + +SECURITY_READ_ONLY_LATE(pt_desc_t *) ptd_root_table; + +pt_desc_t *ptd_free_list MARK_AS_PMAP_DATA = PTD_ENTRY_NULL; +SECURITY_READ_ONLY_LATE(boolean_t) ptd_preboot = TRUE; +unsigned int ptd_free_count MARK_AS_PMAP_DATA = 0; +decl_simple_lock_data(,ptd_free_list_lock MARK_AS_PMAP_DATA) + +/* + * physical page attribute + */ +typedef u_int16_t pp_attr_t; + +#define PP_ATTR_WIMG_MASK 0x003F +#define PP_ATTR_WIMG(x) ((x) & PP_ATTR_WIMG_MASK) + +#define PP_ATTR_REFERENCED 0x0040 +#define PP_ATTR_MODIFIED 0x0080 + +#define PP_ATTR_INTERNAL 0x0100 +#define PP_ATTR_REUSABLE 0x0200 +#define PP_ATTR_ALTACCT 0x0400 +#define PP_ATTR_NOENCRYPT 0x0800 + +#define PP_ATTR_REFFAULT 0x1000 +#define PP_ATTR_MODFAULT 0x2000 + + +SECURITY_READ_ONLY_LATE(pp_attr_t*) pp_attr_table; + + +typedef uint8_t io_attr_t; + +#define IO_ATTR_WIMG_MASK 0x3F +#define IO_ATTR_WIMG(x) ((x) & IO_ATTR_WIMG_MASK) + +SECURITY_READ_ONLY_LATE(io_attr_t*) io_attr_table; + +SECURITY_READ_ONLY_LATE(pmap_paddr_t) vm_first_phys = (pmap_paddr_t) 0; +SECURITY_READ_ONLY_LATE(pmap_paddr_t) vm_last_phys = (pmap_paddr_t) 0; + +SECURITY_READ_ONLY_LATE(pmap_paddr_t) io_rgn_start = 0; +SECURITY_READ_ONLY_LATE(pmap_paddr_t) io_rgn_end = 0; +SECURITY_READ_ONLY_LATE(uint32_t) io_rgn_granule = 0; + +SECURITY_READ_ONLY_LATE(boolean_t) pmap_initialized = FALSE; /* Has pmap_init completed? */ + +SECURITY_READ_ONLY_LATE(uint64_t) pmap_nesting_size_min; +SECURITY_READ_ONLY_LATE(uint64_t) pmap_nesting_size_max; + +SECURITY_READ_ONLY_LATE(vm_map_offset_t) arm_pmap_max_offset_default = 0x0; +#if defined(__arm64__) +SECURITY_READ_ONLY_LATE(vm_map_offset_t) arm64_pmap_max_offset_default = 0x0; +#endif + +/* free address spaces (1 means free) */ +static uint32_t asid_bitmap[MAX_ASID / (sizeof(uint32_t) * NBBY)] MARK_AS_PMAP_DATA; + +#if (__ARM_VMSA__ > 7) +SECURITY_READ_ONLY_LATE(pmap_t) u32_sharedpage_pmap; +#endif + + +#define pa_index(pa) \ + (atop((pa) - vm_first_phys)) + +#define pai_to_pvh(pai) \ + (&pv_head_table[pai]) + +#define pa_valid(x) \ + ((x) >= vm_first_phys && (x) < vm_last_phys) + +/* PTE Define Macros */ + +#define pte_is_wired(pte) \ + (((pte) & ARM_PTE_WIRED_MASK) == ARM_PTE_WIRED) + +#define pte_set_wired(ptep, wired) \ + do { \ + SInt16 *ptd_wiredcnt_ptr; \ + ptd_wiredcnt_ptr = (SInt16 *)&(ptep_get_ptd(ptep)->pt_cnt[ARM_PT_DESC_INDEX(ptep)].wiredcnt); \ + if (wired) { \ + *ptep |= ARM_PTE_WIRED; \ + OSAddAtomic16(1, ptd_wiredcnt_ptr); \ + } else { \ + *ptep &= ~ARM_PTE_WIRED; \ + OSAddAtomic16(-1, ptd_wiredcnt_ptr); \ + } \ + } while(0) + +#define pte_is_ffr(pte) \ + (((pte) & ARM_PTE_WRITEABLE) == ARM_PTE_WRITEABLE) + +#define pte_set_ffr(pte, ffr) \ + do { \ + if (ffr) { \ + pte |= ARM_PTE_WRITEABLE; \ + } else { \ + pte &= ~ARM_PTE_WRITEABLE; \ + } \ + } while(0) + +/* PVE Define Macros */ + +#define pve_next(pve) \ + ((pve)->pve_next) + +#define pve_link_field(pve) \ + (&pve_next(pve)) + +#define pve_link(pp, e) \ + ((pve_next(e) = pve_next(pp)), (pve_next(pp) = (e))) + +#define pve_unlink(pp, e) \ + (pve_next(pp) = pve_next(e)) + +/* bits held in the ptep pointer field */ + +#define pve_get_ptep(pve) \ + ((pve)->pve_ptep) + +#define pve_set_ptep(pve, ptep_new) \ + do { \ + (pve)->pve_ptep = (ptep_new); \ + } while (0) + +/* PTEP Define Macros */ + +#if (__ARM_VMSA__ == 7) + +#define ARM_PT_DESC_INDEX_MASK 0x00000 +#define ARM_PT_DESC_INDEX_SHIFT 0 + + /* + * mask for page descriptor index: 4MB per page table + */ +#define ARM_TT_PT_INDEX_MASK 0xfffU /* mask for page descriptor index: 4MB per page table */ + + /* + * Shift value used for reconstructing the virtual address for a PTE. + */ +#define ARM_TT_PT_ADDR_SHIFT (10U) + +#define ARM_PT_DESC_INDEX(ptep) \ + (((unsigned)(ptep) & ARM_PT_DESC_INDEX_MASK) >> ARM_PT_DESC_INDEX_SHIFT) + +#define ptep_get_ptd(ptep) \ + ((struct pt_desc *)((*((vm_offset_t *)(pai_to_pvh(pa_index((vm_offset_t)(ptep) - gVirtBase + gPhysBase))))) & PVH_LIST_MASK)) + +#define ptep_get_va(ptep) \ + ((((pt_desc_t *) (pvh_list(pai_to_pvh(pa_index((((vm_offset_t)(ptep) & ~0xFFF) - gVirtBase + gPhysBase))))))->pt_map[ARM_PT_DESC_INDEX(ptep)].va)+ ((((unsigned)(ptep)) & ARM_TT_PT_INDEX_MASK)<<ARM_TT_PT_ADDR_SHIFT)) + +#define ptep_get_pmap(ptep) \ + ((((pt_desc_t *) (pvh_list(pai_to_pvh(pa_index((((vm_offset_t)(ptep) & ~0xFFF) - gVirtBase + gPhysBase))))))->pmap)) + + +#else + +#if (ARM_PGSHIFT == 12) +#define ARM_PT_DESC_INDEX_MASK ((PAGE_SHIFT_CONST == ARM_PGSHIFT )? 0x00000ULL : 0x03000ULL) +#define ARM_PT_DESC_INDEX_SHIFT ((PAGE_SHIFT_CONST == ARM_PGSHIFT )? 0 : 12) + /* + * mask for page descriptor index: 2MB per page table + */ +#define ARM_TT_PT_INDEX_MASK (0x0fffULL) + /* + * Shift value used for reconstructing the virtual address for a PTE. + */ +#define ARM_TT_PT_ADDR_SHIFT (9ULL) + + /* TODO: Give this a better name/documentation than "other" */ +#define ARM_TT_PT_OTHER_MASK (0x0fffULL) + +#else + +#define ARM_PT_DESC_INDEX_MASK (0x00000) +#define ARM_PT_DESC_INDEX_SHIFT (0) + /* + * mask for page descriptor index: 32MB per page table + */ +#define ARM_TT_PT_INDEX_MASK (0x3fffULL) + /* + * Shift value used for reconstructing the virtual address for a PTE. + */ +#define ARM_TT_PT_ADDR_SHIFT (11ULL) + + /* TODO: Give this a better name/documentation than "other" */ +#define ARM_TT_PT_OTHER_MASK (0x3fffULL) +#endif + +#define ARM_PT_DESC_INDEX(ptep) \ + (((unsigned)(ptep) & ARM_PT_DESC_INDEX_MASK) >> ARM_PT_DESC_INDEX_SHIFT) + + +#define ptep_get_ptd(ptep) \ + ((struct pt_desc *)((*((vm_offset_t *)(pai_to_pvh(pa_index((vm_offset_t)(ptep) - gVirtBase + gPhysBase))))) & PVH_LIST_MASK)) + +#define ptep_get_va(ptep) \ + ((((pt_desc_t *) (pvh_list(pai_to_pvh(pa_index((((vm_offset_t)(ptep) & ~ARM_TT_PT_OTHER_MASK) - gVirtBase + gPhysBase))))))->pt_map[ARM_PT_DESC_INDEX(ptep)].va)+ ((((unsigned)(ptep)) & ARM_TT_PT_INDEX_MASK)<<ARM_TT_PT_ADDR_SHIFT)) + +#define ptep_get_pmap(ptep) \ + ((((pt_desc_t *) (pvh_list(pai_to_pvh(pa_index((((vm_offset_t)(ptep) & ~ARM_TT_PT_OTHER_MASK) - gVirtBase + gPhysBase))))))->pmap)) + +#endif + + +/* PVH Define Macros */ + +/* pvhead type */ +#define PVH_TYPE_NULL 0x0UL +#define PVH_TYPE_PVEP 0x1UL +#define PVH_TYPE_PTEP 0x2UL +#define PVH_TYPE_PTDP 0x3UL + +#define PVH_TYPE_MASK (0x3UL) +#define PVH_LIST_MASK (~PVH_TYPE_MASK) + +#if (__ARM_VMSA__ == 7) +#define pvh_set_bits(h, b) \ + do { \ + while (!OSCompareAndSwap(*(vm_offset_t *)(h), *(vm_offset_t *)(h) | (b), (vm_offset_t *)(h))); \ + } while (0) + +#define pvh_clear_bits(h, b) \ + do { \ + while (!OSCompareAndSwap(*(vm_offset_t *)(h), *(vm_offset_t *)(h) & ~(b), (vm_offset_t *)(h))); \ + } while (0) +#else +#define pvh_set_bits(h, b) \ + do { \ + while (!OSCompareAndSwap64(*(vm_offset_t *)(h), *(vm_offset_t *)(h) | ((int64_t)b), (vm_offset_t *)(h))); \ + } while (0) + +#define pvh_clear_bits(h, b) \ + do { \ + while (!OSCompareAndSwap64(*(vm_offset_t *)(h), *(vm_offset_t *)(h) & ~((int64_t)b), (vm_offset_t *)(h))); \ + } while (0) +#endif + +#define pvh_test_type(h, b) \ + ((*(vm_offset_t *)(h) & (PVH_TYPE_MASK)) == (b)) + +#define pvh_ptep(h) \ + ((pt_entry_t *)(*(vm_offset_t *)(h) & PVH_LIST_MASK)) + +#define pvh_list(h) \ + ((pv_entry_t *)(*(vm_offset_t *)(h) & PVH_LIST_MASK)) + +#define pvh_bits(h) \ + (*(vm_offset_t *)(h) & PVH_TYPE_MASK) + +#if (__ARM_VMSA__ == 7) +#define pvh_update_head(h, e, t) \ + do { \ + while (!OSCompareAndSwap(*(vm_offset_t *)(h), (vm_offset_t)(e) | (t), (vm_offset_t *)(h))); \ + } while (0) +#else +#define pvh_update_head(h, e, t) \ + do { \ + while (!OSCompareAndSwap64(*(vm_offset_t *)(h), (vm_offset_t)(e) | (t), (vm_offset_t *)(h))); \ + } while (0) +#endif + +#define pvh_add(h, e) \ + do { \ + assert(!pvh_test_type((h), PVH_TYPE_PTEP)); \ + pve_next(e) = pvh_list(h); \ + pvh_update_head((h), (e), PVH_TYPE_PVEP); \ + } while (0) + +#define pvh_remove(h, p, e) \ + do { \ + assert(!PVE_NEXT_IS_ALTACCT(pve_next((e)))); \ + if ((p) == (h)) { \ + if (PVE_NEXT_PTR(pve_next((e))) == PV_ENTRY_NULL) { \ + pvh_update_head((h), PV_ENTRY_NULL, PVH_TYPE_NULL); \ + } else { \ + pvh_update_head((h), PVE_NEXT_PTR(pve_next((e))), PVH_TYPE_PVEP); \ + } \ + } else { \ + /* \ + * PMAP LEDGERS: \ + * preserve the "alternate accounting" bit \ + * when updating "p" (the previous entry's \ + * "pve_next"). \ + */ \ + boolean_t __is_altacct; \ + __is_altacct = PVE_NEXT_IS_ALTACCT(*(p)); \ + *(p) = PVE_NEXT_PTR(pve_next((e))); \ + if (__is_altacct) { \ + PVE_NEXT_SET_ALTACCT((p)); \ + } else { \ + PVE_NEXT_CLR_ALTACCT((p)); \ + } \ + } \ + } while (0) + + +/* PPATTR Define Macros */ + +#define ppattr_set_bits(h, b) \ + do { \ + while (!OSCompareAndSwap16(*(pp_attr_t *)(h), *(pp_attr_t *)(h) | (b), (pp_attr_t *)(h))); \ + } while (0) + +#define ppattr_clear_bits(h, b) \ + do { \ + while (!OSCompareAndSwap16(*(pp_attr_t *)(h), *(pp_attr_t *)(h) & ~(b), (pp_attr_t *)(h))); \ + } while (0) + +#define ppattr_test_bits(h, b) \ + ((*(pp_attr_t *)(h) & (b)) == (b)) + +#define pa_set_bits(x, b) \ + do { \ + if (pa_valid(x)) \ + ppattr_set_bits(&pp_attr_table[pa_index(x)], \ + (b)); \ + } while (0) + +#define pa_test_bits(x, b) \ + (pa_valid(x) ? ppattr_test_bits(&pp_attr_table[pa_index(x)],\ + (b)) : FALSE) + +#define pa_clear_bits(x, b) \ + do { \ + if (pa_valid(x)) \ + ppattr_clear_bits(&pp_attr_table[pa_index(x)], \ + (b)); \ + } while (0) + +#define pa_set_modify(x) \ + pa_set_bits(x, PP_ATTR_MODIFIED) + +#define pa_clear_modify(x) \ + pa_clear_bits(x, PP_ATTR_MODIFIED) + +#define pa_set_reference(x) \ + pa_set_bits(x, PP_ATTR_REFERENCED) + +#define pa_clear_reference(x) \ + pa_clear_bits(x, PP_ATTR_REFERENCED) + + +#define IS_INTERNAL_PAGE(pai) \ + ppattr_test_bits(&pp_attr_table[pai], PP_ATTR_INTERNAL) +#define SET_INTERNAL_PAGE(pai) \ + ppattr_set_bits(&pp_attr_table[pai], PP_ATTR_INTERNAL) +#define CLR_INTERNAL_PAGE(pai) \ + ppattr_clear_bits(&pp_attr_table[pai], PP_ATTR_INTERNAL) + +#define IS_REUSABLE_PAGE(pai) \ + ppattr_test_bits(&pp_attr_table[pai], PP_ATTR_REUSABLE) +#define SET_REUSABLE_PAGE(pai) \ + ppattr_set_bits(&pp_attr_table[pai], PP_ATTR_REUSABLE) +#define CLR_REUSABLE_PAGE(pai) \ + ppattr_clear_bits(&pp_attr_table[pai], PP_ATTR_REUSABLE) + +#define IS_ALTACCT_PAGE(pai, pve_p) \ + (((pve_p) == NULL) \ + ? ppattr_test_bits(&pp_attr_table[pai], PP_ATTR_ALTACCT) \ + : PVE_NEXT_IS_ALTACCT(pve_next((pve_p)))) +#define SET_ALTACCT_PAGE(pai, pve_p) \ + if ((pve_p) == NULL) { \ + ppattr_set_bits(&pp_attr_table[pai], PP_ATTR_ALTACCT); \ + } else { \ + PVE_NEXT_SET_ALTACCT(&pve_next((pve_p))); \ + } +#define CLR_ALTACCT_PAGE(pai, pve_p) \ + if ((pve_p) == NULL) { \ + ppattr_clear_bits(&pp_attr_table[pai], PP_ATTR_ALTACCT);\ + } else { \ + PVE_NEXT_CLR_ALTACCT(&pve_next((pve_p))); \ + } + +#define IS_REFFAULT_PAGE(pai) \ + ppattr_test_bits(&pp_attr_table[pai], PP_ATTR_REFFAULT) +#define SET_REFFAULT_PAGE(pai) \ + ppattr_set_bits(&pp_attr_table[pai], PP_ATTR_REFFAULT) +#define CLR_REFFAULT_PAGE(pai) \ + ppattr_clear_bits(&pp_attr_table[pai], PP_ATTR_REFFAULT) + +#define IS_MODFAULT_PAGE(pai) \ + ppattr_test_bits(&pp_attr_table[pai], PP_ATTR_MODFAULT) +#define SET_MODFAULT_PAGE(pai) \ + ppattr_set_bits(&pp_attr_table[pai], PP_ATTR_MODFAULT) +#define CLR_MODFAULT_PAGE(pai) \ + ppattr_clear_bits(&pp_attr_table[pai], PP_ATTR_MODFAULT) + + +#if (__ARM_VMSA__ == 7) + +#define tte_index(pmap, addr) \ + ttenum((addr)) + +#define tte_get_ptd(tte) \ + ((struct pt_desc *)((*((vm_offset_t *)(pai_to_pvh(pa_index((vm_offset_t)((tte) & ~PAGE_MASK)))))) & PVH_LIST_MASK)) + +#else + +#define tt0_index(pmap, addr) \ + (((addr) & ARM_TT_L0_INDEX_MASK) >> ARM_TT_L0_SHIFT) + +#define tt1_index(pmap, addr) \ + (((addr) & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT) + +#define tt2_index(pmap, addr) \ + (((addr) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT) + +#define tt3_index(pmap, addr) \ + (((addr) & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT) + +#define tte_index(pmap, addr) \ + (((addr) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT) + +#define tte_get_ptd(tte) \ + ((struct pt_desc *)((*((vm_offset_t *)(pai_to_pvh(pa_index((vm_offset_t)((tte) & ~PAGE_MASK)))))) & PVH_LIST_MASK)) + +#endif + +/* + * Lock on pmap system + */ + +#define PMAP_LOCK_INIT(pmap) { \ + simple_lock_init(&(pmap)->lock, 0); \ + } + +#define PMAP_LOCK(pmap) { \ + simple_lock(&(pmap)->lock); \ +} + +#define PMAP_UNLOCK(pmap) { \ + simple_unlock(&(pmap)->lock); \ +} + +#if MACH_ASSERT +#define PMAP_ASSERT_LOCKED(pmap) { \ + simple_lock_assert(&(pmap)->lock, LCK_ASSERT_OWNED); \ +} +#else +#define PMAP_ASSERT_LOCKED(pmap) +#endif + +/* + * Each entry in the pv_head_table is locked by a bit in the + * pv lock array, which is stored in the region preceding pv_head_table. + * The lock bits are accessed by the physical address of the page they lock. + */ +#define LOCK_PVH(index) { \ + hw_lock_bit((hw_lock_bit_t *) \ + ((unsigned int*)pv_head_table)-1-(index>>5), \ + (index&0x1F)); \ + } + +#define UNLOCK_PVH(index) { \ + hw_unlock_bit((hw_lock_bit_t *) \ + ((unsigned int*)pv_head_table)-1-(index>>5), \ + (index&0x1F)); \ + } + +#define ASSERT_PVH_LOCKED(index) { \ + assert(*(((unsigned int*)pv_head_table)-1-(index>>5)) & (1 << (index & 0x1F))); \ +} + +#define PMAP_UPDATE_TLBS(pmap, s, e) { \ + flush_mmu_tlb_region_asid(s, (unsigned)(e - s), pmap); \ +} + +#ifdef __ARM_L1_PTW__ + +#define FLUSH_PTE_RANGE(spte, epte) \ + __asm__ volatile("dsb ish"); + +#define FLUSH_PTE(pte_p) \ + __asm__ volatile("dsb ish"); + +#else + +#define FLUSH_PTE_RANGE(spte, epte) \ + CleanPoU_DcacheRegion((vm_offset_t)spte, \ + (vm_offset_t)epte - (vm_offset_t)spte); + +#define FLUSH_PTE(pte_p) \ + CleanPoU_DcacheRegion((vm_offset_t)pte_p, sizeof(pt_entry_t)); +#endif + +#define WRITE_PTE(pte_p, pte_entry) \ + __unreachable_ok_push \ + if (TEST_PAGE_RATIO_4) { \ + do { \ + if (((unsigned)(pte_p)) & 0x1f) panic("WRITE_PTE\n"); \ + if (((pte_entry) & ~ARM_PTE_COMPRESSED_MASK) == ARM_PTE_EMPTY) { \ + *(pte_p) = (pte_entry); \ + *((pte_p)+1) = (pte_entry); \ + *((pte_p)+2) = (pte_entry); \ + *((pte_p)+3) = (pte_entry); \ + } else { \ + *(pte_p) = (pte_entry); \ + *((pte_p)+1) = (pte_entry) | 0x1000; \ + *((pte_p)+2) = (pte_entry) | 0x2000; \ + *((pte_p)+3) = (pte_entry) | 0x3000; \ + } \ + FLUSH_PTE_RANGE((pte_p),((pte_p)+4)); \ + } while(0); \ + } else { \ + do { \ + *(pte_p) = (pte_entry); \ + FLUSH_PTE(pte_p); \ + } while(0); \ + } \ + __unreachable_ok_pop + +#define WRITE_PTE_FAST(pte_p, pte_entry) \ + __unreachable_ok_push \ + if (TEST_PAGE_RATIO_4) { \ + if (((unsigned)(pte_p)) & 0x1f) panic("WRITE_PTE\n"); \ + if (((pte_entry) & ~ARM_PTE_COMPRESSED_MASK) == ARM_PTE_EMPTY) { \ + *(pte_p) = (pte_entry); \ + *((pte_p)+1) = (pte_entry); \ + *((pte_p)+2) = (pte_entry); \ + *((pte_p)+3) = (pte_entry); \ + } else { \ + *(pte_p) = (pte_entry); \ + *((pte_p)+1) = (pte_entry) | 0x1000; \ + *((pte_p)+2) = (pte_entry) | 0x2000; \ + *((pte_p)+3) = (pte_entry) | 0x3000; \ + } \ + } else { \ + *(pte_p) = (pte_entry); \ + } \ + __unreachable_ok_pop + + +/* + * Other useful macros. + */ +#define current_pmap() \ + (vm_map_pmap(current_thread()->map)) + +#define PMAP_IS_VALID(x) (TRUE) + +#ifdef PMAP_TRACES +unsigned int pmap_trace = 0; + +#define PMAP_TRACE(...) \ + if (pmap_trace) { \ + KDBG_RELEASE(__VA_ARGS__); \ + } +#else +#define PMAP_TRACE(...) KDBG_DEBUG(__VA_ARGS__) +#endif + +#define PMAP_TRACE_CONSTANT(...) KDBG_RELEASE(__VA_ARGS__) + +/* + * Internal function prototypes (forward declarations). + */ + +static void pv_init( + void); + +static boolean_t pv_alloc( + pmap_t pmap, + unsigned int pai, + pv_entry_t **pvepp); + +static void pv_free( + pv_entry_t *pvep); + +static void pv_list_free( + pv_entry_t *pvehp, + pv_entry_t *pvetp, + unsigned int cnt); + +static void ptd_bootstrap( + pt_desc_t *ptdp, unsigned int ptd_cnt); + +static pt_desc_t *ptd_alloc( + pmap_t pmap); + +static void ptd_deallocate( + pt_desc_t *ptdp); + +static void ptd_init( + pt_desc_t *ptdp, pmap_t pmap, vm_map_address_t va, unsigned int ttlevel, pt_entry_t * pte_p); + +static void pmap_zone_init( + void); + +static void pmap_set_reference( + ppnum_t pn); + +ppnum_t pmap_vtophys( + pmap_t pmap, addr64_t va); + +void pmap_switch_user_ttb( + pmap_t pmap); + +static void flush_mmu_tlb_region_asid( + vm_offset_t va, unsigned length, pmap_t pmap); + +static kern_return_t pmap_expand( + pmap_t, vm_map_address_t, unsigned int options, unsigned int level); + +static int pmap_remove_range( + pmap_t, vm_map_address_t, pt_entry_t *, pt_entry_t *, uint32_t *); + +static int pmap_remove_range_options( + pmap_t, vm_map_address_t, pt_entry_t *, pt_entry_t *, uint32_t *, int); + +static tt_entry_t *pmap_tt1_allocate( + pmap_t, vm_size_t, unsigned int); + +#define PMAP_TT_ALLOCATE_NOWAIT 0x1 + +static void pmap_tt1_deallocate( + pmap_t, tt_entry_t *, vm_size_t, unsigned int); + +#define PMAP_TT_DEALLOCATE_NOBLOCK 0x1 + +static kern_return_t pmap_tt_allocate( + pmap_t, tt_entry_t **, unsigned int, unsigned int); + +#define PMAP_TT_ALLOCATE_NOWAIT 0x1 + +static void pmap_tte_deallocate( + pmap_t, tt_entry_t *, unsigned int); + +#define PMAP_TT_L1_LEVEL 0x1 +#define PMAP_TT_L2_LEVEL 0x2 +#define PMAP_TT_L3_LEVEL 0x3 +#if (__ARM_VMSA__ == 7) +#define PMAP_TT_MAX_LEVEL PMAP_TT_L2_LEVEL +#else +#define PMAP_TT_MAX_LEVEL PMAP_TT_L3_LEVEL +#endif + +#ifdef __ARM64_PMAP_SUBPAGE_L1__ +#if (__ARM_VMSA__ <= 7) +#error This is not supported for old-style page tables +#endif +#define PMAP_ROOT_ALLOC_SIZE (((ARM_TT_L1_INDEX_MASK >> ARM_TT_L1_SHIFT) + 1) * sizeof(tt_entry_t)) +#else +#define PMAP_ROOT_ALLOC_SIZE (ARM_PGBYTES) +#endif + +const unsigned int arm_hardware_page_size = ARM_PGBYTES; +const unsigned int arm_pt_desc_size = sizeof(pt_desc_t); +const unsigned int arm_pt_root_size = PMAP_ROOT_ALLOC_SIZE; + +#define PMAP_TT_DEALLOCATE_NOBLOCK 0x1 + +void pmap_init_pte_page_internal( + pmap_t, pt_entry_t *, vm_offset_t, unsigned int , pt_desc_t **); + + +#if (__ARM_VMSA__ > 7) + +static inline tt_entry_t *pmap_tt1e( + pmap_t, vm_map_address_t); + +static inline tt_entry_t *pmap_tt2e( + pmap_t, vm_map_address_t); + +static inline pt_entry_t *pmap_tt3e( + pmap_t, vm_map_address_t); + +static void pmap_unmap_sharedpage32( + pmap_t pmap); + +static void pmap_sharedpage_flush_32_to_64( + void); + +static boolean_t + pmap_is_64bit(pmap_t); + + +#endif +static inline tt_entry_t *pmap_tte( + pmap_t, vm_map_address_t); + +static inline pt_entry_t *pmap_pte( + pmap_t, vm_map_address_t); + +static void pmap_update_cache_attributes_locked( + ppnum_t, unsigned); + +boolean_t arm_clear_fast_fault( + ppnum_t ppnum, + vm_prot_t fault_type); + +static pmap_paddr_t pmap_pages_reclaim( + void); + +static kern_return_t pmap_pages_alloc( + pmap_paddr_t *pa, + unsigned size, + unsigned option); + +#define PMAP_PAGES_ALLOCATE_NOWAIT 0x1 +#define PMAP_PAGES_RECLAIM_NOWAIT 0x2 + +static void pmap_pages_free( + pmap_paddr_t pa, + unsigned size); + + +#define PMAP_SUPPORT_PROTOTYPES(__return_type, __function_name, __function_args, __function_index) \ + static __return_type __function_name##_internal __function_args; + +PMAP_SUPPORT_PROTOTYPES( +kern_return_t, +arm_fast_fault, (pmap_t pmap, + vm_map_address_t va, + vm_prot_t fault_type, + boolean_t from_user), ARM_FAST_FAULT_INDEX); + + +PMAP_SUPPORT_PROTOTYPES( +boolean_t, +arm_force_fast_fault, (ppnum_t ppnum, + vm_prot_t allow_mode, + int options), ARM_FORCE_FAST_FAULT_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +kern_return_t, +mapping_free_prime, (void), MAPPING_FREE_PRIME_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +kern_return_t, +mapping_replenish, (void), MAPPING_REPLENISH_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +boolean_t, +pmap_batch_set_cache_attributes, (ppnum_t pn, + unsigned int cacheattr, + unsigned int page_cnt, + unsigned int page_index, + boolean_t doit, + unsigned int *res), PMAP_BATCH_SET_CACHE_ATTRIBUTES_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +void, +pmap_change_wiring, (pmap_t pmap, + vm_map_address_t v, + boolean_t wired), PMAP_CHANGE_WIRING_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +pmap_t, +pmap_create, (ledger_t ledger, + vm_map_size_t size, + boolean_t is_64bit), PMAP_CREATE_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +void, +pmap_destroy, (pmap_t pmap), PMAP_DESTROY_INDEX); + + + +PMAP_SUPPORT_PROTOTYPES( +kern_return_t, +pmap_enter_options, (pmap_t pmap, + vm_map_address_t v, + ppnum_t pn, + vm_prot_t prot, + vm_prot_t fault_type, + unsigned int flags, + boolean_t wired, + unsigned int options), PMAP_ENTER_OPTIONS_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +vm_offset_t, +pmap_extract, (pmap_t pmap, + vm_map_address_t va), PMAP_EXTRACT_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +ppnum_t, +pmap_find_phys, (pmap_t pmap, + addr64_t va), PMAP_FIND_PHYS_INDEX); + +#if (__ARM_VMSA__ > 7) +PMAP_SUPPORT_PROTOTYPES( +void, +pmap_insert_sharedpage, (pmap_t pmap), PMAP_INSERT_SHAREDPAGE_INDEX); +#endif + + +PMAP_SUPPORT_PROTOTYPES( +boolean_t, +pmap_is_empty, (pmap_t pmap, + vm_map_offset_t va_start, + vm_map_offset_t va_end), PMAP_IS_EMPTY_INDEX); + + +PMAP_SUPPORT_PROTOTYPES( +unsigned int, +pmap_map_cpu_windows_copy, (ppnum_t pn, + vm_prot_t prot, + unsigned int wimg_bits), PMAP_MAP_CPU_WINDOWS_COPY_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +kern_return_t, +pmap_nest, (pmap_t grand, + pmap_t subord, + addr64_t vstart, + addr64_t nstart, + uint64_t size), PMAP_NEST_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +void, +pmap_page_protect_options, (ppnum_t ppnum, + vm_prot_t prot, + unsigned int options), PMAP_PAGE_PROTECT_OPTIONS_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +void, +pmap_protect_options, (pmap_t pmap, + vm_map_address_t start, + vm_map_address_t end, + vm_prot_t prot, + unsigned int options, + void *args), PMAP_PROTECT_OPTIONS_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +kern_return_t, +pmap_query_page_info, (pmap_t pmap, + vm_map_offset_t va, + int *disp_p), PMAP_QUERY_PAGE_INFO_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +boolean_t, +pmap_query_resident, (pmap_t pmap, + vm_map_address_t start, + vm_map_address_t end, + mach_vm_size_t *resident_bytes_p, + mach_vm_size_t *compressed_bytes_p), PMAP_QUERY_RESIDENT_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +void, +pmap_reference, (pmap_t pmap), PMAP_REFERENCE_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +int, +pmap_remove_options, (pmap_t pmap, + vm_map_address_t start, + vm_map_address_t end, + int options), PMAP_REMOVE_OPTIONS_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +kern_return_t, +pmap_return, (boolean_t do_panic, + boolean_t do_recurse), PMAP_RETURN_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +void, +pmap_set_cache_attributes, (ppnum_t pn, + unsigned int cacheattr), PMAP_SET_CACHE_ATTRIBUTES_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +void, +pmap_set_nested, (pmap_t pmap), PMAP_SET_NESTED_INDEX); + +#if MACH_ASSERT +PMAP_SUPPORT_PROTOTYPES( +void, +pmap_set_process, (pmap_t pmap, + int pid, + char *procname), PMAP_SET_PROCESS_INDEX); +#endif + + +PMAP_SUPPORT_PROTOTYPES( +void, +pmap_unmap_cpu_windows_copy, (unsigned int index), PMAP_UNMAP_CPU_WINDOWS_COPY_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +kern_return_t, +pmap_unnest_options, (pmap_t grand, + addr64_t vaddr, + uint64_t size, + unsigned int option), PMAP_UNNEST_OPTIONS_INDEX); + + +PMAP_SUPPORT_PROTOTYPES( +void, +phys_attribute_set, (ppnum_t pn, + unsigned int bits), PHYS_ATTRIBUTE_SET_INDEX); + + +PMAP_SUPPORT_PROTOTYPES( +void, +phys_attribute_clear, (ppnum_t pn, + unsigned int bits, + int options, + void *arg), PHYS_ATTRIBUTE_CLEAR_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +void, +pmap_switch, (pmap_t pmap), PMAP_SWITCH_INDEX); + +PMAP_SUPPORT_PROTOTYPES( +void, +pmap_switch_user_ttb, (pmap_t pmap), PMAP_SWITCH_USER_TTB_INDEX); + + + +void pmap_footprint_suspend(vm_map_t map, + boolean_t suspend); +PMAP_SUPPORT_PROTOTYPES( + void, + pmap_footprint_suspend, (vm_map_t map, + boolean_t suspend), + PMAP_FOOTPRINT_SUSPEND_INDEX); + +#if CONFIG_PGTRACE +boolean_t pgtrace_enabled = 0; + +typedef struct { + queue_chain_t chain; + + /* + pmap - pmap for below addresses + ova - original va page address + cva - clone va addresses for pre, target and post pages + cva_spte - clone saved ptes + range - trace range in this map + cloned - has been cloned or not + */ + pmap_t pmap; + vm_map_offset_t ova; + vm_map_offset_t cva[3]; + pt_entry_t cva_spte[3]; + struct { + pmap_paddr_t start; + pmap_paddr_t end; + } range; + bool cloned; +} pmap_pgtrace_map_t; + +static void pmap_pgtrace_init(void); +static bool pmap_pgtrace_enter_clone(pmap_t pmap, vm_map_offset_t va_page, vm_map_offset_t start, vm_map_offset_t end); +static void pmap_pgtrace_remove_clone(pmap_t pmap, pmap_paddr_t pa_page, vm_map_offset_t va_page); +static void pmap_pgtrace_remove_all_clone(pmap_paddr_t pa); +#endif + +#if (__ARM_VMSA__ > 7) +/* + * The low global vector page is mapped at a fixed alias. + * Since the page size is 16k for H8 and newer we map the globals to a 16k + * aligned address. Readers of the globals (e.g. lldb, panic server) need + * to check both addresses anyway for backward compatibility. So for now + * we leave H6 and H7 where they were. + */ +#if (ARM_PGSHIFT == 14) +#define LOWGLOBAL_ALIAS (LOW_GLOBAL_BASE_ADDRESS + 0x4000) +#else +#define LOWGLOBAL_ALIAS (LOW_GLOBAL_BASE_ADDRESS + 0x2000) +#endif + +#else +#define LOWGLOBAL_ALIAS (0xFFFF1000) +#endif + +long long alloc_tteroot_count __attribute__((aligned(8))) MARK_AS_PMAP_DATA = 0LL; +long long alloc_ttepages_count __attribute__((aligned(8))) MARK_AS_PMAP_DATA = 0LL; +long long alloc_ptepages_count __attribute__((aligned(8))) MARK_AS_PMAP_DATA = 0LL; +long long alloc_pmap_pages_count __attribute__((aligned(8))) = 0LL; + +int pt_fake_zone_index = -1; /* index of pmap fake zone */ + + + +/* + * Allocates and initializes a per-CPU data structure for the pmap. + */ +static void +pmap_cpu_data_init_internal(unsigned int cpu_number) +{ + pmap_cpu_data_t * pmap_cpu_data = NULL; + + pmap_cpu_data = pmap_get_cpu_data(); + pmap_cpu_data->cpu_number = cpu_number; +} + +void +pmap_cpu_data_init(void) +{ + pmap_cpu_data_init_internal(cpu_number()); +} + +static void +pmap_cpu_data_array_init(void) +{ + + pmap_cpu_data_init(); +} + +pmap_cpu_data_t * +pmap_get_cpu_data(void) +{ + pmap_cpu_data_t * pmap_cpu_data = NULL; + + pmap_cpu_data = &getCpuDatap()->cpu_pmap_cpu_data; + + return pmap_cpu_data; +} + + +/* TODO */ +pmap_paddr_t +pmap_pages_reclaim( + void) +{ + boolean_t found_page; + unsigned i; + pt_desc_t *ptdp; + + + /* + * pmap_pages_reclaim() is returning a page by freeing an active pt page. + * To be eligible, a pt page is assigned to a user pmap. It doesn't have any wired pte + * entry and it contains at least one valid pte entry. + * + * In a loop, check for a page in the reclaimed pt page list. + * if one is present, unlink that page and return the physical page address. + * Otherwise, scan the pt page list for an eligible pt page to reclaim. + * If found, invoke pmap_remove_range() on its pmap and address range then + * deallocates that pt page. This will end up adding the pt page to the + * reclaimed pt page list. + * If no eligible page were found in the pt page list, panic. + */ + + simple_lock(&pmap_pages_lock); + pmap_pages_request_count++; + pmap_pages_request_acum++; + + while (1) { + + if (pmap_pages_reclaim_list != (page_free_entry_t *)NULL) { + page_free_entry_t *page_entry; + + page_entry = pmap_pages_reclaim_list; + pmap_pages_reclaim_list = pmap_pages_reclaim_list->next; + simple_unlock(&pmap_pages_lock); + + return((pmap_paddr_t)ml_static_vtop((vm_offset_t)page_entry)); + } + + simple_unlock(&pmap_pages_lock); + + simple_lock(&pt_pages_lock); + ptdp = (pt_desc_t *)queue_first(&pt_page_list); + found_page = FALSE; + + while (!queue_end(&pt_page_list, (queue_entry_t)ptdp)) { + if ((ptdp->pmap != kernel_pmap) + && (ptdp->pmap->nested == FALSE) + && (simple_lock_try(&ptdp->pmap->lock))) { + + unsigned refcnt_acc = 0; + unsigned wiredcnt_acc = 0; + + for (i = 0 ; i < PT_INDEX_MAX ; i++) { + if (ptdp->pt_cnt[i].refcnt & PT_DESC_REFCOUNT) { + /* Do not attempt to free a page that contains an L2 table + * or is currently being operated on by pmap_enter(), + * which can drop the pmap lock. */ + refcnt_acc = 0; + break; + } + refcnt_acc += ptdp->pt_cnt[i].refcnt; + wiredcnt_acc += ptdp->pt_cnt[i].wiredcnt; + } + if ((wiredcnt_acc == 0) && (refcnt_acc != 0)) { + found_page = TRUE; + /* Leave ptdp->pmap locked here. We're about to reclaim + * a tt page from it, so we don't want anyone else messing + * with it while we do that. */ + break; + } + simple_unlock(&ptdp->pmap->lock); + } + ptdp = (pt_desc_t *)queue_next((queue_t)ptdp); + } + if (!found_page) { + panic("pmap_pages_reclaim(): No eligible page in pt_page_list\n"); + } else { + int remove_count = 0; + vm_map_address_t va; + pmap_t pmap; + pt_entry_t *bpte, *epte; + pt_entry_t *pte_p; + tt_entry_t *tte_p; + uint32_t rmv_spte=0; + + simple_unlock(&pt_pages_lock); + pmap = ptdp->pmap; + PMAP_ASSERT_LOCKED(pmap); // pmap lock should be held from loop above + for (i = 0 ; i < PT_INDEX_MAX ; i++) { + va = ptdp->pt_map[i].va; + + tte_p = pmap_tte(pmap, va); + if ((tte_p != (tt_entry_t *) NULL) + && ((*tte_p & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE)) { + +#if (__ARM_VMSA__ == 7) + pte_p = (pt_entry_t *) ttetokv(*tte_p); + bpte = &pte_p[ptenum(va)]; + epte = bpte + PAGE_SIZE/sizeof(pt_entry_t); +#else + pte_p = (pt_entry_t *) ttetokv(*tte_p); + bpte = &pte_p[tt3_index(pmap, va)]; + epte = bpte + PAGE_SIZE/sizeof(pt_entry_t); +#endif + /* + * Use PMAP_OPTIONS_REMOVE to clear any + * "compressed" markers and update the + * "compressed" counter in pmap->stats. + * This means that we lose accounting for + * any compressed pages in this range + * but the alternative is to not be able + * to account for their future decompression, + * which could cause the counter to drift + * more and more. + */ + remove_count += pmap_remove_range_options( + pmap, va, bpte, epte, + &rmv_spte, PMAP_OPTIONS_REMOVE); + if (ptdp->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].refcnt != 0) + panic("pmap_pages_reclaim(): ptdp %p, count %d\n", ptdp, ptdp->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].refcnt); +#if (__ARM_VMSA__ == 7) + pmap_tte_deallocate(pmap, tte_p, PMAP_TT_L1_LEVEL); + flush_mmu_tlb_entry((va & ~ARM_TT_L1_PT_OFFMASK) | (pmap->asid & 0xff)); + flush_mmu_tlb_entry(((va & ~ARM_TT_L1_PT_OFFMASK) + ARM_TT_L1_SIZE) | (pmap->asid & 0xff)); + flush_mmu_tlb_entry(((va & ~ARM_TT_L1_PT_OFFMASK) + 2*ARM_TT_L1_SIZE)| (pmap->asid & 0xff)); + flush_mmu_tlb_entry(((va & ~ARM_TT_L1_PT_OFFMASK) + 3*ARM_TT_L1_SIZE)| (pmap->asid & 0xff)); +#else + pmap_tte_deallocate(pmap, tte_p, PMAP_TT_L2_LEVEL); + flush_mmu_tlb_entry(tlbi_addr(va & ~ARM_TT_L2_OFFMASK) | tlbi_asid(pmap->asid)); +#endif + + if (remove_count > 0) { +#if (__ARM_VMSA__ == 7) + PMAP_UPDATE_TLBS(pmap, va, va+4*ARM_TT_L1_SIZE); +#else + PMAP_UPDATE_TLBS(pmap, va, va+ARM_TT_L2_SIZE); +#endif + } + } + } + // Undo the lock we grabbed when we found ptdp above + PMAP_UNLOCK(pmap); + } + simple_lock(&pmap_pages_lock); + } +} + + +static kern_return_t +pmap_pages_alloc( + pmap_paddr_t *pa, + unsigned size, + unsigned option) +{ + vm_page_t m = VM_PAGE_NULL, m_prev; + + if(option & PMAP_PAGES_RECLAIM_NOWAIT) { + assert(size == PAGE_SIZE); + *pa = pmap_pages_reclaim(); + return KERN_SUCCESS; + } + if (size == PAGE_SIZE) { + while ((m = vm_page_grab()) == VM_PAGE_NULL) { + if(option & PMAP_PAGES_ALLOCATE_NOWAIT) { + return KERN_RESOURCE_SHORTAGE; + } + + VM_PAGE_WAIT(); + } + vm_page_lock_queues(); + vm_page_wire(m, VM_KERN_MEMORY_PTE, TRUE); + vm_page_unlock_queues(); + } + if (size == 2*PAGE_SIZE) { + while (cpm_allocate(size, &m, 0, 1, TRUE, 0) != KERN_SUCCESS) { + if(option & PMAP_PAGES_ALLOCATE_NOWAIT) + return KERN_RESOURCE_SHORTAGE; + + VM_PAGE_WAIT(); + } + } + + *pa = (pmap_paddr_t)ptoa(VM_PAGE_GET_PHYS_PAGE(m)); + + vm_object_lock(pmap_object); + while (m != VM_PAGE_NULL) { + vm_page_insert_wired(m, pmap_object, (vm_object_offset_t) ((ptoa(VM_PAGE_GET_PHYS_PAGE(m))) - gPhysBase), VM_KERN_MEMORY_PTE); + m_prev = m; + m = NEXT_PAGE(m_prev); + *(NEXT_PAGE_PTR(m_prev)) = VM_PAGE_NULL; + } + vm_object_unlock(pmap_object); + + OSAddAtomic(size>>PAGE_SHIFT, &inuse_pmap_pages_count); + OSAddAtomic64(size>>PAGE_SHIFT, &alloc_pmap_pages_count); + + return KERN_SUCCESS; +} + + +static void +pmap_pages_free( + pmap_paddr_t pa, + unsigned size) +{ + simple_lock(&pmap_pages_lock); + + if (pmap_pages_request_count != 0) { + page_free_entry_t *page_entry; + + pmap_pages_request_count--; + page_entry = (page_free_entry_t *)phystokv(pa); + page_entry->next = pmap_pages_reclaim_list; + pmap_pages_reclaim_list = page_entry; + simple_unlock(&pmap_pages_lock); + + return; + } + + simple_unlock(&pmap_pages_lock); + + vm_page_t m; + pmap_paddr_t pa_max; + + OSAddAtomic(-(size>>PAGE_SHIFT), &inuse_pmap_pages_count); + + for (pa_max = pa + size; pa < pa_max; pa = pa + PAGE_SIZE) { + vm_object_lock(pmap_object); + m = vm_page_lookup(pmap_object, (pa - gPhysBase)); + assert(m != VM_PAGE_NULL); + assert(VM_PAGE_WIRED(m)); + vm_page_lock_queues(); + vm_page_free(m); + vm_page_unlock_queues(); + vm_object_unlock(pmap_object); + } +} + +static inline void +PMAP_ZINFO_PALLOC( + pmap_t pmap, int bytes) +{ + pmap_ledger_credit(pmap, task_ledgers.tkm_private, bytes); +} + +static inline void +PMAP_ZINFO_PFREE( + pmap_t pmap, + int bytes) +{ + pmap_ledger_debit(pmap, task_ledgers.tkm_private, bytes); +} + +static inline void +pmap_tt_ledger_credit( + pmap_t pmap, + vm_size_t size) +{ + if (pmap != kernel_pmap) { + pmap_ledger_credit(pmap, task_ledgers.phys_footprint, size); + pmap_ledger_credit(pmap, task_ledgers.page_table, size); + } +} + +static inline void +pmap_tt_ledger_debit( + pmap_t pmap, + vm_size_t size) +{ + if (pmap != kernel_pmap) { + pmap_ledger_debit(pmap, task_ledgers.phys_footprint, size); + pmap_ledger_debit(pmap, task_ledgers.page_table, size); + } +} + +static unsigned int +alloc_asid( + void) +{ + unsigned int asid_bitmap_index; + + simple_lock(&pmaps_lock); + for (asid_bitmap_index = 0; asid_bitmap_index < (MAX_ASID / (sizeof(uint32_t) * NBBY)); asid_bitmap_index++) { + unsigned int temp = ffs(asid_bitmap[asid_bitmap_index]); + if (temp > 0) { + temp -= 1; + asid_bitmap[asid_bitmap_index] &= ~(1 << temp); + simple_unlock(&pmaps_lock); + + /* We should never vend out physical ASID 0 through this method. */ + assert(((asid_bitmap_index * sizeof(uint32_t) * NBBY + temp) % ARM_MAX_ASID) != 0); + + return (asid_bitmap_index * sizeof(uint32_t) * NBBY + temp); + } + } + simple_unlock(&pmaps_lock); + /* + * ToDo: Add code to deal with pmap with no asid panic for now. Not + * an issue with the small config process hard limit + */ + panic("alloc_asid(): out of ASID number"); + return MAX_ASID; +} + +static void +free_asid( + int asid) +{ + /* Don't free up any alias of physical ASID 0. */ + assert((asid % ARM_MAX_ASID) != 0); + + simple_lock(&pmaps_lock); + setbit(asid, (int *) asid_bitmap); + simple_unlock(&pmaps_lock); +} + +#define PV_LOW_WATER_MARK_DEFAULT 0x200 +#define PV_KERN_LOW_WATER_MARK_DEFAULT 0x200 +#define PV_ALLOC_CHUNK_INITIAL 0x200 +#define PV_KERN_ALLOC_CHUNK_INITIAL 0x200 +#define PV_ALLOC_INITIAL_TARGET (PV_ALLOC_CHUNK_INITIAL * 5) +#define PV_KERN_ALLOC_INITIAL_TARGET (PV_KERN_ALLOC_CHUNK_INITIAL) + + +uint32_t pv_free_count MARK_AS_PMAP_DATA = 0; +uint32_t pv_page_count MARK_AS_PMAP_DATA = 0; +uint32_t pv_kern_free_count MARK_AS_PMAP_DATA = 0; + +uint32_t pv_low_water_mark MARK_AS_PMAP_DATA; +uint32_t pv_kern_low_water_mark MARK_AS_PMAP_DATA; +uint32_t pv_alloc_chunk MARK_AS_PMAP_DATA; +uint32_t pv_kern_alloc_chunk MARK_AS_PMAP_DATA; + +thread_t mapping_replenish_thread; +event_t mapping_replenish_event; +event_t pmap_user_pv_throttle_event; +volatile uint32_t mappingrecurse = 0; + +uint64_t pmap_pv_throttle_stat; +uint64_t pmap_pv_throttled_waiters; + +unsigned pmap_mapping_thread_wakeups; +unsigned pmap_kernel_reserve_replenish_stat MARK_AS_PMAP_DATA; +unsigned pmap_user_reserve_replenish_stat MARK_AS_PMAP_DATA; +unsigned pmap_kern_reserve_alloc_stat MARK_AS_PMAP_DATA; + + +static void +pv_init( + void) +{ + simple_lock_init(&pv_free_list_lock, 0); + simple_lock_init(&pv_kern_free_list_lock, 0); + pv_free_list = PV_ENTRY_NULL; + pv_free_count = 0x0U; + pv_kern_free_list = PV_ENTRY_NULL; + pv_kern_free_count = 0x0U; +} + +static inline void PV_ALLOC(pv_entry_t **pv_ep); +static inline void PV_KERN_ALLOC(pv_entry_t **pv_e); +static inline void PV_FREE_LIST(pv_entry_t *pv_eh, pv_entry_t *pv_et, int pv_cnt); +static inline void PV_KERN_FREE_LIST(pv_entry_t *pv_eh, pv_entry_t *pv_et, int pv_cnt); + +static inline void pmap_pv_throttle(pmap_t p); + +static boolean_t +pv_alloc( + pmap_t pmap, + unsigned int pai, + pv_entry_t **pvepp) +{ + PMAP_ASSERT_LOCKED(pmap); + ASSERT_PVH_LOCKED(pai); + PV_ALLOC(pvepp); + if (PV_ENTRY_NULL == *pvepp) { + + if (kernel_pmap == pmap) { + + PV_KERN_ALLOC(pvepp); + + if (PV_ENTRY_NULL == *pvepp) { + pv_entry_t *pv_e; + pv_entry_t *pv_eh; + pv_entry_t *pv_et; + int pv_cnt; + unsigned j; + pmap_paddr_t pa; + kern_return_t ret; + + UNLOCK_PVH(pai); + PMAP_UNLOCK(pmap); + + ret = pmap_pages_alloc(&pa, PAGE_SIZE, PMAP_PAGES_ALLOCATE_NOWAIT); + + if (ret == KERN_RESOURCE_SHORTAGE) { + ret = pmap_pages_alloc(&pa, PAGE_SIZE, PMAP_PAGES_RECLAIM_NOWAIT); + } + + if (ret != KERN_SUCCESS) { + panic("%s: failed to alloc page for kernel, ret=%d, " + "pmap=%p, pai=%u, pvepp=%p", + __FUNCTION__, ret, + pmap, pai, pvepp); + } + + pv_page_count++; + + pv_e = (pv_entry_t *)phystokv(pa); + pv_cnt = 0; + pv_eh = pv_et = PV_ENTRY_NULL; + *pvepp = pv_e; + pv_e++; + + for (j = 1; j < (PAGE_SIZE/sizeof(pv_entry_t)) ; j++) { + pv_e->pve_next = pv_eh; + pv_eh = pv_e; + + if (pv_et == PV_ENTRY_NULL) + pv_et = pv_e; + pv_cnt++; + pv_e++; + } + PV_KERN_FREE_LIST(pv_eh, pv_et, pv_cnt); + PMAP_LOCK(pmap); + LOCK_PVH(pai); + return FALSE; + } + } else { + UNLOCK_PVH(pai); + PMAP_UNLOCK(pmap); + pmap_pv_throttle(pmap); + { + pv_entry_t *pv_e; + pv_entry_t *pv_eh; + pv_entry_t *pv_et; + int pv_cnt; + unsigned j; + pmap_paddr_t pa; + kern_return_t ret; + + ret = pmap_pages_alloc(&pa, PAGE_SIZE, 0); + + if (ret != KERN_SUCCESS) { + panic("%s: failed to alloc page, ret=%d, " + "pmap=%p, pai=%u, pvepp=%p", + __FUNCTION__, ret, + pmap, pai, pvepp); + } + + pv_page_count++; + + pv_e = (pv_entry_t *)phystokv(pa); + pv_cnt = 0; + pv_eh = pv_et = PV_ENTRY_NULL; + *pvepp = pv_e; + pv_e++; + + for (j = 1; j < (PAGE_SIZE/sizeof(pv_entry_t)) ; j++) { + pv_e->pve_next = pv_eh; + pv_eh = pv_e; + + if (pv_et == PV_ENTRY_NULL) + pv_et = pv_e; + pv_cnt++; + pv_e++; + } + PV_FREE_LIST(pv_eh, pv_et, pv_cnt); + } + PMAP_LOCK(pmap); + LOCK_PVH(pai); + return FALSE; + } + } + assert(PV_ENTRY_NULL != *pvepp); + return TRUE; +} + +static void +pv_free( + pv_entry_t *pvep) +{ + PV_FREE_LIST(pvep, pvep, 1); +} + +static void +pv_list_free( + pv_entry_t *pvehp, + pv_entry_t *pvetp, + unsigned int cnt) +{ + PV_FREE_LIST(pvehp, pvetp, cnt); +} + + + +static inline void PV_ALLOC(pv_entry_t **pv_ep) { + assert(*pv_ep == PV_ENTRY_NULL); + simple_lock(&pv_free_list_lock); + /* + * If the kernel reserved pool is low, let non-kernel mappings allocate + * synchronously, possibly subject to a throttle. + */ + if ((pv_kern_free_count >= pv_kern_low_water_mark) && ((*pv_ep = pv_free_list) != 0)) { + pv_free_list = (pv_entry_t *)(*pv_ep)->pve_next; + (*pv_ep)->pve_next = PV_ENTRY_NULL; + pv_free_count--; + } + + simple_unlock(&pv_free_list_lock); + + if ((pv_free_count < pv_low_water_mark) || (pv_kern_free_count < pv_kern_low_water_mark)) { + if (!mappingrecurse && hw_compare_and_store(0,1, &mappingrecurse)) + thread_wakeup(&mapping_replenish_event); + } +} + +static inline void PV_FREE_LIST(pv_entry_t *pv_eh, pv_entry_t *pv_et, int pv_cnt) { + simple_lock(&pv_free_list_lock); + pv_et->pve_next = (pv_entry_t *)pv_free_list; + pv_free_list = pv_eh; + pv_free_count += pv_cnt; + simple_unlock(&pv_free_list_lock); +} + +static inline void PV_KERN_ALLOC(pv_entry_t **pv_e) { + assert(*pv_e == PV_ENTRY_NULL); + simple_lock(&pv_kern_free_list_lock); + + if ((*pv_e = pv_kern_free_list) != 0) { + pv_kern_free_list = (pv_entry_t *)(*pv_e)->pve_next; + (*pv_e)->pve_next = PV_ENTRY_NULL; + pv_kern_free_count--; + pmap_kern_reserve_alloc_stat++; + } + + simple_unlock(&pv_kern_free_list_lock); + + if (pv_kern_free_count < pv_kern_low_water_mark) { + if (!mappingrecurse && hw_compare_and_store(0,1, &mappingrecurse)) { + thread_wakeup(&mapping_replenish_event); + } + } +} + +static inline void PV_KERN_FREE_LIST(pv_entry_t *pv_eh, pv_entry_t *pv_et, int pv_cnt) { + simple_lock(&pv_kern_free_list_lock); + pv_et->pve_next = pv_kern_free_list; + pv_kern_free_list = pv_eh; + pv_kern_free_count += pv_cnt; + simple_unlock(&pv_kern_free_list_lock); +} + +static inline void pmap_pv_throttle(__unused pmap_t p) { + assert(p != kernel_pmap); + /* Apply throttle on non-kernel mappings */ + if (pv_kern_free_count < (pv_kern_low_water_mark / 2)) { + pmap_pv_throttle_stat++; + /* This doesn't need to be strictly accurate, merely a hint + * to eliminate the timeout when the reserve is replenished. + */ + pmap_pv_throttled_waiters++; + assert_wait_timeout(&pmap_user_pv_throttle_event, THREAD_UNINT, 1, 1000 * NSEC_PER_USEC); + thread_block(THREAD_CONTINUE_NULL); + } +} + +/* + * Creates a target number of free pv_entry_t objects for the kernel free list + * and the general free list. + */ +static kern_return_t +mapping_free_prime_internal(void) +{ + unsigned j; + pmap_paddr_t pa; + kern_return_t ret; + pv_entry_t *pv_e; + pv_entry_t *pv_eh; + pv_entry_t *pv_et; + int pv_cnt; + int alloc_options = 0; + int needed_pv_cnt = 0; + int target_pv_free_cnt = 0; + + SECURITY_READ_ONLY_LATE(static boolean_t) mapping_free_prime_internal_called = FALSE; + SECURITY_READ_ONLY_LATE(static boolean_t) mapping_free_prime_internal_done = FALSE; + + if (mapping_free_prime_internal_done) { + return KERN_FAILURE; + } + + if (!mapping_free_prime_internal_called) { + mapping_free_prime_internal_called = TRUE; + + pv_low_water_mark = PV_LOW_WATER_MARK_DEFAULT; + + /* Alterable via sysctl */ + pv_kern_low_water_mark = PV_KERN_LOW_WATER_MARK_DEFAULT; + + pv_kern_alloc_chunk = PV_KERN_ALLOC_CHUNK_INITIAL; + pv_alloc_chunk = PV_ALLOC_CHUNK_INITIAL; + } + + pv_cnt = 0; + pv_eh = pv_et = PV_ENTRY_NULL; + target_pv_free_cnt = PV_ALLOC_INITIAL_TARGET; + + /* + * We don't take the lock to read pv_free_count, as we should not be + * invoking this from a multithreaded context. + */ + needed_pv_cnt = target_pv_free_cnt - pv_free_count; + + if (needed_pv_cnt > target_pv_free_cnt) { + needed_pv_cnt = 0; + } + + while (pv_cnt < needed_pv_cnt) { + ret = pmap_pages_alloc(&pa, PAGE_SIZE, alloc_options); + + assert(ret == KERN_SUCCESS); + + pv_page_count++; + + pv_e = (pv_entry_t *)phystokv(pa); + + for (j = 0; j < (PAGE_SIZE/sizeof(pv_entry_t)) ; j++) { + pv_e->pve_next = pv_eh; + pv_eh = pv_e; + + if (pv_et == PV_ENTRY_NULL) + pv_et = pv_e; + pv_cnt++; + pv_e++; + } + } + + if (pv_cnt) { + PV_FREE_LIST(pv_eh, pv_et, pv_cnt); + } + + pv_cnt = 0; + pv_eh = pv_et = PV_ENTRY_NULL; + target_pv_free_cnt = PV_KERN_ALLOC_INITIAL_TARGET; + + /* + * We don't take the lock to read pv_kern_free_count, as we should not + * be invoking this from a multithreaded context. + */ + needed_pv_cnt = target_pv_free_cnt - pv_kern_free_count; + + if (needed_pv_cnt > target_pv_free_cnt) { + needed_pv_cnt = 0; + } + + while (pv_cnt < needed_pv_cnt) { + + ret = pmap_pages_alloc(&pa, PAGE_SIZE, alloc_options); + + assert(ret == KERN_SUCCESS); + pv_page_count++; + + pv_e = (pv_entry_t *)phystokv(pa); + + for (j = 0; j < (PAGE_SIZE/sizeof(pv_entry_t)) ; j++) { + pv_e->pve_next = pv_eh; + pv_eh = pv_e; + + if (pv_et == PV_ENTRY_NULL) + pv_et = pv_e; + pv_cnt++; + pv_e++; + } + } + + if (pv_cnt) { + PV_KERN_FREE_LIST(pv_eh, pv_et, pv_cnt); + } + + mapping_free_prime_internal_done = TRUE; + return KERN_SUCCESS; +} + +void +mapping_free_prime(void) +{ + kern_return_t kr = KERN_FAILURE; + + kr = mapping_free_prime_internal(); + + if (kr != KERN_SUCCESS) { + panic("%s: failed, kr=%d", __FUNCTION__, kr); + } +} + +void mapping_replenish(void); + +void mapping_adjust(void) { + kern_return_t mres; + + mres = kernel_thread_start_priority((thread_continue_t)mapping_replenish, NULL, MAXPRI_KERNEL, &mapping_replenish_thread); + if (mres != KERN_SUCCESS) { + panic("pmap: mapping_replenish thread creation failed"); + } + thread_deallocate(mapping_replenish_thread); +} + +/* + * Fills the kernel and general PV free lists back up to their low watermarks. + */ +static kern_return_t +mapping_replenish_internal(void) +{ + pv_entry_t *pv_e; + pv_entry_t *pv_eh; + pv_entry_t *pv_et; + int pv_cnt; + unsigned j; + pmap_paddr_t pa; + kern_return_t ret = KERN_SUCCESS; + + while (pv_kern_free_count < pv_kern_low_water_mark) { + pv_cnt = 0; + pv_eh = pv_et = PV_ENTRY_NULL; + + ret = pmap_pages_alloc(&pa, PAGE_SIZE, 0); + assert(ret == KERN_SUCCESS); + + pv_page_count++; + + pv_e = (pv_entry_t *)phystokv(pa); + + for (j = 0; j < (PAGE_SIZE/sizeof(pv_entry_t)) ; j++) { + pv_e->pve_next = pv_eh; + pv_eh = pv_e; + + if (pv_et == PV_ENTRY_NULL) + pv_et = pv_e; + pv_cnt++; + pv_e++; + } + pmap_kernel_reserve_replenish_stat += pv_cnt; + PV_KERN_FREE_LIST(pv_eh, pv_et, pv_cnt); + } + + while (pv_free_count < pv_low_water_mark) { + pv_cnt = 0; + pv_eh = pv_et = PV_ENTRY_NULL; + + ret = pmap_pages_alloc(&pa, PAGE_SIZE, 0); + assert(ret == KERN_SUCCESS); + + pv_page_count++; + + pv_e = (pv_entry_t *)phystokv(pa); + + for (j = 0; j < (PAGE_SIZE/sizeof(pv_entry_t)) ; j++) { + pv_e->pve_next = pv_eh; + pv_eh = pv_e; + + if (pv_et == PV_ENTRY_NULL) + pv_et = pv_e; + pv_cnt++; + pv_e++; + } + pmap_user_reserve_replenish_stat += pv_cnt; + PV_FREE_LIST(pv_eh, pv_et, pv_cnt); + } + + return ret; +} + +/* + * Continuation function that keeps the PV free lists from running out of free + * elements. + */ +__attribute__((noreturn)) +void +mapping_replenish(void) +{ + kern_return_t kr; + + /* We qualify for VM privileges...*/ + current_thread()->options |= TH_OPT_VMPRIV; + + for (;;) { + kr = mapping_replenish_internal(); + + if (kr != KERN_SUCCESS) { + panic("%s: failed, kr=%d", __FUNCTION__, kr); + } + + /* + * Wake threads throttled while the kernel reserve was being replenished. + */ + if (pmap_pv_throttled_waiters) { + pmap_pv_throttled_waiters = 0; + thread_wakeup(&pmap_user_pv_throttle_event); + } + + /* Check if the kernel pool has been depleted since the + * first pass, to reduce refill latency. + */ + if (pv_kern_free_count < pv_kern_low_water_mark) + continue; + /* Block sans continuation to avoid yielding kernel stack */ + assert_wait(&mapping_replenish_event, THREAD_UNINT); + mappingrecurse = 0; + thread_block(THREAD_CONTINUE_NULL); + pmap_mapping_thread_wakeups++; + } +} + + +static void +ptd_bootstrap( + pt_desc_t *ptdp, + unsigned int ptd_cnt) +{ + simple_lock_init(&ptd_free_list_lock, 0); + while (ptd_cnt != 0) { + (*(void **)ptdp) = (void *)ptd_free_list; + ptd_free_list = ptdp; + ptdp++; + ptd_cnt--; + ptd_free_count++; + } + ptd_preboot = FALSE; +} + +static pt_desc_t +*ptd_alloc( + pmap_t pmap) +{ + pt_desc_t *ptdp; + unsigned i; + + if (!ptd_preboot) + simple_lock(&ptd_free_list_lock); + + if (ptd_free_count == 0) { + unsigned int ptd_cnt; + pt_desc_t *ptdp_next; + + if (ptd_preboot) { + ptdp = (pt_desc_t *)avail_start; + avail_start += ARM_PGBYTES; + ptdp_next = ptdp; + ptd_cnt = ARM_PGBYTES/sizeof(pt_desc_t); + } else { + pmap_paddr_t pa; + kern_return_t ret; + + simple_unlock(&ptd_free_list_lock); + + if (pmap_pages_alloc(&pa, PAGE_SIZE, PMAP_PAGES_ALLOCATE_NOWAIT) != KERN_SUCCESS) { + ret = pmap_pages_alloc(&pa, PAGE_SIZE, PMAP_PAGES_RECLAIM_NOWAIT); + assert(ret == KERN_SUCCESS); + } + ptdp = (pt_desc_t *)phystokv(pa); + + simple_lock(&ptd_free_list_lock); + ptdp_next = ptdp; + ptd_cnt = PAGE_SIZE/sizeof(pt_desc_t); + } + + while (ptd_cnt != 0) { + (*(void **)ptdp_next) = (void *)ptd_free_list; + ptd_free_list = ptdp_next; + ptdp_next++; + ptd_cnt--; + ptd_free_count++; + } + } + + if ((ptdp = ptd_free_list) != PTD_ENTRY_NULL) { + ptd_free_list = (pt_desc_t *)(*(void **)ptdp); + ptd_free_count--; + } else { + panic("out of ptd entry\n"); + } + + if (!ptd_preboot) + simple_unlock(&ptd_free_list_lock); + + ptdp->pt_page.next = NULL; + ptdp->pt_page.prev = NULL; + ptdp->pmap = pmap; + + for (i = 0 ; i < PT_INDEX_MAX ; i++) { + ptdp->pt_map[i].va = 0; + ptdp->pt_cnt[i].refcnt = 0; + ptdp->pt_cnt[i].wiredcnt = 0; + } + simple_lock(&pt_pages_lock); + queue_enter(&pt_page_list, ptdp, pt_desc_t *, pt_page); + simple_unlock(&pt_pages_lock); + + pmap_tt_ledger_credit(pmap, sizeof(*ptdp)); + + return(ptdp); +} + +static void +ptd_deallocate( + pt_desc_t *ptdp) +{ + unsigned i; + pmap_t pmap = ptdp->pmap; + + if (ptd_preboot) { + panic("ptd_deallocate(): early boot\n"); + } + for (i = 0 ; i < PT_INDEX_MAX ; i++) { + if (ptdp->pt_cnt[i].refcnt != 0) + panic("ptd_deallocate(): ptdp=%p refcnt=0x%x \n", ptdp, ptdp->pt_cnt[i].refcnt); + } + + if (ptdp->pt_page.next != NULL) { + simple_lock(&pt_pages_lock); + queue_remove(&pt_page_list, ptdp, pt_desc_t *, pt_page); + simple_unlock(&pt_pages_lock); + } + simple_lock(&ptd_free_list_lock); + (*(void **)ptdp) = (void *)ptd_free_list; + ptd_free_list = (pt_desc_t *)ptdp; + ptd_free_count++; + simple_unlock(&ptd_free_list_lock); + pmap_tt_ledger_debit(pmap, sizeof(*ptdp)); +} + +static void +ptd_init( + pt_desc_t *ptdp, + pmap_t pmap, + vm_map_address_t va, + unsigned int level, + pt_entry_t *pte_p) +{ + if (ptdp->pmap != pmap) + panic("ptd_init(): pmap mismatch\n"); + +#if (__ARM_VMSA__ == 7) + assert(level == 2); + ptdp->pt_map[ARM_PT_DESC_INDEX(pte_p)].va = (vm_offset_t) va & ~(ARM_TT_L1_PT_OFFMASK); +#else + if (level == 3) { + ptdp->pt_map[ARM_PT_DESC_INDEX(pte_p)].va = (vm_offset_t) va & ~ARM_TT_L2_OFFMASK ; + } else if (level == 2) + ptdp->pt_map[ARM_PT_DESC_INDEX(pte_p)].va = (vm_offset_t) va & ~ARM_TT_L1_OFFMASK ; +#endif + if (level < PMAP_TT_MAX_LEVEL) + ptdp->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].refcnt = PT_DESC_REFCOUNT; + +} + + +boolean_t +pmap_valid_address( + pmap_paddr_t addr) +{ + return pa_valid(addr); +} + +#if (__ARM_VMSA__ == 7) + +/* + * Given an offset and a map, compute the address of the + * corresponding translation table entry. + */ +static inline tt_entry_t * +pmap_tte(pmap_t pmap, + vm_map_address_t addr) +{ + if (!(tte_index(pmap, addr) < pmap->tte_index_max)) + return (tt_entry_t *)NULL; + return (&pmap->tte[tte_index(pmap, addr)]); +} + + +/* + * Given an offset and a map, compute the address of the + * pte. If the address is invalid with respect to the map + * then PT_ENTRY_NULL is returned (and the map may need to grow). + * + * This is only used internally. + */ +static inline pt_entry_t * +pmap_pte( + pmap_t pmap, + vm_map_address_t addr) +{ + pt_entry_t *ptp; + tt_entry_t *ttp; + tt_entry_t tte; + + ttp = pmap_tte(pmap, addr); + if (ttp == (tt_entry_t *)NULL) + return (PT_ENTRY_NULL); + tte = *ttp; + #if MACH_ASSERT + if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_BLOCK) + panic("Attempt to demote L1 block: pmap=%p, va=0x%llx, tte=0x%llx\n", pmap, (uint64_t)addr, (uint64_t)tte); + #endif + if ((tte & ARM_TTE_TYPE_MASK) != ARM_TTE_TYPE_TABLE) + return (PT_ENTRY_NULL); + ptp = (pt_entry_t *) ttetokv(tte) + ptenum(addr); + return (ptp); +} + +#else + +/* + * Given an offset and a map, compute the address of level 1 translation table entry. + * If the tranlation is invalid then PT_ENTRY_NULL is returned. + */ +static inline tt_entry_t * +pmap_tt1e(pmap_t pmap, + vm_map_address_t addr) +{ +#if __ARM64_TWO_LEVEL_PMAP__ +#pragma unused(pmap, addr) + panic("pmap_tt1e called on a two level pmap"); + return (NULL); +#else + return (&pmap->tte[tt1_index(pmap, addr)]); +#endif +} + +/* + * Given an offset and a map, compute the address of level 2 translation table entry. + * If the tranlation is invalid then PT_ENTRY_NULL is returned. + */ +static inline tt_entry_t * +pmap_tt2e(pmap_t pmap, + vm_map_address_t addr) +{ +#if __ARM64_TWO_LEVEL_PMAP__ + return (&pmap->tte[tt2_index(pmap, addr)]); +#else + tt_entry_t *ttp; + tt_entry_t tte; + + ttp = pmap_tt1e(pmap, addr); + tte = *ttp; + #if MACH_ASSERT + if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) == (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID)) + panic("Attempt to demote L1 block (?!): pmap=%p, va=0x%llx, tte=0x%llx\n", pmap, (uint64_t)addr, (uint64_t)tte); + #endif + if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) != (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID)) + return (PT_ENTRY_NULL); + + ttp = &((tt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt2_index(pmap, addr)]; + return ((tt_entry_t *)ttp); +#endif +} + + +/* + * Given an offset and a map, compute the address of level 3 translation table entry. + * If the tranlation is invalid then PT_ENTRY_NULL is returned. + */ +static inline pt_entry_t * +pmap_tt3e( + pmap_t pmap, + vm_map_address_t addr) +{ + pt_entry_t *ptp; + tt_entry_t *ttp; + tt_entry_t tte; + + /* Level 0 currently unused */ +#if __ARM64_TWO_LEVEL_PMAP__ + ttp = pmap_tt2e(pmap, addr); + tte = *ttp; +#else + /* Get first-level (1GB) entry */ + ttp = pmap_tt1e(pmap, addr); + tte = *ttp; + #if MACH_ASSERT + if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) == (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID)) + panic("Attempt to demote L1 block (?!): pmap=%p, va=0x%llx, tte=0x%llx\n", pmap, (uint64_t)addr, (uint64_t)tte); + #endif + if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) != (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID)) + return (PT_ENTRY_NULL); + + tte = ((tt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt2_index(pmap, addr)]; +#endif +#if MACH_ASSERT + if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) == (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID)) + panic("Attempt to demote L2 block: pmap=%p, va=0x%llx, tte=0x%llx\n", pmap, (uint64_t)addr, (uint64_t)tte); +#endif + if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) != (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID)) { + return (PT_ENTRY_NULL); + } + + /* Get third-level (4KB) entry */ + ptp = &(((pt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt3_index(pmap, addr)]); + return (ptp); +} + + +static inline tt_entry_t * +pmap_tte( + pmap_t pmap, + vm_map_address_t addr) +{ + return(pmap_tt2e(pmap, addr)); +} + + +static inline pt_entry_t * +pmap_pte( + pmap_t pmap, + vm_map_address_t addr) +{ + return(pmap_tt3e(pmap, addr)); +} + +#endif + + +/* + * Map memory at initialization. The physical addresses being + * mapped are not managed and are never unmapped. + * + * For now, VM is already on, we only need to map the + * specified memory. + */ +vm_map_address_t +pmap_map( + vm_map_address_t virt, + vm_offset_t start, + vm_offset_t end, + vm_prot_t prot, + unsigned int flags) +{ + kern_return_t kr; + vm_size_t ps; + + ps = PAGE_SIZE; + while (start < end) { + kr = pmap_enter(kernel_pmap, virt, (ppnum_t)atop(start), + prot, VM_PROT_NONE, flags, FALSE); + + if (kr != KERN_SUCCESS) { + panic("%s: failed pmap_enter, " + "virt=%p, start_addr=%p, end_addr=%p, prot=%#x, flags=%#x", + __FUNCTION__, + (void *) virt, (void *) start, (void *) end, prot, flags); + } + + virt += ps; + start += ps; + } + return (virt); +} + +vm_map_address_t +pmap_map_bd_with_options( + vm_map_address_t virt, + vm_offset_t start, + vm_offset_t end, + vm_prot_t prot, + int32_t options) +{ + pt_entry_t tmplate; + pt_entry_t *ptep; + vm_map_address_t vaddr; + vm_offset_t paddr; + pt_entry_t mem_attr; + + switch (options & PMAP_MAP_BD_MASK) { + case PMAP_MAP_BD_WCOMB: + mem_attr = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITECOMB); +#if (__ARM_VMSA__ > 7) + mem_attr |= ARM_PTE_SH(SH_OUTER_MEMORY); +#else + mem_attr |= ARM_PTE_SH; +#endif + break; + case PMAP_MAP_BD_POSTED: + mem_attr = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_POSTED); + break; + default: + mem_attr = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DISABLE); + break; + } + + tmplate = pa_to_pte(start) | ARM_PTE_AP((prot & VM_PROT_WRITE) ? AP_RWNA : AP_RONA) | + mem_attr | ARM_PTE_TYPE | ARM_PTE_NX | ARM_PTE_PNX | ARM_PTE_AF; + + vaddr = virt; + paddr = start; + while (paddr < end) { + + ptep = pmap_pte(kernel_pmap, vaddr); + if (ptep == PT_ENTRY_NULL) { + panic("pmap_map_bd"); + } + assert(!ARM_PTE_IS_COMPRESSED(*ptep)); + WRITE_PTE(ptep, tmplate); + + pte_increment_pa(tmplate); + vaddr += PAGE_SIZE; + paddr += PAGE_SIZE; + } + + if (end >= start) + flush_mmu_tlb_region(virt, (unsigned)(end - start)); + + return (vaddr); +} + +/* + * Back-door routine for mapping kernel VM at initialization. + * Useful for mapping memory outside the range + * [vm_first_phys, vm_last_phys] (i.e., devices). + * Otherwise like pmap_map. + */ +vm_map_address_t +pmap_map_bd( + vm_map_address_t virt, + vm_offset_t start, + vm_offset_t end, + vm_prot_t prot) +{ + pt_entry_t tmplate; + pt_entry_t *ptep; + vm_map_address_t vaddr; + vm_offset_t paddr; + + /* not cacheable and not buffered */ + tmplate = pa_to_pte(start) + | ARM_PTE_TYPE | ARM_PTE_AF | ARM_PTE_NX | ARM_PTE_PNX + | ARM_PTE_AP((prot & VM_PROT_WRITE) ? AP_RWNA : AP_RONA) + | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DISABLE); + + vaddr = virt; + paddr = start; + while (paddr < end) { + + ptep = pmap_pte(kernel_pmap, vaddr); + if (ptep == PT_ENTRY_NULL) { + panic("pmap_map_bd"); + } + assert(!ARM_PTE_IS_COMPRESSED(*ptep)); + WRITE_PTE(ptep, tmplate); + + pte_increment_pa(tmplate); + vaddr += PAGE_SIZE; + paddr += PAGE_SIZE; + } + + if (end >= start) + flush_mmu_tlb_region(virt, (unsigned)(end - start)); + + return (vaddr); +} + +/* + * Back-door routine for mapping kernel VM at initialization. + * Useful for mapping memory specific physical addresses in early + * boot (i.e., before kernel_map is initialized). + * + * Maps are in the VM_HIGH_KERNEL_WINDOW area. + */ + +vm_map_address_t +pmap_map_high_window_bd( + vm_offset_t pa_start, + vm_size_t len, + vm_prot_t prot) +{ + pt_entry_t *ptep, pte; +#if (__ARM_VMSA__ == 7) + vm_map_address_t va_start = VM_HIGH_KERNEL_WINDOW; + vm_map_address_t va_max = VM_MAX_KERNEL_ADDRESS; +#else + vm_map_address_t va_start = VREGION1_START; + vm_map_address_t va_max = VREGION1_START + VREGION1_SIZE; +#endif + vm_map_address_t va_end; + vm_map_address_t va; + vm_size_t offset; + + offset = pa_start & PAGE_MASK; + pa_start -= offset; + len += offset; + + if (len > (va_max - va_start)) { + panic("pmap_map_high_window_bd: area too large\n"); + } + +scan: + for ( ; va_start < va_max; va_start += PAGE_SIZE) { + ptep = pmap_pte(kernel_pmap, va_start); + assert(!ARM_PTE_IS_COMPRESSED(*ptep)); + if (*ptep == ARM_PTE_TYPE_FAULT) + break; + } + if (va_start > va_max) { + panic("pmap_map_high_window_bd: insufficient pages\n"); + } + + for (va_end = va_start + PAGE_SIZE; va_end < va_start + len; va_end += PAGE_SIZE) { + ptep = pmap_pte(kernel_pmap, va_end); + assert(!ARM_PTE_IS_COMPRESSED(*ptep)); + if (*ptep != ARM_PTE_TYPE_FAULT) { + va_start = va_end + PAGE_SIZE; + goto scan; + } + } + + for (va = va_start; va < va_end; va += PAGE_SIZE, pa_start += PAGE_SIZE) { + ptep = pmap_pte(kernel_pmap, va); + pte = pa_to_pte(pa_start) + | ARM_PTE_TYPE | ARM_PTE_AF | ARM_PTE_NX | ARM_PTE_PNX + | ARM_PTE_AP((prot & VM_PROT_WRITE) ? AP_RWNA : AP_RONA) + | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT); +#if (__ARM_VMSA__ > 7) + pte |= ARM_PTE_SH(SH_OUTER_MEMORY); +#else + pte |= ARM_PTE_SH; +#endif + WRITE_PTE(ptep, pte); + } + PMAP_UPDATE_TLBS(kernel_pmap, va_start, va_start + len); +#if KASAN + kasan_notify_address(va_start, len); +#endif + return va_start; +} + +#define PMAP_ALIGN(addr, align) ((addr) + ((align) - 1) & ~((align) - 1)) + +typedef struct pmap_io_range +{ + uint64_t addr; + uint32_t len; + uint32_t wimg; +} __attribute__((packed)) pmap_io_range_t; + +static unsigned int +pmap_compute_io_rgns(void) +{ + DTEntry entry; + pmap_io_range_t *ranges; + void *prop = NULL; + int err; + unsigned int prop_size; + + err = DTLookupEntry(NULL, "/defaults", &entry); + assert(err == kSuccess); + + if (kSuccess != DTGetProperty(entry, "pmap-io-granule", &prop, &prop_size)) + return 0; + + io_rgn_granule = *((uint32_t*)prop); + + if (kSuccess != DTGetProperty(entry, "pmap-io-ranges", &prop, &prop_size)) + return 0; + + if ((io_rgn_granule == 0) || (io_rgn_granule & PAGE_MASK)) + panic("pmap I/O region granularity is not page-aligned!\n"); + + ranges = prop; + for (unsigned int i = 0; i < (prop_size / sizeof(*ranges)); ++i) { + if ((i == 0) || (ranges[i].addr < io_rgn_start)) + io_rgn_start = ranges[i].addr; + if ((i == 0) || ((ranges[i].addr + ranges[i].len) > io_rgn_end)) + io_rgn_end = ranges[i].addr + ranges[i].len; + } + + if (io_rgn_start & PAGE_MASK) + panic("pmap I/O region start is not page-aligned!\n"); + + if (io_rgn_end & PAGE_MASK) + panic("pmap I/O region end is not page-aligned!\n"); + + if (((io_rgn_start < gPhysBase) && (io_rgn_end >= gPhysBase)) || + ((io_rgn_start < avail_end) && (io_rgn_end >= avail_end))) + panic("pmap I/O region overlaps physical memory!\n"); + + return (unsigned int)((io_rgn_end - io_rgn_start) / io_rgn_granule); +} + +static void +pmap_load_io_rgns(void) +{ + DTEntry entry; + pmap_io_range_t *ranges; + void *prop = NULL; + int err; + unsigned int prop_size; + + if (io_rgn_granule == 0) + return; + + err = DTLookupEntry(NULL, "/defaults", &entry); + assert(err == kSuccess); + + err = DTGetProperty(entry, "pmap-io-ranges", &prop, &prop_size); + assert(err == kSuccess); + + ranges = prop; + for (unsigned int i = 0; i < (prop_size / sizeof(*ranges)); ++i) { + if ((ranges[i].addr - io_rgn_start) % io_rgn_granule) + panic("pmap I/O region %d is not aligned to I/O granularity!\n", i); + if (ranges[i].len % io_rgn_granule) + panic("pmap I/O region %d size is not a multiple of I/O granularity!\n", i); + for (uint32_t offs = 0; offs < ranges[i].len; offs += io_rgn_granule) { + io_attr_table[(ranges[i].addr + offs - io_rgn_start) / io_rgn_granule] = + IO_ATTR_WIMG(ranges[i].wimg); + } + } +} + + +/* + * Bootstrap the system enough to run with virtual memory. + * + * The early VM initialization code has already allocated + * the first CPU's translation table and made entries for + * all the one-to-one mappings to be found there. + * + * We must set up the kernel pmap structures, the + * physical-to-virtual translation lookup tables for the + * physical memory to be managed (between avail_start and + * avail_end). + + * Map the kernel's code and data, and allocate the system page table. + * Page_size must already be set. + * + * Parameters: + * first_avail first available physical page - + * after kernel page tables + * avail_start PA of first managed physical page + * avail_end PA of last managed physical page + */ + +void +pmap_bootstrap( + vm_offset_t vstart) +{ + pmap_paddr_t pmap_struct_start; + vm_size_t pv_head_size; + vm_size_t pv_lock_table_size; + vm_size_t ptd_root_table_size; + vm_size_t pp_attr_table_size; + vm_size_t io_attr_table_size; + unsigned int niorgns; + unsigned int npages; + unsigned int i; + vm_map_offset_t maxoffset; + + +#ifdef PMAP_TRACES + if (PE_parse_boot_argn("-pmap_trace", &pmap_trace, sizeof (pmap_trace))) { + kprintf("Kernel traces for pmap operations enabled\n"); + } +#endif + + /* + * Initialize the kernel pmap. + */ + pmap_stamp = 1; + kernel_pmap->tte = cpu_tte; + kernel_pmap->ttep = cpu_ttep; +#if (__ARM_VMSA__ > 7) + kernel_pmap->min = ARM64_TTBR1_MIN_ADDR; +#else + kernel_pmap->min = VM_MIN_KERNEL_AND_KEXT_ADDRESS; +#endif + kernel_pmap->max = VM_MAX_KERNEL_ADDRESS; + kernel_pmap->wired = 0; + kernel_pmap->ref_count = 1; + kernel_pmap->gc_status = 0; + kernel_pmap->nx_enabled = TRUE; +#ifdef __arm64__ + kernel_pmap->is_64bit = TRUE; +#else + kernel_pmap->is_64bit = FALSE; +#endif + kernel_pmap->stamp = hw_atomic_add(&pmap_stamp, 1); + + kernel_pmap->nested_region_grand_addr = 0x0ULL; + kernel_pmap->nested_region_subord_addr = 0x0ULL; + kernel_pmap->nested_region_size = 0x0ULL; + kernel_pmap->nested_region_asid_bitmap = NULL; + kernel_pmap->nested_region_asid_bitmap_size = 0x0UL; + +#if (__ARM_VMSA__ == 7) + kernel_pmap->tte_index_max = 4*NTTES; +#else + kernel_pmap->tte_index_max = (ARM_PGBYTES / sizeof(tt_entry_t)); +#endif + kernel_pmap->prev_tte = (tt_entry_t *) NULL; + kernel_pmap->cpu_ref = 0; + + PMAP_LOCK_INIT(kernel_pmap); +#if (__ARM_VMSA__ == 7) + simple_lock_init(&kernel_pmap->tt1_lock, 0); +#endif + memset((void *) &kernel_pmap->stats, 0, sizeof(kernel_pmap->stats)); + + /* allocate space for and initialize the bookkeeping structures */ + niorgns = pmap_compute_io_rgns(); + npages = (unsigned int)atop(mem_size); + pp_attr_table_size = npages * sizeof(pp_attr_t); + io_attr_table_size = niorgns * sizeof(io_attr_t); + pv_lock_table_size = npages; + pv_head_size = round_page(sizeof(pv_entry_t *) * npages); +#if (__ARM_VMSA__ == 7) + ptd_root_table_size = sizeof(pt_desc_t) * (1<<((mem_size>>30)+12)); +#else + ptd_root_table_size = sizeof(pt_desc_t) * (1<<((mem_size>>30)+13)); +#endif + + pmap_struct_start = avail_start; + + pp_attr_table = (pp_attr_t *) phystokv(avail_start); + avail_start = PMAP_ALIGN(avail_start + pp_attr_table_size, __alignof(pp_attr_t)); + io_attr_table = (io_attr_t *) phystokv(avail_start); + avail_start = PMAP_ALIGN(avail_start + io_attr_table_size + pv_lock_table_size, __alignof(pv_entry_t*)); + pv_head_table = (pv_entry_t **) phystokv(avail_start); + avail_start = PMAP_ALIGN(avail_start + pv_head_size, __alignof(pt_desc_t)); + ptd_root_table = (pt_desc_t *)phystokv(avail_start); + avail_start = round_page(avail_start + ptd_root_table_size); + + memset((char *)phystokv(pmap_struct_start), 0, avail_start - pmap_struct_start); + + pmap_load_io_rgns(); + ptd_bootstrap(ptd_root_table, (unsigned int)(ptd_root_table_size/sizeof(pt_desc_t))); + + pmap_cpu_data_array_init(); + + vm_first_phys = gPhysBase; + vm_last_phys = trunc_page(avail_end); + + simple_lock_init(&pmaps_lock, 0); + queue_init(&map_pmap_list); + queue_enter(&map_pmap_list, kernel_pmap, pmap_t, pmaps); + queue_init(&tt_pmap_list); + tt_pmap_count = 0; + tt_pmap_max = 0; + free_page_size_tt_list = TT_FREE_ENTRY_NULL; + free_page_size_tt_count = 0; + free_page_size_tt_max = 0; + free_two_page_size_tt_list = TT_FREE_ENTRY_NULL; + free_two_page_size_tt_count = 0; + free_two_page_size_tt_max = 0; + free_tt_list = TT_FREE_ENTRY_NULL; + free_tt_count = 0; + free_tt_max = 0; + + simple_lock_init(&pt_pages_lock, 0); + queue_init(&pt_page_list); + + simple_lock_init(&pmap_pages_lock, 0); + pmap_pages_request_count = 0; + pmap_pages_request_acum = 0; + pmap_pages_reclaim_list = PAGE_FREE_ENTRY_NULL; + + virtual_space_start = vstart; + virtual_space_end = VM_MAX_KERNEL_ADDRESS; + + /* mark all the address spaces in use */ + for (i = 0; i < MAX_ASID / (sizeof(uint32_t) * NBBY); i++) + asid_bitmap[i] = 0xffffffff; + + /* + * The kernel gets ASID 0, and all aliases of it. This is + * important because ASID 0 is global; if we vend ASID 0 + * out to a user pmap, those translations will show up in + * other processes through the TLB. + */ + for (i = 0; i < MAX_ASID; i += ARM_MAX_ASID) { + asid_bitmap[i / (sizeof(uint32_t) * NBBY)] &= ~(1 << (i % (sizeof(uint32_t) * NBBY))); + } + + kernel_pmap->asid = 0; + kernel_pmap->vasid = 0; + + if (PE_parse_boot_argn("arm_maxoffset", &maxoffset, sizeof (maxoffset))) { + maxoffset = trunc_page(maxoffset); + if ((maxoffset >= pmap_max_offset(FALSE, ARM_PMAP_MAX_OFFSET_MIN)) + && (maxoffset <= pmap_max_offset(FALSE, ARM_PMAP_MAX_OFFSET_MAX))) { + arm_pmap_max_offset_default = maxoffset; + } + } +#if defined(__arm64__) + if (PE_parse_boot_argn("arm64_maxoffset", &maxoffset, sizeof (maxoffset))) { + maxoffset = trunc_page(maxoffset); + if ((maxoffset >= pmap_max_offset(TRUE, ARM_PMAP_MAX_OFFSET_MIN)) + && (maxoffset <= pmap_max_offset(TRUE, ARM_PMAP_MAX_OFFSET_MAX))) { + arm64_pmap_max_offset_default = maxoffset; + } + } +#endif + +#if DEVELOPMENT || DEBUG + PE_parse_boot_argn("panic_on_unsigned_execute", &panic_on_unsigned_execute, sizeof (panic_on_unsigned_execute)); +#endif /* DEVELOPMENT || DEBUG */ + + pmap_nesting_size_min = ARM_NESTING_SIZE_MIN; + pmap_nesting_size_max = ARM_NESTING_SIZE_MAX; + + simple_lock_init(&phys_backup_lock, 0); +} + + +void +pmap_virtual_space( + vm_offset_t *startp, + vm_offset_t *endp +) +{ + *startp = virtual_space_start; + *endp = virtual_space_end; +} + + +boolean_t +pmap_virtual_region( + unsigned int region_select, + vm_map_offset_t *startp, + vm_map_size_t *size +) +{ + boolean_t ret = FALSE; +#if __ARM64_PMAP_SUBPAGE_L1__ && __ARM_16K_PG__ + if (region_select == 0) { + /* + * In this config, the bootstrap mappings should occupy their own L2 + * TTs, as they should be immutable after boot. Having the associated + * TTEs and PTEs in their own pages allows us to lock down those pages, + * while allowing the rest of the kernel address range to be remapped. + */ +#if (__ARM_VMSA__ > 7) + *startp = LOW_GLOBAL_BASE_ADDRESS & ~ARM_TT_L2_OFFMASK; +#else +#error Unsupported configuration +#endif + *size = ((VM_MAX_KERNEL_ADDRESS - *startp) & ~PAGE_MASK); + ret = TRUE; + } +#else +#if (__ARM_VMSA__ > 7) + unsigned long low_global_vr_mask = 0; + vm_map_size_t low_global_vr_size = 0; +#endif + + if (region_select == 0) { +#if (__ARM_VMSA__ == 7) + *startp = gVirtBase & 0xFFC00000; + *size = ((virtual_space_start-(gVirtBase & 0xFFC00000)) + ~0xFFC00000) & 0xFFC00000; +#else + /* Round to avoid overlapping with the V=P area; round to at least the L2 block size. */ + if (!TEST_PAGE_SIZE_4K) { + *startp = gVirtBase & 0xFFFFFFFFFE000000; + *size = ((virtual_space_start-(gVirtBase & 0xFFFFFFFFFE000000)) + ~0xFFFFFFFFFE000000) & 0xFFFFFFFFFE000000; + } else { + *startp = gVirtBase & 0xFFFFFFFFFF800000; + *size = ((virtual_space_start-(gVirtBase & 0xFFFFFFFFFF800000)) + ~0xFFFFFFFFFF800000) & 0xFFFFFFFFFF800000; + } +#endif + ret = TRUE; + } + if (region_select == 1) { + *startp = VREGION1_START; + *size = VREGION1_SIZE; + ret = TRUE; + } +#if (__ARM_VMSA__ > 7) + /* We need to reserve a range that is at least the size of an L2 block mapping for the low globals */ + if (!TEST_PAGE_SIZE_4K) { + low_global_vr_mask = 0xFFFFFFFFFE000000; + low_global_vr_size = 0x2000000; + } else { + low_global_vr_mask = 0xFFFFFFFFFF800000; + low_global_vr_size = 0x800000; + } + + if (((gVirtBase & low_global_vr_mask) != LOW_GLOBAL_BASE_ADDRESS) && (region_select == 2)) { + *startp = LOW_GLOBAL_BASE_ADDRESS; + *size = low_global_vr_size; + ret = TRUE; + } + + if (region_select == 3) { + /* In this config, we allow the bootstrap mappings to occupy the same + * page table pages as the heap. + */ + *startp = VM_MIN_KERNEL_ADDRESS; + *size = LOW_GLOBAL_BASE_ADDRESS - *startp; + ret = TRUE; + } +#endif +#endif + return ret; +} + +unsigned int +pmap_free_pages( + void) +{ + return (unsigned int)atop(avail_end - first_avail); +} + + +boolean_t +pmap_next_page_hi( + ppnum_t * pnum) +{ + return pmap_next_page(pnum); +} + + +boolean_t +pmap_next_page( + ppnum_t *pnum) +{ + if (first_avail != avail_end) { + *pnum = (ppnum_t)atop(first_avail); + first_avail += PAGE_SIZE; + return TRUE; + } + return FALSE; +} + + +/* + * Initialize the pmap module. + * Called by vm_init, to initialize any structures that the pmap + * system needs to map virtual memory. + */ +void +pmap_init( + void) +{ + /* + * Protect page zero in the kernel map. + * (can be overruled by permanent transltion + * table entries at page zero - see arm_vm_init). + */ + vm_protect(kernel_map, 0, PAGE_SIZE, TRUE, VM_PROT_NONE); + + pmap_initialized = TRUE; + + pmap_zone_init(); + + + /* + * Initialize the pmap object (for tracking the vm_page_t + * structures for pages we allocate to be page tables in + * pmap_expand(). + */ + _vm_object_allocate(mem_size, pmap_object); + pmap_object->copy_strategy = MEMORY_OBJECT_COPY_NONE; + + pv_init(); + + /* + * The value of hard_maxproc may have been scaled, make sure + * it is still less than the value of MAX_ASID. + */ + assert(hard_maxproc < MAX_ASID); + +#if CONFIG_PGTRACE + pmap_pgtrace_init(); +#endif +} + +boolean_t +pmap_verify_free( + ppnum_t ppnum) +{ + pv_entry_t **pv_h; + int pai; + boolean_t result = TRUE; + pmap_paddr_t phys = ptoa(ppnum); + + assert(phys != vm_page_fictitious_addr); + + if (!pa_valid(phys)) + return (FALSE); + + pai = (int)pa_index(phys); + pv_h = pai_to_pvh(pai); + + result = (pvh_list(pv_h) == PV_ENTRY_NULL); + + return (result); +} + + +/* + * Initialize zones used by pmap. + */ +static void +pmap_zone_init( + void) +{ + /* + * Create the zone of physical maps + * and the physical-to-virtual entries. + */ + pmap_zone = zinit((vm_size_t) sizeof(struct pmap), (vm_size_t) sizeof(struct pmap)*256, + PAGE_SIZE, "pmap"); +} + + +/* + * Create and return a physical map. + * + * If the size specified for the map + * is zero, the map is an actual physical + * map, and may be referenced by the + * hardware. + * + * If the size specified is non-zero, + * the map will be used in software only, and + * is bounded by that size. + */ +static pmap_t +pmap_create_internal( + ledger_t ledger, + vm_map_size_t size, + boolean_t is_64bit) +{ + unsigned i; + pmap_t p; + + /* + * A software use-only map doesn't even need a pmap. + */ + if (size != 0) { + return (PMAP_NULL); + } + + + /* + * Allocate a pmap struct from the pmap_zone. Then allocate + * the translation table of the right size for the pmap. + */ + if ((p = (pmap_t) zalloc(pmap_zone)) == PMAP_NULL) + return (PMAP_NULL); + + if (is_64bit) { + p->min = MACH_VM_MIN_ADDRESS; + p->max = MACH_VM_MAX_ADDRESS; + } else { + p->min = VM_MIN_ADDRESS; + p->max = VM_MAX_ADDRESS; + } + + p->wired = 0; + p->ref_count = 1; + p->gc_status = 0; + p->stamp = hw_atomic_add(&pmap_stamp, 1); + p->nx_enabled = TRUE; + p->is_64bit = is_64bit; + p->nested = FALSE; + p->nested_pmap = PMAP_NULL; + + + ledger_reference(ledger); + p->ledger = ledger; + + PMAP_LOCK_INIT(p); +#if (__ARM_VMSA__ == 7) + simple_lock_init(&p->tt1_lock, 0); +#endif + memset((void *) &p->stats, 0, sizeof(p->stats)); + + p->tt_entry_free = (tt_entry_t *)0; + + p->tte = pmap_tt1_allocate(p, PMAP_ROOT_ALLOC_SIZE, 0); + p->ttep = ml_static_vtop((vm_offset_t)p->tte); + +#if (__ARM_VMSA__ == 7) + p->tte_index_max = NTTES; +#else + p->tte_index_max = (PMAP_ROOT_ALLOC_SIZE / sizeof(tt_entry_t)); +#endif + p->prev_tte = (tt_entry_t *) NULL; + p->cpu_ref = 0; + + /* nullify the translation table */ + for (i = 0; i < p->tte_index_max; i++) + p->tte[i] = ARM_TTE_TYPE_FAULT; + +#ifndef __ARM_L1_PTW__ + CleanPoU_DcacheRegion((vm_offset_t) (p->tte), PMAP_ROOT_ALLOC_SIZE); +#else + __asm__ volatile("dsb ish"); +#endif + /* assign a asid */ + p->vasid = alloc_asid(); + p->asid = p->vasid % ARM_MAX_ASID; + + /* + * initialize the rest of the structure + */ + p->nested_region_grand_addr = 0x0ULL; + p->nested_region_subord_addr = 0x0ULL; + p->nested_region_size = 0x0ULL; + p->nested_region_asid_bitmap = NULL; + p->nested_region_asid_bitmap_size = 0x0UL; + +#if MACH_ASSERT + p->pmap_pid = 0; + strlcpy(p->pmap_procname, "<nil>", sizeof (p->pmap_procname)); +#endif /* MACH_ASSERT */ +#if DEVELOPMENT || DEBUG + p->footprint_suspended = FALSE; + p->footprint_was_suspended = FALSE; +#endif /* DEVELOPMENT || DEBUG */ + + simple_lock(&pmaps_lock); + queue_enter(&map_pmap_list, p, pmap_t, pmaps); + simple_unlock(&pmaps_lock); + + return (p); +} + +pmap_t +pmap_create( + ledger_t ledger, + vm_map_size_t size, + boolean_t is_64bit) +{ + pmap_t pmap; + + PMAP_TRACE(PMAP_CODE(PMAP__CREATE) | DBG_FUNC_START, size, is_64bit); + + pmap = pmap_create_internal(ledger, size, is_64bit); + + PMAP_TRACE(PMAP_CODE(PMAP__CREATE) | DBG_FUNC_END, + VM_KERNEL_ADDRHIDE(pmap)); + + return pmap; +} + +#if MACH_ASSERT +static void +pmap_set_process_internal( + __unused pmap_t pmap, + __unused int pid, + __unused char *procname) +{ +#if MACH_ASSERT + if (pmap == NULL) { + return; + } + + pmap->pmap_pid = pid; + strlcpy(pmap->pmap_procname, procname, sizeof (pmap->pmap_procname)); +#endif +} +#endif + +#if MACH_ASSERT +void +pmap_set_process( + pmap_t pmap, + int pid, + char *procname) +{ + pmap_set_process_internal(pmap, pid, procname); +} + +/* + * We maintain stats and ledgers so that a task's physical footprint is: + * phys_footprint = ((internal - alternate_accounting) + * + (internal_compressed - alternate_accounting_compressed) + * + iokit_mapped + * + purgeable_nonvolatile + * + purgeable_nonvolatile_compressed + * + page_table) + * where "alternate_accounting" includes "iokit" and "purgeable" memory. + */ + +struct { + uint64_t num_pmaps_checked; + + int phys_footprint_over; + ledger_amount_t phys_footprint_over_total; + ledger_amount_t phys_footprint_over_max; + int phys_footprint_under; + ledger_amount_t phys_footprint_under_total; + ledger_amount_t phys_footprint_under_max; + + int internal_over; + ledger_amount_t internal_over_total; + ledger_amount_t internal_over_max; + int internal_under; + ledger_amount_t internal_under_total; + ledger_amount_t internal_under_max; + + int internal_compressed_over; + ledger_amount_t internal_compressed_over_total; + ledger_amount_t internal_compressed_over_max; + int internal_compressed_under; + ledger_amount_t internal_compressed_under_total; + ledger_amount_t internal_compressed_under_max; + + int iokit_mapped_over; + ledger_amount_t iokit_mapped_over_total; + ledger_amount_t iokit_mapped_over_max; + int iokit_mapped_under; + ledger_amount_t iokit_mapped_under_total; + ledger_amount_t iokit_mapped_under_max; + + int alternate_accounting_over; + ledger_amount_t alternate_accounting_over_total; + ledger_amount_t alternate_accounting_over_max; + int alternate_accounting_under; + ledger_amount_t alternate_accounting_under_total; + ledger_amount_t alternate_accounting_under_max; + + int alternate_accounting_compressed_over; + ledger_amount_t alternate_accounting_compressed_over_total; + ledger_amount_t alternate_accounting_compressed_over_max; + int alternate_accounting_compressed_under; + ledger_amount_t alternate_accounting_compressed_under_total; + ledger_amount_t alternate_accounting_compressed_under_max; + + int page_table_over; + ledger_amount_t page_table_over_total; + ledger_amount_t page_table_over_max; + int page_table_under; + ledger_amount_t page_table_under_total; + ledger_amount_t page_table_under_max; + + int purgeable_volatile_over; + ledger_amount_t purgeable_volatile_over_total; + ledger_amount_t purgeable_volatile_over_max; + int purgeable_volatile_under; + ledger_amount_t purgeable_volatile_under_total; + ledger_amount_t purgeable_volatile_under_max; + + int purgeable_nonvolatile_over; + ledger_amount_t purgeable_nonvolatile_over_total; + ledger_amount_t purgeable_nonvolatile_over_max; + int purgeable_nonvolatile_under; + ledger_amount_t purgeable_nonvolatile_under_total; + ledger_amount_t purgeable_nonvolatile_under_max; + + int purgeable_volatile_compressed_over; + ledger_amount_t purgeable_volatile_compressed_over_total; + ledger_amount_t purgeable_volatile_compressed_over_max; + int purgeable_volatile_compressed_under; + ledger_amount_t purgeable_volatile_compressed_under_total; + ledger_amount_t purgeable_volatile_compressed_under_max; + + int purgeable_nonvolatile_compressed_over; + ledger_amount_t purgeable_nonvolatile_compressed_over_total; + ledger_amount_t purgeable_nonvolatile_compressed_over_max; + int purgeable_nonvolatile_compressed_under; + ledger_amount_t purgeable_nonvolatile_compressed_under_total; + ledger_amount_t purgeable_nonvolatile_compressed_under_max; +} pmap_ledgers_drift; +#endif /* MACH_ASSERT */ + +/* + * Retire the given physical map from service. + * Should only be called if the map contains + * no valid mappings. + */ +static void +pmap_destroy_internal( + pmap_t pmap) +{ +#if (__ARM_VMSA__ == 7) + pt_entry_t *ttep; + unsigned int i; + pmap_t tmp_pmap, tt_pmap; + queue_head_t tmp_pmap_list; + + queue_init(&tmp_pmap_list); + simple_lock(&pmaps_lock); + tt_pmap = CAST_DOWN_EXPLICIT(pmap_t, queue_first(&tt_pmap_list)); + while (!queue_end(&tt_pmap_list, (queue_entry_t)tt_pmap)) { + if (tt_pmap->cpu_ref == 0 ) { + tmp_pmap = tt_pmap; + tt_pmap = CAST_DOWN_EXPLICIT(pmap_t, queue_next(&tmp_pmap->pmaps)); + queue_remove(&tt_pmap_list, tmp_pmap, pmap_t, pmaps); + tt_pmap_count--; + queue_enter(&tmp_pmap_list, tmp_pmap, pmap_t, pmaps); + } else { + tmp_pmap = tt_pmap; + tt_pmap = CAST_DOWN_EXPLICIT(pmap_t, queue_next(&tmp_pmap->pmaps)); + } + } + simple_unlock(&pmaps_lock); + + tmp_pmap = CAST_DOWN_EXPLICIT(pmap_t, queue_first(&tmp_pmap_list)); + while (!queue_end(&tmp_pmap_list, (queue_entry_t)tmp_pmap)) { + tt_pmap = tmp_pmap; + tmp_pmap = CAST_DOWN_EXPLICIT(pmap_t, queue_next(&tt_pmap->pmaps)); + queue_remove(&tmp_pmap_list, tt_pmap, pmap_t, pmaps); + if (tt_pmap->tte) { + pmap_tt1_deallocate(pmap, tt_pmap->tte, tt_pmap->tte_index_max*sizeof(tt_entry_t), 0); + tt_pmap->tte = (tt_entry_t *) NULL; + tt_pmap->ttep = 0; + tt_pmap->tte_index_max = 0; + } + if (tt_pmap->prev_tte) { + pmap_tt1_deallocate(pmap, tt_pmap->prev_tte, PMAP_ROOT_ALLOC_SIZE, 0); + tt_pmap->prev_tte = (tt_entry_t *) NULL; + } + assert((tt_free_entry_t*)pmap->tt_entry_free == NULL); + free_asid(tt_pmap->vasid); + + pmap_check_ledgers(tt_pmap); + ledger_dereference(tt_pmap->ledger); + + zfree(pmap_zone, tt_pmap); + } + + if (pmap == PMAP_NULL) + return; + + if (hw_atomic_sub(&pmap->ref_count, 1) != 0) + return; + + simple_lock(&pmaps_lock); + + while (pmap->gc_status & PMAP_GC_INFLIGHT) { + pmap->gc_status |= PMAP_GC_WAIT; + assert_wait((event_t) & pmap->gc_status, THREAD_UNINT); + simple_unlock(&pmaps_lock); + (void) thread_block(THREAD_CONTINUE_NULL); + simple_lock(&pmaps_lock); + + } + + queue_remove(&map_pmap_list, pmap, pmap_t, pmaps); + simple_unlock(&pmaps_lock); + + /* + * Free the memory maps, then the + * pmap structure. + */ + PMAP_LOCK(pmap); + for (i = 0; i < pmap->tte_index_max; i++) { + ttep = &pmap->tte[i]; + if ((*ttep & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) { + pmap_tte_deallocate(pmap, ttep, PMAP_TT_L1_LEVEL); + flush_mmu_tlb_entry((i<<ARM_TT_L1_SHIFT) | (pmap->asid & 0xff)); + } + } + PMAP_UNLOCK(pmap); + + if (pmap->cpu_ref == 0) { + if (pmap->tte) { + pmap_tt1_deallocate(pmap, pmap->tte, pmap->tte_index_max*sizeof(tt_entry_t), 0); + pmap->tte = (tt_entry_t *) NULL; + pmap->ttep = 0; + pmap->tte_index_max = 0; + } + if (pmap->prev_tte) { + pmap_tt1_deallocate(pmap, pmap->prev_tte, PMAP_ROOT_ALLOC_SIZE, 0); + pmap->prev_tte = (tt_entry_t *) NULL; + } + assert((tt_free_entry_t*)pmap->tt_entry_free == NULL); + + /* return its asid to the pool */ + free_asid(pmap->vasid); + pmap_check_ledgers(pmap); + + ledger_dereference(pmap->ledger); + if (pmap->nested_region_asid_bitmap) + kfree(pmap->nested_region_asid_bitmap, pmap->nested_region_asid_bitmap_size*sizeof(unsigned int)); + zfree(pmap_zone, pmap); + } else { + simple_lock(&pmaps_lock); + queue_enter(&tt_pmap_list, pmap, pmap_t, pmaps); + tt_pmap_count++; + if (tt_pmap_count > tt_pmap_max) + tt_pmap_max = tt_pmap_count; + simple_unlock(&pmaps_lock); + } +#else + pt_entry_t *ttep; + pmap_paddr_t pa; + vm_map_address_t c; + + if (pmap == PMAP_NULL) { + return; + } + + if (!pmap->is_64bit) + pmap_unmap_sharedpage32(pmap); + + if (hw_atomic_sub(&pmap->ref_count, 1) == 0) { + + simple_lock(&pmaps_lock); + while (pmap->gc_status & PMAP_GC_INFLIGHT) { + pmap->gc_status |= PMAP_GC_WAIT; + assert_wait((event_t) & pmap->gc_status, THREAD_UNINT); + simple_unlock(&pmaps_lock); + (void) thread_block(THREAD_CONTINUE_NULL); + simple_lock(&pmaps_lock); + } + queue_remove(&map_pmap_list, pmap, pmap_t, pmaps); + simple_unlock(&pmaps_lock); + + /* + * Free the memory maps, then the + * pmap structure. + */ + for (c = pmap->min; c < pmap->max; c += ARM_TT_L2_SIZE) { + ttep = pmap_tt2e(pmap, c); + if ((ttep != PT_ENTRY_NULL) && (*ttep & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) { + PMAP_LOCK(pmap); + pmap_tte_deallocate(pmap, ttep, PMAP_TT_L2_LEVEL); + PMAP_UNLOCK(pmap); + flush_mmu_tlb_entry(tlbi_addr(c) | tlbi_asid(pmap->asid)); + } + } +#if !__ARM64_TWO_LEVEL_PMAP__ + for (c = pmap->min; c < pmap->max; c += ARM_TT_L1_SIZE) { + ttep = pmap_tt1e(pmap, c); + if ((ttep != PT_ENTRY_NULL) && (*ttep & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) { + PMAP_LOCK(pmap); + pmap_tte_deallocate(pmap, ttep, PMAP_TT_L1_LEVEL); + PMAP_UNLOCK(pmap); + } + } +#endif + + if (pmap->tte) { + pa = pmap->ttep; + pmap_tt1_deallocate(pmap, (tt_entry_t *)phystokv(pa), PMAP_ROOT_ALLOC_SIZE, 0); + } + + + assert((tt_free_entry_t*)pmap->tt_entry_free == NULL); + + flush_mmu_tlb_asid((uint64_t)(pmap->asid) << TLBI_ASID_SHIFT); + free_asid(pmap->vasid); + + if (pmap->nested_region_asid_bitmap) { + kfree(pmap->nested_region_asid_bitmap, pmap->nested_region_asid_bitmap_size*sizeof(unsigned int)); + } + + pmap_check_ledgers(pmap); + ledger_dereference(pmap->ledger); + + zfree(pmap_zone, pmap); + } + +#endif +} + +void +pmap_destroy( + pmap_t pmap) +{ + PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_START, + VM_KERNEL_ADDRHIDE(pmap)); + + pmap_destroy_internal(pmap); + + PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_END); +} + + +/* + * Add a reference to the specified pmap. + */ +static void +pmap_reference_internal( + pmap_t pmap) +{ + if (pmap != PMAP_NULL) { + (void) hw_atomic_add(&pmap->ref_count, 1); + } +} + +void +pmap_reference( + pmap_t pmap) +{ + pmap_reference_internal(pmap); +} + +static tt_entry_t * +pmap_tt1_allocate( + pmap_t pmap, + vm_size_t size, + unsigned option) +{ + tt_entry_t *tt1; + tt_free_entry_t *tt1_free; + pmap_paddr_t pa; + vm_address_t va; + vm_address_t va_end; + kern_return_t ret; + + simple_lock(&pmaps_lock); + if ((size == PAGE_SIZE) && (free_page_size_tt_count != 0)) { + free_page_size_tt_count--; + tt1 = (tt_entry_t *)free_page_size_tt_list; + free_page_size_tt_list = ((tt_free_entry_t *)tt1)->next; + simple_unlock(&pmaps_lock); + pmap_tt_ledger_credit(pmap, size); + return (tt_entry_t *)tt1; + }; + if ((size == 2*PAGE_SIZE) && (free_two_page_size_tt_count != 0)) { + free_two_page_size_tt_count--; + tt1 = (tt_entry_t *)free_two_page_size_tt_list; + free_two_page_size_tt_list = ((tt_free_entry_t *)tt1)->next; + simple_unlock(&pmaps_lock); + pmap_tt_ledger_credit(pmap, size); + return (tt_entry_t *)tt1; + }; + if (free_tt_count != 0) { + free_tt_count--; + tt1 = (tt_entry_t *)free_tt_list; + free_tt_list = (tt_free_entry_t *)((tt_free_entry_t *)tt1)->next; + simple_unlock(&pmaps_lock); + pmap_tt_ledger_credit(pmap, size); + return (tt_entry_t *)tt1; + } + + simple_unlock(&pmaps_lock); + + ret = pmap_pages_alloc(&pa, (unsigned)((size < PAGE_SIZE)? PAGE_SIZE : size), ((option & PMAP_TT_ALLOCATE_NOWAIT)? PMAP_PAGES_ALLOCATE_NOWAIT : 0)); + + if(ret == KERN_RESOURCE_SHORTAGE) + return (tt_entry_t *)0; + + + if (size < PAGE_SIZE) { + simple_lock(&pmaps_lock); + + for (va_end = phystokv(pa) + PAGE_SIZE, va = phystokv(pa) + size; va < va_end; va = va+size) { + tt1_free = (tt_free_entry_t *)va; + tt1_free->next = free_tt_list; + free_tt_list = tt1_free; + free_tt_count++; + } + if (free_tt_count > free_tt_max) + free_tt_max = free_tt_count; + + simple_unlock(&pmaps_lock); + } + + /* Always report root allocations in units of PMAP_ROOT_ALLOC_SIZE, which can be obtained by sysctl arm_pt_root_size. + * Depending on the device, this can vary between 512b and 16K. */ + OSAddAtomic((uint32_t)(size / PMAP_ROOT_ALLOC_SIZE), (pmap == kernel_pmap ? &inuse_kernel_tteroot_count : &inuse_user_tteroot_count)); + OSAddAtomic64(size / PMAP_ROOT_ALLOC_SIZE, &alloc_tteroot_count); + pmap_tt_ledger_credit(pmap, size); + + return (tt_entry_t *) phystokv(pa); +} + +static void +pmap_tt1_deallocate( + pmap_t pmap, + tt_entry_t *tt, + vm_size_t size, + unsigned option) +{ + tt_free_entry_t *tt_entry; + + tt_entry = (tt_free_entry_t *)tt; + if (not_in_kdp) + simple_lock(&pmaps_lock); + + if (size < PAGE_SIZE) { + free_tt_count++; + if (free_tt_count > free_tt_max) + free_tt_max = free_tt_count; + tt_entry->next = free_tt_list; + free_tt_list = tt_entry; + } + + if (size == PAGE_SIZE) { + free_page_size_tt_count++; + if (free_page_size_tt_count > free_page_size_tt_max) + free_page_size_tt_max = free_page_size_tt_count; + tt_entry->next = free_page_size_tt_list; + free_page_size_tt_list = tt_entry; + } + + if (size == 2*PAGE_SIZE) { + free_two_page_size_tt_count++; + if (free_two_page_size_tt_count > free_two_page_size_tt_max) + free_two_page_size_tt_max = free_two_page_size_tt_count; + tt_entry->next = free_two_page_size_tt_list; + free_two_page_size_tt_list = tt_entry; + } + + if ((option & PMAP_TT_DEALLOCATE_NOBLOCK) || (!not_in_kdp)) { + if (not_in_kdp) + simple_unlock(&pmaps_lock); + pmap_tt_ledger_debit(pmap, size); + return; + } + + while (free_page_size_tt_count > FREE_PAGE_SIZE_TT_MAX) { + + free_page_size_tt_count--; + tt = (tt_entry_t *)free_page_size_tt_list; + free_page_size_tt_list = ((tt_free_entry_t *)tt)->next; + + simple_unlock(&pmaps_lock); + + pmap_pages_free(ml_static_vtop((vm_offset_t)tt), PAGE_SIZE); + + OSAddAtomic(-(int32_t)(PAGE_SIZE / PMAP_ROOT_ALLOC_SIZE), (pmap == kernel_pmap ? &inuse_kernel_tteroot_count : &inuse_user_tteroot_count)); + + simple_lock(&pmaps_lock); + } + + while (free_two_page_size_tt_count > FREE_TWO_PAGE_SIZE_TT_MAX) { + free_two_page_size_tt_count--; + tt = (tt_entry_t *)free_two_page_size_tt_list; + free_two_page_size_tt_list = ((tt_free_entry_t *)tt)->next; + + simple_unlock(&pmaps_lock); + + pmap_pages_free(ml_static_vtop((vm_offset_t)tt), 2*PAGE_SIZE); + + OSAddAtomic(-2 * (int32_t)(PAGE_SIZE / PMAP_ROOT_ALLOC_SIZE), (pmap == kernel_pmap ? &inuse_kernel_tteroot_count : &inuse_user_tteroot_count)); + + simple_lock(&pmaps_lock); + } + simple_unlock(&pmaps_lock); + pmap_tt_ledger_debit(pmap, size); +} + +static kern_return_t +pmap_tt_allocate( + pmap_t pmap, + tt_entry_t **ttp, + unsigned int level, + unsigned int options) +{ + pmap_paddr_t pa; + *ttp = NULL; + + PMAP_LOCK(pmap); + if ((tt_free_entry_t *)pmap->tt_entry_free != NULL) { + tt_free_entry_t *tt_free_next; + + tt_free_next = ((tt_free_entry_t *)pmap->tt_entry_free)->next; + *ttp = (tt_entry_t *)pmap->tt_entry_free; + pmap->tt_entry_free = (tt_entry_t *)tt_free_next; + } + PMAP_UNLOCK(pmap); + + if (*ttp == NULL) { + pt_desc_t *ptdp; + + /* + * Allocate a VM page for the level x page table entries. + */ + while (pmap_pages_alloc(&pa, PAGE_SIZE, ((options & PMAP_TT_ALLOCATE_NOWAIT)? PMAP_PAGES_ALLOCATE_NOWAIT : 0)) != KERN_SUCCESS) { + if(options & PMAP_OPTIONS_NOWAIT) { + return KERN_RESOURCE_SHORTAGE; + } + VM_PAGE_WAIT(); + } + + if (level < PMAP_TT_MAX_LEVEL) { + OSAddAtomic64(1, &alloc_ttepages_count); + OSAddAtomic(1, (pmap == kernel_pmap ? &inuse_kernel_ttepages_count : &inuse_user_ttepages_count)); + } else { + OSAddAtomic64(1, &alloc_ptepages_count); + OSAddAtomic(1, (pmap == kernel_pmap ? &inuse_kernel_ptepages_count : &inuse_user_ptepages_count)); + } + + pmap_tt_ledger_credit(pmap, PAGE_SIZE); + + PMAP_ZINFO_PALLOC(pmap, PAGE_SIZE); + + ptdp = ptd_alloc(pmap); + *(pt_desc_t **)pai_to_pvh(pa_index(pa)) = ptdp; + + __unreachable_ok_push + if (TEST_PAGE_RATIO_4) { + vm_address_t va; + vm_address_t va_end; + + PMAP_LOCK(pmap); + + for (va_end = phystokv(pa) + PAGE_SIZE, va = phystokv(pa) + ARM_PGBYTES; va < va_end; va = va+ARM_PGBYTES) { + ((tt_free_entry_t *)va)->next = (tt_free_entry_t *)pmap->tt_entry_free; + pmap->tt_entry_free = (tt_entry_t *)va; + } + PMAP_UNLOCK(pmap); + } + __unreachable_ok_pop + + *ttp = (tt_entry_t *)phystokv(pa); + } + + + return KERN_SUCCESS; +} + + +static void +pmap_tt_deallocate( + pmap_t pmap, + tt_entry_t *ttp, + unsigned int level) +{ + pt_desc_t *ptdp; + unsigned pt_acc_cnt; + unsigned i, max_pt_index = PAGE_RATIO; + vm_offset_t free_page=0; + + PMAP_LOCK(pmap); + + ptdp = ptep_get_ptd((vm_offset_t)ttp); + + if (level < PMAP_TT_MAX_LEVEL) { + + if (ptdp->pt_cnt[ARM_PT_DESC_INDEX(ttp)].refcnt == PT_DESC_REFCOUNT) + ptdp->pt_cnt[ARM_PT_DESC_INDEX(ttp)].refcnt = 0; + } + + ptdp->pt_map[ARM_PT_DESC_INDEX(ttp)].va = 0; + + if (ptdp->pt_cnt[ARM_PT_DESC_INDEX(ttp)].refcnt != 0) + panic("pmap_tt_deallocate(): ptdp %p, count %d\n", ptdp, ptdp->pt_cnt[ARM_PT_DESC_INDEX(ttp)].refcnt); + + for (i = 0, pt_acc_cnt = 0 ; i < max_pt_index ; i++) + pt_acc_cnt += ptdp->pt_cnt[i].refcnt; + + if (pt_acc_cnt == 0) { + tt_free_entry_t *tt_free_list = (tt_free_entry_t *)&pmap->tt_entry_free; + unsigned pt_free_entry_cnt = 1; + + while (pt_free_entry_cnt < max_pt_index && tt_free_list) { + tt_free_entry_t *tt_free_list_next; + + tt_free_list_next = tt_free_list->next; + if ((((vm_offset_t)tt_free_list_next) - ((vm_offset_t)ttp & ~PAGE_MASK)) < PAGE_SIZE) { + pt_free_entry_cnt++; + } + tt_free_list = tt_free_list_next; + } + if (pt_free_entry_cnt == max_pt_index) { + tt_free_entry_t *tt_free_list_cur; + + free_page = (vm_offset_t)ttp & ~PAGE_MASK; + tt_free_list = (tt_free_entry_t *)&pmap->tt_entry_free; + tt_free_list_cur = (tt_free_entry_t *)&pmap->tt_entry_free; + + while (tt_free_list_cur) { + tt_free_entry_t *tt_free_list_next; + + tt_free_list_next = tt_free_list_cur->next; + if ((((vm_offset_t)tt_free_list_next) - free_page) < PAGE_SIZE) { + tt_free_list->next = tt_free_list_next->next; + } else { + tt_free_list = tt_free_list_next; + } + tt_free_list_cur = tt_free_list_next; + } + } else { + ((tt_free_entry_t *)ttp)->next = (tt_free_entry_t *)pmap->tt_entry_free; + pmap->tt_entry_free = ttp; + } + } else { + ((tt_free_entry_t *)ttp)->next = (tt_free_entry_t *)pmap->tt_entry_free; + pmap->tt_entry_free = ttp; + } + + PMAP_UNLOCK(pmap); + + if (free_page != 0) { + + ptd_deallocate(ptep_get_ptd((vm_offset_t)free_page)); + *(pt_desc_t **)pai_to_pvh(pa_index(ml_static_vtop(free_page))) = NULL; + pmap_pages_free(ml_static_vtop(free_page), PAGE_SIZE); + if (level < PMAP_TT_MAX_LEVEL) + OSAddAtomic(-1, (pmap == kernel_pmap ? &inuse_kernel_ttepages_count : &inuse_user_ttepages_count)); + else + OSAddAtomic(-1, (pmap == kernel_pmap ? &inuse_kernel_ptepages_count : &inuse_user_ptepages_count)); + PMAP_ZINFO_PFREE(pmap, PAGE_SIZE); + pmap_tt_ledger_debit(pmap, PAGE_SIZE); + } +} + +static void +pmap_tte_deallocate( + pmap_t pmap, + tt_entry_t *ttep, + unsigned int level) +{ + pmap_paddr_t pa; + tt_entry_t tte; + + PMAP_ASSERT_LOCKED(pmap); + + tte = *ttep; + + if (tte == 0) { + panic("pmap_tte_deallocate(): null tt_entry ttep==%p\n", ttep); + } + +#if MACH_ASSERT + if (tte_get_ptd(tte)->pmap != pmap) { + panic("pmap_tte_deallocate(): ptd=%p ptd->pmap=%p pmap=%p \n", + tte_get_ptd(tte), tte_get_ptd(tte)->pmap, pmap); + } +#endif + if (((level+1) == PMAP_TT_MAX_LEVEL) && (tte_get_ptd(tte)->pt_cnt[ARM_PT_DESC_INDEX(ttetokv(*ttep))].refcnt != 0)) { + panic("pmap_tte_deallocate(): pmap=%p ttep=%p ptd=%p refcnt=0x%x \n", pmap, ttep, + tte_get_ptd(tte), (tte_get_ptd(tte)->pt_cnt[ARM_PT_DESC_INDEX(ttetokv(*ttep))].refcnt)); + } + +#if (__ARM_VMSA__ == 7) + { + tt_entry_t *ttep_4M = (tt_entry_t *) ((vm_offset_t)ttep & 0xFFFFFFF0); + unsigned i; + + for (i = 0; i<4; i++, ttep_4M++) + *ttep_4M = (tt_entry_t) 0; + } +#else + *ttep = (tt_entry_t) 0; +#endif + +#ifndef __ARM_L1_PTW__ + CleanPoU_DcacheRegion((vm_offset_t) ttep, sizeof(tt_entry_t)); +#else + __asm__ volatile("dsb ish"); +#endif + if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) { +#if MACH_ASSERT + { + pt_entry_t *pte_p = ((pt_entry_t *) (ttetokv(tte) & ~ARM_PGMASK)); + unsigned i; + + for (i = 0; i < (ARM_PGBYTES / sizeof(*pte_p)); i++,pte_p++) { + if (ARM_PTE_IS_COMPRESSED(*pte_p)) { + panic("pmap_tte_deallocate: tte=0x%llx pmap=%p, pte_p=%p, pte=0x%llx compressed\n", + (uint64_t)tte, pmap, pte_p, (uint64_t)(*pte_p)); + } else if (((*pte_p) & ARM_PTE_TYPE_MASK) != ARM_PTE_TYPE_FAULT) { + panic("pmap_tte_deallocate: tte=0x%llx pmap=%p, pte_p=%p, pte=0x%llx\n", + (uint64_t)tte, pmap, pte_p, (uint64_t)(*pte_p)); + } + } + } +#endif + PMAP_UNLOCK(pmap); + + /* Clear any page offset: we mean to free the whole page, but armv7 TTEs may only be + * aligned on 1K boundaries. We clear the surrounding "chunk" of 4 TTEs above. */ + pa = tte_to_pa(tte) & ~ARM_PGMASK; + pmap_tt_deallocate(pmap, (tt_entry_t *) phystokv(pa), level+1); + PMAP_LOCK(pmap); + } +} + +/* + * Remove a range of hardware page-table entries. + * The entries given are the first (inclusive) + * and last (exclusive) entries for the VM pages. + * The virtual address is the va for the first pte. + * + * The pmap must be locked. + * If the pmap is not the kernel pmap, the range must lie + * entirely within one pte-page. This is NOT checked. + * Assumes that the pte-page exists. + * + * Returns the number of PTE changed, and sets *rmv_cnt + * to the number of SPTE changed. + */ +static int +pmap_remove_range( + pmap_t pmap, + vm_map_address_t va, + pt_entry_t *bpte, + pt_entry_t *epte, + uint32_t *rmv_cnt) +{ + return pmap_remove_range_options(pmap, va, bpte, epte, rmv_cnt, + PMAP_OPTIONS_REMOVE); +} + +#if MACH_ASSERT +int num_reusable_mismatch = 0; +#endif /* MACH_ASSERT */ + +static int +pmap_remove_range_options( + pmap_t pmap, + vm_map_address_t va, + pt_entry_t *bpte, + pt_entry_t *epte, + uint32_t *rmv_cnt, + int options) +{ + pt_entry_t *cpte; + int num_removed, num_unwired; + int num_pte_changed; + int pai = 0; + pmap_paddr_t pa; + int num_external, num_internal, num_reusable; + int num_alt_internal; + uint64_t num_compressed, num_alt_compressed; + + PMAP_ASSERT_LOCKED(pmap); + + num_removed = 0; + num_unwired = 0; + num_pte_changed = 0; + num_external = 0; + num_internal = 0; + num_reusable = 0; + num_compressed = 0; + num_alt_internal = 0; + num_alt_compressed = 0; + + for (cpte = bpte; cpte < epte; + cpte += PAGE_SIZE/ARM_PGBYTES, va += PAGE_SIZE) { + pv_entry_t **pv_h, **pve_pp; + pv_entry_t *pve_p; + pt_entry_t spte; + boolean_t managed=FALSE; + + spte = *cpte; + +#if CONFIG_PGTRACE + if (pgtrace_enabled) { + pmap_pgtrace_remove_clone(pmap, pte_to_pa(spte), va); + } +#endif + + while (!managed) { + if (pmap != kernel_pmap && + (options & PMAP_OPTIONS_REMOVE) && + (ARM_PTE_IS_COMPRESSED(spte))) { + /* + * "pmap" must be locked at this point, + * so this should not race with another + * pmap_remove_range() or pmap_enter(). + */ + + /* one less "compressed"... */ + num_compressed++; + if (spte & ARM_PTE_COMPRESSED_ALT) { + /* ... but it used to be "ALTACCT" */ + num_alt_compressed++; + } + + /* clear marker */ + WRITE_PTE_FAST(cpte, ARM_PTE_TYPE_FAULT); + /* + * "refcnt" also accounts for + * our "compressed" markers, + * so let's update it here. + */ + if (OSAddAtomic16(-1, (SInt16 *) &(ptep_get_ptd(cpte)->pt_cnt[ARM_PT_DESC_INDEX(cpte)].refcnt)) <= 0) + panic("pmap_remove_range_options: over-release of ptdp %p for pte %p\n", ptep_get_ptd(cpte), cpte); + spte = *cpte; + } + /* + * It may be possible for the pte to transition from managed + * to unmanaged in this timeframe; for now, elide the assert. + * We should break out as a consequence of checking pa_valid. + */ + //assert(!ARM_PTE_IS_COMPRESSED(spte)); + pa = pte_to_pa(spte); + if (!pa_valid(pa)) { + break; + } + pai = (int)pa_index(pa); + LOCK_PVH(pai); + spte = *cpte; + pa = pte_to_pa(spte); + if (pai == (int)pa_index(pa)) { + managed =TRUE; + break; // Leave pai locked as we will unlock it after we free the PV entry + } + UNLOCK_PVH(pai); + } + + if (ARM_PTE_IS_COMPRESSED(*cpte)) { + /* + * There used to be a valid mapping here but it + * has already been removed when the page was + * sent to the VM compressor, so nothing left to + * remove now... + */ + continue; + } + + /* remove the translation, do not flush the TLB */ + if (*cpte != ARM_PTE_TYPE_FAULT) { + assert(!ARM_PTE_IS_COMPRESSED(*cpte)); +#if MACH_ASSERT + if (managed && (pmap != kernel_pmap) && (ptep_get_va(cpte) != va)) { + panic("pmap_remove_range_options(): cpte=%p ptd=%p pte=0x%llx va=0x%llx\n", + cpte, ptep_get_ptd(cpte), (uint64_t)*cpte, (uint64_t)va); + } +#endif + WRITE_PTE_FAST(cpte, ARM_PTE_TYPE_FAULT); + num_pte_changed++; + } + + if ((spte != ARM_PTE_TYPE_FAULT) && + (pmap != kernel_pmap)) { + assert(!ARM_PTE_IS_COMPRESSED(spte)); + if (OSAddAtomic16(-1, (SInt16 *) &(ptep_get_ptd(cpte)->pt_cnt[ARM_PT_DESC_INDEX(cpte)].refcnt)) <= 0) + panic("pmap_remove_range_options: over-release of ptdp %p for pte %p\n", ptep_get_ptd(cpte), cpte); + if(rmv_cnt) (*rmv_cnt)++; + } + + if (pte_is_wired(spte)) { + pte_set_wired(cpte, 0); + num_unwired++; + } + /* + * if not managed, we're done + */ + if (!managed) + continue; + /* + * find and remove the mapping from the chain for this + * physical address. + */ + ASSERT_PVH_LOCKED(pai); // Should have been locked when we found the managed PTE above + pv_h = pai_to_pvh(pai); + + if (pvh_test_type(pv_h, PVH_TYPE_PTEP)) { + if (__builtin_expect((cpte != pvh_ptep(pv_h)), 0)) + panic("pmap_remove_range(): cpte=%p (0x%llx) does not match pv_h=%p (%p)\n", cpte, (uint64_t)spte, pv_h, pvh_ptep(pv_h)); + if (IS_ALTACCT_PAGE(pai, PV_ENTRY_NULL)) { + assert(IS_INTERNAL_PAGE(pai)); + num_internal++; + num_alt_internal++; + CLR_ALTACCT_PAGE(pai, PV_ENTRY_NULL); + } else if (IS_INTERNAL_PAGE(pai)) { + if (IS_REUSABLE_PAGE(pai)) { + num_reusable++; + } else { + num_internal++; + } + } else { + num_external++; + } + pvh_update_head(pv_h, PV_ENTRY_NULL, PVH_TYPE_NULL); + } else if (pvh_test_type(pv_h, PVH_TYPE_PVEP)) { + + pve_pp = pv_h; + pve_p = pvh_list(pv_h); + + while (pve_p != PV_ENTRY_NULL && + (pve_get_ptep(pve_p) != cpte)) { + pve_pp = pve_link_field(pve_p); + pve_p = PVE_NEXT_PTR(pve_next(pve_p)); + } + + if (__builtin_expect((pve_p == PV_ENTRY_NULL), 0)) { + UNLOCK_PVH(pai); + panic("pmap_remove_range(): cpte=%p (0x%llx) not in pv_h=%p\n", cpte, (uint64_t)spte, pv_h); + } + +#if MACH_ASSERT + if (kern_feature_override(KF_PMAPV_OVRD) == FALSE) { + pv_entry_t *check_pve_p = PVE_NEXT_PTR(pve_next(pve_p)); + while (check_pve_p != PV_ENTRY_NULL) { + if (pve_get_ptep(check_pve_p) == cpte) { + panic("pmap_remove_range(): duplicate pve entry cpte=%p pmap=%p, pv_h=%p, pve_p=%p, pte=0x%llx, va=0x%llx\n", + cpte, pmap, pv_h, pve_p, (uint64_t)spte, (uint64_t)va); + } + check_pve_p = PVE_NEXT_PTR(pve_next(check_pve_p)); + } + } +#endif + + if (IS_ALTACCT_PAGE(pai, pve_p)) { + assert(IS_INTERNAL_PAGE(pai)); + num_internal++; + num_alt_internal++; + CLR_ALTACCT_PAGE(pai, pve_p); + } else if (IS_INTERNAL_PAGE(pai)) { + if (IS_REUSABLE_PAGE(pai)) { + num_reusable++; + } else { + num_internal++; + } + } else { + num_external++; + } + + pvh_remove(pv_h, pve_pp, pve_p) ; + pv_free(pve_p); + } else { + panic("pmap_remove_range(): unexpected PV head %p, cpte=%p pmap=%p pv_h=%p pte=0x%llx va=0x%llx\n", + *pv_h, cpte, pmap, pv_h, (uint64_t)spte, (uint64_t)va); + } + + UNLOCK_PVH(pai); + num_removed++; + } + + /* + * Update the counts + */ + OSAddAtomic(-num_removed, (SInt32 *) &pmap->stats.resident_count); + pmap_ledger_debit(pmap, task_ledgers.phys_mem, machine_ptob(num_removed)); + + if (pmap != kernel_pmap) { + /* sanity checks... */ +#if MACH_ASSERT + if (pmap->stats.internal < num_internal) { + if ((pmap->stats.internal + pmap->stats.reusable) == + (num_internal + num_reusable)) { + num_reusable_mismatch++; + printf("pmap_remove_range_options(%p,0x%llx,%p,%p,0x%x): num_internal=%d num_removed=%d num_unwired=%d num_external=%d num_reusable=%d num_compressed=%lld num_alt_internal=%d num_alt_compressed=%lld num_pte_changed=%d stats.internal=%d stats.reusable=%d\n", + pmap, + (uint64_t) va, + bpte, + epte, + options, + num_internal, + num_removed, + num_unwired, + num_external, + num_reusable, + num_compressed, + num_alt_internal, + num_alt_compressed, + num_pte_changed, + pmap->stats.internal, + pmap->stats.reusable); + /* slight mismatch: fix it... */ + num_internal = pmap->stats.internal; + num_reusable = pmap->stats.reusable; + } else { + panic("pmap_remove_range_options(%p,0x%llx,%p,%p,0x%x): num_internal=%d num_removed=%d num_unwired=%d num_external=%d num_reusable=%d num_compressed=%lld num_alt_internal=%d num_alt_compressed=%lld num_pte_changed=%d stats.internal=%d stats.reusable=%d", + pmap, + (uint64_t) va, + bpte, + epte, + options, + num_internal, + num_removed, + num_unwired, + num_external, + num_reusable, + num_compressed, + num_alt_internal, + num_alt_compressed, + num_pte_changed, + pmap->stats.internal, + pmap->stats.reusable); + } + } +#endif /* MACH_ASSERT */ + assertf(pmap->stats.external >= num_external, + "pmap=%p num_external=%d stats.external=%d", + pmap, num_external, pmap->stats.external); + assertf(pmap->stats.internal >= num_internal, + "pmap=%p num_internal=%d stats.internal=%d num_reusable=%d stats.reusable=%d", + pmap, + num_internal, pmap->stats.internal, + num_reusable, pmap->stats.reusable); + assertf(pmap->stats.reusable >= num_reusable, + "pmap=%p num_internal=%d stats.internal=%d num_reusable=%d stats.reusable=%d", + pmap, + num_internal, pmap->stats.internal, + num_reusable, pmap->stats.reusable); + assertf(pmap->stats.compressed >= num_compressed, + "pmap=%p num_compressed=%lld num_alt_compressed=%lld stats.compressed=%lld", + pmap, num_compressed, num_alt_compressed, + pmap->stats.compressed); + + /* update pmap stats... */ + OSAddAtomic(-num_unwired, (SInt32 *) &pmap->stats.wired_count); + if (num_external) + OSAddAtomic(-num_external, &pmap->stats.external); + if (num_internal) + OSAddAtomic(-num_internal, &pmap->stats.internal); + if (num_reusable) + OSAddAtomic(-num_reusable, &pmap->stats.reusable); + if (num_compressed) + OSAddAtomic64(-num_compressed, &pmap->stats.compressed); + /* ... and ledgers */ + pmap_ledger_debit(pmap, task_ledgers.wired_mem, machine_ptob(num_unwired)); + pmap_ledger_debit(pmap, task_ledgers.internal, machine_ptob(num_internal)); + pmap_ledger_debit(pmap, task_ledgers.alternate_accounting, machine_ptob(num_alt_internal)); + pmap_ledger_debit(pmap, task_ledgers.alternate_accounting_compressed, machine_ptob(num_alt_compressed)); + pmap_ledger_debit(pmap, task_ledgers.internal_compressed, machine_ptob(num_compressed)); + /* make needed adjustments to phys_footprint */ + pmap_ledger_debit(pmap, task_ledgers.phys_footprint, + machine_ptob((num_internal - + num_alt_internal) + + (num_compressed - + num_alt_compressed))); + } + + /* flush the ptable entries we have written */ + if (num_pte_changed > 0) + FLUSH_PTE_RANGE(bpte, epte); + + return num_pte_changed; +} + + +/* + * Remove the given range of addresses + * from the specified map. + * + * It is assumed that the start and end are properly + * rounded to the hardware page size. + */ +void +pmap_remove( + pmap_t pmap, + vm_map_address_t start, + vm_map_address_t end) +{ + pmap_remove_options(pmap, start, end, PMAP_OPTIONS_REMOVE); +} + +static int +pmap_remove_options_internal(pmap_t pmap, +vm_map_address_t start, +vm_map_address_t end, +int options) +{ + int remove_count = 0; + pt_entry_t *bpte, *epte; + pt_entry_t *pte_p; + tt_entry_t *tte_p; + uint32_t rmv_spte=0; + + PMAP_LOCK(pmap); + + tte_p = pmap_tte(pmap, start); + + if (tte_p == (tt_entry_t *) NULL) { + goto done; + } + + if ((*tte_p & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) { + pte_p = (pt_entry_t *) ttetokv(*tte_p); + bpte = &pte_p[ptenum(start)]; + epte = bpte + ((end - start) >> ARM_TT_LEAF_SHIFT); + + remove_count += pmap_remove_range_options(pmap, start, bpte, epte, + &rmv_spte, options); + +#if (__ARM_VMSA__ == 7) + if (rmv_spte && (ptep_get_ptd(pte_p)->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].refcnt == 0) && + (pmap != kernel_pmap) && (pmap->nested == FALSE)) { + pmap_tte_deallocate(pmap, tte_p, PMAP_TT_L1_LEVEL); + flush_mmu_tlb_entry((start & ~ARM_TT_L1_OFFMASK) | (pmap->asid & 0xff)); + } +#else + if (rmv_spte && (ptep_get_ptd(pte_p)->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].refcnt == 0) && + (pmap != kernel_pmap) && (pmap->nested == FALSE)) { + pmap_tte_deallocate(pmap, tte_p, PMAP_TT_L2_LEVEL); + flush_mmu_tlb_entry(tlbi_addr(start & ~ARM_TT_L2_OFFMASK) | tlbi_asid(pmap->asid)); + } +#endif + } + +done: + PMAP_UNLOCK(pmap); + + return remove_count; +} + +void +pmap_remove_options( + pmap_t pmap, + vm_map_address_t start, + vm_map_address_t end, + int options) +{ + int remove_count = 0; + vm_map_address_t va; + + if (pmap == PMAP_NULL) + return; + + PMAP_TRACE(PMAP_CODE(PMAP__REMOVE) | DBG_FUNC_START, + VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(start), + VM_KERNEL_ADDRHIDE(end)); + +#if MACH_ASSERT + if ((start|end) & PAGE_MASK) { + panic("pmap_remove_options() pmap %p start 0x%llx end 0x%llx\n", + pmap, (uint64_t)start, (uint64_t)end); + } + if ((end < start) || (start < pmap->min) || (end > pmap->max)) { + panic("pmap_remove_options(): invalid address range, pmap=%p, start=0x%llx, end=0x%llx\n", + pmap, (uint64_t)start, (uint64_t)end); + } +#endif + + /* + * Invalidate the translation buffer first + */ + va = start; + while (va < end) { + vm_map_address_t l; + +#if (__ARM_VMSA__ == 7) + l = ((va + ARM_TT_L1_SIZE) & ~ARM_TT_L1_OFFMASK); +#else + l = ((va + ARM_TT_L2_SIZE) & ~ARM_TT_L2_OFFMASK); +#endif + if (l > end) + l = end; + + remove_count += pmap_remove_options_internal(pmap, va, l, options); + + va = l; + } + + + if (remove_count > 0) + PMAP_UPDATE_TLBS(pmap, start, end); + + PMAP_TRACE(PMAP_CODE(PMAP__REMOVE) | DBG_FUNC_END); +} + + +/* + * Remove phys addr if mapped in specified map + */ +void +pmap_remove_some_phys( + __unused pmap_t map, + __unused ppnum_t pn) +{ + /* Implement to support working set code */ +} + + +void +pmap_set_pmap( + pmap_t pmap, +#if !__ARM_USER_PROTECT__ + __unused +#endif + thread_t thread) +{ + pmap_switch(pmap); +#if __ARM_USER_PROTECT__ + if (pmap->tte_index_max == NTTES) { + thread->machine.uptw_ttc = 2; + thread->machine.uptw_ttb = ((unsigned int) pmap->ttep) | TTBR_SETUP; + } else { + thread->machine.uptw_ttc = 1; \ + thread->machine.uptw_ttb = ((unsigned int) pmap->ttep ) | TTBR_SETUP; + } + thread->machine.asid = pmap->asid; +#endif +} + +static void +pmap_flush_core_tlb_asid(pmap_t pmap) +{ +#if (__ARM_VMSA__ == 7) + flush_core_tlb_asid(pmap->asid); +#else + flush_core_tlb_asid(((uint64_t) pmap->asid) << TLBI_ASID_SHIFT); +#endif +} + +static void +pmap_switch_internal( + pmap_t pmap) +{ + pmap_cpu_data_t *cpu_data_ptr = pmap_get_cpu_data(); + uint32_t last_asid_high_bits, asid_high_bits; + pmap_t cur_pmap; + pmap_t cur_user_pmap; + boolean_t do_asid_flush = FALSE; + +#if (__ARM_VMSA__ == 7) + if (not_in_kdp) + simple_lock(&pmap->tt1_lock); +#endif + + cur_pmap = current_pmap(); + cur_user_pmap = cpu_data_ptr->cpu_user_pmap; + + /* Paranoia. */ + assert(pmap->asid < (sizeof(cpu_data_ptr->cpu_asid_high_bits) / sizeof(*cpu_data_ptr->cpu_asid_high_bits))); + + /* Extract the "virtual" bits of the ASIDs (which could cause us to alias). */ + asid_high_bits = pmap->vasid >> ARM_ASID_SHIFT; + last_asid_high_bits = (uint32_t) cpu_data_ptr->cpu_asid_high_bits[pmap->asid]; + + if (asid_high_bits != last_asid_high_bits) { + /* + * If the virtual ASID of the new pmap does not match the virtual ASID + * last seen on this CPU for the physical ASID (that was a mouthful), + * then this switch runs the risk of aliasing. We need to flush the + * TLB for this phyiscal ASID in this case. + */ + cpu_data_ptr->cpu_asid_high_bits[pmap->asid] = (uint8_t) asid_high_bits; + do_asid_flush = TRUE; + } + + if ((cur_user_pmap == cur_pmap) && (cur_pmap == pmap)) { + if (cpu_data_ptr->cpu_user_pmap_stamp == pmap->stamp) { + pmap_switch_user_ttb_internal(pmap); + +#if (__ARM_VMSA__ == 7) + if (not_in_kdp) + simple_unlock(&pmap->tt1_lock); +#endif + + if (do_asid_flush) { + pmap_flush_core_tlb_asid(pmap); + } + + return; + } else + cur_user_pmap = NULL; + } else if ((cur_user_pmap == pmap) && (cpu_data_ptr->cpu_user_pmap_stamp != pmap->stamp)) + cur_user_pmap = NULL; + + pmap_switch_user_ttb_internal(pmap); + + if (do_asid_flush) { + pmap_flush_core_tlb_asid(pmap); + } + +#if (__ARM_VMSA__ == 7) + if (not_in_kdp) + simple_unlock(&pmap->tt1_lock); +#else + if (pmap != kernel_pmap) { + + if (cur_user_pmap != PMAP_NULL) { + /* + * We have a low-address global mapping for the commpage + * for 32-bit processes; flush it if we switch to a 64-bot + * process. + */ + if (pmap_is_64bit(pmap) && !pmap_is_64bit(cur_user_pmap)) { + pmap_sharedpage_flush_32_to_64(); + } + + } else + flush_core_tlb(); + } +#endif +} + +void +pmap_switch( + pmap_t pmap) +{ + pmap_switch_internal(pmap); +} + +void +pmap_page_protect( + ppnum_t ppnum, + vm_prot_t prot) +{ + pmap_page_protect_options(ppnum, prot, 0, NULL); +} + +/* + * Routine: pmap_page_protect_options + * + * Function: + * Lower the permission for all mappings to a given + * page. + */ +static void +pmap_page_protect_options_internal( + ppnum_t ppnum, + vm_prot_t prot, + unsigned int options) +{ + pmap_paddr_t phys = ptoa(ppnum); + pv_entry_t **pv_h; + pv_entry_t *pve_p; + pv_entry_t *pveh_p; + pv_entry_t *pvet_p; + pt_entry_t *pte_p; + int pai; + boolean_t remove; + boolean_t set_NX; + unsigned int pvh_cnt = 0; + + assert(ppnum != vm_page_fictitious_addr); + + /* Only work with managed pages. */ + if (!pa_valid(phys)) { + return; + } + + /* + * Determine the new protection. + */ + switch (prot) { + case VM_PROT_ALL: + return; /* nothing to do */ + case VM_PROT_READ: + case VM_PROT_READ | VM_PROT_EXECUTE: + remove = FALSE; + break; + default: + remove = TRUE; + break; + } + + pai = (int)pa_index(phys); + LOCK_PVH(pai); + pv_h = pai_to_pvh(pai); + + pte_p = PT_ENTRY_NULL; + pve_p = PV_ENTRY_NULL; + pveh_p = PV_ENTRY_NULL; + pvet_p = PV_ENTRY_NULL; + if (pvh_test_type(pv_h, PVH_TYPE_PTEP)) { + pte_p = pvh_ptep(pv_h); + } else if (pvh_test_type(pv_h, PVH_TYPE_PVEP)) { + pve_p = pvh_list(pv_h); + pveh_p = pve_p; + } + + while ((pve_p != PV_ENTRY_NULL) || (pte_p != PT_ENTRY_NULL)) { + vm_map_address_t va; + pmap_t pmap; + pt_entry_t tmplate; + boolean_t update = FALSE; + + if (pve_p != PV_ENTRY_NULL) + pte_p = pve_get_ptep(pve_p); + + pmap = ptep_get_pmap(pte_p); + va = ptep_get_va(pte_p); + + if (pte_p == PT_ENTRY_NULL) { + panic("pmap_page_protect: pmap=%p prot=%d options=%u, pv_h=%p, pveh_p=%p, pve_p=%p, va=0x%llx ppnum: 0x%x\n", + pmap, prot, options, pv_h, pveh_p, pve_p, (uint64_t)va, ppnum); + } else if ((pmap == NULL) || (atop(pte_to_pa(*pte_p)) != ppnum)) { +#if MACH_ASSERT + if (kern_feature_override(KF_PMAPV_OVRD) == FALSE) { + + pv_entry_t *check_pve_p = pveh_p; + while (check_pve_p != PV_ENTRY_NULL) { + if ((check_pve_p != pve_p) && (pve_get_ptep(check_pve_p) == pte_p)) { + panic("pmap_page_protect: duplicate pve entry pte_p=%p pmap=%p prot=%d options=%u, pv_h=%p, pveh_p=%p, pve_p=%p, pte=0x%llx, va=0x%llx ppnum: 0x%x\n", + pte_p, pmap, prot, options, pv_h, pveh_p, pve_p, (uint64_t)*pte_p, (uint64_t)va, ppnum); + } + check_pve_p = PVE_NEXT_PTR(pve_next(check_pve_p)); + } + } +#endif + panic("pmap_page_protect: bad pve entry pte_p=%p pmap=%p prot=%d options=%u, pv_h=%p, pveh_p=%p, pve_p=%p, pte=0x%llx, va=0x%llx ppnum: 0x%x\n", + pte_p, pmap, prot, options, pv_h, pveh_p, pve_p, (uint64_t)*pte_p, (uint64_t)va, ppnum); + } + +#if DEVELOPMENT || DEBUG + if ((prot & VM_PROT_EXECUTE) || !nx_enabled || !pmap->nx_enabled) +#else + if ((prot & VM_PROT_EXECUTE)) +#endif + set_NX = FALSE; + else + set_NX = TRUE; + + /* Remove the mapping if new protection is NONE */ + if (remove) { + boolean_t is_altacct = FALSE; + + if (IS_ALTACCT_PAGE(pai, pve_p)) { + is_altacct = TRUE; + } else { + is_altacct = FALSE; + } + + if (pte_is_wired(*pte_p)) { + pte_set_wired(pte_p, 0); + if (pmap != kernel_pmap) { + pmap_ledger_debit(pmap, task_ledgers.wired_mem, PAGE_SIZE); + OSAddAtomic(-1, (SInt32 *) &pmap->stats.wired_count); + } + } + + if (*pte_p != ARM_PTE_TYPE_FAULT && + pmap != kernel_pmap && + (options & PMAP_OPTIONS_COMPRESSOR) && + IS_INTERNAL_PAGE(pai)) { + assert(!ARM_PTE_IS_COMPRESSED(*pte_p)); + /* mark this PTE as having been "compressed" */ + tmplate = ARM_PTE_COMPRESSED; + if (is_altacct) { + tmplate |= ARM_PTE_COMPRESSED_ALT; + is_altacct = TRUE; + } + } else { + tmplate = ARM_PTE_TYPE_FAULT; + } + + if ((*pte_p != ARM_PTE_TYPE_FAULT) && + tmplate == ARM_PTE_TYPE_FAULT && + (pmap != kernel_pmap)) { + if (OSAddAtomic16(-1, (SInt16 *) &(ptep_get_ptd(pte_p)->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].refcnt)) <= 0) + panic("pmap_page_protect_options(): over-release of ptdp %p for pte %p\n", ptep_get_ptd(pte_p), pte_p); + } + + if (*pte_p != tmplate) { + WRITE_PTE(pte_p, tmplate); + update = TRUE; + } + pvh_cnt++; + pmap_ledger_debit(pmap, task_ledgers.phys_mem, PAGE_SIZE); + OSAddAtomic(-1, (SInt32 *) &pmap->stats.resident_count); + +#if MACH_ASSERT + /* + * We only ever compress internal pages. + */ + if (options & PMAP_OPTIONS_COMPRESSOR) { + assert(IS_INTERNAL_PAGE(pai)); + } +#endif + + if (pmap != kernel_pmap) { + if (IS_REUSABLE_PAGE(pai) && + IS_INTERNAL_PAGE(pai) && + !is_altacct) { + assert(pmap->stats.reusable > 0); + OSAddAtomic(-1, &pmap->stats.reusable); + } else if (IS_INTERNAL_PAGE(pai)) { + assert(pmap->stats.internal > 0); + OSAddAtomic(-1, &pmap->stats.internal); + } else { + assert(pmap->stats.external > 0); + OSAddAtomic(-1, &pmap->stats.external); + } + if ((options & PMAP_OPTIONS_COMPRESSOR) && + IS_INTERNAL_PAGE(pai)) { + /* adjust "compressed" stats */ + OSAddAtomic64(+1, &pmap->stats.compressed); + PMAP_STATS_PEAK(pmap->stats.compressed); + pmap->stats.compressed_lifetime++; + } + + if (IS_ALTACCT_PAGE(pai, pve_p)) { + assert(IS_INTERNAL_PAGE(pai)); + pmap_ledger_debit(pmap, task_ledgers.internal, PAGE_SIZE); + pmap_ledger_debit(pmap, task_ledgers.alternate_accounting, PAGE_SIZE); + if (options & PMAP_OPTIONS_COMPRESSOR) { + pmap_ledger_credit(pmap, task_ledgers.internal_compressed, PAGE_SIZE); + pmap_ledger_credit(pmap, task_ledgers.alternate_accounting_compressed, PAGE_SIZE); + } + + /* + * Cleanup our marker before + * we free this pv_entry. + */ + CLR_ALTACCT_PAGE(pai, pve_p); + + } else if (IS_REUSABLE_PAGE(pai)) { + assert(IS_INTERNAL_PAGE(pai)); + if (options & PMAP_OPTIONS_COMPRESSOR) { + pmap_ledger_credit(pmap, task_ledgers.internal_compressed, PAGE_SIZE); + /* was not in footprint, but is now */ + pmap_ledger_credit(pmap, task_ledgers.phys_footprint, PAGE_SIZE); + } + + } else if (IS_INTERNAL_PAGE(pai)) { + pmap_ledger_debit(pmap, task_ledgers.internal, PAGE_SIZE); + + /* + * Update all stats related to physical footprint, which only + * deals with internal pages. + */ + if (options & PMAP_OPTIONS_COMPRESSOR) { + /* + * This removal is only being done so we can send this page to + * the compressor; therefore it mustn't affect total task footprint. + */ + pmap_ledger_credit(pmap, task_ledgers.internal_compressed, PAGE_SIZE); + } else { + /* + * This internal page isn't going to the compressor, so adjust stats to keep + * phys_footprint up to date. + */ + pmap_ledger_debit(pmap, task_ledgers.phys_footprint, PAGE_SIZE); + } + } else { + /* external page: no impact on ledgers */ + } + } + + if (pve_p != PV_ENTRY_NULL) { + assert(pve_next(pve_p) == PVE_NEXT_PTR(pve_next(pve_p))); + } + + } else { + pt_entry_t spte; + + spte = *pte_p; + + if (pmap == kernel_pmap) + tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RONA)); + else + tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RORO)); + + pte_set_ffr(tmplate, 0); + +#if (__ARM_VMSA__ == 7) + if (set_NX) { + tmplate |= ARM_PTE_NX; + } else { + /* + * While the naive implementation of this would serve to add execute + * permission, this is not how the VM uses this interface, or how + * x86_64 implements it. So ignore requests to add execute permissions. + */ +#if 0 + tmplate &= ~ARM_PTE_NX; +#else + ; +#endif + } +#else + if (set_NX) + tmplate |= ARM_PTE_NX | ARM_PTE_PNX; + else { + /* + * While the naive implementation of this would serve to add execute + * permission, this is not how the VM uses this interface, or how + * x86_64 implements it. So ignore requests to add execute permissions. + */ +#if 0 + if (pmap == kernel_pmap) { + tmplate &= ~ARM_PTE_PNX; + tmplate |= ARM_PTE_NX; + } else { + tmplate &= ~ARM_PTE_NX; + tmplate |= ARM_PTE_PNX; + } +#else + ; +#endif + } +#endif + + + if (*pte_p != ARM_PTE_TYPE_FAULT && + !ARM_PTE_IS_COMPRESSED(*pte_p) && + *pte_p != tmplate) { + WRITE_PTE(pte_p, tmplate); + update = TRUE; + } + } + + /* Invalidate TLBs for all CPUs using it */ + if (update) + PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE); + + pte_p = PT_ENTRY_NULL; + pvet_p = pve_p; + if (pve_p != PV_ENTRY_NULL) { + pvet_p = pve_p; + if (remove) { + assert(pve_next(pve_p) == PVE_NEXT_PTR(pve_next(pve_p))); + } + pve_p = PVE_NEXT_PTR(pve_next(pve_p)); + } + } + + /* if we removed a bunch of entries, take care of them now */ + if (remove) { + pvh_update_head(pv_h, PV_ENTRY_NULL, PVH_TYPE_NULL); + } + + UNLOCK_PVH(pai); + + if (remove && (pveh_p != PV_ENTRY_NULL)) { + pv_list_free(pveh_p, pvet_p, pvh_cnt); + } +} + +void +pmap_page_protect_options( + ppnum_t ppnum, + vm_prot_t prot, + unsigned int options, + __unused void *arg) +{ + pmap_paddr_t phys = ptoa(ppnum); + + assert(ppnum != vm_page_fictitious_addr); + + /* Only work with managed pages. */ + if (!pa_valid(phys)) + return; + + /* + * Determine the new protection. + */ + if (prot == VM_PROT_ALL) { + return; /* nothing to do */ + } + + PMAP_TRACE(PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_START, ppnum, prot); + + pmap_page_protect_options_internal(ppnum, prot, options); + + PMAP_TRACE(PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_END); +} + +/* + * Indicates if the pmap layer enforces some additional restrictions on the + * given set of protections. + */ +bool pmap_has_prot_policy(__unused vm_prot_t prot) +{ + return FALSE; +} + +/* + * Set the physical protection on the + * specified range of this map as requested. + * VERY IMPORTANT: Will not increase permissions. + * VERY IMPORTANT: Only pmap_enter() is allowed to grant permissions. + */ +void +pmap_protect( + pmap_t pmap, + vm_map_address_t b, + vm_map_address_t e, + vm_prot_t prot) +{ + pmap_protect_options(pmap, b, e, prot, 0, NULL); +} + +static void +pmap_protect_options_internal(pmap_t pmap, + vm_map_address_t start, + vm_map_address_t end, + vm_prot_t prot, + unsigned int options, + __unused void *args) +{ + tt_entry_t *tte_p; + pt_entry_t *bpte_p, *epte_p; + pt_entry_t *pte_p; + boolean_t set_NX = TRUE; +#if (__ARM_VMSA__ > 7) + boolean_t set_XO = FALSE; +#endif + boolean_t should_have_removed = FALSE; + +#ifndef __ARM_IC_NOALIAS_ICACHE__ + boolean_t InvalidatePoU_Icache_Done = FALSE; +#endif + +#if DEVELOPMENT || DEBUG + if (options & PMAP_OPTIONS_PROTECT_IMMEDIATE) { + if ((prot & VM_PROT_ALL) == VM_PROT_NONE) { + should_have_removed = TRUE; + } + } else +#endif + { + /* Determine the new protection. */ + switch (prot) { +#if (__ARM_VMSA__ > 7) + case VM_PROT_EXECUTE: + set_XO = TRUE; + /* fall through */ +#endif + case VM_PROT_READ: + case VM_PROT_READ | VM_PROT_EXECUTE: + break; + case VM_PROT_READ | VM_PROT_WRITE: + case VM_PROT_ALL: + return; /* nothing to do */ + default: + should_have_removed = TRUE; + } + } + + if (should_have_removed) { + panic("%s: should have been a remove operation, " + "pmap=%p, start=%p, end=%p, prot=%#x, options=%#x, args=%p", + __FUNCTION__, + pmap, (void *)start, (void *)end, prot, options, args); + } + +#if DEVELOPMENT || DEBUG + if ((prot & VM_PROT_EXECUTE) || !nx_enabled || !pmap->nx_enabled) +#else + if ((prot & VM_PROT_EXECUTE)) +#endif + { + set_NX = FALSE; + } else { + set_NX = TRUE; + } + + PMAP_LOCK(pmap); + tte_p = pmap_tte(pmap, start); + + if ((tte_p != (tt_entry_t *) NULL) && (*tte_p & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) { + bpte_p = (pt_entry_t *) ttetokv(*tte_p); + bpte_p = &bpte_p[ptenum(start)]; + epte_p = bpte_p + arm_atop(end - start); + pte_p = bpte_p; + + for (pte_p = bpte_p; + pte_p < epte_p; + pte_p += PAGE_SIZE/ARM_PGBYTES) { + pt_entry_t spte; +#if DEVELOPMENT || DEBUG + boolean_t force_write = FALSE; +#endif + + spte = *pte_p; + + if ((spte == ARM_PTE_TYPE_FAULT) || + ARM_PTE_IS_COMPRESSED(spte)) { + continue; + } + + pmap_paddr_t pa; + int pai=0; + boolean_t managed=FALSE; + + while (!managed) { + /* + * It may be possible for the pte to transition from managed + * to unmanaged in this timeframe; for now, elide the assert. + * We should break out as a consequence of checking pa_valid. + */ + // assert(!ARM_PTE_IS_COMPRESSED(spte)); + pa = pte_to_pa(spte); + if (!pa_valid(pa)) + break; + pai = (int)pa_index(pa); + LOCK_PVH(pai); + spte = *pte_p; + pa = pte_to_pa(spte); + if (pai == (int)pa_index(pa)) { + managed =TRUE; + break; // Leave the PVH locked as we will unlock it after we free the PTE + } + UNLOCK_PVH(pai); + } + + if ((spte == ARM_PTE_TYPE_FAULT) || + ARM_PTE_IS_COMPRESSED(spte)) { + continue; + } + + pt_entry_t tmplate; + + if (pmap == kernel_pmap) { +#if DEVELOPMENT || DEBUG + if ((options & PMAP_OPTIONS_PROTECT_IMMEDIATE) && (prot & VM_PROT_WRITE)) { + force_write = TRUE; + tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RWNA)); + } else +#endif + { + tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RONA)); + } + } else { +#if DEVELOPMENT || DEBUG + if ((options & PMAP_OPTIONS_PROTECT_IMMEDIATE) && (prot & VM_PROT_WRITE)) { + force_write = TRUE; + tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RWRW)); + } else +#endif + { + tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RORO)); + } + } + + /* + * XXX Removing "NX" would + * grant "execute" access + * immediately, bypassing any + * checks VM might want to do + * in its soft fault path. + * pmap_protect() and co. are + * not allowed to increase + * access permissions. + */ +#if (__ARM_VMSA__ == 7) + if (set_NX) + tmplate |= ARM_PTE_NX; + else { + /* do NOT clear "NX"! */ + } +#else + if (set_NX) + tmplate |= ARM_PTE_NX | ARM_PTE_PNX; + else { + if (pmap == kernel_pmap) { + /* + * TODO: Run CS/Monitor checks here; + * should we be clearing PNX here? Is + * this just for dtrace? + */ + tmplate &= ~ARM_PTE_PNX; + tmplate |= ARM_PTE_NX; + } else { + /* do NOT clear "NX"! */ + tmplate |= ARM_PTE_PNX; + if (set_XO) { + tmplate &= ~ARM_PTE_APMASK; + tmplate |= ARM_PTE_AP(AP_RONA); + } + } + } +#endif + +#if DEVELOPMENT || DEBUG + if (force_write) { + /* + * TODO: Run CS/Monitor checks here. + */ + if (managed) { + /* + * We are marking the page as writable, + * so we consider it to be modified and + * referenced. + */ + pa_set_bits(pa, PP_ATTR_REFERENCED | PP_ATTR_MODIFIED); + tmplate |= ARM_PTE_AF; + + if (IS_REFFAULT_PAGE(pai)) { + CLR_REFFAULT_PAGE(pai); + } + + if (IS_MODFAULT_PAGE(pai)) { + CLR_MODFAULT_PAGE(pai); + } + } + } else if (options & PMAP_OPTIONS_PROTECT_IMMEDIATE) { + /* + * An immediate request for anything other than + * write should still mark the page as + * referenced if managed. + */ + if (managed) { + pa_set_bits(pa, PP_ATTR_REFERENCED); + tmplate |= ARM_PTE_AF; + + if (IS_REFFAULT_PAGE(pai)) { + CLR_REFFAULT_PAGE(pai); + } + } + } +#endif + + /* We do not expect to write fast fault the entry. */ + pte_set_ffr(tmplate, 0); + + /* TODO: Doesn't this need to worry about PNX? */ + if (((spte & ARM_PTE_NX) == ARM_PTE_NX) && (prot & VM_PROT_EXECUTE)) { + CleanPoU_DcacheRegion((vm_offset_t) phystokv(pa), PAGE_SIZE); +#ifdef __ARM_IC_NOALIAS_ICACHE__ + InvalidatePoU_IcacheRegion((vm_offset_t) phystokv(pa), PAGE_SIZE); +#else + if (!InvalidatePoU_Icache_Done) { + InvalidatePoU_Icache(); + InvalidatePoU_Icache_Done = TRUE; + } +#endif + } + + WRITE_PTE_FAST(pte_p, tmplate); + + if (managed) { + ASSERT_PVH_LOCKED(pai); + UNLOCK_PVH(pai); + } + } + + FLUSH_PTE_RANGE(bpte_p, epte_p); + PMAP_UPDATE_TLBS(pmap, start, end); + } + + PMAP_UNLOCK(pmap); +} + +void +pmap_protect_options( + pmap_t pmap, + vm_map_address_t b, + vm_map_address_t e, + vm_prot_t prot, + unsigned int options, + __unused void *args) +{ + vm_map_address_t l, beg; + + if ((b|e) & PAGE_MASK) { + panic("pmap_protect_options() pmap %p start 0x%llx end 0x%llx\n", + pmap, (uint64_t)b, (uint64_t)e); + } + +#if DEVELOPMENT || DEBUG + if (options & PMAP_OPTIONS_PROTECT_IMMEDIATE) { + if ((prot & VM_PROT_ALL) == VM_PROT_NONE) { + pmap_remove_options(pmap, b, e, options); + return; + } + } else +#endif + { + /* Determine the new protection. */ + switch (prot) { + case VM_PROT_EXECUTE: + case VM_PROT_READ: + case VM_PROT_READ | VM_PROT_EXECUTE: + break; + case VM_PROT_READ | VM_PROT_WRITE: + case VM_PROT_ALL: + return; /* nothing to do */ + default: + pmap_remove_options(pmap, b, e, options); + return; + } + } + + PMAP_TRACE(PMAP_CODE(PMAP__PROTECT) | DBG_FUNC_START, + VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(b), + VM_KERNEL_ADDRHIDE(e)); + + beg = b; + + while (beg < e) { + l = ((beg + ARM_TT_TWIG_SIZE) & ~ARM_TT_TWIG_OFFMASK); + + if (l > e) + l = e; + + pmap_protect_options_internal(pmap, beg, l, prot, options, args); + + beg = l; + } + + PMAP_TRACE(PMAP_CODE(PMAP__PROTECT) | DBG_FUNC_END); +} + +/* Map a (possibly) autogenned block */ +kern_return_t +pmap_map_block( + pmap_t pmap, + addr64_t va, + ppnum_t pa, + uint32_t size, + vm_prot_t prot, + int attr, + __unused unsigned int flags) +{ + kern_return_t kr; + addr64_t original_va = va; + uint32_t page; + + for (page = 0; page < size; page++) { + kr = pmap_enter(pmap, va, pa, prot, VM_PROT_NONE, attr, TRUE); + + if (kr != KERN_SUCCESS) { + /* + * This will panic for now, as it is unclear that + * removing the mappings is correct. + */ + panic("%s: failed pmap_enter, " + "pmap=%p, va=%#llx, pa=%u, size=%u, prot=%#x, flags=%#x", + __FUNCTION__, + pmap, va, pa, size, prot, flags); + + pmap_remove(pmap, original_va, va - original_va); + return kr; + } + + va += PAGE_SIZE; + pa++; + } + + return KERN_SUCCESS; +} + +/* + * Insert the given physical page (p) at + * the specified virtual address (v) in the + * target physical map with the protection requested. + * + * If specified, the page will be wired down, meaning + * that the related pte can not be reclaimed. + * + * NB: This is the only routine which MAY NOT lazy-evaluate + * or lose information. That is, this routine must actually + * insert this page into the given map eventually (must make + * forward progress eventually. + */ +kern_return_t +pmap_enter( + pmap_t pmap, + vm_map_address_t v, + ppnum_t pn, + vm_prot_t prot, + vm_prot_t fault_type, + unsigned int flags, + boolean_t wired) +{ + return pmap_enter_options(pmap, v, pn, prot, fault_type, flags, wired, 0, NULL); +} + + +static inline void pmap_enter_pte(pmap_t pmap, pt_entry_t *pte_p, pt_entry_t pte, vm_map_address_t v) +{ + if (pmap != kernel_pmap && ((pte & ARM_PTE_WIRED) != (*pte_p & ARM_PTE_WIRED))) + { + SInt16 *ptd_wiredcnt_ptr = (SInt16 *)&(ptep_get_ptd(pte_p)->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].wiredcnt); + if (pte & ARM_PTE_WIRED) { + OSAddAtomic16(1, ptd_wiredcnt_ptr); + pmap_ledger_credit(pmap, task_ledgers.wired_mem, PAGE_SIZE); + OSAddAtomic(1, (SInt32 *) &pmap->stats.wired_count); + } else { + OSAddAtomic16(-1, ptd_wiredcnt_ptr); + pmap_ledger_debit(pmap, task_ledgers.wired_mem, PAGE_SIZE); + OSAddAtomic(-1, (SInt32 *) &pmap->stats.wired_count); + } + } + if (*pte_p != ARM_PTE_TYPE_FAULT && + !ARM_PTE_IS_COMPRESSED(*pte_p)) { + WRITE_PTE(pte_p, pte); + PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE); + } else { + WRITE_PTE(pte_p, pte); + __asm__ volatile("isb"); + } +} + +static pt_entry_t +wimg_to_pte(unsigned int wimg) +{ + pt_entry_t pte; + + switch (wimg & (VM_WIMG_MASK)) { + case VM_WIMG_IO: + pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DISABLE); + pte |= ARM_PTE_NX | ARM_PTE_PNX; + break; + case VM_WIMG_POSTED: + pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_POSTED); + pte |= ARM_PTE_NX | ARM_PTE_PNX; + break; + case VM_WIMG_WCOMB: + pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITECOMB); + pte |= ARM_PTE_NX | ARM_PTE_PNX; + break; + case VM_WIMG_WTHRU: + pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITETHRU); +#if (__ARM_VMSA__ > 7) + pte |= ARM_PTE_SH(SH_OUTER_MEMORY); +#else + pte |= ARM_PTE_SH; +#endif + break; + case VM_WIMG_COPYBACK: + pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK); +#if (__ARM_VMSA__ > 7) + pte |= ARM_PTE_SH(SH_OUTER_MEMORY); +#else + pte |= ARM_PTE_SH; +#endif + break; + case VM_WIMG_INNERWBACK: + pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_INNERWRITEBACK); +#if (__ARM_VMSA__ > 7) + pte |= ARM_PTE_SH(SH_INNER_MEMORY); +#else + pte |= ARM_PTE_SH; +#endif + break; + default: + pte = ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT); +#if (__ARM_VMSA__ > 7) + pte |= ARM_PTE_SH(SH_OUTER_MEMORY); +#else + pte |= ARM_PTE_SH; +#endif + } + + return pte; +} + +static kern_return_t +pmap_enter_options_internal( + pmap_t pmap, + vm_map_address_t v, + ppnum_t pn, + vm_prot_t prot, + vm_prot_t fault_type, + unsigned int flags, + boolean_t wired, + unsigned int options) +{ + pmap_paddr_t pa = ptoa(pn); + pt_entry_t pte; + pt_entry_t spte; + pt_entry_t *pte_p; + pv_entry_t *pve_p; + boolean_t set_NX; + boolean_t set_XO = FALSE; + boolean_t refcnt_updated; + unsigned int wimg_bits; + boolean_t was_compressed, was_alt_compressed; + + if ((v) & PAGE_MASK) { + panic("pmap_enter_options() pmap %p v 0x%llx\n", + pmap, (uint64_t)v); + } + + if ((prot & VM_PROT_EXECUTE) && (prot & VM_PROT_WRITE) && (pmap == kernel_pmap)) { + panic("pmap_enter_options(): WX request on kernel_pmap"); + } + +#if DEVELOPMENT || DEBUG + if ((prot & VM_PROT_EXECUTE) || !nx_enabled || !pmap->nx_enabled) +#else + if ((prot & VM_PROT_EXECUTE)) +#endif + set_NX = FALSE; + else + set_NX = TRUE; + +#if (__ARM_VMSA__ > 7) + if (prot == VM_PROT_EXECUTE) { + set_XO = TRUE; + } +#endif + + assert(pn != vm_page_fictitious_addr); + + refcnt_updated = FALSE; + pve_p = PV_ENTRY_NULL; + was_compressed = FALSE; + was_alt_compressed = FALSE; + + PMAP_LOCK(pmap); + + /* + * Expand pmap to include this pte. Assume that + * pmap is always expanded to include enough hardware + * pages to map one VM page. + */ + while ((pte_p = pmap_pte(pmap, v)) == PT_ENTRY_NULL) { + /* Must unlock to expand the pmap. */ + PMAP_UNLOCK(pmap); + + kern_return_t kr=pmap_expand(pmap, v, options, PMAP_TT_MAX_LEVEL); + + if(kr) { + return kr; + } + + PMAP_LOCK(pmap); + } + + if (options & PMAP_OPTIONS_NOENTER) { + PMAP_UNLOCK(pmap); + return KERN_SUCCESS; + } + +Pmap_enter_retry: + + spte = *pte_p; + + if (ARM_PTE_IS_COMPRESSED(spte)) { + /* + * "pmap" should be locked at this point, so this should + * not race with another pmap_enter() or pmap_remove_range(). + */ + assert(pmap != kernel_pmap); + + /* one less "compressed" */ + OSAddAtomic64(-1, &pmap->stats.compressed); + pmap_ledger_debit(pmap, task_ledgers.internal_compressed, + PAGE_SIZE); + + was_compressed = TRUE; + if (spte & ARM_PTE_COMPRESSED_ALT) { + was_alt_compressed = TRUE; + pmap_ledger_debit( + pmap, + task_ledgers.alternate_accounting_compressed, + PAGE_SIZE); + } else { + /* was part of the footprint */ + pmap_ledger_debit(pmap, task_ledgers.phys_footprint, PAGE_SIZE); + } + + /* clear "compressed" marker */ + /* XXX is it necessary since we're about to overwrite it ? */ + WRITE_PTE_FAST(pte_p, ARM_PTE_TYPE_FAULT); + spte = ARM_PTE_TYPE_FAULT; + + /* + * We're replacing a "compressed" marker with a valid PTE, + * so no change for "refcnt". + */ + refcnt_updated = TRUE; + } + + if ((spte != ARM_PTE_TYPE_FAULT) && (pte_to_pa(spte) != pa)) { + pmap_remove_range(pmap, v, pte_p, pte_p + 1, 0); + PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE); + } + + pte = pa_to_pte(pa) | ARM_PTE_TYPE; + + /* Don't bother tracking wiring for kernel PTEs. We use ARM_PTE_WIRED to track + * wired memory statistics for user pmaps, but kernel PTEs are assumed + * to be wired in nearly all cases. For VM layer functionality, the wired + * count in vm_page_t is sufficient. */ + if (wired && pmap != kernel_pmap) + pte |= ARM_PTE_WIRED; + +#if (__ARM_VMSA__ == 7) + if (set_NX) + pte |= ARM_PTE_NX; +#else + if (set_NX) + pte |= ARM_PTE_NX | ARM_PTE_PNX; + else { + if (pmap == kernel_pmap) { + pte |= ARM_PTE_NX; + } else { + pte |= ARM_PTE_PNX; + } + } +#endif + + if ((flags & (VM_WIMG_MASK | VM_WIMG_USE_DEFAULT))) + wimg_bits = (flags & (VM_WIMG_MASK | VM_WIMG_USE_DEFAULT)); + else + wimg_bits = pmap_cache_attributes(pn); + + pte |= wimg_to_pte(wimg_bits); + + if (pmap == kernel_pmap) { + if (prot & VM_PROT_WRITE) { + pte |= ARM_PTE_AP(AP_RWNA); + pa_set_bits(pa, PP_ATTR_MODIFIED | PP_ATTR_REFERENCED); + } else { + pte |= ARM_PTE_AP(AP_RONA); + pa_set_bits(pa, PP_ATTR_REFERENCED); + } +#if (__ARM_VMSA__ == 7) + if ((_COMM_PAGE_BASE_ADDRESS <= v) && (v < _COMM_PAGE_BASE_ADDRESS + _COMM_PAGE_AREA_LENGTH)) + pte = (pte & ~(ARM_PTE_APMASK)) | ARM_PTE_AP(AP_RORO); +#endif + } else { + if (!(pmap->nested)) { + pte |= ARM_PTE_NG; + } else if ((pmap->nested_region_asid_bitmap) + && (v >= pmap->nested_region_subord_addr) + && (v < (pmap->nested_region_subord_addr+pmap->nested_region_size))) { + + unsigned int index = (unsigned int)((v - pmap->nested_region_subord_addr) >> ARM_TT_TWIG_SHIFT); + + if ((pmap->nested_region_asid_bitmap) + && testbit(index, (int *)pmap->nested_region_asid_bitmap)) + pte |= ARM_PTE_NG; + } +#if MACH_ASSERT + if (pmap->nested_pmap != NULL) { + vm_map_address_t nest_vaddr; + pt_entry_t *nest_pte_p; + + nest_vaddr = v - pmap->nested_region_grand_addr + pmap->nested_region_subord_addr; + + if ((nest_vaddr >= pmap->nested_region_subord_addr) + && (nest_vaddr < (pmap->nested_region_subord_addr+pmap->nested_region_size)) + && ((nest_pte_p = pmap_pte(pmap->nested_pmap, nest_vaddr)) != PT_ENTRY_NULL) + && (*nest_pte_p != ARM_PTE_TYPE_FAULT) + && (!ARM_PTE_IS_COMPRESSED(*nest_pte_p)) + && (((*nest_pte_p) & ARM_PTE_NG) != ARM_PTE_NG)) { + unsigned int index = (unsigned int)((v - pmap->nested_region_subord_addr) >> ARM_TT_TWIG_SHIFT); + + if ((pmap->nested_pmap->nested_region_asid_bitmap) + && !testbit(index, (int *)pmap->nested_pmap->nested_region_asid_bitmap)) { + + panic("pmap_enter(): Global attribute conflict nest_pte_p=%p pmap=%p v=0x%llx spte=0x%llx \n", + nest_pte_p, pmap, (uint64_t)v, (uint64_t)*nest_pte_p); + } + } + + } +#endif + if (prot & VM_PROT_WRITE) { + + if (pa_valid(pa) && (!pa_test_bits(pa, PP_ATTR_MODIFIED))) { + if (fault_type & VM_PROT_WRITE) { + if (set_XO) + pte |= ARM_PTE_AP(AP_RWNA); + else + pte |= ARM_PTE_AP(AP_RWRW); + pa_set_bits(pa, PP_ATTR_REFERENCED | PP_ATTR_MODIFIED); + } else { + if (set_XO) + pte |= ARM_PTE_AP(AP_RONA); + else + pte |= ARM_PTE_AP(AP_RORO); + pa_set_bits(pa, PP_ATTR_REFERENCED); + pte_set_ffr(pte, 1); + } + } else { + if (set_XO) + pte |= ARM_PTE_AP(AP_RWNA); + else + pte |= ARM_PTE_AP(AP_RWRW); + pa_set_bits(pa, PP_ATTR_REFERENCED); + } + } else { + + if (set_XO) + pte |= ARM_PTE_AP(AP_RONA); + else + pte |= ARM_PTE_AP(AP_RORO); + pa_set_bits(pa, PP_ATTR_REFERENCED); + } + } + + pte |= ARM_PTE_AF; + + volatile uint16_t *refcnt = NULL; + if (pmap != kernel_pmap) { + refcnt = &(ptep_get_ptd(pte_p)->pt_cnt[ARM_PT_DESC_INDEX(pte_p)].refcnt); + /* Mark the PT page active to keep it from being reclaimed. We need this because + * we may drop the PVH and pmap locks later in pmap_enter() if we need to allocate + * a new PV entry. Note that setting this high bit (0x4000) can temporarily + * prevent the refcount underflow checks in pmap_page_protect() and pmap_remove() from + * working. If an underflow should happen during this window, we'll instead get a + * refcount along the lines of 0x3FFF, which will produce a later panic on non-zero + * refcount in pmap_pages_reclaim() or pmap_tt_deallocate(). */ + OSBitOrAtomic16(PT_DESC_REFCOUNT, refcnt); + if (!refcnt_updated) { + OSAddAtomic16(1, (volatile int16_t*)refcnt); + refcnt_updated = TRUE; + } + } + + if (pa_valid(pa)) { + pv_entry_t **pv_h; + int pai; + boolean_t is_altacct, is_internal; + + is_internal = FALSE; + is_altacct = FALSE; + + pai = (int)pa_index(pa); + pv_h = pai_to_pvh(pai); + + LOCK_PVH(pai); +Pmap_enter_loop: + + if (pte == *pte_p) { + /* + * This pmap_enter operation has been completed by another thread + * undo refcnt on pt and return + */ + if (refcnt != NULL) { + assert(refcnt_updated); + if (OSAddAtomic16(-1, (volatile int16_t*)refcnt) <= (int16_t)PT_DESC_REFCOUNT) + panic("pmap_enter(): over-release of ptdp %p for pte %p\n", ptep_get_ptd(pte_p), pte_p); + } + UNLOCK_PVH(pai); + goto Pmap_enter_return; + } else if (pte_to_pa(*pte_p) == pa) { + if (refcnt != NULL) { + assert(refcnt_updated); + if (OSAddAtomic16(-1, (volatile int16_t*)refcnt) <= (int16_t)PT_DESC_REFCOUNT) + panic("pmap_enter(): over-release of ptdp %p for pte %p\n", ptep_get_ptd(pte_p), pte_p); + } + pmap_enter_pte(pmap, pte_p, pte, v); + UNLOCK_PVH(pai); + goto Pmap_enter_return; + } else if (*pte_p != ARM_PTE_TYPE_FAULT) { + /* + * pte has been modified by another thread + * hold refcnt on pt and retry pmap_enter operation + */ + UNLOCK_PVH(pai); + goto Pmap_enter_retry; + } + if (pvh_test_type(pv_h, PVH_TYPE_NULL)) { + pvh_update_head(pv_h, pte_p, PVH_TYPE_PTEP); + /* 1st mapping: see what kind of page it is */ + if (options & PMAP_OPTIONS_INTERNAL) { + SET_INTERNAL_PAGE(pai); + } else { + CLR_INTERNAL_PAGE(pai); + } + if ((options & PMAP_OPTIONS_INTERNAL) && + (options & PMAP_OPTIONS_REUSABLE)) { + SET_REUSABLE_PAGE(pai); + } else { + CLR_REUSABLE_PAGE(pai); + } + if (pmap != kernel_pmap && + ((options & PMAP_OPTIONS_ALT_ACCT) || + PMAP_FOOTPRINT_SUSPENDED(pmap)) && + IS_INTERNAL_PAGE(pai)) { + /* + * Make a note to ourselves that this mapping is using alternative + * accounting. We'll need this in order to know which ledger to + * debit when the mapping is removed. + * + * The altacct bit must be set while the pv head is locked. Defer + * the ledger accounting until after we've dropped the lock. + */ + SET_ALTACCT_PAGE(pai, PV_ENTRY_NULL); + is_altacct = TRUE; + } else { + CLR_ALTACCT_PAGE(pai, PV_ENTRY_NULL); + } + } else { + if (pvh_test_type(pv_h, PVH_TYPE_PTEP)) { + pt_entry_t *pte1_p; + + /* + * convert pvh list from PVH_TYPE_PTEP to PVH_TYPE_PVEP + */ + pte1_p = pvh_ptep(pv_h); + if((pve_p == PV_ENTRY_NULL) && (!pv_alloc(pmap, pai, &pve_p))) { + goto Pmap_enter_loop; + } + pve_set_ptep(pve_p, pte1_p); + pve_p->pve_next = PV_ENTRY_NULL; + + if (IS_ALTACCT_PAGE(pai, PV_ENTRY_NULL)) { + /* + * transfer "altacct" from + * pp_attr to this pve + */ + CLR_ALTACCT_PAGE(pai, PV_ENTRY_NULL); + SET_ALTACCT_PAGE(pai, pve_p); + } + pvh_update_head(pv_h, pve_p, PVH_TYPE_PVEP); + pve_p = PV_ENTRY_NULL; + } + /* + * Set up pv_entry for this new mapping and then + * add it to the list for this physical page. + */ + if((pve_p == PV_ENTRY_NULL) && (!pv_alloc(pmap, pai, &pve_p))) { + goto Pmap_enter_loop; + } + pve_set_ptep(pve_p, pte_p); + pve_p->pve_next = PV_ENTRY_NULL; + + pvh_add(pv_h, pve_p); + + if (pmap != kernel_pmap && + ((options & PMAP_OPTIONS_ALT_ACCT) || + PMAP_FOOTPRINT_SUSPENDED(pmap)) && + IS_INTERNAL_PAGE(pai)) { + /* + * Make a note to ourselves that this + * mapping is using alternative + * accounting. We'll need this in order + * to know which ledger to debit when + * the mapping is removed. + * + * The altacct bit must be set while + * the pv head is locked. Defer the + * ledger accounting until after we've + * dropped the lock. + */ + SET_ALTACCT_PAGE(pai, pve_p); + is_altacct = TRUE; + } + + pve_p = PV_ENTRY_NULL; + } + + pmap_enter_pte(pmap, pte_p, pte, v); + + if (pmap != kernel_pmap) { + if (IS_REUSABLE_PAGE(pai) && + !is_altacct) { + assert(IS_INTERNAL_PAGE(pai)); + OSAddAtomic(+1, &pmap->stats.reusable); + PMAP_STATS_PEAK(pmap->stats.reusable); + } else if (IS_INTERNAL_PAGE(pai)) { + OSAddAtomic(+1, &pmap->stats.internal); + PMAP_STATS_PEAK(pmap->stats.internal); + is_internal = TRUE; + } else { + OSAddAtomic(+1, &pmap->stats.external); + PMAP_STATS_PEAK(pmap->stats.external); + } + } + + UNLOCK_PVH(pai); + + if (pmap != kernel_pmap) { + pmap_ledger_credit(pmap, task_ledgers.phys_mem, PAGE_SIZE); + + if (is_internal) { + /* + * Make corresponding adjustments to + * phys_footprint statistics. + */ + pmap_ledger_credit(pmap, task_ledgers.internal, PAGE_SIZE); + if (is_altacct) { + /* + * If this page is internal and + * in an IOKit region, credit + * the task's total count of + * dirty, internal IOKit pages. + * It should *not* count towards + * the task's total physical + * memory footprint, because + * this entire region was + * already billed to the task + * at the time the mapping was + * created. + * + * Put another way, this is + * internal++ and + * alternate_accounting++, so + * net effect on phys_footprint + * is 0. That means: don't + * touch phys_footprint here. + */ + pmap_ledger_credit(pmap, task_ledgers.alternate_accounting, PAGE_SIZE); + } else { + pmap_ledger_credit(pmap, task_ledgers.phys_footprint, PAGE_SIZE); + } + } + } + + OSAddAtomic(1, (SInt32 *) &pmap->stats.resident_count); + if (pmap->stats.resident_count > pmap->stats.resident_max) + pmap->stats.resident_max = pmap->stats.resident_count; + } else { + pmap_enter_pte(pmap, pte_p, pte, v); + } + +Pmap_enter_return: + +#if CONFIG_PGTRACE + if (pgtrace_enabled) { + // Clone and invalidate original mapping if eligible + for (int i = 0; i < PAGE_RATIO; i++) { + pmap_pgtrace_enter_clone(pmap, v + ARM_PGBYTES*i, 0, 0); + } + } +#endif + + if (pve_p != PV_ENTRY_NULL) + pv_free(pve_p); + + if (refcnt != NULL) + OSBitAndAtomic16(~PT_DESC_REFCOUNT, refcnt); // clear active marker + PMAP_UNLOCK(pmap); + + return KERN_SUCCESS; +} + +kern_return_t +pmap_enter_options( + pmap_t pmap, + vm_map_address_t v, + ppnum_t pn, + vm_prot_t prot, + vm_prot_t fault_type, + unsigned int flags, + boolean_t wired, + unsigned int options, + __unused void *arg) +{ + kern_return_t kr = KERN_FAILURE; + + PMAP_TRACE(PMAP_CODE(PMAP__ENTER) | DBG_FUNC_START, + VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(v), pn, prot); + + kr = pmap_enter_options_internal(pmap, v, pn, prot, fault_type, flags, wired, options); + + PMAP_TRACE(PMAP_CODE(PMAP__ENTER) | DBG_FUNC_END, kr); + + return kr; +} + +/* + * Routine: pmap_change_wiring + * Function: Change the wiring attribute for a map/virtual-address + * pair. + * In/out conditions: + * The mapping must already exist in the pmap. + */ +static void +pmap_change_wiring_internal( + pmap_t pmap, + vm_map_address_t v, + boolean_t wired) +{ + pt_entry_t *pte_p; + pmap_paddr_t pa; + + /* Don't bother tracking wiring for kernel PTEs. We use ARM_PTE_WIRED to track + * wired memory statistics for user pmaps, but kernel PTEs are assumed + * to be wired in nearly all cases. For VM layer functionality, the wired + * count in vm_page_t is sufficient. */ + if (pmap == kernel_pmap) { + return; + } + + PMAP_LOCK(pmap); + pte_p = pmap_pte(pmap, v); + assert(pte_p != PT_ENTRY_NULL); + pa = pte_to_pa(*pte_p); + if (pa_valid(pa)) + LOCK_PVH((int)pa_index(pa)); + + if (wired && !pte_is_wired(*pte_p)) { + pte_set_wired(pte_p, wired); + OSAddAtomic(+1, (SInt32 *) &pmap->stats.wired_count); + pmap_ledger_credit(pmap, task_ledgers.wired_mem, PAGE_SIZE); + } else if (!wired && pte_is_wired(*pte_p)) { + assert(pmap->stats.wired_count >= 1); + pte_set_wired(pte_p, wired); + OSAddAtomic(-1, (SInt32 *) &pmap->stats.wired_count); + pmap_ledger_debit(pmap, task_ledgers.wired_mem, PAGE_SIZE); + } + + if (pa_valid(pa)) + UNLOCK_PVH((int)pa_index(pa)); + + PMAP_UNLOCK(pmap); +} + +void +pmap_change_wiring( + pmap_t pmap, + vm_map_address_t v, + boolean_t wired) +{ + pmap_change_wiring_internal(pmap, v, wired); +} + +static ppnum_t +pmap_find_phys_internal( + pmap_t pmap, + addr64_t va) +{ + ppnum_t ppn=0; + + if (pmap != kernel_pmap) { + PMAP_LOCK(pmap); + } + + ppn = pmap_vtophys(pmap, va); + + if (pmap != kernel_pmap) { + PMAP_UNLOCK(pmap); + } + + return ppn; +} + +ppnum_t +pmap_find_phys( + pmap_t pmap, + addr64_t va) +{ + pmap_paddr_t pa=0; + + if (pmap == kernel_pmap) + pa = mmu_kvtop(va); + else if ((current_thread()->map) && (pmap == vm_map_pmap(current_thread()->map))) + pa = mmu_uvtop(va); + + if (pa) return (ppnum_t)(pa >> PAGE_SHIFT); + + if (not_in_kdp) { + return pmap_find_phys_internal(pmap, va); + } else { + return pmap_vtophys(pmap, va); + } +} + +pmap_paddr_t +kvtophys( + vm_offset_t va) +{ + pmap_paddr_t pa; + + pa = mmu_kvtop(va); + if (pa) return pa; + pa = ((pmap_paddr_t)pmap_vtophys(kernel_pmap, va)) << PAGE_SHIFT; + if (pa) + pa |= (va & PAGE_MASK); + + return ((pmap_paddr_t)pa); +} + +ppnum_t +pmap_vtophys( + pmap_t pmap, + addr64_t va) +{ + if ((va < pmap->min) || (va >= pmap->max)) { + return 0; + } + +#if (__ARM_VMSA__ == 7) + tt_entry_t *tte_p, tte; + pt_entry_t *pte_p; + ppnum_t ppn; + + tte_p = pmap_tte(pmap, va); + if (tte_p == (tt_entry_t *) NULL) + return (ppnum_t) 0; + + tte = *tte_p; + if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) { + pte_p = (pt_entry_t *) ttetokv(tte) + ptenum(va); + ppn = (ppnum_t) atop(pte_to_pa(*pte_p) | (va & ARM_PGMASK)); +#if DEVELOPMENT || DEBUG + if (ppn != 0 && + ARM_PTE_IS_COMPRESSED(*pte_p)) { + panic("pmap_vtophys(%p,0x%llx): compressed pte_p=%p 0x%llx with ppn=0x%x\n", + pmap, va, pte_p, (uint64_t) (*pte_p), ppn); + } +#endif /* DEVELOPMENT || DEBUG */ + } else if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_BLOCK) + if ((tte & ARM_TTE_BLOCK_SUPER) == ARM_TTE_BLOCK_SUPER) + ppn = (ppnum_t) atop(suptte_to_pa(tte) | (va & ARM_TT_L1_SUPER_OFFMASK)); + else + ppn = (ppnum_t) atop(sectte_to_pa(tte) | (va & ARM_TT_L1_BLOCK_OFFMASK)); + else + ppn = 0; +#else + tt_entry_t *ttp; + tt_entry_t tte; + ppnum_t ppn=0; + + /* Level 0 currently unused */ + +#if __ARM64_TWO_LEVEL_PMAP__ + /* We have no L1 entry; go straight to the L2 entry */ + ttp = pmap_tt2e(pmap, va); + tte = *ttp; +#else + /* Get first-level (1GB) entry */ + ttp = pmap_tt1e(pmap, va); + tte = *ttp; + if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) != (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID)) + return (ppn); + + tte = ((tt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt2_index(pmap, va)]; +#endif + if ((tte & ARM_TTE_VALID) != (ARM_TTE_VALID)) + return (ppn); + + if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_BLOCK) { + ppn = (ppnum_t) atop((tte & ARM_TTE_BLOCK_L2_MASK)| (va & ARM_TT_L2_OFFMASK)); + return(ppn); + } + tte = ((tt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt3_index(pmap, va)]; + ppn = (ppnum_t) atop((tte & ARM_PTE_MASK)| (va & ARM_TT_L3_OFFMASK)); +#endif + + return ppn; +} + +static vm_offset_t +pmap_extract_internal( + pmap_t pmap, + vm_map_address_t va) +{ + pmap_paddr_t pa=0; + ppnum_t ppn=0; + + if (pmap == NULL) { + return 0; + } + + PMAP_LOCK(pmap); + + ppn = pmap_vtophys(pmap, va); + + if (ppn != 0) + pa = ptoa(ppn)| ((va) & PAGE_MASK); + + PMAP_UNLOCK(pmap); + + return pa; +} + +/* + * Routine: pmap_extract + * Function: + * Extract the physical page address associated + * with the given map/virtual_address pair. + * + */ +vm_offset_t +pmap_extract( + pmap_t pmap, + vm_map_address_t va) +{ + pmap_paddr_t pa=0; + + if (pmap == kernel_pmap) + pa = mmu_kvtop(va); + else if (pmap == vm_map_pmap(current_thread()->map)) + pa = mmu_uvtop(va); + + if (pa) return pa; + + return pmap_extract_internal(pmap, va); +} + +/* + * pmap_init_pte_page - Initialize a page table page. + */ +void +pmap_init_pte_page( + pmap_t pmap, + pt_entry_t *pte_p, + vm_offset_t va, + unsigned int ttlevel, + boolean_t alloc_ptd) +{ + pt_desc_t *ptdp; + + ptdp = *(pt_desc_t **)pai_to_pvh(pa_index((((vm_offset_t)pte_p) - gVirtBase + gPhysBase))); + + if (ptdp == NULL) { + if (alloc_ptd) { + /* + * This path should only be invoked from arm_vm_init. If we are emulating 16KB pages + * on 4KB hardware, we may already have allocated a page table descriptor for a + * bootstrap request, so we check for an existing PTD here. + */ + ptdp = ptd_alloc(pmap); + *(pt_desc_t **)pai_to_pvh(pa_index((((vm_offset_t)pte_p) - gVirtBase + gPhysBase))) = ptdp; + } else { + panic("pmap_init_pte_page(): pte_p %p\n", pte_p); + } + } + + pmap_init_pte_page_internal(pmap, pte_p, va, ttlevel, &ptdp); +} + +/* + * pmap_init_pte_page_internal - Initialize page table page and page table descriptor + */ +void +pmap_init_pte_page_internal( + pmap_t pmap, + pt_entry_t *pte_p, + vm_offset_t va, + unsigned int ttlevel, + pt_desc_t **ptdp) +{ + bzero(pte_p, ARM_PGBYTES); + // below barrier ensures the page zeroing is visible to PTW before + // it is linked to the PTE of previous level + __asm__ volatile("DMB ST" : : : "memory"); + ptd_init(*ptdp, pmap, va, ttlevel, pte_p); +} + +/* + * pmap_init_pte_static_page - for static mappings to a known contiguous range of pa's + * Called from arm_vm_init(). + */ +void +pmap_init_pte_static_page( + __unused pmap_t pmap, + pt_entry_t * pte_p, + pmap_paddr_t pa) +{ +#if (__ARM_VMSA__ == 7) + unsigned int i; + pt_entry_t *pte_cur; + + for (i = 0, pte_cur = pte_p; + i < (ARM_PGBYTES / sizeof(*pte_p)); + i++, pa += PAGE_SIZE) { + if (pa >= avail_end) { + /* We don't want to map memory xnu does not own through this routine. */ + break; + } + + *pte_cur = pa_to_pte(pa) + | ARM_PTE_TYPE | ARM_PTE_AF | ARM_PTE_SH | ARM_PTE_AP(AP_RONA) + | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT); + pte_cur++; + } +#else + unsigned int i; + pt_entry_t *pte_cur; + pt_entry_t template; + + template = ARM_PTE_TYPE | ARM_PTE_AF | ARM_PTE_SH(SH_OUTER_MEMORY) | ARM_PTE_AP(AP_RONA) | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT) | ARM_PTE_NX; + + for (i = 0, pte_cur = pte_p; + i < (ARM_PGBYTES / sizeof(*pte_p)); + i++, pa += PAGE_SIZE) { + if (pa >= avail_end) { + /* We don't want to map memory xnu does not own through this routine. */ + break; + } + + /* TEST_PAGE_RATIO_4 may be pre-processor defined to 0 */ + __unreachable_ok_push + if (TEST_PAGE_RATIO_4) { + *pte_cur = pa_to_pte(pa) | template; + *(pte_cur+1) = pa_to_pte(pa+0x1000) | template; + *(pte_cur+2) = pa_to_pte(pa+0x2000) | template; + *(pte_cur+3) = pa_to_pte(pa+0x3000) | template; + pte_cur += 4; + } else { + *pte_cur = pa_to_pte(pa) | template; + pte_cur++; + } + __unreachable_ok_pop + } +#endif + bzero(pte_cur, ARM_PGBYTES - ((vm_offset_t)pte_cur - (vm_offset_t)pte_p)); +} + + +/* + * Routine: pmap_expand + * + * Expands a pmap to be able to map the specified virtual address. + * + * Allocates new memory for the default (COARSE) translation table + * entry, initializes all the pte entries to ARM_PTE_TYPE_FAULT and + * also allocates space for the corresponding pv entries. + * + * Nothing should be locked. + */ +static kern_return_t +pmap_expand( + pmap_t pmap, + vm_map_address_t v, + unsigned int options, + unsigned int level) +{ +#if (__ARM_VMSA__ == 7) + vm_offset_t pa; + tt_entry_t *tte_p; + tt_entry_t *tt_p; + unsigned int i; + + + while (tte_index(pmap, v) >= pmap->tte_index_max) { + tte_p = pmap_tt1_allocate(pmap, 2*ARM_PGBYTES, ((options & PMAP_OPTIONS_NOWAIT)? PMAP_TT_ALLOCATE_NOWAIT : 0)); + if (tte_p == (tt_entry_t *)0) + return KERN_RESOURCE_SHORTAGE; + + PMAP_LOCK(pmap); + if (pmap->tte_index_max > NTTES) { + pmap_tt1_deallocate(pmap, tte_p, 2*ARM_PGBYTES, PMAP_TT_DEALLOCATE_NOBLOCK); + PMAP_UNLOCK(pmap); + break; + } + + simple_lock(&pmap->tt1_lock); + for (i = 0; i < pmap->tte_index_max; i++) + tte_p[i] = pmap->tte[i]; + for (i = NTTES; i < 2*NTTES; i++) + tte_p[i] = ARM_TTE_TYPE_FAULT; + + pmap->prev_tte = pmap->tte; + pmap->tte = tte_p; + pmap->ttep = ml_static_vtop((vm_offset_t)pmap->tte); +#ifndef __ARM_L1_PTW__ + CleanPoU_DcacheRegion((vm_offset_t) pmap->tte, 2*NTTES * sizeof(tt_entry_t)); +#else + __builtin_arm_dsb(DSB_ISH); +#endif + pmap->tte_index_max = 2*NTTES; + pmap->stamp = hw_atomic_add(&pmap_stamp, 1); + + for (i = 0; i < NTTES; i++) + pmap->prev_tte[i] = ARM_TTE_TYPE_FAULT; +#ifndef __ARM_L1_PTW__ + CleanPoU_DcacheRegion((vm_offset_t) pmap->prev_tte, NTTES * sizeof(tt_entry_t)); +#else + __builtin_arm_dsb(DSB_ISH); +#endif + + simple_unlock(&pmap->tt1_lock); + PMAP_UNLOCK(pmap); + pmap_set_pmap(pmap, current_thread()); + + } + + if (level == 1) + return (KERN_SUCCESS); + + { + tt_entry_t *tte_next_p; + + PMAP_LOCK(pmap); + pa = 0; + if (pmap_pte(pmap, v) != PT_ENTRY_NULL) { + PMAP_UNLOCK(pmap); + return (KERN_SUCCESS); + } + tte_p = &pmap->tte[ttenum(v & ~ARM_TT_L1_PT_OFFMASK)]; + for (i = 0, tte_next_p = tte_p; i<4; i++) { + if (tte_to_pa(*tte_next_p)) { + pa = tte_to_pa(*tte_next_p); + break; + } + tte_next_p++; + } + pa = pa & ~PAGE_MASK; + if (pa) { + tte_p = &pmap->tte[ttenum(v)]; + *tte_p = pa_to_tte(pa) | (((v >> ARM_TT_L1_SHIFT) & 0x3) << 10) | ARM_TTE_TYPE_TABLE; +#ifndef __ARM_L1_PTW__ + CleanPoU_DcacheRegion((vm_offset_t) tte_p, sizeof(tt_entry_t)); +#endif + PMAP_UNLOCK(pmap); + return (KERN_SUCCESS); + } + PMAP_UNLOCK(pmap); + } + v = v & ~ARM_TT_L1_PT_OFFMASK; + + + while (pmap_pte(pmap, v) == PT_ENTRY_NULL) { + /* + * Allocate a VM page for the level 2 page table entries. + */ + while (pmap_tt_allocate(pmap, &tt_p, PMAP_TT_L2_LEVEL, ((options & PMAP_TT_ALLOCATE_NOWAIT)? PMAP_PAGES_ALLOCATE_NOWAIT : 0)) != KERN_SUCCESS) { + if(options & PMAP_OPTIONS_NOWAIT) { + return KERN_RESOURCE_SHORTAGE; + } + VM_PAGE_WAIT(); + } + + PMAP_LOCK(pmap); + /* + * See if someone else expanded us first + */ + if (pmap_pte(pmap, v) == PT_ENTRY_NULL) { + tt_entry_t *tte_next_p; + + pmap_init_pte_page(pmap, (pt_entry_t *) tt_p, v, PMAP_TT_L2_LEVEL, FALSE); + pa = kvtophys((vm_offset_t)tt_p); +#ifndef __ARM_L1_PTW__ + CleanPoU_DcacheRegion((vm_offset_t) phystokv(pa), PAGE_SIZE); +#endif + tte_p = &pmap->tte[ttenum(v)]; + for (i = 0, tte_next_p = tte_p; i<4; i++) { + *tte_next_p = pa_to_tte(pa) | ARM_TTE_TYPE_TABLE; + tte_next_p++; + pa = pa +0x400; + } +#ifndef __ARM_L1_PTW__ + CleanPoU_DcacheRegion((vm_offset_t) tte_p, 4*sizeof(tt_entry_t)); +#endif + pa = 0x0ULL; + tt_p = (tt_entry_t *)NULL; + } + PMAP_UNLOCK(pmap); + if (tt_p != (tt_entry_t *)NULL) { + pmap_tt_deallocate(pmap, tt_p, PMAP_TT_L2_LEVEL); + tt_p = (tt_entry_t *)NULL; + } + } + return (KERN_SUCCESS); +#else + pmap_paddr_t pa; +#if __ARM64_TWO_LEVEL_PMAP__ + /* If we are using a two level page table, we'll start at L2. */ + unsigned int ttlevel = 2; +#else + /* Otherwise, we start at L1 (we use 3 levels by default). */ + unsigned int ttlevel = 1; +#endif + tt_entry_t *tte_p; + tt_entry_t *tt_p; + + pa = 0x0ULL; + tt_p = (tt_entry_t *)NULL; + + for (; ttlevel < level; ttlevel++) { + + PMAP_LOCK(pmap); + + if (ttlevel == 1) { + if ((pmap_tt2e(pmap, v) == PT_ENTRY_NULL)) { + PMAP_UNLOCK(pmap); + while (pmap_tt_allocate(pmap, &tt_p, PMAP_TT_L2_LEVEL, ((options & PMAP_TT_ALLOCATE_NOWAIT)? PMAP_PAGES_ALLOCATE_NOWAIT : 0)) != KERN_SUCCESS) { + if(options & PMAP_OPTIONS_NOWAIT) { + return KERN_RESOURCE_SHORTAGE; + } + VM_PAGE_WAIT(); + } + PMAP_LOCK(pmap); + if ((pmap_tt2e(pmap, v) == PT_ENTRY_NULL)) { + pmap_init_pte_page(pmap, (pt_entry_t *) tt_p, v, PMAP_TT_L2_LEVEL, FALSE); + pa = kvtophys((vm_offset_t)tt_p); + tte_p = pmap_tt1e( pmap, v); + *tte_p = (pa & ARM_TTE_TABLE_MASK) | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID; + pa = 0x0ULL; + tt_p = (tt_entry_t *)NULL; + if ((pmap == kernel_pmap) && (VM_MIN_KERNEL_ADDRESS < 0x00000000FFFFFFFFULL)) + current_pmap()->tte[v>>ARM_TT_L1_SHIFT] = kernel_pmap->tte[v>>ARM_TT_L1_SHIFT]; + } + + } + } else if (ttlevel == 2) { + if (pmap_tt3e(pmap, v) == PT_ENTRY_NULL) { + PMAP_UNLOCK(pmap); + while (pmap_tt_allocate(pmap, &tt_p, PMAP_TT_L3_LEVEL, ((options & PMAP_TT_ALLOCATE_NOWAIT)? PMAP_PAGES_ALLOCATE_NOWAIT : 0)) != KERN_SUCCESS) { + if(options & PMAP_OPTIONS_NOWAIT) { + return KERN_RESOURCE_SHORTAGE; + } + VM_PAGE_WAIT(); + } + PMAP_LOCK(pmap); + if ((pmap_tt3e(pmap, v) == PT_ENTRY_NULL)) { + pmap_init_pte_page(pmap, (pt_entry_t *) tt_p, v , PMAP_TT_L3_LEVEL, FALSE); + pa = kvtophys((vm_offset_t)tt_p); + tte_p = pmap_tt2e( pmap, v); + *tte_p = (pa & ARM_TTE_TABLE_MASK) | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID; + pa = 0x0ULL; + tt_p = (tt_entry_t *)NULL; + } + } + } + + PMAP_UNLOCK(pmap); + + if (tt_p != (tt_entry_t *)NULL) { + pmap_tt_deallocate(pmap, tt_p, ttlevel+1); + tt_p = (tt_entry_t *)NULL; + } + } + + return (KERN_SUCCESS); +#endif +} + +/* + * Routine: pmap_collect + * Function: + * Garbage collects the physical map system for + * pages which are no longer used. + * Success need not be guaranteed -- that is, there + * may well be pages which are not referenced, but + * others may be collected. + */ +void +pmap_collect(pmap_t pmap) +{ + if (pmap == PMAP_NULL) + return; + +#if 0 + PMAP_LOCK(pmap); + if ((pmap->nested == FALSE) && (pmap != kernel_pmap)) { + /* TODO: Scan for vm page assigned to top level page tables with no reference */ + } + PMAP_UNLOCK(pmap); +#endif + + return; +} + +/* + * Routine: pmap_gc + * Function: + * Pmap garbage collection + * Called by the pageout daemon when pages are scarce. + * + */ +void +pmap_gc( + void) +{ + pmap_t pmap, pmap_next; + boolean_t gc_wait; + + if (pmap_gc_allowed && + (pmap_gc_allowed_by_time_throttle || + pmap_gc_forced)) { + pmap_gc_forced = FALSE; + pmap_gc_allowed_by_time_throttle = FALSE; + simple_lock(&pmaps_lock); + pmap = CAST_DOWN_EXPLICIT(pmap_t, queue_first(&map_pmap_list)); + while (!queue_end(&map_pmap_list, (queue_entry_t)pmap)) { + if (!(pmap->gc_status & PMAP_GC_INFLIGHT)) + pmap->gc_status |= PMAP_GC_INFLIGHT; + simple_unlock(&pmaps_lock); + + pmap_collect(pmap); + + simple_lock(&pmaps_lock); + gc_wait = (pmap->gc_status & PMAP_GC_WAIT); + pmap->gc_status &= ~(PMAP_GC_INFLIGHT|PMAP_GC_WAIT); + pmap_next = CAST_DOWN_EXPLICIT(pmap_t, queue_next(&pmap->pmaps)); + if (gc_wait) { + if (!queue_end(&map_pmap_list, (queue_entry_t)pmap_next)) + pmap_next->gc_status |= PMAP_GC_INFLIGHT; + simple_unlock(&pmaps_lock); + thread_wakeup((event_t) & pmap->gc_status); + simple_lock(&pmaps_lock); + } + pmap = pmap_next; + } + simple_unlock(&pmaps_lock); + } +} + +/* + * Called by the VM to reclaim pages that we can reclaim quickly and cheaply. + */ +void +pmap_release_pages_fast(void) +{ +} + +/* + * By default, don't attempt pmap GC more frequently + * than once / 1 minutes. + */ + +void +compute_pmap_gc_throttle( + void *arg __unused) +{ + pmap_gc_allowed_by_time_throttle = TRUE; +} + +/* + * pmap_attribute_cache_sync(vm_offset_t pa) + * + * Invalidates all of the instruction cache on a physical page and + * pushes any dirty data from the data cache for the same physical page + */ + +kern_return_t +pmap_attribute_cache_sync( + ppnum_t pp, + vm_size_t size, + __unused vm_machine_attribute_t attribute, + __unused vm_machine_attribute_val_t * value) +{ + if (size > PAGE_SIZE) { + panic("pmap_attribute_cache_sync size: 0x%llx\n", (uint64_t)size); + } else + cache_sync_page(pp); + + return KERN_SUCCESS; +} + +/* + * pmap_sync_page_data_phys(ppnum_t pp) + * + * Invalidates all of the instruction cache on a physical page and + * pushes any dirty data from the data cache for the same physical page + */ +void +pmap_sync_page_data_phys( + ppnum_t pp) +{ + cache_sync_page(pp); +} + +/* + * pmap_sync_page_attributes_phys(ppnum_t pp) + * + * Write back and invalidate all cachelines on a physical page. + */ +void +pmap_sync_page_attributes_phys( + ppnum_t pp) +{ + flush_dcache((vm_offset_t) (pp << PAGE_SHIFT), PAGE_SIZE, TRUE); +} + +#if CONFIG_COREDUMP +/* temporary workaround */ +boolean_t +coredumpok( + vm_map_t map, + vm_offset_t va) +{ + pt_entry_t *pte_p; + pt_entry_t spte; + + pte_p = pmap_pte(map->pmap, va); + if (0 == pte_p) + return FALSE; + spte = *pte_p; + return ((spte & ARM_PTE_ATTRINDXMASK) == ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT)); +} +#endif + +void +fillPage( + ppnum_t pn, + unsigned int fill) +{ + unsigned int *addr; + int count; + + addr = (unsigned int *) phystokv(ptoa(pn)); + count = PAGE_SIZE / sizeof(unsigned int); + while (count--) + *addr++ = fill; +} + +extern void mapping_set_mod(ppnum_t pn); + +void +mapping_set_mod( + ppnum_t pn) +{ + pmap_set_modify(pn); +} + +extern void mapping_set_ref(ppnum_t pn); + +void +mapping_set_ref( + ppnum_t pn) +{ + pmap_set_reference(pn); +} + +/* + * Clear specified attribute bits. + * + * Try to force an arm_fast_fault() for all mappings of + * the page - to force attributes to be set again at fault time. + * If the forcing succeeds, clear the cached bits at the head. + * Otherwise, something must have been wired, so leave the cached + * attributes alone. + */ +static void +phys_attribute_clear_internal( + ppnum_t pn, + unsigned int bits, + int options, + void *arg) +{ + pmap_paddr_t pa = ptoa(pn); + vm_prot_t allow_mode = VM_PROT_ALL; + + + if ((bits & PP_ATTR_MODIFIED) && + (options & PMAP_OPTIONS_NOFLUSH) && + (arg == NULL)) { + panic("phys_attribute_clear(0x%x,0x%x,0x%x,%p): " + "should not clear 'modified' without flushing TLBs\n", + pn, bits, options, arg); + } + + assert(pn != vm_page_fictitious_addr); + if (bits & PP_ATTR_REFERENCED) + allow_mode &= ~(VM_PROT_READ | VM_PROT_EXECUTE); + if (bits & PP_ATTR_MODIFIED) + allow_mode &= ~VM_PROT_WRITE; + + if (bits == PP_ATTR_NOENCRYPT) { + /* + * We short circuit this case; it should not need to + * invoke arm_force_fast_fault, so just clear and + * return. On ARM, this bit is just a debugging aid. + */ + pa_clear_bits(pa, bits); + return; + } + + if (arm_force_fast_fault_internal(pn, allow_mode, options)) + pa_clear_bits(pa, bits); + return; +} + +static void +phys_attribute_clear( + ppnum_t pn, + unsigned int bits, + int options, + void *arg) +{ + /* + * Do we really want this tracepoint? It will be extremely chatty. + * Also, should we have a corresponding trace point for the set path? + */ + PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_START, pn, bits); + + phys_attribute_clear_internal(pn, bits, options, arg); + + PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_END); +} + +/* + * Set specified attribute bits. + * + * Set cached value in the pv head because we have + * no per-mapping hardware support for referenced and + * modify bits. + */ +static void +phys_attribute_set_internal( + ppnum_t pn, + unsigned int bits) +{ + pmap_paddr_t pa = ptoa(pn); + assert(pn != vm_page_fictitious_addr); + + + pa_set_bits(pa, bits); + + return; +} + +static void +phys_attribute_set( + ppnum_t pn, + unsigned int bits) +{ + phys_attribute_set_internal(pn, bits); +} + + +/* + * Check specified attribute bits. + * + * use the software cached bits (since no hw support). + */ +static boolean_t +phys_attribute_test( + ppnum_t pn, + unsigned int bits) +{ + pmap_paddr_t pa = ptoa(pn); + assert(pn != vm_page_fictitious_addr); + return pa_test_bits(pa, bits); +} + + +/* + * Set the modify/reference bits on the specified physical page. + */ +void +pmap_set_modify(ppnum_t pn) +{ + phys_attribute_set(pn, PP_ATTR_MODIFIED); +} + + +/* + * Clear the modify bits on the specified physical page. + */ +void +pmap_clear_modify( + ppnum_t pn) +{ + phys_attribute_clear(pn, PP_ATTR_MODIFIED, 0, NULL); +} + + +/* + * pmap_is_modified: + * + * Return whether or not the specified physical page is modified + * by any physical maps. + */ +boolean_t +pmap_is_modified( + ppnum_t pn) +{ + return phys_attribute_test(pn, PP_ATTR_MODIFIED); +} + + +/* + * Set the reference bit on the specified physical page. + */ +static void +pmap_set_reference( + ppnum_t pn) +{ + phys_attribute_set(pn, PP_ATTR_REFERENCED); +} + +/* + * Clear the reference bits on the specified physical page. + */ +void +pmap_clear_reference( + ppnum_t pn) +{ + phys_attribute_clear(pn, PP_ATTR_REFERENCED, 0, NULL); +} + + +/* + * pmap_is_referenced: + * + * Return whether or not the specified physical page is referenced + * by any physical maps. + */ +boolean_t +pmap_is_referenced( + ppnum_t pn) +{ + return phys_attribute_test(pn, PP_ATTR_REFERENCED); +} + +/* + * pmap_get_refmod(phys) + * returns the referenced and modified bits of the specified + * physical page. + */ +unsigned int +pmap_get_refmod( + ppnum_t pn) +{ + return (((phys_attribute_test(pn, PP_ATTR_MODIFIED)) ? VM_MEM_MODIFIED : 0) + | ((phys_attribute_test(pn, PP_ATTR_REFERENCED)) ? VM_MEM_REFERENCED : 0)); +} + +/* + * pmap_clear_refmod(phys, mask) + * clears the referenced and modified bits as specified by the mask + * of the specified physical page. + */ +void +pmap_clear_refmod_options( + ppnum_t pn, + unsigned int mask, + unsigned int options, + void *arg) +{ + unsigned int bits; + + bits = ((mask & VM_MEM_MODIFIED) ? PP_ATTR_MODIFIED : 0) | + ((mask & VM_MEM_REFERENCED) ? PP_ATTR_REFERENCED : 0); + phys_attribute_clear(pn, bits, options, arg); +} + +void +pmap_clear_refmod( + ppnum_t pn, + unsigned int mask) +{ + pmap_clear_refmod_options(pn, mask, 0, NULL); +} + +unsigned int +pmap_disconnect_options( + ppnum_t pn, + unsigned int options, + void *arg) +{ + if ((options & PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED)) { + /* + * On ARM, the "modified" bit is managed by software, so + * we know up-front if the physical page is "modified", + * without having to scan all the PTEs pointing to it. + * The caller should have made the VM page "busy" so noone + * should be able to establish any new mapping and "modify" + * the page behind us. + */ + if (pmap_is_modified(pn)) { + /* + * The page has been modified and will be sent to + * the VM compressor. + */ + options |= PMAP_OPTIONS_COMPRESSOR; + } else { + /* + * The page hasn't been modified and will be freed + * instead of compressed. + */ + } + } + + /* disconnect the page */ + pmap_page_protect_options(pn, 0, options, arg); + + /* return ref/chg status */ + return (pmap_get_refmod(pn)); +} + +/* + * Routine: + * pmap_disconnect + * + * Function: + * Disconnect all mappings for this page and return reference and change status + * in generic format. + * + */ +unsigned int +pmap_disconnect( + ppnum_t pn) +{ + pmap_page_protect(pn, 0); /* disconnect the page */ + return (pmap_get_refmod(pn)); /* return ref/chg status */ +} + +boolean_t +pmap_has_managed_page(ppnum_t first, ppnum_t last) +{ + if (ptoa(first) >= vm_last_phys) return (FALSE); + if (ptoa(last) < vm_first_phys) return (FALSE); + + return (TRUE); +} + +/* + * The state maintained by the noencrypt functions is used as a + * debugging aid on ARM. This incurs some overhead on the part + * of the caller. A special case check in phys_attribute_clear + * (the most expensive path) currently minimizes this overhead, + * but stubbing these functions out on RELEASE kernels yields + * further wins. + */ +boolean_t +pmap_is_noencrypt( + ppnum_t pn) +{ +#if DEVELOPMENT || DEBUG + boolean_t result = FALSE; + + if (!pa_valid(ptoa(pn))) return FALSE; + + result = (phys_attribute_test(pn, PP_ATTR_NOENCRYPT)); + + return result; +#else +#pragma unused(pn) + return FALSE; +#endif +} + +void +pmap_set_noencrypt( + ppnum_t pn) +{ +#if DEVELOPMENT || DEBUG + if (!pa_valid(ptoa(pn))) return; + + phys_attribute_set(pn, PP_ATTR_NOENCRYPT); +#else +#pragma unused(pn) +#endif +} + +void +pmap_clear_noencrypt( + ppnum_t pn) +{ +#if DEVELOPMENT || DEBUG + if (!pa_valid(ptoa(pn))) return; + + phys_attribute_clear(pn, PP_ATTR_NOENCRYPT, 0, NULL); +#else +#pragma unused(pn) +#endif +} + + +void +pmap_lock_phys_page(ppnum_t pn) +{ + int pai; + pmap_paddr_t phys = ptoa(pn); + + if (pa_valid(phys)) { + pai = (int)pa_index(phys); + LOCK_PVH(pai); + } else + simple_lock(&phys_backup_lock); +} + + +void +pmap_unlock_phys_page(ppnum_t pn) +{ + int pai; + pmap_paddr_t phys = ptoa(pn); + + if (pa_valid(phys)) { + pai = (int)pa_index(phys); + UNLOCK_PVH(pai); + } else + simple_unlock(&phys_backup_lock); +} + +static void +pmap_switch_user_ttb_internal( + pmap_t pmap) +{ +#if (__ARM_VMSA__ == 7) + pmap_cpu_data_t *cpu_data_ptr; + + cpu_data_ptr = pmap_get_cpu_data(); + + if ((cpu_data_ptr->cpu_user_pmap != PMAP_NULL) + && (cpu_data_ptr->cpu_user_pmap != kernel_pmap)) { + unsigned int c; + + c = hw_atomic_sub((volatile uint32_t *)&cpu_data_ptr->cpu_user_pmap->cpu_ref, 1); + if ((c == 0) && (cpu_data_ptr->cpu_user_pmap->prev_tte != 0)) { + /* We saved off the old 1-page tt1 in pmap_expand() in case other cores were still using it. + * Now that the user pmap's cpu_ref is 0, we should be able to safely free it.*/ + tt_entry_t *tt_entry; + + tt_entry = cpu_data_ptr->cpu_user_pmap->prev_tte; + cpu_data_ptr->cpu_user_pmap->prev_tte = (tt_entry_t *) NULL; + pmap_tt1_deallocate(cpu_data_ptr->cpu_user_pmap, tt_entry, ARM_PGBYTES, PMAP_TT_DEALLOCATE_NOBLOCK); + } + } + cpu_data_ptr->cpu_user_pmap = pmap; + cpu_data_ptr->cpu_user_pmap_stamp = pmap->stamp; + (void) hw_atomic_add((volatile uint32_t *)&pmap->cpu_ref, 1); + +#if MACH_ASSERT && __ARM_USER_PROTECT__ + { + unsigned int ttbr0_val, ttbr1_val; + __asm__ volatile("mrc p15,0,%0,c2,c0,0\n" : "=r"(ttbr0_val)); + __asm__ volatile("mrc p15,0,%0,c2,c0,1\n" : "=r"(ttbr1_val)); + if (ttbr0_val != ttbr1_val) { + panic("Misaligned ttbr0 %08X\n", ttbr0_val); + } + } +#endif + if (pmap->tte_index_max == NTTES) { + /* Setting TTBCR.N for TTBR0 TTBR1 boundary at 0x40000000 */ + __asm__ volatile("mcr p15,0,%0,c2,c0,2" : : "r"(2)); + __asm__ volatile("isb"); +#if !__ARM_USER_PROTECT__ + set_mmu_ttb(pmap->ttep); +#endif + } else { +#if !__ARM_USER_PROTECT__ + set_mmu_ttb(pmap->ttep); +#endif + /* Setting TTBCR.N for TTBR0 TTBR1 boundary at 0x80000000 */ + __asm__ volatile("mcr p15,0,%0,c2,c0,2" : : "r"(1)); + __asm__ volatile("isb"); +#if MACH_ASSERT && __ARM_USER_PROTECT__ + if (pmap->ttep & 0x1000) { + panic("Misaligned ttbr0 %08X\n", pmap->ttep); + } +#endif + } + +#if !__ARM_USER_PROTECT__ + set_context_id(pmap->asid); +#endif +#else + + pmap_get_cpu_data()->cpu_user_pmap = pmap; + pmap_get_cpu_data()->cpu_user_pmap_stamp = pmap->stamp; + +#if !__arm64__ + set_context_id(pmap->asid); /* Not required */ +#endif + if (pmap == kernel_pmap) { + set_mmu_ttb(invalid_ttep & TTBR_BADDR_MASK); + } else { + set_mmu_ttb((pmap->ttep & TTBR_BADDR_MASK)|(((uint64_t)pmap->asid) << TTBR_ASID_SHIFT)); + } +#endif +} + +void +pmap_switch_user_ttb( + pmap_t pmap) +{ + pmap_switch_user_ttb_internal(pmap); +} + +/* + * Try to "intuit" whether we need to raise a VM_PROT_WRITE fault + * for the given address when a "swp" instruction raised the fault. + * We have to look at the existing pte for the address to see + * if it needs to get bumped, or just added. If just added, do it + * as a read-only mapping first (this could result in extra faults - + * but better that than extra copy-on-write evaluations). + */ + +#if (__ARM_VMSA__ == 7) +boolean_t +arm_swap_readable_type( + vm_map_address_t addr, + unsigned int spsr) +{ + int ap; + pt_entry_t spte; + pt_entry_t *ptep; + + ptep = pmap_pte(current_pmap(), addr); + if (ptep == PT_ENTRY_NULL) + return (FALSE); + + spte = *ptep; + if (spte == ARM_PTE_TYPE_FAULT || + ARM_PTE_IS_COMPRESSED(spte)) + return (FALSE); + + /* get the access permission bitmaps */ + /* (all subpages should be the same) */ + ap = (spte & ARM_PTE_APMASK); + + if (spsr & 0xf) { /* Supervisor mode */ + panic("arm_swap_readable_type supv"); + return TRUE; + } else { /* User mode */ + if ((ap == ARM_PTE_AP(AP_RWRW)) || (ap == ARM_PTE_AP(AP_RORO))) + return (FALSE); + else + return (TRUE); + } +} +#endif + +/* + * Routine: arm_force_fast_fault + * + * Function: + * Force all mappings for this page to fault according + * to the access modes allowed, so we can gather ref/modify + * bits again. + */ +static boolean_t +arm_force_fast_fault_internal( + ppnum_t ppnum, + vm_prot_t allow_mode, + int options) +{ + pmap_paddr_t phys = ptoa(ppnum); + pv_entry_t *pve_p; + pt_entry_t *pte_p; + int pai; + boolean_t result; + pv_entry_t **pv_h; + boolean_t is_reusable, is_internal; + boolean_t ref_fault; + boolean_t mod_fault; + + assert(ppnum != vm_page_fictitious_addr); + + if (!pa_valid(phys)) { + return FALSE; /* Not a managed page. */ + } + + result = TRUE; + ref_fault = FALSE; + mod_fault = FALSE; + pai = (int)pa_index(phys); + LOCK_PVH(pai); + pv_h = pai_to_pvh(pai); + + pte_p = PT_ENTRY_NULL; + pve_p = PV_ENTRY_NULL; + if (pvh_test_type(pv_h, PVH_TYPE_PTEP)) { + pte_p = pvh_ptep(pv_h); + } else if (pvh_test_type(pv_h, PVH_TYPE_PVEP)) { + pve_p = pvh_list(pv_h); + } + + is_reusable = IS_REUSABLE_PAGE(pai); + is_internal = IS_INTERNAL_PAGE(pai); + + while ((pve_p != PV_ENTRY_NULL) || (pte_p != PT_ENTRY_NULL)) { + vm_map_address_t va; + pt_entry_t spte; + pt_entry_t tmplate; + pmap_t pmap; + boolean_t update_pte; + + if (pve_p != PV_ENTRY_NULL) + pte_p = pve_get_ptep(pve_p); + + if (pte_p == PT_ENTRY_NULL) { + panic("pte_p is NULL: pve_p=%p ppnum=0x%x\n", pve_p, ppnum); + } + if (*pte_p == ARM_PTE_EMPTY) { + panic("pte is NULL: pte_p=%p ppnum=0x%x\n", pte_p, ppnum); + } + if (ARM_PTE_IS_COMPRESSED(*pte_p)) { + panic("pte is COMPRESSED: pte_p=%p ppnum=0x%x\n", pte_p, ppnum); + } + + pmap = ptep_get_pmap(pte_p); + va = ptep_get_va(pte_p); + + assert(va >= pmap->min && va < pmap->max); + + if (pte_is_wired(*pte_p) || pmap == kernel_pmap) { + result = FALSE; + break; + } + + spte = *pte_p; + tmplate = spte; + update_pte = FALSE; + + if ((allow_mode & VM_PROT_READ) != VM_PROT_READ) { + /* read protection sets the pte to fault */ + tmplate = tmplate & ~ARM_PTE_AF; + update_pte = TRUE; + ref_fault = TRUE; + } + if ((allow_mode & VM_PROT_WRITE) != VM_PROT_WRITE) { + /* take away write permission if set */ + if (pmap == kernel_pmap) { + if ((tmplate & ARM_PTE_APMASK) == ARM_PTE_AP(AP_RWNA)) { + tmplate = ((tmplate & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RONA)); + } + } else { + if ((tmplate & ARM_PTE_APMASK) == ARM_PTE_AP(AP_RWRW)) { + tmplate = ((tmplate & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RORO)); + } + } + + pte_set_ffr(tmplate, 1); + update_pte = TRUE; + mod_fault = TRUE; + } + + + if (update_pte) { + if (*pte_p != ARM_PTE_TYPE_FAULT && + !ARM_PTE_IS_COMPRESSED(*pte_p)) { + WRITE_PTE(pte_p, tmplate); + PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE); + } else { + WRITE_PTE(pte_p, tmplate); + __asm__ volatile("isb"); + } + } + + /* update pmap stats and ledgers */ + if (IS_ALTACCT_PAGE(pai, pve_p)) { + /* + * We do not track "reusable" status for + * "alternate accounting" mappings. + */ + } else if ((options & PMAP_OPTIONS_CLEAR_REUSABLE) && + is_reusable && + is_internal && + pmap != kernel_pmap) { + /* one less "reusable" */ + assert(pmap->stats.reusable > 0); + OSAddAtomic(-1, &pmap->stats.reusable); + /* one more "internal" */ + OSAddAtomic(+1, &pmap->stats.internal); + PMAP_STATS_PEAK(pmap->stats.internal); + assert(pmap->stats.internal > 0); + pmap_ledger_credit(pmap, + task_ledgers.internal, + machine_ptob(1)); + assert(!IS_ALTACCT_PAGE(pai, pve_p)); + assert(IS_INTERNAL_PAGE(pai)); + pmap_ledger_credit(pmap, + task_ledgers.phys_footprint, + machine_ptob(1)); + + /* + * Avoid the cost of another trap to handle the fast + * fault when we next write to this page: let's just + * handle that now since we already have all the + * necessary information. + */ + { + arm_clear_fast_fault(ppnum, VM_PROT_WRITE); + } + } else if ((options & PMAP_OPTIONS_SET_REUSABLE) && + !is_reusable && + is_internal && + pmap != kernel_pmap) { + /* one more "reusable" */ + OSAddAtomic(+1, &pmap->stats.reusable); + PMAP_STATS_PEAK(pmap->stats.reusable); + assert(pmap->stats.reusable > 0); + /* one less "internal" */ + assert(pmap->stats.internal > 0); + OSAddAtomic(-1, &pmap->stats.internal); + pmap_ledger_debit(pmap, + task_ledgers.internal, + machine_ptob(1)); + assert(!IS_ALTACCT_PAGE(pai, pve_p)); + assert(IS_INTERNAL_PAGE(pai)); + pmap_ledger_debit(pmap, + task_ledgers.phys_footprint, + machine_ptob(1)); + } + + pte_p = PT_ENTRY_NULL; + if (pve_p != PV_ENTRY_NULL) + pve_p = PVE_NEXT_PTR(pve_next(pve_p)); + } + + /* update global "reusable" status for this page */ + if (is_internal) { + if ((options & PMAP_OPTIONS_CLEAR_REUSABLE) && + is_reusable) { + CLR_REUSABLE_PAGE(pai); + } else if ((options & PMAP_OPTIONS_SET_REUSABLE) && + !is_reusable) { + SET_REUSABLE_PAGE(pai); + } + } + + if (mod_fault) { + SET_MODFAULT_PAGE(pai); + } + if (ref_fault) { + SET_REFFAULT_PAGE(pai); + } + + UNLOCK_PVH(pai); + return result; +} + +boolean_t +arm_force_fast_fault( + ppnum_t ppnum, + vm_prot_t allow_mode, + int options, + __unused void *arg) +{ + pmap_paddr_t phys = ptoa(ppnum); + + assert(ppnum != vm_page_fictitious_addr); + + if (!pa_valid(phys)) { + return FALSE; /* Not a managed page. */ + } + + return arm_force_fast_fault_internal(ppnum, allow_mode, options); +} + +/* + * Routine: arm_clear_fast_fault + * + * Function: + * Clear pending force fault for all mappings for this page based on + * the observed fault type, update ref/modify bits. + */ +boolean_t +arm_clear_fast_fault( + ppnum_t ppnum, + vm_prot_t fault_type) +{ + pmap_paddr_t pa = ptoa(ppnum); + pv_entry_t *pve_p; + pt_entry_t *pte_p; + int pai; + boolean_t result; + pv_entry_t **pv_h; + + assert(ppnum != vm_page_fictitious_addr); + + if (!pa_valid(pa)) { + return FALSE; /* Not a managed page. */ + } + + result = FALSE; + pai = (int)pa_index(pa); + ASSERT_PVH_LOCKED(pai); + pv_h = pai_to_pvh(pai); + + pte_p = PT_ENTRY_NULL; + pve_p = PV_ENTRY_NULL; + if (pvh_test_type(pv_h, PVH_TYPE_PTEP)) { + pte_p = pvh_ptep(pv_h); + } else if (pvh_test_type(pv_h, PVH_TYPE_PVEP)) { + pve_p = pvh_list(pv_h); + } + + while ((pve_p != PV_ENTRY_NULL) || (pte_p != PT_ENTRY_NULL)) { + vm_map_address_t va; + pt_entry_t spte; + pt_entry_t tmplate; + pmap_t pmap; + + if (pve_p != PV_ENTRY_NULL) + pte_p = pve_get_ptep(pve_p); + + if (pte_p == PT_ENTRY_NULL) { + panic("pte_p is NULL: pve_p=%p ppnum=0x%x\n", pve_p, ppnum); + } + if (*pte_p == ARM_PTE_EMPTY) { + panic("pte is NULL: pte_p=%p ppnum=0x%x\n", pte_p, ppnum); + } + + pmap = ptep_get_pmap(pte_p); + va = ptep_get_va(pte_p); + + assert(va >= pmap->min && va < pmap->max); + + spte = *pte_p; + tmplate = spte; + + if ((fault_type & VM_PROT_WRITE) && (pte_is_ffr(spte))) { + { + if (pmap == kernel_pmap) + tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RWNA)); + else + tmplate = ((spte & ~ARM_PTE_APMASK) | ARM_PTE_AP(AP_RWRW)); + } + + tmplate |= ARM_PTE_AF; + + pte_set_ffr(tmplate, 0); + pa_set_bits(pa, PP_ATTR_REFERENCED | PP_ATTR_MODIFIED); + + } else if ((fault_type & VM_PROT_READ) && ((spte & ARM_PTE_AF) != ARM_PTE_AF)) { + tmplate = spte | ARM_PTE_AF; + + { + pa_set_bits(pa, PP_ATTR_REFERENCED); + } + } + + + if (spte != tmplate) { + if (spte != ARM_PTE_TYPE_FAULT) { + WRITE_PTE(pte_p, tmplate); + PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE); + } else { + WRITE_PTE(pte_p, tmplate); + __asm__ volatile("isb"); + } + result = TRUE; + } + + pte_p = PT_ENTRY_NULL; + if (pve_p != PV_ENTRY_NULL) + pve_p = PVE_NEXT_PTR(pve_next(pve_p)); + } + return result; +} + +/* + * Determine if the fault was induced by software tracking of + * modify/reference bits. If so, re-enable the mapping (and set + * the appropriate bits). + * + * Returns KERN_SUCCESS if the fault was induced and was + * successfully handled. + * + * Returns KERN_FAILURE if the fault was not induced and + * the function was unable to deal with it. + * + * Returns KERN_PROTECTION_FAILURE if the pmap layer explictly + * disallows this type of access. + */ +static kern_return_t +arm_fast_fault_internal( + pmap_t pmap, + vm_map_address_t va, + vm_prot_t fault_type, + __unused boolean_t from_user) +{ + kern_return_t result = KERN_FAILURE; + pt_entry_t *ptep; + pt_entry_t spte = ARM_PTE_TYPE_FAULT; + int pai; + pmap_paddr_t pa; + + PMAP_LOCK(pmap); + + /* + * If the entry doesn't exist, is completely invalid, or is already + * valid, we can't fix it here. + */ + + ptep = pmap_pte(pmap, va); + if (ptep != PT_ENTRY_NULL) { + spte = *ptep; + + pa = pte_to_pa(spte); + + if ((spte == ARM_PTE_TYPE_FAULT) || + ARM_PTE_IS_COMPRESSED(spte) || + (!pa_valid(pa))) { + PMAP_UNLOCK(pmap); + return result; + } + + pai = (int)pa_index(pa); + LOCK_PVH(pai); + } else { + PMAP_UNLOCK(pmap); + return result; + } + + + if ((IS_REFFAULT_PAGE(pai)) || + ((fault_type & VM_PROT_WRITE) && IS_MODFAULT_PAGE(pai))) { + /* + * An attempted access will always clear ref/mod fault state, as + * appropriate for the fault type. arm_clear_fast_fault will + * update the associated PTEs for the page as appropriate; if + * any PTEs are updated, we redrive the access. If the mapping + * does not actually allow for the attempted access, the + * following fault will (hopefully) fail to update any PTEs, and + * thus cause arm_fast_fault to decide that it failed to handle + * the fault. + */ + if (IS_REFFAULT_PAGE(pai)) { + CLR_REFFAULT_PAGE(pai); + } + if ( (fault_type & VM_PROT_WRITE) && IS_MODFAULT_PAGE(pai)) { + CLR_MODFAULT_PAGE(pai); + } + + if (arm_clear_fast_fault((ppnum_t)atop(pa),fault_type)) { + /* + * Should this preserve KERN_PROTECTION_FAILURE? The + * cost of not doing so is a another fault in a case + * that should already result in an exception. + */ + result = KERN_SUCCESS; + } + } + + UNLOCK_PVH(pai); + PMAP_UNLOCK(pmap); + return result; +} + +kern_return_t +arm_fast_fault( + pmap_t pmap, + vm_map_address_t va, + vm_prot_t fault_type, + __unused boolean_t from_user) +{ + kern_return_t result = KERN_FAILURE; + + if (va < pmap->min || va >= pmap->max) + return result; + + PMAP_TRACE(PMAP_CODE(PMAP__FAST_FAULT) | DBG_FUNC_START, + VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(va), fault_type, + from_user); + +#if (__ARM_VMSA__ == 7) + if (pmap != kernel_pmap) { + pmap_cpu_data_t *cpu_data_ptr = pmap_get_cpu_data(); + pmap_t cur_pmap; + pmap_t cur_user_pmap; + + cur_pmap = current_pmap(); + cur_user_pmap = cpu_data_ptr->cpu_user_pmap; + + if ((cur_user_pmap == cur_pmap) && (cur_pmap == pmap)) { + if (cpu_data_ptr->cpu_user_pmap_stamp != pmap->stamp) { + pmap_set_pmap(pmap, current_thread()); + result = KERN_SUCCESS; + goto done; + } + } + } +#endif + + result = arm_fast_fault_internal(pmap, va, fault_type, from_user); + +#if (__ARM_VMSA__ == 7) +done: +#endif + + PMAP_TRACE(PMAP_CODE(PMAP__FAST_FAULT) | DBG_FUNC_END, result); + + return result; +} + +void +pmap_copy_page( + ppnum_t psrc, + ppnum_t pdst) +{ + bcopy_phys((addr64_t) (ptoa(psrc)), + (addr64_t) (ptoa(pdst)), + PAGE_SIZE); +} + + +/* + * pmap_copy_page copies the specified (machine independent) pages. + */ +void +pmap_copy_part_page( + ppnum_t psrc, + vm_offset_t src_offset, + ppnum_t pdst, + vm_offset_t dst_offset, + vm_size_t len) +{ + bcopy_phys((addr64_t) (ptoa(psrc) + src_offset), + (addr64_t) (ptoa(pdst) + dst_offset), + len); +} + + +/* + * pmap_zero_page zeros the specified (machine independent) page. + */ +void +pmap_zero_page( + ppnum_t pn) +{ + assert(pn != vm_page_fictitious_addr); + bzero_phys((addr64_t) ptoa(pn), PAGE_SIZE); +} + +/* + * pmap_zero_part_page + * zeros the specified (machine independent) part of a page. + */ +void +pmap_zero_part_page( + ppnum_t pn, + vm_offset_t offset, + vm_size_t len) +{ + assert(pn != vm_page_fictitious_addr); + assert(offset + len <= PAGE_SIZE); + bzero_phys((addr64_t) (ptoa(pn) + offset), len); +} + + +/* + * nop in current arm implementation + */ +void +inval_copy_windows( + __unused thread_t t) +{ +} + +void +pmap_map_globals( + void) +{ + pt_entry_t *ptep, pte; + + ptep = pmap_pte(kernel_pmap, LOWGLOBAL_ALIAS); + assert(ptep != PT_ENTRY_NULL); + assert(*ptep == ARM_PTE_EMPTY); + + pte = pa_to_pte(ml_static_vtop((vm_offset_t)&lowGlo)) | AP_RONA | ARM_PTE_NX | ARM_PTE_PNX | ARM_PTE_AF | ARM_PTE_TYPE; + pte |= ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK); +#if (__ARM_VMSA__ > 7) + pte |= ARM_PTE_SH(SH_OUTER_MEMORY); +#else + pte |= ARM_PTE_SH; +#endif + *ptep = pte; + FLUSH_PTE_RANGE(ptep,(ptep+1)); + PMAP_UPDATE_TLBS(kernel_pmap, LOWGLOBAL_ALIAS, LOWGLOBAL_ALIAS + PAGE_SIZE); +} + +vm_offset_t +pmap_cpu_windows_copy_addr(int cpu_num, unsigned int index) +{ + return (vm_offset_t)(CPUWINDOWS_BASE + (PAGE_SIZE * ((CPUWINDOWS_MAX * cpu_num) + index))); +} + +static unsigned int +pmap_map_cpu_windows_copy_internal( + ppnum_t pn, + vm_prot_t prot, + unsigned int wimg_bits) +{ + pt_entry_t *ptep = NULL, pte; + unsigned int cpu_num; + unsigned int i; + vm_offset_t cpu_copywindow_vaddr = 0; + + cpu_num = pmap_get_cpu_data()->cpu_number; + + for (i = 0; i<CPUWINDOWS_MAX; i++) { + cpu_copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_num, i); + ptep = pmap_pte(kernel_pmap, cpu_copywindow_vaddr); + assert(!ARM_PTE_IS_COMPRESSED(*ptep)); + if (*ptep == ARM_PTE_TYPE_FAULT) + break; + } + if (i == CPUWINDOWS_MAX) { + panic("pmap_map_cpu_windows_copy: out of window\n"); + } + + pte = pa_to_pte(ptoa(pn)) | ARM_PTE_TYPE | ARM_PTE_AF | ARM_PTE_NX | ARM_PTE_PNX; + + pte |= wimg_to_pte(wimg_bits); + + if (prot & VM_PROT_WRITE) { + pte |= ARM_PTE_AP(AP_RWNA); + } else { + pte |= ARM_PTE_AP(AP_RONA); + } + + WRITE_PTE(ptep, pte); + /* + * Invalidate tlb. Cover nested cpu_copywindow_vaddr usage with the interrupted context + * in pmap_unmap_cpu_windows_copy() after clearing the pte and before tlb invalidate. + */ + PMAP_UPDATE_TLBS(kernel_pmap, cpu_copywindow_vaddr, cpu_copywindow_vaddr + PAGE_SIZE); + + return(i); +} + +unsigned int +pmap_map_cpu_windows_copy( + ppnum_t pn, + vm_prot_t prot, + unsigned int wimg_bits) +{ + return pmap_map_cpu_windows_copy_internal(pn, prot, wimg_bits); +} + +static void +pmap_unmap_cpu_windows_copy_internal( + unsigned int index) +{ + pt_entry_t *ptep; + unsigned int cpu_num; + vm_offset_t cpu_copywindow_vaddr = 0; + + cpu_num = pmap_get_cpu_data()->cpu_number; + + cpu_copywindow_vaddr = pmap_cpu_windows_copy_addr(cpu_num, index); + __asm__ volatile("dsb sy"); + ptep = pmap_pte(kernel_pmap, cpu_copywindow_vaddr); + WRITE_PTE(ptep, ARM_PTE_TYPE_FAULT); + PMAP_UPDATE_TLBS(kernel_pmap, cpu_copywindow_vaddr, cpu_copywindow_vaddr + PAGE_SIZE); +} + +void +pmap_unmap_cpu_windows_copy( + unsigned int index) +{ + return pmap_unmap_cpu_windows_copy_internal(index); +} + +/* + * Marked a pmap has nested + */ +static void +pmap_set_nested_internal( + pmap_t pmap) +{ + pmap->nested = TRUE; +} + +void +pmap_set_nested( + pmap_t pmap) +{ + pmap_set_nested_internal(pmap); +} + +/* + * kern_return_t pmap_nest(grand, subord, vstart, size) + * + * grand = the pmap that we will nest subord into + * subord = the pmap that goes into the grand + * vstart = start of range in pmap to be inserted + * nstart = start of range in pmap nested pmap + * size = Size of nest area (up to 16TB) + * + * Inserts a pmap into another. This is used to implement shared segments. + * + */ + +static kern_return_t +pmap_nest_internal( + pmap_t grand, + pmap_t subord, + addr64_t vstart, + addr64_t nstart, + uint64_t size) +{ + kern_return_t kr = KERN_FAILURE; + vm_map_offset_t vaddr, nvaddr; + tt_entry_t *stte_p; + tt_entry_t *gtte_p; + unsigned int i; + unsigned int num_tte; + unsigned int nested_region_asid_bitmap_size; + unsigned int* nested_region_asid_bitmap; + int expand_options = 0; + + +#if (__ARM_VMSA__ == 7) + if (((size|vstart|nstart) & ARM_TT_L1_PT_OFFMASK) != 0x0ULL) { + return KERN_INVALID_VALUE; /* Nest 4MB region */ + } +#else + if (((size|vstart|nstart) & (ARM_TT_L2_OFFMASK)) != 0x0ULL) { + panic("pmap_nest() pmap %p has a nested pmap 0x%llx, 0x%llx, 0x%llx\n", grand, vstart, nstart, size); + } +#endif + + if ((grand->nested_pmap != PMAP_NULL) && (grand->nested_pmap != subord)) { + panic("pmap_nest() pmap %p has a nested pmap\n", grand); + } + + if (subord->nested_region_asid_bitmap == NULL) { + nested_region_asid_bitmap_size = (unsigned int)(size>>ARM_TT_TWIG_SHIFT)/(sizeof(unsigned int)*NBBY); + + nested_region_asid_bitmap = kalloc(nested_region_asid_bitmap_size*sizeof(unsigned int)); + bzero(nested_region_asid_bitmap, nested_region_asid_bitmap_size*sizeof(unsigned int)); + + PMAP_LOCK(subord); + if (subord->nested_region_asid_bitmap == NULL) { + subord->nested_region_asid_bitmap = nested_region_asid_bitmap; + subord->nested_region_asid_bitmap_size = nested_region_asid_bitmap_size; + subord->nested_region_subord_addr = nstart; + subord->nested_region_size = (mach_vm_offset_t) size; + nested_region_asid_bitmap = NULL; + } + PMAP_UNLOCK(subord); + if (nested_region_asid_bitmap != NULL) { + kfree(nested_region_asid_bitmap, nested_region_asid_bitmap_size*sizeof(unsigned int)); + } + } + if ((subord->nested_region_subord_addr + subord->nested_region_size) < (nstart+size)) { + uint64_t new_size; + unsigned int new_nested_region_asid_bitmap_size; + unsigned int* new_nested_region_asid_bitmap; + + nested_region_asid_bitmap = NULL; + nested_region_asid_bitmap_size = 0; + new_size = nstart + size - subord->nested_region_subord_addr; + + /* We explicitly add 1 to the bitmap allocation size in order to avoid issues with truncation. */ + new_nested_region_asid_bitmap_size = (unsigned int)((new_size>>ARM_TT_TWIG_SHIFT)/(sizeof(unsigned int)*NBBY)) + 1; + + new_nested_region_asid_bitmap = kalloc(new_nested_region_asid_bitmap_size*sizeof(unsigned int)); + PMAP_LOCK(subord); + if (subord->nested_region_size < new_size) { + bzero(new_nested_region_asid_bitmap, new_nested_region_asid_bitmap_size*sizeof(unsigned int)); + bcopy(subord->nested_region_asid_bitmap, new_nested_region_asid_bitmap, subord->nested_region_asid_bitmap_size); + nested_region_asid_bitmap_size = subord->nested_region_asid_bitmap_size; + nested_region_asid_bitmap = subord->nested_region_asid_bitmap; + subord->nested_region_asid_bitmap = new_nested_region_asid_bitmap; + subord->nested_region_asid_bitmap_size = new_nested_region_asid_bitmap_size; + subord->nested_region_size = new_size; + new_nested_region_asid_bitmap = NULL; + } + PMAP_UNLOCK(subord); + if (nested_region_asid_bitmap != NULL) + kfree(nested_region_asid_bitmap, nested_region_asid_bitmap_size*sizeof(unsigned int)); + if (new_nested_region_asid_bitmap != NULL) + kfree(new_nested_region_asid_bitmap, new_nested_region_asid_bitmap_size*sizeof(unsigned int)); + } + + PMAP_LOCK(subord); + if (grand->nested_pmap == PMAP_NULL) { + grand->nested_pmap = subord; + grand->nested_region_grand_addr = vstart; + grand->nested_region_subord_addr = nstart; + grand->nested_region_size = (mach_vm_offset_t) size; + } else { + if ((grand->nested_region_grand_addr > vstart)) { + panic("pmap_nest() pmap %p : attempt to nest outside the nested region\n", grand); + } + else if ((grand->nested_region_grand_addr + grand->nested_region_size) < (vstart+size)) { + grand->nested_region_size = (mach_vm_offset_t)(vstart - grand->nested_region_grand_addr + size); + } + } + +#if (__ARM_VMSA__ == 7) + nvaddr = (vm_map_offset_t) nstart; + vaddr = (vm_map_offset_t) vstart; + num_tte = size >> ARM_TT_L1_SHIFT; + + for (i = 0; i < num_tte; i++) { + stte_p = pmap_tte(subord, nvaddr); + if ((stte_p == (tt_entry_t *)NULL) || (((*stte_p) & ARM_TTE_TYPE_MASK) != ARM_TTE_TYPE_TABLE)) { + PMAP_UNLOCK(subord); + kr = pmap_expand(subord, nvaddr, expand_options, PMAP_TT_L2_LEVEL); + + if (kr != KERN_SUCCESS) { + PMAP_LOCK(grand); + goto done; + } + + PMAP_LOCK(subord); + } + PMAP_UNLOCK(subord); + PMAP_LOCK(grand); + stte_p = pmap_tte(grand, vaddr); + if (stte_p == (tt_entry_t *)NULL) { + PMAP_UNLOCK(grand); + kr = pmap_expand(grand, vaddr, expand_options, PMAP_TT_L1_LEVEL); + + if (kr != KERN_SUCCESS) { + PMAP_LOCK(grand); + goto done; + } + } else { + PMAP_UNLOCK(grand); + kr = KERN_SUCCESS; + } + PMAP_LOCK(subord); + + + nvaddr += ARM_TT_L1_SIZE; + vaddr += ARM_TT_L1_SIZE; + } + +#else + nvaddr = (vm_map_offset_t) nstart; + num_tte = (unsigned int)(size >> ARM_TT_L2_SHIFT); + + for (i = 0; i < num_tte; i++) { + stte_p = pmap_tt2e(subord, nvaddr); + if (stte_p == PT_ENTRY_NULL || *stte_p == ARM_TTE_EMPTY) { + PMAP_UNLOCK(subord); + kr = pmap_expand(subord, nvaddr, expand_options, PMAP_TT_L3_LEVEL); + + if (kr != KERN_SUCCESS) { + PMAP_LOCK(grand); + goto done; + } + + PMAP_LOCK(subord); + } + nvaddr += ARM_TT_L2_SIZE; + } +#endif + PMAP_UNLOCK(subord); + + /* + * copy tte's from subord pmap into grand pmap + */ + + PMAP_LOCK(grand); + nvaddr = (vm_map_offset_t) nstart; + vaddr = (vm_map_offset_t) vstart; + + +#if (__ARM_VMSA__ == 7) + for (i = 0; i < num_tte; i++) { + + stte_p = pmap_tte(subord, nvaddr); + gtte_p = pmap_tte(grand, vaddr); + *gtte_p = *stte_p; + + nvaddr += ARM_TT_L1_SIZE; + vaddr += ARM_TT_L1_SIZE; + } +#else + for (i = 0; i < num_tte; i++) { + + stte_p = pmap_tt2e(subord, nstart); + gtte_p = pmap_tt2e(grand, vaddr); + if (gtte_p == PT_ENTRY_NULL) { + PMAP_UNLOCK(grand); + kr = pmap_expand(grand, vaddr, expand_options, PMAP_TT_L2_LEVEL); + PMAP_LOCK(grand); + + if (kr != KERN_SUCCESS) { + goto done; + } + + gtte_p = pmap_tt2e(grand, vaddr); + } + *gtte_p = *stte_p; + vaddr += ARM_TT_L2_SIZE; + nstart += ARM_TT_L2_SIZE; + } +#endif + + kr = KERN_SUCCESS; +done: + +#ifndef __ARM_L1_PTW__ + CleanPoU_DcacheRegion((vm_offset_t) pmap_tte(grand, vstart), num_tte * sizeof(tt_entry_t)); +#endif + +#if (__ARM_VMSA__ > 7) + /* + * check for overflow on LP64 arch + */ + assert((size & 0xFFFFFFFF00000000ULL) == 0); +#endif + PMAP_UPDATE_TLBS(grand, vstart, vstart + size); + + PMAP_UNLOCK(grand); + return kr; +} + +kern_return_t pmap_nest( + pmap_t grand, + pmap_t subord, + addr64_t vstart, + addr64_t nstart, + uint64_t size) +{ + kern_return_t kr = KERN_FAILURE; + + PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_START, + VM_KERNEL_ADDRHIDE(grand), VM_KERNEL_ADDRHIDE(subord), + VM_KERNEL_ADDRHIDE(vstart)); + + kr = pmap_nest_internal(grand, subord, vstart, nstart, size); + + PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_END, kr); + + return kr; +} + +/* + * kern_return_t pmap_unnest(grand, vaddr) + * + * grand = the pmap that we will nest subord into + * vaddr = start of range in pmap to be unnested + * size = size of range in pmap to be unnested + * + */ + +kern_return_t +pmap_unnest( + pmap_t grand, + addr64_t vaddr, + uint64_t size) +{ + return(pmap_unnest_options(grand, vaddr, size, 0)); +} + +static kern_return_t +pmap_unnest_options_internal( + pmap_t grand, + addr64_t vaddr, + uint64_t size, + unsigned int option) +{ + vm_map_offset_t start; + vm_map_offset_t addr; + tt_entry_t *tte_p; + unsigned int current_index; + unsigned int start_index; + unsigned int max_index; + unsigned int num_tte; + unsigned int i; + +#if (__ARM_VMSA__ == 7) + if (((size|vaddr) & ARM_TT_L1_PT_OFFMASK) != 0x0ULL) { + panic("pmap_unnest(): unaligned request\n"); + } +#else + if (((size|vaddr) & ARM_TT_L2_OFFMASK) != 0x0ULL) { + panic("pmap_unnest(): unaligned request\n"); + } +#endif + + if ((option & PMAP_UNNEST_CLEAN) == 0) + { + PMAP_LOCK(grand->nested_pmap); + + start = vaddr - grand->nested_region_grand_addr + grand->nested_region_subord_addr ; + start_index = (unsigned int)((vaddr - grand->nested_region_grand_addr) >> ARM_TT_TWIG_SHIFT); + max_index = (unsigned int)(start_index + (size >> ARM_TT_TWIG_SHIFT)); + num_tte = (unsigned int)(size >> ARM_TT_TWIG_SHIFT); + + if (size > grand->nested_region_size) { + panic("pmap_unnest() pmap %p %llu, %llu\n", grand, size, (uint64_t)grand->nested_region_size); + } + + for (current_index = start_index, addr = start; current_index < max_index; current_index++) { + pt_entry_t *bpte, *epte, *cpte; + + + if(!testbit(current_index, (int *)grand->nested_pmap->nested_region_asid_bitmap)) { + + setbit(current_index, (int *)grand->nested_pmap->nested_region_asid_bitmap); + bpte = pmap_pte(grand->nested_pmap, addr); + epte = bpte + (ARM_TT_LEAF_INDEX_MASK>>ARM_TT_LEAF_SHIFT); + + for (cpte = bpte; cpte <= epte; cpte++) { + pmap_paddr_t pa; + int pai=0; + boolean_t managed=FALSE; + pt_entry_t spte; + + if ((*cpte != ARM_PTE_TYPE_FAULT) + && (!ARM_PTE_IS_COMPRESSED(*cpte))) { + + spte = *cpte; + while (!managed) { + pa = pte_to_pa(spte); + if (!pa_valid(pa)) + break; + pai = (int)pa_index(pa); + LOCK_PVH(pai); + spte = *cpte; + pa = pte_to_pa(spte); + if (pai == (int)pa_index(pa)) { + managed =TRUE; + break; // Leave the PVH locked as we'll unlock it after we update the PTE + } + UNLOCK_PVH(pai); + } + + if (((spte & ARM_PTE_NG) != ARM_PTE_NG)) { + + WRITE_PTE(cpte, (spte | ARM_PTE_NG)); + } + + if (managed) + { + ASSERT_PVH_LOCKED(pai); + UNLOCK_PVH(pai); + } + } + } + } + + addr += ARM_TT_TWIG_SIZE; + +#ifndef __ARM_L1_PTW__ + CleanPoU_DcacheRegion((vm_offset_t) pmap_pte(grand->nested_pmap, start), num_tte * sizeof(tt_entry_t)); +#endif + PMAP_UPDATE_TLBS(grand->nested_pmap, start, start + size); + } + + PMAP_UNLOCK(grand->nested_pmap); + } + + PMAP_LOCK(grand); + + /* + * invalidate all pdes for segment at vaddr in pmap grand + */ + start = vaddr; + addr = vaddr; + + num_tte = (unsigned int)(size >> ARM_TT_TWIG_SHIFT); + + for (i = 0; i < num_tte; i++) { + tte_p = pmap_tte(grand, addr); + *tte_p = ARM_TTE_TYPE_FAULT; + + addr += ARM_TT_TWIG_SIZE; + } + +#ifndef __ARM_L1_PTW__ + CleanPoU_DcacheRegion((vm_offset_t) pmap_tte(grand, start), num_tte * sizeof(tt_entry_t)); +#endif + PMAP_UPDATE_TLBS(grand, start, start + size); + + PMAP_UNLOCK(grand); + + return KERN_SUCCESS; +} + +kern_return_t +pmap_unnest_options( + pmap_t grand, + addr64_t vaddr, + uint64_t size, + unsigned int option) +{ + kern_return_t kr = KERN_FAILURE; + + PMAP_TRACE(PMAP_CODE(PMAP__UNNEST) | DBG_FUNC_START, + VM_KERNEL_ADDRHIDE(grand), VM_KERNEL_ADDRHIDE(vaddr)); + + kr = pmap_unnest_options_internal(grand, vaddr, size, option); + + PMAP_TRACE(PMAP_CODE(PMAP__UNNEST) | DBG_FUNC_END, kr); + + return kr; +} + +boolean_t +pmap_adjust_unnest_parameters( + __unused pmap_t p, + __unused vm_map_offset_t *s, + __unused vm_map_offset_t *e) +{ + return TRUE; /* to get to log_unnest_badness()... */ +} + +/* + * disable no-execute capability on + * the specified pmap + */ +#if DEVELOPMENT || DEBUG +void +pmap_disable_NX( + pmap_t pmap) +{ + pmap->nx_enabled = FALSE; +} +#else +void +pmap_disable_NX( + __unused pmap_t pmap) +{ +} +#endif + +void +pt_fake_zone_init( + int zone_index) +{ + pt_fake_zone_index = zone_index; +} + +void +pt_fake_zone_info( + int *count, + vm_size_t *cur_size, vm_size_t *max_size, vm_size_t *elem_size, vm_size_t *alloc_size, + uint64_t *sum_size, int *collectable, int *exhaustable, int *caller_acct) +{ + *count = inuse_pmap_pages_count; + *cur_size = PAGE_SIZE * (inuse_pmap_pages_count); + *max_size = PAGE_SIZE * (inuse_pmap_pages_count + vm_page_inactive_count + vm_page_active_count + vm_page_free_count); + *elem_size = PAGE_SIZE; + *alloc_size = PAGE_SIZE; + *sum_size = (alloc_pmap_pages_count) * PAGE_SIZE; + + *collectable = 1; + *exhaustable = 0; + *caller_acct = 1; +} + +/* + * flush a range of hardware TLB entries. + * NOTE: assumes the smallest TLB entry in use will be for + * an ARM small page (4K). + */ + +#define ARM_FULL_TLB_FLUSH_THRESHOLD 64 +#define ARM64_FULL_TLB_FLUSH_THRESHOLD 256 + +static void +flush_mmu_tlb_region_asid( + vm_offset_t va, + unsigned length, + pmap_t pmap) +{ +#if (__ARM_VMSA__ == 7) + vm_offset_t end = va + length; + uint32_t asid; + + asid = pmap->asid; + + if (length / ARM_SMALL_PAGE_SIZE > ARM_FULL_TLB_FLUSH_THRESHOLD) { + boolean_t flush_all = FALSE; + + if ((asid == 0) || (pmap->nested == TRUE)) + flush_all = TRUE; + if (flush_all) + flush_mmu_tlb(); + else + flush_mmu_tlb_asid(asid); + + return; + } + if (pmap->nested == TRUE) { +#if !__ARM_MP_EXT__ + flush_mmu_tlb(); +#else + va = arm_trunc_page(va); + while (va < end) { + flush_mmu_tlb_mva_entries(va); + va += ARM_SMALL_PAGE_SIZE; + } +#endif + return; + } + va = arm_trunc_page(va) | (asid & 0xff); + flush_mmu_tlb_entries(va, end); + +#else + vm_offset_t end = va + length; + uint32_t asid; + + asid = pmap->asid; + + if ((length >> ARM_TT_L3_SHIFT) > ARM64_FULL_TLB_FLUSH_THRESHOLD) { + boolean_t flush_all = FALSE; + + if ((asid == 0) || (pmap->nested == TRUE)) + flush_all = TRUE; + if (flush_all) + flush_mmu_tlb(); + else + flush_mmu_tlb_asid((uint64_t)asid << TLBI_ASID_SHIFT); + return; + } + va = tlbi_asid(asid) | tlbi_addr(va); + end = tlbi_asid(asid) | tlbi_addr(end); + if (pmap->nested == TRUE) { + flush_mmu_tlb_allentries(va, end); + } else { + flush_mmu_tlb_entries(va, end); + } + +#endif +} + +void +flush_mmu_tlb_region( + vm_offset_t va, + unsigned length) +{ + flush_mmu_tlb_region_asid(va, length, kernel_pmap); +} + +unsigned int +pmap_cache_attributes( + ppnum_t pn) +{ + pmap_paddr_t paddr; + int pai; + unsigned int result; + pp_attr_t pp_attr_current; + + paddr = ptoa(pn); + + if ((paddr >= io_rgn_start) && (paddr < io_rgn_end)) { + unsigned int attr = IO_ATTR_WIMG(io_attr_table[(paddr - io_rgn_start) / io_rgn_granule]); + if (attr) + return attr; + else + return (VM_WIMG_IO); + } + + + if (!pmap_initialized) { + if ((paddr >= gPhysBase) && (paddr < gPhysBase+gPhysSize)) + return (VM_WIMG_DEFAULT); + else + return (VM_WIMG_IO); + } + + + if (!pa_valid(paddr)) + return (VM_WIMG_IO); + + result = VM_WIMG_DEFAULT; + + pai = (int)pa_index(paddr); + + pp_attr_current = pp_attr_table[pai]; + if (pp_attr_current & PP_ATTR_WIMG_MASK) + result = pp_attr_current & PP_ATTR_WIMG_MASK; + return result; +} + +static boolean_t +pmap_batch_set_cache_attributes_internal( + ppnum_t pn, + unsigned int cacheattr, + unsigned int page_cnt, + unsigned int page_index, + boolean_t doit, + unsigned int *res) +{ + pmap_paddr_t paddr; + int pai; + pp_attr_t pp_attr_current; + pp_attr_t pp_attr_template; + unsigned int wimg_bits_prev, wimg_bits_new; + + if (cacheattr & VM_WIMG_USE_DEFAULT) + cacheattr = VM_WIMG_DEFAULT; + + if ((doit == FALSE) && (*res == 0)) { + *res = page_cnt; + if (platform_cache_batch_wimg(cacheattr & (VM_WIMG_MASK), page_cnt<<PAGE_SHIFT) == FALSE) { + return FALSE; + } + } + + paddr = ptoa(pn); + + if (!pa_valid(paddr)) { + panic("pmap_batch_set_cache_attributes(): pn 0x%08x not managed\n", pn); + } + + pai = (int)pa_index(paddr); + + if (doit) + LOCK_PVH(pai); + + pp_attr_current = pp_attr_table[pai]; + wimg_bits_prev = VM_WIMG_DEFAULT; + if (pp_attr_current & PP_ATTR_WIMG_MASK) + wimg_bits_prev = pp_attr_current & PP_ATTR_WIMG_MASK; + + pp_attr_template = (pp_attr_current & ~PP_ATTR_WIMG_MASK) | PP_ATTR_WIMG(cacheattr & (VM_WIMG_MASK)); + + if (doit) + pp_attr_table[pai] = pp_attr_template; + + wimg_bits_new = VM_WIMG_DEFAULT; + if (pp_attr_template & PP_ATTR_WIMG_MASK) + wimg_bits_new = pp_attr_template & PP_ATTR_WIMG_MASK; + + if (doit) { + if (wimg_bits_new != wimg_bits_prev) + pmap_update_cache_attributes_locked(pn, cacheattr); + UNLOCK_PVH(pai); + } else { + if (wimg_bits_new == VM_WIMG_COPYBACK) { + return FALSE; + } + if (wimg_bits_prev == wimg_bits_new) { + *res = *res-1; + if (!platform_cache_batch_wimg(wimg_bits_new, (*res)<<PAGE_SHIFT)) { + return FALSE; + } + } + return TRUE; + } + + if (page_cnt == (page_index+1)) { + wimg_bits_prev = VM_WIMG_COPYBACK; + if (((page_cnt == (page_index+1)) && (wimg_bits_prev != wimg_bits_new)) + && ((wimg_bits_prev == VM_WIMG_COPYBACK) + || ((wimg_bits_prev == VM_WIMG_INNERWBACK) + && (wimg_bits_new != VM_WIMG_COPYBACK)) + || ((wimg_bits_prev == VM_WIMG_WTHRU) + && ((wimg_bits_new != VM_WIMG_COPYBACK) || (wimg_bits_new != VM_WIMG_INNERWBACK))))) { + platform_cache_flush_wimg(wimg_bits_new); + } + } + + return TRUE; +}; + +boolean_t +pmap_batch_set_cache_attributes( + ppnum_t pn, + unsigned int cacheattr, + unsigned int page_cnt, + unsigned int page_index, + boolean_t doit, + unsigned int *res) +{ + return pmap_batch_set_cache_attributes_internal(pn, cacheattr, page_cnt, page_index, doit, res); +} + +static void +pmap_set_cache_attributes_internal( + ppnum_t pn, + unsigned int cacheattr) +{ + pmap_paddr_t paddr; + int pai; + pp_attr_t pp_attr_current; + pp_attr_t pp_attr_template; + unsigned int wimg_bits_prev, wimg_bits_new; + + paddr = ptoa(pn); + + if (!pa_valid(paddr)) { + return; /* Not a managed page. */ + } + + if (cacheattr & VM_WIMG_USE_DEFAULT) + cacheattr = VM_WIMG_DEFAULT; + + pai = (int)pa_index(paddr); + + LOCK_PVH(pai); + + pp_attr_current = pp_attr_table[pai]; + wimg_bits_prev = VM_WIMG_DEFAULT; + if (pp_attr_current & PP_ATTR_WIMG_MASK) + wimg_bits_prev = pp_attr_current & PP_ATTR_WIMG_MASK; + + pp_attr_template = (pp_attr_current & ~PP_ATTR_WIMG_MASK) | PP_ATTR_WIMG(cacheattr & (VM_WIMG_MASK)) ; + + pp_attr_table[pai] = pp_attr_template; + wimg_bits_new = VM_WIMG_DEFAULT; + if (pp_attr_template & PP_ATTR_WIMG_MASK) + wimg_bits_new = pp_attr_template & PP_ATTR_WIMG_MASK; + + if (wimg_bits_new != wimg_bits_prev) + pmap_update_cache_attributes_locked(pn, cacheattr); + + UNLOCK_PVH(pai); + + if ((wimg_bits_prev != wimg_bits_new) + && ((wimg_bits_prev == VM_WIMG_COPYBACK) + || ((wimg_bits_prev == VM_WIMG_INNERWBACK) + && (wimg_bits_new != VM_WIMG_COPYBACK)) + || ((wimg_bits_prev == VM_WIMG_WTHRU) + && ((wimg_bits_new != VM_WIMG_COPYBACK) || (wimg_bits_new != VM_WIMG_INNERWBACK))))) + pmap_sync_page_attributes_phys(pn); + +} + +void +pmap_set_cache_attributes( + ppnum_t pn, + unsigned int cacheattr) +{ + pmap_set_cache_attributes_internal(pn, cacheattr); +} + +void +pmap_update_cache_attributes_locked( + ppnum_t ppnum, + unsigned attributes) +{ + pmap_paddr_t phys = ptoa(ppnum); + pv_entry_t *pve_p; + pt_entry_t *pte_p; + pv_entry_t **pv_h; + pt_entry_t tmplate; + unsigned int pai; + +#if (__ARM_VMSA__ == 7) + #define ARM_PTE_SHMASK ARM_PTE_SH +#endif + +#if __ARM_PTE_PHYSMAP__ + vm_offset_t kva = phystokv(phys); + pte_p = pmap_pte(kernel_pmap, kva); + + tmplate = *pte_p; + tmplate &= ~(ARM_PTE_ATTRINDXMASK | ARM_PTE_SHMASK); + tmplate |= wimg_to_pte(attributes); + + WRITE_PTE(pte_p, tmplate); + PMAP_UPDATE_TLBS(kernel_pmap, kva, kva + PAGE_SIZE); +#endif + + pai = (unsigned int)pa_index(phys); + + pv_h = pai_to_pvh(pai); + + pte_p = PT_ENTRY_NULL; + pve_p = PV_ENTRY_NULL; + if (pvh_test_type(pv_h, PVH_TYPE_PTEP)) { + pte_p = pvh_ptep(pv_h); + } else if (pvh_test_type(pv_h, PVH_TYPE_PVEP)) { + pve_p = pvh_list(pv_h); + pte_p = PT_ENTRY_NULL; + } + + while ((pve_p != PV_ENTRY_NULL) || (pte_p != PT_ENTRY_NULL)) { + vm_map_address_t va; + pmap_t pmap; + + if (pve_p != PV_ENTRY_NULL) + pte_p = pve_get_ptep(pve_p); + + pmap = ptep_get_pmap(pte_p); + va = ptep_get_va(pte_p); + + tmplate = *pte_p; + tmplate &= ~(ARM_PTE_ATTRINDXMASK | ARM_PTE_NX | ARM_PTE_PNX | ARM_PTE_SHMASK); + tmplate |= wimg_to_pte(attributes); + + WRITE_PTE(pte_p, tmplate); + PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE); + + pte_p = PT_ENTRY_NULL; + if (pve_p != PV_ENTRY_NULL) + pve_p = PVE_NEXT_PTR(pve_next(pve_p)); + + } +} + +#if (__ARM_VMSA__ == 7) +vm_map_address_t +pmap_create_sharedpage( + void) +{ + pmap_paddr_t pa; + kern_return_t kr; + + (void) pmap_pages_alloc(&pa, PAGE_SIZE, 0); + memset((char *) phystokv(pa), 0, PAGE_SIZE); + + kr = pmap_enter(kernel_pmap, _COMM_PAGE_BASE_ADDRESS, atop(pa), VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE); + assert(kr == KERN_SUCCESS); + + return((vm_map_address_t)phystokv(pa)); + +} +#else +static void +pmap_update_tt3e( + pmap_t pmap, + vm_address_t address, + tt_entry_t template) +{ + tt_entry_t *ptep, pte; + + ptep = pmap_tt3e(pmap, address); + if (ptep == NULL) { + panic("%s: no ptep?\n", __FUNCTION__); + } + + pte = *ptep; + pte = tte_to_pa(pte) | template; + WRITE_PTE(ptep, pte); +} + +/* Note absence of non-global bit */ +#define PMAP_COMM_PAGE_PTE_TEMPLATE (ARM_PTE_TYPE_VALID \ + | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) \ + | ARM_PTE_SH(SH_INNER_MEMORY) | ARM_PTE_NX \ + | ARM_PTE_PNX | ARM_PTE_AP(AP_RORO) | ARM_PTE_AF) + +vm_map_address_t +pmap_create_sharedpage( + void +) +{ + kern_return_t kr; + pmap_paddr_t pa = 0; + + + kr = pmap_expand(kernel_pmap, _COMM_PAGE64_BASE_ADDRESS, 0, PMAP_TT_L3_LEVEL); + assert(kr == KERN_SUCCESS); + + (void) pmap_pages_alloc(&pa, PAGE_SIZE, 0); + + memset((char *) phystokv(pa), 0, PAGE_SIZE); + + /* + * This is the mapping which U64 will refer to. + * Create with common path, update to be non-global and user-readable. + */ + kr = pmap_enter(kernel_pmap, _COMM_PAGE64_BASE_ADDRESS, (ppnum_t)atop(pa), VM_PROT_READ, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE); + assert(kr == KERN_SUCCESS); + pmap_update_tt3e(kernel_pmap, _COMM_PAGE64_BASE_ADDRESS, PMAP_COMM_PAGE_PTE_TEMPLATE); + + /* + * With PAN enabled kernel drivers can no longer use the previous mapping which is user readable + * They should use the following mapping instead + */ + kr = pmap_expand(kernel_pmap, _COMM_PRIV_PAGE64_BASE_ADDRESS, 0, PMAP_TT_L3_LEVEL); + assert(kr == KERN_SUCCESS); + kr = pmap_enter(kernel_pmap, _COMM_PRIV_PAGE64_BASE_ADDRESS, (ppnum_t)atop(pa), VM_PROT_READ, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE); + assert(kr == KERN_SUCCESS); + + /* + * Preferrable to just use a single entry,but we consume + * the full entry 0 of the L1 page table for U32 (i.e. + * for the 1GB of user address space), so we have no choice + * but to burn another L1 entry for the shared page... unless + * something clever comes to us. For the short term (i.e. + * bringup) do a kind of forced nesting (we don't have a + * notion of nesting more than one pmap, and the shared cache + * wins). In effect, just allocate a pmap, fill it out + * to include the one entry we care about, and then + * use its L1 pointer in U32 TTBR0 page tables. + * + * Note that we update parameters of entry for our unique + * needs (NG entry, etc.). + */ + u32_sharedpage_pmap = pmap_create(NULL, 0x0, FALSE); + assert(u32_sharedpage_pmap != NULL); + kr = pmap_enter(u32_sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS, (ppnum_t)atop(pa), VM_PROT_READ, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE); + assert(kr == KERN_SUCCESS); + pmap_update_tt3e(u32_sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS, PMAP_COMM_PAGE_PTE_TEMPLATE); + + /* For manipulation in kernel, go straight to physical page */ + sharedpage_rw_addr = phystokv(pa); + return((vm_map_address_t)sharedpage_rw_addr); +} + +static void +pmap_insert_sharedpage_internal( + pmap_t pmap) +{ +#if (ARM_PGSHIFT == 14) && !__ARM64_TWO_LEVEL_PMAP__ + kern_return_t kr; +#endif + pt_entry_t *ttep, *src_ttep; +#if _COMM_PAGE_AREA_LENGTH != PAGE_SIZE +#error We assume a single page. +#endif + + if (pmap_is_64bit(pmap)) { + /* Already in kernel pmap */ + return; + } + + PMAP_LOCK(pmap); + + /* + * For 4KB pages, we can force the commpage to nest at the level one + * page table, as each entry is 1GB (i.e, there will be no overlap + * with regular userspace mappings). For 16KB pages, each level one + * entry is 64GB, so we must go to the second level entry (32MB) in + * order to nest. + */ +#if (ARM_PGSHIFT == 12) +#if __ARM64_TWO_LEVEL_PMAP__ +#error A two level page table with a page shift of 12 is not currently supported +#endif + /* Just slam in the L1 entry. */ + ttep = pmap_tt1e(pmap, _COMM_PAGE32_BASE_ADDRESS); + if (*ttep != ARM_PTE_EMPTY) { + panic("%s: Found something mapped at the commpage address?!", __FUNCTION__); + } + + src_ttep = pmap_tt1e(u32_sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS); +#elif (ARM_PGSHIFT == 14) +#if !__ARM64_TWO_LEVEL_PMAP__ + /* Allocate for the L2 entry if necessary, and slam it into place. */ + /* + * As long as we are use a three level page table, the first level + * should always exist, so we don't need to check for it. + */ + while (*pmap_tt1e(pmap, _COMM_PAGE32_BASE_ADDRESS) == ARM_PTE_EMPTY) { + PMAP_UNLOCK(pmap); + + kr = pmap_expand(pmap, _COMM_PAGE32_BASE_ADDRESS, 0, PMAP_TT_L2_LEVEL); + + if (kr != KERN_SUCCESS) { + panic("Failed to pmap_expand for 32-bit commpage, pmap=%p", pmap); + } + + PMAP_LOCK(pmap); + } +#endif + + ttep = pmap_tt2e(pmap, _COMM_PAGE32_BASE_ADDRESS); + if (*ttep != ARM_PTE_EMPTY) { + panic("%s: Found something mapped at the commpage address?!", __FUNCTION__); + } + + src_ttep = pmap_tt2e(u32_sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS); +#endif + + *ttep = *src_ttep; +#ifndef __ARM_L1_PTW__ + CleanPoU_DcacheRegion((vm_offset_t) ttep, sizeof(tt_entry_t)); +#endif + flush_mmu_tlb_region(_COMM_PAGE32_BASE_ADDRESS, PAGE_SIZE); +#if (ARM_PGSHIFT == 12) && !__ARM64_TWO_LEVEL_PMAP__ + flush_mmu_tlb_entry(tlbi_addr(_COMM_PAGE32_BASE_ADDRESS & ~ARM_TT_L1_OFFMASK) | tlbi_asid(pmap->asid)); +#elif (ARM_PGSHIFT == 14) + flush_mmu_tlb_entry(tlbi_addr(_COMM_PAGE32_BASE_ADDRESS & ~ARM_TT_L2_OFFMASK) | tlbi_asid(pmap->asid)); +#endif + + PMAP_UNLOCK(pmap); +} + +static void +pmap_sharedpage_flush_32_to_64( + void) +{ + flush_mmu_tlb_region(_COMM_PAGE32_BASE_ADDRESS, PAGE_SIZE); +} + +static void +pmap_unmap_sharedpage32( + pmap_t pmap) +{ + pt_entry_t *ttep; + +#if _COMM_PAGE_AREA_LENGTH != PAGE_SIZE +#error We assume a single page. +#endif + +#if (ARM_PGSHIFT == 12) +#if __ARM64_TWO_LEVEL_PMAP__ +#error A two level page table with a page shift of 12 is not currently supported +#endif + ttep = pmap_tt1e(pmap, _COMM_PAGE32_BASE_ADDRESS); + if (ttep == NULL) { + return; + } + + /* It had better be mapped to the shared page */ + if (*ttep != ARM_TTE_EMPTY && *ttep != *pmap_tt1e(u32_sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS)) { + panic("%s: Something other than commpage mapped in shared page slot?", __FUNCTION__); + } +#elif (ARM_PGSHIFT == 14) + ttep = pmap_tt2e(pmap, _COMM_PAGE32_BASE_ADDRESS); + if (ttep == NULL) { + return; + } + + /* It had better be mapped to the shared page */ + if (*ttep != ARM_TTE_EMPTY && *ttep != *pmap_tt2e(u32_sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS)) { + panic("%s: Something other than commpage mapped in shared page slot?", __FUNCTION__); + } +#endif + + *ttep = ARM_TTE_EMPTY; + flush_mmu_tlb_region(_COMM_PAGE32_BASE_ADDRESS, PAGE_SIZE); + +#if (ARM_PGSHIFT == 12) +#if __ARM64_TWO_LEVEL_PMAP__ +#error A two level page table with a page shift of 12 is not currently supported +#endif + flush_mmu_tlb_entry(tlbi_addr(_COMM_PAGE32_BASE_ADDRESS & ~ARM_TT_L1_OFFMASK) | tlbi_asid(pmap->asid)); +#elif (ARM_PGSHIFT == 14) + flush_mmu_tlb_entry(tlbi_addr(_COMM_PAGE32_BASE_ADDRESS & ~ARM_TT_L2_OFFMASK) | tlbi_asid(pmap->asid)); +#endif +} + +void +pmap_insert_sharedpage( + pmap_t pmap) +{ + pmap_insert_sharedpage_internal(pmap); +} + +static boolean_t +pmap_is_64bit( + pmap_t pmap) +{ + return (pmap->is_64bit); +} + +#endif + +/* ARMTODO -- an implementation that accounts for + * holes in the physical map, if any. + */ +boolean_t +pmap_valid_page( + ppnum_t pn) { + return pa_valid(ptoa(pn)); +} + +static boolean_t +pmap_is_empty_internal( + pmap_t pmap, + vm_map_offset_t va_start, + vm_map_offset_t va_end) +{ + vm_map_offset_t block_start, block_end; + tt_entry_t *tte_p; + + if (pmap == NULL) { + return TRUE; + } + + if ((pmap != kernel_pmap) && (not_in_kdp)) { + PMAP_LOCK(pmap); + } + +#if (__ARM_VMSA__ == 7) + if (tte_index(pmap, va_end) >= pmap->tte_index_max) { + if ((pmap != kernel_pmap) && (not_in_kdp)) { + PMAP_UNLOCK(pmap); + } + return TRUE; + } + + block_start = va_start; + tte_p = pmap_tte(pmap, block_start); + while (block_start < va_end) { + block_end = (block_start + ARM_TT_L1_SIZE) & ~(ARM_TT_L1_OFFMASK); + if (block_end > va_end) + block_end = va_end; + + if ((*tte_p & ARM_TTE_TYPE_MASK) != 0) { + vm_map_offset_t offset; + ppnum_t phys_page = 0; + + for (offset = block_start; + offset < block_end; + offset += ARM_PGBYTES) { + // This does a pmap_find_phys() lookup but assumes lock is held + phys_page = pmap_vtophys(pmap, offset); + if (phys_page) { + if ((pmap != kernel_pmap) && (not_in_kdp)) { + PMAP_UNLOCK(pmap); + } + return FALSE; + } + } + } + + block_start = block_end; + tte_p++; + } +#else + block_start = va_start; + + while (block_start < va_end) { + pt_entry_t *bpte_p, *epte_p; + pt_entry_t *pte_p; + + block_end = (block_start + ARM_TT_L2_SIZE) & ~ARM_TT_L2_OFFMASK; + if (block_end > va_end) + block_end = va_end; + + tte_p = pmap_tt2e(pmap, block_start); + if ((tte_p != PT_ENTRY_NULL) + && ((*tte_p & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE)) { + + pte_p = (pt_entry_t *) ttetokv(*tte_p); + bpte_p = &pte_p[tt3_index(pmap, block_start)]; + epte_p = bpte_p + (((block_end - block_start) & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT); + + for (pte_p = bpte_p; pte_p < epte_p; pte_p++) { + if (*pte_p != ARM_PTE_EMPTY) { + if ((pmap != kernel_pmap) && (not_in_kdp)) { + PMAP_UNLOCK(pmap); + } + return FALSE; + } + } + } + block_start = block_end; + } +#endif + + if ((pmap != kernel_pmap) && (not_in_kdp)) { + PMAP_UNLOCK(pmap); + } + + return TRUE; +} + +boolean_t +pmap_is_empty( + pmap_t pmap, + vm_map_offset_t va_start, + vm_map_offset_t va_end) +{ + return pmap_is_empty_internal(pmap, va_start, va_end); +} + +vm_map_offset_t pmap_max_offset( + boolean_t is64 __unused, + unsigned int option) +{ + vm_map_offset_t max_offset_ret = 0; + +#if defined(__arm64__) + assert (is64); + vm_map_offset_t min_max_offset = SHARED_REGION_BASE_ARM64 + SHARED_REGION_SIZE_ARM64 + 0x20000000; // end of shared region + 512MB for various purposes + if (option == ARM_PMAP_MAX_OFFSET_DEFAULT) { + max_offset_ret = arm64_pmap_max_offset_default; + } else if (option == ARM_PMAP_MAX_OFFSET_MIN) { + max_offset_ret = min_max_offset; + } else if (option == ARM_PMAP_MAX_OFFSET_MAX) { + max_offset_ret = MACH_VM_MAX_ADDRESS; + } else if (option == ARM_PMAP_MAX_OFFSET_DEVICE) { + if (arm64_pmap_max_offset_default) { + max_offset_ret = arm64_pmap_max_offset_default; + } else if (max_mem > 0xC0000000) { + max_offset_ret = 0x0000000318000000ULL; // Max offset is 12.375GB for devices with > 3GB of memory + } else if (max_mem > 0x40000000) { + max_offset_ret = 0x0000000218000000ULL; // Max offset is 8.375GB for devices with > 1GB and <= 3GB of memory + } else { + max_offset_ret = min_max_offset; + } + } else if (option == ARM_PMAP_MAX_OFFSET_JUMBO) { + max_offset_ret = 0x0000000518000000ULL; // Max offset is 20.375GB for pmaps with special "jumbo" blessing + } else { + panic("pmap_max_offset illegal option 0x%x\n", option); + } + + assert(max_offset_ret >= min_max_offset); + return max_offset_ret; +#else + if (option == ARM_PMAP_MAX_OFFSET_DEFAULT) { + max_offset_ret = arm_pmap_max_offset_default; + } else if (option == ARM_PMAP_MAX_OFFSET_MIN) { + max_offset_ret = 0x66000000; + } else if (option == ARM_PMAP_MAX_OFFSET_MAX) { + max_offset_ret = VM_MAX_ADDRESS; + } else if (option == ARM_PMAP_MAX_OFFSET_DEVICE) { + if (arm_pmap_max_offset_default) { + max_offset_ret = arm_pmap_max_offset_default; + } else if (max_mem > 0x20000000) { + max_offset_ret = 0x80000000; + } else { + max_offset_ret = 0x66000000; + } + } else { + panic("pmap_max_offset illegal option 0x%x\n", option); + } + + return max_offset_ret; +#endif +} + +#if CONFIG_DTRACE +/* + * Constrain DTrace copyin/copyout actions + */ +extern kern_return_t dtrace_copyio_preflight(addr64_t); +extern kern_return_t dtrace_copyio_postflight(addr64_t); + +kern_return_t dtrace_copyio_preflight( + __unused addr64_t va) +{ + if (current_map() == kernel_map) + return KERN_FAILURE; + else + return KERN_SUCCESS; +} + +kern_return_t dtrace_copyio_postflight( + __unused addr64_t va) +{ + return KERN_SUCCESS; +} +#endif /* CONFIG_DTRACE */ + + +void +pmap_flush_context_init(__unused pmap_flush_context *pfc) +{ +} + + +void +pmap_flush( + __unused pmap_flush_context *cpus_to_flush) +{ + /* not implemented yet */ + return; +} + +static boolean_t +pmap_query_resident_internal( + pmap_t pmap, + vm_map_address_t start, + vm_map_address_t end, + mach_vm_size_t *resident_bytes_p, + mach_vm_size_t *compressed_bytes_p) +{ + mach_vm_size_t resident_bytes = 0; + mach_vm_size_t compressed_bytes = 0; + + pt_entry_t *bpte, *epte; + pt_entry_t *pte_p; + tt_entry_t *tte_p; + + if (pmap == NULL) { + return FALSE; + } + + /* Ensure that this request is valid, and addresses exactly one TTE. */ + assert(!(start % ARM_PGBYTES)); + assert(!(end % ARM_PGBYTES)); + assert(end >= start); + assert((end - start) <= (PTE_PGENTRIES * ARM_PGBYTES)); + + PMAP_LOCK(pmap); + tte_p = pmap_tte(pmap, start); + if (tte_p == (tt_entry_t *) NULL) { + PMAP_UNLOCK(pmap); + return FALSE; + } + if ((*tte_p & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_TABLE) { + +#if (__ARM_VMSA__ == 7) + pte_p = (pt_entry_t *) ttetokv(*tte_p); + bpte = &pte_p[ptenum(start)]; + epte = bpte + atop(end - start); +#else + pte_p = (pt_entry_t *) ttetokv(*tte_p); + bpte = &pte_p[tt3_index(pmap, start)]; + epte = bpte + ((end - start) >> ARM_TT_L3_SHIFT); +#endif + + for (; bpte < epte; bpte++) { + if (ARM_PTE_IS_COMPRESSED(*bpte)) { + compressed_bytes += ARM_PGBYTES; + } else if (pa_valid(pte_to_pa(*bpte))) { + resident_bytes += ARM_PGBYTES; + } + } + } + PMAP_UNLOCK(pmap); + + if (compressed_bytes_p) { + *compressed_bytes_p += compressed_bytes; + } + + if (resident_bytes_p) { + *resident_bytes_p += resident_bytes; + } + + return TRUE; +} + +mach_vm_size_t +pmap_query_resident( + pmap_t pmap, + vm_map_address_t start, + vm_map_address_t end, + mach_vm_size_t *compressed_bytes_p) +{ + mach_vm_size_t resident_bytes; + mach_vm_size_t compressed_bytes; + vm_map_address_t va; + + + if (pmap == PMAP_NULL) { + if (compressed_bytes_p) { + *compressed_bytes_p = 0; + } + return 0; + } + + resident_bytes = 0; + compressed_bytes = 0; + + PMAP_TRACE(PMAP_CODE(PMAP__QUERY_RESIDENT) | DBG_FUNC_START, + VM_KERNEL_ADDRHIDE(pmap), VM_KERNEL_ADDRHIDE(start), + VM_KERNEL_ADDRHIDE(end)); + + va = start; + while (va < end) { + vm_map_address_t l; + + l = ((va + ARM_TT_TWIG_SIZE) & ~ARM_TT_TWIG_OFFMASK); + + if (l > end) + l = end; + if (!pmap_query_resident_internal(pmap, va, l, &resident_bytes, compressed_bytes_p)) { + break; + } + + va = l; + } + + if (compressed_bytes_p) { + *compressed_bytes_p = compressed_bytes; + } + + PMAP_TRACE(PMAP_CODE(PMAP__QUERY_RESIDENT) | DBG_FUNC_END, + resident_bytes); + + return resident_bytes; +} + +#if MACH_ASSERT +extern int pmap_ledgers_panic; +static void +pmap_check_ledgers( + pmap_t pmap) +{ + ledger_amount_t bal; + int pid; + char *procname; + boolean_t do_panic; + + if (pmap->pmap_pid == 0) { + /* + * This pmap was not or is no longer fully associated + * with a task (e.g. the old pmap after a fork()/exec() or + * spawn()). Its "ledger" still points at a task that is + * now using a different (and active) address space, so + * we can't check that all the pmap ledgers are balanced here. + * + * If the "pid" is set, that means that we went through + * pmap_set_process() in task_terminate_internal(), so + * this task's ledger should not have been re-used and + * all the pmap ledgers should be back to 0. + */ + return; + } + + do_panic = FALSE; + pid = pmap->pmap_pid; + procname = pmap->pmap_procname; + + pmap_ledgers_drift.num_pmaps_checked++; + + ledger_get_balance(pmap->ledger, + task_ledgers.phys_footprint, + &bal); + if (bal != 0) { +#if DEVELOPMENT || DEBUG +// if (!pmap->footprint_was_suspended) +#endif /* DEVELOPMENT || DEBUG */ + do_panic = TRUE; + printf("LEDGER BALANCE proc %d (%s) " + "\"phys_footprint\" = %lld\n", + pid, procname, bal); + if (bal > 0) { + pmap_ledgers_drift.phys_footprint_over++; + pmap_ledgers_drift.phys_footprint_over_total += bal; + if (bal > pmap_ledgers_drift.phys_footprint_over_max) { + pmap_ledgers_drift.phys_footprint_over_max = bal; + } + } else { + pmap_ledgers_drift.phys_footprint_under++; + pmap_ledgers_drift.phys_footprint_under_total += bal; + if (bal < pmap_ledgers_drift.phys_footprint_under_max) { + pmap_ledgers_drift.phys_footprint_under_max = bal; + } + } + } + ledger_get_balance(pmap->ledger, + task_ledgers.internal, + &bal); + if (bal != 0) { + do_panic = TRUE; + printf("LEDGER BALANCE proc %d (%s) " + "\"internal\" = %lld\n", + pid, procname, bal); + if (bal > 0) { + pmap_ledgers_drift.internal_over++; + pmap_ledgers_drift.internal_over_total += bal; + if (bal > pmap_ledgers_drift.internal_over_max) { + pmap_ledgers_drift.internal_over_max = bal; + } + } else { + pmap_ledgers_drift.internal_under++; + pmap_ledgers_drift.internal_under_total += bal; + if (bal < pmap_ledgers_drift.internal_under_max) { + pmap_ledgers_drift.internal_under_max = bal; + } + } + } + ledger_get_balance(pmap->ledger, + task_ledgers.internal_compressed, + &bal); + if (bal != 0) { + do_panic = TRUE; + printf("LEDGER BALANCE proc %d (%s) " + "\"internal_compressed\" = %lld\n", + pid, procname, bal); + if (bal > 0) { + pmap_ledgers_drift.internal_compressed_over++; + pmap_ledgers_drift.internal_compressed_over_total += bal; + if (bal > pmap_ledgers_drift.internal_compressed_over_max) { + pmap_ledgers_drift.internal_compressed_over_max = bal; + } + } else { + pmap_ledgers_drift.internal_compressed_under++; + pmap_ledgers_drift.internal_compressed_under_total += bal; + if (bal < pmap_ledgers_drift.internal_compressed_under_max) { + pmap_ledgers_drift.internal_compressed_under_max = bal; + } + } + } + ledger_get_balance(pmap->ledger, + task_ledgers.iokit_mapped, + &bal); + if (bal != 0) { + do_panic = TRUE; + printf("LEDGER BALANCE proc %d (%s) " + "\"iokit_mapped\" = %lld\n", + pid, procname, bal); + if (bal > 0) { + pmap_ledgers_drift.iokit_mapped_over++; + pmap_ledgers_drift.iokit_mapped_over_total += bal; + if (bal > pmap_ledgers_drift.iokit_mapped_over_max) { + pmap_ledgers_drift.iokit_mapped_over_max = bal; + } + } else { + pmap_ledgers_drift.iokit_mapped_under++; + pmap_ledgers_drift.iokit_mapped_under_total += bal; + if (bal < pmap_ledgers_drift.iokit_mapped_under_max) { + pmap_ledgers_drift.iokit_mapped_under_max = bal; + } + } + } + ledger_get_balance(pmap->ledger, + task_ledgers.alternate_accounting, + &bal); + if (bal != 0) { + do_panic = TRUE; + printf("LEDGER BALANCE proc %d (%s) " + "\"alternate_accounting\" = %lld\n", + pid, procname, bal); + if (bal > 0) { + pmap_ledgers_drift.alternate_accounting_over++; + pmap_ledgers_drift.alternate_accounting_over_total += bal; + if (bal > pmap_ledgers_drift.alternate_accounting_over_max) { + pmap_ledgers_drift.alternate_accounting_over_max = bal; + } + } else { + pmap_ledgers_drift.alternate_accounting_under++; + pmap_ledgers_drift.alternate_accounting_under_total += bal; + if (bal < pmap_ledgers_drift.alternate_accounting_under_max) { + pmap_ledgers_drift.alternate_accounting_under_max = bal; + } + } + } + ledger_get_balance(pmap->ledger, + task_ledgers.alternate_accounting_compressed, + &bal); + if (bal != 0) { + do_panic = TRUE; + printf("LEDGER BALANCE proc %d (%s) " + "\"alternate_accounting_compressed\" = %lld\n", + pid, procname, bal); + if (bal > 0) { + pmap_ledgers_drift.alternate_accounting_compressed_over++; + pmap_ledgers_drift.alternate_accounting_compressed_over_total += bal; + if (bal > pmap_ledgers_drift.alternate_accounting_compressed_over_max) { + pmap_ledgers_drift.alternate_accounting_compressed_over_max = bal; + } + } else { + pmap_ledgers_drift.alternate_accounting_compressed_under++; + pmap_ledgers_drift.alternate_accounting_compressed_under_total += bal; + if (bal < pmap_ledgers_drift.alternate_accounting_compressed_under_max) { + pmap_ledgers_drift.alternate_accounting_compressed_under_max = bal; + } + } + } + ledger_get_balance(pmap->ledger, + task_ledgers.page_table, + &bal); + if (bal != 0) { + do_panic = TRUE; + printf("LEDGER BALANCE proc %d (%s) " + "\"page_table\" = %lld\n", + pid, procname, bal); + if (bal > 0) { + pmap_ledgers_drift.page_table_over++; + pmap_ledgers_drift.page_table_over_total += bal; + if (bal > pmap_ledgers_drift.page_table_over_max) { + pmap_ledgers_drift.page_table_over_max = bal; + } + } else { + pmap_ledgers_drift.page_table_under++; + pmap_ledgers_drift.page_table_under_total += bal; + if (bal < pmap_ledgers_drift.page_table_under_max) { + pmap_ledgers_drift.page_table_under_max = bal; + } + } + } + ledger_get_balance(pmap->ledger, + task_ledgers.purgeable_volatile, + &bal); + if (bal != 0) { + do_panic = TRUE; + printf("LEDGER BALANCE proc %d (%s) " + "\"purgeable_volatile\" = %lld\n", + pid, procname, bal); + if (bal > 0) { + pmap_ledgers_drift.purgeable_volatile_over++; + pmap_ledgers_drift.purgeable_volatile_over_total += bal; + if (bal > pmap_ledgers_drift.purgeable_volatile_over_max) { + pmap_ledgers_drift.purgeable_volatile_over_max = bal; + } + } else { + pmap_ledgers_drift.purgeable_volatile_under++; + pmap_ledgers_drift.purgeable_volatile_under_total += bal; + if (bal < pmap_ledgers_drift.purgeable_volatile_under_max) { + pmap_ledgers_drift.purgeable_volatile_under_max = bal; + } + } + } + ledger_get_balance(pmap->ledger, + task_ledgers.purgeable_nonvolatile, + &bal); + if (bal != 0) { + do_panic = TRUE; + printf("LEDGER BALANCE proc %d (%s) " + "\"purgeable_nonvolatile\" = %lld\n", + pid, procname, bal); + if (bal > 0) { + pmap_ledgers_drift.purgeable_nonvolatile_over++; + pmap_ledgers_drift.purgeable_nonvolatile_over_total += bal; + if (bal > pmap_ledgers_drift.purgeable_nonvolatile_over_max) { + pmap_ledgers_drift.purgeable_nonvolatile_over_max = bal; + } + } else { + pmap_ledgers_drift.purgeable_nonvolatile_under++; + pmap_ledgers_drift.purgeable_nonvolatile_under_total += bal; + if (bal < pmap_ledgers_drift.purgeable_nonvolatile_under_max) { + pmap_ledgers_drift.purgeable_nonvolatile_under_max = bal; + } + } + } + ledger_get_balance(pmap->ledger, + task_ledgers.purgeable_volatile_compressed, + &bal); + if (bal != 0) { + do_panic = TRUE; + printf("LEDGER BALANCE proc %d (%s) " + "\"purgeable_volatile_compressed\" = %lld\n", + pid, procname, bal); + if (bal > 0) { + pmap_ledgers_drift.purgeable_volatile_compressed_over++; + pmap_ledgers_drift.purgeable_volatile_compressed_over_total += bal; + if (bal > pmap_ledgers_drift.purgeable_volatile_compressed_over_max) { + pmap_ledgers_drift.purgeable_volatile_compressed_over_max = bal; + } + } else { + pmap_ledgers_drift.purgeable_volatile_compressed_under++; + pmap_ledgers_drift.purgeable_volatile_compressed_under_total += bal; + if (bal < pmap_ledgers_drift.purgeable_volatile_compressed_under_max) { + pmap_ledgers_drift.purgeable_volatile_compressed_under_max = bal; + } + } + } + ledger_get_balance(pmap->ledger, + task_ledgers.purgeable_nonvolatile_compressed, + &bal); + if (bal != 0) { + do_panic = TRUE; + printf("LEDGER BALANCE proc %d (%s) " + "\"purgeable_nonvolatile_compressed\" = %lld\n", + pid, procname, bal); + if (bal > 0) { + pmap_ledgers_drift.purgeable_nonvolatile_compressed_over++; + pmap_ledgers_drift.purgeable_nonvolatile_compressed_over_total += bal; + if (bal > pmap_ledgers_drift.purgeable_nonvolatile_compressed_over_max) { + pmap_ledgers_drift.purgeable_nonvolatile_compressed_over_max = bal; + } + } else { + pmap_ledgers_drift.purgeable_nonvolatile_compressed_under++; + pmap_ledgers_drift.purgeable_nonvolatile_compressed_under_total += bal; + if (bal < pmap_ledgers_drift.purgeable_nonvolatile_compressed_under_max) { + pmap_ledgers_drift.purgeable_nonvolatile_compressed_under_max = bal; + } + } + } + + if (do_panic) { + if (pmap_ledgers_panic) { + panic("pmap_destroy(%p) %d[%s] has imbalanced ledgers\n", + pmap, pid, procname); + } else { + printf("pmap_destroy(%p) %d[%s] has imbalanced ledgers\n", + pmap, pid, procname); + } + } + + assert(pmap->stats.resident_count == 0); +#if 00 + assert(pmap->stats.wired_count == 0); +#endif + assert(pmap->stats.device == 0); + assert(pmap->stats.internal == 0); + assert(pmap->stats.external == 0); + assert(pmap->stats.reusable == 0); + assert(pmap->stats.compressed == 0); +} +#endif /* MACH_ASSERT */ + +void pmap_advise_pagezero_range(__unused pmap_t p, __unused uint64_t a) { +} + + +#if CONFIG_PGTRACE +#define PROF_START uint64_t t, nanot;\ + t = mach_absolute_time(); + +#define PROF_END absolutetime_to_nanoseconds(mach_absolute_time()-t, &nanot);\ + kprintf("%s: took %llu ns\n", __func__, nanot); + +#define PMAP_PGTRACE_LOCK(p) \ + do { \ + *(p) = ml_set_interrupts_enabled(false); \ + if (simple_lock_try(&(pmap_pgtrace.lock))) break; \ + ml_set_interrupts_enabled(*(p)); \ + } while (true) + +#define PMAP_PGTRACE_UNLOCK(p) \ + do { \ + simple_unlock(&(pmap_pgtrace.lock)); \ + ml_set_interrupts_enabled(*(p)); \ + } while (0) + +#define PGTRACE_WRITE_PTE(pte_p, pte_entry) \ + do { \ + *(pte_p) = (pte_entry); \ + FLUSH_PTE(pte_p); \ + } while (0) + +#define PGTRACE_MAX_MAP 16 // maximum supported va to same pa + +typedef enum { + UNDEFINED, + PA_UNDEFINED, + VA_UNDEFINED, + DEFINED +} pmap_pgtrace_page_state_t; + +typedef struct { + queue_chain_t chain; + + /* + pa - pa + maps - list of va maps to upper pa + map_pool - map pool + map_waste - waste can + state - state + */ + pmap_paddr_t pa; + queue_head_t maps; + queue_head_t map_pool; + queue_head_t map_waste; + pmap_pgtrace_page_state_t state; +} pmap_pgtrace_page_t; + +static struct { + /* + pages - list of tracing page info + */ + queue_head_t pages; + decl_simple_lock_data(, lock); +} pmap_pgtrace = {}; + +static void pmap_pgtrace_init(void) +{ + queue_init(&(pmap_pgtrace.pages)); + simple_lock_init(&(pmap_pgtrace.lock), 0); + + boolean_t enabled; + + if (PE_parse_boot_argn("pgtrace", &enabled, sizeof(enabled))) { + pgtrace_enabled = enabled; + } +} + +// find a page with given pa - pmap_pgtrace should be locked +inline static pmap_pgtrace_page_t *pmap_pgtrace_find_page(pmap_paddr_t pa) +{ + queue_head_t *q = &(pmap_pgtrace.pages); + pmap_pgtrace_page_t *p; + + queue_iterate(q, p, pmap_pgtrace_page_t *, chain) { + if (p->state == UNDEFINED) { + continue; + } + if (p->state == PA_UNDEFINED) { + continue; + } + if (p->pa == pa) { + return p; + } + } + + return NULL; +} + +// enter clone of given pmap, va page and range - pmap should be locked +static bool pmap_pgtrace_enter_clone(pmap_t pmap, vm_map_offset_t va_page, vm_map_offset_t start, vm_map_offset_t end) +{ + bool ints; + queue_head_t *q = &(pmap_pgtrace.pages); + pmap_paddr_t pa_page; + pt_entry_t *ptep, *cptep; + pmap_pgtrace_page_t *p; + bool found = false; + + PMAP_ASSERT_LOCKED(pmap); + assert(va_page == arm_trunc_page(va_page)); + + PMAP_PGTRACE_LOCK(&ints); + + ptep = pmap_pte(pmap, va_page); + + // target pte should exist + if (!ptep || !(*ptep & ARM_PTE_TYPE_VALID)) { + PMAP_PGTRACE_UNLOCK(&ints); + return false; + } + + queue_head_t *mapq; + queue_head_t *mappool; + pmap_pgtrace_map_t *map = NULL; + + pa_page = pte_to_pa(*ptep); + + // find if we have a page info defined for this + queue_iterate(q, p, pmap_pgtrace_page_t *, chain) { + mapq = &(p->maps); + mappool = &(p->map_pool); + + switch (p->state) { + case PA_UNDEFINED: + queue_iterate(mapq, map, pmap_pgtrace_map_t *, chain) { + if (map->cloned == false && map->pmap == pmap && map->ova == va_page) { + p->pa = pa_page; + map->range.start = start; + map->range.end = end; + found = true; + break; + } + } + break; + + case VA_UNDEFINED: + if (p->pa != pa_page) { + break; + } + queue_iterate(mapq, map, pmap_pgtrace_map_t *, chain) { + if (map->cloned == false) { + map->pmap = pmap; + map->ova = va_page; + map->range.start = start; + map->range.end = end; + found = true; + break; + } + } + break; + + case DEFINED: + if (p->pa != pa_page) { + break; + } + queue_iterate(mapq, map, pmap_pgtrace_map_t *, chain) { + if (map->cloned == true && map->pmap == pmap && map->ova == va_page) { + kprintf("%s: skip existing mapping at va=%llx\n", __func__, va_page); + break; + } else if (map->cloned == true && map->pmap == kernel_pmap && map->cva[1] == va_page) { + kprintf("%s: skip clone mapping at va=%llx\n", __func__, va_page); + break; + } else if (map->cloned == false && map->pmap == pmap && map->ova == va_page) { + // range should be already defined as well + found = true; + break; + } + } + break; + + default: + panic("invalid state p->state=%x\n", p->state); + } + + if (found == true) { + break; + } + } + + // do not clone if no page info found + if (found == false) { + PMAP_PGTRACE_UNLOCK(&ints); + return false; + } + + // copy pre, target and post ptes to clone ptes + for (int i = 0; i < 3; i++) { + ptep = pmap_pte(pmap, va_page + (i-1)*ARM_PGBYTES); + cptep = pmap_pte(kernel_pmap, map->cva[i]); + assert(cptep != NULL); + if (ptep == NULL) { + PGTRACE_WRITE_PTE(cptep, (pt_entry_t)NULL); + } else { + PGTRACE_WRITE_PTE(cptep, *ptep); + } + PMAP_UPDATE_TLBS(kernel_pmap, map->cva[i], map->cva[i]+ARM_PGBYTES); + } + + // get ptes for original and clone + ptep = pmap_pte(pmap, va_page); + cptep = pmap_pte(kernel_pmap, map->cva[1]); + + // invalidate original pte and mark it as a pgtrace page + PGTRACE_WRITE_PTE(ptep, (*ptep | ARM_PTE_PGTRACE) & ~ARM_PTE_TYPE_VALID); + PMAP_UPDATE_TLBS(pmap, map->ova, map->ova+ARM_PGBYTES); + + map->cloned = true; + p->state = DEFINED; + + kprintf("%s: pa_page=%llx va_page=%llx cva[1]=%llx pmap=%p ptep=%p cptep=%p\n", __func__, pa_page, va_page, map->cva[1], pmap, ptep, cptep); + + PMAP_PGTRACE_UNLOCK(&ints); + + return true; +} + +// This function removes trace bit and validate pte if applicable. Pmap must be locked. +static void pmap_pgtrace_remove_clone(pmap_t pmap, pmap_paddr_t pa, vm_map_offset_t va) +{ + bool ints, found = false; + pmap_pgtrace_page_t *p; + pt_entry_t *ptep; + + PMAP_PGTRACE_LOCK(&ints); + + // we must have this page info + p = pmap_pgtrace_find_page(pa); + if (p == NULL) { + goto unlock_exit; + } + + // find matching map + queue_head_t *mapq = &(p->maps); + queue_head_t *mappool = &(p->map_pool); + pmap_pgtrace_map_t *map; + + queue_iterate(mapq, map, pmap_pgtrace_map_t *, chain) { + if (map->pmap == pmap && map->ova == va) { + found = true; + break; + } + } + + if (!found) { + goto unlock_exit; + } + + if (map->cloned == true) { + // Restore back the pte to original state + ptep = pmap_pte(pmap, map->ova); + assert(ptep); + PGTRACE_WRITE_PTE(ptep, *ptep | ARM_PTE_TYPE_VALID); + PMAP_UPDATE_TLBS(pmap, va, va+ARM_PGBYTES); + + // revert clone pages + for (int i = 0; i < 3; i++) { + ptep = pmap_pte(kernel_pmap, map->cva[i]); + assert(ptep != NULL); + PGTRACE_WRITE_PTE(ptep, map->cva_spte[i]); + PMAP_UPDATE_TLBS(kernel_pmap, map->cva[i], map->cva[i]+ARM_PGBYTES); + } + } + + queue_remove(mapq, map, pmap_pgtrace_map_t *, chain); + map->pmap = NULL; + map->ova = (vm_map_offset_t)NULL; + map->cloned = false; + queue_enter_first(mappool, map, pmap_pgtrace_map_t *, chain); + + kprintf("%s: p=%p pa=%llx va=%llx\n", __func__, p, pa, va); + +unlock_exit: + PMAP_PGTRACE_UNLOCK(&ints); +} + +// remove all clones of given pa - pmap must be locked +static void pmap_pgtrace_remove_all_clone(pmap_paddr_t pa) +{ + bool ints; + pmap_pgtrace_page_t *p; + pt_entry_t *ptep; + + PMAP_PGTRACE_LOCK(&ints); + + // we must have this page info + p = pmap_pgtrace_find_page(pa); + if (p == NULL) { + PMAP_PGTRACE_UNLOCK(&ints); + return; + } + + queue_head_t *mapq = &(p->maps); + queue_head_t *mappool = &(p->map_pool); + queue_head_t *mapwaste = &(p->map_waste); + pmap_pgtrace_map_t *map; + + // move maps to waste + while (!queue_empty(mapq)) { + queue_remove_first(mapq, map, pmap_pgtrace_map_t *, chain); + queue_enter_first(mapwaste, map, pmap_pgtrace_map_t*, chain); + } + + PMAP_PGTRACE_UNLOCK(&ints); + + // sanitize maps in waste + queue_iterate(mapwaste, map, pmap_pgtrace_map_t *, chain) { + if (map->cloned == true) { + PMAP_LOCK(map->pmap); + + // restore back original pte + ptep = pmap_pte(map->pmap, map->ova); + assert(ptep); + PGTRACE_WRITE_PTE(ptep, *ptep | ARM_PTE_TYPE_VALID); + PMAP_UPDATE_TLBS(map->pmap, map->ova, map->ova+ARM_PGBYTES); + + // revert clone ptes + for (int i = 0; i < 3; i++) { + ptep = pmap_pte(kernel_pmap, map->cva[i]); + assert(ptep != NULL); + PGTRACE_WRITE_PTE(ptep, map->cva_spte[i]); + PMAP_UPDATE_TLBS(kernel_pmap, map->cva[i], map->cva[i]+ARM_PGBYTES); + } + + PMAP_UNLOCK(map->pmap); + } + + map->pmap = NULL; + map->ova = (vm_map_offset_t)NULL; + map->cloned = false; + } + + PMAP_PGTRACE_LOCK(&ints); + + // recycle maps back to map_pool + while (!queue_empty(mapwaste)) { + queue_remove_first(mapwaste, map, pmap_pgtrace_map_t *, chain); + queue_enter_first(mappool, map, pmap_pgtrace_map_t*, chain); + } + + PMAP_PGTRACE_UNLOCK(&ints); +} + +inline static void pmap_pgtrace_get_search_space(pmap_t pmap, vm_map_offset_t *startp, vm_map_offset_t *endp) +{ + uint64_t tsz; + vm_map_offset_t end; + + if (pmap == kernel_pmap) { + tsz = (get_tcr() >> TCR_T1SZ_SHIFT) & TCR_TSZ_MASK; + *startp = MAX(VM_MIN_KERNEL_ADDRESS, (UINT64_MAX >> (64-tsz)) << (64-tsz)); + *endp = VM_MAX_KERNEL_ADDRESS; + } else { + tsz = (get_tcr() >> TCR_T0SZ_SHIFT) & TCR_TSZ_MASK; + if (tsz == 64) { + end = 0; + } else { + end = ((uint64_t)1 << (64-tsz)) - 1; + } + + *startp = 0; + *endp = end; + } + + assert(*endp > *startp); + + return; +} + +// has pa mapped in given pmap? then clone it +static uint64_t pmap_pgtrace_clone_from_pa(pmap_t pmap, pmap_paddr_t pa, vm_map_offset_t start_offset, vm_map_offset_t end_offset) { + uint64_t ret = 0; + vm_map_offset_t min, max; + vm_map_offset_t cur_page, end_page; + pt_entry_t *ptep; + tt_entry_t *ttep; + tt_entry_t tte; + + pmap_pgtrace_get_search_space(pmap, &min, &max); + + cur_page = arm_trunc_page(min); + end_page = arm_trunc_page(max); + while (cur_page <= end_page) { + vm_map_offset_t add = 0; + + PMAP_LOCK(pmap); + + // skip uninterested space + if (pmap == kernel_pmap && + ((vm_kernel_base <= cur_page && cur_page < vm_kernel_top) || + (vm_kext_base <= cur_page && cur_page < vm_kext_top))) { + add = ARM_PGBYTES; + goto unlock_continue; + } + +#if __ARM64_TWO_LEVEL_PMAP__ + // check whether we can skip l2 + ttep = pmap_tt2e(pmap, cur_page); + assert(ttep); + tte = *ttep; +#else + // check whether we can skip l1 + ttep = pmap_tt1e(pmap, cur_page); + assert(ttep); + tte = *ttep; + if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) != (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID)) { + add = ARM_TT_L1_SIZE; + goto unlock_continue; + } + + // how about l2 + tte = ((tt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt2_index(pmap, cur_page)]; +#endif + if ((tte & (ARM_TTE_TYPE_MASK | ARM_TTE_VALID)) != (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID)) { + add = ARM_TT_L2_SIZE; + goto unlock_continue; + } + + // ptep finally + ptep = &(((pt_entry_t*) phystokv(tte & ARM_TTE_TABLE_MASK))[tt3_index(pmap, cur_page)]); + if (ptep == PT_ENTRY_NULL) { + add = ARM_TT_L3_SIZE; + goto unlock_continue; + } + + if (arm_trunc_page(pa) == pte_to_pa(*ptep)) { + if (pmap_pgtrace_enter_clone(pmap, cur_page, start_offset, end_offset) == true) { + ret++; + } + } + + add = ARM_PGBYTES; + +unlock_continue: + PMAP_UNLOCK(pmap); + + //overflow + if (cur_page + add < cur_page) { + break; + } + + cur_page += add; + } + + + return ret; +} + +// search pv table and clone vas of given pa +static uint64_t pmap_pgtrace_clone_from_pvtable(pmap_paddr_t pa, vm_map_offset_t start_offset, vm_map_offset_t end_offset) +{ + uint64_t ret = 0; + unsigned long pai; + pv_entry_t **pvh; + pt_entry_t *ptep; + pmap_t pmap; + + typedef struct { + queue_chain_t chain; + pmap_t pmap; + vm_map_offset_t va; + } pmap_va_t; + + queue_head_t pmapvaq; + pmap_va_t *pmapva; + + queue_init(&pmapvaq); + + pai = pa_index(pa); + LOCK_PVH(pai); + pvh = pai_to_pvh(pai); + + // collect pmap/va pair from pvh + if (pvh_test_type(pvh, PVH_TYPE_PTEP)) { + ptep = pvh_ptep(pvh); + pmap = ptep_get_pmap(ptep); + + pmapva = (pmap_va_t *)kalloc(sizeof(pmap_va_t)); + pmapva->pmap = pmap; + pmapva->va = ptep_get_va(ptep); + + queue_enter_first(&pmapvaq, pmapva, pmap_va_t *, chain); + + } else if (pvh_test_type(pvh, PVH_TYPE_PVEP)) { + pv_entry_t *pvep; + + pvep = pvh_list(pvh); + while (pvep) { + ptep = pve_get_ptep(pvep); + pmap = ptep_get_pmap(ptep); + + pmapva = (pmap_va_t *)kalloc(sizeof(pmap_va_t)); + pmapva->pmap = pmap; + pmapva->va = ptep_get_va(ptep); + + queue_enter_first(&pmapvaq, pmapva, pmap_va_t *, chain); + + pvep = PVE_NEXT_PTR(pve_next(pvep)); + } + } + + UNLOCK_PVH(pai); + + // clone them while making sure mapping still exists + queue_iterate(&pmapvaq, pmapva, pmap_va_t *, chain) { + PMAP_LOCK(pmapva->pmap); + ptep = pmap_pte(pmapva->pmap, pmapva->va); + if (pte_to_pa(*ptep) == pa) { + if (pmap_pgtrace_enter_clone(pmapva->pmap, pmapva->va, start_offset, end_offset) == true) { + ret++; + } + } + PMAP_UNLOCK(pmapva->pmap); + + kfree(pmapva, sizeof(pmap_va_t)); + } + + return ret; +} + +// allocate a page info +static pmap_pgtrace_page_t *pmap_pgtrace_alloc_page(void) +{ + pmap_pgtrace_page_t *p; + queue_head_t *mapq; + queue_head_t *mappool; + queue_head_t *mapwaste; + pmap_pgtrace_map_t *map; + + p = kalloc(sizeof(pmap_pgtrace_page_t)); + assert(p); + + p->state = UNDEFINED; + + mapq = &(p->maps); + mappool = &(p->map_pool); + mapwaste = &(p->map_waste); + queue_init(mapq); + queue_init(mappool); + queue_init(mapwaste); + + for (int i = 0; i < PGTRACE_MAX_MAP; i++) { + vm_map_offset_t newcva; + pt_entry_t *cptep; + kern_return_t kr; + vm_map_entry_t entry; + + // get a clone va + vm_object_reference(kernel_object); + kr = vm_map_find_space(kernel_map, &newcva, vm_map_round_page(3*ARM_PGBYTES, PAGE_MASK), 0, 0, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_DIAG, &entry); + if (kr != KERN_SUCCESS) { + panic("%s VM couldn't find any space kr=%d\n", __func__, kr); + } + VME_OBJECT_SET(entry, kernel_object); + VME_OFFSET_SET(entry, newcva); + vm_map_unlock(kernel_map); + + // fill default clone page info and add to pool + map = kalloc(sizeof(pmap_pgtrace_map_t)); + for (int j = 0; j < 3; j ++) { + vm_map_offset_t addr = newcva + j * ARM_PGBYTES; + + // pre-expand pmap while preemption enabled + kr = pmap_expand(kernel_pmap, addr, 0, PMAP_TT_MAX_LEVEL); + if (kr != KERN_SUCCESS) { + panic("%s: pmap_expand(kernel_pmap, addr=%llx) returns kr=%d\n", __func__, addr, kr); + } + + cptep = pmap_pte(kernel_pmap, addr); + assert(cptep != NULL); + + map->cva[j] = addr; + map->cva_spte[j] = *cptep; + } + map->range.start = map->range.end = 0; + map->cloned = false; + queue_enter_first(mappool, map, pmap_pgtrace_map_t *, chain); + } + + return p; +} + +// free a page info +static void pmap_pgtrace_free_page(pmap_pgtrace_page_t *p) +{ + queue_head_t *mapq; + queue_head_t *mappool; + queue_head_t *mapwaste; + pmap_pgtrace_map_t *map; + + assert(p); + + mapq = &(p->maps); + mappool = &(p->map_pool); + mapwaste = &(p->map_waste); + + while (!queue_empty(mapq)) { + queue_remove_first(mapq, map, pmap_pgtrace_map_t *, chain); + kfree(map, sizeof(pmap_pgtrace_map_t)); + } + + while (!queue_empty(mappool)) { + queue_remove_first(mappool, map, pmap_pgtrace_map_t *, chain); + kfree(map, sizeof(pmap_pgtrace_map_t)); + } + + while (!queue_empty(mapwaste)) { + queue_remove_first(mapwaste, map, pmap_pgtrace_map_t *, chain); + kfree(map, sizeof(pmap_pgtrace_map_t)); + } + + kfree(p, sizeof(pmap_pgtrace_page_t)); +} + +// construct page infos with the given address range +int pmap_pgtrace_add_page(pmap_t pmap, vm_map_offset_t start, vm_map_offset_t end) +{ + int ret = 0; + pt_entry_t *ptep; + queue_head_t *q = &(pmap_pgtrace.pages); + bool ints; + vm_map_offset_t cur_page, end_page; + + if (start > end) { + kprintf("%s: invalid start=%llx > end=%llx\n", __func__, start, end); + return -1; + } + + PROF_START + + // add each page in given range + cur_page = arm_trunc_page(start); + end_page = arm_trunc_page(end); + while (cur_page <= end_page) { + pmap_paddr_t pa_page = 0; + uint64_t num_cloned = 0; + pmap_pgtrace_page_t *p = NULL, *newp; + bool free_newp = true; + pmap_pgtrace_page_state_t state; + + // do all allocations outside of spinlocks + newp = pmap_pgtrace_alloc_page(); + + // keep lock orders in pmap, kernel_pmap and pgtrace lock + if (pmap != NULL) { + PMAP_LOCK(pmap); + } + if (pmap != kernel_pmap) { + PMAP_LOCK(kernel_pmap); + } + + // addresses are physical if pmap is null + if (pmap == NULL) { + ptep = NULL; + pa_page = cur_page; + state = VA_UNDEFINED; + } else { + ptep = pmap_pte(pmap, cur_page); + if (ptep != NULL) { + pa_page = pte_to_pa(*ptep); + state = DEFINED; + } else { + state = PA_UNDEFINED; + } + } + + // search if we have a page info already + PMAP_PGTRACE_LOCK(&ints); + if (state != PA_UNDEFINED) { + p = pmap_pgtrace_find_page(pa_page); + } + + // add pre-allocated page info if nothing found + if (p == NULL) { + queue_enter_first(q, newp, pmap_pgtrace_page_t *, chain); + p = newp; + free_newp = false; + } + + // now p points what we want + p->state = state; + + queue_head_t *mapq = &(p->maps); + queue_head_t *mappool = &(p->map_pool); + pmap_pgtrace_map_t *map; + vm_map_offset_t start_offset, end_offset; + + // calculate trace offsets in the page + if (cur_page > start) { + start_offset = 0; + } else { + start_offset = start-cur_page; + } + if (cur_page == end_page) { + end_offset = end-end_page; + } else { + end_offset = ARM_PGBYTES-1; + } + + kprintf("%s: pmap=%p cur_page=%llx ptep=%p state=%d start_offset=%llx end_offset=%llx\n", __func__, pmap, cur_page, ptep, state, start_offset, end_offset); + + // fill map info + assert(!queue_empty(mappool)); + queue_remove_first(mappool, map, pmap_pgtrace_map_t *, chain); + if (p->state == PA_UNDEFINED) { + map->pmap = pmap; + map->ova = cur_page; + map->range.start = start_offset; + map->range.end = end_offset; + } else if (p->state == VA_UNDEFINED) { + p->pa = pa_page; + map->range.start = start_offset; + map->range.end = end_offset; + } else if (p->state == DEFINED) { + p->pa = pa_page; + map->pmap = pmap; + map->ova = cur_page; + map->range.start = start_offset; + map->range.end = end_offset; + } else { + panic("invalid p->state=%d\n", p->state); + } + + // not cloned yet + map->cloned = false; + queue_enter(mapq, map, pmap_pgtrace_map_t *, chain); + + // unlock locks + PMAP_PGTRACE_UNLOCK(&ints); + if (pmap != kernel_pmap) { + PMAP_UNLOCK(kernel_pmap); + } + if (pmap != NULL) { + PMAP_UNLOCK(pmap); + } + + // now clone it + if (pa_valid(pa_page)) { + num_cloned = pmap_pgtrace_clone_from_pvtable(pa_page, start_offset, end_offset); + } + if (pmap == NULL) { + num_cloned += pmap_pgtrace_clone_from_pa(kernel_pmap, pa_page, start_offset, end_offset); + } else { + num_cloned += pmap_pgtrace_clone_from_pa(pmap, pa_page, start_offset, end_offset); + } + + // free pre-allocations if we didn't add it to the q + if (free_newp) { + pmap_pgtrace_free_page(newp); + } + + if (num_cloned == 0) { + kprintf("%s: no mapping found for pa_page=%llx but will be added when a page entered\n", __func__, pa_page); + } + + ret += num_cloned; + + // overflow + if (cur_page + ARM_PGBYTES < cur_page) { + break; + } else { + cur_page += ARM_PGBYTES; + } + } + + PROF_END + + return ret; +} + +// delete page infos for given address range +int pmap_pgtrace_delete_page(pmap_t pmap, vm_map_offset_t start, vm_map_offset_t end) +{ + int ret = 0; + bool ints; + queue_head_t *q = &(pmap_pgtrace.pages); + pmap_pgtrace_page_t *p; + vm_map_offset_t cur_page, end_page; + + kprintf("%s start=%llx end=%llx\n", __func__, start, end); + + PROF_START + + pt_entry_t *ptep; + pmap_paddr_t pa_page; + + // remove page info from start to end + cur_page = arm_trunc_page(start); + end_page = arm_trunc_page(end); + while (cur_page <= end_page) { + p = NULL; + + if (pmap == NULL) { + pa_page = cur_page; + } else { + PMAP_LOCK(pmap); + ptep = pmap_pte(pmap, cur_page); + if (ptep == NULL) { + PMAP_UNLOCK(pmap); + goto cont; + } + pa_page = pte_to_pa(*ptep); + PMAP_UNLOCK(pmap); + } + + // remove all clones and validate + pmap_pgtrace_remove_all_clone(pa_page); + + // find page info and delete + PMAP_PGTRACE_LOCK(&ints); + p = pmap_pgtrace_find_page(pa_page); + if (p != NULL) { + queue_remove(q, p, pmap_pgtrace_page_t *, chain); + ret++; + } + PMAP_PGTRACE_UNLOCK(&ints); + + // free outside of locks + if (p != NULL) { + pmap_pgtrace_free_page(p); + } + +cont: + // overflow + if (cur_page + ARM_PGBYTES < cur_page) { + break; + } else { + cur_page += ARM_PGBYTES; + } + } + + PROF_END + + return ret; +} + +kern_return_t pmap_pgtrace_fault(pmap_t pmap, vm_map_offset_t va, arm_saved_state_t *ss) +{ + pt_entry_t *ptep; + pgtrace_run_result_t res; + pmap_pgtrace_page_t *p; + bool ints, found = false; + pmap_paddr_t pa; + + // Quick check if we are interested + ptep = pmap_pte(pmap, va); + if (!ptep || !(*ptep & ARM_PTE_PGTRACE)) { + return KERN_FAILURE; + } + + PMAP_PGTRACE_LOCK(&ints); + + // Check again since access is serialized + ptep = pmap_pte(pmap, va); + if (!ptep || !(*ptep & ARM_PTE_PGTRACE)) { + PMAP_PGTRACE_UNLOCK(&ints); + return KERN_FAILURE; + + } else if ((*ptep & ARM_PTE_TYPE_VALID) == ARM_PTE_TYPE_VALID) { + // Somehow this cpu's tlb has not updated + kprintf("%s Somehow this cpu's tlb has not updated?\n", __func__); + PMAP_UPDATE_TLBS(pmap, va, va+ARM_PGBYTES); + + PMAP_PGTRACE_UNLOCK(&ints); + return KERN_SUCCESS; + } + + // Find if this pa is what we are tracing + pa = pte_to_pa(*ptep); + + p = pmap_pgtrace_find_page(arm_trunc_page(pa)); + if (p == NULL) { + panic("%s Can't find va=%llx pa=%llx from tracing pages\n", __func__, va, pa); + } + + // find if pmap and va are also matching + queue_head_t *mapq = &(p->maps); + queue_head_t *mapwaste = &(p->map_waste); + pmap_pgtrace_map_t *map; + + queue_iterate(mapq, map, pmap_pgtrace_map_t *, chain) { + if (map->pmap == pmap && map->ova == arm_trunc_page(va)) { + found = true; + break; + } + } + + // if not found, search map waste as they are still valid + if (!found) { + queue_iterate(mapwaste, map, pmap_pgtrace_map_t *, chain) { + if (map->pmap == pmap && map->ova == arm_trunc_page(va)) { + found = true; + break; + } + } + } + + if (!found) { + panic("%s Can't find va=%llx pa=%llx from tracing pages\n", __func__, va, pa); + } + + // Decode and run it on the clone map + bzero(&res, sizeof(res)); + pgtrace_decode_and_run(*(uint32_t *)get_saved_state_pc(ss), // instruction + va, map->cva, // fault va and clone page vas + ss, &res); + + // write a log if in range + vm_map_offset_t offset = va - map->ova; + if (map->range.start <= offset && offset <= map->range.end) { + pgtrace_write_log(res); + } + + PMAP_PGTRACE_UNLOCK(&ints); + + // Return to next instruction + set_saved_state_pc(ss, get_saved_state_pc(ss) + sizeof(uint32_t)); + + return KERN_SUCCESS; +} +#endif + +boolean_t +pmap_enforces_execute_only( +#if (__ARM_VMSA__ == 7) + __unused +#endif + pmap_t pmap) +{ +#if (__ARM_VMSA__ > 7) + return (pmap != kernel_pmap); +#else + return FALSE; +#endif +} + +void +pmap_set_jit_entitled( + __unused pmap_t pmap) +{ + return; +} + +static kern_return_t +pmap_query_page_info_internal( + pmap_t pmap, + vm_map_offset_t va, + int *disp_p) +{ + int disp; + pmap_paddr_t pa; + int pai; + pt_entry_t *pte; + pv_entry_t **pv_h, *pve_p; + + if (pmap == PMAP_NULL || pmap == kernel_pmap) { + *disp_p = 0; + return KERN_INVALID_ARGUMENT; + } + + disp = 0; + + PMAP_LOCK(pmap); + + pte = pmap_pte(pmap, va); + if (pte == PT_ENTRY_NULL) { + goto done; + } + + pa = pte_to_pa(*pte); + if (pa == 0) { + if (ARM_PTE_IS_COMPRESSED(*pte)) { + disp |= PMAP_QUERY_PAGE_COMPRESSED; + if (*pte & ARM_PTE_COMPRESSED_ALT) { + disp |= PMAP_QUERY_PAGE_COMPRESSED_ALTACCT; + } + } + } else { + disp |= PMAP_QUERY_PAGE_PRESENT; + pai = (int) pa_index(pa); + if (!pa_valid(pa)) { + goto done; + } + LOCK_PVH(pai); + pv_h = pai_to_pvh(pai); + pve_p = PV_ENTRY_NULL; + if (pvh_test_type(pv_h, PVH_TYPE_PVEP)) { + pve_p = pvh_list(pv_h); + while (pve_p != PV_ENTRY_NULL && + pve_get_ptep(pve_p) != pte) { + pve_p = pvh_list(pv_h); + } + } + if (IS_ALTACCT_PAGE(pai, pve_p)) { + disp |= PMAP_QUERY_PAGE_ALTACCT; + } else if (IS_REUSABLE_PAGE(pai)) { + disp |= PMAP_QUERY_PAGE_REUSABLE; + } else if (IS_INTERNAL_PAGE(pai)) { + disp |= PMAP_QUERY_PAGE_INTERNAL; + } + UNLOCK_PVH(pai); + } + +done: + PMAP_UNLOCK(pmap); + *disp_p = disp; + return KERN_SUCCESS; +} + +kern_return_t +pmap_query_page_info( + pmap_t pmap, + vm_map_offset_t va, + int *disp_p) +{ + return pmap_query_page_info_internal(pmap, va, disp_p); +} + +kern_return_t +pmap_return_internal(__unused boolean_t do_panic, __unused boolean_t do_recurse) +{ + + return KERN_SUCCESS; +} + +kern_return_t +pmap_return(boolean_t do_panic, boolean_t do_recurse) +{ + return pmap_return_internal(do_panic, do_recurse); +} + +static void +pmap_footprint_suspend_internal( + vm_map_t map, + boolean_t suspend) +{ +#if DEVELOPMENT || DEBUG + if (suspend) { + map->pmap->footprint_suspended = TRUE; + map->pmap->footprint_was_suspended = TRUE; + } else { + map->pmap->footprint_suspended = FALSE; + } +#else /* DEVELOPMENT || DEBUG */ + (void) map; + (void) suspend; +#endif /* DEVELOPMENT || DEBUG */ +} +void +pmap_footprint_suspend( + vm_map_t map, + boolean_t suspend) +{ + pmap_footprint_suspend_internal(map, suspend); +} diff --git a/osfmk/arm/pmap.h b/osfmk/arm/pmap.h new file mode 100644 index 000000000..349ebc17f --- /dev/null +++ b/osfmk/arm/pmap.h @@ -0,0 +1,516 @@ +/* + * + * Copyright (c) 2007-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _ARM_PMAP_H_ +#define _ARM_PMAP_H_ 1 + +#include <mach_assert.h> + +#include <arm/proc_reg.h> +#if defined(__arm64__) +#include <arm64/proc_reg.h> +#endif + +/* + * Machine-dependent structures for the physical map module. + */ + +#ifndef ASSEMBLER + +#include <mach/kern_return.h> +#include <mach/machine/vm_types.h> +#include <mach/vm_prot.h> +#include <mach/vm_statistics.h> +#include <mach/machine/vm_param.h> +#include <kern/kern_types.h> +#include <kern/thread.h> +#include <kern/queue.h> + +/* Base address for low globals. */ +#define LOW_GLOBAL_BASE_ADDRESS 0xfffffff000000000ULL + +/* + * This indicates (roughly) where there is free space for the VM + * to use for the heap; this does not need to be precise. + */ +#if __ARM64_PMAP_SUBPAGE_L1__ && __ARM_16K_PG__ +#define KERNEL_PMAP_HEAP_RANGE_START VM_MIN_KERNEL_AND_KEXT_ADDRESS +#else +#define KERNEL_PMAP_HEAP_RANGE_START LOW_GLOBAL_BASE_ADDRESS +#endif + +#if defined(__arm64__) + +typedef uint64_t tt_entry_t; /* translation table entry type */ +#define TT_ENTRY_NULL ((tt_entry_t *) 0) + +typedef uint64_t pt_entry_t; /* page table entry type */ +#define PT_ENTRY_NULL ((pt_entry_t *) 0) + +typedef uint64_t pmap_paddr_t; /* physical address (not ppnum_t) */ + +#elif defined(__arm__) + +typedef uint32_t tt_entry_t; /* translation table entry type */ +#define PT_ENTRY_NULL ((pt_entry_t *) 0) + +typedef uint32_t pt_entry_t; /* page table entry type */ +#define TT_ENTRY_NULL ((tt_entry_t *) 0) + +typedef uint32_t pmap_paddr_t; /* physical address (not ppnum_t) */ + +#else +#error unknown arch +#endif + + +/* superpages */ +#define SUPERPAGE_NBASEPAGES 1 /* No superpages support */ + +/* + * Convert addresses to pages and vice versa. + * No rounding is used. + */ +#define arm_atop(x) (((vm_map_address_t)(x)) >> ARM_PGSHIFT) +#define arm_ptoa(x) (((vm_map_address_t)(x)) << ARM_PGSHIFT) + +/* + * Round off or truncate to the nearest page. These will work + * for either addresses or counts. (i.e. 1 byte rounds to 1 page + * bytes. + */ +#define arm_round_page(x) \ + ((((vm_map_address_t)(x)) + ARM_PGMASK) & ~ARM_PGMASK) +#define arm_trunc_page(x) (((vm_map_address_t)(x)) & ~ARM_PGMASK) + +/* Convert address offset to page table index */ +#define ptenum(a) ((((a) & ARM_TT_LEAF_INDEX_MASK) >> ARM_TT_LEAF_SHIFT)) + +/* + * For setups where the kernel page size does not match the hardware + * page size (assumably, the kernel page size must be a multiple of + * the hardware page size), we will need to determine what the page + * ratio is. + */ +#define PAGE_RATIO ((1 << PAGE_SHIFT) >> ARM_PGSHIFT) +#define TEST_PAGE_RATIO_4 (PAGE_RATIO == 4) + +#if (__ARM_VMSA__ <= 7) +#define NTTES (ARM_PGBYTES / sizeof(tt_entry_t)) +#define NPTES ((ARM_PGBYTES/4) /sizeof(pt_entry_t)) +#else +#define NTTES (ARM_PGBYTES / sizeof(tt_entry_t)) +#define NPTES (ARM_PGBYTES / sizeof(pt_entry_t)) +#endif + +extern void flush_mmu_tlb(void); +extern void flush_core_tlb(void); +#if defined(__arm64__) +extern void flush_mmu_tlb_allentries(uint64_t, uint64_t); +extern void flush_mmu_tlb_entry(uint64_t); +extern void flush_mmu_tlb_entries(uint64_t, uint64_t); +extern void flush_mmu_tlb_asid(uint64_t); +extern void flush_core_tlb_asid(uint64_t); +/* + * TLBI appers to only deal in 4KB page addresses, so give + * it an explicit shift of 12. + */ +#define TLBI_ADDR_SIZE 44 +#define TLBI_ADDR_MASK ((1ULL << TLBI_ADDR_SIZE) - 1) +#define TLBI_ADDR_SHIFT (12) +#define tlbi_addr(x) (((x) >> TLBI_ADDR_SHIFT) & TLBI_ADDR_MASK) + +#define TLBI_ASID_SHIFT 48 +#define TLBI_ASID_SIZE 16 +#define TLBI_ASID_MASK (((1ULL << TLBI_ASID_SIZE) - 1) << TLBI_ASID_SHIFT) +#define tlbi_asid(x) (((uint64_t)x << TLBI_ASID_SHIFT) & TLBI_ASID_MASK) +#else +extern void flush_mmu_tlb_entry(uint32_t); +extern void flush_mmu_tlb_entries(uint32_t, uint32_t); +extern void flush_mmu_tlb_mva_entries(uint32_t); +extern void flush_mmu_tlb_asid(uint32_t); +extern void flush_core_tlb_asid(uint32_t); +#endif +extern void flush_mmu_tlb_region(vm_offset_t va, unsigned length); + +#if defined(__arm64__) +extern uint64_t get_mmu_control(void); +extern void set_mmu_control(uint64_t); +extern uint64_t get_aux_control(void); +extern void set_aux_control(uint64_t); +extern void set_mmu_ttb(uint64_t); +extern void set_mmu_ttb_alternate(uint64_t); +extern uint64_t get_tcr(void); +extern void set_tcr(uint64_t); +#else +extern uint32_t get_mmu_control(void); +extern void set_mmu_control(uint32_t); +extern uint32_t get_aux_control(void); +extern void set_aux_control(uint32_t); +extern void set_mmu_ttb(pmap_paddr_t); +extern void set_mmu_ttb_alternate(pmap_paddr_t); +extern void set_context_id(uint32_t); +#endif + +extern pmap_paddr_t get_mmu_ttb(void); +extern pmap_paddr_t mmu_kvtop(vm_offset_t va); +extern pmap_paddr_t mmu_kvtop_wpreflight(vm_offset_t va); +extern pmap_paddr_t mmu_uvtop(vm_offset_t va); + +#if (__ARM_VMSA__ <= 7) +/* Convert address offset to translation table index */ +#define ttenum(a) ((a) >> ARM_TT_L1_SHIFT) + +/* Convert translation table index to user virtual address */ +#define tteitova(a) ((a) << ARM_TT_L1_SHIFT) + +#define pa_to_suptte(a) ((a) & ARM_TTE_SUPER_L1_MASK) +#define suptte_to_pa(p) ((p) & ARM_TTE_SUPER_L1_MASK) + +#define pa_to_sectte(a) ((a) & ARM_TTE_BLOCK_L1_MASK) +#define sectte_to_pa(p) ((p) & ARM_TTE_BLOCK_L1_MASK) + +#define pa_to_tte(a) ((a) & ARM_TTE_TABLE_MASK) +#define tte_to_pa(p) ((p) & ARM_TTE_TABLE_MASK) + +#define pa_to_pte(a) ((a) & ARM_PTE_PAGE_MASK) +#define pte_to_pa(p) ((p) & ARM_PTE_PAGE_MASK) +#define pte_increment_pa(p) ((p) += ptoa(1)) + +#define ARM_NESTING_SIZE_MIN ((PAGE_SIZE/0x1000)*4*ARM_TT_L1_SIZE) +#define ARM_NESTING_SIZE_MAX ((256*ARM_TT_L1_SIZE)) + +#else + +/* Convert address offset to translation table index */ +#define ttel0num(a) ((a & ARM_TTE_L0_MASK) >> ARM_TT_L0_SHIFT) +#define ttel1num(a) ((a & ARM_TTE_L1_MASK) >> ARM_TT_L1_SHIFT) +#define ttel2num(a) ((a & ARM_TTE_L2_MASK) >> ARM_TT_L2_SHIFT) + +#define pa_to_tte(a) ((a) & ARM_TTE_TABLE_MASK) +#define tte_to_pa(p) ((p) & ARM_TTE_TABLE_MASK) + +#define pa_to_pte(a) ((a) & ARM_PTE_MASK) +#define pte_to_pa(p) ((p) & ARM_PTE_MASK) +#define pte_to_ap(p) (((p) & ARM_PTE_APMASK) >> ARM_PTE_APSHIFT) +#define pte_increment_pa(p) ((p) += ptoa(1)) + +#define ARM_NESTING_SIZE_MIN ((PAGE_SIZE/ARM_PGBYTES)*ARM_TT_L2_SIZE) +#define ARM_NESTING_SIZE_MAX (0x0000000010000000ULL) + +#define TLBFLUSH_SIZE (ARM_TTE_MAX/((sizeof(unsigned int))*BYTE_SIZE)) + +#endif /* __ARM_VMSA__ <= 7 */ + +#define PMAP_GC_INFLIGHT 1 +#define PMAP_GC_WAIT 2 + +/* + * Convert translation/page table entry to kernel virtual address + */ +#define ttetokv(a) (phystokv(tte_to_pa(a))) +#define ptetokv(a) (phystokv(pte_to_pa(a))) + +struct pmap { + tt_entry_t *tte; /* translation table entries */ + pmap_paddr_t ttep; /* translation table physical */ + vm_map_address_t min; /* min address in pmap */ + vm_map_address_t max; /* max address in pmap */ + unsigned int asid; /* address space id */ + unsigned int vasid; /* Virtual address space id */ + unsigned int stamp; /* creation stamp */ + unsigned int wired; /* wired bits */ + volatile uint32_t ref_count; /* pmap reference count */ + unsigned int cpu_ref; /* number of cpus using pmap */ + unsigned int gc_status; /* gc status */ + ledger_t ledger; /* ledger tracking phys mappings */ + decl_simple_lock_data(,lock) /* lock on map */ + struct pmap_statistics stats; /* map statistics */ + queue_chain_t pmaps; /* global list of pmaps */ + tt_entry_t *tt_entry_free; /* free translation table entries */ + tt_entry_t *prev_tte; /* previous translation table */ + unsigned int tte_index_max; /* max tte index in translation table entries */ + boolean_t nx_enabled; /* no execute */ + boolean_t nested; /* is nested */ + boolean_t is_64bit; /* is 64bit */ + struct pmap *nested_pmap; /* nested pmap */ + vm_map_address_t nested_region_grand_addr; + vm_map_address_t nested_region_subord_addr; + vm_map_offset_t nested_region_size; + unsigned int *nested_region_asid_bitmap; + unsigned int nested_region_asid_bitmap_size; + +#if (__ARM_VMSA__ <= 7) + decl_simple_lock_data(,tt1_lock) /* lock on tt1 */ +#endif +#if MACH_ASSERT + int pmap_pid; + char pmap_procname[17]; +#endif /* MACH_ASSERT */ +#if DEVELOPMENT || DEBUG + boolean_t footprint_suspended; + boolean_t footprint_was_suspended; +#endif /* DEVELOPMENT || DEBUG */ +}; + +/* typedef struct pmap *pmap_t; */ +#define PMAP_NULL ((pmap_t) 0) + + +/* + * WIMG control + */ +#define VM_MEM_INNER 0x10 +#define VM_MEM_EARLY_ACK 0x20 + +#define VM_WIMG_DEFAULT (VM_MEM_COHERENT) +#define VM_WIMG_COPYBACK (VM_MEM_COHERENT) +#define VM_WIMG_INNERWBACK (VM_MEM_COHERENT | VM_MEM_INNER) +#define VM_WIMG_IO (VM_MEM_COHERENT | VM_MEM_NOT_CACHEABLE | VM_MEM_GUARDED) +#define VM_WIMG_POSTED (VM_MEM_COHERENT | VM_MEM_NOT_CACHEABLE | VM_MEM_GUARDED | VM_MEM_EARLY_ACK) +#define VM_WIMG_WTHRU (VM_MEM_WRITE_THROUGH | VM_MEM_COHERENT | VM_MEM_GUARDED) +#define VM_WIMG_WCOMB (VM_MEM_NOT_CACHEABLE | VM_MEM_COHERENT) + + +#if VM_DEBUG +extern int pmap_list_resident_pages( + pmap_t pmap, + vm_offset_t *listp, + int space + ); +#else /* #if VM_DEBUG */ +#define pmap_list_resident_pages(pmap, listp, space) (0) +#endif /* #if VM_DEBUG */ + +extern int copysafe(vm_map_address_t from, vm_map_address_t to, uint32_t cnt, int type, uint32_t *bytes_copied); + +/* globals shared between arm_vm_init and pmap */ +extern tt_entry_t *cpu_tte; /* first CPUs translation table (shared with kernel pmap) */ +extern pmap_paddr_t cpu_ttep; /* physical translation table addr */ + +#if __arm64__ +extern void *ropagetable_begin; +extern void *ropagetable_end; +#endif + +#if __arm64__ +extern tt_entry_t *invalid_tte; /* global invalid translation table */ +extern pmap_paddr_t invalid_ttep; /* physical invalid translation table addr */ +#endif + +#define PMAP_CONTEXT(pmap, thread) + +/* + * platform dependent Prototypes + */ +extern void pmap_switch_user_ttb(pmap_t pmap); +extern void pmap_bootstrap(vm_offset_t); +extern vm_map_address_t pmap_ptov(pmap_t, ppnum_t); +extern ppnum_t pmap_find_phys(pmap_t map, addr64_t va); +extern void pmap_set_pmap(pmap_t pmap, thread_t thread); +extern void pmap_collect(pmap_t pmap); +extern void pmap_gc(void); +#if defined(__arm64__) +extern vm_offset_t pmap_extract(pmap_t pmap, vm_map_offset_t va); +#endif + +/* + * Interfaces implemented as macros. + */ + +#define PMAP_SWITCH_USER(th, new_map, my_cpu) { \ + th->map = new_map; \ + pmap_set_pmap(vm_map_pmap(new_map), th); \ +} + +#define pmap_kernel() \ + (kernel_pmap) + +#define pmap_compressed(pmap) \ + ((pmap)->stats.compressed) + +#define pmap_resident_count(pmap) \ + ((pmap)->stats.resident_count) + +#define pmap_resident_max(pmap) \ + ((pmap)->stats.resident_max) + +#define MACRO_NOOP + +#define pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr) \ + MACRO_NOOP + +#define pmap_pageable(pmap, start, end, pageable) \ + MACRO_NOOP + +#define pmap_kernel_va(VA) \ + (((VA) >= VM_MIN_KERNEL_ADDRESS) && ((VA) <= VM_MAX_KERNEL_ADDRESS)) + +#define pmap_attribute(pmap,addr,size,attr,value) \ + (KERN_INVALID_ADDRESS) + +#define copyinmsg(from, to, cnt) \ + copyin(from, to, cnt) + +#define copyoutmsg(from, to, cnt) \ + copyout(from, to, cnt) + +extern pmap_paddr_t kvtophys(vm_offset_t va); + +extern vm_map_address_t pmap_map(vm_map_address_t va, vm_offset_t sa, vm_offset_t ea, vm_prot_t prot, unsigned int flags); +extern vm_map_address_t pmap_map_high_window_bd( vm_offset_t pa, vm_size_t len, vm_prot_t prot); +extern kern_return_t pmap_map_block(pmap_t pmap, addr64_t va, ppnum_t pa, uint32_t size, vm_prot_t prot, int attr, unsigned int flags); +extern void pmap_map_globals(void); + +#define PMAP_MAP_BD_DEVICE 0x1 +#define PMAP_MAP_BD_WCOMB 0x2 +#define PMAP_MAP_BD_POSTED 0x3 +#define PMAP_MAP_BD_MASK 0x3 + +extern vm_map_address_t pmap_map_bd_with_options(vm_map_address_t va, vm_offset_t sa, vm_offset_t ea, vm_prot_t prot, int32_t options); +extern vm_map_address_t pmap_map_bd(vm_map_address_t va, vm_offset_t sa, vm_offset_t ea, vm_prot_t prot); + +extern void pmap_init_pte_page(pmap_t, pt_entry_t *, vm_offset_t, unsigned int ttlevel, boolean_t alloc_ptd); +extern void pmap_init_pte_static_page(pmap_t, pt_entry_t *, pmap_paddr_t); + +extern boolean_t pmap_valid_address(pmap_paddr_t addr); +extern void pmap_disable_NX(pmap_t pmap); +extern void pmap_set_nested(pmap_t pmap); +extern vm_map_address_t pmap_create_sharedpage(void); +extern void pmap_insert_sharedpage(pmap_t pmap); +extern void pmap_protect_sharedpage(void); + +extern vm_offset_t pmap_cpu_windows_copy_addr(int cpu_num, unsigned int index); +extern unsigned int pmap_map_cpu_windows_copy(ppnum_t pn, vm_prot_t prot, unsigned int wimg_bits); +extern void pmap_unmap_cpu_windows_copy(unsigned int index); + +extern void pt_fake_zone_init(int); +extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_size_t *, + uint64_t *, int *, int *, int *); + +extern boolean_t pmap_valid_page(ppnum_t pn); + +#define MACHINE_PMAP_IS_EMPTY 1 +extern boolean_t pmap_is_empty(pmap_t pmap, vm_map_offset_t start, vm_map_offset_t end); + +#define ARM_PMAP_MAX_OFFSET_DEFAULT 0x01 +#define ARM_PMAP_MAX_OFFSET_MIN 0x02 +#define ARM_PMAP_MAX_OFFSET_MAX 0x04 +#define ARM_PMAP_MAX_OFFSET_DEVICE 0x08 +#define ARM_PMAP_MAX_OFFSET_JUMBO 0x10 + +#define ASID_SHIFT (11) /* Shift for the maximum virtual ASID value (2048) */ +#define MAX_ASID (1 << ASID_SHIFT) /* Max supported ASIDs (can be virtual) */ +#define ARM_ASID_SHIFT (8) /* Shift for the maximum ARM ASID value (256) */ +#define ARM_MAX_ASID (1 << ARM_ASID_SHIFT) /* Max ASIDs supported by the hardware */ +#define ASID_VIRT_BITS (ASID_SHIFT - ARM_ASID_SHIFT) /* The number of virtual bits in a virtaul ASID */ +#define NBBY 8 + +extern vm_map_offset_t pmap_max_offset(boolean_t is64, unsigned int option); + +boolean_t pmap_virtual_region(unsigned int region_select, vm_map_offset_t *startp, vm_map_size_t *size); + +boolean_t pmap_enforces_execute_only(pmap_t pmap); + +/* pmap dispatch indices */ +#define ARM_FAST_FAULT_INDEX 0 +#define ARM_FORCE_FAST_FAULT_INDEX 1 +#define MAPPING_FREE_PRIME_INDEX 2 +#define MAPPING_REPLENISH_INDEX 3 +#define PHYS_ATTRIBUTE_CLEAR_INDEX 4 +#define PHYS_ATTRIBUTE_SET_INDEX 5 +#define PMAP_BATCH_SET_CACHE_ATTRIBUTES_INDEX 6 +#define PMAP_CHANGE_WIRING_INDEX 7 +#define PMAP_CREATE_INDEX 8 +#define PMAP_DESTROY_INDEX 9 +#define PMAP_ENTER_OPTIONS_INDEX 10 +#define PMAP_EXTRACT_INDEX 11 +#define PMAP_FIND_PHYS_INDEX 12 +#define PMAP_INSERT_SHAREDPAGE_INDEX 13 +#define PMAP_IS_EMPTY_INDEX 14 +#define PMAP_MAP_CPU_WINDOWS_COPY_INDEX 15 +#define PMAP_MARK_PAGE_AS_PMAP_PAGE_INDEX 16 +#define PMAP_NEST_INDEX 17 +#define PMAP_PAGE_PROTECT_OPTIONS_INDEX 18 +#define PMAP_PROTECT_OPTIONS_INDEX 19 +#define PMAP_QUERY_PAGE_INFO_INDEX 20 +#define PMAP_QUERY_RESIDENT_INDEX 21 +#define PMAP_REFERENCE_INDEX 22 +#define PMAP_REMOVE_OPTIONS_INDEX 23 +#define PMAP_RETURN_INDEX 24 +#define PMAP_SET_CACHE_ATTRIBUTES_INDEX 25 +#define PMAP_SET_NESTED_INDEX 26 +#define PMAP_SET_PROCESS_INDEX 27 +#define PMAP_SWITCH_INDEX 28 +#define PMAP_SWITCH_USER_TTB_INDEX 29 +#define PMAP_UNHINT_KV_ADDR_INDEX 30 +#define PMAP_UNMAP_CPU_WINDOWS_COPY_INDEX 31 +#define PMAP_UNNEST_OPTIONS_INDEX 32 +#define PMAP_FOOTPRINT_SUSPEND_INDEX 33 +#define PMAP_CPU_DATA_INIT_INDEX 34 +#define PMAP_RELEASE_PAGES_TO_KERNEL_INDEX 35 + +#define MAX_PMAP_INDEX 36 + +#define PMAP_INVALID_CPU_NUM (~0U) + +struct pmap_cpu_data { + pmap_t cpu_user_pmap; + unsigned int cpu_number; + unsigned int cpu_user_pmap_stamp; + + /* + * This supports overloading of ARM ASIDs by the pmap. The field needs + * to be wide enough to cover all the virtual bits in a virtual ASID. + * With 256 physical ASIDs, 8-bit fields let us support up to 65536 + * Virtual ASIDs, minus all that would map on to 0 (as 0 is a global + * ASID). + * + * If we were to use bitfield shenanigans here, we could save a bit of + * memory by only having enough bits to support MAX_ASID. However, such + * an implementation would be more error prone. + */ + uint8_t cpu_asid_high_bits[ARM_MAX_ASID]; +}; + +typedef struct pmap_cpu_data pmap_cpu_data_t; + +/* Initialize the pmap per-CPU data for the current CPU. */ +extern void pmap_cpu_data_init(void); + +/* Get the pmap per-CPU data for the current CPU. */ +extern pmap_cpu_data_t * pmap_get_cpu_data(void); + +#define MARK_AS_PMAP_TEXT +#define MARK_AS_PMAP_DATA + +extern kern_return_t pmap_return(boolean_t do_panic, boolean_t do_recurse); + +#endif /* #ifndef ASSEMBLER */ + +#endif /* #ifndef _ARM_PMAP_H_ */ diff --git a/osfmk/arm/proc_reg.h b/osfmk/arm/proc_reg.h new file mode 100644 index 000000000..ca58e18ae --- /dev/null +++ b/osfmk/arm/proc_reg.h @@ -0,0 +1,1084 @@ +/* + * Copyright (c) 2007-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* CMU_ENDHIST */ +/* + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ + +/* + */ + +/* + * Processor registers for ARM + */ +#ifndef _ARM_PROC_REG_H_ +#define _ARM_PROC_REG_H_ + +#if defined (__arm64__) +#include <pexpert/arm64/board_config.h> +#elif defined (__arm__) +#include <pexpert/arm/board_config.h> +#endif + +#if defined (ARMA7) +#define __ARM_ARCH__ 7 +#define __ARM_SUB_ARCH__ CPU_ARCH_ARMv7k +#define __ARM_VMSA__ 7 +#define __ARM_VFP__ 3 +#if defined(__XNU_UP__) +#define __ARM_SMP__ 0 +#else +#define __ARM_SMP__ 1 +/* For SMP kernels, force physical aperture to be mapped at PTE level so that its mappings + * can be updated to reflect cache attribute changes on alias mappings. This prevents + * prefetched physical aperture cachelines from becoming dirty in L1 due to a write to + * an uncached alias mapping on the same core. Subsequent uncached writes from another + * core may not snoop this line, and the dirty line may end up being evicted later to + * effectively overwrite the uncached writes from other cores. */ +#define __ARM_PTE_PHYSMAP__ 1 +#endif +/* __ARMA7_SMP__ controls whether we are consistent with the A7 MP_CORE spec; needed because entities other than + * the xnu-managed processors may need to snoop our cache operations. + */ +#define __ARMA7_SMP__ 1 +#define __ARM_COHERENT_CACHE__ 1 +#define __ARM_L1_PTW__ 1 +#define __ARM_DEBUG__ 7 +#define __ARM_USER_PROTECT__ 1 +#define __ARM_TIME_TIMEBASE_ONLY__ 1 + +#elif defined (APPLECYCLONE) +#define __ARM_ARCH__ 8 +#define __ARM_VMSA__ 8 +#define __ARM_SMP__ 1 +#define __ARM_VFP__ 4 +#define __ARM_COHERENT_CACHE__ 1 +#define __ARM_COHERENT_IO__ 1 +#define __ARM_IC_NOALIAS_ICACHE__ 1 +#define __ARM_L1_PTW__ 1 +#define __ARM_DEBUG__ 7 +#define __ARM_ENABLE_SWAP__ 1 +#define __ARM_V8_CRYPTO_EXTENSIONS__ 1 +#define __ARM64_PMAP_SUBPAGE_L1__ 1 + +#elif defined (APPLETYPHOON) +#define __ARM_ARCH__ 8 +#define __ARM_VMSA__ 8 +#define __ARM_SMP__ 1 +#define __ARM_VFP__ 4 +#define __ARM_COHERENT_CACHE__ 1 +#define __ARM_COHERENT_IO__ 1 +#define __ARM_IC_NOALIAS_ICACHE__ 1 +#define __ARM_L1_PTW__ 1 +#define __ARM_DEBUG__ 7 +#define __ARM_ENABLE_SWAP__ 1 +#define __ARM_V8_CRYPTO_EXTENSIONS__ 1 +#define __ARM64_PMAP_SUBPAGE_L1__ 1 + +#elif defined (APPLETWISTER) +#define __ARM_ARCH__ 8 +#define __ARM_VMSA__ 8 +#define __ARM_SMP__ 1 +#define __ARM_VFP__ 4 +#define __ARM_COHERENT_CACHE__ 1 +#define __ARM_COHERENT_IO__ 1 +#define __ARM_IC_NOALIAS_ICACHE__ 1 +#define __ARM_L1_PTW__ 1 +#define __ARM_DEBUG__ 7 +#define __ARM_ENABLE_SWAP__ 1 +#define __ARM_V8_CRYPTO_EXTENSIONS__ 1 +#define __ARM_16K_PG__ 1 +#define __ARM64_TWO_LEVEL_PMAP__ 1 + +#elif defined (APPLEHURRICANE) +#define __ARM_ARCH__ 8 +#define __ARM_VMSA__ 8 +#define __ARM_SMP__ 1 +#define __ARM_VFP__ 4 +#define __ARM_COHERENT_CACHE__ 1 +#define __ARM_COHERENT_IO__ 1 +#define __ARM_IC_NOALIAS_ICACHE__ 1 +#define __ARM_L1_PTW__ 1 +#define __ARM_DEBUG__ 7 +#define __ARM_ENABLE_SWAP__ 1 +#define __ARM_V8_CRYPTO_EXTENSIONS__ 1 +#define __ARM_16K_PG__ 1 +#define __ARM64_PMAP_SUBPAGE_L1__ 1 +#define __ARM_GLOBAL_SLEEP_BIT__ 1 +#define __ARM_PAN_AVAILABLE__ 1 + +#else +#error processor not supported +#endif + +#if defined(ARM_BOARD_WFE_TIMEOUT_NS) +#define __ARM_ENABLE_WFE_ 1 +#else +#define __ARM_ENABLE_WFE_ 0 +#endif + +#define CONFIG_THREAD_GROUPS 0 + + +#ifdef XNU_KERNEL_PRIVATE + +#if __ARM_VFP__ +#define ARM_VFP_DEBUG 0 +#endif + +#endif + + + +/* + * FSR registers + * + * CPSR: Current Program Status Register + * SPSR: Saved Program Status Registers + * + * 31 30 29 28 27 24 19 16 9 8 7 6 5 4 0 + * +-----------------------------------------------------------+ + * | N| Z| C| V| Q|...| J|...|GE[3:0]|...| E| A| I| F| T| MODE | + * +-----------------------------------------------------------+ + */ + +/* + * Flags + */ +#define PSR_NF 0x80000000 /* Negative/Less than */ +#define PSR_ZF 0x40000000 /* Zero */ +#define PSR_CF 0x20000000 /* Carry/Borrow/Extend */ +#define PSR_VF 0x10000000 /* Overflow */ +#define PSR_QF 0x08000000 /* saturation flag (QADD ARMv5) */ + +/* + * Modified execution mode flags + */ +#define PSR_JF 0x01000000 /* Jazelle flag (BXJ ARMv5) */ +#define PSR_EF 0x00000200 /* mixed-endian flag (SETEND ARMv6) */ +#define PSR_AF 0x00000100 /* precise abort flag (ARMv6) */ +#define PSR_TF 0x00000020 /* thumb flag (BX ARMv4T) */ +#define PSR_TFb 5 /* thumb flag (BX ARMv4T) */ + +/* + * Interrupts + */ +#define PSR_IRQFb 7 /* IRQ : 0 = IRQ enable */ +#define PSR_IRQF 0x00000080 /* IRQ : 0 = IRQ enable */ +#define PSR_FIQF 0x00000040 /* FIQ : 0 = FIQ enable */ + +/* + * CPU mode + */ +#define PSR_USER_MODE 0x00000010 /* User mode */ +#define PSR_FIQ_MODE 0x00000011 /* FIQ mode */ +#define PSR_IRQ_MODE 0x00000012 /* IRQ mode */ +#define PSR_SVC_MODE 0x00000013 /* Supervisor mode */ +#define PSR_ABT_MODE 0x00000017 /* Abort mode */ +#define PSR_UND_MODE 0x0000001B /* Undefined mode */ + +#define PSR_MODE_MASK 0x0000001F +#define PSR_IS_KERNEL(psr) (((psr) & PSR_MODE_MASK) != PSR_USER_MODE) +#define PSR_IS_USER(psr) (((psr) & PSR_MODE_MASK) == PSR_USER_MODE) + +#define PSR_USERDFLT PSR_USER_MODE +#define PSR_USER_MASK (PSR_AF | PSR_IRQF | PSR_FIQF | PSR_MODE_MASK) +#define PSR_USER_SET PSR_USER_MODE + +#define PSR_INTMASK PSR_IRQF /* Interrupt disable */ + +/* + * FPEXC: Floating-Point Exception Register + */ + +#define FPEXC_EX 0x80000000 /* Exception status */ +#define FPEXC_EX_BIT 31 +#define FPEXC_EN 0x40000000 /* VFP : 1 = EN enable */ +#define FPEXC_EN_BIT 30 + + +/* + * FPSCR: Floating-point Status and Control Register + */ + +#define FPSCR_DN 0x02000000 /* Default NaN */ +#define FPSCR_FZ 0x01000000 /* Flush to zero */ + +#define FPSCR_DEFAULT FPSCR_DN | FPSCR_FZ + + +/* + * FSR registers + * + * IFSR: Instruction Fault Status Register + * DFSR: Data Fault Status Register + */ +#define FSR_ALIGN 0x00000001 /* Alignment */ +#define FSR_DEBUG 0x00000002 /* Debug (watch/break) */ +#define FSR_ICFAULT 0x00000004 /* Fault on instruction cache maintenance */ +#define FSR_SFAULT 0x00000005 /* Translation Section */ +#define FSR_PFAULT 0x00000007 /* Translation Page */ +#define FSR_SACCESS 0x00000003 /* Section access */ +#define FSR_PACCESS 0x00000006 /* Page Access */ +#define FSR_SDOM 0x00000009 /* Domain Section */ +#define FSR_PDOM 0x0000000B /* Domain Page */ +#define FSR_SPERM 0x0000000D /* Permission Section */ +#define FSR_PPERM 0x0000000F /* Permission Page */ +#define FSR_EXT 0x00001000 /* External (Implementation Defined Classification) */ + +#define FSR_MASK 0x0000040F /* Valid bits */ +#define FSR_ALIGN_MASK 0x0000040D /* Valid bits to check align */ + +#define DFSR_WRITE 0x00000800 /* write data abort fault */ + +#if defined (ARMA7) || defined (APPLE_ARM64_ARCH_FAMILY) + +#define TEST_FSR_VMFAULT(status) \ + (((status) == FSR_PFAULT) \ + || ((status) == FSR_PPERM) \ + || ((status) == FSR_SFAULT) \ + || ((status) == FSR_SPERM) \ + || ((status) == FSR_ICFAULT) \ + || ((status) == FSR_SACCESS) \ + || ((status) == FSR_PACCESS)) + +#else + +#error Incompatible CPU type configured + +#endif + +/* + * Cache configuration + */ + +#if defined (ARMA7) + +/* I-Cache */ +#define MMU_I_CLINE 5 /* cache line size as 1<<MMU_I_CLINE (32) */ + +/* D-Cache */ +#define MMU_CSIZE 15 /* cache size as 1<<MMU_CSIZE (32K) */ +#define MMU_CLINE 6 /* cache line size as 1<<MMU_CLINE (64) */ +#define MMU_NWAY 2 /* set associativity 1<<MMU_NWAY (4) */ +#define MMU_I7SET 6 /* cp15 c7 set incrementer 1<<MMU_I7SET */ +#define MMU_I7WAY 30 /* cp15 c7 way incrementer 1<<MMU_I7WAY */ + +#define MMU_SWAY (MMU_CSIZE - MMU_NWAY) /* set size 1<<MMU_SWAY */ +#define MMU_NSET (MMU_SWAY - MMU_CLINE) /* lines per way 1<<MMU_NSET */ + +#define __ARM_L2CACHE__ 1 + +#define L2_CSIZE __ARM_L2CACHE_SIZE_LOG__ /* cache size as 1<<MMU_CSIZE */ +#define L2_CLINE 6 /* cache line size as 1<<MMU_CLINE (64) */ +#define L2_NWAY 3 /* set associativity 1<<MMU_NWAY (8) */ +#define L2_I7SET 6 /* cp15 c7 set incrementer 1<<MMU_I7SET */ +#define L2_I7WAY 29 /* cp15 c7 way incrementer 1<<MMU_I7WAY */ +#define L2_I9WAY 29 /* cp15 c9 way incrementer 1<<MMU_I9WAY */ + +#define L2_SWAY (L2_CSIZE - L2_NWAY) /* set size 1<<MMU_SWAY */ +#define L2_NSET (L2_SWAY - L2_CLINE) /* lines per way 1<<MMU_NSET */ + +#elif defined (APPLECYCLONE) + +/* I-Cache */ +#define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ + +/* D-Cache */ +#define MMU_CSIZE 16 /* cache size as 1<<MMU_CSIZE (64K) */ +#define MMU_CLINE 6 /* cache line size as 1<<MMU_CLINE (64) */ +#define MMU_NWAY 1 /* set associativity 1<<MMU_NWAY (2) */ +#define MMU_I7SET 6 /* cp15 c7 set incrementer 1<<MMU_I7SET */ +#define MMU_I7WAY 31 /* cp15 c7 way incrementer 1<<MMU_I7WAY */ +#define MMU_I9WAY 31 /* cp15 c9 way incrementer 1<<MMU_I9WAY */ + +#define MMU_SWAY (MMU_CSIZE - MMU_NWAY) /* set size 1<<MMU_SWAY */ +#define MMU_NSET (MMU_SWAY - MMU_CLINE) /* lines per way 1<<MMU_NSET */ + +#define __ARM_L2CACHE__ 1 + +#define L2_CSIZE __ARM_L2CACHE_SIZE_LOG__ /* cache size as 1<<L2_CSIZE */ +#define L2_CLINE 6 /* cache line size as 1<<L2_CLINE (64) */ +#define L2_NWAY 3 /* set associativity 1<<L2_NWAY (8) */ +#define L2_I7SET 6 /* cp15 c7 set incrementer 1<<L2_I7SET */ +#define L2_I7WAY 29 /* cp15 c7 way incrementer 1<<L2_I7WAY */ +#define L2_I9WAY 29 /* cp15 c9 way incrementer 1<<L2_I9WAY */ + +#define L2_SWAY (L2_CSIZE - L2_NWAY) /* set size 1<<L2_SWAY */ +#define L2_NSET (L2_SWAY - L2_CLINE) /* lines per way 1<<L2_NSET */ + +#elif defined (APPLETYPHOON) + +/* I-Cache */ +#define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ + +/* D-Cache */ +#define MMU_CSIZE 16 /* cache size as 1<<MMU_CSIZE (64K) */ +#define MMU_CLINE 6 /* cache line size as 1<<MMU_CLINE (64) */ +#define MMU_NWAY 1 /* set associativity 1<<MMU_NWAY (2) */ +#define MMU_I7SET 6 /* cp15 c7 set incrementer 1<<MMU_I7SET */ +#define MMU_I7WAY 31 /* cp15 c7 way incrementer 1<<MMU_I7WAY */ +#define MMU_I9WAY 31 /* cp15 c9 way incrementer 1<<MMU_I9WAY */ + +#define MMU_SWAY (MMU_CSIZE - MMU_NWAY) /* set size 1<<MMU_SWAY */ +#define MMU_NSET (MMU_SWAY - MMU_CLINE) /* lines per way 1<<MMU_NSET */ + +#define __ARM_L2CACHE__ 1 + +#define L2_CSIZE __ARM_L2CACHE_SIZE_LOG__ /* cache size as 1<<L2_CSIZE */ +#define L2_CLINE 6 /* cache line size as 1<<L2_CLINE (64) */ +#define L2_NWAY 3 /* set associativity 1<<L2_NWAY (8) */ +#define L2_I7SET 6 /* cp15 c7 set incrementer 1<<L2_I7SET */ +#define L2_I7WAY 29 /* cp15 c7 way incrementer 1<<L2_I7WAY */ +#define L2_I9WAY 29 /* cp15 c9 way incrementer 1<<L2_I9WAY */ + +#define L2_SWAY (L2_CSIZE - L2_NWAY) /* set size 1<<L2_SWAY */ +#define L2_NSET (L2_SWAY - L2_CLINE) /* lines per way 1<<L2_NSET */ + +#elif defined (APPLETWISTER) + +/* I-Cache */ +#define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ + +/* D-Cache */ +#define MMU_CSIZE 16 /* cache size as 1<<MMU_CSIZE (64K) */ +#define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ +#define MMU_NWAY 2 /* set associativity 1<<MMU_NWAY (4) */ +#define MMU_I7SET 6 /* cp15 c7 set incrementer 1<<MMU_I7SET */ +#define MMU_I7WAY 30 /* cp15 c7 way incrementer 1<<MMU_I7WAY */ +#define MMU_I9WAY 30 /* cp15 c9 way incrementer 1<<MMU_I9WAY */ + +#define MMU_SWAY (MMU_CSIZE - MMU_NWAY) /* set size 1<<MMU_SWAY */ +#define MMU_NSET (MMU_SWAY - MMU_CLINE) /* lines per way 1<<MMU_NSET */ + +/* L2-Cache */ +#define __ARM_L2CACHE__ 1 + +/* + * For reasons discussed in the platform expert code, we round the reported + * L2 size to 4MB, and adjust the other parameters accordingly. + */ +#define L2_CSIZE __ARM_L2CACHE_SIZE_LOG__ /* cache size as 1<<L2_CSIZE */ +#define L2_CLINE 6 /* cache line size as 1<<L2_CSIZE (64) */ +#define L2_NWAY 4 /* set associativity as 1<<L2_CLINE (16, is actually 12) */ +#define L2_I7SET 6 /* cp15 c7 set incrementer 1<<L2_I7SET */ +#define L2_I7WAY 28 /* cp15 c7 way incrementer 1<<L2_I7WAY */ +#define L2_I9WAY 28 /* cp15 c9 way incremenber 1<<L2_I9WAY */ + +#define L2_SWAY (L2_CSIZE - L2_NWAY) /* set size 1<<L2_SWAY */ +#define L2_NSET (L2_SWAY - L2_CLINE) /* lines per way 1<<L2_NSET */ + +#elif defined (APPLEHURRICANE) + +/* I-Cache */ +#define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ + +/* D-Cache */ +#define MMU_CSIZE 16 /* cache size as 1<<MMU_CSIZE (64K) */ +#define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ +#define MMU_NWAY 2 /* set associativity 1<<MMU_NWAY (4) */ +#define MMU_I7SET 6 /* cp15 c7 set incrementer 1<<MMU_I7SET */ +#define MMU_I7WAY 30 /* cp15 c7 way incrementer 1<<MMU_I7WAY */ +#define MMU_I9WAY 30 /* cp15 c9 way incrementer 1<<MMU_I9WAY */ + +#define MMU_SWAY (MMU_CSIZE - MMU_NWAY) /* set size 1<<MMU_SWAY */ +#define MMU_NSET (MMU_SWAY - MMU_CLINE) /* lines per way 1<<MMU_NSET */ + +/* L2-Cache */ +#define __ARM_L2CACHE__ 1 + +/* + * For reasons discussed in the platform expert code, we round the reported + * L2 size to 4MB, and adjust the other parameters accordingly. + */ +#define L2_CSIZE __ARM_L2CACHE_SIZE_LOG__ /* cache size as 1<<L2_CSIZE */ +#define L2_CLINE 6 /* cache line size as 1<<L2_CSIZE (64) */ +#define L2_NWAY 4 /* set associativity as 1<<L2_CLINE (16, is actually 12) */ +#define L2_I7SET 6 /* cp15 c7 set incrementer 1<<L2_I7SET */ +#define L2_I7WAY 28 /* cp15 c7 way incrementer 1<<L2_I7WAY */ +#define L2_I9WAY 28 /* cp15 c9 way incremenber 1<<L2_I9WAY */ + +#define L2_SWAY (L2_CSIZE - L2_NWAY) /* set size 1<<L2_SWAY */ +#define L2_NSET (L2_SWAY - L2_CLINE) /* lines per way 1<<L2_NSET */ + +#else +#error processor not supported +#endif + + +#if (__ARM_VMSA__ <= 7) + +/* + * SCTLR: System Control Register + */ +/* + * System Control Register (SCTLR) + * + * 31 30 29 28 27 25 24 22 21 20 19 17 15 14 13 12 11 10 5 2 1 0 + * +-+--+---+---+----+-+--+--+--+--+----+---+-+--+-+-+--+--+--+--+--+---+-+------+--+-+-+-+ + * |0|TE|AFE|TRE|NMFI|0|EE|VE|11|FI|UWXN|WXN|1|HA|1|0|RR| V| I| Z|SW|000|1|C15BEN|11|C|A|M| + * +-+--+---+---+----+-+--+--+--+--+----+---+-+--+-+-+--+--+--+--+--+---+-+------+--+-+-+-+ + * + * TE Thumb Exception enable + * AFE Access flag enable + * TRE TEX remap enable + * NMFI Non-maskable FIQ (NMFI) support + * EE Exception Endianness + * VE Interrupt Vectors Enable + * FI Fast interrupts configuration enable + * ITD IT Disable + * UWXN Unprivileged write permission implies PL1 XN + * WXN Write permission implies XN + * HA Hardware Access flag enable + * RR Round Robin select + * V High exception vectors + * I Instruction cache enable + * Z Branch prediction enable + * SW SWP/SWPB enable + * C15BEN CP15 barrier enable + * C Cache enable + * A Alignment check enable + * M MMU enable + */ + +#define SCTLR_RESERVED 0x82DD8394 + +#define SCTLR_ENABLE 0x00000001 /* MMU enable */ +#define SCTLR_ALIGN 0x00000002 /* Alignment check enable */ +#define SCTLR_DCACHE 0x00000004 /* Data or Unified Cache enable */ +#define SCTLR_BEN 0x00000040 /* CP15 barrier enable */ +#define SCTLR_SW 0x00000400 /* SWP/SWPB Enable */ +#define SCTLR_PREDIC 0x00000800 /* Branch prediction enable */ +#define SCTLR_ICACHE 0x00001000 /* Instruction cache enabled. */ +#define SCTLR_HIGHVEC 0x00002000 /* Vector table at 0xffff0000 */ +#define SCTLR_RROBIN 0x00004000 /* Round Robin replacement */ +#define SCTLR_HA 0x00020000 /* Hardware Access flag enable */ +#define SCTLR_NMFI 0x08000000 /* Non-maskable FIQ */ +#define SCTLR_TRE 0x10000000 /* TEX remap enable */ +#define SCTLR_AFE 0x20000000 /* Access flag enable */ +#define SCTLR_TE 0x40000000 /* Thumb Exception enable */ + +#define SCTLR_DEFAULT (SCTLR_AFE|SCTLR_TRE|SCTLR_HIGHVEC|SCTLR_ICACHE|SCTLR_PREDIC|SCTLR_DCACHE|SCTLR_ENABLE) + + +/* + * PRRR: Primary Region Remap Register + * + * 31 24 20 19 18 17 16 0 + * +---------------------------------------------------------------+ + * | NOSn | Res |NS1|NS0|DS1|DS0| TRn | + * +---------------------------------------------------------------+ + */ + +#define PRRR_NS1 0x00080000 +#define PRRR_NS0 0x00040000 +#define PRRR_DS1 0x00020000 +#define PRRR_DS0 0x00010000 +#define PRRR_NOSn_ISH(region) (0x1<<((region)+24)) + +#if defined (ARMA7) +#define PRRR_SETUP (0x1F08022A) +#else +#error processor not supported +#endif + +/* + * NMRR, Normal Memory Remap Register + * + * 30 28 26 24 22 20 18 16 14 12 10 8 6 4 2 0 + * +---------------------------------------------------------------+ + * |OR7|OR6|OR5|OR4|OR3|OR2|OR1|OR0|IR7|IR6|IR5|IR4|IR3|IR2|IR1|IR0| + * +---------------------------------------------------------------+ + */ + +#define NMRR_DISABLED 0x0 /* Non-cacheable */ +#define NMRR_WRITEBACK 0x1 /* Write-Back, Write-Allocate */ +#define NMRR_WRITETHRU 0x2 /* Write-Through, no Write-Allocate */ +#define NMRR_WRITEBACKNO 0x3 /* Write-Back, no Write-Allocate */ + +#if defined (ARMA7) +#define NMRR_SETUP (0x01210121) +#else +#error processor not supported +#endif + +/* + * TTBR: Translation Table Base Register + * + */ + +#define TTBR_IRGN_DISBALED 0x00000000 /* inner non-cacheable */ +#define TTBR_IRGN_WRITEBACK 0x00000040 /* inner write back and allocate */ +#define TTBR_IRGN_WRITETHRU 0x00000001 /* inner write thru */ +#define TTBR_IRGN_WRITEBACKNO 0x00000041 /* inner write back no allocate */ + +#define TTBR_RGN_DISBALED 0x00000000 /* outer non-cacheable */ +#define TTBR_RGN_WRITEBACK 0x00000008 /* outer write back and allocate */ +#define TTBR_RGN_WRITETHRU 0x00000010 /* outer write thru outer cache */ +#define TTBR_RGN_WRITEBACKNO 0x00000018 /* outer write back no allocate */ + +#define TTBR_SHARED 0x00000002 /* Shareable memory atribute */ +#define TTBR_SHARED_NOTOUTER 0x00000020 /* Outer not shareable memory atribute */ + +#if defined (ARMA7) +#define TTBR_SETUP (TTBR_RGN_WRITEBACK|TTBR_IRGN_WRITEBACK|TTBR_SHARED) +#else +#error processor not supported +#endif + +/* + * TTBCR: Translation Table Base Control register + * + * 31 3 2 0 + * +----------+ + * | zero | N | + * +----------+ + * + * If N=0, always use translation table base register 0. Otherwise, if + * bits [31:32-N] of the address are all zero use base register 0. Otherwise, + * use base register 1. + * + * Reading from this register also returns the page table boundary for TTB0. + * Writing to it updates the boundary for TTB0. (0=16KB, 1=8KB, 2=4KB, etc...) + */ + +#define TTBCR_N_1GB_TTB0 0x2 /* 1 GB TTB0, 3GB TTB1 */ +#define TTBCR_N_2GB_TTB0 0x1 /* 2 GB TTB0, 2GB TTB1 */ +#define TTBCR_N_4GB_TTB0 0x0 /* 4 GB TTB0 */ +#define TTBCR_N_MASK 0x3 + + + +/* + * ARM Page Granule + */ +#define ARM_PGSHIFT 12 +#define ARM_PGBYTES (1 << ARM_PGSHIFT) +#define ARM_PGMASK (ARM_PGBYTES-1) + +/* + * DACR: Domain Access Control register + */ + +#define DAC_FAULT 0x0 /* invalid domain - everyone loses */ +#define DAC_CLIENT 0x1 /* client domain - use AP bits */ +#define DAC_RESERVE 0x2 /* reserved domain - undefined */ +#define DAC_MANAGER 0x3 /* manager domain - all access */ +#define DACR_SET(dom, x) ((x)<<((dom)<<1)) + + +#define ARM_DOM_DEFAULT 0 /* domain that forces AP use */ +#define ARM_DAC_SETUP 0x1 + +/* + * ARM 2-level Page Table support + */ + +/* + * Memory Attribute Index + */ +#define CACHE_ATTRINDX_WRITEBACK 0x0 /* cache enabled, buffer enabled */ +#define CACHE_ATTRINDX_WRITECOMB 0x1 /* no cache, buffered writes */ +#define CACHE_ATTRINDX_WRITETHRU 0x2 /* cache enabled, buffer disabled */ +#define CACHE_ATTRINDX_DISABLE 0x3 /* no cache, no buffer */ +#define CACHE_ATTRINDX_INNERWRITEBACK 0x4 /* inner cache enabled, buffer enabled, write allocate */ +#define CACHE_ATTRINDX_POSTED CACHE_ATTRINDX_DISABLE +#define CACHE_ATTRINDX_DEFAULT CACHE_ATTRINDX_WRITEBACK + + +/* + * Access protection bit values + */ +#define AP_RWNA 0x0 /* priv=read-write, user=no-access */ +#define AP_RWRW 0x1 /* priv=read-write, user=read-write */ +#define AP_RONA 0x2 /* priv=read-only , user=no-access */ +#define AP_RORO 0x3 /* priv=read-only , user=read-only */ + +/* + * L1 Translation table + * + * Each translation table is up to 16KB + * 4096 32-bit entries of 1MB of address space. + */ + +#define ARM_TT_L1_SIZE 0x00100000 /* size of area covered by a tte */ +#define ARM_TT_L1_OFFMASK 0x000FFFFF /* offset within an L1 entry */ +#define ARM_TT_L1_TABLE_OFFMASK 0x000FFFFF /* offset within an L1 entry */ +#define ARM_TT_L1_BLOCK_OFFMASK 0x000FFFFF /* offset within an L1 entry */ +#define ARM_TT_L1_SUPER_OFFMASK 0x00FFFFFF /* offset within an L1 entry */ +#define ARM_TT_L1_SHIFT 20 /* page descriptor shift */ +#define ARM_TT_L1_INDEX_MASK 0xfff00000 /* mask for getting index in L1 table from virtual address */ + +#define ARM_TT_L1_PT_SIZE (4 * ARM_TT_L1_SIZE) /* 4 L1 table entries required to consume 1 L2 pagetable page */ +#define ARM_TT_L1_PT_OFFMASK (ARM_TT_L1_PT_SIZE - 1) + +/* + * L2 Translation table + * + * Each translation table is up to 1KB + * 4096 32-bit entries of 1MB (2^30) of address space. + */ + +#define ARM_TT_L2_SIZE 0x00001000 /* size of area covered by a tte */ +#define ARM_TT_L2_OFFMASK 0x00000FFF /* offset within an L2 entry */ +#define ARM_TT_L2_SHIFT 12 /* page descriptor shift */ +#define ARM_TT_L2_INDEX_MASK 0x000ff000 /* mask for getting index in L2 table from virtual address */ + +/* + * Convenience definitions for: + * ARM_TT_LEAF: The last level of the configured page table format. + * ARM_TT_TWIG: The second to last level of the configured page table format. + * + * My apologies to any botanists who may be reading this. + */ +#define ARM_TT_LEAF_SIZE ARM_TT_L2_SIZE +#define ARM_TT_LEAF_OFFMASK ARM_TT_L2_OFFMASK +#define ARM_TT_LEAF_SHIFT ARM_TT_L2_SHIFT +#define ARM_TT_LEAF_INDEX_MASK ARM_TT_L2_INDEX_MASK + +#define ARM_TT_TWIG_SIZE ARM_TT_L1_SIZE +#define ARM_TT_TWIG_OFFMASK ARM_TT_L1_OFFMASK +#define ARM_TT_TWIG_SHIFT ARM_TT_L1_SHIFT +#define ARM_TT_TWIG_INDEX_MASK ARM_TT_L1_INDEX_MASK + +/* + * Level 1 Translation Table Entry + * + * page table entry + * + * 31 10 9 8 5 4 2 0 + * +----------------------+-+----+--+--+--+ + * | page table base addr | |dom |XN|00|01| + * +----------------------+-+----+--+--+--+ + * + * direct (1MB) section entry + * + * 31 20 18 15 12 10 9 8 5 4 2 0 + * +------------+--+-+-+-+---+--+-+----+--+--+--+ + * | base addr |00|G|S|A|TEX|AP| |dom |XN|CB|10| + * +------------+--+-+-+-+---+--+-+----+--+--+--+ + * + * super (16MB) section entry + * + * 31 24 23 18 15 12 10 9 8 5 4 2 0 + * +---------+------+-+-+-+---+--+-+----+--+--+--+ + * |base addr|000001|G|S|A|TEX|AP| |dom |XN|CB|10| + * +---------+------+-+-+-+---+--+-+----+--+--+--+ + * + * where: + * 'G' is the notGlobal bit + * 'S' is the shared bit + * 'A' in the access permission extension (APX) bit + * 'TEX' remap register control bits + * 'AP' is the access protection + * 'dom' is the domain for the translation + * 'XN' is the eXecute Never bit + * 'CB' is the cache/buffer attribute + */ + +#define ARM_TTE_EMPTY 0x00000000 /* unasigned entry */ + +#define ARM_TTE_TYPE_FAULT 0x00000000 /* fault entry type */ +#define ARM_TTE_TYPE_TABLE 0x00000001 /* page table type */ +#define ARM_TTE_TYPE_BLOCK 0x00000002 /* section entry type */ +#define ARM_TTE_TYPE_MASK 0x00000003 /* mask for extracting the type */ + +#define ARM_TTE_BLOCK_NGSHIFT 17 +#define ARM_TTE_BLOCK_NG_MASK 0x00020000 /* mask to determine notGlobal bit */ +#define ARM_TTE_BLOCK_NG 0x00020000 /* value for a per-process mapping */ + +#define ARM_TTE_BLOCK_SHSHIFT 16 +#define ARM_TTE_BLOCK_SH_MASK 0x00010000 /* shared (SMP) mapping mask */ +#define ARM_TTE_BLOCK_SH 0x00010000 /* shared (SMP) mapping */ + +#define ARM_TTE_BLOCK_CBSHIFT 2 +#define ARM_TTE_BLOCK_CB(x) ((x) << ARM_TTE_BLOCK_CBSHIFT) +#define ARM_TTE_BLOCK_CB_MASK (3<< ARM_TTE_BLOCK_CBSHIFT) + +#define ARM_TTE_BLOCK_AP0SHIFT 10 +#define ARM_TTE_BLOCK_AP0 (1<<ARM_TTE_BLOCK_AP0SHIFT) +#define ARM_TTE_BLOCK_AP0_MASK (1<<ARM_TTE_BLOCK_AP0SHIFT) + +#define ARM_TTE_BLOCK_AP1SHIFT 11 +#define ARM_TTE_BLOCK_AP1 (1<<ARM_TTE_BLOCK_AP1SHIFT) +#define ARM_TTE_BLOCK_AP1_MASK (1<<ARM_TTE_BLOCK_AP1SHIFT) + +#define ARM_TTE_BLOCK_AP2SHIFT 15 +#define ARM_TTE_BLOCK_AP2 (1<<ARM_TTE_BLOCK_AP2SHIFT) +#define ARM_TTE_BLOCK_AP2_MASK (1<<ARM_TTE_BLOCK_AP2SHIFT) + + + /* access protections */ +#define ARM_TTE_BLOCK_AP(ap) ((((ap)&0x1)<<ARM_TTE_BLOCK_AP1SHIFT) \ + | ((((ap)>>1)&0x1)<<ARM_TTE_BLOCK_AP2SHIFT)) + + /* mask access protections */ +#define ARM_TTE_BLOCK_APMASK (ARM_TTE_BLOCK_AP1_MASK \ + | ARM_TTE_BLOCK_AP2_MASK) + +#define ARM_TTE_BLOCK_AF ARM_TTE_BLOCK_AP0 /* value for access */ +#define ARM_TTE_BLOCK_AFMASK ARM_TTE_BLOCK_AP0_MASK /* access mask */ + +#define ARM_TTE_TABLE_MASK 0xFFFFFC00 /* mask for a L2 page table entry */ +#define ARM_TTE_TABLE_SHIFT 10 /* shift for L2 page table phys address */ + +#define ARM_TTE_BLOCK_L1_MASK 0xFFF00000 /* mask to extract phys address from L1 section entry */ +#define ARM_TTE_BLOCK_L1_SHIFT 20 /* shift for 1MB section phys address */ + +#define ARM_TTE_SUPER_L1_MASK 0xFF000000 /* mask to extract phys address from L1 super entry */ +#define ARM_TTE_SUPER_L1_SHIFT 24 /* shift for 16MB section phys address */ + +#define ARM_TTE_BLOCK_SUPER 0x00040000 /* make section a 16MB section */ +#define ARM_TTE_BLOCK_SUPER_MASK 0x00F40000 /* make section a 16MB section */ + +#define ARM_TTE_BLOCK_NXSHIFT 4 +#define ARM_TTE_BLOCK_NX 0x00000010 /* section is no execute */ +#define ARM_TTE_BLOCK_NX_MASK 0x00000010 /* mask for extracting no execute bit */ +#define ARM_TTE_BLOCK_PNX ARM_TTE_BLOCK_NX + +#define ARM_TTE_BLOCK_TEX0SHIFT 12 +#define ARM_TTE_BLOCK_TEX0 (1<<ARM_TTE_BLOCK_TEX0SHIFT) +#define ARM_TTE_BLOCK_TEX0_MASK (1<<ARM_TTE_BLOCK_TEX0SHIFT) + +#define ARM_TTE_BLOCK_TEX1SHIFT 13 +#define ARM_TTE_BLOCK_TEX1 (1<<ARM_TTE_BLOCK_TEX1SHIFT) +#define ARM_TTE_BLOCK_TEX1_MASK (1<<ARM_TTE_BLOCK_TEX1SHIFT) + +#define ARM_TTE_BLOCK_TEX2SHIFT 14 +#define ARM_TTE_BLOCK_TEX2 (1<<ARM_TTE_BLOCK_TEX2SHIFT) +#define ARM_TTE_BLOCK_TEX2_MASK (1<<ARM_TTE_BLOCK_TEX2SHIFT) + + + /* mask memory attributes index */ +#define ARM_TTE_BLOCK_ATTRINDX(i) ((((i)&0x3)<<ARM_TTE_BLOCK_CBSHIFT) \ + | ((((i)>>2)&0x1)<<ARM_TTE_BLOCK_TEX0SHIFT)) + + /* mask memory attributes index */ +#define ARM_TTE_BLOCK_ATTRINDXMASK (ARM_TTE_BLOCK_CB_MASK \ + | ARM_TTE_BLOCK_TEX0_MASK) + + +/* + * Level 2 Page table entries + * + * The following page table entry types are possible: + * + * fault page entry + * 31 2 0 + * +----------------------------------------+--+ + * | ignored |00| + * +----------------------------------------+--+ + * + * large (64KB) page entry + * 31 16 15 12 9 6 4 3 2 0 + * +----------------+--+---+-+-+-+---+--+-+-+--+ + * | base phys addr |XN|TEX|G|S|A|000|AP|C|B|01| + * +----------------+--+---+-+-+-+---+--+-+-+--+ + * + * small (4KB) page entry + * 31 12 9 6 4 3 2 1 0 + * +-----------------------+-+-+-+---+--+-+-+-+--+ + * | base phys addr |G|S|A|TEX|AP|C|B|1|XN| + * +-----------------------+-+-+-+---+--+-+-+-+--+ + * + * also where: + * 'XN' is the eXecute Never bit + * 'G' is the notGlobal (process-specific) bit + * 'S' is the shared bit + * 'A' in the access permission extension (ATX) bit + * 'TEX' remap register control bits + * 'AP' is the access protection + * 'dom' is the domain for the translation + * 'C' is the cache attribute + * 'B' is the write buffer attribute + */ + +#define PTE_SHIFT 2 /* shift width of a pte (sizeof(pte) == (1 << PTE_SHIFT)) */ +#define PTE_PGENTRIES (1024 >> PTE_SHIFT) /* number of ptes per page */ + +#define ARM_PTE_EMPTY 0x00000000 /* unasigned - invalid entry */ + +/* markers for (invalid) PTE for a page sent to compressor */ +#define ARM_PTE_COMPRESSED ARM_PTE_TEX1 /* compressed... */ +#define ARM_PTE_COMPRESSED_ALT ARM_PTE_TEX2 /* ... and was "alt_acct" */ +#define ARM_PTE_COMPRESSED_MASK (ARM_PTE_COMPRESSED | ARM_PTE_COMPRESSED_ALT) +#define ARM_PTE_IS_COMPRESSED(x) \ + ((((x) & 0x3) == 0) && /* PTE is not valid... */ \ + ((x) & ARM_PTE_COMPRESSED) && /* ...has "compressed" marker" */ \ + ((!((x) & ~ARM_PTE_COMPRESSED_MASK)) || /* ...no other bits */ \ + (panic("compressed PTE %p 0x%x has extra bits 0x%x: corrupted?", \ + &(x), (x), (x) & ~ARM_PTE_COMPRESSED_MASK), FALSE))) + +#define ARM_PTE_TYPE_FAULT 0x00000000 /* fault entry type */ +#define ARM_PTE_TYPE 0x00000002 /* small page entry type */ +#define ARM_PTE_TYPE_MASK 0x00000002 /* mask to get pte type */ + +#define ARM_PTE_NG_MASK 0x00000800 /* mask to determine notGlobal bit */ +#define ARM_PTE_NG 0x00000800 /* value for a per-process mapping */ + +#define ARM_PTE_SHSHIFT 10 +#define ARM_PTE_SH_MASK 0x00000400 /* shared (SMP) mapping mask */ +#define ARM_PTE_SH 0x00000400 /* shared (SMP) mapping */ + +#define ARM_PTE_CBSHIFT 2 +#define ARM_PTE_CB(x) ((x)<<ARM_PTE_CBSHIFT) +#define ARM_PTE_CB_MASK (0x3<<ARM_PTE_CBSHIFT) + +#define ARM_PTE_AP0SHIFT 4 +#define ARM_PTE_AP0 (1<<ARM_PTE_AP0SHIFT) +#define ARM_PTE_AP0_MASK (1<<ARM_PTE_AP0SHIFT) + +#define ARM_PTE_AP1SHIFT 5 +#define ARM_PTE_AP1 (1<<ARM_PTE_AP1SHIFT) +#define ARM_PTE_AP1_MASK (1<<ARM_PTE_AP1SHIFT) + +#define ARM_PTE_AP2SHIFT 9 +#define ARM_PTE_AP2 (1<<ARM_PTE_AP2SHIFT) +#define ARM_PTE_AP2_MASK (1<<ARM_PTE_AP2SHIFT) + + /* access protections */ +#define ARM_PTE_AP(ap) ((((ap)&0x1)<<ARM_PTE_AP1SHIFT) \ + | ((((ap)>>1)&0x1)<<ARM_PTE_AP2SHIFT)) + + /* mask access protections */ +#define ARM_PTE_APMASK (ARM_PTE_AP1_MASK \ + | ARM_PTE_AP2_MASK) + +#define ARM_PTE_AF ARM_PTE_AP0 /* value for access */ +#define ARM_PTE_AFMASK ARM_PTE_AP0_MASK /* access mask */ + +#define ARM_PTE_PAGE_MASK 0xFFFFF000 /* mask for a small page */ +#define ARM_PTE_PAGE_SHIFT 12 /* page shift for 4KB page */ + +#define ARM_PTE_NXSHIFT 0 +#define ARM_PTE_NX 0x00000001 /* small page no execute */ +#define ARM_PTE_NX_MASK (1<<ARM_PTE_NXSHIFT) + +#define ARM_PTE_PNXSHIFT 0 +#define ARM_PTE_PNX 0x00000000 /* no privilege execute. not impl */ +#define ARM_PTE_PNX_MASK (0<<ARM_PTE_NXSHIFT) + +#define ARM_PTE_TEX0SHIFT 6 +#define ARM_PTE_TEX0 (1<<ARM_PTE_TEX0SHIFT) +#define ARM_PTE_TEX0_MASK (1<<ARM_PTE_TEX0SHIFT) + +#define ARM_PTE_TEX1SHIFT 7 +#define ARM_PTE_TEX1 (1<<ARM_PTE_TEX1SHIFT) +#define ARM_PTE_TEX1_MASK (1<<ARM_PTE_TEX1SHIFT) + +#define ARM_PTE_WRITEABLESHIFT ARM_PTE_TEX1SHIFT +#define ARM_PTE_WRITEABLE ARM_PTE_TEX1 +#define ARM_PTE_WRITEABLE_MASK ARM_PTE_TEX1_MASK + +#define ARM_PTE_TEX2SHIFT 8 +#define ARM_PTE_TEX2 (1<<ARM_PTE_TEX2SHIFT) +#define ARM_PTE_TEX2_MASK (1<<ARM_PTE_TEX2SHIFT) + +#define ARM_PTE_WIREDSHIFT ARM_PTE_TEX2SHIFT +#define ARM_PTE_WIRED ARM_PTE_TEX2 +#define ARM_PTE_WIRED_MASK ARM_PTE_TEX2_MASK + + /* mask memory attributes index */ +#define ARM_PTE_ATTRINDX(indx) ((((indx)&0x3)<<ARM_PTE_CBSHIFT) \ + | ((((indx)>>2)&0x1)<<ARM_PTE_TEX0SHIFT)) + + /* mask memory attributes index */ +#define ARM_PTE_ATTRINDXMASK (ARM_PTE_CB_MASK \ + | ARM_PTE_TEX0_MASK) + +#define ARM_SMALL_PAGE_SIZE (4096) /* 4KB */ +#define ARM_LARGE_PAGE_SIZE (64*1024) /* 64KB */ +#define ARM_SECTION_SIZE (1024*1024) /* 1MB */ +#define ARM_SUPERSECTION_SIZE (16*1024*1024) /* 16MB */ + +#endif + +/* + * Format of the Debug Status and Control Register (DBGDSCR) + */ +#define ARM_DBGDSCR_RXFULL (1 << 30) +#define ARM_DBGDSCR_TXFULL (1 << 29) +#define ARM_DBGDSCR_RXFULL_1 (1 << 27) +#define ARM_DBGDSCR_TXFULL_1 (1 << 26) +#define ARM_DBGDSCR_PIPEADV (1 << 25) +#define ARM_DBGDSCR_INSTRCOMPL_1 (1 << 24) +#define ARM_DBGDSCR_EXTDCCMODE_MASK (3 << 20) +#define ARM_DBGDSCR_EXTDCCMODE_NONBLOCKING (0 << 20) +#define ARM_DBGDSCR_EXTDCCMODE_STALL (1 << 20) +#define ARM_DBGDSCR_EXTDCCMODE_FAST (1 << 20) +#define ARM_DBGDSCR_ADADISCARD (1 << 19) +#define ARM_DBGDSCR_NS (1 << 18) +#define ARM_DBGDSCR_SPNIDDIS (1 << 17) +#define ARM_DBGDSCR_SPIDDIS (1 << 16) +#define ARM_DBGDSCR_MDBGEN (1 << 15) +#define ARM_DBGDSCR_HDBGEN (1 << 14) +#define ARM_DBGDSCR_ITREN (1 << 13) +#define ARM_DBGDSCR_UDCCDIS (1 << 12) +#define ARM_DBGDSCR_INTDIS (1 << 11) +#define ARM_DBGDSCR_DBGACK (1 << 10) +#define ARM_DBGDSCR_DBGNOPWRDWN (1 << 9) +#define ARM_DBGDSCR_UND_1 (1 << 8) +#define ARM_DBGDSCR_ADABORT_1 (1 << 7) +#define ARM_DBGDSCR_SDABORT_1 (1 << 6) +#define ARM_DBGDSCR_MOE_MASK (15 << 2) +#define ARM_DBGDSCR_MOE_HALT_REQUEST (0 << 2) +#define ARM_DBGDSCR_MOE_BREAKPOINT (1 << 2) +#define ARM_DBGDSCR_MOE_ASYNC_WATCHPOINT (2 << 2) +#define ARM_DBGDSCR_MOE_BKPT_INSTRUCTION (3 << 2) +#define ARM_DBGDSCR_MOE_EXT_DEBUG_REQ (4 << 2) +#define ARM_DBGDSCR_MOE_VECTOR_CATCH (5 << 2) +#define ARM_DBGDSCR_MOE_DSIDE_ABORT (6 << 2) +#define ARM_DBGDSCR_MOE_ISIDE_ABORT (7 << 2) +#define ARM_DBGDSCR_MOE_OS_UNLOCK_CATCH (8 << 2) +#define ARM_DBGDSCR_MOE_SYNC_WATCHPOINT (10 << 2) + +#define ARM_DBGDSCR_RESTARTED (1 << 1) +#define ARM_DBGDSCR_HALTED (1 << 0) + +/* + * Format of the Debug & Watchpoint Breakpoint Value and Control Registers + * Using ARMv7 names; ARMv6 and ARMv6.1 are bit-compatible + */ +#define ARM_DBG_VR_ADDRESS_MASK 0xFFFFFFFC /* BVR & WVR */ +#define ARM_DBGBVR_CONTEXTID_MASK 0xFFFFFFFF /* BVR only */ + +#define ARM_DBG_CR_ADDRESS_MASK_MASK 0x1F000000 /* BCR & WCR */ +#define ARM_DBGBCR_MATCH_MASK (1 << 22) /* BCR only */ +#define ARM_DBGBCR_MATCH_MATCH (0 << 22) +#define ARM_DBGBCR_MATCH_MISMATCH (1 << 22) +#define ARM_DBGBCR_TYPE_MASK (1 << 21) /* BCR only */ +#define ARM_DBGBCR_TYPE_IVA (0 << 21) +#define ARM_DBGBCR_TYPE_CONTEXTID (1 << 21) +#define ARM_DBG_CR_LINKED_MASK (1 << 20) /* BCR & WCR */ +#define ARM_DBG_CR_LINKED_LINKED (1 << 20) +#define ARM_DBG_CR_LINKED_UNLINKED (0 << 20) +#define ARM_DBG_CR_LINKED_BRP_MASK 0x000F0000 /* BCR & WCR */ +#define ARM_DBG_CR_SECURITY_STATE_MASK (3 << 14) /* BCR & WCR */ +#define ARM_DBG_CR_SECURITY_STATE_BOTH (0 << 14) +#define ARM_DBG_CR_SECURITY_STATE_NONSECURE (1 << 14) +#define ARM_DBG_CR_SECURITY_STATE_SECURE (2 << 14) +#define ARM_DBGWCR_BYTE_ADDRESS_SELECT_MASK 0x00001FE0 /* WCR only */ +#define ARM_DBG_CR_BYTE_ADDRESS_SELECT_MASK 0x000001E0 /* BCR & WCR */ +#define ARM_DBGWCR_ACCESS_CONTROL_MASK (3 << 3) /* WCR only */ +#define ARM_DBCWCR_ACCESS_CONTROL_LOAD (1 << 3) +#define ARM_DBCWCR_ACCESS_CONTROL_STORE (2 << 3) +#define ARM_DBCWCR_ACCESS_CONTROL_ANY (3 << 3) +#define ARM_DBG_CR_MODE_CONTROL_MASK (3 << 1) /* BCR & WCR */ +#define ARM_DBG_CR_MODE_CONTROL_U_S_S (0 << 1) /* BCR only */ +#define ARM_DBG_CR_MODE_CONTROL_PRIVILEDGED (1 << 1) /* BCR & WCR */ +#define ARM_DBG_CR_MODE_CONTROL_USER (2 << 1) /* BCR & WCR */ +#define ARM_DBG_CR_MODE_CONTROL_ANY (3 << 1) /* BCR & WCR */ +#define ARM_DBG_CR_ENABLE_MASK (1 << 0) /* BCR & WCR */ +#define ARM_DBG_CR_ENABLE_ENABLE (1 << 0) +#define ARM_DBG_CR_ENABLE_DISABLE (0 << 0) + +/* + * Format of the Device Power-down and Reset Status Register (DBGPRSR) + */ +#define ARM_DBGPRSR_STICKY_RESET_STATUS (1 << 3) +#define ARM_DBGPRSR_RESET_STATUS (1 << 2) +#define ARM_DBGPRSR_STICKY_POWERDOWN_STATUS (1 << 1) +#define ARM_DBGPRSR_POWERUP_STATUS (1 << 0) + +/* + * Format of the OS Lock Access (DBGOSLAR) and Lock Access Registers (DBGLAR) + */ +#define ARM_DBG_LOCK_ACCESS_KEY 0xC5ACCE55 + +/* ARMv7 Debug register map */ +#define ARM_DEBUG_OFFSET_DBGDIDR (0x000) +#define ARM_DEBUG_OFFSET_DBGWFAR (0x018) +#define ARM_DEBUG_OFFSET_DBGVCR (0x01C) +#define ARM_DEBUG_OFFSET_DBGECR (0x024) +#define ARM_DEBUG_OFFSET_DBGDSCCR (0x028) +#define ARM_DEBUG_OFFSET_DBGDSMCR (0x02C) +#define ARM_DEBUG_OFFSET_DBGDTRRX (0x080) +#define ARM_DEBUG_OFFSET_DBGITR (0x084) /* Write-only */ +#define ARM_DEBUG_OFFSET_DBGPCSR (0x084) /* Read-only */ +#define ARM_DEBUG_OFFSET_DBGDSCR (0x088) +#define ARM_DEBUG_OFFSET_DBGDTRTX (0x08C) +#define ARM_DEBUG_OFFSET_DBGDRCR (0x090) +#define ARM_DEBUG_OFFSET_DBGBVR (0x100) /* 0x100 - 0x13C */ +#define ARM_DEBUG_OFFSET_DBGBCR (0x140) /* 0x140 - 0x17C */ +#define ARM_DEBUG_OFFSET_DBGWVR (0x180) /* 0x180 - 0x1BC */ +#define ARM_DEBUG_OFFSET_DBGWCR (0x1C0) /* 0x1C0 - 0x1FC */ +#define ARM_DEBUG_OFFSET_DBGOSLAR (0x300) +#define ARM_DEBUG_OFFSET_DBGOSLSR (0x304) +#define ARM_DEBUG_OFFSET_DBGOSSRR (0x308) +#define ARM_DEBUG_OFFSET_DBGPRCR (0x310) +#define ARM_DEBUG_OFFSET_DBGPRSR (0x314) +#define ARM_DEBUG_OFFSET_DBGITCTRL (0xF00) +#define ARM_DEBUG_OFFSET_DBGCLAIMSET (0xFA0) +#define ARM_DEBUG_OFFSET_DBGCLAIMCLR (0xFA4) +#define ARM_DEBUG_OFFSET_DBGLAR (0xFB0) +#define ARM_DEBUG_OFFSET_DBGLSR (0xFB4) +#define ARM_DEBUG_OFFSET_DBGAUTHSTATUS (0xFB8) +#define ARM_DEBUG_OFFSET_DBGDEVID (0xFC8) +#define ARM_DEBUG_OFFSET_DBGDEVTYPE (0xFCC) +#define ARM_DEBUG_OFFSET_DBGPID0 (0xFD0) +#define ARM_DEBUG_OFFSET_DBGPID1 (0xFD4) +#define ARM_DEBUG_OFFSET_DBGPID2 (0xFD8) +#define ARM_DEBUG_OFFSET_DBGPID3 (0xFDA) +#define ARM_DEBUG_OFFSET_DBGPID4 (0xFDC) +#define ARM_DEBUG_OFFSET_DBGCID0 (0xFF0) +#define ARM_DEBUG_OFFSET_DBGCID1 (0xFF4) +#define ARM_DEBUG_OFFSET_DBGCID2 (0xFF8) +#define ARM_DEBUG_OFFSET_DBGCID3 (0xFFA) +#define ARM_DEBUG_OFFSET_DBGCID4 (0xFFC) + +/* + * Media and VFP Feature Register 1 (MVFR1) + */ +#define MVFR_ASIMD_HPFP 0x00100000UL + +#endif /* _ARM_PROC_REG_H_ */ diff --git a/osfmk/arm/rtclock.c b/osfmk/arm/rtclock.c new file mode 100644 index 000000000..e4e304375 --- /dev/null +++ b/osfmk/arm/rtclock.c @@ -0,0 +1,495 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * @APPLE_FREE_COPYRIGHT@ + */ +/* + * File: arm/rtclock.c + * Purpose: Routines for handling the machine dependent + * real-time clock. + */ + +#include <mach/mach_types.h> + +#include <kern/clock.h> +#include <kern/thread.h> +#include <kern/macro_help.h> +#include <kern/spl.h> +#include <kern/timer_queue.h> + +#include <kern/host_notify.h> + +#include <machine/commpage.h> +#include <machine/machine_routines.h> +#include <arm/exception.h> +#include <arm/cpu_data_internal.h> +#if __arm64__ +#include <arm64/proc_reg.h> +#elif __arm__ +#include <arm/proc_reg.h> +#else +#error Unsupported arch +#endif +#include <arm/rtclock.h> + +#include <IOKit/IOPlatformExpert.h> +#include <libkern/OSAtomic.h> + +#include <sys/kdebug.h> + +#define MAX_TIMEBASE_TRIES 10 + +int rtclock_init(void); + +static int +deadline_to_decrementer(uint64_t deadline, + uint64_t now); +static void +timebase_callback(struct timebase_freq_t * freq); + +#if DEVELOPMENT || DEBUG +uint32_t absolute_time_validation = 1; +#endif + +/* + * Configure the real-time clock device at boot + */ +void +rtclock_early_init(void) +{ + PE_register_timebase_callback(timebase_callback); +#if DEVELOPMENT || DEBUG + uint32_t tmp_mv = 1; + if (kern_feature_override(KF_MATV_OVRD)) { + absolute_time_validation = 0; + } + if (PE_parse_boot_argn("timebase_validation", &tmp_mv, sizeof(tmp_mv))) { + if (tmp_mv == 0) { + absolute_time_validation = 0; + } + } +#endif +} + +static void +timebase_callback(struct timebase_freq_t * freq) +{ + unsigned long numer, denom; + uint64_t t64_1, t64_2; + uint32_t divisor; + + if (freq->timebase_den < 1 || freq->timebase_den > 4 || + freq->timebase_num < freq->timebase_den) + panic("rtclock timebase_callback: invalid constant %ld / %ld", + freq->timebase_num, freq->timebase_den); + + denom = freq->timebase_num; + numer = freq->timebase_den * NSEC_PER_SEC; + // reduce by the greatest common denominator to minimize overflow + if (numer > denom) { + t64_1 = numer; + t64_2 = denom; + } else { + t64_1 = denom; + t64_2 = numer; + } + while (t64_2 != 0) { + uint64_t temp = t64_2; + t64_2 = t64_1 % t64_2; + t64_1 = temp; + } + numer /= t64_1; + denom /= t64_1; + + rtclock_timebase_const.numer = (uint32_t)numer; + rtclock_timebase_const.denom = (uint32_t)denom; + divisor = (uint32_t)(freq->timebase_num / freq->timebase_den); + + rtclock_sec_divisor = divisor; + rtclock_usec_divisor = divisor / USEC_PER_SEC; +} + +/* + * Initialize the system clock device for the current cpu + */ +int +rtclock_init(void) +{ + uint64_t abstime; + cpu_data_t * cdp; + + clock_timebase_init(); + ml_init_lock_timeout(); + + cdp = getCpuDatap(); + + abstime = mach_absolute_time(); + cdp->rtcPop = EndOfAllTime; /* Init Pop time */ + timer_resync_deadlines(); /* Start the timers going */ + + return (1); +} + +uint64_t +mach_absolute_time(void) +{ +#if DEVELOPMENT || DEBUG + if (__improbable(absolute_time_validation == 1)) { + static volatile uint64_t s_last_absolute_time = 0; + uint64_t new_absolute_time, old_absolute_time; + int attempts = 0; + + /* ARM 64: We need a dsb here to ensure that the load of s_last_absolute_time + * completes before the timebase read. Were the load to complete after the + * timebase read, there would be a window for another CPU to update + * s_last_absolute_time and leave us in an inconsistent state. Consider the + * following interleaving: + * + * Let s_last_absolute_time = t0 + * CPU0: Read timebase at t1 + * CPU1: Read timebase at t2 + * CPU1: Update s_last_absolute_time to t2 + * CPU0: Load completes + * CPU0: Update s_last_absolute_time to t1 + * + * This would cause the assertion to fail even though time did not go + * backwards. Thus, we use a dsb to guarantee completion of the load before + * the timebase read. + */ + do { + attempts++; + old_absolute_time = s_last_absolute_time; + +#if __arm64__ + __asm__ volatile("dsb ld" ::: "memory"); +#else + OSSynchronizeIO(); // See osfmk/arm64/rtclock.c +#endif + + new_absolute_time = ml_get_timebase(); + } while (attempts < MAX_TIMEBASE_TRIES && !OSCompareAndSwap64(old_absolute_time, new_absolute_time, &s_last_absolute_time)); + + if (attempts < MAX_TIMEBASE_TRIES && old_absolute_time > new_absolute_time) { + panic("mach_absolute_time returning non-monotonically increasing value 0x%llx (old value 0x%llx\n)\n", + new_absolute_time, old_absolute_time); + } + return new_absolute_time; + } else { + return ml_get_timebase(); + } +#else + return ml_get_timebase(); +#endif +} + +uint64_t +mach_approximate_time(void) +{ +#if __ARM_TIME__ || __ARM_TIME_TIMEBASE_ONLY__ || __arm64__ + /* Hardware supports a fast timestamp, so grab it without asserting monotonicity */ + return ml_get_timebase(); +#else + processor_t processor; + uint64_t approx_time; + + disable_preemption(); + processor = current_processor(); + approx_time = processor->last_dispatch; + enable_preemption(); + + return approx_time; +#endif +} + +void +clock_get_system_microtime(clock_sec_t * secs, + clock_usec_t * microsecs) +{ + absolutetime_to_microtime(mach_absolute_time(), secs, microsecs); +} + +void +clock_get_system_nanotime(clock_sec_t * secs, + clock_nsec_t * nanosecs) +{ + uint64_t abstime; + uint64_t t64; + + abstime = mach_absolute_time(); + *secs = (t64 = abstime / rtclock_sec_divisor); + abstime -= (t64 * rtclock_sec_divisor); + + *nanosecs = (clock_nsec_t)((abstime * NSEC_PER_SEC) / rtclock_sec_divisor); +} + +void +clock_gettimeofday_set_commpage(uint64_t abstime, + uint64_t sec, + uint64_t frac, + uint64_t scale, + uint64_t tick_per_sec) +{ + commpage_set_timestamp(abstime, sec, frac, scale, tick_per_sec); +} + +void +clock_timebase_info(mach_timebase_info_t info) +{ + *info = rtclock_timebase_const; +} + +/* + * Real-time clock device interrupt. + */ +void +rtclock_intr(__unused unsigned int is_user_context) +{ + uint64_t abstime; + cpu_data_t * cdp; + struct arm_saved_state * regs; + unsigned int user_mode; + uintptr_t pc; + + cdp = getCpuDatap(); + + cdp->cpu_stat.timer_cnt++; + cdp->cpu_stat.timer_cnt_wake++; + SCHED_STATS_TIMER_POP(current_processor()); + + assert(!ml_get_interrupts_enabled()); + + abstime = mach_absolute_time(); + + if (cdp->cpu_idle_pop != 0x0ULL) { + if (( cdp->rtcPop-abstime) < cdp->cpu_idle_latency) { + cdp->cpu_idle_pop = 0x0ULL; + while (abstime < cdp->rtcPop) + abstime = mach_absolute_time(); + } else { + ClearIdlePop(FALSE); + } + } + + if ((regs = cdp->cpu_int_state)) { + pc = get_saved_state_pc(regs); + +#if __arm64__ + user_mode = PSR64_IS_USER(get_saved_state_cpsr(regs)); +#else + user_mode = (regs->cpsr & PSR_MODE_MASK) == PSR_USER_MODE; +#endif + } else { + pc = 0; + user_mode = 0; + } + if (abstime >= cdp->rtcPop) { + /* Log the interrupt service latency (-ve value expected by tool) */ + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + MACHDBG_CODE(DBG_MACH_EXCP_DECI, 0) | DBG_FUNC_NONE, + -(abstime - cdp->rtcPop), + user_mode ? pc : VM_KERNEL_UNSLIDE(pc), user_mode, 0, 0); + } + + /* call the generic etimer */ + timer_intr(user_mode, pc); +} + +static int +deadline_to_decrementer(uint64_t deadline, + uint64_t now) +{ + uint64_t delt; + + if (deadline <= now) + return DECREMENTER_MIN; + else { + delt = deadline - now; + + return (delt >= (DECREMENTER_MAX + 1)) ? DECREMENTER_MAX : ((delt >= (DECREMENTER_MIN + 1)) ? (int)delt : DECREMENTER_MIN); + } +} + +/* + * Request a decrementer pop + */ +int +setPop(uint64_t time) +{ + int delay_time; + uint64_t current_time; + cpu_data_t * cdp; + + cdp = getCpuDatap(); + current_time = mach_absolute_time(); + + delay_time = deadline_to_decrementer(time, current_time); + cdp->rtcPop = delay_time + current_time; + + ml_set_decrementer((uint32_t) delay_time); + + return (delay_time); +} + +/* + * Request decrementer Idle Pop. Return true if set + */ +boolean_t +SetIdlePop(void) +{ + int delay_time; + uint64_t time; + uint64_t current_time; + cpu_data_t * cdp; + + cdp = getCpuDatap(); + current_time = mach_absolute_time(); + + if (((cdp->rtcPop < current_time) || + (cdp->rtcPop - current_time) < cdp->cpu_idle_latency)) + return FALSE; + + time = cdp->rtcPop - cdp->cpu_idle_latency; + + delay_time = deadline_to_decrementer(time, current_time); + cdp->cpu_idle_pop = delay_time + current_time; + ml_set_decrementer((uint32_t) delay_time); + + return TRUE; +} + +/* + * Clear decrementer Idle Pop + */ +void +ClearIdlePop( + boolean_t wfi) +{ +#if !__arm64__ +#pragma unused(wfi) +#endif + cpu_data_t * cdp; + + cdp = getCpuDatap(); + cdp->cpu_idle_pop = 0x0ULL; + +#if __arm64__ + /* + * Don't update the HW timer if there's a pending + * interrupt (we can lose interrupt assertion); + * we want to take the interrupt right now and update + * the deadline from the handler). + * + * ARM64_TODO: consider this more carefully. + */ + if (!(wfi && ml_get_timer_pending())) +#endif + { + setPop(cdp->rtcPop); + } +} + +void +absolutetime_to_microtime(uint64_t abstime, + clock_sec_t * secs, + clock_usec_t * microsecs) +{ + uint64_t t64; + + *secs = t64 = abstime / rtclock_sec_divisor; + abstime -= (t64 * rtclock_sec_divisor); + + *microsecs = (uint32_t)(abstime / rtclock_usec_divisor); +} + +void +absolutetime_to_nanoseconds(uint64_t abstime, + uint64_t * result) +{ + uint64_t t64; + + *result = (t64 = abstime / rtclock_sec_divisor) * NSEC_PER_SEC; + abstime -= (t64 * rtclock_sec_divisor); + *result += (abstime * NSEC_PER_SEC) / rtclock_sec_divisor; +} + +void +nanoseconds_to_absolutetime(uint64_t nanosecs, + uint64_t * result) +{ + uint64_t t64; + + *result = (t64 = nanosecs / NSEC_PER_SEC) * rtclock_sec_divisor; + nanosecs -= (t64 * NSEC_PER_SEC); + *result += (nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC; +} + +void +nanotime_to_absolutetime(clock_sec_t secs, + clock_nsec_t nanosecs, + uint64_t * result) +{ + *result = ((uint64_t) secs * rtclock_sec_divisor) + + ((uint64_t) nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC; +} + +void +clock_interval_to_absolutetime_interval(uint32_t interval, + uint32_t scale_factor, + uint64_t * result) +{ + uint64_t nanosecs = (uint64_t) interval * scale_factor; + uint64_t t64; + + *result = (t64 = nanosecs / NSEC_PER_SEC) * rtclock_sec_divisor; + nanosecs -= (t64 * NSEC_PER_SEC); + *result += (nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC; +} + +void +machine_delay_until(uint64_t interval, + uint64_t deadline) +{ +#pragma unused(interval) + uint64_t now; + + do { +#if __ARM_ENABLE_WFE_ +#if __arm64__ + if (arm64_wfe_allowed()) +#endif /* __arm64__ */ + { + __builtin_arm_wfe(); + } +#endif /* __ARM_ENABLE_WFE_ */ + + now = mach_absolute_time(); + } while (now < deadline); +} diff --git a/osfmk/arm/rtclock.h b/osfmk/arm/rtclock.h new file mode 100644 index 000000000..fb051b2ab --- /dev/null +++ b/osfmk/arm/rtclock.h @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * @APPLE_FREE_COPYRIGHT@ + */ + +#ifndef _ARM_RTCLOCK_H_ +#define _ARM_RTCLOCK_H_ + +#include <mach/boolean.h> +#include <mach/mach_types.h> +#include <mach/mach_time.h> +#include <arm/machine_routines.h> + +#define EndOfAllTime 0xFFFFFFFFFFFFFFFFULL +#define DECREMENTER_MAX 0x7FFFFFFFUL +#define DECREMENTER_MIN 0xAUL + +typedef struct _rtclock_data_ { + uint32_t rtc_sec_divisor; + uint32_t rtc_usec_divisor; + mach_timebase_info_data_t rtc_timebase_const; + union { + uint64_t abstime; + struct { + uint32_t low; + uint32_t high; + } abstime_val; + } rtc_base; + union { + uint64_t abstime; + struct { + uint32_t low; + uint32_t high; + } abstime_val; + } rtc_adj; + tbd_ops_data_t rtc_timebase_func; + + /* Only needed for AIC manipulation */ + vm_offset_t rtc_timebase_addr; + vm_offset_t rtc_timebase_val; + +} rtclock_data_t; + +extern rtclock_data_t RTClockData; +#define rtclock_sec_divisor RTClockData.rtc_sec_divisor +#define rtclock_usec_divisor RTClockData.rtc_usec_divisor +#define rtclock_timebase_const RTClockData.rtc_timebase_const +#define rtclock_base_abstime RTClockData.rtc_base.abstime +#define rtclock_base_abstime_low RTClockData.rtc_base.abstime_val.low +#define rtclock_base_abstime_high RTClockData.rtc_base.abstime_val.high +#define rtclock_adj_abstime RTClockData.rtc_adj.abstime +#define rtclock_adj_abstime_low RTClockData.rtc_adj.abstime_val.low +#define rtclock_adj_abstime_high RTClockData.rtc_adj.abstime_val.high +#define rtclock_timebase_func RTClockData.rtc_timebase_func + +/* Only needed for AIC manipulation */ +#define rtclock_timebase_addr RTClockData.rtc_timebase_addr +#define rtclock_timebase_val RTClockData.rtc_timebase_val + +extern uint64_t arm_timer_slop_max; + +extern void rtclock_intr(unsigned int); +extern boolean_t SetIdlePop(void); + +extern void ClearIdlePop(boolean_t); +extern void rtclock_early_init(void); + +#endif /* _ARM_RTCLOCK_H_ */ diff --git a/osfmk/arm/sched_param.h b/osfmk/arm/sched_param.h new file mode 100644 index 000000000..a3d20dc47 --- /dev/null +++ b/osfmk/arm/sched_param.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ + +/* + */ + +/* + * Scheduler parameters. + */ + +#ifndef _ARM_SCHED_PARAM_H_ +#define _ARM_SCHED_PARAM_H_ + +#endif /* _ARM_SCHED_PARAM_H_ */ diff --git a/osfmk/arm/setjmp.h b/osfmk/arm/setjmp.h new file mode 100644 index 000000000..a3a2f5ead --- /dev/null +++ b/osfmk/arm/setjmp.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ +/* + */ + +/* + * Setjmp/longjmp buffer for ARM. + */ +#ifndef _ARM_SETJMP_H_ +#define _ARM_SETJMP_H_ + +typedef struct jmp_buf { + int jmp_buf[28]; +} jmp_buf_t; + +#endif /* _ARM_SETJMP_H_ */ diff --git a/osfmk/arm/simple_lock.h b/osfmk/arm/simple_lock.h new file mode 100644 index 000000000..0fb708ede --- /dev/null +++ b/osfmk/arm/simple_lock.h @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ + +#ifdef KERNEL_PRIVATE + +#ifndef _ARM_SIMPLE_LOCK_TYPES_H_ +#define _ARM_SIMPLE_LOCK_TYPES_H_ + +#ifdef KERNEL_PRIVATE +#include <mach/boolean.h> +#include <kern/kern_types.h> + +#include <sys/appleapiopts.h> +#ifdef MACH_KERNEL_PRIVATE +#include <arm/hw_lock_types.h> +#include <arm/locks.h> +#include <mach_ldebug.h> +#endif + +#ifdef MACH_KERNEL_PRIVATE + +typedef uint32_t hw_lock_bit_t; + +extern void hw_lock_bit( + hw_lock_bit_t *, + unsigned int); + +extern void hw_unlock_bit( + hw_lock_bit_t *, + unsigned int); + +extern unsigned int hw_lock_bit_try( + hw_lock_bit_t *, + unsigned int); + +extern unsigned int hw_lock_bit_to( + hw_lock_bit_t *, + unsigned int, + uint32_t); + +#define hw_lock_bit_held(l,b) (((*(l))&(1<<b))!=0) + + +extern uint32_t LockTimeOut; /* Number of hardware ticks of a lock timeout */ +extern uint32_t LockTimeOutUsec; /* Number of microseconds for lock timeout */ + +/* + * USLOCK_DEBUG is broken on ARM and has been disabled. + * There are no callers to any of the usld_lock functions and data structures + * don't match between between usimple_lock_data_t and lck_spin_t +*/ + +/* +#if MACH_LDEBUG +#define USLOCK_DEBUG 1 +#else +#define USLOCK_DEBUG 0 +#endif +*/ + +#if !USLOCK_DEBUG + +typedef lck_spin_t usimple_lock_data_t, *usimple_lock_t; + +#else + +typedef struct uslock_debug { + void *lock_pc; /* pc where lock operation began */ + void *lock_thread; /* thread that acquired lock */ + unsigned long duration[2]; + unsigned short state; + unsigned char lock_cpu; + void *unlock_thread; /* last thread to release lock */ + unsigned char unlock_cpu; + void *unlock_pc; /* pc where lock operation ended */ +} uslock_debug; + +typedef struct { + hw_lock_data_t interlock; /* must be first... see lock.c */ + unsigned short lock_type; /* must be second... see lock.c */ +#define USLOCK_TAG 0x5353 + uslock_debug debug; +} usimple_lock_data_t, *usimple_lock_t; + +#endif /* USLOCK_DEBUG */ + +#else + +#if defined(__arm__) +typedef struct slock { + unsigned int lock_data[10]; +} usimple_lock_data_t, *usimple_lock_t; +#elif defined(__arm64__) +/* + * ARM64_TODO: this is quite a waste of space (and a + * poorly packed data structure). See if anyone's + * using these outside of osfmk. + * NOTE: only osfmk uses this structure in xnu-2624 + */ +typedef struct slock { + uint64_t lock_data[9]; +} usimple_lock_data_t, *usimple_lock_t; +#else +#error Unknown architecture. +#endif + +#endif /* MACH_KERNEL_PRIVATE */ + +#define USIMPLE_LOCK_NULL ((usimple_lock_t) 0) + +#if !defined(decl_simple_lock_data) + +typedef usimple_lock_data_t *simple_lock_t; +typedef usimple_lock_data_t simple_lock_data_t; + +#define decl_simple_lock_data(class,name) \ + class simple_lock_data_t name; + +#endif /* !defined(decl_simple_lock_data) */ + +#ifdef MACH_KERNEL_PRIVATE + +#define MACHINE_SIMPLE_LOCK + +extern void arm_usimple_lock_init(simple_lock_t, __unused unsigned short); + +#define simple_lock_init(l,t) arm_usimple_lock_init(l,t) +#define simple_lock(l) lck_spin_lock(l) +#define simple_unlock(l) lck_spin_unlock(l) +#define simple_lock_try(l) lck_spin_try_lock(l) +#define simple_lock_try_lock_loop(l) simple_lock(l) +#define simple_lock_addr(l) (&(l)) +#define simple_lock_assert(l,t) lck_spin_assert(l,t) +#define kdp_simple_lock_is_acquired(l) kdp_lck_spin_is_acquired(l) + +#endif /* MACH_KERNEL_PRIVATE */ +#endif /* KERNEL_PRIVATE */ + +#endif /* !_ARM_SIMPLE_LOCK_TYPES_H_ */ + +#endif /* KERNEL_PRIVATE */ diff --git a/osfmk/arm/smp.h b/osfmk/arm/smp.h new file mode 100644 index 000000000..069b5fd52 --- /dev/null +++ b/osfmk/arm/smp.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2014 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef _ARM_SMP_H_ +#define _ARM_SMP_H_ + +#include <arm/proc_reg.h> + +#define __SMP__ __ARM_SMP__ +#define __AMP__ __ARM_AMP__ + +#endif /* _ARM_SMP_H_ */ diff --git a/osfmk/arm/start.s b/osfmk/arm/start.s new file mode 100644 index 000000000..ced8c0d3e --- /dev/null +++ b/osfmk/arm/start.s @@ -0,0 +1,434 @@ +/* + * Copyright (c) 2007-2014 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include <arm/asm.h> +#include <arm/proc_reg.h> +#include <mach_kdp.h> +#include "assym.s" + + .text + .align 12 + + .align 2 + .globl EXT(resume_idle_cpu) +LEXT(resume_idle_cpu) + // r0 set to BootArgs phys address + // r1 set to cpu data phys address + LOAD_ADDR(lr, arm_init_idle_cpu) + b L_start_cpu_0 + + .globl EXT(start_cpu) +LEXT(start_cpu) + // r0 set to BootArgs phys address + // r1 set to cpu data phys address + LOAD_ADDR(lr, arm_init_cpu) + b L_start_cpu_0 + +L_start_cpu_0: + cpsid if // Disable IRQ FIQ + + // Turn on L1 I-Cache, Branch prediction early + mcr p15, 0, r11, c7, c5, 0 // invalidate the icache + isb // before moving on + mrc p15, 0, r11, c1, c0, 0 // read mmu control into r11 + orr r11, r11, #(SCTLR_ICACHE | SCTLR_PREDIC) // enable i-cache, b-prediction + mcr p15, 0, r11, c1, c0, 0 // set mmu control + dsb // ensure mmu settings are inplace + isb // before moving on + + // Get the kernel's phys & virt addr, and size from BootArgs + ldr r8, [r0, BA_PHYS_BASE] // Get the phys base in r8 + ldr r9, [r0, BA_VIRT_BASE] // Get the virt base in r9 + ldr r10, [r0, BA_MEM_SIZE] // Get the mem size in r10 + + // Set the base of the translation table into the MMU + ldr r4, [r0, BA_TOP_OF_KERNEL_DATA] // Get the top of kernel data + orr r5, r4, #(TTBR_SETUP & 0x00FF) // Setup PTWs memory attribute + orr r5, r5, #(TTBR_SETUP & 0xFF00) // Setup PTWs memory attribute + mcr p15, 0, r5, c2, c0, 0 // write kernel to translation table base 0 + mcr p15, 0, r5, c2, c0, 1 // also to translation table base 1 + mov r5, #TTBCR_N_1GB_TTB0 // identify the split between 0 and 1 + mcr p15, 0, r5, c2, c0, 2 // and set up the translation control reg + ldr r2, [r1, CPU_NUMBER_GS] // Get cpu number + mcr p15, 0, r2, c13, c0, 3 // Write TPIDRURO + ldr sp, [r1, CPU_INTSTACK_TOP] // Get interrupt stack top + sub sp, sp, SS_SIZE // Set stack pointer + sub r0, r1, r8 // Convert to virtual address + add r0, r0, r9 + b join_start + + .align 2 + .globl EXT(_start) +LEXT(_start) + // r0 has the boot-args pointer + // r1 set to zero + mov r1, #0 + LOAD_ADDR(lr, arm_init) + cpsid if // Disable IRQ FIQ + + // Turn on L1 I-Cache, Branch prediction early + mcr p15, 0, r11, c7, c5, 0 // invalidate the icache + isb // before moving on + mrc p15, 0, r11, c1, c0, 0 // read mmu control into r11 + orr r11, r11, #(SCTLR_ICACHE | SCTLR_PREDIC) // enable i-cache, b-prediction + mcr p15, 0, r11, c1, c0, 0 // set mmu control + dsb // ensure mmu settings are inplace + isb // before moving on + + // Get the kernel's phys & virt addr, and size from boot_args. + ldr r8, [r0, BA_PHYS_BASE] // Get the phys base in r8 + ldr r9, [r0, BA_VIRT_BASE] // Get the virt base in r9 + ldr r10, [r0, BA_MEM_SIZE] // Get the mem size in r10 + +#define LOAD_PHYS_ADDR(reg, label) \ + LOAD_ADDR(reg, label); \ + sub reg, reg, r9; \ + add reg, reg, r8 + + // Take this opportunity to patch the targets for the exception vectors + LOAD_ADDR(r4, fleh_reset) + LOAD_PHYS_ADDR(r5, ExceptionVectorsTable) + str r4, [r5] + LOAD_ADDR(r4, fleh_undef) + add r5, #4 + str r4, [r5] + LOAD_ADDR(r4, fleh_swi) + add r5, #4 + str r4, [r5] + LOAD_ADDR(r4, fleh_prefabt) + add r5, #4 + str r4, [r5] + LOAD_ADDR(r4, fleh_dataabt) + add r5, #4 + str r4, [r5] + LOAD_ADDR(r4, fleh_addrexc) + add r5, #4 + str r4, [r5] + LOAD_ADDR(r4, fleh_irq) + add r5, #4 + str r4, [r5] + LOAD_ADDR(r4, fleh_decirq) + add r5, #4 + str r4, [r5] + + // arm_init_tramp is sensitive, so for the moment, take the opportunity to store the + // virtual address locally, so that we don't run into issues retrieving it later. + // This is a pretty miserable solution, but it should be enough for the moment + LOAD_ADDR(r4, arm_init_tramp) + adr r5, arm_init_tramp_addr + str r4, [r5] + +#undef LOAD_PHYS_ADDR + + // Set the base of the translation table into the MMU + ldr r4, [r0, BA_TOP_OF_KERNEL_DATA] // Get the top of kernel data + orr r5, r4, #(TTBR_SETUP & 0x00FF) // Setup PTWs memory attribute + orr r5, r5, #(TTBR_SETUP & 0xFF00) // Setup PTWs memory attribute + mcr p15, 0, r5, c2, c0, 0 // write kernel to translation table base 0 + mcr p15, 0, r5, c2, c0, 1 // also to translation table base 1 + mov r5, #TTBCR_N_1GB_TTB0 // identify the split between 0 and 1 + mcr p15, 0, r5, c2, c0, 2 // and set up the translation control reg + + // Mark the entries invalid in the 4 page trampoline translation table + // Mark the entries invalid in the 4 page CPU translation table + // Mark the entries invalid in the one page table for the final 1MB (if used) + // Mark the entries invalid in the one page table for HIGH_EXC_VECTORS + mov r5, r4 // local copy of base + mov r11, #ARM_TTE_TYPE_FAULT // invalid entry template + mov r2, PGBYTES >> 2 // number of ttes/page + add r2, r2, r2, LSL #2 // 8 ttes + 2 ptes to clear. Multiply by 5... + mov r2, r2, LSL #1 // ...then multiply by 2 +invalidate_tte: + str r11, [r5] // store the invalid tte + add r5, r5, #4 // increment tte pointer + subs r2, r2, #1 // decrement count + bne invalidate_tte + + // create default section tte template + mov r6, #ARM_TTE_TYPE_BLOCK // use block mapping entries + mov r7, #(ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_DEFAULT) & 0xFF) + orr r7, r7, #(ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_DEFAULT) & 0xFF00) + orr r7, r7, #(ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_DEFAULT) & 0xF0000) + orr r6, r6, r7 // with default cache attrs + mov r7, #ARM_TTE_BLOCK_AP(AP_RWNA) // Set kernel rw, user no access + orr r7, r7, #(ARM_TTE_BLOCK_AP(AP_RWNA) & 0xFF00) + orr r7, r7, #(ARM_TTE_BLOCK_AP(AP_RWNA) & 0xF0000) + orr r6, r6, r7 // Set RWNA protection + + orr r6, r6, #ARM_TTE_BLOCK_AF // Set access protection + orr r6, r6, #ARM_TTE_BLOCK_SH // Set shareability + + // Set up the V=P mapping for the 1 MB section around the current pc + lsr r7, pc, #ARM_TT_L1_SHIFT // Extract tte index for pc addr + add r5, r4, r7, LSL #2 // convert tte index to tte pointer + lsl r7, r7, #ARM_TT_L1_SHIFT // Truncate pc to 1MB aligned addr + orr r11, r7, r6 // make tte entry value + str r11, [r5] // store tte + + // Set up the virtual mapping for the kernel using 1Mb direct section TTE entries + mov r7, r8 // Save original phys base + add r5, r4, r9, LSR #ARM_TT_L1_SHIFT-2 // convert vaddr to tte pointer + mov r3, #ARM_TT_L1_SIZE // set 1MB boundary + +mapveqp: + cmp r3, r10 // Check if we're beyond the last 1MB section + bgt mapveqpL2 // If so, a coarse entry is required + + orr r11, r7, r6 // make tte entry value + str r11, [r5], #4 // store tte and move to next + add r7, r7, #ARM_TT_L1_SIZE // move to next phys addr + subs r10, r10, #ARM_TT_L1_SIZE // subtract tte size + bne mapveqp + b doneveqp // end is 1MB aligned, and we're done + +mapveqpL2: + // The end is not 1MB aligned, so steal a page and set up L2 entries within + + // Coarse entry first + add r6, r4, PGBYTES * 8 // add L2 offset + mov r11, r6 + + orr r6, #ARM_TTE_TYPE_TABLE // coarse entry + + str r6, [r5] // store coarse tte entry + + // Fill in the L2 entries + mov r5, r11 + + // create pte template + mov r2, #ARM_PTE_TYPE // default pte type + orr r2, r2, #(ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT) & 0xff) // with default cache attrs + orr r2, r2, #(ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT) & 0xff00) + orr r2, r2, #(ARM_PTE_AP(AP_RWNA) & 0xff) // with default cache attrs + orr r2, r2, #(ARM_PTE_AP(AP_RWNA) & 0xff00) + orr r2, r2, #ARM_PTE_AF // Set access + orr r2, r2, #ARM_PTE_SH // Set shareability + +storepte: + orr r11, r7, r2 // make pte entry value + str r11, [r5], #4 // store pte and move to next + add r7, r7, PGBYTES // move to next phys addr + subs r10, r10, PGBYTES // subtract pte size + bne storepte + +doneveqp: + // Insert page table page for high address exception vectors into translation table + mov r5, #0xff000000 // part of virt HIGH_EXC_VECTORS (HACK!) + orr r5, r5, #0x00ff0000 // rest of virt HIGH_EXC_VECTORS (HACK!) + mov r5, r5, LSR #ARM_TT_L1_SHIFT // convert virt addr to index + add r5, r4, r5, LSL #2 // convert to tte pointer + + add r6, r4, PGBYTES * 9 // get page table base (past 4 + 4 + 1 tte/pte pages) + mov r7, #(ARM_TTE_TABLE_MASK & 0xFFFF) // ARM_TTE_TABLE_MASK low halfword + movt r7, #(ARM_TTE_TABLE_MASK >> 16) // ARM_TTE_TABLE_MASK top halfword + and r11, r6, r7 // apply mask + orr r11, r11, #ARM_TTE_TYPE_TABLE // mark it as a coarse page table + str r11, [r5] // store tte entry for page table + + // create pte template + mov r2, #ARM_PTE_TYPE // pte type + orr r2, r2, #(ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT) & 0x00ff) // default cache attrs + orr r2, r2, #(ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT) & 0xff00) + orr r2, r2, #(ARM_PTE_AP(AP_RWNA) & 0x00ff) // set RWNA protection + orr r2, r2, #(ARM_PTE_AP(AP_RWNA) & 0xff00) + orr r2, r2, #ARM_PTE_AF // Set access + orr r2, r2, #ARM_PTE_SH // Set shareability + + // Now initialize the page table entry for the exception vectors + mov r5, #0xff000000 // part of HIGH_EXC_VECTORS + orr r5, r5, #0x00ff0000 // rest of HIGH_EXC_VECTORS + mov r7, #(ARM_TT_L2_INDEX_MASK & 0xFFFF) // ARM_TT_L2_INDEX_MASK low halfword + movt r7, #(ARM_TT_L2_INDEX_MASK >> 16) // ARM_TT_L2_INDEX_MASK top halfword + and r5, r5, r7 // mask for getting index + mov r5, r5, LSR #ARM_TT_L2_SHIFT // get page table index + add r5, r6, r5, LSL #2 // convert to pte pointer + + LOAD_ADDR(r11, ExceptionVectorsBase) // get address of vectors addr + sub r11, r11, r9 // convert to physical address + add r11, r11, r8 + + mov r7, #(ARM_PTE_PAGE_MASK & 0xFFFF) // ARM_PTE_PAGE_MASK low halfword + movt r7, #(ARM_PTE_PAGE_MASK >> 16) // ARM_PTE_PAGE_MASK top halfword + and r11, r11, r7 // insert masked address into pte + orr r11, r11, r2 // add template bits + str r11, [r5] // store pte by base and index + + // clean the dcache + mov r11, #0 +cleanflushway: +cleanflushline: + mcr p15, 0, r11, c7, c14, 2 // cleanflush dcache line by way/set + add r11, r11, #1 << MMU_I7SET // increment set index + tst r11, #1 << (MMU_NSET + MMU_I7SET) // look for overflow + beq cleanflushline + bic r11, r11, #1 << (MMU_NSET + MMU_I7SET) // clear set overflow + adds r11, r11, #1 << MMU_I7WAY // increment way + bcc cleanflushway // loop + +#if __ARM_L2CACHE__ + // Invalidate L2 cache + mov r11, #2 +invall2flushway: +invall2flushline: + mcr p15, 0, r11, c7, c14, 2 // Invalidate dcache line by way/set + add r11, r11, #1 << L2_I7SET // increment set index + tst r11, #1 << (L2_NSET + L2_I7SET) // look for overflow + beq invall2flushline + bic r11, r11, #1 << (L2_NSET + L2_I7SET) // clear set overflow + adds r11, r11, #1 << L2_I7WAY // increment way + bcc invall2flushway // loop + +#endif + + mov r11, #0 + mcr p15, 0, r11, c13, c0, 3 // Write TPIDRURO + LOAD_ADDR(sp, intstack_top) // Get interrupt stack top + sub sp, sp, SS_SIZE // Set stack pointer + sub r0, r0, r8 // Convert to virtual address + add r0, r0, r9 + +join_start: + // kernel page table is setup + // lr set to return handler function virtual address + // r0 set to return handler argument virtual address + // sp set to interrupt context stack virtual address + + // Cpu specific configuration + +#ifdef ARMA7 +#if __ARMA7_SMP__ + mrc p15, 0, r11, c1, c0, 1 + orr r11, r11, #(1<<6) // SMP + mcr p15, 0, r11, c1, c0, 1 + isb +#endif +#endif + + mrs r11, cpsr // Get cpsr + bic r11, #0x100 // Allow async aborts + msr cpsr_x, r11 // Update cpsr + + mov r11, #0 + mcr p15, 0, r11, c8, c7, 0 // invalidate all TLB entries + mcr p15, 0, r11, c7, c5, 0 // invalidate the icache + + // set DACR + mov r11, #(ARM_DAC_SETUP & 0xFFFF) // ARM_DAC_SETUP low halfword + movt r11, #(ARM_DAC_SETUP >> 16) // ARM_DAC_SETUP top halfword + mcr p15, 0, r11, c3, c0, 0 // write to dac register + + // Set PRRR + mov r11, #(PRRR_SETUP & 0xFFFF) // PRRR_SETUP low halfword + movt r11, #(PRRR_SETUP >> 16) // PRRR_SETUP top halfword + mcr p15, 0, r11, c10,c2,0 // write to PRRR register + + // Set NMRR + mov r11, #(NMRR_SETUP & 0xFFFF) // NMRR_SETUP low halfword + movt r11, #(NMRR_SETUP >> 16) // NMRR_SETUP top halfword + mcr p15, 0, r11, c10,c2,1 // write to NMRR register + + // set SCTLR + mrc p15, 0, r11, c1, c0, 0 // read system control + + bic r11, r11, #SCTLR_ALIGN // force off alignment exceptions + mov r7, #(SCTLR_AFE|SCTLR_TRE) // Access flag, TEX remap + orr r7, r7, #(SCTLR_HIGHVEC | SCTLR_ICACHE | SCTLR_PREDIC) + orr r7, r7, #(SCTLR_DCACHE | SCTLR_ENABLE) +#if (__ARM_ENABLE_SWAP__ == 1) + orr r7, r7, #SCTLR_SW // SWP/SWPB Enable +#endif + orr r11, r11, r7 // or in the default settings + mcr p15, 0, r11, c1, c0, 0 // set mmu control + + dsb // ensure mmu settings are inplace + isb // before moving on + +#if __ARM_VFP__ + // Initialize the VFP coprocessors. + mrc p15, 0, r2, c1, c0, 2 // read coprocessor control register + mov r3, #15 // 0xF + orr r2, r2, r3, LSL #20 // enable 10 and 11 + mcr p15, 0, r2, c1, c0, 2 // write coprocessor control register + isb +#endif /* __ARM_VFP__ */ + + // Running virtual. Prepare to call init code + cmp r1, #0 // Test if invoked from start + beq join_start_1 // Branch if yes + ldr r7, arm_init_tramp_addr // Load trampoline address + bx r7 // Branch to virtual trampoline address + + // Loading the virtual address for arm_init_tramp is a rather ugly + // problem. There is probably a better solution, but for the moment, + // patch the address in locally so that loading it is trivial +arm_init_tramp_addr: + .long 0 + .globl EXT(arm_init_tramp) +LEXT(arm_init_tramp) + mrc p15, 0, r5, c2, c0, 0 // Read to translation table base 0 + add r5, r5, PGBYTES * 4 // get kernel page table base (past 4 boot tte pages) + mcr p15, 0, r5, c2, c0, 0 // write kernel to translation table base 0 + mcr p15, 0, r5, c2, c0, 1 // also to translation table base 1 + isb + mov r5, #0 + mcr p15, 0, r5, c8, c7, 0 // Flush all TLB entries + dsb // ensure mmu settings are inplace + isb // before moving on + +join_start_1: +#if __ARM_VFP__ + // Enable VFP for the bootstrap thread context. + // VFP is enabled for the arm_init path as we may + // execute VFP code before we can handle an undef. + fmrx r2, fpexc // get fpexc + orr r2, #FPEXC_EN // set the enable bit + fmxr fpexc, r2 // set fpexc + mov r2, #FPSCR_DEFAULT // set default fpscr + fmxr fpscr, r2 // set fpscr +#endif /* __ARM_VFP__ */ + + mov r7, #0 // Set stack frame 0 + bx lr + +LOAD_ADDR_GEN_DEF(arm_init) +LOAD_ADDR_GEN_DEF(arm_init_cpu) +LOAD_ADDR_GEN_DEF(arm_init_idle_cpu) +LOAD_ADDR_GEN_DEF(arm_init_tramp) +LOAD_ADDR_GEN_DEF(fleh_reset) +LOAD_ADDR_GEN_DEF(ExceptionVectorsTable) +LOAD_ADDR_GEN_DEF(fleh_undef) +LOAD_ADDR_GEN_DEF(fleh_swi) +LOAD_ADDR_GEN_DEF(fleh_prefabt) +LOAD_ADDR_GEN_DEF(fleh_dataabt) +LOAD_ADDR_GEN_DEF(fleh_addrexc) +LOAD_ADDR_GEN_DEF(fleh_irq) +LOAD_ADDR_GEN_DEF(fleh_decirq) + +#include "globals_asm.h" + +/* vim: set ts=4: */ diff --git a/osfmk/arm/status.c b/osfmk/arm/status.c new file mode 100644 index 000000000..94128c4bf --- /dev/null +++ b/osfmk/arm/status.c @@ -0,0 +1,873 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include <debug.h> +#include <mach/mach_types.h> +#include <mach/kern_return.h> +#include <mach/thread_status.h> +#include <kern/thread.h> +#include <kern/kalloc.h> +#include <arm/vmparam.h> +#include <arm/cpu_data_internal.h> +#include <arm/proc_reg.h> + +struct arm_vfpv2_state +{ + __uint32_t __r[32]; + __uint32_t __fpscr; + +}; + +typedef struct arm_vfpv2_state arm_vfpv2_state_t; + +#define ARM_VFPV2_STATE_COUNT ((mach_msg_type_number_t) \ + (sizeof (arm_vfpv2_state_t)/sizeof(uint32_t))) + + +/* + * Forward definitions + */ +void + thread_set_child(thread_t child, int pid); + +void + thread_set_parent(thread_t parent, int pid); + +/* + * Maps state flavor to number of words in the state: + */ +/* __private_extern__ */ +unsigned int _MachineStateCount[] = { + /* FLAVOR_LIST */ 0, + ARM_THREAD_STATE_COUNT, + ARM_VFP_STATE_COUNT, + ARM_EXCEPTION_STATE_COUNT, + ARM_DEBUG_STATE_COUNT +}; + +extern zone_t ads_zone; + +/* + * Routine: machine_thread_get_state + * + */ +kern_return_t +machine_thread_get_state( + thread_t thread, + thread_flavor_t flavor, + thread_state_t tstate, + mach_msg_type_number_t * count) +{ + +#define machine_thread_get_state_kprintf(x...) /* kprintf("machine_thread_get + * _state: " x) */ + + switch (flavor) { + case THREAD_STATE_FLAVOR_LIST: + if (*count < 4) + return (KERN_INVALID_ARGUMENT); + + tstate[0] = ARM_THREAD_STATE; + tstate[1] = ARM_VFP_STATE; + tstate[2] = ARM_EXCEPTION_STATE; + tstate[3] = ARM_DEBUG_STATE; + *count = 4; + break; + + case ARM_THREAD_STATE:{ + struct arm_thread_state *state; + struct arm_saved_state *saved_state; + arm_unified_thread_state_t *unified_state; + + unsigned int i; + if (*count < ARM_THREAD_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + + if (*count == ARM_UNIFIED_THREAD_STATE_COUNT) { + unified_state = (arm_unified_thread_state_t *) tstate; + state = &unified_state->ts_32; + unified_state->ash.flavor = ARM_THREAD_STATE32; + unified_state->ash.count = ARM_THREAD_STATE32_COUNT; + } else { + state = (struct arm_thread_state *) tstate; + } + saved_state = &thread->machine.PcbData; + + state->sp = saved_state->sp; + state->lr = saved_state->lr; + state->pc = saved_state->pc; + state->cpsr = saved_state->cpsr; + for (i = 0; i < 13; i++) + state->r[i] = saved_state->r[i]; + machine_thread_get_state_kprintf("machine_thread_get_state: pc 0x%x r0 0x%x sp 0x%x\n", + state->pc, state->r[0], state->sp); + + if (*count != ARM_UNIFIED_THREAD_STATE_COUNT) { + *count = ARM_THREAD_STATE_COUNT; + } + break; + } + case ARM_EXCEPTION_STATE:{ + struct arm_exception_state *state; + struct arm_saved_state *saved_state; + + if (*count < ARM_EXCEPTION_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + + state = (struct arm_exception_state *) tstate; + saved_state = &thread->machine.PcbData; + + state->exception = saved_state->exception; + state->fsr = saved_state->fsr; + state->far = saved_state->far; + + *count = ARM_EXCEPTION_STATE_COUNT; + break; + } + case ARM_VFP_STATE:{ +#if __ARM_VFP__ + struct arm_vfp_state *state; + struct arm_vfpsaved_state *saved_state; + unsigned int i; + unsigned int max; + + if (*count < ARM_VFP_STATE_COUNT) { + if (*count < ARM_VFPV2_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + else + *count = ARM_VFPV2_STATE_COUNT; + } + + if (*count == ARM_VFPV2_STATE_COUNT) + max = 32; + else + max = 64; + + state = (struct arm_vfp_state *) tstate; + saved_state = find_user_vfp(thread); + + state->fpscr = saved_state->fpscr; + for (i = 0; i < max; i++) + state->r[i] = saved_state->r[i]; + +#endif + break; + } + case ARM_DEBUG_STATE:{ + arm_debug_state_t *state; + arm_debug_state_t *thread_state; + + if (*count < ARM_DEBUG_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + + state = (arm_debug_state_t *) tstate; + thread_state = find_debug_state(thread); + + if (thread_state == NULL) + bzero(state, sizeof(arm_debug_state_t)); + else + bcopy(thread_state, state, sizeof(arm_debug_state_t)); + + *count = ARM_DEBUG_STATE_COUNT; + break; + } + + default: + return (KERN_INVALID_ARGUMENT); + } + return (KERN_SUCCESS); +} + + +/* + * Routine: machine_thread_get_kern_state + * + */ +kern_return_t +machine_thread_get_kern_state( + thread_t thread, + thread_flavor_t flavor, + thread_state_t tstate, + mach_msg_type_number_t * count) +{ + +#define machine_thread_get_kern_state_kprintf(x...) /* kprintf("machine_threa + * d_get_kern_state: " + * x) */ + + /* + * This works only for an interrupted kernel thread + */ + if (thread != current_thread() || getCpuDatap()->cpu_int_state == NULL) + return KERN_FAILURE; + + switch (flavor) { + case ARM_THREAD_STATE:{ + struct arm_thread_state *state; + struct arm_saved_state *saved_state; + unsigned int i; + if (*count < ARM_THREAD_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + + state = (struct arm_thread_state *) tstate; + saved_state = getCpuDatap()->cpu_int_state; + + state->sp = saved_state->sp; + state->lr = saved_state->lr; + state->pc = saved_state->pc; + state->cpsr = saved_state->cpsr; + for (i = 0; i < 13; i++) + state->r[i] = saved_state->r[i]; + machine_thread_get_kern_state_kprintf("machine_thread_get_state: pc 0x%x r0 0x%x sp 0x%x\n", + state->pc, state->r[0], state->sp); + *count = ARM_THREAD_STATE_COUNT; + break; + } + default: + return (KERN_INVALID_ARGUMENT); + } + return (KERN_SUCCESS); +} + +extern long long arm_debug_get(void); + +/* + * Routine: machine_thread_set_state + * + */ +kern_return_t +machine_thread_set_state( + thread_t thread, + thread_flavor_t flavor, + thread_state_t tstate, + mach_msg_type_number_t count) +{ + +#define machine_thread_set_state_kprintf(x...) /* kprintf("machine_thread_set + * _state: " x) */ + + switch (flavor) { + case ARM_THREAD_STATE:{ + struct arm_thread_state *state; + struct arm_saved_state *saved_state; + arm_unified_thread_state_t *unified_state; + int old_psr; + + if (count < ARM_THREAD_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + + if (count == ARM_UNIFIED_THREAD_STATE_COUNT) { + unified_state = (arm_unified_thread_state_t *) tstate; + state = &unified_state->ts_32; + } else { + state = (struct arm_thread_state *) tstate; + } + saved_state = &thread->machine.PcbData; + old_psr = saved_state->cpsr; + memcpy((char *) saved_state, (char *) state, sizeof(*state)); + /* + * do not allow privileged bits of the PSR to be + * changed + */ + saved_state->cpsr = (saved_state->cpsr & ~PSR_USER_MASK) | (old_psr & PSR_USER_MASK); + + machine_thread_set_state_kprintf("machine_thread_set_state: pc 0x%x r0 0x%x sp 0x%x\n", + state->pc, state->r[0], state->sp); + break; + } + case ARM_VFP_STATE:{ +#if __ARM_VFP__ + struct arm_vfp_state *state; + struct arm_vfpsaved_state *saved_state; + unsigned int i; + unsigned int max; + + if (count < ARM_VFP_STATE_COUNT) { + if (count < ARM_VFPV2_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + else + count = ARM_VFPV2_STATE_COUNT; + } + + if (count == ARM_VFPV2_STATE_COUNT) + max = 32; + else + max = 64; + + state = (struct arm_vfp_state *) tstate; + saved_state = find_user_vfp(thread); + + saved_state->fpscr = state->fpscr; + for (i = 0; i < max; i++) + saved_state->r[i] = state->r[i]; + +#endif + break; + } + case ARM_EXCEPTION_STATE:{ + + if (count < ARM_EXCEPTION_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + + break; + } + case ARM_DEBUG_STATE:{ + arm_debug_state_t *state; + arm_debug_state_t *thread_state; + boolean_t enabled = FALSE; + unsigned int i; + + if (count < ARM_DEBUG_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + + state = (arm_debug_state_t *) tstate; + thread_state = find_debug_state(thread); + + if (count < ARM_DEBUG_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + + for (i = 0; i < 16; i++) { + /* do not allow context IDs to be set */ + if (((state->bcr[i] & ARM_DBGBCR_TYPE_MASK) != ARM_DBGBCR_TYPE_IVA) + || ((state->bcr[i] & ARM_DBG_CR_LINKED_MASK) != ARM_DBG_CR_LINKED_UNLINKED) + || ((state->wcr[i] & ARM_DBGBCR_TYPE_MASK) != ARM_DBGBCR_TYPE_IVA) + || ((state->wcr[i] & ARM_DBG_CR_LINKED_MASK) != ARM_DBG_CR_LINKED_UNLINKED)) { + return KERN_PROTECTION_FAILURE; + } + if ((((state->bcr[i] & ARM_DBG_CR_ENABLE_MASK) == ARM_DBG_CR_ENABLE_ENABLE)) + || ((state->wcr[i] & ARM_DBG_CR_ENABLE_MASK) == ARM_DBG_CR_ENABLE_ENABLE)) { + enabled = TRUE; + } + } + + if (!enabled) { + if (thread_state != NULL) + { + void *pTmp = thread->machine.DebugData; + thread->machine.DebugData = NULL; + zfree(ads_zone, pTmp); + } + } + else + { + if (thread_state == NULL) + thread_state = zalloc(ads_zone); + + for (i = 0; i < 16; i++) { + /* set appropriate priviledge; mask out unknown bits */ + thread_state->bcr[i] = (state->bcr[i] & (ARM_DBG_CR_ADDRESS_MASK_MASK + | ARM_DBGBCR_MATCH_MASK + | ARM_DBG_CR_BYTE_ADDRESS_SELECT_MASK + | ARM_DBG_CR_ENABLE_MASK)) + | ARM_DBGBCR_TYPE_IVA + | ARM_DBG_CR_LINKED_UNLINKED + | ARM_DBG_CR_SECURITY_STATE_BOTH + | ARM_DBG_CR_MODE_CONTROL_USER; + thread_state->bvr[i] = state->bvr[i] & ARM_DBG_VR_ADDRESS_MASK; + thread_state->wcr[i] = (state->wcr[i] & (ARM_DBG_CR_ADDRESS_MASK_MASK + | ARM_DBGWCR_BYTE_ADDRESS_SELECT_MASK + | ARM_DBGWCR_ACCESS_CONTROL_MASK + | ARM_DBG_CR_ENABLE_MASK)) + | ARM_DBG_CR_LINKED_UNLINKED + | ARM_DBG_CR_SECURITY_STATE_BOTH + | ARM_DBG_CR_MODE_CONTROL_USER; + thread_state->wvr[i] = state->wvr[i] & ARM_DBG_VR_ADDRESS_MASK; + } + + if (thread->machine.DebugData == NULL) + thread->machine.DebugData = thread_state; + } + + if (thread == current_thread()) { + arm_debug_set(thread_state); + } + + break; + } + + default: + return (KERN_INVALID_ARGUMENT); + } + return (KERN_SUCCESS); +} + +/* + * Routine: machine_thread_state_initialize + * + */ +kern_return_t +machine_thread_state_initialize( + thread_t thread) +{ + struct arm_saved_state *savestate; + + savestate = (struct arm_saved_state *) & thread->machine.PcbData; + bzero((char *) savestate, sizeof(struct arm_saved_state)); + savestate->cpsr = PSR_USERDFLT; + +#if __ARM_VFP__ + vfp_state_initialize(&thread->machine.uVFPdata); + vfp_state_initialize(&thread->machine.kVFPdata); +#endif + + thread->machine.DebugData = NULL; + + return KERN_SUCCESS; +} + +#if __ARM_VFP__ +void +vfp_state_initialize(struct arm_vfpsaved_state *vfp_state) +{ + /* Set default VFP state to RunFast mode: + * + * - flush-to-zero mode + * - default NaN mode + * - no enabled exceptions + * + * On the VFP11, this allows the use of floating point without + * trapping to support code, which we do not provide. With + * the Cortex-A8, this allows the use of the (much faster) NFP + * pipeline for single-precision operations. + */ + + bzero(vfp_state, sizeof(*vfp_state)); + vfp_state->fpscr = FPSCR_DEFAULT; +} +#endif /* __ARM_VFP__ */ + + +/* + * Routine: machine_thread_dup + * + */ +kern_return_t +machine_thread_dup( + thread_t self, + thread_t target) +{ + struct arm_saved_state *self_saved_state; + struct arm_saved_state *target_saved_state; + +#if __ARM_VFP__ + struct arm_vfpsaved_state *self_vfp_state; + struct arm_vfpsaved_state *target_vfp_state; +#endif + + target->machine.cthread_self = self->machine.cthread_self; + target->machine.cthread_data = self->machine.cthread_data; + + self_saved_state = &self->machine.PcbData; + target_saved_state = &target->machine.PcbData; + bcopy(self_saved_state, target_saved_state, sizeof(struct arm_saved_state)); + +#if __ARM_VFP__ + self_vfp_state = &self->machine.uVFPdata; + target_vfp_state = &target->machine.uVFPdata; + bcopy(self_vfp_state, target_vfp_state, sizeof(struct arm_vfpsaved_state)); +#endif + + return (KERN_SUCCESS); +} + +/* + * Routine: get_user_regs + * + */ +struct arm_saved_state * +get_user_regs( + thread_t thread) +{ + return (&thread->machine.PcbData); +} + +/* + * Routine: find_user_regs + * + */ +struct arm_saved_state * +find_user_regs( + thread_t thread) +{ + return get_user_regs(thread); +} + +/* + * Routine: find_kern_regs + * + */ +struct arm_saved_state * +find_kern_regs( + thread_t thread) +{ + /* + * This works only for an interrupted kernel thread + */ + if (thread != current_thread() || getCpuDatap()->cpu_int_state == NULL) + return ((struct arm_saved_state *) NULL); + else + return (getCpuDatap()->cpu_int_state); + +} + +#if __ARM_VFP__ +/* + * Find the user state floating point context. If there is no user state context, + * we just return a 0. + */ + +struct arm_vfpsaved_state * +find_user_vfp( + thread_t thread) +{ + return &thread->machine.uVFPdata; +} +#endif /* __ARM_VFP__ */ + +arm_debug_state_t * +find_debug_state( + thread_t thread) +{ + return thread->machine.DebugData; +} + +/* + * Routine: thread_userstack + * + */ +kern_return_t +thread_userstack( + __unused thread_t thread, + int flavor, + thread_state_t tstate, + unsigned int count, + mach_vm_offset_t * user_stack, + int *customstack, + __unused boolean_t is64bit +) +{ + + switch (flavor) { + case ARM_THREAD_STATE: + { + struct arm_thread_state *state; + + + if (count < ARM_THREAD_STATE_COUNT) + return (KERN_INVALID_ARGUMENT); + + if (customstack) + *customstack = 0; + state = (struct arm_thread_state *) tstate; + + if (state->sp) { + *user_stack = CAST_USER_ADDR_T(state->sp); + if (customstack) + *customstack = 1; + } else { + *user_stack = CAST_USER_ADDR_T(USRSTACK); + } + } + break; + + default: + return (KERN_INVALID_ARGUMENT); + } + + return (KERN_SUCCESS); +} + +/* + * thread_userstackdefault: + * + * Return the default stack location for the + * thread, if otherwise unknown. + */ +kern_return_t +thread_userstackdefault( + mach_vm_offset_t *default_user_stack, + boolean_t is64bit __unused) +{ + *default_user_stack = USRSTACK; + + return (KERN_SUCCESS); +} + +/* + * Routine: thread_setuserstack + * + */ +void +thread_setuserstack(thread_t thread, mach_vm_address_t user_stack) +{ + struct arm_saved_state *sv; + +#define thread_setuserstack_kprintf(x...) /* kprintf("thread_setuserstac + * k: " x) */ + + sv = get_user_regs(thread); + + sv->sp = user_stack; + + thread_setuserstack_kprintf("stack %x\n", sv->sp); + + return; +} + +/* + * Routine: thread_adjuserstack + * + */ +uint64_t +thread_adjuserstack(thread_t thread, int adjust) +{ + struct arm_saved_state *sv; + + sv = get_user_regs(thread); + + sv->sp += adjust; + + return sv->sp; +} + +/* + * Routine: thread_setentrypoint + * + */ +void +thread_setentrypoint(thread_t thread, mach_vm_offset_t entry) +{ + struct arm_saved_state *sv; + +#define thread_setentrypoint_kprintf(x...) /* kprintf("thread_setentrypoi + * nt: " x) */ + + sv = get_user_regs(thread); + + sv->pc = entry; + + thread_setentrypoint_kprintf("entry %x\n", sv->pc); + + return; +} + +/* + * Routine: thread_entrypoint + * + */ +kern_return_t +thread_entrypoint( + __unused thread_t thread, + int flavor, + thread_state_t tstate, + __unused unsigned int count, + mach_vm_offset_t * entry_point +) +{ + switch (flavor) { + case ARM_THREAD_STATE: + { + struct arm_thread_state *state; + + state = (struct arm_thread_state *) tstate; + + /* + * If a valid entry point is specified, use it. + */ + if (state->pc) { + *entry_point = CAST_USER_ADDR_T(state->pc); + } else { + *entry_point = CAST_USER_ADDR_T(VM_MIN_ADDRESS); + } + } + break; + + default: + return (KERN_INVALID_ARGUMENT); + } + + return (KERN_SUCCESS); +} + + +/* + * Routine: thread_set_child + * + */ +void +thread_set_child( + thread_t child, + int pid) +{ + struct arm_saved_state *child_state; + + child_state = get_user_regs(child); + + child_state->r[0] = (uint_t) pid; + child_state->r[1] = 1ULL; +} + + +/* + * Routine: thread_set_parent + * + */ +void +thread_set_parent( + thread_t parent, + int pid) +{ + struct arm_saved_state *parent_state; + + parent_state = get_user_regs(parent); + + parent_state->r[0] = pid; + parent_state->r[1] = 0; +} + + +struct arm_act_context { + struct arm_saved_state ss; +#if __ARM_VFP__ + struct arm_vfpsaved_state vfps; +#endif +}; + +/* + * Routine: act_thread_csave + * + */ +void * +act_thread_csave(void) +{ + struct arm_act_context *ic; + kern_return_t kret; + unsigned int val; + + ic = (struct arm_act_context *) kalloc(sizeof(struct arm_act_context)); + + if (ic == (struct arm_act_context *) NULL) + return ((void *) 0); + + val = ARM_THREAD_STATE_COUNT; + kret = machine_thread_get_state(current_thread(), + ARM_THREAD_STATE, + (thread_state_t) & ic->ss, + &val); + if (kret != KERN_SUCCESS) { + kfree(ic, sizeof(struct arm_act_context)); + return ((void *) 0); + } +#if __ARM_VFP__ + val = ARM_VFP_STATE_COUNT; + kret = machine_thread_get_state(current_thread(), + ARM_VFP_STATE, + (thread_state_t) & ic->vfps, + &val); + if (kret != KERN_SUCCESS) { + kfree(ic, sizeof(struct arm_act_context)); + return ((void *) 0); + } +#endif + return (ic); +} + +/* + * Routine: act_thread_catt + * + */ +void +act_thread_catt(void *ctx) +{ + struct arm_act_context *ic; + kern_return_t kret; + + ic = (struct arm_act_context *) ctx; + + if (ic == (struct arm_act_context *) NULL) + return; + + kret = machine_thread_set_state(current_thread(), + ARM_THREAD_STATE, + (thread_state_t) & ic->ss, + ARM_THREAD_STATE_COUNT); + if (kret != KERN_SUCCESS) + goto out; + +#if __ARM_VFP__ + kret = machine_thread_set_state(current_thread(), + ARM_VFP_STATE, + (thread_state_t) & ic->vfps, + ARM_VFP_STATE_COUNT); + if (kret != KERN_SUCCESS) + goto out; +#endif +out: + kfree(ic, sizeof(struct arm_act_context)); +} + +/* + * Routine: act_thread_catt + * + */ +void +act_thread_cfree(void *ctx) +{ + kfree(ctx, sizeof(struct arm_act_context)); +} + +kern_return_t +thread_set_wq_state32(thread_t thread, thread_state_t tstate) +{ + arm_thread_state_t *state; + struct arm_saved_state *saved_state; + thread_t curth = current_thread(); + spl_t s=0; + + saved_state = &thread->machine.PcbData; + state = (arm_thread_state_t *)tstate; + + if (curth != thread) { + s = splsched(); + thread_lock(thread); + } + + /* + * do not zero saved_state, it can be concurrently accessed + * and zero is not a valid state for some of the registers, + * like sp. + */ + thread_state32_to_saved_state(state, saved_state); + saved_state->cpsr = PSR_USERDFLT; + + if (curth != thread) { + thread_unlock(thread); + splx(s); + } + + return KERN_SUCCESS; +} diff --git a/osfmk/arm/status_shared.c b/osfmk/arm/status_shared.c new file mode 100644 index 000000000..42d7a1235 --- /dev/null +++ b/osfmk/arm/status_shared.c @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2012 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include <debug.h> +#include <mach/mach_types.h> +#include <mach/kern_return.h> +#include <mach/thread_status.h> +#include <kern/thread.h> +#include <kern/kalloc.h> +#include <arm/vmparam.h> +#include <arm/cpu_data_internal.h> + +/* + * Copy values from saved_state to ts32. + */ +void +saved_state_to_thread_state32(const arm_saved_state_t *saved_state, arm_thread_state32_t *ts32) +{ + uint32_t i; + + assert(is_saved_state32(saved_state)); + + ts32->lr = (uint32_t)get_saved_state_lr(saved_state); + ts32->sp = (uint32_t)get_saved_state_sp(saved_state); + ts32->pc = (uint32_t)get_saved_state_pc(saved_state); + ts32->cpsr = get_saved_state_cpsr(saved_state); + for (i = 0; i < 13; i++) + ts32->r[i] = (uint32_t)get_saved_state_reg(saved_state, i); +} + +/* + * Copy values from ts32 to saved_state. + */ +void +thread_state32_to_saved_state(const arm_thread_state32_t *ts32, arm_saved_state_t *saved_state) +{ + uint32_t i; + + assert(is_saved_state32(saved_state)); + + set_saved_state_lr(saved_state, ts32->lr); + set_saved_state_sp(saved_state, ts32->sp); + set_saved_state_pc(saved_state, ts32->pc); + +#if defined(__arm64__) + set_saved_state_cpsr(saved_state, (ts32->cpsr & ~PSR64_MODE_MASK) | PSR64_MODE_RW_32); +#elif defined(__arm__) + set_saved_state_cpsr(saved_state, (ts32->cpsr & ~PSR_USER_MASK) | (ts32->cpsr & PSR_USER_MASK)); +#else +#error Unknown architecture. +#endif + + for (i = 0; i < 13; i++) + set_saved_state_reg(saved_state, i, ts32->r[i]); +} + + diff --git a/osfmk/arm/strlcpy.c b/osfmk/arm/strlcpy.c new file mode 100644 index 000000000..e06f4173b --- /dev/null +++ b/osfmk/arm/strlcpy.c @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2011 Apple, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include "string.h" + +#undef strlcpy +size_t +strlcpy(char * dst, const char * src, size_t maxlen) { + const size_t srclen = strlen(src); + if (srclen + 1 < maxlen) { + memcpy(dst, src, srclen + 1); + } else if (maxlen != 0) { + memcpy(dst, src, maxlen - 1); + dst[maxlen-1] = '\0'; + } + return srclen; +} diff --git a/osfmk/arm/strlen.s b/osfmk/arm/strlen.s new file mode 100644 index 000000000..05d779709 --- /dev/null +++ b/osfmk/arm/strlen.s @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2011 Apple, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include <arm/arch.h> +.syntax unified +.code 32 +.globl _strlen + +#define addr r0 +#define word r1 +#define temp r2 +#define mask r3 +#define save ip +#define indx r0 + +.macro IfWordDoesntContainNUL_SetZ +#if defined _ARM_ARCH_6 +// In each word of the string, we check for NUL bytes via a saturating +// unsigned subtraction of each byte from 0x1. The result of this is +// non-zero if and only if the corresponding byte in the string is NUL. +// Simply using a TST instruction checks all four bytes for NULs in one +// go. + uqsub8 temp, mask, word + tst temp, temp +#else +// If we're on armv5, we do not have the uqsub8 instruction, so we need +// to use a different test for NUL. Instead, we compute: +// +// byte - 0x1 & ~byte +// +// and test the high-order bit. If it is set, then byte is NUL. Just +// as with the other test, this can be applied simultaneously to all +// bytes in a word. + sub temp, word, mask + bic temp, temp, word + tst temp, mask, lsl #7 +#endif +.endm + +.text +.align 4 +.long 0x0 // padding +.long 0x01010101 // mask for use in finding NULs +_strlen: +// Establish stack frame, load mask that we will use to find NUL bytes, +// and set aside a copy of the pointer to the string. + push {r7,lr} + mov r7, sp + ldr mask, (_strlen-4) + add save, addr, #4 + +// Load the aligned word that contains the start of the string, then OR +// 0x01 into any bytes that preceed the start of the string to prevent +// false positives when we check for NUL bytes. + and temp, addr, #3 + bic addr, addr, #3 + lsl temp, temp, #3 + ldr word, [addr], #4 + rsb temp, temp, #32 + orr word, word, mask, lsr temp + +// Check if the string ends in the first word. If so, don't load any +// more of the string; instead jump directly to the cleanup code. + IfWordDoesntContainNUL_SetZ + bne 1f + +.align 4 +// Load one word of the string on each iteration, and check it for NUL +// bytes. If a NUL is found, fall through into the cleanup code. +0: ldr word, [addr], #4 + IfWordDoesntContainNUL_SetZ + beq 0b + +// The last word that we loaded contained a NUL. Subtracting the saved +// pointer from the current pointer gives us the number of bytes from +// the start of the string to the word containing the NUL. +1: sub indx, addr, save +#if defined _ARM_ARCH_6 +// To that we add the index of the first NUL byte in the word, computed +// using REV and CLZ followed by a shift. + rev temp, temp + clz temp, temp + add indx, indx, temp, lsr #3 +#else +// armv5 does not have the REV instruction, so instead we find the +// index of the NUL byte in word with a linear search. + tst word, #0x000000ff + addne indx, #1 + tstne word, #0x0000ff00 + addne indx, #1 + tstne word, #0x00ff0000 + addne indx, #1 +#endif + pop {r7,pc} diff --git a/osfmk/arm/strncmp.s b/osfmk/arm/strncmp.s new file mode 100644 index 000000000..a58185565 --- /dev/null +++ b/osfmk/arm/strncmp.s @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2010, 2011 Apple, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +.text +.syntax unified +.code 32 +.globl _strncmp +// int strncmp(const char *s1, const char *s2, size_t n); +// +// Returns zero if the two NUL-terminated strings s1 and s2 are equal up to +// n characters. Otherwise, returns the difference between the first two +// characters that do not match, interpreted as unsigned integers. + +#define ESTABLISH_FRAME \ + push {r4-r7,lr} ;\ + add r7, sp, #12 ;\ + push {r8,r10} +#define CLEAR_FRAME \ + pop {r8,r10} ;\ + pop {r4-r7,lr} + +.align 3 +.long 0, 0x01010101 +_strncmp: +// If n < 16, jump straight to the byte-by-byte comparison loop. + cmp r2, #16 + blo L_byteCompareLoop +// Load a character from each string and advance the pointers. If the loaded +// characters are unequal or NUL, return their difference. +0: ldrb r3, [r0],#1 + ldrb ip, [r1],#1 + sub r2, #1 + cmp r3, #1 + cmphs r3, ip + bne L_earlyReturn +// If the address of the next character from s1 does not have word alignment, +// continue with the character-by-character comparison. Otherwise, fall +// through into the word-by-word comparison path. + tst r0, #3 + bne 0b + +// We have not encountered a NUL or a mismatch, and s1 has word alignment. +// Establish a frame, since we're going to need additional registers anyway. + ESTABLISH_FRAME + ldr lr, (_strncmp-4) + +// Word align s2, and place the remainder in r10. Compute the right- and +// left-shifts to extract each word that we will compare to the other source +// from the aligned words that we load: +// +// aligned s2 to be loaded on next iteration +// | "true" s2 | +// v v v +// +---+---+---+---+ +---+---+---+---+ +// | 0 | 1 | 2 | 3 | | 4 | 5 | 6 | 7 | +// +---+---+---+---+ +---+---+---+---+ +// ^-----------------^ +// to be compared on next iteration + and r10, r1, #3 + bic r1, r1, #3 + mov r10, r10, lsl #3 + rsb r6, r10,#32 + +// Subtract the number of bytes of the initial word load from s2 that will +// actually be used from n. + sub r2, r2, r6, lsr #3 + +// Load the first aligned word of s2. OR 0x01 into any bytes that preceed the +// "true s2", to prevent our check for NUL from generating a false positive. +// Then check for NUL, and jump to the byte-by-byte comparison loop after +// unwinding the pointers if we enounter one. + ldr r8, [r1],#4 + orr r8, r8, lr, lsr r6 + sub r3, r8, lr + bic r3, r3, r8 + tst r3, lr, lsl #7 + mov r5, r8, lsr r10 + bne L_unwindLoopPreload + +.align 3 +L_wordCompareLoop: +// If n < 4, abort the word compare loop before we load any more data. + subs r2, r2, #4 + blo L_nIsLessThanFour +// Load the next aligned word of s2 and check if it contains any NUL bytes. +// Load the next aligned word of s1, and extract the corresponding bytes from +// the two words of s2 loaded in this and the previous iteration of the loop. +// Compare these two words. +// If no NUL or mismatched words have been encountered, continue the loop. + ldr r8, [r1],#4 +#if defined _ARM_ARCH_6 + uqsub8 r3, lr, r8 + tst r3, r3 + ldr ip, [r0],#4 +#else + sub r3, r8, lr + bic r3, r3, r8 + ldr ip, [r0],#4 + tst r3, lr, lsl #7 +#endif + orr r4, r5, r8, lsl r6 + cmpeq ip, r4 + mov r5, r8, lsr r10 + beq L_wordCompareLoop + +// Either we have encountered a NUL, or we have found a mismatch between s1 +// and s2. Unwind the pointers and use a byte-by-byte comparison loop. + sub r0, r0, #4 + sub r1, r1, #4 +L_nIsLessThanFour: + add r2, r2, #4 +L_unwindLoopPreload: + sub r1, r1, r6, lsr #3 + add r2, r2, r6, lsr #3 + CLEAR_FRAME + +L_byteCompareLoop: +// If n-- == 0, we have exhausted the allowed number of comparisons, and need +// to return zero without additional loads. + subs r2, r2, #1 + movlo r0, #0 + bxlo lr +// Load a character from each string and advance the pointers. If the loaded +// characters are unequal or NUL, return their difference. + ldrb r3, [r0],#1 + ldrb ip, [r1],#1 + cmp r3, #1 + cmpcs r3, ip + beq L_byteCompareLoop + +L_earlyReturn: +// Return the difference between the last two characters loaded. + sub r0, r3, ip + bx lr diff --git a/osfmk/arm/strncpy.c b/osfmk/arm/strncpy.c new file mode 100644 index 000000000..5ee1847a6 --- /dev/null +++ b/osfmk/arm/strncpy.c @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2011 Apple, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include "string.h" + +#undef strncpy +char * +strncpy(char * dst, const char * src, size_t maxlen) { + const size_t srclen = strnlen(src, maxlen); + if (srclen < maxlen) { + memcpy(dst, src, srclen); + memset(dst+srclen, 0, maxlen - srclen); + } else { + memcpy(dst, src, maxlen); + } + return dst; +} diff --git a/osfmk/arm/strnlen.s b/osfmk/arm/strnlen.s new file mode 100644 index 000000000..49788471f --- /dev/null +++ b/osfmk/arm/strnlen.s @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2011 Apple, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include <arm/arch.h> +.syntax unified +.code 32 +.globl _strnlen + +#define addr r0 +#define maxl r1 +#define temp r2 +#define mask r3 +#define save ip +#define word lr +#define byte lr +#define indx r0 + +.macro IfHS_and_WordDoesntContainNUL_SetZ +#if defined _ARM_ARCH_6 +// In each word of the string, we check for NUL bytes via a saturating +// unsigned subtraction of each byte from 0x1. The result of this is +// non-zero if and only if the corresponding byte in the string is NUL. +// Simply using a TST instruction checks all four bytes for NULs in one +// go. + uqsub8 temp, mask, word + tsths temp, temp +#else +// If we're on armv5, we do not have the uqsub8 instruction, so we need +// to use a different test for NUL. Instead, we compute: +// +// byte - 0x1 & ~byte +// +// and test the high-order bit. If it is set, then byte is NUL. Just +// as with the other test, this can be applied simultaneously to all +// bytes in a word. + sub temp, word, mask + bic temp, temp, word + tsths temp, mask, lsl #7 +#endif +.endm + +.text +.align 3 +.long 0x0 // padding +.long 0x01010101 // mask for use in finding NULs +_strnlen: +// Establish stack frame, load mask that we will use to find NUL bytes, +// and set aside a copy of the pointer to the string. Subtract 4 from +// the maxlen, and jump into a byte-by-byte search if this requires a +// borrow, as we cannot use a word-by-word search in that case. + push {r7,lr} + mov r7, sp + ldr mask, (_strnlen-4) + add save, addr, #4 + subs maxl, maxl, #4 + blo L_bytewiseSearch + +// Load the aligned word that contains the start of the string, then OR +// 0x01 into any bytes that preceed the start to prevent false positives +// when we check for NUL bytes. Additionally, add the number of unused +// bytes to maxlen. + and temp, addr, #3 + bic addr, addr, #3 + add maxl, maxl, temp + lsl temp, temp, #3 + ldr word, [addr], #4 + rsb temp, temp, #32 + orr word, word, mask, lsr temp + + subs maxl, maxl, #4 + IfHS_and_WordDoesntContainNUL_SetZ + bne 1f + +.align 4 +0: ldr word, [addr], #4 + subs maxl, maxl, #4 + IfHS_and_WordDoesntContainNUL_SetZ + beq 0b + +.align 4 +// Either the last word that we loaded contained a NUL, or we will +// exceed maxlen before we finish the next word in the string. Determine +// which case we are in by repeating the check for NUL, and branch if +// there was not a NUL byte. Padding ensures that we don't have two +// branches in a single 16-byte fetch group, as this interferes with +// branch prediction on Swift. +1: tst temp, temp + beq L_bytewiseSearch + +// The last word that we loaded contained a NUL. Subtracting the saved +// pointer from the current pointer gives us the number of bytes from +// the start of the string to the word containing the NUL. + sub indx, addr, save +#if defined _ARM_ARCH_6 +// To that we add the index of the first NUL byte in the word, computed +// using REV and CLZ followed by a shift. + rev temp, temp + clz temp, temp + add indx, indx, temp, lsr #3 +#else +// armv5 does not have the REV instruction, so instead we find the +// index of the NUL byte in word with a linear search. + tst word, #0x000000ff + addne indx, #1 + tstne word, #0x0000ff00 + addne indx, #1 + tstne word, #0x00ff0000 + addne indx, #1 +#endif + pop {r7,pc} + +.align 4 +L_bytewiseSearch: +// Restore maxlen (the last thing that happened before we branched here +// was that we subtracted 4 from maxlen), and adjust the saved string +// pointer. Then we do a simple byte-by-byte search until we either +// reach the end of the string or maxlen reaches zero, at which point +// the length to return is simply the difference between the current +// and saved pointers. + adds maxl, maxl, #4 + sub save, save, #4 + beq 1f +0: ldrb byte, [addr] + cmp byte, #0 + addhi addr, #1 + subshi maxl, #1 + bhi 0b +1: sub indx, addr, save + pop {r7,pc} diff --git a/osfmk/arm/task.h b/osfmk/arm/task.h new file mode 100644 index 000000000..e545bd36c --- /dev/null +++ b/osfmk/arm/task.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ + +/* + * Mach Operating System + * Copyright (c) 1991,1990,1989 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ + +/* + * Machine dependant task fields + */ + +#define MACHINE_TASK \ + void* task_debug; + + diff --git a/osfmk/arm/thread.h b/osfmk/arm/thread.h new file mode 100644 index 000000000..e270512a2 --- /dev/null +++ b/osfmk/arm/thread.h @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990,1989 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ +/* + */ + +#ifndef _ARM_THREAD_H_ +#define _ARM_THREAD_H_ + +#include <mach/mach_types.h> +#include <mach/boolean.h> +#include <mach/arm/vm_types.h> +#include <mach/thread_status.h> + +#ifdef MACH_KERNEL_PRIVATE +#include <arm/cpu_data.h> +#include <arm/proc_reg.h> +#endif + +#if __ARM_VFP__ + +#define VFPSAVE_ALIGN 16 +#define VFPSAVE_ATTRIB __attribute__ ((aligned (VFPSAVE_ALIGN))) +#define THREAD_ALIGN VFPSAVE_ALIGN + +/* + * vector floating point saved state + */ +struct arm_vfpsaved_state { + uint32_t r[64]; + uint32_t fpscr; + uint32_t fpexc; +}; +#endif + +struct perfcontrol_state { + uint64_t opaque[8] __attribute__((aligned(8))); +}; + +/* + * Maps state flavor to number of words in the state: + */ +extern unsigned int _MachineStateCount[]; + +#ifdef MACH_KERNEL_PRIVATE +#if __arm64__ +typedef arm_context_t machine_thread_kernel_state; +#else +typedef struct arm_saved_state machine_thread_kernel_state; +#endif +#include <kern/thread_kernel_state.h> + +struct machine_thread { +#if __arm64__ + arm_context_t *contextData; /* allocated user context */ + arm_saved_state_t *upcb; /* pointer to user GPR state */ + arm_neon_saved_state_t *uNeon; /* pointer to user VFP state */ +#elif __arm__ + struct arm_saved_state PcbData; +#if __ARM_VFP__ + struct arm_vfpsaved_state uVFPdata VFPSAVE_ATTRIB; + struct arm_vfpsaved_state kVFPdata VFPSAVE_ATTRIB; +#endif /* __ARM_VFP__ */ + +#else +#error Unknown arch +#endif +#if __ARM_USER_PROTECT__ + unsigned int uptw_ttc; + unsigned int uptw_ttb; + unsigned int kptw_ttb; + unsigned int asid; +#endif + + vm_offset_t kstackptr; /* top of kernel stack */ + struct cpu_data *CpuDatap; /* current per cpu data */ + unsigned int preemption_count; /* preemption count */ + + arm_debug_state_t *DebugData; + mach_vm_address_t cthread_self; /* for use of cthread package */ + mach_vm_address_t cthread_data; /* for use of cthread package */ + + struct perfcontrol_state perfctrl_state; +#if __arm64__ + uint64_t energy_estimate_nj; +#endif + +#if INTERRUPT_MASKED_DEBUG + uint64_t intmask_timestamp; /* timestamp of when interrupts were masked */ +#endif +}; +#endif + +extern struct arm_saved_state *get_user_regs(thread_t); +extern struct arm_saved_state *find_user_regs(thread_t); +extern struct arm_saved_state *find_kern_regs(thread_t); +extern struct arm_vfpsaved_state *find_user_vfp(thread_t); +#if defined(__arm__) +extern arm_debug_state_t *find_debug_state(thread_t); +#elif defined(__arm64__) +extern arm_debug_state32_t *find_debug_state32(thread_t); +extern arm_debug_state64_t *find_debug_state64(thread_t); +extern arm_neon_saved_state_t *get_user_neon_regs(thread_t); +#else +#error unknown arch +#endif + +#define FIND_PERFCONTROL_STATE(th) (&th->machine.perfctrl_state) + +#ifdef MACH_KERNEL_PRIVATE +#if __ARM_VFP__ +extern void vfp_state_initialize(struct arm_vfpsaved_state *vfp_state); +extern void vfp_save(struct arm_vfpsaved_state *vfp_ss); +extern void vfp_load(struct arm_vfpsaved_state *vfp_ss); +extern void toss_live_vfp(void *vfp_fc); +#endif /* __ARM_VFP__ */ +extern void arm_debug_set(arm_debug_state_t *debug_state); +#if defined(__arm64__) +extern void arm_debug_set32(arm_debug_state_t *debug_state); +extern void arm_debug_set64(arm_debug_state_t *debug_state); + +kern_return_t handle_get_arm_thread_state( + thread_state_t tstate, + mach_msg_type_number_t * count, + const arm_saved_state_t *saved_state); +kern_return_t handle_get_arm32_thread_state( + thread_state_t tstate, + mach_msg_type_number_t * count, + const arm_saved_state_t *saved_state); +kern_return_t handle_get_arm64_thread_state( + thread_state_t tstate, + mach_msg_type_number_t * count, + const arm_saved_state_t *saved_state); + +kern_return_t handle_set_arm_thread_state( + const thread_state_t tstate, + mach_msg_type_number_t count, + arm_saved_state_t *saved_state); +kern_return_t handle_set_arm32_thread_state( + const thread_state_t tstate, + mach_msg_type_number_t count, + arm_saved_state_t *saved_state); +kern_return_t handle_set_arm64_thread_state( + const thread_state_t tstate, + mach_msg_type_number_t count, + arm_saved_state_t *saved_state); +#endif +#endif /* MACH_KERNEL_PRIVATE */ + +extern void *act_thread_csave(void); +extern void act_thread_catt(void *ctx); +extern void act_thread_cfree(void *ctx); + +/* + * Return address of the function that called current function, given + * address of the first parameter of current function. + */ +#define GET_RETURN_PC(addr) (((vm_offset_t *)0)) + +/* + * Defining this indicates that MD code will supply an exception() + * routine, conformant with kern/exception.c (dependency alert!) + * but which does wonderfully fast, machine-dependent magic. + */ +#define MACHINE_FAST_EXCEPTION 1 + +#endif /* _ARM_THREAD_H_ */ diff --git a/osfmk/arm/trap.c b/osfmk/arm/trap.c new file mode 100644 index 000000000..617c652ab --- /dev/null +++ b/osfmk/arm/trap.c @@ -0,0 +1,897 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include <kern/debug.h> +#include <mach_kdp.h> +#include <machine/endian.h> +#include <mach/mach_types.h> +#include <mach/boolean.h> +#include <mach/vm_prot.h> +#include <mach/vm_types.h> +#include <mach/mach_traps.h> + +#include <mach/exception.h> +#include <mach/kern_return.h> +#include <mach/vm_param.h> +#include <mach/message.h> +#include <mach/machine/thread_status.h> + +#include <vm/vm_page.h> +#include <vm/pmap.h> +#include <vm/vm_fault.h> +#include <vm/vm_kern.h> + +#include <kern/ast.h> +#include <kern/thread.h> +#include <kern/task.h> +#include <kern/sched_prim.h> + +#include <sys/kdebug.h> + +#include <arm/trap.h> +#include <arm/caches_internal.h> +#include <arm/cpu_data_internal.h> +#include <arm/machdep_call.h> +#include <arm/machine_routines.h> +#include <arm/misc_protos.h> +#include <arm/setjmp.h> +#include <arm/proc_reg.h> + +/* + * External function prototypes. + */ +#include <kern/syscall_sw.h> +#include <kern/host.h> +#include <kern/processor.h> + + +#if CONFIG_DTRACE +extern kern_return_t dtrace_user_probe(arm_saved_state_t* regs, unsigned int instr); +extern boolean_t dtrace_tally_fault(user_addr_t); + +/* Traps for userland processing. Can't include bsd/sys/fasttrap_isa.h, so copy and paste the trap instructions + over from that file. Need to keep these in sync! */ +#define FASTTRAP_ARM_INSTR 0xe7ffdefc +#define FASTTRAP_THUMB_INSTR 0xdefc + +#define FASTTRAP_ARM_RET_INSTR 0xe7ffdefb +#define FASTTRAP_THUMB_RET_INSTR 0xdefb + +/* See <rdar://problem/4613924> */ +perfCallback tempDTraceTrapHook = NULL; /* Pointer to DTrace fbt trap hook routine */ +#endif + +#define COPYIN(dst, src, size) \ + ((regs->cpsr & PSR_MODE_MASK) != PSR_USER_MODE) ? \ + copyin_kern(dst, src, size) \ + : \ + copyin(dst, src, size) + +#define COPYOUT(src, dst, size) \ + ((regs->cpsr & PSR_MODE_MASK) != PSR_USER_MODE) ? \ + copyout_kern(src, dst, size) \ + : \ + copyout(src, dst, size) + +/* Second-level exception handlers forward declarations */ +void sleh_undef(struct arm_saved_state *, struct arm_vfpsaved_state *); +void sleh_abort(struct arm_saved_state *, int); +static kern_return_t sleh_alignment(struct arm_saved_state *); +static void panic_with_thread_kernel_state(const char *msg, arm_saved_state_t *regs); + + +volatile perfCallback perfTrapHook = NULL; /* Pointer to CHUD trap hook routine */ + +int sleh_alignment_count = 0; +int trap_on_alignment_fault = 0; + +/* + * Routine: sleh_undef + * Function: Second level exception handler for undefined exception + */ + +void +sleh_undef(struct arm_saved_state * regs, struct arm_vfpsaved_state * vfp_ss __unused) +{ + exception_type_t exception = EXC_BAD_INSTRUCTION; + mach_exception_data_type_t code[2] = {EXC_ARM_UNDEFINED}; + mach_msg_type_number_t codeCnt = 2; + thread_t thread = current_thread(); + vm_offset_t recover; + + recover = thread->recover; + thread->recover = 0; + + getCpuDatap()->cpu_stat.undef_ex_cnt++; + + /* Inherit the interrupt masks from previous */ + if (!(regs->cpsr & PSR_INTMASK)) + ml_set_interrupts_enabled(TRUE); + +#if CONFIG_DTRACE + if (tempDTraceTrapHook) { + if (tempDTraceTrapHook(exception, regs, 0, 0) == KERN_SUCCESS) { + /* + * If it succeeds, we are done... + */ + goto exit; + } + } + + /* Check to see if we've hit a userland probe */ + if ((regs->cpsr & PSR_MODE_MASK) == PSR_USER_MODE) { + if (regs->cpsr & PSR_TF) { + uint16_t instr; + + if(COPYIN((user_addr_t)(regs->pc), (char *)&instr,(vm_size_t)(sizeof(uint16_t))) != KERN_SUCCESS) + goto exit; + + if (instr == FASTTRAP_THUMB_INSTR || instr == FASTTRAP_THUMB_RET_INSTR) { + if (dtrace_user_probe(regs, instr) == KERN_SUCCESS) + /* If it succeeds, we are done... */ + goto exit; + } + } else { + uint32_t instr; + + if(COPYIN((user_addr_t)(regs->pc), (char *)&instr,(vm_size_t)(sizeof(uint32_t))) != KERN_SUCCESS) + goto exit; + + if (instr == FASTTRAP_ARM_INSTR || instr == FASTTRAP_ARM_RET_INSTR) { + if (dtrace_user_probe(regs, instr) == KERN_SUCCESS) + /* If it succeeds, we are done... */ + goto exit; + } + } + } +#endif /* CONFIG_DTRACE */ + + + if (regs->cpsr & PSR_TF) { + unsigned short instr; + + if(COPYIN((user_addr_t)(regs->pc), (char *)&instr,(vm_size_t)(sizeof(unsigned short))) != KERN_SUCCESS) + goto exit; + + if (IS_THUMB32(instr)) { + unsigned int instr32; + + instr32 = (instr<<16); + + if(COPYIN((user_addr_t)(((unsigned short *) (regs->pc))+1), (char *)&instr,(vm_size_t)(sizeof(unsigned short))) != KERN_SUCCESS) + goto exit; + + instr32 |= instr; + code[1] = instr32; + +#if __ARM_VFP__ + if (IS_THUMB_VFP(instr32)) { + /* We no longer manage FPEXC beyond bootstrap, so verify that VFP is still enabled. */ + if (!get_vfp_enabled()) + panic("VFP was disabled (thumb); VFP should always be enabled"); + } +#endif + } else { + /* I don't believe we have any 16 bit VFP instructions, so just set code[1]. */ + code[1] = instr; + + if (IS_THUMB_GDB_TRAP(instr)) { + exception = EXC_BREAKPOINT; + code[0] = EXC_ARM_BREAKPOINT; + } + } + } else { + uint32_t instr; + + if(COPYIN((user_addr_t)(regs->pc), (char *)&instr,(vm_size_t)(sizeof(uint32_t))) != KERN_SUCCESS) + goto exit; + + code[1] = instr; +#if __ARM_VFP__ + if (IS_ARM_VFP(instr)) { + /* We no longer manage FPEXC beyond bootstrap, so verify that VFP is still enabled. */ + if (!get_vfp_enabled()) + panic("VFP was disabled (arm); VFP should always be enabled"); + } +#endif + + if (IS_ARM_GDB_TRAP(instr)) { + exception = EXC_BREAKPOINT; + code[0] = EXC_ARM_BREAKPOINT; + } + } + + if (!((regs->cpsr & PSR_MODE_MASK) == PSR_USER_MODE)) { + boolean_t intr; + + intr = ml_set_interrupts_enabled(FALSE); + + if (exception == EXC_BREAKPOINT) { + /* Save off the context here (so that the debug logic + * can see the original state of this thread). + */ + vm_offset_t kstackptr = current_thread()->machine.kstackptr; + *((arm_saved_state_t *) kstackptr) = *regs; + + DebuggerCall(exception, regs); + (void) ml_set_interrupts_enabled(intr); + goto exit; + } + panic_context(exception, (void *)regs, "undefined kernel instruction\n" + "r0: 0x%08x r1: 0x%08x r2: 0x%08x r3: 0x%08x\n" + "r4: 0x%08x r5: 0x%08x r6: 0x%08x r7: 0x%08x\n" + "r8: 0x%08x r9: 0x%08x r10: 0x%08x r11: 0x%08x\n" + "r12: 0x%08x sp: 0x%08x lr: 0x%08x pc: 0x%08x\n" + "cpsr: 0x%08x fsr: 0x%08x far: 0x%08x\n", + regs->r[0], regs->r[1], regs->r[2], regs->r[3], + regs->r[4], regs->r[5], regs->r[6], regs->r[7], + regs->r[8], regs->r[9], regs->r[10], regs->r[11], + regs->r[12], regs->sp, regs->lr, regs->pc, + regs->cpsr, regs->fsr, regs->far); + + (void) ml_set_interrupts_enabled(intr); + + } else { + exception_triage(exception, code, codeCnt); + /* NOTREACHED */ + } + +exit: + if (recover) + thread->recover = recover; +} + +/* + * Routine: sleh_abort + * Function: Second level exception handler for abort(Pref/Data) + */ + +void +sleh_abort(struct arm_saved_state * regs, int type) +{ + int status; + int debug_status=0; + int spsr; + int exc; + mach_exception_data_type_t codes[2]; + vm_map_t map; + vm_map_address_t vaddr; + vm_map_address_t fault_addr; + vm_prot_t fault_type; + kern_return_t result; + vm_offset_t recover; + thread_t thread = current_thread(); + boolean_t intr; + + recover = thread->recover; + thread->recover = 0; + + status = regs->fsr & FSR_MASK; + spsr = regs->cpsr; + + /* The DSFR/IFSR.ExT bit indicates "IMPLEMENTATION DEFINED" classification. + * Allow a platform-level error handler to decode it. + */ + if ((regs->fsr) & FSR_EXT) { + cpu_data_t *cdp = getCpuDatap(); + + if (cdp->platform_error_handler != (platform_error_handler_t) NULL) { + (*(platform_error_handler_t)cdp->platform_error_handler) (cdp->cpu_id, 0); + /* If a platform error handler is registered, expect it to panic, not fall through */ + panic("Unexpected return from platform_error_handler"); + } + } + + /* Done with asynchronous handling; re-enable here so that subsequent aborts are taken as early as possible. */ + reenable_async_aborts(); + + if (ml_at_interrupt_context()) + panic_with_thread_kernel_state("sleh_abort at interrupt context", regs); + + fault_addr = vaddr = regs->far; + + if (type == T_DATA_ABT) { + getCpuDatap()->cpu_stat.data_ex_cnt++; + } else { /* T_PREFETCH_ABT */ + getCpuDatap()->cpu_stat.instr_ex_cnt++; + fault_type = VM_PROT_READ | VM_PROT_EXECUTE; + } + + if (status == FSR_DEBUG) + debug_status = arm_debug_read_dscr() & ARM_DBGDSCR_MOE_MASK; + + /* Inherit the interrupt masks from previous */ + if (!(spsr & PSR_INTMASK)) + ml_set_interrupts_enabled(TRUE); + + if (type == T_DATA_ABT) { + /* + * Now that interrupts are reenabled, we can perform any needed + * copyin operations. + * + * Because we have reenabled interrupts, any instruction copy + * must be a copyin, even on UP systems. + */ + + if (regs->fsr & DFSR_WRITE) { + fault_type = (VM_PROT_READ | VM_PROT_WRITE); + /* Cache operations report faults as write access, change these to read access */ + /* Cache operations are invoked from arm mode for now */ + if (!(regs->cpsr & PSR_TF)) { + unsigned int ins; + + if(COPYIN((user_addr_t)(regs->pc), (char *)&ins,(vm_size_t)(sizeof(unsigned int))) != KERN_SUCCESS) + goto exit; + + if (arm_mcr_cp15(ins) || arm_mcrr_cp15(ins)) + fault_type = VM_PROT_READ; + } + } else { + fault_type = VM_PROT_READ; + /* + * DFSR is not getting the "write" bit set + * when a swp instruction is encountered (even when it is + * a write fault. + */ + if (!(regs->cpsr & PSR_TF)) { + unsigned int ins; + + if(COPYIN((user_addr_t)(regs->pc), (char *)&ins,(vm_size_t)(sizeof(unsigned int))) != KERN_SUCCESS) + goto exit; + + if ((ins & ARM_SWP_MASK) == ARM_SWP) + fault_type = VM_PROT_WRITE; + } + } + } + + if ((spsr & PSR_MODE_MASK) != PSR_USER_MODE) { + /* Fault in kernel mode */ + + if ((status == FSR_DEBUG) + && ((debug_status == ARM_DBGDSCR_MOE_ASYNC_WATCHPOINT) || (debug_status == ARM_DBGDSCR_MOE_SYNC_WATCHPOINT)) + && (recover != 0) && (getCpuDatap()->cpu_user_debug != 0)) { + /* If we hit a watchpoint in kernel mode, probably in a copyin/copyout which we don't want to + * abort. Turn off watchpoints and keep going; we'll turn them back on in load_and_go_user. + */ + arm_debug_set(NULL); + goto exit; + } + + if ((type == T_PREFETCH_ABT) || (status == FSR_DEBUG)) { + + intr = ml_set_interrupts_enabled(FALSE); + if (status == FSR_DEBUG) { + DebuggerCall(EXC_BREAKPOINT, regs); + (void) ml_set_interrupts_enabled(intr); + goto exit; + } + panic_context(EXC_BAD_ACCESS, (void*)regs, "sleh_abort: prefetch abort in kernel mode: fault_addr=0x%x\n" + "r0: 0x%08x r1: 0x%08x r2: 0x%08x r3: 0x%08x\n" + "r4: 0x%08x r5: 0x%08x r6: 0x%08x r7: 0x%08x\n" + "r8: 0x%08x r9: 0x%08x r10: 0x%08x r11: 0x%08x\n" + "r12: 0x%08x sp: 0x%08x lr: 0x%08x pc: 0x%08x\n" + "cpsr: 0x%08x fsr: 0x%08x far: 0x%08x\n", + fault_addr, + regs->r[0], regs->r[1], regs->r[2], regs->r[3], + regs->r[4], regs->r[5], regs->r[6], regs->r[7], + regs->r[8], regs->r[9], regs->r[10], regs->r[11], + regs->r[12], regs->sp, regs->lr, regs->pc, + regs->cpsr, regs->fsr, regs->far); + + (void) ml_set_interrupts_enabled(intr); + + } else if (TEST_FSR_VMFAULT(status)) { + +#if CONFIG_DTRACE + if (thread->options & TH_OPT_DTRACE) { /* Executing under dtrace_probe? */ + if (dtrace_tally_fault(fault_addr)) { /* Should a fault under dtrace be ignored? */ + /* Point to next instruction */ + regs->pc += ((regs->cpsr & PSR_TF) && !IS_THUMB32(*((uint16_t*) (regs->pc)))) ? 2 : 4; + goto exit; + } else { + intr = ml_set_interrupts_enabled(FALSE); + panic_context(EXC_BAD_ACCESS, (void *)regs, "Unexpected page fault under dtrace_probe" + "r0: 0x%08x r1: 0x%08x r2: 0x%08x r3: 0x%08x\n" + "r4: 0x%08x r5: 0x%08x r6: 0x%08x r7: 0x%08x\n" + "r8: 0x%08x r9: 0x%08x r10: 0x%08x r11: 0x%08x\n" + "r12: 0x%08x sp: 0x%08x lr: 0x%08x pc: 0x%08x\n" + "cpsr: 0x%08x fsr: 0x%08x far: 0x%08x\n", + regs->r[0], regs->r[1], regs->r[2], regs->r[3], + regs->r[4], regs->r[5], regs->r[6], regs->r[7], + regs->r[8], regs->r[9], regs->r[10], regs->r[11], + regs->r[12], regs->sp, regs->lr, regs->pc, + regs->cpsr, regs->fsr, regs->far); + + (void) ml_set_interrupts_enabled(intr); + + goto exit; + } + } +#endif + + if (VM_KERNEL_ADDRESS(vaddr) || thread == THREAD_NULL) + map = kernel_map; + else + map = thread->map; + + /* check to see if it is just a pmap ref/modify fault */ + result = arm_fast_fault(map->pmap, trunc_page(fault_addr), fault_type, FALSE); + if (result == KERN_SUCCESS) + goto exit; + + /* + * We have to "fault" the page in. + */ + result = vm_fault(map, fault_addr, + fault_type, + FALSE /* change_wiring */, VM_KERN_MEMORY_NONE, + (map == kernel_map) ? THREAD_UNINT : THREAD_ABORTSAFE, NULL, 0); + + if (result == KERN_SUCCESS) { + goto exit; + } else { + /* + * If we have a recover handler, invoke it now. + */ + if (recover != 0) { + regs->pc = (register_t) (recover & ~0x1); + regs->cpsr = (regs->cpsr & ~PSR_TF) | ((recover & 0x1) << PSR_TFb); + goto exit; + } + } + } else if ((status & FSR_ALIGN_MASK) == FSR_ALIGN) { + result = sleh_alignment(regs); + if (result == KERN_SUCCESS) { + goto exit; + } else { + intr = ml_set_interrupts_enabled(FALSE); + + panic_context(EXC_BAD_ACCESS, (void *)regs, "unaligned kernel data access: pc=0x%08x fault_addr=0x%x\n" + "r0: 0x%08x r1: 0x%08x r2: 0x%08x r3: 0x%08x\n" + "r4: 0x%08x r5: 0x%08x r6: 0x%08x r7: 0x%08x\n" + "r8: 0x%08x r9: 0x%08x r10: 0x%08x r11: 0x%08x\n" + "r12: 0x%08x sp: 0x%08x lr: 0x%08x pc: 0x%08x\n" + "cpsr: 0x%08x fsr: 0x%08x far: 0x%08x\n", + regs->pc, fault_addr, + regs->r[0], regs->r[1], regs->r[2], regs->r[3], + regs->r[4], regs->r[5], regs->r[6], regs->r[7], + regs->r[8], regs->r[9], regs->r[10], regs->r[11], + regs->r[12], regs->sp, regs->lr, regs->pc, + regs->cpsr, regs->fsr, regs->far); + + (void) ml_set_interrupts_enabled(intr); + + goto exit; + } + + } + intr = ml_set_interrupts_enabled(FALSE); + + panic_context(EXC_BAD_ACCESS, (void *)regs, "kernel abort type %d: fault_type=0x%x, fault_addr=0x%x\n" + "r0: 0x%08x r1: 0x%08x r2: 0x%08x r3: 0x%08x\n" + "r4: 0x%08x r5: 0x%08x r6: 0x%08x r7: 0x%08x\n" + "r8: 0x%08x r9: 0x%08x r10: 0x%08x r11: 0x%08x\n" + "r12: 0x%08x sp: 0x%08x lr: 0x%08x pc: 0x%08x\n" + "cpsr: 0x%08x fsr: 0x%08x far: 0x%08x\n", + type, fault_type, fault_addr, + regs->r[0], regs->r[1], regs->r[2], regs->r[3], + regs->r[4], regs->r[5], regs->r[6], regs->r[7], + regs->r[8], regs->r[9], regs->r[10], regs->r[11], + regs->r[12], regs->sp, regs->lr, regs->pc, + regs->cpsr, regs->fsr, regs->far); + + (void) ml_set_interrupts_enabled(intr); + + goto exit; + } + /* Fault in user mode */ + + if (TEST_FSR_VMFAULT(status)) { + map = thread->map; + +#if CONFIG_DTRACE + if (thread->options & TH_OPT_DTRACE) { /* Executing under dtrace_probe? */ + if (dtrace_tally_fault(fault_addr)) { /* Should a user mode fault under dtrace be ignored? */ + if (recover) { + regs->pc = recover; + } else { + intr = ml_set_interrupts_enabled(FALSE); + + panic_context(EXC_BAD_ACCESS, (void *)regs, "copyin/out has no recovery point" + "r0: 0x%08x r1: 0x%08x r2: 0x%08x r3: 0x%08x\n" + "r4: 0x%08x r5: 0x%08x r6: 0x%08x r7: 0x%08x\n" + "r8: 0x%08x r9: 0x%08x r10: 0x%08x r11: 0x%08x\n" + "r12: 0x%08x sp: 0x%08x lr: 0x%08x pc: 0x%08x\n" + "cpsr: 0x%08x fsr: 0x%08x far: 0x%08x\n", + regs->r[0], regs->r[1], regs->r[2], regs->r[3], + regs->r[4], regs->r[5], regs->r[6], regs->r[7], + regs->r[8], regs->r[9], regs->r[10], regs->r[11], + regs->r[12], regs->sp, regs->lr, regs->pc, + regs->cpsr, regs->fsr, regs->far); + + (void) ml_set_interrupts_enabled(intr); + } + goto exit; + } else { + intr = ml_set_interrupts_enabled(FALSE); + + panic_context(EXC_BAD_ACCESS, (void*)regs, "Unexpected UMW page fault under dtrace_probe" + "r0: 0x%08x r1: 0x%08x r2: 0x%08x r3: 0x%08x\n" + "r4: 0x%08x r5: 0x%08x r6: 0x%08x r7: 0x%08x\n" + "r8: 0x%08x r9: 0x%08x r10: 0x%08x r11: 0x%08x\n" + "r12: 0x%08x sp: 0x%08x lr: 0x%08x pc: 0x%08x\n" + "cpsr: 0x%08x fsr: 0x%08x far: 0x%08x\n", + regs->r[0], regs->r[1], regs->r[2], regs->r[3], + regs->r[4], regs->r[5], regs->r[6], regs->r[7], + regs->r[8], regs->r[9], regs->r[10], regs->r[11], + regs->r[12], regs->sp, regs->lr, regs->pc, + regs->cpsr, regs->fsr, regs->far); + + (void) ml_set_interrupts_enabled(intr); + + goto exit; + } + } +#endif + + /* check to see if it is just a pmap ref/modify fault */ + result = arm_fast_fault(map->pmap, trunc_page(fault_addr), fault_type, TRUE); + if (result != KERN_SUCCESS) { + /* + * We have to "fault" the page in. + */ + result = vm_fault(map, fault_addr, fault_type, + FALSE /* change_wiring */, VM_KERN_MEMORY_NONE, + THREAD_ABORTSAFE, NULL, 0); + } + if (result == KERN_SUCCESS || result == KERN_ABORTED) { + goto exception_return; + } + exc = EXC_BAD_ACCESS; + codes[0] = result; + } else if ((status & FSR_ALIGN_MASK) == FSR_ALIGN) { + if (sleh_alignment(regs) == KERN_SUCCESS) { + goto exception_return; + } + exc = EXC_BAD_ACCESS; + codes[0] = EXC_ARM_DA_ALIGN; + } else if (status == FSR_DEBUG) { + exc = EXC_BREAKPOINT; + codes[0] = EXC_ARM_DA_DEBUG; + } else if ((status == FSR_SDOM) || (status == FSR_PDOM)) { + exc = EXC_BAD_ACCESS; + codes[0] = KERN_INVALID_ADDRESS; + } else { + exc = EXC_BAD_ACCESS; + codes[0] = KERN_FAILURE; + } + + codes[1] = vaddr; + exception_triage(exc, codes, 2); + /* NOTREACHED */ + +exception_return: + if (recover) + thread->recover = recover; + thread_exception_return(); + /* NOTREACHED */ + +exit: + if (recover) + thread->recover = recover; + return; +} + + +/* + * Routine: sleh_alignment + * Function: Second level exception handler for alignment data fault + */ + +static kern_return_t +sleh_alignment(struct arm_saved_state * regs) +{ + unsigned int status; + unsigned int ins; + unsigned int rd_index; + unsigned int base_index; + unsigned int paddr; + void *src; + unsigned int reg_list; + unsigned int pre; + unsigned int up; + unsigned int write_back; + kern_return_t rc = KERN_SUCCESS; + + getCpuDatap()->cpu_stat.unaligned_cnt++; + + /* Do not try to emulate in modified execution states */ + if (regs->cpsr & (PSR_EF | PSR_JF)) + return KERN_NOT_SUPPORTED; + + /* Disallow emulation of kernel instructions */ + if ((regs->cpsr & PSR_MODE_MASK) != PSR_USER_MODE) + return KERN_NOT_SUPPORTED; + + +#define ALIGN_THRESHOLD 1024 + if ((sleh_alignment_count++ & (ALIGN_THRESHOLD - 1)) == + (ALIGN_THRESHOLD - 1)) + kprintf("sleh_alignment: %d more alignment faults: %d total\n", + ALIGN_THRESHOLD, sleh_alignment_count); + + if ((trap_on_alignment_fault != 0) + && (sleh_alignment_count % trap_on_alignment_fault == 0)) + return KERN_NOT_SUPPORTED; + + status = regs->fsr; + paddr = regs->far; + + if (regs->cpsr & PSR_TF) { + unsigned short ins16; + + /* Get aborted instruction */ +#if __ARM_SMP__ || __ARM_USER_PROTECT__ + if(COPYIN((user_addr_t)(regs->pc), (char *)&ins16,(vm_size_t)(sizeof(uint16_t))) != KERN_SUCCESS) { + /* Failed to fetch instruction, return success to re-drive the exception */ + return KERN_SUCCESS; + } +#else + ins16 = *(unsigned short *) (regs->pc); +#endif + + /* + * Map multi-word Thumb loads and stores to their ARM + * equivalents. + * Don't worry about single-word instructions, since those are + * handled in hardware. + */ + + reg_list = ins16 & 0xff; + if (reg_list == 0) + return KERN_NOT_SUPPORTED; + + if (((ins16 & THUMB_STR_1_MASK) == THUMB_LDMIA) || + ((ins16 & THUMB_STR_1_MASK) == THUMB_STMIA)) { + base_index = (ins16 >> 8) & 0x7; + ins = 0xE8800000 | (base_index << 16) | reg_list; + if ((ins16 & THUMB_STR_1_MASK) == THUMB_LDMIA) + ins |= (1 << 20); + if (((ins16 & THUMB_STR_1_MASK) == THUMB_STMIA) || + !(reg_list & (1 << base_index))) + ins |= (1 << 21); + } else if ((ins16 & THUMB_PUSH_MASK) == THUMB_POP) { + unsigned int r = (ins16 >> 8) & 1; + ins = 0xE8BD0000 | (r << 15) | reg_list; + } else if ((ins16 & THUMB_PUSH_MASK) == THUMB_PUSH) { + unsigned int r = (ins16 >> 8) & 1; + ins = 0xE92D0000 | (r << 14) | reg_list; + } else { + return KERN_NOT_SUPPORTED; + } + } else { + /* Get aborted instruction */ +#if __ARM_SMP__ || __ARM_USER_PROTECT__ + if(COPYIN((user_addr_t)(regs->pc), (char *)&ins,(vm_size_t)(sizeof(unsigned int))) != KERN_SUCCESS) { + /* Failed to fetch instruction, return success to re-drive the exception */ + return KERN_SUCCESS; + } +#else + ins = *(unsigned int *) (regs->pc); +#endif + } + + /* Don't try to emulate unconditional instructions */ + if ((ins & 0xF0000000) == 0xF0000000) + return KERN_NOT_SUPPORTED; + + pre = (ins >> 24) & 1; + up = (ins >> 23) & 1; + reg_list = ins & 0xffff; + write_back = (ins >> 21) & 1; + base_index = (ins >> 16) & 0xf; + + if ((ins & ARM_BLK_MASK) == ARM_STM) { /* STM or LDM */ + int reg_count = 0; + int waddr; + + for (rd_index = 0; rd_index < 16; rd_index++) { + if (reg_list & (1 << rd_index)) + reg_count++; + } + + paddr = regs->r[base_index]; + + switch (ins & (ARM_POST_INDEXING | ARM_INCREMENT)) { + /* Increment after */ + case ARM_INCREMENT: + waddr = paddr + reg_count * 4; + break; + + /* Increment before */ + case ARM_POST_INDEXING | ARM_INCREMENT: + waddr = paddr + reg_count * 4; + paddr += 4; + break; + + /* Decrement after */ + case 0: + waddr = paddr - reg_count * 4; + paddr = waddr + 4; + break; + + /* Decrement before */ + case ARM_POST_INDEXING: + waddr = paddr - reg_count * 4; + paddr = waddr; + break; + + default: + waddr = 0; + } + + for (rd_index = 0; rd_index < 16; rd_index++) { + if (reg_list & (1 << rd_index)) { + src = ®s->r[rd_index]; + + if ((ins & (1 << 20)) == 0) /* STM */ + rc = COPYOUT(src, paddr, 4); + else /* LDM */ + rc = COPYIN(paddr, src, 4); + + if (rc != KERN_SUCCESS) + break; + + paddr += 4; + } + } + + paddr = waddr; + } else { + rc = 1; + } + + if (rc == KERN_SUCCESS) { + if (regs->cpsr & PSR_TF) + regs->pc += 2; + else + regs->pc += 4; + + if (write_back) + regs->r[base_index] = paddr; + } + return (rc); +} + + +#ifndef NO_KDEBUG +/* XXX quell warnings */ +void syscall_trace(struct arm_saved_state * regs); +void syscall_trace_exit(unsigned int, unsigned int); +void mach_syscall_trace(struct arm_saved_state * regs, unsigned int call_number); +void mach_syscall_trace_exit(unsigned int retval, unsigned int call_number); +void interrupt_trace(struct arm_saved_state * regs); +void interrupt_trace_exit(void); + +/* called from the fleh_swi handler, if TRACE_SYSCALL is enabled */ +void +syscall_trace( + struct arm_saved_state * regs) +{ + kprintf("syscall: %d\n", regs->r[12]); +} + +void +syscall_trace_exit( + unsigned int r0, + unsigned int r1) +{ + kprintf("syscall exit: 0x%x 0x%x\n", r0, r1); +} + +void +mach_syscall_trace( + struct arm_saved_state * regs, + unsigned int call_number) +{ + int i, argc; + int kdarg[3] = {0, 0, 0}; + + argc = mach_trap_table[call_number].mach_trap_arg_count; + + if (argc > 3) + argc = 3; + + for (i = 0; i < argc; i++) + kdarg[i] = (int) regs->r[i]; + + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + MACHDBG_CODE(DBG_MACH_EXCP_SC, (call_number)) | DBG_FUNC_START, + kdarg[0], kdarg[1], kdarg[2], 0, 0); + +} + +void +mach_syscall_trace_exit( + unsigned int retval, + unsigned int call_number) +{ + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + MACHDBG_CODE(DBG_MACH_EXCP_SC, (call_number)) | DBG_FUNC_END, + retval, 0, 0, 0, 0); +} + +void +interrupt_trace( + struct arm_saved_state * regs) +{ +#define UMODE(rp) (((rp)->cpsr & PSR_MODE_MASK) == PSR_USER_MODE) + + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_START, + 0, UMODE(regs) ? regs->pc : VM_KERNEL_UNSLIDE(regs->pc), + UMODE(regs), 0, 0); +} + +void +interrupt_trace_exit( + void) +{ + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_END, + 0, 0, 0, 0, 0); +} +#endif + +/* XXX quell warnings */ +void interrupt_stats(void); + +/* This is called from locore.s directly. We only update per-processor interrupt counters in this function */ +void +interrupt_stats(void) +{ + SCHED_STATS_INTERRUPT(current_processor()); +} + +static void +panic_with_thread_kernel_state(const char *msg, struct arm_saved_state *regs) +{ + panic_context(0, (void*)regs, "%s (saved state:%p)\n" + "r0: 0x%08x r1: 0x%08x r2: 0x%08x r3: 0x%08x\n" + "r4: 0x%08x r5: 0x%08x r6: 0x%08x r7: 0x%08x\n" + "r8: 0x%08x r9: 0x%08x r10: 0x%08x r11: 0x%08x\n" + "r12: 0x%08x sp: 0x%08x lr: 0x%08x pc: 0x%08x\n" + "cpsr: 0x%08x fsr: 0x%08x far: 0x%08x\n", + msg, regs, + regs->r[0], regs->r[1], regs->r[2], regs->r[3], + regs->r[4], regs->r[5], regs->r[6], regs->r[7], + regs->r[8], regs->r[9], regs->r[10], regs->r[11], + regs->r[12], regs->sp, regs->lr, regs->pc, + regs->cpsr, regs->fsr, regs->far); + +} diff --git a/osfmk/arm/trap.h b/osfmk/arm/trap.h new file mode 100644 index 000000000..fe15c2733 --- /dev/null +++ b/osfmk/arm/trap.h @@ -0,0 +1,284 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ +/* + */ + +#ifndef _ARM_TRAP_H_ +#define _ARM_TRAP_H_ + +/* + * Hardware trap vectors for ARM. + */ + +#define T_RESET 0 +#define T_UNDEF 1 +#define T_SWI 2 +#define T_PREFETCH_ABT 3 +#define T_DATA_ABT 4 +#define T_IRQ 6 +#define T_FIQ 7 +#define T_PMU 8 + + +#define TRAP_NAMES "reset", "undefined instruction", "software interrupt", \ + "prefetch abort", "data abort", "irq interrupt", \ + "fast interrupt", "perfmon" + +/* + * Page-fault trap codes. + */ +#define T_PF_PROT 0x1 /* protection violation */ +#define T_PF_WRITE 0x2 /* write access */ +#define T_PF_USER 0x4 /* from user state */ + +#if !defined(ASSEMBLER) && defined(MACH_KERNEL) + +#include <arm/thread.h> + +#define GDB_TRAP_INSTR1 0xe7ffdefe +#define GDB_TRAP_INSTR2 0xe7ffdeff + +#define ARM_GDB_INSTR1 GDB_TRAP_INSTR1 +#define ARM_GDB_INSTR2 GDB_TRAP_INSTR2 + +#define IS_ARM_GDB_TRAP(op) \ + (((op) == ARM_GDB_INSTR1) || ((op) == ARM_GDB_INSTR2)) + +#define THUMB_GDB_INSTR1 (GDB_TRAP_INSTR1 & 0xFFFF) +#define THUMB_GDB_INSTR2 (GDB_TRAP_INSTR2 & 0xFFFF) + +#define IS_THUMB_GDB_TRAP(op) \ + (((op) == THUMB_GDB_INSTR1) || ((op) == THUMB_GDB_INSTR2)) + + +#define ARM_STR 0x04000000 /* STR */ +#define ARM_STRH 0x000000B0 /* STRH */ +#define ARM_STRH_MASK 0x0E1000F0 /* STRH MASK */ +#define ARM_SDX_MASK 0x0C100000 /* SINGLE DATA TRANSFER */ +#define ARM_SNGL_DX_MASK 0x0C000000 /* SINGLE DATA TRANSFER MASK */ +#define ARM_SDX 0x04000000 + +#define ARM_STM 0x08000000 /* STM */ +#define ARM_BDX_MASK 0x0E100000 /* BLOCK DATA TRANSFER */ +#define ARM_BLK_MASK 0x0E000000 /* BLOCK DATA TRANSFER */ +#define ARM_BDX 0x08000000 /* BLOCK DATA TRANSFER */ + +#define ARM_WRITE_BACK 0x00200000 +#define ARM_BASE_REG 0x000F0000 +#define ARM_INCREMENT 0x00800000 + +#define ARM_STC 0x0C000000 /* STC */ +#define ARM_CDX_MASK ARM_BDX_MASK /* COPROCESSOR DATA TRANSFER */ +#define ARM_CBLK_MASK ARM_BLK_MASK +#define ARM_CDX 0x0C000000 /* COPROCESSOR DATA TRANSFER */ + +#define ARM_SWP 0x01000090 /* SWP */ +#define ARM_SWP_MASK 0x0FB00FF0 /* SWP */ + +#define ARM_POST_INDEXING 0x01000000 +#define ARM_IMMEDIATE 0x02000000 +#define ARM_LSL 0 +#define ARM_LSR 1 +#define ARM_ASR 2 +#define ARM_ROR 3 + +#define MCR_MASK 0x0F100F10 +#define MCR_CP15 0x0E000F10 +#define MCRR_MASK 0x0FF00F00 +#define MCRR_CP15 0x0C400F00 + +#define arm_mcr_cp15(op) (((op)&MCR_MASK) == 0x0E000F10) +#define arm_mcrr_cp15(op) (((op)&0x0FF00F00) == 0x0C400F00) + +#define IS_THUMB32(op) ( \ + (((op) & 0xE000) == 0xE000) && (((op) & 0x1800) != 0x0000)) + +#define THUMB_LDR_1_MASK 0x8800 /* (1) forms of LD* instructions */ +#define THUMB_STR_1_MASK 0xF800 /* (1) forms of ST* instructions */ +#define THUMB_STR_2_MASK 0xFE00 /* (2) forms of ST* instructions */ +#define THUMB_STR_3_MASK 0xF800 /* (3) forms of ST* instructions */ +#define THUMB_PUSH_MASK 0xFE00 /* PUSH instruction */ + +#define THUMB_LDRH_1 0x8800 /* LDRH(1) */ +#define THUMB_STMIA 0xC000 /* STMIA */ +#define THUMB_STR_1 0x6000 /* STR(1) */ +#define THUMB_STR_2 0x5000 /* STR(2) */ +#define THUMB_STR_3 0x9000 /* STR(3) */ +#define THUMB_STRB_1 0x7000 /* STRB(1) */ +#define THUMB_STRB_2 0x5400 /* STRB(2) */ +#define THUMB_STRH_1 0x8000 /* STRH(1) */ +#define THUMB_STRH_2 0x5200 /* STRH(2) */ +#define THUMB_PUSH 0xB400 /* PUSH */ +#define THUMB_LDMIA 0xC800 /* LDMIA */ +#define THUMB_POP 0xBC00 /* POP */ + + +/* + * Shifts, masks, and other values for load/store multiple decoding; largely needed for + * supporting misaligned accesses. + */ +#define THUMB_STR_1_BASE_OFFSET 8 /* Offset of the base register field */ +#define THUMB_PUSH_EXTRA_OFFSET 8 /* Offset of the "extra" register field */ +#define ARM_STM_BASE_OFFSET 16 /* Offset of the base register field */ +#define ARM_STM_LOAD_OFFSET 20 /* Offset of the load flag */ +#define ARM_STM_WBACK_OFFSET 21 /* Offset of the writeback flag */ +#define ARM_STM_INCR_OFFSET 23 /* Offset of the increment flag */ +#define ARM_STM_BEFORE_OFFSET 24 /* Offset of the pre-index flag */ +#define ARM_REG_LIST_LR_OFFSET 14 /* Offset of LR in the register list */ +#define ARM_REG_LIST_PC_OFFSET 15 /* Offset of PC in the register list */ + +#define THUMB_STR_REG_LIST_MASK 0x000000FF /* Offset of the reg list is 0 */ +#define THUMB_STR_1_BASE_MASK 0x00000700 +#define THUMB_PUSH_EXTRA_MASK 0x00000100 +#define ARM_STM_REG_LIST_MASK 0x0000FFFF /* Offset of the reg list is 0 */ +#define ARM_STM_BASE_MASK 0x000F0000 +#define ARM_STM_LOAD_MASK 0x00100000 +#define ARM_STM_WBACK_MASK 0x00200000 +#define ARM_STM_INCR_MASK 0x00800000 +#define ARM_STM_BEFORE_MASK 0x01000000 +#define ARM_COND_MASK 0xF0000000 /* Mask for the condition code */ + +#define ARM_COND_UNCOND 0xF0000000 /* Instruction does not support condition codes */ + +#define ARM_SIMD_MASK0 0xFE000000 +#define ARM_SIMD_CODE0 0xF2000000 + +#define ARM_VFP_MASK0 0x0F000E10 +#define ARM_VFP_CODE0 0x0E000A00 + +#define ARM_SIMD_VFP_MASK0 0x0E000E00 +#define ARM_SIMD_VFP_CODE0 0x0C000A00 +#define ARM_SIMD_VFP_MASK1 0xFF100000 +#define ARM_SIMD_VFP_CODE1 0xF4000000 +#define ARM_SIMD_VFP_MASK2 0x0F000E10 +#define ARM_SIMD_VFP_CODE2 0x0E000A10 +#define ARM_SIMD_VFP_MASK3 0x0FE00E00 +#define ARM_SIMD_VFP_CODE3 0x0C400A00 + +#define IS_ARM_VFP(op) ( \ + (((op) & ARM_SIMD_MASK0) == ARM_SIMD_CODE0) \ + ||(((op) & ARM_VFP_MASK0) == ARM_VFP_CODE0) \ + ||(((op) & ARM_SIMD_VFP_MASK0) == ARM_SIMD_VFP_CODE0) \ + ||(((op) & ARM_SIMD_VFP_MASK1) == ARM_SIMD_VFP_CODE1) \ + ||(((op) & ARM_SIMD_VFP_MASK2) == ARM_SIMD_VFP_CODE2) \ + || (((op) & ARM_SIMD_VFP_MASK3) == ARM_SIMD_VFP_CODE3)) + +#define THUMB_SIMD_MASK0 0xEF000000 +#define THUMB_SIMD_CODE0 0xEF000000 + +#define THUMB_VFP_MASK0 0xEF000E10 +#define THUMB_VFP_CODE0 0xEE000A00 + +#define THUMB_SIMD_VFP_MASK0 0xEE000E00 +#define THUMB_SIMD_VFP_CODE0 0xEC000A00 +#define THUMB_SIMD_VFP_MASK1 0xFF100000 +#define THUMB_SIMD_VFP_CODE1 0xF9000000 +#define THUMB_SIMD_VFP_MASK2 0xEF000E10 +#define THUMB_SIMD_VFP_CODE2 0xEE000A10 +#define THUMB_SIMD_VFP_MASK3 0xEFE00E00 +#define THUMB_SIMD_VFP_CODE3 0xEC400A00 + +#define IS_THUMB_VFP(op) ( \ + (((op) & THUMB_SIMD_MASK0) == THUMB_SIMD_CODE0 ) \ + || (((op) & THUMB_VFP_MASK0) == THUMB_VFP_CODE0 ) \ + || (((op) & THUMB_SIMD_VFP_MASK0) == THUMB_SIMD_VFP_CODE0 ) \ + || (((op) & THUMB_SIMD_VFP_MASK1) == THUMB_SIMD_VFP_CODE1 ) \ + || (((op) & THUMB_SIMD_VFP_MASK2) == THUMB_SIMD_VFP_CODE2 ) \ + || (((op) & THUMB_SIMD_VFP_MASK3) == THUMB_SIMD_VFP_CODE3)) + +extern boolean_t arm_swap_readable_type(vm_map_address_t, unsigned int /* spsr */); +extern boolean_t arm_force_fast_fault(ppnum_t, vm_prot_t, int, void *); +extern kern_return_t arm_fast_fault(pmap_t, vm_map_address_t, vm_prot_t, boolean_t); + +/* + * Determines if the aborted instruction is read or write operation + */ +#define arm_fault_type(op,spsr,vaddr) \ + (((((op)&ARM_CDX_MASK) == ARM_STC) || \ + (((op)&ARM_STRH_MASK) == ARM_STRH) || \ + (((op)&ARM_BDX_MASK) == ARM_STM) || \ + (((op)&ARM_SDX_MASK) == ARM_STR) || \ + ((((op)&ARM_SWP_MASK) == ARM_SWP) && \ + arm_swap_readable_type(vaddr,spsr))) ? \ + (VM_PROT_WRITE|VM_PROT_READ) : (VM_PROT_READ)) + +#define thumb_fault_type(op,spsr,vaddr) \ + (((((op)&THUMB_STR_1_MASK) == THUMB_STMIA) || \ + (((op)&THUMB_STR_1_MASK) == THUMB_STR_1) || \ + (((op)&THUMB_STR_2_MASK) == THUMB_STR_2) || \ + (((op)&THUMB_STR_3_MASK) == THUMB_STR_3) || \ + (((op)&THUMB_STR_1_MASK) == THUMB_STRB_1) || \ + (((op)&THUMB_STR_2_MASK) == THUMB_STRB_2) || \ + (((op)&THUMB_STR_1_MASK) == THUMB_STRH_1) || \ + (((op)&THUMB_STR_2_MASK) == THUMB_STRH_2) || \ + (((op)&THUMB_PUSH_MASK) == THUMB_PUSH)) ? \ + (VM_PROT_WRITE|VM_PROT_READ) : (VM_PROT_READ)) + +typedef kern_return_t (*perfCallback)( + int trapno, + struct arm_saved_state *ss, + uintptr_t *, + int); + +typedef kern_return_t (*perfASTCallback)(ast_t reasons, ast_t *myast); + +extern volatile perfCallback perfTrapHook; +extern volatile perfASTCallback perfASTHook; +extern volatile perfCallback perfIntHook; + +#endif /* !ASSEMBLER && MACH_KERNEL */ + +#endif /* _ARM_TRAP_H_ */ diff --git a/osfmk/arm/vm_tuning.h b/osfmk/arm/vm_tuning.h new file mode 100644 index 000000000..728de1775 --- /dev/null +++ b/osfmk/arm/vm_tuning.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ +/* + */ +/* + * File: arm/vm_tuning.h + * + * VM tuning parameters for arm (without reference bits). + */ + +#ifndef _ARM_VM_TUNING_H_ +#define _ARM_VM_TUNING_H_ + +#endif /* _ARM_VM_TUNING_H_ */ diff --git a/osfmk/arm/xpr.h b/osfmk/arm/xpr.h new file mode 100644 index 000000000..b6151ddca --- /dev/null +++ b/osfmk/arm/xpr.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ + +/* + * Machine dependent module for the XPR tracing facility. + */ + +#define XPR_TIMESTAMP (0) diff --git a/osfmk/arm64/Makefile b/osfmk/arm64/Makefile new file mode 100644 index 000000000..ad75e8a11 --- /dev/null +++ b/osfmk/arm64/Makefile @@ -0,0 +1,31 @@ +export MakeInc_cmd=${SRCROOT}/makedefs/MakeInc.cmd +export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def +export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule +export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir + +include $(MakeInc_cmd) +include $(MakeInc_def) + +ARM_HEADER_FILES = \ + lowglobals.h \ + machine_cpuid.h \ + machine_machdep.h \ + proc_reg.h + +INSTALL_MD_DIR = arm64 + +INSTALL_MD_LCL_LIST = + +INSTALL_MD_LIST = + +INSTALL_KF_MD_LIST = $(ARM_HEADER_FILES) + +INSTALL_KF_MD_LCL_LIST = machine_kpc.h monotonic.h pgtrace.h $(ARM_HEADER_FILES) + +EXPORT_MD_LIST = machine_cpuid.h machine_kpc.h monotonic.h proc_reg.h pgtrace.h + + +EXPORT_MD_DIR = arm64 + +include $(MakeInc_rule) +include $(MakeInc_dir) diff --git a/osfmk/arm64/WKdmCompress_16k.s b/osfmk/arm64/WKdmCompress_16k.s new file mode 100644 index 000000000..7588c9f40 --- /dev/null +++ b/osfmk/arm64/WKdmCompress_16k.s @@ -0,0 +1,634 @@ +/* + * Copyright (c) 2000-2014 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +/* + This file contains arm64 hand optimized implementation of WKdm memory page compressor. + + int WKdm_compress (WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, unsigned int bytes_budget); + + input : + src_buf : address of input page (length = 1024 words) + dest_buf : address of output buffer (may not be 16-byte aligned) + scratch : a 16-byte aligned 4k bytes scratch memory provided by the caller, + bytes_budget : a given byte target in compression + + output : + + if the input buffer can be compressed within the given byte budget, the dest_buf is written with compressed data and the function returns with number of bytes for the compressed data + o.w., the function returns -1 to signal that the input data can not be compressed with the given byte budget. + During the scan and tag process, each word that can not be compressed will be written to dest_buf, followed by a 12-bytes header + 256-bytes tag area. + When the functions returns -1, dest_buf is filled with all those words that can not be compressed and should be considered undefined. + The worst-case scenario is that all words can not be compressed. Hence, the minimum size requirement for dest_buf should be 12+256+4096 = 4364 bytes to prevent from memory fault. + + The 4th argument bytes_budget is the target compress budget in bytes. + Should the input page can be compressed within the budget, the compressed data is written to *dest_buf, and the function returns the number of compressed bytes. + Otherwise, the function returns -1 (to signal to the caller that the page can not be compressed). + + WKdm Compression algorithm is briefly stated as follows: + + There is a dynamically updated dictionary consisting of 16 words. Each dictionary word is initialized to 1 at the point of entry to the function. + For a nonzero input word x, its 8-bits (10-bits scaled up) is used to determine a corresponding word from the dictionary, represented by dict_index (4-bits) and dict_word (32-bits). + a. k = (x>>10)&255; // 8-bit hash table index + b. dict_index = hashTable[k]; // 4-bit dictionary index, hashTable[] is fixed + c. dict_word = dictionary[dict_index]; // 32-bit dictionary word, dictionary[] is dynamically updated + + Each input word x is classified/tagged into 4 classes : + 0 : x = 0 + 1 : (x>>10) == (dict_word>>10), bits 10:31 of the input word match a dictionary word + 2 : (x>>10) != (dict_word>>10), the above condition (22 higher bits matched) is not met, meaning a dictionary miss + 3 : (x == dict_word), the exact input word is in the dictionary + + For each class, different numbers of bits are needed for the decompressor to reproduce the original input word. + 0 : 2-bits tag (32->2 compression) + 1 : 2-bits tag + 4-bits dict_index + 10-bits lower bits (32->16 compression) + 2 : 2-bits tag + 32-bits new word (32->34 expansion) + 3 : 2-bits tag + 4-bits dict_index (32->6 compression) + + It is obvious now that WKdm compress algorithm works well for pages where there are lots of zero words (32->2) and/or there are freqeunt repeats of some word patterns (32->6). + + the output bit stream (*dest_buf) consists of + a. 12 bytes header + b. 256 bytes for 1024 packed tags + c. (varying number of) words for new words not matched to dictionary word. + d. (varying number of) 32-bit words for packed 4-bit dict_indices (for class 1 and 3) + e. (varying number of) 32-bit words for packed 10-bit low bits (for class 1) + + the header is actually of 3 words that specify the ending offset (in 32-bit words) from the start of the bit stream of c,d,e, respectively. + Note that there might be padding bits in d (if the number of dict_indices does not divide by 8), and there are 2/12/22 padding bits for packing 3/2/1 low 10-bits in a 32-bit word. + + + The WKdm compress algorithm 1st runs a scan and classification pass, tagging and write unpacked data into temporary buffers. It follows by packing those data into the output buffer. + + The temp buffers are + + uint8_t tempTagsArray[1024]; // temporary saving for tags before final packing + uint8_t tempQPosArray[1024]; // temporary saving for dict_indices before final packing + uint16_t tempLowBitsArray[1024]; // temporary saving for partially matched lower 10 bits before final packing + + Since the new words (that can not matched fully or partially to the dictionary) are stored right after the header and the tags section and need no packing, we directly write them to + the destination buffer. + + uint32_t *new_word = dest_buf+3+64; // 3 words for header, 64 words for tags, new words come right after the tags. + + Now since we are given a byte budget for this compressor, we can monitor the byte (or bit) usage on the fly in the scanning and tagging pass. + + byte_count -= 12 + 256; // bit budget minus header and tags + + whenever an input word is classified as class + + 2 : byte_count -= 4; + + the compress function can early exit (return -1) should the page can not be compressed with the given byte budget (i.e., byte_count <= 0). + + without showing the bit budget management, the pseudo code is given as follows: + + uint8_t *tags=tempTagsArray; + uint8_t *dict=tempQPosArray; + uint8_t *partial=tempLowBitsArray; + + for (i=0;i<1024;i++) { + x = *src_buf++; + if (x == 0) { // zero, 2-bits tag + *tags++ = 0; + } else { + + // find dict_index and dict_word from x + k = (x>>10)&255; + dict_index = hashTable[k]; + dict_word = dictionary[dict_index]; + + if (dict_word == x) { // exactly match + // 2-bits tag + 4-bits table index + *tags++ = 3; + *dict++ = dict_index; + } else if (((x^dict_word)>>10)==0) { // 22 higher bits matched + // 2-bits tag + 4-bits table index + 10-bits lower partial + *tags++ = 1; + *dict++ = dict_index; + *partial++ = x &0x3ff; + dictionary[dict_index] = x; + } else { // not matched + // 2-bits tag + 32-bits new word + *tags++ = 2; + *new_word++ = x; + dictionary[dict_index] = x; + } + } + } + + after this classification/tagging pass is completed, the 3 temp buffers are packed into the output *dest_buf: + + 1. 1024 tags are packed into 256 bytes right after the 12-bytes header + 2. dictionary indices (4-bits each) are packed into are right after the new words section + 3. 3 low 10-bits are packed into a 32-bit word, this is after the dictionary indices section. + + cclee, 11/9/12 + + Added zero page, single value page, sparse page, early abort optimizations + rsrini, 09/14/14 +*/ + +#define PAGES_SIZE_IN_KBYTES 16 + +#ifndef PAGES_SIZE_IN_KBYTES +#define PAGES_SIZE_IN_KBYTES 4 +#endif + +#if !((PAGES_SIZE_IN_KBYTES==4) || (PAGES_SIZE_IN_KBYTES==16)) +#error "Only PAGES_SIZE_IN_KBYTES = 4 or 16 is supported" +#endif + + + .text + .align 4 + +/* + int WKdm_compress (WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, unsigned int bytes_budget); +*/ + +.globl _WKdm_compress_16k +_WKdm_compress_16k: + +/* + ------------------------- symbolizing register use ----------------------------------- +*/ + #define src_buf x0 + #define next_input_word x0 + #define dest_buf x1 + #define scratch x2 + #define byte_count x3 + #define next_tag x4 + #define tempTagsArray x2 // scratch + #define dictionary x5 + #define remaining x6 + #define next_full_patt x7 + #define dict_location x8 + #define wdict_location w8 + #define next_qp x9 + #define hashTable x10 + #define tempQPosArray x11 + #define next_low_bits x12 + +/* + this arm64 assembly code is ported from x86_64 assembly code, + therefore need such symbolization to quickly reuse the x86_64 assembly code + for these intermediate/temporary register use +*/ + #define rax x13 + #define eax w13 + #define rcx x14 + #define ecx w14 + #define rdx x15 + #define edx w15 + #define rdi x0 /* after some point, x0/rdi becomes free other usage */ + + +/* + ------------------------- scratch memory -------------------------------------- + + need 16*4 (dictionary) + 256*4 (tempTagsArray) + 256*4 (tempQPosArray) + 1024*4 (tempLowBitsArray) + total 6208 bytes + [sp,#0] : dictionary + [scratch,#0] : tempTagsArray + [scratch,#1024] : tempQPosArray + [scratch,#2048] : tempLowBitsArray +*/ + +#define scale (PAGES_SIZE_IN_KBYTES/4) + +#define SV_RETURN 0 // return value when SV, ZV page is found +#define MZV_MAGIC 17185 // magic value used to identify MZV page encoding +#define CHKPT_BYTES 416 // for early aborts: checkpoint after processing this many bytes. Must be in range [4..4096] +#define CHKPT_WORDS (CHKPT_BYTES/4) // checkpoint bytes in words +#define CHKPT_TAG_BYTES (CHKPT_BYTES/16) // size of the tags for CHKPT_BYTES of data +#define CHKPT_SHRUNK_BYTES 426 // for early aborts: max size of compressed stream to allow further processing .. + // .. to disable early aborts, set CHKPT_SHRUNK_BYTES to 4096 +#if CHKPT_BYTES > 4096 + #error CHKPT_BYTES must be <= 4096 +#endif +#if CHKPT_BYTES < 4 + #error CHKPT_BYTES must be >= 4 +#endif + +#if KERNEL + sub sp, sp, #64 + st1.4s {v0,v1,v2,v3},[sp] +#endif + + sub sp, sp, #64 // allocate for dictionary + mov dictionary, sp // use x5 to point to sp, so we can use sub xd, xn, sp + + sub sp, sp, #64 // allocate space for saving callee-saved registers + mov x15, sp + stp x20, x21, [x15, #0] // save x20, x21 + stp x22, x23, [x15, #16] // save x22, x23 + stp x24, x25, [x15, #32] // save x24, x25 + stp x26, x27, [x15, #48] // save x26, x27 + +/* + ------- entwined statck space allocation, registers set up, and PRELOAD_DICTIONARY ------------------- +*/ + + // NOTE: ALL THE DICTIONARY VALUES MUST BE INITIALIZED TO ZERO + // THIS IS NEEDED TO EFFICIENTLY DETECT SINGLE VALUE PAGES + mov next_tag, tempTagsArray // &tempTagsArray[0] + add next_qp, scratch, #(1024*scale) // next_qp + mov remaining, #(CHKPT_WORDS*scale) // remaining input words .. initially set to checkpoint + add next_full_patt, dest_buf, #(12+256*scale) // dest_buf + [TAGS_AREA_OFFSET + (num_input_words / 16)]*4 + sub byte_count, byte_count, #(12+256*scale) // bit_count - header - tags + add next_low_bits, scratch, #(2048*scale) // &tempLowBitsArray[0] + stp xzr, xzr, [dictionary, #0] // initialize dictionary + adrp hashTable, _hashLookupTable@GOTPAGE + stp xzr, xzr, [dictionary, #16] // initialize dictionary + stp xzr, xzr, [dictionary, #32] // initialize dictionary + ldr hashTable, [hashTable, _hashLookupTable@GOTPAGEOFF] + stp xzr, xzr, [dictionary, #48] // initialize dictionary + +#define EARLYCHECK 0 +#define NORMAL 1 + +#define mode w20 +#define start_next_full_patt x21 +#define start_next_input_word x22 +#define start_next_low_bits x23 +#define r11 x24 +#define r13 x25 +#define byte_budget x26 +#define start_next_qp tempQPosArray + + add tempQPosArray, scratch, #(1024*scale) // &tempQPosArray[0] + mov mode, EARLYCHECK // indicate we are yet to evaluate the early aborts + mov start_next_full_patt, next_full_patt // remember the start of next_full_patt + mov start_next_input_word, next_input_word // remember the start of next_input_word + mov start_next_low_bits, next_low_bits // remember the start of next_low_bit + add byte_budget, byte_count, #(12+256*scale) // remember the byte budget + + b L_loop + + .align 4, 0x90 + + /* we've just detected a zero input word in edx */ +L_RECORD_ZERO: + strb edx, [next_tag], #1 // *next_tag++ = ZERO; edx is used as input word, and if we are here edx = 0 + subs remaining, remaining, #1 // remaing--; + b.le CHECKPOINT // if remaining = 0, break + + /* -------------- scan/tag pass loop ------------------------- */ +L_loop: + + /* load new input word to edx */ + ldr edx, [next_input_word], #4 + cbz edx, L_RECORD_ZERO // if (input_word==0) RECORD_ZERO + + /* + now the input word edx is nonzero, we next find the corresponding dictionary word (eax) and dict_location + */ + ubfm eax, edx, #10, #17 + ldrb wdict_location, [hashTable, rax] // HASH_TO_DICT_BYTE_OFFSET(input_word) + ldr eax, [dictionary, dict_location] // dict_word = *dict_location; + + /* detect whether we match input to its corresponding dictionary word */ + eor eax, eax, edx // dict_word vs input_word + cbz eax, L_RECORD_EXACT // if identical, RECORD_EXACT + lsr eax, eax, #10 // HIGH_BITS(dict_word^input_word) + cbz eax, L_RECORD_PARTIAL // if identical, RECORD_PARTIAL + +L_RECORD_MISS: +/* + if we are here, the input word can not be derived from the dictionary, + we write the input word as a new word, + and update the dictionary with this new word +*/ + subs byte_count, byte_count, #4 // byte_count -= 4 + b.le L_budgetExhausted // return -1 to signal this page is not compressable + str edx, [next_full_patt], #4 // *next_full_patt++ = input_word; + mov eax, #2 // tag for MISS + subs remaining, remaining, #1 // remaing--; + str edx, [dictionary, dict_location] // *dict_location = input_word + strb eax, [next_tag], #1 // *next_tag++ = 2 for miss + b.gt L_loop // // if remaining > 0, repeat + b CHECKPOINT + +L_done_search: + + // SET_QPOS_AREA_START(dest_buf,next_full_patt); + /* 1st word in dest_buf header = 4-byte offset (from start) of end of new word section */ + + sub rax, next_full_patt, dest_buf // next_full_patt - dest_buf + lsr eax, eax, #2 // offset in 4-bytes + str eax, [dest_buf] // dest_buf[0] = next_full_patt - dest_buf + + /* -------------------------- packing 1024 tags into 256 bytes ----------------------------------------*/ + // boundary_tmp = WK_pack_2bits(tempTagsArray, (WK_word *) next_tag, dest_buf + HEADER_SIZE_IN_WORDS); + + add rdi, dest_buf, #12 // dest_buf + mov rcx, tempTagsArray // &tempTagsArray[0] + +L_pack_2bits: + ld1.2s {v0,v1,v2,v3},[rcx],#32 + + shl.2d v1,v1,#4 + shl.2d v3,v3,#4 + + orr.8b v0, v0, v1 + orr.8b v2, v2, v3 + + ushr.2d v1, v0, #30 + ushr.2d v3, v2, #30 + + orr.8b v0, v0, v1 + orr.8b v2, v2, v3 + + zip1.2s v0, v0, v2 + st1.2s {v0},[rdi],#8 + cmp next_tag, rcx + b.hi L_pack_2bits + + /* --------------------------------- packing 4-bits dict indices into dest_buf ---------------------------------- */ + + /* 1st, round up number of 4-bits dict_indices to a multiple of 8 and fill in 0 if needed */ + sub rax, next_qp, tempQPosArray // eax = num_bytes_to_pack = next_qp - (char *) tempQPosArray; + add eax, eax, #7 // num_bytes_to_pack+7 + lsr eax, eax, #3 // num_packed_words = (num_bytes_to_pack + 7) >> 3 + add rcx, tempQPosArray, rax, lsl #3 // endQPosArray = tempQPosArray + 2*num_source_words + lsl rax, rax, #2 + subs byte_count, byte_count, rax + b.lt L_budgetExhausted + + cmp rcx, next_qp // endQPosArray vs next_qp + b.ls 2f // if (next_qp >= endQPosArray) skip the following zero paddings + sub rax, rcx, next_qp + mov edx, #0 + tst eax, #4 + b.eq 1f + str edx, [next_qp], #4 +1: tst eax, #2 + b.eq 1f + strh edx, [next_qp], #2 +1: tst eax, #1 + b.eq 2f + strb edx, [next_qp], #1 +2: + mov rdi, next_full_patt // next_full_patt + cmp rcx, tempQPosArray // endQPosArray vs tempQPosArray + ldr eax, [dest_buf] + b.ls L20 // if (endQPosArray <= tempQPosArray) skip the following + mov rdx, tempQPosArray // tempQPosArray + + /* packing 4-bits dict indices into dest_buf */ +L_pack_4bits: + ldr rax, [rdx], #8 // src_next[1]:src_next[0] + orr rax, rax, rax, lsr #28 // eax = src_next[0] | (src_next[1] << 4) + cmp rcx, rdx // source_end vs src_next + str eax, [rdi], #4 // *dest_next++ = temp; + b.hi L_pack_4bits // while (src_next < source_end) repeat the loop + + // SET_LOW_BITS_AREA_START(dest_buf,boundary_tmp); + sub rax, rdi, dest_buf // boundary_tmp - dest_buf + lsr eax, eax, #2 // boundary_tmp - dest_buf in words +L20: + str eax, [dest_buf,#4] // dest_buf[1] = boundary_tmp - dest_buf + + + + /* --------------------------- packing 3 10-bits low bits into a 32-bit word in dest_buf[] ----------------------------------------- */ + + add rcx, scratch, #(2048*scale) // tempLowBitsArray + sub rdx, next_low_bits, rcx // next_low_bits - tempLowBitsArray (in bytes) + lsr rdx, rdx, #1 // num_tenbits_to_pack (in half-words) + subs edx, edx, #3 // pre-decrement num_tenbits_to_pack by 3 + b.lt 1f // if num_tenbits_to_pack < 3, skip the following loop +0: + subs byte_count, byte_count, #4 // byte_count -= 4 + b.le L_budgetExhausted // return -1 to signal this page is not compressable + subs edx, edx, #3 // num_tenbits_to_pack-=3 + ldr rax, [rcx], #6 + bfm rax, rax, #58, #9 // pack 1st toward 2nd + bfm rax, rax, #58, #25 // pack 1st/2nd toward 3rd + lsr rax, rax, #12 + str eax, [rdi], #4 // pack w0,w1,w2 into 1 dest_buf word + b.ge 0b // if no less than 3 elements, back to loop head + +1: adds edx, edx, #3 // post-increment num_tenbits_to_pack by 3 + b.eq 3f // if num_tenbits_to_pack is a multiple of 3, skip the following + subs byte_count, byte_count, #4 // byte_count -= 4 + b.le L_budgetExhausted // return -1 to signal this page is not compressable + ldrh eax,[rcx] // w0 + subs edx, edx, #1 // num_tenbits_to_pack-- + b.eq 2f // + ldrh edx, [rcx, #2] // w1 + orr eax, eax, edx, lsl #10 // w0 | (w1<<10) + +2: str eax, [rdi], #4 // write the final dest_buf word + +3: sub rax, rdi, dest_buf // boundary_tmp - dest_buf + lsr eax, eax, #2 // boundary_tmp - dest_buf in terms of words + str eax, [dest_buf, #8] // SET_LOW_BITS_AREA_END(dest_buf,boundary_tmp) + lsl w0, eax, #2 // boundary_tmp - dest_buf in terms of bytes + +L_done: + + // restore registers and return + mov x15, sp + ldp x20, x21, [x15, #0] // restore x20, x21 + ldp x22, x23, [x15, #16] // restore x22, x23 + ldp x24, x25, [x15, #32] // restore x24, x25 + ldp x26, x27, [x15, #48] // restore x26, x27 + add sp, sp, #128 // deallocate for dictionary + saved register space + +#if KERNEL + ld1.4s {v0,v1,v2,v3},[sp],#64 +#endif + ret lr + + .align 4 +L_budgetExhausted: + mov x0, #-1 + b L_done + + + .align 4,0x90 +L_RECORD_EXACT: +/* + we have an exact match of the input word to its corresponding dictionary word + write tag/dict_index to the temorary buffers +*/ + mov eax, #3 + lsr w14, wdict_location, #2 // divide by 4 for word offset + subs remaining, remaining, #1 // remaing--; + strb eax, [next_tag], #1 // *next_tag++ = 3 for exact + strb w14, [next_qp], #1 // *next_qp = word offset (4-bit) + b.gt L_loop + b CHECKPOINT // if remaining = 0, break + + .align 4,0x90 +L_RECORD_PARTIAL: +/* + we have a partial (high 22-bits) match of the input word to its corresponding dictionary word + write tag/dict_index/low 10 bits to the temorary buffers +*/ + mov ecx, #1 + strb ecx, [next_tag], #1 // *next_tag++ = 1 for partial matched + str edx, [dictionary, dict_location] // *dict_location = input_word; + subs remaining, remaining, #1 // remaing--; + lsr eax, wdict_location, #2 // offset in 32-bit word + and edx, edx, #1023 // lower 10 bits + strb eax, [next_qp], #1 // update *next_qp++ + strh edx, [next_low_bits], #2 // save next_low_bits++ + b.gt L_loop + +CHECKPOINT: + + cbz mode, L_check_compression_ratio // if this this an early abort check.. + +L_check_zero_page: + + cmp start_next_full_patt, next_full_patt // check if any dictionary misses in page + b.ne L_check_single_value_page + + cmp start_next_qp, next_qp // check if any partial or exact dictionary matches + b.ne L_check_single_value_page + + mov x0, #SV_RETURN // Magic return value + b L_done + +L_check_single_value_page: + + sub rax, next_full_patt, start_next_full_patt // get # dictionary misses + lsr rax, rax, #2 + + sub r11, next_qp, start_next_qp // get # dictionary hits (exact + partial) + + sub r13, next_low_bits, start_next_low_bits // get # dictionary partial hits + lsr r13, r13, #1 + + // Single value page if one of the follwoing is true: + // partial == 0 AND hits == 1023(for 4K page) AND miss == 1 AND tag[0] == 2 (i.e. miss) + // partial == 1 AND hits == 1024(for 4K page) AND tag[0] == 1 (i.e. partial) + // + cbnz r13, 1f // were there 0 partial hits? + + cmp r11, #(256*PAGES_SIZE_IN_KBYTES - 1) // were there 1023 dictionary hits + b.ne 1f + + cmp rax, #1 // was there exacly 1 dictionary miss? + b.ne 1f + + ldrb edx, [tempTagsArray] // read the very 1st tag + cmp edx, #2 // was the very 1st tag a miss? + b.eq L_is_single_value_page + +1: + cmp r13, #1 // was there 1 partial hit? + b.ne L_check_mostly_zero + + cmp r11, #(256*PAGES_SIZE_IN_KBYTES) // were there 1024 dictionary hits + b.ne L_check_mostly_zero + + ldrb edx, [tempTagsArray] // read the very 1st tag + cmp edx, #1 // was the very 1st tag a partial? + b.ne L_check_mostly_zero + +L_is_single_value_page: + + mov x0, #SV_RETURN // Magic return value + b L_done + +L_check_mostly_zero: + // how much space will the sparse packer take? + add rax, rax, r11 // rax += (next_qp - start_next_qp) + mov rdx, #6 + mov rcx, #4 + madd r11, rax, rdx, rcx // r11 = rax * 6 (i.e. 4 byte word + 2 byte offset) + 4 byte for header + + sub rax, next_low_bits, start_next_low_bits // get bytes consumed by lower-10 bits + mov rdx, #1365 + mul rax, rax, rdx + + sub rdx, next_full_patt, start_next_full_patt // get bytes consumed by dictionary misses + add rax, rdx, rax, lsr #11 // rax = 2/3*(next_low_bits - start_next_low_bits) + (next_full_patt - start_next_full_patt) + + sub rdx, next_qp, start_next_qp + add rax, rax, rdx, lsr #1 // rax += (next_qp - start_next_qp)/2 + add rax, rax, #(12+256*scale) // rax += bytes taken by the header + tags + + cmp rax, r11 // is the default packer the better option? + b.lt L_done_search + + cmp r11, byte_budget // can the sparse packer fit into the given budget? + b.gt L_budgetExhausted + +L_sparse_packer: + mov edx, #MZV_MAGIC + str edx, [dest_buf], #4 // header to indicate a sparse packer + + mov rdx, #0 // rdx = byte offset in src of non-0 word +1: + ldr rax, [start_next_input_word, rdx] // rax = read dword + cbnz rax, 5f // is dword != 0 +3: + add rdx, rdx, #8 // 8 more bytes have been processed +4: + cmp rdx, #(4096*scale) // has the entire page been processed + b.ne 1b + mov x0, r11 // store the size of the compressed stream + b L_done + +5: + cbz eax, 6f // is lower word == 0 + str eax, [dest_buf], #4 // store the non-0 word in the dest buffer + strh edx, [dest_buf], #2 // store the byte index +6: + lsr rax, rax, 32 // get the upper word into position + cbz eax, 3b // is dword == 0 + add rdx, rdx, #4 + str eax, [dest_buf], #4 // store the non-0 word in the dest buffer + strh edx, [dest_buf], #2 // store the byte index + add rdx, rdx, #4 + b 4b + +L_check_compression_ratio: + + mov mode, NORMAL + mov remaining, #((1024 - CHKPT_WORDS)*scale) // remaining input words to process + cbz remaining, CHECKPOINT // if there are no remaining words to process + + sub rax, next_low_bits, start_next_low_bits // get bytes consumed by lower-10 bits + mov rdx, #1365 + mul rax, rax, rdx + + sub rdx, next_full_patt, start_next_full_patt // get bytes consumed by dictionary misses + add rax, rdx, rax, lsr #11 // rax = 2/3*(next_low_bits - start_next_low_bits) + (next_full_patt - start_next_full_patt) + + sub rdx, next_qp, start_next_qp + add rax, rax, rdx, lsr #1 // rax += (next_qp - start_next_qp)/2 + subs rax, rax, #((CHKPT_SHRUNK_BYTES - CHKPT_TAG_BYTES)*scale) + // rax += CHKPT_TAG_BYTES; rax -= CHKPT_SHRUNK_BYTES + + b.gt L_budgetExhausted // if rax is > 0, we need to early abort + b L_loop // we are done diff --git a/osfmk/arm64/WKdmCompress_4k.s b/osfmk/arm64/WKdmCompress_4k.s new file mode 100644 index 000000000..2ac438768 --- /dev/null +++ b/osfmk/arm64/WKdmCompress_4k.s @@ -0,0 +1,632 @@ +/* + * Copyright (c) 2000-2014 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +/* + This file contains arm64 hand optimized implementation of WKdm memory page compressor. + + int WKdm_compress (WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, unsigned int bytes_budget); + + input : + src_buf : address of input page (length = 1024 words) + dest_buf : address of output buffer (may not be 16-byte aligned) + scratch : a 16-byte aligned 4k bytes scratch memory provided by the caller, + bytes_budget : a given byte target in compression + + output : + + if the input buffer can be compressed within the given byte budget, the dest_buf is written with compressed data and the function returns with number of bytes for the compressed data + o.w., the function returns -1 to signal that the input data can not be compressed with the given byte budget. + During the scan and tag process, each word that can not be compressed will be written to dest_buf, followed by a 12-bytes header + 256-bytes tag area. + When the functions returns -1, dest_buf is filled with all those words that can not be compressed and should be considered undefined. + The worst-case scenario is that all words can not be compressed. Hence, the minimum size requirement for dest_buf should be 12+256+4096 = 4364 bytes to prevent from memory fault. + + The 4th argument bytes_budget is the target compress budget in bytes. + Should the input page can be compressed within the budget, the compressed data is written to *dest_buf, and the function returns the number of compressed bytes. + Otherwise, the function returns -1 (to signal to the caller that the page can not be compressed). + + WKdm Compression algorithm is briefly stated as follows: + + There is a dynamically updated dictionary consisting of 16 words. Each dictionary word is initialized to 1 at the point of entry to the function. + For a nonzero input word x, its 8-bits (10-bits scaled up) is used to determine a corresponding word from the dictionary, represented by dict_index (4-bits) and dict_word (32-bits). + a. k = (x>>10)&255; // 8-bit hash table index + b. dict_index = hashTable[k]; // 4-bit dictionary index, hashTable[] is fixed + c. dict_word = dictionary[dict_index]; // 32-bit dictionary word, dictionary[] is dynamically updated + + Each input word x is classified/tagged into 4 classes : + 0 : x = 0 + 1 : (x>>10) == (dict_word>>10), bits 10:31 of the input word match a dictionary word + 2 : (x>>10) != (dict_word>>10), the above condition (22 higher bits matched) is not met, meaning a dictionary miss + 3 : (x == dict_word), the exact input word is in the dictionary + + For each class, different numbers of bits are needed for the decompressor to reproduce the original input word. + 0 : 2-bits tag (32->2 compression) + 1 : 2-bits tag + 4-bits dict_index + 10-bits lower bits (32->16 compression) + 2 : 2-bits tag + 32-bits new word (32->34 expansion) + 3 : 2-bits tag + 4-bits dict_index (32->6 compression) + + It is obvious now that WKdm compress algorithm works well for pages where there are lots of zero words (32->2) and/or there are freqeunt repeats of some word patterns (32->6). + + the output bit stream (*dest_buf) consists of + a. 12 bytes header + b. 256 bytes for 1024 packed tags + c. (varying number of) words for new words not matched to dictionary word. + d. (varying number of) 32-bit words for packed 4-bit dict_indices (for class 1 and 3) + e. (varying number of) 32-bit words for packed 10-bit low bits (for class 1) + + the header is actually of 3 words that specify the ending offset (in 32-bit words) from the start of the bit stream of c,d,e, respectively. + Note that there might be padding bits in d (if the number of dict_indices does not divide by 8), and there are 2/12/22 padding bits for packing 3/2/1 low 10-bits in a 32-bit word. + + + The WKdm compress algorithm 1st runs a scan and classification pass, tagging and write unpacked data into temporary buffers. It follows by packing those data into the output buffer. + + The temp buffers are + + uint8_t tempTagsArray[1024]; // temporary saving for tags before final packing + uint8_t tempQPosArray[1024]; // temporary saving for dict_indices before final packing + uint16_t tempLowBitsArray[1024]; // temporary saving for partially matched lower 10 bits before final packing + + Since the new words (that can not matched fully or partially to the dictionary) are stored right after the header and the tags section and need no packing, we directly write them to + the destination buffer. + + uint32_t *new_word = dest_buf+3+64; // 3 words for header, 64 words for tags, new words come right after the tags. + + Now since we are given a byte budget for this compressor, we can monitor the byte (or bit) usage on the fly in the scanning and tagging pass. + + byte_count -= 12 + 256; // bit budget minus header and tags + + whenever an input word is classified as class + + 2 : byte_count -= 4; + + the compress function can early exit (return -1) should the page can not be compressed with the given byte budget (i.e., byte_count <= 0). + + without showing the bit budget management, the pseudo code is given as follows: + + uint8_t *tags=tempTagsArray; + uint8_t *dict=tempQPosArray; + uint8_t *partial=tempLowBitsArray; + + for (i=0;i<1024;i++) { + x = *src_buf++; + if (x == 0) { // zero, 2-bits tag + *tags++ = 0; + } else { + + // find dict_index and dict_word from x + k = (x>>10)&255; + dict_index = hashTable[k]; + dict_word = dictionary[dict_index]; + + if (dict_word == x) { // exactly match + // 2-bits tag + 4-bits table index + *tags++ = 3; + *dict++ = dict_index; + } else if (((x^dict_word)>>10)==0) { // 22 higher bits matched + // 2-bits tag + 4-bits table index + 10-bits lower partial + *tags++ = 1; + *dict++ = dict_index; + *partial++ = x &0x3ff; + dictionary[dict_index] = x; + } else { // not matched + // 2-bits tag + 32-bits new word + *tags++ = 2; + *new_word++ = x; + dictionary[dict_index] = x; + } + } + } + + after this classification/tagging pass is completed, the 3 temp buffers are packed into the output *dest_buf: + + 1. 1024 tags are packed into 256 bytes right after the 12-bytes header + 2. dictionary indices (4-bits each) are packed into are right after the new words section + 3. 3 low 10-bits are packed into a 32-bit word, this is after the dictionary indices section. + + cclee, 11/9/12 + + Added zero page, single value page, sparse page, early abort optimizations + rsrini, 09/14/14 +*/ + +#ifndef PAGES_SIZE_IN_KBYTES +#define PAGES_SIZE_IN_KBYTES 4 +#endif + +#if !((PAGES_SIZE_IN_KBYTES==4) || (PAGES_SIZE_IN_KBYTES==16)) +#error "Only PAGES_SIZE_IN_KBYTES = 4 or 16 is supported" +#endif + + + .text + .align 4 + +/* + int WKdm_compress (WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, unsigned int bytes_budget); +*/ + +.globl _WKdm_compress_4k +_WKdm_compress_4k: + +/* + ------------------------- symbolizing register use ----------------------------------- +*/ + #define src_buf x0 + #define next_input_word x0 + #define dest_buf x1 + #define scratch x2 + #define byte_count x3 + #define next_tag x4 + #define tempTagsArray x2 // scratch + #define dictionary x5 + #define remaining x6 + #define next_full_patt x7 + #define dict_location x8 + #define wdict_location w8 + #define next_qp x9 + #define hashTable x10 + #define tempQPosArray x11 + #define next_low_bits x12 + +/* + this arm64 assembly code is ported from x86_64 assembly code, + therefore need such symbolization to quickly reuse the x86_64 assembly code + for these intermediate/temporary register use +*/ + #define rax x13 + #define eax w13 + #define rcx x14 + #define ecx w14 + #define rdx x15 + #define edx w15 + #define rdi x0 /* after some point, x0/rdi becomes free other usage */ + + +/* + ------------------------- scratch memory -------------------------------------- + + need 16*4 (dictionary) + 256*4 (tempTagsArray) + 256*4 (tempQPosArray) + 1024*4 (tempLowBitsArray) + total 6208 bytes + [sp,#0] : dictionary + [scratch,#0] : tempTagsArray + [scratch,#1024] : tempQPosArray + [scratch,#2048] : tempLowBitsArray +*/ + +#define scale (PAGES_SIZE_IN_KBYTES/4) + +#define SV_RETURN 0 // return value when SV, ZV page is found +#define MZV_MAGIC 17185 // magic value used to identify MZV page encoding +#define CHKPT_BYTES 416 // for early aborts: checkpoint after processing this many bytes. Must be in range [4..4096] +#define CHKPT_WORDS (CHKPT_BYTES/4) // checkpoint bytes in words +#define CHKPT_TAG_BYTES (CHKPT_BYTES/16) // size of the tags for CHKPT_BYTES of data +#define CHKPT_SHRUNK_BYTES 426 // for early aborts: max size of compressed stream to allow further processing .. + // .. to disable early aborts, set CHKPT_SHRUNK_BYTES to 4096 +#if CHKPT_BYTES > 4096 + #error CHKPT_BYTES must be <= 4096 +#endif +#if CHKPT_BYTES < 4 + #error CHKPT_BYTES must be >= 4 +#endif + +#if KERNEL + sub sp, sp, #64 + st1.4s {v0,v1,v2,v3},[sp] +#endif + + sub sp, sp, #64 // allocate for dictionary + mov dictionary, sp // use x5 to point to sp, so we can use sub xd, xn, sp + + sub sp, sp, #64 // allocate space for saving callee-saved registers + mov x15, sp + stp x20, x21, [x15, #0] // save x20, x21 + stp x22, x23, [x15, #16] // save x22, x23 + stp x24, x25, [x15, #32] // save x24, x25 + stp x26, x27, [x15, #48] // save x26, x27 + +/* + ------- entwined statck space allocation, registers set up, and PRELOAD_DICTIONARY ------------------- +*/ + + // NOTE: ALL THE DICTIONARY VALUES MUST BE INITIALIZED TO ZERO + // THIS IS NEEDED TO EFFICIENTLY DETECT SINGLE VALUE PAGES + mov next_tag, tempTagsArray // &tempTagsArray[0] + add next_qp, scratch, #(1024*scale) // next_qp + mov remaining, #(CHKPT_WORDS*scale) // remaining input words .. initially set to checkpoint + add next_full_patt, dest_buf, #(12+256*scale) // dest_buf + [TAGS_AREA_OFFSET + (num_input_words / 16)]*4 + sub byte_count, byte_count, #(12+256*scale) // bit_count - header - tags + add next_low_bits, scratch, #(2048*scale) // &tempLowBitsArray[0] + stp xzr, xzr, [dictionary, #0] // initialize dictionary + adrp hashTable, _hashLookupTable@GOTPAGE + stp xzr, xzr, [dictionary, #16] // initialize dictionary + stp xzr, xzr, [dictionary, #32] // initialize dictionary + ldr hashTable, [hashTable, _hashLookupTable@GOTPAGEOFF] + stp xzr, xzr, [dictionary, #48] // initialize dictionary + +#define EARLYCHECK 0 +#define NORMAL 1 + +#define mode w20 +#define start_next_full_patt x21 +#define start_next_input_word x22 +#define start_next_low_bits x23 +#define r11 x24 +#define r13 x25 +#define byte_budget x26 +#define start_next_qp tempQPosArray + + add tempQPosArray, scratch, #(1024*scale) // &tempQPosArray[0] + mov mode, EARLYCHECK // indicate we are yet to evaluate the early aborts + mov start_next_full_patt, next_full_patt // remember the start of next_full_patt + mov start_next_input_word, next_input_word // remember the start of next_input_word + mov start_next_low_bits, next_low_bits // remember the start of next_low_bit + add byte_budget, byte_count, #(12+256*scale) // remember the byte budget + + b L_loop + + .align 4, 0x90 + + /* we've just detected a zero input word in edx */ +L_RECORD_ZERO: + strb edx, [next_tag], #1 // *next_tag++ = ZERO; edx is used as input word, and if we are here edx = 0 + subs remaining, remaining, #1 // remaing--; + b.le CHECKPOINT // if remaining = 0, break + + /* -------------- scan/tag pass loop ------------------------- */ +L_loop: + + /* load new input word to edx */ + ldr edx, [next_input_word], #4 + cbz edx, L_RECORD_ZERO // if (input_word==0) RECORD_ZERO + + /* + now the input word edx is nonzero, we next find the corresponding dictionary word (eax) and dict_location + */ + ubfm eax, edx, #10, #17 + ldrb wdict_location, [hashTable, rax] // HASH_TO_DICT_BYTE_OFFSET(input_word) + ldr eax, [dictionary, dict_location] // dict_word = *dict_location; + + /* detect whether we match input to its corresponding dictionary word */ + eor eax, eax, edx // dict_word vs input_word + cbz eax, L_RECORD_EXACT // if identical, RECORD_EXACT + lsr eax, eax, #10 // HIGH_BITS(dict_word^input_word) + cbz eax, L_RECORD_PARTIAL // if identical, RECORD_PARTIAL + +L_RECORD_MISS: +/* + if we are here, the input word can not be derived from the dictionary, + we write the input word as a new word, + and update the dictionary with this new word +*/ + subs byte_count, byte_count, #4 // byte_count -= 4 + b.le L_budgetExhausted // return -1 to signal this page is not compressable + str edx, [next_full_patt], #4 // *next_full_patt++ = input_word; + mov eax, #2 // tag for MISS + subs remaining, remaining, #1 // remaing--; + str edx, [dictionary, dict_location] // *dict_location = input_word + strb eax, [next_tag], #1 // *next_tag++ = 2 for miss + b.gt L_loop // // if remaining > 0, repeat + b CHECKPOINT + +L_done_search: + + // SET_QPOS_AREA_START(dest_buf,next_full_patt); + /* 1st word in dest_buf header = 4-byte offset (from start) of end of new word section */ + + sub rax, next_full_patt, dest_buf // next_full_patt - dest_buf + lsr eax, eax, #2 // offset in 4-bytes + str eax, [dest_buf] // dest_buf[0] = next_full_patt - dest_buf + + /* -------------------------- packing 1024 tags into 256 bytes ----------------------------------------*/ + // boundary_tmp = WK_pack_2bits(tempTagsArray, (WK_word *) next_tag, dest_buf + HEADER_SIZE_IN_WORDS); + + add rdi, dest_buf, #12 // dest_buf + mov rcx, tempTagsArray // &tempTagsArray[0] + +L_pack_2bits: + ld1.2s {v0,v1,v2,v3},[rcx],#32 + + shl.2d v1,v1,#4 + shl.2d v3,v3,#4 + + orr.8b v0, v0, v1 + orr.8b v2, v2, v3 + + ushr.2d v1, v0, #30 + ushr.2d v3, v2, #30 + + orr.8b v0, v0, v1 + orr.8b v2, v2, v3 + + zip1.2s v0, v0, v2 + st1.2s {v0},[rdi],#8 + cmp next_tag, rcx + b.hi L_pack_2bits + + /* --------------------------------- packing 4-bits dict indices into dest_buf ---------------------------------- */ + + /* 1st, round up number of 4-bits dict_indices to a multiple of 8 and fill in 0 if needed */ + sub rax, next_qp, tempQPosArray // eax = num_bytes_to_pack = next_qp - (char *) tempQPosArray; + add eax, eax, #7 // num_bytes_to_pack+7 + lsr eax, eax, #3 // num_packed_words = (num_bytes_to_pack + 7) >> 3 + add rcx, tempQPosArray, rax, lsl #3 // endQPosArray = tempQPosArray + 2*num_source_words + lsl rax, rax, #2 + subs byte_count, byte_count, rax + b.lt L_budgetExhausted + + cmp rcx, next_qp // endQPosArray vs next_qp + b.ls 2f // if (next_qp >= endQPosArray) skip the following zero paddings + sub rax, rcx, next_qp + mov edx, #0 + tst eax, #4 + b.eq 1f + str edx, [next_qp], #4 +1: tst eax, #2 + b.eq 1f + strh edx, [next_qp], #2 +1: tst eax, #1 + b.eq 2f + strb edx, [next_qp], #1 +2: + mov rdi, next_full_patt // next_full_patt + cmp rcx, tempQPosArray // endQPosArray vs tempQPosArray + ldr eax, [dest_buf] + b.ls L20 // if (endQPosArray <= tempQPosArray) skip the following + mov rdx, tempQPosArray // tempQPosArray + + /* packing 4-bits dict indices into dest_buf */ +L_pack_4bits: + ldr rax, [rdx], #8 // src_next[1]:src_next[0] + orr rax, rax, rax, lsr #28 // eax = src_next[0] | (src_next[1] << 4) + cmp rcx, rdx // source_end vs src_next + str eax, [rdi], #4 // *dest_next++ = temp; + b.hi L_pack_4bits // while (src_next < source_end) repeat the loop + + // SET_LOW_BITS_AREA_START(dest_buf,boundary_tmp); + sub rax, rdi, dest_buf // boundary_tmp - dest_buf + lsr eax, eax, #2 // boundary_tmp - dest_buf in words +L20: + str eax, [dest_buf,#4] // dest_buf[1] = boundary_tmp - dest_buf + + + + /* --------------------------- packing 3 10-bits low bits into a 32-bit word in dest_buf[] ----------------------------------------- */ + + add rcx, scratch, #(2048*scale) // tempLowBitsArray + sub rdx, next_low_bits, rcx // next_low_bits - tempLowBitsArray (in bytes) + lsr rdx, rdx, #1 // num_tenbits_to_pack (in half-words) + subs edx, edx, #3 // pre-decrement num_tenbits_to_pack by 3 + b.lt 1f // if num_tenbits_to_pack < 3, skip the following loop +0: + subs byte_count, byte_count, #4 // byte_count -= 4 + b.le L_budgetExhausted // return -1 to signal this page is not compressable + subs edx, edx, #3 // num_tenbits_to_pack-=3 + ldr rax, [rcx], #6 + bfm rax, rax, #58, #9 // pack 1st toward 2nd + bfm rax, rax, #58, #25 // pack 1st/2nd toward 3rd + lsr rax, rax, #12 + str eax, [rdi], #4 // pack w0,w1,w2 into 1 dest_buf word + b.ge 0b // if no less than 3 elements, back to loop head + +1: adds edx, edx, #3 // post-increment num_tenbits_to_pack by 3 + b.eq 3f // if num_tenbits_to_pack is a multiple of 3, skip the following + subs byte_count, byte_count, #4 // byte_count -= 4 + b.le L_budgetExhausted // return -1 to signal this page is not compressable + ldrh eax,[rcx] // w0 + subs edx, edx, #1 // num_tenbits_to_pack-- + b.eq 2f // + ldrh edx, [rcx, #2] // w1 + orr eax, eax, edx, lsl #10 // w0 | (w1<<10) + +2: str eax, [rdi], #4 // write the final dest_buf word + +3: sub rax, rdi, dest_buf // boundary_tmp - dest_buf + lsr eax, eax, #2 // boundary_tmp - dest_buf in terms of words + str eax, [dest_buf, #8] // SET_LOW_BITS_AREA_END(dest_buf,boundary_tmp) + lsl w0, eax, #2 // boundary_tmp - dest_buf in terms of bytes + +L_done: + + // restore registers and return + mov x15, sp + ldp x20, x21, [x15, #0] // restore x20, x21 + ldp x22, x23, [x15, #16] // restore x22, x23 + ldp x24, x25, [x15, #32] // restore x24, x25 + ldp x26, x27, [x15, #48] // restore x26, x27 + add sp, sp, #128 // deallocate for dictionary + saved register space + +#if KERNEL + ld1.4s {v0,v1,v2,v3},[sp],#64 +#endif + ret lr + + .align 4 +L_budgetExhausted: + mov x0, #-1 + b L_done + + + .align 4,0x90 +L_RECORD_EXACT: +/* + we have an exact match of the input word to its corresponding dictionary word + write tag/dict_index to the temorary buffers +*/ + mov eax, #3 + lsr w14, wdict_location, #2 // divide by 4 for word offset + subs remaining, remaining, #1 // remaing--; + strb eax, [next_tag], #1 // *next_tag++ = 3 for exact + strb w14, [next_qp], #1 // *next_qp = word offset (4-bit) + b.gt L_loop + b CHECKPOINT // if remaining = 0, break + + .align 4,0x90 +L_RECORD_PARTIAL: +/* + we have a partial (high 22-bits) match of the input word to its corresponding dictionary word + write tag/dict_index/low 10 bits to the temorary buffers +*/ + mov ecx, #1 + strb ecx, [next_tag], #1 // *next_tag++ = 1 for partial matched + str edx, [dictionary, dict_location] // *dict_location = input_word; + subs remaining, remaining, #1 // remaing--; + lsr eax, wdict_location, #2 // offset in 32-bit word + and edx, edx, #1023 // lower 10 bits + strb eax, [next_qp], #1 // update *next_qp++ + strh edx, [next_low_bits], #2 // save next_low_bits++ + b.gt L_loop + +CHECKPOINT: + + cbz mode, L_check_compression_ratio // if this this an early abort check.. + +L_check_zero_page: + + cmp start_next_full_patt, next_full_patt // check if any dictionary misses in page + b.ne L_check_single_value_page + + cmp start_next_qp, next_qp // check if any partial or exact dictionary matches + b.ne L_check_single_value_page + + mov x0, #SV_RETURN // Magic return value + b L_done + +L_check_single_value_page: + + sub rax, next_full_patt, start_next_full_patt // get # dictionary misses + lsr rax, rax, #2 + + sub r11, next_qp, start_next_qp // get # dictionary hits (exact + partial) + + sub r13, next_low_bits, start_next_low_bits // get # dictionary partial hits + lsr r13, r13, #1 + + // Single value page if one of the follwoing is true: + // partial == 0 AND hits == 1023(for 4K page) AND miss == 1 AND tag[0] == 2 (i.e. miss) + // partial == 1 AND hits == 1024(for 4K page) AND tag[0] == 1 (i.e. partial) + // + cbnz r13, 1f // were there 0 partial hits? + + cmp r11, #(256*PAGES_SIZE_IN_KBYTES - 1) // were there 1023 dictionary hits + b.ne 1f + + cmp rax, #1 // was there exacly 1 dictionary miss? + b.ne 1f + + ldrb edx, [tempTagsArray] // read the very 1st tag + cmp edx, #2 // was the very 1st tag a miss? + b.eq L_is_single_value_page + +1: + cmp r13, #1 // was there 1 partial hit? + b.ne L_check_mostly_zero + + cmp r11, #(256*PAGES_SIZE_IN_KBYTES) // were there 1024 dictionary hits + b.ne L_check_mostly_zero + + ldrb edx, [tempTagsArray] // read the very 1st tag + cmp edx, #1 // was the very 1st tag a partial? + b.ne L_check_mostly_zero + +L_is_single_value_page: + + mov x0, #SV_RETURN // Magic return value + b L_done + +L_check_mostly_zero: + // how much space will the sparse packer take? + add rax, rax, r11 // rax += (next_qp - start_next_qp) + mov rdx, #6 + mov rcx, #4 + madd r11, rax, rdx, rcx // r11 = rax * 6 (i.e. 4 byte word + 2 byte offset) + 4 byte for header + + sub rax, next_low_bits, start_next_low_bits // get bytes consumed by lower-10 bits + mov rdx, #1365 + mul rax, rax, rdx + + sub rdx, next_full_patt, start_next_full_patt // get bytes consumed by dictionary misses + add rax, rdx, rax, lsr #11 // rax = 2/3*(next_low_bits - start_next_low_bits) + (next_full_patt - start_next_full_patt) + + sub rdx, next_qp, start_next_qp + add rax, rax, rdx, lsr #1 // rax += (next_qp - start_next_qp)/2 + add rax, rax, #(12+256*scale) // rax += bytes taken by the header + tags + + cmp rax, r11 // is the default packer the better option? + b.lt L_done_search + + cmp r11, byte_budget // can the sparse packer fit into the given budget? + b.gt L_budgetExhausted + +L_sparse_packer: + mov edx, #MZV_MAGIC + str edx, [dest_buf], #4 // header to indicate a sparse packer + + mov rdx, #0 // rdx = byte offset in src of non-0 word +1: + ldr rax, [start_next_input_word, rdx] // rax = read dword + cbnz rax, 5f // is dword != 0 +3: + add rdx, rdx, #8 // 8 more bytes have been processed +4: + cmp rdx, #(4096*scale) // has the entire page been processed + b.ne 1b + mov x0, r11 // store the size of the compressed stream + b L_done + +5: + cbz eax, 6f // is lower word == 0 + str eax, [dest_buf], #4 // store the non-0 word in the dest buffer + strh edx, [dest_buf], #2 // store the byte index +6: + lsr rax, rax, 32 // get the upper word into position + cbz eax, 3b // is dword == 0 + add rdx, rdx, #4 + str eax, [dest_buf], #4 // store the non-0 word in the dest buffer + strh edx, [dest_buf], #2 // store the byte index + add rdx, rdx, #4 + b 4b + +L_check_compression_ratio: + + mov mode, NORMAL + mov remaining, #((1024 - CHKPT_WORDS)*scale) // remaining input words to process + cbz remaining, CHECKPOINT // if there are no remaining words to process + + sub rax, next_low_bits, start_next_low_bits // get bytes consumed by lower-10 bits + mov rdx, #1365 + mul rax, rax, rdx + + sub rdx, next_full_patt, start_next_full_patt // get bytes consumed by dictionary misses + add rax, rdx, rax, lsr #11 // rax = 2/3*(next_low_bits - start_next_low_bits) + (next_full_patt - start_next_full_patt) + + sub rdx, next_qp, start_next_qp + add rax, rax, rdx, lsr #1 // rax += (next_qp - start_next_qp)/2 + subs rax, rax, #((CHKPT_SHRUNK_BYTES - CHKPT_TAG_BYTES)*scale) + // rax += CHKPT_TAG_BYTES; rax -= CHKPT_SHRUNK_BYTES + + b.gt L_budgetExhausted // if rax is > 0, we need to early abort + b L_loop // we are done diff --git a/osfmk/arm64/WKdmData.s b/osfmk/arm64/WKdmData.s new file mode 100644 index 000000000..2f2736bbe --- /dev/null +++ b/osfmk/arm64/WKdmData.s @@ -0,0 +1,330 @@ +/* + * Copyright (c) 2000-2014 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + + +.globl _hashLookupTable + .const + .align 5 +_hashLookupTable: + .byte 0 + .byte 52 + .byte 8 + .byte 56 + .byte 16 + .byte 12 + .byte 28 + .byte 20 + .byte 4 + .byte 36 + .byte 48 + .byte 24 + .byte 44 + .byte 40 + .byte 32 + .byte 60 + .byte 8 + .byte 12 + .byte 28 + .byte 20 + .byte 4 + .byte 60 + .byte 16 + .byte 36 + .byte 24 + .byte 48 + .byte 44 + .byte 32 + .byte 52 + .byte 56 + .byte 40 + .byte 12 + .byte 8 + .byte 48 + .byte 16 + .byte 52 + .byte 60 + .byte 28 + .byte 56 + .byte 32 + .byte 20 + .byte 24 + .byte 36 + .byte 40 + .byte 44 + .byte 4 + .byte 8 + .byte 40 + .byte 60 + .byte 32 + .byte 20 + .byte 44 + .byte 4 + .byte 36 + .byte 52 + .byte 24 + .byte 16 + .byte 56 + .byte 48 + .byte 12 + .byte 28 + .byte 16 + .byte 8 + .byte 40 + .byte 36 + .byte 28 + .byte 32 + .byte 12 + .byte 4 + .byte 44 + .byte 52 + .byte 20 + .byte 24 + .byte 48 + .byte 60 + .byte 56 + .byte 40 + .byte 48 + .byte 8 + .byte 32 + .byte 28 + .byte 36 + .byte 4 + .byte 44 + .byte 20 + .byte 56 + .byte 60 + .byte 24 + .byte 52 + .byte 16 + .byte 12 + .byte 12 + .byte 4 + .byte 48 + .byte 20 + .byte 8 + .byte 52 + .byte 16 + .byte 60 + .byte 24 + .byte 36 + .byte 44 + .byte 28 + .byte 56 + .byte 40 + .byte 32 + .byte 36 + .byte 20 + .byte 24 + .byte 60 + .byte 40 + .byte 44 + .byte 52 + .byte 16 + .byte 32 + .byte 4 + .byte 48 + .byte 8 + .byte 28 + .byte 56 + .byte 12 + .byte 28 + .byte 32 + .byte 40 + .byte 52 + .byte 36 + .byte 16 + .byte 20 + .byte 48 + .byte 8 + .byte 4 + .byte 60 + .byte 24 + .byte 56 + .byte 44 + .byte 12 + .byte 8 + .byte 36 + .byte 24 + .byte 28 + .byte 16 + .byte 60 + .byte 20 + .byte 56 + .byte 32 + .byte 40 + .byte 48 + .byte 12 + .byte 4 + .byte 44 + .byte 52 + .byte 44 + .byte 40 + .byte 12 + .byte 56 + .byte 8 + .byte 36 + .byte 24 + .byte 60 + .byte 28 + .byte 48 + .byte 4 + .byte 32 + .byte 20 + .byte 16 + .byte 52 + .byte 60 + .byte 12 + .byte 24 + .byte 36 + .byte 8 + .byte 4 + .byte 16 + .byte 56 + .byte 48 + .byte 44 + .byte 40 + .byte 52 + .byte 32 + .byte 20 + .byte 28 + .byte 32 + .byte 12 + .byte 36 + .byte 28 + .byte 24 + .byte 56 + .byte 40 + .byte 16 + .byte 52 + .byte 44 + .byte 4 + .byte 20 + .byte 60 + .byte 8 + .byte 48 + .byte 48 + .byte 52 + .byte 12 + .byte 20 + .byte 32 + .byte 44 + .byte 36 + .byte 28 + .byte 4 + .byte 40 + .byte 24 + .byte 8 + .byte 56 + .byte 60 + .byte 16 + .byte 36 + .byte 32 + .byte 8 + .byte 40 + .byte 4 + .byte 52 + .byte 24 + .byte 44 + .byte 20 + .byte 12 + .byte 28 + .byte 48 + .byte 56 + .byte 16 + .byte 60 + .byte 4 + .byte 52 + .byte 60 + .byte 48 + .byte 20 + .byte 16 + .byte 56 + .byte 44 + .byte 24 + .byte 8 + .byte 40 + .byte 12 + .byte 32 + .byte 28 + .byte 36 + .byte 24 + .byte 32 + .byte 12 + .byte 4 + .byte 20 + .byte 16 + .byte 60 + .byte 36 + .byte 28 + .byte 8 + .byte 52 + .byte 40 + .byte 48 + .byte 44 + .byte 56 + + .globl _table_2bits +_table_2bits: + .word 0 + .word -2 + .word -4 + .word -6 + .word 0x03030303 + .word 0x03030303 + .word 0x03030303 + .word 0x03030303 + + .globl _table_4bits +_table_4bits: + .word 0 + .word -4 + .word 0 + .word -4 + .word 0x0f0f0f0f + .word 0x0f0f0f0f + .word 0x0f0f0f0f + .word 0x0f0f0f0f + + .globl _table_10bits +_table_10bits: + .word 6 + .word 0 + .word 6 + .word 0 + .word 0 + .word -20 + .word 0 + .word -20 + .word (1023<<16) + .word 0 + .word (1023<<16) + .word 0 + .word 1023 + .word 1023 + .word 1023 + .word 1023 diff --git a/osfmk/arm64/WKdmDecompress_16k.s b/osfmk/arm64/WKdmDecompress_16k.s new file mode 100644 index 000000000..b3def749b --- /dev/null +++ b/osfmk/arm64/WKdmDecompress_16k.s @@ -0,0 +1,428 @@ +/* + * Copyright (c) 2000-2014 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +/* + This file contains arm64 hand optimized implementation of WKdm memory page decompressor. + + void WKdm_decompress (WK_word* src_buf, WK_word* dest_buf, WK_word *scratch, __unused__ unsigned int words); + + input : + src_buf : address of input compressed data buffer + dest_buf : address of output decompressed buffer + scratch : an 8-k bytes scratch mempro provided by the caller + words : this argument is used by the mostly-zero-value decoder + + output : + + the input buffer is decompressed and the dest_buf is written with decompressed data. + + Am algorithm description of the WKdm compress and bit stream format can be found in the WKdm Compress arm64 assembly code WKdmCompress.s + + The bit stream (*src_buf) consists of + a. 12 bytes header + b. 256 bytes for 1024 packed tags + c. (varying number of) words for new words not matched to dictionary word. + d. (varying number of) 32-bit words for packed 4-bit dict_indices (for class 1 and 3) + e. (varying number of) 32-bit words for packed 10-bit low bits (for class 1) + + where the header (of 3 words) specifies the ending boundaries (in 32-bit words) of the bit stream of c,d,e, respectively. + + The decompressor 1st unpacking the bit stream component b/d/e into temorary buffers. Then it sequentially decodes the decompressed word as follows + + for (i=0;i<1024;i++) { + tag = *next_tag++ + switch (tag) { + case 0 : *dest_buf++ = 0; break; + case 1 : dict_word = dictionary[*dict_index]; dictionary[*dict_index++] = *dest_buf++ = dict_word&0xfffffc00 | *LowBits++; break; + case 2 : x = *new_word++; k = (x>>10)&255; k = hashTable[k]; dictionary[k] = *dest_buf++ = x; break; + case 3 : *dest_buf++ = dictionary[*dict_index++]; break; + } + + cclee, Nov 9, '12 + + Added zero page, single value page, sparse page, early abort optimizations + rsrini, 09/14/14 +*/ + +#define PAGES_SIZE_IN_KBYTES 16 +#define MZV_MAGIC 17185 // magic value used to identify MZV page encoding + +#ifndef PAGES_SIZE_IN_KBYTES +#define PAGES_SIZE_IN_KBYTES 4 +#endif + +#if !((PAGES_SIZE_IN_KBYTES==4) || (PAGES_SIZE_IN_KBYTES==16)) +#error "Only PAGES_SIZE_IN_KBYTES = 4 or 16 is supported" +#endif + +#define scale (PAGES_SIZE_IN_KBYTES/4) + + + .align 4 + .text + +/* + void WKdm_decompress (WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, unsigned int bytes); +*/ + + .globl _WKdm_decompress_16k +_WKdm_decompress_16k: + + /* + -------- symbolizing registers -------- + the arm64 code was ported from x86_64 so we name some registers that are used as temp variables with x86_64 register names. + */ + + #define src_buf x0 + #define dest_buf x1 + #define scratch x2 + #define n_bytes x3 + #define dictionary sp + #define rax x13 + #define eax w13 + #define rbx x4 + #define ebx w4 + #define rcx x5 + #define ecx w5 + #define rdx x6 + #define edx w6 + #define tags_counter x7 + #define next_tag x12 + #define r8 x8 + #define r9 x9 + #define r10 x10 + #define r11 x11 + #define r12 x12 + + /* + + ------ scratch memory for local variables --------- + + [sp,#0] : dictionary + [scratch,#0] : tempTagsArray + [scratch,#1024] : tempQPosArray + [scratch,#2048] : tempLowBitsArray + + */ + +#if KERNEL + sub rax, sp, #96 + sub sp, sp, #96 + st1.4s {v0,v1,v2},[rax],#48 + st1.4s {v3,v4,v5},[rax],#48 +#endif + + sub sp, sp, #64 + + ldr eax, [src_buf] // read the 1st word from the header + mov ecx, #MZV_MAGIC + cmp eax, ecx // is the alternate packer used (i.e. is MZV page)? + b.ne L_default_decompressor // default decompressor was used + + // Mostly Zero Page Handling... + // { + add src_buf, src_buf, 4 // skip the header + mov rax, dest_buf + mov rcx, #(PAGES_SIZE_IN_KBYTES*1024) // number of bytes to zero out +1: + dc zva, rax // zero 64 bytes. since dest_buf is a page, it will be 4096 or 16384 byte aligned + add rax, rax, #64 + dc zva, rax + add rax, rax, #64 + dc zva, rax + add rax, rax, #64 + dc zva, rax + add rax, rax, #64 + subs rcx, rcx, #256 + b.ne 1b + + mov r12, #4 // current byte position in src to read from + mov rdx, #0 +2: + ldr eax, [src_buf], #4 // get the word + ldrh edx, [src_buf], #2 // get the index + str eax, [dest_buf, rdx] // store non-0 word in the destination buffer + add r12, r12, #6 // 6 more bytes processed + cmp r12, n_bytes // finished processing all the bytes? + b.ne 2b + b L_done + // } + +L_default_decompressor: + + /* + ---------------------- set up registers and PRELOAD_DICTIONARY --------------------------------- + */ + // NOTE: ALL THE DICTIONARY VALUES MUST BE INITIALIZED TO ZERO TO MIRROR THE COMPRESSOR + adrp rbx, _table_2bits@GOTPAGE + stp xzr, xzr, [dictionary, #0] + add r10, src_buf, #(12+256*scale) // TAGS_AREA_END + stp xzr, xzr, [dictionary, #16] + add rax, src_buf, #12 // TAGS_AREA_START + ldr rbx, [rbx, _table_2bits@GOTPAGEOFF] + stp xzr, xzr, [dictionary, #32] + mov rcx, scratch // tempTagsArray + stp xzr, xzr, [dictionary, #48] + ld1.4s {v0,v1},[rbx] + + + /* + ------------------------------ unpacking bit stream ---------------------------------- + */ + + // WK_unpack_2bits(TAGS_AREA_START(src_buf), TAGS_AREA_END(src_buf), tempTagsArray); +/* + unpacking 16 2-bit tags (from a 32-bit word) into 16 bytes + for arm64, this can be done by + 1. read the input 32-bit word into GPR w + 2. duplicate GPR into 4 elements in a vector register v0 + 3. ushl.4s vd, v0, vshift where vshift = {0, -2, -4, -6} + 4. and.4s vd, vd, vmask where vmask = 0x03030303030303030303030303030303 +*/ + +L_WK_unpack_2bits: + ldr q5, [rax], #16 // read 4 32-bit words for 64 2-bit tags + dup.4s v2, v5[0] // duplicate to 4 elements + dup.4s v3, v5[1] // duplicate to 4 elements + dup.4s v4, v5[2] // duplicate to 4 elements + dup.4s v5, v5[3] // duplicate to 4 elements + ushl.4s v2, v2, v0 // v1 = {0, -2, -4, -6} + ushl.4s v3, v3, v0 // v1 = {0, -2, -4, -6} + ushl.4s v4, v4, v0 // v1 = {0, -2, -4, -6} + ushl.4s v5, v5, v0 // v1 = {0, -2, -4, -6} + and.16b v2, v2, v1 // v2 = {3,3,...,3} + and.16b v3, v3, v1 // v2 = {3,3,...,3} + and.16b v4, v4, v1 // v2 = {3,3,...,3} + and.16b v5, v5, v1 // v2 = {3,3,...,3} + cmp r10, rax // TAGS_AREA_END vs TAGS_AREA_START + st1.4s {v2,v3,v4,v5}, [rcx], #64 // write 64 tags into tempTagsArray + b.hi L_WK_unpack_2bits // if not reach TAGS_AREA_END, repeat L_WK_unpack_2bits + + + // WK_unpack_4bits(QPOS_AREA_START(src_buf), QPOS_AREA_END(src_buf), tempQPosArray); + + ldp w8, w9, [src_buf] // WKdm header qpos start and end + adrp rbx, _table_4bits@GOTPAGE + subs x14, r9, r8 // x14 = (QPOS_AREA_END - QPOS_AREA_START)/4 + add r8, src_buf, r8, lsl #2 // QPOS_AREA_START + add r9, src_buf, r9, lsl #2 // QPOS_AREA_END + + b.ls 1f // if QPOS_AREA_END <= QPOS_AREA_START, skip L_WK_unpack_4bits + ldr rbx, [rbx, _table_4bits@GOTPAGEOFF] + add rcx, scratch, #(1024*scale) // tempQPosArray + ld1.4s {v0,v1},[rbx] + subs w14, w14, #1 + b.ls 2f // do loop of 2 only if w14 >= 5 +L_WK_unpack_4bits: + ldr d2, [r8], #8 // read a 32-bit word for 8 4-bit positions + subs w14, w14, #2 + zip1.4s v2, v2, v2 + ushl.4s v2, v2, v0 // v1 = {0, -4, 0, -4} + and.16b v2, v2, v1 // v2 = {15,15,...,15} + str q2, [rcx], #16 + b.hi L_WK_unpack_4bits +2: + adds w14, w14, #1 + b.le 1f + + ldr s3, [r8], #4 // read a 32-bit word for 8 4-bit positions + dup.2s v2, v3[0] // duplicate to 2 elements + ushl.2s v2, v2, v0 // v1 = {0, -4} + and.8b v2, v2, v1 // v2 = {15,15,...,15} + str d2, [rcx], #8 // write 16 tags into tempTagsArray + +1: + + // WK_unpack_3_tenbits(LOW_BITS_AREA_START(src_buf), LOW_BITS_AREA_END(src_buf), tempLowBitsArray); + + ldr eax, [src_buf,#8] // LOW_BITS_AREA_END offset + add r8, src_buf, rax, lsl #2 // LOW_BITS_AREA_END + add rcx, scratch, #(2048*scale) // tempLowBitsArray +#if (scale==1) + add r11, scratch, #(4096*scale-2) // final tenbits for the rare case +#else + add r11, scratch, #(4096*scale) // final tenbits for the rare case + sub r11, r11, #2 +#endif + subs r8, r8, r9 // LOW_BITS_AREA_START vs LOW_BITS_AREA_END + b.ls 1f // if START>=END, skip L_WK_unpack_3_tenbits + + adrp rbx, _table_10bits@GOTPAGE + ldr rbx, [rbx, _table_10bits@GOTPAGEOFF] + ld1.4s {v0,v1,v2,v3},[rbx] + + /* + a very rare case : 1024 tenbits, 1023 + 1 -> 341 + final 1 that is padded with 2 zeros + since the scratch memory is 4k (2k for this section), we need to pay attention to the last case + so we don't overwrite to the scratch memory + + we 1st do a single 3_tenbits, followed by 2x_3_tenbits loop, and detect whether the last 3_tenbits + hits the raee case + */ +#if 1 + subs r8, r8, #4 // pre-decrement by 8 + ldr s4, [r9], #4 // read 32-bit words for 3 low 10-bits + zip1.4s v4, v4, v4 // bits 0-63 contain first triplet twice, bits 64-127 contain second triplet twice. + ushl.4s v5, v4, v0 // v0 = {6, 0, 6, 0}, places second element of triplets into bits 16-25 and 80-89. + ushl.4s v4, v4, v1 // v1 = {0, -20, 0, -20}, places third element of triplets into bits 32-41 and 96-105. + and.16b v5, v5, v2 // v2 = {0, 1023, 0, 0, 0, 1023, 0, 0}, isolate second element of triplets. + and.16b v4, v4, v3 // v3 = {1023, 0, 1023, 0, 1023, 0, 1023, 0}, isolate first and third elements of triplets + orr.16b v4, v4, v5 // combine data + str d4, [rcx], #6 // write 3 low 10-bits + b.eq 1f +#endif + + subs r8, r8, #8 // pre-decrement by 8 + b.lt L_WK_unpack_3_tenbits + +L_WK_unpack_2x_3_tenbits: + ldr d4, [r9], #8 // read 2 32-bit words for a pair of 3 low 10-bits + zip1.4s v4, v4, v4 // bits 0-63 contain first triplet twice, bits 64-127 contain second triplet twice. + ushl.4s v5, v4, v0 // v0 = {6, 0, 6, 0}, places second element of triplets into bits 16-25 and 80-89. + ushl.4s v4, v4, v1 // v1 = {0, -20, 0, -20}, places third element of triplets into bits 32-41 and 96-105. + and.16b v5, v5, v2 // v2 = {0, 1023, 0, 0, 0, 1023, 0, 0}, isolate second element of triplets. + and.16b v4, v4, v3 // v3 = {1023, 0, 1023, 0, 1023, 0, 1023, 0}, isolate first and third elements of triplets + orr.16b v4, v4, v5 // combine data + ins v5.d[0], v4.d[1] + str d4, [rcx], #6 // write 3 low 10-bits + str d5, [rcx], #6 // write 3 low 10-bits + + subs r8, r8, #8 + b.ge L_WK_unpack_2x_3_tenbits // repeat loop if LOW_BITS_AREA_END > next_word + + tst r8, #4 + b.eq 1f + +L_WK_unpack_3_tenbits: + ldr s4, [r9] // read 32-bit words for 3 low 10-bits + zip1.4s v4, v4, v4 // bits 0-63 contain first triplet twice, bits 64-127 contain second triplet twice. + ushl.4s v5, v4, v0 // v0 = {6, 0, 6, 0}, places second element of triplets into bits 16-25 and 80-89. + ushl.4s v4, v4, v1 // v1 = {0, -20, 0, -20}, places third element of triplets into bits 32-41 and 96-105. + and.16b v5, v5, v2 // v2 = {0, 1023, 0, 0, 0, 1023, 0, 0}, isolate second element of triplets. + and.16b v4, v4, v3 // v3 = {1023, 0, 1023, 0, 1023, 0, 1023, 0}, isolate first and third elements of triplets + orr.16b v4, v4, v5 // combine data +#if 0 + str d4, [rcx] // write 3 low 10-bits +#else + cmp rcx, r11 + b.eq 2f + str d4, [rcx] // write 3 low 10-bits + b 1f +2: + str h4, [rcx] // write final 1 low 10-bits +#endif +1: + + /* + set up before going to the main decompress loop + */ + mov next_tag, scratch // tempTagsArray + add r8, scratch, #(1024*scale) // next_qpos + add r11, scratch, #(2048*scale) // tempLowBitsArray + adrp rbx, _hashLookupTable@GOTPAGE + mov tags_counter, #(1024*scale) // tag_area_end + ldr rbx, [rbx, _hashLookupTable@GOTPAGEOFF] + + b L_next + + .align 4,0x90 +L_ZERO_TAG: + /* + we can only get here if w9 = 0, meaning this is a zero tag + *dest_buf++ = 0; + */ + str w9, [dest_buf], #4 // *dest_buf++ = 0 + subs tags_counter, tags_counter, #1 // next_tag vs tag_area_end + b.ls L_done // if next_tag >= tag_area_end, we're done + + /* WKdm decompress main loop */ +L_next: + ldrb w9, [next_tag], #1 // new tag + cbz w9, L_ZERO_TAG + cmp w9, #2 // partial match tag ? + b.eq L_MISS_TAG + b.gt L_EXACT_TAG + +L_PARTIAL_TAG: + /* + this is a partial match: + dict_word = dictionary[*dict_index]; + dictionary[*dict_index++] = *dest_buf++ = dict_word&0xfffffc00 | *LowBits++; + */ + + ldrb edx, [r8], #1 // qpos = *next_qpos++ + ldrh ecx, [r11], #2 // lower 10-bits from *next_low_bits++ + ldr eax, [dictionary, rdx, lsl #2] // read dictionary word + bfm eax, ecx, #0, #9 // pad the lower 10-bits from *next_low_bits + str eax, [dictionary,rdx,lsl #2] // *dict_location = newly formed word + str eax, [dest_buf], #4 // *dest_buf++ = newly formed word + subs tags_counter, tags_counter, #1 // next_tag vs tag_area_end + b.gt L_next // repeat loop until next_tag==tag_area_end + +L_done: + + // release stack memory, restore registers, and return + add sp, sp, #64 // deallocate for dictionary +#if KERNEL + ld1.4s {v0,v1,v2},[sp],#48 + ld1.4s {v3,v4,v5},[sp],#48 +#endif + ret lr + + .align 4,0x90 +L_MISS_TAG: + /* + this is a dictionary miss. + x = *new_word++; + k = (x>>10)&255; + k = hashTable[k]; + dictionary[k] = *dest_buf++ = x; + */ + ldr eax, [r10], #4 // w = *next_full_patt++ + ubfm edx, eax, #10, #17 // 8-bit hash table index + str eax, [dest_buf], #4 // *dest_buf++ = word + ldrb edx, [rbx, rdx] // qpos + str eax, [dictionary,rdx] // dictionary[qpos] = word + subs tags_counter, tags_counter, #1 // next_tag vs tag_area_end + b.gt L_next // repeat the loop + b L_done // if next_tag >= tag_area_end, we're done + + .align 4,0x90 +L_EXACT_TAG: + /* + this is an exact match; + *dest_buf++ = dictionary[*dict_index++]; + */ + ldrb eax, [r8], #1 // qpos = *next_qpos++ + ldr eax, [dictionary,rax,lsl #2] // w = dictionary[qpos] + str eax, [dest_buf], #4 // *dest_buf++ = w + subs tags_counter, tags_counter, #1 // next_tag vs tag_area_end + b.gt L_next // repeat the loop + b L_done // if next_tag >= tag_area_end, we're done + + diff --git a/osfmk/arm64/WKdmDecompress_4k.s b/osfmk/arm64/WKdmDecompress_4k.s new file mode 100644 index 000000000..1b399ea33 --- /dev/null +++ b/osfmk/arm64/WKdmDecompress_4k.s @@ -0,0 +1,428 @@ +/* + * Copyright (c) 2000-2014 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +/* + This file contains arm64 hand optimized implementation of WKdm memory page decompressor. + + void WKdm_decompress (WK_word* src_buf, WK_word* dest_buf, WK_word *scratch, __unused__ unsigned int words); + + input : + src_buf : address of input compressed data buffer + dest_buf : address of output decompressed buffer + scratch : an 8-k bytes scratch mempro provided by the caller + words : this argument is not used in the implementation + (The 4th argument is, in fact, used by the Mostly Zero Value decoder) + + output : + + the input buffer is decompressed and the dest_buf is written with decompressed data. + + Am algorithm description of the WKdm compress and bit stream format can be found in the WKdm Compress arm64 assembly code WKdmCompress.s + + The bit stream (*src_buf) consists of + a. 12 bytes header + b. 256 bytes for 1024 packed tags + c. (varying number of) words for new words not matched to dictionary word. + d. (varying number of) 32-bit words for packed 4-bit dict_indices (for class 1 and 3) + e. (varying number of) 32-bit words for packed 10-bit low bits (for class 1) + + where the header (of 3 words) specifies the ending boundaries (in 32-bit words) of the bit stream of c,d,e, respectively. + + The decompressor 1st unpacking the bit stream component b/d/e into temorary buffers. Then it sequentially decodes the decompressed word as follows + + for (i=0;i<1024;i++) { + tag = *next_tag++ + switch (tag) { + case 0 : *dest_buf++ = 0; break; + case 1 : dict_word = dictionary[*dict_index]; dictionary[*dict_index++] = *dest_buf++ = dict_word&0xfffffc00 | *LowBits++; break; + case 2 : x = *new_word++; k = (x>>10)&255; k = hashTable[k]; dictionary[k] = *dest_buf++ = x; break; + case 3 : *dest_buf++ = dictionary[*dict_index++]; break; + } + + cclee, Nov 9, '12 + + Added zero page, single value page, sparse page, early abort optimizations + rsrini, 09/14/14 +*/ + +#define MZV_MAGIC 17185 // magic value used to identify MZV page encoding + +#ifndef PAGES_SIZE_IN_KBYTES +#define PAGES_SIZE_IN_KBYTES 4 +#endif + +#if !((PAGES_SIZE_IN_KBYTES==4) || (PAGES_SIZE_IN_KBYTES==16)) +#error "Only PAGES_SIZE_IN_KBYTES = 4 or 16 is supported" +#endif + +#define scale (PAGES_SIZE_IN_KBYTES/4) + + + .align 4 + .text + +/* + void WKdm_decompress (WK_word* src_buf, WK_word* dest_buf, WK_word* scratch, unsigned int bytes); +*/ + + .globl _WKdm_decompress_4k +_WKdm_decompress_4k: + + /* + -------- symbolizing registers -------- + the arm64 code was ported from x86_64 so we name some registers that are used as temp variables with x86_64 register names. + */ + + #define src_buf x0 + #define dest_buf x1 + #define scratch x2 + #define n_bytes x3 + #define dictionary sp + #define rax x13 + #define eax w13 + #define rbx x4 + #define ebx w4 + #define rcx x5 + #define ecx w5 + #define rdx x6 + #define edx w6 + #define tags_counter x7 + #define next_tag x12 + #define r8 x8 + #define r9 x9 + #define r10 x10 + #define r11 x11 + #define r12 x12 + + /* + + ------ scratch memory for local variables --------- + + [sp,#0] : dictionary + [scratch,#0] : tempTagsArray + [scratch,#1024] : tempQPosArray + [scratch,#2048] : tempLowBitsArray + + */ + +#if KERNEL + sub rax, sp, #96 + sub sp, sp, #96 + st1.4s {v0,v1,v2},[rax],#48 + st1.4s {v3,v4,v5},[rax],#48 +#endif + + sub sp, sp, #64 + + ldr eax, [src_buf] // read the 1st word from the header + mov ecx, #MZV_MAGIC + cmp eax, ecx // is the alternate packer used (i.e. is MZV page)? + b.ne L_default_decompressor // default decompressor was used + + // Mostly Zero Page Handling... + // { + add src_buf, src_buf, 4 // skip the header + mov rax, dest_buf + mov rcx, #(PAGES_SIZE_IN_KBYTES*1024) // number of bytes to zero out +1: + dc zva, rax // zero 64 bytes. since dest_buf is a page, it will be 4096 or 16384 byte aligned + add rax, rax, #64 + dc zva, rax + add rax, rax, #64 + dc zva, rax + add rax, rax, #64 + dc zva, rax + add rax, rax, #64 + subs rcx, rcx, #256 + b.ne 1b + + mov r12, #4 // current byte position in src to read from + mov rdx, #0 +2: + ldr eax, [src_buf], #4 // get the word + ldrh edx, [src_buf], #2 // get the index + str eax, [dest_buf, rdx] // store non-0 word in the destination buffer + add r12, r12, #6 // 6 more bytes processed + cmp r12, n_bytes // finished processing all the bytes? + b.ne 2b + b L_done + // } + +L_default_decompressor: + + /* + ---------------------- set up registers and PRELOAD_DICTIONARY --------------------------------- + */ + // NOTE: ALL THE DICTIONARY VALUES MUST BE INITIALIZED TO ZERO TO MIRROR THE COMPRESSOR + adrp rbx, _table_2bits@GOTPAGE + stp xzr, xzr, [dictionary, #0] + add r10, src_buf, #(12+256*scale) // TAGS_AREA_END + stp xzr, xzr, [dictionary, #16] + add rax, src_buf, #12 // TAGS_AREA_START + ldr rbx, [rbx, _table_2bits@GOTPAGEOFF] + stp xzr, xzr, [dictionary, #32] + mov rcx, scratch // tempTagsArray + stp xzr, xzr, [dictionary, #48] + ld1.4s {v0,v1},[rbx] + + + /* + ------------------------------ unpacking bit stream ---------------------------------- + */ + + // WK_unpack_2bits(TAGS_AREA_START(src_buf), TAGS_AREA_END(src_buf), tempTagsArray); +/* + unpacking 16 2-bit tags (from a 32-bit word) into 16 bytes + for arm64, this can be done by + 1. read the input 32-bit word into GPR w + 2. duplicate GPR into 4 elements in a vector register v0 + 3. ushl.4s vd, v0, vshift where vshift = {0, -2, -4, -6} + 4. and.4s vd, vd, vmask where vmask = 0x03030303030303030303030303030303 +*/ + +L_WK_unpack_2bits: + ldr q5, [rax], #16 // read 4 32-bit words for 64 2-bit tags + dup.4s v2, v5[0] // duplicate to 4 elements + dup.4s v3, v5[1] // duplicate to 4 elements + dup.4s v4, v5[2] // duplicate to 4 elements + dup.4s v5, v5[3] // duplicate to 4 elements + ushl.4s v2, v2, v0 // v1 = {0, -2, -4, -6} + ushl.4s v3, v3, v0 // v1 = {0, -2, -4, -6} + ushl.4s v4, v4, v0 // v1 = {0, -2, -4, -6} + ushl.4s v5, v5, v0 // v1 = {0, -2, -4, -6} + and.16b v2, v2, v1 // v2 = {3,3,...,3} + and.16b v3, v3, v1 // v2 = {3,3,...,3} + and.16b v4, v4, v1 // v2 = {3,3,...,3} + and.16b v5, v5, v1 // v2 = {3,3,...,3} + cmp r10, rax // TAGS_AREA_END vs TAGS_AREA_START + st1.4s {v2,v3,v4,v5}, [rcx], #64 // write 64 tags into tempTagsArray + b.hi L_WK_unpack_2bits // if not reach TAGS_AREA_END, repeat L_WK_unpack_2bits + + + // WK_unpack_4bits(QPOS_AREA_START(src_buf), QPOS_AREA_END(src_buf), tempQPosArray); + + ldp w8, w9, [src_buf] // WKdm header qpos start and end + adrp rbx, _table_4bits@GOTPAGE + subs x14, r9, r8 // x14 = (QPOS_AREA_END - QPOS_AREA_START)/4 + add r8, src_buf, r8, lsl #2 // QPOS_AREA_START + add r9, src_buf, r9, lsl #2 // QPOS_AREA_END + + b.ls 1f // if QPOS_AREA_END <= QPOS_AREA_START, skip L_WK_unpack_4bits + ldr rbx, [rbx, _table_4bits@GOTPAGEOFF] + add rcx, scratch, #(1024*scale) // tempQPosArray + ld1.4s {v0,v1},[rbx] + subs w14, w14, #1 + b.ls 2f // do loop of 2 only if w14 >= 5 +L_WK_unpack_4bits: + ldr d2, [r8], #8 // read a 32-bit word for 8 4-bit positions + subs w14, w14, #2 + zip1.4s v2, v2, v2 + ushl.4s v2, v2, v0 // v1 = {0, -4, 0, -4} + and.16b v2, v2, v1 // v2 = {15,15,...,15} + str q2, [rcx], #16 + b.hi L_WK_unpack_4bits +2: + adds w14, w14, #1 + b.le 1f + + ldr s3, [r8], #4 // read a 32-bit word for 8 4-bit positions + dup.2s v2, v3[0] // duplicate to 2 elements + ushl.2s v2, v2, v0 // v1 = {0, -4} + and.8b v2, v2, v1 // v2 = {15,15,...,15} + str d2, [rcx], #8 // write 16 tags into tempTagsArray + +1: + + // WK_unpack_3_tenbits(LOW_BITS_AREA_START(src_buf), LOW_BITS_AREA_END(src_buf), tempLowBitsArray); + + ldr eax, [src_buf,#8] // LOW_BITS_AREA_END offset + add r8, src_buf, rax, lsl #2 // LOW_BITS_AREA_END + add rcx, scratch, #(2048*scale) // tempLowBitsArray +#if (scale==1) + add r11, scratch, #(4096*scale-2) // final tenbits for the rare case +#else + add r11, scratch, #(4096*scale) // final tenbits for the rare case + sub r11, r11, #2 +#endif + subs r8, r8, r9 // LOW_BITS_AREA_START vs LOW_BITS_AREA_END + b.ls 1f // if START>=END, skip L_WK_unpack_3_tenbits + + adrp rbx, _table_10bits@GOTPAGE + ldr rbx, [rbx, _table_10bits@GOTPAGEOFF] + ld1.4s {v0,v1,v2,v3},[rbx] + + /* + a very rare case : 1024 tenbits, 1023 + 1 -> 341 + final 1 that is padded with 2 zeros + since the scratch memory is 4k (2k for this section), we need to pay attention to the last case + so we don't overwrite to the scratch memory + + we 1st do a single 3_tenbits, followed by 2x_3_tenbits loop, and detect whether the last 3_tenbits + hits the raee case + */ +#if 1 + subs r8, r8, #4 // pre-decrement by 8 + ldr s4, [r9], #4 // read 32-bit words for 3 low 10-bits + zip1.4s v4, v4, v4 // bits 0-63 contain first triplet twice, bits 64-127 contain second triplet twice. + ushl.4s v5, v4, v0 // v0 = {6, 0, 6, 0}, places second element of triplets into bits 16-25 and 80-89. + ushl.4s v4, v4, v1 // v1 = {0, -20, 0, -20}, places third element of triplets into bits 32-41 and 96-105. + and.16b v5, v5, v2 // v2 = {0, 1023, 0, 0, 0, 1023, 0, 0}, isolate second element of triplets. + and.16b v4, v4, v3 // v3 = {1023, 0, 1023, 0, 1023, 0, 1023, 0}, isolate first and third elements of triplets + orr.16b v4, v4, v5 // combine data + str d4, [rcx], #6 // write 3 low 10-bits + b.eq 1f +#endif + + subs r8, r8, #8 // pre-decrement by 8 + b.lt L_WK_unpack_3_tenbits + +L_WK_unpack_2x_3_tenbits: + ldr d4, [r9], #8 // read 2 32-bit words for a pair of 3 low 10-bits + zip1.4s v4, v4, v4 // bits 0-63 contain first triplet twice, bits 64-127 contain second triplet twice. + ushl.4s v5, v4, v0 // v0 = {6, 0, 6, 0}, places second element of triplets into bits 16-25 and 80-89. + ushl.4s v4, v4, v1 // v1 = {0, -20, 0, -20}, places third element of triplets into bits 32-41 and 96-105. + and.16b v5, v5, v2 // v2 = {0, 1023, 0, 0, 0, 1023, 0, 0}, isolate second element of triplets. + and.16b v4, v4, v3 // v3 = {1023, 0, 1023, 0, 1023, 0, 1023, 0}, isolate first and third elements of triplets + orr.16b v4, v4, v5 // combine data + ins v5.d[0], v4.d[1] + str d4, [rcx], #6 // write 3 low 10-bits + str d5, [rcx], #6 // write 3 low 10-bits + + subs r8, r8, #8 + b.ge L_WK_unpack_2x_3_tenbits // repeat loop if LOW_BITS_AREA_END > next_word + + tst r8, #4 + b.eq 1f + +L_WK_unpack_3_tenbits: + ldr s4, [r9] // read 32-bit words for 3 low 10-bits + zip1.4s v4, v4, v4 // bits 0-63 contain first triplet twice, bits 64-127 contain second triplet twice. + ushl.4s v5, v4, v0 // v0 = {6, 0, 6, 0}, places second element of triplets into bits 16-25 and 80-89. + ushl.4s v4, v4, v1 // v1 = {0, -20, 0, -20}, places third element of triplets into bits 32-41 and 96-105. + and.16b v5, v5, v2 // v2 = {0, 1023, 0, 0, 0, 1023, 0, 0}, isolate second element of triplets. + and.16b v4, v4, v3 // v3 = {1023, 0, 1023, 0, 1023, 0, 1023, 0}, isolate first and third elements of triplets + orr.16b v4, v4, v5 // combine data +#if 0 + str d4, [rcx] // write 3 low 10-bits +#else + cmp rcx, r11 + b.eq 2f + str d4, [rcx] // write 3 low 10-bits + b 1f +2: + str h4, [rcx] // write final 1 low 10-bits +#endif +1: + + /* + set up before going to the main decompress loop + */ + mov next_tag, scratch // tempTagsArray + add r8, scratch, #(1024*scale) // next_qpos + add r11, scratch, #(2048*scale) // tempLowBitsArray + adrp rbx, _hashLookupTable@GOTPAGE + mov tags_counter, #(1024*scale) // tag_area_end + ldr rbx, [rbx, _hashLookupTable@GOTPAGEOFF] + + b L_next + + .align 4,0x90 +L_ZERO_TAG: + /* + we can only get here if w9 = 0, meaning this is a zero tag + *dest_buf++ = 0; + */ + str w9, [dest_buf], #4 // *dest_buf++ = 0 + subs tags_counter, tags_counter, #1 // next_tag vs tag_area_end + b.ls L_done // if next_tag >= tag_area_end, we're done + + /* WKdm decompress main loop */ +L_next: + ldrb w9, [next_tag], #1 // new tag + cbz w9, L_ZERO_TAG + cmp w9, #2 // partial match tag ? + b.eq L_MISS_TAG + b.gt L_EXACT_TAG + +L_PARTIAL_TAG: + /* + this is a partial match: + dict_word = dictionary[*dict_index]; + dictionary[*dict_index++] = *dest_buf++ = dict_word&0xfffffc00 | *LowBits++; + */ + + ldrb edx, [r8], #1 // qpos = *next_qpos++ + ldrh ecx, [r11], #2 // lower 10-bits from *next_low_bits++ + ldr eax, [dictionary, rdx, lsl #2] // read dictionary word + bfm eax, ecx, #0, #9 // pad the lower 10-bits from *next_low_bits + str eax, [dictionary,rdx,lsl #2] // *dict_location = newly formed word + str eax, [dest_buf], #4 // *dest_buf++ = newly formed word + subs tags_counter, tags_counter, #1 // next_tag vs tag_area_end + b.gt L_next // repeat loop until next_tag==tag_area_end + +L_done: + + // release stack memory, restore registers, and return + add sp, sp, #64 // deallocate for dictionary +#if KERNEL + ld1.4s {v0,v1,v2},[sp],#48 + ld1.4s {v3,v4,v5},[sp],#48 +#endif + ret lr + + .align 4,0x90 +L_MISS_TAG: + /* + this is a dictionary miss. + x = *new_word++; + k = (x>>10)&255; + k = hashTable[k]; + dictionary[k] = *dest_buf++ = x; + */ + ldr eax, [r10], #4 // w = *next_full_patt++ + ubfm edx, eax, #10, #17 // 8-bit hash table index + str eax, [dest_buf], #4 // *dest_buf++ = word + ldrb edx, [rbx, rdx] // qpos + str eax, [dictionary,rdx] // dictionary[qpos] = word + subs tags_counter, tags_counter, #1 // next_tag vs tag_area_end + b.gt L_next // repeat the loop + b L_done // if next_tag >= tag_area_end, we're done + + .align 4,0x90 +L_EXACT_TAG: + /* + this is an exact match; + *dest_buf++ = dictionary[*dict_index++]; + */ + ldrb eax, [r8], #1 // qpos = *next_qpos++ + ldr eax, [dictionary,rax,lsl #2] // w = dictionary[qpos] + str eax, [dest_buf], #4 // *dest_buf++ = w + subs tags_counter, tags_counter, #1 // next_tag vs tag_area_end + b.gt L_next // repeat the loop + b L_done // if next_tag >= tag_area_end, we're done + + diff --git a/osfmk/arm64/alternate_debugger.c b/osfmk/arm64/alternate_debugger.c new file mode 100644 index 000000000..0fa80fe36 --- /dev/null +++ b/osfmk/arm64/alternate_debugger.c @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2012 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#if ALTERNATE_DEBUGGER + +/* + +The alternate debugger feature is enabled by setting the boot arg "alternate_debugger_init" +to the size of memory that should be set aside for the debugger. The boot arg +"alternate_debugger_init_pages" is used to allocate more vmpages that the alternate debugger +may use to do additional VA->PA mappings. The boot-arg "alternate_debugger_pause_for_load_at_boot" +will halt the system so that the debugger can be loaded early in the boot cycle -- once the +alternate debugger code is loaded, a register must be set to a 1 to continue the boot process. + +Here's an example: +nvram boot-arg="alternate_debugger_init=0x800000 alternate_debugger_init_pages=0x8000 alternate_debugger_pause_for_load_at_boot=1" + +The low memory global lgAltDebugger will contain the address of the allocated memory for +the alternate debugger. On arm64, the address of this low memory global is 0xffffff8000002048. + +At any point after the low memory global is non-zero, Astris may be used to halt the cpu +and load the alternate debugger: + +If no alternate debugger is given, but alternate_debugger_init has been specified, and the +kernel debugger is entered, the string ">MT<" is printed and normal processing continues. + +Anytime the alternate debugger is entered, the osversion string is modified to start with "ALT" +so that panic reports can clearly indicated that some kernel poking may have occurred, and +the panic should be weighted accordingly. + +*/ + +#include <arm64/alternate_debugger.h> + +#include <kern/kalloc.h> +#include <arm64/lowglobals.h> +#include <arm/caches_internal.h> +#include <kern/cpu_data.h> +#include <arm/pmap.h> +#include <pexpert/pexpert.h> +#include <vm/vm_map.h> +#include <vm/vm_kern.h> +#include <libkern/version.h> + +void kprintf(const char *fmt, ...); + + +static mach_vm_address_t alt_code; +static mach_vm_size_t alt_size; +static mach_vm_address_t alt_pages; +static mach_vm_size_t alt_pages_size; + +typedef void (*t_putc_fn)(char c); +typedef void (*t_call_altdbg_fn)(mach_vm_size_t size, mach_vm_address_t pages, mach_vm_size_t pages_size, t_putc_fn putc_address ); + +// used as a temporary alternate debugger until another is loaded +extern void alternate_debugger_just_return(__unused mach_vm_size_t size, __unused mach_vm_address_t pages, __unused mach_vm_size_t pages_size, t_putc_fn putc_address); +extern void *alternate_debugger_just_return_end; + +// public entry to the alternate debugger +void alternate_debugger_enter(void) +{ + if ( alt_code != 0 ) { + disable_preemption(); + + printf("########## Going to call ALTERNATE DEBUGGER\n"); + + // make sure it isn't in the cache + assert((alt_size & 0xFFFFFFFF00000000) == 0); + flush_dcache(alt_code, (unsigned int)alt_size, 0); + + // set the code to execute + pmap_protect(kernel_map->pmap, alt_code, alt_code+alt_size, VM_PROT_READ|VM_PROT_EXECUTE); + + // black-spot the OS version for any panic reports that occur because of entering the alternate debugger + if ( *osversion ) { + memcpy(osversion, "ALT", 3); // Version set, stomp on the begining of it + } else { + strncpy(osversion, "ALT - Version Not Set Yet", OSVERSIZE); + } + + kprintf("########## Calling ALTERNATE DEBUGGER (size %lld, pages 0x%llx, pages_size 0x%llx, putc %p\n", alt_size, alt_pages, alt_pages_size, &consdebug_putc_unbuffered); + ((t_call_altdbg_fn)alt_code)(alt_size, alt_pages, alt_pages_size, &consdebug_putc_unbuffered); + kprintf("########## Returned from calling ALTERNATE DEBUGGER\n"); + + enable_preemption(); + } +} + +// public entry to check boot args and init accordingly +void alternate_debugger_init(void) +{ + // use the alternate debugger + if( PE_parse_boot_argn("alternate_debugger_init", (void*)&alt_size, sizeof(alt_size)) ) + { + vm_offset_t alt_va = 0; + + kprintf("########## ALTERNATE_DEBUGGER\n"); + + PE_parse_boot_argn("alternate_debugger_init_pages", (void*)&alt_pages_size, sizeof(alt_pages_size)); + + alt_size = vm_map_round_page(alt_size, + VM_MAP_PAGE_MASK(kernel_map)); + alt_pages_size = vm_map_round_page(alt_pages_size, + VM_MAP_PAGE_MASK(kernel_map)); + + kern_return_t kr = KERN_SUCCESS; + kr = kmem_alloc_contig(kernel_map, &alt_va, alt_size, VM_MAP_PAGE_MASK(kernel_map), 0, 0, KMA_NOPAGEWAIT | KMA_KOBJECT | KMA_LOMEM, VM_KERN_MEMORY_DIAG); + if( kr != KERN_SUCCESS) + { + kprintf("########## ALTERNATE_DEBUGGER FAILED kmem_alloc_contig with %d\n", kr); + alt_va = 0; + } + else { + if ( alt_pages_size ) { + alt_pages = (vm_offset_t) kalloc((vm_size_t) alt_pages_size); + } + } + + kprintf("########## Initializing ALTERNATE DEBUGGER : [alloc size 0x%llx @0x%lx] [pages_size 0x%llx @0x%llx] -- lowmem pointer at %p\n", + alt_size, alt_va, alt_pages_size, alt_pages, &lowGlo.lgAltDebugger ); + + if ( alt_va ) { + uintptr_t just_return_size = (uintptr_t)&alternate_debugger_just_return_end - (uintptr_t)&alternate_debugger_just_return; + assert(just_return_size <= alt_size); // alt_size is page-rounded, just_return_size should be much less than a page. + // install a simple return vector + memcpy((void*)alt_va, &alternate_debugger_just_return, just_return_size); + + // code is ready, enable the pointers to it + lowGlo.lgAltDebugger = alt_code = alt_va; + +#if 1 + // DEBUG for BRING-UP testing + unsigned int alt_init_test; + if(PE_parse_boot_argn("alternate_debugger_pause_for_load_at_boot", &alt_init_test, sizeof(alt_init_test)) ) { + + // debug!! + kprintf("########## Waiting for ALTERNATE DEBUGGER to load (in file %s).... to continue, set register to 1", __FILE__ ); + volatile int ii = 0; + while(!ii) + ; + kprintf("\n"); + alternate_debugger_enter(); + } +#endif + } + } +} + +#endif /* ALTERNATE_DEBUGGER */ diff --git a/osfmk/arm64/alternate_debugger.h b/osfmk/arm64/alternate_debugger.h new file mode 100644 index 000000000..22be4c09e --- /dev/null +++ b/osfmk/arm64/alternate_debugger.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2012 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#ifndef _ALTERNATE_DEBUGGER_H_ +#define _ALTERNATE_DEBUGGER_H_ + +#if ALTERNATE_DEBUGGER + +#include <kern/kalloc.h> + +__BEGIN_DECLS + +void alternate_debugger_init(void); +void alternate_debugger_enter(void); + +__END_DECLS + +#endif /* ALTERNATE_DEBUGGER */ + +#endif /* _ALTERNATE_DEBUGGER_H_ */ + diff --git a/osfmk/arm64/alternate_debugger_asm.s b/osfmk/arm64/alternate_debugger_asm.s new file mode 100644 index 000000000..97f381063 --- /dev/null +++ b/osfmk/arm64/alternate_debugger_asm.s @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include <arm64/asm.h> + +#if ALTERNATE_DEBUGGER + .text +/* void alternate_debugger_just_return(__unused mach_vm_size_t size, __unused mach_vm_address_t pages, __unused mach_vm_size_t pages_size, t_putc_fn putc_address) */ + .align 2 + .globl EXT(alternate_debugger_just_return) +LEXT(alternate_debugger_just_return) + sub sp, sp, #0x20 + stp x29, x30, [sp, #0x10] + add x29, sp, #0x10 + str x3, [sp, #0x8] + mov w0, #0xa + mov x1, x3 + blr x1 // (*putc_address)('\n'); + orr w0, wzr, #0x3e + ldr x1, [sp, #0x8] + blr x1 // (*putc_address)('>'); + mov w0, #0x4d + ldr x1, [sp, #0x8] + blr x1 // (*putc_address)('M'); + mov w0, #0x54 + ldr x1, [sp, #0x8] + blr x1 // (*putc_address)('T'); + orr w0, wzr, #0x3c + ldr x1, [sp, #0x8] + blr x1 // (*putc_address)('<'); + mov w0, #0xa + ldr x1, [sp, #0x8] + ldp x29, x30, [sp, #0x10] + add sp, sp, #0x20 + br x1 // (*putc_address)('\n'); + .align 2 + .globl EXT(alternate_debugger_just_return_end) +LEXT(alternate_debugger_just_return_end) + +#endif /* ALTERNATE_DEBUGGER */ diff --git a/osfmk/arm64/arm_vm_init.c b/osfmk/arm64/arm_vm_init.c new file mode 100644 index 000000000..dc0cb7430 --- /dev/null +++ b/osfmk/arm64/arm_vm_init.c @@ -0,0 +1,1203 @@ +/* + * Copyright (c) 2007-2011 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include <mach_debug.h> +#include <mach_kdp.h> +#include <debug.h> + +#include <mach/vm_types.h> +#include <mach/vm_param.h> +#include <kern/misc_protos.h> +#include <kern/assert.h> +#include <vm/vm_kern.h> +#include <vm/vm_page.h> +#include <vm/pmap.h> + +#include <arm64/proc_reg.h> +#include <arm64/lowglobals.h> +#include <arm/cpu_data_internal.h> +#include <arm/misc_protos.h> +#include <pexpert/arm64/boot.h> + +#include <libkern/kernel_mach_header.h> +#include <libkern/section_keywords.h> + +#if KASAN +extern vm_offset_t shadow_pbase; +extern vm_offset_t shadow_ptop; +extern vm_offset_t physmap_vbase; +extern vm_offset_t physmap_vtop; +#endif + +/* + * Denotes the end of xnu. + */ +extern void *last_kernel_symbol; + +/* + * KASLR parameters + */ +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_base; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_top; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kext_base; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kext_top; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_stext; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_etext; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_slide; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_slid_base; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_kernel_slid_top; + +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_stext; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_etext; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_sdata; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_edata; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_sinfo; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_prelink_einfo; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_slinkedit; +SECURITY_READ_ONLY_LATE(vm_offset_t) vm_elinkedit; + +/* Used by <mach/arm/vm_param.h> */ +SECURITY_READ_ONLY_LATE(unsigned long) gVirtBase; +SECURITY_READ_ONLY_LATE(unsigned long) gPhysBase; +SECURITY_READ_ONLY_LATE(unsigned long) gPhysSize; + + +/* + * NOTE: mem_size is bogus on large memory machines. + * We will pin it to 0x80000000 if there is more than 2 GB + * This is left only for compatibility and max_mem should be used. + */ +vm_offset_t mem_size; /* Size of actual physical memory present + * minus any performance buffer and possibly + * limited by mem_limit in bytes */ +uint64_t mem_actual; /* The "One True" physical memory size + * actually, it's the highest physical + * address + 1 */ +uint64_t max_mem; /* Size of physical memory (bytes), adjusted + * by maxmem */ +uint64_t sane_size; /* Memory size to use for defaults + * calculations */ +/* This no longer appears to be used; kill it? */ +addr64_t vm_last_addr = VM_MAX_KERNEL_ADDRESS; /* Highest kernel + * virtual address known + * to the VM system */ + +SECURITY_READ_ONLY_LATE(static vm_offset_t) segTEXTB; +SECURITY_READ_ONLY_LATE(static unsigned long) segSizeTEXT; + + +SECURITY_READ_ONLY_LATE(static vm_offset_t) segDATACONSTB; +SECURITY_READ_ONLY_LATE(static unsigned long) segSizeDATACONST; + +SECURITY_READ_ONLY_LATE(static vm_offset_t) segTEXTEXECB; +SECURITY_READ_ONLY_LATE(static unsigned long) segSizeTEXTEXEC; + +SECURITY_READ_ONLY_LATE(static vm_offset_t) segDATAB; +SECURITY_READ_ONLY_LATE(static unsigned long) segSizeDATA; + + +SECURITY_READ_ONLY_LATE(static vm_offset_t) segLINKB; +SECURITY_READ_ONLY_LATE(static unsigned long) segSizeLINK; + +SECURITY_READ_ONLY_LATE(static vm_offset_t) segKLDB; +SECURITY_READ_ONLY_LATE(static unsigned long) segSizeKLD; +SECURITY_READ_ONLY_LATE(static vm_offset_t) segLASTB; +SECURITY_READ_ONLY_LATE(static unsigned long) segSizeLAST; + +SECURITY_READ_ONLY_LATE(vm_offset_t) segPRELINKTEXTB; +SECURITY_READ_ONLY_LATE(unsigned long) segSizePRELINKTEXT; + +SECURITY_READ_ONLY_LATE(static vm_offset_t) segPLKTEXTEXECB; +SECURITY_READ_ONLY_LATE(static unsigned long) segSizePLKTEXTEXEC; + +SECURITY_READ_ONLY_LATE(static vm_offset_t) segPLKDATACONSTB; +SECURITY_READ_ONLY_LATE(static unsigned long) segSizePLKDATACONST; + +SECURITY_READ_ONLY_LATE(static vm_offset_t) segPRELINKDATAB; +SECURITY_READ_ONLY_LATE(static unsigned long) segSizePRELINKDATA; + +SECURITY_READ_ONLY_LATE(static vm_offset_t) segPLKLLVMCOVB = 0; +SECURITY_READ_ONLY_LATE(static unsigned long) segSizePLKLLVMCOV = 0; + +SECURITY_READ_ONLY_LATE(static vm_offset_t) segPLKLINKEDITB; +SECURITY_READ_ONLY_LATE(static unsigned long) segSizePLKLINKEDIT; + +SECURITY_READ_ONLY_LATE(static vm_offset_t) segPRELINKINFOB; +SECURITY_READ_ONLY_LATE(static unsigned long) segSizePRELINKINFO; + +SECURITY_READ_ONLY_LATE(static boolean_t) use_contiguous_hint = TRUE; + +SECURITY_READ_ONLY_LATE(unsigned) PAGE_SHIFT_CONST; + +SECURITY_READ_ONLY_LATE(vm_offset_t) end_kern; +SECURITY_READ_ONLY_LATE(vm_offset_t) etext; +SECURITY_READ_ONLY_LATE(vm_offset_t) sdata; +SECURITY_READ_ONLY_LATE(vm_offset_t) edata; + +vm_offset_t alloc_ptpage(boolean_t map_static); +SECURITY_READ_ONLY_LATE(vm_offset_t) ropage_next; + +/* + * Bootstrap the system enough to run with virtual memory. + * Map the kernel's code and data, and allocate the system page table. + * Page_size must already be set. + * + * Parameters: + * first_avail: first available physical page - + * after kernel page tables + * avail_start: PA of first physical page + * avail_end: PA of last physical page + */ +SECURITY_READ_ONLY_LATE(vm_offset_t) first_avail; +SECURITY_READ_ONLY_LATE(vm_offset_t) static_memory_end; +SECURITY_READ_ONLY_LATE(pmap_paddr_t) avail_start; +SECURITY_READ_ONLY_LATE(pmap_paddr_t) avail_end; + +#define MEM_SIZE_MAX 0x100000000ULL + +#if defined(KERNEL_INTEGRITY_KTRR) +#if __ARM64_TWO_LEVEL_PMAP__ +/* We could support this configuration, but it adds memory overhead. */ +#error This configuration is not supported +#endif +#endif + +/* + * This rounds the given address up to the nearest boundary for a PTE contiguous + * hint. + */ +static vm_offset_t +round_up_pte_hint_address(vm_offset_t address) +{ + vm_offset_t hint_size = ARM_PTE_SIZE << ARM_PTE_HINT_ENTRIES_SHIFT; + return ((address + (hint_size - 1)) & ~(hint_size - 1)); +} + +/* allocate a page for a page table: we support static and dynamic mappings. + * + * returns a virtual address for the allocated page + * + * for static mappings, we allocate from the region ropagetable_begin to ro_pagetable_end-1, + * which is defined in the DATA_CONST segment and will be protected RNX when vm_prot_finalize runs. + * + * for dynamic mappings, we allocate from avail_start, which should remain RWNX. + */ + +vm_offset_t alloc_ptpage(boolean_t map_static) { + vm_offset_t vaddr; + +#if !(defined(KERNEL_INTEGRITY_KTRR)) + map_static = FALSE; +#endif + + if (!ropage_next) { + ropage_next = (vm_offset_t)&ropagetable_begin; + } + + if (map_static) { + assert(ropage_next < (vm_offset_t)&ropagetable_end); + + vaddr = ropage_next; + ropage_next += ARM_PGBYTES; + + return vaddr; + } else { + vaddr = phystokv(avail_start); + avail_start += ARM_PGBYTES; + + return vaddr; + } +} + +#if DEBUG + +void dump_kva_l2(vm_offset_t tt_base, tt_entry_t *tt, int indent, uint64_t *rosz_out, uint64_t *rwsz_out); + +void dump_kva_l2(vm_offset_t tt_base, tt_entry_t *tt, int indent, uint64_t *rosz_out, uint64_t *rwsz_out) { + unsigned int i; + boolean_t cur_ro, prev_ro = 0; + int start_entry = -1; + tt_entry_t cur, prev = 0; + pmap_paddr_t robegin = kvtophys((vm_offset_t)&ropagetable_begin); + pmap_paddr_t roend = kvtophys((vm_offset_t)&ropagetable_end); + boolean_t tt_static = kvtophys((vm_offset_t)tt) >= robegin && + kvtophys((vm_offset_t)tt) < roend; + + for(i=0; i<TTE_PGENTRIES; i++) { + int tte_type = tt[i] & ARM_TTE_TYPE_MASK; + cur = tt[i] & ARM_TTE_TABLE_MASK; + + if (tt_static) { + /* addresses mapped by this entry are static if it is a block mapping, + * or the table was allocated from the RO page table region */ + cur_ro = (tte_type == ARM_TTE_TYPE_BLOCK) || (cur >= robegin && cur < roend); + } else { + cur_ro = 0; + } + + if ((cur == 0 && prev != 0) || (cur_ro != prev_ro && prev != 0)) { // falling edge + uintptr_t start,end,sz; + + start = (uintptr_t)start_entry << ARM_TT_L2_SHIFT; + start += tt_base; + end = ((uintptr_t)i << ARM_TT_L2_SHIFT) - 1; + end += tt_base; + + sz = end - start + 1; + printf("%*s0x%08x_%08x-0x%08x_%08x %s (%luMB)\n", + indent*4, "", + (uint32_t)(start >> 32),(uint32_t)start, + (uint32_t)(end >> 32),(uint32_t)end, + prev_ro ? "Static " : "Dynamic", + (sz >> 20)); + + if (prev_ro) { + *rosz_out += sz; + } else { + *rwsz_out += sz; + } + } + + if ((prev == 0 && cur != 0) || cur_ro != prev_ro) { // rising edge: set start + start_entry = i; + } + + prev = cur; + prev_ro = cur_ro; + } +} + +void dump_kva_space() { + uint64_t tot_rosz=0, tot_rwsz=0; + int ro_ptpages, rw_ptpages; + pmap_paddr_t robegin = kvtophys((vm_offset_t)&ropagetable_begin); + pmap_paddr_t roend = kvtophys((vm_offset_t)&ropagetable_end); + boolean_t root_static = kvtophys((vm_offset_t)cpu_tte) >= robegin && + kvtophys((vm_offset_t)cpu_tte) < roend; + uint64_t kva_base = ~((1ULL << (64 - T1SZ_BOOT)) - 1); + + printf("Root page table: %s\n", root_static ? "Static" : "Dynamic"); + +#if !__ARM64_TWO_LEVEL_PMAP__ + for(unsigned int i=0; i<TTE_PGENTRIES; i++) { + pmap_paddr_t cur; + boolean_t cur_ro; + uintptr_t start,end; + uint64_t rosz = 0, rwsz = 0; + + if ((cpu_tte[i] & ARM_TTE_VALID) == 0) + continue; + + cur = cpu_tte[i] & ARM_TTE_TABLE_MASK; + start = (uint64_t)i << ARM_TT_L1_SHIFT; + start = start + kva_base; + end = start + (ARM_TT_L1_SIZE - 1); + cur_ro = cur >= robegin && cur < roend; + + printf("0x%08x_%08x-0x%08x_%08x %s\n", + (uint32_t)(start >> 32),(uint32_t)start, + (uint32_t)(end >> 32),(uint32_t)end, + cur_ro ? "Static " : "Dynamic"); + + dump_kva_l2(start, (tt_entry_t*)phystokv(cur), 1, &rosz, &rwsz); + tot_rosz += rosz; + tot_rwsz += rwsz; + } +#else + dump_kva_l2(kva_base, cpu_tte, 0, &tot_rosz, &tot_rwsz); +#endif /* !_ARM64_TWO_LEVEL_PMAP__ */ + + printf("L2 Address space mapped: Static %lluMB Dynamic %lluMB Total %lluMB\n", + tot_rosz >> 20, + tot_rwsz >> 20, + (tot_rosz >> 20) + (tot_rwsz >> 20)); + + ro_ptpages = (int)((ropage_next - (vm_offset_t)&ropagetable_begin) >> ARM_PGSHIFT); + rw_ptpages = (int)(lowGlo.lgStaticSize >> ARM_PGSHIFT); + printf("Pages used: static %d dynamic %d\n", ro_ptpages, rw_ptpages); +} + +#endif /* DEBUG */ + +#if defined(KERNEL_INTEGRITY_KTRR) +extern void bootstrap_instructions; + +/* + * arm_replace_identity_map takes the V=P map that we construct in start.s + * and repurposes it in order to have it map only the page we need in order + * to turn on the MMU. This prevents us from running into issues where + * KTRR will cause us to fault on executable block mappings that cross the + * KTRR boundary. + */ +static void arm_replace_identity_map(boot_args * args) +{ + vm_offset_t addr; + pmap_paddr_t paddr; + +#if !__ARM64_TWO_LEVEL_PMAP__ + pmap_paddr_t l1_ptp_phys = 0; + tt_entry_t *l1_ptp_virt = NULL; + tt_entry_t *tte1 = NULL; +#endif + pmap_paddr_t l2_ptp_phys = 0; + tt_entry_t *l2_ptp_virt = NULL; + tt_entry_t *tte2 = NULL; + pmap_paddr_t l3_ptp_phys = 0; + pt_entry_t *l3_ptp_virt = NULL; + pt_entry_t *ptep = NULL; + + addr = ((vm_offset_t)&bootstrap_instructions) & ~ARM_PGMASK; + paddr = kvtophys(addr); + + /* + * The V=P page tables (at the time this comment was written) start + * after the last bit of kernel data, and consist of 1 to 2 pages. + * Grab references to those pages, and allocate an L3 page. + */ +#if !__ARM64_TWO_LEVEL_PMAP__ + l1_ptp_phys = args->topOfKernelData; + l1_ptp_virt = (tt_entry_t *)phystokv(l1_ptp_phys); + tte1 = &l1_ptp_virt[(((paddr) & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT)]; + + l2_ptp_phys = l1_ptp_phys + ARM_PGBYTES; +#else + l2_ptp_phys = args->topOfKernelData; +#endif + l2_ptp_virt = (tt_entry_t *)phystokv(l2_ptp_phys); + tte2 = &l2_ptp_virt[(((paddr) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT)]; + + l3_ptp_virt = (pt_entry_t *)alloc_ptpage(FALSE); + l3_ptp_phys = kvtophys((vm_offset_t)l3_ptp_virt); + ptep = &l3_ptp_virt[(((paddr) & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT)]; + + /* + * Replace the large V=P mapping with a mapping that provides only the + * mappings needed to turn on the MMU. + */ +#if !__ARM64_TWO_LEVEL_PMAP__ + bzero(l1_ptp_virt, ARM_PGBYTES); + *tte1 = ARM_TTE_BOOT_TABLE | (l2_ptp_phys & ARM_TTE_TABLE_MASK); +#endif + bzero(l2_ptp_virt, ARM_PGBYTES); + *tte2 = ARM_TTE_BOOT_TABLE | (l3_ptp_phys & ARM_TTE_TABLE_MASK); + + *ptep = (paddr & ARM_PTE_MASK) | + ARM_PTE_TYPE_VALID | + ARM_PTE_SH(SH_OUTER_MEMORY) | + ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | + ARM_PTE_AF | + ARM_PTE_AP(AP_RONA) | + ARM_PTE_NX; +} +#endif /* defined(KERNEL_INTEGRITY_KTRR)*/ + +/* + * arm_vm_page_granular_helper updates protections at the L3 level. It will (if + * neccessary) allocate a page for the L3 table and update the corresponding L2 + * entry. Then, it will iterate over the L3 table, updating protections as necessary. + * This expects to be invoked on a L2 entry or sub L2 entry granularity, so this should + * not be invoked from a context that does not do L2 iteration separately (basically, + * don't call this except from arm_vm_page_granular_prot). + */ +static void +arm_vm_page_granular_helper(vm_offset_t start, vm_offset_t _end, vm_offset_t va, + int pte_prot_APX, int pte_prot_XN, int forceCoarse, + pt_entry_t **deferred_pte, pt_entry_t *deferred_ptmp) +{ + if (va & ARM_TT_L2_OFFMASK) { /* ragged edge hanging over a ARM_TT_L2_SIZE boundary */ +#if __ARM64_TWO_LEVEL_PMAP__ + tt_entry_t *tte2; +#else + tt_entry_t *tte1, *tte2; +#endif + tt_entry_t tmplate; + pmap_paddr_t pa; + pt_entry_t *ppte, *recursive_pte = NULL, ptmp, recursive_ptmp = 0; + addr64_t ppte_phys; + unsigned i; + + va &= ~ARM_TT_L2_OFFMASK; + pa = va - gVirtBase + gPhysBase; + +#if __ARM64_TWO_LEVEL_PMAP__ + tte2 = &cpu_tte[(((va) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT)]; +#else + tte1 = &cpu_tte[(((va) & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT)]; + tte2 = &((tt_entry_t*) phystokv((*tte1) & ARM_TTE_TABLE_MASK))[(((va) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT)]; +#endif + + tmplate = *tte2; + + if (ARM_TTE_TYPE_TABLE == (tmplate & ARM_TTE_TYPE_MASK)) { + /* pick up the existing page table. */ + ppte = (pt_entry_t *)phystokv((tmplate & ARM_TTE_TABLE_MASK)); + } else { + // TTE must be reincarnated COARSE. + ppte = (pt_entry_t*)alloc_ptpage(TRUE); + ppte_phys = kvtophys((vm_offset_t)ppte); + + pmap_init_pte_static_page(kernel_pmap, ppte, pa); + + *tte2 = pa_to_tte(ppte_phys) | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID; + } + + /* Apply the desired protections to the specified page range */ + for (i = 0; i <= (ARM_TT_L3_INDEX_MASK>>ARM_TT_L3_SHIFT); i++) { + if ((start <= va) && (va < _end)) { + + ptmp = pa | ARM_PTE_AF | ARM_PTE_SH(SH_OUTER_MEMORY) | ARM_PTE_TYPE; + ptmp = ptmp | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT); + ptmp = ptmp | ARM_PTE_AP(pte_prot_APX); + ptmp = ptmp | ARM_PTE_NX; + + if (pte_prot_XN) { + ptmp = ptmp | ARM_PTE_PNX; + } + + /* + * If we can, apply the contiguous hint to this range. The hint is + * applicable if we are not trying to create per-page mappings and + * if the current address falls within a hint-sized range that will + * be fully covered by this mapping request. + */ + if ((va >= round_up_pte_hint_address(start)) && (round_up_pte_hint_address(va + 1) < _end) && + !forceCoarse && use_contiguous_hint) { + ptmp |= ARM_PTE_HINT; + } + + if ((pt_entry_t*)(phystokv(pa)) == ppte) { + assert(recursive_pte == NULL); + /* This assert should be reenabled as part of rdar://problem/30149465 */ + assert(!forceCoarse); + recursive_pte = &ppte[i]; + recursive_ptmp = ptmp; + } else if ((deferred_pte != NULL) && (&ppte[i] == &recursive_pte[1])) { + assert(*deferred_pte == NULL); + assert(deferred_ptmp != NULL); + *deferred_pte = &ppte[i]; + *deferred_ptmp = ptmp; + } else { + ppte[i] = ptmp; + } + } + + va += ARM_PGBYTES; + pa += ARM_PGBYTES; + } + if (recursive_pte != NULL) + *recursive_pte = recursive_ptmp; + } +} + +/* + * arm_vm_page_granular_prot updates protections by iterating over the L2 entries and + * changing them. If a particular chunk necessitates L3 entries (for reasons of + * alignment or length, or an explicit request that the entry be fully expanded), we + * hand off to arm_vm_page_granular_helper to deal with the L3 chunk of the logic. + * + * Note that counterintuitively a forceCoarse request is a request to expand the entries + * out to L3, i.e. to make *finer* grained mappings. That comes from historical arm32 + * nomenclature in which the 4K granule is "coarse" vs. the 1K "fine" granule (which we + * don't use). + */ +static void +arm_vm_page_granular_prot(vm_offset_t start, unsigned long size, + int tte_prot_XN, int pte_prot_APX, int pte_prot_XN, int forceCoarse) +{ + pt_entry_t *deferred_pte = NULL, deferred_ptmp = 0; + vm_offset_t _end = start + size; + vm_offset_t align_start = (start + ARM_TT_L2_OFFMASK) & ~ARM_TT_L2_OFFMASK; + + if (size == 0x0UL) + return; + + if (align_start > _end) { + arm_vm_page_granular_helper(start, _end, start, pte_prot_APX, pte_prot_XN, forceCoarse, NULL, NULL); + return; + } + + arm_vm_page_granular_helper(start, align_start, start, pte_prot_APX, pte_prot_XN, forceCoarse, &deferred_pte, &deferred_ptmp); + + while ((_end - align_start) >= ARM_TT_L2_SIZE) { + if (forceCoarse) + arm_vm_page_granular_helper(align_start, align_start+ARM_TT_L2_SIZE, align_start + 1, + pte_prot_APX, pte_prot_XN, forceCoarse, NULL, NULL); + else { +#if __ARM64_TWO_LEVEL_PMAP__ + tt_entry_t *tte2; +#else + tt_entry_t *tte1, *tte2; +#endif + tt_entry_t tmplate; + +#if __ARM64_TWO_LEVEL_PMAP__ + tte2 = &cpu_tte[((align_start & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT)]; +#else + tte1 = &cpu_tte[((align_start & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT)]; + tte2 = &((tt_entry_t*) phystokv((*tte1) & ARM_TTE_TABLE_MASK))[((align_start & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT)]; +#endif + + tmplate = *tte2; + + tmplate = (tmplate & ~ARM_TTE_BLOCK_APMASK) | ARM_TTE_BLOCK_AP(pte_prot_APX); + tmplate = tmplate | ARM_TTE_BLOCK_NX; + if (tte_prot_XN) + tmplate = tmplate | ARM_TTE_BLOCK_PNX; + + *tte2 = tmplate; + } + align_start += ARM_TT_L2_SIZE; + } + + if (align_start < _end) + arm_vm_page_granular_helper(align_start, _end, _end, pte_prot_APX, pte_prot_XN, forceCoarse, &deferred_pte, &deferred_ptmp); + + if (deferred_pte != NULL) + *deferred_pte = deferred_ptmp; +} + +static inline void +arm_vm_page_granular_RNX(vm_offset_t start, unsigned long size, int forceCoarse) +{ + arm_vm_page_granular_prot(start, size, 1, AP_RONA, 1, forceCoarse); +} + +static inline void +arm_vm_page_granular_ROX(vm_offset_t start, unsigned long size, int forceCoarse) +{ + arm_vm_page_granular_prot(start, size, 0, AP_RONA, 0, forceCoarse); +} + +static inline void +arm_vm_page_granular_RWNX(vm_offset_t start, unsigned long size, int forceCoarse) +{ + arm_vm_page_granular_prot(start, size, 1, AP_RWNA, 1, forceCoarse); +} + +static inline void +arm_vm_page_granular_RWX(vm_offset_t start, unsigned long size, int forceCoarse) +{ + arm_vm_page_granular_prot(start, size, 0, AP_RWNA, 0, forceCoarse); +} + +void +arm_vm_prot_init(boot_args * args) +{ + /* + * Enforce W^X protections on sections that have been identified so far. This will be + * further refined for each KEXT's TEXT and DATA segments in readPrelinkedExtensions() + */ + bool use_small_page_mappings = FALSE; + + /* + * First off, we'll create mappings for any physical memory preceeding the kernel TEXT. + * This is memory that we want to give to the VM; this will be accomplished through an + * ml_static_mfree call in arm_vm_prot_finalize. This allows the pmap/vm bootstrap + * routines to assume they will have a physically contiguous chunk of memory to deal + * with during bootstrap, while reclaiming this memory later. + */ + arm_vm_page_granular_RWNX(gVirtBase, segPRELINKTEXTB - gVirtBase, use_small_page_mappings); // Memory for the VM + + /* Map coalesced kext TEXT segment RWNX for now */ + arm_vm_page_granular_RWNX(segPRELINKTEXTB, segSizePRELINKTEXT, FALSE); // Refined in OSKext::readPrelinkedExtensions + + /* Map coalesced kext DATA_CONST segment RWNX (could be empty) */ + arm_vm_page_granular_RWNX(segPLKDATACONSTB, segSizePLKDATACONST, FALSE); // Refined in OSKext::readPrelinkedExtensions + + /* Map coalesced kext TEXT_EXEC segment RWX (could be empty) */ + arm_vm_page_granular_ROX(segPLKTEXTEXECB, segSizePLKTEXTEXEC, FALSE); // Refined in OSKext::readPrelinkedExtensions + + /* if new segments not present, set space between PRELINK_TEXT and xnu TEXT to RWNX + * otherwise we no longer expecting any space between the coalesced kext read only segments and xnu rosegments + */ + if (!segSizePLKDATACONST && !segSizePLKTEXTEXEC) { + arm_vm_page_granular_RWNX(segPRELINKTEXTB + segSizePRELINKTEXT, segTEXTB - (segPRELINKTEXTB + segSizePRELINKTEXT), FALSE); + } else { + /* + * If we have the new segments, we should still protect the gap between kext + * read-only pages and kernel read-only pages, in the event that this gap + * exists. + */ + if ((segPLKDATACONSTB + segSizePLKDATACONST) < segTEXTB) { + arm_vm_page_granular_RWNX(segPLKDATACONSTB + segSizePLKDATACONST, segTEXTB - (segPLKDATACONSTB + segSizePLKDATACONST), FALSE); + } + } + + /* + * Protection on kernel text is loose here to allow shenanigans early on. These + * protections are tightened in arm_vm_prot_finalize(). This is necessary because + * we currently patch LowResetVectorBase in cpu.c. + * + * TEXT segment contains mach headers and other non-executable data. This will become RONX later. + */ + arm_vm_page_granular_RNX(segTEXTB, segSizeTEXT, FALSE); + + /* Can DATACONST start out and stay RNX? + * NO, stuff in this segment gets modified during startup (viz. mac_policy_init()/mac_policy_list) + * Make RNX in prot_finalize + */ + arm_vm_page_granular_RWNX(segDATACONSTB, segSizeDATACONST, FALSE); + + /* TEXTEXEC contains read only executable code: becomes ROX in prot_finalize */ + arm_vm_page_granular_RWX(segTEXTEXECB, segSizeTEXTEXEC, FALSE); + + + /* DATA segment will remain RWNX */ + arm_vm_page_granular_RWNX(segDATAB, segSizeDATA, FALSE); + + arm_vm_page_granular_ROX(segKLDB, segSizeKLD, FALSE); + arm_vm_page_granular_RWNX(segLINKB, segSizeLINK, FALSE); + arm_vm_page_granular_ROX(segLASTB, segSizeLAST, FALSE); // __LAST may be empty, but we cannot assume this + + arm_vm_page_granular_RWNX(segPRELINKDATAB, segSizePRELINKDATA, FALSE); // Prelink __DATA for kexts (RW data) + + if (segSizePLKLLVMCOV > 0) + arm_vm_page_granular_RWNX(segPLKLLVMCOVB, segSizePLKLLVMCOV, FALSE); // LLVM code coverage data + + arm_vm_page_granular_RWNX(segPLKLINKEDITB, segSizePLKLINKEDIT, use_small_page_mappings); // Coalesced kext LINKEDIT segment + + arm_vm_page_granular_RWNX(segPRELINKINFOB, segSizePRELINKINFO, FALSE); /* PreLinkInfoDictionary */ + arm_vm_page_granular_RWNX(end_kern, phystokv(args->topOfKernelData) - end_kern, use_small_page_mappings); /* Device Tree, RAM Disk (if present), bootArgs */ + + /* + * This is offset by 4 pages to make room for the boot page tables; we could probably + * include them in the overall mapping, but we'll be paranoid for now. + */ + vm_offset_t extra = 0; +#if KASAN + /* add the KASAN stolen memory to the physmap */ + extra = shadow_ptop - shadow_pbase; + + /* record the extent of the physmap */ + physmap_vbase = phystokv(args->topOfKernelData) + ARM_PGBYTES * 4; + physmap_vtop = static_memory_end; +#endif + arm_vm_page_granular_RNX(phystokv(args->topOfKernelData), ARM_PGBYTES * 4, FALSE); // Boot page tables; they should not be mutable. + arm_vm_page_granular_RWNX(phystokv(args->topOfKernelData) + ARM_PGBYTES * 4, + extra + static_memory_end - ((phystokv(args->topOfKernelData) + ARM_PGBYTES * 4)), use_small_page_mappings); // rest of physmem +} + +void +arm_vm_prot_finalize(boot_args * args) +{ +#pragma unused(args) + /* + * At this point, we are far enough along in the boot process that it will be + * safe to free up all of the memory preceeding the kernel. It may in fact + * be safe to do this earlier. + * + * This keeps the memory in the V-to-P mapping, but advertises it to the VM + * as usable. + */ + + /* + * if old style PRELINK segment exists, free memory before it, and after it before XNU text + * otherwise we're dealing with a new style kernel cache, so we should just free the + * memory before PRELINK_TEXT segment, since the rest of the KEXT read only data segments + * should be immediately followed by XNU's TEXT segment + */ + + ml_static_mfree(gVirtBase, segPRELINKTEXTB - gVirtBase); + + if (!segSizePLKDATACONST && !segSizePLKTEXTEXEC) { + /* If new segments not present, PRELINK_TEXT is not dynamically sized, free DRAM between it and xnu TEXT */ + ml_static_mfree(segPRELINKTEXTB + segSizePRELINKTEXT, segTEXTB - (segPRELINKTEXTB + segSizePRELINKTEXT)); + } + + /* + * LowResetVectorBase patching should be done by now, so tighten executable + * protections. + */ + arm_vm_page_granular_ROX(segTEXTEXECB, segSizeTEXTEXEC, FALSE); + + /* tighten permissions on kext read only data and code */ + if (segSizePLKDATACONST && segSizePLKTEXTEXEC) { + arm_vm_page_granular_RNX(segPRELINKTEXTB, segSizePRELINKTEXT, FALSE); + arm_vm_page_granular_ROX(segPLKTEXTEXECB, segSizePLKTEXTEXEC, FALSE); + arm_vm_page_granular_RNX(segPLKDATACONSTB, segSizePLKDATACONST, FALSE); + } + +#if defined(KERNEL_INTEGRITY_KTRR) + /* + * __LAST,__pinst should no longer be executable. + */ + arm_vm_page_granular_RNX(segLASTB, segSizeLAST, FALSE); + + /* + * Must wait until all other region permissions are set before locking down DATA_CONST + * as the kernel static page tables live in DATA_CONST on KTRR enabled systems + * and will become immutable. + */ +#endif + arm_vm_page_granular_RNX(segDATACONSTB, segSizeDATACONST, FALSE); + +#ifndef __ARM_L1_PTW__ + FlushPoC_Dcache(); +#endif + flush_mmu_tlb(); +} + +#define TBI_USER 0x1 +#define TBI_KERNEL 0x2 + +boolean_t user_tbi = TRUE; + +/* + * TBI (top-byte ignore) is an ARMv8 feature for ignoring the top 8 bits of + * address accesses. It can be enabled separately for TTBR0 (user) and + * TTBR1 (kernel). We enable it by default for user only, but allow both + * to be controlled by the 'tbi' boot-arg. + */ +static void +set_tbi(void) +{ + uint64_t old_tcr, new_tcr; + int tbi = 0; + + if (PE_parse_boot_argn("tbi", &tbi, sizeof(tbi))) + user_tbi = ((tbi & TBI_USER) == TBI_USER); + old_tcr = new_tcr = get_tcr(); + new_tcr |= (user_tbi) ? TCR_TBI0_TOPBYTE_IGNORED : 0; + new_tcr |= (tbi & TBI_KERNEL) ? TCR_TBI1_TOPBYTE_IGNORED : 0; + + if (old_tcr != new_tcr) { + set_tcr(new_tcr); + sysreg_restore.tcr_el1 = new_tcr; + } +} + +void +arm_vm_init(uint64_t memory_size, boot_args * args) +{ +#if !__ARM64_TWO_LEVEL_PMAP__ + vm_map_address_t va_l1, va_l1_end; + pmap_paddr_t pa_l1; + tt_entry_t *cpu_l1_tte; +#else + /* + * If we are using two level page tables, rather than the + * 3 level page tables that xnu defaults to for ARM64, + * then a great deal of the code in this path becomes + * redundant. As a result, most of the logic having to + * do with L1 pages will be excluded from such + * configurations in this function. + */ +#endif + vm_map_address_t va_l2, va_l2_end; + pmap_paddr_t pa_l2; + tt_entry_t *cpu_l2_tte; + pmap_paddr_t boot_ttep; + tt_entry_t *boot_tte; + uint64_t mem_segments; + vm_offset_t ptpage_vaddr; + + + /* + * Get the virtual and physical memory base from boot_args. + */ + gVirtBase = args->virtBase; + gPhysBase = args->physBase; + gPhysSize = args->memSize; + mem_size = args->memSize; + if ((memory_size != 0) && (mem_size > memory_size)) + mem_size = memory_size; + if (mem_size > MEM_SIZE_MAX ) + mem_size = MEM_SIZE_MAX; + static_memory_end = gVirtBase + mem_size; + + boot_ttep = args->topOfKernelData; + boot_tte = (tt_entry_t *) phystokv(boot_ttep); + + /* + * Four pages: + * TTBR0 L1, TTBR0 L2 - 1:1 bootstrap mapping. + * TTBR1 L1, TTBR1 L2 - kernel mapping + */ + avail_start = boot_ttep + 4*ARM_PGBYTES; + +#if defined(KERNEL_INTEGRITY_KTRR) + arm_replace_identity_map(args); +#endif + + /* Initialize invalid tte page */ + invalid_tte = (tt_entry_t *)alloc_ptpage(TRUE); + invalid_ttep = kvtophys((vm_offset_t)invalid_tte); + bzero(invalid_tte, ARM_PGBYTES); + + /* + * Initialize l1 page table page + */ +#if __ARM64_TWO_LEVEL_PMAP__ + /* + * If we're using a two level page table, we still need to + * set the cpu_ttep to avail_start, as this will be the root + * of our page table regardless of how many levels we are + * using. + */ +#endif + cpu_tte = (tt_entry_t *)alloc_ptpage(TRUE); + cpu_ttep = kvtophys((vm_offset_t)cpu_tte); + bzero(cpu_tte, ARM_PGBYTES); + + avail_end = gPhysBase + mem_size; + + /* + * Initialize l1 and l2 page table pages : + * map physical memory at the kernel base virtual address + * cover the kernel dynamic address range section + * + * the so called physical aperture should be statically mapped + */ + +#if !__ARM64_TWO_LEVEL_PMAP__ + pa_l1 = gPhysBase; + va_l1 = gVirtBase; + va_l1_end = gVirtBase + mem_size; +#if KASAN + /* add the KASAN stolen memory to the physmap */ + va_l1_end = gVirtBase + (shadow_ptop - gPhysBase); +#endif + cpu_l1_tte = cpu_tte + ((va_l1 & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT); + + while (va_l1 < va_l1_end) { + tt_entry_t *new_tte = (tt_entry_t *)alloc_ptpage(TRUE); + /* Allocate a page and setup L1 Table TTE in L1 */ + *cpu_l1_tte = (kvtophys((vm_offset_t)new_tte) & ARM_TTE_TABLE_MASK) | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID; + bzero((void *)new_tte, ARM_PGBYTES); + + va_l2 = va_l1; + + if (((va_l1 & ~ARM_TT_L1_OFFMASK)+ARM_TT_L1_SIZE) < va_l1) { + /* If this is the last L1 entry, it must cover the last mapping. */ + va_l2_end = va_l1_end; + } else { + va_l2_end = MIN((va_l1 & ~ARM_TT_L1_OFFMASK)+ARM_TT_L1_SIZE, va_l1_end); + } + + pa_l2 = pa_l1; + cpu_l2_tte = ((tt_entry_t *) phystokv(((*cpu_l1_tte) & ARM_TTE_TABLE_MASK))) + ((va_l1 & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT); +#else + va_l2 = gVirtBase; + va_l2_end = gVirtBase + mem_size; + pa_l2 = gPhysBase; + cpu_l2_tte = cpu_tte + ((va_l2 & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT); + +#if KASAN + /* add the KASAN stolen memory to the physmap */ + va_l2_end = gVirtBase + (shadow_ptop - gPhysBase); +#endif + +#endif + + while (va_l2 < va_l2_end) { + /* Set up L2 Block TTE in L2 */ + *cpu_l2_tte = (pa_l2 & ARM_TTE_BLOCK_L2_MASK) | ARM_TTE_TYPE_BLOCK + | ARM_TTE_VALID | ARM_TTE_BLOCK_AF + | ARM_TTE_BLOCK_AP(AP_RWNA) | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) + | ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK); + va_l2 += ARM_TT_L2_SIZE; + pa_l2 += ARM_TT_L2_SIZE; + cpu_l2_tte++; + } +#if !__ARM64_TWO_LEVEL_PMAP__ + cpu_l1_tte++; + va_l1 = va_l2; + pa_l1 = pa_l2; + } +#endif + + /* + * Now retrieve addresses for end, edata, and etext from MACH-O headers + */ + segPRELINKTEXTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PRELINK_TEXT", &segSizePRELINKTEXT); + segPLKDATACONSTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PLK_DATA_CONST", &segSizePLKDATACONST); + segPLKTEXTEXECB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PLK_TEXT_EXEC", &segSizePLKTEXTEXEC); + segTEXTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__TEXT", &segSizeTEXT); + segDATACONSTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__DATA_CONST", &segSizeDATACONST); + segTEXTEXECB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__TEXT_EXEC", &segSizeTEXTEXEC); + segDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__DATA", &segSizeDATA); + segLINKB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__LINKEDIT", &segSizeLINK); + segKLDB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__KLD", &segSizeKLD); + segPRELINKDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PRELINK_DATA", &segSizePRELINKDATA); + segPRELINKINFOB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PRELINK_INFO", &segSizePRELINKINFO); + segPLKLLVMCOVB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PLK_LLVM_COV", &segSizePLKLLVMCOV); + segPLKLINKEDITB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__PLK_LINKEDIT", &segSizePLKLINKEDIT); + segLASTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header, "__LAST", &segSizeLAST); + + (void) PE_parse_boot_argn("use_contiguous_hint", &use_contiguous_hint, sizeof(use_contiguous_hint)); + assert(segSizePRELINKTEXT < 0x03000000); /* 23355738 */ + + /* if one of the new segments is present, the other one better be as well */ + if (segSizePLKDATACONST || segSizePLKTEXTEXEC) { + assert(segSizePLKDATACONST && segSizePLKTEXTEXEC); + } + + etext = (vm_offset_t) segTEXTB + segSizeTEXT; + sdata = (vm_offset_t) segDATAB; + edata = (vm_offset_t) segDATAB + segSizeDATA; + end_kern = round_page(getlastaddr()); /* Force end to next page */ + + vm_set_page_size(); + + vm_kernel_base = segTEXTB; + vm_kernel_top = (vm_offset_t) &last_kernel_symbol; + vm_kext_base = segPRELINKTEXTB; + vm_kext_top = vm_kext_base + segSizePRELINKTEXT; + + vm_prelink_stext = segPRELINKTEXTB; + if (!segSizePLKTEXTEXEC && !segSizePLKDATACONST) { + vm_prelink_etext = segPRELINKTEXTB + segSizePRELINKTEXT; + } else { + vm_prelink_etext = segPRELINKTEXTB + segSizePRELINKTEXT + segSizePLKDATACONST + segSizePLKTEXTEXEC; + } + vm_prelink_sinfo = segPRELINKINFOB; + vm_prelink_einfo = segPRELINKINFOB + segSizePRELINKINFO; + vm_slinkedit = segLINKB; + vm_elinkedit = segLINKB + segSizeLINK; + + vm_prelink_sdata = segPRELINKDATAB; + vm_prelink_edata = segPRELINKDATAB + segSizePRELINKDATA; + + arm_vm_prot_init(args); + + + /* + * Initialize the page tables for the low globals: + * cover this address range: + * LOW_GLOBAL_BASE_ADDRESS + 2MB + */ +#if __ARM64_TWO_LEVEL_PMAP__ + va_l2 = LOW_GLOBAL_BASE_ADDRESS; + cpu_l2_tte = cpu_tte + ((va_l2 & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT); +#else + va_l1 = va_l2 = LOW_GLOBAL_BASE_ADDRESS; + cpu_l1_tte = cpu_tte + ((va_l1 & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT); + cpu_l2_tte = ((tt_entry_t *) phystokv(((*cpu_l1_tte) & ARM_TTE_TABLE_MASK))) + ((va_l2 & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT); +#endif + ptpage_vaddr = alloc_ptpage(TRUE); + *cpu_l2_tte = (kvtophys(ptpage_vaddr) & ARM_TTE_TABLE_MASK) | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID | ARM_TTE_TABLE_PXN | ARM_TTE_TABLE_XN; + bzero((void *)ptpage_vaddr, ARM_PGBYTES); + + /* + * Initialize l2 page table pages : + * cover this address range: + * KERNEL_DYNAMIC_ADDR - VM_MAX_KERNEL_ADDRESS + */ +#if !__ARM64_TWO_LEVEL_PMAP__ + va_l1 = (gVirtBase+MEM_SIZE_MAX+ ~0xFFFFFFFFFF800000ULL) & 0xFFFFFFFFFF800000ULL; + va_l1_end = VM_MAX_KERNEL_ADDRESS; + cpu_l1_tte = cpu_tte + ((va_l1 & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT); + + while (va_l1 < va_l1_end) { + if (*cpu_l1_tte == ARM_TTE_EMPTY) { + /* Allocate a page and setup L1 Table TTE in L1 */ + ptpage_vaddr = alloc_ptpage(TRUE); + *cpu_l1_tte = (kvtophys(ptpage_vaddr) & ARM_TTE_TABLE_MASK) | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID | ARM_TTE_TABLE_PXN | ARM_TTE_TABLE_XN; + bzero((void *)ptpage_vaddr, ARM_PGBYTES); + } + + if ((va_l1 + ARM_TT_L1_SIZE) < va_l1) { + /* If this is the last L1 entry, it must cover the last mapping. */ + break; + } + + va_l1 += ARM_TT_L1_SIZE; + cpu_l1_tte++; + } +#endif + +#if KASAN + kasan_init(); +#endif + + set_mmu_ttb(invalid_ttep & TTBR_BADDR_MASK); + set_mmu_ttb_alternate(cpu_ttep & TTBR_BADDR_MASK); + set_tbi(); + flush_mmu_tlb(); + + /* + * TODO: We're hardcoding the expected virtual TEXT base here; + * that gives us an ugly dependency on a linker argument in + * the make files. Clean this up, so we don't hardcode it + * twice; this is nothing but trouble. + */ + sane_size = mem_size - (avail_start - gPhysBase); + max_mem = mem_size; + vm_kernel_slid_base = segPRELINKTEXTB; + vm_kernel_slid_top = vm_prelink_einfo; + vm_kernel_slide = segTEXTB-0xfffffff007004000; + vm_kernel_stext = segTEXTB; + assert(segDATACONSTB == segTEXTB + segSizeTEXT); + assert(segTEXTEXECB == segDATACONSTB + segSizeDATACONST); + vm_kernel_etext = segTEXTB + segSizeTEXT + segSizeDATACONST + segSizeTEXTEXEC; + + pmap_bootstrap((gVirtBase+MEM_SIZE_MAX+ ~0xFFFFFFFFFF800000ULL) & 0xFFFFFFFFFF800000ULL); + + /* + * Initialize l3 page table pages : + * cover this address range: + * 2MB + FrameBuffer size + 10MB for each 256MB segment + */ + + mem_segments = (mem_size + 0x0FFFFFFF) >> 28; + +#if !__ARM64_TWO_LEVEL_PMAP__ + va_l1 = (gVirtBase+MEM_SIZE_MAX+ ~0xFFFFFFFFFF800000ULL) & 0xFFFFFFFFFF800000ULL; + va_l1_end = va_l1 + ((2 + (mem_segments * 10)) << 20); + va_l1_end += round_page(args->Video.v_height * args->Video.v_rowBytes); + va_l1_end = (va_l1_end + 0x00000000007FFFFFULL) & 0xFFFFFFFFFF800000ULL; + + cpu_l1_tte = cpu_tte + ((va_l1 & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT); + + while (va_l1 < va_l1_end) { + + va_l2 = va_l1; + + if (((va_l1 & ~ARM_TT_L1_OFFMASK)+ARM_TT_L1_SIZE) < va_l1) { + /* If this is the last L1 entry, it must cover the last mapping. */ + va_l2_end = va_l1_end; + } else { + va_l2_end = MIN((va_l1 & ~ARM_TT_L1_OFFMASK)+ARM_TT_L1_SIZE, va_l1_end); + } + + cpu_l2_tte = ((tt_entry_t *) phystokv(((*cpu_l1_tte) & ARM_TTE_TABLE_MASK))) + ((va_l2 & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT); +#else + va_l2 = (gVirtBase+MEM_SIZE_MAX+ ~0xFFFFFFFFFF800000ULL) & 0xFFFFFFFFFF800000ULL; + va_l2_end = va_l2 + ((2 + (mem_segments * 10)) << 20); + va_l2_end += round_page(args->Video.v_height * args->Video.v_rowBytes); + va_l2_end = (va_l2_end + 0x00000000007FFFFFULL) & 0xFFFFFFFFFF800000ULL; + cpu_l2_tte = cpu_tte + ((va_l2 & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT); +#endif + + while (va_l2 < va_l2_end) { + pt_entry_t * ptp; + pmap_paddr_t ptp_phys; + + /* Allocate a page and setup L3 Table TTE in L2 */ + ptp = (pt_entry_t *) alloc_ptpage(FALSE); + ptp_phys = (pmap_paddr_t)kvtophys((vm_offset_t)ptp); + + pmap_init_pte_page(kernel_pmap, ptp, va_l2, 3, TRUE); + + *cpu_l2_tte = (pa_to_tte (ptp_phys)) | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID | ARM_TTE_TABLE_PXN | ARM_TTE_TABLE_XN; + + va_l2 += ARM_TT_L2_SIZE; + cpu_l2_tte++; + }; +#if !__ARM64_TWO_LEVEL_PMAP__ + va_l1 = va_l2_end; + cpu_l1_tte++; + } +#endif + + /* + * Initialize l3 page table pages : + * cover this address range: + * (VM_MAX_KERNEL_ADDRESS & CPUWINDOWS_BASE_MASK) - VM_MAX_KERNEL_ADDRESS + */ +#if !__ARM64_TWO_LEVEL_PMAP__ + va_l1 = VM_MAX_KERNEL_ADDRESS & CPUWINDOWS_BASE_MASK; + va_l1_end = VM_MAX_KERNEL_ADDRESS; + + cpu_l1_tte = cpu_tte + ((va_l1 & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT); + + while (va_l1 < va_l1_end) { + + va_l2 = va_l1; + + if (((va_l1 & ~ARM_TT_L1_OFFMASK)+ARM_TT_L1_SIZE) < va_l1) { + /* If this is the last L1 entry, it must cover the last mapping. */ + va_l2_end = va_l1_end; + } else { + va_l2_end = MIN((va_l1 & ~ARM_TT_L1_OFFMASK)+ARM_TT_L1_SIZE, va_l1_end); + } + + cpu_l2_tte = ((tt_entry_t *) phystokv(((*cpu_l1_tte) & ARM_TTE_TABLE_MASK))) + ((va_l2 & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT); +#else + va_l2 = VM_MAX_KERNEL_ADDRESS & CPUWINDOWS_BASE_MASK; + va_l2_end = VM_MAX_KERNEL_ADDRESS; + cpu_l2_tte = cpu_tte + ((va_l2 & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT); +#endif + + while (va_l2 < va_l2_end) { + pt_entry_t * ptp; + pmap_paddr_t ptp_phys; + + /* Allocate a page and setup L3 Table TTE in L2 */ + ptp = (pt_entry_t *) alloc_ptpage(FALSE); + ptp_phys = (pmap_paddr_t)kvtophys((vm_offset_t)ptp); + + pmap_init_pte_page(kernel_pmap, ptp, va_l2, 3, TRUE); + + *cpu_l2_tte = (pa_to_tte (ptp_phys)) | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID | ARM_TTE_TABLE_PXN | ARM_TTE_TABLE_XN; + + va_l2 += ARM_TT_L2_SIZE; + cpu_l2_tte++; + }; +#if !__ARM64_TWO_LEVEL_PMAP__ + va_l1 = va_l2_end; + cpu_l1_tte++; + } +#endif + +#if __ARM64_PMAP_SUBPAGE_L1__ && __ARM_16K_PG__ + /* + * In this configuration, the bootstrap mappings (arm_vm_init) and + * the heap mappings occupy separate L1 regions. Explicitly set up + * the heap L1 allocations here. + */ + va_l1 = VM_MIN_KERNEL_ADDRESS & ~ARM_TT_L1_OFFMASK; + cpu_l1_tte = cpu_tte + ((va_l1 & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT); + + while ((va_l1 >= (VM_MIN_KERNEL_ADDRESS & ~ARM_TT_L1_OFFMASK)) && (va_l1 < VM_MAX_KERNEL_ADDRESS)) { + /* + * If the L1 entry has not yet been allocated, allocate it + * now and treat it as a heap table. + */ + if (*cpu_l1_tte == ARM_TTE_EMPTY) { + tt_entry_t *new_tte = (tt_entry_t*)alloc_ptpage(FALSE); + bzero(new_tte, ARM_PGBYTES); + *cpu_l1_tte = (kvtophys((vm_offset_t)new_tte) & ARM_TTE_TABLE_MASK) | ARM_TTE_TYPE_TABLE | ARM_TTE_VALID | ARM_TTE_TABLE_PXN | ARM_TTE_TABLE_XN; + } + + cpu_l1_tte++; + va_l1 += ARM_TT_L1_SIZE; + } +#endif + + /* + * Adjust avail_start so that the range that the VM owns + * starts on a PAGE_SIZE aligned boundary. + */ + avail_start = (avail_start + PAGE_MASK) & ~PAGE_MASK; + + + first_avail = avail_start; + patch_low_glo_static_region(args->topOfKernelData, avail_start - args->topOfKernelData); +} + diff --git a/osfmk/arm64/asm.h b/osfmk/arm64/asm.h new file mode 100644 index 000000000..f756f22ae --- /dev/null +++ b/osfmk/arm64/asm.h @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990,1989 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ + +#ifndef _ARM_ASM_H_ +#define _ARM_ASM_H_ + +#include <arm/arch.h> + +#ifndef __arm64__ +#error Why are we including this? +#endif + +/* There is another definition of ALIGN for .c sources */ +#ifdef __ASSEMBLER__ +#define ALIGN 2 +#endif /* ASSEMBLER */ + +#ifndef FALIGN +#define FALIGN ALIGN +#endif + +#define LB(x,n) n +#if __STDC__ +#ifndef __NO_UNDERSCORES__ +#define LCL(x) L ## x +#define EXT(x) _ ## x +#define LEXT(x) _ ## x ## : +#else +#define LCL(x) .L ## x +#define EXT(x) x +#define LEXT(x) x ## : +#endif +#define LBc(x,n) n ## : +#define LBb(x,n) n ## b +#define LBf(x,n) n ## f +#else /* __STDC__ */ +#ifndef __NO_UNDERSCORES__ +#define LCL(x) L/**/x +#define EXT(x) _/**/x +#define LEXT(x) _/**/x/**/: +#else /* __NO_UNDERSCORES__ */ +#define LCL(x) .L/**/x +#define EXT(x) x +#define LEXT(x) x/**/: +#endif /* __NO_UNDERSCORES__ */ +#define LBc(x,n) n/**/: +#define LBb(x,n) n/**/b +#define LBf(x,n) n/**/f +#endif /* __STDC__ */ + +#define String .asciz +#define Value .word +#define Times(a,b) (a*b) +#define Divide(a,b) (a/b) + +#ifdef __ASSEMBLER__ +#if MACH_KDB +#include <ddb/stab.h> +/* + * This pseudo-assembler line is added so that there will be at least + * one N_SO entry in the symbol stable to define the current file name. + */ +#endif /* MACH_KDB */ + +/* + * Multiline macros must use .macro syntax for now, + * as there is no ARM64 statement separator. + */ +.macro ENTRY + .align FALIGN + .globl _$0 + _$0 : +.endmacro + +.macro ENTRY2 + .align FALIGN + .globl _$0 + .globl _$1 + _$0 : + _$1 : +.endmacro + +.macro READ_THREAD + mrs $0, TPIDR_EL1 +.endmacro + +.macro BRANCH_EXTERN + b _$0 +.endmacro + +.macro CALL_EXTERN + bl _$0 +.endmacro + +.macro MOV64 + movk $0, #((($1) >> 48) & 0x000000000000FFFF), lsl #48 + movk $0, #((($1) >> 32) & 0x000000000000FFFF), lsl #32 + movk $0, #((($1) >> 16) & 0x000000000000FFFF), lsl #16 + movk $0, #((($1) >> 00) & 0x000000000000FFFF), lsl #00 +.endmacro + +#define PUSH_FRAME \ + stp fp, lr, [sp, #-16]! %% \ + mov fp, sp %% + +#define POP_FRAME \ + mov sp, fp %% \ + ldp fp, lr, [sp], #16 %% + +#define EXT(x) _ ## x + +#ifdef XNU_KERNEL_PRIVATE +.macro PANIC_UNIMPLEMENTED + bl _panic_unimplemented +.endmacro +#endif + +#else /* NOT __ASSEMBLER__ */ + +/* These defines are here for .c files that wish to reference global symbols + * within __asm__ statements. + */ +#ifndef __NO_UNDERSCORES__ +#define CC_SYM_PREFIX "_" +#else +#define CC_SYM_PREFIX "" +#endif /* __NO_UNDERSCORES__ */ +#endif /* __ASSEMBLER__ */ + +#ifdef __ASSEMBLER__ + +# define BRANCH_EXTERN(x) b EXT(x) + +#endif /* __ASSEMBLER__ */ + +#endif /* _ARM_ASM_H_ */ diff --git a/osfmk/arm64/bcopy.s b/osfmk/arm64/bcopy.s new file mode 100644 index 000000000..01f33d61e --- /dev/null +++ b/osfmk/arm64/bcopy.s @@ -0,0 +1,296 @@ +/* + * Copyright (c) 2012 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + * + * This file implements the following functions for the arm64 architecture. + * + * void bcopy(const void * source, + * void * destination, + * size_t length); + * + * void *memmove(void * destination, + * const void * source, + * size_t n); + * + * void *memcpy(void * restrict destination, + * const void * restrict source, + * size_t n); + * + * All copy n successive bytes from source to destination. Memmove and memcpy + * return destination, whereas bcopy has no return value. Copying takes place + * as if it were through a temporary buffer -- after return destination + * contains exactly the bytes from source, even if the buffers overlap (this is + * not required of memcpy by the C standard; its behavior is undefined if the + * buffers overlap, but we are holding ourselves to the historical behavior of + * this function on MacOS). + */ + +#include "asm.h" + +.globl _bcopy +.globl _ovbcopy +.globl _memcpy +.globl _memmove + +/***************************************************************************** + * Macros * + *****************************************************************************/ + +#define kSmallCopy 64 + +/***************************************************************************** + * Entrypoints * + *****************************************************************************/ + +.text +.align 5 +_bcopy: +_ovbcopy: +// Translate bcopy into memcpy by swapping the first and second arguments. + mov x3, x0 + mov x0, x1 + mov x1, x3 + +.align 4 +_memcpy: +_memmove: +// Our preference is to copy the data in ascending address order, but if the +// buffers overlap such that the beginning of the destination buffer aliases +// the end of the source buffer, we need to copy in descending address order +// instead to preserve the memmove semantics. We detect this case with the +// test: +// +// destination - source < length (unsigned compare) +// +// If the address of the source buffer is higher than the address of the +// destination buffer, this arithmetic can overflow, but the overflowed value +// can only be smaller than length if the buffers do not overlap, so we don't +// need to worry about false positives due to the overflow (they happen, but +// only in cases where copying in either order is correct). + PUSH_FRAME + sub x3, x0, x1 + cmp x3, x2 + b.cc L_reverse + mov x3, x0 // copy destination pointer + cmp x2, #(kSmallCopy) + b.cc L_forwardSmallCopy + +/***************************************************************************** + * Forward large copy * + *****************************************************************************/ + +// Load the first 32 bytes from src, and compute the number of bytes to the +// first 32-byte aligned location in dst. Even though we are going to copy +// 32 bytes, only those preceeding that 32-byte location "count" towards +// reducing the length of the buffer or advancing the pointers. We will need +// to issue the first load from the advanced src pointer BEFORE the store to +// the unmodified dst pointer. + add x3, x3, #32 + and x3, x3, #-32 // aligned dst + ldp x12,x13,[x1] + ldp x14,x15,[x1, #16] + sub x5, x3, x0 // bytes between original dst and aligned dst + add x1, x1, x5 // update src pointer + +// At this point, data in the following registers is in flight: +// +// x0 original dst pointer +// x1 corresponding location in src buffer. +// x2 length from aligned location in dst to end of buffer. This is +// guaranteed to be >= (64 - 32). +// x3 aligned location in dst buffer. +// x12:x15 first 32 bytes of src buffer. +// +// We now load 32 bytes from x1, and store 32 bytes from x12:x15 to x3. The +// store *may* overlap the first 32 bytes of the load, so in order to get +// correct memmove semantics, the first 32 byte load must occur before the +// store. +// +// After loading these 32 bytes, we advance x1, and decrement the length by +// 64. If the remaining length of the buffer was less than 64, then we jump +// directly to the cleanup path. + ldp x8, x9, [x1] + ldp x10,x11,[x1, #16] + add x1, x1, #32 + sub x2, x2, x5 // update length + stp x12,x13,[x0] // initial unaligned store + stp x14,x15,[x0, #16] // initial unaligned store + subs x2, x2, #64 + b.ls L_forwardCleanup + +L_forwardCopyLoop: +// Main copy loop: +// +// 1. store the 32 bytes loaded in the previous loop iteration +// 2. advance the destination pointer +// 3. load the next 32 bytes +// 4. advance the source pointer +// 5. subtract 32 from the length +// +// The loop is terminated when 32 or fewer bytes remain to be loaded. Those +// trailing 1-32 bytes will be copied in the loop cleanup. + stnp x8, x9, [x3] + stnp x10,x11,[x3, #16] + add x3, x3, #32 + ldnp x8, x9, [x1] + ldnp x10,x11,[x1, #16] + add x1, x1, #32 + subs x2, x2, #32 + b.hi L_forwardCopyLoop + +L_forwardCleanup: +// There are 32 bytes in x8-x11 that were loaded in the previous loop +// iteration, which need to be stored to [x3,x3+32). In addition, between +// 0 and 32 more bytes need to be copied from x1 to x3 + 32. The exact +// number of bytes to copy is x2 + 32. Instead of using smaller conditional +// copies, we simply copy 32 unaligned bytes from x1+x2 to 64+x3+x2. +// This copy may overlap with the first store, so the loads must come before +// the store of the data from the previous loop iteration. + add x1, x1, x2 + ldp x12,x13,[x1] + ldp x14,x15,[x1, #16] + stp x8, x9, [x3] + stp x10,x11,[x3, #16] + add x3, x3, x2 + stp x12,x13,[x3, #32] + stp x14,x15,[x3, #48] + POP_FRAME + ret + +/***************************************************************************** + * forward small copy * + *****************************************************************************/ + +// Copy one quadword at a time until less than 8 bytes remain to be copied. +// At the point of entry to L_forwardSmallCopy, the "calling convention" +// is as follows: +// +// x0 pointer to first byte of destination +// x1 pointer to first byte of source +// x2 length of buffers +// x3 pointer to first byte of destination +0: ldr x6, [x1],#8 + str x6, [x3],#8 +L_forwardSmallCopy: + subs x2, x2, #8 + b.cs 0b + adds x2, x2, #8 + b.eq 2f +1: ldrb w6, [x1],#1 + strb w6, [x3],#1 + subs x2, x2, #1 + b.ne 1b +2: POP_FRAME + ret + +/***************************************************************************** + * Reverse copy engines * + *****************************************************************************/ + +// The reverse copy engines are identical in every way to the forward copy +// engines, except in that they do everything backwards. For this reason, they +// are somewhat more sparsely commented than the forward copy loops. I have +// tried to only comment things that might be somewhat surprising in how they +// differ from the forward implementation. +// +// The one important thing to note is that (almost without fail), x1 and x3 +// will point to ONE BYTE BEYOND the "right-hand edge" of the active buffer +// throughout these copy loops. They are initially advanced to that position +// in the L_reverse jump island. Because of this, whereas the forward copy +// loops generally follow a "copy data, then advance pointers" scheme, in the +// reverse copy loops, we advance the pointers, then copy the data. + +L_reverse: +// As a minor optimization, we early out if dst == src. + cbz x3, L_return +// advance both pointers to the ends of their respective buffers before +// jumping into the appropriate reverse copy loop. + add x4, x0, x2 + add x1, x1, x2 + cmp x2, #(kSmallCopy) + b.cc L_reverseSmallCopy + +/***************************************************************************** + * Reverse large copy * + *****************************************************************************/ + + ldp x12,x13,[x1, #-16] + ldp x14,x15,[x1, #-32] + sub x3, x4, #1 // In the forward copy, we used dst+32 & -32 + and x3, x3, #-32 // to find an aligned location in the dest + sub x5, x4, x3 // buffer. Here we use dst-1 & -32 instead, + sub x1, x1, x5 // because we are going backwards. + sub x2, x2, x5 + ldp x8, x9, [x1, #-16] + ldp x10,x11,[x1, #-32] + stp x12,x13,[x4, #-16] + stp x14,x15,[x4, #-32] + sub x1, x1, #32 + subs x2, x2, #64 + b.ls L_reverseCleanup + +L_reverseCopyLoop: + stnp x8, x9, [x3, #-16] + stnp x10,x11,[x3, #-32] + sub x3, x3, #32 + ldnp x8, x9, [x1, #-16] + ldnp x10,x11,[x1, #-32] + sub x1, x1, #32 + subs x2, x2, #32 + b.hi L_reverseCopyLoop + +L_reverseCleanup: + sub x1, x1, x2 + ldp x12,x13,[x1, #-16] + ldp x14,x15,[x1, #-32] + stp x8, x9, [x3, #-16] + stp x10,x11,[x3, #-32] + stp x12,x13,[x0, #16] // In the forward copy, we need to compute the + stp x14,x15,[x0] // address of these stores, but here we already + POP_FRAME // have a pointer to the start of the buffer. + ret + +/***************************************************************************** + * reverse small copy * + *****************************************************************************/ + +0: ldr x6, [x1,#-8]! + str x6, [x4,#-8]! +L_reverseSmallCopy: + subs x2, x2, #8 + b.cs 0b + adds x2, x2, #8 + b.eq 2f +1: ldrb w6, [x1,#-1]! + strb w6, [x4,#-1]! + subs x2, x2, #1 + b.ne 1b +2: POP_FRAME + ret + +L_return: + POP_FRAME + ret diff --git a/osfmk/arm64/bsd_arm64.c b/osfmk/arm64/bsd_arm64.c new file mode 100644 index 000000000..726b12bd3 --- /dev/null +++ b/osfmk/arm64/bsd_arm64.c @@ -0,0 +1,227 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifdef MACH_BSD +#include <mach_debug.h> +#include <mach_ldebug.h> + +#include <mach/kern_return.h> +#include <mach/mach_traps.h> +#include <mach/vm_param.h> + +#include <kern/counters.h> +#include <kern/cpu_data.h> +#include <arm/cpu_data_internal.h> +#include <kern/mach_param.h> +#include <kern/task.h> +#include <kern/thread.h> +#include <kern/sched_prim.h> +#include <kern/misc_protos.h> +#include <kern/assert.h> +#include <kern/spl.h> +#include <kern/syscall_sw.h> +#include <ipc/ipc_port.h> +#include <vm/vm_kern.h> +#include <mach/thread_status.h> +#include <vm/pmap.h> + +#include <sys/kdebug.h> + +#include <sys/syscall.h> + +extern void throttle_lowpri_io(int); +void mach_syscall(struct arm_saved_state*); +typedef kern_return_t (*mach_call_t)(void *); + +struct mach_call_args { + syscall_arg_t arg1; + syscall_arg_t arg2; + syscall_arg_t arg3; + syscall_arg_t arg4; + syscall_arg_t arg5; + syscall_arg_t arg6; + syscall_arg_t arg7; + syscall_arg_t arg8; + syscall_arg_t arg9; +}; + +static void +arm_set_mach_syscall_ret(struct arm_saved_state *state, int retval) +{ + if (is_saved_state32(state)) { + saved_state32(state)->r[0] = retval; + } else { + saved_state64(state)->x[0] = retval; + } +} + +static kern_return_t +arm_get_mach_syscall_args(struct arm_saved_state *state, struct mach_call_args *dest, const mach_trap_t *trapp) +{ + uint32_t reg_count; + + if (is_saved_state32(state)) { + /* The trap table entry defines the number of 32-bit words to be copied in from userspace. */ + reg_count = trapp->mach_trap_u32_words; + + /* + * We get 7 contiguous words; r0-r6, hop over r7 + * (frame pointer), optionally r8 + */ + if (reg_count <= 7) { + bcopy((char*)saved_state32(state), (char*)dest, sizeof(uint32_t) * reg_count); + } else if (reg_count <= 9) { + bcopy((char*)saved_state32(state), (char*)dest, sizeof(uint32_t) * 7); + bcopy((char*)&saved_state32(state)->r[8], ((char*)dest) + sizeof(uint32_t) * 7, + reg_count - 7); + } else { + panic("Trap with %d words of args? We only support 9.", reg_count); + } + +#if CONFIG_REQUIRES_U32_MUNGING + trapp->mach_trap_arg_munge32(dest); +#else +#error U32 mach traps on ARM64 kernel requires munging +#endif + } else { + assert(is_saved_state64(state)); + bcopy((char*)saved_state64(state), (char*)dest, trapp->mach_trap_arg_count * sizeof(uint64_t)); + } + + return KERN_SUCCESS; +} + +kern_return_t +thread_setsinglestep(__unused thread_t thread, __unused int on) +{ + return (KERN_FAILURE); /* XXX TODO */ +} + +#if CONFIG_DTRACE + +vm_offset_t dtrace_get_cpu_int_stack_top(void); + +vm_offset_t +dtrace_get_cpu_int_stack_top(void) +{ + return getCpuDatap()->intstack_top; +} +#endif /* CONFIG_DTRACE */ +extern const char *mach_syscall_name_table[]; + +/* ARM64_TODO: remove this. still TODO?*/ +extern struct proc* current_proc(void); +extern int proc_pid(struct proc*); + +void +mach_syscall(struct arm_saved_state *state) +{ + kern_return_t retval; + mach_call_t mach_call; + struct mach_call_args args = { 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + int call_number = get_saved_state_svc_number(state); + int64_t exc_code; + int argc; + + struct uthread *ut = get_bsdthread_info(current_thread()); + uthread_reset_proc_refcount(ut); + + assert(call_number < 0); /* Otherwise it would be a Unix syscall */ + call_number = -call_number; + + if (call_number >= MACH_TRAP_TABLE_COUNT) { + goto bad; + } + + DEBUG_KPRINT_SYSCALL_MACH( + "mach_syscall: code=%d(%s) (pid %d, tid %lld)\n", + call_number, mach_syscall_name_table[call_number], + proc_pid(current_proc()), thread_tid(current_thread())); + +#if DEBUG_TRACE + kprintf("mach_syscall(0x%08x) code=%d\n", state, call_number); +#endif + + mach_call = (mach_call_t)mach_trap_table[call_number].mach_trap_function; + + if (mach_call == (mach_call_t)kern_invalid) { + DEBUG_KPRINT_SYSCALL_MACH( + "mach_syscall: kern_invalid 0x%x\n", call_number); + goto bad; + } + + argc = mach_trap_table[call_number].mach_trap_arg_count; + if (argc) { + retval = arm_get_mach_syscall_args(state, &args, &mach_trap_table[call_number]); + if (retval != KERN_SUCCESS) { + arm_set_mach_syscall_ret(state, retval); + + DEBUG_KPRINT_SYSCALL_MACH( + "mach_syscall: retval=0x%x\n", retval); + return; + } + } + + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + MACHDBG_CODE(DBG_MACH_EXCP_SC, (call_number)) | DBG_FUNC_START, + args.arg1, args.arg2, args.arg3, args.arg4, 0); + + retval = mach_call(&args); + + DEBUG_KPRINT_SYSCALL_MACH("mach_syscall: retval=0x%x (pid %d, tid %lld)\n", retval, + proc_pid(current_proc()), thread_tid(current_thread())); + + KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, + MACHDBG_CODE(DBG_MACH_EXCP_SC,(call_number)) | DBG_FUNC_END, + retval, 0, 0, 0, 0); + + arm_set_mach_syscall_ret(state, retval); + + throttle_lowpri_io(1); + +#if DEBUG || DEVELOPMENT + kern_allocation_name_t + prior __assert_only = thread_get_kernel_state(current_thread())->allocation_name; + assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior)); +#endif /* DEBUG || DEVELOPMENT */ + +#if PROC_REF_DEBUG + if (__improbable(uthread_get_proc_refcount(ut) != 0)) { + panic("system call returned with uu_proc_refcount != 0"); + } +#endif + + return; + +bad: + exc_code = call_number; + exception_triage(EXC_SYSCALL, &exc_code, 1); + /* NOTREACHED */ + panic("Returned from exception_triage()?\n"); +} +#endif /* MACH_BSD */ diff --git a/osfmk/arm64/bzero.s b/osfmk/arm64/bzero.s new file mode 100644 index 000000000..c2f084e47 --- /dev/null +++ b/osfmk/arm64/bzero.s @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2012 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + * + * This file implements the following functions for the arm64 architecture: + * + * void bzero(void *buffer, size_t length); + * void __bzero(void *buffer, size_t length); + * void *memset(void *buffer, int value, size_t length); + * + * The first two zero-fill a buffer. The third fills the buffer with the low + * byte of its second argument. + */ + +#include "asm.h" + +.globl _bzero +.globl ___bzero +.globl _memset +.globl _secure_memset + +/***************************************************************************** + * bzero entrypoint * + *****************************************************************************/ + +.text +.align 4 +_bzero: +___bzero: + PUSH_FRAME + mov x2, x1 + eor x1, x1, x1 + mov x3, x0 + cmp x2, #128 + b.cc L_memsetSmall + +/***************************************************************************** + * Large buffer zero engine * + *****************************************************************************/ + +L_bzeroLarge: +// Write the first 64 bytes of the buffer without regard to alignment, then +// advance x3 to point to a cacheline-aligned location within the buffer, and +// decrement the length accordingly. + stp x1, x1, [x0] + stp x1, x1, [x0, #16] + stp x1, x1, [x0, #32] + stp x1, x1, [x0, #48] + add x3, x0, #64 + and x3, x3, #-64 + add x2, x2, x0 // end of buffer + add x4, x3, #64 // end of first cacheline to zero + subs x2, x2, x4 // if the end of the buffer comes first, jump + b.ls 1f // directly to the cleanup pass. +0: dc zva, x3 // zero cacheline + add x3, x3, #64 // increment pointer + subs x2, x2, #64 // decrement length + b.hi 0b +1: add x3, x3, x2 // back up pointer to (end of buffer) - 64. + stp x1, x1, [x3] // and store 64 bytes to reach end of buffer. + stp x1, x1, [x3, #16] + stp x1, x1, [x3, #32] + stp x1, x1, [x3, #48] + POP_FRAME + ret + +/***************************************************************************** + * memset entrypoint * + *****************************************************************************/ + +.align 4 +/* + * It is important that secure_memset remains defined in assembly to avoid + * compiler optimizations. + */ +_secure_memset: +_memset: + PUSH_FRAME + and x1, x1, #0xff + orr x3, xzr,#0x0101010101010101 + mul x1, x1, x3 + mov x3, x0 + cmp x2, #64 + b.cc L_memsetSmall + +/***************************************************************************** + * Large buffer store engine * + *****************************************************************************/ + +L_memsetLarge: +// Write the first 64 bytes of the buffer without regard to alignment, then +// advance x3 to point to an aligned location within the buffer, and +// decrement the length accordingly. + stp x1, x1, [x0] + add x3, x0, #16 + and x3, x3, #-16 + add x2, x2, x0 // end of buffer + add x4, x3, #64 // end of first aligned 64-byte store + subs x2, x2, x4 // if the end of the buffer comes first, jump + b.ls 1f // directly to the cleanup store. +0: stnp x1, x1, [x3] + stnp x1, x1, [x3, #16] + stnp x1, x1, [x3, #32] + stnp x1, x1, [x3, #48] + add x3, x3, #64 + subs x2, x2, #64 + b.hi 0b +1: add x3, x3, x2 // back up pointer to (end of buffer) - 64. + stp x1, x1, [x3] + stp x1, x1, [x3, #16] + stp x1, x1, [x3, #32] + stp x1, x1, [x3, #48] + POP_FRAME + ret + +/***************************************************************************** + * Small buffer store engine * + *****************************************************************************/ + +0: str x1, [x3],#8 +L_memsetSmall: + subs x2, x2, #8 + b.cs 0b + adds x2, x2, #8 + b.eq 2f +1: strb w1, [x3],#1 + subs x2, x2, #1 + b.ne 1b +2: POP_FRAME + ret diff --git a/osfmk/arm64/caches_asm.s b/osfmk/arm64/caches_asm.s new file mode 100644 index 000000000..463d39067 --- /dev/null +++ b/osfmk/arm64/caches_asm.s @@ -0,0 +1,369 @@ +/* + * Copyright (c) 2010-2013 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include <machine/asm.h> +#include <arm64/proc_reg.h> +#include <arm/pmap.h> +#include <sys/errno.h> +#include "assym.s" + +/* + * void InvalidatePoU_Icache(void) + * + * Invalidate i-cache + */ + .text + .align 2 + .globl EXT(InvalidatePoU_Icache) + .globl EXT(invalidate_mmu_icache) +LEXT(InvalidatePoU_Icache) +LEXT(invalidate_mmu_icache) + ic ialluis // Invalidate icache + dsb sy + isb sy + ret + +/* + * void InvalidatePoU_IcacheRegion(vm_offset_t va, unsigned length) + * + * Invalidate icache region + */ + .text + .align 2 + .globl EXT(InvalidatePoU_IcacheRegion) +LEXT(InvalidatePoU_IcacheRegion) + mov x9, #((1<<MMU_I_CLINE)-1) + and x2, x0, x9 + bic x0, x0, x9 // Cached aligned + add x1, x1, x2 + sub x1, x1, #1 + lsr x1, x1, #MMU_I_CLINE // Set cache line counter +L_ipui_loop: + ic ivau, x0 // Invalidate icache line + add x0, x0, #1<<MMU_I_CLINE // Get next cache aligned addr + subs x1, x1, #1 // Decrementer cache line counter + b.pl L_ipui_loop // Loop in counter not null + dsb sy + isb sy + ret + + +/* + * void CleanPoC_Dcache(void) + * + * Clean all d-caches + */ + .text + .align 2 + .globl EXT(CleanPoC_Dcache) + .globl EXT(clean_mmu_dcache) +LEXT(CleanPoC_Dcache) +#if defined(APPLE_ARM64_ARCH_FAMILY) + /* "Fully Coherent." */ +#else /* !defined(APPLE_ARM64_ARCH_FAMILY) */ + mov x0, #0 + mov x9, #(1 << MMU_I7SET) + mov x10, #(1 << (MMU_NSET + MMU_I7SET)) + mov x11, #(1 << MMU_I7WAY) +L_cpcd_dcacheway: +L_cpcd_dcacheline: + dc csw, x0 // clean dcache line by way/set + add x0, x0, x9 // increment set index + tst x0, #(1 << (MMU_NSET + MMU_I7SET)) // look for overflow + b.eq L_cpcd_dcacheline + bic x0, x0, x10 // clear set overflow + adds x0, x0, x11 // increment way + b.cc L_cpcd_dcacheway // loop +#if __ARM_L2CACHE__ + mov x0, #2 + mov x9, #(1 << L2_I7SET) + mov x10, #(1 << (L2_NSET + L2_I7SET)) + mov x11, #(1 << L2_I7WAY) +L_cpcd_l2dcacheway: +L_cpcd_l2dcacheline: + dc csw, x0 // clean dcache line by way/set + add x0, x0, x9 // increment set index + tst x0, #(1 << (L2_NSET + L2_I7SET)) // look for overflow + b.eq L_cpcd_l2dcacheline + bic x0, x0, x10 // clear set overflow + adds x0, x0, x11 // increment way + b.cc L_cpcd_l2dcacheway // loop +#endif +#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */ + dsb sy + ret + +/* + * void CleanPoU_Dcache(void) + * + * Clean D-cache to Point of Unification + */ + .text + .align 2 + .globl EXT(CleanPoU_Dcache) +LEXT(CleanPoU_Dcache) +#if defined(APPLE_ARM64_ARCH_FAMILY) + /* "Fully Coherent." */ +#else /* !defined(APPLE_ARM64_ARCH_FAMILY) */ +#error CleanPoU_Dcache needs an implementation +#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */ + dsb sy + ret + +/* + * void CleanPoU_DcacheRegion(vm_offset_t va, unsigned length) + * + * Clean d-cache region to Point of Unification + */ + .text + .align 2 + .globl EXT(CleanPoU_DcacheRegion) +LEXT(CleanPoU_DcacheRegion) +#if defined(APPLE_ARM64_ARCH_FAMILY) + /* "Fully Coherent." */ +#else /* !defined(APPLE_ARM64_ARCH_FAMILY) */ + mov x9, #((1<<MMU_CLINE)-1) + and x2, x0, x9 + bic x0, x0, x9 // Cached aligned + add x1, x1, x2 + sub x1, x1, #1 + lsr x1, x1, #MMU_CLINE // Set cache line counter +L_cpudr_loop: + dc cvau, x0 // Clean dcache line to PoU + add x0, x0, #(1<<MMU_CLINE) // Get next cache aligned addr + subs x1, x1, #1 // Decrementer cache line counter + b.pl L_cpudr_loop // Loop in counter not null +#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */ + dsb sy + ret + +/* + * void CleanPoC_DcacheRegion_internal(vm_offset_t va, unsigned length) + * + * Clean d-cache region to Point of Coherency + */ + .text + .align 2 +LEXT(CleanPoC_DcacheRegion_internal) + PUSH_FRAME + mov x9, #((1<<MMU_CLINE)-1) + and x2, x0, x9 + bic x0, x0, x9 // Cached aligned + add x1, x1, x2 + sub x1, x1, #1 + lsr x1, x1, #MMU_CLINE // Set cache line counter + dsb sy +L_cpcdr_loop: +#if defined(APPLE_ARM64_ARCH_FAMILY) + // It may be tempting to clean the cache (dc cvac), + // but see Cyclone UM 5.3.8.3 -- it's always a NOP on Cyclone. + // + // Clean & Invalidate, however, will work as long as HID4.DisDCMvaOps isn't set. + dc civac, x0 // Clean & Invalidate dcache line to PoC +#else + dc cvac, x0 // Clean dcache line to PoC +#endif + add x0, x0, #(1<<MMU_CLINE) // Get next cache aligned addr + subs x1, x1, #1 // Decrementer cache line counter + b.pl L_cpcdr_loop // Loop in counter not null + dsb sy + POP_FRAME + ret + +/* + * void CleanPoC_DcacheRegion(vm_offset_t va, unsigned length) + * + * Clean d-cache region to Point of Coherency + */ + .text + .align 2 + .globl EXT(CleanPoC_DcacheRegion) +LEXT(CleanPoC_DcacheRegion) +#if defined(APPLE_ARM64_ARCH_FAMILY) + /* "Fully Coherent." */ + dsb sy + ret +#else /* !defined(APPLE_ARM64_ARCH_FAMILY) */ + b EXT(CleanPoC_DcacheRegion_internal) +#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */ + +/* + * void CleanPoC_DcacheRegion_Force(vm_offset_t va, unsigned length) + * + * Clean d-cache region to Point of Coherency - when you really + * need to flush even on coherent platforms, e.g. panic log + */ +.text + .align 2 + .globl EXT(CleanPoC_DcacheRegion_Force) +LEXT(CleanPoC_DcacheRegion_Force) + b EXT(CleanPoC_DcacheRegion_internal) + +/* + * void FlushPoC_Dcache(void) + * + * Clean and Invalidate dcaches to Point of Coherency + */ + .text + .align 2 + .globl EXT(FlushPoC_Dcache) +LEXT(FlushPoC_Dcache) +#if defined(APPLE_ARM64_ARCH_FAMILY) + /* "Fully Coherent." */ +#else /* !defined(APPLE_ARM64_ARCH_FAMILY) */ + mov x0, #0 + mov x9, #(1 << MMU_I7SET) + mov x10, #(1 << (MMU_NSET + MMU_I7SET)) + mov x11, #(1 << MMU_I7WAY) +L_fpcd_dcacheway: +L_fpcd_dcacheline: + dc cisw, x0 // clean invalidate dcache line by way/set + add x0, x0, x9 // increment set index + tst x0, #(1 << (MMU_NSET + MMU_I7SET)) // look for overflow + b.eq L_fpcd_dcacheline + bic x0, x0, x10 // clear set overflow + adds x0, x0, x11 // increment way + b.cc L_fpcd_dcacheway // loop +#if __ARM_L2CACHE__ + mov x0, #2 + mov x9, #(1 << L2_I7SET) + mov x10, #(1 << (L2_NSET + L2_I7SET)) + mov x11, #(1 << L2_I7WAY) +L_fpcd_l2dcacheway: +L_fpcd_l2dcacheline: + dc cisw, x0 // clean invalide dcache line by way/set + add x0, x0, x9 // increment set index + tst x0, #(1 << (L2_NSET + L2_I7SET)) // look for overflow + b.eq L_fpcd_l2dcacheline + bic x0, x0, x10 // clear set overflow + adds x0, x0, x11 // increment way + b.cc L_fpcd_l2dcacheway // loop +#endif +#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */ + dsb sy + ret + +/* + * void FlushPoU_Dcache(void) + * + * Flush D-cache to Point of Unification + */ + .text + .align 2 + .globl EXT(FlushPoU_Dcache) +LEXT(FlushPoU_Dcache) +#if defined(APPLE_ARM64_ARCH_FAMILY) + /* "Fully Coherent." */ +#else /* !defined(APPLE_ARM64_ARCH_FAMILY) */ + mov x0, #0 + mov x9, #(1 << MMU_I7SET) + mov x10, #(1 << (MMU_NSET + MMU_I7SET)) + mov x11, #(1 << MMU_I7WAY) +L_fpud_way: +L_fpud_line: + dc cisw, x0 // clean invalidate dcache line by way/set + add x0, x0, x9 // increment set index + tst x0, #1 << (MMU_NSET + MMU_I7SET) // look for overflow + b.eq L_fpud_line + bic x0, x0, x10 // clear set overflow + adds x0, x0, x11 // increment way + b.cc L_fpud_way // loop +#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */ + dsb sy + ret + +/* + * void FlushPoC_DcacheRegion(vm_offset_t va, unsigned length) + * + * Clean and Invalidate d-cache region to Point of Coherency + */ + .text + .align 2 + .globl EXT(FlushPoC_DcacheRegion) +LEXT(FlushPoC_DcacheRegion) +#if defined(APPLE_ARM64_ARCH_FAMILY) + /* "Fully Coherent." */ +#else /* !defined(APPLE_ARM64_ARCH_FAMILY) */ + mov x9, #((1<<MMU_CLINE)-1) + and x2, x0, x9 + bic x0, x0, x9 // Cached aligned + add x1, x1, x2 + sub x1, x1, #1 + lsr x1, x1, #MMU_CLINE // Set cache line counter +L_fpcdr_loop: + dc civac, x0 // Clean invalidate dcache line to PoC + add x0, x0, #(1<<MMU_CLINE) // Get next cache aligned addr + subs x1, x1, #1 // Decrementer cache line counter + b.pl L_fpcdr_loop // Loop in counter not null +#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */ + dsb sy + ret + +/* + * void flush_dcache64(addr64_t addr, unsigned length, boolean_t phys) + */ + .text + .align 2 + .globl EXT(flush_dcache64) +LEXT(flush_dcache64) + BRANCH_EXTERN flush_dcache + +/* + * void clean_dcache64(addr64_t addr, unsigned length, boolean_t phys) + */ + .text + .align 2 + .globl EXT(clean_dcache64) +LEXT(clean_dcache64) + BRANCH_EXTERN clean_dcache + +/* + * void invalidate_icache(vm_offset_t va, unsigned length, boolean_t phys) + * void invalidate_icache64(addr64_t va, unsigned length, boolean_t phys) + */ + .text + .align 2 + .globl EXT(invalidate_icache64) + .globl EXT(invalidate_icache) +LEXT(invalidate_icache64) +LEXT(invalidate_icache) + cmp w2, #0 // Is it physical? + b.eq Lcall_invalidate_worker + adrp x2, _gPhysBase@page + add x2, x2, _gPhysBase@pageoff + ldr x2, [x2] + sub x0, x0, x2 + adrp x2, _gVirtBase@page + add x2, x2, _gVirtBase@pageoff + ldr x2, [x2] + add x0, x0, x2 +Lcall_invalidate_worker: + b EXT(InvalidatePoU_IcacheRegion) + + +/* vim: set ts=4: */ diff --git a/osfmk/arm64/copyio.c b/osfmk/arm64/copyio.c new file mode 100644 index 000000000..7d7974d5d --- /dev/null +++ b/osfmk/arm64/copyio.c @@ -0,0 +1,311 @@ +/* + * Copyright (c) 2012-2013 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include <arm/cpu_data_internal.h> +#include <arm/misc_protos.h> +#include <kern/thread.h> +#include <sys/errno.h> +#include <vm/pmap.h> +#include <vm/vm_map.h> +#include <san/kasan.h> + +extern int _bcopyin(const char *src, char *dst, vm_size_t len); +extern int _bcopyinstr(const char *src, char *dst, vm_size_t max, vm_size_t *actual); +extern int _bcopyout(const char *src, char *dst, vm_size_t len); +extern int _copyin_word(const char *src, uint64_t *dst, vm_size_t len); + +extern pmap_t kernel_pmap; +extern boolean_t arm_pan_enabled; + +typedef enum copyio_type { + COPYIO_IN, + COPYIO_IN_WORD, + COPYIO_INSTR, + COPYIO_OUT, +} copyio_type_t; + +int +copyio_check_user_addr(user_addr_t user_addr, vm_size_t nbytes) +{ + if (nbytes && (user_addr + nbytes <= user_addr)) + return EFAULT; + + if ((user_addr + nbytes) > vm_map_max(current_thread()->map)) + return EFAULT; + + return 0; +} + +static inline void +user_access_enable(void) +{ +#if __ARM_PAN_AVAILABLE__ + if (arm_pan_enabled) { + __builtin_arm_wsr("pan", 0); + } +#endif /* __ARM_PAN_AVAILABLE__ */ +} + +static inline void +user_access_disable(void) +{ +#if __ARM_PAN_AVAILABLE__ + if (arm_pan_enabled) { + __builtin_arm_wsr("pan", 1); + } +#endif /* __ARM_PAN_AVAILABLE__ */ +} + +static int +copyio(copyio_type_t copytype, const char *src, char *dst, + vm_size_t nbytes, vm_size_t *lencopied) +{ + int result = 0; + vm_size_t bytes_copied = 0; + + /* Reject TBI addresses */ + if (copytype == COPYIO_OUT) { + if ((uintptr_t)dst & TBI_MASK) + return EINVAL; + } else { + if ((uintptr_t)src & TBI_MASK) + return EINVAL; + } + + if (!nbytes) { + return 0; + } + +#if KASAN + /* For user copies, asan-check the kernel-side buffer */ + if (copytype == COPYIO_IN || copytype == COPYIO_INSTR || copytype == COPYIO_IN_WORD) { + __asan_storeN((uintptr_t)dst, nbytes); + } else if (copytype == COPYIO_OUT) { + __asan_loadN((uintptr_t)src, nbytes); + } +#endif + + user_access_enable(); + + /* Select copy routines based on direction: + * COPYIO_IN - Use unprivileged loads to read from user address + * COPYIO_OUT - Use unprivleged stores to write to user address + */ + + switch (copytype) { + case COPYIO_IN: + result = _bcopyin(src, dst, nbytes); + break; + case COPYIO_INSTR: + result = _bcopyinstr(src, dst, nbytes, &bytes_copied); + if (result != EFAULT) { + *lencopied = bytes_copied; + } + break; + case COPYIO_IN_WORD: + result = _copyin_word(src, (uint64_t *)(uintptr_t)dst, nbytes); + break; + case COPYIO_OUT: + result = _bcopyout(src, dst, nbytes); + break; + default: + result = EINVAL; + } + + user_access_disable(); + return result; +} + +int +copyin_kern(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes) +{ + bcopy((const char*)(uintptr_t)user_addr, kernel_addr, nbytes); + + return 0; +} + +int +copyout_kern(const char *kernel_addr, user_addr_t user_addr, vm_size_t nbytes) +{ + bcopy(kernel_addr, (char *)(uintptr_t)user_addr, nbytes); + + return 0; +} + +int +copyin(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes) +{ + int result; + + if (user_addr >= VM_MIN_KERNEL_ADDRESS || user_addr + nbytes >= VM_MIN_KERNEL_ADDRESS) { + if (current_thread()->map->pmap == kernel_pmap) + return copyin_kern(user_addr, kernel_addr, nbytes); + else + return EFAULT; + } + + if (nbytes >= 4096) { + result = copyin_validate(user_addr, (uintptr_t)kernel_addr, nbytes); + if (result) return result; + } + + result = copyio_check_user_addr(user_addr, nbytes); + + if (result) return result; + + return copyio(COPYIO_IN, (const char *)(uintptr_t)user_addr, kernel_addr, nbytes, NULL); +} + +/* + * copyin_word + * Read an aligned value from userspace as a single memory transaction. + * This function supports userspace synchronization features + */ +int +copyin_word(const user_addr_t user_addr, uint64_t *kernel_addr, vm_size_t nbytes) +{ + int result; + + /* Verify sizes */ + if ((nbytes != 4) && (nbytes != 8)) + return EINVAL; + + /* Test alignment */ + if (user_addr & (nbytes - 1)) + return EINVAL; + + /* Address must be user */ + if (user_addr >= VM_MIN_KERNEL_ADDRESS || user_addr + nbytes >= VM_MIN_KERNEL_ADDRESS) + return EFAULT; + + result = copyio_check_user_addr(user_addr, nbytes); + if (result) + return result; + + return copyio(COPYIO_IN_WORD, (const char *)user_addr, (char *)(uintptr_t)kernel_addr, nbytes, NULL); +} + +int +copyinstr(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes, vm_size_t *lencopied) +{ + int result; + + if (user_addr >= VM_MIN_KERNEL_ADDRESS || user_addr + nbytes >= VM_MIN_KERNEL_ADDRESS) { + return EFAULT; + } + + result = copyio_check_user_addr(user_addr, nbytes); + + if (result) return result; + + if (!nbytes) { + return ENAMETOOLONG; + } + + return copyio(COPYIO_INSTR, (const char *)(uintptr_t)user_addr, kernel_addr, nbytes, lencopied); +} + +int +copyout(const void *kernel_addr, user_addr_t user_addr, vm_size_t nbytes) +{ + int result; + + if (user_addr >= VM_MIN_KERNEL_ADDRESS || user_addr + nbytes >= VM_MIN_KERNEL_ADDRESS) { + if (current_thread()->map->pmap == kernel_pmap) + return copyout_kern(kernel_addr, user_addr, nbytes); + else + return EFAULT; + } + + if (nbytes >= 4096) { + result = copyout_validate((uintptr_t)kernel_addr, user_addr, nbytes); + if (result) return result; + } + + result = copyio_check_user_addr(user_addr, nbytes); + + if (result) return result; + + return copyio(COPYIO_OUT, kernel_addr, (char *)(uintptr_t)user_addr, nbytes, NULL); +} + + +/* + * Copy sizes bigger than this value will cause a kernel panic. + * + * Yes, this is an arbitrary fixed limit, but it's almost certainly + * a programming error to be copying more than this amount between + * user and wired kernel memory in a single invocation on this + * platform. + */ +const int copysize_limit_panic = (64 * 1024 * 1024); + +/* + * Validate the arguments to copy{in,out} on this platform. + * + * Called when nbytes is "large" e.g. more than a page. Such sizes are + * infrequent, and very large sizes are likely indications of attempts + * to exploit kernel programming errors (bugs). + */ +static int +copy_validate(const user_addr_t user_addr, + uintptr_t kernel_addr, vm_size_t nbytes) +{ + uintptr_t kernel_addr_last = kernel_addr + nbytes; + + if (kernel_addr < VM_MIN_KERNEL_ADDRESS || + kernel_addr > VM_MAX_KERNEL_ADDRESS || + kernel_addr_last < kernel_addr || + kernel_addr_last > VM_MAX_KERNEL_ADDRESS) + panic("%s(%p, %p, %lu) - kaddr not in kernel", __func__, + (void *)user_addr, (void *)kernel_addr, nbytes); + + user_addr_t user_addr_last = user_addr + nbytes; + + if (user_addr_last < user_addr || user_addr_last > VM_MIN_KERNEL_ADDRESS) + return (EFAULT); + + if (__improbable(nbytes > copysize_limit_panic)) + panic("%s(%p, %p, %lu) - transfer too large", __func__, + (void *)user_addr, (void *)kernel_addr, nbytes); + + return (0); +} + +int +copyin_validate(const user_addr_t ua, uintptr_t ka, vm_size_t nbytes) +{ + return (copy_validate(ua, ka, nbytes)); +} + +int +copyout_validate(uintptr_t ka, const user_addr_t ua, vm_size_t nbytes) +{ + return (copy_validate(ua, ka, nbytes)); +} + diff --git a/osfmk/arm64/cpu.c b/osfmk/arm64/cpu.c new file mode 100644 index 000000000..6e0d5ed52 --- /dev/null +++ b/osfmk/arm64/cpu.c @@ -0,0 +1,864 @@ +/* + * Copyright (c) 2007-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * File: arm64/cpu.c + * + * cpu specific routines + */ + +#include <pexpert/arm64/board_config.h> +#include <kern/kalloc.h> +#include <kern/machine.h> +#include <kern/cpu_number.h> +#include <kern/thread.h> +#include <kern/timer_queue.h> +#include <arm/cpu_data.h> +#include <arm/cpuid.h> +#include <arm/caches_internal.h> +#include <arm/cpu_data_internal.h> +#include <arm/cpu_internal.h> +#include <arm/misc_protos.h> +#include <arm/machine_cpu.h> +#include <arm/rtclock.h> +#include <arm64/proc_reg.h> +#include <mach/processor_info.h> +#include <vm/pmap.h> +#include <vm/vm_kern.h> +#include <vm/vm_map.h> +#include <pexpert/arm/protos.h> +#include <pexpert/device_tree.h> +#include <sys/kdebug.h> +#include <arm/machine_routines.h> + +#include <machine/atomic.h> + +#include <san/kasan.h> + +#if KPC +#include <kern/kpc.h> +#endif + +#if MONOTONIC +#include <kern/monotonic.h> +#endif /* MONOTONIC */ + +extern boolean_t idle_enable; +extern uint64_t wake_abstime; + +#if WITH_CLASSIC_S2R +void sleep_token_buffer_init(void); +#endif + + +extern uintptr_t resume_idle_cpu; +extern uintptr_t start_cpu; + +extern void __attribute__((noreturn)) arm64_prepare_for_sleep(void); +extern void arm64_force_wfi_clock_gate(void); +#if (defined(APPLECYCLONE) || defined(APPLETYPHOON)) +// <rdar://problem/15827409> CPU1 Stuck in WFIWT Because of MMU Prefetch +extern void cyclone_typhoon_prepare_for_wfi(void); +extern void cyclone_typhoon_return_from_wfi(void); +#endif + + +vm_address_t start_cpu_paddr; + +sysreg_restore_t sysreg_restore __attribute__((section("__DATA, __const"))) = { + .tcr_el1 = TCR_EL1_BOOT, +}; + + +// wfi - wfi mode +// 0 : disabled +// 1 : normal +// 2 : overhead simulation (delay & flags) +static int wfi = 1; + +#if DEVELOPMENT || DEBUG + +// wfi_flags +// 1 << 0 : flush L1s +// 1 << 1 : flush TLBs +static int wfi_flags = 0; + +// wfi_delay - delay ticks after wfi exit +static uint64_t wfi_delay = 0; + +#endif /* DEVELOPMENT || DEBUG */ + +#if __ARM_GLOBAL_SLEEP_BIT__ +volatile boolean_t arm64_stall_sleep = TRUE; +#endif + +#if WITH_CLASSIC_S2R +/* + * These must be aligned to avoid issues with calling bcopy_phys on them before + * we are done with pmap initialization. + */ +static const uint8_t __attribute__ ((aligned(8))) suspend_signature[] = {'X', 'S', 'O', 'M', 'P', 'S', 'U', 'S'}; +static const uint8_t __attribute__ ((aligned(8))) running_signature[] = {'X', 'S', 'O', 'M', 'N', 'N', 'U', 'R'}; +#endif + +#if WITH_CLASSIC_S2R +static vm_offset_t sleepTokenBuffer = (vm_offset_t)NULL; +#endif +static boolean_t coresight_debug_enabled = FALSE; + + +static void +configure_coresight_registers(cpu_data_t *cdp) +{ + uint64_t addr; + int i; + + assert(cdp); + + /* + * ARMv8 coresight registers are optional. If the device tree did not + * provide cpu_regmap_paddr, assume that coresight registers are not + * supported. + */ + if (cdp->cpu_regmap_paddr) { + for (i = 0; i < CORESIGHT_REGIONS; ++i) { + /* Skip CTI; these registers are debug-only (they are + * not present on production hardware), and there is + * at least one known Cyclone errata involving CTI + * (rdar://12802966). We have no known clients that + * need the kernel to unlock CTI, so it is safer + * to avoid doing the access. + */ + if (i == CORESIGHT_CTI) + continue; + /* Skip debug-only registers on production chips */ + if (((i == CORESIGHT_ED) || (i == CORESIGHT_UTT)) && !coresight_debug_enabled) + continue; + + if (!cdp->coresight_base[i]) { + addr = cdp->cpu_regmap_paddr + CORESIGHT_OFFSET(i); + cdp->coresight_base[i] = (vm_offset_t)ml_io_map(addr, CORESIGHT_SIZE); + + /* + * At this point, failing to io map the + * registers is considered as an error. + */ + if (!cdp->coresight_base[i]) { + panic("unable to ml_io_map coresight regions"); + } + } + /* Unlock EDLAR, CTILAR, PMLAR */ + if (i != CORESIGHT_UTT) + *(volatile uint32_t *)(cdp->coresight_base[i] + ARM_DEBUG_OFFSET_DBGLAR) = ARM_DBG_LOCK_ACCESS_KEY; + } + } +} + + +/* + * Routine: cpu_bootstrap + * Function: + */ +void +cpu_bootstrap(void) +{ +} + +/* + * Routine: cpu_sleep + * Function: + */ +void +cpu_sleep(void) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + pmap_switch_user_ttb(kernel_pmap); + cpu_data_ptr->cpu_active_thread = current_thread(); + cpu_data_ptr->cpu_reset_handler = (uintptr_t) start_cpu_paddr; + cpu_data_ptr->cpu_flags |= SleepState; + cpu_data_ptr->cpu_user_debug = NULL; +#if KPC + kpc_idle(); +#endif /* KPC */ +#if MONOTONIC + mt_cpu_down(cpu_data_ptr); +#endif /* MONOTONIC */ + + CleanPoC_Dcache(); + + PE_cpu_machine_quiesce(cpu_data_ptr->cpu_id); + +} + +/* + * Routine: cpu_idle + * Function: + */ +void __attribute__((noreturn)) +cpu_idle(void) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + uint64_t new_idle_timeout_ticks = 0x0ULL, lastPop; + + if ((!idle_enable) || (cpu_data_ptr->cpu_signal & SIGPdisabled)) + Idle_load_context(); + if (!SetIdlePop()) + Idle_load_context(); + lastPop = cpu_data_ptr->rtcPop; + + pmap_switch_user_ttb(kernel_pmap); + cpu_data_ptr->cpu_active_thread = current_thread(); + if (cpu_data_ptr->cpu_user_debug) + arm_debug_set(NULL); + cpu_data_ptr->cpu_user_debug = NULL; + + if (cpu_data_ptr->cpu_idle_notify) + ((processor_idle_t) cpu_data_ptr->cpu_idle_notify) (cpu_data_ptr->cpu_id, TRUE, &new_idle_timeout_ticks); + + if (cpu_data_ptr->idle_timer_notify != 0) { + if (new_idle_timeout_ticks == 0x0ULL) { + /* turn off the idle timer */ + cpu_data_ptr->idle_timer_deadline = 0x0ULL; + } else { + /* set the new idle timeout */ + clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline); + } + timer_resync_deadlines(); + if (cpu_data_ptr->rtcPop != lastPop) + SetIdlePop(); + } + +#if KPC + kpc_idle(); +#endif +#if MONOTONIC + mt_cpu_idle(cpu_data_ptr); +#endif /* MONOTONIC */ + + if (wfi) { + platform_cache_idle_enter(); + +#if DEVELOPMENT || DEBUG + // When simulating wfi overhead, + // force wfi to clock gating only + if (wfi == 2) { + arm64_force_wfi_clock_gate(); + } +#endif /* DEVELOPMENT || DEBUG */ + +#if defined(APPLECYCLONE) || defined(APPLETYPHOON) + // <rdar://problem/15827409> CPU1 Stuck in WFIWT Because of MMU Prefetch + cyclone_typhoon_prepare_for_wfi(); +#endif + __builtin_arm_dsb(DSB_SY); + __builtin_arm_wfi(); + +#if defined(APPLECYCLONE) || defined(APPLETYPHOON) + // <rdar://problem/15827409> CPU1 Stuck in WFIWT Because of MMU Prefetch + cyclone_typhoon_return_from_wfi(); +#endif + +#if DEVELOPMENT || DEBUG + // Handle wfi overhead simulation + if (wfi == 2) { + uint64_t deadline; + + // Calculate wfi delay deadline + clock_absolutetime_interval_to_deadline(wfi_delay, &deadline); + + // Flush L1 caches + if ((wfi_flags & 1) != 0) { + InvalidatePoU_Icache(); + FlushPoC_Dcache(); + } + + // Flush TLBs + if ((wfi_flags & 2) != 0) { + flush_core_tlb(); + } + + // Wait for the ballance of the wfi delay + clock_delay_until(deadline); + } +#endif /* DEVELOPMENT || DEBUG */ + + platform_cache_idle_exit(); + } + + ClearIdlePop(TRUE); + + cpu_idle_exit(); +} + +/* + * Routine: cpu_idle_exit + * Function: + */ +void +cpu_idle_exit(void) +{ + uint64_t new_idle_timeout_ticks = 0x0ULL; + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + assert(exception_stack_pointer() != 0); + + /* Back from WFI, unlock OSLAR and EDLAR. */ + configure_coresight_registers(cpu_data_ptr); + +#if KPC + kpc_idle_exit(); +#endif + +#if MONOTONIC + mt_cpu_run(cpu_data_ptr); +#endif /* MONOTONIC */ + + pmap_switch_user_ttb(cpu_data_ptr->cpu_active_thread->map->pmap); + + if (cpu_data_ptr->cpu_idle_notify) + ((processor_idle_t) cpu_data_ptr->cpu_idle_notify) (cpu_data_ptr->cpu_id, FALSE, &new_idle_timeout_ticks); + + if (cpu_data_ptr->idle_timer_notify != 0) { + if (new_idle_timeout_ticks == 0x0ULL) { + /* turn off the idle timer */ + cpu_data_ptr->idle_timer_deadline = 0x0ULL; + } else { + /* set the new idle timeout */ + clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline); + } + timer_resync_deadlines(); + } + + Idle_load_context(); +} + +void +cpu_init(void) +{ + cpu_data_t *cdp = getCpuDatap(); + arm_cpu_info_t *cpu_info_p; + + assert(exception_stack_pointer() != 0); + + if (cdp->cpu_type != CPU_TYPE_ARM64) { + + cdp->cpu_type = CPU_TYPE_ARM64; + + timer_call_queue_init(&cdp->rtclock_timer.queue); + cdp->rtclock_timer.deadline = EndOfAllTime; + + if (cdp == &BootCpuData) { + do_cpuid(); + do_cacheid(); + do_mvfpid(); + } else { + /* + * We initialize non-boot CPUs here; the boot CPU is + * dealt with as part of pmap_bootstrap. + */ + pmap_cpu_data_init(); + } + /* ARM_SMP: Assuming identical cpu */ + do_debugid(); + + cpu_info_p = cpuid_info(); + + /* switch based on CPU's reported architecture */ + switch (cpu_info_p->arm_info.arm_arch) { + case CPU_ARCH_ARMv8: + cdp->cpu_subtype = CPU_SUBTYPE_ARM64_V8; + break; + default: + //cdp->cpu_subtype = CPU_SUBTYPE_ARM64_ALL; + /* this panic doesn't work this early in startup */ + panic("Unknown CPU subtype..."); + break; + } + + cdp->cpu_threadtype = CPU_THREADTYPE_NONE; + } + cdp->cpu_stat.irq_ex_cnt_wake = 0; + cdp->cpu_stat.ipi_cnt_wake = 0; + cdp->cpu_stat.timer_cnt_wake = 0; + cdp->cpu_running = TRUE; + cdp->cpu_sleep_token_last = cdp->cpu_sleep_token; + cdp->cpu_sleep_token = 0x0UL; +#if KPC + kpc_idle_exit(); +#endif /* KPC */ +#if MONOTONIC + mt_cpu_up(cdp); +#endif /* MONOTONIC */ +} + +cpu_data_t * +cpu_data_alloc(boolean_t is_boot_cpu) +{ + cpu_data_t *cpu_data_ptr = NULL; + + if (is_boot_cpu) + cpu_data_ptr = &BootCpuData; + else { + void *irq_stack = NULL; + void *exc_stack = NULL; + void *fiq_stack = NULL; + + if ((kmem_alloc(kernel_map, (vm_offset_t *)&cpu_data_ptr, sizeof(cpu_data_t), VM_KERN_MEMORY_CPU)) != KERN_SUCCESS) + goto cpu_data_alloc_error; + + bzero((void *)cpu_data_ptr, sizeof(cpu_data_t)); + + if ((irq_stack = kalloc(INTSTACK_SIZE)) == 0) + goto cpu_data_alloc_error; + cpu_data_ptr->intstack_top = (vm_offset_t)irq_stack + INTSTACK_SIZE ; + cpu_data_ptr->istackptr = cpu_data_ptr->intstack_top; + + if ((exc_stack = kalloc(PAGE_SIZE)) == 0) + goto cpu_data_alloc_error; + cpu_data_ptr->excepstack_top = (vm_offset_t)exc_stack + PAGE_SIZE ; + cpu_data_ptr->excepstackptr = cpu_data_ptr->excepstack_top; + + if ((fiq_stack = kalloc(PAGE_SIZE)) == 0) + goto cpu_data_alloc_error; + cpu_data_ptr->fiqstack_top = (vm_offset_t)fiq_stack + PAGE_SIZE ; + cpu_data_ptr->fiqstackptr = cpu_data_ptr->fiqstack_top; + } + + cpu_data_ptr->cpu_processor = cpu_processor_alloc(is_boot_cpu); + if (cpu_data_ptr->cpu_processor == (struct processor *)NULL) + goto cpu_data_alloc_error; + + return cpu_data_ptr; + +cpu_data_alloc_error: + panic("cpu_data_alloc() failed\n"); + return (cpu_data_t *)NULL; +} + + +void +cpu_data_free(cpu_data_t *cpu_data_ptr) +{ + if (cpu_data_ptr == &BootCpuData) + return; + + cpu_processor_free( cpu_data_ptr->cpu_processor); + kfree( (void *)(cpu_data_ptr->intstack_top - INTSTACK_SIZE), INTSTACK_SIZE); + kfree( (void *)(cpu_data_ptr->fiqstack_top - PAGE_SIZE), PAGE_SIZE); + kmem_free(kernel_map, (vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t)); +} + +void +cpu_data_init(cpu_data_t *cpu_data_ptr) +{ + uint32_t i; + + cpu_data_ptr->cpu_flags = 0; + cpu_data_ptr->interrupts_enabled = 0; + cpu_data_ptr->cpu_int_state = 0; + cpu_data_ptr->cpu_pending_ast = AST_NONE; + cpu_data_ptr->cpu_cache_dispatch = (void *) 0; + cpu_data_ptr->rtcPop = EndOfAllTime; + cpu_data_ptr->rtclock_datap = &RTClockData; + cpu_data_ptr->cpu_user_debug = NULL; + + + cpu_data_ptr->cpu_base_timebase = 0; + cpu_data_ptr->cpu_idle_notify = (void *) 0; + cpu_data_ptr->cpu_idle_latency = 0x0ULL; + cpu_data_ptr->cpu_idle_pop = 0x0ULL; + cpu_data_ptr->cpu_reset_type = 0x0UL; + cpu_data_ptr->cpu_reset_handler = 0x0UL; + cpu_data_ptr->cpu_reset_assist = 0x0UL; + cpu_data_ptr->cpu_regmap_paddr = 0x0ULL; + cpu_data_ptr->cpu_phys_id = 0x0UL; + cpu_data_ptr->cpu_l2_access_penalty = 0; + cpu_data_ptr->cpu_cluster_type = CLUSTER_TYPE_SMP; + cpu_data_ptr->cpu_cluster_id = 0; + cpu_data_ptr->cpu_l2_id = 0; + cpu_data_ptr->cpu_l2_size = 0; + cpu_data_ptr->cpu_l3_id = 0; + cpu_data_ptr->cpu_l3_size = 0; + + cpu_data_ptr->cpu_signal = SIGPdisabled; + +#if DEBUG || DEVELOPMENT + cpu_data_ptr->failed_xcall = NULL; + cpu_data_ptr->failed_signal = 0; + cpu_data_ptr->failed_signal_count = 0; +#endif + + cpu_data_ptr->cpu_get_fiq_handler = NULL; + cpu_data_ptr->cpu_tbd_hardware_addr = NULL; + cpu_data_ptr->cpu_tbd_hardware_val = NULL; + cpu_data_ptr->cpu_get_decrementer_func = NULL; + cpu_data_ptr->cpu_set_decrementer_func = NULL; + cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH; + cpu_data_ptr->cpu_sleep_token_last = 0x00000000UL; + cpu_data_ptr->cpu_xcall_p0 = NULL; + cpu_data_ptr->cpu_xcall_p1 = NULL; + + for (i = 0; i < CORESIGHT_REGIONS; ++i) { + cpu_data_ptr->coresight_base[i] = 0; + } + + pmap_cpu_data_t * pmap_cpu_data_ptr = &cpu_data_ptr->cpu_pmap_cpu_data; + + pmap_cpu_data_ptr->cpu_user_pmap = (struct pmap *) NULL; + pmap_cpu_data_ptr->cpu_user_pmap_stamp = 0; + pmap_cpu_data_ptr->cpu_number = PMAP_INVALID_CPU_NUM; + + for (i = 0; i < (sizeof(pmap_cpu_data_ptr->cpu_asid_high_bits) / sizeof(*pmap_cpu_data_ptr->cpu_asid_high_bits)); i++) { + pmap_cpu_data_ptr->cpu_asid_high_bits[i] = 0; + } + cpu_data_ptr->halt_status = CPU_NOT_HALTED; +} + +kern_return_t +cpu_data_register(cpu_data_t *cpu_data_ptr) +{ + int cpu = cpu_data_ptr->cpu_number; + +#if KASAN + for (int i = 0; i < CPUWINDOWS_MAX; i++) { + kasan_notify_address_nopoison(pmap_cpu_windows_copy_addr(cpu, i), PAGE_SIZE); + } +#endif + + CpuDataEntries[cpu].cpu_data_vaddr = cpu_data_ptr; + CpuDataEntries[cpu].cpu_data_paddr = (void *)ml_vtophys( (vm_offset_t)cpu_data_ptr); + return KERN_SUCCESS; + +} + +kern_return_t +cpu_start(int cpu) +{ + cpu_data_t *cpu_data_ptr = CpuDataEntries[cpu].cpu_data_vaddr; + + kprintf("cpu_start() cpu: %d\n", cpu); + + if (cpu == cpu_number()) { + cpu_machine_init(); + configure_coresight_registers(cpu_data_ptr); + } else { + thread_t first_thread; + + cpu_data_ptr->cpu_reset_handler = (vm_offset_t) start_cpu_paddr; + + cpu_data_ptr->cpu_pmap_cpu_data.cpu_user_pmap = NULL; + + if (cpu_data_ptr->cpu_processor->next_thread != THREAD_NULL) + first_thread = cpu_data_ptr->cpu_processor->next_thread; + else + first_thread = cpu_data_ptr->cpu_processor->idle_thread; + cpu_data_ptr->cpu_active_thread = first_thread; + first_thread->machine.CpuDatap = cpu_data_ptr; + + configure_coresight_registers(cpu_data_ptr); + + flush_dcache((vm_offset_t)&CpuDataEntries[cpu], sizeof(cpu_data_entry_t), FALSE); + flush_dcache((vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t), FALSE); + (void) PE_cpu_start(cpu_data_ptr->cpu_id, (vm_offset_t)NULL, (vm_offset_t)NULL); + } + + return KERN_SUCCESS; +} + + +void +cpu_timebase_init(boolean_t from_boot) +{ + cpu_data_t *cdp = getCpuDatap(); + + if (cdp->cpu_get_fiq_handler == NULL) { + cdp->cpu_get_fiq_handler = rtclock_timebase_func.tbd_fiq_handler; + cdp->cpu_get_decrementer_func = rtclock_timebase_func.tbd_get_decrementer; + cdp->cpu_set_decrementer_func = rtclock_timebase_func.tbd_set_decrementer; + cdp->cpu_tbd_hardware_addr = (void *)rtclock_timebase_addr; + cdp->cpu_tbd_hardware_val = (void *)rtclock_timebase_val; + } + + if (!from_boot && (cdp == &BootCpuData)) { + /* + * When we wake from sleep, we have no guarantee about the state + * of the hardware timebase. It may have kept ticking across sleep, or + * it may have reset. + * + * To deal with this, we calculate an offset to the clock that will + * produce a timebase value wake_abstime at the point the boot + * CPU calls cpu_timebase_init on wake. + * + * This ensures that mach_absolute_time() stops ticking across sleep. + */ + rtclock_base_abstime = wake_abstime - ml_get_hwclock(); + } + + cdp->cpu_decrementer = 0x7FFFFFFFUL; + cdp->cpu_timebase = 0x0UL; + cdp->cpu_base_timebase = rtclock_base_abstime; +} + +int +cpu_cluster_id(void) +{ + return (getCpuDatap()->cpu_cluster_id); +} + +__attribute__((noreturn)) +void +ml_arm_sleep(void) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + if (cpu_data_ptr == &BootCpuData) { + cpu_data_t *target_cdp; + int cpu; + int max_cpu; + + max_cpu = ml_get_max_cpu_number(); + for (cpu=0; cpu <= max_cpu; cpu++) { + target_cdp = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr; + + if ((target_cdp == NULL) || (target_cdp == cpu_data_ptr)) + continue; + + while (target_cdp->cpu_sleep_token != ARM_CPU_ON_SLEEP_PATH); + } + + /* + * Now that the other cores have entered the sleep path, set + * the abstime value we'll use when we resume. + */ + wake_abstime = ml_get_timebase(); + } else { + CleanPoU_Dcache(); + } + + cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH; + + if (cpu_data_ptr == &BootCpuData) { +#if WITH_CLASSIC_S2R + // Classic suspend to RAM writes the suspend signature into the + // sleep token buffer so that iBoot knows that it's on the warm + // boot (wake) path (as opposed to the cold boot path). Newer SoC + // do not go through SecureROM/iBoot on the warm boot path. The + // reconfig engine script brings the CPU out of reset at the kernel's + // reset vector which points to the warm boot initialization code. + if(sleepTokenBuffer != (vm_offset_t) NULL) { + platform_cache_shutdown(); + bcopy((const void *)suspend_signature, (void *)sleepTokenBuffer, sizeof(SleepToken)); + } + else { + panic("No sleep token buffer"); + } +#endif + +#if __ARM_GLOBAL_SLEEP_BIT__ + /* Allow other CPUs to go to sleep. */ + arm64_stall_sleep = FALSE; + __builtin_arm_dmb(DMB_ISH); +#endif + + /* Architectural debug state: <rdar://problem/12390433>: + * Grab debug lock EDLAR and clear bit 0 in EDPRCR, + * tell debugger to not prevent power gating . + */ + if (cpu_data_ptr->coresight_base[CORESIGHT_ED]) { + *(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGLAR) = ARM_DBG_LOCK_ACCESS_KEY; + *(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGPRCR) = 0; + } + +#if MONOTONIC + mt_sleep(); +#endif /* MONOTONIC */ + /* ARM64-specific preparation */ + arm64_prepare_for_sleep(); + } else { +#if __ARM_GLOBAL_SLEEP_BIT__ + /* + * With the exception of the CPU revisions listed above, our ARM64 CPUs have a + * global register to manage entering deep sleep, as opposed to a per-CPU + * register. We cannot update this register until all CPUs are ready to enter + * deep sleep, because if a CPU executes WFI outside of the deep sleep context + * (by idling), it will hang (due to the side effects of enabling deep sleep), + * which can hang the sleep process or cause memory corruption on wake. + * + * To avoid these issues, we'll stall on this global value, which CPU0 will + * manage. + */ + while (arm64_stall_sleep) { + __builtin_arm_wfe(); + } +#endif + CleanPoU_DcacheRegion((vm_offset_t) cpu_data_ptr, sizeof(cpu_data_t)); + + /* Architectural debug state: <rdar://problem/12390433>: + * Grab debug lock EDLAR and clear bit 0 in EDPRCR, + * tell debugger to not prevent power gating . + */ + if (cpu_data_ptr->coresight_base[CORESIGHT_ED]) { + *(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGLAR) = ARM_DBG_LOCK_ACCESS_KEY; + *(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGPRCR) = 0; + } + + /* ARM64-specific preparation */ + arm64_prepare_for_sleep(); + } +} + +void +cpu_machine_idle_init(boolean_t from_boot) +{ + static vm_address_t resume_idle_cpu_paddr = (vm_address_t)NULL; + cpu_data_t *cpu_data_ptr = getCpuDatap(); + + if (from_boot) { + unsigned long jtag = 0; + int wfi_tmp = 1; + uint32_t production = 1; + DTEntry entry; + + if (PE_parse_boot_argn("jtag", &jtag, sizeof (jtag))) { + if (jtag != 0) + idle_enable = FALSE; + else + idle_enable = TRUE; + } else + idle_enable = TRUE; + + PE_parse_boot_argn("wfi", &wfi_tmp, sizeof (wfi_tmp)); + + // bits 7..0 give the wfi type + switch (wfi_tmp & 0xff) { + case 0 : + // disable wfi + wfi = 0; + break; + +#if DEVELOPMENT || DEBUG + case 2 : + // wfi overhead simulation + // 31..16 - wfi delay is us + // 15..8 - flags + // 7..0 - 2 + wfi = 2; + wfi_flags = (wfi_tmp >> 8) & 0xFF; + nanoseconds_to_absolutetime(((wfi_tmp >> 16) & 0xFFFF) * NSEC_PER_MSEC, &wfi_delay); + break; +#endif /* DEVELOPMENT || DEBUG */ + + case 1 : + default : + // do nothing + break; + } + + ResetHandlerData.assist_reset_handler = 0; + ResetHandlerData.cpu_data_entries = ml_static_vtop((vm_offset_t)CpuDataEntries); + +#ifdef MONITOR + monitor_call(MONITOR_SET_ENTRY, (uintptr_t)ml_static_vtop((vm_offset_t)&LowResetVectorBase), 0, 0); +#elif !defined(NO_MONITOR) +#error MONITOR undefined, WFI power gating may not operate correctly +#endif /* MONITOR */ + + // Determine if we are on production or debug chip + if (kSuccess == DTLookupEntry(NULL, "/chosen", &entry)) { + unsigned int size; + void *prop; + + if (kSuccess == DTGetProperty(entry, "effective-production-status-ap", &prop, &size)) + if (size == 4) + bcopy(prop, &production, size); + } + if (!production) { +#if defined(APPLE_ARM64_ARCH_FAMILY) + // Enable coresight debug registers on debug-fused chips + coresight_debug_enabled = TRUE; +#endif + } + + start_cpu_paddr = ml_static_vtop((vm_offset_t)&start_cpu); + resume_idle_cpu_paddr = ml_static_vtop((vm_offset_t)&resume_idle_cpu); + } + +#if WITH_CLASSIC_S2R + if (cpu_data_ptr == &BootCpuData) { + static addr64_t SleepToken_low_paddr = (addr64_t)NULL; + if (sleepTokenBuffer != (vm_offset_t) NULL) { + SleepToken_low_paddr = ml_vtophys(sleepTokenBuffer); + } + else { + panic("No sleep token buffer"); + } + + bcopy_phys((addr64_t)ml_static_vtop((vm_offset_t)running_signature), + SleepToken_low_paddr, sizeof(SleepToken)); + flush_dcache((vm_offset_t)SleepToken, sizeof(SleepToken), TRUE); + }; +#endif + + cpu_data_ptr->cpu_reset_handler = resume_idle_cpu_paddr; + clean_dcache((vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t), FALSE); +} + +_Atomic uint32_t cpu_idle_count = 0; + +void +machine_track_platform_idle(boolean_t entry) +{ + if (entry) + (void)__c11_atomic_fetch_add(&cpu_idle_count, 1, __ATOMIC_RELAXED); + else + (void)__c11_atomic_fetch_sub(&cpu_idle_count, 1, __ATOMIC_RELAXED); +} + +#if WITH_CLASSIC_S2R +void +sleep_token_buffer_init(void) +{ + cpu_data_t *cpu_data_ptr = getCpuDatap(); + DTEntry entry; + size_t size; + void **prop; + + if ((cpu_data_ptr == &BootCpuData) && (sleepTokenBuffer == (vm_offset_t) NULL)) { + /* Find the stpage node in the device tree */ + if (kSuccess != DTLookupEntry(0, "stram", &entry)) + return; + + if (kSuccess != DTGetProperty(entry, "reg", (void **)&prop, (unsigned int *)&size)) + return; + + /* Map the page into the kernel space */ + sleepTokenBuffer = ml_io_map(((vm_offset_t *)prop)[0], ((vm_size_t *)prop)[1]); + } +} +#endif + diff --git a/osfmk/arm64/cswitch.s b/osfmk/arm64/cswitch.s new file mode 100644 index 000000000..e3a0cb317 --- /dev/null +++ b/osfmk/arm64/cswitch.s @@ -0,0 +1,239 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +#include <machine/asm.h> +#include <arm64/machine_machdep.h> +#include <arm64/proc_reg.h> +#include "assym.s" + +/* + * save_general_registers + * + * Saves variable registers to kernel PCB. + * arg0 - thread_kernel_state pointer + * arg1 - Scratch register + */ + +.macro save_general_registers +/* AAPCS-64 Page 14 + * + * A subroutine invocation must preserve the contents of the registers r19-r29 + * and SP. We also save IP0 and IP1, as machine_idle uses IP0 for saving the LR. + */ + stp x16, x17, [$0, SS64_X16] + stp x19, x20, [$0, SS64_X19] + stp x21, x22, [$0, SS64_X21] + stp x23, x24, [$0, SS64_X23] + stp x25, x26, [$0, SS64_X25] + stp x27, x28, [$0, SS64_X27] + stp fp, lr, [$0, SS64_FP] + mov $1, sp + str $1, [$0, SS64_SP] + +/* AAPCS-64 Page 14 + * + * Registers d8-d15 (s8-s15) must be preserved by a callee across subroutine + * calls; the remaining registers (v0-v7, v16-v31) do not need to be preserved + * (or should be preserved by the caller). + */ + str d8, [$0, NS64_D8] + str d9, [$0, NS64_D9] + str d10,[$0, NS64_D10] + str d11,[$0, NS64_D11] + str d12,[$0, NS64_D12] + str d13,[$0, NS64_D13] + str d14,[$0, NS64_D14] + str d15,[$0, NS64_D15] +.endmacro + +/* + * load_general_registers + * + * Loads variable registers from kernel PCB. + * arg0 - thread_kernel_state pointer + * arg1 - Scratch register + */ +.macro load_general_registers + ldp x16, x17, [$0, SS64_X16] + ldp x19, x20, [$0, SS64_X19] + ldp x21, x22, [$0, SS64_X21] + ldp x23, x24, [$0, SS64_X23] + ldp x25, x26, [$0, SS64_X25] + ldp x27, x28, [$0, SS64_X27] + ldp fp, lr, [$0, SS64_FP] + ldr $1, [$0, SS64_SP] + mov sp, $1 + + ldr d8, [$0, NS64_D8] + ldr d9, [$0, NS64_D9] + ldr d10,[$0, NS64_D10] + ldr d11,[$0, NS64_D11] + ldr d12,[$0, NS64_D12] + ldr d13,[$0, NS64_D13] + ldr d14,[$0, NS64_D14] + ldr d15,[$0, NS64_D15] +.endmacro + +/* + * set_thread_registers + * + * Updates thread registers during context switch + * arg0 - New thread pointer + * arg1 - Scratch register + * arg2 - Scratch register + */ +.macro set_thread_registers + msr TPIDR_EL1, $0 // Write new thread pointer to TPIDR_EL1 + ldr $1, [$0, TH_CTH_SELF] // Get cthread pointer + mrs $2, TPIDRRO_EL0 // Extract cpu number from TPIDRRO_EL0 + and $2, $2, #(MACHDEP_CPUNUM_MASK) + orr $2, $1, $2 // Save new cthread/cpu to TPIDRRO_EL0 + msr TPIDRRO_EL0, $2 + ldr $1, [$0, TH_CTH_DATA] // Get new cthread data pointer + msr TPIDR_EL0, $1 // Save data pointer to TPIDRRW_EL0 + /* ARM64_TODO Reserve x18 until we decide what to do with it */ + mov x18, $1 // ... and trash reserved x18 +.endmacro + + +/* + * void machine_load_context(thread_t thread) + * + * Load the context for the first thread to run on a + * cpu, and go. + */ + .text + .align 2 + .globl EXT(machine_load_context) + +LEXT(machine_load_context) + set_thread_registers x0, x1, x2 + ldr x1, [x0, TH_KSTACKPTR] // Get top of kernel stack + load_general_registers x1, x2 + mov x0, xzr // Clear argument to thread_continue + ret + +/* + * void Call_continuation( void (*continuation)(void), + * void *param, + * wait_result_t wresult, + * vm_offset_t stack_ptr) + */ + .text + .align 5 + .globl EXT(Call_continuation) + +LEXT(Call_continuation) + mrs x4, TPIDR_EL1 // Get the current thread pointer + + /* ARM64_TODO arm loads the kstack top instead of arg4. What should we use? */ + ldr x5, [x4, TH_KSTACKPTR] // Get the top of the kernel stack + mov sp, x5 // Set stack pointer + + mov fp, xzr // Clear the frame pointer + mov x4, x0 // Load the continuation + mov x0, x1 // Set the first parameter + mov x1, x2 // Set the wait result arg + blr x4 // Branch to the continuation + mrs x0, TPIDR_EL1 // Get the current thread pointer + b EXT(thread_terminate) // Kill the thread + + +/* + * thread_t Switch_context(thread_t old, + * void (*cont)(void), + * thread_t new) + */ + .text + .align 5 + .globl EXT(Switch_context) + +LEXT(Switch_context) + cbnz x1, Lswitch_threads // Skip saving old state if blocking on continuation + ldr x3, [x0, TH_KSTACKPTR] // Get the old kernel stack top + save_general_registers x3, x4 +Lswitch_threads: + set_thread_registers x2, x3, x4 + ldr x3, [x2, TH_KSTACKPTR] + load_general_registers x3, x4 + ret + +/* + * thread_t Shutdown_context(void (*doshutdown)(processor_t), processor_t processor) + * + */ + .text + .align 5 + .globl EXT(Shutdown_context) + +LEXT(Shutdown_context) + mrs x10, TPIDR_EL1 // Get thread pointer + ldr x11, [x10, TH_KSTACKPTR] // Get the top of the kernel stack + save_general_registers x11, x12 + msr DAIFSet, #(DAIFSC_FIQF | DAIFSC_IRQF) // Disable interrupts + ldr x11, [x10, ACT_CPUDATAP] // Get current cpu + ldr x12, [x11, CPU_ISTACKPTR] // Switch to interrupt stack + mov sp, x12 + b EXT(cpu_doshutdown) + + +/* + * thread_t Idle_context(void) + * + */ + .text + .align 5 + .globl EXT(Idle_context) + +LEXT(Idle_context) + mrs x0, TPIDR_EL1 // Get thread pointer + ldr x1, [x0, TH_KSTACKPTR] // Get the top of the kernel stack + save_general_registers x1, x2 + ldr x1, [x0, ACT_CPUDATAP] // Get current cpu + ldr x2, [x1, CPU_ISTACKPTR] // Switch to interrupt stack + mov sp, x2 + b EXT(cpu_idle) + +/* + * thread_t Idle_context(void) + * + */ + .text + .align 5 + .globl EXT(Idle_load_context) + +LEXT(Idle_load_context) + mrs x0, TPIDR_EL1 // Get thread pointer + ldr x1, [x0, TH_KSTACKPTR] // Get the top of the kernel stack + load_general_registers x1, x2 + ret + + .align 2 + .globl EXT(machine_set_current_thread) +LEXT(machine_set_current_thread) + set_thread_registers x0, x1, x2 + ret diff --git a/osfmk/arm64/dbgwrap.c b/osfmk/arm64/dbgwrap.c new file mode 100644 index 000000000..282432450 --- /dev/null +++ b/osfmk/arm64/dbgwrap.c @@ -0,0 +1,283 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include <arm/cpu_data_internal.h> +#include <arm/dbgwrap.h> +#include <arm64/proc_reg.h> +#include <machine/atomic.h> +#include <pexpert/arm64/board_config.h> + +#define DBGWRAP_REG_OFFSET 0 +#define DBGWRAP_DBGHALT (1ULL << 31) +#define DBGWRAP_DBGACK (1ULL << 28) + +#define EDDTRRX_REG_OFFSET 0x80 +#define EDITR_REG_OFFSET 0x84 +#define EDSCR_REG_OFFSET 0x88 +#define EDSCR_TXFULL (1ULL << 29) +#define EDSCR_ITE (1ULL << 24) +#define EDSCR_MA (1ULL << 20) +#define EDSCR_ERR (1ULL << 6) +#define EDDTRTX_REG_OFFSET 0x8C +#define EDRCR_REG_OFFSET 0x90 +#define EDRCR_CSE (1ULL << 2) +#define EDPRSR_REG_OFFSET 0x314 +#define EDPRSR_OSLK (1ULL << 5) + +#define MAX_EDITR_RETRIES 16 + +/* Older SoCs require 32-bit accesses for DBGWRAP; + * newer ones require 64-bit accesses. */ +#ifdef HAS_32BIT_DBGWRAP +typedef uint32_t dbgwrap_reg_t; +#else +typedef uint64_t dbgwrap_reg_t; +#endif + +#if DEVELOPMENT || DEBUG +#define MAX_STUFFED_INSTRS 64 +uint32_t stuffed_instrs[MAX_STUFFED_INSTRS]; +volatile uint32_t stuffed_instr_count = 0; +#endif + +static volatile uint32_t halt_from_cpu = (uint32_t)-1; + +boolean_t +ml_dbgwrap_cpu_is_halted(int cpu_index) +{ + cpu_data_t *cdp = cpu_datap(cpu_index); + if ((cdp == NULL) || (cdp->coresight_base[CORESIGHT_UTT] == 0)) + return FALSE; + + return ((*(volatile dbgwrap_reg_t *)(cdp->coresight_base[CORESIGHT_UTT] + DBGWRAP_REG_OFFSET) & DBGWRAP_DBGACK) != 0); +} + +dbgwrap_status_t +ml_dbgwrap_wait_cpu_halted(int cpu_index, uint64_t timeout_ns) +{ + cpu_data_t *cdp = cpu_datap(cpu_index); + if ((cdp == NULL) || (cdp->coresight_base[CORESIGHT_UTT] == 0)) + return DBGWRAP_ERR_UNSUPPORTED; + + volatile dbgwrap_reg_t *dbgWrapReg = (volatile dbgwrap_reg_t *)(cdp->coresight_base[CORESIGHT_UTT] + DBGWRAP_REG_OFFSET); + + uint64_t interval; + nanoseconds_to_absolutetime(timeout_ns, &interval); + uint64_t deadline = mach_absolute_time() + interval; + while (!(*dbgWrapReg & DBGWRAP_DBGACK)) { + if (mach_absolute_time() > deadline) + return DBGWRAP_ERR_HALT_TIMEOUT; + } + + return DBGWRAP_SUCCESS; +} + +dbgwrap_status_t +ml_dbgwrap_halt_cpu(int cpu_index, uint64_t timeout_ns) +{ + cpu_data_t *cdp = cpu_datap(cpu_index); + if ((cdp == NULL) || (cdp->coresight_base[CORESIGHT_UTT] == 0)) + return DBGWRAP_ERR_UNSUPPORTED; + + /* Only one cpu is allowed to initiate the halt sequence, to prevent cpus from cross-halting + * each other. The first cpu to request a halt may then halt any and all other cpus besides itself. */ + int curcpu = cpu_number(); + if (cpu_index == curcpu) + return DBGWRAP_ERR_SELF_HALT; + + if (!hw_compare_and_store((uint32_t)-1, (unsigned int)curcpu, &halt_from_cpu) && + (halt_from_cpu != (uint32_t)curcpu)) + return DBGWRAP_ERR_INPROGRESS; + + volatile dbgwrap_reg_t *dbgWrapReg = (volatile dbgwrap_reg_t *)(cdp->coresight_base[CORESIGHT_UTT] + DBGWRAP_REG_OFFSET); + + if (ml_dbgwrap_cpu_is_halted(cpu_index)) + return DBGWRAP_WARN_ALREADY_HALTED; + + /* Clear all other writable bits besides dbgHalt; none of the power-down or reset bits must be set. */ + *dbgWrapReg = DBGWRAP_DBGHALT; + + if (timeout_ns != 0) { + dbgwrap_status_t stat = ml_dbgwrap_wait_cpu_halted(cpu_index, timeout_ns); + return stat; + } + else + return DBGWRAP_SUCCESS; +} + +static void +ml_dbgwrap_stuff_instr(cpu_data_t *cdp, uint32_t instr, uint64_t timeout_ns, dbgwrap_status_t *status) +{ + if (*status < 0) + return; + + volatile uint32_t *editr = (volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + EDITR_REG_OFFSET); + volatile uint32_t *edscr = (volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + EDSCR_REG_OFFSET); + volatile uint32_t *edrcr = (volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + EDRCR_REG_OFFSET); + + int retries = 0; + + uint64_t interval; + nanoseconds_to_absolutetime(timeout_ns, &interval); + uint64_t deadline = mach_absolute_time() + interval; + +#if DEVELOPMENT || DEBUG + uint32_t stuffed_instr_index = hw_atomic_add(&stuffed_instr_count, 1); + stuffed_instrs[(stuffed_instr_index - 1) % MAX_STUFFED_INSTRS] = instr; +#endif + + do { + *editr = instr; + volatile uint32_t edscr_val; + while (!((edscr_val = *edscr) & EDSCR_ITE)) { + if (mach_absolute_time() > deadline) { + *status = DBGWRAP_ERR_INSTR_TIMEOUT; + return; + } + if (edscr_val & EDSCR_ERR) + break; + } + if (edscr_val & EDSCR_ERR) { + /* If memory access mode was enable by a debugger, clear it. + * This will cause ERR to be set on any attempt to use EDITR. */ + if (edscr_val & EDSCR_MA) + *edscr = edscr_val & ~EDSCR_MA; + *edrcr = EDRCR_CSE; + ++retries; + } else + break; + } while (retries < MAX_EDITR_RETRIES); + + if (retries >= MAX_EDITR_RETRIES) { + *status = DBGWRAP_ERR_INSTR_ERROR; + return; + } +} + +static uint64_t +ml_dbgwrap_read_dtr(cpu_data_t *cdp, uint64_t timeout_ns, dbgwrap_status_t *status) +{ + if (*status < 0) + return 0; + + uint64_t interval; + nanoseconds_to_absolutetime(timeout_ns, &interval); + uint64_t deadline = mach_absolute_time() + interval; + + /* Per armv8 debug spec, writes to DBGDTR_EL0 on target cpu will set EDSCR.TXFull, + * with bits 63:32 available in EDDTRRX and bits 31:0 availabe in EDDTRTX. */ + volatile uint32_t *edscr = (volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + EDSCR_REG_OFFSET); + + while (!(*edscr & EDSCR_TXFULL)) { + if (*edscr & EDSCR_ERR) { + *status = DBGWRAP_ERR_INSTR_ERROR; + return 0; + } + if (mach_absolute_time() > deadline) { + *status = DBGWRAP_ERR_INSTR_TIMEOUT; + return 0; + } + } + + uint32_t dtrrx = *((volatile uint32_t*)(cdp->coresight_base[CORESIGHT_ED] + EDDTRRX_REG_OFFSET)); + uint32_t dtrtx = *((volatile uint32_t*)(cdp->coresight_base[CORESIGHT_ED] + EDDTRTX_REG_OFFSET)); + + return (((uint64_t)dtrrx << 32) | dtrtx); +} + +dbgwrap_status_t +ml_dbgwrap_halt_cpu_with_state(int cpu_index, uint64_t timeout_ns, dbgwrap_thread_state_t *state) +{ + cpu_data_t *cdp = cpu_datap(cpu_index); + if ((cdp == NULL) || (cdp->coresight_base[CORESIGHT_ED] == 0)) + return DBGWRAP_ERR_UNSUPPORTED; + + /* Ensure memory-mapped coresight registers can be written */ + *((volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGLAR)) = ARM_DBG_LOCK_ACCESS_KEY; + + dbgwrap_status_t status = ml_dbgwrap_halt_cpu(cpu_index, timeout_ns); + + /* A core that is not fully powered (e.g. idling in wfi) can still be halted; the dbgwrap + * register and certain coresight registers such EDPRSR are in the always-on domain. + * However, EDSCR/EDITR are not in the always-on domain and will generate a parity abort + * on read. EDPRSR can be safely read in all cases, and the OS lock defaults to being set + * but we clear it first thing, so use that to detect the offline state. */ + if (*((volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + EDPRSR_REG_OFFSET)) & EDPRSR_OSLK) { + bzero(state, sizeof(*state)); + return DBGWRAP_WARN_CPU_OFFLINE; + } + + uint32_t instr; + + for (unsigned int i = 0; i < (sizeof(state->x) / sizeof(state->x[0])); ++i) { + instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | i; // msr DBGDTR0, x<i> + ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, &status); + state->x[i] = ml_dbgwrap_read_dtr(cdp, timeout_ns, &status); + } + + instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | 29; // msr DBGDTR0, fp + ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, &status); + state->fp = ml_dbgwrap_read_dtr(cdp, timeout_ns, &status); + + instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | 30; // msr DBGDTR0, lr + ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, &status); + state->lr = ml_dbgwrap_read_dtr(cdp, timeout_ns, &status); + + /* Stack pointer (x31) can't be used as a register operand for msr; register 31 is treated as xzr + * rather than sp when used as the transfer operand there. Instead, load sp into a GPR + * we've already saved off and then store that register in the DTR. I've chosen x18 + * as the temporary GPR since it's reserved by the arm64 ABI and unused by xnu, so overwriting + * it poses the least risk of causing trouble for external debuggers. */ + + instr = (0x91U << 24) | (31 << 5) | 18; // mov x18, sp + ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, &status); + instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | 18; // msr DBGDTR0, x18 + ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, &status); + state->sp = ml_dbgwrap_read_dtr(cdp, timeout_ns, &status); + + /* reading PC (e.g. through adr) is undefined in debug state. Instead use DLR_EL0, + * which contains PC at time of entry into debug state.*/ + + instr = (0xD53U << 20) | (1 << 19) | (3 << 16) | (4 << 12) | (5 << 8) | (1 << 5) | 18; // mrs x18, DLR_EL0 + ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, &status); + instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | 18; // msr DBGDTR0, x18 + ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, &status); + state->pc = ml_dbgwrap_read_dtr(cdp, timeout_ns, &status); + + /* reading CPSR is undefined in debug state. Instead use DSPSR_EL0, + * which contains CPSR at time of entry into debug state.*/ + instr = (0xD53U << 20) | (1 << 19) | (3 << 16) | (4 << 12) | (5 << 8) | 18; // mrs x18, DSPSR_EL0 + ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, &status); + instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | 18; // msr DBGDTR0, x18 + ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, &status); + state->cpsr = (uint32_t)ml_dbgwrap_read_dtr(cdp, timeout_ns, &status); + + return status; +} + + diff --git a/osfmk/arm64/genassym.c b/osfmk/arm64/genassym.c new file mode 100644 index 000000000..5c06aabce --- /dev/null +++ b/osfmk/arm64/genassym.c @@ -0,0 +1,430 @@ +/* + * Copyright (c) 2007 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +/* + * @OSF_COPYRIGHT@ + */ +/* + * Mach Operating System + * Copyright (c) 1991,1990 Carnegie Mellon University + * All Rights Reserved. + * + * Permission to use, copy, modify and distribute this software and its + * documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR + * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie Mellon + * the rights to redistribute these changes. + */ + +#include <stddef.h> + +#include <mach_ldebug.h> + +/* + * Pass field offsets to assembly code. + */ +#include <kern/ast.h> +#include <kern/thread.h> +#include <kern/task.h> +#include <kern/locks.h> +#include <ipc/ipc_space.h> +#include <ipc/ipc_port.h> +#include <ipc/ipc_pset.h> +#include <kern/host.h> +#include <kern/misc_protos.h> +#include <kern/syscall_sw.h> +#include <arm/thread.h> +#include <mach/arm/vm_param.h> +#include <arm/misc_protos.h> +#include <arm/pmap.h> +#include <arm/trap.h> +#include <arm/cpu_data_internal.h> +#include <arm/cpu_capabilities.h> +#include <arm/cpu_internal.h> +#include <arm/rtclock.h> +#include <machine/commpage.h> +#include <vm/vm_map.h> +#include <pexpert/arm64/boot.h> +#include <arm64/proc_reg.h> +#include <prng/random.h> + +#if CONFIG_DTRACE +#define NEED_DTRACE_DEFS +#include <../bsd/sys/lockstat.h> +#endif /* CONFIG_DTRACE */ + +/* + * genassym.c is used to produce an + * assembly file which, intermingled with unuseful assembly code, + * has all the necessary definitions emitted. This assembly file is + * then postprocessed with sed to extract only these definitions + * and thus the final assyms.s is created. + * + * This convoluted means is necessary since the structure alignment + * and packing may be different between the host machine and the + * target so we are forced into using the cross compiler to generate + * the values, but we cannot run anything on the target machine. + */ + +#define DECLARE(SYM,VAL) \ + __asm("DEFINITION__define__" SYM ":\t .ascii \"%0\"" : : "n" ((u_long)(VAL))) + + +int main( + int argc, + char ** argv); + +int +main( + int argc, + char **argv) +{ + + DECLARE("T_PREFETCH_ABT", T_PREFETCH_ABT); + DECLARE("T_DATA_ABT", T_DATA_ABT); + + DECLARE("AST_URGENT", AST_URGENT); + DECLARE("AST_PREEMPTION", AST_PREEMPTION); + + DECLARE("TH_RECOVER", offsetof(struct thread, recover)); + DECLARE("TH_CONTINUATION", offsetof(struct thread, continuation)); + DECLARE("TH_KERNEL_STACK", offsetof(struct thread, kernel_stack)); + DECLARE("TH_KSTACKPTR", offsetof(struct thread, machine.kstackptr)); + DECLARE("THREAD_UTHREAD", offsetof(struct thread, uthread)); + + DECLARE("TASK_MACH_EXC_PORT", + offsetof(struct task, exc_actions[EXC_MACH_SYSCALL].port)); + + /* These fields are being added on demand */ + DECLARE("ACT_TASK", offsetof(struct thread, task)); + DECLARE("ACT_CONTEXT", offsetof(struct thread, machine.contextData)); + DECLARE("ACT_UPCB", offsetof(struct thread, machine.upcb)); +// DECLARE("ACT_PCBDATA", offsetof(struct thread, machine.contextData.ss)); + DECLARE("ACT_UNEON", offsetof(struct thread, machine.uNeon)); +// DECLARE("ACT_NEONDATA", offsetof(struct thread, machine.contextData.ns)); + DECLARE("TH_CTH_SELF", offsetof(struct thread, machine.cthread_self)); + DECLARE("TH_CTH_DATA", offsetof(struct thread, machine.cthread_data)); + DECLARE("ACT_PREEMPT_CNT", offsetof(struct thread, machine.preemption_count)); + DECLARE("ACT_CPUDATAP", offsetof(struct thread, machine.CpuDatap)); + DECLARE("ACT_MAP", offsetof(struct thread, map)); + DECLARE("ACT_DEBUGDATA", offsetof(struct thread, machine.DebugData)); + DECLARE("TH_IOTIER_OVERRIDE", offsetof(struct thread, iotier_override)); + DECLARE("TH_RWLOCK_CNT", offsetof(struct thread, rwlock_count)); + DECLARE("TH_SCHED_FLAGS", offsetof(struct thread, sched_flags)); + DECLARE("TH_SFLAG_RW_PROMOTED_BIT", TH_SFLAG_RW_PROMOTED_BIT); + + DECLARE("TH_MACH_SYSCALLS", offsetof(struct thread, syscalls_mach)); + DECLARE("TH_UNIX_SYSCALLS", offsetof(struct thread, syscalls_unix)); + DECLARE("TASK_BSD_INFO", offsetof(struct task, bsd_info)); + + DECLARE("MACH_TRAP_TABLE_COUNT", MACH_TRAP_TABLE_COUNT); + DECLARE("MACH_TRAP_TABLE_ENTRY_SIZE", sizeof(mach_trap_t)); + + DECLARE("MAP_PMAP", offsetof(struct _vm_map, pmap)); + + DECLARE("ARM_CONTEXT_SIZE", sizeof(arm_context_t)); + + DECLARE("CONTEXT_SS", offsetof(arm_context_t, ss)); + DECLARE("SS_FLAVOR", offsetof(arm_context_t, ss.ash.flavor)); + DECLARE("ARM_SAVED_STATE32", ARM_SAVED_STATE32); + DECLARE("ARM_SAVED_STATE64", ARM_SAVED_STATE64); + DECLARE("ARM_SAVED_STATE64_COUNT", ARM_SAVED_STATE64_COUNT); + + DECLARE("SS32_W0", offsetof(arm_context_t, ss.ss_32.r[0])); + DECLARE("SS32_W2", offsetof(arm_context_t, ss.ss_32.r[2])); + DECLARE("SS32_W4", offsetof(arm_context_t, ss.ss_32.r[4])); + DECLARE("SS32_W6", offsetof(arm_context_t, ss.ss_32.r[6])); + DECLARE("SS32_W8", offsetof(arm_context_t, ss.ss_32.r[8])); + DECLARE("SS32_W10", offsetof(arm_context_t, ss.ss_32.r[10])); + DECLARE("SS32_W12", offsetof(arm_context_t, ss.ss_32.r[12])); + DECLARE("SS32_SP", offsetof(arm_context_t, ss.ss_32.sp)); + DECLARE("SS32_LR", offsetof(arm_context_t, ss.ss_32.lr)); + DECLARE("SS32_PC", offsetof(arm_context_t, ss.ss_32.pc)); + DECLARE("SS32_CPSR", offsetof(arm_context_t, ss.ss_32.cpsr)); + DECLARE("SS32_VADDR", offsetof(arm_context_t, ss.ss_32.far)); + DECLARE("SS32_STATUS", offsetof(arm_context_t, ss.ss_32.esr)); + + DECLARE("SS64_X0", offsetof(arm_context_t, ss.ss_64.x[0])); + DECLARE("SS64_X2", offsetof(arm_context_t, ss.ss_64.x[2])); + DECLARE("SS64_X4", offsetof(arm_context_t, ss.ss_64.x[4])); + DECLARE("SS64_X6", offsetof(arm_context_t, ss.ss_64.x[6])); + DECLARE("SS64_X8", offsetof(arm_context_t, ss.ss_64.x[8])); + DECLARE("SS64_X10", offsetof(arm_context_t, ss.ss_64.x[10])); + DECLARE("SS64_X12", offsetof(arm_context_t, ss.ss_64.x[12])); + DECLARE("SS64_X14", offsetof(arm_context_t, ss.ss_64.x[14])); + DECLARE("SS64_X16", offsetof(arm_context_t, ss.ss_64.x[16])); + DECLARE("SS64_X18", offsetof(arm_context_t, ss.ss_64.x[18])); + DECLARE("SS64_X19", offsetof(arm_context_t, ss.ss_64.x[19])); + DECLARE("SS64_X20", offsetof(arm_context_t, ss.ss_64.x[20])); + DECLARE("SS64_X21", offsetof(arm_context_t, ss.ss_64.x[21])); + DECLARE("SS64_X22", offsetof(arm_context_t, ss.ss_64.x[22])); + DECLARE("SS64_X23", offsetof(arm_context_t, ss.ss_64.x[23])); + DECLARE("SS64_X24", offsetof(arm_context_t, ss.ss_64.x[24])); + DECLARE("SS64_X25", offsetof(arm_context_t, ss.ss_64.x[25])); + DECLARE("SS64_X26", offsetof(arm_context_t, ss.ss_64.x[26])); + DECLARE("SS64_X27", offsetof(arm_context_t, ss.ss_64.x[27])); + DECLARE("SS64_X28", offsetof(arm_context_t, ss.ss_64.x[28])); + DECLARE("SS64_FP", offsetof(arm_context_t, ss.ss_64.fp)); + DECLARE("SS64_LR", offsetof(arm_context_t, ss.ss_64.lr)); + DECLARE("SS64_SP", offsetof(arm_context_t, ss.ss_64.sp)); + DECLARE("SS64_PC", offsetof(arm_context_t, ss.ss_64.pc)); + DECLARE("SS64_CPSR", offsetof(arm_context_t, ss.ss_64.cpsr)); + DECLARE("SS64_FAR", offsetof(arm_context_t, ss.ss_64.far)); + DECLARE("SS64_ESR", offsetof(arm_context_t, ss.ss_64.esr)); + + DECLARE("CONTEXT_NS", offsetof(arm_context_t, ns)); + DECLARE("NS_FLAVOR", offsetof(arm_context_t, ns.nsh.flavor)); + DECLARE("NS_COUNT", offsetof(arm_context_t, ns.nsh.count)); + DECLARE("ARM_NEON_SAVED_STATE32", ARM_NEON_SAVED_STATE32); + DECLARE("ARM_NEON_SAVED_STATE64", ARM_NEON_SAVED_STATE64); + DECLARE("ARM_NEON_SAVED_STATE64_COUNT", ARM_NEON_SAVED_STATE64_COUNT); + + DECLARE("NS32_Q0", offsetof(arm_context_t, ns.ns_32.v.q[0])); + DECLARE("NS32_Q2", offsetof(arm_context_t, ns.ns_32.v.q[2])); + DECLARE("NS32_Q4", offsetof(arm_context_t, ns.ns_32.v.q[4])); + DECLARE("NS32_Q6", offsetof(arm_context_t, ns.ns_32.v.q[6])); + DECLARE("NS32_Q8", offsetof(arm_context_t, ns.ns_32.v.q[8])); + DECLARE("NS32_Q10", offsetof(arm_context_t, ns.ns_32.v.q[10])); + DECLARE("NS32_Q12", offsetof(arm_context_t, ns.ns_32.v.q[12])); + DECLARE("NS32_Q14", offsetof(arm_context_t, ns.ns_32.v.q[14])); + DECLARE("NS32_FPSR", offsetof(arm_context_t, ns.ns_32.fpsr)); + DECLARE("NS32_FPCR", offsetof(arm_context_t, ns.ns_32.fpcr)); + + DECLARE("NS64_D8", offsetof(arm_context_t, ns.ns_64.v.d[8])); + DECLARE("NS64_D9", offsetof(arm_context_t, ns.ns_64.v.d[9])); + DECLARE("NS64_D10", offsetof(arm_context_t, ns.ns_64.v.d[10])); + DECLARE("NS64_D11", offsetof(arm_context_t, ns.ns_64.v.d[11])); + DECLARE("NS64_D12", offsetof(arm_context_t, ns.ns_64.v.d[12])); + DECLARE("NS64_D13", offsetof(arm_context_t, ns.ns_64.v.d[13])); + DECLARE("NS64_D14", offsetof(arm_context_t, ns.ns_64.v.d[14])); + DECLARE("NS64_D15", offsetof(arm_context_t, ns.ns_64.v.d[15])); + + DECLARE("NS64_Q0", offsetof(arm_context_t, ns.ns_64.v.q[0])); + DECLARE("NS64_Q2", offsetof(arm_context_t, ns.ns_64.v.q[2])); + DECLARE("NS64_Q4", offsetof(arm_context_t, ns.ns_64.v.q[4])); + DECLARE("NS64_Q6", offsetof(arm_context_t, ns.ns_64.v.q[6])); + DECLARE("NS64_Q8", offsetof(arm_context_t, ns.ns_64.v.q[8])); + DECLARE("NS64_Q10", offsetof(arm_context_t, ns.ns_64.v.q[10])); + DECLARE("NS64_Q12", offsetof(arm_context_t, ns.ns_64.v.q[12])); + DECLARE("NS64_Q14", offsetof(arm_context_t, ns.ns_64.v.q[14])); + DECLARE("NS64_Q16", offsetof(arm_context_t, ns.ns_64.v.q[16])); + DECLARE("NS64_Q18", offsetof(arm_context_t, ns.ns_64.v.q[18])); + DECLARE("NS64_Q20", offsetof(arm_context_t, ns.ns_64.v.q[20])); + DECLARE("NS64_Q22", offsetof(arm_context_t, ns.ns_64.v.q[22])); + DECLARE("NS64_Q24", offsetof(arm_context_t, ns.ns_64.v.q[24])); + DECLARE("NS64_Q26", offsetof(arm_context_t, ns.ns_64.v.q[26])); + DECLARE("NS64_Q28", offsetof(arm_context_t, ns.ns_64.v.q[28])); + DECLARE("NS64_Q30", offsetof(arm_context_t, ns.ns_64.v.q[30])); + DECLARE("NS64_FPSR", offsetof(arm_context_t, ns.ns_64.fpsr)); + DECLARE("NS64_FPCR", offsetof(arm_context_t, ns.ns_64.fpcr)); + + DECLARE("PGBYTES", ARM_PGBYTES); + DECLARE("PGSHIFT", ARM_PGSHIFT); + DECLARE("PGMASK", ARM_PGMASK); + + + DECLARE("VM_MIN_ADDRESS", VM_MIN_ADDRESS); + DECLARE("VM_MAX_ADDRESS", VM_MAX_ADDRESS); + DECLARE("VM_MIN_KERNEL_ADDRESS", VM_MIN_KERNEL_ADDRESS); + DECLARE("VM_MAX_KERNEL_ADDRESS", VM_MAX_KERNEL_ADDRESS); + DECLARE("KERNELBASE", VM_MIN_KERNEL_ADDRESS); + DECLARE("KERNEL_STACK_SIZE", KERNEL_STACK_SIZE); + DECLARE("TBI_MASK", TBI_MASK); + + DECLARE("KERN_INVALID_ADDRESS", KERN_INVALID_ADDRESS); + + + DECLARE("MAX_CPUS", MAX_CPUS); + + DECLARE("cdeSize", + sizeof(struct cpu_data_entry)); + + DECLARE("cdSize", + sizeof(struct cpu_data)); + + DECLARE("CPU_ACTIVE_THREAD", + offsetof(cpu_data_t, cpu_active_thread)); + DECLARE("CPU_ACTIVE_STACK", + offsetof(cpu_data_t, cpu_active_stack)); + DECLARE("CPU_ISTACKPTR", + offsetof(cpu_data_t, istackptr)); + DECLARE("CPU_INTSTACK_TOP", + offsetof(cpu_data_t, intstack_top)); + DECLARE("CPU_EXCEPSTACKPTR", + offsetof(cpu_data_t, excepstackptr)); + DECLARE("CPU_EXCEPSTACK_TOP", + offsetof(cpu_data_t, excepstack_top)); + DECLARE("CPU_FIQSTACKPTR", + offsetof(cpu_data_t, fiqstackptr)); + DECLARE("CPU_FIQSTACK_TOP", + offsetof(cpu_data_t, fiqstack_top)); + DECLARE("CPU_NUMBER_GS", + offsetof(cpu_data_t,cpu_number)); + DECLARE("CPU_IDENT", + offsetof(cpu_data_t,cpu_ident)); + DECLARE("CPU_RUNNING", + offsetof(cpu_data_t,cpu_running)); + DECLARE("CPU_MCOUNT_OFF", + offsetof(cpu_data_t,cpu_mcount_off)); + DECLARE("CPU_PENDING_AST", + offsetof(cpu_data_t,cpu_pending_ast)); + DECLARE("CPU_PROCESSOR", + offsetof(cpu_data_t,cpu_processor)); + DECLARE("CPU_CACHE_DISPATCH", + offsetof(cpu_data_t,cpu_cache_dispatch)); + DECLARE("CPU_BASE_TIMEBASE", + offsetof(cpu_data_t,cpu_base_timebase)); + DECLARE("CPU_DECREMENTER", + offsetof(cpu_data_t,cpu_decrementer)); + DECLARE("CPU_GET_DECREMENTER_FUNC", + offsetof(cpu_data_t,cpu_get_decrementer_func)); + DECLARE("CPU_SET_DECREMENTER_FUNC", + offsetof(cpu_data_t,cpu_set_decrementer_func)); + DECLARE("CPU_GET_FIQ_HANDLER", + offsetof(cpu_data_t,cpu_get_fiq_handler)); + DECLARE("CPU_TBD_HARDWARE_ADDR", + offsetof(cpu_data_t,cpu_tbd_hardware_addr)); + DECLARE("CPU_TBD_HARDWARE_VAL", + offsetof(cpu_data_t,cpu_tbd_hardware_val)); + DECLARE("CPU_INT_STATE", + offsetof(cpu_data_t,cpu_int_state)); + DECLARE("INTERRUPT_HANDLER", + offsetof(cpu_data_t,interrupt_handler)); + DECLARE("INTERRUPT_TARGET", + offsetof(cpu_data_t,interrupt_target)); + DECLARE("INTERRUPT_REFCON", + offsetof(cpu_data_t,interrupt_refCon)); + DECLARE("INTERRUPT_NUB", + offsetof(cpu_data_t,interrupt_nub)); + DECLARE("INTERRUPT_SOURCE", + offsetof(cpu_data_t,interrupt_source)); + DECLARE("CPU_USER_DEBUG", + offsetof(cpu_data_t, cpu_user_debug)); + DECLARE("CPU_STAT_IRQ", + offsetof(cpu_data_t, cpu_stat.irq_ex_cnt)); + DECLARE("CPU_STAT_IRQ_WAKE", + offsetof(cpu_data_t, cpu_stat.irq_ex_cnt_wake)); + DECLARE("CPU_RESET_HANDLER", + offsetof(cpu_data_t, cpu_reset_handler)); + DECLARE("CPU_RESET_ASSIST", + offsetof(cpu_data_t, cpu_reset_assist)); + DECLARE("CPU_REGMAP_PADDR", + offsetof(cpu_data_t, cpu_regmap_paddr)); + DECLARE("CPU_PHYS_ID", + offsetof(cpu_data_t, cpu_phys_id)); + DECLARE("RTCLOCK_DATAP", + offsetof(cpu_data_t, rtclock_datap)); + + DECLARE("RTCLOCKDataSize", + sizeof(rtclock_data_t)); + DECLARE("RTCLOCK_ADJ_ABSTIME_LOW", + offsetof(rtclock_data_t, rtc_adj.abstime_val.low)); + DECLARE("RTCLOCK_ADJ_ABSTIME_HIGH", + offsetof(rtclock_data_t, rtc_adj.abstime_val.high)); + DECLARE("RTCLOCK_BASE_ABSTIME_LOW", + offsetof(rtclock_data_t, rtc_base.abstime_val.low)); + DECLARE("RTCLOCK_BASE_ABSTIME_HIGH", + offsetof(rtclock_data_t, rtc_base.abstime_val.high)); + + DECLARE("SIGPdec", SIGPdec); + + DECLARE("rhdSize", + sizeof(struct reset_handler_data)); +#if WITH_CLASSIC_S2R || !__arm64__ + DECLARE("stSize", + sizeof(SleepToken)); +#endif + + DECLARE("CPU_DATA_ENTRIES", offsetof(struct reset_handler_data, cpu_data_entries)); + DECLARE("ASSIST_RESET_HANDLER", offsetof(struct reset_handler_data, assist_reset_handler)); + + DECLARE("CPU_DATA_PADDR", offsetof(struct cpu_data_entry, cpu_data_paddr)); + + + DECLARE("INTSTACK_SIZE", INTSTACK_SIZE); + + DECLARE("TIMER_TSTAMP", + offsetof(struct timer, tstamp)); + DECLARE("THREAD_TIMER", + offsetof(struct processor, processor_data.thread_timer)); + DECLARE("KERNEL_TIMER", + offsetof(struct processor, processor_data.kernel_timer)); + DECLARE("SYSTEM_STATE", + offsetof(struct processor, processor_data.system_state)); + DECLARE("USER_STATE", + offsetof(struct processor, processor_data.user_state)); + DECLARE("CURRENT_STATE", + offsetof(struct processor, processor_data.current_state)); + + DECLARE("SYSTEM_TIMER", + offsetof(struct thread, system_timer)); + DECLARE("USER_TIMER", + offsetof(struct thread, user_timer)); + +#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME + DECLARE("PRECISE_USER_KERNEL_TIME", + offsetof(struct thread, precise_user_kernel_time)); +#endif + + DECLARE("BA_VIRT_BASE", + offsetof(struct boot_args, virtBase)); + DECLARE("BA_PHYS_BASE", + offsetof(struct boot_args, physBase)); + DECLARE("BA_MEM_SIZE", + offsetof(struct boot_args, memSize)); + DECLARE("BA_TOP_OF_KERNEL_DATA", + offsetof(struct boot_args, topOfKernelData)); + DECLARE("BA_DEVICE_TREE", + offsetof(struct boot_args, deviceTreeP)); + DECLARE("BA_DEVICE_TREE_LENGTH", + offsetof(struct boot_args, deviceTreeLength)); + + DECLARE("ENTROPY_INDEX_PTR", + offsetof(entropy_data_t, index_ptr)); + DECLARE("ENTROPY_BUFFER", + offsetof(entropy_data_t, buffer)); + DECLARE("ENTROPY_DATA_SIZE", sizeof(struct entropy_data)); + + DECLARE("SR_RESTORE_TCR_EL1", offsetof(struct sysreg_restore, tcr_el1)); + + + return (0); +} diff --git a/osfmk/arm64/kpc.c b/osfmk/arm64/kpc.c new file mode 100644 index 000000000..b1eae91fe --- /dev/null +++ b/osfmk/arm64/kpc.c @@ -0,0 +1,1135 @@ +/* + * Copyright (c) 2012-2016 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include <arm/cpu_data_internal.h> +#include <arm/cpu_internal.h> +#include <kern/kalloc.h> +#include <kern/kpc.h> +#include <kern/thread.h> +#include <kern/processor.h> +#include <mach/mach_types.h> +#include <machine/machine_routines.h> +#include <stdint.h> +#include <sys/errno.h> + +#if MONOTONIC +#include <kern/monotonic.h> +#endif /* MONOTONIC */ + +/* + * PMCs 8 and 9 were added to Hurricane and to maintain the existing bit + * positions of the other PMCs, their configuration bits start at position 32. + */ +#define PMCR_PMC_8_9_OFFSET (32) +#define PMCR_PMC_8_9_SHIFT(PMC) (((PMC) - 8) + PMCR_PMC_8_9_OFFSET) +#define PMCR_PMC_SHIFT(PMC) (((PMC) <= 7) ? (PMC) : \ + PMCR_PMC_8_9_SHIFT(PMC)) + +/* + * PMCR0 controls enabling, interrupts, and overflow of performance counters. + */ + +/* PMC is enabled */ +#define PMCR0_PMC_ENABLE_MASK(PMC) (UINT64_C(0x1) << PMCR_PMC_SHIFT(PMC)) +#define PMCR0_PMC_DISABLE_MASK(PMC) (~PMCR0_PMC_ENABLE_MASK(PMC)) + +/* how interrupts are generated on PMIs */ +#define PMCR0_INTGEN_SHIFT (8) +#define PMCR0_INTGEN_MASK (UINT64_C(0x7) << PMCR0_INTGEN_SHIFT) +#define PMCR0_INTGEN_OFF (UINT64_C(0) << PMCR0_INTGEN_SHIFT) +#define PMCR0_INTGEN_PMI (UINT64_C(1) << PMCR0_INTGEN_SHIFT) +#define PMCR0_INTGEN_AIC (UINT64_C(2) << PMCR0_INTGEN_SHIFT) +#define PMCR0_INTGEN_DBG_HLT (UINT64_C(3) << PMCR0_INTGEN_SHIFT) +#define PMCR0_INTGEN_FIQ (UINT64_C(4) << PMCR0_INTGEN_SHIFT) + +/* 10 unused */ + +/* set by hardware if PMI was generated */ +#define PMCR0_PMAI_SHIFT (11) +#define PMCR0_PMAI_MASK (UINT64_C(1) << PMCR0_PMAI_SHIFT) + +/* overflow on a PMC generates an interrupt */ +#define PMCR0_PMI_OFFSET (12) +#define PMCR0_PMI_SHIFT(PMC) (PMCR0_PMI_OFFSET + PMCR_PMC_SHIFT(PMC)) +#define PMCR0_PMI_ENABLE_MASK(PMC) (UINT64_C(1) << PMCR0_PMI_SHIFT(PMC)) +#define PMCR0_PMI_DISABLE_MASK(PMC) (~PMCR0_PMI_ENABLE_MASK(PMC)) + +/* disable counting when a PMI is signaled (except for AIC interrupts) */ +#define PMCR0_DISCNT_SHIFT (20) +#define PMCR0_DISCNT_ENABLE_MASK (UINT64_C(1) << PMCR0_DISCNT_SHIFT) +#define PMCR0_DISCNT_DISABLE_MASK (~PMCR0_DISCNT_ENABLE_MASK) + +/* 21 unused */ + +/* block PMIs until ERET retires */ +#define PMCR0_WFRFE_SHIFT (22) +#define PMCR0_WFRFE_ENABLE_MASK (UINT64_C(1) << PMCR0_WFRE_SHIFT) +#define PMCR0_WFRFE_DISABLE_MASK (~PMCR0_WFRFE_ENABLE_MASK) + +/* count global L2C events */ +#define PMCR0_L2CGLOBAL_SHIFT (23) +#define PMCR0_L2CGLOBAL_ENABLE_MASK (UINT64_C(1) << PMCR0_L2CGLOBAL_SHIFT) +#define PMCR0_L2CGLOBAL_DISABLE_MASK (~PMCR0_L2CGLOBAL_ENABLE_MASK) + +/* allow user mode access to configuration registers */ +#define PMCR0_USEREN_SHIFT (30) +#define PMCR0_USEREN_ENABLE_MASK (UINT64_C(1) << PMCR0_USEREN_SHIFT) +#define PMCR0_USEREN_DISABLE_MASK (~PMCR0_USEREN_ENABLE_MASK) + +/* force the CPMU clocks in case of a clocking bug */ +#define PMCR0_CLKEN_SHIFT (31) +#define PMCR0_CLKEN_ENABLE_MASK (UINT64_C(1) << PMCR0_USEREN_SHIFT) +#define PMCR0_CLKEN_DISABLE_MASK (~PMCR0_CLKEN_ENABLE_MASK) + +/* 32 - 44 mirror the low bits for PMCs 8 and 9 */ + +/* PMCR1 enables counters in different processor modes */ + +#define PMCR1_EL0_A32_OFFSET (0) +#define PMCR1_EL0_A64_OFFSET (8) +#define PMCR1_EL1_A64_OFFSET (16) +#define PMCR1_EL3_A64_OFFSET (24) + +#define PMCR1_EL0_A32_SHIFT(PMC) (PMCR1_EL0_A32_OFFSET + PMCR_PMC_SHIFT(PMC)) +#define PMCR1_EL0_A64_SHIFT(PMC) (PMCR1_EL0_A64_OFFSET + PMCR_PMC_SHIFT(PMC)) +#define PMCR1_EL1_A64_SHIFT(PMC) (PMCR1_EL1_A64_OFFSET + PMCR_PMC_SHIFT(PMC)) +#define PMCR1_EL3_A64_SHIFT(PMC) (PMCR1_EL0_A64_OFFSET + PMCR_PMC_SHIFT(PMC)) + +#define PMCR1_EL0_A32_ENABLE_MASK(PMC) (UINT64_C(1) << PMCR1_EL0_A32_SHIFT(PMC)) +#define PMCR1_EL0_A64_ENABLE_MASK(PMC) (UINT64_C(1) << PMCR1_EL0_A64_SHIFT(PMC)) +#define PMCR1_EL1_A64_ENABLE_MASK(PMC) (UINT64_C(1) << PMCR1_EL1_A64_SHIFT(PMC)) +/* PMCR1_EL3_A64 is not supported on PMCs 8 and 9 */ +#if NO_MONITOR +#define PMCR1_EL3_A64_ENABLE_MASK(PMC) UINT64_C(0) +#else +#define PMCR1_EL3_A64_ENABLE_MASK(PMC) (UINT64_C(1) << PMCR1_EL3_A64_SHIFT(PMC)) +#endif + +#define PMCR1_EL_ALL_ENABLE_MASK(PMC) (PMCR1_EL0_A32_ENABLE_MASK(PMC) | \ + PMCR1_EL0_A64_ENABLE_MASK(PMC) | \ + PMCR1_EL1_A64_ENABLE_MASK(PMC) | \ + PMCR1_EL3_A64_ENABLE_MASK(PMC)) +#define PMCR1_EL_ALL_DISABLE_MASK(PMC) (~PMCR1_EL_ALL_ENABLE_MASK(PMC)) + +/* PMESR0 and PMESR1 are event selection registers */ + +/* PMESR0 selects which event is counted on PMCs 2, 3, 4, and 5 */ +/* PMESR1 selects which event is counted on PMCs 6, 7, 8, and 9 */ + +#define PMESR_PMC_WIDTH (8) +#define PMESR_PMC_MASK (UINT8_MAX) +#define PMESR_SHIFT(PMC, OFF) (8 * ((PMC) - (OFF))) +#define PMESR_EVT_MASK(PMC, OFF) (PMESR_PMC_MASK << PMESR_SHIFT(PMC, OFF)) +#define PMESR_EVT_CLEAR(PMC, OFF) (~PMESR_EVT_MASK(PMC, OFF)) + +#define PMESR_EVT_DECODE(PMESR, PMC, OFF) \ + (((PMESR) >> PMESR_SHIFT(PMC, OFF)) & PMESR_PMC_MASK) +#define PMESR_EVT_ENCODE(EVT, PMC, OFF) \ + (((EVT) & PMESR_PMC_MASK) << PMESR_SHIFT(PMC, OFF)) + +/* system registers in the CPMU */ + +#define SREG_PMCR0 "S3_1_c15_c0_0" +#define SREG_PMCR1 "S3_1_c15_c1_0" +#define SREG_PMCR2 "S3_1_c15_c2_0" +#define SREG_PMCR3 "S3_1_c15_c3_0" +#define SREG_PMCR4 "S3_1_c15_c4_0" +#define SREG_PMESR0 "S3_1_c15_c5_0" +#define SREG_PMESR1 "S3_1_c15_c6_0" +#define SREG_PMSR "S3_1_c15_c13_0" +#define SREG_OPMAT0 "S3_1_c15_c7_0" +#define SREG_OPMAT1 "S3_1_c15_c8_0" +#define SREG_OPMSK0 "S3_1_c15_c9_0" +#define SREG_OPMSK1 "S3_1_c15_c10_0" + +#define SREG_PMC0 "S3_2_c15_c0_0" +#define SREG_PMC1 "S3_2_c15_c1_0" +#define SREG_PMC2 "S3_2_c15_c2_0" +#define SREG_PMC3 "S3_2_c15_c3_0" +#define SREG_PMC4 "S3_2_c15_c4_0" +#define SREG_PMC5 "S3_2_c15_c5_0" +#define SREG_PMC6 "S3_2_c15_c6_0" +#define SREG_PMC7 "S3_2_c15_c7_0" +#define SREG_PMC8 "S3_2_c15_c9_0" +#define SREG_PMC9 "S3_2_c15_c10_0" + +#if !defined(APPLECYCLONE) +#define SREG_PMMMAP "S3_2_c15_c15_0" +#define SREG_PMTRHLD2 "S3_2_c15_c14_0" +#define SREG_PMTRHLD4 "S3_2_c15_c13_0" +#define SREG_PMTRHLD6 "S3_2_c15_c12_0" +#endif + +/* + * The low 8 bits of a configuration words select the event to program on + * PMESR{0,1}. Bits 16-19 are mapped to PMCR1 bits. + */ +#define CFGWORD_EL0A32EN_MASK (0x10000) +#define CFGWORD_EL0A64EN_MASK (0x20000) +#define CFGWORD_EL1EN_MASK (0x40000) +#define CFGWORD_EL3EN_MASK (0x80000) +#define CFGWORD_ALLMODES_MASK (0xf0000) + +/* ACC offsets for PIO */ +#define ACC_CPMU_PMC0_OFFSET (0x200) +#define ACC_CPMU_PMC8_OFFSET (0x280) + +/* + * Macros for reading and writing system registers. + * + * SR must be one of the SREG_* defines above. + */ +#define SREG_WRITE(SR, V) __asm__ volatile("msr " SR ", %0 ; isb" : : "r"(V)) +#define SREG_READ(SR) ({ uint64_t VAL; \ + __asm__ volatile("mrs %0, " SR : "=r"(VAL)); \ + VAL; }) + +/* + * Configuration registers that can be controlled by RAWPMU: + * + * All: PMCR2-4, OPMAT0-1, OPMSK0-1. + * Typhoon/Twister/Hurricane: PMMMAP, PMTRHLD2/4/6. + */ +#if defined(APPLECYCLONE) +#define RAWPMU_CONFIG_COUNT 7 +#else +#define RAWPMU_CONFIG_COUNT 11 +#endif + +/* TODO: allocate dynamically */ +static uint64_t saved_PMCR[MAX_CPUS][2]; +static uint64_t saved_PMESR[MAX_CPUS][2]; +static uint64_t saved_RAWPMU[MAX_CPUS][RAWPMU_CONFIG_COUNT]; +static uint64_t saved_counter[MAX_CPUS][KPC_MAX_COUNTERS]; +static uint64_t kpc_running_cfg_pmc_mask = 0; +static uint32_t kpc_running_classes = 0; +static uint32_t kpc_configured = 0; + +static int first_time = 1; + +/* + * The whitelist is disabled by default on development/debug kernel. This can + * be changed via the kpc.disable_whitelist sysctl. The whitelist is enabled on + * release kernel and cannot be disabled. + */ +#if DEVELOPMENT || DEBUG +static boolean_t whitelist_disabled = TRUE; +#else +static boolean_t whitelist_disabled = FALSE; +#endif + +/* List of counter events that are allowed externally */ +static kpc_config_t whitelist[] = { + 0, /* NO_EVENT */ + +#if defined(APPLECYCLONE) + 0x02, /* CORE_CYCLE */ + 0x19, /* BIU_UPSTREAM_CYCLE */ + 0x1a, /* BIU_DOWNSTREAM_CYCLE */ + 0x22, /* L2C_AGENT_LD */ + 0x23, /* L2C_AGENT_LD_MISS */ + 0x24, /* L2C_AGENT_ST */ + 0x25, /* L2C_AGENT_ST_MISS */ + 0x78, /* INST_A32 */ + 0x79, /* INST_THUMB */ + 0x7a, /* INST_A64 */ + 0x7b, /* INST_BRANCH */ + 0xb4, /* SYNC_DC_LOAD_MISS */ + 0xb5, /* SYNC_DC_STORE_MISS */ + 0xb6, /* SYNC_DTLB_MISS */ + 0xb9, /* SYNC_ST_HIT_YNGR_LD */ + 0xc0, /* SYNC_BR_ANY_MISP */ + 0xce, /* FED_IC_MISS_DEM */ + 0xcf, /* FED_ITLB_MISS */ + +#elif defined(APPLETYPHOON) + 0x02, /* CORE_CYCLE */ + 0x13, /* BIU_UPSTREAM_CYCLE */ + 0x14, /* BIU_DOWNSTREAM_CYCLE */ + 0x1a, /* L2C_AGENT_LD */ + 0x1b, /* L2C_AGENT_LD_MISS */ + 0x1c, /* L2C_AGENT_ST */ + 0x1d, /* L2C_AGENT_ST_MISS */ + 0x8a, /* INST_A32 */ + 0x8b, /* INST_THUMB */ + 0x8c, /* INST_A64 */ + 0x8d, /* INST_BRANCH */ + 0xbf, /* SYNC_DC_LOAD_MISS */ + 0xc0, /* SYNC_DC_STORE_MISS */ + 0xc1, /* SYNC_DTLB_MISS */ + 0xc4, /* SYNC_ST_HIT_YNGR_LD */ + 0xcb, /* SYNC_BR_ANY_MISP */ + 0xd3, /* FED_IC_MISS_DEM */ + 0xd4, /* FED_ITLB_MISS */ + +#elif defined(APPLETWISTER) || defined(APPLEHURRICANE) + 0x02, /* CORE_CYCLE */ + 0x1a, /* L2C_AGENT_LD */ + 0x1b, /* L2C_AGENT_LD_MISS */ + 0x1c, /* L2C_AGENT_ST */ + 0x1d, /* L2C_AGENT_ST_MISS */ + 0x8a, /* INST_A32 */ + 0x8b, /* INST_THUMB */ + 0x8c, /* INST_A64 */ + 0x8d, /* INST_BRANCH */ + 0xbf, /* SYNC_DC_LOAD_MISS */ + 0xc0, /* SYNC_DC_STORE_MISS */ + 0xc1, /* SYNC_DTLB_MISS */ + 0xc4, /* SYNC_ST_HIT_YNGR_LD */ + 0xcb, /* SYNC_BR_ANY_MISP */ + 0xd3, /* FED_IC_MISS_DEM */ + 0xd4, /* FED_ITLB_MISS */ + +#else + /* An unknown CPU gets a trivial { NO_EVENT } whitelist. */ +#endif +}; +#define WHITELIST_COUNT (sizeof(whitelist)/sizeof(*whitelist)) + +static boolean_t +config_in_whitelist(kpc_config_t cfg) +{ + unsigned int i; + + for (i = 0; i < WHITELIST_COUNT; i++) { + if (cfg == whitelist[i]) { + return TRUE; + } + } + + return FALSE; +} + +#ifdef KPC_DEBUG +static void dump_regs(void) +{ + uint64_t val; + kprintf("PMCR0 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMCR0)); + kprintf("PMCR1 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMCR1)); + kprintf("PMCR2 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMCR2)); + kprintf("PMCR3 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMCR3)); + kprintf("PMCR4 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMCR4)); + kprintf("PMESR0 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMESR0)); + kprintf("PMESR1 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMESR1)); + + kprintf("PMC0 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC0)); + kprintf("PMC1 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC1)); + kprintf("PMC2 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC2)); + kprintf("PMC3 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC3)); + kprintf("PMC4 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC4)); + kprintf("PMC5 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC5)); + kprintf("PMC6 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC6)); + kprintf("PMC7 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC7)); + +#if (KPC_ARM64_CONFIGURABLE_COUNT > 6) + kprintf("PMC8 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC8)); + kprintf("PMC9 = 0x%" PRIx64 "\n", SREG_READ(SREG_PMC9)); +#endif +} +#endif + +static boolean_t +enable_counter(uint32_t counter) +{ + int cpuid = cpu_number(); + uint64_t pmcr0 = 0, intgen_type; + boolean_t counter_running, pmi_enabled, intgen_correct, enabled; + + pmcr0 = SREG_READ(SREG_PMCR0) | 0x3 /* leave the fixed counters enabled for monotonic */; + + counter_running = (pmcr0 & PMCR0_PMC_ENABLE_MASK(counter)) != 0; + pmi_enabled = (pmcr0 & PMCR0_PMI_ENABLE_MASK(counter)) != 0; + + /* TODO this should use the PMI path rather than AIC for the interrupt + * as it is faster + */ + intgen_type = PMCR0_INTGEN_AIC; + intgen_correct = (pmcr0 & PMCR0_INTGEN_MASK) == intgen_type; + + enabled = counter_running && pmi_enabled && intgen_correct; + + if (!enabled) { + pmcr0 |= PMCR0_PMC_ENABLE_MASK(counter); + pmcr0 |= PMCR0_PMI_ENABLE_MASK(counter); + pmcr0 &= ~PMCR0_INTGEN_MASK; + pmcr0 |= intgen_type; + + SREG_WRITE(SREG_PMCR0, pmcr0); + } + + saved_PMCR[cpuid][0] = pmcr0; + return enabled; +} + +static boolean_t +disable_counter(uint32_t counter) +{ + uint64_t pmcr0; + boolean_t enabled; + int cpuid = cpu_number(); + + if (counter < 2) { + return true; + } + + pmcr0 = SREG_READ(SREG_PMCR0) | 0x3; + enabled = (pmcr0 & PMCR0_PMC_ENABLE_MASK(counter)) != 0; + + if (enabled) { + pmcr0 &= PMCR0_PMC_DISABLE_MASK(counter); + SREG_WRITE(SREG_PMCR0, pmcr0); + } + + saved_PMCR[cpuid][0] = pmcr0; + return enabled; +} + +/* + * Enable counter in processor modes determined by configuration word. + */ +static void +set_modes(uint32_t counter, kpc_config_t cfgword) +{ + uint64_t bits = 0; + int cpuid = cpu_number(); + + if (cfgword & CFGWORD_EL0A32EN_MASK) { + bits |= PMCR1_EL0_A32_ENABLE_MASK(counter); + } + if (cfgword & CFGWORD_EL0A64EN_MASK) { + bits |= PMCR1_EL0_A64_ENABLE_MASK(counter); + } + if (cfgword & CFGWORD_EL1EN_MASK) { + bits |= PMCR1_EL1_A64_ENABLE_MASK(counter); + } +#if !NO_MONITOR + if (cfgword & CFGWORD_EL3EN_MASK) { + bits |= PMCR1_EL3_A64_ENABLE_MASK(counter); + } +#endif + + /* + * Backwards compatibility: Writing a non-zero configuration word with + * all zeros in bits 16-19 is interpreted as enabling in all modes. + * This matches the behavior when the PMCR1 bits weren't exposed. + */ + if (bits == 0 && cfgword != 0) { + bits = PMCR1_EL_ALL_ENABLE_MASK(counter); + } + + uint64_t pmcr1 = SREG_READ(SREG_PMCR1); + pmcr1 &= PMCR1_EL_ALL_DISABLE_MASK(counter); + pmcr1 |= bits; + pmcr1 |= 0x30303; /* monotonic compatibility */ + SREG_WRITE(SREG_PMCR1, pmcr1); + saved_PMCR[cpuid][1] = pmcr1; +} + +static uint64_t +read_counter(uint32_t counter) +{ + switch (counter) { + // case 0: return SREG_READ(SREG_PMC0); + // case 1: return SREG_READ(SREG_PMC1); + case 2: return SREG_READ(SREG_PMC2); + case 3: return SREG_READ(SREG_PMC3); + case 4: return SREG_READ(SREG_PMC4); + case 5: return SREG_READ(SREG_PMC5); + case 6: return SREG_READ(SREG_PMC6); + case 7: return SREG_READ(SREG_PMC7); +#if (KPC_ARM64_CONFIGURABLE_COUNT > 6) + case 8: return SREG_READ(SREG_PMC8); + case 9: return SREG_READ(SREG_PMC9); +#endif + default: return 0; + } +} + +static void +write_counter(uint32_t counter, uint64_t value) +{ + switch (counter) { + // case 0: SREG_WRITE(SREG_PMC0, value); break; + // case 1: SREG_WRITE(SREG_PMC1, value); break; + case 2: SREG_WRITE(SREG_PMC2, value); break; + case 3: SREG_WRITE(SREG_PMC3, value); break; + case 4: SREG_WRITE(SREG_PMC4, value); break; + case 5: SREG_WRITE(SREG_PMC5, value); break; + case 6: SREG_WRITE(SREG_PMC6, value); break; + case 7: SREG_WRITE(SREG_PMC7, value); break; +#if (KPC_ARM64_CONFIGURABLE_COUNT > 6) + case 8: SREG_WRITE(SREG_PMC8, value); break; + case 9: SREG_WRITE(SREG_PMC9, value); break; +#endif + default: break; + } +} + +uint32_t +kpc_rawpmu_config_count(void) +{ + return RAWPMU_CONFIG_COUNT; +} + +int +kpc_get_rawpmu_config(kpc_config_t *configv) +{ + configv[0] = SREG_READ(SREG_PMCR2); + configv[1] = SREG_READ(SREG_PMCR3); + configv[2] = SREG_READ(SREG_PMCR4); + configv[3] = SREG_READ(SREG_OPMAT0); + configv[4] = SREG_READ(SREG_OPMAT1); + configv[5] = SREG_READ(SREG_OPMSK0); + configv[6] = SREG_READ(SREG_OPMSK1); +#if RAWPMU_CONFIG_COUNT > 7 + configv[7] = SREG_READ(SREG_PMMMAP); + configv[8] = SREG_READ(SREG_PMTRHLD2); + configv[9] = SREG_READ(SREG_PMTRHLD4); + configv[10] = SREG_READ(SREG_PMTRHLD6); +#endif + return 0; +} + +static int +kpc_set_rawpmu_config(kpc_config_t *configv) +{ + SREG_WRITE(SREG_PMCR2, configv[0]); + SREG_WRITE(SREG_PMCR3, configv[1]); + SREG_WRITE(SREG_PMCR4, configv[2]); + SREG_WRITE(SREG_OPMAT0, configv[3]); + SREG_WRITE(SREG_OPMAT1, configv[4]); + SREG_WRITE(SREG_OPMSK0, configv[5]); + SREG_WRITE(SREG_OPMSK1, configv[6]); +#if RAWPMU_CONFIG_COUNT > 7 + SREG_WRITE(SREG_PMMMAP, configv[7]); + SREG_WRITE(SREG_PMTRHLD2, configv[8]); + SREG_WRITE(SREG_PMTRHLD4, configv[9]); + SREG_WRITE(SREG_PMTRHLD6, configv[10]); +#endif + return 0; +} + +static void +save_regs(void) +{ + int cpuid = cpu_number(); + + __asm__ volatile("dmb ish"); + + assert(ml_get_interrupts_enabled() == FALSE); + + /* Save current PMCR0/1 values. PMCR2-4 are in the RAWPMU set. */ + saved_PMCR[cpuid][0] = SREG_READ(SREG_PMCR0) | 0x3; + + /* Save event selections. */ + saved_PMESR[cpuid][0] = SREG_READ(SREG_PMESR0); + saved_PMESR[cpuid][1] = SREG_READ(SREG_PMESR1); + + kpc_get_rawpmu_config(saved_RAWPMU[cpuid]); + + /* Disable the counters. */ + // SREG_WRITE(SREG_PMCR0, clear); + + /* Finally, save state for each counter*/ + for (int i = 2; i < KPC_ARM64_PMC_COUNT; i++) { + saved_counter[cpuid][i] = read_counter(i); + } +} + +static void +restore_regs(void) +{ + int cpuid = cpu_number(); + + /* Restore PMESR values. */ + SREG_WRITE(SREG_PMESR0, saved_PMESR[cpuid][0]); + SREG_WRITE(SREG_PMESR1, saved_PMESR[cpuid][1]); + + kpc_set_rawpmu_config(saved_RAWPMU[cpuid]); + + /* Restore counter values */ + for (int i = 2; i < KPC_ARM64_PMC_COUNT; i++) { + write_counter(i, saved_counter[cpuid][i]); + } + + /* Restore PMCR0/1 values (with PMCR0 last to enable). */ + SREG_WRITE(SREG_PMCR1, saved_PMCR[cpuid][1] | 0x30303); + SREG_WRITE(SREG_PMCR0, saved_PMCR[cpuid][0] | 0x3); +} + +static uint64_t +get_counter_config(uint32_t counter) +{ + uint64_t pmesr; + + switch (counter) { + case 2: /* FALLTHROUGH */ + case 3: /* FALLTHROUGH */ + case 4: /* FALLTHROUGH */ + case 5: + pmesr = PMESR_EVT_DECODE(SREG_READ(SREG_PMESR0), counter, 2); + break; + case 6: /* FALLTHROUGH */ + case 7: +#if (KPC_ARM64_CONFIGURABLE_COUNT > 6) + /* FALLTHROUGH */ + case 8: /* FALLTHROUGH */ + case 9: +#endif + pmesr = PMESR_EVT_DECODE(SREG_READ(SREG_PMESR1), counter, 6); + break; + default: + pmesr = 0; + break; + } + + kpc_config_t config = pmesr; + + uint64_t pmcr1 = SREG_READ(SREG_PMCR1); + + if (pmcr1 & PMCR1_EL0_A32_ENABLE_MASK(counter)) { + config |= CFGWORD_EL0A32EN_MASK; + } + if (pmcr1 & PMCR1_EL0_A64_ENABLE_MASK(counter)) { + config |= CFGWORD_EL0A64EN_MASK; + } + if (pmcr1 & PMCR1_EL1_A64_ENABLE_MASK(counter)) { + config |= CFGWORD_EL1EN_MASK; +#if NO_MONITOR + config |= CFGWORD_EL3EN_MASK; +#endif + } +#if !NO_MONITOR + if (pmcr1 & PMCR1_EL3_A64_ENABLE_MASK(counter)) { + config |= CFGWORD_EL3EN_MASK; + } +#endif + + return config; +} + +static void +set_counter_config(uint32_t counter, uint64_t config) +{ + int cpuid = cpu_number(); + uint64_t pmesr = 0; + + switch (counter) { + case 2: /* FALLTHROUGH */ + case 3: /* FALLTHROUGH */ + case 4: /* FALLTHROUGH */ + case 5: + pmesr = SREG_READ(SREG_PMESR0); + pmesr &= PMESR_EVT_CLEAR(counter, 2); + pmesr |= PMESR_EVT_ENCODE(config, counter, 2); + SREG_WRITE(SREG_PMESR0, pmesr); + saved_PMESR[cpuid][0] = pmesr; + break; + + case 6: /* FALLTHROUGH */ + case 7: +#if KPC_ARM64_CONFIGURABLE_COUNT > 6 + /* FALLTHROUGH */ + case 8: /* FALLTHROUGH */ + case 9: +#endif + pmesr = SREG_READ(SREG_PMESR1); + pmesr &= PMESR_EVT_CLEAR(counter, 6); + pmesr |= PMESR_EVT_ENCODE(config, counter, 6); + SREG_WRITE(SREG_PMESR1, pmesr); + saved_PMESR[cpuid][1] = pmesr; + break; + default: + break; + } + + set_modes(counter, config); +} + +/* internal functions */ + +void +kpc_arch_init(void) +{ +} + +boolean_t +kpc_is_running_fixed(void) +{ + return (kpc_running_classes & KPC_CLASS_FIXED_MASK) == KPC_CLASS_FIXED_MASK; +} + +boolean_t +kpc_is_running_configurable(uint64_t pmc_mask) +{ + assert(kpc_popcount(pmc_mask) <= kpc_configurable_count()); + return ((kpc_running_classes & KPC_CLASS_CONFIGURABLE_MASK) == KPC_CLASS_CONFIGURABLE_MASK) && + ((kpc_running_cfg_pmc_mask &